Update BPE
Summary: Pull Request resolved:
@@ -9,6 +9,10 @@ from fairseq import file_utils
from fairseq.data.encoders import register_bpe
+DEFAULT_ENCODER_JSON = 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json'
+DEFAULT_VOCAB_BPE = 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe'
+
+
@register_bpe('gpt2')
class GPT2BPE(object):
@@ -16,16 +20,20 @@ class GPT2BPE(object):
def add_args(parser):
# fmt: off
parser.add_argument('--gpt2-encoder-json', type=str,
- default='https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json',
+ default=DEFAULT_ENCODER_JSON,
help='path to encoder.json')
parser.add_argument('--gpt2-vocab-bpe', type=str,
- default='https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe',
+ default=DEFAULT_VOCAB_BPE,
help='path to vocab.bpe')
# fmt: on
def __init__(self, args):
- encoder_json = file_utils.cached_path(args.gpt2_encoder_json)
- vocab_bpe = file_utils.cached_path(args.gpt2_vocab_bpe)
+ encoder_json = file_utils.cached_path(
+ getattr(args, 'gpt2_encoder_json', DEFAULT_ENCODER_JSON)
+ )
+ vocab_bpe = file_utils.cached_path(
+ getattr(args, 'gpt2_vocab_bpe', DEFAULT_VOCAB_BPE)
+ )
self.bpe = get_encoder(encoder_json, vocab_bpe)
def encode(self, x: str) -> str:
