fix the file issues of en-de-nmt-small.py
fix the file issues of en-de-nmt-small.py by scripts/get_en_de.sh
@@ -87,10 +87,10 @@ base_params = {
train_params = {
"data_layer": ParallelTextDataLayer,
"data_layer_params": {
- "src_vocab_file": data_root+"vocab.bpe.32000",
- "tgt_vocab_file": data_root+"vocab.bpe.32000",
- "source_file": data_root+"train.tok.clean.bpe.32000.en",
- "target_file": data_root+"train.tok.clean.bpe.32000.de",
+ "src_vocab_file": data_root + "m_common.vocab",
+ "tgt_vocab_file": data_root + "m_common.vocab",
+ "source_file": data_root + "train.clean.en.shuffled.BPE_common.32K.tok",
+ "target_file": data_root + "train.clean.de.shuffled.BPE_common.32K.tok",
"delimiter": " ",
"shuffle": True,
"shuffle_buffer_size": 25000,
@@ -105,10 +105,10 @@ eval_params = {
"batch_size_per_gpu": 32,
"data_layer": ParallelTextDataLayer,
"data_layer_params": {
- "src_vocab_file": data_root+"vocab.bpe.32000",
- "tgt_vocab_file": data_root+"vocab.bpe.32000",
- "source_file": data_root+"newstest2013.tok.bpe.32000.en",
- "target_file": data_root+"newstest2013.tok.bpe.32000.de",
+ "src_vocab_file": data_root+"m_common.vocab",
+ "tgt_vocab_file": data_root+"m_common.vocab",
+ "source_file": data_root+"wmt13-en-de.src.BPE_common.32K.tok",
+ "target_file": data_root+"wmt13-en-de.ref.BPE_common.32K.tok",
"delimiter": " ",
"shuffle": False,
"repeat": True,
@@ -142,11 +142,11 @@ infer_params = {
},
"data_layer": ParallelTextDataLayer,
"data_layer_params": {
- "src_vocab_file": data_root+"vocab.bpe.32000",
- "tgt_vocab_file": data_root+"vocab.bpe.32000",
- "source_file": data_root+"newstest2014.tok.bpe.32000.en",
+ "src_vocab_file": data_root+"m_common.vocab",
+ "tgt_vocab_file": data_root+"m_common.vocab",
+ "source_file": data_root+"wmt14-en-de.src.BPE_common.32K.tok",
# this is intentional
- "target_file": data_root+"newstest2014.tok.bpe.32000.en",
+ "target_file": data_root+"wmt14-en-de.src.BPE_common.32K.tok",
"delimiter": " ",
"shuffle": False,
"repeat": False,
