Use onmt-build-vocab in WMT script
@@ -141,12 +141,5 @@ fi
# Let's finish and clean up
mv wmt$sl$tl.model data/wmt$sl$tl.model
-# We keep the first field of the vocab file generated by SentencePiece and remove the first line <unk>
-cut -f 1 wmt$sl$tl.vocab | tail -n +2 > data/wmt$sl$tl.vocab.tmp
-
-# we add the <blank> word in first position, needed for OpenNMT-TF
-sed -i '1i<blank>' data/wmt$sl$tl.vocab.tmp
-
-# Last tweak we replace the empty line supposed to be the "tab" character (removed by the cut above)
-perl -pe '$/=""; s/\n\n/\n\t\n/;' data/wmt$sl$tl.vocab.tmp > data/wmt$sl$tl.vocab
-rm data/wmt$sl$tl.vocab.tmp
+# Prepare vocabulary for OpenNMT-tf
+onmt-build-vocab --from_format sentencepiece --from_vocab wmt$sl$tl.vocab --save_vocab data/wmt$sl$tl.vocab
