Fixed the issue that no space in string converted from tensor
Summary: Pull Request resolved:
@@ -176,7 +176,7 @@ def batch_by_size(
def process_bpe_symbol(sentence: str, bpe_symbol: str):
if bpe_symbol == 'sentencepiece':
- sentence = sentence.replace('\u2581', ' ').strip()
+ sentence = sentence.replace(' ','').replace('\u2581', ' ').strip()
elif bpe_symbol is not None:
sentence = (sentence + ' ').replace(bpe_symbol, '').rstrip()
return sentence
