head controll char
@@ -109,6 +109,11 @@ class TransformersWordPiecer(Pipe):
def _align(self, segment, wp_tokens, *, offset=0):
spacy_tokens = [w.text for w in segment]
a2b, _ = get_alignments(spacy_tokens, wp_tokens)
+
+ # Head of wp_tokens is sometimes controll character (e.g. "_" for xlnet tokenizer),
+ # so insert 0 to head when it is missed.
+ if a2b and (not a2b[0] or a2b[0] != 0):
+ a2b[0].insert(0, 0)
a2b = [[i + offset for i in a] for a in a2b]
return wp_tokens, a2b
