restore head attach
@@ -110,6 +110,13 @@ class TransformersWordPiecer(Pipe):
spacy_tokens = [w.text for w in segment]
a2b, _ = get_alignments(spacy_tokens, wp_tokens)
+ # Head of wp_tokens is sometimes controll character (e.g. "_" for xlnet tokenizer),
+ # so insert 0 to head when it is missed.
+ for a in a2b:
+ if a:
+ if a[0] != 0:
+ a.insert(0, 0)
+ break
a2b = [[i + offset for i in a] for a in a2b]
return wp_tokens, a2b
