fix head attach
@@ -110,10 +110,6 @@ class TransformersWordPiecer(Pipe):
spacy_tokens = [w.text for w in segment]
a2b, _ = get_alignments(spacy_tokens, wp_tokens)
- # Head of wp_tokens is sometimes controll character (e.g. "_" for xlnet tokenizer),
- # so insert 0 to head when it is missed.
- if a2b and (not a2b[0] or a2b[0] != 0):
- a2b[0].insert(0, 0)
a2b = [[i + offset for i in a] for a in a2b]
return wp_tokens, a2b
