Changed tokenizer to add infix when infix_start is offset
@@ -241,9 +241,8 @@ cdef class Tokenizer:
for match in matches:
infix_start = match.start()
infix_end = match.end()
- if infix_start == start:
- continue
+ if infix_start != start:
span = string[start:infix_start]
tokens.push_back(self.vocab.get(tokens.mem, span), False)
