Convert custom user_data to token extension format for Japanese tokenizer
* Convert custom user_data to token extension format
Convert the user_data values so that they can be loaded as custom token
extensions for `inflection`, `reading_form`, `sub_tokens`, and `lemma`.
* Reset Underscore state in ja tokenizer tests
@@ -145,8 +145,7 @@ class JapaneseTokenizer(DummyTokenizer):
dtokens, spaces = get_dtokens_and_spaces(dtokens, text)
# create Doc with tag bi-gram based part-of-speech identification rules
- words, tags, inflections, lemmas, readings, sub_tokens_list = zip(*dtokens) if dtokens else [[]] * 6
- sub_tokens_list = list(sub_tokens_list)
+ words = [dtoken.surface for dtoken in dtokens]
doc = Doc(self.vocab, words=words, spaces=spaces)
next_pos = None # for bi-gram rules
for idx, (token, dtoken) in enumerate(zip(doc, dtokens)):
@@ -158,14 +157,14 @@ class JapaneseTokenizer(DummyTokenizer):
token.pos, next_pos = resolve_pos(
token.orth_,
dtoken.tag,
- tags[idx + 1] if idx + 1 < len(tags) else None
+ dtokens[idx + 1].tag if idx + 1 < len(dtokens) else None
)
# if there's no lemma info (it's an unk) just use the surface
token.lemma_ = dtoken.lemma if dtoken.lemma else dtoken.surface
-
- doc.user_data["inflections"] = inflections
- doc.user_data["reading_forms"] = readings
- doc.user_data["sub_tokens"] = sub_tokens_list
+ doc.user_data[('._.', 'inflection', token.idx, None)] = dtoken.inf
+ doc.user_data[('._.', 'reading_form', token.idx, None)] = dtoken.reading
+ doc.user_data[('._.', 'sub_tokens', token.idx, None)] = dtoken.sub_tokens
+ doc.user_data[('._.', 'lemma', token.idx, None)] = token.lemma_
return doc
