Handle out-of-vocab words
Wasn't handling words out of the tokenizer dictionary vocabulary
properly. This adds a fix and test for that. -POLM
@@ -66,11 +66,15 @@ def detailed_tokens(tokenizer, text):
node = node.next # first node is beginning of sentence and empty, skip it
words = []
while node.posid != 0:
+ surface = node.surface
+ base = surface
parts = node.feature.split(',')
pos = ','.join(parts[0:4])
+
+ if len(parts) > 6:
+ # this information is only available for words in the tokenizer dictionary
reading = parts[6]
base = parts[7]
- surface = parts[8]
words.append( ShortUnitWord(surface, base, pos) )
node = node.next
