Fix to add_ngram function and docstring
@@ -48,13 +48,13 @@ def add_ngram(sequences, token_indice, ngram_range=2):
>>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
>>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017, (7, 9, 2): 2018}
>>> add_ngram(sequences, token_indice, ngram_range=3)
- [[1, 3, 4, 5, 1337], [1, 3, 7, 9, 2, 1337, 2018]]
+ [[1, 3, 4, 5, 1337, 2017], [1, 3, 7, 9, 2, 1337, 42, 2018]]
"""
new_sequences = []
for input_list in sequences:
new_list = input_list[:]
- for i in range(len(new_list) - ngram_range + 1):
for ngram_value in range(2, ngram_range + 1):
+ for i in range(len(new_list) - ngram_value + 1):
ngram = tuple(new_list[i:i + ngram_value])
if ngram in token_indice:
new_list.append(token_indice[ngram])
