fix bug caused by deleting keys as we iterated through dict, closes
@@ -21,15 +21,18 @@ class CanopyIndex(TextIndex): # pragma: no cover
N = len(self.index._docweight)
threshold = int(max(1000, N * 0.05))
- self._wids_dict = {}
-
- bucket = self.index.family.IF.Bucket
- for wid, docs in self.index._wordinfo.items():
- if len(docs) > threshold:
+ stop_words = [
+ wid for wid, docs in self.index._wordinfo.items() if len(docs) > threshold
+ ]
+ for wid in stop_words:
word = self.lexicon._words[wid]
logger.info("Removing stop word {}".format(word))
+ del self.lexicon._words[wid]
del self.index._wordinfo[wid]
- continue
+
+ bucket = self.index.family.IF.Bucket
+ self._wids_dict = {}
+ for wid, docs in self.index._wordinfo.items():
if isinstance(docs, dict):
docs = bucket(docs)
idf = numpy.log1p(N / len(docs))
