Avoid repetition of "vocab = set(vocabulary)"
Avoid repetitive execution of ```vocab = set(vocabulary)``` when iterating through each document.
@@ -160,8 +160,8 @@ class Preprocessing:
else:
labels = [line.strip() for line in open(labels_path, 'r').readlines()]
- for i, doc, label in zip(range(len(docs)), docs, labels):
vocab = set(vocabulary)
+ for i, doc, label in zip(range(len(docs)), docs, labels):
new_doc = [w for w in doc.split() if w in vocab]
if len(new_doc) > self.min_doc_words:
final_docs.append(new_doc)
@@ -179,8 +179,8 @@ class Preprocessing:
final_labels.append(label)
document_indexes.append(i)
else:
- for i, doc in enumerate(docs):
vocab = set(vocabulary)
+ for i, doc in enumerate(docs):
new_doc = [w for w in doc.split() if w in vocab]
if len(new_doc) > self.min_doc_words:
final_docs.append(new_doc)
