fixing partitioning for ETM
@@ -39,7 +39,7 @@ class ETM(Abstract_Model):
self.test_tokens, self.test_counts = None, None
self.valid_tokens, self.valid_counts = None, None
self.train_tokens, self.train_counts, self.vocab = None, None, None
- self.use_partitions = True
+ self.use_partitions = False
self.model = None
self.optimizer = None
self.embeddings = None
@@ -65,7 +65,7 @@ class ETM(Abstract_Model):
return result
def set_model(self, dataset, hyperparameters):
- #if self.use_partitions:
+ if self.use_partitions:
train_data, validation_data, testing_data = \
dataset.get_partitioned_corpus(use_validation=True)
@@ -79,7 +79,7 @@ class ETM(Abstract_Model):
self.train_tokens, self.train_counts, self.test_tokens, self.test_counts, self.valid_tokens, \
self.valid_counts = self.preprocess(vocab2id, data_corpus_train, data_corpus_test, data_corpus_val)
- '''
+
else:
data_corpus = [' '.join(i) for i in dataset.get_corpus()]
vocab = dataset.get_vocabulary()
@@ -87,7 +87,7 @@ class ETM(Abstract_Model):
vocab2id = {w: i for i, w in enumerate(vocab)}
self.train_tokens, self.train_counts = self.preprocess(vocab2id, data_corpus, None)
- '''
+
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.set_default_hyperparameters(hyperparameters)
@@ -182,6 +182,9 @@ class ETM(Abstract_Model):
print('*' * 100)
# VALIDATION ###
+ if self.valid_tokens is None:
+ return True
+ else:
model = self.model.to(self.device)
model.eval()
with torch.no_grad():
