changed grounded
@@ -456,7 +456,7 @@ class MultiTaskModel(nn.Module):
out = {}; d_1, d_2 = 1024, 2048;
# embed the sentence, embed the image, map and classify
- sent_embs, sent_mask = self.sent_encoder(batch['input1'])
+ sent_emb, sent_mask = self.sent_encoder(batch['input1'])
image_map = nn.Linear(d_1, d_2).cuda(); sent_transform = image_map(sent_emb)
ids = batch['ids'].cpu().squeeze(-1); ids = list(ids.data.numpy());
labels = batch['labels'].cpu().squeeze(-1); labels = [int(item) for item in labels.data.numpy()]
@@ -467,6 +467,10 @@ class MultiTaskModel(nn.Module):
init_emb = task.img_encoder.forward(int(img_id)).data.numpy()[0]
seq.append(torch.tensor(init_emb, dtype=torch.float)); true.append(label)
img_emb = torch.stack(seq, dim=0);
+
+ batch_size = len(labels)
+ sent_transform = sent_transform.view(batch_size, -1)
+ image_map = nn.Linear(list(sent_transform.size())[-1], d_2).cuda(); sent_transform = image_map(sent_transform)
'''
cos = nn.SmoothL1Loss()
cos = nn.MSELoss()
@@ -476,9 +480,10 @@ class MultiTaskModel(nn.Module):
cos = nn.CosineEmbeddingLoss(); flags = Variable(torch.ones(len(labels)))
out['loss'] = cos(torch.tensor(sent_transform, dtype=torch.float), torch.tensor(img_emb, dtype=torch.float), flags)
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
-
+ sim = cos(torch.tensor(sent_transform, dtype=torch.float), torch.tensor(img_emb, dtype=torch.float))
classifier = nn.Linear(len(labels), len(labels))
- out['logits'] = classifier(sim)
+ logits = classifier(sim)
+ out['logits'] = logits
preds = [1 if item > 0 else 0 for item in logits.data.numpy()]
acc = [1 if preds[i] == labels[i] else 0 for i in range(len(labels))]
