grounded change
@@ -472,50 +472,35 @@ class MultiTaskModel(nn.Module):
return out
def _grounded_classification_forward(self, batch, task):
- out = {}
- d_1, d_2 = 1024, 2048
+ out = {}; d_1, d_2 = self.sent_encoder.output_dim, 2048;
# embed the sentence, embed the image, map and classify
sent_emb, sent_mask = self.sent_encoder(batch['input1'])
- image_map = nn.Linear(d_1, d_2).cuda()
- sent_transform = image_map(sent_emb)
- ids = batch['ids'].cpu().squeeze(-1)
- ids = list(ids.data.numpy())
- labels = batch['labels'].cpu().squeeze(-1)
- labels = [int(item) for item in labels.data.numpy()]
+ image_map = nn.Linear(d_1, d_2).cuda(); sent_transform = image_map(sent_emb)
+ ids = batch['ids'].cpu().squeeze(-1); ids = list(ids.data.numpy());
+ labels = batch['labels'].cpu().squeeze(-1); labels = [int(item) for item in labels.data.numpy()]
seq, true = [], []
for i in range(len(ids)):
img_id, label = ids[i], labels[i]
init_emb = task.img_encoder.forward(int(img_id)).data.numpy()[0]
- seq.append(torch.tensor(init_emb, dtype=torch.float))
- true.append(label)
- img_emb = torch.stack(seq, dim=0)
+ seq.append(torch.tensor(init_emb, dtype=torch.float)); true.append(label)
+ img_emb = torch.stack(seq, dim=0);
batch_size = len(labels)
sent_transform = sent_transform.view(batch_size, -1)
- image_map = nn.Linear(list(sent_transform.size())[-1], d_2).cuda()
- sent_transform = image_map(sent_transform)
+ image_map = nn.Linear(list(sent_transform.size())[-1], d_2).cuda(); sent_transform = image_map(sent_transform)
+
'''
cos = nn.SmoothL1Loss()
cos = nn.MSELoss()
cos = nn.L1Loss()
out['loss'] = cos(sent_emb, torch.tensor(img_emb, requires_grad=False))
'''
- cos = nn.CosineEmbeddingLoss()
- flags = Variable(torch.ones(len(labels)))
- out['loss'] = cos(
- torch.tensor(
- sent_transform, dtype=torch.float), torch.tensor(
- img_emb, dtype=torch.float), flags)
+ cos = nn.CosineEmbeddingLoss(); flags = Variable(torch.ones(len(labels)))
+ out['loss'] = cos(torch.tensor(sent_transform, dtype=torch.float), torch.tensor(img_emb, dtype=torch.float), flags)
cos = nn.CosineSimilarity(dim=1, eps=1e-6)
- sim = cos(
- torch.tensor(
- sent_transform,
- dtype=torch.float),
- torch.tensor(
- img_emb,
- dtype=torch.float))
+ sim = cos(torch.tensor(sent_transform, dtype=torch.float), torch.tensor(img_emb, dtype=torch.float))
classifier = nn.Linear(len(labels), len(labels))
logits = classifier(sim)
out['logits'] = logits
