fixed loss
@@ -553,7 +553,7 @@ class MultiTaskModel(nn.Module):
elif isinstance(task, SequenceGenerationTask):
out = self._seq_gen_forward(batch, task, predict)
elif isinstance(task, (GroundedTask, GroundedSWTask)):
- out = self._grounded_forward(batch, task, predict)
+ out = self._grounded_ranking_bce_forward(batch, task, predict)
elif isinstance(task, RankingTask):
out = self._ranking_forward(batch, task, predict)
else:
@@ -938,6 +938,47 @@ class MultiTaskModel(nn.Module):
task.scorer1(np.mean(acc))
return out
+ def _grounded_ranking_bce_forward(self, batch, task, predict):
+ ''' Binary Cross Entropy Loss
+ Create sentence, image representation.
+ '''
+
+ out, neg = {}, []
+ sent_emb, sent_mask = self.sent_encoder(batch['input1'], task)
+ batch_size = get_batch_size(batch)
+ out['n_exs'] = batch_size
+ sent_pooler = self._get_classifier(task)
+ sent_rep = sent_pooler(sent_emb, sent_mask)
+ loss_fn = nn.L1Loss()
+ ids = batch['ids'].cpu().squeeze(-1).data.numpy().tolist()
+ img_seq = []
+
+ for img_idx in ids:
+ img_rep = task.img_encoder.forward(int(img_idx))[0]
+ img_seq.append(torch.tensor(img_rep, dtype=torch.float32).cuda())
+
+ img_emb = torch.stack(img_seq, dim=0);
+ sent1_rep = sent_rep; sent2_rep = img_emb
+
+ sent1_rep = F.normalize(sent1_rep, 2, 1)
+ sent2_rep = F.normalize(sent2_rep, 2, 1)
+ mat_mul = torch.mm(sent1_rep, torch.transpose(sent2_rep, 0,1))
+ labels = torch.eye(len(mat_mul))
+
+ scale = 1/(len(mat_mul) - 1)
+ weights = scale * torch.ones(mat_mul.shape) - (scale-1) * torch.eye(len(mat_mul))
+ weights = weights.view(-1).cuda()
+
+ mat_mul = mat_mul.view(-1)
+ labels = labels.view(-1).cuda()
+ pred = F.sigmoid(mat_mul).round()
+
+ out['loss'] = loss_fn(mat_mul, labels)
+ total_correct = torch.sum(pred == labels)
+ batch_acc = total_correct.item()/len(labels)
+ task.scorer1.__call__(batch_acc)
+
+ return out
def get_elmo_mixing_weights(self, tasks=[]):
''' Get elmo mixing weights from text_field_embedder,
