change CNN encoder for shapeworld
@@ -790,9 +790,21 @@ class CNNEncoder(Model):
super(CNNEncoder, self).__init__(model_name)
self.model_name = model_name
self.model = self._load_model(model_name)
+
+
+ # New loader
+ '''
+ self.feat_dict = self._load_features_from_json(path, 'train')
+ self.feat_dict.update(self._load_features_from_json(path, 'val'))
+ self.feat_dict.update(self._load_features_from_json(path, 'test'))
+ '''
+
+ '''
+ # Old loader
self.feat_dict = self._load_features(path, 'train')
self.feat_dict.update(self._load_features(path, 'val'))
self.feat_dict.update(self._load_features(path, 'test'))
+ '''
def _load_model(self, model_name):
if model_name == 'alexnet':
@@ -803,17 +815,33 @@ class CNNEncoder(Model):
model = resnet101(pretrained=True)
return model
+ def _load_features_from_json(self, path, dataset):
+ print('Loading CNN features for: ' + str(dataset))
+ if dataset == 'train':
+ f = open('/nfs/jsalt/home/roma/CNN/' + dataset + '_lines.json', 'r')
+ feat_dict = {}
+ for line in f:
+ temp = json.loads(line)
+ feat_dict[temp['img_id']] = temp['feat']
+ else:
+ f = open('/nfs/jsalt/home/roma/CNN/' + dataset + '.json', 'r')
+ for line in f:
+ feat_dict = json.loads(line)
+
+ return feat_dict
+
def _load_features(self, path, dataset):
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = datasets.ImageFolder(
- path + dataset,
+ path,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
+
train_loader = torch.utils.data.DataLoader(
train_dataset)
@@ -830,7 +858,6 @@ class CNNEncoder(Model):
for i, (input, target) in enumerate(train_loader):
x = self.model.forward(input)
feat_dict[rev_class[i]] = x.data
- print(dataset + ' CNN features loaded!')
return feat_dict
def forward(self, img_id):
@@ -838,5 +865,8 @@ class CNNEncoder(Model):
Args:
- img_id that maps image -> sentence pairs in respective datasets.
"""
- # already computed tensor from pretrained
- return self.feat_dict[str(img_id)]
+
+ f = open('/nfs/jsalt/home/roma/CNN/feat/' + str(img_id) + '.json', 'r')
+ for line in f: feat_dict = json.loads(line)
+ return feat_dict['feat']
+
