move labels to eval_labels instead of removing completely
@@ -137,8 +137,10 @@ class DialogTeacher(Teacher):
self.episode_done = action['episode_done']
action['id'] = self.getID()
self.lastY = action.get('labels', None)
- if not self.datatype.startswith('train'):
- action.pop('labels', None)
+ if not self.datatype.startswith('train') and 'labels' in action:
+ # move labels to eval field so not used for training
+ # but this way the model can use the labels for perplexity or loss
+ action['eval_labels'] = action.pop('labels')
return action
# Return transformed metrics showing total examples and accuracy if avail.
