full doc sentence teacher
@@ -377,3 +377,53 @@ class SentenceLabelsTeacher(IndexTeacher):
}
return action
+
+
+class FulldocSentenceLabelsTeacher(FulldocTeacher):
+ """Teacher which contains the question as the text, the sentences as the
+ label candidates, and the label as the sentence containing the answer.
+
+ Some punctuation may be removed for tokenization purposes.
+ """
+ def __init__(self, opt, shared=None):
+ super().__init__(opt, shared)
+
+ try:
+ import nltk
+ except ImportError:
+ raise ImportError('Please install nltk (e.g. pip install nltk).')
+ # nltk-specific setup
+ st_path = 'tokenizers/punkt/{0}.pickle'.format('english')
+ try:
+ self.sent_tok = nltk.data.load(st_path)
+ except LookupError:
+ nltk.download('punkt')
+ self.sent_tok = nltk.data.load(st_path)
+
+ def get(self, episode_idx, entry_idx=None):
+ episode = self.episodes[episode_idx][entry_idx]
+ entry = {'episode_done': episode['episode_done']}
+ context = ' '.join(episode['text'].split('\n')[:-1]).replace('\xa0', ' ')
+ question = episode['text'].split('\n')[-1]
+ entry['text'] = question
+ label_field = 'labels' if 'labels' in episode else 'eval_labels'
+ answers = []
+ for answer in episode[label_field]:
+ new_answer = answer.replace(
+ '.', '').replace('?', '').replace('!', '')
+ context = context.replace(answer, new_answer)
+ answers.append(new_answer)
+ sentences = self.sent_tok.tokenize(context)
+ entry[label_field] = []
+ for sentence in sentences:
+ for answer in answers:
+ if answer in sentence and sentence not in entry[label_field]:
+ entry[label_field].append(sentence)
+
+
+ entry['label_candidates'] = sentences
+
+ if entry[label_field] == []:
+ entry = {'episode_done': episode['episode_done']}
+
+ return entry
