Verify data script checks for missing labels in label_cands
* verify checks for missing labels in label_cands
* fix + lint
* Update verify_data.py
use 'if x is True' or 'if x is False' instead of ==.
@@ -36,12 +36,25 @@ def report(world, counts, log_time):
'missing_labels': counts['missing_labels'],
'missing_label_candidates': counts['missing_label_candidates'],
'empty_label_candidates': counts['empty_label_candidates'],
+ 'label_candidates_with_missing_label': counts[
+ 'label_candidates_with_missing_label'
+ ],
}
text, log = log_time.log(report['exs'], world.num_examples(), log)
return text, log
+def warn(txt, act, opt):
+ if opt.get('display_examples'):
+ print(txt + ":\n" + str(act))
+ else:
+ warn_once(txt)
+
+
def verify(opt, printargs=None, print_parser=None):
+ if opt['datatype'] == 'train':
+ print("[ note: changing datatype from train to train:ordered ]")
+ opt['datatype'] = 'train:ordered'
# create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
@@ -56,6 +69,7 @@ def verify(opt, printargs=None, print_parser=None):
counts['missing_labels'] = 0
counts['missing_label_candidates'] = 0
counts['empty_label_candidates'] = 0
+ counts['label_candidates_with_missing_label'] = 0
# Show some example dialogs.
while not world.epoch_done():
@@ -63,20 +77,36 @@ def verify(opt, printargs=None, print_parser=None):
act = world.acts[0]
if 'text' not in act:
- warn_once("warning: missing text field")
+ warn("warning: missing text field:\n", act, opt)
counts['missing_text'] += 1
if 'labels' not in act and 'eval_labels' not in act:
- warn_once("warning: missing labels/eval_labels field")
+ warn("warning: missing labels/eval_labels field:\n", act, opt)
counts['missing_labels'] += 1
else:
if 'label_candidates' not in act:
counts['missing_label_candidates'] += 1
else:
+ labels = act.get('labels', act.get('eval_labels'))
+ is_label_cand = {}
+ for l in labels:
+ is_label_cand[l] = False
for c in act['label_candidates']:
if c == '':
- warn_once("warning: empty string label_candidate")
+ warn("warning: empty string label_candidate:\n", act, opt)
counts['empty_label_candidates'] += 1
+ if c in is_label_cand:
+ if is_label_cand[c] is True:
+ warn(
+ "warning: label mentioned twice in candidate_labels:\n",
+ act,
+ opt,
+ )
+ is_label_cand[c] = True
+ for l, has in is_label_cand.items():
+ if has is False:
+ warn("warning: label missing in candidate_labels:\n", act, opt)
+ counts['label_candidates_with_missing_label'] += 1
if log_time.time() > log_every_n_secs:
text, log = report(world, counts, log_time)
@@ -85,9 +115,11 @@ def verify(opt, printargs=None, print_parser=None):
try:
# print dataset size if available
- print('[ loaded {} episodes with a total of {} examples ]'.format(
+ print(
+ '[ loaded {} episodes with a total of {} examples ]'.format(
world.num_episodes(), world.num_examples()
- ))
+ )
+ )
except Exception:
pass
return report(world, counts, log_time)
