fix data cleaning bug in cornell movie dataset
splitting on spaces fails when a character has both a first and last name. In such cases this leads to the unintentional inclusion of the field separator +++$+++ at the beginning of a line of dialogue.
@@ -19,8 +19,8 @@ def create_fb_format(lines_file, convo_file, outpath):
codecs.register_error('strict', codecs.ignore_errors)
with codecs.open(lines_file, 'r') as f:
for line in f:
- l = line.split(' ')
- lines[l[0]] = ' '.join(l[8:]).strip('\n').replace('\t', ' ')
+ l = line.split(' +++$+++ ')
+ lines[l[0]] = ' '.join(l[4:]).strip('\n').replace('\t', ' ')
cnt = 0
with codecs.open(convo_file, 'r') as f:
