convert bash command to python code
"- Azure Machine Learning Workspace\n",
"- Azure Machine Learning SDK\n",
"\n",
- "To run rouge evaluation, please refer to the section of compute_rouge_perl in [summarization_evaluation.ipynb](summarization_evaluation.ipynb). "
+ "To run rouge evaluation, please refer to the section of compute_rouge_perl in [summarization_evaluation.ipynb](summarization_evaluation.ipynb). \n",
+ "\n",
+ "You can run this notebook on CPU-only machines."
]
},
{
"outputs": [],
"source": [
"ENTRY_SCRIPT = \"extractive_summarization_cnndm_distributed_train.py\"\n",
- "!mkdir -p {PROJECT_FOLDER}\n",
- "!python ../../tools/generate_conda_file.py --gpu --name {CONDA_ENV_NAME}\n",
- "!cp ./nlp_gpu.yaml {PROJECT_FOLDER}\n",
- "!cp {ENTRY_SCRIPT} {PROJECT_FOLDER}\n",
- "!cp -r ../../utils_nlp {PROJECT_FOLDER}"
+ "os.makedirs(PROJECT_FOLDER, exist_ok=True)\n",
+ "os.system(\"python ../../tools/generate_conda_file.py --gpu --name {}\".format(CONDA_ENV_NAME))\n",
+ "os.system(\"cp ./nlp_gpu.yaml {}\".format(PROJECT_FOLDER))\n",
+ "os.system(\"cp {} {}\".format(ENTRY_SCRIPT, PROJECT_FOLDER))\n",
+ "os.system(\"cp -r ../../utils_nlp {}\".format(PROJECT_FOLDER))"
]
},
{
"outputs": [],
"source": [
"# need to clear the local output dir as the ds.download won't download if the path exists \n",
- "!rm -rf {LOCAL_OUTPUT_DIR}/* "
+ "os.system(\"rm -rf {}/*\".format(LOCAL_OUTPUT_DIR))"
]
},
{
"metadata": {},
"outputs": [],
"source": [
+ "# the script uses <q> as sentence separator so it can write the prediction into the files properly\n",
+ "# here we need to replace <q> with \"\\n\" to prepare for evalation\n",
+ "# removing the ending \"\\n\" is also a preparation step for evalution.\n",
"prediction = []\n",
"with open(os.path.join(LOCAL_OUTPUT_DIR, f'{TARGET_OUTPUT_DIR}{SUMMARY_FILENAME}'), \"r\") as filehandle:\n",
" for cnt, line in enumerate(filehandle):\n",
- " prediction.append(line[0:-1]) # remove the ending \"\\n\""
+ " prediction.append(line[0:-1].replace(\"<q>\", \"\\n\")) # remove the ending \"\\n\""
]
},
{
"for i in ext_sum_test:\n",
" source.append(i[\"src_txt\"]) \n",
" temp_target.append(\" \".join(j) for j in i['tgt']) \n",
- "target = [''.join(i) for i in list(temp_target)]"
+ "target = ['\\n'.join(i) for i in list(temp_target)]"
]
},
{
