Add evaluation.
"%autoreload 2\n",
"import time\n",
"from utils_nlp.dataset.cnndm import CNNDMSummarizationDatasetOrg\n",
- "from utils_nlp.models import S2SAbsSumProcessor, S2SAbstractiveSummarizer"
+ "from utils_nlp.models import S2SAbsSumProcessor, S2SAbstractiveSummarizer\n",
+ "from utils_nlp.eval import compute_rouge_python"
]
},
{
"outputs": [],
"source": [
"start = time.time()\n",
- "train_ds, test_ds = CNNDMSummarizationDatasetOrg(top_n=100)\n",
+ "train_ds, test_ds = CNNDMSummarizationDatasetOrg()\n",
"print(time.time() - start)"
]
},
" for line in res:\n",
" f.write(line + '\\n')"
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluation\n",
+ "We provide utility functions for evaluating summarization tasks and details can be found in the [summarization evaluation notebook](./summarization_evaluation.ipynb). \n",
+ "For the settings in this notebook with QUICK_RUN=False, you should get ROUGE scores close to the following numbers: \n",
+ "{'rouge-1': {'f': 0.37109626943068647,\n",
+ " 'p': 0.4692792272280924,\n",
+ " 'r': 0.33322322114381886}, \n",
+ " 'rouge-2': {'f': 0.1690495786379728,\n",
+ " 'p': 0.21782900161918375,\n",
+ " 'r': 0.15079122430118444}, \n",
+ " 'rouge-l': {'f': 0.2671310062443078,\n",
+ " 'p': 0.3414039392451434,\n",
+ " 'r': 0.2392756715930202}}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "compute_rouge_python(cand=res, ref=test_ds.get_target())"
+ ]
}
],
"metadata": {
