add check to eval for tolerance
@@ -50,7 +50,6 @@ class EntityRankingJob(EvaluationJob):
for f in Job.job_created_hooks:
f(self)
-
def _prepare(self):
super()._prepare()
"""Construct all indexes needed to run."""
@@ -220,8 +219,16 @@ class EntityRankingJob(EvaluationJob):
o_in_chunk_mask = (chunk_start <= o) & (o < chunk_end)
o_in_chunk = (o[o_in_chunk_mask] - chunk_start).long()
s_in_chunk = (s[s_in_chunk_mask] - chunk_start).long()
- scores_sp[o_in_chunk_mask, o_in_chunk] = o_true_scores[o_in_chunk_mask]
- scores_po[s_in_chunk_mask, s_in_chunk] = s_true_scores[s_in_chunk_mask]
+
+ # check that scoring is consistent up to configured tolerance
+ # if this is not the case, evaluation metrics may be artificially inflated
+ close_check = torch.allclose(scores_sp[o_in_chunk_mask, o_in_chunk], o_true_scores[o_in_chunk_mask],
+ rtol=self.tie_rtol, atol=self.tie_atol)
+ close_check &= torch.allclose(scores_po[s_in_chunk_mask, s_in_chunk], s_true_scores[s_in_chunk_mask],
+ rtol=self.tie_rtol, atol=self.tie_atol)
+ if not close_check:
+ raise ValueError("Tie-handling tolerances set too low! "
+ "The same triples scored twice did not fall within the configured tolerance.")
# now compute the rankings (assumes order: None, _filt, _filt_test)
for ranking in rankings:
