Make cleanup of memmapped scores be more DRY
@@ -173,20 +173,9 @@ class DedupeMatching(IntegralMatching):
pairs = self.pairs(data)
pair_scores = self.score(pairs)
clusters = self.cluster(pair_scores, threshold)
-
clusters = self._add_singletons(data, clusters)
-
clusters = list(clusters)
-
- try:
- mmap_file = pair_scores.filename # type: ignore
- except AttributeError:
- pass
- else:
- del pair_scores
- if mmap_file:
- os.remove(mmap_file)
-
+ _cleanup_scores(pair_scores)
return clusters
def _add_singletons(self, data: Data, clusters: Clusters) -> Clusters:
@@ -525,16 +514,7 @@ class RecordLinkMatching(IntegralMatching):
links = pair_scores[pair_scores["score"] > threshold]
links = list(links)
-
- try:
- mmap_file = pair_scores.filename # type: ignore
- except AttributeError:
- pass
- else:
- del pair_scores
- if mmap_file:
- os.remove(mmap_file)
-
+ _cleanup_scores(pair_scores)
return links
def one_to_one(self, scores: Scores, threshold: float = 0.0) -> Links:
@@ -1488,3 +1468,14 @@ def flatten_training(
y.extend([encoded_y] * len(pairs))
return examples, numpy.array(y)
+
+
+def _cleanup_scores(arr: Scores) -> None:
+ try:
+ mmap_file = arr.filename # type: ignore
+ except AttributeError:
+ pass
+ else:
+ del arr
+ if mmap_file:
+ os.remove(mmap_file)
