flake8 convenience.py
@@ -9,11 +9,13 @@ import sys
from dedupe.core import randomPairs, randomPairsMatch
from dedupe.canonical import getCanonicalRep
+
def unique(seq):
seen = set()
seen_add = seen.add
return [x for x in seq if not (x in seen or seen_add(x))]
+
def consoleLabel(deduper): # pragma: no cover
'''
Command line interface for presenting and labeling training pairs
@@ -125,7 +127,6 @@ def trainingDataLink(data_1, data_2, common_key, training_size=50000) : # pragma
then they are distinct records.
'''
-
identified_records = collections.defaultdict(lambda: [[], []])
matched_pairs = set()
distinct_pairs = set()
@@ -148,7 +149,8 @@ def trainingDataLink(data_1, data_2, common_key, training_size=50000) : # pragma
in randomPairsMatch(len(data_1), len(data_2),
training_size)]
- distinct_pairs = (pair for pair in random_pairs if pair not in matched_pairs)
+ distinct_pairs = (
+ pair for pair in random_pairs if pair not in matched_pairs)
matched_records = [(data_1[key_1], data_2[key_2])
for key_1, key_2 in matched_pairs]
@@ -184,7 +186,6 @@ def trainingDataDedupe(data, common_key, training_size=50000) : # pragma: nocove
then they are distinct records.
'''
-
identified_records = collections.defaultdict(list)
matched_pairs = set()
distinct_pairs = set()
@@ -223,7 +224,6 @@ def trainingDataDedupe(data, common_key, training_size=50000) : # pragma: nocove
return training_pairs
-
def canonicalize(record_cluster): # pragma: nocover
"""
Constructs a canonical representation of a duplicate cluster by
