test lambda estimation in degenerate case when all links are matches
@@ -173,7 +173,8 @@ def test_prob_rr_match_link_only_multitable():
deterministic_rules = ["l.first_name = r.first_name", "l.surname = r.surname"]
- linker = DuckDBLinker([df_1, df_2, df_3, df_4], settings)
+ dfs = [df_1, df_2, df_3, df_4]
+ linker = DuckDBLinker(dfs, settings)
linker.estimate_probability_two_random_records_match(
deterministic_rules, recall=1.0
)
@@ -183,6 +184,15 @@ def test_prob_rr_match_link_only_multitable():
# 4*3 + 4*5 + 4*7 + 3*5 + 3*7 + 5*7 = 131 comparisons
assert pytest.approx(prob) == 6 / 131
+ # if we define all record pairs to be a match match, then the probability should be 1
+ dfs = list(map(lambda df: df.assign(city = "Brighton"), dfs))
+ linker = DuckDBLinker(dfs, settings)
+ linker.estimate_probability_two_random_records_match(
+ ["l.city = r.city"], recall=1.0
+ )
+ prob = linker._settings_obj._probability_two_random_records_match
+ assert prob == 1
+
def test_prob_rr_match_link_and_dedupe_multitable():
df_1 = pd.DataFrame(
@@ -232,7 +242,8 @@ def test_prob_rr_match_link_and_dedupe_multitable():
deterministic_rules = ["l.first_name = r.first_name", "l.surname = r.surname"]
- linker = DuckDBLinker([df_1, df_2, df_3, df_4], settings)
+ dfs = [df_1, df_2, df_3, df_4]
+ linker = DuckDBLinker(dfs, settings)
linker.estimate_probability_two_random_records_match(
deterministic_rules, recall=1.0
)
@@ -241,3 +252,11 @@ def test_prob_rr_match_link_and_dedupe_multitable():
# 10 matches (1 John, 3 Mary, 2 Jones (ignoring already matched Mary), 1 Taylor, 3 Graham, 0 Roberts (ignoring already counted Graham))
# (3 + 4 + 5 + 7)(3 + 4 + 5 + 7 - 1)/2 = 171 comparisons
assert pytest.approx(prob) == 10 / 171
+
+ dfs = list(map(lambda df: df.assign(city = "Brighton"), dfs))
+ linker = DuckDBLinker(dfs, settings)
+ linker.estimate_probability_two_random_records_match(
+ ["l.city = r.city"], recall=1.0
+ )
+ prob = linker._settings_obj._probability_two_random_records_match
+ assert prob == 1
