falsely passing blocking tests - looking at multi-table linking
@@ -140,9 +140,9 @@ def test_blocking_records_accuracy():
]
settings = {"link_type": "link_and_dedupe"}
- settings = DuckDBLinker([df, df], settings)
+ linker_settings = DuckDBLinker([df, df], settings)
validate_blocking_output(
- settings,
+ linker_settings,
expected_out={
"row_count": [13591, 53472, 137280],
"cumulative_rows": [13591, 67063, 204343],
@@ -152,9 +152,9 @@ def test_blocking_records_accuracy():
)
settings = {"link_type": "link_only"}
- settings = DuckDBLinker([df, df], settings)
+ linker_settings = DuckDBLinker([df, df], settings)
validate_blocking_output(
- settings,
+ linker_settings,
expected_out={
"row_count": [7257, 27190, 68640],
"cumulative_rows": [7257, 34447, 103087],
@@ -162,3 +162,30 @@ def test_blocking_records_accuracy():
},
blocking_rules=blocking_rules,
)
+
+ # now multi-table
+ # still link only
+ linker_settings = DuckDBLinker([df, df, df], settings)
+ validate_blocking_output(
+ linker_settings,
+ expected_out={
+ # number of links per block simply related to two-frame case
+ "row_count": [3*7257, 3*27190, 3*68640],
+ "cumulative_rows": [3*7257, 3*7257 + 3*27190, 3*7257 + 3*27190 + 3*68640],
+ "cartesian": 1_000_000_000 #3_000_000,
+ },
+ blocking_rules=blocking_rules,
+ )
+
+ settings = {"link_type": "link_and_dedupe"}
+ linker_settings = DuckDBLinker([df, df, df], settings)
+ validate_blocking_output(
+ linker_settings,
+ expected_out={
+ # and as above,
+ "row_count": [31272, 120993, 308880],
+ "cumulative_rows": [31272, 31272 + 120993, 31272 + 120993 + 308880],
+ "cartesian": 1_000_000_000 + 3*1000*999//2 #(3000 * 2999)//2,
+ },
+ blocking_rules=blocking_rules,
+ )
