fixing udf registration code after classpaths were removed, updating error logging
@@ -244,17 +244,16 @@ class SparkLinker(Linker):
# will for loop through this list to register UDFs.
# List is a tuple of structure (UDF Name, class path, spark return type)
udfs_register = [
- ("jaro_winkler_sim", "JaroWinklerSimilarity", DoubleType()),
- ("jaccard_sim", "JaccardSimilarity", DoubleType()),
- ("cosine_distance", "CosineDistance", DoubleType()),
- ("Dmetaphone", "DoubleMetaphone", StringType()),
- ("Dmetaphone", "DoubleMetaphone", StringType()),
- ("DmetaphoneAlt", "DoubleMetaphoneAlt", StringType()),
+ ("jaro_winkler_sim", "uk.gov.moj.dash.linkage.JaroWinklerSimilarity", DoubleType()),
+ ("jaccard_sim", "uk.gov.moj.dash.linkage.JaccardSimilarity", DoubleType()),
+ ("cosine_distance", "uk.gov.moj.dash.linkage.CosineDistance", DoubleType()),
+ ("Dmetaphone", "uk.gov.moj.dash.linkage.DoubleMetaphone", StringType()),
+ ("DmetaphoneAlt", "uk.gov.moj.dash.linkage.DoubleMetaphoneAlt", StringType()),
]
try:
for udf in udfs_register:
self.spark.udf.registerJavaFunction(*udf)
- except AnalysisException:
+ except AnalysisException as e:
logger.warning(
"Unable to load custom Spark SQL functions such as jaro_winkler from "
"the jar that's provided with Splink.\n"
@@ -264,6 +263,8 @@ class SparkLinker(Linker):
"You will not be able to use these functions in your linkage.\n"
"You can find the location of the jar by calling the following function"
":\nfrom splink.spark.jar_location import similarity_jar_location"
+ "\n\nFull error:\n"
+ f"{e}"
)
def _table_to_splink_dataframe(self, templated_name, physical_name):
