Added DataFrame from pyspark and tidied up code
-from .charts import load_chart_definition, altair_if_installed_else_json
-
+from pyspark.sql.dataframe import DataFrame
from pyspark.sql.functions import col, count, when
+from .charts import load_chart_definition, altair_if_installed_else_json
+
import pandas as pd
-def missingness_chart(df):
+def missingness_chart(df: DataFrame):
"""Produce bar chart of missingness in standardised nodes
Args:
df (DataFrame): Input Spark dataframe
@@ -20,19 +21,18 @@ def missingness_chart(df):
# Data for plot
-
# Count and percentage of nulls in each columns as pandas dataframe
df_nulls = df.select([count(when(col(c).isNull(), c)).alias(c) for c in df.columns])
pd_nulls = df_nulls.toPandas()
pd_nulls = pd.melt(pd_nulls)
- pd_nulls['record_count'] = df.count()
- pd_nulls['percentage'] = round(pd_nulls['value']*100/ pd_nulls['record_count'], 1)
- pd_nulls['percentage'] = pd_nulls['percentage'].astype(str) + "%"
+ pd_nulls["record_count"] = df.count()
+ pd_nulls["percentage"] = round(pd_nulls['value']*100/pd_nulls["record_count"], 1)
+ pd_nulls["percentage"] = pd_nulls['percentage'].astype(str) + "%"
# Add data to JSON chart definition
- missingness_chart_def["data"]["values"] = pd_nulls.to_dict('records')
+ missingness_chart_def["data"]["values"] = pd_nulls.to_dict("records")
return altair_if_installed_else_json(missingness_chart_def)
