added tuple support for span classifier conversion and improved error messasges
@@ -23,9 +23,9 @@ class DataConversionUtils:
@staticmethod
def except_invalid_question_data_format(cols):
raise ValueError(
- f'You input data format is invaid for question answering with span classification.'
- f'Make sure you have at least 2 columns in you dataset '
- f'Refer to <DOCS TOTO> for more details{cols} ')
+ f'You input data format is invalid for question answering with span classification.'
+ f'Make sure you have at least 2 columns in you dataset, named context/question for pandas Dataframes'
+ f'For Strings/Iterables/Tuples make sure to use the format `question|||context` or (question,context) ' )
@staticmethod
def sdf_to_sdf(data, spark_sess, raw_text_column='text'):
@@ -70,6 +70,33 @@ class DataConversionUtils:
'origin_index': [0]}, index=[0]))
return sdf, [], output_datatype
+ @staticmethod
+ def question_tuple_to_sdf(data, spark_sess):
+ """Casting str to spark and add index col. This is a bit inefficient. Casting follow # inefficient, str->pd->spark->pd , we can could first pd"""
+ output_datatype = 'string'
+ question, context = data[0], data[1]
+ sdf = spark_sess.createDataFrame(pd.DataFrame({NLP_FEATURES.RAW_QUESTION: question,
+ NLP_FEATURES.RAW_QUESTION_CONTEXT: context,
+ 'origin_index': [0]}, index=[0]))
+ return sdf, [], output_datatype
+
+ @staticmethod
+ def question_tuple_iterable_to_sdf(data, spark_sess):
+ """Casting str to spark and add index col. This is a bit inefficient. Casting follow # inefficient, str->pd->spark->pd , we can could first pd"""
+ output_datatype = 'string'
+
+ if len(data) == 0:
+ DataConversionUtils.except_invalid_question_data_format(data)
+ if len(data[0]) != 2:
+ DataConversionUtils.except_invalid_question_data_format(data)
+
+ question, context = zip(*[(d[0], d[1]) for d in data])
+
+ sdf = spark_sess.createDataFrame(pd.DataFrame({NLP_FEATURES.RAW_QUESTION: question,
+ NLP_FEATURES.RAW_QUESTION_CONTEXT: context,
+ 'origin_index': [0]}, index=list(range(len(question)))))
+ return sdf, [], output_datatype
+
@staticmethod
def question_str_iterable_to_sdf(data, spark_sess):
"""Casting str to spark and add index col. This is a bit inefficient. Casting follow # inefficient, str->pd->spark->pd , we can could first pd"""
@@ -225,19 +252,17 @@ class DataConversionUtils:
return DataConversionUtils.question_sdf_to_sdf(data, spark_sess)
elif isinstance(data, pd.DataFrame):
return DataConversionUtils.question_pdf_to_sdf(data, spark_sess)
- elif isinstance(data, pd.Series):
- pass
- elif isinstance(data, np.ndarray):
- pass
+ elif isinstance(data, tuple):
+ return DataConversionUtils.question_tuple_to_sdf(data, spark_sess)
elif isinstance(data, str):
return DataConversionUtils.question_str_to_sdf(data, spark_sess)
- elif isinstance(data, list):
+ elif isinstance(data, (list, pd.Series, np.ndarray)):
+ if isinstance(data[0], tuple):
+ return DataConversionUtils.question_tuple_iterable_to_sdf(data, spark_sess)
+ elif isinstance(data[0], str):
return DataConversionUtils.question_str_iterable_to_sdf(data, spark_sess)
- else:
- pass
-
- except Exception as err:
- pass
+ except:
+ ValueError("Data could not be converted to Spark Dataframe for internal conversion.")
else:
try:
if isinstance(data, pyspark.sql.dataframe.DataFrame):
