extractor confs added
@@ -16,9 +16,11 @@ class SparkNLPExtractorConfig:
get_full_meta :bool # get all keys and vals from base emta map
get_annotator_type :bool # Get Annotator Type
unpack_single_list :bool # Should unpack the result field. Only set true for annotators that return exactly one element in their result, like Document classifier! This will convert list with just 1 element into just their element in the final pandas representation
- full_meta_blacklist :List[str] # Blacklist some keys which should not be fetched from full_meta
- meta_blacklist :List[str] # Blacklist some keys which should not be fetched from meta
-
+ meta_white_list :List[str] # Whitelist some keys which should be fetched from meta map
+ meta_black_list :List[str] # black_list some keys which should not be fetched from meta map
+ # { # dict of finisher extractor methods, which will be applied to specific fields aber finishing up base extraction
+ # 'field' : extractor_method
+ # }
def get_default_full_extractor_config(output_col_prefix='DEFAULT'):
return SparkNLPExtractorConfig(
output_col_prefix = output_col_prefix,
@@ -28,8 +30,8 @@ def get_default_full_extractor_config(output_col_prefix='DEFAULT'):
get_result = True,
get_meta = True,
get_full_meta = True,
- full_meta_blacklist = [],
- meta_blacklist = [],
+ meta_white_list = [],
+ meta_black_list = [],
get_annotator_type = True,
unpack_single_list = False
)
@@ -43,9 +45,9 @@ def get_default_document_extractor_config():
get_result = True,
get_meta = False,
get_full_meta = False,
- full_meta_blacklist = [],
- meta_blacklist = [],
- get_annotator_type = True,
+ meta_white_list = [],
+ meta_black_list = [],
+ get_annotator_type = False,
unpack_single_list = False
)
@@ -59,13 +61,29 @@ def get_default_word_embedding_extractor_config():
get_result = True,
get_meta = False,
get_full_meta = False,
- full_meta_blacklist = [],
- meta_blacklist = [],
- get_annotator_type = True,
+ meta_white_list = [],
+ meta_black_list = [],
+ get_annotator_type = False,
+ unpack_single_list = False
+ )
+
+def get_default_NER_extractor_config():
+ return SparkNLPExtractorConfig(
+ output_col_prefix = 'NER',
+ get_begin = False,
+ get_end = False,
+ get_embeds = False,
+ get_result = True,
+ get_meta = True,
+ get_full_meta = False,
+ meta_white_list = ['confidence'],
+ meta_black_list = [],
+ get_annotator_type = False,
unpack_single_list = False
)
+
def get_default_named_word_embedding_extractor_config(output_col_prefix='DEFAULT'):
"""Prefixes _word_embedding to input name"""
return SparkNLPExtractorConfig(
@@ -76,11 +94,8 @@ def get_default_named_word_embedding_extractor_config(output_col_prefix='DEFAULT
get_result = True,
get_meta = False,
get_full_meta = False,
- full_meta_blacklist = [],
- meta_blacklist = [],
- full_meta_blacklist = [],
- meta_blacklist = [],
-
+ meta_black_list = [],
+ meta_white_list = [],
get_annotator_type = False,
unpack_single_list = False
)
\ No newline at end of file
