Wordsegmenter updates
@@ -5,11 +5,26 @@ class WordSegmenter:
def get_default_model():
return WordSegmenterModel()\
.setInputCols(["document"]) \
- .setOutputCol("seg_token")
+ .setOutputCol("token")
@staticmethod
def get_pretrained_model(name, language):
return WordSegmenterModel.pretrained(name,language) \
.setInputCols(["document"]) \
- .setOutputCol("seg_token")
+ .setOutputCol("token")
+
+ @staticmethod
+ def get_default_model_for_lang(language):
+ name = WordSegmenter.get_default_word_seg_for_lang(language)
+
+ return WordSegmenterModel.pretrained(name,language) \
+ .setInputCols(["document"]) \
+ .setOutputCol("token")
+
+
+ @staticmethod
+ def get_default_word_seg_for_lang(language):
+ import nlu
+ # get default reference
+ return nlu.NameSpace.pretrained_models_references[language][language+'.'+'segment_words']
\ No newline at end of file
