add preprocessing test
@@ -25,7 +25,7 @@ def data_dir(root_dir):
return root_dir + "/../preprocessed_datasets/"
-def test_preprocessing(data_dir):
+def test_preprocessing_custom_stops(data_dir):
texts_path = data_dir+"/sample_texts/unprepr_docs.txt"
p = Preprocessing(vocabulary=None, max_features=None, remove_punctuation=True, punctuation=".,?:",
lemmatize=False, stopword_list=['am', 'are', 'this', 'that'],
@@ -37,6 +37,18 @@ def test_preprocessing(data_dir):
dataset.save(data_dir+"/sample_texts/")
dataset.load_custom_dataset_from_folder(data_dir + "/sample_texts")
+def test_preprocessing_english_stops(data_dir):
+ texts_path = data_dir+"/sample_texts/unprepr_docs.txt"
+ p = Preprocessing(vocabulary=None, max_features=None, remove_punctuation=True,
+ lemmatize=False, stopword_list='english',
+ min_chars=2, min_words_docs=1)
+ dataset = p.preprocess_dataset(
+ documents_path=texts_path,
+ )
+
+ dataset.save(data_dir+"/sample_texts/")
+ dataset.load_custom_dataset_from_folder(data_dir + "/sample_texts")
+
def test_load_20ng():
data_home = get_data_home(data_home=None)
