Raise error when custom_stopwords and ignored_stopwords have elements in common
@@ -63,7 +63,7 @@ def lower_text(text: str):
return text.lower()
-def filter_groups(token: str, excluded_stopwords: list = None) -> str:
+def filter_groups(token: str, ignored_stopwords: list = None) -> str:
"""
Given ``token`` str and a list of groups of words
that were concatenated into tokens, reverses the tokens
@@ -72,21 +72,20 @@ def filter_groups(token: str, excluded_stopwords: list = None) -> str:
Parameters
----------
token : string
- excluded_stopwords : list of strings
+ ignored_stopwords : list of strings
Returns
-------
string
"""
- if excluded_stopwords:
- for group in excluded_stopwords:
- if token == ''.join(group.split()):
+ if ignored_stopwords:
+ for group in ignored_stopwords:
+ if token == group.replace(" ", ""):
token = group
-
return token
-def ungroup_excluded_stopwords(tokens: list, excluded_stopwords: list = None) -> list:
+def ungroup_ignored_stopwords(tokens: list, ignored_stopwords: list = None) -> list:
"""
Given ``tokens`` list of str and a list of groups of words
that are concatenated in tokens, reverses the tokens to
@@ -95,38 +94,46 @@ def ungroup_excluded_stopwords(tokens: list, excluded_stopwords: list = None) ->
Parameters
----------
tokens : list of strings
- excluded_stopwords : list of strings
+ ignored_stopwords : list of strings
Returns
-------
list of strings
"""
- return [filter_groups(token, excluded_stopwords) for token in tokens]
+ return [filter_groups(token, ignored_stopwords) for token in tokens]
-def remove_stopwords(text: str, lang: str, custom_stopwords: list = None, excluded_stopwords: list = None) -> str:
+def remove_stopwords(text: str, lang: str, custom_stopwords: list = None, ignored_stopwords: list = None) -> str:
"""
Given ``text`` str, remove classic stopwords for a given language and
custom stopwords given as a list. Words and groups of words from
- excluded_stopwords list are ignored during stopwords removal.
+ ignored_stopwords list are ignored during stopwords removal.
Parameters
----------
text : string
lang : string
custom_stopwords : list of strings
- excluded_stopwords : list of strings
+ ignored_stopwords : list of strings
Returns
-------
string
+
+ Raises
+ -------
+ ValueError
+ if ``custom_stopwords`` and ``ignored_stopwords`` have common elements.
"""
+ if custom_stopwords and ignored_stopwords:
+ if len(set(custom_stopwords) & set(ignored_stopwords)) > 0:
+ raise ValueError("You are trying to add and remove a stopword at the same time !")
stopwords = get_stopwords(lang)
- if excluded_stopwords:
+ if ignored_stopwords:
keyword_processor = KeywordProcessor()
- singletons_to_keep = [x for x in excluded_stopwords if len(x.split()) == 1]
- for group_of_words in excluded_stopwords:
+ singletons_to_keep = [x for x in ignored_stopwords if len(x.split()) == 1]
+ for group_of_words in ignored_stopwords:
keyword_processor.add_keyword(group_of_words, ''.join(group_of_words.split()))
text = keyword_processor.replace_keywords(text)
else:
@@ -142,7 +149,7 @@ def remove_stopwords(text: str, lang: str, custom_stopwords: list = None, exclud
else:
tokens = text.split()
tokens = [t for t in tokens if (t not in stopwords or t in singletons_to_keep)]
- tokens = ungroup_excluded_stopwords(tokens, excluded_stopwords)
+ tokens = ungroup_ignored_stopwords(tokens, ignored_stopwords)
return ' '.join(tokens)
