Log Probabilities and adding Laplace smoothing
"""File for miscellaneous utility functions and constants."""
from collections import deque
-from functools import reduce
+from functools import reduce, lru_cache
import operator
import math
import os
@@ -598,19 +598,6 @@ class PaddingUtils(object):
return
-class ProbabilityDistribution(dict):
- def __init__(self, dict_agent, N=None, missingfn=None):
- self = dict_agent.freqs
- self.N = float(N or sum(self.itervalues()))
- self.missingfn = lambda k, N: 1./N
-
- def __call__(self, key):
- if key in self:
- return self[key]/self.N
- else:
- return self.missingfn(key, self.N)
-
-
class OffensiveLanguageDetector(object):
"""Tries to detect offensive language in text.
@@ -678,17 +665,6 @@ class OffensiveLanguageDetector(object):
if mod_p not in self.white_list:
self.add_phrase(mod_p)
- def memo(f):
- # Memoize function f
- table = {}
-
- def fmemo(*args):
- if args not in table:
- table[args] = f(*args)
- return table[args]
- fmemo.memo = table
- return fmemo
-
def add_phrase(self, phrase):
"""Add a single phrase to the filter."""
toks = self.tokenize(phrase)
@@ -741,7 +717,13 @@ class OffensiveLanguageDetector(object):
def str_segment(self, text, dict_agent):
- @self.memo
+ # Total number of word tokens
+ N = sum(dict_agent.freq.values())
+
+ # Number of distinct words in the Vocab
+ V = len(dict_agent.freq)
+
+ @lru_cache(maxsize=None)
def segment(text):
# Return a list of words that is the best segmentation of text.
if not text:
@@ -751,30 +733,32 @@ class OffensiveLanguageDetector(object):
def splits(text, L=20):
# Returns a list of all possible first and remainder tuples where
- # first +" "+ rem = tex
candidates = []
for i in range(min(len(text), L)):
candidates.append((text[:i+1], text[i+1:]))
return candidates
def prob_words(words):
- # Returns Naive Bayes probability for a sequence of words
- return product(prob_dist(w) for w in words)
+ # Returns probability for a sequence of words
+ return add(prob(w) for w in words)
- def product(nums):
+ def add(nums):
# Return the product of a sequence of numbers
- return reduce(operator.mul, nums, 1)
+ return reduce(operator.add, nums, 1)
def avoid_long_words(word, N):
return 10./(N * 10**len(word))
- N = dict_agent.maxtokens
+ def laplace_smoothing(word, N, V):
+ return math.log((dict_agent.freq[word] + 1)/(N + V))
+
+ def prob(word, missingfn = laplace_smoothing):
+ if word in dict_agent.freq:
+ return math.log(dict_agent.freq[word]/N)
+ else:
+ return missingfn(word, N, V)
+
- prob_dist = ProbabilityDistribution(
- dict_agent,
- N,
- avoid_long_words
- )
return segment(text)
@@ -1101,4 +1085,3 @@ def warn_once(msg, warningtype=None):
if msg not in _seen_warnings:
_seen_warnings.add(msg)
warnings.warn(msg, warningtype, stacklevel=2)
-
