Support arbitrary in-graph text tokenizer
@@ -172,12 +172,34 @@ def tokens_to_chars(tokens):
return chars
+def tokenize(tokenizer, text, delimiter=" "):
+ """Tokenizes text.
+
+ This is an in-graph transformation.
+
+ Args:
+ tokenizer: A callable with the signature `(unicode string) -> list of unicode strings`.
+ text: A `tf.Tensor` containing the text to tokenize.
+ delimiter: The delimiter used to delimit tokens. Note that this character is
+ reserved and should not be part of any tokens.
+
+ Returns:
+ A `tf.Tensor` containing the tokenized text using `delimiter` as separator.
+ """
+ def wrapper(text):
+ unicode_text = text.decode("utf-8")
+ unicode_tokens = tokenizer(unicode_text)
+ return delimiter.join(unicode_tokens).encode("utf-8")
+ return tf.py_func(wrapper, [text], tf.string)
+
+
@six.add_metaclass(abc.ABCMeta)
class TextInputter(Inputter):
"""An abstract inputter that processes text."""
- def __init__(self):
+ def __init__(self, tokenizer=None):
super(TextInputter, self).__init__()
+ self.tokenizer = tokenizer
def get_length(self, data):
return data["length"]
@@ -197,6 +219,8 @@ class TextInputter(Inputter):
if "tokens" not in data:
text = data["raw"]
+ if self.tokenizer is not None:
+ text = tokenize(self.tokenizer, text)
tokens = tf.string_split([text]).values
length = tf.shape(tokens)[0]
@@ -222,7 +246,8 @@ class WordEmbedder(TextInputter):
embedding_size=None,
embedding_file_key=None,
trainable=True,
- dropout=0.0):
+ dropout=0.0,
+ tokenizer=None):
"""Initializes the parameters of the word embedder.
Args:
@@ -235,8 +260,10 @@ class WordEmbedder(TextInputter):
format and behavior.
trainable: If `False`, do not optimize embeddings.
dropout: The probability to drop units in the embedding.
+ tokenizer: An optional callable to tokenize the input text. If `None`,
+ the text is tokenized on spaces. See also `tokenize`.
"""
- super(WordEmbedder, self).__init__()
+ super(WordEmbedder, self).__init__(tokenizer=tokenizer)
self.vocabulary_file_key = vocabulary_file_key
self.embedding_size = embedding_size
@@ -318,7 +345,8 @@ class CharConvEmbedder(TextInputter):
num_outputs,
kernel_size=5,
stride=3,
- dropout=0.0):
+ dropout=0.0,
+ tokenizer=None):
"""Initializes the parameters of the character convolution embedder.
Args:
@@ -329,8 +357,10 @@ class CharConvEmbedder(TextInputter):
kernel_size: Length of the convolution window.
stride: Length of the convolution stride.
dropout: The probability to drop units in the embedding.
+ tokenizer: An optional callable to tokenize the input text. If `None`,
+ the text is tokenized on spaces. See also `tokenize`.
"""
- super(CharConvEmbedder, self).__init__()
+ super(CharConvEmbedder, self).__init__(tokenizer=tokenizer)
self.vocabulary_file_key = vocabulary_file_key
self.embedding_size = embedding_size
