Register get_pytt_pooler_output model
@@ -43,6 +43,20 @@ def fine_tune_class_vector(nr_class, *, exclusive_classes=True, **cfg):
Pooling(mean_pool),
)
+@register_model("fine_tune_pooler_output")
+def fine_tune_pooler_output(nr_class, *, exclusive_classes=True, **cfg):
+ """Select features from the class-vectors from the last hidden state,
+ softmax them, and then mean-pool them to produce one feature per vector.
+ The gradients of the class vectors are incremented in the backward pass,
+ to allow fine-tuning.
+ """
+ return chain(
+ get_pytt_pooler_output,
+ flatten_add_lengths,
+ with_getitem(0, Softmax(nr_class, cfg["token_vector_width"])),
+ Pooling(mean_pool),
+ )
+
@layerize
def get_pytt_class_tokens(docs, drop=0.0):
@@ -65,7 +79,7 @@ def get_pytt_class_tokens(docs, drop=0.0):
def backprop_pytt_class_tokens(d_outputs, sgd=None):
for doc, dY in zip(docs, d_outputs):
- if doc._.pytt_d_last_hidden_state is None:
+ if doc._.pytt_d_last_hidden_state.size == 0:
xp = get_array_module(doc._.pytt_last_hidden_state)
grads = xp.zeros(doc._.pytt_last_hidden_state.shape, dtype="f")
doc._.pytt_d_last_hidden_state = grads
@@ -77,6 +91,26 @@ def get_pytt_class_tokens(docs, drop=0.0):
return outputs, backprop_pytt_class_tokens
+@layerize
+def get_pytt_pooler_output(docs, drop=0.0):
+ """Output a List[array], where the array is the class vector
+ for each sentence in the document. To backprop, we increment the values
+ in the doc._.pytt_d_last_hidden_state array.
+ """
+ outputs = [doc._.pytt_pooler_output for doc in docs]
+
+ def backprop_pytt_pooler_output(d_outputs, sgd=None):
+ for doc, dY in zip(docs, d_outputs):
+ if doc._.pytt_d_pooler_output.size == 0:
+ xp = get_array_module(doc._.pytt_pooler_output)
+ grads = xp.zeros(doc._.pytt_pooler_output.shape, dtype="f")
+ doc._.pytt_d_pooler_output = grads
+ doc._.pytt_d_pooler_output += dY
+ return None
+
+ return outputs, backprop_pytt_pooler_output
+
+
@layerize
def get_pytt_last_hidden(docs, drop=0.0):
"""Output a List[array], where the array is the last hidden vector vector
