Clean missing imports
from thinc.api import layerize, chain, flatten_add_lengths, with_getitem
-from thinc.t2v import Pooling, mean_pool, sum_pool, max_pool
-from thinc.v2v import Softmax, Affine, Model, Maxout
-from thinc.misc import LayerNorm
+from thinc.t2v import Pooling, mean_pool
+from thinc.v2v import Softmax, Affine, Model
from thinc.neural.util import get_array_module
@@ -40,14 +39,15 @@ def fine_tune_class_vector(nr_class, *, exclusive_classes=True, **cfg):
return chain(
get_pytt_class_tokens,
flatten_add_lengths,
- with_getitem(0, chain(
- Affine(cfg["token_vector_width"], cfg["token_vector_width"]),
- tanh)),
+ with_getitem(
+ 0, chain(Affine(cfg["token_vector_width"], cfg["token_vector_width"]), tanh)
+ ),
Pooling(mean_pool),
Affine(2, cfg["token_vector_width"], drop_factor=0),
- softmax
+ softmax,
)
+
@register_model("fine_tune_pooler_output")
def fine_tune_pooler_output(nr_class, *, exclusive_classes=True, **cfg):
"""Select features from the class-vectors from the last hidden state,
@@ -141,21 +141,25 @@ def get_pytt_last_hidden(docs, drop=0.0):
@layerize
-def softmax(X, drop=0.):
+def softmax(X, drop=0.0):
ops = Model.ops
Y = ops.softmax(X)
+
def backprop_softmax(dY, sgd=None):
dX = ops.backprop_softmax(Y, dY)
return dX
+
return Y, backprop_softmax
@layerize
-def tanh(X, drop=0.):
+def tanh(X, drop=0.0):
xp = get_array_module(X)
Y = xp.tanh(X)
+
def backprop_tanh(dY, sgd=None):
one = Y.dtype.type(1)
- dX = dY * (1 - Y * Y)
+ dX = dY * (one - Y * Y)
return dX
+
return Y, backprop_tanh
