Clean up misplaced alignment test
@@ -5,6 +5,7 @@ import numpy
import pytest
from spacy_pytorch_transformers.util import align_word_pieces
+from thinc.neural.optimizers import Adam
@pytest.fixture
def name():
@@ -29,17 +30,5 @@ def test_bert_wrapper_from_pretrained(name, ids):
assert len(outputs) == 2
assert outputs.last_hidden_state.shape == (1, 6, 768)
assert outputs.pooler_output.shape == (1, 768)
- backprop(outputs)
-
-@pytest.mark.parametrize("text,spacy_tokens", [
- ("hello world", ["hello", "world"]),
- ("aaaa aaaaa", ["aaaa", "aaaaa"]),
- ("##!! hello world", ["##!!", "hello", "world"]),
- ("hello world ##!!", ["hello", "world", "##!!"])
-])
-def test_alignment(tokenizer, text, spacy_tokens):
- wp_tokens = tokenizer.tokenize(text)
- alignment = align_word_pieces(spacy_tokens, wp_tokens)
- assert len(alignment) == len(spacy_tokens)
- for wp_offsets in alignment:
- assert len(wp_offsets) >= 1
+ optimizer = Adam(model.ops, 0.001)
+ backprop(outputs, sgd=optimizer)
