Update joint-core-bert config
# data is passed in sentence-by-sentence via some prior preprocessing.
gold_preproc = true
# Limitations on training document length or number of examples.
-max_length = 0
+max_length = 1000
limit = 0
# Data augmentation
orth_variant_level = 0.0
@@ -14,12 +14,11 @@ dropout = 0.1
# Controls early-stopping.
patience = 10000
eval_frequency = 200
-max_epochs = 100
+max_epochs = 10000
# Other settings
seed = 0
-accumulate_gradient = 4
+accumulate_gradient = 2
width = 768
-use_gpu = 0
# Control how scores are printed and checkpoints are evaluated.
scores = ["tags_acc", "uas", "las", "ents_f"]
score_weights = {"las": 0.4, "ents_f": 0.4, "tags_acc": 0.2}
@@ -30,8 +29,8 @@ vectors = null
[training.batch_size]
@schedules = "compounding.v1"
-start = 4
-stop = 4
+start = 500
+stop = 500
compound = 1.001
[optimizer]
@@ -74,7 +73,7 @@ factory = "parser"
# The old NER model wasn't ideal for transformers. This component ditches
# the transition-based machinery and imitation-learning objective for a simpler
# tagging approach. A validity mask is used to ensure sequences are valid.
-factory = "simple_ner"
+factory = "ner"
# This loads the Huggingface Transformers model. The transformer is applied
@@ -82,18 +81,13 @@ factory = "simple_ner"
# longer documents.
[nlp.pipeline.transformer.model]
@architectures = "spacy.TransformerByName.v2"
-name = "distilbert-base-uncased"
-fast_tokenizer = false
+name = "bert-base-cased"
+fast_tokenizer = true
[nlp.pipeline.transformer.model.get_spans]
# You can set a custom strategy for preparing spans from the batch, e.g. you
-# can predict over sentences. Here we predict over windows of 256 tokens, with
-# a stride of 192 -- so there's an overlap of 64 tokens between each window.
-# The tokens will receive vectors that average over their wordpieces, so there's
-# no problem with having the same token in multiple spans.
-@layers = "spacy-transformers.strided_spans.v1"
-window = 256
-stride = 192
+# can predict over sentences. Here we predict over the whole document.
+@layers = "spacy-transformers.get_doc_spans.v1"
[nlp.pipeline.tagger.model]
@architectures = "spacy.Tagger.v1"
@@ -101,11 +95,16 @@ stride = 192
[nlp.pipeline.parser.model]
@architectures = "spacy.TransitionBasedParser.v1"
nr_feature_tokens = 8
-hidden_width = 256
+hidden_width = 128
maxout_pieces = 3
+use_upper = false
[nlp.pipeline.ner.model]
-@architectures = "spacy.BiluoTagger.v1"
+@architectures = "spacy.TransitionBasedParser.v1"
+nr_feature_tokens = 3
+hidden_width = 128
+maxout_pieces = 3
+use_upper = false
# These "listener" layers are connected to the transformer pipeline component
# in order to achieve multi-task learning across the pipeline.
