Get cls and sep markets in sentence wp tokens
@@ -146,6 +146,8 @@ def get_defaults(lang):
def get_wp_start(span):
+ if isinstance(span, Token):
+ span = span.doc[span.i:span.i+1]
for token in span:
if token._.pytt_alignment:
wp_start = token._.pytt_alignment[0]
@@ -160,6 +162,8 @@ def get_wp_start(span):
def get_wp_end(span):
+ if isinstance(span, Token):
+ span = span.doc[span.i:span.i+1]
for token in reversed(span):
if token._.pytt_alignment:
wp_end = token._.pytt_alignment[-1]
@@ -175,9 +179,18 @@ def get_wp_end(span):
def get_span_wp_getter(attr):
- def span_getter(span):
+ def span_alignment_getter(span):
return [token._.get(attr) for token in span]
+ def span_getter(span):
+ start = span._.pytt_start
+ end = span._.pytt_end
+ doc_values = span.doc._.get(attr)
+ return doc_values[start:end+1]
+
+ if attr == "pytt_alignment":
+ return span_alignment_getter
+ else:
return span_getter
@@ -188,8 +201,9 @@ def get_token_wp_getter(attr):
def token_wordpiece_getter(token):
doc_values = token.doc._.get(attr)
- indices = token._.pytt_alignment
- return [doc_values[i] for i in indices]
+ start = token._.pytt_start
+ end = token._.pytt_end
+ return [doc_values[i] for i in range(start, end+1)]
if attr == "pytt_alignment":
return token_alignment_getter
@@ -202,10 +216,10 @@ def get_span_tok2vec_getter(attr):
doc_activations = span.doc._.get(attr)
if doc_activations is None:
return None
- wp_start = span[0]._.pytt_alignment[0]
- wp_end = span[-1]._.pytt_alignment[-1]
+ wp_start = span[0]._.pytt_wp_start
+ wp_end = span[-1]._.pytt_wp_end
if wp_start is not None and wp_end is not None:
- return doc_activations[wp_start:wp_end]
+ return doc_activations[wp_start:wp_end+1]
else:
# Return empty slice.
return doc_activations[0:0]
