Fix alignment if tokens excluded from spans
@@ -52,8 +52,12 @@ def apply_alignment(ops: Ops, align: Ragged, X: Floats2d) -> Tuple[Ragged, Calla
def get_token_positions(spans: List[Span]) -> Dict[Tuple[Token, int], int]:
token_positions: Dict[Token, int] = {}
+ seen_docs = set()
for span in spans:
- for token in span:
+ if span.doc in seen_docs:
+ continue
+ seen_docs.add(span.doc)
+ for token in span.doc:
if token not in token_positions:
token_positions[token] = len(token_positions)
return token_positions
