fix dataclasses import
import copy
+import dataclasses
import logging
from typing import Any, Dict, List, Optional, Tuple, Iterable
@@ -427,9 +428,6 @@ class PretrainedTransformerTokenizer(Tokenizer):
# Make sure we don't change the input parameters
tokens2 = copy.deepcopy(tokens2)
- # We add special tokens and also set token type ids.
- import dataclasses
-
if tokens2 is None:
return (
self.single_sequence_start_tokens
