RJQ: change reader for attribute_masking_processor test
import unittest
from forte.data.data_pack import DataPack
-from forte.data.readers import CoNLL03Reader
+from forte.data.readers import ConllUDReader
from forte.pipeline import Pipeline
from forte.processors import AttributeMasker
from ft.onto.base_ontology import Token
@@ -12,27 +12,27 @@ class TestAttributeMaskingProcessor(unittest.TestCase):
def test_without_attribute_masker(self):
pl = Pipeline[DataPack]()
- pl.set_reader(CoNLL03Reader())
+ pl.set_reader(ConllUDReader())
pl.initialize()
- for pack in pl.process_dataset("data_samples/conll03/"):
+ for pack in pl.process_dataset("data_samples/conll_ud/"):
entries = pack.get(Token)
for entry in entries:
- self.assertIsNotNone(entry.ner)
+ self.assertIsNotNone(entry.pos)
def test_attribute_masker(self):
pl = Pipeline[DataPack]()
- pl.set_reader(CoNLL03Reader())
+ pl.set_reader(ConllUDReader())
config = {
"kwargs": {
- Token: ["ner"]
+ Token: ["pos"]
}
}
pl.add(component=AttributeMasker(), config=config)
pl.initialize()
- for pack in pl.process_dataset("data_samples/conll03/"):
+ for pack in pl.process_dataset("data_samples/conll_ud/"):
entries = pack.get(Token)
for entry in entries:
- self.assertIsNone(entry.ner)
+ self.assertIsNone(entry.pos)
