clean up code in test loader
-import os
import configparser
-
import json
+import os
import unittest
-
from fastNLP.loader.config_loader import ConfigSection, ConfigLoader
from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, POSDatasetLoader, LMDatasetLoader
+
class TestConfigLoader(unittest.TestCase):
def test_case_ConfigLoader(self):
@@ -33,7 +32,7 @@ class TestConfigLoader(unittest.TestCase):
return dict
test_arg = ConfigSection()
- ConfigLoader("config", "").load_config(os.path.join("./test/loader", "config"), {"test": test_arg})
+ ConfigLoader("config").load_config(os.path.join("./test/loader", "config"), {"test": test_arg})
section = read_section_from_config(os.path.join("./test/loader", "config"), "test")
for sec in section:
@@ -54,18 +53,18 @@ class TestConfigLoader(unittest.TestCase):
class TestDatasetLoader(unittest.TestCase):
def test_case_TokenizeDatasetLoader(self):
- loader = TokenizeDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8")
+ loader = TokenizeDatasetLoader("./test/data_for_tests/cws_pku_utf_8")
data = loader.load_pku(max_seq_len=32)
print("pass TokenizeDatasetLoader test!")
def test_case_POSDatasetLoader(self):
- loader = POSDatasetLoader("people", "./test/data_for_tests/people.txt")
+ loader = POSDatasetLoader("./test/data_for_tests/people.txt")
data = loader.load()
datas = loader.load_lines()
print("pass POSDatasetLoader test!")
def test_case_LMDatasetLoader(self):
- loader = LMDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8")
+ loader = LMDatasetLoader("./test/data_for_tests/cws_pku_utf_8")
data = loader.load()
datas = loader.load_lines()
print("pass TokenizeDatasetLoader test!")
\ No newline at end of file
