Add more explanation for data format; Make `initialize` as the first method; Move code related to doc to resource managment block;
@@ -42,11 +42,23 @@ class OpenIEReader(PackReader):
The related source code for generating this dataset can be found
`here
<https://github.com/gabrielStanovsky/oie-benchmark>`__.
- To use this Reader, you must follow the dataset format described
- `here
+ To use this Reader, you must follow the dataset format. Each line in
+ the dataset should contain following fields:
+
+ .. code-block:: none
+
+ <sentence>\t<predicate_head>\t<full_predicate>\t<arg1>\t<arg2>....
+
+ You can also find the dataset format `here
<https://github.com/gabrielStanovsky/oie-benchmark/tree/master/oie_corpus>`__.
"""
+ def initialize(self, resources: Resources, configs: Config):
+ super().initialize(resources, configs)
+
+ if configs.oie_file_extension is None:
+ raise ProcessorConfigError("OIE dataset file extension not found")
+
def _collect(self, *args, **kwargs) -> Iterator[Any]:
# pylint: disable = unused-argument
r"""Should be called with param ``oie_directory`` which is a path to a
@@ -69,13 +81,11 @@ class OpenIEReader(PackReader):
def _parse_pack(self, file_path: str) -> Iterator[DataPack]:
pack = self.new_pack()
- with open(file_path, "r", encoding="utf8") as f:
- doc = f.readlines()
-
text = ""
offset = 0
- for line in doc:
+ with open(file_path, "r", encoding="utf8") as f:
+ for line in f:
line = line.strip()
if line != "":
oie_component = line.split("\t")
@@ -91,7 +101,8 @@ class OpenIEReader(PackReader):
# Add predicate.
predicate_mention = PredicateMention(pack,
offset,
- offset + len(predicate))
+ offset
+ + len(predicate))
offset += len(predicate) + 1
text += predicate + " "
@@ -123,9 +134,3 @@ class OpenIEReader(PackReader):
'oie_file_extension': 'oie'
})
return config
-
- def initialize(self, resources: Resources, configs: Config):
- super().initialize(resources, configs)
-
- if configs.oie_file_extension is None:
- raise ProcessorConfigError("OIE dataset file extension not found")
