Use a fast read path when possible.
@@ -505,6 +505,9 @@ class KgtkReader(KgtkBase, ClosableIter[typing.List[str]]):
is_edge_file: bool = attr.ib(validator=attr.validators.instance_of(bool), default=False)
is_node_file: bool = attr.ib(validator=attr.validators.instance_of(bool), default=False)
+ # Use the fast reading path?
+ use_fast_path: bool = attr.ib(validator=attr.validators.instance_of(bool), default=False)
+
# Reject file
reject_file: typing.Optional[typing.TextIO] = attr.ib(default=None)
reject_line_count: int = attr.ib(validator=attr.validators.instance_of(int), default=0)
@@ -627,6 +630,29 @@ class KgtkReader(KgtkBase, ClosableIter[typing.List[str]]):
label_column_idx,
node2_column_idx,
id_column_idx), file=error_file, flush=True)
+ # Decide whether or not to use the fast read path. This code
+ # assumes that the options will not change beyond this point.
+ input_format: str
+ if options.input_format is None:
+ input_format = KgtkReaderOptions.INPUT_FORMAT_KGTK
+ else:
+ input_format = options.input_format
+
+ use_fast_path: bool
+ if options.record_limit is None and \
+ options.tail_count is None and \
+ options.initial_skip_count == 0 and \
+ options.every_nth_record <= 1 and \
+ not options.repair_and_validate_lines and \
+ not options.repair_and_validate_values and \
+ input_format != KgtkReaderOptions.INPUT_FORMAT_CSV and \
+ options.implied_label is None:
+ use_fast_path = True
+ if verbose:
+ print("KgtkReader: using the fast read path.", file=error_file, flush=True)
+ else:
+ use_fast_path = False
+
if is_edge_file:
# We'll instantiate an EdgeReader, which is a subclass of KgtkReader.
# The EdgeReader import is deferred to avoid circular imports.
@@ -664,6 +690,7 @@ class KgtkReader(KgtkBase, ClosableIter[typing.List[str]]):
value_options=value_options,
is_edge_file=is_edge_file,
is_node_file=is_node_file,
+ use_fast_path=use_fast_path,
verbose=verbose,
very_verbose=very_verbose,
)
@@ -923,6 +950,29 @@ class KgtkReader(KgtkBase, ClosableIter[typing.List[str]]):
def nextrow(self)-> typing.List[str]:
row: typing.List[str]
+ if self.use_fast_path:
+ # This is the fast read path.
+
+ # Read a line from the source
+ line: str
+ try:
+
+ line = next(self.source) # Will throw StopIteration
+ except StopIteration as e:
+ # Close the input file!
+ #
+ # TODO: implement a close() routine and/or whatever it takes to support "with".
+ self.source.close() # Do we need to guard against repeating this call?
+ raise e
+
+ # Count the data line read and passed on:
+ self.data_lines_read += 1
+ self.data_lines_passed += 1
+
+ # Strip the end-of-line characters and split the fields:
+ return line.rstrip("\r\n").split(self.options.column_separator)
+
+ else:
repair_and_validate_lines: bool = self.options.repair_and_validate_lines
repair_and_validate_values: bool = self.options.repair_and_validate_values
