refactor(tx): TransactionJSONStorage.get_all_transactions using scandir
instead of listdir
Benchmark MacOS to read a directory with 205,828 transactions:
Before
CPU times: user 38 s, sys: 19.2 s, total: 57.2 s
Wall time: 2min 8s
After
CPU times: user 22.8 s, sys: 12.2 s, total: 35 s
Wall time: 1min 29s
@@ -196,14 +196,16 @@ class TransactionJSONStorage(TransactionStorage):
yield tx
path = self.path
- files = os.listdir(path)
pattern = r'tx_[\dabcdef]{64}\.json'
+ re_pattern = re.compile(pattern)
- for f in files:
- if re.match(pattern, f):
+ with os.scandir(path) as it:
+ for f in it:
+ if re_pattern.match(f.name):
# TODO Return a proxy that will load the transaction only when it is used.
- hash_hex = f[3:-5]
- transaction = self.get_transaction_by_hash(hash_hex)
+ with open(f.path, 'r') as json_file:
+ dict_data = json.loads(json_file.read())
+ transaction = self.load(dict_data)
yield transaction
def get_count_tx_blocks(self):
