competitve with pgsql
@@ -14,6 +14,7 @@ import multiprocessing
import random
import warnings
import os
+import tempfile
from collections import defaultdict, OrderedDict
import shelve
@@ -255,32 +256,43 @@ class DedupeMatching(Matching):
def _blockData(self, data_d):
- blocks = shelve.open('foo', 'n')
+ blocks, file_path = tempfile.mkstemp()
+ os.close(blocks)
+
+ blocks = shelve.open(file_path, 'n',
+ protocol=pickle.HIGHEST_PROTOCOL)
if not self.loaded_indices:
self.blocker.indexAll(data_d)
- block_numbers = core.Enumerator(start=0)
block_groups = itertools.groupby(self.blocker(viewitems(data_d)),
lambda x: x[1])
for record_id, block in block_groups:
record = data_d[record_id]
- block_ids = sorted(block_numbers[block_key]
- for block_key, _ in block)
+ block_ids = sorted(block_key for block_key, _ in block)
while block_ids:
- id = str(block_ids.pop())
+ id = block_ids.pop()
if id in blocks:
blocks[id] += [(record_id, record, set(block_ids))]
else:
blocks[id] = [(record_id, record, set(block_ids))]
+ if not self.loaded_indices:
+ self.blocker.resetIndices()
+
+ blocks.close()
+ blocks = shelve.open(file_path, 'r',
+ protocol=pickle.HIGHEST_PROTOCOL)
+
for block in viewvalues(blocks):
if len(block) > 1:
yield block
blocks.close()
+ os.remove(file_path)
+
def _checkBlock(self, block):
if block:
