Fix another memory leak
import logging as log
import time
import multiprocessing
+import uuid
from psycopg2.extras import DictCursor, Json
from ingestion_server.indexer import database_connect
from urllib.parse import urlparse
@@ -168,7 +169,8 @@ def clean_data(table):
)
log.info('Running cleanup on selection "{}"'.format(cleanup_selection))
conn = database_connect()
- iter_cur = conn.cursor(cursor_factory=DictCursor)
+ cursor_name = '{}-{}'.format(table, str(uuid.uuid4()))
+ with conn.cursor(name=cursor_name, cursor_factory=DictCursor) as iter_cur:
iter_cur.itersize = CLEANUP_BUFFER_SIZE
iter_cur.execute(cleanup_selection)
@@ -205,7 +207,6 @@ def clean_data(table):
.format(num_cleaned))
jobs = []
batch = iter_cur.fetchmany(size=CLEANUP_BUFFER_SIZE)
- iter_cur.close()
end_time = time.time()
cleanup_time = end_time - start_time
log.info('Cleaned all records in {} seconds'.format(
