remove duplicate rows
@@ -420,6 +420,7 @@ class Parser():
record.append(fields_lookup[field.lower()][0](endpoint))
# remove rows that are all zero or 'NO DATA'
if not nonzero or not all(item == '0' or item == 'NO DATA' for item in record):
+ if not (unique and record in records):
records.append(record)
# remove columns that are all zero or 'NO DATA'
