Verify columns.
@@ -7,6 +7,7 @@ import logging
import os
import pathlib
import numpy as np
+from collections import defaultdict, Counter
from networkml.featurizers.main import Featurizer
@@ -122,11 +123,17 @@ class CSVToFeatures():
featurizer = Featurizer()
rows = featurizer.main(features, rows, features_path)
- header = set()
+ rowcounts = Counter()
for row in rows:
for r in row:
- header.update(r.keys())
- header = list(header)
+ for header_key in r:
+ rowcounts[header_key] += 1
+ rowcompare = defaultdict(set)
+ for header_key, header_count in rowcounts.items():
+ if header_key != 'host_key':
+ rowcompare[header_count].add(header_key)
+ assert len(rowcompare) == 1, 'inconsistent featurizer row counts: %s' % rowcompare
+ header = list(rowcounts.keys())
columns = [np.array(row) for row in rows]
np_array = np.vstack(columns)
