replaced h5diff with John's updated version
import h5py
-from numpy import isclose
+from numpy import isclose, ndarray, issubdtype, floating
class H5Diff:
@@ -16,11 +16,14 @@ class H5Diff:
test_object = obj
ref_object = self.ref_file.get(name)
- assert ref_object is not None
+ assert ref_object is not None, f"Element {ref_object} did not exist in test file"
# If it's a dataset, compare the actual contents
if type(test_object) == h5py._hl.dataset.Dataset:
+ if name in self.excluded_datasets:
+ return None
+
# These are the names of the dtypes corresponding to some h5py Reference objects.
# Reference objects are pointers to other parts of the h5 file
# I'm not sure if it's meaningful to directly compare them, I think comparing all the "actual" data
@@ -37,7 +40,6 @@ class H5Diff:
x for i, x in enumerate(test_object[()][0]) if not test_object[()].dtype.names[i] in ref_names
]
- print(ref_object[()])
non_reference_ref_elements = [
x for i, x in enumerate(ref_object[()][0]) if not ref_object[()].dtype.names[i] in ref_names
]
@@ -50,16 +52,24 @@ class H5Diff:
# things being compared are arrays it becomes an element-wise comparison.
# If the latter is the case, then we need to ensure ALL elements are true.
- # TODO: This may be checking numbers which differ up to floating point accuracy
comparison = non_reference_test_elements == non_reference_ref_elements
- if non_reference_ref_elements.dtype == 'float64':
- comparison = isclose(non_reference_test_elements, non_reference_ref_elements)
+ # If you're comparing to datasets that are arrays of floats, then they may differ up to floating point precision.
+ # So, use numpy's 'isclose' to check if they're close within the default tolerance of 1e-8.
+ if type(non_reference_ref_elements) is ndarray and issubdtype(non_reference_ref_elements.dtype, floating):
+ comparison = isclose(non_reference_test_elements, non_reference_ref_elements, atol=self.float_thresh)
if type(comparison) == bool:
- assert non_reference_test_elements == non_reference_ref_elements
+ assert non_reference_test_elements == non_reference_ref_elements, f"Elements didn't match in {name}"
else:
- assert comparison.all()
+ try:
+ assert comparison.all(), f"Elements didn't match in {name}"
+
+ except AssertionError as e:
+ print(non_reference_ref_elements)
+ print(non_reference_test_elements)
+ print(non_reference_ref_elements[~comparison] - non_reference_test_elements[~comparison])
+ raise e
# If it's a group, do nothing
# TODO: Is it sufficient to check only the datasets? The groups should just be organizational units
@@ -78,14 +88,16 @@ class H5Diff:
# for flipping the logic.
# But if it's flipped, then if there's extra junk in the test file, that wouldn't be revealed.
# This works for the time being, and as an **extremely** janky hack fix, you could do it both ways...
- def check(self):
+ def check(self, float_thresh=1e-8, excluded_datasets=[]):
+ self.float_thresh = float_thresh
+ self.excluded_datasets = excluded_datasets
self.test_file.visititems(self.check_in_ref)
# TODO: Add some simple argument handler here
-# if __name__ == "__main__":
-#
-# diff = H5Diff("./assign_ref.h5", "./analysis_tocompare.h5")
-#
-# diff.check()
+if __name__ == "__main__":
+
+ diff = H5Diff("./assign_ref.h5", "./analysis_tocompare.h5")
+
+ diff.check()
