rate limit the canary
+import time
+
import numpy as np
import tensorflow as tf
from cleverhans.utils_tf import infer_devices
+last_run = None
+
+
def run_canary():
"""
Runs some code that will crash if the GPUs / GPU driver are suffering from
@@ -14,6 +19,14 @@ def run_canary():
# crashes after your edits. Due to the transient nature of the GPU bug it is
# not possible to unit test the canary in our continuous integration system.
+ global last_run
+ current = time.time()
+ if last_run is None or current - last_run > 3600:
+ last_run = current
+ else:
+ # Run the canary at most once per hour
+ return
+
# Try very hard not to let the canary affect the graph for the rest of the
# python process
canary_graph = tf.Graph()
