Restart watching on the 410 error (resource version too old)
@@ -74,20 +74,29 @@ async def watcher(
scheduler = await aiojobs.create_scheduler(limit=10)
queues = {}
try:
+ while True:
# Make a Kubernetes call to watch for the events via the API.
w = kubernetes.watch.Watch()
api = kubernetes.client.CustomObjectsApi()
- stream = w.stream(api.list_cluster_custom_object, resource.group, resource.version, resource.plural)
+ api_fn = api.list_cluster_custom_object
+ stream = w.stream(api_fn, resource.group, resource.version, resource.plural)
async for event in _async_wrapper(stream):
+ # "410 Gone" is for the "resource version too old" error, we must restart watching.
+ # The resource versions are lost by k8s after few minutes (as per the official doc).
+ # The error occurs when there is nothing happening for few minutes. This is normal.
+ if event['type'] == 'ERROR' and event['object']['code'] == 410:
+ logger.debug("Restarting the watch-stream for %r", resource)
+ break # out of for-cycle, to the while-true-cycle.
+
# Other watch errors should be fatal for the operator.
if event['type'] == 'ERROR':
raise Exception(f"Error in the watch-stream: {event['object']}")
# Ensure that the event is something we understand and can handle.
if event['type'] not in ['ADDED', 'MODIFIED', 'DELETED']:
- logger.warn("Unsupported event type received, ignoring: %r", event)
+ logger.warn("Ignoring an unsupported event type: %r", event)
continue
# Filter out all unrelated events as soon as possible (before queues), and silently.
