separate 'retry' status for launches that fail but will be retried
so 'failure' status only indicates user-facing failures
@@ -445,12 +445,18 @@ class BuildHandler(BaseHandler):
server_info = await launcher.launch(image=self.image_name, username=username)
LAUNCH_TIME.labels(status='success', retries=i, **self.metric_labels).observe(time.perf_counter() - launch_starttime)
except Exception as e:
- LAUNCH_TIME.labels(status='failure', retries=i, **self.metric_labels).observe(time.perf_counter() - launch_starttime)
+ if i + 1 == launcher.retries:
+ status = 'failure'
+ else:
+ status = 'retry'
+ LAUNCH_TIME.labels(status=status, retries=i, **self.metric_labels).observe(time.perf_counter() - launch_starttime)
+
if i + 1 == launcher.retries:
# last attempt failed, let it raise
raise
- else:
- app_log.error("Retrying launch after failure: %s", e)
+
+ # not the last attempt, try again
+ app_log.error("Retrying launch after error: %s", e)
await self.emit({
'phase': 'launching',
'message': 'Launch attempt {} failed, retrying...\n'.format(i + 1),
