Fix abort sometimes not working, fixes
@@ -742,17 +742,11 @@ class Scheduler(object):
cursor = self.conn.cursor()
cursor.execute('''
- WITH all_aborts AS (
- DELETE FROM "abort" RETURNING job_id, user_id
- ), jobs_to_abort AS (
- SELECT j.id, j.state, user_id FROM all_aborts
+ SELECT j.id, a.user_id
+ FROM abort a
JOIN job j
- ON all_aborts.job_id = j.id
- AND j.state not in ('finished', 'failure', 'error', 'killed')
- AND cluster_name = %s
- )
-
- SELECT id, user_id FROM jobs_to_abort WHERE state in ('scheduled', 'running', 'queued')
+ ON a.job_id = j.id
+ AND j.cluster_name = %s
''', [cluster_name])
aborts = cursor.fetchall()
@@ -761,6 +755,7 @@ class Scheduler(object):
for abort in aborts:
job_id = abort[0]
user_id = abort[1]
+
self.kube_delete_job(job_id)
self.upload_console(job_id)
@@ -788,6 +783,14 @@ class Scheduler(object):
""", [message, job_id])
cursor.close()
+ # Forget abort
+ cursor = self.conn.cursor()
+ cursor.execute('''
+ DELETE FROM "abort"
+ WHERE job_id = %s
+ ''', [job_id])
+ cursor.close()
+
def handle_timeouts(self):
cursor = self.conn.cursor()
cursor.execute('''
