uncordon nodes at start of image-cleaner
check if node is still cordoned from a previous run of image-cleaner that crashed,
uncordon if so
move cordon logging to cordon/uncordon functions
uncordon before crashing
@@ -20,6 +20,9 @@ logging.basicConfig(format='%(asctime)s %(message)s',
level=logging.INFO)
+annotation_key = "hub.jupyter.org/image-cleaner-cordoned"
+
+
def get_absolute_size(path):
"""
Directory size in bytes
@@ -91,9 +94,16 @@ def get_docker_images(client):
def cordon(kube, node):
"""cordon a kubernetes node"""
+ logging.info(f"Cordoning node {node}")
kube.patch_node(
node,
{
+ # record that we are the one responsible for cordoning
+ "metadata": {
+ "annotations": {
+ annotation_key: "true",
+ },
+ },
"spec": {
"unschedulable": True,
},
@@ -103,9 +113,17 @@ def cordon(kube, node):
def uncordon(kube, node):
"""uncordon a kubernetes node"""
+ logging.info(f"Uncordoning node {node}")
kube.patch_node(
node,
{
+ # clear annotation since we're no longer the reason for cordoning,
+ # if the node ever does become cordoned
+ "metadata": {
+ "annotations": {
+ annotation_key: None,
+ },
+ },
"spec": {
"unschedulable": False,
},
@@ -124,7 +142,12 @@ def main():
kubernetes.config.load_kube_config()
kube = kubernetes.client.CoreV1Api()
# verify that we can talk to the node
- kube.read_node(node)
+ node_info = kube.read_node(node)
+ # recover from possible crash!
+ if node_info.spec.unschedulable and node_info.metadata.annotations.get(annotation_key):
+ logging.warning(f"Node {node} still cordoned, possibly leftover from earlier crash of image-cleaner")
+ uncordon(kube, node)
+
path_to_check = os.getenv('PATH_TO_CHECK', '/var/lib/docker')
interval = float(os.getenv('IMAGE_GC_INTERVAL', '300'))
@@ -169,7 +192,6 @@ def main():
images_before = len(images)
if node:
- logging.info(f"Cordoning node {node}")
cordon(kube, node)
deleted = 0
@@ -177,7 +199,6 @@ def main():
while images and get_used(path_to_check) > gc_low:
# Ensure the node is still cordoned
if node:
- logging.info(f"Cordoning node {node}")
cordon(kube, node)
# Remove biggest image
image = images.pop(0)
@@ -203,16 +224,23 @@ def main():
elif e.status_code == 404:
logging.info(f'{name} not found, probably already deleted')
else:
+ if node:
+ # uncordon before giving up
+ uncordon(kube, node)
raise
except requests.exceptions.ReadTimeout:
logging.warning(f'Timeout removing {name}')
# Delay longer after a timeout, which indicates that Docker is overworked
time.sleep(max(delay, 30))
+ except Exception:
+ if node:
+ # uncordon before giving up
+ uncordon(kube, node)
+ raise
else:
deleted += 1
if node:
- logging.info(f"Uncordoning node {node}")
uncordon(kube, node)
# log what we did and how long it took
