Update namespace monitor with exception handling
@@ -3,6 +3,7 @@ import time
from kubernetes.client.rest import ApiException
from django.conf import settings
+from django.db import InterfaceError, ProgrammingError, OperationalError
from clusters.models import Cluster
from events.management.commands._base_monitor import BaseMonitorCommand
@@ -13,6 +14,18 @@ from polyaxon_k8s.manager import K8SManager
class Command(BaseMonitorCommand):
help = 'Watch namespace warning and errors events.'
+ def get_cluster_or_wait(self, log_sleep_interval):
+ max_trials = 10
+ trials = 0
+ while trials < max_trials:
+ try:
+ return Cluster.load()
+ except (InterfaceError, ProgrammingError, OperationalError) as e:
+ namespace.logger.exception("Database is not synced yet %s\n", e)
+ trials += 1
+ time.sleep(log_sleep_interval * 2)
+ return None
+
def handle(self, *args, **options):
log_sleep_interval = options['log_sleep_interval']
self.stdout.write(
@@ -20,7 +33,11 @@ class Command(BaseMonitorCommand):
"log sleep interval: `{}`.".format(log_sleep_interval),
ending='\n')
k8s_manager = K8SManager(namespace=settings.K8S_NAMESPACE, in_cluster=True)
- cluster = Cluster.load()
+ cluster = self.get_cluster_or_wait(log_sleep_interval)
+ if not cluster:
+ # End process
+ return
+
while True:
try:
namespace.run(k8s_manager, cluster)
