Core should take action when a microservice is observed to be down
@@ -15,8 +15,9 @@ from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.services.core.service_registry.service_registry import ServiceRegistry
from foglamp.common.service_record import ServiceRecord
from foglamp.services.core import connect
+from foglamp.services.core import server
-__author__ = "Ashwin Gopalakrishnan"
+__author__ = "Ashwin Gopalakrishnan, Amarendra K Sinha"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"
@@ -32,10 +33,13 @@ class Monitor(object):
_DEFAULT_MAX_ATTEMPTS = 15
+ _DEFAULT_RESTART_FAILED = "auto"
+ """Restart failed microservice - manual/auto"""
+
_logger = None
def __init__(self):
- self._logger = logger.setup(__name__, level=20)
+ self._logger = logger.setup(__name__)
self._monitor_loop_task = None # type: asyncio.Task
"""Task for :meth:`_monitor_loop`, to ensure it has finished"""
@@ -45,6 +49,10 @@ class Monitor(object):
"""Timeout for a response from any given micro-service"""
self._max_attempts = None # type: int
"""Number of max attempts for finding a heartbeat of service"""
+ self._restart_failed = None # type: str
+ """Restart failed microservice - manual/auto"""
+
+ self.restarted_services = []
async def _sleep(self, sleep_time):
await asyncio.sleep(sleep_time)
@@ -63,9 +71,22 @@ class Monitor(object):
for service_record in ServiceRegistry.all():
if service_record._id not in check_count:
check_count.update({service_record._id: 1})
+
# Try ping if service status is either running or doubtful (i.e. give service a chance to recover)
- if service_record._status not in [ServiceRecord.Status.Running, ServiceRecord.Status.Unresponsive]:
+ if service_record._status not in [ServiceRecord.Status.Running,
+ ServiceRecord.Status.Unresponsive,
+ ServiceRecord.Status.Failed]:
continue
+
+ self._logger.debug("Service: {} Status: {}".format(service_record._name, service_record._status))
+
+ if service_record._status == ServiceRecord.Status.Failed:
+ if self._restart_failed == "auto":
+ if service_record._id not in self.restarted_services:
+ self.restarted_services.append(service_record._id)
+ asyncio.ensure_future(self.restart_service(service_record))
+ continue
+
try:
url = "{}://{}:{}/foglamp/service/ping".format(
service_record._protocol, service_record._address, service_record._management_port)
@@ -121,6 +142,11 @@ class Monitor(object):
"type": "integer",
"default": str(self._DEFAULT_MAX_ATTEMPTS)
},
+ "restart_failed": {
+ "description": "Restart failed microservice - manual/auto",
+ "type": "string",
+ "default": self._DEFAULT_RESTART_FAILED
+ }
}
storage_client = connect.get_storage_async()
@@ -132,6 +158,12 @@ class Monitor(object):
self._sleep_interval = int(config['sleep_interval']['value'])
self._ping_timeout = int(config['ping_timeout']['value'])
self._max_attempts = int(config['max_attempts']['value'])
+ self._restart_failed = config['restart_failed']['value']
+
+ async def restart_service(self, service_record):
+ schedule = await server.Server.scheduler.get_schedule_by_name(service_record._name)
+ await server.Server.scheduler.queue_task(schedule.schedule_id)
+ self.restarted_services.remove(service_record._id)
async def start(self):
await self._read_config()
