Serialize server management operations
This blocks a given ServiceManager client from performing an
operation or checking server state until pending operations
are finished.
@@ -13,6 +13,7 @@ from psutil import NoSuchProcess
from tornado import gen
from tornado.httpclient import AsyncHTTPClient
from tornado.ioloop import IOLoop, PeriodicCallback
+from tornado.locks import Lock as AsyncLock
from tornado.options import options
from appscale.common.async_retrying import retry_data_watch_coroutine
@@ -113,9 +114,13 @@ class DatastoreServer(Server):
self._stdout = None
self._verbose = verbose
+ # Serializes start, stop, and monitor operations.
+ self._management_lock = AsyncLock()
+
@gen.coroutine
def ensure_running(self):
""" Checks to make sure the server is still running. """
+ with (yield self._management_lock.acquire()):
yield self._wait_for_service(timeout=self.STATUS_TIMEOUT)
@staticmethod
@@ -138,7 +143,8 @@ class DatastoreServer(Server):
@gen.coroutine
def start(self):
""" Starts a new datastore server. """
- if self.state in (ServerStates.STARTING, ServerStates.RUNNING):
+ with (yield self._management_lock.acquire()):
+ if self.state == ServerStates.RUNNING:
return
self.state = ServerStates.STARTING
@@ -155,20 +161,19 @@ class DatastoreServer(Server):
['cgexec', '-g', ':'.join(DATASTORE_CGROUP)] + start_cmd,
stdout=self._stdout, stderr=subprocess.STDOUT)
- # Wait for server to bind to port before making HTTP requests.
- yield gen.sleep(1)
yield self._wait_for_service(timeout=self.START_TIMEOUT)
self.state = ServerStates.RUNNING
@gen.coroutine
def stop(self):
""" Stops an existing datastore server. """
- if self.state in (ServerStates.STOPPING, ServerStates.STOPPED):
+ with (yield self._management_lock.acquire()):
+ if self.state == ServerStates.STOPPED:
return
self.state = ServerStates.STOPPING
try:
- self._cleanup()
+ yield self._cleanup()
finally:
self.state = ServerStates.STOPPED
