Update instrumentation middleware so we correctly track timing info for
stream requests.
Previously we didn't correctly record timing for long running stream
requests (middleware returns immediately and not at the end of
response).
@@ -22,6 +22,7 @@ from webob import Request
from st2common.metrics.base import CounterWithTimer
from st2common.metrics.base import get_driver
+from st2common.util.date import get_datetime_utc_now
class RequestInstrumentationMiddleware(object):
@@ -62,15 +63,35 @@ class RequestInstrumentationMiddleware(object):
# correctly set the counter when the connection is closed / full response is returned.
# See http://eventlet.net/doc/modules/wsgi.html#non-standard-extension-to-support-post-
# hooks for details
+
+ # Increase request counter
+ key = '%s.request' % (self._service_name)
+ metrics_driver.inc_counter(key)
+
+ # Increase "total number of connections" gauge
metrics_driver.inc_gauge('stream.connections', 1)
+ start_time = get_datetime_utc_now()
+
def hook(env):
+ # Hook which is called at the very end after all the response has been sent and
+ # connection closed
+ time_delta = (get_datetime_utc_now() - start_time)
+ duration = time_delta.total_seconds()
+
+ # Send total request time
+ metrics_driver.time(key, duration)
+
+ # Decrease "current number of connections" gauge
metrics_driver.dec_gauge('stream.connections', 1)
environ['eventlet.posthooks'].append((hook, (), {}))
+ return self.app(environ, start_response)
+ else:
# Track and time current number of processing requests
key = '%s.request' % (self._service_name)
+
with CounterWithTimer(key=key):
return self.app(environ, start_response)
