Adding a basic metrics service and some metrics to track

Author: matburt

Pipfile

@@ -12,6 +12,7 @@ flake8 = "*"
 
 [packages]
 python-dateutil = "*"
+prometheus-client = "*"
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -5,6 +5,8 @@
 import os
 import ssl
 
+from prometheus_client import Counter, Gauge
+
 from .entrypoints import run_as_node, run_as_controller, run_as_ping, run_as_send
 from .exceptions import ReceptorRuntimeError, ReceptorConfigError
 
@@ -30,6 +32,12 @@
     },
 }
 
+messages_received_counter = Counter("incoming_messages", "Messages received from Receptor Peers")
+connected_peers_guage = Gauge("connected_peers", "Number of active peer connections")
+work_counter = Counter("work_events", "A count of the number of work events that have been received")
+active_work_gauge = Gauge("active_work", "Amount of work currently being performed")
+route_counter = Counter("route_events", "A count of the number of messages that have been routed elsewhere in the mesh")
+
 
 def py_class(class_spec):
     if class_spec not in SINGLETONS:
@@ -137,6 +145,21 @@ def __init__(self, args=None):
             value_type='bool',
             hint='Disable the server function and only connect to configured peers',
         )
+        self.add_config_option(
+            section='node',
+            key='stats_enable',
+            default_value=None,
+            set_value=True,
+            value_type='bool',
+            hint="Enable Prometheus style stats port",
+        )
+        self.add_config_option(
+            section='node',
+            key='stats_port',
+            default_value=8889,
+            value_type='int',
+            hint='Port to listen for requests to show stats',
+        )
         self.add_config_option(
             section='node',
             key='ping_interval',
@@ -192,6 +215,21 @@ def __init__(self, args=None):
             value_type='path',
             hint='Path to control socket for controller commands.',
         )
+        self.add_config_option(
+            section='controller',
+            key='stats_enable',
+            default_value=None,
+            set_value=True,
+            value_type='bool',
+            hint="Enable Prometheus style stats port",
+        )
+        self.add_config_option(
+            section='controller',
+            key='stats_port',
+            default_value=8889,
+            value_type='int',
+            hint='Port to listen for requests to show stats',
+        )
         self.add_config_option(
             section='ping',
             key='count',

receptor/config.py

@@ -4,6 +4,8 @@
 import sys
 import time
 
+from prometheus_client import start_http_server
+
 from .receptor import Receptor
 from . import controller
 from . import exceptions
@@ -15,12 +17,18 @@
 def run_as_controller(config):
     receptor = Receptor(config)
     logger.info(f'Starting up as node ID {receptor.node_id}')
+    if config.controller_stats_enable:
+        logger.info(f'Starting stats on port {config.controller_stats_port}')
+        start_http_server(config.controller_stats_port)
     controller.mainloop(receptor, config.controller_socket_path)
 
 
 def run_as_node(config):
     receptor = Receptor(config)
     logger.info(f'Running as Receptor node with ID: {receptor.node_id}')
+    if config.node_stats_enable:
+        logger.info(f'Starting stats on port {config.node_stats_port}')
+        start_http_server(config.node_stats_port)
     node.mainloop(receptor, config.node_ping_interval)
 
 

receptor/entrypoints.py

@@ -10,6 +10,7 @@
 
 from .messages import envelope
 from .exceptions import ReceptorBufferError
+from .config import connected_peers_guage
 
 logger = logging.getLogger(__name__)
 
@@ -74,9 +75,11 @@ def connection_made(self, transport):
         self.transport = transport
         self.greeted = False
         self.incoming_buffer = DataBuffer()
+        connected_peers_guage.inc()
         self.loop.create_task(self.wait_greeting())
 
     def connection_lost(self, exc):
+        connected_peers_guage.dec()
         self.receptor.remove_connection(self)
 
     def data_received(self, data):
@@ -171,10 +174,12 @@ def __init__(self, receptor, loop):
 
     def connection_made(self, transport):
         self.transport = transport
+        connected_peers_guage.inc()
         if self not in self.receptor.controller_connections:
             self.receptor.controller_connections.append(self)
 
     def connection_lost(self, exc):
+        connected_peers_guage.dec()
         if self in self.receptor.controller_connections:
             self.receptor.controller_connections.remove(self)
 

receptor/protocol.py

@@ -19,6 +19,7 @@
     long_description_content_type='text/markdown',
     python_requires=">=3.6",
     install_requires=[
+        "prometheus_client==0.7.1",
     ],
     zip_safe=False,
     entry_points={