Add fail2ban monitor (#12251)

* Add fail2ban monitor

Author: cdrini

compose.production.yaml

@@ -358,6 +358,8 @@ services:
     volumes:
       # Needed to inspect other docker containers
       - /var/run/docker.sock:/var/run/docker.sock
+      # Needed to inspect fail2ban since it runs on the host
+      - /var/run/fail2ban:/var/run/fail2ban
     logging:
       options:
         max-size: "512m"

docker/Dockerfile.olbase

@@ -24,6 +24,8 @@ RUN apt-get -qq update && apt-get install -y \
     libffi-dev \
     curl \
     screen \
+# fail2ban used for monitoring nginx logs and banning abusive IPs
+    fail2ban \
 # Editors (for our convenience)
     vim \
     emacs \

scripts/monitoring/fail2ban_monitor.py

@@ -0,0 +1,20 @@
+from scripts.monitoring.utils import bash_run
+
+
+def get_fail2ban_counts(jail: str) -> tuple[int, int]:
+    """
+    Returns (currently_failed, currently_banned) counts for the given fail2ban jail.
+    """
+    result = bash_run(
+        f"fail2ban-client status {jail} | grep 'Currently'",
+        capture_output=True,
+    )
+    failed = 0
+    banned = 0
+    for line in result.stdout.splitlines():
+        line = line.strip()
+        if "Currently failed:" in line:
+            failed = int(line.split(":")[-1].strip())
+        elif "Currently banned:" in line:
+            banned = int(line.split(":")[-1].strip())
+    return failed, banned

scripts/monitoring/monitor.py

@@ -10,6 +10,7 @@
 
 import httpx
 
+from scripts.monitoring.fail2ban_monitor import get_fail2ban_counts
 from scripts.monitoring.solr_updater_monitor import get_solr_updater_lag_event
 from scripts.monitoring.utils import (
     GraphiteEvent,
@@ -172,6 +173,29 @@ async def monitor_empty_homepage():
         ).submit(GRAPHITE_URL)
 
 
+@limit_server(["ol-www0"], scheduler)
+@scheduler.scheduled_job('interval', seconds=60)
+def monitor_fail2ban():
+    """Logs fail2ban nginx-429 jail stats (currently failed and banned counts)."""
+    failed, banned = get_fail2ban_counts("nginx-429")
+    ts = int(time.time())
+    GraphiteEvent.submit_many(
+        [
+            GraphiteEvent(
+                path="stats.ol.fail2ban.nginx-429.failed",
+                value=float(failed),
+                timestamp=ts,
+            ),
+            GraphiteEvent(
+                path="stats.ol.fail2ban.nginx-429.banned",
+                value=float(banned),
+                timestamp=ts,
+            ),
+        ],
+        GRAPHITE_URL,
+    )
+
+
 @limit_server(["ol-home0"], scheduler)
 @scheduler.scheduled_job('interval', seconds=60)
 async def monitor_solr_updater_lag():

scripts/monitoring/tests/test_fail2ban_monitor.py

@@ -0,0 +1,26 @@
+from unittest.mock import MagicMock, patch
+
+from scripts.monitoring.fail2ban_monitor import get_fail2ban_counts
+
+FAKE_FAIL2BAN_OUTPUT = """
+Status for the jail: nginx-429
+|- Filter
+|  |- Currently failed: 193
+|  |- Total failed:     56440304
+|  `- File list:        /1/var/log/nginx/error.log
+`- Actions
+   |- Currently banned: 141
+   |- Total banned:     661976
+   `- Banned IP list:   08.07.04.02 04.06.02.09
+"""
+
+
+def test_get_fail2ban_counts():
+    mock_result = MagicMock()
+    mock_result.stdout = FAKE_FAIL2BAN_OUTPUT
+
+    with patch("scripts.monitoring.fail2ban_monitor.bash_run", return_value=mock_result):
+        failed, banned = get_fail2ban_counts("nginx-429")
+
+    assert failed == 193
+    assert banned == 141