feat: support for monitor external api/file added

Author: codeperfectplus

app.py

@@ -1,13 +1,14 @@
 from src.config import app
 from src import routes
-from src.background_task import start_website_monitoring, monitor_settings
+from src.background_task import start_website_monitoring, monitor_settings, fetch_file_metrics_task
 import os
 
 # background thread to monitor system settings changes
 print("FLASK_ENV: ", os.getenv('FLASK_ENV'))
-if os.getenv('FLASK_ENV') == 'production':
-    monitor_settings()  # Starts monitoring for system logging changes
-    start_website_monitoring()  # Starts pinging active websites
+# if os.getenv('FLASK_ENV') == 'production':
+start_website_monitoring()  # Starts pinging active websites
+fetch_file_metrics_task()
+monitor_settings()  # Starts monitoring for system logging changes
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=5000, debug=True)

src/background_task/__init__.py

@@ -1,4 +1,5 @@
 from src.background_task.monitor_website import start_website_monitoring
 from src.background_task.log_system_info import monitor_settings
+from src.background_task.prometheus_helper import fetch_file_metrics_task
 
-__all__ = ["start_website_monitoring", "monitor_settings"]
+__all__ = ["start_website_monitoring", "monitor_settings", "fetch_file_metrics_task"]

src/background_task/prometheus_helper.py

@@ -0,0 +1,52 @@
+from flask import Flask, Response
+from prometheus_client import Gauge, generate_latest
+import os
+import time
+from src.models import ExternalMonitornig
+from src.config import app
+import threading
+from src.logger import logger
+
+# Define a Gauge metric with a label 'key' to store multiple values from the file
+file_metric = Gauge('external_metrics', 'Value read from file for each key', ['key'])
+
+def read_file_and_update_metric(file_path: str) -> None:    
+    """Reads a file and updates metrics based on its content."""
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as file:
+            for line in file:
+                try:
+                    key, value = line.strip().split(':')
+                    file_metric.labels(key.strip()).set(float(value.strip()))
+                except ValueError as ve:
+                    logger.error(f"Value error processing line '{line}': {ve}")
+                except Exception as e:
+                    logger.error(f"Error processing line '{line}': {e}")
+    else:
+        logger.warning(f"File {file_path} does not exist")
+
+def fetch_file_metrics(sleep_duration: int = 5) -> None:
+    """Background task to read file paths from the database and update metrics."""
+    while True:
+        with app.app_context():
+            file_paths = ExternalMonitornig.query.all()
+            current_keys = {sample.labels['key'] for sample in file_metric.collect()[0].samples}
+            new_keys = set()
+
+            for file_path in file_paths:
+                logger.info(f"Reading file: {file_path.file_path}")
+                read_file_and_update_metric(file_path.file_path)
+                new_keys.update({key.strip() for line in open(file_path.file_path) for key in line.split(':')[0].strip()})
+
+            # Remove metrics for keys that are no longer in the database
+            for key in current_keys:
+                if key not in new_keys:
+                    file_metric.remove(key)
+
+        time.sleep(sleep_duration)
+
+def fetch_file_metrics_task() -> None:
+    """Starts the background task in a separate thread."""
+    thread = threading.Thread(target=fetch_file_metrics)
+    thread.daemon = True
+    thread.start()

src/config.py

@@ -26,7 +26,8 @@
 os.makedirs(DB_DIR, exist_ok=True)
 
 # Configure the SQLite database
-app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{DB_DIR}/systemguard.db"
+# app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{DB_DIR}/systemguard.db"
+app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///systemguard.db"
 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 app.config['SECRET_KEY'] = 'secret'
 

src/models/__init__.py

@@ -9,6 +9,7 @@
 from src.models.system_information import SystemInformation
 from src.models.user_profile import UserProfile
 from src.models.monitored_website import MonitoredWebsite
+from src.models.prometheus_model import ExternalMonitornig
 from flask_login import current_user
 from src.logger import logger
 from werkzeug.security import generate_password_hash

src/models/prometheus_model.py

@@ -0,0 +1,7 @@
+from src.config import db
+
+
+class ExternalMonitornig(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    file_path = db.Column(db.String(100), nullable=False)
+    is_active = db.Column(db.Boolean, default=True)

src/routes/prometheus.py

@@ -1,11 +1,45 @@
-from flask import Blueprint, Response
+from flask import Blueprint, Response, request, jsonify, render_template, flash, redirect, url_for
 from prometheus_client import generate_latest
-from src.config import app
+
+from src.config import app, db
+from src.models import ExternalMonitornig
+
 
 # Define the Prometheus Blueprint
 prometheus_bp = Blueprint('prometheus', __name__)
 
 # Define a route to serve Prometheus metrics
 @app.route('/metrics')
 def metrics():
-    return Response(generate_latest(), mimetype='text/plain')
+    output = generate_latest()
+    output = '\n'.join([line for line in output.decode().split('\n') if not line.startswith('#') and line])
+    return Response(output, mimetype='text/plain')
+
+# post request to add file path
+@app.route('/prometheus/add_file_path', methods=['GET', 'POST'])
+def add_file_path():
+    if request.method == 'POST':
+        
+        file_path = request.form.get('file_path')
+        # save into the ExternalMonitornig table
+        new_task = ExternalMonitornig(file_path=file_path)
+        # commit the changes
+        db.session.add(new_task)
+        db.session.commit()
+        
+        # read_file_and_update_metric(file_path=file_path)
+        return redirect(url_for('add_file_path'))
+    
+    data = ExternalMonitornig.query.all()
+    return render_template('prometheus/add_file_path.html',  data=data)
+
+
+# post request to delete file path
+@app.route('/prometheus/delete_file_path/<int:id>', methods=['POST'])
+def delete_file_path(id):
+    file_path = ExternalMonitornig.query.get_or_404(id)
+    db.session.delete(file_path)
+    db.session.commit()
+    flash('File path deleted successfully!', 'success')
+    return redirect(url_for('add_file_path'))
+

src/templates/prometheus/add_file_path.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>External File for Monitoring Prometheus</title>
+</head>
+
+<body>
+    <div class="container">
+        <h1>Add File Path</h1>
+        <form method="POST" action="{{ url_for('add_file_path') }}">
+            <label for="file-path">File Path</label>
+            <input type="text" id="file-path" name="file_path" placeholder="Enter file path" required>
+            <button type="submit">Submit</button>
+        </form>
+        <div class="message" id="message"></div>
+    </div>
+
+    <!-- show a list of external monitoring file path from db -->
+    <!-- /prometheus/delete_file_path/<int:id> -->
+    {% if data %}
+        {% for item in data %}
+            <div>
+                <p>{{ item.file_path }}</p>
+                <form method="POST" action="{{ url_for('delete_file_path', id=item.id) }}">
+                    <button type="submit">Delete</button>
+                </form>
+            </div>
+        {% endfor %}
+    {% endif %}
+</body>
+
+</html>