sqlite replaced with prometheus for time series metrics

Author: codeperfectplus

app.py

@@ -4,11 +4,10 @@
 import os
 
 # background thread to monitor system settings changes
-# print("FLASK_ENV: ", os.getenv('FLASK_ENV'))
-# # if os.getenv('FLASK_ENV') == 'production':
-# start_website_monitoring()  # Starts pinging active websites
-# fetch_file_metrics_task()
-monitor_settings()  # Starts monitoring for system logging changes
+if os.getenv('FLASK_ENV') == 'production':
+    start_website_monitoring()  # Starts pinging active websites
+    fetch_file_metrics_task()
+    monitor_settings()  # Starts monitoring for system logging changes
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=5000, debug=True)

prometheus_config/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 30s  # How often Prometheus scrapes the target
+
+scrape_configs:
+  - job_name: 'flask_app_metrics'  # Scraping metrics from Flask app
+    static_configs:
+      - targets: ['192.168.1.10:5050']

setup.sh

@@ -853,10 +853,12 @@ start_server() {
 
 # Install function
 install() {
+    create_dir "$EXTRACT_DIR"
+    # PROMETHEUS_INSTALL_SCRIPT
     message_box "$APP_NAME Installer $INSATLLER_VERSION" 0
     message_box "Welcome on board: $(echo "$USER_NAME" | sed 's/.*/\u&/')" 3
     check_dependencies
-    create_dir "$EXTRACT_DIR"
+    
     message_box "Choose the installation method\nNote: Release is recommended for production use." 0
     message_box "1. Release (More Stable Version)\n2. Git Repository (Pre-Release Version)\n3. Source Code (Current Directory)" 0
 
@@ -878,6 +880,11 @@ install() {
         exit 1
         ;;
     esac
+        PROMETHEUS_INSTALL_SCRIPT=$(find "$EXTRACT_DIR" -name prometheus.sh) || {
+        log "ERROR" "Prometheus installation script not found."
+        exit 1
+    }
+    sudo -u "$USER_NAME" bash "$PROMETHEUS_INSTALL_SCRIPT"
     start_server
     message_box "The $APP_NAME server is running at $HOST_URL" 0
     # open_browser

src/background_task/log_system_info.py

@@ -8,12 +8,13 @@
 from sqlalchemy.exc import SQLAlchemyError
 from prometheus_client import Counter, Gauge
 import os, time
-from influxdb_client import Point, WritePrecision
 
 from src.logger import logger
-from src.config import influx_client, bucket, write_api
+from src.influxdb_config import bucket, write_api
+from src.utils import get_system_username
 # Flag to track if logging is already scheduled
 is_logging_scheduled = False
+fetch_system_info_interval = 30
 
 # Initialize Prometheus metrics
 metrics = {
@@ -44,7 +45,7 @@ def log_system_info():
 
             log_system_info_to_db()
             logger.debug("System information logged successfully.")
-            schedule_next_log()
+            schedule_next_log(interval=fetch_system_info_interval)
 
         except Exception as e:
             logger.error(f"Error during system info logging: {e}", exc_info=True)
@@ -63,7 +64,7 @@ def is_logging_enabled():
         return False
 
 
-def schedule_next_log(interval=60):
+def schedule_next_log(interval=10):
     """
     Schedules the next logging event after the specified interval (in seconds).
     """
@@ -80,8 +81,9 @@ def log_system_info_to_db():
 
             # Update Prometheus metrics
             update_prometheus_metrics(system_info)
+
             # Store system information in InfluxDB
-            store_system_info_in_influxdb(system_info)
+            # store_system_info_in_influxdb(system_info)
 
             # Store system information in the database
             # store_system_info_in_db(system_info)
@@ -128,34 +130,34 @@ def store_system_info_in_db(system_info):
     db.session.add(system_log)
     db.session.commit()
 
-def store_system_info_in_influxdb(system_info):
-    """
-    Stores the collected system information into the InfluxDB with proper error handling.
-    """
-    try:
-        # Create a data point for system information
-        point = (
-            Point("system_info")
-            .tag("host", os.environ.get("HOSTNAME", "unknown"))
-            .field("cpu_percent", system_info["cpu_percent"])
-            .field("memory_percent", system_info["memory_percent"])
-            .field("battery_percent", system_info["battery_percent"])
-            .field("network_sent", system_info["network_sent"])
-            .field("network_received", system_info["network_received"])
-            .field("dashboard_memory_usage", system_info["dashboard_memory_usage"])
-            .field("cpu_frequency", system_info["cpu_frequency"])
-            .field("current_temp", system_info["current_temp"])
-            .time(int(time.time() * 1_000_000_000), WritePrecision.NS)  # Nanosecond precision
-        )
-
-        # Write the data point to InfluxDB
-        write_api.write(bucket=bucket, record=point)
-        logger.info("Successfully wrote system information to InfluxDB")
-
-    except ValueError as ve:
-        logger.error(f"Value error while storing system info: {ve}")
-    except Exception as e:
-        logger.error(f"An unexpected error occurred: {e}", exc_info=True)
+# def store_system_info_in_influxdb(system_info):
+#     """
+#     Stores the collected system information into the InfluxDB with proper error handling.
+#     """
+#     try:
+#         # Create a data point for system information
+#         point = (
+#             Point("system_info")
+#             .tag("host", get_system_username())
+#             .field("cpu_percent", system_info["cpu_percent"])
+#             .field("memory_percent", system_info["memory_percent"])
+#             .field("battery_percent", system_info["battery_percent"])
+#             .field("network_sent", system_info["network_sent"])
+#             .field("network_received", system_info["network_received"])
+#             .field("dashboard_memory_usage", system_info["dashboard_memory_usage"])
+#             .field("cpu_frequency", system_info["cpu_frequency"])
+#             .field("current_temp", system_info["current_temp"])
+#             .time(int(time.time() * 1_000_000_000))
+#         )
+
+#         # Write the data point to InfluxDB
+#         write_api.write(bucket=bucket, record=point)
+#         logger.info("Successfully wrote system information to InfluxDB")
+
+#     except ValueError as ve:
+#         logger.error(f"Value error while storing system info: {ve}")
+#     except Exception as e:
+#         logger.error(f"An unexpected error occurred: {e}", exc_info=True)
 
 
 

src/config.py

@@ -39,26 +39,6 @@
 db = SQLAlchemy(app)
 migrate = Migrate(app, db)
 
-# influx db configuration
-
-INFLUXDB_TOKEN=""
-print("Token: ", INFLUXDB_TOKEN)
-if not INFLUXDB_TOKEN:
-    raise ValueError("Please set the INFLUXDB_TOKEN environment variable.")
-org = "systemguard"
-url = "http://localhost:8086"
-bucket="system_metrics"
-try:
-    influx_client = InfluxDBClient(url=url, token=INFLUXDB_TOKEN, org=org)
-    bucket = "system_metrics"
-    write_api = influx_client.write_api(write_options=SYNCHRONOUS)
-    query_api = influx_client.query_api()
-    logger.info("Connected to InfluxDB successfully")
-
-except Exception as e:
-    logger.error(f"Failed to connect to InfluxDB: {e}")
-    raiseclient = InfluxDBClient(url=url, token=INFLUXDB_TOKEN, org=org)
-
 # Define global variables for templates
 app.jinja_env.globals.update(
     title=APP_NAME,

src/helper.py

@@ -33,12 +33,7 @@ def get_ip_address():
 
         return ip_address
     except (IndexError, subprocess.CalledProcessError) as e:
-        print(f"Error occurred: {e}")
         return None
-    
-
-import os
-import subprocess
 
 def check_installation_information():
     # Output dictionary to store results
@@ -82,7 +77,6 @@ def check_installation_information():
     # Check for updates
     try:
         result = subprocess.run(["git", "status", "-uno"], capture_output=True, text=True, check=True)
-        print(result.stdout)
         if "Your branch is up to date" in result.stdout:
             output["update_available"] = False
         else:

src/influxdb_config.py

@@ -0,0 +1,22 @@
+from influxdb_client import InfluxDBClient, Point, WritePrecision
+from influxdb_client.client.write_api import SYNCHRONOUS
+import os
+from src.logger import logger
+
+
+# influx db configuration
+org = "systemguard"
+url = "http://localhost:8086"
+bucket="system_metrics"
+INFLUXDB_TOKEN=os.getenv('INFLUXDB_TOKEN')
+print("INFLUXDB_TOKEN: ", INFLUXDB_TOKEN)
+
+try:
+    influx_client = InfluxDBClient(url=url, token=INFLUXDB_TOKEN, org=org)
+    bucket = "system_metrics"
+    write_api = influx_client.write_api(write_options=SYNCHRONOUS)
+    query_api = influx_client.query_api()
+    logger.info("Connected to InfluxDB successfully")
+except Exception as e:
+    logger.error(f"Failed to connect to InfluxDB: {e}")
+    raiseclient = InfluxDBClient(url=url, token=INFLUXDB_TOKEN, org=org)