feat: influx support added

Author: codeperfectplus

.gitignore

@@ -164,3 +164,5 @@ cython_debug/
 
 predefine_user.json
 src/assets/predefine_user.json
+
+influxdb_data

app.py

@@ -6,9 +6,9 @@
 # background thread to monitor system settings changes
 # print("FLASK_ENV: ", os.getenv('FLASK_ENV'))
 # # if os.getenv('FLASK_ENV') == 'production':
-# start_website_monitoring()  # Starts pinging active websites
-# fetch_file_metrics_task()
-# monitor_settings()  # Starts monitoring for system logging changes
+start_website_monitoring()  # Starts pinging active websites
+fetch_file_metrics_task()
+monitor_settings()  # Starts monitoring for system logging changes
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=5000, debug=True)

install_influx.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Define variables
+INFLUXDB_VERSION="latest"
+CONTAINER_NAME="influxdb"
+NETWORK_NAME="influx_network"
+DATA_DIR="./influxdb_data"
+INFLUXDB_USER="admin"                # Change this to your desired username
+INFLUXDB_PASSWORD="admin_passwor"         # Change this to your desired password
+INFLUXDB_ORG="systemguard"                # Change this to your desired organization name
+INFLUXDB_BUCKET="system_metrics"     # Change this to your desired initial bucket (database) name
+INFLUXDB_TOKEN="GixPn4ZmYuPyzPtj2XivfALmtHUaatBTK85ZjSe78P8NdbM_J9426tTPoqnbQDCok_NgBreOaqzdmoGxJ3UfsA=="
+
+# Export the INFLUXDB_TOKEN to be used later in your app
+export "INFLUXDB_TOKEN"=$INFLUXDB_TOKEN
+
+# Create a data directory for InfluxDB
+rm -rf $DATA_DIR
+mkdir -p $DATA_DIR
+
+# Stop and remove existing InfluxDB container if it exists
+if [ "$(docker ps -aq -f name=$CONTAINER_NAME)" ]; then
+    echo "Stopping existing InfluxDB container..."
+    docker stop $CONTAINER_NAME
+    echo "Removing existing InfluxDB container..."
+    docker rm -f $CONTAINER_NAME
+fi
+
+# Kill any process using port 8086
+if sudo lsof -i :8086; then
+    echo "Killing process using port 8086..."
+    sudo fuser -k 8086/tcp
+fi
+
+# Create a Docker network
+echo "Creating Docker network..."
+docker network create $NETWORK_NAME || true  # Avoid error if the network already exists
+
+# Pull the InfluxDB Docker image
+echo "Pulling InfluxDB Docker image..."
+docker pull influxdb:$INFLUXDB_VERSION
+
+# Run the InfluxDB container with authentication and initial setup to skip onboarding
+echo "Running InfluxDB container..."
+docker run -d \
+  --name $CONTAINER_NAME \
+  --network $NETWORK_NAME \
+  -p 8086:8086 \
+  -v $PWD/$DATA_DIR:/var/lib/influxdb2 \
+  -e DOCKER_INFLUXDB_INIT_MODE=setup \
+  -e DOCKER_INFLUXDB_INIT_USERNAME=$INFLUXDB_USER \
+  -e DOCKER_INFLUXDB_INIT_PASSWORD=$INFLUXDB_PASSWORD \
+  -e DOCKER_INFLUXDB_INIT_ORG=$INFLUXDB_ORG \
+  -e DOCKER_INFLUXDB_INIT_BUCKET=$INFLUXDB_BUCKET \
+  -e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=$INFLUXDB_TOKEN \
+  influxdb:$INFLUXDB_VERSION
+
+# Output completion message
+echo "InfluxDB setup completed! Access it at http://localhost:8086 with your credentials."

src/background_task/log_system_info.py

@@ -7,6 +7,8 @@
 from src.models import GeneralSettings, SystemInformation
 from sqlalchemy.exc import SQLAlchemyError
 from prometheus_client import Counter, Gauge
+import os, time
+from influxdb_client import Point, WritePrecision
 
 from src.logger import logger
 from src.config import influx_client, bucket, write_api
@@ -78,9 +80,11 @@ def log_system_info_to_db():
 
             # Update Prometheus metrics
             update_prometheus_metrics(system_info)
+            # Store system information in InfluxDB
+            store_system_info_in_influxdb(system_info)
 
             # Store system information in the database
-            store_system_info_in_db(system_info)
+            # store_system_info_in_db(system_info)
             logger.info("System information logged to database.")
 
         except SQLAlchemyError as db_err:
@@ -124,6 +128,36 @@ def store_system_info_in_db(system_info):
     db.session.add(system_log)
     db.session.commit()
 
+def store_system_info_in_influxdb(system_info):
+    """
+    Stores the collected system information into the InfluxDB with proper error handling.
+    """
+    try:
+        # Create a data point for system information
+        point = (
+            Point("system_info")
+            .tag("host", os.environ.get("HOSTNAME", "unknown"))
+            .field("cpu_percent", system_info["cpu_percent"])
+            .field("memory_percent", system_info["memory_percent"])
+            .field("battery_percent", system_info["battery_percent"])
+            .field("network_sent", system_info["network_sent"])
+            .field("network_received", system_info["network_received"])
+            .field("dashboard_memory_usage", system_info["dashboard_memory_usage"])
+            .field("cpu_frequency", system_info["cpu_frequency"])
+            .field("current_temp", system_info["current_temp"])
+            .time(int(time.time() * 1_000_000_000), WritePrecision.NS)  # Nanosecond precision
+        )
+
+        # Write the data point to InfluxDB
+        write_api.write(bucket=bucket, record=point)
+        logger.info("Successfully wrote system information to InfluxDB")
+
+    except ValueError as ve:
+        logger.error(f"Value error while storing system info: {ve}")
+    except Exception as e:
+        logger.error(f"An unexpected error occurred: {e}", exc_info=True)
+
+
 
 def monitor_settings():
     """

src/config.py

@@ -2,6 +2,10 @@
 from flask import Flask, render_template
 from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
+import os, time
+from influxdb_client import InfluxDBClient, Point, WritePrecision
+from influxdb_client.client.write_api import SYNCHRONOUS
+
 
 from src.logger import logger
 from src.helper import get_system_node_name, get_ip_address
@@ -26,15 +30,35 @@
 os.makedirs(DB_DIR, exist_ok=True)
 
 # Configure the SQLite database
-# app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{DB_DIR}/systemguard.db"
-app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///systemguard.db"
+app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{DB_DIR}/systemguard.db"
+# app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///systemguard.db"
 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 app.config['SECRET_KEY'] = 'secret'
 
 # Initialize the database
 db = SQLAlchemy(app)
 migrate = Migrate(app, db)
 
+# influx db configuration
+
+INFLUXDB_TOKEN=""
+print("Token: ", INFLUXDB_TOKEN)
+if not INFLUXDB_TOKEN:
+    raise ValueError("Please set the INFLUXDB_TOKEN environment variable.")
+org = "systemguard"
+url = "http://localhost:8086"
+bucket="_monitoring"
+try:
+    influx_client = InfluxDBClient(url=url, token=INFLUXDB_TOKEN, org=org)
+    bucket = "_monitoring"
+    write_api = influx_client.write_api(write_options=SYNCHRONOUS)
+    query_api = influx_client.query_api()
+    logger.info("Connected to InfluxDB successfully")
+
+except Exception as e:
+    logger.error(f"Failed to connect to InfluxDB: {e}")
+    raiseclient = InfluxDBClient(url=url, token=INFLUXDB_TOKEN, org=org)
+
 # Define global variables for templates
 app.jinja_env.globals.update(
     title=APP_NAME,

src/models/__init__.py

@@ -18,9 +18,39 @@
 
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+# Context processor for injecting settings into templates
+@app.context_processor
+def inject_settings():
+    if current_user.is_anonymous:
+        user_dashboard_settings = UserDashboardSettings(user_id=0)
+        card_settings = None
+        page_toggles_settings = None
+        general_settings = None
+        return dict(
+            user_dashboard_settings=user_dashboard_settings,
+            card_settings=card_settings,
+            page_toggles_settings=page_toggles_settings,
+            general_settings=general_settings,
+        )
+    general_settings = GeneralSettings.query.first()
+    card_settings = UserCardSettings.query.filter_by(user_id=current_user.id).first()
+    user_dashboard_settings = UserDashboardSettings.query.filter_by(
+        user_id=current_user.id
+    ).first()  # Retrieve user-specific user_dashboard_settings from DB
+    page_toggles_settings = PageToggleSettings.query.filter_by(
+        user_id=current_user.id
+    ).first()
+    all_settings = dict(
+        user_dashboard_settings=user_dashboard_settings,
+        general_settings=general_settings,
+        card_settings=card_settings,
+        page_toggles_settings=page_toggles_settings,
+    )
+    return all_settings
+
 with app.app_context():
     # Check if tables already exist
-    if not db.inspect(db.engine).has_table('user_profile'):  # Use an important table to check existence
+    if not db.inspect(db.engine).has_table('users'):  # Use an important table to check existence
         logger.info("Creating tables")
         db.create_all()
 
@@ -70,32 +100,3 @@
     else:
         logger.info("Tables already exist. Skipping creation.")
 
-# Context processor for injecting settings into templates
-@app.context_processor
-def inject_settings():
-    if current_user.is_anonymous:
-        user_dashboard_settings = UserDashboardSettings(user_id=0)
-        card_settings = None
-        page_toggles_settings = None
-        general_settings = None
-        return dict(
-            user_dashboard_settings=user_dashboard_settings,
-            card_settings=card_settings,
-            page_toggles_settings=page_toggles_settings,
-            general_settings=general_settings,
-        )
-    general_settings = GeneralSettings.query.first()
-    card_settings = UserCardSettings.query.filter_by(user_id=current_user.id).first()
-    user_dashboard_settings = UserDashboardSettings.query.filter_by(
-        user_id=current_user.id
-    ).first()  # Retrieve user-specific user_dashboard_settings from DB
-    page_toggles_settings = PageToggleSettings.query.filter_by(
-        user_id=current_user.id
-    ).first()
-    all_settings = dict(
-        user_dashboard_settings=user_dashboard_settings,
-        general_settings=general_settings,
-        card_settings=card_settings,
-        page_toggles_settings=page_toggles_settings,
-    )
-    return all_settings

src/routes/api.py

@@ -4,8 +4,10 @@
 from src.models import SystemInformation, UserDashboardSettings
 from src.utils import _get_system_info, get_os_release_info, get_os_info, get_cached_value
 from datetime import datetime, timedelta
+from flask import request, jsonify
 import gc
 
+from src.config import query_api, bucket
 api_bp = blueprints.Blueprint("api", __name__)
 
 @app.route("/api/system-info", methods=["GET"])
@@ -109,6 +111,105 @@ def graph_data_api():
         # Handle and log the error for debugging purposes
         return jsonify({'error': 'An error occurred while fetching the graph data', 'details': str(e)}), 500
 
+@app.route('/api/v2/graphs_data', methods=['GET'])
+@login_required
+def graph_data_api_v2():
+    try:
+        current_time = datetime.now()
+        # Get the time filter from query parameters
+        time_filter = request.args.get('filter', default='1 day')
+
+        # Determine the start time based on the filter
+        time_deltas = {
+            '5 minutes': '-5m',
+            '15 minutes': '-15m',
+            '30 minutes': '-30m',
+            '1 hour': '-1h',
+            '3 hours': '-3h',
+            '6 hours': '-6h',
+            '12 hours': '-12h',
+            '1 day': '-1d',
+            '2 days': '-2d',
+            '3 days': '-3d',
+            '1 week': '-1w',
+            '1 month': '-30d',
+            '3 months': '-90d',
+        }
+
+        # Get the start time for the query
+        time_range = time_deltas.get(time_filter, '-1d')
+
+        # Build the InfluxDB query
+        flux_query = f"""
+        from(bucket: "{bucket}")
+        |> range(start: {time_range})
+        |> filter(fn: (r) => r._measurement == "system_info")
+        |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
+        """
+
+        # Execute the query
+        tables = query_api.query(flux_query)
+
+        # Initialize lists for the data
+        time_data = []
+        cpu_data = []
+        memory_data = []
+        battery_data = []
+        network_sent_data = []
+        network_received_data = []
+        dashboard_memory_usage = []
+        cpu_frequency = []
+        current_temp = []
+
+        # Parse the results
+        for table in tables:
+            for record in table.records:
+                time_data.append(record.values.get("_time", None))
+                # Extract each field by key (handle missing fields gracefully)
+                cpu_data.append(record.values.get("cpu_percent", None))
+                memory_data.append(record.values.get("memory_percent", None))
+                battery_data.append(record.values.get("battery_percent", None))
+                network_sent_data.append(record.values.get("network_sent", None))
+                network_received_data.append(record.values.get("network_received", None))
+                dashboard_memory_usage.append(record.values.get("dashboard_memory_usage", None))
+                cpu_frequency.append(record.values.get("cpu_frequency", None))
+                current_temp.append(record.values.get("current_temp", None))
+
+        # Return the data as JSON
+        response = jsonify({
+            "time": time_data,
+            "cpu": cpu_data,
+            "memory": memory_data,
+            "battery": battery_data,
+            "network_sent": network_sent_data,
+            "network_received": network_received_data,
+            "dashboard_memory_usage": dashboard_memory_usage,
+            "cpu_frequency": cpu_frequency,
+            "current_temp": current_temp,
+            "current_time": current_time
+        })
+
+        print(time_data)
+        # Clean up large data structures
+        del tables
+        del time_data
+        del cpu_data
+        del memory_data
+        del battery_data
+        del network_sent_data
+        del network_received_data
+        del dashboard_memory_usage
+        del cpu_frequency
+        del current_temp
+
+        gc.collect()
+
+        return response, 200
+
+    except Exception as e:
+        # Handle and log the error for debugging purposes
+        return jsonify({'error': 'An error occurred while fetching the graph data', 'details': str(e)}), 500
+    
 @app.route('/api/v1/refresh-interval', methods=['GET', 'POST'])
 @login_required
 def manage_refresh_interval():

src/static/js/graphs.js

@@ -12,7 +12,7 @@ function fetchDataAndRenderCharts() {
     console.log('Stored Filter Value:', storedFilterValue);
 
     // Fetch data with the selected time filter
-    fetch(`/api/v1/graphs_data?filter=${storedFilterValue}`)
+    fetch(`/api/v2/graphs_data?filter=${storedFilterValue}`)
         .then(response => response.json())
         .then(data => {
             const cpuData = data.cpu;