Feat/nfstream migration

.env.example

@@ -1,6 +1,18 @@
+# Database credentials (consumed by docker-compose for the postgres service
+# and woven into SQLALCHEMY_DATABASE_URI below). Change these before any
+# non-local deployment.
+POSTGRES_USER=adns
+POSTGRES_PASSWORD=adns_password
+POSTGRES_DB=adns
+
 # Backend / API
 SQLALCHEMY_DATABASE_URI=postgresql://adns:adns_password@127.0.0.1/adns
 ADNS_REDIS_URL=redis://127.0.0.1:6379/0
+
+# Admin token for the network-response endpoints (/block_ip, /unblock_ip,
+# /killswitch). Leave blank to keep these endpoints disabled. When set, callers
+# must send `Authorization: Bearer <token>` (or `X-Admin-Token: <token>`).
+ADNS_ADMIN_TOKEN=
 ADNS_RQ_QUEUE=flow_scores
 ADNS_RQ_JOB_TIMEOUT=120
 ADNS_RQ_BATCH_SIZE=100

.gitattributes

@@ -0,0 +1 @@
+api/model_artifacts/*.joblib filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-installer.yml

@@ -0,0 +1,60 @@
+name: Build Windows Installer
+
+on:
+  push:
+    tags:
+      - "v*"          # trigger on version tags: v1.0.0, v1.2.3, etc.
+  workflow_dispatch:  # allow manual runs from the Actions tab
+
+jobs:
+  build:
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: frontend/adns-frontend/package-lock.json
+
+      - name: Build React frontend
+        working-directory: frontend/adns-frontend
+        run: |
+          npm ci
+          npm run build
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install Python dependencies
+        run: pip install -r requirements-desktop.txt pyinstaller
+
+      - name: Build with PyInstaller
+        run: pyinstaller ADNS.spec
+
+      - name: Build installer with Inno Setup
+        run: |
+          $iscc = "C:\Program Files (x86)\Inno Setup 6\ISCC.exe"
+          & $iscc installer.iss
+        shell: pwsh
+
+      - name: Upload installer artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ADNS_installer
+          path: Output\ADNS_installer.exe
+          retention-days: 30
+
+      - name: Create GitHub Release
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: Output\ADNS_installer.exe
+          generate_release_notes: true

.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  api-tests:
+    name: API tests (pytest)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install test dependencies
+        run: pip install -r api/requirements-test.txt
+      - name: Run tests
+        working-directory: api
+        run: python -m pytest
+
+  frontend-build:
+    name: Frontend lint & build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend/adns-frontend
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: frontend/adns-frontend/package-lock.json
+      - name: Install dependencies
+        run: npm ci
+      - name: Lint
+        run: npm run lint
+      - name: Build
+        run: npm run build

.gitignore

@@ -1,13 +1,31 @@
+# Python
 __pycache__/
 *.pyc
-.venv/
+*.pyo
+*.pyd
+*.egg-info/
+.env
 venv/
-node_modules/
+.venv/
+
+# SQLite dev DB
+api/instance/
+
+# PyInstaller output
+dist/
 build/
-.env
-*.env
-*.pem
-data/
-outputs/
-.vs/
-**/.vs/
+
+# Model artifacts are tracked via Git LFS (.gitattributes), not ignored.
+
+# Frontend build
+frontend/adns-frontend/dist/
+frontend/adns-frontend/node_modules/
+node_modules/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# IDE
+.vscode/
+.idea/

ADNS.spec

@@ -0,0 +1,145 @@
+# PyInstaller spec for ADNS desktop application.
+# Run from repo root after:
+#   npm run build  (inside frontend/adns-frontend)
+#   pip install -r requirements-desktop.txt
+
+import os
+
+# Bundle npcap installer if present in repo root (download from https://npcap.com).
+# Npcap (the packet capture driver) must still be installed on the target machine.
+_npcap_datas = []
+_npcap_installer = os.path.join(os.path.abspath("."), "npcap-installer.exe")
+if os.path.isfile(_npcap_installer):
+    _npcap_datas.append((_npcap_installer, "."))
+
+from PyInstaller.utils.hooks import collect_all, collect_data_files
+import importlib.util as _ilu
+
+block_cipher = None
+
+# Collect all sklearn files — it uses a lot of data files and Cython extensions
+sklearn_datas, sklearn_binaries, sklearn_hiddenimports = collect_all("sklearn")
+webview_datas, webview_binaries, webview_hiddenimports = collect_all("webview")
+pystray_datas, pystray_binaries, pystray_hiddenimports = collect_all("pystray")
+
+# collect_all bundles xgboost Python files but misses the native DLL on Windows — add it explicitly.
+import xgboost as _xgb
+xgboost_datas, xgboost_binaries, xgboost_hiddenimports = collect_all("xgboost")
+_xgb_lib = os.path.join(os.path.dirname(_xgb.__file__), "lib", "xgboost.dll")
+if os.path.isfile(_xgb_lib) and not any(_xgb_lib == src for src, _ in xgboost_binaries):
+    xgboost_binaries.append((_xgb_lib, "xgboost/lib"))
+
+# NFStream: collect Python package + _lib_engine.pyd (CFFI extension at site-packages root).
+# collect_all("nfstream") finds only the Python package — the native .pyd is one level up.
+nfstream_datas, nfstream_binaries_pkg, nfstream_hiddenimports = collect_all("nfstream")
+_lib_engine_spec = _ilu.find_spec("_lib_engine")
+_nfstream_extra_binaries = []
+if _lib_engine_spec and _lib_engine_spec.origin:
+    _nfstream_extra_binaries.append((_lib_engine_spec.origin, "."))
+
+a = Analysis(
+    ["launcher.py"],
+    pathex=["api", "ml"],    # api: 'from app import ...' resolves; ml: adns_flows package
+    binaries=(sklearn_binaries + webview_binaries + pystray_binaries + xgboost_binaries
+              + nfstream_binaries_pkg + _nfstream_extra_binaries),
+    datas=[
+        # React production build
+        ("frontend/adns-frontend/dist", "dist"),
+        # Trained model artifacts — REQUIRED for detection.  Absent model blocks
+        # /capture/autostart with HTTP 503 (not silent).  See api/model_runner.py.
+        # nfstream_model.joblib is tracked via Git LFS; run `git lfs pull` after clone.
+        ("api/model_artifacts", "model_artifacts"),
+        # Flask app source files (all modules in api/)
+        ("api/*.py", "api"),
+        # App icon (used by the desktop shortcut)
+        ("assets/icon.ico", "assets"),
+    ] + sklearn_datas + webview_datas + pystray_datas + xgboost_datas
+      + nfstream_datas + _npcap_datas,
+    hiddenimports=[
+        # Flask ecosystem
+        "flask_cors",
+        "flask_sqlalchemy",
+        "sqlalchemy.dialects.sqlite",
+        "sqlalchemy.dialects.sqlite.pysqlite",
+        "sqlalchemy.dialects.postgresql",
+        "sqlalchemy.dialects.postgresql.base",
+        "sqlalchemy.pool.impl",
+        # ML stack
+        "joblib",
+        "numpy",
+        "pandas",
+        # pywebview Windows backends
+        "webview.platforms.winforms",
+        "webview.platforms.edgechromium",
+        "clr",
+        "pystray",
+        "pystray._win32",
+        "PIL",
+        "PIL.Image",
+        # adns_flows shared extractor (ml/adns_flows/ — on pathex, but api/*.py are data
+        # files so PyInstaller won't trace their imports automatically)
+        "adns_flows",
+        "adns_flows.schema",
+        "adns_flows.extract_nfstream",
+        "adns_flows.nfstream_config",
+        "adns_flows.extract",
+        "adns_flows.assemble",
+        # NFStream serving module (api/ data file, imports not auto-traced)
+        "serving_nfstream",
+        # NFStream sub-modules (collect_all may miss lazy-imported ones)
+        "nfstream",
+        "nfstream.streamer",
+        "nfstream.meter",
+        "nfstream.plugin",
+        "nfstream.engine",
+        "nfstream.utils",
+        # multiprocessing spawn protocol (NFStream meter workers use spawn on Windows)
+        "multiprocessing.spawn",
+        "multiprocessing.forkserver",
+        "multiprocessing.popen_spawn_win32",
+    ] + sklearn_hiddenimports + webview_hiddenimports + pystray_hiddenimports
+      + xgboost_hiddenimports + nfstream_hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=["pyi_hooks/rthook_nfstream_npcap.py"],
+    excludes=[
+        "psycopg2",
+        "psycopg2_binary",
+        "alembic",
+        "tkinter",
+        "matplotlib",
+        "IPython",
+        "jupyter",
+    ],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name="ADNS",
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=False,        # no terminal window on Windows
+    icon="assets/icon.ico",
+)
+
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name="ADNS",
+)