From d94d45890868fb106706cbd225a760f327139d21 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 21 May 2026 15:49:26 -0500 Subject: [PATCH 1/6] Update udev rules to create /dev/ptpN device nodes for nsim_ptp devices - Changed the udev rule to create real device nodes instead of symlinks, ensuring compatibility with linuxptp tools. - Updated installation script and README to reflect the change from symlinks to device nodes. - Added a new script for creating device nodes inside containers where udev rules do not run, enhancing usability in containerized environments. --- .github/workflows/ci.yml | 14 +++++++-- 99-nsim-ptp.rules | 11 +++++-- README.md | 21 +++++++++----- install-udev-rule.sh | 2 +- scripts/nsim-ptp-container-setup.sh | 45 +++++++++++++++++++++++++++++ scripts/test-utm-ubuntu.sh | 12 +++++--- 6 files changed, 88 insertions(+), 17 deletions(-) create mode 100755 scripts/nsim-ptp-container-setup.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28167ef..f9c8346 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,9 +100,17 @@ jobs: set -euo pipefail ls /sys/class/nsim_ptp/ ls -la /dev/nsim_ptp* /dev/ptp* 2>/dev/null || echo "No PTP devices found" - readlink /dev/ptp0 2>/dev/null | grep -q nsim_ptp \ - && echo "OK: /dev/ptp0 -> $(readlink /dev/ptp0)" \ - || echo "WARN: /dev/ptp0 symlink not found" + if [ -e /dev/ptp0 ] && [ -e /dev/nsim_ptp0 ]; then + PTP_RDEV=$(stat -c '%t:%T' /dev/ptp0) + NSIM_RDEV=$(stat -c '%t:%T' /dev/nsim_ptp0) + if [ "$PTP_RDEV" = "$NSIM_RDEV" ]; then + echo "OK: /dev/ptp0 ($PTP_RDEV) matches /dev/nsim_ptp0" + else + echo "WARN: /dev/ptp0 ($PTP_RDEV) != /dev/nsim_ptp0 ($NSIM_RDEV)" + fi + else + echo "WARN: /dev/ptp0 or /dev/nsim_ptp0 not found" + fi - name: Smoke test — verify ethtool PHC if: steps.load.outputs.load_ok == 'true' diff --git a/99-nsim-ptp.rules b/99-nsim-ptp.rules index 38ac875..cb5ad73 100644 --- a/99-nsim-ptp.rules +++ b/99-nsim-ptp.rules @@ -1,4 +1,4 @@ -# Create /dev/ptp* compat symlinks for nsim_ptp devices so that +# Create /dev/ptpN compat device nodes for nsim_ptp devices so that # linuxptp tools (ptp4l, phc2sys, ts2phc) find clocks at the # standard path reported by ethtool -T. # @@ -6,8 +6,15 @@ # colliding with the kernel's built-in PTP subsystem, but ethtool # still reports integer PHC indices that userspace maps to /dev/ptpN. # +# We use RUN+=mknod instead of SYMLINK+= because real device nodes +# propagate into containers (Kind nodes, Kubernetes pods), whereas +# udev symlinks only exist on the host devtmpfs. +# # MODE="0666" is needed because the non-standard major number (234) # is not in the default cgroup device allowlist; containers and # systemd-sandboxed services (DevicePolicy=closed) would get # "Operation not permitted" without world-readable permissions. -SUBSYSTEM=="nsim_ptp", KERNEL=="nsim_ptp[0-9]*", MODE="0666", SYMLINK+="ptp%n" +SUBSYSTEM=="nsim_ptp", KERNEL=="nsim_ptp[0-9]*", MODE="0666", \ + RUN+="/bin/sh -c 'rm -f /dev/ptp%n; mknod /dev/ptp%n c %M %m; chmod 666 /dev/ptp%n'" +SUBSYSTEM=="nsim_ptp", KERNEL=="nsim_ptp[0-9]*", ACTION=="remove", \ + RUN+="/bin/rm -f /dev/ptp%n" diff --git a/README.md b/README.md index e4cbd25..89dd252 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ sudo modprobe netdevsim pci_bus_nr=0x1f ## udev Rule -Install the udev rule to create `/dev/ptp*` compat symlinks: +Install the udev rule to create `/dev/ptp*` compat device nodes: ```bash sudo cp 99-nsim-ptp.rules /etc/udev/rules.d/ @@ -79,8 +79,10 @@ sudo udevadm control --reload-rules The `nsim_ptp` module registers a separate device class (`/dev/nsim_ptp*`) to avoid colliding with the kernel's built-in PTP subsystem. The udev rule -creates standard `/dev/ptpN` symlinks so that `ethtool -T` PHC indices -resolve correctly for `ptp4l` and other linuxptp tools. +creates real `/dev/ptpN` device nodes (via `mknod`, same major:minor) so +that `ethtool -T` PHC indices resolve correctly for `ptp4l` and other +linuxptp tools. Real device nodes propagate into containers, unlike +symlinks which only exist on the host devtmpfs. ## Usage @@ -120,10 +122,15 @@ When using netdevsim inside containers or Kubernetes pods: `char-ptp` class devices. Override with a drop-in: `DeviceAllow=char-* rw` and `DevicePolicy=auto`. -- **Kubernetes pods:** The `/dev/ptp*` symlinks from the udev rule - propagate to kind node containers but NOT into Kubernetes pods. The - actual `/dev/nsim_ptp*` device nodes do appear in pods. Create the - symlinks inside pods: `for i in 0 1 2 ...; do ln -sf nsim_ptp$i /dev/ptp$i; done` +- **Kubernetes pods:** The udev rule creates real `/dev/ptpN` device + nodes (not symlinks) so they propagate into containers. If your pod + still doesn't see them (e.g. the container runtime only allowlists + `/dev/nsim_ptp*`), run the helper script inside the pod: + + ```bash + scripts/nsim-ptp-container-setup.sh # one-shot + scripts/nsim-ptp-container-setup.sh --watch # sidecar loop + ``` ## Local UTM VMs (macOS) diff --git a/install-udev-rule.sh b/install-udev-rule.sh index d7ba87f..5fa0a43 100755 --- a/install-udev-rule.sh +++ b/install-udev-rule.sh @@ -1,5 +1,5 @@ #!/bin/bash -# POST_INSTALL hook for DKMS: install the nsim_ptp udev symlink rule. +# POST_INSTALL hook for DKMS: install the nsim_ptp udev device-node rule. set -e RULE_SRC="${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/source/99-nsim-ptp.rules" diff --git a/scripts/nsim-ptp-container-setup.sh b/scripts/nsim-ptp-container-setup.sh new file mode 100755 index 0000000..3ab3e20 --- /dev/null +++ b/scripts/nsim-ptp-container-setup.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Create /dev/ptpN device nodes inside containers where udev rules don't run. +# +# Containers see /dev/nsim_ptpN (propagated by the runtime) but NOT the +# /dev/ptpN nodes created by the host's udev rule. This script recreates +# them so linuxptp tools, ethtool, and the ptp-operator find the devices +# at the standard paths. +# +# Usage: +# As an init container: scripts/nsim-ptp-container-setup.sh +# From a DaemonSet: scripts/nsim-ptp-container-setup.sh --watch +# +# With --watch the script loops every 5 s, useful as a sidecar. +set -euo pipefail + +create_nodes() { + local created=0 + for dev in /dev/nsim_ptp*; do + [ -e "$dev" ] || continue + local n="${dev##*nsim_ptp}" + local target="/dev/ptp${n}" + + if [ -e "$target" ]; then + continue + fi + + local maj min + maj=$(stat -c '%t' "$dev" 2>/dev/null) || continue + min=$(stat -c '%T' "$dev" 2>/dev/null) || continue + mknod "$target" c "0x$maj" "0x$min" 2>/dev/null || true + chmod 666 "$target" 2>/dev/null || true + echo "created $target (c $maj:$min)" + created=$((created + 1)) + done + return $created +} + +create_nodes + +if [ "${1:-}" = "--watch" ]; then + while true; do + sleep 5 + create_nodes 2>/dev/null || true + done +fi diff --git a/scripts/test-utm-ubuntu.sh b/scripts/test-utm-ubuntu.sh index 982ae2f..f417db1 100755 --- a/scripts/test-utm-ubuntu.sh +++ b/scripts/test-utm-ubuntu.sh @@ -630,7 +630,7 @@ vm_ssh "sudo dkms install --force ${DKMS_PKG}/${DKMS_VER}" vm_ssh "dkms status" # --------------------------------------------------------------------------- -# 11. Install udev rule for /dev/ptp* compat symlinks +# 11. Install udev rule for /dev/ptp* compat device nodes # --------------------------------------------------------------------------- log "Installing nsim_ptp udev rule ..." vm_ssh "sudo cp ${REMOTE_DKMS}/99-nsim-ptp.rules /etc/udev/rules.d/ && \ @@ -680,12 +680,16 @@ else # Verify the bus device appeared ls /sys/bus/netdevsim/devices/netdevsim1/ - # Verify PTP clock (nsim_ptp class + /dev/ptp compat symlink) + # Verify PTP clock (nsim_ptp class + /dev/ptp compat device node) ls /sys/class/nsim_ptp/ ls -la /dev/nsim_ptp* /dev/ptp* 2>/dev/null - # Verify /dev/ptp* symlinks point to nsim_ptp devices - readlink /dev/ptp0 | grep -q nsim_ptp && echo \" /dev/ptp0 -> nsim_ptp OK\" + # Verify /dev/ptpN has the same major:minor as /dev/nsim_ptpN + PTP_RDEV=\$(stat -c '%t:%T' /dev/ptp0 2>/dev/null || true) + NSIM_RDEV=\$(stat -c '%t:%T' /dev/nsim_ptp0 2>/dev/null || true) + [ -n \"\$PTP_RDEV\" ] && [ \"\$PTP_RDEV\" = \"\$NSIM_RDEV\" ] \\ + && echo \" /dev/ptp0 matches /dev/nsim_ptp0 (rdev \$PTP_RDEV)\" \\ + || echo \" WARN: /dev/ptp0 rdev mismatch or missing\" # Verify ethtool reports a valid PHC IFACE=\$(ls /sys/bus/pci/devices/000\${PCI_ADDR}/net/ 2>/dev/null | head -1) From af730d004857c00408c12b8b2a3481805e6c0e7f Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 28 May 2026 15:52:08 -0500 Subject: [PATCH 2/6] Refactor CI workflow to streamline ptp-operator image builds and e2e tests - Updated the CI configuration to clarify that ptp-operator images are built and e2e tests are run on every PR and workflow_dispatch. - Removed conditional checks for workflow_dispatch in the ptp-images and ptp-test jobs to simplify the workflow structure. --- .github/workflows/ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9c8346..8c0dda6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,5 @@ -# Full CI: build DKMS modules, run smoke tests on every PR. -# On workflow_dispatch, also run ptp-operator e2e tests — build images -# once (ptp-images), then run each scenario in parallel (ptp-test matrix). +# Full CI: build DKMS modules, run smoke tests, build ptp-operator images, +# and run ptp-operator e2e tests on every PR and workflow_dispatch. name: CI @@ -141,10 +140,8 @@ jobs: # ------------------------------------------------------------------ # ptp-operator: build images once and upload as artifact - # (workflow_dispatch only) # ------------------------------------------------------------------ ptp-images: - if: github.event_name == 'workflow_dispatch' needs: dkms-test runs-on: ubuntu-22.04 timeout-minutes: 45 @@ -197,10 +194,8 @@ jobs: # ------------------------------------------------------------------ # ptp-operator: one parallel runner per scenario - # (workflow_dispatch only) # ------------------------------------------------------------------ ptp-test: - if: github.event_name == 'workflow_dispatch' needs: ptp-images runs-on: ${{ matrix.os }} timeout-minutes: 90 From 631b4844e14ee02cc9c2fa167a677dc3a82c27ae Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 28 May 2026 16:01:46 -0500 Subject: [PATCH 3/6] Remove unused container setup script and add architecture docs Drop scripts/nsim-ptp-container-setup.sh (never integrated into any workflow) and clean up references in README.md and ARCHITECTURE.md. --- ARCHITECTURE.md | 333 ++++++++++++++++++++++++++++ README.md | 9 +- scripts/nsim-ptp-container-setup.sh | 45 ---- 3 files changed, 334 insertions(+), 53 deletions(-) create mode 100644 ARCHITECTURE.md delete mode 100755 scripts/nsim-ptp-container-setup.sh diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..b8fa74d --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,333 @@ +# netdevsim-dkms Architecture + +Out-of-tree DKMS package that brings an enhanced **netdevsim** kernel +module — with fake PCI device simulation, DPLL/GNSS emulation, PTP +hardware clock support, and logical clock ID sharing — to standard +Ubuntu runners without requiring custom kernel builds or dedicated +hardware. + +Source: [github.com/redhat-cne/netdevsim-dkms](https://github.com/redhat-cne/netdevsim-dkms) +Based on: Linux 6.9.5 kernel sources + +--- + +## Why This Exists + +The upstream kernel ships `netdevsim`, `ptp`, and `dpll` subsystems +either built-in (`=y`) or as modules (`=m`). The upstream versions +lack simulation features needed by `ptp-operator` CI (mock PTP clocks, +EXTTS, DPLL pin emulation, fake PCI topology). Building a custom +kernel for every CI run is impractical, so this DKMS package installs +enhanced modules alongside the stock kernel. + +The key challenge is **symbol collision**: loading a second `ptp.ko` or +`dpll.ko` would conflict with the kernel's own copies. This is solved +by the **nsim\_ symbol prefix** layer (see below). + +--- + +## Module Map + +``` +┌──────────────────────────────────────────────────────────────┐ +│ netdevsim.ko │ +│ Fake PCI bus · devlink · ethtool · FIB · BPF offload · │ +│ IPsec · MACsec · hwstats · UDP tunnels · psample · DPLL │ +│ │ +│ Imports: nsim_ptp_*, nsim_dpll_*, nsim_mock_phc_* │ +├──────────────┬───────────────────────────┬───────────────────┤ +│ nsim_ptp.ko │ nsim_ptp_mock.ko │ nsim_dpll.ko │ +│ │ │ │ +│ PTP core: │ Mock PTP clock: │ DPLL subsystem: │ +│ clock reg, │ kref lifecycle, │ device/pin reg, │ +│ chardev, │ 2-pin layout, │ netlink, │ +│ sysfs, │ EXTTS simulation, │ multicast_allns, │ +│ vclock │ logical clock IDs │ change ntf │ +│ │ │ │ +│ Exports: │ Exports: │ Exports: │ +│ nsim_ptp_* │ nsim_mock_phc_* │ nsim_dpll_* │ +└──────────────┴───────────────────────────┴───────────────────┘ +``` + +| Module | Source dir | Built from | Description | +|--------|-----------|------------|-------------| +| `nsim_ptp.ko` | `ptp/` | `ptp_clock.c`, `ptp_chardev.c`, `ptp_sysfs.c`, `ptp_vclock.c` | PTP core with `nsim_ptp_class` export | +| `nsim_ptp_mock.ko` | `ptp/` | `ptp_mock.c` | Mock PTP hardware clock with EXTTS simulation | +| `nsim_dpll.ko` | `dpll/` | `dpll_core.c`, `dpll_netlink.c`, `dpll_nl.c` | DPLL subsystem with cross-namespace genetlink | +| `netdevsim.ko` | `netdevsim/` | 14 source files (see below) | Main simulation driver | + +### netdevsim.ko components + +| File | Feature | Kernel config gate | +|------|---------|-------------------| +| `netdev.c` | Net device, ethtool, module entry | always | +| `dev.c` | Device lifecycle, devlink, debugfs | always | +| `bus.c` | Simulated bus, sysfs new/del device | always | +| `fakepci.c` | Fake PCI host bridge and device | always | +| `ethtool.c` | Ethtool ops, PHC index reporting | always | +| `fib.c` | FIB offload simulation | always | +| `health.c` | Devlink health reporters | always | +| `hwstats.c` | Hardware statistics simulation | always | +| `udp_tunnels.c` | UDP tunnel NIC offload simulation | always | +| `dpll.c` | DPLL pin wiring into netdevsim | always | +| `bpf.c` | BPF program offload | `CONFIG_BPF_SYSCALL` | +| `ipsec.c` | IPsec / XFRM offload | `CONFIG_XFRM_OFFLOAD` | +| `psample.c` | Packet sampling | `CONFIG_PSAMPLE` | +| `macsec.c` | MACsec offload | `CONFIG_MACSEC` | + +--- + +## Build Order and Symbol Dependencies + +``` + ┌─────────┐ + │ ptp/ │ ← builds first + │ nsim_ptp│ ← exports nsim_ptp_clock_register, nsim_ptp_class, ... + │ nsim_ptp│ nsim_mock_phc_create, nsim_mock_phc_index, ... + │ _mock │ + └────┬────┘ + │ Module.symvers + ┌────▼────┐ + │ dpll/ │ ← builds second + │nsim_dpll│ ← exports nsim_dpll_device_register, nsim_dpll_pin_get, ... + └────┬────┘ + │ Module.symvers + ┌────▼──────┐ + │ netdevsim/│ ← builds last, KBUILD_EXTRA_SYMBOLS from ptp + dpll + │ netdevsim │ ← imports nsim_ptp_* and nsim_dpll_*, no own exports + └───────────┘ +``` + +The top-level `Makefile` enforces this order: + +```makefile +all: + $(MAKE) -C $(KDIR) M=$(PWD)/ptp ... modules + $(MAKE) -C $(KDIR) M=$(PWD)/dpll ... modules + $(MAKE) -C $(KDIR) M=$(PWD)/netdevsim \ + KBUILD_EXTRA_SYMBOLS="$(PWD)/ptp/Module.symvers $(PWD)/dpll/Module.symvers" \ + modules +``` + +--- + +## Symbol Renaming (nsim\_ prefix) + +`include/nsim_rename.h` contains ~70 `#define` macros that rename every +exported and link-visible symbol: + +```c +#define ptp_clock_register nsim_ptp_clock_register +#define dpll_device_register nsim_dpll_device_register +#define mock_phc_create nsim_mock_phc_create +// ... etc +``` + +This is included transitively by every source file via: + +``` +source.c → netdevsim.h (or ptp_private.h) → dkms_compat.h → nsim_rename.h +``` + +The C source files remain **unmodified** from the 6.9.5 kernel; all +renaming happens at the preprocessor level. + +--- + +## Kernel Version Compatibility Layer + +`include/dkms_compat.h` provides shims so the 6.9.5 source compiles on +kernels from **6.8** (Ubuntu 22.04) through **6.17** (Ubuntu 24.04 HWE): + +| Shim | Kernel boundary | What changed | +|------|----------------|--------------| +| `HAVE_POSIX_CLOCK_CONTEXT` | >= 6.8 | PTP chardev gets per-fd context struct | +| `HAVE_PTP_CLOCK_EXTOFF` | >= 6.9 | External-timestamp-as-offset support | +| `PTP_EXTTS_EVENT_VALID` / `PTP_EXT_OFFSET` | < 6.9 → stub 0 | Flags didn't exist in 6.8 | +| `DPLL_DEVICE_CONST` / `DPLL_PIN_CONST` | >= 6.8 | Const-qualified DPLL ops pointers | +| `DPLL_NO_LOCK_STATUS_ERROR` | < 6.9 | Lock status error enum didn't exist | +| `genlmsg_multicast_allns` | < 6.9 / >= 6.12 | Signature changed 3 times across versions | +| `nla_put_sint` | < 6.7 | Inline fallback for missing helper | +| `hrtimer_init` → `hrtimer_setup` | >= 6.15 | Old API removed in favor of combined init | +| `CYCLECOUNTER_READ_CONST` | >= 6.14 | Const qualifier removed from read callback | +| `no_llseek` → `noop_llseek` | >= 6.12 | Symbol removed from fs.h | +| `HAVE_DEBUGFS_GET_AUX` | >= 6.16 | `debugfs_real_fops` removed, new aux API | +| `HAVE_XFRMDEV_OPS_DEV_PARAM` | >= 6.16 | xfrm callbacks gained `net_device *` param | +| `UDP_TUNNEL_NIC_INFO_MAY_SLEEP` | >= 6.17 → stub 0 | Flag removed from tunnel infrastructure | +| `NSIM_ETHTOOL_TS_INFO` | >= 6.11 | `ethtool_ts_info` → `kernel_ethtool_ts_info` | + +Each block is guarded by `LINUX_VERSION_CODE` so the code compiles +cleanly on the native 6.9.x tree as well (no shims active). + +--- + +## DKMS Packaging + +`dkms.conf` registers four modules: + +``` +Module 0: nsim_ptp.ko from ptp/ +Module 1: nsim_ptp_mock.ko from ptp/ +Module 2: nsim_dpll.ko from dpll/ +Module 3: netdevsim.ko from netdevsim/ +``` + +The `POST_INSTALL` hook (`install-udev-rule.sh`) copies the udev rule +into `/etc/udev/rules.d/` and reloads. + +### udev Rule + +`99-nsim-ptp.rules` creates real `/dev/ptpN` device nodes (via `mknod`) +with the same major:minor as the corresponding `/dev/nsim_ptpN` devices. +Real device nodes are used instead of symlinks so they propagate into +containers (Kind nodes, Kubernetes pods): + +``` +SUBSYSTEM=="nsim_ptp", KERNEL=="nsim_ptp[0-9]*", MODE="0666", \ + RUN+="/bin/sh -c 'rm -f /dev/ptp%n; mknod /dev/ptp%n c %M %m; chmod 666 /dev/ptp%n'" +SUBSYSTEM=="nsim_ptp", KERNEL=="nsim_ptp[0-9]*", ACTION=="remove", \ + RUN+="/bin/rm -f /dev/ptp%n" +``` + +`MODE="0666"` is needed because the non-standard major number (234) +falls outside default cgroup device allowlists. Without it, containers +and systemd-sandboxed services would get `EPERM`. + +--- + +## Module Loading Order + +```bash +sudo modprobe gnss # optional, needed by netdevsim +sudo modprobe nsim_ptp # PTP core +sudo modprobe nsim_ptp_mock # mock clocks +sudo modprobe nsim_dpll # DPLL subsystem +sudo modprobe netdevsim pci_bus_nr=0x1f # main driver +``` + +Creating a simulated device: + +```bash +echo "1 0000:1f:02.0 1" | sudo tee /sys/bus/netdevsim/new_device +``` + +Format: ` [num_ports]` + +--- + +## Directory Layout + +``` +netdevsim-dkms/ +├── .github/workflows/ci.yml # GitHub Actions CI +├── include/ +│ ├── dkms_compat.h # Kernel version shims (6.8 → 6.17) +│ ├── nsim_rename.h # nsim_ symbol prefix macros +│ └── linux/ +│ ├── ptp_clock_kernel.h # PTP kernel API header +│ └── ptp_mock.h # Mock PHC API header +├── ptp/ # nsim_ptp.ko + nsim_ptp_mock.ko +│ ├── Makefile +│ ├── ptp_clock.c # PTP core: register, events, index +│ ├── ptp_chardev.c # /dev/nsim_ptpN character device +│ ├── ptp_sysfs.c # Sysfs attributes +│ ├── ptp_vclock.c # Virtual clocks +│ ├── ptp_mock.c # Mock PHC: EXTTS, logical clock IDs +│ └── ptp_private.h # Shared PTP internal header +├── dpll/ # nsim_dpll.ko +│ ├── Makefile +│ ├── dpll_core.c # Device/pin registration +│ ├── dpll_netlink.c # Genetlink interface, change ntf +│ ├── dpll_nl.c # Generated netlink policy +│ └── dpll_*.h # Internal headers +├── netdevsim/ # netdevsim.ko +│ ├── Makefile +│ ├── netdevsim.h # Central struct nsim_dev +│ ├── netdev.c # Module entry, net_device ops +│ ├── dev.c # Device lifecycle, devlink +│ ├── bus.c # Simulated bus, sysfs +│ ├── fakepci.c # Fake PCI host bridge +│ ├── ethtool.c # Ethtool ops +│ ├── fib.c # FIB offload +│ ├── dpll.c # DPLL pin wiring +│ ├── health.c # Devlink health +│ ├── hwstats.c # Hardware statistics +│ ├── udp_tunnels.c # UDP tunnel offload +│ ├── bpf.c # BPF offload (optional) +│ ├── ipsec.c # IPsec/XFRM offload (optional) +│ ├── psample.c # Packet sampling (optional) +│ └── macsec.c # MACsec offload (optional) +├── scripts/ +│ └── test-utm-ubuntu.sh # Local macOS testing via UTM +├── Makefile # Top-level: ordered build, tarball, RPM +├── dkms.conf # DKMS registration (4 modules) +├── install-udev-rule.sh # POST_INSTALL hook +├── 99-nsim-ptp.rules # udev: /dev/ptpN device nodes (mknod, same major:minor) +├── LICENSE # GPL-2.0 +└── README.md +``` + +--- + +## CI Architecture + +### netdevsim-dkms CI (`ci.yml`) + +Runs in the [netdevsim-dkms](https://github.com/redhat-cne/netdevsim-dkms) +repository on every PR and workflow_dispatch: + +``` + ┌─────────────────────────────┐ + │ dkms-test │ + │ matrix: [22.04, 24.04] │ + │ │ + │ Clone → Build DKMS → │ + │ Load modules → Smoke test │ + └──────────┬──────────────────┘ + │ (workflow_dispatch only) + ┌──────────▼──────────────────┐ + │ ptp-images │ + │ ubuntu-22.04 │ + │ │ + │ Build ptp-operator images │ + │ Upload tarball artifact │ + └──────────┬──────────────────┘ + │ + ┌──────────▼──────────────────┐ + │ ptp-test │ + │ matrix: [22.04, 24.04] │ + │ × [oc, bc, ...] │ + │ │ + │ Install DKMS + load images │ + │ Deploy kind cluster │ + │ Run ptp-operator scenario │ + └─────────────────────────────┘ +``` + +### ptp-operator CI (`netdevsim-ci.yml`) + +Runs in the [ptp-operator](https://github.com/k8snetworkplumbingwg/ptp-operator) +repository — inverted: ptp-operator is checked out, netdevsim-dkms is +cloned for DKMS source only. Full e2e runs on every PR (no OIDC or +cloud credentials needed). + +--- + +## Container and Kubernetes Notes + +| Issue | Solution | +|-------|----------| +| Non-standard major (234) blocked by cgroup | udev rule sets `MODE="0666"` | +| `systemd DevicePolicy=closed` | Drop-in: `DeviceAllow=char-* rw`, `DevicePolicy=auto` | +| `/dev/ptpN` missing in pods | udev rule creates real device nodes (mknod); check cgroup allowlist | +| `chrony` vs `chronyd` service name | `run-tests.sh` handles both portably | + +--- + +## Supported Kernel Versions + +| Ubuntu | Kernel | Status | +|--------|--------|--------| +| 22.04 (HWE) | 6.8.x | Tested in CI | +| 24.04 (HWE) | 6.17.x | Tested in CI | +| 6.9.x (native) | 6.9.x | Source origin — no shims needed | diff --git a/README.md b/README.md index 89dd252..d0aade7 100644 --- a/README.md +++ b/README.md @@ -123,14 +123,7 @@ When using netdevsim inside containers or Kubernetes pods: `DeviceAllow=char-* rw` and `DevicePolicy=auto`. - **Kubernetes pods:** The udev rule creates real `/dev/ptpN` device - nodes (not symlinks) so they propagate into containers. If your pod - still doesn't see them (e.g. the container runtime only allowlists - `/dev/nsim_ptp*`), run the helper script inside the pod: - - ```bash - scripts/nsim-ptp-container-setup.sh # one-shot - scripts/nsim-ptp-container-setup.sh --watch # sidecar loop - ``` + nodes (not symlinks) so they propagate into containers. ## Local UTM VMs (macOS) diff --git a/scripts/nsim-ptp-container-setup.sh b/scripts/nsim-ptp-container-setup.sh deleted file mode 100755 index 3ab3e20..0000000 --- a/scripts/nsim-ptp-container-setup.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# Create /dev/ptpN device nodes inside containers where udev rules don't run. -# -# Containers see /dev/nsim_ptpN (propagated by the runtime) but NOT the -# /dev/ptpN nodes created by the host's udev rule. This script recreates -# them so linuxptp tools, ethtool, and the ptp-operator find the devices -# at the standard paths. -# -# Usage: -# As an init container: scripts/nsim-ptp-container-setup.sh -# From a DaemonSet: scripts/nsim-ptp-container-setup.sh --watch -# -# With --watch the script loops every 5 s, useful as a sidecar. -set -euo pipefail - -create_nodes() { - local created=0 - for dev in /dev/nsim_ptp*; do - [ -e "$dev" ] || continue - local n="${dev##*nsim_ptp}" - local target="/dev/ptp${n}" - - if [ -e "$target" ]; then - continue - fi - - local maj min - maj=$(stat -c '%t' "$dev" 2>/dev/null) || continue - min=$(stat -c '%T' "$dev" 2>/dev/null) || continue - mknod "$target" c "0x$maj" "0x$min" 2>/dev/null || true - chmod 666 "$target" 2>/dev/null || true - echo "created $target (c $maj:$min)" - created=$((created + 1)) - done - return $created -} - -create_nodes - -if [ "${1:-}" = "--watch" ]; then - while true; do - sleep 5 - create_nodes 2>/dev/null || true - done -fi From 297fd543eeb375b1ffbb3a43712ee622abc66b83 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 28 May 2026 16:20:17 -0500 Subject: [PATCH 4/6] Point CI to upstream ptp-operator repo --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c0dda6..a788ef2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,8 @@ on: env: DKMS_PKG: netdevsim DKMS_VER: "6.9.5" - PTP_REPO: https://github.com/edcdavid/ptp-operator-upstream.git - PTP_BRANCH: netdevsim-dkms + PTP_REPO: https://github.com/k8snetworkplumbingwg/ptp-operator.git + PTP_BRANCH: main jobs: # ------------------------------------------------------------------ From 4175c1a87edfed677a46c91b0397c2ab20e2716b Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 28 May 2026 16:21:18 -0500 Subject: [PATCH 5/6] Add concurrency group to cancel stale CI runs per PR --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a788ef2..374c15d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,10 @@ on: pull_request: workflow_dispatch: +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: DKMS_PKG: netdevsim DKMS_VER: "6.9.5" From 5f26780d2ce8197f586ac4ad7512e668529d9420 Mon Sep 17 00:00:00 2001 From: David Elie-Dit-Cosaque Date: Thu, 28 May 2026 17:58:15 -0500 Subject: [PATCH 6/6] Prevent podman image GC during parallel builds Keep intermediate containers and layers alive so manifest pushes don't reference deleted image IDs when builds run concurrently. --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 374c15d..05db1d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,6 +183,9 @@ jobs: set -x export PATH=/usr/local/go/bin:/root/go/bin:$PATH export KIND_EXPERIMENTAL_PROVIDER=podman + export BUILDAH_LAYERS=true + export BUILDAH_HISTORY=true + export PODMAN_BUILD_OPTS="--rm=false --force-rm=false" VM_IP=$(hostname -I | awk '{print $1}') cd /root/ptp-operator/scripts