Add unified lab validation interface for Docker and PR testing

dhalperi · dhalperi · commit 68bb91e26b2e · 2025-07-29T16:59:03.000-07:00
- Modify CI workflow to accept multiple Batfish/Pybatfish sources:
  - Git refs (existing behavior + PR testing)
  - Docker images (for release testing)
  - PyPI versions with test/prod distinction
- Create reusable workflow for external repositories
- Add PR comment trigger template for batfish repo
- Add Docker release integration example
- Support lab filtering for selective testing
- Maintain backwards compatibility with existing workflows
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -7,6 +7,30 @@ on:
     branches: [ main, develop ]
   workflow_dispatch:
   workflow_call:
+    inputs:
+      batfish_ref:
+        description: "Git ref for Batfish version to test"
+        required: false
+        default: "master"
+        type: string
+      batfish_docker:
+        description: "Docker image for Batfish (e.g., batfish/batfish:2024.12.20.1234)"
+        required: false
+        type: string
+      pybatfish_ref:
+        description: "Git ref for Pybatfish version to test"
+        required: false
+        default: "master"
+        type: string
+      pybatfish_version:
+        description: "PyPI version for Pybatfish (e.g., 1.2.3)"
+        required: false
+        type: string
+      pybatfish_pypi_repo:
+        description: "PyPI repository (test or prod)"
+        required: false
+        default: "prod"
+        type: string
 
 jobs:
   lint:
@@ -83,16 +107,17 @@ jobs:
         echo "labs=$labs" >> $GITHUB_OUTPUT
         echo "Discovered labs: $labs"
 
-  # Build Batfish JAR once and cache it
+  # Build Batfish JAR once and cache it (only if not using Docker)
   build-batfish:
     runs-on: ubuntu-latest
+    if: inputs.batfish_docker == ''
     steps:
     - name: Checkout Batfish
       uses: actions/checkout@v4
       with:
         repository: batfish/batfish
         path: batfish
-        ref: master
+        ref: ${{ inputs.batfish_ref || 'master' }}
 
     - name: Set up Java 17
       uses: actions/setup-java@v4
@@ -128,8 +153,9 @@ jobs:
   # Matrix job to run each lab in parallel
   lab-integration-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 20  # Reduced since we don't build JAR
+    timeout-minutes: 20
     needs: [discover-labs, build-batfish]
+    if: always() # Run even if build-batfish was skipped (for Docker mode)
     strategy:
       matrix:
         lab: ${{ fromJson(needs.discover-labs.outputs.labs) }}
@@ -155,6 +181,7 @@ jobs:
         java-version: '17'
 
     - name: Download Batfish artifacts
+      if: inputs.batfish_docker == ''
       uses: actions/download-artifact@v4
       with:
         name: batfish-artifacts
@@ -164,28 +191,54 @@ jobs:
       working-directory: lab-validation
       run: |
         python -m pip install --upgrade pip
+
+        # Install base dependencies from requirements.txt
         pip install -r requirements.txt
+
+        # Install Pybatfish based on input parameters (may override version from requirements.txt)
+        if [ "${{ inputs.pybatfish_version }}" != "" ]; then
+          if [ "${{ inputs.pybatfish_pypi_repo }}" = "test" ]; then
+            pip install -i https://test.pypi.org/simple --extra-index-url https://pypi.org/simple pybatfish==${{ inputs.pybatfish_version }}
+          else
+            pip install pybatfish==${{ inputs.pybatfish_version }}
+          fi
+        else
+          # Default: install from git ref (override requirements.txt version)
+          PYBF_REF="${{ inputs.pybatfish_ref || 'master' }}"
+          pip install git+https://github.com/batfish/pybatfish.git@${PYBF_REF}
+        fi
+
         pip install -r requirements-dev.txt
         pip install -e .
 
     - name: Start Batfish service
       run: |
-        # Extract questions archive
-        tar -xzf questions.tgz
-
-        # Start Batfish using simple allinone mode (like pybatfish CI)
-        coordinator_args=(\
-          -templatedirs=questions \
-          -periodassignworkms=5 \
-        )
-        allinone_args=(\
-          -runclient=false \
-          -coordinatorargs="$(echo -n "${coordinator_args[@]}")" \
-        )
-        java -cp allinone.jar org.batfish.allinone.Main "${allinone_args[@]}" > batfish.log 2>&1 &
-
-        # Wait for service to start
-        sleep 10
+        if [ "${{ inputs.batfish_docker }}" != "" ]; then
+          # Start Batfish using Docker container
+          docker run -d --name batfish \
+            -p 9996:9996 \
+            ${{ inputs.batfish_docker }}
+
+          # Wait for service to start
+          sleep 15
+        else
+          # Extract questions archive and start from JAR
+          tar -xzf questions.tgz
+
+          # Start Batfish using simple allinone mode (like pybatfish CI)
+          coordinator_args=(\
+            -templatedirs=questions \
+            -periodassignworkms=5 \
+          )
+          allinone_args=(\
+            -runclient=false \
+            -coordinatorargs="$(echo -n "${coordinator_args[@]}")" \
+          )
+          java -cp allinone.jar org.batfish.allinone.Main "${allinone_args[@]}" > batfish.log 2>&1 &
+
+          # Wait for service to start
+          sleep 10
+        fi
 
     - name: Run lab integration test - ${{ matrix.lab }}
       working-directory: lab-validation
@@ -196,7 +249,11 @@ jobs:
       if: failure()
       run: |
         echo "=== Batfish logs ==="
-        tail -100 batfish.log || echo "No batfish log found"
+        if [ "${{ inputs.batfish_docker }}" != "" ]; then
+          docker logs batfish || echo "No Docker container logs found"
+        else
+          tail -100 batfish.log || echo "No batfish log found"
+        fi
 
   # Summary job for required status checks
   all-labs-passed:
diff --git a/.github/workflows/docker-release-integration-example.yml b/.github/workflows/docker-release-integration-example.yml
@@ -0,0 +1,30 @@
+# This file is an example of how to integrate lab validation into the docker release process
+# Add this job to docker/.github/workflows/reusable-upload.yml after artifact building
+
+  # Add this job to the docker upload workflow after dev artifacts are built
+  lab-validation-test:
+    runs-on: ubuntu-latest
+    needs: [dev_image_upload, dev_whl_upload]
+    if: inputs.BATFISH_GITHUB_BATFISH_REPO == 'batfish/batfish' && inputs.BATFISH_GITHUB_PYBATFISH_REPO == 'batfish/pybatfish'
+    steps:
+    - name: Call lab validation with test artifacts
+      uses: batfish/lab-validation/.github/workflows/reusable-lab-validation.yml@main
+      with:
+        batfish_docker: "batfish/batfish:${{ inputs.test_tag }}"
+        pybatfish_version: "${{ inputs.bf_version }}"
+        pybatfish_pypi_repo: "test"
+
+  # Update prod upload jobs to depend on lab validation passing
+  prod_image_upload:
+    if: inputs.queue_prod_release && inputs.BATFISH_GITHUB_BATFISH_REPO == 'batfish/batfish' && inputs.BATFISH_GITHUB_PYBATFISH_REPO == 'batfish/pybatfish'
+    environment: Release Containers
+    runs-on: ubuntu-latest
+    needs: [lab-validation-test]  # Add this dependency
+    # ... rest of existing job
+
+  prod_whl_upload:
+    if: inputs.queue_prod_release && inputs.BATFISH_GITHUB_BATFISH_REPO == 'batfish/batfish' && inputs.BATFISH_GITHUB_PYBATFISH_REPO == 'batfish/pybatfish'
+    environment: Release Pybatfish
+    runs-on: ubuntu-latest
+    needs: [lab-validation-test]  # Add this dependency
+    # ... rest of existing job
diff --git a/.github/workflows/pr-comment-lab-validation.yml b/.github/workflows/pr-comment-lab-validation.yml
@@ -0,0 +1,106 @@
+# This file is a template for adding lab validation to Batfish PRs via comments
+# Copy this to batfish/.github/workflows/ to enable PR-triggered lab validation
+
+name: PR Comment Lab Validation
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  # Check if comment is on a PR and from authorized user
+  check-comment:
+    runs-on: ubuntu-latest
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/test-labs')
+    outputs:
+      should_run: ${{ steps.check.outputs.should_run }}
+      lab_filter: ${{ steps.parse.outputs.lab_filter }}
+      pr_ref: ${{ steps.pr.outputs.ref }}
+    steps:
+    - name: Check permissions
+      id: check
+      run: |
+        # Only allow repo collaborators to trigger lab validation
+        # You may want to customize this permission check
+        echo "should_run=true" >> $GITHUB_OUTPUT
+
+    - name: Get PR ref
+      id: pr
+      run: |
+        PR_NUMBER=${{ github.event.issue.number }}
+        echo "ref=refs/pull/$PR_NUMBER/head" >> $GITHUB_OUTPUT
+
+    - name: Parse lab filter
+      id: parse
+      run: |
+        COMMENT="${{ github.event.comment.body }}"
+        # Extract lab filter from comment (e.g., "/test-labs eos_*" -> "eos_*")
+        if [[ "$COMMENT" =~ /test-labs[[:space:]]+([^[:space:]]+) ]]; then
+          LAB_FILTER="${BASH_REMATCH[1]}"
+        else
+          LAB_FILTER=""
+        fi
+        echo "lab_filter=$LAB_FILTER" >> $GITHUB_OUTPUT
+        echo "Parsed lab filter: '$LAB_FILTER'"
+
+  # Add reaction to indicate the workflow started
+  react-start:
+    runs-on: ubuntu-latest
+    needs: check-comment
+    if: needs.check-comment.outputs.should_run == 'true'
+    steps:
+    - name: Add eyes reaction
+      run: |
+        curl -X POST \
+          -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
+          -d '{"content":"eyes"}'
+
+  # Call the lab validation workflow
+  run-lab-validation:
+    needs: [check-comment, react-start]
+    if: needs.check-comment.outputs.should_run == 'true'
+    uses: batfish/lab-validation/.github/workflows/reusable-lab-validation.yml@main
+    with:
+      batfish_ref: ${{ needs.check-comment.outputs.pr_ref }}
+      pybatfish_ref: master
+      lab_filter: ${{ needs.check-comment.outputs.lab_filter }}
+
+  # React based on results
+  react-result:
+    runs-on: ubuntu-latest
+    needs: [check-comment, run-lab-validation]
+    if: always() && needs.check-comment.outputs.should_run == 'true'
+    steps:
+    - name: Add success reaction
+      if: needs.run-lab-validation.result == 'success'
+      run: |
+        curl -X POST \
+          -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
+          -d '{"content":"+1"}'
+
+    - name: Add failure reaction
+      if: needs.run-lab-validation.result == 'failure'
+      run: |
+        curl -X POST \
+          -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
+          -d '{"content":"-1"}'
+
+    - name: Comment with results
+      run: |
+        if [ "${{ needs.run-lab-validation.result }}" = "success" ]; then
+          COMMENT="✅ Lab validation passed!"
+        else
+          COMMENT="❌ Lab validation failed. Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
+        fi
+
+        curl -X POST \
+          -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          https://api.github.com/repos/${{ github.repository }}/issues/comments \
+          -d "{\"body\":\"$COMMENT\"}"
diff --git a/.github/workflows/reusable-lab-validation.yml b/.github/workflows/reusable-lab-validation.yml
