fix(ci): exclude golden tests from matrix + fix beta channel version pin #66
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Visual Regression | |
| # ───────────────────────────────────────────────────────────────────────────── | |
| # Golden-test stability contract | |
| # | |
| # All jobs MUST run on the SAME pinned runner image and the SAME Flutter | |
| # version so that font metrics, sub-pixel rounding, and Skia rasterisation | |
| # are byte-identical between the machine that generated the reference PNGs | |
| # and the machine that compares against them. | |
| # | |
| # Pinning strategy: | |
| # • runs-on: ubuntu-22.04 — exact LTS release, never rolls | |
| # • flutter-version: <exact tag> — exact Flutter SDK tag | |
| # • Noto font family installed — consistent CJK / Arabic / Latin glyphs | |
| # | |
| # To update the Flutter pin: bump FLUTTER_VERSION here AND re-run | |
| # the "Update Goldens" workflow_dispatch job so references are regenerated | |
| # on the new version before the pin is merged. | |
| # ───────────────────────────────────────────────────────────────────────────── | |
| env: | |
| FLUTTER_VERSION: "3.41.5" # ← single source of truth; bump here + update-goldens | |
| FONT_PACKAGES: >- | |
| fonts-noto | |
| fonts-noto-cjk | |
| fonts-noto-color-emoji | |
| fonts-roboto | |
| on: | |
| pull_request: | |
| branches: [main, develop] | |
| push: | |
| branches: [main] | |
| workflow_dispatch: # manual trigger for update-goldens job | |
| jobs: | |
| # ── Compare against stored references ────────────────────────────────────── | |
| golden: | |
| name: Golden Tests | |
| runs-on: ubuntu-22.04 # pinned — do NOT change to ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install pinned Noto fonts | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y --no-install-recommends ${{ env.FONT_PACKAGES }} | |
| # Rebuild font cache so Flutter picks up the newly installed fonts | |
| fc-cache -fv | |
| echo "Installed fonts:" | |
| fc-list | grep -E "Noto|Roboto" | sort | |
| - name: Setup Flutter (pinned version) | |
| uses: subosito/flutter-action@v2 | |
| with: | |
| flutter-version: ${{ env.FLUTTER_VERSION }} | |
| channel: stable | |
| cache: true | |
| - name: Get dependencies | |
| run: flutter pub get | |
| - name: Run golden tests | |
| id: golden | |
| run: | | |
| flutter test test/golden/ \ | |
| --reporter json \ | |
| 2>&1 | tee golden_results.json || true | |
| FAILURES=$(python3 -c " | |
| import json, sys | |
| data = open('golden_results.json').read() | |
| errors = data.count('\"result\":\"error\"') | |
| print(errors) | |
| " 2>/dev/null || echo 0) | |
| echo "failures=$FAILURES" >> "$GITHUB_OUTPUT" | |
| if [ "$FAILURES" -gt "0" ]; then | |
| echo "::error::$FAILURES golden test(s) failed — pixel mismatch detected" | |
| exit 1 | |
| fi | |
| - name: Upload failure diffs | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: golden-failures-${{ github.run_number }} | |
| path: | | |
| test/golden/goldens/failures/ | |
| golden_results.json | |
| retention-days: 14 | |
| - name: Post PR comment on failure | |
| if: failure() && github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const failures = '${{ steps.golden.outputs.failures }}'; | |
| const flutterVersion = process.env.FLUTTER_VERSION; | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: [ | |
| '## ❌ Visual Regression Detected', | |
| '', | |
| `**${failures} golden test(s) failed** on Flutter ${flutterVersion} / ubuntu-22.04.`, | |
| '', | |
| 'The rendered output no longer matches the reference images.', | |
| '', | |
| '**If the change is intentional**, regenerate the goldens on the', | |
| 'same platform via the _Update Goldens_ workflow dispatch, or run', | |
| 'locally in Docker:', | |
| '```bash', | |
| 'docker run --rm \\', | |
| ' -v $(pwd):/workspace -w /workspace \\', | |
| ' ghcr.io/cirruslabs/flutter:' + flutterVersion + ' \\', | |
| ' bash -c "apt-get update -qq && \\', | |
| ' apt-get install -y fonts-noto fonts-noto-cjk fonts-roboto && \\', | |
| ' flutter pub get && \\', | |
| ' flutter test test/golden/ --update-goldens"', | |
| 'git add test/golden/goldens/', | |
| 'git commit -m "chore: update golden references (Flutter ' + flutterVersion + ')"', | |
| '```', | |
| '', | |
| '> ⚠️ Always regenerate goldens on **ubuntu-22.04** with **Flutter', | |
| `> ${flutterVersion}** to keep references pixel-stable across machines.`, | |
| ].join('\n'), | |
| }); | |
| # ── Regenerate reference images (manual workflow_dispatch only) ──────────── | |
| update-goldens: | |
| name: Update Goldens (manual) | |
| runs-on: ubuntu-22.04 # MUST match the comparison job above | |
| if: github.event_name == 'workflow_dispatch' | |
| permissions: | |
| contents: write # required to push the regenerated golden PNGs | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install pinned Noto fonts | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y --no-install-recommends ${{ env.FONT_PACKAGES }} | |
| fc-cache -fv | |
| - name: Setup Flutter (pinned version) | |
| uses: subosito/flutter-action@v2 | |
| with: | |
| flutter-version: ${{ env.FLUTTER_VERSION }} | |
| channel: stable | |
| cache: true | |
| - name: Get dependencies | |
| run: flutter pub get | |
| - name: Regenerate goldens | |
| run: flutter test test/golden/ --update-goldens | |
| - name: Commit updated goldens | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add test/golden/goldens/ | |
| git diff --staged --quiet || \ | |
| git commit -m "chore: regenerate golden references (Flutter ${{ env.FLUTTER_VERSION }}) [skip ci]" | |
| git push |