JUnit summary with failures

Author: stephanos

.github/actions/get-job-id/action.yml

@@ -31,27 +31,40 @@ runs:
       run: |
         set -euo pipefail
         job_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100"
-        json=$(curl -sSL \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer $GITHUB_TOKEN" \
-          "$job_url")
+        job_id=""
+        json=""
+        for attempt in 1 2 3 4 5 6; do
+          json=$(curl -sSL \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer $GITHUB_TOKEN" \
+            "$job_url")
 
-        # Prefer matching both job name and the current runner name to disambiguate matrix jobs
-        job_id=$(jq -r --arg name "$JOB_NAME" --arg runner "$RUNNER_NAME" '
-          (.jobs // [])
-          | map(select(.name == $name and (.runner_name // "") == $runner))
-          | (.[0].id // empty)
-        ' <<< "$json" )
-
-        # Fallback: match by name only
-        if [ -z "${job_id:-}" ]; then
-          job_id=$(jq -r --arg name "$JOB_NAME" '
-            (.jobs // []) | map(select(.name == $name)) | (.[0].id // empty)
+          # Prefer matching both job name and the current runner name to disambiguate matrix jobs
+          job_id=$(jq -r --arg name "$JOB_NAME" --arg runner "$RUNNER_NAME" '
+            (.jobs // [])
+            | map(select(.name == $name and (.runner_name // "") == $runner))
+            | (.[0].id // empty)
           ' <<< "$json" )
-        fi
+
+          # Fallback: match by name only
+          if [ -z "${job_id:-}" ]; then
+            job_id=$(jq -r --arg name "$JOB_NAME" '
+              (.jobs // []) | map(select(.name == $name)) | (.[0].id // empty)
+            ' <<< "$json" )
+          fi
+
+          if [ -n "${job_id:-}" ] && [ "$job_id" != "null" ]; then
+            break
+          fi
+
+          if [ "$attempt" -lt 6 ]; then
+            echo "::notice::Job ID for '$JOB_NAME' not visible yet (attempt $attempt/6); retrying in 5s"
+            sleep 5
+          fi
+        done
 
         if [ -z "${job_id:-}" ] || [ "$job_id" = "null" ]; then
-          echo "::error::Failed to resolve job ID for name '$JOB_NAME' on runner '$RUNNER_NAME' in run '$RUN_ID'" >&2
+          echo "::error::Failed to resolve job ID for name '$JOB_NAME' on runner '$RUNNER_NAME' in run '$RUN_ID' after retries" >&2
           exit 1
         fi
         echo "job_id=$job_id" >> "$GITHUB_OUTPUT"

.github/workflows/run-tests.yml

@@ -314,9 +314,18 @@ jobs:
           path: ~/.cache/go-build
           key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
 
+      - name: Get job ID
+        id: get_job_id
+        uses: ./.github/actions/get-job-id
+        with:
+          job_name: Unit test
+          run_id: ${{ github.run_id }}
+
       - name: Run unit tests
         timeout-minutes: 20
         run: TEST_TIMEOUT=15m ./develop/github/monitor_test.sh make unit-test-coverage
+        env:
+          GITHUB_JOB_ID: ${{ steps.get_job_id.outputs.job_id }}
 
       - name: Print memory snapshot
         if: always()
@@ -328,16 +337,6 @@ jobs:
           [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
             CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
 
-      - name: Generate test summary
-        uses: mikepenz/action-junit-report@v6
-        if: ${{ !cancelled() }}
-        with:
-          report_paths: ./.testoutput/junit.*.xml
-          detailed_summary: true
-          check_annotations: false
-          annotate_only: true
-          skip_annotations: true
-
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v5
         with:
@@ -354,13 +353,6 @@ jobs:
           flags: unit-test
           report_type: test_results
 
-      - name: Get job ID
-        id: get_job_id
-        uses: ./.github/actions/get-job-id
-        with:
-          job_name: Unit test
-          run_id: ${{ github.run_id }}
-
       - name: Upload test results to GitHub
         # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
         uses: actions/upload-artifact@v6
@@ -414,9 +406,18 @@ jobs:
           # shellcheck disable=SC2046
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up --wait $(docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} ps --services)
 
+      - name: Get job ID
+        id: get_job_id
+        uses: ./.github/actions/get-job-id
+        with:
+          job_name: Integration test
+          run_id: ${{ github.run_id }}
+
       - name: Run integration test
         timeout-minutes: 15
         run: ./develop/github/monitor_test.sh make integration-test-coverage
+        env:
+          GITHUB_JOB_ID: ${{ steps.get_job_id.outputs.job_id }}
 
       - name: Print memory snapshot
         if: always()
@@ -428,16 +429,6 @@ jobs:
           [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
             CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
 
-      - name: Generate test summary
-        uses: mikepenz/action-junit-report@v6
-        if: ${{ !cancelled() }}
-        with:
-          report_paths: ./.testoutput/junit.*.xml
-          detailed_summary: true
-          check_annotations: false
-          annotate_only: true
-          skip_annotations: true
-
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v5
         with:
@@ -454,13 +445,6 @@ jobs:
           flags: integration-test
           report_type: test_results
 
-      - name: Get job ID
-        id: get_job_id
-        uses: ./.github/actions/get-job-id
-        with:
-          job_name: Integration test
-          run_id: ${{ github.run_id }}
-
       - name: Upload test results to GitHub
         # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
         uses: actions/upload-artifact@v6
@@ -547,6 +531,7 @@ jobs:
         timeout-minutes: ${{ matrix.github_timeout }}
         run: ./develop/github/monitor_test.sh ${{ matrix.cmd }}
         env:
+          GITHUB_JOB_ID: ${{ steps.get_job_id.outputs.job_id }}
           TEST_TOTAL_SHARDS: ${{ matrix.total_shards }}
           TEST_SHARD_INDEX: ${{ matrix.total_shards && matrix.shard_index }} # guard with total_shards to avoid falsy eval of shard_index=0
           TEST_ARGS: "${{ matrix.test_args }}"
@@ -567,16 +552,6 @@ jobs:
           [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
             CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
 
-      - name: Generate test summary
-        uses: mikepenz/action-junit-report@v6
-        if: ${{ !cancelled() }}
-        with:
-          report_paths: ./.testoutput/junit.*.xml
-          detailed_summary: true
-          check_annotations: false
-          annotate_only: true
-          skip_annotations: true
-
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v5
         with:

tools/testrunner/junit.go

@@ -13,6 +13,11 @@ import (
 	"github.com/jstemmer/go-junit-report/v2/junit"
 )
 
+// alertsSuiteName is the JUnit suite name used for structural alerts (data
+// races, panics, fatal errors). It is also used by collectSummaryRows to
+// distinguish alert entries from regular test failures.
+const alertsSuiteName = "ALERTS"
+
 type junitReport struct {
 	junit.Testsuites
 	path          string
@@ -83,19 +88,22 @@ func (j *junitReport) appendAlertsSuite(alerts []alert) {
 		if p := primaryTestName(a.Tests); p != "" {
 			name = fmt.Sprintf("%s — in %s", name, p)
 		}
-		// Include only test names for context, not the full log details to avoid XML malformation
-		var details string
+		var sb strings.Builder
+		if a.Details != "" {
+			sb.WriteString(sanitizeXML(a.Details))
+			sb.WriteByte('\n')
+		}
 		if len(a.Tests) > 0 {
-			details = fmt.Sprintf("Detected in tests:\n\t%s", strings.Join(a.Tests, "\n\t"))
+			fmt.Fprintf(&sb, "Detected in tests:\n\t%s", strings.Join(a.Tests, "\n\t"))
 		}
-		r := &junit.Result{Message: string(a.Kind), Data: details}
+		r := &junit.Result{Message: string(a.Kind), Data: strings.TrimRight(sb.String(), "\n")}
 		cases = append(cases, junit.Testcase{
 			Name:    name,
 			Failure: r,
 		})
 	}
 	suite := junit.Testsuite{
-		Name:      "ALERTS",
+		Name:      alertsSuiteName,
 		Failures:  len(cases),
 		Tests:     len(cases),
 		Testcases: cases,
@@ -105,6 +113,21 @@ func (j *junitReport) appendAlertsSuite(alerts []alert) {
 	j.Tests += suite.Tests
 }
 
+// sanitizeXML removes characters that are invalid in XML 1.0. Go's xml.Encoder
+// escapes <, >, & etc., but control characters other than \t, \n, \r are not
+// legal XML and cause parsers to reject the document.
+func sanitizeXML(s string) string {
+	return strings.Map(func(r rune) rune {
+		if r == '\t' || r == '\n' || r == '\r' {
+			return r
+		}
+		if r < 0x20 || r == 0xFFFE || r == 0xFFFF {
+			return -1
+		}
+		return r
+	}, s)
+}
+
 // dedupeAlerts removes duplicate alerts (e.g., repeated across retries) based
 // on kind and details while preserving the first-seen order.
 func dedupeAlerts(alerts []alert) []alert {
@@ -224,6 +247,9 @@ func mergeReports(reports []*junitReport) (*junitReport, error) {
 					// Discard test case parents since they provide no value.
 					continue
 				}
+				if testCase.Failure != nil && testCase.Failure.Data != "" {
+					testCase.Failure.Data = parseFailureDetails(testCase.Failure.Data)
+				}
 				testCase.Name += suffix
 				newSuite.Testcases = append(newSuite.Testcases, testCase)
 			}

tools/testrunner/junit_test.go

@@ -73,6 +73,19 @@ func TestMergeReports_SingleReport(t *testing.T) {
 	require.Len(t, testNames, 5)
 	require.NotContains(t, testNames, "TestCallbacksSuite")
 	require.NotContains(t, testNames, "TestCallbacksSuite/TestWorkflowNexusCallbacks_CarriedOver")
+	var failureData string
+	for _, tc := range suites[0].Testcases {
+		if tc.Name == "TestCallbacksSuite/TestWorkflowCallbacks_InvalidArgument" {
+			require.NotNil(t, tc.Failure)
+			failureData = tc.Failure.Data
+			break
+		}
+	}
+	require.NotEmpty(t, failureData)
+	require.Contains(t, failureData, "Error Trace:")
+	require.Contains(t, failureData, "expected: 1")
+	require.NotContains(t, failureData, "=== RUN")
+	require.NotContains(t, failureData, "--- FAIL:")
 }
 
 func TestMergeReports_MultipleReports(t *testing.T) {