JUnit summary with failures

Author: stephanos

.github/actions/get-job-id/action.yml

@@ -30,28 +30,44 @@ runs:
         # The script will first try to match by (name, runner_name), then fall back to name-only.
       run: |
         set -euo pipefail
+
         job_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100"
-        json=$(curl -sSL \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer $GITHUB_TOKEN" \
-          "$job_url")
+        max_retries=6
+        job_id=""
+        json=""
 
-        # Prefer matching both job name and the current runner name to disambiguate matrix jobs
-        job_id=$(jq -r --arg name "$JOB_NAME" --arg runner "$RUNNER_NAME" '
-          (.jobs // [])
-          | map(select(.name == $name and (.runner_name // "") == $runner))
-          | (.[0].id // empty)
-        ' <<< "$json" )
+        for ((attempt = 1; attempt <= max_retries; attempt++)); do
+          json=$(curl -sSL \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer $GITHUB_TOKEN" \
+            "$job_url")
 
-        # Fallback: match by name only
-        if [ -z "${job_id:-}" ]; then
-          job_id=$(jq -r --arg name "$JOB_NAME" '
-            (.jobs // []) | map(select(.name == $name)) | (.[0].id // empty)
+          # Prefer matching both job name and the current runner name to disambiguate matrix jobs
+          job_id=$(jq -r --arg name "$JOB_NAME" --arg runner "$RUNNER_NAME" '
+            (.jobs // [])
+            | map(select(.name == $name and (.runner_name // "") == $runner))
+            | (.[0].id // empty)
           ' <<< "$json" )
-        fi
+
+          # Fallback: match by name only
+          if [ -z "${job_id:-}" ]; then
+            job_id=$(jq -r --arg name "$JOB_NAME" '
+              (.jobs // []) | map(select(.name == $name)) | (.[0].id // empty)
+            ' <<< "$json" )
+          fi
+
+          if [ -n "${job_id:-}" ] && [ "$job_id" != "null" ]; then
+            break
+          fi
+
+          if [ "$attempt" -lt "$max_retries" ]; then
+            echo "::notice::Job ID for '$JOB_NAME' not visible yet (attempt $attempt/$max_retries); retrying in 5s"
+            sleep 5
+          fi
+        done
 
         if [ -z "${job_id:-}" ] || [ "$job_id" = "null" ]; then
-          echo "::error::Failed to resolve job ID for name '$JOB_NAME' on runner '$RUNNER_NAME' in run '$RUN_ID'" >&2
+          echo "::error::Failed to resolve job ID for name '$JOB_NAME' on runner '$RUNNER_NAME' in run '$RUN_ID' after retries" >&2
           exit 1
         fi
         echo "job_id=$job_id" >> "$GITHUB_OUTPUT"

.github/workflows/run-tests.yml

@@ -314,6 +314,13 @@ jobs:
           path: ~/.cache/go-build
           key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}
 
+      - name: Get job ID
+        id: get_job_id
+        uses: ./.github/actions/get-job-id
+        with:
+          job_name: Unit test
+          run_id: ${{ github.run_id }}
+
       - name: Run unit tests
         timeout-minutes: 20
         run: TEST_TIMEOUT=15m ./develop/github/monitor_test.sh make unit-test-coverage
@@ -328,15 +335,13 @@ jobs:
           [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
             CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
 
-      - name: Generate test summary
-        uses: mikepenz/action-junit-report@v6
+      - name: Write test summary
         if: ${{ !cancelled() }}
-        with:
-          report_paths: ./.testoutput/junit.*.xml
-          detailed_summary: true
-          check_annotations: false
-          annotate_only: true
-          skip_annotations: true
+        run: |
+          summary="$(make -s write-test-summary)"
+          if [ -n "$summary" ]; then
+            printf '%s\n' "$summary" > "$GITHUB_STEP_SUMMARY"
+          fi
 
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v5
@@ -354,13 +359,6 @@ jobs:
           flags: unit-test
           report_type: test_results
 
-      - name: Get job ID
-        id: get_job_id
-        uses: ./.github/actions/get-job-id
-        with:
-          job_name: Unit test
-          run_id: ${{ github.run_id }}
-
       - name: Upload test results to GitHub
         # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
         uses: actions/upload-artifact@v6
@@ -414,6 +412,13 @@ jobs:
           # shellcheck disable=SC2046
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up --wait $(docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} ps --services)
 
+      - name: Get job ID
+        id: get_job_id
+        uses: ./.github/actions/get-job-id
+        with:
+          job_name: Integration test
+          run_id: ${{ github.run_id }}
+
       - name: Run integration test
         timeout-minutes: 15
         run: ./develop/github/monitor_test.sh make integration-test-coverage
@@ -428,15 +433,13 @@ jobs:
           [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
             CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
 
-      - name: Generate test summary
-        uses: mikepenz/action-junit-report@v6
+      - name: Write test summary
         if: ${{ !cancelled() }}
-        with:
-          report_paths: ./.testoutput/junit.*.xml
-          detailed_summary: true
-          check_annotations: false
-          annotate_only: true
-          skip_annotations: true
+        run: |
+          summary="$(make -s write-test-summary)"
+          if [ -n "$summary" ]; then
+            printf '%s\n' "$summary" > "$GITHUB_STEP_SUMMARY"
+          fi
 
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v5
@@ -454,13 +457,6 @@ jobs:
           flags: integration-test
           report_type: test_results
 
-      - name: Get job ID
-        id: get_job_id
-        uses: ./.github/actions/get-job-id
-        with:
-          job_name: Integration test
-          run_id: ${{ github.run_id }}
-
       - name: Upload test results to GitHub
         # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
         uses: actions/upload-artifact@v6
@@ -567,15 +563,13 @@ jobs:
           [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
             CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
 
-      - name: Generate test summary
-        uses: mikepenz/action-junit-report@v6
+      - name: Write test summary
         if: ${{ !cancelled() }}
-        with:
-          report_paths: ./.testoutput/junit.*.xml
-          detailed_summary: true
-          check_annotations: false
-          annotate_only: true
-          skip_annotations: true
+        run: |
+          summary="$(make -s write-test-summary)"
+          if [ -n "$summary" ]; then
+            printf '%s\n' "$summary" > "$GITHUB_STEP_SUMMARY"
+          fi
 
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v5

Makefile

@@ -557,6 +557,10 @@ report-test-crash: $(TEST_OUTPUT_ROOT)
 		--junitfile=$(TEST_OUTPUT_ROOT)/junit.crash.xml \
 		--crashreportname=$(CRASH_REPORT_NAME)
 
+write-test-summary: $(TEST_OUTPUT_ROOT)
+	@go run ./cmd/tools/test-runner write-summary \
+		--junit-glob=$(TEST_OUTPUT_ROOT)/junit.*.xml
+
 ##### Schema #####
 install-schema-cass-es: temporal-cassandra-tool install-schema-es
 	@printf $(COLOR) "Install Cassandra schema..."

tools/testrunner/junit.go

@@ -13,6 +13,12 @@ import (
 	"github.com/jstemmer/go-junit-report/v2/junit"
 )
 
+// alertsSuiteName is the JUnit suite name used for structural alerts (data
+// races, panics, fatal errors). It is also used by collectSummaryRows to
+// distinguish alert entries from regular test failures.
+const alertsSuiteName = "ALERTS"
+const junitAlertDetailsMaxBytes = 64 * 1024
+
 type junitReport struct {
 	junit.Testsuites
 	path          string
@@ -83,19 +89,22 @@ func (j *junitReport) appendAlertsSuite(alerts []alert) {
 		if p := primaryTestName(a.Tests); p != "" {
 			name = fmt.Sprintf("%s — in %s", name, p)
 		}
-		// Include only test names for context, not the full log details to avoid XML malformation
-		var details string
+		var sb strings.Builder
+		if a.Details != "" {
+			sb.WriteString(truncateAlertDetails(sanitizeXML(a.Details)))
+			sb.WriteByte('\n')
+		}
 		if len(a.Tests) > 0 {
-			details = fmt.Sprintf("Detected in tests:\n\t%s", strings.Join(a.Tests, "\n\t"))
+			fmt.Fprintf(&sb, "Detected in tests:\n\t%s", strings.Join(a.Tests, "\n\t"))
 		}
-		r := &junit.Result{Message: string(a.Kind), Data: details}
+		r := &junit.Result{Message: string(a.Kind), Data: strings.TrimRight(sb.String(), "\n")}
 		cases = append(cases, junit.Testcase{
 			Name:    name,
 			Failure: r,
 		})
 	}
 	suite := junit.Testsuite{
-		Name:      "ALERTS",
+		Name:      alertsSuiteName,
 		Failures:  len(cases),
 		Tests:     len(cases),
 		Testcases: cases,
@@ -105,6 +114,35 @@ func (j *junitReport) appendAlertsSuite(alerts []alert) {
 	j.Tests += suite.Tests
 }
 
+// sanitizeXML removes characters that are invalid in XML 1.0. Go's xml.Encoder
+// escapes <, >, & etc., but control characters other than \t, \n, \r are not
+// legal XML and cause parsers to reject the document.
+func sanitizeXML(s string) string {
+	return strings.Map(func(r rune) rune {
+		if r == '\t' || r == '\n' || r == '\r' {
+			return r
+		}
+		if r < 0x20 || r == 0xFFFE || r == 0xFFFF {
+			return -1
+		}
+		return r
+	}, s)
+}
+
+func truncateAlertDetails(s string) string {
+	if len(s) <= junitAlertDetailsMaxBytes {
+		return s
+	}
+	const marker = "\n... (truncated) ...\n"
+	if junitAlertDetailsMaxBytes <= len(marker) {
+		return s[:junitAlertDetailsMaxBytes]
+	}
+	remaining := junitAlertDetailsMaxBytes - len(marker)
+	head := remaining / 2
+	tail := remaining - head
+	return s[:head] + marker + s[len(s)-tail:]
+}
+
 // dedupeAlerts removes duplicate alerts (e.g., repeated across retries) based
 // on kind and details while preserving the first-seen order.
 func dedupeAlerts(alerts []alert) []alert {
@@ -224,6 +262,11 @@ func mergeReports(reports []*junitReport) (*junitReport, error) {
 					// Discard test case parents since they provide no value.
 					continue
 				}
+				if testCase.Failure != nil && testCase.Failure.Data != "" {
+					if details := parseFailureDetails(testCase.Failure.Data); details != noFailureDetails {
+						testCase.Failure.Data = details
+					}
+				}
 				testCase.Name += suffix
 				newSuite.Testcases = append(newSuite.Testcases, testCase)
 			}

tools/testrunner/junit_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"os"
 	"slices"
+	"strings"
 	"testing"
 
 	"github.com/jstemmer/go-junit-report/v2/junit"
@@ -73,6 +74,19 @@ func TestMergeReports_SingleReport(t *testing.T) {
 	require.Len(t, testNames, 5)
 	require.NotContains(t, testNames, "TestCallbacksSuite")
 	require.NotContains(t, testNames, "TestCallbacksSuite/TestWorkflowNexusCallbacks_CarriedOver")
+	var failureData string
+	for _, tc := range suites[0].Testcases {
+		if tc.Name == "TestCallbacksSuite/TestWorkflowCallbacks_InvalidArgument" {
+			require.NotNil(t, tc.Failure)
+			failureData = tc.Failure.Data
+			break
+		}
+	}
+	require.NotEmpty(t, failureData)
+	require.Contains(t, failureData, "Error Trace:")
+	require.Contains(t, failureData, "expected: 1")
+	require.NotContains(t, failureData, "=== RUN")
+	require.NotContains(t, failureData, "--- FAIL:")
 }
 
 func TestMergeReports_MultipleReports(t *testing.T) {
@@ -164,6 +178,56 @@ func TestAppendAlertsSuite(t *testing.T) {
 	requireReportEquals(t, "testdata/junit-alerts-output.xml", out.Name())
 }
 
+func TestAppendAlertsSuite_TruncatesLargeDetails(t *testing.T) {
+	j := &junitReport{}
+	details := "panic: large panic\n" + strings.Repeat("x", junitAlertDetailsMaxBytes) + "\ntrailing sentinel"
+	j.appendAlertsSuite([]alert{{
+		Kind:    alertKindPanic,
+		Summary: "large panic",
+		Details: details,
+		Tests:   []string{"TestLargePanic"},
+	}})
+
+	require.Len(t, j.Suites, 1)
+	require.Len(t, j.Suites[0].Testcases, 1)
+	got := j.Suites[0].Testcases[0].Failure.Data
+	require.Contains(t, got, "... (truncated) ...")
+	require.Contains(t, got, "panic: large panic")
+	require.Contains(t, got, "trailing sentinel")
+	require.LessOrEqual(t, len(got), junitAlertDetailsMaxBytes+len("\nDetected in tests:\n\tTestLargePanic"))
+}
+
+func TestMergeReports_PreservesOriginalFailureDataWhenExtractionFindsNothing(t *testing.T) {
+	report := &junitReport{
+		Testsuites: junit.Testsuites{
+			Suites: []junit.Testsuite{
+				{
+					Name: "suite",
+					Testcases: []junit.Testcase{
+						{
+							Name: "TestStandalone",
+							Failure: &junit.Result{
+								Message: "Failed",
+								Data:    "plain failure output with no recognizable block",
+							},
+						},
+					},
+					Failures: 1,
+					Tests:    1,
+				},
+			},
+			Failures: 1,
+			Tests:    1,
+		},
+	}
+
+	merged, err := mergeReports([]*junitReport{report})
+	require.NoError(t, err)
+	require.Len(t, merged.Suites, 1)
+	require.Len(t, merged.Suites[0].Testcases, 1)
+	require.Equal(t, "plain failure output with no recognizable block", merged.Suites[0].Testcases[0].Failure.Data)
+}
+
 func collectTestNames(suites []junit.Testsuite) []string {
 	var testNames []string
 	for _, suite := range suites {