Skip to content

Commit 9e6b18d

Browse files
committed
JUnit summary with failures
1 parent 956f6ec commit 9e6b18d

19 files changed

Lines changed: 918 additions & 106 deletions

.github/actions/get-job-id/action.yml

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,28 +30,44 @@ runs:
3030
# The script will first try to match by (name, runner_name), then fall back to name-only.
3131
run: |
3232
set -euo pipefail
33+
3334
job_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?per_page=100"
34-
json=$(curl -sSL \
35-
-H "Accept: application/vnd.github+json" \
36-
-H "Authorization: Bearer $GITHUB_TOKEN" \
37-
"$job_url")
35+
max_retries=6
36+
job_id=""
37+
json=""
3838
39-
# Prefer matching both job name and the current runner name to disambiguate matrix jobs
40-
job_id=$(jq -r --arg name "$JOB_NAME" --arg runner "$RUNNER_NAME" '
41-
(.jobs // [])
42-
| map(select(.name == $name and (.runner_name // "") == $runner))
43-
| (.[0].id // empty)
44-
' <<< "$json" )
39+
for ((attempt = 1; attempt <= max_retries; attempt++)); do
40+
json=$(curl -sSL \
41+
-H "Accept: application/vnd.github+json" \
42+
-H "Authorization: Bearer $GITHUB_TOKEN" \
43+
"$job_url")
4544
46-
# Fallback: match by name only
47-
if [ -z "${job_id:-}" ]; then
48-
job_id=$(jq -r --arg name "$JOB_NAME" '
49-
(.jobs // []) | map(select(.name == $name)) | (.[0].id // empty)
45+
# Prefer matching both job name and the current runner name to disambiguate matrix jobs
46+
job_id=$(jq -r --arg name "$JOB_NAME" --arg runner "$RUNNER_NAME" '
47+
(.jobs // [])
48+
| map(select(.name == $name and (.runner_name // "") == $runner))
49+
| (.[0].id // empty)
5050
' <<< "$json" )
51-
fi
51+
52+
# Fallback: match by name only
53+
if [ -z "${job_id:-}" ]; then
54+
job_id=$(jq -r --arg name "$JOB_NAME" '
55+
(.jobs // []) | map(select(.name == $name)) | (.[0].id // empty)
56+
' <<< "$json" )
57+
fi
58+
59+
if [ -n "${job_id:-}" ] && [ "$job_id" != "null" ]; then
60+
break
61+
fi
62+
63+
if [ "$attempt" -lt "$max_retries" ]; then
64+
echo "::notice::Job ID for '$JOB_NAME' not visible yet (attempt $attempt/$max_retries); retrying in 5s"
65+
sleep 5
66+
fi
67+
done
5268
5369
if [ -z "${job_id:-}" ] || [ "$job_id" = "null" ]; then
54-
echo "::error::Failed to resolve job ID for name '$JOB_NAME' on runner '$RUNNER_NAME' in run '$RUN_ID'" >&2
70+
echo "::error::Failed to resolve job ID for name '$JOB_NAME' on runner '$RUNNER_NAME' in run '$RUN_ID' after retries" >&2
5571
exit 1
5672
fi
5773
echo "job_id=$job_id" >> "$GITHUB_OUTPUT"

.github/workflows/run-tests.yml

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -328,15 +328,13 @@ jobs:
328328
[ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
329329
CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
330330
331-
- name: Generate test summary
332-
uses: mikepenz/action-junit-report@v6
331+
- name: Write test summary
333332
if: ${{ !cancelled() }}
334-
with:
335-
report_paths: ./.testoutput/junit.*.xml
336-
detailed_summary: true
337-
check_annotations: false
338-
annotate_only: true
339-
skip_annotations: true
333+
run: |
334+
summary="$(make -s print-test-summary)"
335+
if [ -n "$summary" ]; then
336+
printf '%s\n' "$summary" > "$GITHUB_STEP_SUMMARY"
337+
fi
340338
341339
- name: Upload code coverage to Codecov
342340
uses: codecov/codecov-action@v5
@@ -428,15 +426,13 @@ jobs:
428426
[ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
429427
CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
430428
431-
- name: Generate test summary
432-
uses: mikepenz/action-junit-report@v6
429+
- name: Write test summary
433430
if: ${{ !cancelled() }}
434-
with:
435-
report_paths: ./.testoutput/junit.*.xml
436-
detailed_summary: true
437-
check_annotations: false
438-
annotate_only: true
439-
skip_annotations: true
431+
run: |
432+
summary="$(make -s print-test-summary)"
433+
if [ -n "$summary" ]; then
434+
printf '%s\n' "$summary" > "$GITHUB_STEP_SUMMARY"
435+
fi
440436
441437
- name: Upload code coverage to Codecov
442438
uses: codecov/codecov-action@v5
@@ -567,15 +563,13 @@ jobs:
567563
[ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
568564
CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash
569565
570-
- name: Generate test summary
571-
uses: mikepenz/action-junit-report@v6
566+
- name: Write test summary
572567
if: ${{ !cancelled() }}
573-
with:
574-
report_paths: ./.testoutput/junit.*.xml
575-
detailed_summary: true
576-
check_annotations: false
577-
annotate_only: true
578-
skip_annotations: true
568+
run: |
569+
summary="$(make -s print-test-summary)"
570+
if [ -n "$summary" ]; then
571+
printf '%s\n' "$summary" > "$GITHUB_STEP_SUMMARY"
572+
fi
579573
580574
- name: Upload code coverage to Codecov
581575
uses: codecov/codecov-action@v5

Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -557,6 +557,10 @@ report-test-crash: $(TEST_OUTPUT_ROOT)
557557
--junitfile=$(TEST_OUTPUT_ROOT)/junit.crash.xml \
558558
--crashreportname=$(CRASH_REPORT_NAME)
559559

560+
print-test-summary: $(TEST_OUTPUT_ROOT)
561+
@go run ./cmd/tools/test-runner print-summary \
562+
--junit-glob=$(TEST_OUTPUT_ROOT)/junit.*.xml
563+
560564
##### Schema #####
561565
install-schema-cass-es: temporal-cassandra-tool install-schema-es
562566
@printf $(COLOR) "Install Cassandra schema..."

tools/testrunner/junit.go

Lines changed: 64 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,18 @@ import (
1313
"github.com/jstemmer/go-junit-report/v2/junit"
1414
)
1515

16+
// alertsSuiteName is the JUnit suite name used for structural alerts (data
17+
// races, panics, fatal errors).
18+
const alertsSuiteName = "ALERTS"
19+
20+
const junitAlertDetailsMaxBytes = 64 * 1024
21+
22+
const (
23+
failureKindFailed = "Failed"
24+
failureKindTimeout = "TIMEOUT"
25+
failureKindCrash = "CRASH"
26+
)
27+
1628
type junitReport struct {
1729
junit.Testsuites
1830
path string
@@ -33,6 +45,9 @@ func (j *junitReport) read() error {
3345
return nil
3446
}
3547

48+
// generateStatic creates synthetic failures for non-JUnit failure modes such as
49+
// timeouts and crashes. Failure.Message is the canonical failure kind for these
50+
// synthetic rows (for example TIMEOUT or CRASH), and Failure.Data is left empty.
3651
func generateStatic(names []string, suffix string, message string) *junitReport {
3752
var testcases []junit.Testcase
3853
for _, name := range names {
@@ -77,25 +92,32 @@ func (j *junitReport) appendAlertsSuite(alerts []alert) {
7792
if len(alerts) == 0 {
7893
return
7994
}
95+
96+
// Convert alerts to JUnit test cases.
8097
var cases []junit.Testcase
8198
for _, a := range alerts {
8299
name := fmt.Sprintf("%s: %s", a.Kind, a.Summary)
83100
if p := primaryTestName(a.Tests); p != "" {
84101
name = fmt.Sprintf("%s — in %s", name, p)
85102
}
86-
// Include only test names for context, not the full log details to avoid XML malformation
87-
var details string
103+
var sb strings.Builder
104+
if a.Details != "" {
105+
sb.WriteString(truncateAlertDetails(sanitizeXML(a.Details)))
106+
sb.WriteByte('\n')
107+
}
88108
if len(a.Tests) > 0 {
89-
details = fmt.Sprintf("Detected in tests:\n\t%s", strings.Join(a.Tests, "\n\t"))
109+
fmt.Fprintf(&sb, "Detected in tests:\n\t%s", strings.Join(a.Tests, "\n\t"))
90110
}
91-
r := &junit.Result{Message: string(a.Kind), Data: details}
111+
r := &junit.Result{Message: string(a.Kind), Data: strings.TrimRight(sb.String(), "\n")}
92112
cases = append(cases, junit.Testcase{
93113
Name: name,
94114
Failure: r,
95115
})
96116
}
117+
118+
// Append the alerts suite to the report.
97119
suite := junit.Testsuite{
98-
Name: "ALERTS",
120+
Name: alertsSuiteName,
99121
Failures: len(cases),
100122
Tests: len(cases),
101123
Testcases: cases,
@@ -105,6 +127,30 @@ func (j *junitReport) appendAlertsSuite(alerts []alert) {
105127
j.Tests += suite.Tests
106128
}
107129

130+
// sanitizeXML removes characters that are invalid in XML 1.0. Go's xml.Encoder
131+
// escapes <, >, & etc., but control characters other than \t, \n, \r are not
132+
// legal XML and cause parsers to reject the document.
133+
func sanitizeXML(s string) string {
134+
return strings.Map(func(r rune) rune {
135+
if r == '\t' || r == '\n' || r == '\r' {
136+
return r
137+
}
138+
if r < 0x20 || r == 0xFFFE || r == 0xFFFF {
139+
return -1
140+
}
141+
return r
142+
}, s)
143+
}
144+
145+
// truncateAlertDetails keeps alert payloads from bloating the JUnit artifact.
146+
func truncateAlertDetails(s string) string {
147+
if len(s) <= junitAlertDetailsMaxBytes {
148+
return s
149+
}
150+
const marker = "\n... (truncated) ...\n"
151+
return s[:junitAlertDetailsMaxBytes-len(marker)] + marker
152+
}
153+
108154
// dedupeAlerts removes duplicate alerts (e.g., repeated across retries) based
109155
// on kind and details while preserving the first-seen order.
110156
func dedupeAlerts(alerts []alert) []alert {
@@ -224,6 +270,19 @@ func mergeReports(reports []*junitReport) (*junitReport, error) {
224270
// Discard test case parents since they provide no value.
225271
continue
226272
}
273+
274+
// Parse failure details from Failure.Data, if present.
275+
if testCase.Failure != nil && testCase.Failure.Data != "" {
276+
if details := parseFailureDetails(testCase.Failure.Data); details != noFailureDetails {
277+
testCase.Failure.Data = details
278+
}
279+
}
280+
281+
// Normalize ordinary failures to failureKindFailed;
282+
// alert messages keep their specific kind.
283+
if testCase.Failure != nil && suite.Name != alertsSuiteName {
284+
testCase.Failure.Message = failureKindFailed
285+
}
227286
testCase.Name += suffix
228287
newSuite.Testcases = append(newSuite.Testcases, testCase)
229288
}

0 commit comments

Comments
 (0)