From b39579ef4cf5dd2e85ca65bf31dad8b566e54402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 13:39:06 +0200 Subject: [PATCH 1/8] feat: support live replay test reporters --- src/__tests__/cli-capture.ts | 20 +- src/__tests__/cli-network.test.ts | 72 +++++- src/cli-test-reporters/default.ts | 36 ++- src/cli-test-reporters/registry.ts | 11 + src/cli-test-reporters/types.ts | 13 +- src/cli-test.ts | 51 ++++- src/cli.ts | 61 ++++- src/cli/commands/generic.ts | 3 +- src/cli/commands/router-types.ts | 2 + src/cli/commands/router.ts | 1 + src/daemon/client/daemon-client-progress.ts | 221 ++++++++++--------- src/daemon/client/daemon-client-transport.ts | 18 +- src/daemon/client/daemon-client.ts | 10 +- website/docs/docs/replay-e2e.md | 60 +++-- 14 files changed, 411 insertions(+), 168 deletions(-) diff --git a/src/__tests__/cli-capture.ts b/src/__tests__/cli-capture.ts index 6f0f4b3e8..00af4db2e 100644 --- a/src/__tests__/cli-capture.ts +++ b/src/__tests__/cli-capture.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { runCli } from '../cli.ts'; -import type { DaemonRequest, DaemonResponse } from '../daemon/client/daemon-client.ts'; +import type { DaemonRequest, DaemonResponse, sendToDaemon } from '../daemon/client/daemon-client.ts'; import { installIsolatedCliTestEnv } from './cli-test-env.ts'; class ExitSignal extends Error { @@ -15,6 +15,7 @@ class ExitSignal extends Error { } export type CapturedDaemonRequest = Omit; +type DaemonTransportOptions = Parameters[1]; export type CapturedCliRun = { code: number | null; @@ -28,11 +29,17 @@ export type CliCaptureOptions = { env?: Record; stateDirPrefix?: string; passthroughBufferWrites?: boolean; - sendToDaemon?: (req: CapturedDaemonRequest) => Promise; + sendToDaemon?: ( + req: CapturedDaemonRequest, + options?: DaemonTransportOptions, + ) => Promise; defaultResponse?: DaemonResponse; }; -type CliCaptureResponder = (req: CapturedDaemonRequest) => Promise; +type CliCaptureResponder = ( + req: CapturedDaemonRequest, + options?: DaemonTransportOptions, +) => Promise; export async function runCliCapture( argv: string[], @@ -82,10 +89,13 @@ export async function runCliCapture( return true; }) as typeof process.stderr.write; - const sendToDaemon = async (req: CapturedDaemonRequest): Promise => { + const sendToDaemon = async ( + req: CapturedDaemonRequest, + daemonOptions?: DaemonTransportOptions, + ): Promise => { calls.push(req); if (options.sendToDaemon) { - return await options.sendToDaemon(req); + return await options.sendToDaemon(req, daemonOptions); } return options.defaultResponse ?? { ok: true, data: {} }; }; diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index f9515f5ea..b8fdc0bc4 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -771,12 +771,24 @@ test('test command loads custom reporter modules', async () => { 'utf8', ); - const result = await runCliCapture(['test', './suite', '--reporter', reporterPath], async () => - makeReplaySuiteResponse(), + const result = await runCliCapture( + ['test', './suite', '--reporter', reporterPath], + async (_req, options) => { + options?.onProgress?.({ + type: 'replay-test', + file: '/tmp/01-pass.ad', + status: 'pass', + index: 1, + total: 1, + durationMs: 10, + }); + return makeReplaySuiteResponse(); + }, ); assert.equal(result.code, null); assert.doesNotMatch(result.stdout, /Test summary:/); + assert.doesNotMatch(result.stderr, /✓ 01-pass\.ad/); assert.deepEqual(JSON.parse(await fs.readFile(outputPath, 'utf8')), { total: 3, failed: 1, @@ -785,3 +797,59 @@ test('test command loads custom reporter modules', async () => { await fs.rm(tmpDir, { recursive: true, force: true }); } }); + +test('test command streams progress to custom reporter modules', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-live-reporter-test-')); + const reporterPath = path.join(tmpDir, 'live-reporter.mjs'); + + try { + await fs.writeFile( + reporterPath, + [ + 'export default {', + " name: 'live-custom',", + ' onProgress(event, context) {', + ' if (event.type !== "replay-test") return;', + ' context.stderr.write(`live:${event.status}:${event.stepIndex ?? 0}/${event.stepTotal ?? 0}\\n`);', + ' },', + ' onSuiteEnd(suite, context) {', + ' context.stdout.write(`final:${suite.total}\\n`);', + ' },', + ' getExitCode() { return 0; },', + '};', + ].join('\n'), + 'utf8', + ); + + const result = await runCliCapture( + ['test', './suite', '--reporter', reporterPath], + async (_req, options) => { + options?.onProgress?.({ + type: 'replay-test', + file: '/tmp/01-pass.ad', + status: 'progress', + index: 1, + total: 1, + stepIndex: 1, + stepTotal: 2, + }); + options?.onProgress?.({ + type: 'replay-test', + file: '/tmp/01-pass.ad', + status: 'pass', + index: 1, + total: 1, + durationMs: 10, + }); + return makeReplaySuiteResponse(); + }, + ); + + assert.equal(result.code, null); + assert.equal(result.stdout, 'final:3\n'); + assert.match(result.stderr, /live:progress:1\/2/); + assert.match(result.stderr, /live:pass:0\/0/); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/src/cli-test-reporters/default.ts b/src/cli-test-reporters/default.ts index 4ba612ff7..125c307c0 100644 --- a/src/cli-test-reporters/default.ts +++ b/src/cli-test-reporters/default.ts @@ -1,5 +1,6 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; import { replayTestFailureStepLines } from '../cli-test-trace.ts'; +import { createReplayTestProgressRenderer } from '../cli-test-progress.ts'; import { formatDurationSeconds } from '../utils/duration-format.ts'; import { colorize, supportsColor } from '../utils/output.ts'; import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; @@ -19,8 +20,19 @@ import { } from './format.ts'; export function createDefaultReplayTestReporter(): ReplayTestReporter { + let progressRenderer: ReturnType | undefined; return { name: 'default', + onProgress: (event, context) => { + progressRenderer ??= createReplayTestProgressRenderer({ + verbose: context.debug, + liveProgress: context.stderr.isTTY && !process.env.CI, + columns: context.stderr.columns, + }); + const output = progressRenderer.render(event); + if (!output) return; + context.stderr.write(output.newline ? `${output.text}\n` : output.text); + }, onSuiteEnd: (suite, context) => renderReplayTestSummary(suite, context), getExitCode: getReplayTestExitCode, }; @@ -31,7 +43,7 @@ function renderReplayTestSummary( context: ReplayTestReporterContext, ): void { const flaky = data.tests.filter(isFlakyReplayTestResult); - context.writeStdout(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); + context.stdout.write(`${formatReplayTestSummaryLine(data, flaky.length)}\n`); renderFailureDetails(data.tests.filter(isFailedReplayTestResult), context); renderFlakyTestSummary(flaky, context); } @@ -82,10 +94,10 @@ function renderFlakyTestSummary( context: ReplayTestReporterContext, ): void { if (results.length === 0) return; - context.writeStdout('\n'); - context.writeStdout('Flaky tests:\n'); + context.stdout.write('\n'); + context.stdout.write('Flaky tests:\n'); for (const result of results) { - context.writeStdout( + context.stdout.write( ` ${replayFlakyStatusIcon()} ${replayTestDisplayNameWithFile(result)} after ${result.attempts} attempts${formatFlakyReplayDurationSuffix(result)}\n`, ); for (const failure of result.attemptFailures ?? []) { @@ -93,7 +105,7 @@ function renderFlakyTestSummary( typeof failure.durationMs === 'number' ? ` (${formatDurationSeconds(failure.durationMs)})` : ''; - context.writeStdout( + context.stdout.write( ` attempt ${failure.attempt} failed${attemptDuration}: ${failure.message}\n`, ); } @@ -105,10 +117,10 @@ function renderFailureDetails( context: ReplayTestReporterContext, ): void { if (results.length === 0) return; - context.writeStdout('\n'); - context.writeStdout('Failures:\n'); + context.stdout.write('\n'); + context.stdout.write('Failures:\n'); for (const result of results) { - context.writeStdout(` ${replayTestDisplayNameWithFile(result)}\n`); + context.stdout.write(` ${replayTestDisplayNameWithFile(result)}\n`); renderReplayFailureBody(result, context, ' '); } } @@ -119,14 +131,14 @@ function renderReplayFailureBody( indent: string, ): void { const fileLine = replayTestFailureFileLine(result); - if (fileLine) context.writeStdout(`${indent}${fileLine}\n`); - context.writeStdout(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); + if (fileLine) context.stdout.write(`${indent}${fileLine}\n`); + context.stdout.write(`${indent}${result.error?.message ?? 'Unknown test failure'}\n`); for (const line of replayFailureConsoleLines(result)) { - context.writeStdout(`${indent}${line}\n`); + context.stdout.write(`${indent}${line}\n`); } if (!context.debug) return; for (const line of replayTestFailureStepLines(result)) { - context.writeStdout(`${indent}${line}\n`); + context.stdout.write(`${indent}${line}\n`); } } diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index 5baf0e2c3..8ba590d17 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -1,4 +1,5 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; +import type { RequestProgressEvent } from '../daemon/request-progress.ts'; import { createCustomReplayTestReporter } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; @@ -25,6 +26,16 @@ export async function runReplayTestReporters( } } +export function runReplayTestReporterProgress( + reporters: ReplayTestReporter[], + event: RequestProgressEvent, + context: ReplayTestReporterContext, +): void { + for (const reporter of reporters) { + reporter.onProgress?.(event, context); + } +} + export function getReplayTestReporterExitCode( reporters: ReplayTestReporter[], suite: ReplaySuiteResult, diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index e6753c94e..a68082674 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -3,11 +3,18 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; export type ReplayTestReporterContext = { debug?: boolean; - writeStdout(text: string): void; + stdout: ReplayTestReporterStream; + stderr: ReplayTestReporterStream; mkdir(path: string): void; writeFile(path: string, contents: string): void; }; +export type ReplayTestReporterStream = { + isTTY: boolean; + columns?: number; + write(text: string): void; +}; + export type ReplayTestReporterLoadContext = { spec: string; modulePath: string; @@ -15,10 +22,6 @@ export type ReplayTestReporterLoadContext = { export type ReplayTestReporter = { name: string; - /** - * Reserved for live reporter support. The CLI currently invokes reporters after the final - * ReplaySuiteResult is available, so custom reporters should use onSuiteEnd for now. - */ onProgress?(event: RequestProgressEvent, context: ReplayTestReporterContext): void; onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Promise | void; getExitCode?(suite: ReplaySuiteResult): number | undefined; diff --git a/src/cli-test.ts b/src/cli-test.ts index ef4200a14..28d734786 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -1,33 +1,74 @@ import fs from 'node:fs'; +import type { RequestProgressEvent } from './daemon/request-progress.ts'; import type { ReplaySuiteResult } from './daemon/types.ts'; import { getReplayTestReporterExitCode, resolveReplayTestReporters, + runReplayTestReporterProgress, runReplayTestReporters, } from './cli-test-reporters/registry.ts'; -import type { ReplayTestReporterContext } from './cli-test-reporters/types.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './cli-test-reporters/types.ts'; import { printJson } from './utils/output.ts'; +export type ReplayTestReporterRuntime = { + reporters: ReplayTestReporter[]; + context: ReplayTestReporterContext; + onProgress(event: RequestProgressEvent): void; +}; + export async function renderReplayTestResponse(options: { suite: ReplaySuiteResult; json?: boolean; debug?: boolean; reporter?: string[]; reportJunit?: string; + reporterRuntime?: ReplayTestReporterRuntime; }): Promise { const { suite, json, debug, reporter, reportJunit } = options; - const reporters = await resolveReplayTestReporters({ reporters: reporter, reportJunit, json }); - await runReplayTestReporters(reporters, suite, createReplayTestReporterContext({ debug })); + const runtime = + options.reporterRuntime ?? + (await createReplayTestReporterRuntime({ debug, reporter, reportJunit, json })); + await runReplayTestReporters(runtime.reporters, suite, runtime.context); if (json) { printJson({ success: true, data: suite }); } - return getReplayTestReporterExitCode(reporters, suite); + return getReplayTestReporterExitCode(runtime.reporters, suite); +} + +export async function createReplayTestReporterRuntime(options: { + debug?: boolean; + reporter?: string[]; + reportJunit?: string; + json?: boolean; +}): Promise { + const reporters = await resolveReplayTestReporters({ + reporters: options.reporter, + reportJunit: options.reportJunit, + json: options.json, + }); + const context = createReplayTestReporterContext({ debug: options.debug }); + return { + reporters, + context, + onProgress(event) { + runReplayTestReporterProgress(reporters, event, context); + }, + }; } function createReplayTestReporterContext(options: { debug?: boolean }): ReplayTestReporterContext { return { debug: options.debug, - writeStdout: (text) => process.stdout.write(text), + stdout: { + isTTY: process.stdout.isTTY === true, + columns: process.stdout.columns, + write: (text) => process.stdout.write(text), + }, + stderr: { + isTTY: process.stderr.isTTY === true, + columns: process.stderr.columns, + write: (text) => process.stderr.write(text), + }, mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }), writeFile: (filePath, contents) => fs.writeFileSync(filePath, contents, 'utf8'), }; diff --git a/src/cli.ts b/src/cli.ts index a6a1d5652..ea0e0e706 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,6 +6,8 @@ import { pathToFileURL } from 'node:url'; import { sendToDaemon } from './daemon/client/daemon-client.ts'; import fs from 'node:fs'; import type { BatchStep } from './client/client-types.ts'; +import { createReplayTestReporterRuntime } from './cli-test.ts'; +import type { ReplayTestReporterRuntime } from './cli-test.ts'; import { createAgentDeviceClient, type AgentDeviceClientConfig, @@ -40,6 +42,11 @@ type CliDeps = { sendToDaemon: typeof sendToDaemon; }; +type CliDaemonTransport = typeof sendToDaemon; +type CliDaemonRequest = Parameters[0]; +type CliDaemonTransportOptions = Parameters[1]; +type ClientDaemonRequest = Parameters[0]; + const DEFAULT_CLI_DEPS: CliDeps = { sendToDaemon, }; @@ -277,7 +284,7 @@ export async function runCli(argv: string[], deps: CliDeps = DEFAULT_CLI_DEPS): const materializationClient = createAgentDeviceClient( buildClientConfig(effectiveFlags, resolvedRuntime, connectionMetadata), { - transport: deps.sendToDaemon as AgentDeviceDaemonTransport, + transport: createClientDaemonTransport(deps.sendToDaemon), }, ); const materialized = await materializeRemoteConnectionForCommand({ @@ -321,13 +328,24 @@ export async function runCli(argv: string[], deps: CliDeps = DEFAULT_CLI_DEPS): debugOutputEnabled && !effectiveFlags.json && !remoteDaemonBaseUrl ? startDaemonLogTail(daemonPaths.logPath) : null; + const replayTestReporterRuntime = + command === 'test' + ? await createReplayTestReporterRuntime({ + debug: debugOutputEnabled, + verbose: effectiveFlags.verbose, + json: effectiveFlags.json, + reporter: effectiveFlags.reporter, + reportJunit: effectiveFlags.reportJunit, + }) + : undefined; const client = createAgentDeviceClient( buildClientConfig(effectiveFlags, resolvedRuntime, connectionMetadata), { transport: createCliDaemonTransport({ command, flags: effectiveFlags, - transport: deps.sendToDaemon as AgentDeviceDaemonTransport, + replayTestReporterRuntime, + transport: deps.sendToDaemon, }), }, ); @@ -354,6 +372,7 @@ export async function runCli(argv: string[], deps: CliDeps = DEFAULT_CLI_DEPS): flags: { ...effectiveFlags, batchSteps }, client, debug: debugOutputEnabled, + replayTestReporterRuntime, }) ) { return; @@ -370,6 +389,7 @@ export async function runCli(argv: string[], deps: CliDeps = DEFAULT_CLI_DEPS): flags: effectiveFlags, client, debug: debugOutputEnabled, + replayTestReporterRuntime, }) ) { return; @@ -557,18 +577,37 @@ function hasExplicitMetroRuntimeOverrides(explicitFlagKeys: Set): boole function createCliDaemonTransport(options: { command: string; flags: CliFlags; - transport: AgentDeviceDaemonTransport; + replayTestReporterRuntime?: ReplayTestReporterRuntime; + transport: CliDaemonTransport; }): AgentDeviceDaemonTransport { - const { command, flags, transport } = options; - if (flags.json) return transport; + const { command, flags, replayTestReporterRuntime, transport } = options; + if (flags.json) return createClientDaemonTransport(transport); return async (req) => - await transport({ - ...req, - meta: { - ...req.meta, - requestProgress: command === 'test' ? 'replay-test' : 'command', + await sendClientRequestToCliTransport( + transport, + { + ...req, + meta: { + ...req.meta, + requestProgress: command === 'test' ? 'replay-test' : 'command', + }, }, - }); + command === 'test' && replayTestReporterRuntime + ? { onProgress: replayTestReporterRuntime.onProgress } + : undefined, + ); +} + +function createClientDaemonTransport(transport: CliDaemonTransport): AgentDeviceDaemonTransport { + return async (req) => await sendClientRequestToCliTransport(transport, req); +} + +async function sendClientRequestToCliTransport( + transport: CliDaemonTransport, + req: ClientDaemonRequest, + options?: CliDaemonTransportOptions, +): ReturnType { + return await transport(req as CliDaemonRequest, options); } function guessSessionFromArgv(argv: string[]): string | null { diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index 4aad24134..008785b5c 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -47,7 +47,7 @@ function writeGenericCliOutput( command: ClientBackedCliCommandName, flags: CliFlags, data: CommandRequestResult, - options: { debug?: boolean } = {}, + options: Pick = {}, ): Promise | number { if (command === 'test') { return renderReplayTestResponse({ @@ -56,6 +56,7 @@ function writeGenericCliOutput( json: flags.json, reporter: flags.reporter, reportJunit: flags.reportJunit, + reporterRuntime: options.replayTestReporterRuntime, }); } writeCommandOutput(flags, data, () => diff --git a/src/cli/commands/router-types.ts b/src/cli/commands/router-types.ts index 8824e7f3c..b87a60cc5 100644 --- a/src/cli/commands/router-types.ts +++ b/src/cli/commands/router-types.ts @@ -1,12 +1,14 @@ import type { CliFlags } from '../parser/cli-flags.ts'; import type { AgentDeviceClient } from '../../client/client.ts'; import type { CliCommandName } from '../../command-catalog.ts'; +import type { ReplayTestReporterRuntime } from '../../cli-test.ts'; export type ClientCommandParams = { positionals: string[]; flags: CliFlags; client: AgentDeviceClient; debug?: boolean; + replayTestReporterRuntime?: ReplayTestReporterRuntime; }; /** diff --git a/src/cli/commands/router.ts b/src/cli/commands/router.ts index 650bfacbe..1e8472ad4 100644 --- a/src/cli/commands/router.ts +++ b/src/cli/commands/router.ts @@ -34,6 +34,7 @@ export async function tryRunClientBackedCommand(params: { flags: CliFlags; client: AgentDeviceClient; debug?: boolean; + replayTestReporterRuntime?: ClientCommandParams['replayTestReporterRuntime']; }): Promise { const flags = { ...params.flags }; const dedicatedHandler = diff --git a/src/daemon/client/daemon-client-progress.ts b/src/daemon/client/daemon-client-progress.ts index 0e5d52e38..ef5f86e0e 100644 --- a/src/daemon/client/daemon-client-progress.ts +++ b/src/daemon/client/daemon-client-progress.ts @@ -2,49 +2,102 @@ import http from 'node:http'; import type { Socket } from 'node:net'; import { AppError } from '../../kernel/errors.ts'; import type { DaemonRequest, DaemonResponse } from '../types.ts'; -import type { RequestProgressEvent } from '../request-progress.ts'; +import type { RequestProgressEvent, RequestProgressSink } from '../request-progress.ts'; import { consumeTextLines } from '../../utils/line-stream.ts'; -import { - createReplayTestProgressRenderer, - type ReplayTestProgressRender, -} from '../../cli-test-progress.ts'; import { isDaemonProgressEnvelope, isDaemonResponseEnvelope, shouldStreamRequestProgress, } from '../request-progress-protocol.ts'; -type RequestProgressRenderer = { - render(event: RequestProgressEvent): ReplayTestProgressRender | undefined; +type ProgressLineReader = { + handleLine(line: string): boolean; }; -function createRequestProgressRenderer(req: DaemonRequest): RequestProgressRenderer { - const replayProgressRenderer = createReplayTestProgressRenderer({ - verbose: Boolean(req.flags?.verbose || req.meta?.debug), - liveProgress: shouldRenderLiveProgress(), - columns: process.stderr.columns, - }); - return { - render(event) { - if (event.type === 'command') { - return { text: event.message, newline: true }; - } - return replayProgressRenderer.render(event); - }, - }; -} +type ProgressResponseFormat = 'socket-legacy' | 'ndjson-envelope'; -function writeRequestProgressEvent( +function emitProgressEvent( event: RequestProgressEvent, - renderer: RequestProgressRenderer, + options: { + onProgress?: RequestProgressSink; + }, ): void { - const output = renderer.render(event); - if (!output) return; - process.stderr.write(output.newline ? `${output.text}\n` : output.text); + if (options.onProgress) { + options.onProgress(event); + return; + } + if (event.type === 'command') { + process.stderr.write(`${event.message}\n`); + } } -function shouldRenderLiveProgress(): boolean { - return process.stderr.isTTY === true && !process.env.CI; +function createInvalidDaemonResponseError( + req: DaemonRequest, + line: string, + cause?: unknown, +): AppError { + return new AppError( + 'COMMAND_FAILED', + 'Invalid daemon response', + { + requestId: req.meta?.requestId, + line, + }, + cause instanceof Error ? cause : undefined, + ); +} + +function createProgressLineReader(options: { + req: DaemonRequest; + onProgress?: RequestProgressSink; + responseFormat: ProgressResponseFormat; + onResponse(response: DaemonResponse): void; + onError(error: unknown): void; +}): ProgressLineReader { + const finishWithError = (error: unknown): true => { + options.onError(error); + return true; + }; + + return { + handleLine(line) { + let message: unknown; + try { + message = JSON.parse(line) as unknown; + } catch (error) { + return finishWithError(createInvalidDaemonResponseError(options.req, line, error)); + } + + if (isDaemonProgressEnvelope(message)) { + try { + emitProgressEvent(message.event, { + onProgress: options.onProgress, + }); + return false; + } catch (error) { + return finishWithError(error); + } + } + + if (isDaemonResponseEnvelope(message)) { + options.onResponse(message.response); + return true; + } + + if (options.responseFormat === 'socket-legacy') { + options.onResponse(message as DaemonResponse); + return true; + } + + return finishWithError( + createInvalidDaemonResponseError( + options.req, + line, + new Error('Missing daemon progress response envelope'), + ), + ); + }, + }; } export function shouldReadDaemonProgressStream( @@ -63,6 +116,7 @@ export function readDaemonSocketProgressResponse( socket: Socket, options: { req: DaemonRequest; + onProgress?: RequestProgressSink; isSettled: () => boolean; resolve: (response: DaemonResponse) => void; reject: (error: unknown) => void; @@ -71,22 +125,20 @@ export function readDaemonSocketProgressResponse( ): void { const { req, isSettled, resolve, reject, clearTimeout } = options; let buffer = ''; - const progressRenderer = createRequestProgressRenderer(req); - - const rejectInvalidLine = (line: string, error: unknown) => { - clearTimeout(); - reject( - new AppError( - 'COMMAND_FAILED', - 'Invalid daemon response', - { - requestId: req.meta?.requestId, - line, - }, - error instanceof Error ? error : undefined, - ), - ); - }; + const lineReader = createProgressLineReader({ + req, + onProgress: options.onProgress, + responseFormat: 'socket-legacy', + onResponse(response) { + clearTimeout(); + resolve(response); + socket.end(); + }, + onError(error) { + clearTimeout(); + reject(error); + }, + }); socket.setEncoding('utf8'); socket.on('data', (chunk) => { @@ -94,21 +146,7 @@ export function readDaemonSocketProgressResponse( const parsed = consumeTextLines(buffer, chunk); buffer = parsed.buffer; for (const line of parsed.lines) { - try { - const message = JSON.parse(line) as unknown; - if (isDaemonProgressEnvelope(message)) { - writeRequestProgressEvent(message.event, progressRenderer); - continue; - } - const response = isDaemonResponseEnvelope(message) ? message.response : message; - clearTimeout(); - resolve(response as DaemonResponse); - socket.end(); - return; - } catch (error) { - rejectInvalidLine(line, error); - return; - } + if (lineReader.handleLine(line)) return; } }); } @@ -117,6 +155,7 @@ export function readDaemonHttpProgressResponse( res: http.IncomingMessage, options: { req: DaemonRequest; + onProgress?: RequestProgressSink; handleResponseBody: (body: string) => void; reject: (error: unknown) => void; clearTimeout: () => void; @@ -125,42 +164,21 @@ export function readDaemonHttpProgressResponse( const { req, handleResponseBody, reject, clearTimeout } = options; let buffer = ''; let settled = false; - const progressRenderer = createRequestProgressRenderer(req); - const rejectInvalidLine = (line: string, error: unknown) => { - settled = true; - clearTimeout(); - reject( - new AppError( - 'COMMAND_FAILED', - 'Invalid daemon response', - { - requestId: req.meta?.requestId, - line, - }, - error instanceof Error ? error : undefined, - ), - ); - }; - - const handleLine = (line: string): boolean => { - try { - const message = JSON.parse(line) as unknown; - if (isDaemonProgressEnvelope(message)) { - writeRequestProgressEvent(message.event, progressRenderer); - return false; - } - if (isDaemonResponseEnvelope(message)) { - settled = true; - clearTimeout(); - handleResponseBody(JSON.stringify(message.response)); - return true; - } - throw new Error('Missing daemon progress response envelope'); - } catch (error) { - rejectInvalidLine(line, error); - return true; - } - }; + const lineReader = createProgressLineReader({ + req, + onProgress: options.onProgress, + responseFormat: 'ndjson-envelope', + onResponse(response) { + settled = true; + clearTimeout(); + handleResponseBody(JSON.stringify(response)); + }, + onError(error) { + settled = true; + clearTimeout(); + reject(error); + }, + }); res.setEncoding('utf8'); res.on('data', (chunk) => { @@ -168,21 +186,16 @@ export function readDaemonHttpProgressResponse( const parsed = consumeTextLines(buffer, chunk); buffer = parsed.buffer; for (const line of parsed.lines) { - if (line && handleLine(line)) return; + if (line && lineReader.handleLine(line)) return; } }); res.on('end', () => { if (settled) return; const line = buffer.trim(); - if (line && handleLine(line)) return; + if (line && lineReader.handleLine(line)) return; settled = true; clearTimeout(); - reject( - new AppError('COMMAND_FAILED', 'Invalid daemon response', { - requestId: req.meta?.requestId, - line, - }), - ); + reject(createInvalidDaemonResponseError(req, line)); }); res.on('error', (error) => { if (settled) return; diff --git a/src/daemon/client/daemon-client-transport.ts b/src/daemon/client/daemon-client-transport.ts index fd7fe0817..5a7c7f4b2 100644 --- a/src/daemon/client/daemon-client-transport.ts +++ b/src/daemon/client/daemon-client-transport.ts @@ -17,8 +17,12 @@ import { handleRequestTimeout } from './daemon-client-timeout.ts'; import { isRemoteDaemon, type DaemonInfo } from './daemon-client-metadata.ts'; import { DAEMON_RPC_PROTOCOL_VERSION } from '../http-health.ts'; import { readVersion } from '../../utils/version.ts'; +import type { RequestProgressSink } from '../request-progress.ts'; type ResolvedDaemonTransport = 'socket' | 'http'; +type SendRequestOptions = { + onProgress?: RequestProgressSink; +}; const LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS = 500; const REMOTE_DAEMON_HEALTHCHECK_TIMEOUT_MS = 3000; @@ -173,14 +177,15 @@ export async function sendRequest( preference: DaemonTransportPreference, statePaths: DaemonPaths, timeoutMs: number | undefined, + options: SendRequestOptions = {}, ): Promise { const transport = chooseTransport(info, preference); try { - return await sendRequestWithTransport(info, req, statePaths, timeoutMs, transport); + return await sendRequestWithTransport(info, req, statePaths, timeoutMs, transport, options); } catch (error) { const fallback = chooseAutoFallbackTransport(info, preference, transport); if (!fallback || !isSafeAutoTransportFallbackError(error, transport)) throw error; - return await sendRequestWithTransport(info, req, statePaths, timeoutMs, fallback); + return await sendRequestWithTransport(info, req, statePaths, timeoutMs, fallback, options); } } @@ -190,10 +195,11 @@ async function sendRequestWithTransport( statePaths: DaemonPaths, timeoutMs: number | undefined, transport: ResolvedDaemonTransport, + options: SendRequestOptions, ): Promise { return transport === 'http' - ? await sendHttpRequest(info, req, statePaths, timeoutMs) - : await sendSocketRequest(info, req, statePaths, timeoutMs); + ? await sendHttpRequest(info, req, statePaths, timeoutMs, options) + : await sendSocketRequest(info, req, statePaths, timeoutMs, options); } function chooseTransport( @@ -294,6 +300,7 @@ async function sendSocketRequest( req: DaemonRequest, statePaths: DaemonPaths, timeoutMs: number | undefined, + options: SendRequestOptions, ): Promise { const port = info.port; if (!port) throw new AppError('COMMAND_FAILED', DAEMON_SOCKET_ENDPOINT_UNAVAILABLE_MESSAGE); @@ -324,6 +331,7 @@ async function sendSocketRequest( readDaemonSocketProgressResponse(socket, { req, + onProgress: options.onProgress, isSettled: () => settled, clearTimeout: () => { if (timeoutHandle) clearTimeout(timeoutHandle); @@ -356,6 +364,7 @@ async function sendHttpRequest( req: DaemonRequest, statePaths: DaemonPaths, timeoutMs: number | undefined, + options: SendRequestOptions, ): Promise { const rpcUrl = info.baseUrl ? new URL(buildDaemonHttpUrl(info.baseUrl, 'rpc')) @@ -387,6 +396,7 @@ async function sendHttpRequest( if (shouldReadDaemonProgressStream(req, res.headers?.['content-type'])) { readDaemonHttpProgressResponse(res, { req, + onProgress: options.onProgress, reject, clearTimeout: () => { if (timeoutHandle) clearTimeout(timeoutHandle); diff --git a/src/daemon/client/daemon-client.ts b/src/daemon/client/daemon-client.ts index cff7ffbfe..0559e5f9a 100644 --- a/src/daemon/client/daemon-client.ts +++ b/src/daemon/client/daemon-client.ts @@ -2,6 +2,7 @@ import type { DaemonRequest as SharedDaemonRequest, DaemonResponse as SharedDaemonResponse, } from '../types.ts'; +import type { RequestProgressSink } from '../request-progress.ts'; import { createRequestId, emitDiagnostic, withDiagnosticTimer } from '../../utils/diagnostics.ts'; import { INTERNAL_COMMANDS, PUBLIC_COMMANDS } from '../../command-catalog.ts'; import { prepareRemoteRequestArtifacts } from '../../remote/daemon-artifacts.ts'; @@ -27,8 +28,14 @@ export { canConnectSocket } from './daemon-client-transport.ts'; export { shouldResetDaemonAfterRequestTimeout } from './daemon-client-timeout.ts'; export type DaemonRequest = SharedDaemonRequest; export type DaemonResponse = SharedDaemonResponse; +type DaemonTransportOptions = { + onProgress?: RequestProgressSink; +}; -export async function sendToDaemon(req: Omit): Promise { +export async function sendToDaemon( + req: Omit, + options: DaemonTransportOptions = {}, +): Promise { const requestId = req.meta?.requestId ?? createRequestId(); const debug = Boolean(req.meta?.debug || req.flags?.verbose); const settings = resolveClientSettings(req); @@ -90,6 +97,7 @@ export async function sendToDaemon(req: Omit): Promise/...`. Each attempt writes `replay.ad`, `result.txt`, and `replay-timing.ndjson`. Failed attempts also keep copied logs and artifact files when the replay produced them. - `replay-timing.ndjson` records attempt, cleanup, and per-step start/stop events with durations. Upload it from CI even for passing runs when comparing local and CI performance. - Timeouts are cooperative: the runner marks the attempt failed at the timeout boundary, then gives the underlying replay a short grace period to stop before session cleanup. -- The default text reporter streams one-line `pass`, `fail`, or `skip` progress on stderr as each suite entry finishes or retries. Each line includes current/total suite position and elapsed seconds such as `pass 3/6 ... duration=12.34s`, then the final summary prints failed tests and passed-on-retry flaky tests; use `--verbose` to print every final result. +- The default text reporter streams live progress on stderr while a suite runs, then prints the final summary, failed tests, and passed-on-retry flaky tests. Use `--verbose` to include final step traces for completed tests. - `--reporter` is repeatable. Built-ins are `default` for the console summary and `junit:` for JUnit XML. Passing any explicit reporter list replaces the implicit default reporter, so include `--reporter default` when you also want terminal output. `--report-junit ` remains a compatibility alias for `--reporter junit:`. - When `--fail-fast` and retries are both set, the current test still consumes its retries before the suite stops. ### Custom test reporters -Custom reporters are CLI-only presentation adapters. The daemon still returns the structured replay suite result; reporters run in the local CLI process after the suite finishes. +Custom reporters are CLI-only presentation adapters. The daemon streams progress and returns the structured replay suite result; reporters run in the local CLI process and can render both live progress and final output. ```bash agent-device test ./workflows --reporter ./scripts/replay-reporter.mjs ``` -Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive load context. Reporter hooks receive an IO context with `writeStdout`, `mkdir`, and `writeFile` helpers: +Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive load context. Reporter hooks receive an IO context with `stdout` and `stderr` streams: ```js // scripts/replay-reporter.mjs export default function createReporter(loadContext) { return { name: 'summary-file', + onTestStep(test, context) { + context.stderr.write(`running ${test.file} ${test.stepIndex}/${test.stepTotal}\n`); + }, onSuiteEnd(suite, context) { - context.writeFile( - './tmp/report.txt', - JSON.stringify( - { - total: suite.total, - passed: suite.passed, - failed: suite.failed, - modulePath: loadContext.modulePath, - }, - null, - 2, - ), + context.stdout.write( + `finished ${suite.total} tests from ${loadContext.modulePath}\n`, ); }, getExitCode(suite) { @@ -138,11 +131,42 @@ export default function createReporter(loadContext) { } ``` -Reporter modules export a factory function: +For a live terminal reporter that prints each completed test as an emoji, title, and duration: + +```js +// scripts/emoji-reporter.mjs +export default { + name: 'emoji-status', + onTestResult(test, context) { + if (!['pass', 'fail', 'skip'].includes(test.status)) return; + + const icon = + test.status === 'pass' ? '✅' : test.status === 'fail' ? '❌' : '⏭️'; + const title = test.title?.trim() || test.file; + const duration = + typeof test.durationMs === 'number' + ? ` ${formatSeconds(test.durationMs)}` + : ''; + + context.stderr.write(`${icon} ${title}${duration}\n`); + }, +}; + +function formatSeconds(durationMs) { + return `${(durationMs / 1000).toFixed(2)}s`; +} +``` + +TypeScript reporters can import the public types from `agent-device`: ```ts -const createReporter = () => ({ +import type { ReplayTestReporterFactory } from 'agent-device'; + +const createReporter: ReplayTestReporterFactory = () => ({ name: 'typed-reporter', + onTestStart(test, context) { + context.stderr.write(`started ${test.file}\n`); + }, onSuiteEnd(suite) { // Write artifacts, annotations, or summaries from suite. }, @@ -153,7 +177,7 @@ export default createReporter; The CLI loads reporter modules with Node dynamic `import()`. Use `.mjs` or `.js` files at runtime; for TypeScript, compile the reporter to JavaScript before passing it to `--reporter`. Loading `.ts` files directly depends on Node's type-stripping behavior and is not part of the supported reporter contract. -The supported hook today is final-result reporting through `onSuiteEnd`. `getExitCode` can override whether the finished suite exits successfully; when no reporter supplies one, failed tests exit with `1`. The `onProgress` hook is part of the reporter interface for live reporters, but the CLI currently invokes reporters after the suite result is available. +`onSuiteStart`, `onTestStart`, `onTestStep`, and `onTestResult` receive streamed replay progress while the daemon request is running. Generic command progress frames are not exposed to test reporters. `onSuiteEnd` receives the final suite result. `getExitCode` can override whether the finished suite exits successfully; when no reporter supplies one, failed tests exit with `1`. ## Parametrise `.ad` scripts From 4130e692a1e8c8a3b9a4e728f76b5a70653a2635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 13:47:34 +0200 Subject: [PATCH 2/8] refactor: simplify replay progress readers --- src/cli-test.ts | 32 ++++++++++++++------- src/daemon/client/daemon-client-progress.ts | 6 ++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/cli-test.ts b/src/cli-test.ts index 28d734786..978b788b8 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -7,7 +7,11 @@ import { runReplayTestReporterProgress, runReplayTestReporters, } from './cli-test-reporters/registry.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from './cli-test-reporters/types.ts'; +import type { + ReplayTestReporter, + ReplayTestReporterContext, + ReplayTestReporterStream, +} from './cli-test-reporters/types.ts'; import { printJson } from './utils/output.ts'; export type ReplayTestReporterRuntime = { @@ -16,6 +20,12 @@ export type ReplayTestReporterRuntime = { onProgress(event: RequestProgressEvent): void; }; +type ReporterWritableStream = { + isTTY?: boolean; + columns?: number; + write(text: string): boolean; +}; + export async function renderReplayTestResponse(options: { suite: ReplaySuiteResult; json?: boolean; @@ -59,17 +69,17 @@ export async function createReplayTestReporterRuntime(options: { function createReplayTestReporterContext(options: { debug?: boolean }): ReplayTestReporterContext { return { debug: options.debug, - stdout: { - isTTY: process.stdout.isTTY === true, - columns: process.stdout.columns, - write: (text) => process.stdout.write(text), - }, - stderr: { - isTTY: process.stderr.isTTY === true, - columns: process.stderr.columns, - write: (text) => process.stderr.write(text), - }, + stdout: createReplayTestReporterStream(process.stdout), + stderr: createReplayTestReporterStream(process.stderr), mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }), writeFile: (filePath, contents) => fs.writeFileSync(filePath, contents, 'utf8'), }; } + +function createReplayTestReporterStream(stream: ReporterWritableStream): ReplayTestReporterStream { + return { + isTTY: stream.isTTY === true, + columns: stream.columns, + write: (text) => stream.write(text), + }; +} diff --git a/src/daemon/client/daemon-client-progress.ts b/src/daemon/client/daemon-client-progress.ts index ef5f86e0e..f21ebcb4a 100644 --- a/src/daemon/client/daemon-client-progress.ts +++ b/src/daemon/client/daemon-client-progress.ts @@ -16,6 +16,12 @@ type ProgressLineReader = { type ProgressResponseFormat = 'socket-legacy' | 'ndjson-envelope'; +type ProgressLineReader = { + handleLine(line: string): boolean; +}; + +type ProgressResponseFormat = 'socket-legacy' | 'ndjson-envelope'; + function emitProgressEvent( event: RequestProgressEvent, options: { From 69014abc6fb3f4192ca120df9daae0ef455cf8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 14:10:40 +0200 Subject: [PATCH 3/8] fix: preserve verbose replay reporter progress --- src/__tests__/cli-network.test.ts | 128 ++++++++++++++++++++++++++++++ src/cli-test-reporters/default.ts | 2 +- src/cli-test-reporters/types.ts | 1 + src/cli-test.ts | 17 +++- src/cli/commands/generic.ts | 7 +- website/docs/docs/replay-e2e.md | 2 +- 6 files changed, 150 insertions(+), 7 deletions(-) diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index b8fdc0bc4..0ca0f2fa7 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -282,6 +282,87 @@ test('test command --verbose omits step telemetry for passing tests without debu } }); +test('test command --verbose includes step telemetry in completed progress output', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-live-verbose-')); + const artifactsDir = path.join(tmpDir, 'auth-flow'); + const attemptDir = path.join(artifactsDir, 'attempt-1'); + await fs.mkdir(attemptDir, { recursive: true }); + await fs.writeFile( + path.join(attemptDir, 'replay-timing.ndjson'), + [ + { + type: 'replay_action_start', + step: 1, + line: 3, + command: '__maestroTapOn', + positionals: ['text="Log in"'], + }, + { + type: 'replay_action_stop', + step: 1, + line: 3, + command: '__maestroTapOn', + ok: true, + durationMs: 250, + }, + ] + .map((entry) => JSON.stringify(entry)) + .join('\n'), + ); + + try { + const result = await runCliCapture(['test', './suite', '--verbose'], async (_req, options) => { + options?.onProgress?.({ + type: 'replay-test', + file: '/tmp/auth-flow.yml', + title: 'Authentication flow', + status: 'pass', + index: 1, + total: 1, + durationMs: 500, + attempt: 1, + artifactsDir, + }); + return { + ok: true, + data: { + total: 1, + executed: 1, + passed: 1, + failed: 0, + skipped: 0, + notRun: 0, + durationMs: 500, + failures: [], + tests: [ + { + file: '/tmp/auth-flow.yml', + title: 'Authentication flow', + session: 'default:test:suite:1', + status: 'passed', + durationMs: 500, + finalAttemptDurationMs: 500, + attempts: 1, + artifactsDir, + replayed: 1, + healed: 0, + }, + ], + }, + }; + }); + + assert.equal(result.code, null); + assert.equal(result.calls[0]?.meta?.debug, false); + assert.match(result.stderr, /✓ Authentication flow 0\.5s/); + assert.match(result.stderr, /steps:/); + assert.match(result.stderr, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/); + assert.doesNotMatch(result.stdout, /steps:/); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); + test('test command --verbose omits nested passing step telemetry', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-')); const artifactsDir = path.join(tmpDir, 'material-top-tabs'); @@ -853,3 +934,50 @@ test('test command streams progress to custom reporter modules', async () => { await fs.rm(tmpDir, { recursive: true, force: true }); } }); + +test('test command reuses custom reporter instance for progress and final output', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-stateful-reporter-test-')); + const reporterPath = path.join(tmpDir, 'stateful-reporter.mjs'); + + try { + await fs.writeFile( + reporterPath, + [ + 'export default function createReporter() {', + ' const seen = [];', + ' return {', + " name: 'stateful-custom',", + ' onProgress(event) {', + ' if (event.type === "replay-test") seen.push(event.status);', + ' },', + ' onSuiteEnd(_suite, context) {', + ' context.stdout.write(`seen:${seen.join(",")}\\n`);', + ' },', + ' getExitCode() { return 0; },', + ' };', + '}', + ].join('\n'), + 'utf8', + ); + + const result = await runCliCapture( + ['test', './suite', '--reporter', reporterPath], + async (_req, options) => { + options?.onProgress?.({ + type: 'replay-test', + file: '/tmp/01-pass.ad', + status: 'pass', + index: 1, + total: 1, + durationMs: 10, + }); + return makeReplaySuiteResponse(); + }, + ); + + assert.equal(result.code, null); + assert.equal(result.stdout, 'seen:pass\n'); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/src/cli-test-reporters/default.ts b/src/cli-test-reporters/default.ts index 125c307c0..ae4e827a8 100644 --- a/src/cli-test-reporters/default.ts +++ b/src/cli-test-reporters/default.ts @@ -25,7 +25,7 @@ export function createDefaultReplayTestReporter(): ReplayTestReporter { name: 'default', onProgress: (event, context) => { progressRenderer ??= createReplayTestProgressRenderer({ - verbose: context.debug, + verbose: context.verbose, liveProgress: context.stderr.isTTY && !process.env.CI, columns: context.stderr.columns, }); diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index a68082674..f95c35504 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -3,6 +3,7 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; export type ReplayTestReporterContext = { debug?: boolean; + verbose?: boolean; stdout: ReplayTestReporterStream; stderr: ReplayTestReporterStream; mkdir(path: string): void; diff --git a/src/cli-test.ts b/src/cli-test.ts index 978b788b8..0ec08043c 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -30,14 +30,15 @@ export async function renderReplayTestResponse(options: { suite: ReplaySuiteResult; json?: boolean; debug?: boolean; + verbose?: boolean; reporter?: string[]; reportJunit?: string; reporterRuntime?: ReplayTestReporterRuntime; }): Promise { - const { suite, json, debug, reporter, reportJunit } = options; + const { suite, json, debug, verbose, reporter, reportJunit } = options; const runtime = options.reporterRuntime ?? - (await createReplayTestReporterRuntime({ debug, reporter, reportJunit, json })); + (await createReplayTestReporterRuntime({ debug, verbose, reporter, reportJunit, json })); await runReplayTestReporters(runtime.reporters, suite, runtime.context); if (json) { printJson({ success: true, data: suite }); @@ -47,6 +48,7 @@ export async function renderReplayTestResponse(options: { export async function createReplayTestReporterRuntime(options: { debug?: boolean; + verbose?: boolean; reporter?: string[]; reportJunit?: string; json?: boolean; @@ -56,7 +58,10 @@ export async function createReplayTestReporterRuntime(options: { reportJunit: options.reportJunit, json: options.json, }); - const context = createReplayTestReporterContext({ debug: options.debug }); + const context = createReplayTestReporterContext({ + debug: options.debug, + verbose: options.verbose, + }); return { reporters, context, @@ -66,9 +71,13 @@ export async function createReplayTestReporterRuntime(options: { }; } -function createReplayTestReporterContext(options: { debug?: boolean }): ReplayTestReporterContext { +function createReplayTestReporterContext(options: { + debug?: boolean; + verbose?: boolean; +}): ReplayTestReporterContext { return { debug: options.debug, + verbose: options.verbose ?? options.debug, stdout: createReplayTestReporterStream(process.stdout), stderr: createReplayTestReporterStream(process.stderr), mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }), diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index 008785b5c..a8ebd14a5 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -17,6 +17,7 @@ export async function runGenericClientBackedCommand({ flags, client, debug, + replayTestReporterRuntime, }: ClientCommandParams & { command: ClientBackedCliCommandName }): Promise { const { result, cliOutput } = await runCliCommandWithOutput({ client, @@ -35,7 +36,10 @@ export async function runGenericClientBackedCommand({ if (cliOutput) { writeCliOutput(flags, cliOutput); } else { - const exitCode = await writeGenericCliOutput(command, flags, result, { debug }); + const exitCode = await writeGenericCliOutput(command, flags, result, { + debug, + replayTestReporterRuntime, + }); if (exitCode !== 0) { process.exit(exitCode); } @@ -53,6 +57,7 @@ function writeGenericCliOutput( return renderReplayTestResponse({ suite: data as ReplaySuiteResult, debug: options.debug, + verbose: flags.verbose, json: flags.json, reporter: flags.reporter, reportJunit: flags.reportJunit, diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index feed86c72..74a5b3f8e 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -97,7 +97,7 @@ agent-device test ./workflows --reporter default --reporter junit:./tmp/junit.xm - By default, suite artifacts are written under `.agent-device/test-artifacts//...`. Each attempt writes `replay.ad`, `result.txt`, and `replay-timing.ndjson`. Failed attempts also keep copied logs and artifact files when the replay produced them. - `replay-timing.ndjson` records attempt, cleanup, and per-step start/stop events with durations. Upload it from CI even for passing runs when comparing local and CI performance. - Timeouts are cooperative: the runner marks the attempt failed at the timeout boundary, then gives the underlying replay a short grace period to stop before session cleanup. -- The default text reporter streams live progress on stderr while a suite runs, then prints the final summary, failed tests, and passed-on-retry flaky tests. Use `--verbose` to include final step traces for completed tests. +- The default text reporter streams live progress on stderr while a suite runs, then prints the final summary, failed tests, and passed-on-retry flaky tests. Use `--verbose` to include step traces in completed-test progress output. - `--reporter` is repeatable. Built-ins are `default` for the console summary and `junit:` for JUnit XML. Passing any explicit reporter list replaces the implicit default reporter, so include `--reporter default` when you also want terminal output. `--report-junit ` remains a compatibility alias for `--reporter junit:`. - When `--fail-fast` and retries are both set, the current test still consumes its retries before the suite stops. From 1b3e775e16f87f97fa403f79d2c2f8695648337d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 14:25:17 +0200 Subject: [PATCH 4/8] feat: expose semantic replay reporter hooks --- src/__tests__/cli-network.test.ts | 18 +++-- src/__tests__/cli-test-progress.test.ts | 83 ++++----------------- src/cli-test-progress.ts | 98 ++++++++++++------------- src/cli-test-reporters/custom.ts | 5 +- src/cli-test-reporters/default.ts | 36 ++++++--- src/cli-test-reporters/progress.ts | 57 ++++++++++++++ src/cli-test-reporters/registry.ts | 63 +++++++++++++++- src/cli-test-reporters/types.ts | 53 ++++++++++++- src/index.ts | 51 +++++++++++++ website/docs/docs/replay-e2e.md | 25 ++++--- 10 files changed, 331 insertions(+), 158 deletions(-) create mode 100644 src/cli-test-reporters/progress.ts diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 0ca0f2fa7..8b191d6bc 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -867,7 +867,7 @@ test('test command loads custom reporter modules', async () => { }, ); - assert.equal(result.code, null); + assert.equal(result.code, 1); assert.doesNotMatch(result.stdout, /Test summary:/); assert.doesNotMatch(result.stderr, /✓ 01-pass\.ad/); assert.deepEqual(JSON.parse(await fs.readFile(outputPath, 'utf8')), { @@ -889,9 +889,11 @@ test('test command streams progress to custom reporter modules', async () => { [ 'export default {', " name: 'live-custom',", - ' onProgress(event, context) {', - ' if (event.type !== "replay-test") return;', - ' context.stderr.write(`live:${event.status}:${event.stepIndex ?? 0}/${event.stepTotal ?? 0}\\n`);', + ' onTestStep(test, context) {', + ' context.stderr.write(`live:progress:${test.stepIndex ?? 0}/${test.stepTotal ?? 0}\\n`);', + ' },', + ' onTestResult(test, context) {', + ' context.stderr.write(`live:${test.status}:0/0\\n`);', ' },', ' onSuiteEnd(suite, context) {', ' context.stdout.write(`final:${suite.total}\\n`);', @@ -926,7 +928,7 @@ test('test command streams progress to custom reporter modules', async () => { }, ); - assert.equal(result.code, null); + assert.equal(result.code, 1); assert.equal(result.stdout, 'final:3\n'); assert.match(result.stderr, /live:progress:1\/2/); assert.match(result.stderr, /live:pass:0\/0/); @@ -947,8 +949,8 @@ test('test command reuses custom reporter instance for progress and final output ' const seen = [];', ' return {', " name: 'stateful-custom',", - ' onProgress(event) {', - ' if (event.type === "replay-test") seen.push(event.status);', + ' onTestResult(test) {', + ' seen.push(test.status);', ' },', ' onSuiteEnd(_suite, context) {', ' context.stdout.write(`seen:${seen.join(",")}\\n`);', @@ -975,7 +977,7 @@ test('test command reuses custom reporter instance for progress and final output }, ); - assert.equal(result.code, null); + assert.equal(result.code, 1); assert.equal(result.stdout, 'seen:pass\n'); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); diff --git a/src/__tests__/cli-test-progress.test.ts b/src/__tests__/cli-test-progress.test.ts index 143e11e9e..63e8b3ea3 100644 --- a/src/__tests__/cli-test-progress.test.ts +++ b/src/__tests__/cli-test-progress.test.ts @@ -4,7 +4,7 @@ import { createReplayTestProgressRenderer, formatReplayTestProgressEvent, } from '../cli-test-progress.ts'; -import type { RequestProgressEvent } from '../daemon/request-progress.ts'; +import type { ReplayTestResult } from '../cli-test-reporters/types.ts'; function withStreamTty(stream: NodeJS.WriteStream, isTTY: boolean, run: () => T): T { const descriptor = Object.getOwnPropertyDescriptor(stream, 'isTTY'); @@ -18,53 +18,10 @@ function withStreamTty(stream: NodeJS.WriteStream, isTTY: boolean, run: () => } } -test('formatReplayTestProgressEvent suppresses replay suite start context', () => { - const line = formatReplayTestProgressEvent({ - type: 'replay-test-suite', - status: 'start', - total: 4, - runnable: 3, - skipped: 1, - artifactsDir: '/tmp/replay-suite', - shardMode: 'split', - shardCount: 2, - }); - - assert.equal(line, undefined); -}); - -test('formatReplayTestProgressEvent suppresses replay test start context', () => { - const line = formatReplayTestProgressEvent({ - type: 'replay-test', - file: '/tmp/auth-flow.yml', - title: 'Authentication flow', - status: 'start', - index: 2, - total: 5, - session: 'maestro-test:test:suite:2:attempt-1', - artifactsDir: '/tmp/replay-suite/auth-flow', - shardIndex: 1, - shardCount: 2, - deviceId: 'E140A942-965C-4A92-AC63-F3B23756BE02', - }); - - assert.equal(line, undefined); -}); - -test('formatReplayTestProgressEvent ignores unknown progress event types', () => { - const line = formatReplayTestProgressEvent({ - type: 'future-progress-event', - status: 'start', - } as unknown as RequestProgressEvent); - - assert.equal(line, undefined); -}); - test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', () => { - const cases: Array<{ event: RequestProgressEvent; expected: RegExp }> = [ + const cases: Array<{ event: ReplayTestResult; expected: RegExp }> = [ { event: { - type: 'replay-test', file: '/tmp/01-login.ad', status: 'pass', index: 1, @@ -77,7 +34,6 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', }, { event: { - type: 'replay-test', file: '/tmp/02-checkout.ad', status: 'fail', index: 2, @@ -92,7 +48,6 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', }, { event: { - type: 'replay-test', file: '/tmp/03-payment.ad', title: 'Payment flow', status: 'fail', @@ -111,7 +66,6 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', }, { event: { - type: 'replay-test', file: '/tmp/05-sharded.ad', title: 'Sharded flow', status: 'pass', @@ -128,7 +82,6 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', }, { event: { - type: 'replay-test', file: '/tmp/04-skip.ad', status: 'skip', index: 4, @@ -153,7 +106,6 @@ test('formatReplayTestProgressEvent colors stderr progress rows when stdout is p const line = withStreamTty(process.stdout, false, () => withStreamTty(process.stderr, true, () => formatReplayTestProgressEvent({ - type: 'replay-test', file: '/tmp/01-pass.ad', status: 'pass', index: 1, @@ -181,14 +133,15 @@ test('createReplayTestProgressRenderer dims live step progress when color is ena try { const renderer = createReplayTestProgressRenderer({ liveProgress: true }); const rendered = renderer.render({ - type: 'replay-test', - file: '/tmp/checkout.yaml', - title: 'Checkout flow', - status: 'progress', - index: 1, - total: 1, - stepIndex: 3, - stepTotal: 20, + type: 'test-step', + test: { + file: '/tmp/checkout.yaml', + title: 'Checkout flow', + index: 1, + total: 1, + stepIndex: 3, + stepTotal: 20, + }, }); assert.deepEqual(rendered, { @@ -209,17 +162,13 @@ test('formatReplayTestProgressEvent colors completed result markers when color i process.env.FORCE_COLOR = '1'; delete process.env.NO_COLOR; try { - formatReplayTestProgressEvent({ - type: 'replay-test-suite', - status: 'start', - total: 3, - runnable: 3, - skipped: 0, - artifactsDir: '/tmp/replay-suite', + const renderer = createReplayTestProgressRenderer(); + renderer.render({ + type: 'suite-start', + suite: { total: 3, runnable: 3, skipped: 0, artifactsDir: '/tmp/replay-suite' }, }); assert.equal( formatReplayTestProgressEvent({ - type: 'replay-test', file: '/tmp/01-pass.ad', status: 'pass', index: 1, @@ -231,7 +180,6 @@ test('formatReplayTestProgressEvent colors completed result markers when color i ); assert.equal( formatReplayTestProgressEvent({ - type: 'replay-test', file: '/tmp/02-flaky.yml', title: 'Retry flow', status: 'pass', @@ -243,7 +191,6 @@ test('formatReplayTestProgressEvent colors completed result markers when color i '\u001B[33m✓\u001B[39m Retry flow \u001B[33m0.03s\u001B[39m', ); const failedLine = formatReplayTestProgressEvent({ - type: 'replay-test', file: '/tmp/03-fail.ad', title: 'Checkout failure', status: 'fail', diff --git a/src/cli-test-progress.ts b/src/cli-test-progress.ts index f34169d38..19f981769 100644 --- a/src/cli-test-progress.ts +++ b/src/cli-test-progress.ts @@ -1,12 +1,15 @@ import path from 'node:path'; -import type { RequestProgressEvent } from './daemon/request-progress.ts'; import { replayTestStepLines } from './cli-test-trace.ts'; import type { ReplaySuiteTestResult } from './daemon/types.ts'; +import type { + ReplayTestReporterProgressEvent, + ReplayTestResult, + ReplayTestStep, +} from './cli-test-reporters/types.ts'; import { formatDurationSeconds } from './utils/duration-format.ts'; import { colorize, supportsColor } from './utils/output.ts'; -type ReplayTestCaseProgressEvent = Extract; -type ReplayTestProgressFormatOptions = { +export type ReplayTestProgressFormatOptions = { verbose?: boolean; liveProgress?: boolean; columns?: number; @@ -18,7 +21,7 @@ export type ReplayTestProgressRender = { }; export type ReplayTestProgressRenderer = { - render(event: RequestProgressEvent): ReplayTestProgressRender | undefined; + render(event: ReplayTestReporterProgressEvent): ReplayTestProgressRender | undefined; }; export function createReplayTestProgressRenderer( @@ -28,28 +31,26 @@ export function createReplayTestProgressRenderer( let hasLiveProgressLine = false; return { render(event) { - if (event.type === 'replay-test-suite') { + if (event.type === 'suite-start') { completedKeys.clear(); hasLiveProgressLine = false; return undefined; } - if (event.type !== 'replay-test') { - return undefined; - } - if (event.status === 'progress') { + if (event.type === 'test-step') { if (!options.liveProgress) return undefined; hasLiveProgressLine = true; return { - text: clearLinePrefix(formatReplayTestLiveProgressLine(event, options)), + text: clearLinePrefix(formatReplayTestLiveProgressLine(event.test, options)), newline: false, }; } - if (isReplayTestCompletionProgressEvent(event)) { - const key = replayTestCompletionProgressKey(event); + if (event.type !== 'test-result') return undefined; + if (isReplayTestCompletionProgressEvent(event.test)) { + const key = replayTestCompletionProgressKey(event.test); if (completedKeys.has(key)) return undefined; completedKeys.add(key); } - const line = formatReplayTestProgressEvent(event, options); + const line = formatReplayTestProgressEvent(event.test, options); if (!line) return undefined; const text = hasLiveProgressLine ? clearLinePrefix(line) : line; hasLiveProgressLine = false; @@ -59,20 +60,9 @@ export function createReplayTestProgressRenderer( } export function formatReplayTestProgressEvent( - event: RequestProgressEvent, + event: ReplayTestResult, options: ReplayTestProgressFormatOptions = {}, ): string | undefined { - if (event.type !== 'replay-test') { - return undefined; - } - return formatReplayTestCaseProgressEvent(event, options); -} - -function formatReplayTestCaseProgressEvent( - event: ReplayTestCaseProgressEvent, - options: ReplayTestProgressFormatOptions, -): string | undefined { - if (event.status === 'start' || event.status === 'progress') return undefined; if (event.status === 'fail' && event.retrying) return undefined; const lines = [formatReplayTestCaseSummaryLine(event)]; addReplayTestCaseDetailLines(lines, event, options); @@ -82,8 +72,14 @@ function formatReplayTestCaseProgressEvent( return lines.join('\n'); } +export function replayTestStatusIcon(status: ReplayTestResult['status']): string { + if (status === 'pass') return '✓'; + if (status === 'fail') return '⨯'; + return '-'; +} + function formatReplayTestLiveProgressLine( - event: ReplayTestCaseProgressEvent, + event: ReplayTestStep, options: ReplayTestProgressFormatOptions, ): string { const title = event.title?.trim(); @@ -106,7 +102,7 @@ function formatReplayTestLiveProgressLine( } function formatReplayTestLiveProgressStepSuffix( - event: ReplayTestCaseProgressEvent, + event: ReplayTestStep, options: { useColor?: boolean } = {}, ): string { const stepIndex = event.stepIndex ?? 0; @@ -117,7 +113,7 @@ function formatReplayTestLiveProgressStepSuffix( function addReplayTestCaseDetailLines( lines: string[], - event: ReplayTestCaseProgressEvent, + event: ReplayTestResult, options: ReplayTestProgressFormatOptions, ): void { if (shouldSuppressReplayTestCaseDetailLines(event, options)) return; @@ -130,33 +126,30 @@ function addReplayTestCaseDetailLines( } function shouldSuppressReplayTestCaseDetailLines( - event: ReplayTestCaseProgressEvent, + event: ReplayTestResult, options: ReplayTestProgressFormatOptions, ): boolean { return options.verbose === true && event.status === 'fail'; } -function replayTestProgressFailureFileLine(event: ReplayTestCaseProgressEvent): string | undefined { +function replayTestProgressFailureFileLine(event: ReplayTestResult): string | undefined { return event.status === 'fail' && event.title?.trim() ? ` file: ${path.basename(event.file)}` : undefined; } -function replayTestProgressMessageLine(event: ReplayTestCaseProgressEvent): string | undefined { +function replayTestProgressMessageLine(event: ReplayTestResult): string | undefined { const message = event.message?.replace(/\s+/g, ' ').trim(); if (!message) return undefined; return ` ${event.status === 'fail' ? `failed at: ${message}` : message}`; } -function appendReplayTestProgressHintLine( - lines: string[], - event: ReplayTestCaseProgressEvent, -): void { +function appendReplayTestProgressHintLine(lines: string[], event: ReplayTestResult): void { const hint = event.hint?.replace(/\s+/g, ' ').trim(); if (event.status === 'fail' && hint) lines.push(` hint: ${hint}`); } -function replayTestProgressFailureContextLines(event: ReplayTestCaseProgressEvent): string[] { +function replayTestProgressFailureContextLines(event: ReplayTestResult): string[] { if (event.status !== 'fail' || event.retrying) return []; const lines: string[] = []; if (event.session) lines.push(` session: ${event.session}`); @@ -164,7 +157,7 @@ function replayTestProgressFailureContextLines(event: ReplayTestCaseProgressEven return lines; } -function formatReplayTestCaseSummaryLine(event: ReplayTestCaseProgressEvent): string { +function formatReplayTestCaseSummaryLine(event: ReplayTestResult): string { const useColor = supportsColor(process.stderr); const statusLabel = formatReplayTestProgressStatusLabel(event); const name = formatReplayTestProgressName(event); @@ -174,21 +167,20 @@ function formatReplayTestCaseSummaryLine(event: ReplayTestCaseProgressEvent): st return `${statusLabel} ${name}${shardSuffix}${durationSuffix}`; } -function formatReplayTestProgressName(event: ReplayTestCaseProgressEvent): string { +function formatReplayTestProgressName(event: ReplayTestResult | ReplayTestStep): string { const title = event.title?.trim(); const file = path.basename(event.file); return title ? title : file; } -function formatReplayTestProgressStatusLabel(event: ReplayTestCaseProgressEvent): string { +function formatReplayTestProgressStatusLabel(event: ReplayTestResult): string { const useColor = supportsColor(process.stderr); + const icon = replayTestStatusIcon(event.status); if (event.status === 'pass') { const format = event.attempt && event.attempt > 1 ? 'yellow' : 'green'; - return useColor ? colorizeProgressMarker('✓', format) : '✓'; + return useColor ? colorizeProgressMarker(icon, format) : icon; } - if (event.status === 'fail') return useColor ? colorizeProgressMarker('⨯', 'red') : '⨯'; - if (event.status === 'progress') return '⊙'; - return useColor ? colorizeProgressMarker('-', 'dim') : '-'; + return useColor ? colorizeProgressMarker(icon, event.status === 'fail' ? 'red' : 'dim') : icon; } function colorizeProgressMarker(text: string, format: Parameters[1]): string { @@ -196,7 +188,7 @@ function colorizeProgressMarker(text: string, format: Parameters { return { ...replayTestProgressResultBase(event), @@ -274,7 +268,7 @@ function buildPassedReplayTestProgressResult( } function buildFailedReplayTestProgressResult( - event: ReplayTestCaseProgressEvent, + event: ReplayTestResult, ): Extract { return { ...replayTestProgressResultBase(event), @@ -283,7 +277,7 @@ function buildFailedReplayTestProgressResult( }; } -function replayTestProgressResultBase(event: ReplayTestCaseProgressEvent) { +function replayTestProgressResultBase(event: ReplayTestResult) { return { file: event.file, title: event.title, diff --git a/src/cli-test-reporters/custom.ts b/src/cli-test-reporters/custom.ts index 0d2061cd1..e84710e59 100644 --- a/src/cli-test-reporters/custom.ts +++ b/src/cli-test-reporters/custom.ts @@ -12,7 +12,10 @@ type CustomReporterModule = { }; const OPTIONAL_REPORTER_HOOKS = [ - 'onProgress', + 'onSuiteStart', + 'onTestStart', + 'onTestStep', + 'onTestResult', 'onSuiteEnd', 'getExitCode', ] as const satisfies readonly (keyof ReplayTestReporter)[]; diff --git a/src/cli-test-reporters/default.ts b/src/cli-test-reporters/default.ts index ae4e827a8..2271d0425 100644 --- a/src/cli-test-reporters/default.ts +++ b/src/cli-test-reporters/default.ts @@ -3,7 +3,11 @@ import { replayTestFailureStepLines } from '../cli-test-trace.ts'; import { createReplayTestProgressRenderer } from '../cli-test-progress.ts'; import { formatDurationSeconds } from '../utils/duration-format.ts'; import { colorize, supportsColor } from '../utils/output.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import type { + ReplayTestReporter, + ReplayTestReporterContext, + ReplayTestReporterProgressEvent, +} from './types.ts'; import { getReplayTestExitCode, isDefinedString, @@ -21,17 +25,29 @@ import { export function createDefaultReplayTestReporter(): ReplayTestReporter { let progressRenderer: ReturnType | undefined; + const renderProgress = ( + event: ReplayTestReporterProgressEvent, + context: ReplayTestReporterContext, + ) => { + progressRenderer ??= createReplayTestProgressRenderer({ + verbose: context.verbose, + liveProgress: context.stderr.isTTY && !process.env.CI, + columns: context.stderr.columns, + }); + const output = progressRenderer.render(event); + if (!output) return; + context.stderr.write(output.newline ? `${output.text}\n` : output.text); + }; return { name: 'default', - onProgress: (event, context) => { - progressRenderer ??= createReplayTestProgressRenderer({ - verbose: context.verbose, - liveProgress: context.stderr.isTTY && !process.env.CI, - columns: context.stderr.columns, - }); - const output = progressRenderer.render(event); - if (!output) return; - context.stderr.write(output.newline ? `${output.text}\n` : output.text); + onSuiteStart: (suite, context) => { + renderProgress({ type: 'suite-start', suite }, context); + }, + onTestStep: (test, context) => { + renderProgress({ type: 'test-step', test }, context); + }, + onTestResult: (test, context) => { + renderProgress({ type: 'test-result', test }, context); }, onSuiteEnd: (suite, context) => renderReplayTestSummary(suite, context), getExitCode: getReplayTestExitCode, diff --git a/src/cli-test-reporters/progress.ts b/src/cli-test-reporters/progress.ts new file mode 100644 index 000000000..ebbb7be87 --- /dev/null +++ b/src/cli-test-reporters/progress.ts @@ -0,0 +1,57 @@ +import type { RequestProgressEvent } from '../daemon/request-progress.ts'; +import type { ReplayTestReporterProgressEvent } from './types.ts'; + +export function toReplayTestReporterProgressEvent( + event: RequestProgressEvent, +): ReplayTestReporterProgressEvent | undefined { + if (event.type === 'command') return undefined; + if (event.type === 'replay-test-suite') { + return { + type: 'suite-start', + suite: { + total: event.total, + runnable: event.runnable, + skipped: event.skipped, + artifactsDir: event.artifactsDir, + shardMode: event.shardMode, + shardCount: event.shardCount, + }, + }; + } + const test = { + file: event.file, + title: event.title, + index: event.index, + total: event.total, + attempt: event.attempt, + maxAttempts: event.maxAttempts, + session: event.session, + artifactsDir: event.artifactsDir, + shardIndex: event.shardIndex, + shardCount: event.shardCount, + deviceId: event.deviceId, + deviceName: event.deviceName, + }; + if (event.status === 'start') return { type: 'test-start', test }; + if (event.status === 'progress') { + return { + type: 'test-step', + test: { + ...test, + stepIndex: event.stepIndex, + stepTotal: event.stepTotal, + }, + }; + } + return { + type: 'test-result', + test: { + ...test, + status: event.status, + durationMs: event.durationMs, + retrying: event.retrying, + message: event.message, + hint: event.hint, + }, + }; +} diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index 8ba590d17..f3f4c5636 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -4,8 +4,9 @@ import { createCustomReplayTestReporter } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; import { createJunitReplayTestReporter } from './junit.ts'; +import { toReplayTestReporterProgressEvent } from './progress.ts'; import { buildReplayTestReporterSpecs, type ReplayTestReporterSpec } from './spec.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import type { Awaitable, ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; export async function resolveReplayTestReporters(options: { reporters?: string[]; @@ -30,21 +31,75 @@ export function runReplayTestReporterProgress( reporters: ReplayTestReporter[], event: RequestProgressEvent, context: ReplayTestReporterContext, +): void { + const reporterEvent = toReplayTestReporterProgressEvent(event); + if (!reporterEvent) return; + if (reporterEvent.type === 'suite-start') { + runReplayTestReporterHook(reporters, 'onSuiteStart', context, (reporter) => + reporter.onSuiteStart?.(reporterEvent.suite, context), + ); + return; + } + + if (reporterEvent.type === 'test-start') { + runReplayTestReporterHook(reporters, 'onTestStart', context, (reporter) => + reporter.onTestStart?.(reporterEvent.test, context), + ); + return; + } + + if (reporterEvent.type === 'test-step') { + runReplayTestReporterHook(reporters, 'onTestStep', context, (reporter) => + reporter.onTestStep?.(reporterEvent.test, context), + ); + return; + } + + runReplayTestReporterHook(reporters, 'onTestResult', context, (reporter) => + reporter.onTestResult?.(reporterEvent.test, context), + ); +} + +function runReplayTestReporterHook( + reporters: ReplayTestReporter[], + hookName: keyof ReplayTestReporter, + context: ReplayTestReporterContext, + run: (reporter: ReplayTestReporter) => Awaitable | undefined, ): void { for (const reporter of reporters) { - reporter.onProgress?.(event, context); + try { + const result = run(reporter); + if (result && typeof result === 'object' && 'then' in result) { + void result.catch((error: unknown) => + reportReplayTestReporterHookError(reporter, hookName, context, error), + ); + } + } catch (error) { + reportReplayTestReporterHookError(reporter, hookName, context, error); + } } } +function reportReplayTestReporterHookError( + reporter: ReplayTestReporter, + hookName: keyof ReplayTestReporter, + context: ReplayTestReporterContext, + error: unknown, +): void { + const message = error instanceof Error ? error.message : String(error); + context.stderr.write(`Reporter ${reporter.name} ${String(hookName)} failed: ${message}\n`); +} + export function getReplayTestReporterExitCode( reporters: ReplayTestReporter[], suite: ReplaySuiteResult, ): number { + const exitCodes = [getReplayTestExitCode(suite)]; for (const reporter of reporters) { const exitCode = reporter.getExitCode?.(suite); - if (exitCode !== undefined) return exitCode; + if (exitCode !== undefined) exitCodes.push(exitCode); } - return getReplayTestExitCode(suite); + return Math.max(...exitCodes); } async function resolveReplayTestReporter( diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index f95c35504..7ad04607a 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -1,6 +1,7 @@ -import type { RequestProgressEvent } from '../daemon/request-progress.ts'; import type { ReplaySuiteResult } from '../daemon/types.ts'; +export type Awaitable = T | Promise; + export type ReplayTestReporterContext = { debug?: boolean; verbose?: boolean; @@ -21,10 +22,56 @@ export type ReplayTestReporterLoadContext = { modulePath: string; }; +export type ReplayTestSuiteStart = { + total: number; + runnable: number; + skipped: number; + artifactsDir: string; + shardMode?: 'all' | 'split'; + shardCount?: number; +}; + +export type ReplayTestCase = { + file: string; + title?: string; + index: number; + total: number; + attempt?: number; + maxAttempts?: number; + session?: string; + artifactsDir?: string; + shardIndex?: number; + shardCount?: number; + deviceId?: string; + deviceName?: string; +}; + +export type ReplayTestStep = ReplayTestCase & { + stepIndex?: number; + stepTotal?: number; +}; + +export type ReplayTestResult = ReplayTestCase & { + status: 'pass' | 'fail' | 'skip'; + durationMs?: number; + retrying?: boolean; + message?: string; + hint?: string; +}; + +export type ReplayTestReporterProgressEvent = + | { type: 'suite-start'; suite: ReplayTestSuiteStart } + | { type: 'test-start'; test: ReplayTestCase } + | { type: 'test-step'; test: ReplayTestStep } + | { type: 'test-result'; test: ReplayTestResult }; + export type ReplayTestReporter = { name: string; - onProgress?(event: RequestProgressEvent, context: ReplayTestReporterContext): void; - onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Promise | void; + onSuiteStart?(suite: ReplayTestSuiteStart, context: ReplayTestReporterContext): Awaitable; + onTestStart?(test: ReplayTestCase, context: ReplayTestReporterContext): Awaitable; + onTestStep?(test: ReplayTestStep, context: ReplayTestReporterContext): Awaitable; + onTestResult?(test: ReplayTestResult, context: ReplayTestReporterContext): Awaitable; + onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Awaitable; getExitCode?(suite: ReplaySuiteResult): number | undefined; }; diff --git a/src/index.ts b/src/index.ts index 6e366e0a8..9f1c5ce5f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,8 +2,59 @@ export { createAgentDeviceClient } from './client/client.ts'; export { createLocalArtifactAdapter } from './io.ts'; export { AppError, isAgentDeviceError, normalizeAgentDeviceError } from './kernel/errors.ts'; export { centerOfRect } from './kernel/snapshot.ts'; +export { createDefaultReplayTestReporter } from './cli-test-reporters/default.ts'; +export { + createReplayTestProgressRenderer, + replayTestStatusIcon, +} from './cli-test-progress.ts'; +export { formatDurationSeconds as formatReplayTestDuration } from './utils/duration-format.ts'; export type { + ArtifactAdapter, + ArtifactDescriptor, + CreateTempFileOptions, + FileInputRef, + FileOutputRef, + LocalArtifactAdapterOptions, + OutputVisibility, + ReserveOutputOptions, + ReservedOutputFile, + ResolveInputOptions, + ResolvedInputFile, + TemporaryFile, +} from './io.ts'; + +export type { AppErrorCode, NormalizedError } from './kernel/errors.ts'; + +export type { + Awaitable, + ReplayTestReporter, + ReplayTestReporterContext, + ReplayTestReporterFactory, + ReplayTestReporterLoadContext, + ReplayTestReporterProgressEvent, + ReplayTestReporterStream, + ReplayTestCase, + ReplayTestResult, + ReplayTestStep, + ReplayTestSuiteStart, +} from './cli-test-reporters/types.ts'; + +export type { + ReplayTestProgressFormatOptions, + ReplayTestProgressRender, + ReplayTestProgressRenderer, +} from './cli-test-progress.ts'; + +export type { CommandResult } from './core/command-descriptor/command-result.ts'; +export type { ResponseLevel } from './kernel/contracts.ts'; +export type { BootCommandResult, ShutdownCommandResult } from './contracts/device.ts'; +export type { ViewportCommandResult } from './contracts/viewport.ts'; + +export type { + AgentDeviceClient, + AgentDeviceClientConfig, + AgentDeviceCommandClient, AgentDeviceDaemonTransport, AlertCommandResult, AppListOptions, diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 74a5b3f8e..6b6df4268 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -135,37 +135,38 @@ For a live terminal reporter that prints each completed test as an emoji, title, ```js // scripts/emoji-reporter.mjs +import { formatReplayTestDuration, replayTestStatusIcon } from 'agent-device'; + export default { name: 'emoji-status', onTestResult(test, context) { - if (!['pass', 'fail', 'skip'].includes(test.status)) return; - - const icon = - test.status === 'pass' ? '✅' : test.status === 'fail' ? '❌' : '⏭️'; + const icon = replayTestStatusIcon(test.status); const title = test.title?.trim() || test.file; const duration = typeof test.durationMs === 'number' - ? ` ${formatSeconds(test.durationMs)}` + ? ` ${formatReplayTestDuration(test.durationMs)}` : ''; context.stderr.write(`${icon} ${title}${duration}\n`); }, }; - -function formatSeconds(durationMs) { - return `${(durationMs / 1000).toFixed(2)}s`; -} ``` TypeScript reporters can import the public types from `agent-device`: ```ts import type { ReplayTestReporterFactory } from 'agent-device'; +import { createReplayTestProgressRenderer } from 'agent-device'; const createReporter: ReplayTestReporterFactory = () => ({ name: 'typed-reporter', - onTestStart(test, context) { - context.stderr.write(`started ${test.file}\n`); + onSuiteStart(suite, context) { + context.stderr.write(`starting ${suite.runnable} tests\n`); + }, + onTestResult(test, context) { + const renderer = createReplayTestProgressRenderer(); + const output = renderer.render({ type: 'test-result', test }); + if (output) context.stderr.write(`${output.text}\n`); }, onSuiteEnd(suite) { // Write artifacts, annotations, or summaries from suite. @@ -177,7 +178,7 @@ export default createReporter; The CLI loads reporter modules with Node dynamic `import()`. Use `.mjs` or `.js` files at runtime; for TypeScript, compile the reporter to JavaScript before passing it to `--reporter`. Loading `.ts` files directly depends on Node's type-stripping behavior and is not part of the supported reporter contract. -`onSuiteStart`, `onTestStart`, `onTestStep`, and `onTestResult` receive streamed replay progress while the daemon request is running. Generic command progress frames are not exposed to test reporters. `onSuiteEnd` receives the final suite result. `getExitCode` can override whether the finished suite exits successfully; when no reporter supplies one, failed tests exit with `1`. +Live reporter hooks are semantic: `onSuiteStart`, `onTestStart`, `onTestStep`, and `onTestResult` run while the daemon request is active; generic command progress frames are not exposed to test reporters. `onSuiteEnd` receives the final suite result. `getExitCode` can override whether the finished suite exits successfully; the highest reporter-provided exit code wins, and failed tests exit with `1` when no reporter raises the code further. ## Parametrise `.ad` scripts From bde17a5ca6c12012fb7176270e0fbb621fd70c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 14:30:41 +0200 Subject: [PATCH 5/8] refactor: trim replay reporter context --- src/cli-test-reporters/junit.ts | 15 ++++++--------- src/cli-test-reporters/registry.ts | 20 ++++++++++++-------- src/cli-test-reporters/types.ts | 17 ++++++++--------- src/cli-test.ts | 3 --- src/index.ts | 1 - website/docs/docs/replay-e2e.md | 22 +++++++++++++++++++--- 6 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/cli-test-reporters/junit.ts b/src/cli-test-reporters/junit.ts index 979c76fcf..faf0808ec 100644 --- a/src/cli-test-reporters/junit.ts +++ b/src/cli-test-reporters/junit.ts @@ -1,7 +1,8 @@ +import fs from 'node:fs'; import path from 'node:path'; import type { ReplaySuiteResult, ReplaySuiteTestResult } from '../daemon/types.ts'; import { AppError } from '../kernel/errors.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import type { ReplayTestReporter } from './types.ts'; import { appendOptionalLine, appendReplayErrorDetails, @@ -22,7 +23,7 @@ export function createJunitReplayTestReporter(reportPath: string | undefined): R const outputPath = readJunitReportPath(reportPath); return { name: 'junit', - onSuiteEnd: (suite, context) => writeReplayJunitReport(outputPath, suite, context), + onSuiteEnd: (suite) => writeReplayJunitReport(outputPath, suite), getExitCode: getReplayTestExitCode, }; } @@ -35,15 +36,11 @@ function readJunitReportPath(reportPath: string | undefined): string { ); } -function writeReplayJunitReport( - reportPath: string, - suite: ReplaySuiteResult, - context: ReplayTestReporterContext, -): void { +function writeReplayJunitReport(reportPath: string, suite: ReplaySuiteResult): void { const directory = path.dirname(reportPath); try { - context.mkdir(directory); - context.writeFile(reportPath, buildReplayJunitXml(suite)); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(reportPath, buildReplayJunitXml(suite), 'utf8'); } catch (error) { const message = error instanceof Error ? error.message : String(error); throw new AppError( diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index f3f4c5636..0e39a530f 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -6,7 +6,11 @@ import { getReplayTestExitCode } from './format.ts'; import { createJunitReplayTestReporter } from './junit.ts'; import { toReplayTestReporterProgressEvent } from './progress.ts'; import { buildReplayTestReporterSpecs, type ReplayTestReporterSpec } from './spec.ts'; -import type { Awaitable, ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; + +type ReplayTestReporterLiveHook = 'onSuiteStart' | 'onTestStart' | 'onTestStep' | 'onTestResult'; + +type ReporterHookResult = void | Promise; export async function resolveReplayTestReporters(options: { reporters?: string[]; @@ -62,9 +66,9 @@ export function runReplayTestReporterProgress( function runReplayTestReporterHook( reporters: ReplayTestReporter[], - hookName: keyof ReplayTestReporter, + hookName: ReplayTestReporterLiveHook, context: ReplayTestReporterContext, - run: (reporter: ReplayTestReporter) => Awaitable | undefined, + run: (reporter: ReplayTestReporter) => ReporterHookResult | undefined, ): void { for (const reporter of reporters) { try { @@ -82,7 +86,7 @@ function runReplayTestReporterHook( function reportReplayTestReporterHookError( reporter: ReplayTestReporter, - hookName: keyof ReplayTestReporter, + hookName: ReplayTestReporterLiveHook, context: ReplayTestReporterContext, error: unknown, ): void { @@ -94,12 +98,12 @@ export function getReplayTestReporterExitCode( reporters: ReplayTestReporter[], suite: ReplaySuiteResult, ): number { - const exitCodes = [getReplayTestExitCode(suite)]; + let exitCode = getReplayTestExitCode(suite); for (const reporter of reporters) { - const exitCode = reporter.getExitCode?.(suite); - if (exitCode !== undefined) exitCodes.push(exitCode); + const reporterExitCode = reporter.getExitCode?.(suite); + if (reporterExitCode !== undefined) exitCode = Math.max(exitCode, reporterExitCode); } - return Math.max(...exitCodes); + return exitCode; } async function resolveReplayTestReporter( diff --git a/src/cli-test-reporters/types.ts b/src/cli-test-reporters/types.ts index 7ad04607a..c3c3eeacf 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/cli-test-reporters/types.ts @@ -1,14 +1,10 @@ import type { ReplaySuiteResult } from '../daemon/types.ts'; -export type Awaitable = T | Promise; - export type ReplayTestReporterContext = { debug?: boolean; verbose?: boolean; stdout: ReplayTestReporterStream; stderr: ReplayTestReporterStream; - mkdir(path: string): void; - writeFile(path: string, contents: string): void; }; export type ReplayTestReporterStream = { @@ -67,11 +63,14 @@ export type ReplayTestReporterProgressEvent = export type ReplayTestReporter = { name: string; - onSuiteStart?(suite: ReplayTestSuiteStart, context: ReplayTestReporterContext): Awaitable; - onTestStart?(test: ReplayTestCase, context: ReplayTestReporterContext): Awaitable; - onTestStep?(test: ReplayTestStep, context: ReplayTestReporterContext): Awaitable; - onTestResult?(test: ReplayTestResult, context: ReplayTestReporterContext): Awaitable; - onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): Awaitable; + onSuiteStart?( + suite: ReplayTestSuiteStart, + context: ReplayTestReporterContext, + ): void | Promise; + onTestStart?(test: ReplayTestCase, context: ReplayTestReporterContext): void | Promise; + onTestStep?(test: ReplayTestStep, context: ReplayTestReporterContext): void | Promise; + onTestResult?(test: ReplayTestResult, context: ReplayTestReporterContext): void | Promise; + onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): void | Promise; getExitCode?(suite: ReplaySuiteResult): number | undefined; }; diff --git a/src/cli-test.ts b/src/cli-test.ts index 0ec08043c..bf3ff1a1b 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -1,4 +1,3 @@ -import fs from 'node:fs'; import type { RequestProgressEvent } from './daemon/request-progress.ts'; import type { ReplaySuiteResult } from './daemon/types.ts'; import { @@ -80,8 +79,6 @@ function createReplayTestReporterContext(options: { verbose: options.verbose ?? options.debug, stdout: createReplayTestReporterStream(process.stdout), stderr: createReplayTestReporterStream(process.stderr), - mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }), - writeFile: (filePath, contents) => fs.writeFileSync(filePath, contents, 'utf8'), }; } diff --git a/src/index.ts b/src/index.ts index 9f1c5ce5f..ecf7fcb90 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,7 +27,6 @@ export type { export type { AppErrorCode, NormalizedError } from './kernel/errors.ts'; export type { - Awaitable, ReplayTestReporter, ReplayTestReporterContext, ReplayTestReporterFactory, diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 6b6df4268..495017f80 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -109,10 +109,13 @@ Custom reporters are CLI-only presentation adapters. The daemon streams progress agent-device test ./workflows --reporter ./scripts/replay-reporter.mjs ``` -Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive load context. Reporter hooks receive an IO context with `stdout` and `stderr` streams: +Reporter modules can export a reporter object, `reporter`, `createReporter`, or a default factory. Factories receive load context. Reporter hooks receive replay test domain objects and an IO context with `stdout` and `stderr` streams: ```js // scripts/replay-reporter.mjs +import fs from 'node:fs'; +import path from 'node:path'; + export default function createReporter(loadContext) { return { name: 'summary-file', @@ -120,8 +123,21 @@ export default function createReporter(loadContext) { context.stderr.write(`running ${test.file} ${test.stepIndex}/${test.stepTotal}\n`); }, onSuiteEnd(suite, context) { - context.stdout.write( - `finished ${suite.total} tests from ${loadContext.modulePath}\n`, + context.stdout.write(`finished ${suite.total} tests\n`); + fs.mkdirSync('./tmp', { recursive: true }); + fs.writeFileSync( + path.join('./tmp', 'report.txt'), + JSON.stringify( + { + total: suite.total, + passed: suite.passed, + failed: suite.failed, + modulePath: loadContext.modulePath, + }, + null, + 2, + ), + 'utf8', ); }, getExitCode(suite) { From 42981b300c9be110ca9d9aea30517b5225c2d82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 14:46:26 +0200 Subject: [PATCH 6/8] refactor: trim reporter progress internals --- src/__tests__/cli-capture.ts | 6 +- src/__tests__/cli-test-progress.test.ts | 34 ++-- src/__tests__/daemon-client-progress.test.ts | 160 ++++++++++--------- src/cli-test-progress.ts | 4 +- src/cli-test-reporters/registry.ts | 2 + src/daemon/client/daemon-client-progress.ts | 6 - src/index.ts | 50 ------ src/utils/__tests__/daemon-client.test.ts | 62 ++++--- website/docs/docs/replay-e2e.md | 18 +-- 9 files changed, 154 insertions(+), 188 deletions(-) diff --git a/src/__tests__/cli-capture.ts b/src/__tests__/cli-capture.ts index 00af4db2e..1315548b0 100644 --- a/src/__tests__/cli-capture.ts +++ b/src/__tests__/cli-capture.ts @@ -2,7 +2,11 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { runCli } from '../cli.ts'; -import type { DaemonRequest, DaemonResponse, sendToDaemon } from '../daemon/client/daemon-client.ts'; +import type { + DaemonRequest, + DaemonResponse, + sendToDaemon, +} from '../daemon/client/daemon-client.ts'; import { installIsolatedCliTestEnv } from './cli-test-env.ts'; class ExitSignal extends Error { diff --git a/src/__tests__/cli-test-progress.test.ts b/src/__tests__/cli-test-progress.test.ts index 63e8b3ea3..f66bc34ab 100644 --- a/src/__tests__/cli-test-progress.test.ts +++ b/src/__tests__/cli-test-progress.test.ts @@ -1,9 +1,6 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { - createReplayTestProgressRenderer, - formatReplayTestProgressEvent, -} from '../cli-test-progress.ts'; +import { createReplayTestProgressRenderer } from '../cli-test-progress.ts'; import type { ReplayTestResult } from '../cli-test-reporters/types.ts'; function withStreamTty(stream: NodeJS.WriteStream, isTTY: boolean, run: () => T): T { @@ -18,7 +15,15 @@ function withStreamTty(stream: NodeJS.WriteStream, isTTY: boolean, run: () => } } -test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', () => { +function renderTestResult( + event: ReplayTestResult, + options?: Parameters[0], +): string | undefined { + return createReplayTestProgressRenderer(options).render({ type: 'test-result', test: event }) + ?.text; +} + +test('createReplayTestProgressRenderer renders pass, retry, fail, and skip cases', () => { const cases: Array<{ event: ReplayTestResult; expected: RegExp }> = [ { event: { @@ -93,11 +98,11 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases', ]; for (const { event, expected } of cases) { - assert.match(formatReplayTestProgressEvent(event) ?? '', expected); + assert.match(renderTestResult(event) ?? '', expected); } }); -test('formatReplayTestProgressEvent colors stderr progress rows when stdout is piped', () => { +test('createReplayTestProgressRenderer colors stderr progress rows when stdout is piped', () => { const originalForceColor = process.env.FORCE_COLOR; const originalNoColor = process.env.NO_COLOR; delete process.env.FORCE_COLOR; @@ -105,7 +110,7 @@ test('formatReplayTestProgressEvent colors stderr progress rows when stdout is p try { const line = withStreamTty(process.stdout, false, () => withStreamTty(process.stderr, true, () => - formatReplayTestProgressEvent({ + renderTestResult({ file: '/tmp/01-pass.ad', status: 'pass', index: 1, @@ -156,19 +161,14 @@ test('createReplayTestProgressRenderer dims live step progress when color is ena } }); -test('formatReplayTestProgressEvent colors completed result markers when color is enabled', () => { +test('createReplayTestProgressRenderer colors completed result markers when color is enabled', () => { const originalForceColor = process.env.FORCE_COLOR; const originalNoColor = process.env.NO_COLOR; process.env.FORCE_COLOR = '1'; delete process.env.NO_COLOR; try { - const renderer = createReplayTestProgressRenderer(); - renderer.render({ - type: 'suite-start', - suite: { total: 3, runnable: 3, skipped: 0, artifactsDir: '/tmp/replay-suite' }, - }); assert.equal( - formatReplayTestProgressEvent({ + renderTestResult({ file: '/tmp/01-pass.ad', status: 'pass', index: 1, @@ -179,7 +179,7 @@ test('formatReplayTestProgressEvent colors completed result markers when color i '\u001B[32m✓\u001B[39m 01-pass.ad \u001B[33m0.01s\u001B[39m', ); assert.equal( - formatReplayTestProgressEvent({ + renderTestResult({ file: '/tmp/02-flaky.yml', title: 'Retry flow', status: 'pass', @@ -190,7 +190,7 @@ test('formatReplayTestProgressEvent colors completed result markers when color i }), '\u001B[33m✓\u001B[39m Retry flow \u001B[33m0.03s\u001B[39m', ); - const failedLine = formatReplayTestProgressEvent({ + const failedLine = renderTestResult({ file: '/tmp/03-fail.ad', title: 'Checkout failure', status: 'fail', diff --git a/src/__tests__/daemon-client-progress.test.ts b/src/__tests__/daemon-client-progress.test.ts index 7dd625b97..c67d8f877 100644 --- a/src/__tests__/daemon-client-progress.test.ts +++ b/src/__tests__/daemon-client-progress.test.ts @@ -3,6 +3,7 @@ import { EventEmitter } from 'node:events'; import type { Socket } from 'node:net'; import { test } from 'vitest'; import type { DaemonRequest, DaemonResponse } from '../daemon/types.ts'; +import type { RequestProgressEvent } from '../daemon/request-progress.ts'; import { readDaemonSocketProgressResponse } from '../daemon/client/daemon-client-progress.ts'; import { AppError } from '../kernel/errors.ts'; @@ -13,6 +14,8 @@ type MockSocket = EventEmitter & { setEncoding: (encoding: BufferEncoding) => MockSocket; }; +type ReplayTestProgressEvent = Extract; + function createMockSocket(): MockSocket { const socket = new EventEmitter() as MockSocket; socket.ended = false; @@ -31,11 +34,13 @@ function createMockSocket(): MockSocket { function readSocketProgressResponse( socket: MockSocket, req: DaemonRequest, + onProgress?: (event: RequestProgressEvent) => void, ): Promise { let settled = false; return new Promise((resolve, reject) => { readDaemonSocketProgressResponse(socket as unknown as Socket, { req, + onProgress, isSettled: () => settled, clearTimeout: () => {}, resolve: (response) => { @@ -50,33 +55,54 @@ function readSocketProgressResponse( }); } -function withStderrTerminal(params: { isTTY: boolean; columns: number }, run: () => T): T { - const stderr = process.stderr as typeof process.stderr & { - isTTY?: boolean; - columns?: number; - }; - const mutableStderr = stderr as unknown as Record; - const originalIsTTY = Object.getOwnPropertyDescriptor(stderr, 'isTTY'); - const originalColumns = Object.getOwnPropertyDescriptor(stderr, 'columns'); - try { - Object.defineProperty(stderr, 'isTTY', { - configurable: true, - value: params.isTTY, - }); - Object.defineProperty(stderr, 'columns', { - configurable: true, - value: params.columns, - }); - return run(); - } finally { - if (originalIsTTY) Object.defineProperty(stderr, 'isTTY', originalIsTTY); - else delete mutableStderr.isTTY; - if (originalColumns) Object.defineProperty(stderr, 'columns', originalColumns); - else delete mutableStderr.columns; - } +function replayProgressLine(stepIndex: number): string { + return JSON.stringify({ + type: 'progress', + event: { + type: 'replay-test', + file: '/tmp/tab-view-coverflow.yml', + title: 'Tab View - Coverflow', + status: 'progress', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 1, + stepIndex, + stepTotal: 10, + }, + }); +} + +function replayPassLine(): string { + return JSON.stringify({ + type: 'progress', + event: { + type: 'replay-test', + file: '/tmp/tab-view-coverflow.yml', + title: 'Tab View - Coverflow', + status: 'pass', + index: 1, + total: 1, + attempt: 1, + maxAttempts: 1, + durationMs: 17_800, + }, + }); } -test('readDaemonSocketProgressResponse parses split progress lines before response envelopes', async () => { +function responseLine(data: Record): string { + return JSON.stringify({ + type: 'response', + response: { ok: true, data }, + }); +} + +function replayTestEvent(event: RequestProgressEvent | undefined): ReplayTestProgressEvent { + assert.equal(event?.type, 'replay-test'); + return event as ReplayTestProgressEvent; +} + +test('readDaemonSocketProgressResponse forwards split progress lines before response envelopes', async () => { const socket = createMockSocket(); const req: DaemonRequest = { session: 'default', @@ -87,6 +113,7 @@ test('readDaemonSocketProgressResponse parses split progress lines before respon meta: { requestId: 'req-socket-progress', requestProgress: 'replay-test' }, }; let stderr = ''; + const events: RequestProgressEvent[] = []; const originalStderrWrite = process.stderr.write.bind(process.stderr); try { @@ -95,7 +122,7 @@ test('readDaemonSocketProgressResponse parses split progress lines before respon return true; }) as typeof process.stderr.write; - const responsePromise = readSocketProgressResponse(socket, req); + const responsePromise = readSocketProgressResponse(socket, req, (event) => events.push(event)); const progressLine = JSON.stringify({ type: 'progress', event: { @@ -123,7 +150,13 @@ test('readDaemonSocketProgressResponse parses split progress lines before respon assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); assert.equal(socket.encoding, 'utf8'); assert.equal(socket.ended, true); - assert.match(stderr, /✓ Login flow 1\.23s/); + assert.equal(stderr, ''); + assert.equal(events.length, 1); + const event = events[0]; + assert.equal(event?.type, 'replay-test'); + if (event?.type === 'replay-test') { + assert.equal(event.title, 'Login flow'); + } } finally { process.stderr.write = originalStderrWrite; } @@ -175,7 +208,7 @@ test('readDaemonSocketProgressResponse renders generic command progress', async } }); -test('readDaemonSocketProgressResponse rewrites live progress and clears it for final result', async () => { +test('readDaemonSocketProgressResponse forwards replay progress events to the sink', async () => { const socket = createMockSocket(); const req: DaemonRequest = { session: 'default', @@ -186,68 +219,41 @@ test('readDaemonSocketProgressResponse rewrites live progress and clears it for meta: { requestId: 'req-live-progress', requestProgress: 'replay-test' }, }; let stderr = ''; + const events: RequestProgressEvent[] = []; const originalStderrWrite = process.stderr.write.bind(process.stderr); - const originalCi = process.env.CI; try { - delete process.env.CI; (process.stderr as any).write = ((chunk: unknown) => { stderr += String(chunk); return true; }) as typeof process.stderr.write; - const responsePromise = withStderrTerminal({ isTTY: true, columns: 53 }, () => - readSocketProgressResponse(socket, req), + const responsePromise = readSocketProgressResponse(socket, req, (event) => events.push(event)); + socket.emit( + 'data', + [ + replayProgressLine(3), + replayProgressLine(4), + replayPassLine(), + responseLine({ via: 'socket-progress' }), + ].join('\n') + '\n', ); - const progress = (stepIndex: number) => - JSON.stringify({ - type: 'progress', - event: { - type: 'replay-test', - file: '/tmp/tab-view-coverflow.yml', - title: 'Tab View - Coverflow', - status: 'progress', - index: 1, - total: 1, - attempt: 1, - maxAttempts: 1, - stepIndex, - stepTotal: 10, - }, - }); - const pass = JSON.stringify({ - type: 'progress', - event: { - type: 'replay-test', - file: '/tmp/tab-view-coverflow.yml', - title: 'Tab View - Coverflow', - status: 'pass', - index: 1, - total: 1, - attempt: 1, - maxAttempts: 1, - durationMs: 17_800, - }, - }); - const responseLine = JSON.stringify({ - type: 'response', - response: { ok: true, data: { via: 'socket-progress' } }, - }); - - socket.emit('data', `${progress(3)}\n${progress(4)}\n${pass}\n${responseLine}\n`); assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); - assert.ok(stderr.includes('\r\u001B[2K⊙ Tab View - Coverflow [3/10]')); - assert.ok(stderr.includes('\r\u001B[2K⊙ Tab View - Coverflow [4/10]')); - assert.ok(stderr.includes('\r\u001B[2K✓ Tab View - Coverflow 17.8s\n')); + assert.equal(stderr, ''); + assert.deepEqual( + events.map((event) => replayTestEvent(event).status), + ['progress', 'progress', 'pass'], + ); + assert.equal(replayTestEvent(events[0]).stepIndex, 3); + assert.equal(replayTestEvent(events[1]).stepIndex, 4); + assert.equal(replayTestEvent(events[2]).durationMs, 17_800); } finally { - if (typeof originalCi === 'string') process.env.CI = originalCi; - else delete process.env.CI; process.stderr.write = originalStderrWrite; } }); -test('readDaemonSocketProgressResponse suppresses live progress outside interactive terminals', async () => { +test('readDaemonSocketProgressResponse does not render replay progress without a sink', async () => { const socket = createMockSocket(); const req: DaemonRequest = { session: 'default', @@ -266,9 +272,7 @@ test('readDaemonSocketProgressResponse suppresses live progress outside interact return true; }) as typeof process.stderr.write; - const responsePromise = withStderrTerminal({ isTTY: false, columns: 53 }, () => - readSocketProgressResponse(socket, req), - ); + const responsePromise = readSocketProgressResponse(socket, req); const progress = JSON.stringify({ type: 'progress', event: { @@ -306,7 +310,7 @@ test('readDaemonSocketProgressResponse suppresses live progress outside interact socket.emit('data', `${progress}\n${pass}\n${responseLine}\n`); assert.deepEqual(await responsePromise, { ok: true, data: { via: 'socket-progress' } }); - assert.equal(stderr, '✓ Tab View - Coverflow 17.8s\n'); + assert.equal(stderr, ''); } finally { process.stderr.write = originalStderrWrite; } diff --git a/src/cli-test-progress.ts b/src/cli-test-progress.ts index 19f981769..35e06c8e3 100644 --- a/src/cli-test-progress.ts +++ b/src/cli-test-progress.ts @@ -59,7 +59,7 @@ export function createReplayTestProgressRenderer( }; } -export function formatReplayTestProgressEvent( +function formatReplayTestProgressEvent( event: ReplayTestResult, options: ReplayTestProgressFormatOptions = {}, ): string | undefined { @@ -72,7 +72,7 @@ export function formatReplayTestProgressEvent( return lines.join('\n'); } -export function replayTestStatusIcon(status: ReplayTestResult['status']): string { +function replayTestStatusIcon(status: ReplayTestResult['status']): string { if (status === 'pass') return '✓'; if (status === 'fail') return '⨯'; return '-'; diff --git a/src/cli-test-reporters/registry.ts b/src/cli-test-reporters/registry.ts index 0e39a530f..667b39835 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/cli-test-reporters/registry.ts @@ -74,6 +74,8 @@ function runReplayTestReporterHook( try { const result = run(reporter); if (result && typeof result === 'object' && 'then' in result) { + // Progress hooks run from synchronous daemon stream readers, so async work + // is observed for errors but not awaited before later hooks. void result.catch((error: unknown) => reportReplayTestReporterHookError(reporter, hookName, context, error), ); diff --git a/src/daemon/client/daemon-client-progress.ts b/src/daemon/client/daemon-client-progress.ts index f21ebcb4a..ef5f86e0e 100644 --- a/src/daemon/client/daemon-client-progress.ts +++ b/src/daemon/client/daemon-client-progress.ts @@ -16,12 +16,6 @@ type ProgressLineReader = { type ProgressResponseFormat = 'socket-legacy' | 'ndjson-envelope'; -type ProgressLineReader = { - handleLine(line: string): boolean; -}; - -type ProgressResponseFormat = 'socket-legacy' | 'ndjson-envelope'; - function emitProgressEvent( event: RequestProgressEvent, options: { diff --git a/src/index.ts b/src/index.ts index ecf7fcb90..6e366e0a8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,58 +2,8 @@ export { createAgentDeviceClient } from './client/client.ts'; export { createLocalArtifactAdapter } from './io.ts'; export { AppError, isAgentDeviceError, normalizeAgentDeviceError } from './kernel/errors.ts'; export { centerOfRect } from './kernel/snapshot.ts'; -export { createDefaultReplayTestReporter } from './cli-test-reporters/default.ts'; -export { - createReplayTestProgressRenderer, - replayTestStatusIcon, -} from './cli-test-progress.ts'; -export { formatDurationSeconds as formatReplayTestDuration } from './utils/duration-format.ts'; export type { - ArtifactAdapter, - ArtifactDescriptor, - CreateTempFileOptions, - FileInputRef, - FileOutputRef, - LocalArtifactAdapterOptions, - OutputVisibility, - ReserveOutputOptions, - ReservedOutputFile, - ResolveInputOptions, - ResolvedInputFile, - TemporaryFile, -} from './io.ts'; - -export type { AppErrorCode, NormalizedError } from './kernel/errors.ts'; - -export type { - ReplayTestReporter, - ReplayTestReporterContext, - ReplayTestReporterFactory, - ReplayTestReporterLoadContext, - ReplayTestReporterProgressEvent, - ReplayTestReporterStream, - ReplayTestCase, - ReplayTestResult, - ReplayTestStep, - ReplayTestSuiteStart, -} from './cli-test-reporters/types.ts'; - -export type { - ReplayTestProgressFormatOptions, - ReplayTestProgressRender, - ReplayTestProgressRenderer, -} from './cli-test-progress.ts'; - -export type { CommandResult } from './core/command-descriptor/command-result.ts'; -export type { ResponseLevel } from './kernel/contracts.ts'; -export type { BootCommandResult, ShutdownCommandResult } from './contracts/device.ts'; -export type { ViewportCommandResult } from './contracts/viewport.ts'; - -export type { - AgentDeviceClient, - AgentDeviceClientConfig, - AgentDeviceCommandClient, AgentDeviceDaemonTransport, AlertCommandResult, AppListOptions, diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index 0514daf60..62965e210 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -23,6 +23,7 @@ import { shouldResetDaemonAfterRequestTimeout, } from '../../daemon/client/daemon-client.ts'; import { resolveDaemonPaths } from '../../daemon/config.ts'; +import type { RequestProgressEvent } from '../../daemon/request-progress.ts'; import { isProcessAlive, readProcessCommand, @@ -443,7 +444,7 @@ test('sendToDaemon reuses reachable local socket daemon metadata', async (t) => } }); -test('sendToDaemon prints replay test progress before the socket response', async () => { +test('sendToDaemon forwards replay test progress before the socket response', async () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-socket-progress-')); const artifactsDir = path.join(stateDir, 'login-flow'); const attemptDir = path.join(artifactsDir, 'attempt-2'); @@ -486,6 +487,7 @@ test('sendToDaemon prints replay test progress before the socket response', asyn .join('\n'), ); let stderr = ''; + const progressEvents: RequestProgressEvent[] = []; const originalStderrWrite = process.stderr.write.bind(process.stderr); const originalCreateConnection = net.createConnection; let createConnectionCalls = 0; @@ -552,20 +554,26 @@ test('sendToDaemon prints replay test progress before the socket response', asyn writeCurrentDaemonInfo(stateDir, { port: 65_530, transport: 'socket' }); - const response = await sendToDaemon({ - session: 'default', - command: 'test', - positionals: ['/tmp/replays'], - flags: { stateDir, daemonTransport: 'socket', verbose: true }, - meta: { requestId: 'req-progress', requestProgress: 'replay-test' }, - }); + const response = await sendToDaemon( + { + session: 'default', + command: 'test', + positionals: ['/tmp/replays'], + flags: { stateDir, daemonTransport: 'socket', verbose: true }, + meta: { requestId: 'req-progress', requestProgress: 'replay-test' }, + }, + { onProgress: (event) => progressEvents.push(event) }, + ); assert.deepEqual(response, { ok: true, data: { via: 'socket' } }); - assert.match(stderr, /✓ Login flow 1\.23s/); - assert.equal(stderr.match(/✓ Login flow 1\.23s/g)?.length, 1); - assert.match(stderr, /steps \(attempt 2\):/); - assert.match(stderr, /open "Demo" \(line 3, 0\.25s\)/); - assert.match(stderr, /assertVisible "text=\\"Home\\"" "3000" \(line 4, 0\.75s\)/); + assert.equal(stderr, ''); + assert.equal(progressEvents.length, 2); + const progressEvent = progressEvents[0]; + assert.equal(progressEvent?.type, 'replay-test'); + if (progressEvent?.type === 'replay-test') { + assert.equal(progressEvent.title, 'Login flow'); + assert.equal(progressEvent.artifactsDir, artifactsDir); + } } finally { (net as unknown as { createConnection: typeof net.createConnection }).createConnection = originalCreateConnection; @@ -574,9 +582,10 @@ test('sendToDaemon prints replay test progress before the socket response', asyn } }); -test('sendToDaemon prints replay test progress before the HTTP NDJSON response', async () => { +test('sendToDaemon forwards replay test progress before the HTTP NDJSON response', async () => { let stderr = ''; const seenPaths: string[] = []; + const progressEvents: RequestProgressEvent[] = []; const originalStderrWrite = process.stderr.write.bind(process.stderr); const originalHttpRequest = http.request; (http as unknown as { request: typeof http.request }).request = (( @@ -655,18 +664,27 @@ test('sendToDaemon prints replay test progress before the HTTP NDJSON response', }) as typeof process.stderr.write; await withRemoteDaemonEnv(async () => { - const response = await sendToDaemon({ - session: 'default', - command: 'test', - positionals: ['/tmp/replays'], - flags: {}, - meta: { requestId: 'req-http-progress', requestProgress: 'replay-test' }, - }); + const response = await sendToDaemon( + { + session: 'default', + command: 'test', + positionals: ['/tmp/replays'], + flags: {}, + meta: { requestId: 'req-http-progress', requestProgress: 'replay-test' }, + }, + { onProgress: (event) => progressEvents.push(event) }, + ); assert.deepEqual(response, { ok: true, data: { via: 'http-progress' } }); }); assert.deepEqual(seenPaths, ['GET /agent-device/health', 'POST /agent-device/rpc']); - assert.match(stderr, /✓ Payments flow 2\.50s/); + assert.equal(stderr, ''); + assert.equal(progressEvents.length, 1); + const progressEvent = progressEvents[0]; + assert.equal(progressEvent?.type, 'replay-test'); + if (progressEvent?.type === 'replay-test') { + assert.equal(progressEvent.title, 'Payments flow'); + } } finally { (http as unknown as { request: typeof http.request }).request = originalHttpRequest; process.stderr.write = originalStderrWrite; diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 495017f80..f8348034b 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -151,16 +151,15 @@ For a live terminal reporter that prints each completed test as an emoji, title, ```js // scripts/emoji-reporter.mjs -import { formatReplayTestDuration, replayTestStatusIcon } from 'agent-device'; - export default { name: 'emoji-status', onTestResult(test, context) { - const icon = replayTestStatusIcon(test.status); + const icon = + test.status === 'pass' ? '✓' : test.status === 'fail' ? '⨯' : '-'; const title = test.title?.trim() || test.file; const duration = typeof test.durationMs === 'number' - ? ` ${formatReplayTestDuration(test.durationMs)}` + ? ` ${(test.durationMs / 1000).toFixed(2)}s` : ''; context.stderr.write(`${icon} ${title}${duration}\n`); @@ -168,21 +167,16 @@ export default { }; ``` -TypeScript reporters can import the public types from `agent-device`: +TypeScript reporters use the same object shape; compile them to JavaScript before passing them to `--reporter`: ```ts -import type { ReplayTestReporterFactory } from 'agent-device'; -import { createReplayTestProgressRenderer } from 'agent-device'; - -const createReporter: ReplayTestReporterFactory = () => ({ +const createReporter = () => ({ name: 'typed-reporter', onSuiteStart(suite, context) { context.stderr.write(`starting ${suite.runnable} tests\n`); }, onTestResult(test, context) { - const renderer = createReplayTestProgressRenderer(); - const output = renderer.render({ type: 'test-result', test }); - if (output) context.stderr.write(`${output.text}\n`); + context.stderr.write(`${test.status} ${test.title ?? test.file}\n`); }, onSuiteEnd(suite) { // Write artifacts, annotations, or summaries from suite. From aa4d7bf7c99931b463dce89e93329a38155f71f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 30 Jun 2026 19:15:59 +0200 Subject: [PATCH 7/8] refactor: move replay test reporting under replay --- src/cli.ts | 4 ++-- src/cli/commands/generic.ts | 2 +- src/cli/commands/router-types.ts | 2 +- .../test/__tests__/progress.test.ts} | 4 ++-- .../test/__tests__/reporters-spec.test.ts} | 5 +---- src/{cli-test-progress.ts => replay/test/progress.ts} | 10 +++++----- .../test/reporters}/custom.ts | 2 +- .../test/reporters}/default.ts | 10 +++++----- .../test/reporters}/format.ts | 2 +- .../test/reporters}/junit.ts | 4 ++-- .../test/reporters}/progress.ts | 2 +- .../test/reporters}/registry.ts | 4 ++-- .../test/reporters}/spec.ts | 2 +- .../test/reporters}/types.ts | 2 +- src/{cli-test.ts => replay/test/reporting.ts} | 10 +++++----- src/{cli-test-trace.ts => replay/test/trace.ts} | 4 ++-- 16 files changed, 33 insertions(+), 36 deletions(-) rename src/{__tests__/cli-test-progress.test.ts => replay/test/__tests__/progress.test.ts} (97%) rename src/{__tests__/cli-test-reporters-spec.test.ts => replay/test/__tests__/reporters-spec.test.ts} (91%) rename src/{cli-test-progress.ts => replay/test/progress.ts} (97%) rename src/{cli-test-reporters => replay/test/reporters}/custom.ts (98%) rename src/{cli-test-reporters => replay/test/reporters}/default.ts (94%) rename src/{cli-test-reporters => replay/test/reporters}/format.ts (98%) rename src/{cli-test-reporters => replay/test/reporters}/junit.ts (98%) rename src/{cli-test-reporters => replay/test/reporters}/progress.ts (94%) rename src/{cli-test-reporters => replay/test/reporters}/registry.ts (96%) rename src/{cli-test-reporters => replay/test/reporters}/spec.ts (97%) rename src/{cli-test-reporters => replay/test/reporters}/types.ts (96%) rename src/{cli-test.ts => replay/test/reporting.ts} (89%) rename src/{cli-test-trace.ts => replay/test/trace.ts} (98%) diff --git a/src/cli.ts b/src/cli.ts index ea0e0e706..c21904d8c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,8 +6,8 @@ import { pathToFileURL } from 'node:url'; import { sendToDaemon } from './daemon/client/daemon-client.ts'; import fs from 'node:fs'; import type { BatchStep } from './client/client-types.ts'; -import { createReplayTestReporterRuntime } from './cli-test.ts'; -import type { ReplayTestReporterRuntime } from './cli-test.ts'; +import { createReplayTestReporterRuntime } from './replay/test/reporting.ts'; +import type { ReplayTestReporterRuntime } from './replay/test/reporting.ts'; import { createAgentDeviceClient, type AgentDeviceClientConfig, diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index a8ebd14a5..0cb0f0421 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -1,5 +1,5 @@ import type { CommandRequestResult } from '../../client/client.ts'; -import { renderReplayTestResponse } from '../../cli-test.ts'; +import { renderReplayTestResponse } from '../../replay/test/reporting.ts'; import { runCliCommandWithOutput } from '../../commands/cli-runner.ts'; import type { CommandName } from '../../commands/command-metadata.ts'; import type { CliOutput } from '../../commands/command-contract.ts'; diff --git a/src/cli/commands/router-types.ts b/src/cli/commands/router-types.ts index b87a60cc5..7775dcc5b 100644 --- a/src/cli/commands/router-types.ts +++ b/src/cli/commands/router-types.ts @@ -1,7 +1,7 @@ import type { CliFlags } from '../parser/cli-flags.ts'; import type { AgentDeviceClient } from '../../client/client.ts'; import type { CliCommandName } from '../../command-catalog.ts'; -import type { ReplayTestReporterRuntime } from '../../cli-test.ts'; +import type { ReplayTestReporterRuntime } from '../../replay/test/reporting.ts'; export type ClientCommandParams = { positionals: string[]; diff --git a/src/__tests__/cli-test-progress.test.ts b/src/replay/test/__tests__/progress.test.ts similarity index 97% rename from src/__tests__/cli-test-progress.test.ts rename to src/replay/test/__tests__/progress.test.ts index f66bc34ab..aaa62d327 100644 --- a/src/__tests__/cli-test-progress.test.ts +++ b/src/replay/test/__tests__/progress.test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { createReplayTestProgressRenderer } from '../cli-test-progress.ts'; -import type { ReplayTestResult } from '../cli-test-reporters/types.ts'; +import { createReplayTestProgressRenderer } from '../progress.ts'; +import type { ReplayTestResult } from '../reporters/types.ts'; function withStreamTty(stream: NodeJS.WriteStream, isTTY: boolean, run: () => T): T { const descriptor = Object.getOwnPropertyDescriptor(stream, 'isTTY'); diff --git a/src/__tests__/cli-test-reporters-spec.test.ts b/src/replay/test/__tests__/reporters-spec.test.ts similarity index 91% rename from src/__tests__/cli-test-reporters-spec.test.ts rename to src/replay/test/__tests__/reporters-spec.test.ts index b0b4c4d3d..0ca1e4067 100644 --- a/src/__tests__/cli-test-reporters-spec.test.ts +++ b/src/replay/test/__tests__/reporters-spec.test.ts @@ -1,9 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { - buildReplayTestReporterSpecs, - parseReplayTestReporterSpec, -} from '../cli-test-reporters/spec.ts'; +import { buildReplayTestReporterSpecs, parseReplayTestReporterSpec } from '../reporters/spec.ts'; test('parses built-in reporter shorthand specs', () => { assert.deepEqual(parseReplayTestReporterSpec('default'), { diff --git a/src/cli-test-progress.ts b/src/replay/test/progress.ts similarity index 97% rename from src/cli-test-progress.ts rename to src/replay/test/progress.ts index 35e06c8e3..ea934d802 100644 --- a/src/cli-test-progress.ts +++ b/src/replay/test/progress.ts @@ -1,13 +1,13 @@ import path from 'node:path'; -import { replayTestStepLines } from './cli-test-trace.ts'; -import type { ReplaySuiteTestResult } from './daemon/types.ts'; +import { replayTestStepLines } from './trace.ts'; +import type { ReplaySuiteTestResult } from '../../daemon/types.ts'; import type { ReplayTestReporterProgressEvent, ReplayTestResult, ReplayTestStep, -} from './cli-test-reporters/types.ts'; -import { formatDurationSeconds } from './utils/duration-format.ts'; -import { colorize, supportsColor } from './utils/output.ts'; +} from './reporters/types.ts'; +import { formatDurationSeconds } from '../../utils/duration-format.ts'; +import { colorize, supportsColor } from '../../utils/output.ts'; export type ReplayTestProgressFormatOptions = { verbose?: boolean; diff --git a/src/cli-test-reporters/custom.ts b/src/replay/test/reporters/custom.ts similarity index 98% rename from src/cli-test-reporters/custom.ts rename to src/replay/test/reporters/custom.ts index e84710e59..4309c9839 100644 --- a/src/cli-test-reporters/custom.ts +++ b/src/replay/test/reporters/custom.ts @@ -1,7 +1,7 @@ import os from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -import { AppError } from '../kernel/errors.ts'; +import { AppError } from '../../../kernel/errors.ts'; import type { ReplayTestReporter, ReplayTestReporterFactory } from './types.ts'; import type { ReplayTestReporterSpec } from './spec.ts'; diff --git a/src/cli-test-reporters/default.ts b/src/replay/test/reporters/default.ts similarity index 94% rename from src/cli-test-reporters/default.ts rename to src/replay/test/reporters/default.ts index 2271d0425..905c200d5 100644 --- a/src/cli-test-reporters/default.ts +++ b/src/replay/test/reporters/default.ts @@ -1,8 +1,8 @@ -import type { ReplaySuiteResult } from '../daemon/types.ts'; -import { replayTestFailureStepLines } from '../cli-test-trace.ts'; -import { createReplayTestProgressRenderer } from '../cli-test-progress.ts'; -import { formatDurationSeconds } from '../utils/duration-format.ts'; -import { colorize, supportsColor } from '../utils/output.ts'; +import type { ReplaySuiteResult } from '../../../daemon/types.ts'; +import { replayTestFailureStepLines } from '../trace.ts'; +import { createReplayTestProgressRenderer } from '../progress.ts'; +import { formatDurationSeconds } from '../../../utils/duration-format.ts'; +import { colorize, supportsColor } from '../../../utils/output.ts'; import type { ReplayTestReporter, ReplayTestReporterContext, diff --git a/src/cli-test-reporters/format.ts b/src/replay/test/reporters/format.ts similarity index 98% rename from src/cli-test-reporters/format.ts rename to src/replay/test/reporters/format.ts index d465271ad..77ce15d7d 100644 --- a/src/cli-test-reporters/format.ts +++ b/src/replay/test/reporters/format.ts @@ -1,5 +1,5 @@ import path from 'node:path'; -import type { ReplaySuiteTestResult } from '../daemon/types.ts'; +import type { ReplaySuiteTestResult } from '../../../daemon/types.ts'; export type PassedReplayTestResult = Extract; export type FailedReplayTestResult = Extract; diff --git a/src/cli-test-reporters/junit.ts b/src/replay/test/reporters/junit.ts similarity index 98% rename from src/cli-test-reporters/junit.ts rename to src/replay/test/reporters/junit.ts index faf0808ec..785f50dd2 100644 --- a/src/cli-test-reporters/junit.ts +++ b/src/replay/test/reporters/junit.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; -import type { ReplaySuiteResult, ReplaySuiteTestResult } from '../daemon/types.ts'; -import { AppError } from '../kernel/errors.ts'; +import type { ReplaySuiteResult, ReplaySuiteTestResult } from '../../../daemon/types.ts'; +import { AppError } from '../../../kernel/errors.ts'; import type { ReplayTestReporter } from './types.ts'; import { appendOptionalLine, diff --git a/src/cli-test-reporters/progress.ts b/src/replay/test/reporters/progress.ts similarity index 94% rename from src/cli-test-reporters/progress.ts rename to src/replay/test/reporters/progress.ts index ebbb7be87..ce82836d8 100644 --- a/src/cli-test-reporters/progress.ts +++ b/src/replay/test/reporters/progress.ts @@ -1,4 +1,4 @@ -import type { RequestProgressEvent } from '../daemon/request-progress.ts'; +import type { RequestProgressEvent } from '../../../daemon/request-progress.ts'; import type { ReplayTestReporterProgressEvent } from './types.ts'; export function toReplayTestReporterProgressEvent( diff --git a/src/cli-test-reporters/registry.ts b/src/replay/test/reporters/registry.ts similarity index 96% rename from src/cli-test-reporters/registry.ts rename to src/replay/test/reporters/registry.ts index 667b39835..6d1c5f9bd 100644 --- a/src/cli-test-reporters/registry.ts +++ b/src/replay/test/reporters/registry.ts @@ -1,5 +1,5 @@ -import type { ReplaySuiteResult } from '../daemon/types.ts'; -import type { RequestProgressEvent } from '../daemon/request-progress.ts'; +import type { ReplaySuiteResult } from '../../../daemon/types.ts'; +import type { RequestProgressEvent } from '../../../daemon/request-progress.ts'; import { createCustomReplayTestReporter } from './custom.ts'; import { createDefaultReplayTestReporter } from './default.ts'; import { getReplayTestExitCode } from './format.ts'; diff --git a/src/cli-test-reporters/spec.ts b/src/replay/test/reporters/spec.ts similarity index 97% rename from src/cli-test-reporters/spec.ts rename to src/replay/test/reporters/spec.ts index 720d9a8f2..728fbed51 100644 --- a/src/cli-test-reporters/spec.ts +++ b/src/replay/test/reporters/spec.ts @@ -1,4 +1,4 @@ -import { AppError } from '../kernel/errors.ts'; +import { AppError } from '../../../kernel/errors.ts'; export type ReplayTestReporterSpec = | { diff --git a/src/cli-test-reporters/types.ts b/src/replay/test/reporters/types.ts similarity index 96% rename from src/cli-test-reporters/types.ts rename to src/replay/test/reporters/types.ts index c3c3eeacf..157ab2a51 100644 --- a/src/cli-test-reporters/types.ts +++ b/src/replay/test/reporters/types.ts @@ -1,4 +1,4 @@ -import type { ReplaySuiteResult } from '../daemon/types.ts'; +import type { ReplaySuiteResult } from '../../../daemon/types.ts'; export type ReplayTestReporterContext = { debug?: boolean; diff --git a/src/cli-test.ts b/src/replay/test/reporting.ts similarity index 89% rename from src/cli-test.ts rename to src/replay/test/reporting.ts index bf3ff1a1b..ebfc26041 100644 --- a/src/cli-test.ts +++ b/src/replay/test/reporting.ts @@ -1,17 +1,17 @@ -import type { RequestProgressEvent } from './daemon/request-progress.ts'; -import type { ReplaySuiteResult } from './daemon/types.ts'; +import type { RequestProgressEvent } from '../../daemon/request-progress.ts'; +import type { ReplaySuiteResult } from '../../daemon/types.ts'; import { getReplayTestReporterExitCode, resolveReplayTestReporters, runReplayTestReporterProgress, runReplayTestReporters, -} from './cli-test-reporters/registry.ts'; +} from './reporters/registry.ts'; import type { ReplayTestReporter, ReplayTestReporterContext, ReplayTestReporterStream, -} from './cli-test-reporters/types.ts'; -import { printJson } from './utils/output.ts'; +} from './reporters/types.ts'; +import { printJson } from '../../utils/output.ts'; export type ReplayTestReporterRuntime = { reporters: ReplayTestReporter[]; diff --git a/src/cli-test-trace.ts b/src/replay/test/trace.ts similarity index 98% rename from src/cli-test-trace.ts rename to src/replay/test/trace.ts index c44c2c97d..15d0c6440 100644 --- a/src/cli-test-trace.ts +++ b/src/replay/test/trace.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; -import type { ReplaySuiteTestResult } from './daemon/types.ts'; -import { formatDurationSeconds } from './utils/duration-format.ts'; +import type { ReplaySuiteTestResult } from '../../daemon/types.ts'; +import { formatDurationSeconds } from '../../utils/duration-format.ts'; type ReplayActionStartTrace = { type: 'replay_action_start'; From 7e6bdcb3fcbcd737354de9d66de86e7300eae5e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 30 Jun 2026 18:42:51 +0000 Subject: [PATCH 8/8] refactor: make live replay reporter hooks synchronous and simplify dispatch Live reporter hooks (onSuiteStart/onTestStart/onTestStep/onTestResult) were typed as `void | Promise` but fired from the synchronous daemon progress stream reader without being awaited, so a stateful async reporter could receive onSuiteEnd before its live work settled. Type them as `void` to make the contract honest; onSuiteEnd stays awaited for async flushing. A returned promise from a misbehaving custom JS reporter is still caught so it cannot crash the CLI with an unhandled rejection, but it is documented as unsupported and not awaited. Collapse the four near-identical per-event hook dispatch branches into a single table-driven path, and document the synchronous-hook and exit-code-escalation contracts. Add a regression test covering a throwing live hook. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XXHAYxWpvSzqc6CtneYL8J --- src/__tests__/cli-network.test.ts | 45 +++++++++++++++++ src/replay/test/reporters/registry.ts | 72 +++++++++++++-------------- src/replay/test/reporters/types.ts | 15 +++--- website/docs/docs/replay-e2e.md | 2 +- 4 files changed, 88 insertions(+), 46 deletions(-) diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 8b191d6bc..784970b94 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -983,3 +983,48 @@ test('test command reuses custom reporter instance for progress and final output await fs.rm(tmpDir, { recursive: true, force: true }); } }); + +test('test command surfaces a throwing live reporter hook without aborting the run', async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-throwing-reporter-test-')); + const reporterPath = path.join(tmpDir, 'throwing-reporter.mjs'); + + try { + await fs.writeFile( + reporterPath, + [ + 'export default {', + " name: 'throwing-custom',", + ' onTestResult() {', + " throw new Error('boom');", + ' },', + ' onSuiteEnd(suite, context) {', + ' context.stdout.write(`final:${suite.total}\\n`);', + ' },', + ' getExitCode() { return 0; },', + '};', + ].join('\n'), + 'utf8', + ); + + const result = await runCliCapture( + ['test', './suite', '--reporter', reporterPath], + async (_req, options) => { + options?.onProgress?.({ + type: 'replay-test', + file: '/tmp/01-pass.ad', + status: 'pass', + index: 1, + total: 1, + durationMs: 10, + }); + return makeReplaySuiteResponse(); + }, + ); + + assert.equal(result.code, 1); + assert.equal(result.stdout, 'final:3\n'); + assert.match(result.stderr, /Reporter throwing-custom onTestResult failed: boom/); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/src/replay/test/reporters/registry.ts b/src/replay/test/reporters/registry.ts index 6d1c5f9bd..f43fa1c09 100644 --- a/src/replay/test/reporters/registry.ts +++ b/src/replay/test/reporters/registry.ts @@ -6,11 +6,20 @@ import { getReplayTestExitCode } from './format.ts'; import { createJunitReplayTestReporter } from './junit.ts'; import { toReplayTestReporterProgressEvent } from './progress.ts'; import { buildReplayTestReporterSpecs, type ReplayTestReporterSpec } from './spec.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from './types.ts'; +import type { + ReplayTestReporter, + ReplayTestReporterContext, + ReplayTestReporterProgressEvent, +} from './types.ts'; type ReplayTestReporterLiveHook = 'onSuiteStart' | 'onTestStart' | 'onTestStep' | 'onTestResult'; -type ReporterHookResult = void | Promise; +const LIVE_HOOK_BY_EVENT = { + 'suite-start': 'onSuiteStart', + 'test-start': 'onTestStart', + 'test-step': 'onTestStep', + 'test-result': 'onTestResult', +} as const satisfies Record; export async function resolveReplayTestReporters(options: { reporters?: string[]; @@ -38,44 +47,14 @@ export function runReplayTestReporterProgress( ): void { const reporterEvent = toReplayTestReporterProgressEvent(event); if (!reporterEvent) return; - if (reporterEvent.type === 'suite-start') { - runReplayTestReporterHook(reporters, 'onSuiteStart', context, (reporter) => - reporter.onSuiteStart?.(reporterEvent.suite, context), - ); - return; - } - - if (reporterEvent.type === 'test-start') { - runReplayTestReporterHook(reporters, 'onTestStart', context, (reporter) => - reporter.onTestStart?.(reporterEvent.test, context), - ); - return; - } - - if (reporterEvent.type === 'test-step') { - runReplayTestReporterHook(reporters, 'onTestStep', context, (reporter) => - reporter.onTestStep?.(reporterEvent.test, context), - ); - return; - } - - runReplayTestReporterHook(reporters, 'onTestResult', context, (reporter) => - reporter.onTestResult?.(reporterEvent.test, context), - ); -} - -function runReplayTestReporterHook( - reporters: ReplayTestReporter[], - hookName: ReplayTestReporterLiveHook, - context: ReplayTestReporterContext, - run: (reporter: ReplayTestReporter) => ReporterHookResult | undefined, -): void { + const hookName = LIVE_HOOK_BY_EVENT[reporterEvent.type]; for (const reporter of reporters) { try { - const result = run(reporter); - if (result && typeof result === 'object' && 'then' in result) { - // Progress hooks run from synchronous daemon stream readers, so async work - // is observed for errors but not awaited before later hooks. + const result = invokeReplayTestReporterLiveHook(reporter, reporterEvent, context); + if (result instanceof Promise) { + // Live hooks are synchronous by contract and not awaited; a custom reporter + // that returns a promise anyway has its rejection surfaced here so it cannot + // crash the CLI with an unhandled rejection. void result.catch((error: unknown) => reportReplayTestReporterHookError(reporter, hookName, context, error), ); @@ -86,6 +65,23 @@ function runReplayTestReporterHook( } } +function invokeReplayTestReporterLiveHook( + reporter: ReplayTestReporter, + event: ReplayTestReporterProgressEvent, + context: ReplayTestReporterContext, +): unknown { + switch (event.type) { + case 'suite-start': + return reporter.onSuiteStart?.(event.suite, context); + case 'test-start': + return reporter.onTestStart?.(event.test, context); + case 'test-step': + return reporter.onTestStep?.(event.test, context); + case 'test-result': + return reporter.onTestResult?.(event.test, context); + } +} + function reportReplayTestReporterHookError( reporter: ReplayTestReporter, hookName: ReplayTestReporterLiveHook, diff --git a/src/replay/test/reporters/types.ts b/src/replay/test/reporters/types.ts index 157ab2a51..1254e0c58 100644 --- a/src/replay/test/reporters/types.ts +++ b/src/replay/test/reporters/types.ts @@ -63,13 +63,14 @@ export type ReplayTestReporterProgressEvent = export type ReplayTestReporter = { name: string; - onSuiteStart?( - suite: ReplayTestSuiteStart, - context: ReplayTestReporterContext, - ): void | Promise; - onTestStart?(test: ReplayTestCase, context: ReplayTestReporterContext): void | Promise; - onTestStep?(test: ReplayTestStep, context: ReplayTestReporterContext): void | Promise; - onTestResult?(test: ReplayTestResult, context: ReplayTestReporterContext): void | Promise; + // Live hooks are synchronous: they run from the daemon progress stream reader as + // events arrive and are not awaited, so any returned promise is fire-and-forget and + // may not settle before `onSuiteEnd`. Keep per-event work synchronous and flush + // async work from `onSuiteEnd`, which the CLI awaits before exiting. + onSuiteStart?(suite: ReplayTestSuiteStart, context: ReplayTestReporterContext): void; + onTestStart?(test: ReplayTestCase, context: ReplayTestReporterContext): void; + onTestStep?(test: ReplayTestStep, context: ReplayTestReporterContext): void; + onTestResult?(test: ReplayTestResult, context: ReplayTestReporterContext): void; onSuiteEnd?(suite: ReplaySuiteResult, context: ReplayTestReporterContext): void | Promise; getExitCode?(suite: ReplaySuiteResult): number | undefined; }; diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index f8348034b..70ff506e4 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -188,7 +188,7 @@ export default createReporter; The CLI loads reporter modules with Node dynamic `import()`. Use `.mjs` or `.js` files at runtime; for TypeScript, compile the reporter to JavaScript before passing it to `--reporter`. Loading `.ts` files directly depends on Node's type-stripping behavior and is not part of the supported reporter contract. -Live reporter hooks are semantic: `onSuiteStart`, `onTestStart`, `onTestStep`, and `onTestResult` run while the daemon request is active; generic command progress frames are not exposed to test reporters. `onSuiteEnd` receives the final suite result. `getExitCode` can override whether the finished suite exits successfully; the highest reporter-provided exit code wins, and failed tests exit with `1` when no reporter raises the code further. +Live reporter hooks are semantic: `onSuiteStart`, `onTestStart`, `onTestStep`, and `onTestResult` run while the daemon request is active; generic command progress frames are not exposed to test reporters. These live hooks are synchronous — they run from the progress stream as events arrive and are not awaited, so keep their work synchronous and defer anything async to `onSuiteEnd`, which the CLI awaits before exiting. `onSuiteEnd` receives the final suite result. `getExitCode` can only raise the suite exit code, never lower it: the highest reporter-provided code wins and failed tests still exit with `1` when no reporter raises it further, so a reporter cannot mask a failing suite. ## Parametrise `.ad` scripts