diff --git a/ts/docs/plans/vscode-devx/README.md b/ts/docs/plans/vscode-devx/README.md index 07f218bbe1..a71938395a 100644 --- a/ts/docs/plans/vscode-devx/README.md +++ b/ts/docs/plans/vscode-devx/README.md @@ -20,13 +20,15 @@ ### Companion docs (architecture, agent-drivability, status) -| Doc | What it answers | -| ------------------------------------ | ------------------------------------------------------------------------------------------------------------ | -| [DESIGN.md](./DESIGN.md) | Architecture; **§3.0** the guiding principle — headless core, thin presenters, three audiences. | -| [USER-STORY.md](./USER-STORY.md) | The authoring loop and the three interaction modes (human / AI-agent / hybrid). | -| [STUDIO-AGENT.md](./STUDIO-AGENT.md) | The `studio` agent **action-surface reference** (groups A–F, tiers, approval boundary). Phasing lives in 05. | -| [STATUS.md](./STATUS.md) | What's built, known issues, and the ready-to-start next slices (pointing into the 05 §11 phasing). | -| [QUICKSTART.md](./QUICKSTART.md) | Build/test/run commands and where each piece lives. | +| Doc | What it answers | +| -------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| [DESIGN.md](./DESIGN.md) | Architecture; **§3.0** the guiding principle — headless core, thin presenters, three audiences. | +| [USER-STORY.md](./USER-STORY.md) | The authoring loop and the three interaction modes (human / AI-agent / hybrid). | +| [STUDIO-AGENT.md](./STUDIO-AGENT.md) | The `studio` agent **action-surface reference** (groups A–F, tiers, approval boundary). Phasing lives in 05. | +| [STATUS.md](./STATUS.md) | What's built, known issues, and the ready-to-start next slices (pointing into the 05 §11 phasing). | +| [replay-l4b-design.md](./replay-l4b-design.md) | The replay **fidelity depth ladder** — sandbox-convergence framing and the build-from-git-ref (L4b) first-slice plan (Steps 1–3). | +| [impact-report-multi-variant-design.md](./impact-report-multi-variant-design.md) | The Impact Report **breadth axis** — Baseline + N Variants compare matrix and regression bisect. | +| [QUICKSTART.md](./QUICKSTART.md) | Build/test/run commands and where each piece lives. | --- diff --git a/ts/docs/plans/vscode-devx/STATUS.md b/ts/docs/plans/vscode-devx/STATUS.md index b6203e3fd4..d838b168ed 100644 --- a/ts/docs/plans/vscode-devx/STATUS.md +++ b/ts/docs/plans/vscode-devx/STATUS.md @@ -4,8 +4,9 @@ > the **intended** end state), this file tracks what is **actually built today** > and what remains. Update it whenever a capability changes state. -Branch: feature work has merged to `main` via PR #2468; ongoing work continues -on `dev/talzacc/typeagent_studio_part3` (the real replay path). +Branch: earlier feature work merged to `main` via PR #2468; ongoing work +continues on the `dev/talzacc/typeagent_studio_part*` stack (currently part5 — +replay fidelity + Impact Report UX). ## Capability matrix @@ -13,24 +14,72 @@ Legend: ✅ done · 🟡 partial · ❌ not started. "Wired to dispatcher" = backed by the real TypeAgent dispatcher/engine rather than an in-memory stand-in. -| Capability | Core logic | UI | Wired to dispatcher | Tested | -| -------------------------------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------------------ | -------------------------------------------------------------------------- | ------ | -| Sandboxes (lifecycle, agent load/unload) | ✅ in-memory | ✅ tree view (channel-backed; agent runtime is source of truth) | ❌ in-memory only (no subprocess/isolated dispatcher) | ✅ | -| Sandbox persistence across reload/restart | ✅ | ✅ (auto-restore) | n/a | ✅ | -| Corpora (federation: in-repo / captures / external / feedback) | ✅ file-backed | ✅ tree view | n/a | ✅ | -| Event Log (structured event stream) | 🟡 in-memory ring buffer | ✅ tree view (+ channel-backed source) | ❌ most emit sites unwired | ✅ | -| Agent health (status bar + findings) | 🟡 heuristic/filesystem checks | ✅ status bar | ❌ no real schema parse / grammar compile | ✅ | -| Collisions (cross-schema grammar overlap) | ✅ real NFA scanner over compiled `.ag.json` | ✅ tree view + Skipped group + auto-scan (+ channel-backed source) | n/a (reads compiled grammars) | ✅ | -| Feedback (thumbs up/down → corpus) | ✅ | ✅ command | n/a | ✅ | -| Replay / compare engine | ✅ static-grammar, schema-enriched (L1) & construction-cache (L2) resolvers | ✅ Impact Report webview (`ActionDelta[]`) | 🟡 grammar + live construction cache (L1–L2); no two-version build (L3–L4) | ✅ | -| Onboarding bridge (snapshot/restore, stale detection) | ✅ | ✅ commands | ❌ in-memory bridge | ✅ | -| Repo-root detection (find `packages/agents`) | ✅ | ✅ warn toast + status bar | n/a | ✅ | -| Webview infrastructure (`webviewKit`) | ✅ CSP/nonce host + typed protocol | ✅ singleton-panel host | — | ✅ | -| Impact Report webview | ✅ `replayCorpus` over channel | ✅ context header, A/B controls, durable state | 🟡 grammar + construction-cache replay (L1–L2) | ✅ | -| Player corpus capture | ❌ | ❌ | ❌ | ❌ | -| Schema Studio | ❌ | ❌ | ❌ | ❌ | -| Live Trace | ❌ | ❌ | ❌ | ❌ | -| `agr-language` / `vscode-shell` refactor onto core | 🟡 dependency edge only | — | ❌ no behavioral integration | ❌ | +| Capability | Core logic | UI | Wired to dispatcher | Tested | +| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------ | +| Sandboxes (lifecycle, agent load/unload) | ✅ in-memory | ✅ tree view (channel-backed; agent runtime is source of truth) | ❌ in-memory only (no subprocess/isolated dispatcher) | ✅ | +| Sandbox persistence across reload/restart | ✅ | ✅ (auto-restore) | n/a | ✅ | +| Corpora (federation: in-repo / captures / external / feedback) | ✅ file-backed | ✅ tree view | n/a | ✅ | +| Event Log (structured event stream) | 🟡 in-memory ring buffer | ✅ tree view (+ channel-backed source) | ❌ most emit sites unwired | ✅ | +| Agent health (status bar + findings) | 🟡 heuristic/filesystem checks | ✅ status bar | ❌ no real schema parse / grammar compile | ✅ | +| Collisions (cross-schema grammar overlap) | ✅ real NFA scanner over compiled `.ag.json` | ✅ tree view + Skipped group + auto-scan (+ channel-backed source) | n/a (reads compiled grammars) | ✅ | +| Feedback (thumbs up/down → corpus) | ✅ | ✅ command | n/a | ✅ | +| Replay / compare engine | ✅ schema-enriched grammar (L1), construction-cache (L2), selectable two-mode (grammar/cache) + opt-in live wildcard validation (L4a) | ✅ Impact Report webview (`ActionDelta[]`) | 🟡 grammar + live construction cache + working-tree wildcard validation; no two-version build-from-ref (L4b, deferred to P-7) | ✅ | +| Onboarding bridge (snapshot/restore, stale detection) | ✅ | ✅ commands | ❌ in-memory bridge | ✅ | +| Repo-root detection (find `packages/agents`) | ✅ | ✅ warn toast + status bar | n/a | ✅ | +| Webview infrastructure (`webviewKit`) | ✅ CSP/nonce host + typed protocol | ✅ singleton-panel host | — | ✅ | +| Impact Report webview | ✅ `replayCorpus` over channel | ✅ context header, A/B controls, Grammar/Cache + Validate toggles, durable state | 🟡 grammar + construction-cache + working-tree wildcard validation (L1–L2, L4a) | ✅ | +| Corpus capture (real utterances → actions, any agent) | ❌ | ❌ | ❌ | ❌ | +| Schema Studio | ❌ | ❌ | ❌ | ❌ | +| Live Trace | ❌ | ❌ | ❌ | ❌ | +| `agr-language` / `vscode-shell` refactor onto core | 🟡 dependency edge only | — | ❌ no behavioral integration | ❌ | + +## Roadmap at a glance — gate spine + backlog + +This is the single at-a-glance tracker. The MVP is fenced by five acceptance +gates ([`04-mvp-slice.md` §3](./04-mvp-slice.md)); phase sequencing lives in +[`05-implementation-plan.md` §11](./05-implementation-plan.md#11-phasing--concrete-sequencing). +The **gate spine** is the critical path; the **backlog** holds everything +deliberately off it, each row tagged with its parent gate and precondition so it +is clear why it waits. There is no second roadmap — depth (L4b) and breadth +(multi-variant / multi-agent) are backlog rows here, not a parallel plan. + +### Gate spine (MVP critical path) + +| Gate | Journey | Capability | Phase | Status | +| ----- | -------------------- | ------------------------------------------------------- | ----- | -------------------------------------------------------------------------------- | +| **A** | J1 Stand up an agent | New-Agent Wizard | P-2 | ❌ not started (onboarding bridge ✅) | +| **B** | J2 Tune schema | Schema Studio | P-4 | ❌ not started | +| **C** | J4 Find a regression | Impact Report ≥ 80% red/green on hand-labelled `player` | P-3 | 🟡 **long pole** — engine L1–L4a ✅; corpus capture + predicate + measurement ❌ | +| **D** | J5 Debug a trace | Trace Viewer | P-3 | ❌ not started | +| **E** | J6 Observe live | Live Trace + status bar | P-5 | ❌ not started | + +**Critical path now: close Gate C** — corpus capture (the agent-agnostic capture +path; `player` is just the anchor set the gate is scored on) → predicate tuning → +run the ≥ 80% validation. It is the only _tunable_ gate and §7's top risk. + +### Backlog (off the critical path — tagged, not a separate plan) + +Each item is a child of a gate/journey; **none is required to pass A–E**. Track +tags: **depth** = how faithfully one replay side is realized (improves Gate C +accuracy); **breadth** = how many things are compared at once (multiplies cells); +**infra** = enabling plumbing. + +| Item | Parent | Track | Precondition | Status | +| ------------------------------------------------------------------------- | ----------- | ------------------ | ----------------------------------- | ------------------------------ | +| Per-side fidelity matrix (which layers ran on each side + skip reasons) | Gate C / J4 | depth | — | ✅ shipped (L4b Step 1) | +| "Sandbox A / Sandbox B" relabel of the A/B columns | Gate C / J4 | depth | — | optional cosmetic (L4b Step 2) | +| **L4b** build-from-ref sandboxes (real compiled ref side) | Gate C / J4 | depth (P-7) | Gate C banked | deferred flagged epic (Step 3) | +| Multi-variant compare (Baseline vs N variants; bisect / first-divergence) | J4 | breadth — versions | Gate C banked; must preserve Gate C | post-MVP | +| Multi-agent / multi-corpus replay (`code`, 684 utterances) | J4 | breadth — agents | Gate C banked | post-MVP (§2 exclude) | +| Active-sandbox selector + per-sandbox scoping | — | infra | single-sandbox E2E | P-7a | +| Sandbox copy-on-write overlay (true sandbox-local A/B) | — | infra | P-7a | P-7b | + +Why depth and breadth are distinct dials on the same `replayCorpus` (F4.1) +primitive: **depth** raises the fidelity of each compared _cell_ (and thus Gate C +accuracy); the two **breadth** axes raise the _number_ of cells (versions × +agents). They compose and are independent. Detailed slice breakdowns live in the +design docs ([the L4b sandbox-convergence plan](./replay-l4b-design.md) and +[the multi-variant compare design](./impact-report-multi-variant-design.md)). ## The long pole @@ -41,7 +90,7 @@ Impact Report) is **progressing but not yet closed**: builder, singleton-panel host, typed host↔webview protocol, browser-neutral replay view model) and the Impact Report webview are built and tested — the foundation Schema Studio, Wizard, Trace, and Live Trace will reuse. -- **Replay has climbed the fidelity ladder to L2.** Beyond the original identity +- **Replay has climbed the fidelity ladder to L4a.** Beyond the original identity resolver, replay now runs **static-grammar** matching, **schema-enriched grammar** matching (L1: the agent's grammar is enriched with checked-variable metadata from its action schema and matched through the real `GrammarStore`), @@ -52,8 +101,12 @@ Impact Report) is **progressing but not yet closed**: a phantom hit). The construction cache is consulted for the **working tree only** (caches are runtime artifacts, never committed / never read at a git ref) and degrades cleanly to L1 when no live cache is found or it has gone - stale. Results are still indicative for grammar-resolved rows — **L3–L4** - (deterministic dispatch, build-from-git-ref two-version sandboxes) remain, and + stale. Two replay **modes** are now selectable in the Impact Report (grammar-only + vs. construction-cache), and an opt-in **live wildcard validation** pass (L4a) + runs the agent's real `validateWildcardMatch` over working-tree wildcard matches + (`timer`/`list`; fail-open + diagnostics). Results are still indicative for + grammar-resolved rows — **L4b** (build-from-git-ref two-version sandboxes) + remains and is **deferred to P-7** (post-Gate-C, per the implementation plan); there is no git-worktree build of two versions yet. - **No capture-to-corpus path.** `vscode-shell` depends on core but doesn't use it; without capture the Impact Report would have no real labelled corpus. @@ -167,11 +220,17 @@ Ready to start (smallest → larger): Impact Report webview that drives `replayCorpus` over the service channel and renders the `ActionDelta[]` contract. The webview never opens a socket (webview → extension host → channel → agent runtime). -3. **Player corpus capture** — wire `vscode-shell` request/feedback IDs into the - core corpus. +3. **Corpus capture** — wire `vscode-shell` request/feedback IDs into the + core corpus. Agent-agnostic capture path (works for any agent); `player` is + simply the first corpus captured because Gate C is scored on it. 4. **One real replay path** — one agent, one utterance, working tree vs. HEAD, real dispatch; validate the Impact Report `ActionDelta[]` contract (which the - agent's `ValidateChange` and the webview both consume). + agent's `ValidateChange` and the webview both consume). **Largely done:** the + grammar replay path (L1 schema-enriched, L2 construction-cache, two selectable + modes, L4a opt-in wildcard validation) is live and validated against the + contract. The remaining fidelity rung **L4b** (build-from-ref two-version + sandboxes) is **deferred to P-7** (post-Gate-C). **Live priority is now #3 + (corpus capture) → Gate C measurement** — the headline acceptance bar. 5. **Active-sandbox selector + per-sandbox scoping** (plan phase **P-7a**; sequenced **after** the single-sandbox E2E closes) — collisions and corpora are intrinsically per-sandbox (a collision is a function of the co-loaded agent diff --git a/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md b/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md new file mode 100644 index 0000000000..fa0fee2b6e --- /dev/null +++ b/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md @@ -0,0 +1,288 @@ +# Impact Report — multi-variant compare (Baseline + N Variants) — design + +**Builds on:** the existing binary A/B Impact Report (replay engine, `replayCorpus`, +`ActionDelta`, `ReplaySummary`) and the sandbox-convergence framing in +[`replay-l4b-design.md`](./replay-l4b-design.md) (A/B reframed as "Sandbox A / Sandbox B"). + +## One-line idea + +Replace the fixed two-target compare (A vs B) with **one Baseline compared against +an ordered list of Variants** (A vs B1, B2, … Bn), rendered as a matrix. Binary +A/B becomes the N=1 special case. + +## Why — the journeys this unlocks + +- **Bisect a regression.** "Which of these N commits changed the answer?" Baseline = + known-good ref; variants = suspect commits. One run instead of N binary compares. +- **Track against history.** Working tree vs the last few releases at once. +- **Compare candidate fixes.** One baseline vs several feature branches. +- **Fidelity sweep.** Same version, varied replay options (cache on/off, validation + on/off) — see how the answer moves as fidelity climbs. + +## Two orthogonal axes (this is the key framing) + +The Impact Report has two independent dimensions: + +- **Depth / realization axis** (the L4b ladder): how faithfully ONE side is + realized — grammar → schema enrichment → construction cache → wildcard validation + → full built dispatch. This is [`replay-l4b-design.md`](./replay-l4b-design.md) (Steps 1–3). +- **Breadth axis** (THIS doc): how MANY comparison targets sit beside the baseline. + +They compose. Each variant is a sandbox realized at some depth and carries its own +fidelity descriptor. Multi-variant needs **none** of the deferred build epic — it +works entirely at today's L1 grammar fidelity and gets richer for free as the depth +axis advances. + +``` + depth (realization, L4b) ──▶ + breadth B0(baseline) grammar | +cache | +validation | built + (this B1 grammar | +cache | … + doc) B2 grammar | … + │ … + ▼ +``` + +## Grounding in the MVP plan and the gates (A–E) + +Canonical source: [`04-mvp-slice.md`](./04-mvp-slice.md) §3 (gates) + §2 (excludes); +journeys in [`02-journeys.md`](./02-journeys.md). + +The MVP is fenced by five gates; the load-bearing one for replay is: + +- **Gate C — headline / validation gate (journey J4, persona P4 "Regression Owner").** + The Impact Report, run with the default "likely-bad change" predicate (F4.4) on a + **hand-labelled `player` regression set**, agrees with developer judgment on **≥ 80% + of rows** (red = regression, green = improvement), under the **deterministic + `needs-explanation` miss policy**. Evaluated on the **binary** compare + (`working tree` vs `HEAD~1`). This is "the single hardest gate." + +Where multi-variant sits relative to the gates: + +- **It is POST-MVP and on none of A–E's critical path.** Every gate (A stand-up, B + schema loop, C headline, D drill-in, E live mirror) is satisfied by the binary + compare. Multi-variant adds **breadth**, which no MVP gate requires. +- **It is the sibling of the existing §2 exclude.** §2 already defers + **multi-agent / multi-corpus simultaneous replay** (one agent → many agents) as + post-MVP. Multi-variant is the _other_ replay-scaling axis — **one agent → many + versions**. Both are post-MVP breadth extensions of the same `replayCorpus` engine; + they are independent (you can have either without the other). +- **It must PRESERVE Gate C, not move it.** Because binary is exactly N=1, the + slice-1 result-shape refactor has to keep single-variant `ActionDelta`/row semantics + byte-for-byte identical, so the ≥ 80% agreement number doesn't shift. The F4.4 + red/green predicate and the deterministic `needs-explanation` policy run **per cell** + unchanged — so each variant column carries the same Gate-C-validated semantics. +- **It is the natural post-MVP evolution of the J4 headline.** J4 is literally "find a + regression." The bisect / first-divergence feature (range-expand `good..HEAD`, + per-row first `=`→`Δ` column = the introducing commit) is the strongest realization + of that journey — it turns "is there a regression vs HEAD~1" into "which commit + introduced it," for persona P4, once the MVP gate is banked. + +### Two scaling axes vs one depth axis (the 2×2) + +`replayCorpus` (F4.1) is the primitive. Three independent dials sit on top: + +| Dial | What it scales | Doc | MVP status | +| --------------------------------- | --------------------------------------------------------------------- | ------------------------------------------------ | ---------------------- | +| **Fidelity / depth** (L1–L4b) | how faithfully each _side_ is realized → improves Gate C row accuracy | [`replay-l4b-design.md`](./replay-l4b-design.md) | L1–L3 in; L4b deferred | +| **Breadth — versions** (this) | how many _variants_ beside the baseline | this doc | post-MVP | +| **Breadth — agents/corpora** (§2) | how many _agents_ replayed at once | [`04-mvp-slice.md` §2](./04-mvp-slice.md) | post-MVP | + +Depth improves the _quality_ of each cell (and thus Gate C); the two breadth axes +multiply the _number_ of cells. Multi-variant only touches breadth-versions, composes +with depth per-cell, and is orthogonal to breadth-agents. + +## The reframe: Baseline + Variants + +- **A → Baseline**: the fixed reference column. +- **B → Variants**: an ordered list of comparison targets. +- A variant is not just a version — it is **version + replay options**: + +```ts +interface ReplayVariant { + version: VersionSpec; // { kind:"git"; ref } | { kind:"workingTree" } + mode: StudioReplayMode; // "nfa-grammar" | "completionBased-cache" + validateWildcards: boolean; + label?: string; // display label; defaults from version/options +} +``` + +Folding options into the variant unifies _version_ comparison and _fidelity_ +comparison under one abstraction — the option-sweep journey is then free. + +## Data model generalization + +The engine's binary pairs (`actionA`/`actionB`, `cacheStateA`/`cacheStateB`, +`collisionsA`/`collisionsB`, `latencyA`/`latencyB`) become baseline + a variants +array. Sketch: + +```ts +interface VariantCell { + action?: unknown; + cacheState: ReplayCacheState; + collisions: CollisionDetectedEvent[]; + latency: number; + requestId: string; + relation: "equal" | "changed" | "new-match" | "lost-match"; // vs baseline + fidelity: FidelityReport; // per-variant (composes with the depth axis) +} + +interface RowResult { + utterance: string; + utteranceId: string; + source: CorpusSource; + baseline: { + action?: unknown; + cacheState: ReplayCacheState; + collisions: CollisionDetectedEvent[]; + latency: number; + requestId: string; + }; + variants: VariantCell[]; // index-aligned with the request's variants[] +} + +interface VariantSummary { + // one per variant, vs baseline + equalCount: number; + changedCount: number; + newMatchCount: number; + lostMatchCount: number; + collisionDelta: number; + duration: number; +} +``` + +- **Baseline is computed once** and reused across all variants → cost is linear in N, + not N². +- `variants.length === 1` serializes/renders identically to today's A/B. + +### Coordination with L4b Step 1 (important — shape it once) + +L4b Step 1 plans to add `sideFidelity: { A: FidelityReport; B: FidelityReport }` to +`StudioReplayResult`. That is the exact `{A, B}` pair this redesign turns into +`baseline + variants[]`. To avoid refactoring the same field twice, when Step 1 +lands the fidelity descriptor should be modelled **per sandbox** — +`baselineFidelity: FidelityReport` + `variants[i].fidelity` — rather than a fixed +A/B pair. Even before multi-variant ships, expressing it as "baseline + a list of +one" makes the later generalization a no-op on the type. + +## UI design + +### Action bar + +``` +Baseline:[▾ Working tree] Variants:[HEAD ✕][HEAD~1 ✕][feat/x ✕][＋] ⚙ options ▶ Run +``` + +- Variants are chips; `＋` opens today's version picker. The swap button retires + (the compare is asymmetric now). +- Per-variant options (mode/validation) hang off each chip's own little gear, or a + shared default with per-chip overrides. + +### Per-variant summary strip + +``` + HEAD HEAD~1 feat/x + ✓ 48 = · 2Δ ✓ 47 = · 3Δ ⚠ 31 = · 19Δ · 4− +``` + +One card per variant: equal/changed/new/lost counts vs baseline; click to focus. + +### Matrix (primary view) + +``` +Utterance │ Baseline │ HEAD │ HEAD~1 │ feat/x +"play despacito by fonsi" │ playTrack│ = │ = │ Δ +"turn on the lights" │ toggle │ = │ Δ │ Δ +"set a 5 minute timer" │ setTimer │ = │ = │ = +"queue some jazz" │ (none) │ = │ = │ + new +``` + +- Sticky baseline column; one column per variant; horizontal scroll past ~4–5. +- Cell glyph relative to baseline — `=` equal, `Δ` changed, `+` new, `−` lost — + reusing today's vocabulary, just per column. +- Click a **cell** → drawer with the full baseline-vs-that-variant action diff + (reuses today's binary detail view verbatim). + +### Two view modifiers + +- **Divergence filter** — hide rows that are `=` across every variant (essential at + scale; most rows are stable). +- **Row fingerprint** (compact) — collapse columns to `●●○` per row (matches B1,B2; + differs B3) when there are many variants. + +## The payoff: ordered variants → first-divergence / bisect + +If variants are an ordered chronological range (old→new), the matrix answers "find +the regression" directly: per row, the column where it first flips `=`→`Δ` **is the +commit that changed it**. + +- **Range-expand source**: give a range `good..HEAD`; auto-create one variant per + commit (capped, e.g. ≤8); highlight each row's first-divergence column. +- Per-row "introduced at →" annotation pointing at that commit. + +This makes the headline journey ("I edited grammar; which commit regressed it?") a +single run. + +## Where variants come from + +1. **Manual** — add ref / pick commit / working tree (today's picker, N times). +2. **Range-expand** — `A..B` → a variant per commit (bisect mode above). +3. **Option sweep** — same version, varied `mode`/`validateWildcards` (fidelity + sweep). Free once options live on the variant. + +## Cost, perf, back-compat + +- Baseline computed once; each variant is one extra pass. Cap N with a warning; + range-expand caps commits. +- **Progressive fill** — stream columns in as each variant completes (the matrix is + column-addressable) rather than blocking on the whole sweep. +- **Cache by `(agent, version, mode)`** so adding a variant doesn't recompute + existing columns. +- Fully backward compatible: N=1 renders exactly like today; the simple two-target + bar is the default and the matrix is progressive disclosure via `＋`. + +## Delivery slices + +1. **Result-shape generalization** — baseline + `variants[]` in the core result + types + engine, N=1 rendering identical. Pure refactor; the risk-bearing slice. + Best landed together with (or immediately after) L4b Step 1 so the fidelity + descriptor is shaped per-sandbox once. +2. **Matrix UI** — variant chips + per-variant summary cards + matrix columns + (manual variants only). +3. **Divergence filter + cell drill-down drawer** (reuses binary detail view). +4. **Range-expand + first-divergence** (the bisect payoff). +5. **Option-sweep variants** (fidelity sweep). + +## Where it fits in the work plan + +A **new workstream on the breadth-versions axis**, parallel to the L4b realization +ladder, and **gated behind the MVP**: + +- **Precondition: Gate C is banked first.** Multi-variant is post-MVP (§2 sibling of + the deferred multi-agent axis). Do not start it until the binary Impact Report has + passed Gate C on the hand-labelled `player` set — otherwise slice 1's refactor risks + perturbing the very rows the gate measures. +- **Depends on:** the sandbox framing — best sequenced _after_ L4b Steps 1–2 + (per-side fidelity readout + "Sandbox A/B" relabel), because multi-variant is the + natural generalization of "Sandbox A / Sandbox B" to "Baseline + N variant + sandboxes," and slice 1 shares the `StudioReplayResult` fidelity refactor with + Step 1. +- **Independent of:** L4b Step 3 (build-from-ref) and the §2 multi-agent axis. + Multi-variant ships entirely at L1 grammar fidelity and simply gets richer cells as + the depth axis advances. +- **Serves:** persona P4 (Quality / Regression Owner) and journey J4 — it extends the + validated headline ("find a regression vs HEAD~1") into bisect ("which commit + introduced it"). +- **Sequencing recommendation:** [MVP Gate C banked] → L4b Step 1 (shape fidelity + per-sandbox) → L4b Step 2 (relabel) → this slice 1 (baseline + variants[] result + shape) → this slice 2 (matrix) → slices 3–5. Step 3 (build epic) stays deferred and + orthogonal. + +## Open questions + +- Per-variant options vs one shared option set with overrides — how much per-chip + control is worth the UI weight? +- Default column cap (N) and range-expand commit cap? +- Is symmetric variant-vs-variant comparison ever wanted, or is baseline-relative + always sufficient? (Baseline-relative keeps the model simple; a "distinct + outcomes per row" cluster count could cover the rest without full N×N.) diff --git a/ts/docs/plans/vscode-devx/replay-l4b-design.md b/ts/docs/plans/vscode-devx/replay-l4b-design.md new file mode 100644 index 0000000000..11a5d61b19 --- /dev/null +++ b/ts/docs/plans/vscode-devx/replay-l4b-design.md @@ -0,0 +1,173 @@ +# Replay L4b — build-from-git-ref — first-slice design + +**Builds on:** L1 (schema-enriched grammar), L2 (cache, working-tree), L3 (deterministic dispatch), +L4a (live working-tree wildcard validation). This is the deepest rung of the replay fidelity ladder. + +## The honest scope problem + +The fidelity ladder classifies L4 as the largest, fully greenfield rung — multi-week and behind a +flag — needing a product steer before it is attempted. A faithful full L4b is genuinely multi-week +and should NOT be attempted in one pass. So this doc scopes a **tractable first slice** plus the +decision that gates which slice to build. + +## What actually changes between two versions (verified) + +- **Grammar (`.agr`)** — committed, versionable. Already compared at a ref via `git show`. +- **Action schema (`.ts` source)** — committed, versionable. Parsed from SOURCE (no build needed: + `parseActionSchemaSource` reads TS text), already read at a ref via `git show` (L1 enrichment). +- **Validator / handler code (`.ts` → built `.js`)** — committed source, but only the **working + tree** is BUILT. L4a runs the real `validateWildcardMatch` only on the working-tree side because + that's the only side whose JS exists. The git-ref side can't run validators without a build. +- **Construction cache (`.json`)** — NOT committed (runtime artifact). Can NEVER be read at a ref. + So "frozen cache at a ref" is impossible by construction; L4b cannot deliver it. + +⇒ The ONLY thing a full build-from-ref unlocks that we don't already have is **running the ref's +compiled validator/handler JS** (and a freshly-warmed cache, which isn't a frozen artifact anyway). + +## Current materialization limits (what a worktree would fix) + +- `git show :` reads ONE file at a time. Imported `.agr` grammars (a grammar that + `import`s another `.agr`) aren't resolvable at a ref — each import needs its own `git show` with a + versioned FileLoader (deferred since the real-replay slice). The doc notes 57/65 `.agr` have + imports but **none import another `.agr`**, so this is rare. +- The `ReplayActionResolver` contract has **no `dispose()`** — `engine.ts` `cancel()` just flips a + flag; the runtime consumes rows without a `finally` cleanup hook. Any worktree/checkout slice MUST + add a disposer first (a hard prerequisite before any build-from-ref work). + +## Three candidate first slices + +### Option A — Worktree materializer + resolver `dispose()` lifecycle (infra-first, NO build) + +Replace per-file `git show` with a single `git worktree add --detach ` per git-ref side; +read all grammar/schema/paramSpec files from the checkout; `git worktree remove` in a new +`dispose()`/`finally` path threaded through resolver → runtime. + +- **Unlocks:** imported-`.agr` + whole-tree schema at a ref; establishes the disposer lifecycle that + the full build rung mandates. De-risks Option C's hardest plumbing. +- **Honest gain:** modest today (imported-`.agr` is rare), but it is the **required foundation** for + any build-from-ref and removes the "no disposer" blocker. +- **Cost:** small–medium. Main risks: worktree cleanup on crash, lock contention with the user's live + repo, Windows path/locking. Mitigated by `--detach` (no branch checkout) + a temp dir under the OS + temp root + best-effort `git worktree prune` on dispose. + +### Option B — Ref-side wildcard validation via "unchanged-validator" shortcut (NO build) + +Run L4a on the git-ref side too, but ONLY when `git diff ..worktree -- ` is +empty (the agent's validator JS is unchanged between the ref and the built working tree), so the +already-built working-tree module faithfully represents the ref's validator. If the validator +changed, skip ref-side validation (honest, labelled "ref validation: skipped (validator changed)"). + +- **Unlocks:** symmetric wildcard validation for the common case (validator untouched, only grammar/ + schema edited — exactly the regression journey) with ZERO build machinery. +- **Honest gain:** real for the headline journey; limited to validators (timer/list). +- **Cost:** small. Risk: the "unchanged ⇒ safe to reuse working-tree module" inference must be + conservative (compare the whole built package's source closure, not just one file, or restrict to + the allowlisted agents whose validator lives in one known file). + +### Option C — Full build-from-ref (the real L4b, multi-week, behind a flag) + +`git worktree add` → `pnpm install --filter ...` → scoped `tsc -b`/`asc`/`agc` → load the +built dist via a provider pointed at the worktree → full deterministic dispatch at the ref. + +- **Unlocks:** true arbitrary-ref full-dispatch compare. +- **Cost:** multi-week, fully greenfield. Minutes per run (install + build). Many failure modes + (install flakiness, native deps, disk). Clearly a flagged epic, not a one-session slice. + +## Recommendation + +Ship **Option A** now (the lifecycle + worktree foundation — the mandatory step 1 of the L4 epic), +and optionally **Option B** as the first user-visible fidelity win on top of it. Defer **Option C**'s +install/build orchestration to a tracked, flagged epic, started only if a concrete journey needs +ref-vs-ref full dispatch. + +## Open decision + +Which first slice? **A (infra foundation)**, **A+B (foundation + ref-side validation win)**, or +**commit to C (the multi-week build epic) now**? + +--- + +## Design review — verdict & resolution + +- **Option A — CUT as standalone / defer.** Worktree + disposer is infra ahead of need; `git show` + is strictly better while we only READ committed source (no checkout cleanup, no repo-lock + contention vs the user's live checkout, no Windows file-lock/crash-recovery burden). Build the + lifecycle only when something real needs disposing (i.e. inside C). +- **Option B — SHIP ONLY IF very conservative + clearly labelled.** "Validator file unchanged" is + NOT sound alone (a changed transitive import / helper / lockfile / tsconfig / data file silently + breaks it). Acceptable only if the whole validator source CLOSURE is unchanged, only for the tiny + timer/list validators, and surfaced as "opportunistic," not full ref fidelity. If closure + detection starts to grow → cut (it becomes a partial rebuild system). +- **Option C — DEFER to a flagged epic** pending a real product journey. Guardrails before starting: + explicit flag; cache built refs by commit SHA; hard timeout + clear failure modes; disposable + worktree lifecycle first; per-agent build recipes (not guessed generic builds); strong UI + labelling when a ref can't be built. +- **Better immediate slice:** ship **fidelity transparency** — per side/version report which + fidelity layers ran (grammar / schema enrichment / construction cache / wildcard validation / + dispatch) and the exact skip reason, plus a preflight "build-from-ref would add X (unsupported + today)." Honest user value now, near-zero risk, and the right scaffold that makes L4b's eventual + payoff visible/measurable. + +**Resolution:** sandboxes are the design's intended EXECUTION CONTAINERS (today they +are metadata-only shells); a VersionSpec is a _recipe_ and L4b is the _build_ that realizes a recipe +into a populated sandbox. The UI evolves "pick 2 commits" → "pick 2 sandboxes" along a single +**realization axis**, in three incremental steps. Near-term = Steps 1–2 (transparency, no build); +Step 3 (build-from-ref) stays the deferred flagged epic. + +## The sandbox-convergence plan (chosen direction) + +``` +VersionSpec (recipe) L4b build (Step 3) Sandbox (instance) + {git, ref} ──▶ worktree → tsc/asc → load dist ──▶ built agent + dispatcher + cache + {workingTree} ──▶ (already built locally) ──▶ the live agent +``` + +### Step 1 — per-side fidelity readout (SHIP NOW, no build, no new replay behavior) + +Surface signals the resolver ALREADY computes. Add to `StudioReplayResult`: + +```ts +interface FidelityReport { + realization: "source" | "built-live"; // git ref ⇒ source; working tree ⇒ built-live + layers: Record< + | "grammar" + | "schemaEnrichment" + | "constructionCache" + | "wildcardValidation" + | "dispatch", + { status: "ran" | "skipped" | "unavailable"; reason?: string } + >; +} +interface StudioReplayResult { + /* …existing… */ sideFidelity: { A: FidelityReport; B: FidelityReport }; +} +``` + +Derive from existing `methodA`/`methodB` (`static-grammar`/`schema-grammar`/`construction-cache`), +`grammarResolver.enriched`, `wildcardValidation`, and `versionA/B.kind`. Pure mapping + unit test. +Webview: collapsible Fidelity matrix (5 rows × A/B) + a preflight line when a side is `source`-only +and a richer realization is theoretically available. Reuse `renderValidationNote` styling. + +### Step 2 — relabel A/B as "Sandbox A / Sandbox B" (light) + +A/B group header becomes Sandbox A / Sandbox B with a `recipe · realization` sub-label. Cements the +"2 sandboxes" model. Mostly labels on top of Step 1's descriptor. + +### Step 3 — L4b build-from-ref (DEFERRED, flagged epic = old Option C) + +Picker gains a "build from ref" opt-in → host realizes a real sandbox (worktree → build → load) → +ref side climbs the Step-1 matrix to `built`. Needs the resolver `dispose()`/`finally` lifecycle, +per-agent build recipes, SHA-keyed build cache, timeouts. Subprocess-mode + the L4a RPC-context +boundary remain the hardest sub-problem (context-dependent agents). + +## Implementation plan — near-term (Steps 1–2) + +1. **core/studioRuntimeCore.ts** — add `SideFidelity`/`FidelityReport` types + a pure + `deriveSideFidelity(methodA, methodB, enriched, wildcardValidation, versionA, versionB)` mapper; + populate `result.sideFidelity` (and in the error-result path). Unit test the mapper. +2. **webviewKit/protocol.ts** — carry `sideFidelity` on `replay.result` (additive/optional). +3. **webviewKit/client/impactReport.ts + media/impactReport.css** — Fidelity matrix panel + + preflight line + Sandbox A/B relabel. Render test. +4. **Verify**: core `tsc -b` + jest, studio `tsx --test`, studio-service `npm test`, prettier, + esbuild. No commit/push without explicit user consent. + Deferred (Step 3): worktree materializer, scoped build, resolver `dispose()`, ref-side execution. diff --git a/ts/packages/studio-service/src/runtime.ts b/ts/packages/studio-service/src/runtime.ts index 0dcc7965b4..d8510b54b7 100644 --- a/ts/packages/studio-service/src/runtime.ts +++ b/ts/packages/studio-service/src/runtime.ts @@ -14,6 +14,10 @@ import { FileWorkspaceState, studioWorkspaceStateFile, } from "./fileWorkspaceState.js"; +import { + createDefaultWildcardValidatorResolver, + canValidateWildcards, +} from "./wildcardValidation.js"; /** * Candidate repository roots for Studio to inspect, most-specific first. The @@ -130,11 +134,18 @@ export function getStudioRuntime(repoRoot?: string): StudioRuntime { // Recover a pre-canonical snapshot for this workspace, if any, before // the runtime reads its persisted state. migrateLegacyWorkspaceState(stateFile, resolved); - runtime = createStudioRuntimeCore({ - workspaceState: new FileWorkspaceState(stateFile), - globalStorageFsPath: studioProfileDir(), - workspaceFolderFsPaths: candidates, - }); + runtime = createStudioRuntimeCore( + { + workspaceState: new FileWorkspaceState(stateFile), + globalStorageFsPath: studioProfileDir(), + workspaceFolderFsPaths: candidates, + }, + { + resolveWildcardValidator: + createDefaultWildcardValidatorResolver(), + resolveCanValidateWildcards: canValidateWildcards, + }, + ); runtimeCache.set(key, runtime); } return runtime; diff --git a/ts/packages/studio-service/src/studioRpcHandlers.ts b/ts/packages/studio-service/src/studioRpcHandlers.ts index cc8a80b913..147b66177d 100644 --- a/ts/packages/studio-service/src/studioRpcHandlers.ts +++ b/ts/packages/studio-service/src/studioRpcHandlers.ts @@ -68,6 +68,9 @@ export function createStudioInvokeHandlers( async listCorpusAgents(repoRoot) { return conn.getRuntime(repoRoot).listCorpusAgents(); }, + async canValidateWildcards(repoRoot, agent) { + return conn.getRuntime(repoRoot).canValidateWildcards(agent); + }, async listCorpusEntries(repoRoot, agent) { return conn.getRuntime(repoRoot).listCorpusEntries(agent); }, diff --git a/ts/packages/studio-service/src/wildcardValidation.ts b/ts/packages/studio-service/src/wildcardValidation.ts new file mode 100644 index 0000000000..e0eb891ad0 --- /dev/null +++ b/ts/packages/studio-service/src/wildcardValidation.ts @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Production wiring for the replay wildcard validator. + * + * The core `@typeagent/core` runtime exposes the validation algorithm but is + * dependency-light and does NOT know how to load a real agent module. The real + * loader lives in `default-agent-provider` (the dispatcher's agent providers), + * which drags in the whole dispatcher + every agent — far too heavy to bundle + * into the standalone, esbuild-bundled Studio service. + * + * So we load it **lazily, by dynamic import, marked external in the service + * bundle**: it resolves from `node_modules` on the in-repo `typeagent-studio + * serve` dev path, and cleanly throws → no-ops in the packaged `.vsix` (which + * ships without `node_modules`). Wildcard validation is opt-in and fail-open, so + * when the provider is unavailable replay simply stays grammar-only. + * + * NOTE: `default-agent-provider` is intentionally NOT a package.json dependency + * of this service. `default-agent-provider` aggregates every agent including + * `studio-agent`, which depends on this service — declaring the dep here closes + * a build cycle (studio-service → default-agent-provider → studio-agent → + * studio-service). Because the import is external + dynamic, the dependency is + * owned by the bundling host (`typeagent-studio`, a leaf nothing depends on), + * whose `node_modules` the external import resolves against at runtime. + */ + +import { + createWildcardMatchValidator, + type ReplayAppAgentLoader, + type ReplayValidatableAgent, + type WildcardMatchValidator, +} from "@typeagent/core/runtime"; + +/** Minimal structural view of an `AppAgentProvider` (avoids a type-only dep). */ +interface AppAgentProviderLike { + getAppAgentNames(): string[]; + loadAppAgent(agentName: string): Promise; + unloadAppAgent(agentName: string): Promise; +} + +interface DefaultAgentProviderModule { + getDefaultAppAgentProviders( + instanceDir: string | undefined, + ): AppAgentProviderLike[]; +} + +let providersPromise: Promise | undefined; + +/** + * Lazily import `default-agent-provider` and build the default providers once. + * Returns `undefined` (cached) when the module can't be resolved — e.g. inside + * the packaged extension where `node_modules` was stripped — so validation + * silently degrades to grammar-only instead of erroring. + */ +async function loadProviders(): Promise { + if (providersPromise === undefined) { + providersPromise = (async () => { + try { + // Indirect the specifier through a variable so TypeScript does + // not statically resolve the module: `default-agent-provider` is + // deliberately not a build-time dependency of this service (it + // would close a studio-service → default-agent-provider → + // studio-agent → studio-service build cycle). It is resolved at + // runtime from the bundling extension's `node_modules`, and the + // result is cast to the local structural type below. + const specifier = "default-agent-provider"; + const mod = (await import( + specifier + )) as DefaultAgentProviderModule; + return mod.getDefaultAppAgentProviders(undefined); + } catch { + return undefined; + } + })(); + } + return providersPromise; +} + +function findProvider( + providers: AppAgentProviderLike[], + agentName: string, +): AppAgentProviderLike | undefined { + return providers.find((p) => { + try { + return p.getAppAgentNames().includes(agentName); + } catch { + return false; + } + }); +} + +/** A {@link ReplayAppAgentLoader} backed by the dispatcher's agent providers. */ +function createDefaultLoader(): ReplayAppAgentLoader { + return { + async loadAppAgent(agentName) { + const providers = await loadProviders(); + if (providers === undefined) { + throw new Error( + "default-agent-provider unavailable (packaged build?)", + ); + } + const provider = findProvider(providers, agentName); + if (provider === undefined) { + throw new Error(`no provider owns agent "${agentName}"`); + } + return provider.loadAppAgent(agentName); + }, + async unloadAppAgent(agentName) { + const providers = await loadProviders(); + if (providers === undefined) { + return; + } + const provider = findProvider(providers, agentName); + await provider?.unloadAppAgent(agentName); + }, + }; +} + +/** + * Whether wildcard validation can actually run for an agent: the agent loads and + * exposes a `validateWildcardMatch`. Drives the Impact Report's validation + * toggle — when false (no validator, or the provider is unavailable in the + * packaged build) there is nothing to run, so the toggle is disabled. The agent + * is loaded only to inspect for the method, then unloaded; any failure reports + * not-validatable so the toggle stays off. + */ +export async function canValidateWildcards( + agentName: string, +): Promise { + const providers = await loadProviders(); + if (providers === undefined) { + return false; + } + const provider = findProvider(providers, agentName); + if (provider === undefined) { + return false; + } + let loaded = false; + try { + const agent = await provider.loadAppAgent(agentName); + loaded = true; + return typeof agent.validateWildcardMatch === "function"; + } catch { + return false; + } finally { + if (loaded) { + try { + await provider.unloadAppAgent(agentName); + } catch { + // Best-effort cleanup; the run-time validator reloads as needed. + } + } + } +} + +/** + * Build the runtime's `resolveWildcardValidator` option: returns a validator for + * any agent, backed by the lazy default loader. The validator fail-opens + * (load-failed / no-validator / errored diagnostics) so a run never fabricates a + * lost match; whether validation is attempted at all is the operator's per-run + * toggle, which the UI disables for agents that have no validator to run. The + * dynamic import is deferred until a wildcard match actually triggers a lookup, + * so a run that never hits a wildcard never pays for the provider. + */ +export function createDefaultWildcardValidatorResolver(): ( + agentName: string, +) => Promise { + const loader = createDefaultLoader(); + return async (agentName) => + createWildcardMatchValidator(agentName, { loader }); +} diff --git a/ts/packages/studio-service/test/studioService.spec.ts b/ts/packages/studio-service/test/studioService.spec.ts index fbfc9930dd..6759d9f59f 100644 --- a/ts/packages/studio-service/test/studioService.spec.ts +++ b/ts/packages/studio-service/test/studioService.spec.ts @@ -97,6 +97,7 @@ function createRuntimeStub(): { }, queryRecentEvents: async () => [], listCorpusAgents: async () => ["player", "calendar"], + canValidateWildcards: async (agent: string) => agent === "player", listCorpusEntries: async (agent: string) => [ { utterance: `hello ${agent}`, source: "in-repo" }, ], @@ -266,6 +267,17 @@ describe("studio service channel (in-memory rpc)", () => { expect(result.rows[0].utterance).toBe("play jazz"); }); + it("canValidateWildcards round-trips the per-agent capability", async () => { + const stub = createRuntimeStub(); + const { client } = wireClientServer(stub); + expect( + await client.invoke("canValidateWildcards", undefined, "player"), + ).toBe(true); + expect( + await client.invoke("canValidateWildcards", undefined, "calendar"), + ).toBe(false); + }); + it("corpus/feedback methods round-trip", async () => { const stub = createRuntimeStub(); const { client } = wireClientServer(stub); diff --git a/ts/packages/typeagent-core/src/replay/grammarResolver.ts b/ts/packages/typeagent-core/src/replay/grammarResolver.ts index 82ab4481c3..278a4060af 100644 --- a/ts/packages/typeagent-core/src/replay/grammarResolver.ts +++ b/ts/packages/typeagent-core/src/replay/grammarResolver.ts @@ -62,6 +62,7 @@ import type { ConstructionCacheLayer, ConstructionCacheStatus, } from "./constructionCacheResolver.js"; +import type { WildcardMatchValidator } from "./wildcardValidator.js"; const execFileAsync = promisify(execFile); @@ -177,6 +178,17 @@ export interface CreateGrammarReplayResolverOptions { * a git ref. A `stale`/`absent` layer leaves behavior at the grammar match. */ constructionCache?: ConstructionCacheLayer; + /** + * Runs the agent's real `validateWildcardMatch` over each candidate grammar + * match that captured a wildcard, mirroring the dispatcher's post-match + * `getValidatedMatches`. When provided, the **working-tree** side validates + * the ranked match list and resolves the first match the agent accepts (a + * rejected wildcard match falls through, exactly as the dispatcher falls back + * to a lower match / the LLM). Working-tree only; never run at a git ref + * (loading arbitrary-ref agent code is out of scope). Fail-open: only an + * explicit `false` drops a match. + */ + wildcardValidator?: WildcardMatchValidator; } /** @@ -204,6 +216,13 @@ export interface GrammarReplayResolver extends ReplayActionResolver { * label; `"stale"`/`"absent"` fall back to the grammar method label. */ readonly constructionCacheStatus: ConstructionCacheStatus | undefined; + /** + * Whether the wildcard validator actually ran on at least one match this run + * (i.e. a wildcard match occurred on the working-tree side and the validator + * was consulted). Drives the run's "+ wildcard validation" reporting. Always + * `false` when no validator was supplied. + */ + readonly wildcardValidationApplied: boolean; } let entitiesRegistered = false; @@ -387,6 +406,41 @@ function topMatchAction(results: MatchResult[]): unknown { return actions.length > 0 ? actions[0].action : undefined; } +/** + * Mirror the dispatcher's `getValidatedMatches` for the wildcard step: walk the + * heuristically-ranked matches and return the action of the first one the agent + * accepts. A match with no wildcard capture (`wildcardCharCount === 0`) is + * accepted without consulting the validator (exactly as the dispatcher + * short-circuits); a wildcard match is dropped only on an explicit `false` + * verdict, and the walk continues to the next candidate (the dispatcher's + * fall-back-to-a-lower-match behavior). Returns `action: undefined` when every + * candidate was rejected — the row then becomes `needs-explanation`, the + * deterministic stand-in for the dispatcher falling back to the LLM. + * + * Exported for unit testing the ranked-selection contract. + */ +export async function selectValidatedMatchAction( + results: MatchResult[], + validator: WildcardMatchValidator, +): Promise<{ action: unknown; consulted: boolean }> { + let consulted = false; + for (const result of results) { + const actions = result.match.actions; + if (actions.length === 0) { + continue; + } + if (result.wildcardCharCount === 0) { + return { action: actions[0].action, consulted }; + } + consulted = true; + const outcome = await validator.validateMatch(actions); + if (!outcome.rejected) { + return { action: actions[0].action, consulted }; + } + } + return { action: undefined, consulted }; +} + /** * Build a {@link GrammarReplayResolver}. Construct ONE per replay run so the * per-side grammar store never outlives a single `(versionA, versionB)` pair. @@ -397,6 +451,8 @@ export function createGrammarReplayResolver( const { target } = opts; const now = opts.now ?? (() => Date.now()); const constructionCache = opts.constructionCache; + const wildcardValidator = opts.wildcardValidator; + let wildcardValidationApplied = false; // Construction-cache consult is faithful to the dispatcher only for the live // working tree, and only when the cache's namespace hash still matches. const constructionActive = @@ -431,6 +487,10 @@ export function createGrammarReplayResolver( return constructionCache?.status; }, + get wildcardValidationApplied(): boolean { + return wildcardValidationApplied; + }, + async prepare(versionA, versionB): Promise { // Build A then B so the first failure (by side) is the one reported. const a = await build(versionA, "A"); @@ -483,13 +543,35 @@ export function createGrammarReplayResolver( } const results = g.store.match(entry.utterance); + + // Wildcard validation: on the working-tree side, run the + // agent's real `validateWildcardMatch` over the ranked candidates and + // resolve the first accepted match — mirroring the dispatcher's + // post-match `getValidatedMatches`. Never on a git ref (we can't load + // arbitrary-ref agent code). When no validator is supplied this is a + // plain top-match resolution (unchanged behavior). + const validateHere = + wildcardValidator !== undefined && + version.kind === "workingTree"; + let rawAction: unknown; + if (validateHere) { + const validated = await selectValidatedMatchAction( + results, + wildcardValidator!, + ); + rawAction = validated.action; + if (validated.consulted) { + wildcardValidationApplied = true; + } + } else { + rawAction = + results.length > 0 ? topMatchAction(results) : undefined; + } + const latencyMs = now() - t0; const action = - results.length > 0 - ? normalizeAction( - target.schemaName, - topMatchAction(results), - ) + rawAction !== undefined + ? normalizeAction(target.schemaName, rawAction) : undefined; if (action === undefined) { diff --git a/ts/packages/typeagent-core/src/replay/wildcardValidator.ts b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts new file mode 100644 index 0000000000..c890ade187 --- /dev/null +++ b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * Wildcard-match validation for the replay path. + * + * The dispatcher's only beyond-grammar determinism is a post-match step: for + * each candidate match that captured a wildcard, it calls the agent's + * `validateWildcardMatch(action, sessionContext)` and DROPS the match if the + * agent returns `false` (then falls back to a lower-ranked match or the LLM). + * + * Grammar matching alone cannot reproduce this, so a `.agr` that still matches + * an utterance can look unchanged in replay even though the real dispatcher + * would reject the wildcard value. This module runs the agent's REAL validator + * so that axis shows up in the Impact Report. + * + * Scope and guarantees: + * - **Working-tree side only.** The caller never runs this for a git ref. + * - **Built module.** {@link ReplayAppAgentLoader.loadAppAgent} returns the + * agent's BUILT module, so uncommitted validator `.ts` edits are reflected + * only after a rebuild (grammar edits are read from source by the grammar + * resolver). + * - **Fail-OPEN.** Only an explicit `=== false` from the agent rejects a match. + * A missing method, load failure, or thrown validator all ACCEPT the match and + * record a diagnostic — replay must never fabricate a lost match from + * infrastructure noise. + * - **Entity-wildcard validation is out of scope** (it needs conversation + * memory replay does not have). + * + * Which agents run here is decided by the host, not this module: the host only + * builds a validator for agents that declare they are replay-safe. This module + * always loads and runs the validator it is asked to build. + */ + +/** + * Minimal structural view of an app agent for wildcard validation. Typed + * structurally (rather than importing `@typeagent/agent-sdk`) so the + * dependency-light core package does not take on the agent SDK; the real + * `AppAgent.validateWildcardMatch` satisfies this shape. + */ +export interface ReplayValidatableAgent { + validateWildcardMatch?( + action: unknown, + context: unknown, + ): Promise | boolean; +} + +/** + * Loads (and optionally unloads) a built app-agent module by name. The default + * production implementation wraps the dispatcher's `getDefaultAppAgentProviders` + * (which lives outside this dependency-light package), so the loader is injected + * by the host. Tests inject a fake. + */ +export interface ReplayAppAgentLoader { + loadAppAgent(agentName: string): Promise; + /** Best-effort cleanup; many agents default to `execMode: "separate"`. */ + unloadAppAgent?(agentName: string): Promise; +} + +/** + * Why a match was accepted without a genuine `validateWildcardMatch === false` + * verdict — recorded so the run can honestly report that validation degraded. + */ +export type WildcardValidationDiagnostic = + | "no-validator" + | "load-failed" + | "errored"; + +export interface WildcardValidationOutcome { + /** True ONLY when the agent's validator returned an explicit `false`. */ + rejected: boolean; + /** Set when the verdict was fail-open rather than a real validator result. */ + diagnostic?: WildcardValidationDiagnostic; +} + +export interface WildcardMatchValidator { + /** + * Run the agent's validator over every action in one candidate match, + * mirroring the dispatcher's `validateWildcardMatch`: reject as soon as any + * action's validator returns `false`; otherwise accept. Fail-open on any + * error (records a diagnostic, returns `rejected: false`). + */ + validateMatch( + actions: readonly unknown[], + ): Promise; + /** Whether the agent module was actually loaded (drives unload + reporting). */ + readonly loaded: boolean; + /** All fail-open reasons seen this run (for the run-level summary/tooltip). */ + readonly diagnostics: ReadonlySet; + /** Best-effort `unloadAppAgent`; safe to call when nothing was loaded. */ + dispose(): Promise; +} + +export interface CreateWildcardMatchValidatorOptions { + loader: ReplayAppAgentLoader; +} + +/** + * A no-op `SessionContext`-shaped stub for replay validation. Deterministic + * validators ignore the context entirely; the stub still exposes + * `agentContext: {}` — NOT `undefined` — so that any in-process validator that + * defensively reads `context.agentContext.` degrades gracefully instead of + * throwing. The interactive methods throw, since a validator that calls them is + * out of the supported set and should fail-open via the wrapper's try/catch. + */ +export function createReplaySessionContextStub(): Record { + const unsupported = (name: string) => () => { + throw new Error( + `SessionContext.${name} is not available during replay wildcard validation`, + ); + }; + return { + agentContext: {}, + sessionStorage: undefined, + instanceStorage: undefined, + sessionContextId: "studio-replay-validation", + notify: () => {}, + beginAgentThread: unsupported("beginAgentThread"), + popupQuestion: unsupported("popupQuestion"), + toggleTransientAgent: unsupported("toggleTransientAgent"), + addDynamicAgent: unsupported("addDynamicAgent"), + removeDynamicAgent: unsupported("removeDynamicAgent"), + forceCleanupDynamicAgent: unsupported("forceCleanupDynamicAgent"), + reloadAgentSchema: unsupported("reloadAgentSchema"), + }; +} + +/** Pull the `action` payloads out of a grammar match's `actions` array. */ +export function matchActionPayloads(actions: readonly unknown[]): unknown[] { + return actions.map((a) => + a !== null && typeof a === "object" && "action" in a + ? (a as { action: unknown }).action + : a, + ); +} + +/** + * Build a {@link WildcardMatchValidator} for one agent. The agent module is + * loaded lazily on the first {@link WildcardMatchValidator.validateMatch} that + * actually needs it (so a run that never hits a wildcard match never loads the + * module), then cached for the rest of the run. + */ +export function createWildcardMatchValidator( + agentName: string, + options: CreateWildcardMatchValidatorOptions, +): WildcardMatchValidator { + const { loader } = options; + const diagnostics = new Set(); + const stubContext = createReplaySessionContextStub(); + + // null = not loaded yet; { agent } once a load attempt resolved (agent may + // be undefined when the load failed, so we never retry a failed load). + let loadState: { agent: ReplayValidatableAgent | undefined } | undefined; + let didLoad = false; + + async function ensureAgent(): Promise { + if (loadState !== undefined) { + return loadState.agent; + } + try { + const agent = await loader.loadAppAgent(agentName); + didLoad = true; + loadState = { agent }; + return agent; + } catch { + diagnostics.add("load-failed"); + loadState = { agent: undefined }; + return undefined; + } + } + + function note(d: WildcardValidationDiagnostic): WildcardValidationOutcome { + diagnostics.add(d); + return { rejected: false, diagnostic: d }; + } + + return { + get loaded(): boolean { + return didLoad; + }, + get diagnostics(): ReadonlySet { + return diagnostics; + }, + + async validateMatch(actions): Promise { + const agent = await ensureAgent(); + if (agent === undefined) { + return { rejected: false, diagnostic: "load-failed" }; + } + if (typeof agent.validateWildcardMatch !== "function") { + return note("no-validator"); + } + for (const action of matchActionPayloads(actions)) { + let verdict: boolean; + try { + verdict = await agent.validateWildcardMatch( + action, + stubContext, + ); + } catch { + // Fail-open: a throwing validator must never fabricate a + // lost match. Record it and accept. + diagnostics.add("errored"); + continue; + } + if (verdict === false) { + return { rejected: true }; + } + } + return { rejected: false }; + }, + + async dispose(): Promise { + if (!didLoad || loader.unloadAppAgent === undefined) { + return; + } + try { + await loader.unloadAppAgent(agentName); + } catch { + // Best-effort cleanup. + } + }, + }; +} diff --git a/ts/packages/typeagent-core/src/runtime/index.ts b/ts/packages/typeagent-core/src/runtime/index.ts index 05cd0eb224..687c3f8dba 100644 --- a/ts/packages/typeagent-core/src/runtime/index.ts +++ b/ts/packages/typeagent-core/src/runtime/index.ts @@ -16,3 +16,20 @@ export * from "./studioRegistryProtocol.js"; export * from "./studioServiceAuth.js"; export * from "./studioWorkspaceId.js"; export { getDefaultPhaseInputs } from "./onboardingPhaseInputs.js"; + +// Replay wildcard-match validation. Re-exported here so the service host can +// wire a real `ReplayAppAgentLoader` (the production loader lives outside this +// dependency-light package) without reaching into `replay/`. +export { + createWildcardMatchValidator, + createReplaySessionContextStub, + matchActionPayloads, +} from "../replay/wildcardValidator.js"; +export type { + ReplayAppAgentLoader, + ReplayValidatableAgent, + WildcardMatchValidator, + WildcardValidationOutcome, + WildcardValidationDiagnostic, + CreateWildcardMatchValidatorOptions, +} from "../replay/wildcardValidator.js"; diff --git a/ts/packages/typeagent-core/src/runtime/studioProtocol.ts b/ts/packages/typeagent-core/src/runtime/studioProtocol.ts index f92efbb7c1..e5285ccbeb 100644 --- a/ts/packages/typeagent-core/src/runtime/studioProtocol.ts +++ b/ts/packages/typeagent-core/src/runtime/studioProtocol.ts @@ -72,6 +72,16 @@ export type StudioServiceInvokeFunctions = { ): Promise; /** Corpus agents available for replay in this workspace. */ listCorpusAgents(repoRoot?: string): Promise; + /** + * Whether wildcard validation can run for `agent` in replay — the agent + * loads and exposes a `validateWildcardMatch`. The Impact Report reads this + * before a run to disable its validation toggle when there is nothing to + * run. + */ + canValidateWildcards( + repoRoot: string | undefined, + agent: string, + ): Promise; /** * Federated corpus entries for an agent (in-repo, captures, external, * feedback) — what the Corpus tree expands. diff --git a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts index e0529f34d6..c09fd2ccb8 100644 --- a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts +++ b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts @@ -63,12 +63,17 @@ import { ReplayVersionBuildError, type ReplayRunError, type GrammarReplayTarget, + type GrammarReplayResolver, } from "../replay/grammarResolver.js"; import { computeWorkingTreeSchemaHash, loadConstructionCacheLayer, type ConstructionCacheLayer, } from "../replay/constructionCacheResolver.js"; +import type { + WildcardMatchValidator, + WildcardValidationDiagnostic, +} from "../replay/wildcardValidator.js"; import type { CollisionDetectedEvent } from "../events/index.js"; import { getDefaultPhaseInputs } from "./onboardingPhaseInputs.js"; import { @@ -233,6 +238,21 @@ export interface PackagingHealthGateResult { checkedAgent?: string; } +/** + * Which deterministic dispatch path replay models: + * - `nfa-grammar` (default) — grammar-only matching through the real NFA grammar + * store, symmetric for both versions. The construction cache is NOT consulted + * (faithful to the dispatcher's `grammarSystem: "nfa"` mode, where cache + * validation is intentionally skipped). The cleanest signal for "what did my + * `.agr` edit change", since A and B are equivalent environments. + * - `completionBased-cache` — the live working-tree side additionally consults + * the agent's real per-session construction cache before the grammar match + * (faithful to the dispatcher's default `completionBased` mode). Asymmetric: + * only the working-tree side reads the cache, so a cache hit can mask or fake a + * grammar regression. Opt-in for "what would my default dispatcher likely do". + */ +export type StudioReplayMode = "nfa-grammar" | "completionBased-cache"; + /** Request shape for {@link StudioRuntime.replayCorpus}. Versions and miss * policy default to a deterministic working-tree self-compare. */ export interface StudioReplayRequest { @@ -241,6 +261,17 @@ export interface StudioReplayRequest { versionA?: VersionSpec; versionB?: VersionSpec; missPolicy?: ReplayMissPolicy; + /** Deterministic dispatch path to model. Defaults to `nfa-grammar`. */ + mode?: StudioReplayMode; + /** + * Opt-in: additionally run the agent's real `validateWildcardMatch` over the + * working-tree side's wildcard grammar matches (the dispatcher's post-match + * validation step). Default off. Only takes effect when the runtime was given + * a `resolveWildcardValidator` and the agent's manifest declares it + * replay-safe; otherwise it is a no-op. Working-tree side only; fail-open. See + * {@link StudioReplayResult.wildcardValidation}. + */ + validateWildcards?: boolean; } /** @@ -270,6 +301,45 @@ export type StudioReplayMethod = | "schema-grammar" | "construction-cache"; +/** + * The deterministic fidelity layers a replay side can exercise, mirroring the + * dispatcher's path: grammar match → schema enrichment → construction cache → + * wildcard validation → full dispatch. The Impact Report surfaces a per-side + * matrix of these so the run is honest about exactly what it ran (and what + * building from a ref would add) instead of over-claiming fidelity. + */ +export type FidelityLayer = + | "grammar" + | "schemaEnrichment" + | "constructionCache" + | "wildcardValidation" + | "dispatch"; + +export type FidelityLayerStatus = "ran" | "skipped" | "unavailable"; + +export interface FidelityLayerReport { + status: FidelityLayerStatus; + /** Short human reason (shown as hover detail). */ + reason: string; +} + +/** + * How fully a side's version was realized: `built-live` (the working tree — real + * compiled agent code) or `source` (only grammar/schema text materialized at a + * git ref via `git show`; no build, so live-only layers can't run). + */ +export type FidelityRealization = "built-live" | "source"; + +export interface FidelityReport { + realization: FidelityRealization; + layers: Record; +} + +export interface SideFidelity { + A: FidelityReport; + B: FidelityReport; +} + export interface StudioReplayResult { runId: string; summary: ReplaySummary; @@ -286,12 +356,217 @@ export interface StudioReplayResult { methodA: StudioReplayMethod; methodB: StudioReplayMethod; /** - * Set when a version failed to materialize or compile. The run is aborted + * Set when the opt-in wildcard validation actually consulted the agent's + * `validateWildcardMatch` (a wildcard match occurred on the working-tree + * side). Omitted when validation was off or never reached a wildcard match. + */ + wildcardValidation?: StudioWildcardValidationInfo; + /** + * Per-side fidelity descriptor: how each version was realized and which + * deterministic layers actually ran. Derived purely from signals the replay + * already computed (per-side method, mode, version kind, the wildcard + * validation pass) so the Impact Report can show an honest "what ran" matrix. + */ + sideFidelity: SideFidelity; + /** * with an empty summary rather than emitting fabricated regression rows. */ error?: ReplayRunError; } +/** + * Reports the opt-in wildcard-validation pass. Only present when validation was + * enabled AND consulted on at least one wildcard match. `diagnostics` lists fail-open reasons (e.g. the agent's module + * couldn't load, or its validator threw), so the UI can show that validation + * degraded rather than silently claiming full fidelity. + */ +export interface StudioWildcardValidationInfo { + applied: boolean; + diagnostics: WildcardValidationDiagnostic[]; +} + +const FIDELITY_DISPATCH_REASON = + "Full dispatcher path (LLM translation + action handlers) is not run — replay is deterministic grammar/cache matching only."; + +export interface DeriveSideFidelityInput { + versionA: VersionSpec; + versionB: VersionSpec; + methodA: StudioReplayMethod; + methodB: StudioReplayMethod; + mode: StudioReplayMode; + validateWildcards: boolean; + wildcardValidation?: StudioWildcardValidationInfo; +} + +/** + * Derive the per-side {@link SideFidelity} from signals the replay already + * produced. Pure and deterministic so it is unit-testable and reusable by any + * client. Wildcard validation only ever runs on the working-tree side, so the + * single `wildcardValidation` info (when present) is interpreted per side. + */ +export function deriveSideFidelity( + input: DeriveSideFidelityInput, +): SideFidelity { + return { + A: deriveFidelityReport( + input.versionA, + input.methodA, + input.mode, + input.validateWildcards, + input.wildcardValidation, + ), + B: deriveFidelityReport( + input.versionB, + input.methodB, + input.mode, + input.validateWildcards, + input.wildcardValidation, + ), + }; +} + +function deriveFidelityReport( + version: VersionSpec, + method: StudioReplayMethod, + mode: StudioReplayMode, + validateWildcards: boolean, + wildcardValidation: StudioWildcardValidationInfo | undefined, +): FidelityReport { + const isWorkingTree = version.kind === "workingTree"; + return { + realization: isWorkingTree ? "built-live" : "source", + layers: { + grammar: fidelityGrammarLayer(method), + schemaEnrichment: fidelitySchemaEnrichmentLayer(method), + constructionCache: fidelityConstructionCacheLayer( + method, + isWorkingTree, + mode, + ), + wildcardValidation: fidelityWildcardValidationLayer( + isWorkingTree, + validateWildcards, + wildcardValidation, + ), + dispatch: { + status: "unavailable", + reason: FIDELITY_DISPATCH_REASON, + }, + }, + }; +} + +function fidelityGrammarLayer(method: StudioReplayMethod): FidelityLayerReport { + if (method === "identity") { + return { + status: "skipped", + reason: "Identity baseline — surfaces each entry's captured action without evaluating a grammar.", + }; + } + return { + status: "ran", + reason: "Utterances matched through the real grammar store (NFA + sortMatches), the same engine the dispatcher uses.", + }; +} + +function fidelitySchemaEnrichmentLayer( + method: StudioReplayMethod, +): FidelityLayerReport { + if (method === "schema-grammar" || method === "construction-cache") { + return { + status: "ran", + reason: "Grammar enriched with checked_wildcard metadata from the agent's action schema before compilation.", + }; + } + if (method === "static-grammar") { + return { + status: "skipped", + reason: "Agent action-schema not discoverable at this version — matched the bare grammar (still indicative).", + }; + } + return { + status: "skipped", + reason: "Identity baseline — no grammar evaluated.", + }; +} + +function fidelityConstructionCacheLayer( + method: StudioReplayMethod, + isWorkingTree: boolean, + mode: StudioReplayMode, +): FidelityLayerReport { + if (method === "construction-cache") { + return { + status: "ran", + reason: "Consulted the agent's live construction cache (the dispatcher's first check), hash-gated to the current schema.", + }; + } + if (!isWorkingTree) { + return { + status: "unavailable", + reason: "Construction caches are runtime artifacts, never committed — they cannot be read at a git ref.", + }; + } + if (mode === "completionBased-cache") { + return { + status: "skipped", + reason: "No live cache was discovered, or it was stale versus the current schema → grammar fallback.", + }; + } + return { + status: "skipped", + reason: "Grammar mode — the construction cache is intentionally not consulted (keeps matches A/B-symmetric).", + }; +} + +function fidelityWildcardValidationLayer( + isWorkingTree: boolean, + validateWildcards: boolean, + wildcardValidation: StudioWildcardValidationInfo | undefined, +): FidelityLayerReport { + if (!isWorkingTree) { + return { + status: "unavailable", + reason: "The agent's validator code isn't built at a git ref — validation runs on the working-tree side only.", + }; + } + if (!validateWildcards) { + return { + status: "skipped", + reason: "Validate toggle is off.", + }; + } + if (wildcardValidation?.applied === true) { + const diagnostics = wildcardValidation.diagnostics; + if (diagnostics.includes("load-failed")) { + return { + status: "unavailable", + reason: "Agent module couldn't be loaded (e.g. a packaged build ships no agent code) — matches fell back to the grammar.", + }; + } + if (diagnostics.includes("no-validator")) { + return { + status: "skipped", + reason: "Agent exposes no validateWildcardMatch — validation made no change.", + }; + } + if (diagnostics.includes("errored")) { + return { + status: "ran", + reason: "Validator threw on a match; the match was kept fail-open so the run stayed grammar-faithful.", + }; + } + return { + status: "ran", + reason: "Ran the agent's real validateWildcardMatch over its wildcard matches; rejected matches were dropped.", + }; + } + return { + status: "skipped", + reason: "Validation enabled but did not run — no wildcard match occurred, or the agent isn't validatable.", + }; +} + export interface StudioCollisionScanRequest { /** * Agent package names to scan. Defaults to every agent currently loaded @@ -446,6 +721,12 @@ export interface StudioRuntime { * union of agents loaded across running sandboxes. */ listCorpusAgents(): Promise; + /** + * Whether wildcard validation can actually run for `agent` in replay — the + * agent loads and exposes a `validateWildcardMatch`. Lets the Impact Report + * disable its validation toggle when there is nothing to run. + */ + canValidateWildcards(agent: string): Promise; /** Federated corpus entries for an agent (in-repo, captures, external, feedback). */ listCorpusEntries(agent: string): Promise; /** @@ -546,6 +827,39 @@ export interface CreateStudioRuntimeOptions { * deterministic identity replay over each entry's captured `expectedAction`. */ replayResolver?: ReplayActionResolver; + /** + * Resolves the live construction-cache layer for the working-tree side of a + * replay. Injected so tests can exercise the mode gating (the default + * discovers the dispatcher's live cache from the instance dir, which a test + * can't fabricate). Only consulted in `completionBased-cache` mode. + */ + resolveConstructionCache?: ( + target: GrammarReplayTarget, + ) => Promise; + /** + * Builds the agent's wildcard-match validator for an opt-in + * (`validateWildcards`) replay run. Returns `undefined` when the host can't + * validate this agent (e.g. the agent's manifest doesn't declare it + * replay-safe, or no agent loader is available), in which case replay stays + * grammar-only. Injected because the production loader pulls the + * dispatcher's agent providers, which live outside this dependency-light + * package; tests inject a fake-loader-backed validator. The runtime disposes + * the returned validator at run end. + */ + resolveWildcardValidator?: ( + agentName: string, + ) => + | WildcardMatchValidator + | undefined + | Promise; + /** + * Reports whether wildcard validation can run for an agent — it loads and + * exposes a `validateWildcardMatch` — so `canValidateWildcards` can answer + * the Impact Report before a run. Injected for the same reason as + * `resolveWildcardValidator`: loading agent modules lives outside this + * dependency-light package. Absent ⇒ no agent is treated as validatable. + */ + resolveCanValidateWildcards?: (agentName: string) => Promise; collisions?: CollisionService; /** * Scans agents' compiled grammars for collisions. Injected so tests can @@ -586,6 +900,8 @@ function abortedReplayResult( options: Parameters[0], error: ReplayRunError, method: StudioReplayMethod = "static-grammar", + mode: StudioReplayMode = "nfa-grammar", + validateWildcards = false, ): StudioReplayResult { const runId = `replay-error-${Date.now().toString(36)}`; return { @@ -597,6 +913,14 @@ function abortedReplayResult( method, methodA: method, methodB: method, + sideFidelity: deriveSideFidelity({ + versionA: options.versionA, + versionB: options.versionB, + methodA: method, + methodB: method, + mode, + validateWildcards, + }), summary: { runId, agent: options.agent, @@ -968,6 +1292,11 @@ export function createStudioRuntimeCore( } return [...agents].sort((a, b) => a.localeCompare(b)); }, + async canValidateWildcards(agent) { + return ( + (await options.resolveCanValidateWildcards?.(agent)) === true + ); + }, async listCorpusEntries(agent) { return corpus.list(agent); }, @@ -995,6 +1324,7 @@ export function createStudioRuntimeCore( return feedback.list(filter); }, async replayCorpus(request) { + const mode: StudioReplayMode = request.mode ?? "nfa-grammar"; const replayOptions = { agent: request.agent, corpus: request.corpus ?? {}, @@ -1012,6 +1342,10 @@ export function createStudioRuntimeCore( let method: StudioReplayMethod = "identity"; let methodA: StudioReplayMethod = "identity"; let methodB: StudioReplayMethod = "identity"; + // Opt-in wildcard validator; kept in scope so its diagnostics + // can be read after the run and it can be disposed (unload the agent). + let wildcardValidator: WildcardMatchValidator | undefined; + let activeGrammarResolver: GrammarReplayResolver | undefined; if ( resolver === undefined && replayOptions.missPolicy === "needs-explanation" && @@ -1024,12 +1358,33 @@ export function createStudioRuntimeCore( ); if (target !== undefined) { // Best-effort consult of the agent's live per-session - // construction cache for the working-tree side. Hash-gated to - // the current schema exactly as the dispatcher gates it, so a - // schema edit invalidates the cached constructions (→ stale → - // grammar fallback) rather than reporting a phantom cache hit. + // construction cache for the working-tree side — but ONLY in + // `completionBased-cache` mode. In the default `nfa-grammar` + // mode the dispatcher does not consult the cache (grammar + // rules alone decide the match), so replay stays grammar-only + // and A/B-symmetric. Hash-gated to the current schema exactly + // as the dispatcher gates it, so a schema edit invalidates the + // cached constructions (→ stale → grammar fallback) rather + // than reporting a phantom cache hit. const constructionCache = - await resolveConstructionCacheLayer(target); + mode === "completionBased-cache" + ? await ( + options.resolveConstructionCache ?? + resolveConstructionCacheLayer + )(target) + : undefined; + + // Opt-in: build the agent's wildcard validator so the + // working-tree side runs the dispatcher's real post-match + // `validateWildcardMatch`. `undefined` when the host has no + // loader or the agent doesn't opt in — replay then stays + // grammar-only (a silent no-op, not an error). + wildcardValidator = + request.validateWildcards === true + ? await options.resolveWildcardValidator?.( + replayOptions.agent, + ) + : undefined; const grammarResolver = createGrammarReplayResolver({ target, @@ -1037,7 +1392,11 @@ export function createStudioRuntimeCore( ...(constructionCache !== undefined ? { constructionCache } : {}), + ...(wildcardValidator !== undefined + ? { wildcardValidator } + : {}), }); + activeGrammarResolver = grammarResolver; const labelFor = (): StudioReplayMethod => grammarResolver.constructionCacheStatus === "valid" ? "construction-cache" @@ -1080,6 +1439,7 @@ export function createStudioRuntimeCore( ? "schema-grammar" : "static-grammar"; } catch (err) { + await wildcardValidator?.dispose(); if (err instanceof ReplayVersionBuildError) { return abortedReplayResult( replayOptions, @@ -1093,6 +1453,8 @@ export function createStudioRuntimeCore( message: err.message, }, labelFor(), + mode, + request.validateWildcards === true, ); } throw err; @@ -1107,10 +1469,26 @@ export function createStudioRuntimeCore( }); const rows: ActionDelta[] = []; - for await (const row of handle.rows) { - rows.push(row); + try { + for await (const row of handle.rows) { + rows.push(row); + } + } finally { + await wildcardValidator?.dispose(); } const summary = await handle.summary; + // Surface the wildcard-validation pass only when it actually ran on a + // wildcard match (so a run that never hit a wildcard doesn't claim a + // validation it never performed). + const wildcardValidation: StudioWildcardValidationInfo | undefined = + activeGrammarResolver?.wildcardValidationApplied === true + ? { + applied: true, + diagnostics: [ + ...(wildcardValidator?.diagnostics ?? []), + ], + } + : undefined; return { runId: handle.runId, summary, @@ -1118,6 +1496,20 @@ export function createStudioRuntimeCore( method, methodA, methodB, + sideFidelity: deriveSideFidelity({ + versionA: replayOptions.versionA, + versionB: replayOptions.versionB, + methodA, + methodB, + mode, + validateWildcards: request.validateWildcards === true, + ...(wildcardValidation !== undefined + ? { wildcardValidation } + : {}), + }), + ...(wildcardValidation !== undefined + ? { wildcardValidation } + : {}), }; }, reportCollision(event) { diff --git a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts index f2c47d9b9c..5ed9fb935b 100644 --- a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts +++ b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts @@ -17,9 +17,17 @@ import { normalizeGrammarAction, resolveGrammarReplayTarget, ReplayVersionBuildError, + selectValidatedMatchAction, type GrammarReplayTarget, } from "../src/replay/grammarResolver.js"; import type { ConstructionCacheLayer } from "../src/replay/constructionCacheResolver.js"; +import { + createWildcardMatchValidator, + type ReplayAppAgentLoader, + type ReplayValidatableAgent, + type WildcardMatchValidator, +} from "../src/replay/wildcardValidator.js"; +import type { MatchResult } from "agent-cache"; const GRAMMAR_V1 = [ " = | ;", @@ -490,3 +498,209 @@ describe("resolveGrammarReplayTarget + schema enrichment (L1)", () => { }); }); }); + +// --------------------------------------------------------------------------- +// Working-tree wildcard validation +// --------------------------------------------------------------------------- + +// A grammar whose rule captures a greedy wildcard (so a match has +// wildcardCharCount > 0 and is subject to validateWildcardMatch), plus a +// non-wildcard rule (validator never consulted). +const WILDCARD_GRAMMAR = [ + " = | ;", + ' = play $(trackName:wildcard) -> { actionName: "playTrack", parameters: { trackName } };', + ' = pause -> { actionName: "pause" };', +].join("\n"); + +function wildcardLoader(agent: ReplayValidatableAgent): ReplayAppAgentLoader { + return { + async loadAppAgent() { + return agent; + }, + async unloadAppAgent() {}, + }; +} + +/** A validator (for "demo") that rejects a given track name. */ +function rejectTrackValidator(reject: string): WildcardMatchValidator { + const agent: ReplayValidatableAgent = { + async validateWildcardMatch(a) { + const params = (a as { parameters?: { trackName?: string } }) + .parameters; + return params?.trackName !== reject; + }, + }; + return createWildcardMatchValidator("demo", { + loader: wildcardLoader(agent), + }); +} + +describe("grammar resolver wildcard validation", () => { + let dir: string; + + beforeEach(() => { + dir = tempDir(); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + function wildcardTarget(): GrammarReplayTarget { + const grammarPath = path.join(dir, "demo.agr"); + writeFileSync(grammarPath, WILDCARD_GRAMMAR, "utf8"); + return { + agent: "demo", + schemaName: "demo", + grammarFilePath: grammarPath, + }; + } + + it("drops a wildcard match the agent rejects (→ needs-explanation)", async () => { + const validator = rejectTrackValidator("despacito"); + const resolver = createGrammarReplayResolver({ + target: wildcardTarget(), + wildcardValidator: validator, + }); + await resolver.prepare( + { kind: "workingTree" }, + { kind: "workingTree" }, + ); + const res = await resolver.resolve( + entry("e1", "play despacito"), + { kind: "workingTree" }, + "A", + ); + expect(res.cacheState).toBe("needs-explanation"); + expect(res.action).toBeUndefined(); + expect(resolver.wildcardValidationApplied).toBe(true); + await validator.dispose(); + }); + + it("keeps a wildcard match the agent accepts", async () => { + const validator = rejectTrackValidator("nope"); + const resolver = createGrammarReplayResolver({ + target: wildcardTarget(), + wildcardValidator: validator, + }); + await resolver.prepare( + { kind: "workingTree" }, + { kind: "workingTree" }, + ); + const res = await resolver.resolve( + entry("e1", "play despacito"), + { kind: "workingTree" }, + "A", + ); + expect(res.cacheState).toBe("hit"); + expect(res.action).toMatchObject({ + schemaName: "demo", + actionName: "playTrack", + parameters: { trackName: "despacito" }, + }); + await validator.dispose(); + }); + + it("does not consult the validator for a non-wildcard match", async () => { + const validator = rejectTrackValidator("despacito"); + const resolver = createGrammarReplayResolver({ + target: wildcardTarget(), + wildcardValidator: validator, + }); + await resolver.prepare( + { kind: "workingTree" }, + { kind: "workingTree" }, + ); + const res = await resolver.resolve( + entry("e1", "pause"), + { kind: "workingTree" }, + "A", + ); + expect(res.cacheState).toBe("hit"); + expect(res.action).toEqual({ schemaName: "demo", actionName: "pause" }); + // No wildcard match occurred, so validation never "applied". + expect(resolver.wildcardValidationApplied).toBe(false); + await validator.dispose(); + }); + + it("behaves as plain top-match resolution when no validator is supplied", async () => { + const resolver = createGrammarReplayResolver({ + target: wildcardTarget(), + }); + await resolver.prepare( + { kind: "workingTree" }, + { kind: "workingTree" }, + ); + const res = await resolver.resolve( + entry("e1", "play despacito"), + { kind: "workingTree" }, + "A", + ); + expect(res.cacheState).toBe("hit"); + expect(res.action).toMatchObject({ actionName: "playTrack" }); + expect(resolver.wildcardValidationApplied).toBe(false); + }); +}); + +describe("selectValidatedMatchAction (ranked-list contract)", () => { + function match(actionName: string, wildcardCharCount: number): MatchResult { + return { + type: "grammar", + match: { + actions: [{ action: { schemaName: "demo", actionName } }], + }, + matchedCount: 1, + wildcardCharCount, + nonOptionalCount: 0, + implicitParameterCount: 0, + entityWildcardPropertyNames: [], + } as unknown as MatchResult; + } + + function rejectingValidator( + rejectActionNames: string[], + ): WildcardMatchValidator { + const agent: ReplayValidatableAgent = { + async validateWildcardMatch(a) { + return !rejectActionNames.includes( + (a as { actionName: string }).actionName, + ); + }, + }; + return createWildcardMatchValidator("demo", { + loader: wildcardLoader(agent), + }); + } + + it("falls through a rejected top wildcard match to a surviving candidate", async () => { + // Top match (wildcard) is rejected; the 2nd wildcard match passes — so + // this is NOT a lost match, mirroring the dispatcher's fall-back. + const validator = rejectingValidator(["top"]); + const result = await selectValidatedMatchAction( + [match("top", 5), match("second", 3)], + validator, + ); + expect(result.consulted).toBe(true); + expect(result.action).toMatchObject({ actionName: "second" }); + }); + + it("returns no action when every wildcard candidate is rejected", async () => { + const validator = rejectingValidator(["a", "b"]); + const result = await selectValidatedMatchAction( + [match("a", 2), match("b", 4)], + validator, + ); + expect(result.action).toBeUndefined(); + expect(result.consulted).toBe(true); + }); + + it("accepts a non-wildcard match without consulting the validator", async () => { + const validator = rejectingValidator(["top"]); + const result = await selectValidatedMatchAction( + [match("top", 0)], + validator, + ); + expect(result.action).toMatchObject({ actionName: "top" }); + expect(result.consulted).toBe(false); + }); +}); diff --git a/ts/packages/typeagent-core/test/sideFidelity.spec.ts b/ts/packages/typeagent-core/test/sideFidelity.spec.ts new file mode 100644 index 0000000000..c2ad95bb58 --- /dev/null +++ b/ts/packages/typeagent-core/test/sideFidelity.spec.ts @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + deriveSideFidelity, + type DeriveSideFidelityInput, + type FidelityLayer, + type FidelityLayerStatus, +} from "../src/runtime/studioRuntimeCore.js"; + +const workingTree = { kind: "workingTree" } as const; +const gitRef = { kind: "git", ref: "HEAD" } as const; + +function baseInput( + overrides: Partial = {}, +): DeriveSideFidelityInput { + return { + versionA: workingTree, + versionB: gitRef, + methodA: "schema-grammar", + methodB: "schema-grammar", + mode: "nfa-grammar", + validateWildcards: false, + ...overrides, + }; +} + +function statusOf( + input: DeriveSideFidelityInput, + side: "A" | "B", + layer: FidelityLayer, +): FidelityLayerStatus { + return deriveSideFidelity(input)[side].layers[layer].status; +} + +describe("deriveSideFidelity", () => { + it("marks realization by version kind", () => { + const fidelity = deriveSideFidelity(baseInput()); + expect(fidelity.A.realization).toBe("built-live"); + expect(fidelity.B.realization).toBe("source"); + }); + + it("every layer carries a non-empty reason", () => { + const fidelity = deriveSideFidelity(baseInput()); + for (const side of [fidelity.A, fidelity.B]) { + for (const report of Object.values(side.layers)) { + expect(report.reason.length).toBeGreaterThan(0); + } + } + }); + + it("dispatch is always unavailable", () => { + expect(statusOf(baseInput(), "A", "dispatch")).toBe("unavailable"); + expect(statusOf(baseInput(), "B", "dispatch")).toBe("unavailable"); + }); + + it("grammar runs except for identity", () => { + expect(statusOf(baseInput(), "A", "grammar")).toBe("ran"); + const identity = baseInput({ methodA: "identity" }); + expect(statusOf(identity, "A", "grammar")).toBe("skipped"); + }); + + it("schema enrichment runs only for schema/cache methods", () => { + expect(statusOf(baseInput(), "A", "schemaEnrichment")).toBe("ran"); + const cache = baseInput({ methodA: "construction-cache" }); + expect(statusOf(cache, "A", "schemaEnrichment")).toBe("ran"); + const bare = baseInput({ methodA: "static-grammar" }); + expect(statusOf(bare, "A", "schemaEnrichment")).toBe("skipped"); + }); + + it("construction cache is unavailable at a git ref", () => { + expect(statusOf(baseInput(), "B", "constructionCache")).toBe( + "unavailable", + ); + }); + + it("construction cache runs only when the method used it", () => { + const cache = baseInput({ + versionA: workingTree, + methodA: "construction-cache", + }); + expect(statusOf(cache, "A", "constructionCache")).toBe("ran"); + }); + + it("construction cache is skipped in grammar mode on the working tree", () => { + const grammar = baseInput({ + versionA: workingTree, + methodA: "schema-grammar", + mode: "nfa-grammar", + }); + expect(statusOf(grammar, "A", "constructionCache")).toBe("skipped"); + }); + + it("construction cache is skipped on a stale cache-mode working tree", () => { + const stale = baseInput({ + versionA: workingTree, + methodA: "schema-grammar", + mode: "completionBased-cache", + }); + expect(statusOf(stale, "A", "constructionCache")).toBe("skipped"); + }); + + it("wildcard validation is unavailable at a git ref", () => { + expect(statusOf(baseInput(), "B", "wildcardValidation")).toBe( + "unavailable", + ); + }); + + it("wildcard validation is skipped when the toggle is off", () => { + expect( + statusOf( + baseInput({ validateWildcards: false }), + "A", + "wildcardValidation", + ), + ).toBe("skipped"); + }); + + it("wildcard validation runs when applied with no diagnostics", () => { + const applied = baseInput({ + validateWildcards: true, + wildcardValidation: { applied: true, diagnostics: [] }, + }); + expect(statusOf(applied, "A", "wildcardValidation")).toBe("ran"); + }); + + it("wildcard validation maps load-failed to unavailable", () => { + const degraded = baseInput({ + validateWildcards: true, + wildcardValidation: { + applied: true, + diagnostics: ["load-failed"], + }, + }); + expect(statusOf(degraded, "A", "wildcardValidation")).toBe( + "unavailable", + ); + }); + + it("wildcard validation maps no-validator to skipped", () => { + for (const diagnostic of ["no-validator"] as const) { + const skipped = baseInput({ + validateWildcards: true, + wildcardValidation: { + applied: true, + diagnostics: [diagnostic], + }, + }); + expect(statusOf(skipped, "A", "wildcardValidation")).toBe( + "skipped", + ); + } + }); + + it("wildcard validation maps errored to ran (fail-open)", () => { + const errored = baseInput({ + validateWildcards: true, + wildcardValidation: { applied: true, diagnostics: ["errored"] }, + }); + expect(statusOf(errored, "A", "wildcardValidation")).toBe("ran"); + }); + + it("wildcard validation is skipped when enabled but never applied", () => { + const enabled = baseInput({ validateWildcards: true }); + expect(statusOf(enabled, "A", "wildcardValidation")).toBe("skipped"); + }); +}); diff --git a/ts/packages/typeagent-core/test/wildcardValidator.spec.ts b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts new file mode 100644 index 0000000000..c4cbea5941 --- /dev/null +++ b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + createWildcardMatchValidator, + createReplaySessionContextStub, + matchActionPayloads, + type ReplayAppAgentLoader, + type ReplayValidatableAgent, +} from "../src/replay/wildcardValidator.js"; + +/** A loader over a fixed map of fake agents, recording load/unload calls. */ +function fakeLoader( + agents: Record, + log?: { loaded: string[]; unloaded: string[] }, +): ReplayAppAgentLoader { + return { + async loadAppAgent(name) { + log?.loaded.push(name); + const agent = agents[name]; + if (agent === undefined) { + throw new Error(`no such agent: ${name}`); + } + return agent; + }, + async unloadAppAgent(name) { + log?.unloaded.push(name); + }, + }; +} + +/** One grammar-match action wrapper `{ action }`, as `match.actions` carries. */ +function action(actionName: string, parameters?: Record) { + return { + action: { + schemaName: "demo", + actionName, + ...(parameters !== undefined ? { parameters } : {}), + }, + }; +} + +describe("createWildcardMatchValidator", () => { + it("rejects when the agent validator returns false", async () => { + const agent: ReplayValidatableAgent = { + async validateWildcardMatch(a) { + return (a as { actionName: string }).actionName !== "bad"; + }, + }; + const validator = createWildcardMatchValidator("timer", { + loader: fakeLoader({ timer: agent }), + }); + expect((await validator.validateMatch([action("good")])).rejected).toBe( + false, + ); + expect((await validator.validateMatch([action("bad")])).rejected).toBe( + true, + ); + }); + + it("rejects the whole match if ANY action's validator returns false", async () => { + const agent: ReplayValidatableAgent = { + async validateWildcardMatch(a) { + return (a as { actionName: string }).actionName !== "bad"; + }, + }; + const validator = createWildcardMatchValidator("timer", { + loader: fakeLoader({ timer: agent }), + }); + const outcome = await validator.validateMatch([ + action("good"), + action("bad"), + ]); + expect(outcome.rejected).toBe(true); + }); + + it("fails open with a diagnostic when the validator throws", async () => { + const agent: ReplayValidatableAgent = { + async validateWildcardMatch() { + throw new Error("boom"); + }, + }; + const validator = createWildcardMatchValidator("timer", { + loader: fakeLoader({ timer: agent }), + }); + const outcome = await validator.validateMatch([action("x")]); + expect(outcome.rejected).toBe(false); + expect([...validator.diagnostics]).toContain("errored"); + }); + + it("fails open with a diagnostic when the module fails to load", async () => { + const validator = createWildcardMatchValidator("timer", { + // Loader has no "timer" → loadAppAgent throws. + loader: fakeLoader({}), + }); + const outcome = await validator.validateMatch([action("x")]); + expect(outcome.rejected).toBe(false); + expect(outcome.diagnostic).toBe("load-failed"); + expect([...validator.diagnostics]).toContain("load-failed"); + // A failed load is not retried. + await validator.validateMatch([action("y")]); + }); + + it("accepts (no-op) when the agent has no validateWildcardMatch", async () => { + const validator = createWildcardMatchValidator("timer", { + loader: fakeLoader({ timer: {} }), + }); + const outcome = await validator.validateMatch([action("x")]); + expect(outcome.rejected).toBe(false); + expect(outcome.diagnostic).toBe("no-validator"); + }); + + it("does not throw for an in-process validator reading agentContext (empty stub)", async () => { + // A hypothetical in-process validator that reads context.agentContext + // and returns true when its client is absent. Our stub uses + // `agentContext: {}` (NOT undefined), so this must not throw. + const playerLike: ReplayValidatableAgent = { + async validateWildcardMatch(_a, context) { + const spotify = ( + context as { agentContext?: { spotify?: unknown } } + ).agentContext?.spotify; + if (spotify === undefined) { + return true; + } + return false; + }, + }; + const validator = createWildcardMatchValidator("player", { + loader: fakeLoader({ player: playerLike }), + }); + const outcome = await validator.validateMatch([action("playTrack")]); + expect(outcome.rejected).toBe(false); + expect(outcome.diagnostic).toBeUndefined(); + }); + + it("loads the module lazily and only once, then unloads on dispose", async () => { + const log = { loaded: [] as string[], unloaded: [] as string[] }; + const agent: ReplayValidatableAgent = { + async validateWildcardMatch() { + return true; + }, + }; + const validator = createWildcardMatchValidator("timer", { + loader: fakeLoader({ timer: agent }, log), + }); + // Not loaded until first validateMatch. + expect(log.loaded).toEqual([]); + await validator.validateMatch([action("a")]); + await validator.validateMatch([action("b")]); + expect(log.loaded).toEqual(["timer"]); // exactly once + expect(validator.loaded).toBe(true); + await validator.dispose(); + expect(log.unloaded).toEqual(["timer"]); + }); + + it("dispose is a no-op when nothing was loaded", async () => { + const log = { loaded: [] as string[], unloaded: [] as string[] }; + const validator = createWildcardMatchValidator("timer", { + loader: fakeLoader({}, log), + }); + await validator.dispose(); + expect(log.unloaded).toEqual([]); + }); +}); + +describe("createReplaySessionContextStub", () => { + it("exposes an empty (non-undefined) agentContext", () => { + const stub = createReplaySessionContextStub(); + expect(stub.agentContext).toEqual({}); + }); + + it("throws on interactive methods (validators must not call them)", () => { + const stub = createReplaySessionContextStub(); + expect(() => (stub.popupQuestion as () => unknown)()).toThrow(); + }); +}); + +describe("matchActionPayloads", () => { + it("unwraps `{ action }` entries and passes through bare values", () => { + expect( + matchActionPayloads([{ action: { actionName: "x" } }, 42]), + ).toEqual([{ actionName: "x" }, 42]); + }); +}); diff --git a/ts/packages/typeagent-studio/esbuild.mjs b/ts/packages/typeagent-studio/esbuild.mjs index 75af51206f..a2d4b05074 100644 --- a/ts/packages/typeagent-studio/esbuild.mjs +++ b/ts/packages/typeagent-studio/esbuild.mjs @@ -100,7 +100,16 @@ const serviceConfig = { entryPoints: ["../studio-service/src/main.ts"], bundle: true, outfile: "dist/studio-service.js", - external: ["vscode"], + // `default-agent-provider` is dynamically imported by the replay wildcard + // validator (fidelity rung L4a). It drags in the whole dispatcher + every + // agent, so it must NOT be inlined here. Marked external, it resolves from + // `node_modules` on the in-repo `typeagent-studio serve` dev path and simply + // fails the dynamic import (→ grammar-only replay) in the packaged `.vsix`. + // The dependency is declared on THIS extension (not studio-service) so it + // resolves against the extension's `node_modules` here, and to avoid a build + // cycle: studio-service is aggregated by studio-agent, which default-agent + // -provider in turn depends on. See studio-service/src/wildcardValidation.ts. + external: ["vscode", "default-agent-provider"], alias: nodeBundleAlias, format: "cjs", platform: "node", diff --git a/ts/packages/typeagent-studio/media/impactReport.css b/ts/packages/typeagent-studio/media/impactReport.css index c23c19e484..e4eee84dad 100644 --- a/ts/packages/typeagent-studio/media/impactReport.css +++ b/ts/packages/typeagent-studio/media/impactReport.css @@ -107,6 +107,9 @@ body { .codicon-sync::before { content: "\ea77"; } +.codicon-settings-gear::before { + content: "\eb51"; +} /* --- Action bar (the native editor toolbar) ------------------------------ */ .action-bar { @@ -307,6 +310,164 @@ body { background: var(--vscode-button-hoverBackground); } +/* A lit two-state toggle (e.g. the wildcard-validation switch) when active. */ +.tool-button.is-active { + color: var(--vscode-inputOption-activeForeground, var(--vscode-foreground)); + background: var(--vscode-inputOption-activeBackground); + border-color: var(--vscode-inputOption-activeBorder, transparent); +} + +/* --- Replay-options popover + switches ----------------------------------- */ +.settings-group { + position: relative; +} + +.settings-popover { + position: absolute; + top: calc(100% + 4px); + right: 0; + z-index: 30; + display: flex; + flex-direction: column; + gap: 12px; + min-width: 220px; + padding: 12px 14px; + background: var( + --vscode-editorWidget-background, + var(--vscode-menu-background) + ); + color: var(--vscode-editorWidget-foreground, var(--vscode-menu-foreground)); + border: 1px solid + var( + --vscode-editorWidget-border, + var(--vscode-menu-border, var(--vscode-widget-border)) + ); + border-radius: 4px; + box-shadow: 0 2px 8px var(--vscode-widget-shadow, rgba(0, 0, 0, 0.36)); +} + +.switch-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 18px; +} + +.switch-label { + font-size: 12px; + white-space: nowrap; +} + +.switch { + position: relative; + flex: 0 0 auto; + width: 52px; + height: 22px; + padding: 0; + border: none; + border-radius: 11px; + cursor: pointer; + background: var(--vscode-input-background, rgba(128, 128, 128, 0.25)); + box-shadow: inset 0 0 0 1px + var(--vscode-checkbox-border, var(--vscode-input-border, transparent)); + transition: background 0.15s ease; +} + +.switch.is-on { + background: var(--vscode-button-background); + box-shadow: none; +} + +.switch:focus-visible { + outline: 1px solid var(--vscode-focusBorder); + outline-offset: 2px; +} + +.switch:disabled { + opacity: 0.5; + cursor: default; +} + +.switch-knob { + position: absolute; + top: 2px; + left: 2px; + width: 18px; + height: 18px; + border-radius: 50%; + background: var(--vscode-input-foreground, #fff); + transition: + transform 0.15s ease, + background 0.15s ease; +} + +.switch.is-on .switch-knob { + transform: translateX(30px); + background: var(--vscode-button-foreground, #fff); +} + +.switch-text { + position: absolute; + top: 0; + height: 22px; + line-height: 22px; + font-size: 9px; + font-weight: 600; + letter-spacing: 0.04em; + pointer-events: none; +} + +.switch-on { + left: 8px; + color: var(--vscode-button-foreground, #fff); + opacity: 0; +} + +.switch-off { + right: 7px; + color: var(--vscode-descriptionForeground); + opacity: 1; +} + +.switch.is-on .switch-on { + opacity: 1; +} + +.switch.is-on .switch-off { + opacity: 0; +} + +/* A switch and its explanatory caption grouped as one popover item. */ +.switch-group { + display: flex; + flex-direction: column; + gap: 6px; +} + +/* Caption under the wildcard-validation switch: states whether the scoped agent + is replay-safe so the toggle's effect is clear before a run. */ +.switch-note { + display: flex; + align-items: flex-start; + gap: 6px; + max-width: 240px; + font-size: 11px; + line-height: 1.4; + color: var(--vscode-descriptionForeground); +} + +.switch-note.is-caution { + color: var( + --vscode-notificationsWarningIcon-foreground, + var(--vscode-editorWarning-foreground) + ); +} + +.switch-note .codicon { + flex: 0 0 auto; + margin-top: 1px; +} + /* --- Sub-bar: provenance + run status ------------------------------------ */ .sub-bar { display: flex; @@ -348,6 +509,24 @@ body { text-overflow: ellipsis; } +/* The wildcard-validation outcome of the last run; sits next to provenance. + Tinted by severity: a clean pass is neutral, a degraded/unavailable pass + warns so the report never silently claims a fidelity it didn't reach. */ +.validation-note { + display: inline-flex; + align-items: center; + gap: 5px; + white-space: nowrap; + color: var(--vscode-descriptionForeground); +} + +.validation-note.is-degraded { + color: var( + --vscode-editorWarning-foreground, + var(--vscode-descriptionForeground) + ); +} + /* --- Inline notification (errors) ---------------------------------------- */ .notification { display: flex; @@ -375,6 +554,85 @@ body { flex: 0 0 auto; } +/* --- Per-side fidelity matrix -------------------------------------------- */ +.fidelity-panel { + margin: 8px 12px 0; +} + +.fidelity-panel[hidden] { + display: none; +} + +.fidelity-details { + border: 1px solid var(--vscode-panel-border); + border-radius: var(--ta-radius); + background: var(--vscode-sideBar-background, var(--vscode-editor-background)); + padding: 0 10px; +} + +.fidelity-summary { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 7px 0; + cursor: pointer; + color: var(--vscode-descriptionForeground); + font-size: 12px; + user-select: none; +} + +.fidelity-summary::-webkit-details-marker { + display: none; +} + +.fidelity-grid { + display: grid; + grid-template-columns: 1fr auto auto; + gap: 4px 14px; + padding: 4px 0 10px; + font-size: 12px; +} + +.fidelity-head { + color: var(--vscode-descriptionForeground); + font-weight: 600; + text-transform: uppercase; + font-size: 10px; + letter-spacing: 0.04em; +} + +.fidelity-grid .fidelity-head:not(:first-child), +.fidelity-grid .fidelity-cell { + justify-self: center; + text-align: center; +} + +.fidelity-layer { + color: var(--vscode-foreground); +} + +.fidelity-cell { + display: inline-flex; + align-items: center; + gap: 4px; + white-space: nowrap; +} + +.fidelity-cell.is-ran { + color: var(--vscode-testing-iconPassed, var(--vscode-charts-green, #3fb950)); +} + +.fidelity-cell.is-skipped { + color: var(--vscode-descriptionForeground); +} + +.fidelity-cell.is-unavailable { + color: var( + --vscode-editorWarning-foreground, + var(--vscode-descriptionForeground) + ); +} + /* --- Scrolling content area ---------------------------------------------- */ .content { flex: 1 1 auto; diff --git a/ts/packages/typeagent-studio/package.json b/ts/packages/typeagent-studio/package.json index 688930af77..cba1c7600d 100644 --- a/ts/packages/typeagent-studio/package.json +++ b/ts/packages/typeagent-studio/package.json @@ -503,6 +503,7 @@ "@typeagent/core": "workspace:*", "@typeagent/websocket-utils": "workspace:*", "debug": "^4.4.0", + "default-agent-provider": "workspace:*", "ws": "^8.21.0" }, "devDependencies": { diff --git a/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts b/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts index aaf6ea899f..767fc3f98c 100644 --- a/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts +++ b/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts @@ -108,15 +108,19 @@ export class CollisionsTreeProvider async getChildren(row?: CollisionRow): Promise { if (!row) { - // Show the native loading bar while connecting; once connected, - // render scanned collisions (or the "no collisions" placeholder). - await this.whenConnected(); - if ( - !this.connected && - this.entries.length === 0 && - this.skipped.length === 0 - ) { - return []; + // Render the last known scan immediately so a mid-session + // disconnect keeps results visible instead of dropping back to the + // loading bar. Only gate on the connection (showing the native + // loading bar) during the initial load, before anything is scanned. + if (this.entries.length === 0 && this.skipped.length === 0) { + await this.whenConnected(); + if ( + !this.connected && + this.entries.length === 0 && + this.skipped.length === 0 + ) { + return []; + } } return buildCollisionRows(this.entries, this.skipped); } diff --git a/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts b/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts index c172cb0099..614c039258 100644 --- a/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts +++ b/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts @@ -75,11 +75,15 @@ export class EventLogTreeProvider if (row) { return []; } - // Show the native loading bar while connecting; once connected, render - // buffered events (or the "no events yet" placeholder). - await this.whenConnected(); - if (!this.connected && this.entries.length === 0) { - return []; + // Render buffered events immediately so a mid-session disconnect keeps + // the ring visible instead of dropping back to the loading bar. Only + // gate on the connection (showing the native loading bar) during the + // initial load, when nothing has been buffered yet. + if (this.entries.length === 0) { + await this.whenConnected(); + if (!this.connected && this.entries.length === 0) { + return []; + } } return buildEventLogRows(this.entries); } diff --git a/ts/packages/typeagent-studio/src/extension.ts b/ts/packages/typeagent-studio/src/extension.ts index e13a3c3d82..25c6568485 100644 --- a/ts/packages/typeagent-studio/src/extension.ts +++ b/ts/packages/typeagent-studio/src/extension.ts @@ -327,6 +327,11 @@ export function activate(context: vscode.ExtensionContext): void { ); const corpusTree = new CorpusTreeProvider(serviceRuntime); + // The Corpora tree refreshes only on explicit in-extension actions (seeding + // an in-repo file, adding an external source, recording feedback) and the + // manual Refresh command -- each of those calls corpusTree.refresh() + // directly. We intentionally do not watch the filesystem for out-of-band + // edits to *.utterances.jsonl. context.subscriptions.push( corpusTree, vscode.window.registerTreeDataProvider(CORPUS_VIEW_ID, corpusTree), diff --git a/ts/packages/typeagent-studio/src/impactReportView.ts b/ts/packages/typeagent-studio/src/impactReportView.ts index 1c95d67a2a..026902cdf9 100644 --- a/ts/packages/typeagent-studio/src/impactReportView.ts +++ b/ts/packages/typeagent-studio/src/impactReportView.ts @@ -59,9 +59,11 @@ export function openImpactReport( // Subscription to the shared connection's state, disposed with the panel. let stateSub: { dispose(): void } | undefined; // The last completed result + its request id. Re-posted whenever the webview - // signals `ready` so a run that finished while the iframe was torn down (the - // panel is `retainContextWhenHidden: false`, so hidden panels drop posts) is - // recovered on reveal — the webview dedupes by request id. + // signals `ready` so a run whose result arrived before the webview was + // listening (e.g. the first load, or a full extension reload) is recovered + // on the next `ready` — the webview dedupes by request id. With the panel + // retaining context while hidden, navigate-away/back no longer reloads it, + // so this mainly guards the initial-load and extension-reload cases. let lastResult: | { requestId: number; @@ -82,6 +84,11 @@ export function openImpactReport( title: `Impact Report — ${agent}`, scriptPath: ["dist", "webview", "impactReport.js"], stylePath: ["media", "impactReport.css"], + // The panel holds a live service connection and a rendered result, and + // the agent/connection context is re-pushed only via a single reveal + // `ready` handshake. Retain the context so navigating away and back + // keeps that state instead of tearing the webview down and reloading. + retainContextWhenHidden: true, onMessage: (raw) => void handleMessage(raw), onDispose: () => { stateSub?.dispose(); @@ -126,7 +133,13 @@ export function openImpactReport( post({ type: "status", text: "Connecting to the studio service…" }); const c = await ensureClient(); if (!c) { - post({ type: "init", agent, connected: false, available: false }); + post({ + type: "init", + agent, + connected: false, + available: false, + canValidateWildcards: false, + }); return; } // Confirm this agent still has a corpus to replay so the webview can @@ -138,7 +151,22 @@ export function openImpactReport( } catch { // Connected but listing failed; treat as unavailable for now. } - post({ type: "init", agent, connected: true, available }); + // Check whether wildcard validation can actually run for this agent (it + // has a validator), so the webview can disable the toggle when there is + // nothing to run. A lookup failure stays conservative (toggle disabled). + let canValidateWildcards = false; + try { + canValidateWildcards = await c.canValidateWildcards(agent); + } catch { + // Leave the toggle disabled rather than offering a no-op. + } + post({ + type: "init", + agent, + connected: true, + available, + canValidateWildcards, + }); // Recover a result computed while the panel was hidden/reloaded. if (lastResult) { post({ @@ -360,6 +388,14 @@ export function openImpactReport( // free of LLM calls. versionA: msg.versionA, versionB: msg.versionB, + // The webview's mode toggle selects which deterministic dispatch + // path to model (grammar-only vs construction-cache-first); the + // runtime defaults unknown/missing to the cache-free baseline. + mode: msg.mode, + // Opt-in wildcard validation: the runtime only acts on it when a + // `resolveWildcardValidator` is wired and the agent's manifest + // declares it replay-safe; otherwise it is a no-op. + validateWildcards: msg.validateWildcards, missPolicy: "needs-explanation", }); post({ diff --git a/ts/packages/typeagent-studio/src/studioServiceClient.ts b/ts/packages/typeagent-studio/src/studioServiceClient.ts index e8597d964d..4f1451ede2 100644 --- a/ts/packages/typeagent-studio/src/studioServiceClient.ts +++ b/ts/packages/typeagent-studio/src/studioServiceClient.ts @@ -230,6 +230,11 @@ export class StudioServiceClient { return this.rpc.invoke("listCorpusAgents", this.repoRoot); } + /** Whether wildcard validation can run for `agent` (it has a validator). */ + canValidateWildcards(agent: string): Promise { + return this.rpc.invoke("canValidateWildcards", this.repoRoot, agent); + } + /** Federated corpus entries for an agent (Corpus tree expansion). */ listCorpusEntries(agent: string): Promise { return this.rpc.invoke("listCorpusEntries", this.repoRoot, agent); diff --git a/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts b/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts index c2d7024baf..8d78d47586 100644 --- a/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts +++ b/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts @@ -24,8 +24,10 @@ import { impactEmptyState, allRowsEqual, IMPACT_FILTER_ORDER, + toFidelityMatrix, type ReplayRowStatus, } from "../webviewKit/replayViewModel.js"; +import type { SideFidelity } from "@typeagent/core/runtime"; function row(overrides: Partial): ActionDelta { return { @@ -422,3 +424,63 @@ test("impactEmptyState gives first-run guidance", () => { assert.ok(/base/i.test(state.hint)); assert.ok(/compare/i.test(state.hint)); }); + +function fidelityReport( + realization: SideFidelity["A"]["realization"], + overrides: Partial = {}, +): SideFidelity["A"] { + const ran = { status: "ran", reason: "ran reason" } as const; + return { + realization, + layers: { + grammar: { ...ran }, + schemaEnrichment: { ...ran }, + constructionCache: { ...ran }, + wildcardValidation: { ...ran }, + dispatch: { status: "unavailable", reason: "no dispatch" }, + ...overrides, + }, + }; +} + +test("toFidelityMatrix returns undefined when no descriptor", () => { + assert.equal(toFidelityMatrix(undefined), undefined); +}); + +test("toFidelityMatrix produces one row per layer in order", () => { + const view = toFidelityMatrix({ + A: fidelityReport("built-live"), + B: fidelityReport("built-live"), + }); + assert.ok(view); + assert.deepEqual( + view!.rows.map((r) => r.layer), + [ + "Grammar match", + "Schema enrichment", + "Construction cache", + "Wildcard validation", + "Full dispatch", + ], + ); + assert.equal(view!.realizationA, "built (live)"); + assert.equal(view!.realizationB, "built (live)"); +}); + +test("toFidelityMatrix carries per-side status and reason", () => { + const view = toFidelityMatrix({ + A: fidelityReport("built-live"), + B: fidelityReport("source", { + constructionCache: { + status: "unavailable", + reason: "no cache at a ref", + }, + }), + }); + assert.ok(view); + const cacheRow = view!.rows.find((r) => r.layer === "Construction cache"); + assert.ok(cacheRow); + assert.equal(cacheRow!.a.status, "ran"); + assert.equal(cacheRow!.b.status, "unavailable"); + assert.equal(cacheRow!.b.reason, "no cache at a ref"); +}); diff --git a/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts b/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts index dd9af1a35a..4ed754bd8b 100644 --- a/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts +++ b/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts @@ -32,6 +32,7 @@ export function stubInvokeHandlers( clearCollisions: async () => 0, queryRecentEvents: async () => [], listCorpusAgents: async () => [], + canValidateWildcards: async () => false, listCorpusEntries: async () => [], seedInRepoCorpus: async () => ({ path: "/repo/ts/corpus", diff --git a/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts b/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts index d72ca3076f..57d2a104dc 100644 --- a/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts +++ b/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts @@ -1182,3 +1182,168 @@ test("scanGrammarCollisions defaults to agents loaded across sandboxes", async ( assert.deepEqual(received, []); }); + +// --- Replay mode gating (Level B) ----------------------------------------- +// `replayCorpus` consults the live construction cache only in +// `completionBased-cache` mode; the default `nfa-grammar` mode matches against +// the grammar alone. These tests scaffold a real single-schema agent so the +// auto-engaged grammar path runs, and inject a spy cache resolver so the gating +// is observable without a real on-disk construction cache. + +const GATING_GRAMMAR = [ + " = | ;", + ' = pause -> { actionName: "pause" };', + ' = resume -> { actionName: "resume" };', +].join("\n"); + +const GATING_SCHEMA_TS = [ + "export type DemoActions = PauseAction | ResumeAction;", + "", + "// Pause playback.", + 'export type PauseAction = { actionName: "pause" };', + "", + "// Resume playback.", + 'export type ResumeAction = { actionName: "resume" };', + "", +].join("\n"); + +const GATING_MANIFEST = JSON.stringify({ + schema: { + originalSchemaFile: "./demoSchema.ts", + schemaType: { action: "DemoActions" }, + }, +}); + +/** + * Scaffold a temp repo whose `packages/agents/demo` is a discoverable + * single-schema agent (grammar + schema + manifest), so `resolveRepoRoot` and + * `resolveGrammarReplayTarget` engage the real grammar replay path. + */ +async function scaffoldGatingRepo(): Promise { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "studio-gating-")); + const agentDir = path.join( + repoRoot, + "packages", + "agents", + "demo", + "src", + "agent", + ); + await fs.mkdir(agentDir, { recursive: true }); + await fs.writeFile( + path.join(agentDir, "demoSchema.agr"), + GATING_GRAMMAR, + "utf8", + ); + await fs.writeFile( + path.join(agentDir, "demoSchema.ts"), + GATING_SCHEMA_TS, + "utf8", + ); + await fs.writeFile( + path.join(agentDir, "demoManifest.json"), + GATING_MANIFEST, + "utf8", + ); + return repoRoot; +} + +function gatingEntry(id: string, utterance: string): CorpusEntry { + return { + id, + utterance, + agent: "demo", + source: "in-repo", + provenance: { sourceUri: `mem://${id}` }, + }; +} + +/** A spy `resolveConstructionCache` that records consults and returns a valid + * layer resolving "pause" from the cache. */ +function spyCacheResolver(record: { consulted: boolean }) { + return async () => { + record.consulted = true; + return { + status: "valid" as const, + cacheFilePath: "/stub/constructions.json", + schemaName: "demo", + currentHash: "hash=", + cachedHash: "hash=", + match(utterance: string) { + return utterance === "pause" + ? { schemaName: "demo", actionName: "pause" } + : undefined; + }, + }; + }; +} + +test("replayCorpus skips the construction cache in the default nfa-grammar mode", async () => { + const repoRoot = await scaffoldGatingRepo(); + try { + const corpus = new StubCorpusService([gatingEntry("p", "pause")]); + const record = { consulted: false }; + const runtime = createStudioRuntimeCore( + createContext([repoRoot]).context, + { corpus, resolveConstructionCache: spyCacheResolver(record) }, + ); + + const result = await runtime.replayCorpus({ + agent: "demo", + missPolicy: "needs-explanation", + mode: "nfa-grammar", + }); + + assert.equal(record.consulted, false); + assert.notEqual(result.method, "construction-cache"); + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + } +}); + +test("replayCorpus omitting the mode defaults to grammar-only (no cache consult)", async () => { + const repoRoot = await scaffoldGatingRepo(); + try { + const corpus = new StubCorpusService([gatingEntry("p", "pause")]); + const record = { consulted: false }; + const runtime = createStudioRuntimeCore( + createContext([repoRoot]).context, + { corpus, resolveConstructionCache: spyCacheResolver(record) }, + ); + + const result = await runtime.replayCorpus({ + agent: "demo", + missPolicy: "needs-explanation", + }); + + assert.equal(record.consulted, false); + assert.notEqual(result.method, "construction-cache"); + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + } +}); + +test("replayCorpus consults the construction cache in completionBased-cache mode", async () => { + const repoRoot = await scaffoldGatingRepo(); + try { + const corpus = new StubCorpusService([gatingEntry("p", "pause")]); + const record = { consulted: false }; + const runtime = createStudioRuntimeCore( + createContext([repoRoot]).context, + { corpus, resolveConstructionCache: spyCacheResolver(record) }, + ); + + const result = await runtime.replayCorpus({ + agent: "demo", + missPolicy: "needs-explanation", + mode: "completionBased-cache", + }); + + assert.equal(record.consulted, true); + // A valid cache layer for the working-tree side surfaces as the + // construction-cache method. + assert.equal(result.method, "construction-cache"); + } finally { + await fs.rm(repoRoot, { recursive: true, force: true }); + } +}); diff --git a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts index 946b89dd75..313163e4f8 100644 --- a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts +++ b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts @@ -17,7 +17,8 @@ test("parseWebviewMessage accepts well-formed messages", () => { type: "pickVersion", side: "b", }); - // Missing/invalid version fields coerce to the working tree. + // Missing/invalid version fields coerce to the working tree; an absent mode + // defaults to the grammar-only baseline. assert.deepEqual( parseWebviewMessage({ type: "run", requestId: 3, agent: "player" }), { @@ -26,6 +27,8 @@ test("parseWebviewMessage accepts well-formed messages", () => { agent: "player", versionA: { kind: "workingTree" }, versionB: { kind: "workingTree" }, + mode: "nfa-grammar", + validateWildcards: false, }, ); // String version fields are coerced (legacy text-field / test seam). @@ -36,6 +39,8 @@ test("parseWebviewMessage accepts well-formed messages", () => { agent: "player", versionA: "HEAD", versionB: "working tree", + mode: "completionBased-cache", + validateWildcards: true, }), { type: "run", @@ -43,9 +48,13 @@ test("parseWebviewMessage accepts well-formed messages", () => { agent: "player", versionA: { kind: "git", ref: "HEAD" }, versionB: { kind: "workingTree" }, + mode: "completionBased-cache", + validateWildcards: true, }, ); - // Typed specs from a picker selection are validated and taken as-is. + // Typed specs from a picker selection are validated and taken as-is; an + // unknown mode value falls back to the grammar-only baseline, and a + // non-boolean validateWildcards falls back to off. assert.deepEqual( parseWebviewMessage({ type: "run", @@ -53,6 +62,8 @@ test("parseWebviewMessage accepts well-formed messages", () => { agent: "player", versionA: { kind: "git", ref: "v1.0" }, versionB: { kind: "workingTree" }, + mode: "bogus-mode", + validateWildcards: "yes", }), { type: "run", @@ -60,6 +71,8 @@ test("parseWebviewMessage accepts well-formed messages", () => { agent: "player", versionA: { kind: "git", ref: "v1.0" }, versionB: { kind: "workingTree" }, + mode: "nfa-grammar", + validateWildcards: false, }, ); // Non-string / malformed version fields fall back to the working tree. @@ -77,6 +90,8 @@ test("parseWebviewMessage accepts well-formed messages", () => { agent: "player", versionA: { kind: "workingTree" }, versionB: { kind: "workingTree" }, + mode: "nfa-grammar", + validateWildcards: false, }, ); }); diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts index 88e02b3183..3ea4bb1b42 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts @@ -10,7 +10,12 @@ * `vscode`, or node built-ins (so it bundles for the browser). */ -import type { StudioReplayResult } from "@typeagent/core/runtime"; +import type { + StudioReplayResult, + StudioReplayMode, + StudioWildcardValidationInfo, + WildcardValidationDiagnostic, +} from "@typeagent/core/runtime"; import type { ActionDelta } from "@typeagent/core/replay"; import type { HostToWebviewMessage, @@ -33,10 +38,12 @@ import { formatProvenanceLine, formatVersionProvenance, toActionDiff, + toFidelityMatrix, type ImpactRow, type ReplayRowStatus, type ResolvedVersion, type RunProvenance, + type FidelityCell, } from "../replayViewModel.js"; /** Default base (A): the last commit — the baseline of the regression journey. */ @@ -64,6 +71,8 @@ interface PanelState { selectedAgent?: string; versionA?: ResolvedVersion; versionB?: ResolvedVersion; + mode?: StudioReplayMode; + validateWildcards?: boolean; lastResult?: PersistedResult; } interface VsCodeApi { @@ -113,6 +122,40 @@ const activeFilters = defaultImpactFilters(); let currentAgent: string | undefined; let versionA: ResolvedVersion = DEFAULT_VERSION_A; let versionB: ResolvedVersion = DEFAULT_VERSION_B; +// Which deterministic dispatch path the next run models. `nfa-grammar` (default) +// matches both sides against the compiled grammar only — symmetric, no cache. +// `completionBased-cache` lets the working-tree side consult the live +// construction cache first (the default-dispatcher path). Persisted with the +// version selection so a reload keeps the chosen mode. +let mode: StudioReplayMode = "nfa-grammar"; + +// Opt-in wildcard validation. When lit, the run asks the host to additionally +// run the agent's real `validateWildcardMatch` over the working-tree side's +// wildcard matches. Off by default and only available when the agent actually +// has a validator to run (`agentCanValidate`); persisted with the rest of the +// selection so a reload keeps the choice. The host/runtime fail-open it when no +// validator runs and report that back via `wildcardValidation.diagnostics`, +// which `renderValidationNote` surfaces. +let validateWildcards = false; +// Whether wildcard validation can run for the scoped agent (it has a validator), +// from the host `init`. Enables the toggle; when false the toggle is disabled +// because there is nothing to run. +let agentCanValidate = false; + +/** Concise hover copy for the Cache on/off toggle. */ +const CACHE_TOOLTIP: Record<"on" | "off", string> = { + on: "Cache on: the working-tree side consults the live construction cache before the grammar. Click to turn off.", + off: "Cache off: both sides match grammar only and stay symmetric. Click to turn on.", +}; +/** Concise hover copy for the Wildcard validation on/off toggle. */ +const VALIDATE_TOOLTIP: Record<"on" | "off", string> = { + on: "Wildcard validation on: the working-tree side runs the agent's real validator over wildcard matches. Click to turn off.", + off: "Wildcard validation off: matches come from the grammar alone. Click to turn on.", +}; +/** Hover copy when the toggle is disabled because the agent has no wildcard + * validator to run in replay. */ +const VALIDATE_DISABLED_TOOLTIP = + "Wildcard validation unavailable: this agent has no validateWildcardMatch to run in replay."; const root = document.getElementById("root")!; @@ -143,6 +186,44 @@ const swapButton = toolButton("arrow-swap", "Swap A and B", () => swapVersions(), ); swapButton.title = "Swap the base (A) and compare (B) versions."; + +// Replay options (construction cache, wildcard validation) are advanced knobs +// that most runs leave at their defaults, so they live behind a gear popover +// rather than cluttering the toolbar. Each is a VS Code-themed on/off switch +// (see `.switch` in the stylesheet); the gear lights up while the popover open. +const cacheSwitch = settingsSwitch( + "Cache", + () => mode === "completionBased-cache", + () => toggleCache(), + (on) => CACHE_TOOLTIP[on ? "on" : "off"], +); +const validateSwitch = settingsSwitch( + "Wildcard validation", + () => validateWildcards, + () => toggleValidateWildcards(), + (on) => + agentCanValidate + ? VALIDATE_TOOLTIP[on ? "on" : "off"] + : VALIDATE_DISABLED_TOOLTIP, +); +const settingsPopover = el("div", "settings-popover"); +settingsPopover.setAttribute("role", "group"); +settingsPopover.setAttribute("aria-label", "Replay options"); +settingsPopover.hidden = true; +// The validation switch carries a caption: a note when there's no validator to +// run, and a caution when validation is turned on (the real validator may have +// side effects or be non-deterministic, so replay results may not reproduce). +const validateNote = el("div", "switch-note"); +const validateGroup = el("div", "switch-group"); +validateGroup.append(validateSwitch.row, validateNote); +settingsPopover.append(cacheSwitch.row, validateGroup); +const settingsButton = toolButton("settings-gear", "Replay options", () => + toggleSettingsPopover(), +); +settingsButton.setAttribute("aria-haspopup", "true"); +settingsButton.setAttribute("aria-expanded", "false"); +const settingsGroup = el("div", "bar-group settings-group"); +settingsGroup.append(settingsButton, settingsPopover); const runButton = toolButton("play", "Run", () => runReplay(), { primary: true, text: "Run", @@ -159,6 +240,8 @@ actionBar.append( group(codicon("library"), agentNameEl), separator(), group(versionAButton.button, swapButton, versionBButton.button), + separator(), + settingsGroup, spacer(), runButton, connectionPill, @@ -168,19 +251,30 @@ actionBar.append( // commits each side ran against) and the live status text. const subBar = el("div", "sub-bar"); const provenanceEl = el("span", "provenance"); +const validationEl = el("span", "validation-note"); const statusEl = el("span", "status"); -subBar.append(provenanceEl, statusEl); +subBar.append(provenanceEl, validationEl, statusEl); -// The scrolling content region: an inline error notification, the status filter, -// first-run guidance, the results list, and the drill-in detail pane. +// The scrolling content region: an inline error notification, the per-side +// fidelity matrix, the status filter, first-run guidance, the results list, and +// the drill-in detail pane. const contentEl = el("div", "content"); const notificationEl = el("div", "notification"); +const fidelityEl = el("div", "fidelity-panel"); +fidelityEl.hidden = true; const filtersEl = el("div", "filters"); const emptyStateEl = el("div", "empty-state"); const tableWrap = el("div", "table-wrap"); const detailEl = el("div", "detail-pane"); detailEl.hidden = true; -contentEl.append(notificationEl, filtersEl, emptyStateEl, tableWrap, detailEl); +contentEl.append( + notificationEl, + fidelityEl, + filtersEl, + emptyStateEl, + tableWrap, + detailEl, +); root.append(actionBar, subBar, contentEl); @@ -188,6 +282,27 @@ setControlsEnabled(false); renderConnection("connecting"); restoreSelection(); renderVersionButtons(); +cacheSwitch.render(); +validateSwitch.render(); +renderValidateNote(); + +// Dismiss the replay-options popover on an outside click or Escape, the way a +// native menu behaves. The gear's own click is inside `settingsGroup`, so it +// toggles without this handler immediately re-closing it. +document.addEventListener("click", (event) => { + if ( + !settingsPopover.hidden && + !settingsGroup.contains(event.target as Node) + ) { + toggleSettingsPopover(false); + } +}); +document.addEventListener("keydown", (event) => { + if (event.key === "Escape" && !settingsPopover.hidden) { + toggleSettingsPopover(false); + settingsButton.focus(); + } +}); // --- Messaging ------------------------------------------------------------ window.addEventListener("message", (event: MessageEvent) => { @@ -196,6 +311,15 @@ window.addEventListener("message", (event: MessageEvent) => { case "init": currentAgent = msg.agent; renderAgentName(); + // Wildcard validation can only run when the agent has a validator; + // otherwise force the request off and disable the toggle. When a + // validator exists, keep the persisted choice (default off). + agentCanValidate = msg.canValidateWildcards; + if (!agentCanValidate) { + validateWildcards = false; + } + validateSwitch.render(); + renderValidateNote(); controlsAvailable = msg.connected && msg.available; setControlsEnabled(controlsAvailable); renderEmptyState(); @@ -259,11 +383,18 @@ function runReplay(): void { } requestId += 1; latestRequestId = requestId; - persistState({ selectedAgent: agent, versionA, versionB }); + persistState({ + selectedAgent: agent, + versionA, + versionB, + mode, + validateWildcards, + }); setControlsEnabled(false); setStatus(`Replaying ${agent}…`); clearNotification(); provenanceEl.textContent = ""; + validationEl.textContent = ""; filtersEl.textContent = ""; filtersEl.hidden = true; emptyStateEl.hidden = true; @@ -277,9 +408,66 @@ function runReplay(): void { agent, versionA: versionA.spec, versionB: versionB.spec, + mode, + validateWildcards, }); } +/** Flip the construction cache on/off and persist the choice. */ +function toggleCache(): void { + mode = mode === "nfa-grammar" ? "completionBased-cache" : "nfa-grammar"; + cacheSwitch.render(); + persistState({ mode }); +} + +/** Toggle opt-in wildcard validation and persist the choice. */ +function toggleValidateWildcards(): void { + validateWildcards = !validateWildcards; + validateSwitch.render(); + renderValidateNote(); + persistState({ validateWildcards }); +} + +/** Caption the validation switch for the scoped agent. When there's no validator + * to run, show a neutral note explaining why the toggle is disabled. When + * validation is turned on, caution that the real validator may have side effects + * or be non-deterministic, so replay results may not reproduce. Otherwise hide + * the caption. */ +function renderValidateNote(): void { + validateNote.textContent = ""; + if (!agentCanValidate) { + validateNote.hidden = false; + validateNote.classList.remove("is-caution"); + validateNote.append( + codicon("info"), + text("No wildcard validator to run for this agent."), + ); + return; + } + if (!validateWildcards) { + validateNote.hidden = true; + validateNote.classList.remove("is-caution"); + return; + } + validateNote.hidden = false; + validateNote.classList.add("is-caution"); + validateNote.append( + codicon("warning"), + text( + "May be unsafe: the validator can have side effects or be non-deterministic, so replay results may not reproduce.", + ), + ); +} + +/** Open/close the replay-options popover. Pass `force` to set a specific + * state; omit to flip the current one. */ +function toggleSettingsPopover(force?: boolean): void { + const open = force ?? settingsPopover.hidden; + settingsPopover.hidden = !open; + settingsButton.classList.toggle("is-active", open); + settingsButton.setAttribute("aria-expanded", String(open)); +} + /** Apply a version selection from the host picker to one side. */ function applyVersionPick(side: ReplaySide, resolved: ResolvedVersion): void { if (side === "a") { @@ -314,6 +502,10 @@ function renderResult( // second table instead of upgrading the snapshot in place. clearNotification(); provenanceEl.textContent = ""; + validationEl.textContent = ""; + validationEl.classList.remove("is-degraded"); + fidelityEl.textContent = ""; + fidelityEl.hidden = true; filtersEl.textContent = ""; filtersEl.hidden = true; emptyStateEl.hidden = true; @@ -347,6 +539,14 @@ function renderResult( renderProvenance(provenance); } + // Surface the opt-in wildcard-validation outcome, including an honest + // "unavailable" state when the validator couldn't load (e.g. packaged build). + renderValidationNote(result.wildcardValidation); + + // The per-side fidelity matrix: which deterministic layers actually ran on + // each version. + renderFidelity(result.sideFidelity); + currentRows = toImpactRows(result.rows, result.methodA, result.methodB); currentTotal = result.summary.rowCount; @@ -362,6 +562,148 @@ function renderResult( ); } +/** Paint the wildcard-validation outcome into the sub-bar, or clear it. + * Shown only when validation actually ran on a wildcard match; a clean pass is + * neutral, a degraded/unavailable pass warns so the report stays honest. */ +function renderValidationNote( + info: StudioWildcardValidationInfo | undefined, +): void { + validationEl.textContent = ""; + validationEl.classList.remove("is-degraded"); + if (!info || !info.applied) { + return; + } + const { icon, label, detail, degraded } = describeValidation( + info.diagnostics, + ); + validationEl.classList.toggle("is-degraded", degraded); + validationEl.title = detail; + validationEl.append(codicon(icon), text(label)); +} + +/** Map the validation diagnostics to a compact label + hover detail. An empty + * diagnostics list means the validator ran cleanly; otherwise we surface the + * most significant fail-open reason so the run never overclaims fidelity. */ +function describeValidation(diagnostics: WildcardValidationDiagnostic[]): { + icon: string; + label: string; + detail: string; + degraded: boolean; +} { + if (diagnostics.length === 0) { + return { + icon: "verified", + label: "wildcard-validated", + detail: + "The working-tree side ran the agent's real validateWildcardMatch " + + "over its wildcard matches; any match the agent rejected was dropped.", + degraded: false, + }; + } + if (diagnostics.includes("load-failed")) { + return { + icon: "warning", + label: "validation unavailable", + detail: + "Wildcard validation was requested but the agent module couldn't be " + + "loaded (e.g. the packaged build ships no agent code), so matches " + + "fell back to the grammar alone.", + degraded: true, + }; + } + if (diagnostics.includes("no-validator")) { + return { + icon: "info", + label: "no validator", + detail: + "This agent exposes no validateWildcardMatch, so validation made " + + "no change to the grammar matches.", + degraded: true, + }; + } + return { + icon: "warning", + label: "validation degraded", + detail: + "The agent's validator threw while checking a wildcard match; the match " + + "was kept (fail-open) so the run stayed grammar-faithful.", + degraded: true, + }; +} + +const FIDELITY_STATUS_ICON: Record = { + ran: "pass-filled", + skipped: "circle-slash", + unavailable: "circle-large-outline", +}; + +/** Render the per-side fidelity matrix into a collapsible panel, or hide it. + * Surfaces which deterministic layers ran on each version (A/B) with a status + * icon + hover reason — so the report is honest about exactly what it + * exercised. */ +function renderFidelity( + sideFidelity: StudioReplayResult["sideFidelity"] | undefined, +): void { + fidelityEl.textContent = ""; + fidelityEl.hidden = true; + const view = toFidelityMatrix(sideFidelity); + if (!view) { + return; + } + + const details = document.createElement("details"); + details.className = "fidelity-details"; + const summary = document.createElement("summary"); + summary.className = "fidelity-summary"; + summary.append( + codicon("layers"), + text( + `Fidelity \u00b7 A: ${view.realizationA} \u00b7 B: ${view.realizationB}`, + ), + ); + details.append(summary); + + const grid = el("div", "fidelity-grid"); + grid.append( + fidelityHeadCell("Layer"), + fidelityHeadCell("A"), + fidelityHeadCell("B"), + ); + for (const row of view.rows) { + grid.append( + fidelityLayerCell(row.layer), + fidelityStatusCell(row.a), + fidelityStatusCell(row.b), + ); + } + details.append(grid); + + fidelityEl.append(details); + fidelityEl.hidden = false; +} + +function fidelityHeadCell(label: string): HTMLElement { + const cell = el("span", "fidelity-head"); + cell.textContent = label; + return cell; +} + +function fidelityLayerCell(label: string): HTMLElement { + const cell = el("span", "fidelity-layer"); + cell.textContent = label; + return cell; +} + +function fidelityStatusCell(cellInfo: FidelityCell): HTMLElement { + const cell = el("span", `fidelity-cell is-${cellInfo.status}`); + cell.title = cellInfo.reason; + cell.append( + codicon(FIDELITY_STATUS_ICON[cellInfo.status]), + text(cellInfo.status), + ); + return cell; +} + /** Paint the status filter chips for the rows of the current result. */ function renderFilters(): void { filtersEl.textContent = ""; @@ -700,6 +1042,12 @@ function restoreSelection(): void { if (state?.versionB) { versionB = state.versionB; } + if (state?.mode) { + mode = state.mode; + } + if (state?.validateWildcards !== undefined) { + validateWildcards = state.validateWildcards === true; + } // The agent is authoritative from the host `init`; until it arrives, show a // neutral placeholder rather than a stale persisted value. renderAgentName(); @@ -754,6 +1102,15 @@ function setControlsEnabled(enabled: boolean): void { swapButton.disabled = !enabled; versionAButton.button.disabled = !enabled; versionBButton.button.disabled = !enabled; + settingsButton.disabled = !enabled; + cacheSwitch.button.disabled = !enabled; + // The validation toggle additionally requires the agent to have a validator + // to run; otherwise it stays disabled so the operator isn't offered a no-op. + validateSwitch.button.disabled = !enabled || !agentCanValidate; + // A run shouldn't leave the options popover hanging open. + if (!enabled) { + toggleSettingsPopover(false); + } } function setStatus(text: string): void { @@ -896,9 +1253,43 @@ function toolButton( return b; } -/** A dropdown-style picker button: an optional side tag (A/B), the current - * selection, and a trailing chevron. Asks the host to open a native QuickPick - * on click. */ +/** A labelled VS Code-themed on/off switch for the replay-options popover. + * `getState`/`onToggle` bind it to a boolean; `tooltipFor` supplies concise + * hover copy per state. Returns the row, the switch button (so callers can + * disable it), and a `render` that repaints from the current state. */ +function settingsSwitch( + label: string, + getState: () => boolean, + onToggle: () => void, + tooltipFor: (on: boolean) => string, +): { row: HTMLElement; button: HTMLButtonElement; render: () => void } { + const row = el("div", "switch-row"); + const labelEl = el("span", "switch-label"); + labelEl.textContent = label; + labelEl.id = `switch-${label.replace(/\s+/g, "-").toLowerCase()}`; + const button = document.createElement("button"); + button.type = "button"; + button.className = "switch"; + button.setAttribute("role", "switch"); + button.setAttribute("aria-labelledby", labelEl.id); + const offText = el("span", "switch-text switch-off"); + offText.textContent = "OFF"; + const onText = el("span", "switch-text switch-on"); + onText.textContent = "ON"; + button.append(offText, onText, el("span", "switch-knob")); + button.addEventListener("click", () => onToggle()); + row.append(labelEl, button); + const render = () => { + const on = getState(); + button.classList.toggle("is-on", on); + button.setAttribute("aria-checked", String(on)); + const tip = tooltipFor(on); + button.title = tip; + row.title = tip; + }; + return { row, button, render }; +} + function picker( description: string, onClick: () => void, diff --git a/ts/packages/typeagent-studio/src/webviewKit/host.ts b/ts/packages/typeagent-studio/src/webviewKit/host.ts index f4a996b686..ec4c359db9 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/host.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/host.ts @@ -15,6 +15,15 @@ export interface WebviewKitPanelOptions { */ instanceKey?: string; title: string; + /** + * Keep the webview's context (DOM + script state + live messaging channel) + * alive while the panel is hidden instead of tearing it down and reloading + * on reveal. Defaults to `false`. Set `true` for panels that hold live + * state (an open connection, a rendered result) so navigating away and back + * doesn't drop the agent/connection context and strand a one-shot reveal + * handshake. Costs memory while hidden. + */ + retainContextWhenHidden?: boolean; /** Path segments (from the extension root) to the client script bundle. */ scriptPath: string[]; /** Path segments (from the extension root) to the stylesheet. */ @@ -31,8 +40,8 @@ export interface WebviewKitPanelOptions { * existing one instead of opening a second — builds a locked-down CSP'd HTML * shell with a per-load nonce, restricts `localResourceRoots` to the bundle + * media dirs, and exposes typed `post`/`dispose`. State restore is left to the - * webview via `setState` — the panel intentionally does not - * `retainContextWhenHidden`. + * webview via `setState`; panels that hold live state can opt into + * `retainContextWhenHidden` so a reveal doesn't tear down and reload them. */ export class WebviewKitPanel { private static readonly live = new Map(); @@ -81,7 +90,8 @@ export class WebviewKitPanel { vscode.ViewColumn.Active, { enableScripts: true, - retainContextWhenHidden: false, + retainContextWhenHidden: + options.retainContextWhenHidden ?? false, localResourceRoots: [scriptRoot, styleRoot], }, ); diff --git a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts index 0e598ae848..2968ec03de 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts @@ -11,7 +11,10 @@ * touches a socket (the security boundary). */ -import type { StudioReplayResult } from "@typeagent/core/runtime"; +import type { + StudioReplayResult, + StudioReplayMode, +} from "@typeagent/core/runtime"; import type { VersionSpec } from "@typeagent/core/replay"; import { coerceVersionSpec, @@ -39,6 +42,11 @@ export type HostToWebviewMessage = /** Whether `agent` has a corpus available to replay (channel * `listCorpusAgents`). */ available: boolean; + /** Whether wildcard validation can run for `agent` — it exposes a + * `validateWildcardMatch` and its module loads. Drives the validation + * toggle: enabled when there is a validator to run, disabled (a no-op) + * otherwise. */ + canValidateWildcards: boolean; } /** A connection/loading status line for the webview to surface. */ | { type: "status"; text: string } @@ -71,6 +79,14 @@ export type WebviewToHostMessage = versionA: VersionSpec; /** Validated compare (B) version spec. */ versionB: VersionSpec; + /** Which deterministic dispatch path to model. */ + mode: StudioReplayMode; + /** + * Opt-in: additionally run the agent's real `validateWildcardMatch` + * over the working-tree side's wildcard matches. Off unless the user + * lit the validation toggle. + */ + validateWildcards: boolean; } /** Ask the host to open a native version QuickPick for one side. */ | { type: "pickVersion"; side: ReplaySide }; @@ -79,6 +95,15 @@ function narrowSide(value: unknown): ReplaySide | undefined { return value === "a" || value === "b" ? value : undefined; } +/** + * Narrow an untrusted mode into a {@link StudioReplayMode}, defaulting unknown + * or missing values to the grammar-only `nfa-grammar` baseline (the safer, + * cache-free default the runtime also falls back to). + */ +function narrowMode(value: unknown): StudioReplayMode { + return value === "completionBased-cache" ? value : "nfa-grammar"; +} + /** Narrow an untrusted value into a {@link WebviewToHostMessage}. */ export function parseWebviewMessage( value: unknown, @@ -100,6 +125,8 @@ export function parseWebviewMessage( agent?: unknown; versionA?: unknown; versionB?: unknown; + mode?: unknown; + validateWildcards?: unknown; }; if ( typeof m.requestId === "number" && @@ -116,6 +143,9 @@ export function parseWebviewMessage( agent: m.agent, versionA: coerceVersionSpec(m.versionA), versionB: coerceVersionSpec(m.versionB), + mode: narrowMode(m.mode), + // Opt-in and conservative: only an explicit `true` enables it. + validateWildcards: m.validateWildcards === true, }; } return undefined; diff --git a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts index eef46be0ac..c039fb960b 100644 --- a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts +++ b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts @@ -15,7 +15,12 @@ import type { ReplayCacheState, VersionSpec, } from "@typeagent/core/replay"; -import type { StudioReplayResult } from "@typeagent/core/runtime"; +import type { + SideFidelity, + FidelityLayer, + FidelityLayerStatus, + StudioReplayResult, +} from "@typeagent/core/runtime"; import { classifyReplayRow, type ReplayRowStatus, @@ -296,6 +301,75 @@ export function toSideMethodLabel(method: StudioReplayMethod): string { return METHOD_LABEL[method]; } +/** One cell of the fidelity matrix: a layer's status on one side plus the + * reason to show on hover. */ +export interface FidelityCell { + status: FidelityLayerStatus; + reason: string; +} + +/** One row of the fidelity matrix: a deterministic layer and its A/B status. */ +export interface FidelityMatrixRow { + /** Display label for the layer, e.g. "Construction cache". */ + layer: string; + a: FidelityCell; + b: FidelityCell; +} + +/** Render-ready per-side fidelity matrix for the Impact Report. */ +export interface FidelityMatrixView { + /** How side A was realized, e.g. "built (live)". */ + realizationA: string; + /** How side B was realized, e.g. "source (git ref)". */ + realizationB: string; + rows: FidelityMatrixRow[]; +} + +const FIDELITY_LAYER_ORDER: { key: FidelityLayer; label: string }[] = [ + { key: "grammar", label: "Grammar match" }, + { key: "schemaEnrichment", label: "Schema enrichment" }, + { key: "constructionCache", label: "Construction cache" }, + { key: "wildcardValidation", label: "Wildcard validation" }, + { key: "dispatch", label: "Full dispatch" }, +]; + +const FIDELITY_REALIZATION_LABEL: Record< + SideFidelity["A"]["realization"], + string +> = { + "built-live": "built (live)", + source: "source (git ref)", +}; + +/** + * Map the core {@link SideFidelity} descriptor into a render-ready matrix: + * one row per deterministic layer with each side's status + reason. Pure + + * browser-neutral. + */ +export function toFidelityMatrix( + sideFidelity: SideFidelity | undefined, +): FidelityMatrixView | undefined { + if (!sideFidelity) { + return undefined; + } + const rows: FidelityMatrixRow[] = FIDELITY_LAYER_ORDER.map( + ({ key, label }) => ({ + layer: label, + a: toFidelityCell(sideFidelity.A.layers[key]), + b: toFidelityCell(sideFidelity.B.layers[key]), + }), + ); + return { + realizationA: FIDELITY_REALIZATION_LABEL[sideFidelity.A.realization], + realizationB: FIDELITY_REALIZATION_LABEL[sideFidelity.B.realization], + rows, + }; +} + +function toFidelityCell(report: FidelityCell): FidelityCell { + return { status: report.status, reason: report.reason }; +} + /** * A version the user selected through a picker: the typed spec the host will run * plus the resolved, human-readable label/tooltip to show in the launch control. diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index dd1667979b..88f1b1c561 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -155,7 +155,7 @@ importers: version: 8.18.1 jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@25.9.3)(ts-node@10.9.2(@types/node@25.9.3)(typescript@5.4.5)) + version: 29.7.0(@types/node@22.19.19)(ts-node@10.9.2(@types/node@22.19.19)(typescript@5.4.5)) prettier: specifier: ^3.5.3 version: 3.5.3 @@ -919,7 +919,7 @@ importers: version: 12.0.2(webpack@5.105.0) jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@22.19.19)(ts-node@10.9.2(@types/node@22.19.19)(typescript@5.4.5)) + version: 29.7.0(@types/node@25.9.3)(ts-node@10.9.2(@types/node@25.9.3)(typescript@5.4.5)) prettier: specifier: ^3.5.3 version: 3.5.3 @@ -5857,6 +5857,9 @@ importers: debug: specifier: ^4.4.0 version: 4.4.3(supports-color@8.1.1) + default-agent-provider: + specifier: workspace:* + version: link:../defaultAgentProvider ws: specifier: ^8.21.0 version: 8.21.0