Skip to content

Commit c215e0e

Browse files
lhy0718lhy0718
committed
Upgrade hypothesis/design nodes and repair smoke coverage
1 parent 7eca870 commit c215e0e

17 files changed

Lines changed: 1615 additions & 418 deletions

‎README.ko.md‎

Lines changed: 226 additions & 132 deletions
Large diffs are not rendered by default.

‎README.md‎

Lines changed: 226 additions & 134 deletions
Large diffs are not rendered by default.

‎src/core/analysis/researchPlanning.ts‎

Lines changed: 658 additions & 0 deletions
Large diffs are not rendered by default.

‎src/core/nodes/designExperiments.ts‎

Lines changed: 191 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -3,110 +3,228 @@ import path from "node:path";
33
import { promises as fs } from "node:fs";
44

55
import { GraphNodeHandler } from "../stateGraph/types.js";
6-
import { runTreeOfThoughts } from "../agents/runtime/tot.js";
76
import { safeRead, writeRunArtifact } from "./helpers.js";
87
import { NodeExecutionDeps } from "./types.js";
98
import { RunContextMemory } from "../memory/runContextMemory.js";
109
import { resolveConstraintProfile } from "../constraintProfile.js";
10+
import { resolveObjectiveMetricProfile } from "../objectiveMetric.js";
11+
import {
12+
designExperimentsFromHypotheses,
13+
DesignInputHypothesis,
14+
ExperimentDesignCandidate
15+
} from "../analysis/researchPlanning.js";
1116

1217
export function createDesignExperimentsNode(deps: NodeExecutionDeps): GraphNodeHandler {
1318
return {
1419
id: "design_experiments",
15-
async execute({ run, graph }) {
20+
async execute({ run }) {
1621
const runContextMemory = new RunContextMemory(run.memoryRefs.runContextPath);
22+
const emitLog = (text: string) => {
23+
deps.eventStream.emit({
24+
type: "OBS_RECEIVED",
25+
runId: run.id,
26+
node: "design_experiments",
27+
payload: { text }
28+
});
29+
};
30+
1731
const constraintProfile = await resolveConstraintProfile({
1832
run,
1933
runContextMemory,
2034
llm: deps.llm,
2135
eventStream: deps.eventStream,
2236
node: "design_experiments"
2337
});
38+
const objectiveMetricProfile = await resolveObjectiveMetricProfile({
39+
run,
40+
runContextMemory,
41+
llm: deps.llm,
42+
eventStream: deps.eventStream,
43+
node: "design_experiments"
44+
});
45+
2446
const hypothesesPath = path.join(".autoresearch", "runs", run.id, "hypotheses.jsonl");
25-
const hypothesesText = await safeRead(hypothesesPath);
26-
const seeds = hypothesesText
27-
.split("\n")
28-
.map((line) => line.trim())
29-
.filter(Boolean)
30-
.slice(0, 4)
31-
.map((line, idx) => {
32-
try {
33-
const obj = JSON.parse(line) as { text?: string };
34-
return obj.text || `hypothesis_${idx + 1}`;
35-
} catch {
36-
return `hypothesis_${idx + 1}`;
37-
}
38-
});
47+
const hypotheses = parseHypotheses(await safeRead(hypothesesPath));
48+
49+
emitLog(`Designing experiments from ${hypotheses.length} hypothesis/hypotheses.`);
50+
const design = await designExperimentsFromHypotheses({
51+
llm: deps.llm,
52+
runTitle: run.title,
53+
runTopic: run.topic,
54+
objectiveMetric: run.objectiveMetric,
55+
hypotheses,
56+
constraintProfile,
57+
objectiveProfile: objectiveMetricProfile,
58+
candidateCount: 3,
59+
onProgress: emitLog
60+
});
3961

40-
const tot = runTreeOfThoughts(seeds, { branchCount: 6, topK: 2 });
41-
const selected = tot.selected[0];
42-
const collectDefaults = constraintProfile.collect;
43-
const paperProfile = constraintProfile.writing;
44-
45-
const planYaml = [
46-
`run_id: ${run.id}`,
47-
`topic: "${escapeQuote(run.topic)}"`,
48-
"objective:",
49-
` metric: "${escapeQuote(run.objectiveMetric)}"`,
50-
"constraints:",
51-
" raw:",
52-
...renderYamlStringList(run.constraints, 2),
53-
" collect_defaults:",
54-
...renderYamlKeyValueObject(
55-
{
56-
last_years: collectDefaults.lastYears,
57-
open_access_pdf: collectDefaults.openAccessPdf,
58-
min_citation_count: collectDefaults.minCitationCount,
59-
publication_types: collectDefaults.publicationTypes
60-
},
61-
2
62-
),
63-
" writing_defaults:",
64-
...renderYamlKeyValueObject(
65-
{
66-
target_venue: paperProfile.targetVenue,
67-
tone_hint: paperProfile.toneHint,
68-
length_hint: paperProfile.lengthHint
69-
},
70-
2
71-
),
72-
" experiment_guidance:",
73-
...renderYamlKeyValueObject(
74-
{
75-
profile_source: constraintProfile.source
76-
},
77-
2
78-
),
79-
" design_notes:",
80-
...renderYamlStringList(constraintProfile.experiment.designNotes, 2),
81-
" implementation_notes:",
82-
...renderYamlStringList(constraintProfile.experiment.implementationNotes, 2),
83-
" evaluation_notes:",
84-
...renderYamlStringList(constraintProfile.experiment.evaluationNotes, 2),
85-
" assumptions:",
86-
...renderYamlStringList(constraintProfile.assumptions, 2),
87-
"hypotheses:",
88-
...tot.selected.map((x) => ` - "${escapeQuote(x.text)}"`),
89-
"execution:",
90-
" container: local",
91-
" timeout_sec: 1800",
92-
" budget:",
93-
" max_tool_calls: 150"
94-
].join("\n");
62+
const planYaml = buildPlanYaml({
63+
run,
64+
hypotheses,
65+
selected: design.selected,
66+
candidates: design.candidates,
67+
constraintProfile,
68+
objectiveProfile: objectiveMetricProfile,
69+
source: design.source
70+
});
9571

9672
const outputPath = await writeRunArtifact(run, "experiment_plan.yaml", planYaml);
9773
await fs.access(outputPath);
98-
await runContextMemory.put("design_experiments.primary", selected?.text || "");
74+
await runContextMemory.put("design_experiments.primary", design.selected.title);
75+
await runContextMemory.put("design_experiments.source", design.source);
76+
await runContextMemory.put("design_experiments.summary", design.summary);
77+
78+
deps.eventStream.emit({
79+
type: "PLAN_CREATED",
80+
runId: run.id,
81+
node: "design_experiments",
82+
payload: {
83+
candidateCount: design.candidates.length,
84+
selectedId: design.selected.id,
85+
source: design.source,
86+
fallbackReason: design.fallbackReason
87+
}
88+
});
89+
90+
emitLog(`Selected design "${design.selected.title}" from ${design.candidates.length} candidate(s) using ${design.source}.`);
9991

10092
return {
10193
status: "success",
102-
summary: `Experiment plan fixed with ${tot.selected.length} shortlisted designs.`,
94+
summary: design.fallbackReason
95+
? `${design.summary} Falling back after: ${design.fallbackReason}`
96+
: design.summary,
10397
needsApproval: true,
10498
toolCallsUsed: 1
10599
};
106100
}
107101
};
108102
}
109103

104+
function parseHypotheses(raw: string): DesignInputHypothesis[] {
105+
const items: Array<DesignInputHypothesis | undefined> = raw
106+
.split("\n")
107+
.map((line) => line.trim())
108+
.filter(Boolean)
109+
.map((line, index) => {
110+
try {
111+
const parsed = JSON.parse(line) as DesignInputHypothesis;
112+
return {
113+
hypothesis_id: parsed.hypothesis_id || `h_${index + 1}`,
114+
text: parsed.text,
115+
score: parsed.score,
116+
evidence_links: parsed.evidence_links
117+
};
118+
} catch {
119+
return undefined;
120+
}
121+
});
122+
return items.filter((item): item is DesignInputHypothesis => item !== undefined && Boolean(item.text));
123+
}
124+
125+
function buildPlanYaml(args: {
126+
run: { id: string; topic: string; objectiveMetric: string; constraints: string[] };
127+
hypotheses: DesignInputHypothesis[];
128+
selected: ExperimentDesignCandidate;
129+
candidates: ExperimentDesignCandidate[];
130+
constraintProfile: Awaited<ReturnType<typeof resolveConstraintProfile>>;
131+
objectiveProfile: Awaited<ReturnType<typeof resolveObjectiveMetricProfile>>;
132+
source: "llm" | "fallback";
133+
}): string {
134+
const collectDefaults = args.constraintProfile.collect;
135+
const paperProfile = args.constraintProfile.writing;
136+
137+
return [
138+
`run_id: ${args.run.id}`,
139+
`topic: "${escapeQuote(args.run.topic)}"`,
140+
"objective:",
141+
` metric: "${escapeQuote(args.run.objectiveMetric)}"`,
142+
` primary_metric: "${escapeQuote(args.objectiveProfile.primaryMetric || "unspecified")}"`,
143+
` target: "${escapeQuote(args.objectiveProfile.targetDescription || "observe and improve")}"`,
144+
"constraints:",
145+
" raw:",
146+
...renderYamlStringList(args.run.constraints, 2),
147+
" collect_defaults:",
148+
...renderYamlKeyValueObject(
149+
{
150+
last_years: collectDefaults.lastYears,
151+
open_access_pdf: collectDefaults.openAccessPdf,
152+
min_citation_count: collectDefaults.minCitationCount,
153+
publication_types: collectDefaults.publicationTypes
154+
},
155+
2
156+
),
157+
" writing_defaults:",
158+
...renderYamlKeyValueObject(
159+
{
160+
target_venue: paperProfile.targetVenue,
161+
tone_hint: paperProfile.toneHint,
162+
length_hint: paperProfile.lengthHint
163+
},
164+
2
165+
),
166+
" experiment_guidance:",
167+
...renderYamlKeyValueObject(
168+
{
169+
profile_source: args.constraintProfile.source,
170+
objective_profile_source: args.objectiveProfile.source,
171+
design_source: args.source
172+
},
173+
2
174+
),
175+
" design_notes:",
176+
...renderYamlStringList(args.constraintProfile.experiment.designNotes, 2),
177+
" implementation_notes:",
178+
...renderYamlStringList(args.constraintProfile.experiment.implementationNotes, 2),
179+
" evaluation_notes:",
180+
...renderYamlStringList(args.constraintProfile.experiment.evaluationNotes, 2),
181+
" assumptions:",
182+
...renderYamlStringList(args.constraintProfile.assumptions, 2),
183+
"hypotheses:",
184+
...args.hypotheses.map((item) => ` - "${escapeQuote(item.text)}"`),
185+
"selected_hypothesis_ids:",
186+
...renderYamlStringList(args.selected.hypothesis_ids, 1),
187+
"selected_design:",
188+
` id: "${escapeQuote(args.selected.id)}"`,
189+
` title: "${escapeQuote(args.selected.title)}"`,
190+
` summary: "${escapeQuote(args.selected.plan_summary)}"`,
191+
" datasets:",
192+
...renderYamlStringList(args.selected.datasets, 2),
193+
" metrics:",
194+
...renderYamlStringList(args.selected.metrics, 2),
195+
" baselines:",
196+
...renderYamlStringList(args.selected.baselines, 2),
197+
" implementation_notes:",
198+
...renderYamlStringList(args.selected.implementation_notes, 2),
199+
" evaluation_steps:",
200+
...renderYamlStringList(args.selected.evaluation_steps, 2),
201+
" risks:",
202+
...renderYamlStringList(args.selected.risks, 2),
203+
" budget_notes:",
204+
...renderYamlStringList(args.selected.budget_notes, 2),
205+
"shortlisted_designs:",
206+
...renderShortlistedDesigns(args.candidates),
207+
"execution:",
208+
" container: local",
209+
" timeout_sec: 1800",
210+
" budget:",
211+
" max_tool_calls: 150"
212+
].join("\n");
213+
}
214+
215+
function renderShortlistedDesigns(candidates: ExperimentDesignCandidate[]): string[] {
216+
if (candidates.length === 0) {
217+
return [' - "none"'];
218+
}
219+
const lines: string[] = [];
220+
for (const candidate of candidates) {
221+
lines.push(` - id: "${escapeQuote(candidate.id)}"`);
222+
lines.push(` title: "${escapeQuote(candidate.title)}"`);
223+
lines.push(` summary: "${escapeQuote(candidate.plan_summary)}"`);
224+
}
225+
return lines;
226+
}
227+
110228
function escapeQuote(text: string): string {
111229
return text.replace(/"/g, "'");
112230
}

0 commit comments

Comments
 (0)