Improve setup and settings provider selection UX

lhy0718 · lhy0718 · commit 8fc212d6d313 · 2026-03-09T08:32:43.000+09:00
diff --git a/src/config.ts b/src/config.ts
@@ -18,7 +18,7 @@ import {
   normalizeOpenAiResponsesReasoningEffort
 } from "./integrations/openai/modelCatalog.js";
 import { ensureDir, fileExists, writeJsonFile } from "./utils/fs.js";
-import { askLine, askRequiredLine, PromptReader } from "./utils/prompt.js";
+import { askChoice, askLine, askRequiredLine, PromptReader } from "./utils/prompt.js";
 
 export interface AppPaths {
   cwd: string;
@@ -134,11 +134,13 @@ export async function runSetupWizard(
     "Default objective metric",
     "state-of-the-art reproducibility"
   );
+  writePrimaryLlmTradeoffGuidance();
   const llmMode = await askPrimaryLlmMode(promptReader);
   const openAiModel =
     llmMode === "openai_api"
       ? await askOpenAiResponsesModel(promptReader)
       : DEFAULT_OPENAI_RESPONSES_MODEL;
+  writePdfAnalysisTradeoffGuidance();
   const pdfAnalysisMode = await askPdfAnalysisMode(promptReader);
   const responsesPdfModel =
     pdfAnalysisMode === "responses_api_pdf"
@@ -356,9 +358,51 @@ function normalizePrimaryLlmMode(value: unknown): "codex_chatgpt_only" | "openai
   return value === "openai_api" ? value : "codex_chatgpt_only";
 }
 
+function writePrimaryLlmTradeoffGuidance(): void {
+  output.write(
+    [
+      "Primary LLM provider trade-off:",
+      "- codex: uses Sign in with ChatGPT, no OpenAI API key needed, best fit for interactive coding and implement_experiments.",
+      "- api: uses OpenAI API models, requires OPENAI_API_KEY, easier to control model choice and structured API behavior, but API usage is billed separately.",
+      ""
+    ].join("\n")
+  );
+}
+
+function writePdfAnalysisTradeoffGuidance(): void {
+  output.write(
+    [
+      "PDF analysis trade-off:",
+      "- codex: downloads PDFs locally and extracts text with local tools; cheaper to operate inside the current Codex flow, but extraction quality depends on local tooling.",
+      "- api: sends PDFs to the OpenAI Responses API; usually better document understanding, but slower and requires OPENAI_API_KEY.",
+      ""
+    ].join("\n")
+  );
+}
+
 async function askPrimaryLlmMode(
   promptReader: PromptReader = askLine
 ): Promise<"codex_chatgpt_only" | "openai_api"> {
+  if (promptReader === askLine) {
+    const answer = await askChoice(
+      "Primary LLM provider",
+      [
+        {
+          label: "codex",
+          value: "codex_chatgpt_only",
+          description: "(ChatGPT sign-in, best for interactive coding)"
+        },
+        {
+          label: "api",
+          value: "openai_api",
+          description: "(OPENAI_API_KEY required, direct API control)"
+        }
+      ],
+      "codex_chatgpt_only"
+    );
+    return answer === "openai_api" ? "openai_api" : "codex_chatgpt_only";
+  }
+
   while (true) {
     const answer = (await promptReader("Primary LLM provider (codex/api)", "codex")).trim().toLowerCase();
     if (!answer || answer === "codex" || answer === "chatgpt" || answer === "codex_chatgpt_only") {
@@ -374,6 +418,26 @@ async function askPrimaryLlmMode(
 async function askPdfAnalysisMode(
   promptReader: PromptReader = askLine
 ): Promise<"codex_text_extract" | "responses_api_pdf"> {
+  if (promptReader === askLine) {
+    const answer = await askChoice(
+      "PDF analysis mode",
+      [
+        {
+          label: "codex",
+          value: "codex_text_extract",
+          description: "(local PDF download + text extraction)"
+        },
+        {
+          label: "api",
+          value: "responses_api_pdf",
+          description: "(Responses API PDF input, richer but slower)"
+        }
+      ],
+      "codex_text_extract"
+    );
+    return answer === "responses_api_pdf" ? "responses_api_pdf" : "codex_text_extract";
+  }
+
   while (true) {
     const answer = (await promptReader("PDF analysis mode (codex/api)", "codex")).trim().toLowerCase();
     if (!answer || answer === "codex") {
diff --git a/src/utils/prompt.ts b/src/utils/prompt.ts
@@ -1,10 +1,16 @@
 import { stdin as input, stdout as output } from "node:process";
-import readline from "node:readline/promises";
+import readline from "node:readline";
+import readlinePromises from "node:readline/promises";
 
 export type PromptReader = (question: string, defaultValue?: string) => Promise<string>;
+export type PromptChoice = {
+  label: string;
+  value: string;
+  description?: string;
+};
 
 export async function askLine(question: string, defaultValue = ""): Promise<string> {
-  const rl = readline.createInterface({ input, output });
+  const rl = readlinePromises.createInterface({ input, output });
   try {
     const suffix = defaultValue ? ` (${defaultValue})` : "";
     const answer = (await rl.question(`${question}${suffix}: `)).trim();
@@ -17,6 +23,96 @@ export async function askLine(question: string, defaultValue = ""): Promise<stri
   }
 }
 
+export async function askChoice(question: string, choices: PromptChoice[], defaultValue?: string): Promise<string> {
+  if (!input.isTTY || !output.isTTY || typeof input.setRawMode !== "function") {
+    const fallbackDefault = defaultValue ?? choices[0]?.value ?? "";
+    const answer = (await askLine(question, fallbackDefault)).trim();
+    const normalized = answer.toLowerCase();
+    const matched = choices.find((choice) => {
+      const label = choice.label.toLowerCase();
+      const value = choice.value.toLowerCase();
+      return normalized === label || normalized === value;
+    });
+    return matched?.value ?? fallbackDefault;
+  }
+
+  const defaultIndex = Math.max(
+    0,
+    choices.findIndex((choice) => choice.value === defaultValue || choice.label === defaultValue)
+  );
+  let selectedIndex = defaultIndex >= 0 ? defaultIndex : 0;
+  let settled = false;
+
+  return await new Promise<string>((resolve, reject) => {
+    const redraw = () => {
+      const lines = [
+        `${question}:`,
+        ...choices.map((choice, index) => {
+          const pointer = index === selectedIndex ? ">" : " ";
+          const suffix = choice.description ? ` ${choice.description}` : "";
+          return `${pointer} ${choice.label}${suffix}`;
+        })
+      ];
+      output.write("\x1b[2K\r");
+      output.write(lines.join("\n"));
+      output.write(`\x1b[${lines.length - 1}A\r`);
+    };
+
+    const cleanup = () => {
+      input.off("keypress", onKeypress);
+      if (input.isTTY && typeof input.setRawMode === "function") {
+        input.setRawMode(false);
+      }
+      output.write(`\x1b[${choices.length}B\r`);
+      output.write("\x1b[2K\r");
+    };
+
+    const finish = (value: string) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      cleanup();
+      output.write(`${question}: ${choices.find((choice) => choice.value === value)?.label ?? value}\n`);
+      resolve(value);
+    };
+
+    const fail = (error: Error) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      cleanup();
+      reject(error);
+    };
+
+    const onKeypress = (_str: string, key: readline.Key) => {
+      if (key.name === "up") {
+        selectedIndex = (selectedIndex - 1 + choices.length) % choices.length;
+        redraw();
+        return;
+      }
+      if (key.name === "down") {
+        selectedIndex = (selectedIndex + 1) % choices.length;
+        redraw();
+        return;
+      }
+      if (key.name === "return" || key.name === "enter") {
+        finish(choices[selectedIndex]?.value ?? choices[0].value);
+        return;
+      }
+      if (key.ctrl && key.name === "c") {
+        fail(new Error("setup aborted"));
+      }
+    };
+
+    readline.emitKeypressEvents(input);
+    input.setRawMode(true);
+    input.on("keypress", onKeypress);
+    redraw();
+  });
+}
+
 export async function askRequiredLine(
   question: string,
   reader: PromptReader = askLine
diff --git a/tests/configEnv.test.ts b/tests/configEnv.test.ts
@@ -82,6 +82,23 @@ async function createWorkspace(): Promise<{ cwd: string; paths: ReturnType<typeo
   return { cwd, paths };
 }
 
+async function captureStdout<T>(fn: () => Promise<T>): Promise<{ result: T; output: string }> {
+  const chunks: string[] = [];
+  const originalWrite = process.stdout.write.bind(process.stdout);
+
+  process.stdout.write = ((chunk: string | Uint8Array) => {
+    chunks.push(typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"));
+    return true;
+  }) as typeof process.stdout.write;
+
+  try {
+    const result = await fn();
+    return { result, output: chunks.join("") };
+  } finally {
+    process.stdout.write = originalWrite;
+  }
+}
+
 afterEach(() => {
   if (ORIGINAL_SEMANTIC_SCHOLAR_API_KEY === undefined) {
     delete process.env.SEMANTIC_SCHOLAR_API_KEY;
@@ -158,6 +175,35 @@ describe("config .env overrides", () => {
     await expect(fs.readFile(paths.configFile, "utf8")).resolves.toContain("project_name: project");
   });
 
+  it("prints codex/api trade-off guidance during first-run setup", async () => {
+    delete process.env.OPENAI_API_KEY;
+    const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "autoresearch-setup-tradeoffs-"));
+    const paths = resolveAppPaths(cwd);
+    const answers = [
+      "project",
+      "Multi-agent collaboration",
+      "recent papers,last 5 years",
+      "reproducibility",
+      "codex",
+      "codex",
+      "semantic-key"
+    ];
+
+    const { output } = await captureStdout(() =>
+      runSetupWizard(paths, async (_question, defaultValue = "") => {
+        const answer = answers.shift();
+        return answer !== undefined ? answer : defaultValue;
+      })
+    );
+
+    expect(output).toContain("Primary LLM provider trade-off:");
+    expect(output).toContain("codex: uses Sign in with ChatGPT");
+    expect(output).toContain("api: uses OpenAI API models");
+    expect(output).toContain("PDF analysis trade-off:");
+    expect(output).toContain("codex: downloads PDFs locally");
+    expect(output).toContain("api: sends PDFs to the OpenAI Responses API");
+  });
+
   it("uses OPENAI_API_KEY from .env when Responses PDF mode is enabled", async () => {
     delete process.env.OPENAI_API_KEY;
     const { cwd } = await createWorkspace();
diff --git a/tests/terminalAppPlanExecution.test.ts b/tests/terminalAppPlanExecution.test.ts
@@ -55,6 +55,66 @@ function makeRun(id = "run-1"): any {
 }
 
 describe("TerminalApp pending natural plan execution", () => {
+  it("uses selection menus for provider and PDF mode in settings", async () => {
+    const saveConfig = vi.fn().mockResolvedValue(undefined);
+    const app = new TerminalApp({
+      config: {
+        papers: { max_results: 100 },
+        providers: {
+          llm_mode: "codex_chatgpt_only",
+          codex: { model: "gpt-5.3-codex", reasoning_effort: "xhigh", fast_mode: false },
+          openai: { model: "gpt-5.4", reasoning_effort: "medium" }
+        },
+        analysis: {
+          pdf_mode: "codex_text_extract",
+          responses_model: "gpt-5.4"
+        },
+        research: {
+          default_topic: "Multi-agent collaboration",
+          default_constraints: ["recent papers", "last 5 years"],
+          default_objective_metric: "state-of-the-art reproducibility"
+        }
+      } as any,
+      runStore: {} as any,
+      titleGenerator: {} as any,
+      codex: {} as any,
+      eventStream: { subscribe: () => () => {} } as any,
+      orchestrator: {} as any,
+      semanticScholarApiKeyConfigured: false,
+      onQuit: () => {},
+      saveConfig
+    }) as any;
+
+    app.render = () => {};
+    app.updateSuggestions = () => {};
+    app.drainQueuedInputs = async () => {};
+    app.askWithinTui = vi
+      .fn()
+      .mockResolvedValueOnce("Pilot topic")
+      .mockResolvedValueOnce("recent papers,last 5 years")
+      .mockResolvedValueOnce("reproducibility");
+    app.openSelectionMenu = vi
+      .fn()
+      .mockResolvedValueOnce("codex_chatgpt_only")
+      .mockResolvedValueOnce("codex_text_extract");
+
+    await app.handleSettings();
+
+    expect(app.openSelectionMenu).toHaveBeenNthCalledWith(
+      1,
+      "Select primary LLM provider",
+      expect.any(Array),
+      "codex_chatgpt_only"
+    );
+    expect(app.openSelectionMenu).toHaveBeenNthCalledWith(
+      2,
+      "Select PDF analysis mode",
+      expect.any(Array),
+      "codex_text_extract"
+    );
+    expect(saveConfig).toHaveBeenCalledTimes(1);
+  });
+
   it("answers collected-paper count questions directly instead of arming a collect command", async () => {
     const app = makeApp();
     const run = makeRun("run-count");
