Merge branch 'benchmark_openai' - Add OpenAI-compatible provider support (#114)

Co-Authored-By: KeinNiemand <KeinNiemand@users.noreply.github.com>
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: 3 people

benchmark/cli.py

@@ -16,7 +16,11 @@
 from benchmark.runner import BenchmarkRunner, quick_benchmark, full_benchmark
 from benchmark.results.storage import ResultsStorage
 from benchmark.wiki.generator import WikiGenerator
-from benchmark.translator import get_available_ollama_models, get_available_openrouter_models
+from benchmark.translator import (
+    get_available_ollama_models,
+    get_available_openrouter_models,
+    get_available_openai_models,
+)
 
 
 # ANSI color codes for terminal output
@@ -75,6 +79,8 @@ def cmd_run(args: argparse.Namespace) -> int:
     evaluator_provider = getattr(args, 'evaluator_provider', DEFAULT_EVALUATOR_PROVIDER)
     config = BenchmarkConfig.from_cli_args(
         openrouter_key=args.openrouter_key,
+        openai_key=args.openai_key,
+        openai_endpoint=args.openai_endpoint,
         poe_key=args.poe_key,
         evaluator_model=args.evaluator,
         ollama_endpoint=args.ollama_endpoint,
@@ -101,6 +107,14 @@ def cmd_run(args: argparse.Namespace) -> int:
             # Extract model IDs
             models = [m["id"] if isinstance(m, dict) else m for m in models_data[:10]]
             print(colored(f"Found {len(models_data)} models. Using top 10: {', '.join(models[:3])}...", Colors.GREEN))
+        elif provider == "openai":
+            print(colored("Fetching available OpenAI-compatible models...", Colors.CYAN))
+            models_data = asyncio.run(get_available_openai_models(config))
+            if not models_data:
+                log_callback("error", "No OpenAI-compatible models available.")
+                return 1
+            models = [m["id"] if isinstance(m, dict) else m for m in models_data[:10]]
+            print(colored(f"Found {len(models_data)} models. Using top 10: {', '.join(models[:3])}...", Colors.GREEN))
         else:
             print(colored("Detecting available Ollama models...", Colors.CYAN))
             models = asyncio.run(get_available_ollama_models(config))
@@ -288,7 +302,12 @@ def cmd_models(args: argparse.Namespace) -> int:
     """List available models for benchmarking."""
     print_banner()
 
-    config = BenchmarkConfig.from_cli_args(openrouter_key=args.openrouter_key)
+    config = BenchmarkConfig.from_cli_args(
+        openrouter_key=args.openrouter_key,
+        openai_key=args.openai_key,
+        openai_endpoint=args.openai_endpoint,
+        translation_provider=args.provider,
+    )
     provider = args.provider
 
     if provider == "openrouter":
@@ -322,6 +341,32 @@ def cmd_models(args: argparse.Namespace) -> int:
         print(colored("Tip: Use -m to specify models, e.g.:", Colors.YELLOW))
         print("  python -m benchmark.cli run -p openrouter -m anthropic/claude-sonnet-4 openai/gpt-4o")
 
+    elif provider == "openai":
+        print(colored("Fetching OpenAI-compatible models...\n", Colors.CYAN))
+        models = asyncio.run(get_available_openai_models(config))
+
+        if not models:
+            log_callback("error", "Failed to fetch OpenAI-compatible models")
+            return 1
+
+        print(colored(f"Available OpenAI-Compatible Models ({len(models)}):\n", Colors.BOLD))
+        print(f"{'Model ID':<50} {'Owner':<20}")
+        print("-" * 72)
+
+        for model in models[:50]:
+            if isinstance(model, dict):
+                model_id = model.get("id", "unknown")
+                owned_by = model.get("owned_by", "unknown")
+            else:
+                model_id = model
+                owned_by = "unknown"
+
+            print(f"{model_id:<50} {owned_by:<20}")
+
+        print()
+        print(colored("Tip: Use -m and --openai-endpoint to specify a backend, e.g.:", Colors.YELLOW))
+        print("  python -m benchmark.cli run -p openai --openai-endpoint http://localhost:8080/v1 -m your-model")
+
     else:
         print(colored("Detecting Ollama models...\n", Colors.CYAN))
         models = asyncio.run(get_available_ollama_models(config))
@@ -566,6 +611,9 @@ def create_parser() -> argparse.ArgumentParser:
   # Quick benchmark with Ollama (local models)
   python -m benchmark.cli run --openrouter-key YOUR_KEY
 
+    # Quick benchmark with an OpenAI-compatible backend
+    python -m benchmark.cli run --provider openai --openai-endpoint http://localhost:8080/v1 -m your-model
+
   # Quick benchmark with OpenRouter (cloud models)
   python -m benchmark.cli run --provider openrouter --openrouter-key YOUR_KEY
 
@@ -578,6 +626,9 @@ def create_parser() -> argparse.ArgumentParser:
   # Specific OpenRouter models
   python -m benchmark.cli run -p openrouter -m anthropic/claude-sonnet-4 openai/gpt-4o -l fr de ja
 
+    # Specific OpenAI-compatible backend and models
+    python -m benchmark.cli run -p openai --openai-endpoint http://localhost:8080/v1 -m qwen2.5-14b-instruct
+
   # Generate wiki pages
   python -m benchmark.cli wiki
 
@@ -594,6 +645,7 @@ def create_parser() -> argparse.ArgumentParser:
         "-m", "--models",
         nargs="+",
         help="Models to benchmark. For Ollama: model names (e.g., llama3:8b). "
+             "For OpenAI-compatible backends: model IDs (e.g., gpt-4o or local server model names). "
              "For OpenRouter: model IDs (e.g., anthropic/claude-sonnet-4). "
              "If not specified, auto-detects available models."
     )
@@ -609,9 +661,17 @@ def create_parser() -> argparse.ArgumentParser:
     )
     run_parser.add_argument(
         "-p", "--provider",
-        choices=["ollama", "openrouter"],
+        choices=["ollama", "openai", "openrouter"],
         default="ollama",
-        help="Translation provider: 'ollama' (local, default) or 'openrouter' (cloud, 200+ models)"
+        help="Translation provider: 'ollama' (local, default), 'openai' (OpenAI-compatible), or 'openrouter' (cloud, 200+ models)"
+    )
+    run_parser.add_argument(
+        "--openai-key",
+        help="API key for OpenAI-compatible translation backends. Can also be set via OPENAI_API_KEY env var."
+    )
+    run_parser.add_argument(
+        "--openai-endpoint",
+        help="OpenAI-compatible chat completions endpoint or /v1 base URL. Can also be set via OPENAI_API_ENDPOINT env var."
     )
     run_parser.add_argument(
         "--openrouter-key",
@@ -696,10 +756,18 @@ def create_parser() -> argparse.ArgumentParser:
     models_parser = subparsers.add_parser("models", help="List available models for benchmarking")
     models_parser.add_argument(
         "-p", "--provider",
-        choices=["ollama", "openrouter"],
+        choices=["ollama", "openai", "openrouter"],
         default="ollama",
         help="Provider to list models for (default: ollama)"
     )
+    models_parser.add_argument(
+        "--openai-key",
+        help="API key for listing models from an OpenAI-compatible endpoint"
+    )
+    models_parser.add_argument(
+        "--openai-endpoint",
+        help="OpenAI-compatible endpoint to query for available models"
+    )
     models_parser.add_argument(
         "--openrouter-key",
         help="OpenRouter API key (required for listing OpenRouter models)"

benchmark/config.py

@@ -2,7 +2,7 @@
 Benchmark configuration module.
 
 Defines configuration settings for the benchmark system including:
-- Ollama settings for translation
+- Ollama/OpenAI-compatible settings for translation
 - OpenRouter settings for evaluation
 - File paths and defaults
 """
@@ -99,6 +99,30 @@ class OpenRouterConfig:
     site_name: str = "TranslateBookWithLLM Benchmark"
 
 
+@dataclass
+class OpenAICompatibleConfig:
+    """Configuration for OpenAI-compatible translation provider."""
+
+    api_key: Optional[str] = field(
+        default_factory=lambda: os.getenv("OPENAI_API_KEY")
+    )
+    endpoint: str = field(
+        default_factory=lambda: os.getenv(
+            "OPENAI_API_ENDPOINT",
+            "https://api.openai.com/v1/chat/completions"
+        )
+    )
+    default_model: str = field(
+        default_factory=lambda: os.getenv("OPENAI_MODEL", "gpt-4o-mini")
+    )
+    context_window: int = field(
+        default_factory=lambda: int(os.getenv("OPENAI_NUM_CTX", os.getenv("OLLAMA_NUM_CTX", "2048")))
+    )
+    timeout: int = field(
+        default_factory=lambda: int(os.getenv("OPENAI_REQUEST_TIMEOUT", os.getenv("REQUEST_TIMEOUT", "900")))
+    )
+
+
 @dataclass
 class PoeConfig:
     """Configuration for Poe evaluation provider."""
@@ -155,6 +179,7 @@ class BenchmarkConfig:
     """Main benchmark configuration aggregating all sub-configs."""
 
     ollama: OllamaConfig = field(default_factory=OllamaConfig)
+    openai: OpenAICompatibleConfig = field(default_factory=OpenAICompatibleConfig)
     openrouter: OpenRouterConfig = field(default_factory=OpenRouterConfig)
     poe: PoeConfig = field(default_factory=PoeConfig)
     paths: PathConfig = field(default_factory=PathConfig)
@@ -163,7 +188,7 @@ class BenchmarkConfig:
     source_language: str = "English"
     quick_languages: list = field(default_factory=lambda: DEFAULT_QUICK_LANGUAGES.copy())
 
-    # Translation provider ("ollama" or "openrouter")
+    # Translation provider ("ollama", "openai", or "openrouter")
     translation_provider: str = "ollama"
 
     # Evaluator provider ("openrouter" or "poe")
@@ -182,6 +207,8 @@ def from_env(cls) -> "BenchmarkConfig":
     def from_cli_args(
         cls,
         openrouter_key: Optional[str] = None,
+        openai_key: Optional[str] = None,
+        openai_endpoint: Optional[str] = None,
         evaluator_model: Optional[str] = None,
         ollama_endpoint: Optional[str] = None,
         translation_provider: Optional[str] = None,
@@ -195,6 +222,9 @@ def from_cli_args(
         if openrouter_key:
             config.openrouter.api_key = openrouter_key
 
+        if openai_key:
+            config.openai.api_key = openai_key
+
         if poe_key:
             config.poe.api_key = poe_key
 
@@ -205,6 +235,9 @@ def from_cli_args(
         if ollama_endpoint:
             config.ollama.endpoint = ollama_endpoint
 
+        if openai_endpoint:
+            config.openai.endpoint = openai_endpoint
+
         if translation_provider:
             config.translation_provider = translation_provider.lower()
 
@@ -245,17 +278,23 @@ def validate(self) -> list[str]:
                 "Set OPENROUTER_API_KEY in .env or use --openrouter-key"
             )
 
+        if self.translation_provider == "openai" and not self.openai.endpoint:
+            errors.append(
+                "OpenAI-compatible endpoint not configured. Required for translation. "
+                "Set OPENAI_API_ENDPOINT in .env or use --openai-endpoint"
+            )
+
         if not self.paths.languages_file.exists():
             errors.append(f"Languages file not found: {self.paths.languages_file}")
 
         if not self.paths.reference_texts_file.exists():
             errors.append(f"Reference texts file not found: {self.paths.reference_texts_file}")
 
         # Validate translation provider
-        if self.translation_provider not in ("ollama", "openrouter"):
+        if self.translation_provider not in ("ollama", "openai", "openrouter"):
             errors.append(
                 f"Invalid translation provider: {self.translation_provider}. "
-                "Must be 'ollama' or 'openrouter'"
+                "Must be 'ollama', 'openai', or 'openrouter'"
             )
 
         return errors

benchmark/runner.py

@@ -3,7 +3,7 @@
 
 Coordinates the complete benchmark workflow:
 1. Load languages and reference texts
-2. Run translations with specified Ollama models
+2. Run translations with specified provider models
 3. Evaluate translations with OpenRouter
 4. Track progress and handle resumption
 5. Generate results
@@ -25,6 +25,7 @@
 from benchmark.translator import (
     BenchmarkTranslator, TranslationRequest,
     test_ollama_connection, get_available_ollama_models,
+    test_openai_translation_connection, get_available_openai_models,
     test_openrouter_translation_connection, get_available_openrouter_models
 )
 from benchmark.evaluator import (
@@ -206,6 +207,12 @@ async def validate_setup(self) -> tuple[bool, list[str]]:
                 errors.append(f"OpenRouter (translation): {or_trans_msg}")
             else:
                 self._log("info", f"OpenRouter (translation): {or_trans_msg}")
+        elif self.config.translation_provider == "openai":
+            openai_ok, openai_msg = await test_openai_translation_connection(self.config)
+            if not openai_ok:
+                errors.append(f"OpenAI-compatible (translation): {openai_msg}")
+            else:
+                self._log("info", f"OpenAI-compatible (translation): {openai_msg}")
         else:
             # Test Ollama connection
             ollama_ok, ollama_msg = await test_ollama_connection(self.config)
@@ -241,7 +248,7 @@ def _generate_jobs(
         Generate translation jobs, skipping already completed ones.
 
         Args:
-            models: List of Ollama model names
+            models: List of provider model names
             languages: List of target languages
             texts: List of reference texts
             existing_results: Results from a previous run (for resumption)
@@ -280,7 +287,7 @@ async def run(
         Execute a complete benchmark run.
 
         Args:
-            models: List of Ollama model names to benchmark
+            models: List of provider model names to benchmark
             language_codes: Language codes to test (None = quick test set)
             resume_run: Optional previous run to resume
 
@@ -432,7 +439,7 @@ async def quick_benchmark(
 
     Args:
         config: Benchmark configuration
-        models: Optional list of models (defaults to available Ollama models)
+        models: Optional list of models (defaults to auto-detected provider models)
         log_callback: Optional logging callback
 
     Returns:
@@ -447,9 +454,16 @@ async def quick_benchmark(
 
     # Get models if not specified
     if models is None:
-        models = await get_available_ollama_models(config)
+        if config.translation_provider == "openrouter":
+            provider_models = await get_available_openrouter_models(config)
+            models = [m["id"] if isinstance(m, dict) else m for m in provider_models]
+        elif config.translation_provider == "openai":
+            provider_models = await get_available_openai_models(config)
+            models = [m["id"] if isinstance(m, dict) else m for m in provider_models]
+        else:
+            models = await get_available_ollama_models(config)
         if not models:
-            raise RuntimeError("No Ollama models available")
+            raise RuntimeError(f"No {config.translation_provider} models available")
         # Limit to first 3 models for quick benchmark
         models = models[:3]
 
@@ -466,7 +480,7 @@ async def full_benchmark(
 
     Args:
         config: Benchmark configuration
-        models: List of Ollama models to benchmark
+        models: List of provider models to benchmark
         log_callback: Optional logging callback
 
     Returns: