Skip to content

Personal AI, On Personal Devices

Python
6.1 KB
Raw Blame
"""CLI for generating framework-comparison eval configs from `_template.toml`.

Usage:
    python -m openjarvis.evals.comparison.make_configs \\
        --framework hermes --model qwen-9b --benchmark gaia
"""

from __future__ import annotations

import json
from pathlib import Path
from typing import Dict, Optional

import click

# ---------------------------------------------------------------------------
# Domain catalogs - single source of truth for what's a valid combo.
# Extend these dicts to add new frameworks / models / benchmarks.
# ---------------------------------------------------------------------------

FRAMEWORKS: Dict[str, Dict[str, str]] = {
    "openjarvis": {"backend_id": "jarvis-agent"},
    "openjarvis-distilled": {"backend_id": "jarvis-agent"},
    "hermes": {"backend_id": "hermes"},
    "openclaw": {"backend_id": "openclaw"},
}

MODELS: Dict[str, Dict[str, object]] = {
    "claude-opus-46": {
        "model_id": "claude-opus-4-6",
        "model_pretty": "Claude Opus 4.6",
        "engine": "cloud",
        "num_gpus": 0,
        "max_tokens": 8192,
    },
    "qwen-9b": {
        "model_id": "Qwen/Qwen3.5-9B",
        "model_pretty": "Qwen3.5-9B",
        "engine": "vllm",
        "num_gpus": 1,
        "max_tokens": 8192,
    },
    "qwen-122b": {
        "model_id": "Qwen/Qwen3.5-122B",
        "model_pretty": "Qwen3.5-122B",
        "engine": "vllm",
        "num_gpus": 4,
        "max_tokens": 8192,
    },
}

BENCHMARKS: Dict[str, Dict[str, object]] = {
    "toolcall15": {
        "max_samples": 15,
        "tools": ["think", "calculator"],
        "temperature": 0.0,
    },
    "pinchbench": {
        "max_samples": 23,
        "tools": ["think", "code_interpreter", "web_search", "file_read"],
        "temperature": 0.6,
    },
    "livecodebench": {
        "max_samples": 100,
        "tools": ["think", "code_interpreter"],
        "temperature": 0.2,
    },
    "taubench": {
        "max_samples": 100,
        "tools": ["think"],
        "temperature": 0.0,
    },
    "taubench-telecom": {
        "max_samples": 40,
        "tools": ["think"],
        "temperature": 0.0,
    },
    "gaia": {
        "max_samples": 50,
        "tools": [
            "think",
            "calculator",
            "code_interpreter",
            "web_search",
            "file_read",
        ],
        "temperature": 0.6,
    },
    "liveresearchbench": {
        "max_samples": 50,
        "tools": ["think", "web_search"],
        "temperature": 0.6,
    },
    "deepresearchbench": {
        "max_samples": 80,
        "tools": ["think", "web_search"],
        "temperature": 0.6,
    },
}

DEFAULT_JUDGE_MODEL = "gpt-5-mini-2025-08-07"


def _template_path() -> Path:
    return (
        Path(__file__).resolve().parents[1]
        / "configs"
        / "framework_comparison"
        / "_template.toml"
    )


def _default_output_dir() -> Path:
    return _template_path().parent


def materialize_config(
    framework: str,
    model: str,
    benchmark: str,
    output_dir: Optional[Path] = None,
    judge_model: str = DEFAULT_JUDGE_MODEL,
) -> Path:
    """Materialize one TOML config from the template."""
    if framework not in FRAMEWORKS:
        raise ValueError(f"unknown framework {framework!r}; valid: {list(FRAMEWORKS)}")
    if model not in MODELS:
        raise ValueError(f"unknown model {model!r}; valid: {list(MODELS)}")
    if benchmark not in BENCHMARKS:
        raise ValueError(f"unknown benchmark {benchmark!r}; valid: {list(BENCHMARKS)}")

    fwk = FRAMEWORKS[framework]
    mdl = MODELS[model]
    bnk = BENCHMARKS[benchmark]

    template = _template_path().read_text()
    sentinel = "# --- TEMPLATE BEGIN ---\n"
    if sentinel in template:
        template = template.split(sentinel, 1)[1]
    rendered = (
        template.replace("{{benchmark}}", benchmark)
        .replace("{{framework}}", framework)
        .replace("{{framework_id}}", fwk["backend_id"])
        .replace("{{model_pretty}}", str(mdl["model_pretty"]))
        .replace("{{model_id}}", str(mdl["model_id"]))
        .replace("{{model_slug}}", model)
        .replace("{{engine}}", str(mdl["engine"]))
        .replace("{{num_gpus}}", str(mdl["num_gpus"]))
        .replace("{{tools}}", json.dumps(bnk["tools"]))
        .replace("{{max_samples}}", str(bnk["max_samples"]))
        .replace("{{temperature}}", str(bnk["temperature"]))
        .replace("{{max_tokens}}", str(mdl["max_tokens"]))
        .replace("{{judge_model}}", judge_model)
    )

    out_dir = output_dir if output_dir is not None else _default_output_dir()
    out_dir.mkdir(parents=True, exist_ok=True)
    filename = f"{benchmark}-{framework}-{model}.toml"
    out_path = out_dir / filename
    out_path.write_text(rendered)
    return out_path


@click.command()
@click.option("--framework", type=click.Choice(list(FRAMEWORKS)))
@click.option("--model", type=click.Choice(list(MODELS)))
@click.option("--benchmark", type=click.Choice(list(BENCHMARKS)))
@click.option(
    "--output-dir",
    type=click.Path(file_okay=False, path_type=Path),
    default=None,
)
@click.option(
    "--all-tier1",
    is_flag=True,
    help="Materialize the full configs grid (4 frameworks x 8 benchmarks x 3 models)",
)
def main(
    framework: Optional[str],
    model: Optional[str],
    benchmark: Optional[str],
    output_dir: Optional[Path],
    all_tier1: bool,
) -> None:
    """Generate eval config TOMLs for the framework-comparison experiment."""
    if all_tier1:
        tier1_models = ["claude-opus-46", "qwen-9b", "qwen-122b"]
        count = 0
        for f in FRAMEWORKS:
            for m in tier1_models:
                for b in BENCHMARKS:
                    p = materialize_config(f, m, b, output_dir)
                    click.echo(f"  wrote {p.name}")
                    count += 1
        click.echo(f"\nWrote {count} configs.")
        return

    if not framework or not model or not benchmark:
        raise click.UsageError(
            "Specify --framework + --model + --benchmark, or pass --all-tier1"
        )
    p = materialize_config(framework, model, benchmark, output_dir)
    click.echo(f"wrote {p}")


if __name__ == "__main__":
    main()