Skip to content

JiuwenSwarm is an intelligent AI Agent built on openJiuwen. It extends the powerful capabilities of large language models directly to your fingertips through various communication apps you use daily.

Python
7.1 KB
Raw Blame
# coding: utf-8
"""Score the full SkillsBench task set once (no train/val/test cut, no training).

Agent runs on the host via Gateway; verifier uses Docker
(``SKILLSBENCH_VERIFIER_MODE=docker``).

Usage (venv activated, Gateway up)::

    python scripts/skillsbench_docker_score_all.py
    python scripts/skillsbench_docker_score_all.py --exec-timeout 1800 --output-dir ./outputs/my_run
"""

from __future__ import annotations

import argparse
import json
import logging
import os
import sys
from datetime import datetime, timezone
from pathlib import Path

logger = logging.getLogger(__name__)


def _setup_logging() -> None:
    """INFO to stdout, WARNING+ to stderr, message-only format (matches the old prints).

    Handlers live on this script's logger only (propagate off) so library logs
    keep flowing through their own configuration, unduplicated.
    """
    logger.setLevel(logging.INFO)
    logger.propagate = False
    out = logging.StreamHandler(sys.stdout)
    out.addFilter(lambda record: record.levelno < logging.WARNING)
    err = logging.StreamHandler(sys.stderr)
    err.setLevel(logging.WARNING)
    logger.addHandler(out)
    logger.addHandler(err)


def _repo_root() -> Path:
    return Path(__file__).resolve().parents[1]


def _build_all_split(tasks_dir: Path, split_root: Path) -> int:
    """Write the all-in-train split; return the task count (0 when *tasks_dir* is empty)."""
    from openjiuwen.agent_evolving.skill_train.envs.skillsbench.dataloader import (
        load_task_records,
    )

    items = load_task_records(tasks_dir)
    if not items:
        return 0

    for name in ("train", "val", "test"):
        folder = split_root / name
        folder.mkdir(parents=True, exist_ok=True)
        payload = items if name == "train" else []
        (folder / "items.json").write_text(
            json.dumps(payload, ensure_ascii=False, indent=2),
            encoding="utf-8",
        )
    logger.info(f"[skillsbench] all-tasks split: n={len(items)} -> {split_root} (all in train/)")
    return len(items)


def _summarize(results_path: Path, out_path: Path) -> dict:
    rows = []
    if results_path.is_file():
        for line_no, line in enumerate(results_path.read_text(encoding="utf-8").splitlines(), 1):
            if not line.strip():
                continue
            try:
                rows.append(json.loads(line))
            except json.JSONDecodeError:
                # results.jsonl is appended incrementally; a killed run can leave a
                # truncated last line. Skip it so the summary still gets written
                # (the library resume reader tolerates the same).
                logger.warning(f"[skillsbench] skip corrupt line {line_no} in {results_path}")
    n = len(rows)
    hard = sum(float(r.get("hard") or 0) for r in rows) / n if n else 0.0
    soft = sum(float(r.get("soft") or 0) for r in rows) / n if n else 0.0
    fails: dict[str, int] = {}
    for r in rows:
        fr = str(r.get("fail_reason") or ("ok" if r.get("hard") else "unknown"))
        if len(fr) > 80:
            fr = fr[:80] + "..."
        fails[fr] = fails.get(fr, 0) + 1
    summary = {
        "verifier_mode": os.environ.get("SKILLSBENCH_VERIFIER_MODE", "docker"),
        "n": n,
        "hard": hard,
        "soft": soft,
        "fail_reasons": fails,
        "results_jsonl": str(results_path),
        "items": [
            {
                "id": r.get("id"),
                "hard": r.get("hard"),
                "soft": r.get("soft"),
                "fail_reason": r.get("fail_reason"),
                "agent_ok": r.get("agent_ok"),
            }
            for r in rows
        ],
    }
    out_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
    logger.info(f"[skillsbench] TOTAL n={n} hard={hard:.4f} soft={soft:.4f}")
    logger.info(f"[skillsbench] wrote {out_path}")
    return summary


def main(argv: list[str] | None = None) -> int:
    _setup_logging()
    root = _repo_root()
    parser = argparse.ArgumentParser(description="SkillsBench full-dataset docker score (no training).")
    parser.add_argument(
        "--tasks-dir",
        default="",
        help="Tasks dir; defaults to $SKILLSBENCH_TASKS, then $SKILLSBENCH_ROOT/tasks.",
    )
    parser.add_argument("--output-dir", default="")
    parser.add_argument("--exec-timeout", type=int, default=int(os.environ.get("EXEC_TIMEOUT", "1800")))
    parser.add_argument("--workers", type=int, default=1)
    parser.add_argument("--skill-init", default="")
    args = parser.parse_args(argv)

    os.environ["SKILLSBENCH_VERIFIER_MODE"] = os.environ.get("SKILLSBENCH_VERIFIER_MODE") or "docker"

    tasks_str = args.tasks_dir.strip() or os.environ.get("SKILLSBENCH_TASKS", "").strip()
    if not tasks_str:
        root_env = os.environ.get("SKILLSBENCH_ROOT", "").strip()
        if not root_env:
            logger.error(
                "[skillsbench] tasks dir unknown: pass --tasks-dir or set SKILLSBENCH_TASKS / SKILLSBENCH_ROOT"
            )
            return 2
        tasks_str = str(Path(root_env) / "tasks")
    tasks_dir = Path(tasks_str)

    stamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
    output_dir = Path(
        args.output_dir
        or (root / "outputs" / f"skillsbench_docker_score_all_{stamp}")
    ).resolve()
    output_dir.mkdir(parents=True, exist_ok=True)
    split_root = output_dir / "_all_tasks_split"
    n_tasks = _build_all_split(tasks_dir, split_root)
    if n_tasks == 0:
        logger.error(f"No SkillsBench tasks with task.md under {Path(args.tasks_dir)}")
        return 2

    from openjiuwen.agent_evolving.skill_train import TrainLaunchOptions, run_offline_eval

    opts = TrainLaunchOptions(
        env_name="skillsbench",
        target_backend="jiuwenswarm_cli_exec",
        skill_init=args.skill_init,
        split_dir=str(split_root),
        output_dir=str(output_dir),
        exec_timeout=args.exec_timeout,
        workers=args.workers,
        use_gate=False,
        extra_env_kwargs={
            "split_mode": "split_dir",
            "verifier_mode": os.environ["SKILLSBENCH_VERIFIER_MODE"],
            "data_path": "",
        },
    )
    logger.info(
        f"[skillsbench] verifier_mode={os.environ['SKILLSBENCH_VERIFIER_MODE']} "
        f"output={output_dir} n_tasks=all-in-train (eval-only, no optimizer required)"
    )
    # run_offline_eval + jiuwenswarm_cli_exec skips optimizer credential checks.
    summary = run_offline_eval(opts, split="train")
    logger.info(json.dumps(summary, ensure_ascii=False, indent=2))

    results = output_dir / "results.jsonl"
    if not results.is_file():
        # eval may nest under predictions parent; sorted so a reused --output-dir
        # cannot pick a stale file nondeterministically.
        candidates = sorted(output_dir.rglob("results.jsonl"))
        if candidates:
            results = candidates[0]
            logger.info(f"[skillsbench] results.jsonl found at {results}")
    if not results.is_file():
        logger.error(f"[skillsbench] no results.jsonl under {output_dir}")
        return 1
    _summarize(results, output_dir / "summary.json")
    return 0


if __name__ == "__main__":
    sys.exit(main())