Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
13.7 KB
Raw Blame
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Transform a craft-bench dataset into planner or implementer mode.

Takes any craft-bench dataset directory and generates a derived dataset with
modified instructions and scoring for the iterative planning pipeline.

Planner mode: agent explores repo, writes plan.md + plan.json, scored by
file/symbol recall against gold files extracted from solve.sh.

Implementer mode: agent receives a plan from a prior planner run, implements
the changes, scored by existing F2P tests.

Usage:
    python scripts/planning/craft_pipeline.py \\
        --source harbor-tasks/iterative-planning \\
        --mode planner \\
        --output /tmp/planner-dataset

    python scripts/planning/craft_pipeline.py \\
        --source harbor-tasks/iterative-planning \\
        --mode implementer \\
        --plans-dir jobs/planner-opus/ \\
        --output /tmp/impl-dataset

    python scripts/planning/craft_pipeline.py \\
        --source harbor-tasks/iterative-planning \\
        --mode planner \\
        --filter 'scrapy__*' \\
        --output /tmp/planning-only
"""

import argparse
import fnmatch
import json
import logging
import os
import re
import shutil
import stat
from pathlib import Path

logger = logging.getLogger(__name__)


PLANNER_INSTRUCTION_TEMPLATE = """\
You are working in the {repo_hint} repository. The source code is in `/repo/`.

Produce an implementation plan for the task below. Another engineer will \
implement your plan. Explore the codebase with your tools (read, grep, glob) \
but do not modify any files.

Write two files when done:

1. `/logs/agent/plan.md` - the implementation plan.
2. `/logs/agent/plan.json` - a structured summary:
```json
{{
  "files_to_touch": ["path/to/file.py", "..."],
  "symbols_to_modify": ["ClassName.method_name", "..."]
}}
```

## Task

{spec}
"""

IMPLEMENTER_INSTRUCTION_TEMPLATE = """\
Implement the following plan. The codebase is at `/repo/`.

# Plan

{plan_content}

---

# Original Task

{original_instruction}
"""

PLANNER_TEST_SH = """\
#!/bin/bash
mkdir -p /logs/verifier
python3 /tests/score.py
"""

PLANNER_SCORE_PY = r"""#!/usr/bin/env python3
import json
from pathlib import Path

log_dir = Path("/logs/verifier")
log_dir.mkdir(parents=True, exist_ok=True)
agent_dir = Path("/logs/agent")

plan_md = agent_dir / "plan.md"
plan_json = agent_dir / "plan.json"
metadata = json.load(open("/tests/metadata.json"))
gold_files = set(metadata.get("src_files", []))

results = {"plan_md_exists": plan_md.exists(), "plan_json_exists": plan_json.exists()}

if not plan_md.exists():
    print("No plan.md found")
    results["reward"] = 0.0
    results["error"] = "no_plan_md"
elif not plan_json.exists():
    print("No plan.json found")
    results["reward"] = 0.0
    results["error"] = "no_plan_json"
else:
    try:
        plan = json.loads(plan_json.read_text())
        planned_files = set(plan.get("files_to_touch", []))
    except (json.JSONDecodeError, KeyError) as e:
        print(f"Invalid plan.json: {e}")
        results["reward"] = 0.0
        results["error"] = f"invalid_json: {e}"
        planned_files = set()

    if planned_files and gold_files:
        file_matched = planned_files & gold_files
        file_recall = len(file_matched) / len(gold_files)
        results["planned_files"] = sorted(planned_files)
        results["gold_files"] = sorted(gold_files)
        results["matched_files"] = sorted(file_matched)
        results["file_recall"] = round(file_recall, 3)

    SKIP_SYMBOLS = {"__init__", "__repr__", "__str__", "__eq__", "__hash__",
                    "__new__", "__del__", "__enter__", "__exit__"}
    raw_planned = plan.get("symbols_to_modify", [])
    raw_gold = metadata.get("gold_symbols", [])
    planned_sym = set(s.split(".")[-1].lstrip("_") for s in raw_planned) - SKIP_SYMBOLS
    gold_sym = set(s.lstrip("_") for s in raw_gold) - SKIP_SYMBOLS
    if planned_sym and gold_sym:
        sym_matched = planned_sym & gold_sym
        sym_recall = len(sym_matched) / len(gold_sym)
        results["planned_symbols"] = sorted(raw_planned)
        results["gold_symbols"] = sorted(raw_gold)
        results["matched_symbols"] = sorted(sym_matched)
        results["symbol_recall"] = round(sym_recall, 3)

    results["reward"] = results.get("file_recall", 0.0)
    results["plan_text_length"] = len(plan_md.read_text()) if plan_md.exists() else 0

    print(f"File recall:   {results.get('file_recall', 0):.3f}")
    print(f"Symbol recall: {results.get('symbol_recall', 'N/A')}")
    print(f"Reward:        {results['reward']:.3f}")

(log_dir / "reward.txt").write_text(f"{results['reward']:.3f}")
json.dump(results, open(log_dir / "results.json", "w"), indent=2)
"""


def extract_gold_files_from_solve(solve_sh_path):
    """Parse solve.sh to extract source files the oracle modifies."""
    files = []
    text = solve_sh_path.read_text()
    for line in text.splitlines():
        line = line.strip()
        m = re.match(r"git checkout FETCH_HEAD -- (.+)", line)
        if m:
            files.extend(m.group(1).split())
        m = re.match(r"rm -f (.+)", line)
        if m:
            files.extend(m.group(1).split())
    return [f for f in files if not _is_test_path(f)]


def _is_test_path(path):
    lower = path.lower()
    return "test" in lower or "fixture" in lower


def extract_spec_from_instruction(instruction_path, strip_test_hints=False):
    """Extract the spec text from instruction.md, stripping the ## Environment footer."""
    text = instruction_path.read_text()
    env_marker = "## Environment"
    if env_marker in text:
        text = text[: text.index(env_marker)].rstrip()
    if strip_test_hints:
        lines = text.splitlines()
        lines = [ln for ln in lines if not re.match(r"^Run (?:tests|the test)", ln.strip())]
        text = "\n".join(lines).rstrip()
    return text


def guess_repo_from_dockerfile(dockerfile_path):
    """Try to extract the repo name from a git clone line in the Dockerfile."""
    text = dockerfile_path.read_text()
    m = re.search(r"git clone.*github\.com/([^/]+/[^/\s.]+)", text)
    if m:
        repo = m.group(1)
        if repo.endswith(".git"):
            repo = repo[:-4]
        return repo
    return "unknown"


def make_executable(path):
    st = os.stat(path)
    os.chmod(path, st.st_mode | stat.S_IEXEC)


def generate_planner_dataset(source_dir, output_dir, task_filter=None):
    """Transform a craft-bench dataset into planner mode."""
    output_dir.mkdir(parents=True, exist_ok=True)
    tasks_generated = []

    for task_dir in sorted(source_dir.iterdir()):
        if not task_dir.is_dir() or task_dir.name.startswith("."):
            continue
        if task_filter and not fnmatch.fnmatch(task_dir.name, task_filter):
            continue

        out_task = output_dir / task_dir.name
        out_task.mkdir(parents=True, exist_ok=True)

        env_src = task_dir / "environment"
        sol_src = task_dir / "solution"
        env_dst = out_task / "environment"
        sol_dst = out_task / "solution"

        if env_dst.exists():
            if env_dst.is_symlink():
                env_dst.unlink()
            else:
                shutil.rmtree(env_dst)
        if sol_dst.exists():
            if sol_dst.is_symlink():
                sol_dst.unlink()
            else:
                shutil.rmtree(sol_dst)

        env_dst.symlink_to(env_src.resolve())
        sol_dst.symlink_to(sol_src.resolve())

        shutil.copy2(task_dir / "task.toml", out_task / "task.toml")

        spec = extract_spec_from_instruction(task_dir / "instruction.md", strip_test_hints=True)
        dockerfile_path = task_dir / "environment" / "Dockerfile"
        repo_hint = guess_repo_from_dockerfile(dockerfile_path)

        (out_task / "instruction.md").write_text(
            PLANNER_INSTRUCTION_TEMPLATE.format(
                repo_hint=repo_hint,
                spec=spec,
            )
        )

        tests_dir = out_task / "tests"
        if tests_dir.exists():
            shutil.rmtree(tests_dir)
        tests_dir.mkdir()

        gold_files = extract_gold_files_from_solve(task_dir / "solution" / "solve.sh")
        gold_symbols = []

        for candidate in ["planning_tasks.json", "planning_tasks_bootstrapped.json"]:
            p = Path(__file__).parent / candidate
            if p.exists():
                planning_tasks_json = p
                break
        else:
            planning_tasks_json = None
        if planning_tasks_json:
            ptasks = json.loads(planning_tasks_json.read_text())
            for ptask in ptasks.values():
                if ptask["task_name"] == task_dir.name:
                    gold_files = ptask.get("src_files", gold_files)
                    gold_symbols = ptask.get("gold_symbols", [])
                    break

        metadata = {"src_files": gold_files, "gold_symbols": gold_symbols}
        (tests_dir / "metadata.json").write_text(json.dumps(metadata, indent=2))

        test_sh = tests_dir / "test.sh"
        test_sh.write_text(PLANNER_TEST_SH)
        make_executable(test_sh)

        (tests_dir / "score.py").write_text(PLANNER_SCORE_PY)

        tasks_generated.append(task_dir.name)
        logger.info("Planner: %s (%d gold files)", task_dir.name, len(gold_files))

    registry = [
        {
            "name": output_dir.name,
            "version": "1.0",
            "description": f"Planner mode ({len(tasks_generated)} tasks)",
            "metrics": [{"type": "mean"}],
            "tasks": [{"name": t, "path": t} for t in tasks_generated],
        }
    ]
    (output_dir / "registry.json").write_text(json.dumps(registry, indent=2) + "\n")

    return tasks_generated


def generate_implementer_dataset(source_dir, plans_dir, output_dir, task_filter=None):
    """Transform a craft-bench dataset into implementer mode."""
    output_dir.mkdir(parents=True, exist_ok=True)
    plans_dir = Path(plans_dir)
    tasks_generated = []
    tasks_skipped = []

    for task_dir in sorted(source_dir.iterdir()):
        if not task_dir.is_dir() or task_dir.name.startswith("."):
            continue
        if task_filter and not fnmatch.fnmatch(task_dir.name, task_filter):
            continue

        plan_path = None
        # Harbor appends a random suffix to task dirs (e.g., hugapi__hug-651__dH4hXAg)
        # Search for directories starting with the task name
        if plans_dir.is_dir():
            for candidate_dir in sorted(plans_dir.iterdir()):
                if not candidate_dir.is_dir():
                    continue
                if not candidate_dir.name.startswith(task_dir.name):
                    continue
                for subpath in ["agent/plan.md", "plan.md"]:
                    candidate = candidate_dir / subpath
                    if candidate.exists():
                        plan_path = candidate
                        break
                if plan_path:
                    break

        if plan_path is None:
            logger.warning("Skipping %s: no plan found in %s", task_dir.name, plans_dir)
            tasks_skipped.append(task_dir.name)
            continue

        plan_content = plan_path.read_text()

        out_task = output_dir / task_dir.name
        out_task.mkdir(parents=True, exist_ok=True)

        for subdir in ("environment", "solution", "tests"):
            src = task_dir / subdir
            dst = out_task / subdir
            if dst.exists():
                if dst.is_symlink():
                    dst.unlink()
                else:
                    shutil.rmtree(dst)
            dst.symlink_to(src.resolve())

        shutil.copy2(task_dir / "task.toml", out_task / "task.toml")

        original_instruction = (task_dir / "instruction.md").read_text()
        (out_task / "instruction.md").write_text(
            IMPLEMENTER_INSTRUCTION_TEMPLATE.format(
                plan_content=plan_content,
                original_instruction=original_instruction,
            )
        )

        tasks_generated.append(task_dir.name)
        logger.info("Implementer: %s (plan: %d chars)", task_dir.name, len(plan_content))

    registry = [
        {
            "name": output_dir.name,
            "version": "1.0",
            "description": f"Implementer mode ({len(tasks_generated)} tasks)",
            "metrics": [{"type": "mean"}],
            "tasks": [{"name": t, "path": t} for t in tasks_generated],
        }
    ]
    (output_dir / "registry.json").write_text(json.dumps(registry, indent=2) + "\n")

    if tasks_skipped:
        logger.warning("Skipped %d tasks (no plan): %s", len(tasks_skipped), tasks_skipped)

    return tasks_generated


def main():
    parser = argparse.ArgumentParser(description="Generate planner/implementer datasets.")
    parser.add_argument("--source", required=True, help="Source dataset directory")
    parser.add_argument("--mode", required=True, choices=["planner", "implementer"])
    parser.add_argument("--output", required=True, help="Output dataset directory")
    parser.add_argument("--plans-dir", help="Plans directory (implementer mode)")
    parser.add_argument("--filter", help="Glob filter for task names (e.g. 'scrapy__*')")
    args = parser.parse_args()

    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

    source_dir = Path(args.source)
    output_dir = Path(args.output)

    if args.mode == "planner":
        tasks = generate_planner_dataset(source_dir, output_dir, args.filter)
    elif args.mode == "implementer":
        if not args.plans_dir:
            logger.error("--plans-dir required for implementer mode")
            return 1
        tasks = generate_implementer_dataset(source_dir, args.plans_dir, output_dir, args.filter)

    print(f"\n{'=' * 60}")
    print(f"{len(tasks)} tasks generated in {output_dir}/")
    print(f"{'=' * 60}")


if __name__ == "__main__":
    main()