Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Run the evaluate step multiple times and apply per-task majority votes."""
from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
from collections import Counter
from datetime import datetime
from pathlib import Path
from typing import Any
VERDICT_PRIORITY = ["accept", "reject", "ERROR", "error", "unknown"]
EVALUATE_STAGE_BY_VERDICT = {
"accept": "promising",
"reject": "rejected",
"ERROR": "evaluated",
"error": "rejected",
}
EVALUATE_ERA_STAGES = {"candidate", "evaluated", "promising", "rejected"}
def _jsonl_write(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w") as f:
for row in rows:
f.write(json.dumps(row, sort_keys=True) + "\n")
def _task_instance_id(task: dict[str, Any]) -> str:
candidate = task.get("candidate_data") or {}
source_meta = candidate.get("source_metadata") or {}
return str(source_meta.get("instance_id") or candidate.get("source_task_id") or "").strip()
def _matches_filters(task_id: str, task: dict[str, Any], args: argparse.Namespace) -> bool:
if args.task_id and task_id not in args.task_id and str(task.get("task_id") or "") not in args.task_id:
return False
if args.repo and str(task.get("repo") or "") != args.repo:
return False
if args.instance_id and _task_instance_id(task) != args.instance_id:
return False
return True
def _clear_trial_eval_fields(task: dict[str, Any]) -> None:
task["stage"] = "candidate"
task["eval_verdict"] = ""
task["eval_reason"] = ""
task["eval_instruction_sketch"] = ""
task["eval_verifier_notes"] = ""
task["in_progress_step"] = ""
task["iteration_log"] = [
entry for entry in task.get("iteration_log", []) if entry.get("step") != "evaluate"
]
usage = task.get("llm_usage")
if isinstance(usage, dict):
usage.pop("evaluate", None)
def prepare_trial_state(input_state: Path, output_state: Path, args: argparse.Namespace) -> list[str]:
state = json.loads(input_state.read_text())
tasks = state.get("tasks")
if not isinstance(tasks, dict):
raise ValueError(f"{input_state}: expected top-level 'tasks' dict")
selected: list[str] = []
for task_id, task in tasks.items():
if not isinstance(task, dict):
continue
task["in_progress_step"] = ""
if _matches_filters(task_id, task, args):
selected.append(task_id)
if args.limit:
selected = selected[: args.limit]
selected_set = set(selected)
for task_id, task in tasks.items():
if not isinstance(task, dict):
continue
if task_id in selected_set:
_clear_trial_eval_fields(task)
elif task.get("stage") == "candidate":
task["stage"] = "rejected"
task["eval_verdict"] = "skip"
task["eval_reason"] = "Skipped by run_evaluate_majority filter"
if not selected:
raise RuntimeError("no tasks matched the requested filters")
output_state.parent.mkdir(parents=True, exist_ok=True)
output_state.write_text(json.dumps(state, indent=2, default=str) + "\n")
return selected
def _latest_evaluate_usage(task: dict[str, Any]) -> dict[str, Any]:
usage = ((task.get("llm_usage") or {}).get("evaluate") or [])
if isinstance(usage, list) and usage:
latest = usage[-1]
if isinstance(latest, dict):
return latest
return {}
def extract_trial_rows(
state_path: Path,
selected_task_ids: list[str],
run_number: int,
) -> list[dict[str, Any]]:
state = json.loads(state_path.read_text())
tasks = state.get("tasks")
if not isinstance(tasks, dict):
raise ValueError(f"{state_path}: expected top-level 'tasks' dict")
rows: list[dict[str, Any]] = []
for task_id in selected_task_ids:
task = tasks.get(task_id)
if not isinstance(task, dict):
continue
rows.append(
{
"run_number": run_number,
"task_id": str(task.get("task_id") or task_id),
"source_task_id": _task_instance_id(task),
"repo": task.get("repo", ""),
"commit_sha": task.get("commit_sha", ""),
"stage": task.get("stage", ""),
"eval_verdict": task.get("eval_verdict", ""),
"eval_reason": task.get("eval_reason", ""),
"eval_instruction_sketch": task.get("eval_instruction_sketch", ""),
"eval_verifier_notes": task.get("eval_verifier_notes", ""),
"usage": _latest_evaluate_usage(task),
}
)
return rows
def _vote_key(row: dict[str, Any]) -> str:
for key in ("source_task_id", "task_id", "commit_sha"):
value = str(row.get(key) or "").strip()
if value:
return value
raise ValueError(f"evaluate row has no source_task_id/task_id/commit_sha: {row}")
def _vote_label(row: dict[str, Any]) -> str:
return str(row.get("eval_verdict") or "unknown").strip() or "unknown"
def _choose_from_tie(
votes: list[dict[str, Any]],
tied_labels: set[str],
tie_policy: str,
) -> dict[str, Any] | None:
if tie_policy == "keep-current":
return None
if tie_policy == "first":
return next(row for row in votes if _vote_label(row) in tied_labels)
if tie_policy == "priority":
for verdict in VERDICT_PRIORITY:
if verdict in tied_labels:
return next(row for row in votes if _vote_label(row) == verdict)
return next(row for row in votes if _vote_label(row) in tied_labels)
raise ValueError(f"unknown tie policy: {tie_policy}")
def aggregate_votes(
run_rows: list[list[dict[str, Any]]],
*,
tie_policy: str = "keep-current",
require_strict_majority: bool = True,
) -> list[dict[str, Any]]:
grouped: dict[str, list[dict[str, Any]]] = {}
key_order: dict[str, int] = {}
for rows in run_rows:
for row in rows:
key = _vote_key(row)
key_order.setdefault(key, len(key_order))
grouped.setdefault(key, []).append(row)
summaries: list[dict[str, Any]] = []
for key, votes in grouped.items():
counts = Counter(_vote_label(row) for row in votes)
top_count = max(counts.values())
top_labels = {label for label, count in counts.items() if count == top_count}
strict_majority = top_count > len(votes) / 2
tie = len(top_labels) > 1
if tie:
chosen = _choose_from_tie(votes, top_labels, tie_policy)
else:
chosen = next(row for row in votes if _vote_label(row) in top_labels)
if require_strict_majority and not strict_majority:
chosen = None
first = votes[0]
summary = {
"key": key,
"task_id": first.get("task_id", ""),
"source_task_id": first.get("source_task_id", ""),
"repo": first.get("repo", ""),
"commit_sha": first.get("commit_sha", ""),
"counts": dict(sorted(counts.items())),
"total_votes": len(votes),
"winner_count": top_count,
"strict_majority": strict_majority,
"tie": tie,
"applied": False,
"votes": votes,
}
if chosen is None:
summary.update(
{
"winner": "",
"eval_verdict": "",
"eval_reason": "",
"eval_instruction_sketch": "",
"eval_verifier_notes": "",
"chosen_run_number": 0,
}
)
else:
summary.update(
{
"winner": _vote_label(chosen),
"eval_verdict": chosen.get("eval_verdict", ""),
"eval_reason": chosen.get("eval_reason", ""),
"eval_instruction_sketch": chosen.get("eval_instruction_sketch", ""),
"eval_verifier_notes": chosen.get("eval_verifier_notes", ""),
"chosen_run_number": chosen.get("run_number", 0),
}
)
summaries.append(summary)
return sorted(summaries, key=lambda row: key_order[row["key"]])
def _task_key_values(task_id: str, task: dict[str, Any]) -> set[str]:
candidate = task.get("candidate_data") or {}
source_meta = candidate.get("source_metadata") or {}
values = {
task_id,
str(task.get("task_id") or ""),
str(task.get("commit_sha") or ""),
str(candidate.get("sha") or ""),
str(candidate.get("source_task_id") or ""),
str(source_meta.get("instance_id") or ""),
}
return {value for value in values if value}
def apply_majority_to_state(
state_path: Path,
summaries: list[dict[str, Any]],
*,
backup_path: Path | None = None,
force_stage: bool = False,
) -> int:
if backup_path is not None:
backup_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(state_path, backup_path)
state = json.loads(state_path.read_text())
tasks = state.get("tasks")
if not isinstance(tasks, dict):
raise ValueError(f"{state_path}: expected top-level 'tasks' dict")
task_index: dict[str, str] = {}
for task_id, task in tasks.items():
if isinstance(task, dict):
for value in _task_key_values(task_id, task):
task_index[value] = task_id
applied = 0
for summary in summaries:
verdict = str(summary.get("eval_verdict") or "")
if not verdict:
continue
task_id = task_index.get(str(summary.get("task_id") or ""))
task_id = task_id or task_index.get(str(summary.get("source_task_id") or ""))
task_id = task_id or task_index.get(str(summary.get("commit_sha") or ""))
task_id = task_id or task_index.get(str(summary.get("key") or ""))
if not task_id:
continue
task = tasks[task_id]
task["eval_verdict"] = verdict
task["eval_reason"] = str(summary.get("eval_reason") or "")
task["eval_instruction_sketch"] = str(summary.get("eval_instruction_sketch") or "")
task["eval_verifier_notes"] = str(summary.get("eval_verifier_notes") or "")
final_stage = EVALUATE_STAGE_BY_VERDICT.get(verdict, "rejected")
if force_stage or str(task.get("stage") or "") in EVALUATE_ERA_STAGES:
task["stage"] = final_stage
task.setdefault("iteration_log", []).append(
{
"timestamp": datetime.now().isoformat(),
"step": "evaluate_majority",
"verdict": verdict,
"reason": task["eval_reason"],
"counts": summary.get("counts", {}),
"total_votes": summary.get("total_votes", 0),
"strict_majority": summary.get("strict_majority", False),
"chosen_run_number": summary.get("chosen_run_number", 0),
}
)
summary["applied"] = True
summary["applied_task_id"] = task_id
applied += 1
state["last_updated"] = datetime.now().isoformat()
state_path.write_text(json.dumps(state, indent=2, default=str) + "\n")
return applied
def _toml_value(value: Any) -> str:
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, int | float):
return str(value)
text = str(value).replace("\\", "\\\\").replace('"', '\\"')
return f'"{text}"'
def _profile_path(args: argparse.Namespace, output_dir: Path) -> Path | None:
if args.profile:
return args.profile
state = json.loads(args.state_json.read_text())
profile_data = state.get("profile_data") or {}
if not isinstance(profile_data, dict) or not profile_data:
return None
profile_path = output_dir / "profile_from_state.toml"
lines = ["[profile]"]
for key, value in sorted(profile_data.items()):
if isinstance(value, str | int | float | bool):
lines.append(f"{key} = {_toml_value(value)}")
profile_path.write_text("\n".join(lines) + "\n")
return profile_path
def _default_output_dir(state_json: Path) -> Path:
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
return state_json.parent / f"evaluate-majority-{timestamp}"
def _evaluate_command(args: argparse.Namespace, state_json: Path, profile_path: Path | None) -> list[str]:
cmd = [
sys.executable,
"-m",
"craft_taskgen.pipeline",
"--resume",
str(state_json),
"--from-step",
"evaluate",
"--concurrency",
str(args.concurrency),
]
if profile_path is not None:
cmd.extend(["--profile", str(profile_path)])
return cmd
def _run_streamed(cmd: list[str], log_path: Path) -> None:
log_path.parent.mkdir(parents=True, exist_ok=True)
with log_path.open("w") as log:
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
)
assert process.stdout is not None
for line in process.stdout:
print(line, end="")
log.write(line)
rc = process.wait()
if rc != 0:
raise RuntimeError(f"evaluate run failed with exit code {rc}; see {log_path}")
def run_trials(args: argparse.Namespace) -> tuple[list[list[dict[str, Any]]], Path]:
output_dir = args.output_dir or _default_output_dir(args.state_json)
output_dir.mkdir(parents=True, exist_ok=True)
profile_path = _profile_path(args, output_dir)
run_rows: list[list[dict[str, Any]]] = []
selected_task_ids: list[str] | None = None
for run_number in range(1, args.runs + 1):
run_label = f"run_{run_number:02d}"
trial_state = output_dir / f"{run_label}_state.json"
trial_log = output_dir / f"{run_label}.log"
trial_rows_path = output_dir / f"{run_label}.jsonl"
selected = prepare_trial_state(args.state_json, trial_state, args)
if selected_task_ids is None:
selected_task_ids = selected
elif selected != selected_task_ids:
raise RuntimeError("selected task set changed between runs")
cmd = _evaluate_command(args, trial_state, profile_path)
print(f"\n=== Evaluate {run_number}/{args.runs}: {' '.join(cmd)} ===")
_run_streamed(cmd, trial_log)
rows = extract_trial_rows(trial_state, selected, run_number)
for row in rows:
row["run_state"] = str(trial_state)
_jsonl_write(trial_rows_path, rows)
run_rows.append(rows)
return run_rows, output_dir
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("state_json", type=Path, help="Pipeline state.json to re-evaluate")
parser.add_argument("-n", "--runs", type=int, default=5, help="Number of evaluate trials to run")
parser.add_argument(
"--concurrency",
type=int,
default=16,
help="Concurrency passed to each evaluate trial",
)
parser.add_argument("--profile", type=Path, default=None, help="Optional pipeline profile TOML")
parser.add_argument("--repo", default="", help="Filter by short repo name")
parser.add_argument("--instance-id", default="", help="Filter by source_task_id / instance_id")
parser.add_argument("--task-id", action="append", default=[], help="Filter by task_id; may be repeated")
parser.add_argument("--limit", type=int, default=0, help="Limit selected tasks after filters")
parser.add_argument("--output-dir", type=Path, default=None, help="Directory for per-run artifacts")
parser.add_argument(
"--summary-jsonl",
type=Path,
default=None,
help="Majority summary JSONL path (default: <output-dir>/majority.jsonl)",
)
parser.add_argument(
"--tie-policy",
choices=["keep-current", "first", "priority"],
default="keep-current",
help="How to handle tied top vote counts",
)
parser.add_argument(
"--allow-plurality",
action="store_true",
help="Apply the top non-tied verdict even if it is not a strict majority",
)
parser.add_argument(
"--force-stage",
action="store_true",
help="Update task stage even for tasks that have progressed beyond evaluate-era stages",
)
parser.add_argument(
"--no-apply",
action="store_true",
help="Only write trial outputs and majority summary; do not update state_json",
)
args = parser.parse_args()
if args.runs < 1:
print("ERROR: --runs must be >= 1", file=sys.stderr)
return 1
try:
run_rows, output_dir = run_trials(args)
summaries = aggregate_votes(
run_rows,
tie_policy=args.tie_policy,
require_strict_majority=not args.allow_plurality,
)
summary_jsonl = args.summary_jsonl or output_dir / "majority.jsonl"
applied = 0
if not args.no_apply:
backup_path = output_dir / f"{args.state_json.name}.before-evaluate-majority"
applied = apply_majority_to_state(
args.state_json,
summaries,
backup_path=backup_path,
force_stage=args.force_stage,
)
_jsonl_write(summary_jsonl, summaries)
except Exception as err:
print(f"ERROR: {err}", file=sys.stderr)
return 1
unresolved = sum(1 for row in summaries if not row.get("applied"))
print(f"\nWrote majority summary: {summary_jsonl}")
if args.no_apply:
print("State update skipped (--no-apply)")
else:
print(f"Applied majority results to {args.state_json}: {applied} task(s)")
backup_path = output_dir / (args.state_json.name + ".before-evaluate-majority")
print(f"Backup before final apply: {backup_path}")
if unresolved:
print(f"Unapplied/tied/no-majority tasks: {unresolved}")
return 0
if __name__ == "__main__":
raise SystemExit(main())