Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
20.8 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Pipeline configuration constants, path resolution, and state model."""

from __future__ import annotations

import json
import os
import re
from dataclasses import asdict, dataclass, field
from datetime import datetime
from enum import Enum
from pathlib import Path

# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------


_CLAUDE_VERSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+_-]*$")


def _resolve_claude_cmd(version: str) -> str:
    """Return pinned Claude binary if installed, else fall back to system 'claude'.

    The version string can come from a TOML profile, so validate it as a bare
    version token (no path separators, no '..') before building a filesystem
    path, preventing path traversal out of the versions directory (CWE-22).
    """
    if not _CLAUDE_VERSION_RE.match(version) or ".." in version:
        raise ValueError(f"Invalid claude_code_version: {version!r}")
    pinned = os.path.expanduser(f"~/.local/share/claude/versions/{version}")
    if os.path.isfile(pinned) and os.access(pinned, os.X_OK):
        return pinned
    return "claude"


CC_VERSION = "2.1.118"
CLAUDE_CMD = _resolve_claude_cmd(CC_VERSION)
DEFAULT_PERMISSION_MODE = "auto"
DEFAULT_MAX_TURNS = 100  # effectively uncapped; timeout is the real safety net
DEFAULT_TIMEOUT = 1200  # 20 min per LLM step
TASK_SUITE_DIR = "harbor-tasks/craft-tools-v4"
HAIKU_MODEL = "azure/anthropic/claude-haiku-4-5"
OPUS_MODEL = "azure/anthropic/claude-opus-4-6"
LLM_STEP_MODEL = "azure/anthropic/claude-opus-4-6"  # azure avoids bedrock context_management rejection
# Smoke-test agent — the Harbor agent trial that actually *solves* the task and
# produces the reward (the primary quality gate). Decoupled from the direct-API
# judge models above: the agent that attempts the task can differ from the
# Opus/GPT judges that triage its trial. Default is codex + GPT-5.5 (cross-agent
# from the Opus deep-dive judge, per the no-self-family-judging design).
# Gateway naming: codex routes via the NVIDIA gateway, whose slugs double the
# vendor segment (e.g. `openai/openai/gpt-5.5`). Reasoning tokens require the
# codex model catalog (see runner._filtered_codex_catalog).
SMOKE_AGENT = "codex"
SMOKE_MODEL = "openai/openai/gpt-5.5"
SMOKE_REASONING_EFFORT = "high"  # "" → fall back to baselines.reasoning_defaults
# Cross-family judge for build-time alignment audit (Opus generates → GPT judges).
# Paper story: no LLM judges output from its own family (arXiv 2410.21819
# self-preference bias mitigation). Defaults to GPT-5.4 via the NVIDIA gateway;
# per-profile override via llm_alignment_model TOML field.
LLM_ALIGNMENT_MODEL = "openai/us/azure/openai/gpt-5.4"
MAX_FIX_ATTEMPTS = 2  # gates retries in build, docker-fix, and classify-fix paths
# Max times a single task may loop back through the pipeline from triage into
# a new Build regeneration. Persists across pipeline iterations so the task
# can't bounce between triage and Build indefinitely. One regen empirically
# captures the conversions; additional regens add cost without recovery.
MAX_TRIAGE_REGENS = 1
# Number of independent build+alignment candidate loops run per task. Each
# loop is build → alignment → (rebuild on leaked/narrow_tests with feedback,
# up to MAX_BUILD_REGENS_PER_CANDIDATE times). The orchestrator picks a
# passing candidate uniformly at random; REJECTED only if all candidates
# fail. N=3 was empirically the highest-yield + lowest-cost setting on the
# rerun-accepts-v2 calibration cohort (Apr 25). Capped at 4.
BUILD_N_CANDIDATES = 3
# Number of retention retries on the alignment judge per evaluation.
# These are re-polls of the SAME instruction (judge-flakiness mitigation,
# accept-on-first-ok). α=1 disables retention bias entirely (single roll,
# no shielding for borderline-leaky verdicts). On Apr 25 calibration, α=1
# combined with N=3 + r=2 produced the best yield/cost. Capped at [1,5].
ALIGNMENT_MAX_RETRIES = 1
# Maximum alignment-feedback-driven Build regens per candidate. After the
# initial build's alignment evaluation, if verdict is leaked/narrow_tests
# the candidate can rebuild with the assessor's evidence in the prompt.
# r=2 means up to 2 such rebuilds per candidate. r=0 disables build-retry
# entirely. On Apr 25 calibration, r=2 paired with N=3 + α=1 was optimal.
# Capped at [0,3].
MAX_BUILD_REGENS_PER_CANDIDATE = 2
MAX_SMOKE_RETRIES = 2  # retry infra failures (not task failures)
DEFAULT_CONCURRENCY = 4
LLM_CONCURRENCY = 4  # parallel claude -p calls
DOCKER_CONCURRENCY = 2  # parallel Docker build/classify/oracle ops
SMOKE_CONCURRENCY = 2  # parallel Harbor smoke tests
MAX_PROMISING_PER_REPO = 3  # cap per-repo builds to maintain diversity
# Pre-execution difficulty prediction (REJECT_MAYBE / REJECT_BAND_B) was removed
# when evaluate moved to direct-API with a binary accept/reject verdict.


def _load_env() -> None:
    """Load .env file into os.environ (required for Harbor API keys)."""
    env_path = os.path.join(os.getcwd(), ".env")
    if not os.path.isfile(env_path):
        print("WARNING: No .env file found — Harbor runs may fail (missing API keys)")
        return
    with open(env_path) as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#") or "=" not in line:
                continue
            key, _, value = line.partition("=")
            key = key.strip()
            value = value.strip().strip("\"'")
            # Expand bare env-var references: ${VAR} or $VAR as the entire value
            if value.startswith("${") and value.endswith("}"):
                value = os.environ.get(value[2:-1], value)
            elif value.startswith("$") and value[1:].isidentifier():
                value = os.environ.get(value[1:], value)
            if key and key not in os.environ:
                os.environ[key] = value
    print(f"Loaded .env ({env_path})")


# ---------------------------------------------------------------------------
# Path resolution — locates reference docs and templates relative to package
# ---------------------------------------------------------------------------

_PACKAGE_ROOT = Path(__file__).resolve().parent.parent.parent


@dataclass
class PipelineContext:
    """Resolved paths for reference docs, templates, and output directories.

    Auto-resolves relative to the craft-taskgen repo root by default.
    All paths are absolute strings for embedding in prompts.
    """

    references_dir: str = ""
    templates_dir: str = ""
    task_suite_dir: str = ""
    craft_bench_dir: str = ""

    def __post_init__(self) -> None:
        if not self.references_dir:
            self.references_dir = str(_PACKAGE_ROOT / "references")
        if not self.templates_dir:
            self.templates_dir = str(_PACKAGE_ROOT / "templates")
        if not self.task_suite_dir:
            self.task_suite_dir = TASK_SUITE_DIR

    @property
    def task_building_guide(self) -> str:
        return os.path.join(self.references_dir, "task-building-guide.md")

    @property
    def craft_bench_skills(self) -> str:
        return os.path.join(self.craft_bench_dir, ".claude", "skills")

    @property
    def template_task_dir(self) -> str:
        return os.path.join(self.templates_dir, "t2v3-CE0266-celery-retry-unification")

    @property
    def instruction_template(self) -> str:
        return os.path.join(self.templates_dir, "structural_template_instruction.md")

    @property
    def instruction_preamble(self) -> str:
        """First line of the instruction structural template — the standard task preamble."""
        path = self.instruction_template
        try:
            with open(path) as f:
                line = f.readline().strip()
        except FileNotFoundError:
            raise RuntimeError(
                f"Instruction template not found: {path}\n"
                "Ensure templates/structural_template_instruction.md exists in the craft-taskgen repo."
            ) from None
        except OSError as e:
            raise RuntimeError(f"Failed to read instruction template {path}: {e}") from e
        if not line:
            raise RuntimeError(f"Instruction template first line is blank: {path}")
        return line


# ---------------------------------------------------------------------------
# Pipeline profile — configurable parameters, loadable from TOML
# ---------------------------------------------------------------------------


@dataclass
class PipelineProfile:
    """Tunable pipeline parameters. Defaults match the module-level constants.

    Load from TOML for named, reproducible configurations. When applied,
    updates the module-level constants so all existing code reads the right values.
    """

    opus_model: str = OPUS_MODEL
    haiku_model: str = HAIKU_MODEL
    llm_step_model: str = LLM_STEP_MODEL
    llm_alignment_model: str = LLM_ALIGNMENT_MODEL
    smoke_agent: str = SMOKE_AGENT
    smoke_model: str = SMOKE_MODEL
    smoke_reasoning_effort: str = SMOKE_REASONING_EFFORT
    claude_code_version: str = CC_VERSION
    max_fix_attempts: int = MAX_FIX_ATTEMPTS
    max_triage_regens: int = MAX_TRIAGE_REGENS
    build_n_candidates: int = BUILD_N_CANDIDATES
    alignment_max_retries: int = ALIGNMENT_MAX_RETRIES
    max_build_regens_per_candidate: int = MAX_BUILD_REGENS_PER_CANDIDATE
    max_smoke_retries: int = MAX_SMOKE_RETRIES
    max_promising_per_repo: int = MAX_PROMISING_PER_REPO
    default_timeout: int = DEFAULT_TIMEOUT
    default_concurrency: int = DEFAULT_CONCURRENCY
    llm_concurrency: int = LLM_CONCURRENCY
    docker_concurrency: int = DOCKER_CONCURRENCY
    smoke_concurrency: int = SMOKE_CONCURRENCY
    task_suite_dir: str = TASK_SUITE_DIR
    skip_smoke: bool = False
    # task.toml resource/timeout values — written mechanically by assemble_task_dir_artifacts
    task_verifier_timeout: int = 600
    task_agent_timeout: int = 3600
    task_build_timeout: int = 900
    task_cpus: int = 2
    task_memory_mb: int = 4096
    task_storage_mb: int = 10240
    task_gpus: int = 0
    task_allow_internet: bool = True

    @classmethod
    def from_toml(cls, path: str) -> PipelineProfile:
        """Load profile from a TOML file. Unknown keys are ignored."""
        import tomllib

        with open(path, "rb") as f:
            data = tomllib.load(f)
        flat: dict = {}
        for section in data.values():
            if isinstance(section, dict):
                flat.update(section)
            else:
                continue
        # Map TOML keys to dataclass field names
        known = {f.name for f in cls.__dataclass_fields__.values()}
        return cls(**{k: v for k, v in flat.items() if k in known})

    def apply(self) -> None:
        """Update module-level constants from this profile. Call once at startup."""
        import craft_taskgen.config as cfg

        cfg.OPUS_MODEL = self.opus_model
        cfg.HAIKU_MODEL = self.haiku_model
        cfg.LLM_STEP_MODEL = self.llm_step_model
        cfg.LLM_ALIGNMENT_MODEL = self.llm_alignment_model
        cfg.SMOKE_AGENT = self.smoke_agent
        cfg.SMOKE_MODEL = self.smoke_model
        cfg.SMOKE_REASONING_EFFORT = self.smoke_reasoning_effort
        cfg.CC_VERSION = self.claude_code_version
        cfg.CLAUDE_CMD = _resolve_claude_cmd(self.claude_code_version)
        cfg.MAX_FIX_ATTEMPTS = self.max_fix_attempts
        cfg.MAX_TRIAGE_REGENS = self.max_triage_regens
        n = self.build_n_candidates
        if n < 1:
            print(f"WARNING: build_n_candidates={n} < 1, clamping to 1")
            n = 1
        elif n > 4:
            print(f"WARNING: build_n_candidates={n} > 4, clamping to 4 (cost guardrail)")
            n = 4
        cfg.BUILD_N_CANDIDATES = n
        a = self.alignment_max_retries
        if a < 1:
            print(f"WARNING: alignment_max_retries={a} < 1, clamping to 1")
            a = 1
        elif a > 5:
            print(f"WARNING: alignment_max_retries={a} > 5, clamping to 5 (cost guardrail)")
            a = 5
        cfg.ALIGNMENT_MAX_RETRIES = a
        r = self.max_build_regens_per_candidate
        if r < 0:
            print(f"WARNING: max_build_regens_per_candidate={r} < 0, clamping to 0")
            r = 0
        elif r > 3:
            print(f"WARNING: max_build_regens_per_candidate={r} > 3, clamping to 3 (cost guardrail)")
            r = 3
        cfg.MAX_BUILD_REGENS_PER_CANDIDATE = r
        cfg.MAX_SMOKE_RETRIES = self.max_smoke_retries
        cfg.MAX_PROMISING_PER_REPO = self.max_promising_per_repo
        cfg.DEFAULT_TIMEOUT = self.default_timeout
        cfg.DEFAULT_CONCURRENCY = self.default_concurrency
        cfg.LLM_CONCURRENCY = self.llm_concurrency
        cfg.DOCKER_CONCURRENCY = self.docker_concurrency
        cfg.SMOKE_CONCURRENCY = self.smoke_concurrency
        cfg.TASK_SUITE_DIR = self.task_suite_dir

    def to_dict(self) -> dict:
        """Serialize for embedding in state.json."""
        return asdict(self)


# Valid --from-step values. Only "select" (index 0) and "evaluate" (index 1)
# change pipeline behavior; everything from "build" onward is handled by the
# per-task stage machine in run_task_pipeline and all resume to the same place.
# The later entries are kept for backward-compat and operator ergonomics —
# someone typing `--from-step oracle` after a resume should still work.
STEP_ORDER = [
    "select",
    "evaluate",
    "build",
    "alignment",
    "assemble_artifacts",
    "build_dockerfile",
    "docker_classify",
    "oracle",
    "smoke",
    "opus_triage",
    "report",
]


# ---------------------------------------------------------------------------
# State model
# ---------------------------------------------------------------------------


class Stage(str, Enum):
    CANDIDATE = "candidate"
    EVALUATED = "evaluated"
    PROMISING = "promising"
    BUILT = "built"
    ALIGNMENT_CHECKED = "alignment_checked"
    TESTS_DISCOVERED = "tests_discovered"
    DOCKERFILE_BUILT = "dockerfile_built"
    F2P_P2P_CLASSIFIED = "f2p_p2p_classified"

    ORACLE_CHECKED = "oracle_checked"
    OPUS_SMOKE_TESTED = "opus_smoke_tested"
    OPUS_TRIAGED = "opus_triaged"
    ACCEPTED = "accepted"
    NEEDS_FIX = "needs_fix"
    REJECTED = "rejected"


@dataclass
class TaskState:
    task_id: str
    repo: str
    commit_sha: str  # merge commit SHA
    description: str
    base_sha: str  # pre-merge base branch HEAD (pr.base.sha from GitHub PR API)
    merge_base_sha: str  # git merge-base base_sha sha — used for Docker clone and patch application
    stage: Stage = Stage.CANDIDATE

    eval_verdict: str = ""
    eval_reason: str = ""
    eval_instruction_sketch: str = ""
    eval_verifier_notes: str = ""  # removed from EVALUATE_SCHEMA; kept for state.json back-compat

    # Per-step LLM call accounting populated by direct-API judges. Key = step
    # name ("evaluate", "build", "alignment", "deep_dive_opus",
    # "fairness_review", "summary"), value = list of {tokens_in,
    # tokens_out, tokens_cached, model, latency_s} per call. Supports
    # cost/latency reporting in dashboard + status.
    llm_usage: dict[str, list[dict]] = field(default_factory=dict)

    task_dir: str = ""
    instruction_words: int = 0

    # Alignment-judge verdict: ok / vague / narrow_tests / leaked / misaligned.
    # Retention-biased 3× retry — attempts list stores each retry's raw verdict
    # so humans can audit disagreement. Task is accepted if any attempt said ok.
    alignment_verdict: str = ""
    alignment_reason: str = ""
    alignment_v4_audit: dict[str, bool] = field(default_factory=dict)
    alignment_attempts: list[dict] = field(default_factory=list)
    # When alignment rejects with `leaked` or `narrow_tests`, its specific
    # evidence quotes are concatenated into this field and fed back to Build
    # for a single regen attempt (loop bounded at 1 per task).
    alignment_feedback: str = ""
    alignment_regen_count: int = 0
    # Per-candidate loser summaries from the parallel build+alignment
    # orchestrator. One entry per non-winning candidate when N>1; empty
    # when N=1 or for tasks whose winner was the only passer. Each dict:
    # {cand_id, outcome, verdict, short_reason (≤500 chars),
    #  regen_count, instruction_words}. Audit-only — no programmatic
    # consumer downstream.
    build_align_losers: list[dict] = field(default_factory=list)

    # F2P/P2P classification results
    f2p_tests: list[str] = field(default_factory=list)
    p2p_tests: list[str] = field(default_factory=list)

    # Oracle check (hard gate — blocks pipeline if not resolved)
    oracle_resolved: bool = False
    oracle_f2p_score: float = 0.0
    oracle_p2p_score: float = 0.0
    oracle_flagged: bool = False
    oracle_flag_reason: str = ""

    opus_score: str = ""
    opus_trial_dir: str = ""

    issues: list[dict] = field(default_factory=list)
    needs_human_review: bool = False
    human_review_reason: str = ""

    # Triage bookkeeping (overwritten each triage pass). `dd_*` counts
    # come from the Opus per-test skip/keep judge; `reviewer_*` fields
    # come from the cross-family fairness-review step.
    # `reviewer_concern_flag` is advisory (task still accepts); Build
    # regen fires only when `reviewer_concern_severity=major` with both
    # evidence fields populated.
    dd_failure_count: int = 0
    dd_dropped_by_skip_filter: int = 0
    dd_dropped_by_reward_filter: int = 0

    reviewer_concern_flag: bool = False
    reviewer_concern_severity: str = ""  # "none", "minor", "major", or ""
    reviewer_concern_reason: str = ""
    reviewer_concern_evidence_quote: str = ""
    reviewer_concern_evidence_test: str = ""

    # Deterministic easiness check (no LLM) — see steps.py::_deterministic_easiness
    # and docs/reference/easiness-heuristics.md. Fires on reward==1.0 when the agent
    # made <= 5 Grep/Read calls on the full (un-tailed) tool sequence.
    # Routes to needs_human_review; never auto-rejects. `easiness_concern`
    # stays as a kept-for-state.json-compatibility boolean no longer
    # populated (was set by the now-removed skeptical reviewer).
    easiness_flag: bool = False
    easiness_reason: str = ""
    easiness_concern: bool = False

    summary: str = ""  # narrative summary generated after pipeline completes

    fix_attempts: int = 0
    fix_history: list[str] = field(default_factory=list)
    # Count of Build regenerations triggered from triage feedback. Persists
    # across pipeline iterations so a task can't bounce between triage and
    # Build indefinitely. Capped by _cfg.MAX_TRIAGE_REGENS.
    triage_regen_count: int = 0
    # Set by triage when a fix agent runs; consumed by run_task_pipeline to route
    # the retry without re-diffing files a second time. "" means no pending fix.
    pending_fix_type: str = ""

    # Per-iteration log: [{iteration, opus_score, issues, fix_applied, timestamp}]
    iteration_log: list[dict] = field(default_factory=list)

    # What step is currently running (empty = idle). Set at step entry, cleared on completion.
    in_progress_step: str = ""

    # Raw candidate dict from the miner (sha, score, source_files, etc.). Written to
    # candidate.json in the task dir during assemble. Empty for tasks loaded from old state.json.
    candidate_data: dict = field(default_factory=dict)


@dataclass
class PipelineState:
    created: str = ""
    last_updated: str = ""
    run_dir: str = ""  # isolated output directory for this pipeline run
    profile_data: dict = field(default_factory=dict)  # profile snapshot for reproducibility
    # Host/input metadata — lets us identify which machine processed which candidates
    # when a large run is split across multiple machines.
    run_info: dict = field(default_factory=dict)
    tasks: dict[str, TaskState] = field(default_factory=dict)

    def save(self, path: str) -> None:
        self.last_updated = datetime.now().isoformat()
        with open(path, "w") as f:
            json.dump(asdict(self), f, indent=2, default=str)

    @classmethod
    def load(cls, path: str) -> PipelineState:
        with open(path) as f:
            data = json.load(f)
        state = cls(
            created=data["created"],
            last_updated=data["last_updated"],
            run_dir=data.get("run_dir", ""),
            profile_data=data.get("profile_data", {}),
            run_info=data.get("run_info", {}),
        )
        known_fields = {f.name for f in TaskState.__dataclass_fields__.values()}
        # Map removed stage values to their replacements for backward compat
        _stage_migration = {"docker_validated": "oracle_checked"}
        for tid, tdata in data.get("tasks", {}).items():
            raw_stage = tdata["stage"]
            tdata["stage"] = Stage(_stage_migration.get(raw_stage, raw_stage))
            # Filter out unknown fields for forward/backward compatibility
            filtered = {k: v for k, v in tdata.items() if k in known_fields}
            state.tasks[tid] = TaskState(**filtered)
        return state