Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
37.7 KB
Raw Blame
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Launch a Harbor baseline run (one agent, one model) against a task dataset.
#
# Harbor itself walks the dataset and runs N trials in parallel
# (--n-concurrent). This launcher just sets up env vars, runs preflight,
# and nohups the single harbor invocation so it survives terminal exit.
#
# Usage:
#   scripts/run-baselines.sh \
#       --tasks-dir ~/projects/craft-bench/harbor-tasks/craft-taskgen-v1a \
#       --agent opencode \
#       --model aws/anthropic/bedrock-claude-haiku-4-5 \
#       --backend gateway \
#       --n-tasks 3
#
# For a local vLLM endpoint:
#   export VLLM_BASE_URL=http://localhost:8000/v1
#   export VLLM_API_KEY=EMPTY
#   scripts/run-baselines.sh \
#       --tasks-dir ~/projects/craft-bench/harbor-tasks/craft-taskgen-v1a \
#       --agent opencode \
#       --model qwen/Qwen3.5-397B-A17B-FP8 \
#       --backend vllm
#
# Exits immediately after spawning harbor. The PID, log path, and kill
# hint are printed. Artifacts land under <output-dir>/.

set -euo pipefail

TASKS_DIR=""
AGENT=""
MODEL=""
BACKEND="gateway"
OUTPUT_DIR=""
TASK_NAME=""           # glob forwarded to harbor's --include-task-name (v0.6.4 renamed)
EXCLUDE_TASK_NAME=""   # glob forwarded to --exclude-task-name
N_TASKS=0              # 0 = no limit
N_CONCURRENT=4
SKIP_PREFLIGHT=0
DRY_RUN=0
FORCE_BUILD=0          # 1 = pass --force-build to harbor (invalidate image cache)

usage() {
    cat <<'EOF'
Launch a Harbor baseline run (one agent, one model) against a task dataset.

Harbor walks the dataset and runs trials in parallel (--n-concurrent).
This launcher sets up env vars, runs preflight, and nohups the harbor
invocation so it survives terminal exit. Prints the PID, log path, and
kill hint, then exits.

Required:
  --tasks-dir PATH          Task dataset directory (e.g. harbor-tasks/craft-taskgen-v1a)
  --agent NAME              One of: claude-code, codex, opencode, openhands-sdk, qwen-coder, pi
  --model ID                Model ID passed to harbor --model (e.g. azure/anthropic/claude-opus-4-6)

Optional:
  --backend gateway|vllm    Inference backend (default: gateway)
  --output-dir PATH         Output dir for harbor job artifacts (default: baselines/<ts>)
  --task-name GLOB[,...]    Forwarded to harbor --include-task-name (include filter).
                            Accepts a comma-separated list; each entry becomes
                            its own --include-task-name flag.
  --exclude-task-name GLOB  Forwarded to harbor --exclude-task-name
  --n-tasks N               Forwarded to harbor --n-tasks (0 = no limit; default: 0)
  --n-concurrent N          Forwarded to harbor --n-concurrent (default: 4)
  --skip-preflight          Skip the preflight check
  --force-build             Pass --force-build to harbor so Docker layer cache
                            is invalidated and task images are rebuilt. Use
                            after Dockerfile edits (e.g. agent-version pins).
  --dry-run                 Print the harbor command without executing

Examples:
  Gateway:
    scripts/run-baselines.sh \
        --tasks-dir ~/projects/craft-bench/harbor-tasks/craft-taskgen-v1a \
        --agent opencode --model aws/anthropic/bedrock-claude-haiku-4-5 \
        --n-tasks 3

  vLLM (requires VLLM_BASE_URL in env):
    VLLM_BASE_URL=http://localhost:8000/v1 VLLM_API_KEY=EMPTY \
      scripts/run-baselines.sh \
          --tasks-dir ~/projects/craft-bench/harbor-tasks/craft-taskgen-v1a \
          --agent opencode --model qwen/Qwen3.5-397B-A17B-FP8 \
          --backend vllm
EOF
    exit "${1:-1}"
}

# Capture original argv (post-shell-quoting) before argparse consumes
# it, for run_manifest.launcher_argv.
ORIGINAL_ARGV_QUOTED=$(printf '%q ' "$@")

while [[ $# -gt 0 ]]; do
    case "$1" in
        --tasks-dir) TASKS_DIR="$2"; shift 2 ;;
        --agent) AGENT="$2"; shift 2 ;;
        --model) MODEL="$2"; shift 2 ;;
        --backend) BACKEND="$2"; shift 2 ;;
        --output-dir) OUTPUT_DIR="$2"; shift 2 ;;
        --task-name) TASK_NAME="$2"; shift 2 ;;
        --exclude-task-name) EXCLUDE_TASK_NAME="$2"; shift 2 ;;
        --n-tasks) N_TASKS="$2"; shift 2 ;;
        --n-concurrent) N_CONCURRENT="$2"; shift 2 ;;
        --skip-preflight) SKIP_PREFLIGHT=1; shift ;;
        --force-build) FORCE_BUILD=1; shift ;;
        --dry-run) DRY_RUN=1; shift ;;
        -h|--help) usage 0 ;;
        *) echo "Unknown arg: $1" >&2; usage 1 ;;
    esac
done

if [[ -z "$TASKS_DIR" || -z "$AGENT" ]]; then
    echo "ERROR: --tasks-dir and --agent are required" >&2
    usage 1
fi
# Validate --agent and classify it. IS_SANITY_AGENT gates the
# model/effort/cap/compaction wiring below — oracle and nop don't
# talk to an LLM, they just orchestrate harbor's own non-LLM agents
# which apply (or skip) the task's reference solution and run the
# verifier.
case "$AGENT" in
    claude-code|codex|opencode|openhands-sdk|qwen-coder|pi) IS_SANITY_AGENT=0 ;;
    oracle|nop) IS_SANITY_AGENT=1 ;;
    *)
        echo "ERROR: --agent must be claude-code, codex, opencode, openhands-sdk, qwen-coder, pi, oracle, or nop (got: $AGENT)" >&2
        exit 1
        ;;
esac
# --model is required for LLM-driven agents but irrelevant for sanity
# agents. If --model is given for sanity agents we drop it (with a
# warning) so the same launcher command shape works across agents in
# CI scripts.
if [[ "$IS_SANITY_AGENT" -eq 1 ]]; then
    if [[ -n "$MODEL" ]]; then
        echo "INFO: --model is ignored for --agent $AGENT (sanity check)" >&2
        MODEL=""
    fi
elif [[ -z "$MODEL" ]]; then
    echo "ERROR: --model is required for --agent $AGENT" >&2
    usage 1
fi
if [[ ! -d "$TASKS_DIR" ]]; then
    echo "ERROR: tasks dir not found: $TASKS_DIR" >&2
    exit 1
fi
if [[ "$BACKEND" != "gateway" && "$BACKEND" != "vllm" ]]; then
    echo "ERROR: --backend must be gateway or vllm (got: $BACKEND)" >&2
    exit 1
fi

# The `nvidia/` / `vllm/` prefixes are opencode-internal dispatch tokens
# (they tell opencode to load @ai-sdk/openai-compatible); they are not
# part of the model identity the endpoint sees. If a user passes either
# form on --model (legacy or explicit), strip it so every downstream
# consumer — preflight, reasoning_defaults lookup, sampling-family
# match, manifest — sees the canonical slug. The launcher re-adds the
# appropriate prefix when constructing harbor's --model (below).
if [[ "$AGENT" == "opencode" ]]; then
    # Strip the dispatch prefix only when it isn't actually the
    # canonical gateway vendor prefix. Gateway-vendor-prefixed slugs are
    # `aws/...`, `azure/...`, `openai/...` (Anthropic, Azure, etc.); the
    # `nvidia/...` and `vllm/...` here are opencode-internal dispatch
    # tokens that need stripping. nvidia-hosted models also start with
    # `nvidia/` (e.g. `nvidia/qwen/qwen3.6-35b-a3b`), so detect them by
    # the SECOND segment: if it's a known gateway-vendor prefix, the
    # leading `nvidia/` is a legacy dispatch token and we strip; if it
    # isn't (e.g. `nvidia/qwen`, `nvidia/zai-org`, `nvidia/nvidia`), the
    # whole slug IS the canonical name and must be preserved.
    if [[ "$MODEL" == nvidia/aws/* || "$MODEL" == nvidia/azure/* || "$MODEL" == nvidia/openai/* ]]; then
        MODEL="${MODEL#nvidia/}"
    elif [[ "$MODEL" == vllm/* ]]; then
        MODEL="${MODEL#vllm/}"
    fi
fi

# openhands-sdk: the SDK adapter passes self.model_name straight to LiteLLM,
# which interprets the first slug segment as a provider name (aws/, azure/,
# nvidia/) and demands provider-specific creds. To route through an
# OpenAI-compatible endpoint (NVIDIA gateway or local vLLM) we prepend
# `openai/` so LiteLLM treats the full slug as opaque against LLM_BASE_URL.
# Strip if the user already wrote it; we re-add below.
if [[ "$AGENT" == "openhands-sdk" && "$MODEL" == openai/* ]]; then
    MODEL="${MODEL#openai/}"
fi

# Harbor's opencode agent splits --model at the first `/` and uses the left
# segment as the provider key; the remainder is the model ID that actually
# goes on the wire. The patched opencode.py treats both `nvidia` and `vllm`
# as openai-compatible providers (different base URLs; same wire protocol).
HARBOR_MODEL="$MODEL"
if [[ "$AGENT" == "opencode" ]]; then
    if [[ "$BACKEND" == "gateway" ]]; then
        HARBOR_MODEL="nvidia/$MODEL"
    elif [[ "$BACKEND" == "vllm" ]]; then
        HARBOR_MODEL="vllm/$MODEL"
    fi
fi
if [[ "$AGENT" == "openhands-sdk" ]]; then
    HARBOR_MODEL="openai/$MODEL"
fi
# pi: harbor's patched pi.py recognizes `nvidia` as a custom
# openai-completions provider (writes per-model models.json at run time
# pointing at OPENAI_BASE_URL). The launcher prepends `nvidia/` as the
# pi-side dispatch token; pi.py strips it before sending on the wire,
# so the gateway sees the canonical slug.
if [[ "$AGENT" == "pi" && "$MODEL" == nvidia/* ]]; then
    # User wrote --model nvidia/...; we still need to prepend another
    # nvidia/ for pi's dispatch. Don't strip — just prepend below.
    :
fi
if [[ "$AGENT" == "pi" ]]; then
    HARBOR_MODEL="nvidia/$MODEL"
fi

TS=$(date +%Y%m%d-%H%M%S)
if [[ -z "$OUTPUT_DIR" ]]; then
    OUTPUT_DIR="baselines/$TS"
fi
mkdir -p "$OUTPUT_DIR"

# Load .env so we can forward ANTHROPIC_*/OPENAI_*/VLLM_* into agent env.
# Parse KEY=VALUE lines instead of `source`-ing the file: `source` executes
# .env as a shell script, so any command in it would run (CWE-94).
if [[ -f .env ]]; then
    while IFS= read -r _envline || [[ -n "$_envline" ]]; do
        [[ "$_envline" =~ ^[[:space:]]*(#|$) ]] && continue
        _envline="${_envline#"${_envline%%[![:space:]]*}"}"   # ltrim
        [[ "$_envline" == export\ * ]] && _envline="${_envline#export }"
        if [[ "$_envline" =~ ^([A-Za-z_][A-Za-z0-9_]*)=(.*)$ ]]; then
            _envkey="${BASH_REMATCH[1]}"
            _envval="${BASH_REMATCH[2]}"
            _envval="${_envval%\"}"; _envval="${_envval#\"}"     # strip paired "
            _envval="${_envval%\'}"; _envval="${_envval#\'}"     # strip paired '
            export "$_envkey=$_envval"
        fi
    done < .env
    unset _envline _envkey _envval
fi

# Pin the in-container LLM judge to the NVIDIA gateway independent of the
# agent's endpoint. Captured BEFORE the vllm branch rewrites OPENAI_*, so the
# task.toml verifier.env block (which reads ${JUDGE_BASE_URL}/${JUDGE_API_KEY})
# keeps pointing at the gateway even when the agent is routed to local vLLM.
# Explicit JUDGE_* overrides win; otherwise fall back to OPENAI_API_BASE /
# OPENAI_BASE_URL / OPENAI_API_KEY from .env.
export JUDGE_BASE_URL="${JUDGE_BASE_URL:-${OPENAI_API_BASE:-${OPENAI_BASE_URL:-}}}"
export JUDGE_API_KEY="${JUDGE_API_KEY:-${OPENAI_API_KEY:-}}"

# Codex only emits reasoning tokens when it recognizes the model slug as
# reasoning-capable. Gateway-routed slugs aren't in codex's built-in catalog,
# so point CODEX_MODEL_CATALOG_JSON at the per-repo catalog (harbor patch
# reads this env var and passes -c model_catalog_json=... to `codex exec`).
#
# Filter to a single-model catalog at launch time. Harbor's codex.py inlines
# the catalog file content into the `bash -c '<setup_command>'` argv on
# docker exec; the full multi-model catalog blows past Linux's 128 KB
# MAX_ARG_STRLEN per argv element (E2BIG: "Argument list too long"), even
# though total ARG_MAX is ~2 MB. A single-model catalog is ~30 KB.
#
# Explicit CODEX_MODEL_CATALOG_JSON override is honored verbatim.
if [[ "$AGENT" == "codex" ]]; then
    DEFAULT_CATALOG="$(cd "$(dirname "$0")/.." && pwd)/patches/codex-model-catalog.json"
    if [[ -z "${CODEX_MODEL_CATALOG_JSON:-}" ]]; then
        if [[ ! -f "$DEFAULT_CATALOG" ]]; then
            echo "ERROR: source catalog not found: $DEFAULT_CATALOG" >&2
            exit 1
        fi
        FILTERED_CATALOG="$OUTPUT_DIR/codex-model-catalog-${MODEL//\//-}.json"
        # Pass paths/slug as argv (read via sys.argv) using a quoted heredoc, so
        # a $MODEL or path containing quotes cannot break out of the Python
        # string literal and inject code (CWE-94/CWE-78).
        if ! python3 - "$DEFAULT_CATALOG" "$MODEL" "$FILTERED_CATALOG" <<'PY'
import json, sys
src_path, slug, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
src = json.load(open(src_path))
matches = [m for m in src.get('models', []) if m.get('slug') == slug]
if not matches:
    sys.stderr.write(f'ERROR: no entry for slug={slug!r} in {src_path}; '
                     'add a row to patches/codex-model-catalog.json before launching.\n')
    sys.exit(1)
json.dump({'models': matches}, open(out_path, 'w'), indent=2)
PY
        then
            exit 1
        fi
        export CODEX_MODEL_CATALOG_JSON="$FILTERED_CATALOG"
    elif [[ ! -f "$CODEX_MODEL_CATALOG_JSON" ]]; then
        echo "WARNING: CODEX_MODEL_CATALOG_JSON=$CODEX_MODEL_CATALOG_JSON not found;" >&2
        echo "         codex will run without reasoning (catalog lookup will fail)." >&2
    fi
fi

if [[ "$SKIP_PREFLIGHT" -eq 0 ]]; then
    echo "Running baseline preflight..."
    PREFLIGHT_ARGS=(--check-baselines --backend "$BACKEND" --agents "$AGENT")
    if [[ "$IS_SANITY_AGENT" -eq 0 ]]; then
        PREFLIGHT_ARGS+=(--model "$MODEL")
    fi
    if ! uv run craft-taskgen-preflight "${PREFLIGHT_ARGS[@]}"; then
        echo "" >&2
        echo "ABORT: preflight failed. Fix issues above or pass --skip-preflight." >&2
        exit 1
    fi

    # Every task.toml in the dataset must declare memory_mb >= 8192. Catches
    # datasets generated against an older default before harbor silently OOMs
    # a trial mid-run. Tasks may declare more than the floor (e.g. 16384 for
    # heavy builds); only values below the floor block.
    MIN_MEMORY_MB=8192
    mismatched=()
    while IFS= read -r toml; do
        actual=$(grep -E '^\s*memory_mb\s*=' "$toml" | head -1 | sed -E 's/.*=\s*([0-9]+).*/\1/')
        if [[ -z "$actual" ]] || (( actual < MIN_MEMORY_MB )); then
            mismatched+=("$(basename "$(dirname "$toml")"): memory_mb=${actual:-<missing>}")
        fi
    done < <(find "$TASKS_DIR" -mindepth 2 -maxdepth 2 -name task.toml)
    if [[ ${#mismatched[@]} -gt 0 ]]; then
        echo "" >&2
        echo "ABORT: ${#mismatched[@]} task.toml file(s) declare memory_mb < $MIN_MEMORY_MB:" >&2
        printf '  %s\n' "${mismatched[@]}" >&2
        echo "Regenerate the dataset (or fix the candidates) so every task.toml declares memory_mb >= $MIN_MEMORY_MB." >&2
        exit 1
    fi
    echo "OK: all task.toml files declare memory_mb >= $MIN_MEMORY_MB"
    echo ""
fi

AGENT_ENV_ARGS=()
_add_env() {
    AGENT_ENV_ARGS+=(--agent-env "$1=$2")
}

# Determinism envs. Applied unconditionally so in-container pytest is
# reproducible: PYTHONHASHSEED pins Python dict iteration order; LC_ALL
# pins locale-dependent string comparisons and sort. Recorded in the
# run manifest under the `determinism` section.
_add_env PYTHONHASHSEED 0
_add_env LC_ALL C.UTF-8

# Sanity agents (oracle/nop) don't speak to an LLM, so they don't need
# any of the gateway/vllm endpoint plumbing or the claude-code opt-outs.
if [[ "$IS_SANITY_AGENT" -eq 0 ]]; then
    if [[ "$BACKEND" == "gateway" ]]; then
        # Secret API keys are NOT passed via --agent-env: that lands on the
        # harbor process argv (visible to `ps`). `source .env` above already
        # exported ANTHROPIC_API_KEY / OPENAI_API_KEY into this process env, so
        # harbor inherits them in os.environ and every installed agent reads
        # them from there (opencode / claude-code read os.environ directly;
        # codex / openhands via _get_env). Only non-secret base URLs are
        # forwarded explicitly.
        _add_env ANTHROPIC_BASE_URL "${ANTHROPIC_BASE_URL:-}"
        _add_env OPENAI_BASE_URL "${OPENAI_API_BASE:-${OPENAI_BASE_URL:-}}"
        _add_env CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC 1
        _add_env CLAUDE_CODE_ATTRIBUTION_HEADER 0
        _add_env CLAUDE_CODE_ENABLE_TELEMETRY 0
        # Disable experimental betas so claude-code does not inject the
        # context_management beta field. Moved here from the harbor
        # claude_code.py patch (env vars reach the agent via --agent-env).
        _add_env CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS 1
        if [[ "$AGENT" == "openhands-sdk" ]]; then
            # SDK adapter reads LLM_API_KEY / LLM_BASE_URL directly (no provider
            # sniffing); ANTHROPIC_* / OPENAI_* are inert for it. Export the key
            # into os.environ (read via _get_env) instead of --agent-env so it
            # stays off the process argv; LLM_BASE_URL is not secret.
            export LLM_API_KEY="${OPENAI_API_KEY:-}"
            _add_env LLM_BASE_URL "${OPENAI_API_BASE:-${OPENAI_BASE_URL:-}}"
        fi
        if [[ "$AGENT" == "qwen-coder" ]]; then
            # Harbor's QwenCode agent reads OPENAI_MODEL inside the container
            # and falls back to "qwen3-coder-plus" if unset. Set it explicitly
            # to the gateway slug so we never silently drift to the default.
            _add_env OPENAI_MODEL "$MODEL"
        fi
        if [[ "$AGENT" == "pi" ]]; then
            # pi's harbor wrapper writes ~/.pi/agent/models.json at run-time
            # with baseUrl baked in from the host's OPENAI_BASE_URL; the
            # container needs OPENAI_API_KEY to authenticate. Forwarded by
            # the gateway branch above already, no extra plumbing needed.
            :
        fi
    else
        if [[ -z "${VLLM_BASE_URL:-}" ]]; then
            echo "ERROR: --backend vllm requires VLLM_BASE_URL in env/.env" >&2
            exit 1
        fi
        # Rewrite localhost / 127.0.0.1 → host.docker.internal so the container
        # can reach a vLLM running on the host. The host-gateway compose
        # override (added to CMD below) maps host.docker.internal → host-gateway,
        # which works on macOS Docker Desktop and on Linux.
        CONTAINER_BASE_URL="${VLLM_BASE_URL//localhost/host.docker.internal}"
        CONTAINER_BASE_URL="${CONTAINER_BASE_URL//127.0.0.1/host.docker.internal}"
        _add_env OPENAI_BASE_URL "$CONTAINER_BASE_URL"
        _add_env OPENAI_API_KEY "${VLLM_API_KEY:-EMPTY}"
        # Map host.docker.internal → host-gateway via harbor's native
        # --extra-docker-compose (replaces the old docker-compose-base.yaml
        # patch, which no longer exists as of harbor a56546f).
        VLLM_NEEDS_HOST_GATEWAY=1
        if [[ "$AGENT" == "openhands-sdk" ]]; then
            _add_env LLM_API_KEY "${VLLM_API_KEY:-EMPTY}"
            _add_env LLM_BASE_URL "$CONTAINER_BASE_URL"
        fi
        if [[ "$AGENT" == "qwen-coder" ]]; then
            _add_env OPENAI_MODEL "$MODEL"
        fi
    fi
fi

if [[ -n "${OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX:-}" ]]; then
    _add_env OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX "$OPENCODE_EXPERIMENTAL_OUTPUT_TOKEN_MAX"
fi

SUITE_NAME="$(basename "$TASKS_DIR")"
JOB_NAME="baseline-${AGENT}-${SUITE_NAME}-${TS}"
LOG="$OUTPUT_DIR/${JOB_NAME}.log"

# Pin the agent CLI version so runs are reproducible. Source of truth is
# src/craft_taskgen/adapters/_docker.py — bump there, pick up here for free.
# Sanity agents (oracle/nop) are pure harbor built-ins with no CLI
# version to pin.
AGENT_VERSION=""
if [[ "$IS_SANITY_AGENT" -eq 0 ]]; then
    AGENT_VERSION=$(uv run python -c "
import sys
from craft_taskgen.adapters._docker import (
    CLAUDE_CODE_VERSION, CODEX_VERSION, OPENCODE_VERSION, OPENHANDS_SDK_VERSION,
    QWEN_CODE_VERSION, PI_VERSION,
)
versions = {
    'claude-code':   CLAUDE_CODE_VERSION,
    'codex':         CODEX_VERSION,
    'opencode':      OPENCODE_VERSION,
    'openhands-sdk': OPENHANDS_SDK_VERSION,
    'qwen-coder':    QWEN_CODE_VERSION,
    'pi':            PI_VERSION,
}
agent = '$AGENT'
if agent not in versions:
    print(f'no pinned version for agent={agent!r} (known: {sorted(versions)})', file=sys.stderr)
    sys.exit(1)
print(versions[agent])
") || exit 1
fi

# Resolve reasoning_effort for this (agent, model) combo. Source of truth:
# src/craft_taskgen/baselines/reasoning_defaults.py. Env-var REASONING_EFFORT_OVERRIDE
# bypasses the table — set to empty string to disable pass-through entirely.
# Sanity agents (oracle/nop) don't reason; leave EFFORT/REASONING_* empty.
EFFORT=""
REASONING_SOURCE=""
REASONING_NOTES=""
if [[ "$IS_SANITY_AGENT" -eq 0 ]]; then
    EFFORT=$(uv run python - "$AGENT" "$MODEL" <<'PY'
import sys
from craft_taskgen.baselines.reasoning_defaults import effort_for
e = effort_for(sys.argv[1], sys.argv[2])
if e:
    print(e)
PY
)
    REASONING_SOURCE="reasoning_defaults"
    if [[ -n "${REASONING_EFFORT_OVERRIDE+x}" ]]; then
        EFFORT="$REASONING_EFFORT_OVERRIDE"
        REASONING_SOURCE="REASONING_EFFORT_OVERRIDE"
    fi
fi

# Per-agent wiring that is ALWAYS applied — independent of effort so the
# manifest values always match what the container received. Effort-
# dependent wiring lives in the second block below.
case "$AGENT" in
    claude-code)
        # Pin auto-compaction threshold explicitly (default per docs is
        # ~95%). Values above default have no effect, so this is the
        # closest-to-off setting Anthropic exposes. Always on — unrelated
        # to whether we're also setting effort.
        # https://code.claude.com/docs/en/env-vars
        _add_env CLAUDE_AUTOCOMPACT_PCT_OVERRIDE 95
        ;;
esac

# Agent-specific effort wiring — without these, effort is silently
# dropped. Only runs when EFFORT is actually set for this (agent, model).
if [[ -n "$EFFORT" ]]; then
    case "$AGENT" in
        codex)
            # Catalog env is already exported earlier in this script (see the
            # CODEX_MODEL_CATALOG_JSON block near the top). Surface its value
            # in the summary so the operator can confirm what's loaded.
            REASONING_NOTES="catalog=${CODEX_MODEL_CATALOG_JSON:-<unset>}"
            ;;
        claude-code)
            # Gateway-routed opus/sonnet drops effort silently without this
            # capability declaration. See
            # https://code.claude.com/docs/en/model-config.
            _CAPS="effort,thinking,adaptive_thinking,interleaved_thinking"
            _add_env ANTHROPIC_DEFAULT_OPUS_MODEL_SUPPORTED_CAPABILITIES "$_CAPS"
            _add_env ANTHROPIC_DEFAULT_SONNET_MODEL_SUPPORTED_CAPABILITIES "$_CAPS"
            # CLAUDE_CODE_EFFORT_LEVEL is the documented env var
            # (https://code.claude.com/docs/en/env-vars) and takes precedence
            # over --effort / the effortLevel setting. Set it alongside the
            # --agent-kwarg reasoning_effort (which harbor renders as
            # --effort) so the effort flows even on CLI versions where the
            # flag name drifts.
            _add_env CLAUDE_CODE_EFFORT_LEVEL "$EFFORT"
            REASONING_NOTES="capabilities=$_CAPS; CLAUDE_CODE_EFFORT_LEVEL=$EFFORT"
            ;;
        opencode)
            # Harbor's opencode patch reads OPENCODE_REASONING_EFFORT on the
            # HOST when building the install.sh (os.environ.get inside
            # .venv/.../harbor/agents/installed/opencode.py), so we export it
            # rather than passing --agent-env (which forwards to the container
            # where the config has already been generated).
            export OPENCODE_REASONING_EFFORT="$EFFORT"
            REASONING_NOTES="OPENCODE_REASONING_EFFORT=$EFFORT (host-exported)"
            ;;
        openhands-sdk)
            # SDK adapter doesn't take reasoning_effort as a kwarg. Reasoning
            # for models like Nemotron-Ultra is configured at the endpoint
            # (vLLM reasoning-parser plugin). Effort here is a no-op.
            REASONING_NOTES="openhands-sdk: no effort kwarg (reasoning is endpoint-side)"
            ;;
        pi)
            # Pi exposes effort as --thinking {off,minimal,low,medium,high,xhigh}
            # (CLI_FLAGS in harbor/agents/installed/pi.py). Harbor surfaces it as
            # the `thinking` agent-kwarg, not `reasoning_effort`. So we rename
            # the kwarg below by setting PI_THINKING and emptying EFFORT so the
            # generic `reasoning_effort=$EFFORT` kwarg isn't appended.
            PI_THINKING="$EFFORT"
            EFFORT=""
            REASONING_NOTES="pi --thinking=$PI_THINKING (mapped from reasoning_defaults)"
            ;;
    esac
fi

# Qwen3 family on opencode: reasoning_effort is a no-op; Qwen3 uses
# server-side enable_thinking (default true with --reasoning-parser qwen3)
# and the HF model card recommends specific sampling parameters for
# thinking mode that are NOT opencode's defaults. Set them here so the
# paper can cite paper-recommended settings (Qwen3 tech report §3.3 +
# Qwen/Qwen3-32B HF card: temperature=0.6, top_p=0.95, top_k=20).
# Harbor reads OPENCODE_BUILD_{TEMPERATURE,TOP_P,TOP_K} on the host and
# writes them into opencode.json agent.build.* (opencode.py:374-391).
# Apply opencode-family sampling defaults per model card. Each entry is
# (family_label, match_test, T, p, k) where match_test is a bash
# pattern tested against "$MODEL". Empty `k` means the model card is
# silent on top_k — don't force a value.
#
# Sources:
#  - Qwen3: T=0.6 p=0.95 k=20    (Qwen3-32B HF card)
#  - MiniMax: T=1.0 p=0.95 k=40  (MiniMax-M2.5 HF card)
#  - Nemotron-Super-V3: T=1.0 p=0.95 — (nemotron-super-v3 model card
#    says "across all tasks", no top_k). Narrow match avoids Ultra/Nano
#    and the older Llama-3.3-Nemotron-Super-49B-v1.
#  - GLM (4.5+): T=1.0 p=0.95 — z.ai docs + HF generation_config.json.
#    top_k unspecified by model card.
apply_opencode_family_sampling() {
    local family="$1" T="$2" p="$3" k="$4"
    export OPENCODE_BUILD_TEMPERATURE="${OPENCODE_BUILD_TEMPERATURE:-$T}"
    export OPENCODE_BUILD_TOP_P="${OPENCODE_BUILD_TOP_P:-$p}"
    if [[ -n "$k" ]]; then
        export OPENCODE_BUILD_TOP_K="${OPENCODE_BUILD_TOP_K:-$k}"
        REASONING_NOTES="${REASONING_NOTES:+$REASONING_NOTES; }${family} sampling params T=${T} p=${p} k=${k}"
    else
        REASONING_NOTES="${REASONING_NOTES:+$REASONING_NOTES; }${family} sampling params T=${T} p=${p}"
    fi
}

if [[ "$AGENT" == "opencode" ]]; then
    if [[ "$MODEL" == *[Qq]wen* ]]; then
        apply_opencode_family_sampling "Qwen3" 0.6 0.95 20
        if [[ -n "${OPENCODE_ENABLE_THINKING:-}" ]]; then
            _add_env OPENCODE_ENABLE_THINKING "$OPENCODE_ENABLE_THINKING"
            REASONING_NOTES="OPENCODE_ENABLE_THINKING=$OPENCODE_ENABLE_THINKING (Qwen3 path, T=0.6 p=0.95 k=20)"
        fi
    elif [[ "$MODEL" == *[Mm]ini[Mm]ax* ]]; then
        apply_opencode_family_sampling "MiniMax" 1.0 0.95 40
    elif [[ "$MODEL" == *[Nn]emotron*[Ss]uper*[Vv]3* || "$MODEL" == *[Nn]emotron*3*[Ss]uper* || "$MODEL" == *[Nn]emotron-v3* ]]; then
        apply_opencode_family_sampling "Nemotron-Super-V3" 1.0 0.95 ""
    elif [[ "$MODEL" == *[Gg][Ll][Mm]* ]]; then
        apply_opencode_family_sampling "GLM" 1.0 0.95 ""
    fi
fi

# Global per-call output-token cap. Source of truth:
# src/craft_taskgen/baselines/output_cap.py. A cap, not a tuning knob —
# no override env. Applied per agent via whatever env the agent honors.
# Sanity agents (oracle/nop) don't generate output and have no cap to
# apply.
OUTPUT_CAP=0
OUTPUT_CAP_NOTES="N/A (sanity agent has no model output)"
if [[ "$IS_SANITY_AGENT" -eq 0 ]]; then
    OUTPUT_CAP=$(uv run python -c "from craft_taskgen.baselines import OUTPUT_TOKEN_CAP; print(OUTPUT_TOKEN_CAP)")
    case "$AGENT" in
        claude-code)
            # CLAUDE_CODE_MAX_OUTPUT_TOKENS (documented at code.claude.com/docs/en/env-vars)
            # is forwarded into the container by harbor (claude_code.py:870-871).
            _add_env CLAUDE_CODE_MAX_OUTPUT_TOKENS "$OUTPUT_CAP"
            OUTPUT_CAP_NOTES="CLAUDE_CODE_MAX_OUTPUT_TOKENS=$OUTPUT_CAP"
            ;;
        opencode)
            # Harbor's opencode agent reads OPENCODE_{BUILD,PLAN}_MAX_TOKENS on the
            # host at config-generation time (opencode.py:374-391) and writes them
            # into opencode.json under agent.{build,plan}.max_tokens.
            export OPENCODE_BUILD_MAX_TOKENS="$OUTPUT_CAP"
            export OPENCODE_PLAN_MAX_TOKENS="$OUTPUT_CAP"
            OUTPUT_CAP_NOTES="OPENCODE_{BUILD,PLAN}_MAX_TOKENS=$OUTPUT_CAP (host-exported)"
            ;;
        codex)
            # Codex's model_max_output_tokens config key is parsed but never
            # applied — openai/codex#4138. No functional cap today.
            OUTPUT_CAP_NOTES="UNCAPPED (openai/codex#4138)"
            ;;
        openhands-sdk)
            # SDK adapter has no max_tokens knob. Per the Ultra evaluator
            # handoff, max_tokens / max_completion_tokens are intentionally
            # dropped server-side (proxy interceptor) so generations aren't
            # capped below the model's max_output_tokens.
            OUTPUT_CAP_NOTES="UNCAPPED at agent (drop max_tokens server-side; see Ultra handoff)"
            ;;
        qwen-coder)
            # Qwen Code CLI honors a `maxOutputTokens` setting in ~/.qwen/settings.json
            # but harbor's install template doesn't plumb it. Runs uncapped.
            OUTPUT_CAP_NOTES="UNCAPPED at agent (no harbor plumbing for ~/.qwen/settings.json)"
            ;;
        pi)
            # Pi has no env var for per-call max_tokens; models.json's
            # maxTokensField only names the request field. Runs uncapped.
            OUTPUT_CAP_NOTES="UNCAPPED at agent (no env var for max_tokens)"
            ;;
    esac
fi

CMD=(
    uv run harbor run
    --path "$TASKS_DIR"
    --agent "$AGENT"
    --jobs-dir "$OUTPUT_DIR"
    --job-name "$JOB_NAME"
    --n-concurrent "$N_CONCURRENT"
    --yes
)
# --model and --agent-kwarg version=... only apply to LLM-driven agents.
# Harbor's oracle/nop agents take no model and have no CLI version.
if [[ -n "$HARBOR_MODEL" ]]; then
    CMD+=(--model "$HARBOR_MODEL")
fi
if [[ -n "$AGENT_VERSION" ]]; then
    CMD+=(--agent-kwarg "version=$AGENT_VERSION")
fi
if [[ -n "$EFFORT" ]]; then
    CMD+=(--agent-kwarg "reasoning_effort=$EFFORT")
fi
if [[ -n "${PI_THINKING:-}" ]]; then
    # pi-specific: --agent-kwarg thinking=<level> (see CLI_FLAGS in pi.py).
    CMD+=(--agent-kwarg "thinking=$PI_THINKING")
fi
# OpenHands SDK iteration cap. Matches the Ultra evaluator-sidecar pin (200).
if [[ "$AGENT" == "openhands-sdk" ]]; then
    CMD+=(--agent-kwarg "max_iterations=$(uv run python -c 'from craft_taskgen.adapters._docker import OPENHANDS_SDK_MAX_ITERATIONS; print(OPENHANDS_SDK_MAX_ITERATIONS)')")
fi
# Per-agent turn cap. Claude Code exposes max_turns via a harbor
# CliFlag (claude_code.py CLI_FLAGS); codex and opencode do not expose
# an equivalent knob — wall-clock timeout is their only cap. Document
# the asymmetry rather than pretending we control it.
MAX_TURNS=""
# Per-agent tool-restriction policy. Surfaced in the manifest via
# agent.disallowed_tools.
DISALLOWED_TOOLS=""
case "$AGENT" in
    claude-code)
        MAX_TURNS=250
        CMD+=(--agent-kwarg "max_turns=$MAX_TURNS")
        # Disable EnterPlanMode + ExitPlanMode by default. Haiku 4.5
        # self-selects into plan mode on ~50% of tasks, explores
        # read-only for 100+ turns, then treats "present plan" as
        # terminal — the trial exits with 0 code edits because
        # Write/Edit are blocked during plan mode and there's no
        # interactive user to approve ExitPlanMode. Repro:
        # plan-on produced 0 edits over 4+ trials; plan-off produced
        # 3-9 edits per trial within the first 50 turns. Mirrors
        # runner.py's smoke-test policy. Override for ablation
        # experiments via DISABLE_PLAN_MODE=0. Harbor's claude_code.py
        # already exposes this as a CliFlag, so no harbor patch needed.
        if [[ "${DISABLE_PLAN_MODE:-1}" == "1" ]]; then
            DISALLOWED_TOOLS="EnterPlanMode,ExitPlanMode"
            CMD+=(--agent-kwarg "disallowed_tools=$DISALLOWED_TOOLS")
        fi
        ;;
esac
if [[ "$FORCE_BUILD" -eq 1 ]]; then
    CMD+=(--force-build)
fi
if [[ "$N_TASKS" -gt 0 ]]; then
    CMD+=(--n-tasks "$N_TASKS")
fi
if [[ -n "$TASK_NAME" ]]; then
    # Allow comma-separated list; expand into one --include-task-name per entry.
    IFS=',' read -ra _TASK_NAMES <<< "$TASK_NAME"
    for _t in "${_TASK_NAMES[@]}"; do
        CMD+=(--include-task-name "$_t")
    done
fi
if [[ -n "$EXCLUDE_TASK_NAME" ]]; then
    CMD+=(--exclude-task-name "$EXCLUDE_TASK_NAME")
fi
if [[ "${VLLM_NEEDS_HOST_GATEWAY:-0}" -eq 1 ]]; then
    # vLLM-on-host reachability: map host.docker.internal → host-gateway via
    # harbor's native --extra-docker-compose (was a docker-compose-base.yaml patch
    # before the a56546f bump). See patches/compose-overrides/host-gateway.yaml.
    CMD+=(--extra-docker-compose "$(cd "$(dirname "$0")/.." && pwd)/patches/compose-overrides/host-gateway.yaml")
fi
CMD+=("${AGENT_ENV_ARGS[@]}")

echo "Launching baseline"
echo "  tasks_dir:    $TASKS_DIR"
if [[ -n "$AGENT_VERSION" ]]; then
    echo "  agent:        $AGENT (pinned @$AGENT_VERSION)"
else
    echo "  agent:        $AGENT (sanity check; no CLI version)"
fi
if [[ -n "$MODEL" ]]; then
    echo "  model:        $MODEL"
fi
if [[ "$HARBOR_MODEL" != "$MODEL" && -n "$HARBOR_MODEL" ]]; then
    case "$AGENT" in
        opencode)      _label="opencode provider dispatch" ;;
        openhands-sdk) _label="openai/ prefix for LiteLLM chat-completions" ;;
        *)             _label="provider prefix" ;;
    esac
    echo "  harbor --model: $HARBOR_MODEL ($_label)"
fi
echo "  backend:      $BACKEND"
if [[ -n "$EFFORT" ]]; then
    echo "  reasoning:    $EFFORT (source: $REASONING_SOURCE; $REASONING_NOTES)"
elif [[ -n "$REASONING_NOTES" ]]; then
    # No effort set, but a sampling-family note fired (Qwen3/MiniMax/Nemotron).
    echo "  reasoning:    unset; $REASONING_NOTES"
else
    echo "  reasoning:    unset (no row in reasoning_defaults for this agent+model)"
fi
if [[ "$IS_SANITY_AGENT" -eq 0 ]]; then
    echo "  output_cap:   $OUTPUT_CAP ($OUTPUT_CAP_NOTES)"
fi
if [[ "$AGENT" == "claude-code" ]]; then
    echo "  disallowed_tools: ${DISALLOWED_TOOLS:-<none> (DISABLE_PLAN_MODE=0)}"
fi
echo "  output_dir:   $OUTPUT_DIR"
echo "  n_concurrent: $N_CONCURRENT"
[[ "$N_TASKS" -gt 0 ]] && echo "  n_tasks:      $N_TASKS"
[[ -n "$TASK_NAME" ]] && echo "  task_name:    $TASK_NAME"
[[ -n "$EXCLUDE_TASK_NAME" ]] && echo "  exclude:      $EXCLUDE_TASK_NAME"
echo "  log:          $LOG"
echo ""

if [[ "$DRY_RUN" -eq 1 ]]; then
    # Redact API keys. Docs recommend dry-run for verification and
    # people paste output into MRs / Slack; leaking raw gateway keys
    # from that output is a real risk. Real launch path below
    # (nohup ${CMD[@]}) is unaffected — keys go into the container
    # via --agent-env with their original values.
    printf '%q ' "${CMD[@]}" | sed -E 's/(_API_KEY=)[^ ]+/\1<redacted>/g'
    echo
    exit 0
fi

# Write the per-job reproducibility manifest BEFORE spawning harbor so
# it's available even if the run crashes early. Harbor writes its own
# result.json next to this; the two files are the full reproduction
# package. Schema + CLI:
# src/craft_taskgen/baselines/run_manifest.py
MANIFEST_DIR="$OUTPUT_DIR/$JOB_NAME"
mkdir -p "$MANIFEST_DIR"
MANIFEST_PATH="$MANIFEST_DIR/run_manifest.json"

# When --backend vllm, also probe /v1/models so the manifest captures
# the served model name, root (HF path), and max_model_len.
MANIFEST_VLLM_ARGS=()
if [[ "$BACKEND" == "vllm" ]]; then
    MANIFEST_VLLM_ARGS+=(--vllm-probe --vllm-api-key "${VLLM_API_KEY:-EMPTY}")
fi

# Manifest base_url: the endpoint THIS agent actually talks to. vllm
# backend goes to VLLM_BASE_URL; claude-code talks to ANTHROPIC_BASE_URL;
# codex and opencode talk to OPENAI_BASE_URL (even through the NVIDIA
# gateway, since that's the route their SDKs hit). Using a single
# priority-ordered fallback would mis-record codex's endpoint as
# ANTHROPIC_BASE_URL whenever both are set.
if [[ "$BACKEND" == "vllm" ]]; then
    MANIFEST_BASE_URL="${VLLM_BASE_URL:-}"
elif [[ "$AGENT" == "claude-code" ]]; then
    MANIFEST_BASE_URL="${ANTHROPIC_BASE_URL:-}"
else
    MANIFEST_BASE_URL="${OPENAI_API_BASE:-${OPENAI_BASE_URL:-}}"
fi

uv run python -m craft_taskgen.baselines.run_manifest \
    --output          "$MANIFEST_PATH" \
    --tasks-dir       "$TASKS_DIR" \
    --agent           "$AGENT" \
    --agent-version   "$AGENT_VERSION" \
    --model           "$MODEL" \
    --backend         "$BACKEND" \
    --base-url        "$MANIFEST_BASE_URL" \
    --effort          "$EFFORT" \
    --reasoning-source "$REASONING_SOURCE" \
    --reasoning-notes "$REASONING_NOTES" \
    --output-cap      "$OUTPUT_CAP" \
    --output-cap-applied "$OUTPUT_CAP_NOTES" \
    --n-tasks         "$N_TASKS" \
    --n-concurrent    "$N_CONCURRENT" \
    --task-name       "$TASK_NAME" \
    --exclude-task-name "$EXCLUDE_TASK_NAME" \
    --launcher-argv   "$0 $ORIGINAL_ARGV_QUOTED" \
    --extra           'determinism.PYTHONHASHSEED="0"' \
    --extra           "determinism.LC_ALL=C.UTF-8" \
    --extra           "harness.max_turns=${MAX_TURNS:-null}" \
    --extra           "compaction.claude_code_pct_override=$([[ "$AGENT" == "claude-code" ]] && echo 95 || echo null)" \
    --extra           "compaction.opencode_auto=$([[ "$AGENT" == "opencode" ]] && echo false || echo null)" \
    --extra           "compaction.codex_auto_compact_token_limit_note=$([[ "$AGENT" == "codex" ]] && echo '\"uncapped (openai/codex#4138)\"' || echo null)" \
    --extra           "agent.harbor_model_arg=$(
        if [[ -n "$HARBOR_MODEL" ]]; then printf '"%s"' "$HARBOR_MODEL"; else echo null; fi
    )" \
    --extra           "agent.disallowed_tools=$(
        if [[ -n "$DISALLOWED_TOOLS" ]]; then printf '"%s"' "$DISALLOWED_TOOLS"; else echo null; fi
    )" \
    "${MANIFEST_VLLM_ARGS[@]}"
echo "  manifest:     $MANIFEST_PATH"
echo ""

# Wrap harbor so the manifest's outcomes field gets flipped from
# "predicted" to "present"/"missing" after harbor exits. Without this,
# outcomes.harbor_result_json points at a file that may never have
# existed (harbor crashed before writing result.json) and the reader
# has no way to tell from the manifest alone.
nohup bash -c '
    "$@"
    rc=$?
    uv run python -m craft_taskgen.baselines.run_manifest \
        --finalize "'"$MANIFEST_PATH"'" \
        --harbor-rc "$rc" || true
    exit "$rc"
' _ "${CMD[@]}" > "$LOG" 2>&1 &
PID=$!
disown $PID 2>/dev/null || true

echo "Started PID $PID"
echo ""
echo "Monitor with:"
echo "  tail -f $LOG"
echo ""
echo "Stop with:"
echo "  kill $PID"