Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
18.2 KB
Raw Blame
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# rerun-tainted.sh — re-run tainted Harbor trials under a network firewall.
#
# Inputs: one or more Harbor run-group dirs (the parent of trial dirs, e.g.
# `data/extracted/jobs/v2-codex-gpt55/baseline-codex-craft-taskgen-v2-...`).
#
# What it does, per run-group dir:
#   1. Runs scripts/check_integrity.py to find trials that fetched upstream
#      source. Tainted trials get archived OUT of the run-group dir to
#      `archive/tainted/<original-relative-path>/` so the scorer doesn't
#      see them. The original directory layout is preserved under the
#      archive root for forensic inspection.
#   2. Reads run_manifest.json to recover the original launcher_argv
#      (tasks-dir, agent, model, backend, output-dir, ...).
#   3. Enables the docker firewall (scripts/docker-firewall.sh), launches
#      run-baselines.sh restricted to the tainted task IDs, waits for
#      harbor to finish, disables the firewall (in an EXIT trap).
#   4. Re-runs check_integrity.py on the new run-group dir. Any rerun
#      trial that is ALSO tainted also gets archived. One pass only —
#      no retry loop.
#   5. Prints a summary.
#
# Usage:
#   sudo ./scripts/rerun-tainted.sh <run-group-dir>...
#   ./scripts/rerun-tainted.sh --dry-run <run-group-dir>...
#
# Flags:
#   --dry-run                Print planned steps; do not modify anything
#                            and do not toggle the firewall. Safe on macOS.
#   --firewall-script PATH   Override default scripts/docker-firewall.sh.
#   -h, --help               This message.
#
# Env vars:
#   ARCHIVE_ROOT             Override the archive destination. Default:
#                            `archive/tainted` (relative to cwd). Tainted
#                            trial dirs are mv'd into this tree, preserving
#                            the original relative path.
#   CODEX_BINARY_PATH        Required when reruning codex tainted trials.
#                            Path to a tarball produced by
#                            scripts/build-codex-prebake.sh.
#
# Sudo policy: the wrapper does NOT call sudo internally. Operator wraps
# the entire invocation with sudo so all child commands run as root. If
# invoked without sudo (and not --dry-run), the wrapper aborts.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

DRY_RUN=0
FIREWALL_SCRIPT="$SCRIPT_DIR/docker-firewall.sh"
RUN_GROUP_DIRS=()

usage() {
    sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'
}

while [[ $# -gt 0 ]]; do
    case "$1" in
        --dry-run)         DRY_RUN=1; shift ;;
        --firewall-script) FIREWALL_SCRIPT="$2"; shift 2 ;;
        -h|--help)         usage; exit 0 ;;
        --)                shift; break ;;
        -*)                echo "ERROR: unknown flag: $1" >&2; usage >&2; exit 2 ;;
        *)                 RUN_GROUP_DIRS+=("$1"); shift ;;
    esac
done

if [[ ${#RUN_GROUP_DIRS[@]} -eq 0 ]]; then
    echo "ERROR: no run-group dirs given" >&2
    usage >&2
    exit 2
fi

# Sudo gate: must be root unless dry-run
if [[ "$DRY_RUN" -eq 0 && "$EUID" -ne 0 ]]; then
    echo "ERROR: re-run as root, e.g.:" >&2
    echo "  sudo $0 ${RUN_GROUP_DIRS[*]}" >&2
    echo "(or pass --dry-run for a no-op preview)" >&2
    exit 2
fi

# uv is typically installed in $SUDO_USER's ~/.local/bin, which sudo's
# default secure_path strips from PATH. Find and prepend it so child
# commands (uv run python ...) work without `sudo -E PATH="$PATH" ...`.
if ! command -v uv >/dev/null 2>&1; then
    candidate_dirs=()
    [[ -n "${SUDO_USER:-}" ]] && candidate_dirs+=("$(getent passwd "$SUDO_USER" 2>/dev/null | cut -d: -f6)/.local/bin")
    candidate_dirs+=("$HOME/.local/bin" "/usr/local/bin" "/opt/uv/bin")
    for d in "${candidate_dirs[@]}"; do
        if [[ -n "$d" && -x "$d/uv" ]]; then
            export PATH="$d:$PATH"
            break
        fi
    done
fi
if ! command -v uv >/dev/null 2>&1; then
    echo "ERROR: uv not found on PATH. Common fix:" >&2
    echo "  sudo env PATH=\"\$HOME/.local/bin:\$PATH\" $0 ${RUN_GROUP_DIRS[*]}" >&2
    exit 2
fi

# Firewall script must exist (or be valid in dry-run, but dry-run still
# wants to print the path).
if [[ ! -f "$FIREWALL_SCRIPT" ]]; then
    echo "ERROR: firewall script not found: $FIREWALL_SCRIPT" >&2
    echo "Use --firewall-script to point at an alternate path." >&2
    exit 2
fi

# Validate run-group dirs and ensure each has a run_manifest.json
for d in "${RUN_GROUP_DIRS[@]}"; do
    if [[ ! -d "$d" ]]; then
        echo "ERROR: not a directory: $d" >&2
        exit 2
    fi
    if [[ ! -f "$d/run_manifest.json" ]]; then
        echo "ERROR: $d has no run_manifest.json (cannot recover launcher args)" >&2
        exit 2
    fi
done

echo "rerun-tainted.sh"
echo "  dry_run:           $DRY_RUN"
echo "  firewall_script:   $FIREWALL_SCRIPT"
echo "  run_group_dirs:"
for d in "${RUN_GROUP_DIRS[@]}"; do
    echo "    $d"
done
echo ""

# ---------------------------------------------------------------------------
# Stage 1: integrity scan and archive
# ---------------------------------------------------------------------------

CSV_TMP="$(mktemp -t rerun-tainted-csv.XXXXXX)"
trap 'rm -f "$CSV_TMP"' EXIT

echo "[1/4] Running integrity check..."
if ! uv run python "$SCRIPT_DIR/check_integrity.py" \
        "${RUN_GROUP_DIRS[@]}" \
        --csv "$CSV_TMP" >/dev/null; then
    echo "ERROR: check_integrity.py failed" >&2
    exit 1
fi

# Parse the tainted rows and group by run-group dir
declare -A TAINTED_TASKS=()    # run_group_dir -> "task1 task2 task3"
declare -A TAINTED_TRIALS=()   # run_group_dir -> "trial_dir1 trial_dir2 ..."

while IFS=, read -r root trial_dir task agent resolved ws wf upstream_url_count any_web fetched_upstream first_url; do
    [[ "$fetched_upstream" == "1" ]] || continue
    # Find which input run-group dir this trial belongs to.
    # trial_dir is absolute or relative path; match against our inputs.
    for rg in "${RUN_GROUP_DIRS[@]}"; do
        rg_abs="$(cd "$rg" && pwd)"
        trial_abs="$(cd "$(dirname "$trial_dir")" 2>/dev/null && pwd || true)/$(basename "$trial_dir")"
        if [[ "$trial_abs" == "$rg_abs"/* ]]; then
            TAINTED_TASKS[$rg]+="$task "
            TAINTED_TRIALS[$rg]+="$trial_dir "
            break
        fi
    done
done < <(tail -n +2 "$CSV_TMP")  # skip header

n_tainted_total=0
for rg in "${!TAINTED_TRIALS[@]}"; do
    # Count words
    n=$(echo "${TAINTED_TRIALS[$rg]}" | wc -w | tr -d ' ')
    n_tainted_total=$((n_tainted_total + n))
done

if [[ $n_tainted_total -eq 0 ]]; then
    echo "  No tainted trials found across input run-group dirs. Done."
    exit 0
fi

echo "  Found $n_tainted_total tainted trial(s) across ${#TAINTED_TRIALS[@]} run-group(s)."

# Codex reruns require CODEX_BINARY_PATH to be set on the host (so the
# patched codex install template can extract the pre-baked tarball
# instead of bootstrapping NVM, which is blocked by the firewall). Warn
# loudly if any of the tainted trials are codex and CODEX_BINARY_PATH is
# missing — the rerun would otherwise fail at install time.
declare -i n_codex_pending=0
for rg in "${!TAINTED_TRIALS[@]}"; do
    manifest="$rg/run_manifest.json"
    agent_name=$(uv run python - "$manifest" <<'PY' 2>/dev/null || true
import json, sys
m = json.load(open(sys.argv[1]))
print(m.get('agent', {}).get('name') or '')
PY
)
    if [[ "$agent_name" == "codex" ]]; then
        n_in_rg=$(echo "${TAINTED_TRIALS[$rg]}" | wc -w | tr -d ' ')
        n_codex_pending=$((n_codex_pending + n_in_rg))
    fi
done
if [[ $n_codex_pending -gt 0 && -z "${CODEX_BINARY_PATH:-}" ]]; then
    echo "ERROR: $n_codex_pending of the tainted trial(s) are from codex," >&2
    echo "       but CODEX_BINARY_PATH is not set. Codex install under firewall" >&2
    echo "       requires a pre-baked node + codex tarball. Build one with:" >&2
    echo "         bash scripts/build-codex-prebake.sh /tmp/codex-prebake.tar.gz" >&2
    echo "       Then re-invoke as:" >&2
    echo "         sudo -E CODEX_BINARY_PATH=/tmp/codex-prebake.tar.gz $0 $*" >&2
    exit 2
fi
if [[ $n_codex_pending -gt 0 ]]; then
    echo "  CODEX_BINARY_PATH=${CODEX_BINARY_PATH} (will be uploaded for codex reruns)"
fi
echo ""

# ---------------------------------------------------------------------------
# Stage 2: archive originals
# ---------------------------------------------------------------------------
#
# Archives go OUTSIDE the input run-group dir so the scorer
# (scripts/summarize_baseline.py) doesn't need any awareness of them. The
# archive mirrors the original path under a top-level "archive/tainted/"
# tree, e.g.:
#     baselines/integrity-test/baseline-haiku-XXX/t2v3-task-yyy/
#  -> archive/tainted/baselines/integrity-test/baseline-haiku-XXX/t2v3-task-yyy/
ARCHIVE_ROOT="${ARCHIVE_ROOT:-archive/tainted}"
echo "[2/4] Archiving tainted trial dirs to $ARCHIVE_ROOT/..."
for rg in "${!TAINTED_TRIALS[@]}"; do
    for trial in ${TAINTED_TRIALS[$rg]}; do
        [[ -n "$trial" ]] || continue
        if [[ ! -d "$trial" ]]; then
            echo "  WARN: trial dir gone: $trial" >&2
            continue
        fi
        # Strip leading ./ if present, normalise to relative path
        rel="${trial#./}"
        archived="$ARCHIVE_ROOT/$rel"
        if [[ "$DRY_RUN" -eq 1 ]]; then
            echo "  (dry-run) would mv $trial -> $archived"
        else
            if [[ -e "$archived" ]]; then
                echo "  WARN: archive target exists, skipping: $archived" >&2
                continue
            fi
            mkdir -p "$(dirname "$archived")"
            mv "$trial" "$archived"
            echo "  archived: $trial -> $archived"
        fi
    done
done
echo ""

# ---------------------------------------------------------------------------
# Stage 3: enable firewall and rerun
# ---------------------------------------------------------------------------

# Track the new run-group dirs that get created so we can integrity-check
# them at the end.
NEW_RUN_GROUPS=()

if [[ "$DRY_RUN" -eq 1 ]]; then
    echo "[3/4] (dry-run) skipping firewall enable + rerun"
    for rg in "${!TAINTED_TASKS[@]}"; do
        tasks="${TAINTED_TASKS[$rg]}"
        echo "  would rerun: $rg with tasks: $tasks"
    done
    echo ""
    echo "[4/4] (dry-run) skipping post-rerun integrity check"
    echo ""
    echo "Dry-run complete. No filesystem changes were made beyond the"
    echo "tempfile $CSV_TMP (cleaned up on exit)."
    exit 0
fi

# Real run: enable firewall, register cleanup, rerun.
#
# Disable codex's server-side web_search tool for the rerun. Codex
# executes web_search at the inference gateway, NOT in the trial
# container, so an iptables firewall in the container can't block it.
# The patched harbor codex.py reads this env var on the host and adds
# `-c web_search="disabled"` to the codex CLI invocation (a runtime
# override that bypasses TOML config nesting). No-op for opencode and
# claude-code.
export CODEX_DISABLE_WEB_SEARCH=1

# Suppress .pyc generation while harbor runs under sudo. Without this,
# every harbor import path under .venv writes root-owned __pycache__
# entries that block subsequent non-root harbor reinstalls / patch
# resets. Harbor itself doesn't need bytecode caching for one-shot runs.
export PYTHONDONTWRITEBYTECODE=1

echo "[3/4] Enabling firewall..."
bash "$FIREWALL_SCRIPT" enable

cleanup_firewall() {
    echo ""
    echo "Cleanup: disabling firewall..."
    bash "$FIREWALL_SCRIPT" disable || echo "  WARN: firewall disable failed; check iptables manually" >&2
}

cleanup_docker_ownership() {
    # When sudo -E preserves HOME, docker buildx writes config to the
    # real user's $HOME/.docker as root. That breaks subsequent
    # `docker compose build` invocations by the real user. Chown back.
    if [[ -n "${SUDO_USER:-}" && -d "$HOME/.docker" ]]; then
        if chown -R "$SUDO_USER" "$HOME/.docker" 2>/dev/null; then
            echo "Cleanup: restored $HOME/.docker ownership to $SUDO_USER"
        fi
    fi
}

cleanup_pycache_ownership() {
    # PYTHONDONTWRITEBYTECODE=1 (set above) should prevent root-owned .pyc
    # files in .venv. As a defensive measure (in case some sub-process
    # unsets it), reclaim any root-owned __pycache__ entries that landed
    # under the project venv during this run.
    if [[ -n "${SUDO_USER:-}" && -d "$REPO_ROOT/.venv" ]]; then
        if find "$REPO_ROOT/.venv" -name __pycache__ -user root -exec chown -R "$SUDO_USER" {} + 2>/dev/null; then
            :
        fi
    fi
}

trap 'cleanup_firewall; cleanup_docker_ownership; cleanup_pycache_ownership; rm -f "$CSV_TMP"' EXIT

# Per run-group, recover launcher_argv and invoke run-baselines.sh.
# run-baselines.sh nohups harbor in the background; we capture the PID it
# prints and wait for it to exit before moving on.
for rg in "${!TAINTED_TASKS[@]}"; do
    manifest="$rg/run_manifest.json"
    echo ""
    echo "  Recovering launcher_argv from $manifest..."

    # Convert tasks list into a single glob pattern via task_name OR-glob;
    # harbor's --task-name accepts a glob, not a list. We launch one
    # baseline run per task to keep this simple and traceable.
    tasks_arr=( ${TAINTED_TASKS[$rg]} )
    # Deduplicate
    mapfile -t tasks_uniq < <(printf '%s\n' "${tasks_arr[@]}" | awk 'NF' | sort -u)

    for task in "${tasks_uniq[@]}"; do
        echo ""
        echo "  Rerunning task: $task"

        # Reconstruct args from launcher_argv. We need: --tasks-dir,
        # --agent, --model, --backend, --output-dir, --n-concurrent, etc.
        # We override --task-name to scope to one task and drop --n-tasks
        # if present. Output-dir we keep so the new run lands under the
        # same parent as the original (a fresh job-name dir will be picked
        # by run-baselines.sh).
        mapfile -t argv_args < <(uv run python - "$manifest" "$task" <<'PY'
import json, sys
manifest_path, task_name = sys.argv[1], sys.argv[2]
m = json.load(open(manifest_path))
argv = m.get('run', {}).get('launcher_argv') or []
# argv[0] is the script path; we'll drop it.
out = []
i = 1
skip_next = False
while i < len(argv):
    a = argv[i]
    if skip_next:
        skip_next = False
        i += 1
        continue
    if a in ('--task-name','--n-tasks','--exclude-task-name'):
        skip_next = True   # skip this and its value
        i += 1
        continue
    out.append(a)
    i += 1
out += ['--task-name', task_name]
for a in out:
    print(a)
PY
)

        echo "    invoking: scripts/run-baselines.sh ${argv_args[*]}"

        # Capture the PID printed by run-baselines.sh so we can wait.
        rb_log="$(mktemp -t rb-launch.XXXXXX)"
        if ! "$SCRIPT_DIR/run-baselines.sh" "${argv_args[@]}" 2>&1 | tee "$rb_log"; then
            echo "    WARN: run-baselines.sh launch failed for task $task" >&2
            rm -f "$rb_log"
            continue
        fi

        # Extract "Started PID NNNN" from output
        pid="$(grep -E '^Started PID ' "$rb_log" | awk '{print $3}' | tail -n1)"
        rm -f "$rb_log"
        if [[ -z "$pid" ]]; then
            echo "    WARN: could not parse PID from run-baselines.sh output for $task" >&2
            continue
        fi
        echo "    waiting for harbor PID $pid to finish..."
        # Poll until process exits
        while kill -0 "$pid" 2>/dev/null; do sleep 5; done
        echo "    harbor PID $pid finished."

        # Discover the new run-group dir. run-baselines.sh creates a
        # subdir under --output-dir matching `baseline-<agent>-<source>-<ts>`.
        # We'll match anything newer than 1 minute under the output-dir.
        out_dir=""
        for ((i=0; i<${#argv_args[@]}; i++)); do
            if [[ "${argv_args[$i]}" == "--output-dir" ]]; then
                out_dir="${argv_args[$((i+1))]}"
                break
            fi
        done
        if [[ -n "$out_dir" && -d "$out_dir" ]]; then
            # newest dir in $out_dir
            new_rg="$(ls -1dt "$out_dir"/*/ 2>/dev/null | head -n1)"
            if [[ -n "$new_rg" ]]; then
                NEW_RUN_GROUPS+=("$new_rg")
                echo "    new run-group: $new_rg"
            fi
        fi
    done
done

# ---------------------------------------------------------------------------
# Stage 4: integrity check on rerun output, archive any still-tainted
# ---------------------------------------------------------------------------
if [[ ${#NEW_RUN_GROUPS[@]} -gt 0 ]]; then
    echo ""
    echo "[4/4] Re-running integrity check on new run-group(s)..."
    if ! uv run python "$SCRIPT_DIR/check_integrity.py" \
            "${NEW_RUN_GROUPS[@]}" \
            --csv "$CSV_TMP" >/dev/null; then
        echo "  WARN: integrity recheck failed" >&2
    fi

    # Archive a rerun only if it BOTH attempted upstream fetch AND passed.
    # Tainted-but-failed reruns are kept: that's the firewall doing its job —
    # the agent tried to cheat, got blocked at the network layer, and failed
    # honestly. Tainted-and-passed reruns mean the firewall didn't block (or
    # the agent succeeded via memorization), so we drop them.
    n_still=0
    n_blocked=0
    while IFS=, read -r root trial_dir task agent resolved ws wf upstream_url_count any_web fetched_upstream first_url; do
        [[ "$fetched_upstream" == "1" ]] || continue
        [[ -d "$trial_dir" ]] || continue
        if [[ "$resolved" == "1" ]]; then
            rel="${trial_dir#./}"
            archived="$ARCHIVE_ROOT/$rel"
            mkdir -p "$(dirname "$archived")"
            mv "$trial_dir" "$archived"
            echo "  rerun cheated AND passed; archived: $trial_dir -> $archived"
            n_still=$((n_still+1))
        else
            echo "  rerun attempted upstream fetch but failed (firewall worked): $trial_dir (kept)"
            n_blocked=$((n_blocked+1))
        fi
    done < <(tail -n +2 "$CSV_TMP")
    echo "  Reruns cheated and passed (archived): $n_still"
    echo "  Reruns attempted-but-blocked-by-firewall (kept):  $n_blocked"
else
    echo ""
    echo "[4/4] No new run-group dirs found; skipping recheck."
fi

echo ""
echo "Done. Summary:"
echo "  tainted originals archived:                      $n_tainted_total"
echo "  reruns launched (run-groups):                    ${#NEW_RUN_GROUPS[@]}"
echo "  reruns that cheated AND passed (archived):       ${n_still:-0}"
echo "  reruns that attempted but were blocked (kept):   ${n_blocked:-0}"
echo ""
echo "Inspect: $(printf '%s ' "${NEW_RUN_GROUPS[@]}")"