The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(runner): restore direct live eval campaigns and reports (#12909)
Dottaaf8439a70b7a
diff --git a/.github/workflows/runner-live-evals.yml b/.github/workflows/runner-live-evals.ymlindex 978bdca02e..9f797ab89d 100644--- a/.github/workflows/runner-live-evals.yml+++ b/.github/workflows/runner-live-evals.yml@@ -4,6 +4,19 @@ on:schedule:- cron: "17 6 * * 0"workflow_dispatch:+ inputs:+ candidate:+ description: "Comma-separated live candidate IDs (for example codex-luna)"+ type: string+ required: false+ case:+ description: "Comma-separated workflow case IDs"+ type: string+ required: false+ limit:+ description: "Maximum executions after candidate/case filtering"+ type: string+ required: falseconcurrency:group: runner-live-evals-${{ github.ref }}@@ -17,6 +30,8 @@ jobs:timeout-minutes: 5permissions:contents: read+ outputs:+ eval_runner: ${{ steps.runner.outputs.runner }}steps:- name: Require default branch and allowlisted numeric actor IDsenv:@@ -50,11 +65,30 @@ jobs:fidone+ - name: Select paid eval runner+ id: runner+ env:+ AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}+ run: |+ set -euo pipefail+ github_runner='ubuntu-latest'+ aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'++ if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then+ echo "runner=$aws_runner" >> "$GITHUB_OUTPUT"+ echo '::notice title=Paid eval routing::Using an ephemeral RunsOn Fleet runner'+ else+ echo "runner=$github_runner" >> "$GITHUB_OUTPUT"+ echo '::notice title=Paid eval routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner'+ fi+live_matrix:name: Balanced provider/model matrixneeds: authorizeif: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true'- runs-on: ubuntu-latest+ # The authorize job selects one of two literal, reviewed runner labels;+ # dispatch inputs and repository variables cannot inject an arbitrary label.+ runs-on: ${{ needs.authorize.outputs.eval_runner }}timeout-minutes: 180permissions:contents: read@@ -81,6 +115,26 @@ jobs:- name: Checkout repositoryuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ - name: Generate private eval-repository token+ id: evals_token+ env:+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}+ GH_REPO: ${{ github.repository }}+ run: |+ set -euo pipefail+ token="$(node .github/scripts/get-bot-token.mjs)"+ echo "::add-mask::$token"+ echo "value=$token" >> "$GITHUB_OUTPUT"++ - name: Checkout canonical Evalbook reporter+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ repository: paperclipai/paperclip-evals+ ref: 0c8dfea0ef71a73e909b59a5c0484554cbee199b+ path: .paperclip-evals+ token: ${{ steps.evals_token.outputs.value }}+ persist-credentials: false+- name: Setup pnpmuses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6with:@@ -114,6 +168,10 @@ jobs:PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }}PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12"PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-week-v1+ PAPERCLIP_EVAL_CANDIDATE: ${{ inputs.candidate }}+ PAPERCLIP_EVAL_CASE: ${{ inputs.case }}+ PAPERCLIP_EVAL_LIMIT: ${{ inputs.limit }}+ PAPERCLIP_EVALBOOK_PROGRAM: ${{ github.workspace }}/.paperclip-evals/evals/paperclip-runner/tools/eval_program.pyrun: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals- name: Publish job summarydiff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.ymlnew file mode 100644index 0000000000..9f6352b033--- /dev/null+++ b/.github/workflows/runner-protocol-live-evals.yml@@ -0,0 +1,679 @@+name: Runner Direct Live Protocol Evals++on:+ schedule:+ - cron: "23 9 * * 0"+ workflow_dispatch:+ inputs:+ target_branch:+ description: "Branch in paperclipai/paperclip to evaluate; trusted orchestration still runs from master"+ type: string+ required: false+ evals_sha:+ description: "Exact 40-character paperclipai/paperclip-evals commit to execute"+ type: string+ required: false+ rosters:+ description: "Comma-separated live roster IDs/files, or all for the complete direct suite"+ type: string+ default: "all"+ required: false+ max_infrastructure_retries:+ description: "Automatic retries only for explicitly retryable infrastructure failures (0-3)"+ type: number+ default: 1++permissions:+ contents: read++concurrency:+ group: runner-protocol-live-evals-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }}+ cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}++jobs:+ authorize:+ name: Authorize paid direct eval campaign+ if: github.event_name != 'schedule' || vars.RUNNER_PROTOCOL_EVAL_NIGHTLY_ENABLED == 'true'+ runs-on: ubuntu-latest+ timeout-minutes: 5+ permissions:+ contents: read+ outputs:+ test_runner: ${{ steps.runner.outputs.runner }}+ max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }}+ max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }}+ target_sha: ${{ steps.target.outputs.sha }}+ target_ref: ${{ steps.target.outputs.ref }}+ evals_sha: ${{ steps.evals.outputs.sha }}+ steps:+ - name: Require default branch and allowlisted numeric actor IDs+ env:+ GH_TOKEN: ${{ github.token }}+ REPOSITORY: ${{ github.repository }}+ REF: ${{ github.ref }}+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}+ ACTOR: ${{ github.actor }}+ ACTOR_ID: ${{ github.actor_id }}+ TRIGGERING_ACTOR: ${{ github.triggering_actor }}+ ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}+ run: |+ set -euo pipefail+ if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then+ echo "Paid direct Runner eval campaigns may run only from the default branch." >&2+ exit 1+ fi+ if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then+ echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2+ exit 1+ fi+ triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"+ if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then+ echo "GitHub actor identity contexts disagree; refusing the paid run." >&2+ exit 1+ fi+ for candidate in "$triggering_actor_id" "$ACTOR_ID"; do+ if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then+ echo "The initiating GitHub account is not authorized to run paid Runner eval campaigns." >&2+ exit 1+ fi+ done++ - name: Resolve requested Paperclip branch to an immutable commit+ id: target+ env:+ GH_TOKEN: ${{ github.token }}+ REPOSITORY: ${{ github.repository }}+ TARGET_BRANCH: ${{ inputs.target_branch || github.event.repository.default_branch }}+ run: |+ set -euo pipefail+ if [ -z "$TARGET_BRANCH" ] || [[ "$TARGET_BRANCH" == refs/* ]]; then+ echo "target_branch must name a branch in this repository without a refs/ prefix." >&2+ exit 1+ fi+ encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')"+ target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)"+ [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]]+ echo "sha=$target_sha" >> "$GITHUB_OUTPUT"+ echo "ref=refs/heads/$TARGET_BRANCH" >> "$GITHUB_OUTPUT"++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ ref: ${{ github.sha }}+ persist-credentials: false++ - name: Generate private eval-repository token+ id: evals_token+ env:+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}+ GH_REPO: ${{ github.repository }}+ run: |+ set -euo pipefail+ token="$(node .github/scripts/get-bot-token.mjs)"+ echo "::add-mask::$token"+ echo "value=$token" >> "$GITHUB_OUTPUT"++ - name: Verify the private eval program is pinned to an exact commit+ id: evals+ env:+ GH_TOKEN: ${{ steps.evals_token.outputs.value }}+ EVALS_SHA: ${{ inputs.evals_sha || vars.RUNNER_PROTOCOL_EVALS_SHA }}+ run: |+ set -euo pipefail+ if ! [[ "$EVALS_SHA" =~ ^[0-9a-f]{40}$ ]]; then+ echo "evals_sha (or RUNNER_PROTOCOL_EVALS_SHA for schedules) must be an exact 40-character commit." >&2+ exit 1+ fi+ resolved="$(gh api -X GET "repos/paperclipai/paperclip-evals/commits/$EVALS_SHA" --jq .sha)"+ test "$resolved" = "$EVALS_SHA"+ echo "sha=$resolved" >> "$GITHUB_OUTPUT"++ - name: Validate retry envelope+ env:+ RETRIES: ${{ github.event_name == 'schedule' && 1 || inputs.max_infrastructure_retries }}+ run: |+ set -euo pipefail+ [[ "$RETRIES" =~ ^[0-3]$ ]]++ - name: Select paid test runner+ id: runner+ env:+ AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}+ run: |+ set -euo pipefail+ if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then+ {+ echo 'runner=runs-on/fleet=paperclip-public-pr-x64/env=public-ci'+ echo 'max_parallel_default=100'+ echo 'max_parallel_limit=100'+ } >> "$GITHUB_OUTPUT"+ else+ {+ echo 'runner=ubuntu-latest'+ echo 'max_parallel_default=32'+ echo 'max_parallel_limit=57'+ } >> "$GITHUB_OUTPUT"+ fi++ catalog:+ name: Pin and fan out the direct Evalbook roster+ needs: authorize+ runs-on: ubuntu-latest+ timeout-minutes: 10+ permissions:+ contents: read+ outputs:+ matrix_0: ${{ steps.catalog.outputs.matrix_0 }}+ matrix_1: ${{ steps.catalog.outputs.matrix_1 }}+ matrix_1_present: ${{ steps.catalog.outputs.matrix_1_present }}+ max_parallel_per_shard: ${{ steps.catalog.outputs.max_parallel_per_shard }}+ selected: ${{ steps.catalog.outputs.selected }}+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ ref: ${{ github.sha }}+ persist-credentials: false++ - name: Generate private eval-repository token+ id: evals_token+ env:+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}+ GH_REPO: ${{ github.repository }}+ run: |+ set -euo pipefail+ token="$(node .github/scripts/get-bot-token.mjs)"+ echo "::add-mask::$token"+ echo "value=$token" >> "$GITHUB_OUTPUT"++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ repository: paperclipai/paperclip-evals+ ref: ${{ needs.authorize.outputs.evals_sha }}+ path: .paperclip-evals+ token: ${{ steps.evals_token.outputs.value }}+ persist-credentials: false++ - name: Build the two bounded roster-plus-case matrices+ id: catalog+ env:+ PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}+ PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }}+ MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }}+ MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }}+ ROSTERS: ${{ inputs.rosters || 'all' }}+ run: |+ set -euo pipefail+ if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -lt 2 ] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then+ echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 2 through $MAX_PARALLEL_LIMIT for the two-shard direct suite." >&2+ exit 1+ fi+ node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog \+ --evals-root .paperclip-evals \+ --rosters "$ROSTERS" \+ --campaign-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \+ --max-parallel "$MAX_PARALLEL" \+ --output runner-protocol-eval-catalog.json++ - name: Upload immutable campaign catalog+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7+ with:+ name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }}+ path: runner-protocol-eval-catalog.json+ retention-days: 30+ if-no-files-found: error++ build_runner:+ name: Build portable direct-eval runner once+ needs: [authorize, catalog]+ runs-on: ${{ needs.authorize.outputs.test_runner }}+ timeout-minutes: 30+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ ref: ${{ needs.authorize.outputs.target_sha }}+ persist-credentials: false++ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24++ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6+ env:+ NPM_CONFIG_AUDIT: "false"+ NPM_CONFIG_FUND: "false"+ NPM_CONFIG_UPDATE_NOTIFIER: "false"+ with:+ version: 9.15.4++ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24+ cache: pnpm++ - run: pnpm install --frozen-lockfile --ignore-scripts++ - name: Build runner CLI, daemon, and canonical attempt viewer+ run: |+ set -euo pipefail+ pnpm --filter @paperclipai/paperclip-runner build:typescript+ pnpm --filter @paperclipai/paperclip-runner build:runner-binaries+ pnpm --filter @paperclipai/paperclip-runner build:issue-thread++ - name: Package a portable provider runtime+ run: |+ set -euo pipefail+ mkdir -p "$RUNNER_TEMP/runner-protocol-build/package" "$RUNNER_TEMP/runner-protocol-build/portable"+ pnpm --dir packages/paperclip-runner pack \+ --pack-destination "$RUNNER_TEMP/runner-protocol-build/package"+ package="$(find "$RUNNER_TEMP/runner-protocol-build/package" -maxdepth 1 -type f -name '*.tgz' -print -quit)"+ test -f "$package"+ npm install --prefix "$RUNNER_TEMP/runner-protocol-build/portable" --omit=dev "$package"+ cp "$package" "$RUNNER_TEMP/runner-protocol-build/paperclip-runner.tgz"+ cp packages/paperclip-runner/runner/target/debug/paperclip-runnerd "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd"+ cp -R packages/paperclip-runner/dist-issue-thread "$RUNNER_TEMP/runner-protocol-build/dist-issue-thread"+ test -f "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js"+ test -x "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd"+ tar --create --gzip --file runner-protocol-build.tar.gz -C "$RUNNER_TEMP/runner-protocol-build" .+ sha256sum runner-protocol-build.tar.gz > runner-protocol-build.tar.gz.sha256++ - name: Upload immutable portable runner+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7+ with:+ name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }}+ path: |+ runner-protocol-build.tar.gz+ runner-protocol-build.tar.gz.sha256+ retention-days: 1+ compression-level: 0+ if-no-files-found: error++ eval_shard_0:+ name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }}+ needs: [authorize, catalog, build_runner]+ runs-on: ${{ needs.authorize.outputs.test_runner }}+ timeout-minutes: 18+ permissions:+ contents: read+ id-token: write+ environment:+ name: runner-e2e-paid+ strategy:+ fail-fast: false+ max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }}+ matrix: ${{ fromJSON(needs.catalog.outputs.matrix_0) }}+ steps: &direct_eval_steps+ - name: Reauthorize paid execution before provider access+ env:+ GH_TOKEN: ${{ github.token }}+ REF: ${{ github.ref }}+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}+ ACTOR: ${{ github.actor }}+ ACTOR_ID: ${{ github.actor_id }}+ TRIGGERING_ACTOR: ${{ github.triggering_actor }}+ ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}+ run: |+ set -euo pipefail+ test "$REF" = "refs/heads/$DEFAULT_BRANCH"+ triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"+ test "$triggering_actor_id" = "$ACTOR_ID" || test "$TRIGGERING_ACTOR" != "$ACTOR"+ for candidate in "$triggering_actor_id" "$ACTOR_ID"; do+ jq -e --argjson candidate "$candidate" 'type == "array" and index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null+ done++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ ref: ${{ github.sha }}+ persist-credentials: false++ - name: Generate private eval-repository token+ id: evals_token+ env:+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}+ GH_REPO: ${{ github.repository }}+ run: |+ set -euo pipefail+ token="$(node .github/scripts/get-bot-token.mjs)"+ echo "::add-mask::$token"+ echo "value=$token" >> "$GITHUB_OUTPUT"++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ repository: paperclipai/paperclip-evals+ ref: ${{ needs.authorize.outputs.evals_sha }}+ path: .paperclip-evals+ token: ${{ steps.evals_token.outputs.value }}+ persist-credentials: false++ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24++ - name: Download portable runner+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8+ with:+ name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }}+ path: runner-protocol-build++ - name: Verify and extract portable runner+ run: |+ set -euo pipefail+ cd runner-protocol-build+ sha256sum --check runner-protocol-build.tar.gz.sha256+ mkdir extracted+ tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted+ test -x extracted/paperclip-runnerd++ - name: Prepare short-lived AgentCore web identity+ if: matrix.credentialName == 'AWS_AGENTCORE_OIDC'+ env:+ AGENTCORE_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN }}+ run: |+ set -euo pipefail+ test -n "$AGENTCORE_ROLE_ARN"+ token="$(curl --fail --silent --show-error \+ -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \+ "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" | jq -r .value)"+ test -n "$token"+ echo "::add-mask::$token"+ token_file="$RUNNER_TEMP/runner-protocol-agentcore-token"+ printf '%s' "$token" > "$token_file"+ chmod 600 "$token_file"+ {+ echo "AWS_WEB_IDENTITY_TOKEN_FILE=$token_file"+ echo "AWS_ROLE_ARN=$AGENTCORE_ROLE_ARN"+ echo "AWS_ROLE_SESSION_NAME=runner-protocol-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"+ } >> "$GITHUB_ENV"++ - name: Run one immutable direct protocol cell+ id: direct_eval+ env:+ CELL_ID: ${{ matrix.cellId }}+ ROSTER_FILE: ${{ matrix.rosterFile }}+ CASE_ID: ${{ matrix.caseId }}+ CREDENTIAL_NAME: ${{ matrix.credentialName }}+ PROVIDER: ${{ matrix.provider }}+ MAX_INFRASTRUCTURE_RETRIES: ${{ github.event_name == 'schedule' && 1 || inputs.max_infrastructure_retries }}+ OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }}+ ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }}+ OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }}+ PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID }}+ PAPERCLIP_CLAUDE_MANAGED_AGENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_ID }}+ PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION }}+ PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID }}+ AWS_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }}+ AWS_DEFAULT_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }}+ PAPERCLIP_AWS_AGENTCORE_PROFILE_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_PROFILE_ID }}+ PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID }}+ PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN }}+ PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION }}+ PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN }}+ PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER }}+ PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN }}+ PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN }}+ PAPERCLIP_AWS_AGENTCORE_MEMORY_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ID }}+ PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN }}+ PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET }}+ PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX }}+ PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN }}+ PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION }}+ run: |+ set -euo pipefail+ mkdir -p cell-output/runs+ if [ "$CREDENTIAL_NAME" != "AWS_AGENTCORE_OIDC" ]; then+ test -n "${!CREDENTIAL_NAME:-}"+ fi+ if [ "$PROVIDER" = "claude_managed" ]; then+ test -n "$PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID"+ test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_ID"+ test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION"+ test -n "$PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID"+ fi+ set +e+ python3 .paperclip-evals/evals/paperclip-runner/tools/run_live_roster.py run \+ --roster ".paperclip-evals/evals/paperclip-runner/rosters/$ROSTER_FILE" \+ --case "$CASE_ID" \+ --runner-cli runner-protocol-build/extracted/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js \+ --runner-package runner-protocol-build/extracted/paperclip-runner.tgz \+ --runnerd runner-protocol-build/extracted/paperclip-runnerd \+ --runs-root cell-output/runs \+ --max-infrastructure-retries "$MAX_INFRASTRUCTURE_RETRIES" \+ --run-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${CELL_ID}"+ status=$?+ set -e+ CELL_EXIT_CODE="$status" node --input-type=module <<'NODE'+ import { writeFileSync } from "node:fs";+ writeFileSync("cell-output/cell.json", `${JSON.stringify({+ schema: "paperclip.runner-protocol-eval.cell/v1",+ cellId: process.env.CELL_ID,+ rosterFile: process.env.ROSTER_FILE,+ caseId: process.env.CASE_ID,+ exitCode: Number(process.env.CELL_EXIT_CODE),+ }, null, 2)}\n`, { mode: 0o600 });+ NODE+ exit "$status"++ - name: Upload access-controlled cell attempt+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7+ with:+ name: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.cellId }}+ path: cell-output/+ retention-days: 30+ if-no-files-found: error++ eval_shard_1:+ name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }}+ if: needs.catalog.outputs.matrix_1_present == 'true'+ needs: [authorize, catalog, build_runner]+ runs-on: ${{ needs.authorize.outputs.test_runner }}+ timeout-minutes: 18+ permissions:+ contents: read+ id-token: write+ environment:+ name: runner-e2e-paid+ strategy:+ fail-fast: false+ max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }}+ matrix: ${{ fromJSON(needs.catalog.outputs.matrix_1) }}+ steps: *direct_eval_steps++ report:+ name: Merge attempts and render canonical Evalbook+ if: always() && !cancelled() && needs.catalog.result == 'success' && needs.build_runner.result == 'success'+ needs: [authorize, catalog, build_runner, eval_shard_0, eval_shard_1]+ runs-on: ubuntu-latest+ timeout-minutes: 30+ permissions:+ actions: read+ contents: read+ outputs:+ public_report_ready: ${{ steps.public_report.outputs.ready }}+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ ref: ${{ github.sha }}+ persist-credentials: false++ - name: Generate private eval-repository token+ id: evals_token+ env:+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}+ GH_REPO: ${{ github.repository }}+ run: |+ set -euo pipefail+ token="$(node .github/scripts/get-bot-token.mjs)"+ echo "::add-mask::$token"+ echo "value=$token" >> "$GITHUB_OUTPUT"++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ repository: paperclipai/paperclip-evals+ ref: ${{ needs.authorize.outputs.evals_sha }}+ path: .paperclip-evals+ token: ${{ steps.evals_token.outputs.value }}+ persist-credentials: false++ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24++ - name: Download immutable campaign catalog+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8+ with:+ name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }}+ path: runner-protocol-catalog++ - name: Download portable runner and viewer+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8+ with:+ name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }}+ path: runner-protocol-build++ - name: Download every access-controlled cell+ id: download_cells+ continue-on-error: true+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8+ with:+ pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-*+ path: downloaded-runner-protocol-evals+ merge-multiple: false++ - name: Retry cell download after artifact transport failure+ if: steps.download_cells.outcome == 'failure'+ continue-on-error: true+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8+ with:+ pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-*+ path: downloaded-runner-protocol-evals+ merge-multiple: false++ - name: Materialize an empty download root when every cell failed early+ run: mkdir -p downloaded-runner-protocol-evals++ - name: Verify portable viewer+ run: |+ set -euo pipefail+ cd runner-protocol-build+ sha256sum --check runner-protocol-build.tar.gz.sha256+ mkdir extracted+ tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted+ test -f extracted/dist-issue-thread/index.html++ - name: Aggregate every expected cell, including missing infrastructure cells+ env:+ PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}+ PAPERCLIP_PROTOCOL_EVAL_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}+ PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }}+ PAPERCLIP_PROTOCOL_EVAL_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}+ run: |+ node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs aggregate \+ --catalog runner-protocol-catalog/runner-protocol-eval-catalog.json \+ --downloads downloaded-runner-protocol-evals \+ --evals-root .paperclip-evals \+ --runs-out runner-protocol-merged/runs \+ --campaign-out runner-protocol-merged/campaign.json++ - name: Render the access-controlled canonical Evalbook report+ run: |+ python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \+ --runs-root runner-protocol-merged/runs \+ --output runner-protocol-merged/report \+ --viewer-root runner-protocol-build/extracted/dist-issue-thread \+ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \+ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json+ cp runner-protocol-merged/campaign.json runner-protocol-merged/report/campaign.json++ - name: Render the same canonical grid from a public-safe evidence projection+ run: |+ node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs sanitize \+ --runs-root runner-protocol-merged/runs \+ --output runner-protocol-merged/public-runs+ python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \+ --runs-root runner-protocol-merged/public-runs \+ --output runner-protocol-merged/public-report \+ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \+ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json+ cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json++ - name: Enforce the static public allowlist+ id: public_report+ run: |+ node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report");'+ echo "ready=true" >> "$GITHUB_OUTPUT"++ - name: Add campaign result to the workflow summary+ run: |+ {+ echo '## Runner direct live protocol evals'+ echo+ jq -r '"- Cells: \(.totals.passed)/\(.totals.selected) passed\n- Behavior failures: \(.totals.behaviorFailures)\n- Infrastructure failures: \(.totals.infrastructureFailures)\n- Paperclip: `\(.source.paperclip.sha)`\n- Evals: `\(.source.evals.sha)`"' runner-protocol-merged/campaign.json+ } >> "$GITHUB_STEP_SUMMARY"++ - name: Upload access-controlled canonical Evalbook and raw attempts+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7+ with:+ name: runner-protocol-eval-report-${{ github.run_id }}-${{ github.run_attempt }}+ path: runner-protocol-merged/+ retention-days: 30+ if-no-files-found: error++ - name: Upload publisher-only sanitized Evalbook+ if: steps.public_report.outputs.ready == 'true'+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7+ with:+ name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }}+ path: runner-protocol-merged/public-report/+ retention-days: 1+ if-no-files-found: error++ - name: Enforce complete green campaign+ if: always()+ run: jq -e '.complete == true and .allPassed == true' runner-protocol-merged/campaign.json >/dev/null++ publish_history:+ name: Publish immutable Evalbook and mutable campaign index+ needs: [authorize, catalog, report]+ if: always() && needs.report.outputs.public_report_ready == 'true'+ runs-on: ubuntu-latest+ timeout-minutes: 15+ concurrency:+ group: runner-protocol-eval-history-publish+ cancel-in-progress: false+ permissions:+ contents: read+ id-token: write+ environment:+ name: runner-e2e-history+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ # AWS credentials can execute only the publisher from the trusted workflow revision.+ ref: ${{ github.sha }}+ persist-credentials: false++ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24++ - name: Download only the sanitized canonical Evalbook+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8+ with:+ name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }}+ path: runner-protocol-public-report++ - name: Exchange GitHub OIDC identity for scoped AWS credentials+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6+ with:+ role-to-assume: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_ROLE_ARN || vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }}+ aws-region: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_REGION || vars.RUNNER_E2E_HISTORY_AWS_REGION }}++ - name: Publish versioned report and refresh the root index+ env:+ PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report+ RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET || vars.RUNNER_E2E_HISTORY_S3_BUCKET }}+ RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX || 'runner-protocol-evals' }}+ RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL || vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}+ run: node packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjsdiff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.mdindex e0560b1dde..a628694615 100644--- a/packages/paperclip-runner/README.md+++ b/packages/paperclip-runner/README.md@@ -190,7 +190,25 @@ Live console provider-backed routes are loopback-only and reject wildcard/LANbinds. Browser mutations require same-origin Fetch Metadata, matching Origin,and JSON content; see the protocol-server tutorial for direct `curl` examples.-## Live, chaos, and AWS AgentCore operations+## Direct live protocol qualification++The canonical direct live protocol suite lives in the separate+`paperclip-evals` repository under `evals/paperclip-runner/`. Its+`live-mini.json` roster is the complete 35-case Codex qualification lane. Build+this package's TypeScript output, release `paperclip-runnerd`, package tarball,+and `dist-issue-thread` viewer, then use the roster runner documented in that+repository. The package ships the required orchestration entry point as+`paperclip-runner-eval-session` (`dist/cli/eval-session.js`). Evalbook owns the+consistent HTML matrix and read-only attempt drill-down pages.++The hosted full-campaign workflow, parallel matrix, credential boundaries,+canonical report merge, and versioned S3 index are documented in+[`docs/runner-protocol-live-evals.md`](docs/runner-protocol-live-evals.md).++This direct protocol qualification is separate from the stress-derived Runner+workflow schedule below and from the full-stack browser model E2E suite.++## Stress-derived workflow, chaos, and AWS AgentCore operationsThe deterministic workflow scorer and the chaos schedule do not requireprovider credentials:@@ -209,13 +227,22 @@ additional scheduling after the observed campaign total reaches that value:```shPAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD=12 \+ PAPERCLIP_EVALS_ROOT=/path/to/paperclip-evals \pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals++# Run two scheduled native Codex executions only.+PAPERCLIP_EVALS_ROOT=/path/to/paperclip-evals \+ pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals -- \+ --candidate codex-luna --limit 2```GitHub-hosted live campaigns additionally require the default branch, anallowlisted numeric actor ID, the protected `runner-e2e-paid` environment, and-an explicit repository variable before scheduled runs are enabled. Uploaded-reports contain redacted observations and trace digests, not raw provider+an explicit repository variable before scheduled runs are enabled. Manual+dispatches accept the same candidate, case, and execution-limit selectors. The+paid job uses the reviewed RunsOn Fleet label when `RUNNER_E2E_AWS_ENABLED=true`+and otherwise stays on `ubuntu-latest`. Uploaded reports contain redacted+observations and trace digests, not raw providerframes, prompts, credentials, tool arguments, or hidden reasoning.The AgentCore proof-of-concept uses an AWS CLI v2 profile to provision a@@ -234,6 +261,21 @@ pnpm --filter @paperclipai/paperclip-runner smoke:capability:aws-agentcorepnpm --filter @paperclipai/paperclip-runner aws-agentcore:destroy -- --yes```+To admit the hosted direct-eval workflow, provision with the account-local+GitHub Actions OIDC provider and keep the default exact repository and protected+environment binding:++```sh+pnpm --filter @paperclipai/paperclip-runner aws-agentcore:provision -- \+ --aws-profile paperclip-dev \+ --github-oidc-provider-arn arn:aws:iam::<account-id>:oidc-provider/token.actions.githubusercontent.com+```++This adds only `repo:paperclipai/paperclip:environment:runner-e2e-paid` as a+web-identity subject on the scoped invocation role. The generated nonsecret+profile records that role as both the local invocation role and the hosted+execution role.+Provisioning can incur Bedrock, AgentCore Runtime/Memory, storage, and privatenetworking charges. Provisioning refuses to modify a colliding stack unless itsPaperclip ownership tags and template description match. A verified@@ -256,8 +298,8 @@ recorded lab unless `--force` is also supplied.| `check:clean-consumers` | Pack the runner and install its root, evals, and testing exports in a clean consumer. || `test:eval-slice` | Run the credential-free eval bundle, scoring, and behavior/fault slice. || `test:runner-workflow-evals` | Run the deterministic provider-neutral workflow matrix. |-| `report:runner-workflow-evals` | Validate deterministic results and write local reports only when every scoreable result passes. |-| `report:runner-live-evals` | Execute the paid forty-execution provider schedule with qualification and campaign-cost guards. |+| `report:runner-workflow-evals` | Validate deterministic fail-closed results and write JSON, Markdown, JUnit, and GitHub-safe reports. |+| `report:runner-live-evals` | Execute the paid provider schedule and render its immutable attempts with the canonical `paperclip-evals` HTML grid. || `report:runner-chaos-evals` | Write the credential-free eight-scenario chaos schedule. || `test:aws-agentcore-provisioning` | Validate the AgentCore template and wrapper safety contracts without provisioning. || `aws-agentcore:provision` / `probe` / `lab` / `destroy` | Manage the scoped AgentCore proof-of-concept lifecycle. |diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.mdnew file mode 100644index 0000000000..2cdc74774b--- /dev/null+++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md@@ -0,0 +1,148 @@+# Direct live Runner protocol evals++This is the provider-backed, one-turn protocol qualification layer in+`paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally+separate from both the browser full-stack model E2E and the stress-derived+workflow schedule in `runner-workflow-evals.md`.++The canonical unit of work is one live roster plus one authored case. A full+campaign selects every `rosters/live-*.json` file at one immutable+`paperclip-evals` commit. That includes the complete 35-case provider rosters+and the smaller ACPX Codex control roster. The native resume reliability gate+is not a normal one-turn roster: it requires its separately governed external+resource campaign and remains opt-in.++Managed-provider evidence identifies the immutable deployed provider artifact:+Claude Managed uses its Agent version, while AgentCore uses its qualification+revision. Transport API or beta versions remain separate protocol metadata and+must not replace that runtime identity in a report.++## Hosted campaign++Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow+from the default branch and provide:++- `target_branch`: the Paperclip branch to build and test;+- `evals_sha`: an exact 40-character commit from+ `paperclipai/paperclip-evals`;+- `rosters`: `all` for the entire direct suite, or a comma-separated diagnostic+ subset;+- `max_infrastructure_retries`: zero through three, applied only when an+ attempt explicitly reports a retryable infrastructure failure.++The authorization job resolves the Paperclip branch to a commit and verifies+the supplied eval commit before any checkout. A short-lived bot token generated+from `COMMITPERCLIP_KEY` authorizes each checkout of the private eval repository;+the token is masked and is never forwarded to a provider process. The workflow+uses the same numeric actor+allowlist, protected `runner-e2e-paid` environment, RunsOn fleet selector, and+`RUNNER_E2E_MAX_PARALLEL` ceiling as the full-stack E2E workflow. Two balanced+GitHub matrices keep each matrix below GitHub's 256-job limit while keeping+their combined concurrency at or below that shared ceiling. Runner TypeScript,+the native daemon, provider dependencies, and the attempt viewer are built+once and reused by every cell. Because the complete suite requires two matrix+shards, this workflow accepts a shared concurrency ceiling from 2 through 100.++`all` is intentionally literal. A disabled driver, missing remote profile, or+unavailable provider is retained as an infrastructure result; it is not+silently omitted. In particular, the ACPX Pi roster remains visible while Pi+is disabled in the current Runner. Use a roster subset only for diagnosis, not+to claim the full campaign is green.++A provider turn that reaches a durable failed, interrupted, or otherwise+non-completed terminal still produces an attempt artifact and is scored as a+behavior result. It is an infrastructure failure only when the harness cannot+produce usable evidence, such as an unavailable provider, invalid profile, or+transport failure. Automatic retries therefore never rerun a measured behavior+failure merely to improve its score.++## Required protected configuration++The paid jobs read only the credential selected for each roster:++- `OPENAI_API_KEY` for native Codex and ACPX Codex;+- `ANTHROPIC_API_KEY` for ACPX Claude and Claude Managed;+- `OPENROUTER_API_KEY` for native OpenCode and ACPX Pi;+- short-lived GitHub OIDC workload identity for AWS AgentCore.++Claude Managed also requires the four nonsecret+`PAPERCLIP_CLAUDE_MANAGED_*` profile variables. AgentCore requires the+nonsecret `PAPERCLIP_AWS_AGENTCORE_*` profile variables, including+`PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN` and the immutable+`PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION`; the eval fails closed when+that deployed revision differs from the pinned roster config. The workflow+writes the GitHub OIDC token to a mode-`0600` file and never forwards long-lived+AWS access keys.+Provision the AgentCore stack with `--github-oidc-provider-arn` so that scoped+role admits only the `paperclipai/paperclip` repository's protected+`runner-e2e-paid` environment as its web-identity subject.+Scheduled runs additionally require `RUNNER_PROTOCOL_EVAL_NIGHTLY_ENABLED=true`+and the pinned `RUNNER_PROTOCOL_EVALS_SHA` repository variable.++## Reports and history++Each cell uploads its immutable run directory to an access-controlled Actions+artifact. The trusted report job merges all expected cells, represents missing+cell artifacts as infrastructure failures, and invokes the report program from+the pinned eval commit. The full artifact contains the canonical Evalbook grid,+read-only Runner issue-thread attempt pages, and raw immutable run records.++Public publishing uses a separate projection and a separate trusted OIDC job.+The projection retains model/config identity, status, usage totals, and check+outcomes but removes provider session identifiers, transcripts, semantic-tool+payloads, state revisions, traces, remote profile identities, and raw failure+text. The same Evalbook `report` command renders that projection, so the public+grid and test pages have the standard Evalbook layout. The publisher rejects+scripts, remote resources, symlinks, unknown paths, broken links, raw session+fields, and credential-shaped values.++S3 publication is additive:++```text+runner-protocol-evals/+ index.html+ history.json+ latest.json+ latest-green.json+ campaigns/+ gha-<run-id>-<run-attempt>/+ index.html+ latest.html+ inventory.html+ tests/*.html+ attempts/*.html+ campaign.json+ bundle-manifest.json+```++Campaign files use immutable cache headers and a digest manifest. Reusing a+campaign ID with different bytes fails closed. Only the root history and+pointer files are mutable, and the publisher never deletes objects. The root+history retains at most 200 records, reserving one record for the latest green+campaign when it would otherwise fall outside that window so its pointer stays+valid.++The publishing job uses dedicated `RUNNER_PROTOCOL_EVAL_HISTORY_*` variables+when present and falls back to the existing Runner E2E history role, region,+bucket, and public base URL. Its default top-level prefix is+`runner-protocol-evals`, distinct from `runner-e2e`. The AWS role must allow+additive writes and reads for that prefix.++## Local publisher checks++These tests make no provider or AWS calls:++```sh+pnpm --filter @paperclipai/paperclip-runner test:runner-protocol-eval-publish+```++To inspect the catalog without executing it, point the command at a local+evals checkout:++```sh+pnpm --filter @paperclipai/paperclip-runner \+ report:runner-protocol-eval:catalog -- \+ --evals-root /path/to/paperclip-evals \+ --campaign-id gha-1-1 \+ --output /tmp/runner-protocol-eval-catalog.json+```diff --git a/packages/paperclip-runner/docs/runner-workflow-evals.md b/packages/paperclip-runner/docs/runner-workflow-evals.mdindex 19b5357329..ceca8f0387 100644--- a/packages/paperclip-runner/docs/runner-workflow-evals.md+++ b/packages/paperclip-runner/docs/runner-workflow-evals.md@@ -6,7 +6,9 @@ capability inventory, capability cases, and existing scoring/report readers.The workspace-private `@paperclipai/paperclip-eval-kernel` package owns onlystructural scenario-by-candidate orchestration. Runner-specific cases,-observations, scoring, traceability, and report rendering remain package-local.+observations, scoring, and traceability remain package-local. The HTML matrix is+rendered by the canonical `paperclip-evals` report program so live results use+the same grid and drill-down pages as the direct Runner eval suite.## Lanes@@ -14,9 +16,9 @@ observations, scoring, traceability, and report rendering remain package-local.runs the credential-free PR gate over sanitized Codex, OpenCode, and ACPXnormalization fixtures.- `pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals`- validates the deterministic report and writes JSON, Markdown, JUnit, and- GitHub-safe artifacts under `.paperclip-local/evals/workflows/` only when all- scoreable fixture results pass. It makes no network requests.+ validates the deterministic fail-closed fixture matrix and writes JSON,+ Markdown, JUnit, and GitHub-safe artifacts under+ `.paperclip-local/evals/workflows/`. It makes no network requests.- `pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals` runsthe balanced forty-execution schedule against real provider sessions. Livecandidate failures are trend-only; missing credentials, qualification@@ -30,10 +32,38 @@ qualification variable names, and budgets. Credentials remain in theenvironment. `PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD` must be a positive finitenumber and defaults to 12 USD for scheduled runs.+Live executions export one immutable Evalbook attempt per workflow/candidate to+`.paperclip-local/evals/workflows/evalbook-runs/`, then invoke+`evals/paperclip-runner/tools/eval_program.py report` from a `paperclip-evals`+checkout. That program writes the canonical matrix to+`.paperclip-local/evals/workflows/index.html`, plus `latest.html`, test pages,+and attempt pages. Set `PAPERCLIP_EVALBOOK_PROGRAM` to the program's absolute+path or `PAPERCLIP_EVALS_ROOT` to its repository root. Conventional sibling+worktree locations are discovered automatically. GitHub Actions checks out a+pinned `paperclip-evals` revision, so every hosted run uses the same reviewed+report implementation rather than a copied or package-local renderer.+Filtered reports with only a few candidates expand their result columns to the+available viewport, keeping the PASS, FAIL, and INFRA labels visible without a+horizontal scroll. Full matrices retain the canonical scrollable grid and+sticky test-name column.++The exported artifact contains only the safe workflow observation and+scorecard. Prompts, credentials, raw provider frames, tool arguments, and+reasoning remain excluded. `evalbook-manifest.json` records the generator path+and SHA-256 digest used for the render.++Local and manual GitHub runs can bound paid execution with comma-separated+`--candidate` and `--case` selectors plus `--limit`. For example,+`report:runner-live-evals -- --candidate codex-luna --limit 2` executes only+the first two Codex entries in that week's validated schedule. Subsets receive+a distinct bundle identity and do not contaminate full-campaign trend history.+The hosted live workflow is default-branch-only and requires an allowlistednumeric actor plus the protected `runner-e2e-paid` environment. Scheduled runsalso remain disabled until `RUNNER_LIVE_EVALS_NIGHTLY_ENABLED` is explicitly-set to `true`.+set to `true`. The paid job uses the full-stack workflow's literal runner+selection: `RUNNER_E2E_AWS_ENABLED=true` routes it to the RunsOn Fleet, and any+other value uses `ubuntu-latest`.## Trace and reasoning safetydiff --git a/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yamlindex a7cbd72a1c..afa68ed17f 100644--- a/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml+++ b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml@@ -14,6 +14,19 @@ Parameters:Type: StringDescription: Stable IAM user or role ARN allowed to assume the Paperclip invocation role.AllowedPattern: "^arn:aws(-[^:]+)?:iam::[0-9]{12}:(role|user)/.+$"+ GitHubOidcProviderArn:+ Type: String+ Default: ""+ Description: Optional account-local GitHub Actions OIDC provider for hosted Runner evals.+ AllowedPattern: "^$|^arn:aws(-[^:]+)?:iam::[0-9]{12}:oidc-provider/token[.]actions[.]githubusercontent[.]com$"+ GitHubRepository:+ Type: String+ Default: paperclipai/paperclip+ AllowedPattern: "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"+ GitHubEnvironment:+ Type: String+ Default: runner-e2e-paid+ AllowedPattern: "^[A-Za-z0-9_.-]+$"BedrockModelId:Type: StringDefault: global.anthropic.claude-sonnet-4-6@@ -53,6 +66,7 @@ Parameters:Conditions:IsPrivate: !Equals [!Ref DeploymentMode, private]+ HasGitHubOidcTrust: !Not [!Equals [!Ref GitHubOidcProviderArn, ""]]Resources:ContextEncryptionKey:@@ -304,6 +318,17 @@ Resources:Principal:AWS: !Ref TrustedRunnerPrincipalArnAction: sts:AssumeRole+ - !If+ - HasGitHubOidcTrust+ - Effect: Allow+ Principal:+ Federated: !Ref GitHubOidcProviderArn+ Action: sts:AssumeRoleWithWebIdentity+ Condition:+ StringEquals:+ "token.actions.githubusercontent.com:aud": sts.amazonaws.com+ "token.actions.githubusercontent.com:sub": !Sub repo:${GitHubRepository}:environment:${GitHubEnvironment}+ - !Ref AWS::NoValuePolicies:- PolicyName: InvokeAndOperatePinnedHarnessPolicyDocument:diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.jsonindex d69f6d9739..c6d02304fc 100644--- a/packages/paperclip-runner/package.json+++ b/packages/paperclip-runner/package.json@@ -114,12 +114,15 @@"test:capability-inventory": "node scripts/check-capability-inventory.test.mjs","test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts","test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval",- "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts",+ "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && node --test scripts/render-runner-workflow-evalbook.test.mjs && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts",+ "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs","check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs","report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs","report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs","report:runner-workflow-evals": "pnpm run build:typescript && node scripts/run-runner-workflow-evals.mjs","report:runner-live-evals": "pnpm run build:typescript && pnpm run build:runner-binaries && node scripts/run-runner-live-eval-schedule.mjs --mode nightly --execute",+ "report:runner-protocol-eval:catalog": "node scripts/runner-protocol-eval-campaign.mjs catalog",+ "report:runner-protocol-eval:publish": "node scripts/publish-runner-protocol-eval-history.mjs","report:runner-chaos-evals": "pnpm run build:typescript && node scripts/run-runner-live-eval-schedule.mjs --mode chaos","check:conformance-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output","check:replay-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core replay_fixture_parity",diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rsindex 6c77d15241..9bb1784012 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs@@ -31,6 +31,48 @@ fn send(value: Value) -> io::Result<()> {stdout.flush()}+fn send_split_event_burst(state: &FakeState) -> io::Result<()> {+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");+ for index in 0..96 {+ send(json!({+ "method": "item/agentMessage/delta",+ "params": {+ "threadId": state.thread_id,+ "turnId": turn_id,+ "itemId": "split-burst-message",+ "delta": format!("first-{index} "),+ }+ }))?;+ }+ send(json!({+ "id": "split-burst-tool",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": turn_id,+ "callId": "split-burst-semantic-call",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))+}++fn finish_split_event_burst(state: &FakeState) -> io::Result<()> {+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");+ for index in 0..48 {+ send(json!({+ "method": "item/agentMessage/delta",+ "params": {+ "threadId": state.thread_id,+ "turnId": turn_id,+ "itemId": "split-burst-message",+ "delta": format!("second-{index} "),+ }+ }))?;+ }+ Ok(())+}+fn load_state(path: &Path) -> FakeState {fs::read(path).ok()@@ -483,6 +525,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {.any(|value| value == "--opencode-proxy-runtime-question");let emit_runtime_elicitation = args.iter().any(|value| value == "--runtime-elicitation");let emit_structured_activity = args.iter().any(|value| value == "--structured-activity");+ let emit_split_event_burst = args.iter().any(|value| value == "--split-event-burst");let require_skill_instructions = args.iter().any(|value| value == "--include-skill-instructions");@@ -703,6 +746,15 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {finish_turn(&state_path, &mut state, "completed")?;continue;}+ if message.get("method").is_none() && message.get("id") == Some(&json!("split-burst-tool"))+ {+ if message.pointer("/result/success") != Some(&json!(true)) {+ return Err("split event burst semantic tool failed".into());+ }+ finish_split_event_burst(&state)?;+ finish_turn(&state_path, &mut state, "completed")?;+ continue;+ }if message.get("method").is_none() && message.get("id") == Some(&json!("tool-request-1")) {if message.pointer("/result/success") == Some(&json!(false)) {log_call(call_log.as_deref(), "tool-response:failure")?;@@ -1110,6 +1162,8 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {} else if emit_structured_activity {send_structured_activity(&state)?;finish_turn(&state_path, &mut state, "completed")?;+ } else if emit_split_event_burst {+ send_split_event_burst(&state)?;} else if emit_question {send_question(&state)?;} else if !hold_turn {diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rsindex dcdfb5ed0e..0370c7bb90 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs@@ -140,7 +140,7 @@ impl ManagedProviderDescriptor {fn version(&self) -> &str {match self {- Self::ClaudeManaged(config) => &config.beta_version,+ Self::ClaudeManaged(config) => &config.agent_version,Self::AwsAgentcore(config) => &config.qualification_revision,}}@@ -2477,6 +2477,35 @@ mod tests {);}+ #[test]+ fn claude_runtime_identity_uses_the_pinned_agent_version() {+ let descriptor = ManagedProviderDescriptor::ClaudeManaged(ClaudeManagedProviderConfig {+ model: QUALIFIED_CLAUDE_MODEL.to_owned(),+ profile_id: "profile-1".to_owned(),+ anthropic_agent_id: "agent-1".to_owned(),+ agent_version: "17".to_owned(),+ environment_id: "environment-1".to_owned(),+ beta_version: QUALIFIED_CLAUDE_BETA.to_owned(),+ max_session_list_cost_usd: 1.0,+ instructions: "Complete the supplied task.".to_owned(),+ runtime_context: None,+ });++ assert_eq!(descriptor.version(), "17");+ assert_eq!(+ session_event_payload(+ &descriptor,+ &ProviderRuntimeIdentity::RemoteService {+ service: "anthropic_managed_agents".to_owned(),+ provider_session_id: "session-17".to_owned(),+ process_id: None,+ },+ )+ .pointer("/providerDescriptor/providerVersion"),+ Some(&json!("17"))+ );+ }+#[test]fn agentcore_usage_snapshot_accepts_only_bounded_reconciliation_states() {let usage = json!({diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rsindex 6ac380da5d..21eca672f7 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs@@ -261,7 +261,7 @@ impl VerifiedProcessLaunch {}#[cfg(target_os = "macos")]{- let executable = materialize_executable(artifact)?;+ let executable = materialize_bound_executable(artifact)?;args.push(executable.path.to_string_lossy().into_owned());temporary_executables.push(executable);}@@ -290,6 +290,7 @@ struct InheritedCommand {#[cfg(target_os = "macos")]struct TemporaryExecutable {path: PathBuf,+ cleanup_directory: Option<PathBuf>,_file: File,}@@ -297,13 +298,16 @@ struct TemporaryExecutable {impl Drop for TemporaryExecutable {fn drop(&mut self) {let _ = fs::remove_file(&self.path);+ if let Some(directory) = &self.cleanup_directory {+ let _ = fs::remove_dir(directory);+ }}}#[cfg(target_os = "macos")]-fn materialize_executable(+fn verified_executable_parent(artifact: &VerifiedProcessArtifact,-) -> Result<TemporaryExecutable, LocalRunnerError> {+) -> Result<&Path, LocalRunnerError> {let directory = artifact.display_path.parent().ok_or_else(|| {LocalRunnerError::invalid(format!("verified process artifact {} has no parent directory",@@ -318,10 +322,15 @@ fn materialize_executable(directory.display())));}- let path = directory.join(format!(- ".paperclip-verified-executable-{}",- Uuid::new_v4().simple()- ));+ Ok(directory)+}++#[cfg(target_os = "macos")]+fn materialize_executable_at(+ artifact: &VerifiedProcessArtifact,+ path: PathBuf,+ cleanup_directory: Option<PathBuf>,+) -> Result<TemporaryExecutable, LocalRunnerError> {let mut writable = OpenOptions::new().read(true).write(true)@@ -375,6 +384,7 @@ fn materialize_executable(drop(writable);Ok(TemporaryExecutable {path: path.clone(),+ cleanup_directory,_file: file,})})();@@ -384,6 +394,40 @@ fn materialize_executable(result}+#[cfg(target_os = "macos")]+fn materialize_executable(+ artifact: &VerifiedProcessArtifact,+) -> Result<TemporaryExecutable, LocalRunnerError> {+ let directory = verified_executable_parent(artifact)?;+ let path = directory.join(format!(+ ".paperclip-verified-executable-{}",+ Uuid::new_v4().simple()+ ));+ materialize_executable_at(artifact, path, None)+}++#[cfg(target_os = "macos")]+fn materialize_bound_executable(+ artifact: &VerifiedProcessArtifact,+) -> Result<TemporaryExecutable, LocalRunnerError> {+ let parent = verified_executable_parent(artifact)?;+ let directory = parent.join(format!(+ ".paperclip-verified-executable-{}",+ Uuid::new_v4().simple()+ ));+ fs::create_dir(&directory).map_err(|error| snapshot_error(&artifact.display_path, error))?;+ if let Err(error) = fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)) {+ let _ = fs::remove_dir(&directory);+ return Err(snapshot_error(&artifact.display_path, error));+ }+ let result =+ materialize_executable_at(artifact, directory.join("launch"), Some(directory.clone()));+ if result.is_err() {+ let _ = fs::remove_dir(directory);+ }+ result+}+#[cfg(any(target_os = "linux", target_os = "macos"))]fn inherited_artifact(artifact: &VerifiedProcessArtifact,@@ -941,4 +985,50 @@ mod tests {assert!(inherited.args[2].starts_with("/dev/fd/"));fs::remove_dir_all(directory).unwrap();}++ #[cfg(target_os = "macos")]+ #[test]+ fn executable_arguments_use_the_private_nested_launch_contract() {+ let nonce = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap()+ .as_nanos();+ let directory = std::env::temp_dir().join(format!(+ "paperclip-executable-argument-launch-{}-{nonce}",+ std::process::id()+ ));+ fs::create_dir_all(&directory).unwrap();+ let program = verified_artifact(&directory.join("node"), b"node");+ let executable = verified_artifact(&directory.join("opencode"), b"opencode");+ let launch = VerifiedProcessLaunch::new(+ program,+ vec![VerifiedProcessArgument::ExecutableArtifact(executable)],+ );++ let inherited = launch.inherited_command().unwrap();+ let command = PathBuf::from(&inherited.args[0]);+ let private_directory = command.parent().unwrap().to_path_buf();+ assert_eq!(command.file_name().unwrap(), "launch");+ assert!(private_directory+ .file_name()+ .unwrap()+ .to_string_lossy()+ .starts_with(".paperclip-verified-executable-"));+ assert_eq!(+ fs::symlink_metadata(&private_directory)+ .unwrap()+ .permissions()+ .mode()+ & 0o777,+ 0o700+ );+ assert_eq!(+ fs::symlink_metadata(&command).unwrap().permissions().mode() & 0o777,+ 0o500+ );++ drop(inherited);+ assert!(!private_directory.exists());+ fs::remove_dir_all(directory).unwrap();+ }}diff --git a/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjsindex eae5284f90..2bfdbe6a0b 100644--- a/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs+++ b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs@@ -82,6 +82,10 @@ test("AgentCore template has closed development/private resources and explicit cassert.match(source, /Sid: DecryptPinnedRuntimeContext[\s\S]*Action: kms:Decrypt[\s\S]*Resource: !GetAtt ContextEncryptionKey\.Arn/);const invocationRole = source.slice(source.indexOf("RunnerInvocationRole:"), source.indexOf("PrivateVpc:"));assert.doesNotMatch(invocationRole, /Action:\s+(?:-\s+)?(?:s3|kms):\*/);+ assert.match(invocationRole, /Action: sts:AssumeRoleWithWebIdentity/);+ assert.match(invocationRole, /token\.actions\.githubusercontent\.com:aud["']?: sts\.amazonaws\.com/);+ assert.match(invocationRole, /token\.actions\.githubusercontent\.com:sub["']?: !Sub repo:\$\{GitHubRepository\}:environment:\$\{GitHubEnvironment\}/);+ assert.doesNotMatch(invocationRole, /repo:\*|environment:\*/);assert.match(source, /\$\{AgentHarness\.Arn\}\/harness-endpoint\/\$\{HarnessEndpointName\}/);assert.match(source, /\$\{AgentHarness\.Arn\}\/runtime-endpoint\/\$\{HarnessEndpointName\}/);assert.match(source, /BedrockMarketplaceProductId/);@@ -136,10 +140,13 @@ test("AgentCore wrapper is valid shell and writes only nonsecret profile metadatassert.match(source, /--query endpoint\.status/);assert.match(source, /o\.endpoint\?\.arn/);assert.match(source, /--marketplace-product-id/);+ assert.match(source, /--github-oidc-provider-arn/);+ assert.match(source, /GitHubOidcProviderArn=\$GITHUB_OIDC_PROVIDER_ARN/);assert.match(source, /BedrockMarketplaceProductId=\$MARKETPLACE_PRODUCT_ID/);assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=\$context_bucket/);assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=\$context_prefix/);assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=\$context_kms_key_arn/);+ assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN=\$role_arn/);assert.match(source, /ContextPrefix=\$CONTEXT_PREFIX/);assert.match(source, /s3 rm "s3:\/\/\$context_bucket\/\$context_prefix\/assets\/" --recursive/);assert.ok(source.indexOf("delete-harness-endpoint") < source.lastIndexOf("cloudformation delete-stack"));diff --git a/packages/paperclip-runner/scripts/aws-agentcore.sh b/packages/paperclip-runner/scripts/aws-agentcore.shindex 26f4e493d8..c782a22e18 100644--- a/packages/paperclip-runner/scripts/aws-agentcore.sh+++ b/packages/paperclip-runner/scripts/aws-agentcore.sh@@ -22,6 +22,9 @@ MODEL_ID="global.anthropic.claude-sonnet-4-6"MARKETPLACE_PRODUCT_ID="prod-ffvjxvh4ltq64"MARKETPLACE_PRODUCT_ID_EXPLICIT=falseTRUSTED_PRINCIPAL=""+GITHUB_OIDC_PROVIDER_ARN="${PAPERCLIP_AWS_AGENTCORE_GITHUB_OIDC_PROVIDER_ARN:-}"+GITHUB_REPOSITORY="paperclipai/paperclip"+GITHUB_ENVIRONMENT="runner-e2e-paid"CONTEXT_PREFIX=""DRY_RUN=falseFORCE=false@@ -38,6 +41,9 @@ usage() {" --model MODEL_ID Bedrock-native model ID" \" --marketplace-product-id ID exact AWS Marketplace product ID for the model" \" --principal ARN stable IAM role/user trusted to assume runner role" \+ " --github-oidc-provider-arn ARN optional account-local GitHub Actions OIDC provider" \+ " --github-repository OWNER/REPO exact repository admitted by OIDC trust (default: $GITHUB_REPOSITORY)" \+ " --github-environment NAME exact protected environment admitted by OIDC trust (default: $GITHUB_ENVIRONMENT)" \" --context-prefix PFX S3 prefix dedicated to this qualified profile" \" --dry-run validate and print changes without deployment" \" --replace-failed-stack explicitly replace an owned ROLLBACK_COMPLETE stack" \@@ -55,6 +61,9 @@ while [[ $# -gt 0 ]]; do--model) MODEL_ID="$2"; shift 2 ;;--marketplace-product-id) MARKETPLACE_PRODUCT_ID="$2"; MARKETPLACE_PRODUCT_ID_EXPLICIT=true; shift 2 ;;--principal) TRUSTED_PRINCIPAL="$2"; shift 2 ;;+ --github-oidc-provider-arn) GITHUB_OIDC_PROVIDER_ARN="$2"; shift 2 ;;+ --github-repository) GITHUB_REPOSITORY="$2"; shift 2 ;;+ --github-environment) GITHUB_ENVIRONMENT="$2"; shift 2 ;;--context-prefix) CONTEXT_PREFIX="$2"; shift 2 ;;--dry-run) DRY_RUN=true; shift ;;--replace-failed-stack) REPLACE_FAILED_STACK=true; shift ;;@@ -93,6 +102,14 @@ if [[ ! "$MARKETPLACE_PRODUCT_ID" =~ ^prod-[a-z0-9]+$ ]]; thenprintf 'Marketplace product ID must use the AWS prod-* form.\n' >&2exit 2fi+if [[ -n "$GITHUB_OIDC_PROVIDER_ARN" && ! "$GITHUB_OIDC_PROVIDER_ARN" =~ ^arn:aws(-[^:]+)?:iam::[0-9]{12}:oidc-provider/token\.actions\.githubusercontent\.com$ ]]; then+ printf 'GitHub OIDC provider must be the account-local token.actions.githubusercontent.com provider ARN.\n' >&2+ exit 2+fi+if [[ ! "$GITHUB_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ || ! "$GITHUB_ENVIRONMENT" =~ ^[A-Za-z0-9_.-]+$ ]]; then+ printf 'GitHub OIDC trust requires exact OWNER/REPO and environment names.\n' >&2+ exit 2+fiif [[ -z "$CONTEXT_PREFIX" ]]; then CONTEXT_PREFIX="paperclip/agentcore/$STACK_NAME"; fiif [[ ! "$CONTEXT_PREFIX" =~ ^[a-z0-9][a-z0-9/_-]{1,127}$ || "$CONTEXT_PREFIX" == /* || "$CONTEXT_PREFIX" == */ || "$CONTEXT_PREFIX" == *//* || "/$CONTEXT_PREFIX/" == */./* || "/$CONTEXT_PREFIX/" == */../* ]]; thenprintf 'Context prefix must be a safe, relative S3 key prefix.\n' >&2@@ -279,6 +296,14 @@ provision() {caller_arn="$(printf '%s' "$identity" | json_field Arn)"account_id="$(printf '%s' "$identity" | json_field Account)"partition="$(printf '%s' "$caller_arn" | cut -d: -f2)"+ if [[ -n "$GITHUB_OIDC_PROVIDER_ARN" ]]; then+ local expected_github_oidc_provider_arn="arn:$partition:iam::$account_id:oidc-provider/token.actions.githubusercontent.com"+ if [[ "$GITHUB_OIDC_PROVIDER_ARN" != "$expected_github_oidc_provider_arn" ]]; then+ printf 'GitHub OIDC provider must belong to the current AWS account and partition.\n' >&2+ exit 1+ fi+ aws_cli iam get-open-id-connect-provider --open-id-connect-provider-arn "$GITHUB_OIDC_PROVIDER_ARN" >/dev/null+ fimodel_resource_arn="arn:$partition:bedrock:$AWS_REGION_NAME:$account_id:inference-profile/$MODEL_ID"foundation_model_id="$(printf '%s' "$MODEL_ID" | sed -E 's/^(global|us|eu|apac)\.//')"foundation_model_resource_arn="arn:$partition:bedrock:*::foundation-model/$foundation_model_id"@@ -342,7 +367,11 @@ provision() {"EnvironmentName=development" "DeploymentMode=$DEPLOYMENT_MODE" \"HarnessEndpointName=$ENDPOINT_NAME" \"ContextPrefix=$CONTEXT_PREFIX" \- "TrustedRunnerPrincipalArn=$TRUSTED_PRINCIPAL" "BedrockModelId=$MODEL_ID" \+ "TrustedRunnerPrincipalArn=$TRUSTED_PRINCIPAL" \+ "GitHubOidcProviderArn=$GITHUB_OIDC_PROVIDER_ARN" \+ "GitHubRepository=$GITHUB_REPOSITORY" \+ "GitHubEnvironment=$GITHUB_ENVIRONMENT" \+ "BedrockModelId=$MODEL_ID" \"BedrockModelResourceArn=$model_resource_arn" \"BedrockFoundationModelResourceArn=$foundation_model_resource_arn" \"BedrockMarketplaceProductId=$MARKETPLACE_PRODUCT_ID" \@@ -400,6 +429,7 @@ provision() {"PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN=$memory_arn" \"PAPERCLIP_AWS_AGENTCORE_MEMORY_ID=$memory_id" \"PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN=$role_arn" \+ "PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN=$role_arn" \"PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=$context_bucket" \"PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=$context_prefix" \"PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=$context_kms_key_arn" \diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjsnew file mode 100644index 0000000000..b014d76686--- /dev/null+++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs@@ -0,0 +1,547 @@+#!/usr/bin/env node+import { createHash } from "node:crypto";+import { execFile } from "node:child_process";+import {+ lstat,+ mkdtemp,+ mkdir,+ readFile,+ readdir,+ stat,+ writeFile,+} from "node:fs/promises";+import { tmpdir } from "node:os";+import { extname, join, relative, resolve, sep } from "node:path";+import { promisify } from "node:util";++const execFileAsync = promisify(execFile);+const SAFE_CAMPAIGN = /^gha-[1-9][0-9]*-[1-9][0-9]*$/;+const SAFE_REPORT_PATHS = [+ /^(?:index|latest|inventory|real-server)\.html$/,+ /^tests\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/,+ /^attempts\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/,+ /^campaign\.json$/,+];+const CREDENTIAL_PATTERNS = [+ /\bAKIA[0-9A-Z]{16}\b/u,+ /\bsk-[A-Za-z0-9_-]{20,}\b/u,+ /\bBearer\s+[A-Za-z0-9._~-]{16,}\b/iu,+ /["'](?:providerSessionId|sessionId)["']\s*:/u,+];+const ACTIVE_HTML_PATTERNS = [+ /<script\b/iu,+ /<iframe\b/iu,+ /<object\b/iu,+ /<embed\b/iu,+ /<form\b/iu,+ /\son[a-z]+\s*=/iu,+ /javascript\s*:/iu,+ /(?:src|href)\s*=\s*["'](?:https?:)?\/\//iu,+];+const MAX_HISTORY_CAMPAIGNS = 200;++function json(value) {+ return `${JSON.stringify(value, null, 2)}\n`;+}++function html(value) {+ return String(value ?? "")+ .replaceAll("&", "&")+ .replaceAll("<", "<")+ .replaceAll(">", ">")+ .replaceAll('"', """)+ .replaceAll("'", "'");+}++async function loadObject(path) {+ const value = JSON.parse(await readFile(path, "utf8"));+ if (value === null || Array.isArray(value) || typeof value !== "object") {+ throw new Error(`Expected a JSON object: ${path}`);+ }+ return value;+}++export function validateProtocolEvalHistoryDestination({+ bucket,+ prefix,+ publicBaseUrl,+}) {+ if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) {+ throw new Error(+ "RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET is not a valid bucket name",+ );+ }+ const normalizedPrefix = String(prefix ?? "").replace(/^\/+|\/+$/g, "");+ if (+ !normalizedPrefix ||+ normalizedPrefix+ .split("/")+ .some((segment) => !segment || segment === "." || segment === "..")+ ) {+ throw new Error(+ "RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX must be a safe non-empty key prefix",+ );+ }+ const url = new URL(publicBaseUrl);+ if (+ url.protocol !== "https:" ||+ url.username ||+ url.password ||+ url.search ||+ url.hash+ ) {+ throw new Error(+ "RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL",+ );+ }+ return {+ bucket,+ prefix: normalizedPrefix,+ publicBaseUrl: url.href.replace(/\/$/, ""),+ };+}++export function isPublicProtocolEvalPath(relativePath) {+ if (+ relativePath.includes("\\") ||+ relativePath.startsWith("/") ||+ relativePath+ .split("/")+ .some((segment) => !segment || segment === "." || segment === "..")+ ) {+ return false;+ }+ return SAFE_REPORT_PATHS.some((pattern) => pattern.test(relativePath));+}++async function relativeFiles(root, current = root) {+ const entries = await readdir(current, { withFileTypes: true });+ const files = [];+ for (const entry of entries) {+ const absolute = join(current, entry.name);+ if (entry.isSymbolicLink())+ throw new Error(`Refusing public report symlink ${absolute}`);+ if (entry.isDirectory())+ files.push(...(await relativeFiles(root, absolute)));+ else if (entry.isFile())+ files.push(relative(root, absolute).split(sep).join("/"));+ else throw new Error(`Refusing unusual public report path ${absolute}`);+ }+ return files.sort();+}++function internalHtmlHrefs(content) {+ return [...content.matchAll(/href\s*=\s*["']([^"']+)["']/giu)]+ .map((match) => match[1])+ .filter((href) => href && !href.startsWith("#"));+}++export async function validatePublicProtocolEvalReport(reportRoot) {+ const root = resolve(reportRoot);+ const files = await relativeFiles(root);+ if (!files.includes("index.html") || !files.includes("campaign.json")) {+ throw new Error(+ "Public protocol eval report requires index.html and campaign.json",+ );+ }+ for (const file of files) {+ if (!isPublicProtocolEvalPath(file)) {+ throw new Error(+ `Refusing non-allowlisted public protocol eval path ${file}`,+ );+ }+ const absolute = resolve(root, ...file.split("/"));+ const metadata = await stat(absolute);+ if (metadata.size === 0 || metadata.size > 12 * 1024 * 1024) {+ throw new Error(+ `Public protocol eval file exceeds its size boundary: ${file}`,+ );+ }+ const content = await readFile(absolute, "utf8");+ for (const pattern of CREDENTIAL_PATTERNS) {+ if (pattern.test(content))+ throw new Error(+ `Public report contains credential/session material: ${file}`,+ );+ }+ if (extname(file) !== ".html") continue;+ for (const pattern of ACTIVE_HTML_PATTERNS) {+ if (pattern.test(content))+ throw new Error(+ `Public report contains active or remote HTML: ${file}`,+ );+ }+ for (const href of internalHtmlHrefs(content)) {+ const clean = href.split("#", 1)[0].split("?", 1)[0];+ const target = resolve(+ root,+ ...file.split("/").slice(0, -1),+ ...clean.split("/"),+ );+ const rel = relative(root, target).split(sep).join("/");+ if (+ !isPublicProtocolEvalPath(rel) ||+ !(await lstat(target).catch(() => null))?.isFile()+ ) {+ throw new Error(+ `Public report contains a broken or unsafe link in ${file}: ${href}`,+ );+ }+ }+ }+ const campaign = await loadObject(join(root, "campaign.json"));+ if (+ campaign.schema !== "paperclip.runner-protocol-eval.campaign/v1" ||+ !SAFE_CAMPAIGN.test(String(campaign.campaignId ?? ""))+ ) {+ throw new Error("Public report campaign metadata is invalid");+ }+ return { files, campaign };+}++export async function createProtocolEvalBundleManifest(reportRoot, campaignId) {+ if (!SAFE_CAMPAIGN.test(campaignId))+ throw new Error("Unsafe protocol eval campaign ID");+ const { files, campaign } =+ await validatePublicProtocolEvalReport(reportRoot);+ if (campaign.campaignId !== campaignId)+ throw new Error("Report campaign ID does not match publication target");+ const entries = await Promise.all(+ files.map(async (file) => {+ const absolute = resolve(reportRoot, ...file.split("/"));+ const [content, metadata] = await Promise.all([+ readFile(absolute),+ stat(absolute),+ ]);+ return {+ path: file,+ sha256: createHash("sha256").update(content).digest("hex"),+ bytes: metadata.size,+ };+ }),+ );+ return {+ schema: "paperclip.runner-protocol-eval.bundle/v1",+ campaignId,+ bundleDigest: createHash("sha256")+ .update(JSON.stringify(entries))+ .digest("hex"),+ files: entries,+ };+}++export function emptyProtocolEvalHistory() {+ return {+ schema: "paperclip.runner-protocol-eval.history/v1",+ updatedAt: new Date(0).toISOString(),+ latestCampaignId: null,+ latestGreenCampaignId: null,+ campaigns: [],+ };+}++export function protocolEvalHistoryRecord(campaign, publicRoot) {+ return {+ campaignId: campaign.campaignId,+ generatedAt: campaign.generatedAt,+ publicUrl: `${publicRoot}/campaigns/${encodeURIComponent(campaign.campaignId)}/`,+ complete: campaign.complete === true,+ allPassed: campaign.allPassed === true,+ totals: campaign.totals,+ rosters: campaign.rosters,+ source: campaign.source,+ };+}++export function mergeProtocolEvalHistory(history, record) {+ if (history.schema !== "paperclip.runner-protocol-eval.history/v1") {+ throw new Error("Unsupported protocol eval history schema");+ }+ const existing = history.campaigns.find(+ (item) => item.campaignId === record.campaignId,+ );+ if (existing && JSON.stringify(existing) !== JSON.stringify(record)) {+ throw new Error(+ `Immutable campaign history changed for ${record.campaignId}`,+ );+ }+ const campaigns = existing+ ? history.campaigns+ : [...history.campaigns, record];+ campaigns.sort((left, right) =>+ right.generatedAt.localeCompare(left.generatedAt),+ );+ const latest = campaigns[0] ?? null;+ const latestGreen =+ campaigns.find((campaign) => campaign.complete && campaign.allPassed) ??+ null;+ const retained = campaigns.slice(0, MAX_HISTORY_CAMPAIGNS);+ if (+ latestGreen &&+ !retained.some(+ (campaign) => campaign.campaignId === latestGreen.campaignId,+ )+ ) {+ retained[retained.length - 1] = latestGreen;+ }+ return {+ schema: history.schema,+ updatedAt: new Date().toISOString(),+ latestCampaignId: latest?.campaignId ?? null,+ latestGreenCampaignId: latestGreen?.campaignId ?? null,+ campaigns: retained,+ };+}++export function buildProtocolEvalPointers(history) {+ const byId = new Map(+ history.campaigns.map((campaign) => [campaign.campaignId, campaign]),+ );+ const project = (id) => {+ const campaign = id ? byId.get(id) : null;+ return campaign+ ? {+ campaignId: campaign.campaignId,+ generatedAt: campaign.generatedAt,+ publicUrl: campaign.publicUrl,+ paperclipSha: campaign.source?.paperclip?.sha ?? null,+ evalsSha: campaign.source?.evals?.sha ?? null,+ }+ : null;+ };+ return {+ latest: {+ schema: "paperclip.runner-protocol-eval.pointer/v1",+ updatedAt: history.updatedAt,+ campaign: project(history.latestCampaignId),+ },+ latestGreen: {+ schema: "paperclip.runner-protocol-eval.pointer/v1",+ updatedAt: history.updatedAt,+ campaign: project(history.latestGreenCampaignId),+ },+ };+}++function date(value) {+ return new Intl.DateTimeFormat("en-US", {+ dateStyle: "medium",+ timeStyle: "short",+ timeZone: "UTC",+ }).format(new Date(value));+}++export function renderProtocolEvalHistoryIndex(history) {+ const rows = history.campaigns.length+ ? history.campaigns+ .map((campaign) => {+ const status =+ campaign.complete && campaign.allPassed ? "passed" : "failed";+ const rosters = campaign.rosters+ .map(+ (roster) =>+ `${html(roster.model)} · ${roster.passed}/${roster.selected}`,+ )+ .join("<br>");+ return `<tr><td><a href="${html(campaign.publicUrl)}"><code>${html(campaign.campaignId)}</code></a><small>${html(date(campaign.generatedAt))} UTC</small></td><td><span class="status ${status}">${status}</span></td><td><strong>${html(campaign.totals.passed)}/${html(campaign.totals.selected)}</strong><small>${html(campaign.totals.behaviorFailures)} behavior · ${html(campaign.totals.infrastructureFailures)} infrastructure</small></td><td>${rosters}</td><td><code>${html(campaign.source?.paperclip?.sha?.slice(0, 8) ?? "unknown")}</code><small>evals ${html(campaign.source?.evals?.sha?.slice(0, 8) ?? "unknown")}</small></td><td><a href="${html(campaign.publicUrl)}">Open Evalbook →</a></td></tr>`;+ })+ .join("")+ : '<tr><td colspan="6" class="empty">No campaigns have been published yet.</td></tr>';+ const latest = history.campaigns.find(+ (campaign) => campaign.campaignId === history.latestCampaignId,+ );+ const latestGreen = history.campaigns.find(+ (campaign) => campaign.campaignId === history.latestGreenCampaignId,+ );+ return `<!doctype html>+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><meta name="color-scheme" content="light dark"><title>Runner protocol eval campaigns · Paperclip</title>+<style>:root{color-scheme:light dark;--bg:#fff;--fg:#172019;--muted:#667069;--line:#dfe3dc;--raised:#f8f9f6;--pass:#17603a;--pass-bg:#e5f3e9;--fail:#942f2b;--fail-bg:#f9e5e3} @media(prefers-color-scheme:dark){:root{--bg:#141413;--fg:#fafafa;--muted:#aaa;--line:#ffffff1f;--raised:#1c1c1b;--pass:#65d58c;--pass-bg:#22c55e1f;--fail:#ff7770;--fail-bg:#dc26262e}} *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.5 ui-sans-serif,system-ui,sans-serif}main{width:min(1560px,calc(100% - 48px));margin:48px auto 72px}h1{margin:0;font-size:clamp(34px,4vw,56px);line-height:1.05;letter-spacing:-.04em}p{max-width:760px;color:var(--muted);font-size:16px}a{color:inherit;text-underline-offset:3px}.pointers{display:flex;gap:10px;margin:28px 0 18px}.pointers a{padding:8px 11px;border:1px solid var(--line);border-radius:8px;background:var(--raised);text-decoration:none}.table{overflow:auto;border:1px solid var(--line);border-radius:12px}table{width:100%;border-collapse:collapse}th,td{padding:13px 14px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}th{background:var(--raised);color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}small{display:block;margin-top:4px;color:var(--muted);font-size:10px}.status{display:inline-block;padding:3px 8px;border-radius:99px;font-size:10px;font-weight:750;text-transform:uppercase}.passed{color:var(--pass);background:var(--pass-bg)}.failed{color:var(--fail);background:var(--fail-bg)}.empty{padding:48px;text-align:center;color:var(--muted)}footer{margin-top:18px;color:var(--muted);font-size:11px}@media(max-width:700px){main{width:calc(100% - 28px);margin-top:28px}}</style></head>+<body><main><div><small>Paperclip quality engineering</small><h1>Runner protocol eval campaigns</h1><p>Versioned direct live-runner Evalbook reports. Full provider transcripts, session identifiers, state, and raw tool evidence remain in access-controlled workflow artifacts.</p></div>+<nav class="pointers">${latest ? `<a href="${html(latest.publicUrl)}">Latest · ${html(latest.campaignId)}</a>` : ""}${latestGreen ? `<a href="${html(latestGreen.publicUrl)}">Latest green · ${html(latestGreen.campaignId)}</a>` : ""}</nav>+<div class="table"><table><thead><tr><th>Campaign</th><th>Status</th><th>Cells</th><th>Models / rosters</th><th>Source</th><th></th></tr></thead><tbody>${rows}</tbody></table></div><footer>Updated ${html(date(history.updatedAt))} UTC · Immutable campaign bundles · Canonical Evalbook layout with public-safe evidence projections</footer></main></body></html>`;+}++function awsObject(bucket, key) {+ return `s3://${bucket}/${key}`;+}++async function objectExists(bucket, key) {+ try {+ await execFileAsync("aws", [+ "s3api",+ "head-object",+ "--bucket",+ bucket,+ "--key",+ key,+ ]);+ return true;+ } catch (error) {+ const detail = String(error?.stderr ?? error?.message ?? error);+ if (/\b(?:404|Not Found|NoSuchKey)\b/iu.test(detail)) return false;+ throw new Error(+ `Unable to inspect protocol eval history object: ${detail.slice(0, 400)}`,+ );+ }+}++async function downloadJson(bucket, key, destination) {+ if (!(await objectExists(bucket, key))) return null;+ await execFileAsync("aws", [+ "s3",+ "cp",+ awsObject(bucket, key),+ destination,+ "--only-show-errors",+ ]);+ return loadObject(destination);+}++async function uploadFile(bucket, key, file, cacheControl) {+ const contentType =+ extname(file) === ".html" ? "text/html; charset=utf-8" : "application/json";+ await execFileAsync("aws", [+ "s3",+ "cp",+ file,+ awsObject(bucket, key),+ "--only-show-errors",+ "--content-type",+ contentType,+ "--cache-control",+ cacheControl,+ ]);+}++async function uploadImmutableReport(bucket, prefix, reportRoot) {+ const cacheControl = "public,max-age=31536000,immutable";+ await execFileAsync("aws", [+ "s3",+ "cp",+ reportRoot,+ awsObject(bucket, prefix),+ "--recursive",+ "--only-show-errors",+ "--cache-control",+ cacheControl,+ "--exclude",+ "*.json",+ ]);+ await uploadFile(+ bucket,+ `${prefix}/campaign.json`,+ resolve(reportRoot, "campaign.json"),+ cacheControl,+ );+}++export async function publishProtocolEvalHistory({ reportRoot, destination }) {+ const validatedDestination =+ validateProtocolEvalHistoryDestination(destination);+ const { campaign } = await validatePublicProtocolEvalReport(reportRoot);+ const manifest = await createProtocolEvalBundleManifest(+ reportRoot,+ campaign.campaignId,+ );+ const temporary = await mkdtemp(+ join(tmpdir(), "runner-protocol-eval-history-"),+ );+ const historyKey = `${validatedDestination.prefix}/history.json`;+ const history = mergeProtocolEvalHistory(+ (await downloadJson(+ validatedDestination.bucket,+ historyKey,+ join(temporary, "history.json"),+ )) ?? emptyProtocolEvalHistory(),+ protocolEvalHistoryRecord(+ campaign,+ `${validatedDestination.publicBaseUrl}/${validatedDestination.prefix}`,+ ),+ );+ const campaignPrefix = `${validatedDestination.prefix}/campaigns/${campaign.campaignId}`;+ const manifestKey = `${campaignPrefix}/bundle-manifest.json`;+ const existing = await downloadJson(+ validatedDestination.bucket,+ manifestKey,+ join(temporary, "existing-manifest.json"),+ );+ if (existing && existing.bundleDigest !== manifest.bundleDigest) {+ throw new Error(+ `Immutable campaign ${campaign.campaignId} already exists with a different digest`,+ );+ }+ if (!existing) {+ await uploadImmutableReport(+ validatedDestination.bucket,+ campaignPrefix,+ reportRoot,+ );+ const manifestFile = join(temporary, "bundle-manifest.json");+ await writeFile(manifestFile, json(manifest));+ await uploadFile(+ validatedDestination.bucket,+ manifestKey,+ manifestFile,+ "public,max-age=31536000,immutable",+ );+ }+ const pointers = buildProtocolEvalPointers(history);+ const mutable = {+ "history.json": history,+ "latest.json": pointers.latest,+ "latest-green.json": pointers.latestGreen,+ };+ for (const [name, value] of Object.entries(mutable)) {+ const file = join(temporary, name);+ await writeFile(file, json(value));+ await uploadFile(+ validatedDestination.bucket,+ `${validatedDestination.prefix}/${name}`,+ file,+ "no-cache",+ );+ }+ const index = join(temporary, "index.html");+ await writeFile(index, renderProtocolEvalHistoryIndex(history));+ await uploadFile(+ validatedDestination.bucket,+ `${validatedDestination.prefix}/index.html`,+ index,+ "no-cache",+ );+ return {+ campaignId: campaign.campaignId,+ bundleDigest: manifest.bundleDigest,+ historySize: history.campaigns.length,+ };+}++async function main() {+ const result = await publishProtocolEvalHistory({+ reportRoot: resolve(+ process.env.PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR ??+ "runner-protocol-eval-public-report",+ ),+ destination: {+ bucket: process.env.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET ?? "",+ prefix:+ process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX ??+ "runner-protocol-evals",+ publicBaseUrl:+ process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL ?? "",+ },+ });+ console.log(+ `Published immutable protocol eval campaign ${result.campaignId} (${result.bundleDigest}) and ${result.historySize} history record(s)`,+ );+}++if (+ process.argv[1] &&+ resolve(process.argv[1]) === resolve(import.meta.filename)+) {+ await main().catch((error) => {+ console.error(error instanceof Error ? error.message : String(error));+ process.exitCode = 1;+ });+}diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjsnew file mode 100644index 0000000000..835144fdc5--- /dev/null+++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs@@ -0,0 +1,221 @@+import assert from "node:assert/strict";+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import test from "node:test";++import {+ buildProtocolEvalPointers,+ createProtocolEvalBundleManifest,+ emptyProtocolEvalHistory,+ isPublicProtocolEvalPath,+ mergeProtocolEvalHistory,+ protocolEvalHistoryRecord,+ renderProtocolEvalHistoryIndex,+ validateProtocolEvalHistoryDestination,+ validatePublicProtocolEvalReport,+} from "./publish-runner-protocol-eval-history.mjs";++const roots = [];+test.afterEach(async () => {+ await Promise.all(+ roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),+ );+});++function campaign(overrides = {}) {+ return {+ schema: "paperclip.runner-protocol-eval.campaign/v1",+ campaignId: "gha-42-1",+ generatedAt: "2026-09-05T00:00:00.000Z",+ source: {+ paperclip: { sha: "a".repeat(40), ref: "refs/heads/master" },+ evals: { repository: "paperclipai/paperclip-evals", sha: "b".repeat(40) },+ },+ complete: true,+ allPassed: true,+ totals: {+ selected: 35,+ passed: 35,+ behaviorFailures: 0,+ infrastructureFailures: 0,+ },+ rosters: [+ {+ rosterId: "protocol-live-mini",+ model: "gpt-5.4-mini",+ provider: "codex",+ driver: "codex_app_server",+ selected: 35,+ passed: 35,+ },+ ],+ results: [],+ ...overrides,+ };+}++async function reportFixture() {+ const root = await mkdtemp(join(tmpdir(), "runner-protocol-public-"));+ roots.push(root);+ await Promise.all([+ mkdir(join(root, "tests"), { recursive: true }),+ mkdir(join(root, "attempts"), { recursive: true }),+ ]);+ await Promise.all([+ writeFile(join(root, "campaign.json"), JSON.stringify(campaign())),+ writeFile(+ join(root, "index.html"),+ '<!doctype html><a href="tests/get-context.html">test</a><a href="attempts/attempt-01.html">attempt</a>',+ ),+ writeFile(+ join(root, "tests/get-context.html"),+ '<a href="../index.html">overview</a>',+ ),+ writeFile(+ join(root, "attempts/attempt-01.html"),+ '<a href="../index.html">overview</a>',+ ),+ ]);+ return root;+}++test("accepts only credential-free HTTPS destinations and the dedicated prefix", () => {+ assert.deepEqual(+ validateProtocolEvalHistoryDestination({+ bucket: "paperclip-public-reports",+ prefix: "/runner-protocol-evals/",+ publicBaseUrl: "https://reports.paperclip.ing/",+ }),+ {+ bucket: "paperclip-public-reports",+ prefix: "runner-protocol-evals",+ publicBaseUrl: "https://reports.paperclip.ing",+ },+ );+ assert.throws(+ () =>+ validateProtocolEvalHistoryDestination({+ bucket: "bad",+ prefix: "../evals",+ publicBaseUrl: "http://example.test",+ }),+ /prefix|HTTPS/,+ );+});++test("keeps publication to the canonical static Evalbook surface", () => {+ for (const file of [+ "index.html",+ "latest.html",+ "inventory.html",+ "tests/get-context.html",+ "attempts/attempt-01.html",+ "campaign.json",+ ]) {+ assert.equal(isPublicProtocolEvalPath(file), true, file);+ }+ for (const file of [+ "attempts/attempt-01/index.html",+ "runs/attempt/artifact.json",+ "provider-trace.log",+ "../secret",+ "assets/app.js",+ ]) {+ assert.equal(isPublicProtocolEvalPath(file), false, file);+ }+});++test("validates inert linked reports and creates deterministic manifests", async () => {+ const root = await reportFixture();+ const result = await validatePublicProtocolEvalReport(root);+ assert.equal(result.files.length, 4);+ const first = await createProtocolEvalBundleManifest(root, "gha-42-1");+ const second = await createProtocolEvalBundleManifest(root, "gha-42-1");+ assert.deepEqual(second, first);+ assert.match(first.bundleDigest, /^[0-9a-f]{64}$/);+});++test("rejects scripts, remote resources, raw sessions, secret-shaped values, and broken links", async () => {+ for (const bad of [+ "<script>alert(1)</script>",+ '<a href="https://attacker.example">remote</a>',+ '<pre>{"providerSessionId":"session"}</pre>',+ `<pre>${"sk-" + "a".repeat(24)}</pre>`,+ '<a href="missing.html">broken</a>',+ ]) {+ const root = await reportFixture();+ await writeFile(join(root, "index.html"), bad);+ await assert.rejects(validatePublicProtocolEvalReport(root));+ }+});++test("retains immutable history and independent latest-green pointers", () => {+ const green = protocolEvalHistoryRecord(+ campaign(),+ "https://reports.example/runner-protocol-evals",+ );+ const red = protocolEvalHistoryRecord(+ campaign({+ campaignId: "gha-43-1",+ generatedAt: "2026-09-05T01:00:00.000Z",+ allPassed: false,+ totals: {+ selected: 35,+ passed: 34,+ behaviorFailures: 1,+ infrastructureFailures: 0,+ },+ rosters: [{ ...campaign().rosters[0], passed: 34 }],+ }),+ "https://reports.example/runner-protocol-evals",+ );+ let history = mergeProtocolEvalHistory(emptyProtocolEvalHistory(), green);+ history = mergeProtocolEvalHistory(history, red);+ assert.equal(history.latestCampaignId, "gha-43-1");+ assert.equal(history.latestGreenCampaignId, "gha-42-1");+ const pointers = buildProtocolEvalPointers(history);+ assert.equal(pointers.latest.campaign.campaignId, "gha-43-1");+ assert.equal(pointers.latestGreen.campaign.campaignId, "gha-42-1");+ const index = renderProtocolEvalHistoryIndex(history);+ assert.match(index, /Runner protocol eval campaigns/);+ assert.match(index, /Open Evalbook/);+ assert.match(index, /34\/35/);+ assert.doesNotMatch(index, /private-session-id|sk-[A-Za-z0-9_-]{20,}/);+ assert.throws(+ () => mergeProtocolEvalHistory(history, { ...green, allPassed: false }),+ /Immutable campaign history changed/,+ );+});++test("retains the latest green pointer outside the 200 newest campaigns", () => {+ const green = protocolEvalHistoryRecord(+ campaign(),+ "https://reports.example/runner-protocol-evals",+ );+ let history = mergeProtocolEvalHistory(emptyProtocolEvalHistory(), green);+ for (let index = 0; index < 201; index += 1) {+ history = mergeProtocolEvalHistory(+ history,+ protocolEvalHistoryRecord(+ campaign({+ campaignId: `gha-${index + 43}-1`,+ generatedAt: new Date(+ Date.UTC(2026, 8, 5, 1, 0, index),+ ).toISOString(),+ allPassed: false,+ }),+ "https://reports.example/runner-protocol-evals",+ ),+ );+ }++ assert.equal(history.campaigns.length, 200);+ assert.equal(history.latestCampaignId, "gha-243-1");+ assert.equal(history.latestGreenCampaignId, "gha-42-1");+ assert.equal(history.campaigns.at(-1).campaignId, "gha-42-1");+ assert.equal(+ buildProtocolEvalPointers(history).latestGreen.campaign.campaignId,+ "gha-42-1",+ );+});diff --git a/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjsnew file mode 100644index 0000000000..9ebafc19a4--- /dev/null+++ b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs@@ -0,0 +1,352 @@+import { createHash } from "node:crypto";+import { spawn } from "node:child_process";+import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises";+import { resolve } from "node:path";++const EVAL_PROGRAM_RELATIVE_PATH =+ "evals/paperclip-runner/tools/eval_program.py";++function json(value) {+ return `${JSON.stringify(value, null, 2)}\n`;+}++function sha256(value) {+ return createHash("sha256").update(value).digest("hex");+}++function safeSegment(value) {+ const segment = String(value)+ .trim()+ .replaceAll(/[^0-9A-Za-z._-]+/g, "-")+ .replaceAll(/^-+|-+$/g, "");+ if (!segment) throw new Error("Evalbook attempt identity is empty");+ return segment;+}++function candidateDescriptor(report, candidateId) {+ const descriptor =+ report.bundle.providerVersions?.[candidateId] ?? candidateId;+ const separator = descriptor.indexOf(":");+ return separator < 0+ ? { driver: "unknown driver", model: descriptor }+ : {+ driver: descriptor.slice(0, separator),+ model: descriptor.slice(separator + 1),+ };+}++function scoreChecks(result) {+ const dimensionChecks = Object.values(result.scorecard.dimensions).map(+ (dimension) => ({+ id: dimension.dimension,+ kind: "workflow_dimension",+ passed: dimension.passed === true,+ detail:+ dimension.score === null+ ? dimension.reasons.join("; ") || "not scored"+ : `score ${dimension.score}${+ dimension.reasons.length === 0+ ? ""+ : `; ${dimension.reasons.join("; ")}`+ }`,+ evidenceRefs: [],+ }),+ );+ const observationChecks = [+ ["lifecycle", result.observation.lifecycle?.checks ?? []],+ ["continuation", result.observation.continuation?.checks ?? []],+ ["presentation", result.observation.presentation?.checks ?? []],+ ].flatMap(([group, checks]) =>+ checks.map((check) => ({+ id: `${group}.${check.id}`,+ kind: `${group}_check`,+ passed: check.passed === true,+ detail: check.reason ?? (check.passed ? "passed" : "failed"),+ evidenceRefs: [],+ })),+ );+ return [...dimensionChecks, ...observationChecks];+}++function disposition(result) {+ if (result.scorecard.overall.passed === true) return "passed";+ if (+ result.observation.classification === "infrastructure_failure" ||+ result.observation.classification === "skipped"+ ) {+ return "infrastructure_failure";+ }+ return "behavior_failure";+}++function infrastructureErrors(result) {+ if (disposition(result) !== "infrastructure_failure") return [];+ return [+ result.observation.failure?.message ??+ `workflow execution was ${result.observation.classification}`,+ ];+}++export function runnerWorkflowEvalbookAttempt({+ report,+ result,+ caseDefinition,+}) {+ const { driver, model } = candidateDescriptor(report, result.candidateId);+ const identity = {+ generatedAt: report.generatedAt,+ bundleId: report.bundle.id,+ caseId: result.scenarioId,+ candidateId: result.candidateId,+ };+ const timestamp = safeSegment(report.generatedAt);+ const attemptId = `${timestamp}-${safeSegment(result.scenarioId)}-${safeSegment(+ result.candidateId,+ )}-${sha256(JSON.stringify(identity)).slice(0, 10)}`;+ const checks = scoreChecks(result);+ const costUsd = result.observation.metrics.costUsd;+ const infrastructure =+ disposition(result) === "infrastructure_failure"+ ? result.observation.failure+ : undefined;+ const artifact = {+ schema: "paperclip-runner/workflow-eval-artifact/v1",+ attemptId,+ createdAt: report.generatedAt,+ requestedModel: model,+ provider: result.observation.provider,+ driver,+ providerVersion:+ report.bundle.runnerBuild ?? report.bundle.runnerVersion ?? "unknown",+ providerSessionId: result.observation.base.trace.sessionId,+ retainedSession: false,+ retainedSessionStatus: "not retained by safe workflow eval projection",+ usage: {+ agentTurns: result.observation.metrics.attempts,+ inputTokens: result.observation.metrics.totalTokens,+ cachedInputTokens: 0,+ outputTokens: 0,+ reasoningTokens: 0,+ ...(costUsd === undefined+ ? {}+ : { estimatedCostNanodollars: Math.round(costUsd * 1_000_000_000) }),+ },+ turn: {+ status:+ result.observation.classification === "completed"+ ? "completed"+ : "failed",+ },+ snapshot: {+ createdAt: report.generatedAt,+ providerModel: {+ id: model,+ provider: result.observation.provider,+ },+ transcript: [],+ evidence: [],+ },+ devtools: { revisions: [] },+ ...(infrastructure === undefined+ ? {}+ : {+ infrastructureFailure: {+ class: infrastructure.code,+ category: infrastructure.category,+ retryable: infrastructure.retryable,+ },+ }),+ workflow: {+ bundle: report.bundle,+ observation: result.observation,+ scorecard: result.scorecard,+ },+ };+ const score = {+ schema: "paperclip-runner/eval-score/v1",+ attemptId,+ caseId: result.scenarioId,+ checks,+ disposition: disposition(result),+ infrastructureErrors: infrastructureErrors(result),+ passed: result.scorecard.overall.passed === true,+ digest: `sha256:${sha256(JSON.stringify({ identity, checks }))}`,+ };+ const evalCase = {+ schema: "paperclip-runner/workflow-eval-case/v1",+ id: result.scenarioId,+ title: caseDefinition?.title ?? result.scenarioId,+ description: caseDefinition+ ? `Runner workflow case. Tags: ${caseDefinition.tags.join(", ")}.`+ : "Runner workflow case.",+ prompt:+ "Redacted by the safe Runner workflow eval projection; inspect the authored workflow case for orchestration steps.",+ fixture: "runner-workflow-live-harness",+ authority: {+ controlPlaneOwned: result.observation.base.controlPlaneOwned,+ },+ checks: Object.entries(caseDefinition?.assertions ?? {}).map(+ ([id, expected]) => ({+ id,+ kind: "workflow_assertion",+ expected,+ }),+ ),+ ...(caseDefinition === undefined+ ? {}+ : { workflowDefinition: caseDefinition }),+ };+ const config = {+ schema: "paperclip-runner/workflow-eval-config/v1",+ id: result.candidateId,+ model,+ provider: result.observation.provider,+ driver,+ runnerVersion: report.bundle.runnerVersion,+ runnerBuild: report.bundle.runnerBuild,+ promptPolicyId: report.bundle.promptPolicyId,+ };+ return { attemptId, artifact, score, case: evalCase, config };+}++async function writeImmutable(path, value) {+ const content = json(value);+ try {+ const existing = await readFile(path, "utf8");+ if (existing !== content) {+ throw new Error(`Immutable Evalbook record changed: ${path}`);+ }+ } catch (error) {+ if (error?.code !== "ENOENT") throw error;+ await writeFile(path, content, { flag: "wx", mode: 0o600 });+ }+}++export async function writeRunnerWorkflowEvalbookAttempts({+ report,+ runsRoot,+ caseForId,+}) {+ await mkdir(runsRoot, { recursive: true });+ const attempts = [];+ for (const result of report.results) {+ const attempt = runnerWorkflowEvalbookAttempt({+ report,+ result,+ caseDefinition: caseForId?.(result.scenarioId),+ });+ const directory = resolve(runsRoot, attempt.attemptId);+ await mkdir(directory, { recursive: true, mode: 0o700 });+ await Promise.all([+ writeImmutable(resolve(directory, "artifact.json"), attempt.artifact),+ writeImmutable(resolve(directory, "score.json"), attempt.score),+ writeImmutable(resolve(directory, "case.json"), attempt.case),+ writeImmutable(resolve(directory, "config.json"), attempt.config),+ ]);+ attempts.push(attempt.attemptId);+ }+ return attempts;+}++async function existingPath(paths) {+ for (const path of paths.filter(Boolean)) {+ try {+ await access(path);+ return resolve(path);+ } catch {+ // Continue through the explicit and conventional checkout locations.+ }+ }+ return null;+}++export async function resolveCanonicalEvalProgram(packageRoot, environment) {+ const fromRoot = environment.PAPERCLIP_EVALS_ROOT+ ? resolve(environment.PAPERCLIP_EVALS_ROOT, EVAL_PROGRAM_RELATIVE_PATH)+ : null;+ const program = await existingPath([+ environment.PAPERCLIP_EVALBOOK_PROGRAM,+ fromRoot,+ resolve(packageRoot, "../../.paperclip-evals", EVAL_PROGRAM_RELATIVE_PATH),+ resolve(+ packageRoot,+ "../../../paperclip-evals",+ EVAL_PROGRAM_RELATIVE_PATH,+ ),+ resolve(+ packageRoot,+ "../../../../paperclip-evals",+ EVAL_PROGRAM_RELATIVE_PATH,+ ),+ ]);+ if (program !== null) return program;+ throw new Error(+ `Canonical Evalbook generator not found. Set PAPERCLIP_EVALBOOK_PROGRAM to ${EVAL_PROGRAM_RELATIVE_PATH} in a paperclip-evals checkout.`,+ );+}++async function run(command, args) {+ await new Promise((accept, reject) => {+ const child = spawn(command, args, { stdio: "inherit" });+ child.once("error", reject);+ child.once("exit", (code, signal) => {+ if (code === 0) accept();+ else+ reject(+ new Error(+ `${command} exited ${code ?? `for signal ${signal ?? "unknown"}`}`,+ ),+ );+ });+ });+}++export async function renderRunnerWorkflowWithCanonicalEvalbook({+ packageRoot,+ outputDirectory,+ report,+ caseForId,+ environment = process.env,+}) {+ const program = await resolveCanonicalEvalProgram(packageRoot, environment);+ const runsRoot = resolve(outputDirectory, "evalbook-runs");+ await rm(runsRoot, { recursive: true, force: true });+ const attempts = await writeRunnerWorkflowEvalbookAttempts({+ report,+ runsRoot,+ caseForId,+ });+ await Promise.all([+ rm(resolve(outputDirectory, "attempts"), { recursive: true, force: true }),+ rm(resolve(outputDirectory, "tests"), { recursive: true, force: true }),+ rm(resolve(outputDirectory, "index.html"), { force: true }),+ rm(resolve(outputDirectory, "latest.html"), { force: true }),+ rm(resolve(outputDirectory, "live-report.html"), { force: true }),+ rm(resolve(outputDirectory, "deterministic-report.html"), { force: true }),+ ]);+ await run(environment.PYTHON ?? "python3", [+ program,+ "report",+ "--runs-root",+ runsRoot,+ "--output",+ outputDirectory,+ ]);+ const programBytes = await readFile(program);+ const manifest = {+ schema: "paperclip.runner.workflow-evalbook-render.v1",+ generatedAt: report.generatedAt,+ generator: {+ path: program,+ sha256: `sha256:${sha256(programBytes)}`,+ },+ sourceReport: "live-report.json",+ attempts,+ index: resolve(outputDirectory, "index.html"),+ };+ await writeFile(+ resolve(outputDirectory, "evalbook-manifest.json"),+ json(manifest),+ );+ return manifest;+}diff --git a/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjsnew file mode 100644index 0000000000..54fe993633--- /dev/null+++ b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs@@ -0,0 +1,134 @@+import assert from "node:assert/strict";+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { resolve } from "node:path";+import test from "node:test";++import {+ runnerWorkflowEvalbookAttempt,+ writeRunnerWorkflowEvalbookAttempts,+} from "./render-runner-workflow-evalbook.mjs";++function fixture(overrides = {}) {+ const report = {+ generatedAt: "2026-09-05T16:30:00.000Z",+ bundle: {+ id: "runner-live-v2-test",+ runnerVersion: "1.2.3",+ runnerBuild: "abc123",+ promptPolicyId: "runner-live-workflow-v1",+ providerVersions: {+ "codex-luna": "codex_app_server:gpt-5.6-luna",+ },+ },+ results: [],+ };+ const result = {+ scenarioId: "verification-policy",+ candidateId: "codex-luna",+ observation: {+ classification: "completed",+ provider: "codex",+ base: {+ controlPlaneOwned: false,+ trace: { sessionId: "eval-session-1" },+ },+ lifecycle: {+ checks: [{ id: "terminal-authority", passed: true }],+ },+ continuation: { checks: [] },+ presentation: { checks: [] },+ metrics: {+ attempts: 1,+ totalTokens: 1234,+ costUsd: 0.0025,+ },+ ...overrides.observation,+ },+ scorecard: {+ dimensions: {+ semantic_outcome: {+ dimension: "semantic_outcome",+ score: 1,+ passed: true,+ reasons: [],+ },+ },+ overall: { score: 1, passed: true },+ ...overrides.scorecard,+ },+ };+ report.results.push(result);+ return { report, result };+}++test("maps a workflow result to the canonical immutable-attempt inputs", () => {+ const { report, result } = fixture();+ const attempt = runnerWorkflowEvalbookAttempt({+ report,+ result,+ caseDefinition: {+ title: "Verify before completion",+ tags: ["verification"],+ },+ });++ assert.match(attempt.attemptId, /verification-policy-codex-luna/);+ assert.equal(attempt.artifact.requestedModel, "gpt-5.6-luna");+ assert.equal(attempt.artifact.driver, "codex_app_server");+ assert.equal(attempt.artifact.usage.estimatedCostNanodollars, 2_500_000);+ assert.equal(attempt.score.disposition, "passed");+ assert.equal(attempt.config.id, "codex-luna");+ assert.equal(attempt.case.title, "Verify before completion");+ assert.match(attempt.case.prompt, /Redacted/);+});++test("preserves unscored infrastructure semantics for the grid", () => {+ const { report, result } = fixture({+ observation: {+ classification: "infrastructure_failure",+ failure: {+ code: "provider_timeout",+ category: "provider",+ retryable: true,+ message: "provider timed out",+ },+ },+ scorecard: {+ overall: { score: null, passed: null },+ },+ });+ const attempt = runnerWorkflowEvalbookAttempt({ report, result });++ assert.equal(attempt.score.disposition, "infrastructure_failure");+ assert.deepEqual(attempt.score.infrastructureErrors, ["provider timed out"]);+ assert.deepEqual(attempt.artifact.infrastructureFailure, {+ class: "provider_timeout",+ category: "provider",+ retryable: true,+ });+});++test("writes idempotent attempt records without raw provider content", async () => {+ const root = await mkdtemp(resolve(tmpdir(), "runner-workflow-evalbook-"));+ try {+ const { report } = fixture();+ const options = {+ report,+ runsRoot: root,+ caseForId: () => ({ title: "Verify", tags: ["verification"] }),+ };+ const first = await writeRunnerWorkflowEvalbookAttempts(options);+ const second = await writeRunnerWorkflowEvalbookAttempts(options);+ assert.deepEqual(second, first);++ const artifact = JSON.parse(+ await readFile(resolve(root, first[0], "artifact.json"), "utf8"),+ );+ assert.deepEqual(artifact.snapshot.transcript, []);+ assert.deepEqual(artifact.snapshot.evidence, []);+ assert.equal(artifact.workflow.observation.provider, "codex");+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});diff --git a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjsindex 19217335e3..e78643209f 100644--- a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs+++ b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs@@ -9,6 +9,8 @@ import {} from "node:fs/promises";import { resolve } from "node:path";+import { renderRunnerWorkflowWithCanonicalEvalbook } from "./render-runner-workflow-evalbook.mjs";+const packageRoot = resolve(import.meta.dirname, "..");const evals = await import(resolve(packageRoot, "dist/eval/index.js"));const packageManifest = JSON.parse(@@ -21,8 +23,7 @@ if (mode !== "nightly" && mode !== "chaos")throw new Error(`unsupported workflow eval schedule mode: ${mode}`);const now = process.env.PAPERCLIP_EVAL_GENERATED_AT ?? new Date().toISOString();const rotationDay = Number(- process.env.PAPERCLIP_EVAL_ROTATION_DAY ??- evals.runnerLiveRotationWeek(now),+ process.env.PAPERCLIP_EVAL_ROTATION_DAY ?? evals.runnerLiveRotationWeek(now),);const seed =process.env.PAPERCLIP_EVAL_SCHEDULE_SEED ?? "runner-live-seven-week-v1";@@ -35,6 +36,51 @@ const historyDirectory = resolve();await mkdir(outputDirectory, { recursive: true });+function selectorValues(flag, environmentName) {+ const values = [];+ for (let index = 0; index < process.argv.length; index += 1) {+ if (process.argv[index] !== flag) continue;+ const value = process.argv[index + 1];+ if (!value || value.startsWith("--")) {+ throw new Error(`${flag} requires a comma-separated value`);+ }+ values.push(value);+ }+ if (process.env[environmentName]) values.push(process.env[environmentName]);+ return [+ ...new Set(+ values.flatMap((value) =>+ value+ .split(",")+ .map((entry) => entry.trim())+ .filter(Boolean),+ ),+ ),+ ];+}++function selectionLimit() {+ const index = process.argv.indexOf("--limit");+ const raw =+ index < 0 ? process.env.PAPERCLIP_EVAL_LIMIT : process.argv[index + 1];+ if (raw === undefined || raw === "") return undefined;+ const value = Number(raw);+ if (!Number.isSafeInteger(value) || value <= 0) {+ throw new Error("--limit must be a positive integer");+ }+ return value;+}++const selection = {+ candidateIds: selectorValues("--candidate", "PAPERCLIP_EVAL_CANDIDATE"),+ caseIds: selectorValues("--case", "PAPERCLIP_EVAL_CASE"),+ limit: selectionLimit(),+};+const selectionActive =+ selection.candidateIds.length > 0 ||+ selection.caseIds.length > 0 ||+ selection.limit !== undefined;+function safeBundleId(schedule) {const runnerBuild =process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version;@@ -51,6 +97,13 @@ function safeBundleId(schedule) {reasoningEffort,}),),+ ...(selectionActive+ ? {+ selectedExecutions: schedule.entries.map(+ (entry) => entry.executionId,+ ),+ }+ : {}),});return `runner-live-v2-${createHash("sha256").update(identity).digest("hex").slice(0, 16)}`;}@@ -110,7 +163,7 @@ async function retainHistory(report) {}if (mode === "nightly") {- const schedule = evals.buildRunnerLiveEvalSchedule({+ const fullSchedule = evals.buildRunnerLiveEvalSchedule({seed,rotationDay,generatedAt: now,@@ -120,9 +173,12 @@ if (mode === "nightly") {throw new Error(`live schedule coverage failed: ${JSON.stringify(coverage)}`,);+ const schedule = selectionActive+ ? evals.selectRunnerLiveEvalSchedule(fullSchedule, selection)+ : fullSchedule;await writeFile(resolve(outputDirectory, "nightly-schedule.json"),- `${JSON.stringify({ schedule, coverage }, null, 2)}\n`,+ `${JSON.stringify({ schedule, coverage, selection: selectionActive ? selection : null }, null, 2)}\n`,);if (!execute) {process.stdout.write(@@ -230,9 +286,15 @@ if (mode === "nightly") {`${evals.renderRunnerWorkflowGitHubSummary(report, alerts)}\n## Previous compatible bundle\n\n${comparison === null ? "No compatible prior bundle is available." : `Pass-rate delta: ${comparison.passRateDelta}; overall delta: ${comparison.overallDelta}.`}\n`,),]);+ await renderRunnerWorkflowWithCanonicalEvalbook({+ packageRoot,+ outputDirectory,+ report,+ caseForId: evals.runnerWorkflowCase,+ });await retainHistory(report);process.stdout.write(- `Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s).\n`,+ `Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s). Open ${resolve(outputDirectory, "index.html")}.\n`,);} else {const payload = {diff --git a/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjsindex 80273bcd87..e52c720ad3 100644--- a/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs+++ b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs@@ -3,30 +3,62 @@ import { resolve } from "node:path";const packageRoot = resolve(import.meta.dirname, "..");const evals = await import(resolve(packageRoot, "dist/eval/index.js"));-const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));-const traceability = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8"));-const traceabilitySummary = evals.validateStressTraceabilityManifest(traceability);-const results = await evals.runDeterministicRunnerWorkflowMatrix({ bundleId: "runner-workflows-deterministic-v1" });+const packageManifest = JSON.parse(+ await readFile(resolve(packageRoot, "package.json"), "utf8"),+);+const traceability = JSON.parse(+ await readFile(+ resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"),+ "utf8",+ ),+);+const traceabilitySummary =+ evals.validateStressTraceabilityManifest(traceability);+const results = await evals.runDeterministicRunnerWorkflowMatrix({+ bundleId: "runner-workflows-deterministic-v1",+});const report = evals.buildRunnerWorkflowEvalReport({source: "deterministic",bundle: {id: "runner-workflows-deterministic-v1",runnerVersion: packageManifest.version,promptPolicyId: "stress-sanitized-v1",- providerVersions: { codex: "fixture-v1", opencode: "fixture-v1", acpx: "fixture-v1" },+ providerVersions: {+ codex: "fixture-v1",+ opencode: "fixture-v1",+ acpx: "fixture-v1",+ },},results,traceability: traceabilitySummary,});-if (report.aggregate.passed !== report.aggregate.scoreable) {- throw new Error(`Runner workflow evals passed ${report.aggregate.passed}/${report.aggregate.scoreable}`);-}-const outputDirectory = resolve(packageRoot, ".paperclip-local/evals/workflows");+const outputDirectory = resolve(+ packageRoot,+ ".paperclip-local/evals/workflows",+);await mkdir(outputDirectory, { recursive: true });await Promise.all([- writeFile(resolve(outputDirectory, "deterministic-report.json"), `${JSON.stringify(report, null, 2)}\n`),- writeFile(resolve(outputDirectory, "deterministic-report.md"), evals.renderRunnerWorkflowMarkdown(report)),- writeFile(resolve(outputDirectory, "deterministic-report.junit.xml"), evals.renderRunnerWorkflowJUnit(report)),- writeFile(resolve(outputDirectory, "github-summary.md"), evals.renderRunnerWorkflowGitHubSummary(report)),+ writeFile(+ resolve(outputDirectory, "deterministic-report.json"),+ `${JSON.stringify(report, null, 2)}\n`,+ ),+ writeFile(+ resolve(outputDirectory, "deterministic-report.md"),+ evals.renderRunnerWorkflowMarkdown(report),+ ),+ writeFile(+ resolve(outputDirectory, "deterministic-report.junit.xml"),+ evals.renderRunnerWorkflowJUnit(report),+ ),+ writeFile(+ resolve(outputDirectory, "github-summary.md"),+ evals.renderRunnerWorkflowGitHubSummary(report),+ ),]);-process.stdout.write(`Runner workflow evals passed: ${report.aggregate.passed}/${report.aggregate.scoreable}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`);+const outcome =+ report.aggregate.passed === report.aggregate.scoreable+ ? "all scoreable executions passed"+ : `${report.aggregate.passed}/${report.aggregate.scoreable} scoreable executions passed; deterministic fixtures exercised fail-closed scoring`;+process.stdout.write(+ `Runner workflow eval report ready: ${outcome}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`,+);diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjsnew file mode 100644index 0000000000..7314b63748--- /dev/null+++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs@@ -0,0 +1,698 @@+#!/usr/bin/env node+import { createHash } from "node:crypto";+import {+ appendFile,+ cp,+ lstat,+ mkdir,+ readFile,+ readdir,+ rm,+ writeFile,+} from "node:fs/promises";+import { basename, dirname, join, relative, resolve, sep } from "node:path";++const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/;+const ATTEMPT_FILES = new Set([+ "request.json",+ "artifact.json",+ "score.json",+ "compiled-fixture.json",+ "case.json",+ "config.json",+]);++function json(value) {+ return `${JSON.stringify(value, null, 2)}\n`;+}++async function loadObject(path) {+ const value = JSON.parse(await readFile(path, "utf8"));+ if (value === null || Array.isArray(value) || typeof value !== "object") {+ throw new Error(`Expected a JSON object: ${path}`);+ }+ return value;+}++function safeId(value, label = "identifier") {+ const result = String(value ?? "");+ if (!SAFE_ID.test(result)) throw new Error(`Unsafe ${label}: ${result}`);+ return result;+}++function inside(root, candidate, label) {+ const rel = relative(resolve(root), resolve(candidate));+ if (!rel || rel === ".." || rel.startsWith(`..${sep}`)) {+ throw new Error(`${label} escapes its declared root`);+ }+ return resolve(candidate);+}++export function credentialForConfig(config) {+ if (config.provider === "opencode") return "OPENROUTER_API_KEY";+ if (config.provider === "claude_managed") return "ANTHROPIC_API_KEY";+ if (config.provider === "aws_agentcore") return "AWS_AGENTCORE_OIDC";+ if (config.provider === "codex" || config.provider === undefined) {+ return "OPENAI_API_KEY";+ }+ if (config.provider === "acpx") {+ if (config.acpxAgent === "pi") return "OPENROUTER_API_KEY";+ if (config.acpxAgent === "claude") return "ANTHROPIC_API_KEY";+ if (config.acpxAgent === "codex") return "OPENAI_API_KEY";+ }+ throw new Error(+ `No credential policy for ${config.provider ?? "codex"}/${config.acpxAgent ?? "default"}`,+ );+}++function parseRosterSelection(value) {+ if (!value?.trim() || value.trim() === "all") return null;+ const selected = value+ .split(",")+ .map((entry) => entry.trim())+ .filter(Boolean);+ if (selected.length === 0 || new Set(selected).size !== selected.length) {+ throw new Error("Roster selection must contain unique comma-separated IDs");+ }+ return new Set(selected);+}++export async function buildProtocolEvalCatalog({+ evalsRoot,+ rosterSelection = "all",+ campaignId,+ source = {},+ maxParallel = 100,+}) {+ safeId(campaignId, "campaign ID");+ if (+ !Number.isSafeInteger(maxParallel) ||+ maxParallel < 2 ||+ maxParallel > 100+ ) {+ throw new Error("maxParallel must be an integer from 2 through 100");+ }+ const programRoot = resolve(evalsRoot, "evals/paperclip-runner");+ const rosterRoot = resolve(programRoot, "rosters");+ const selected = parseRosterSelection(rosterSelection);+ const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true }))+ .filter(+ (entry) =>+ entry.isFile() &&+ entry.name.startsWith("live-") &&+ entry.name.endsWith(".json"),+ )+ .map((entry) => entry.name)+ .sort();+ const rosters = [];+ for (const rosterFile of rosterFiles) {+ const rosterPath = resolve(rosterRoot, rosterFile);+ const roster = await loadObject(rosterPath);+ const rosterId = safeId(roster.id, "roster ID");+ if (selected && !selected.has(rosterId) && !selected.has(rosterFile)) {+ continue;+ }+ if (roster.schema !== "paperclip-runner/live-roster/v1") {+ throw new Error(`Unsupported live roster schema in ${rosterFile}`);+ }+ if (!Array.isArray(roster.cases) || roster.cases.length === 0) {+ throw new Error(`Live roster ${rosterId} has no cases`);+ }+ const configPath = inside(+ programRoot,+ resolve(rosterRoot, String(roster.config ?? "")),+ `Config for ${rosterId}`,+ );+ const config = await loadObject(configPath);+ const credentialName = credentialForConfig(config);+ const cases = roster.cases.map((caseId) => safeId(caseId, "case ID"));+ if (new Set(cases).size !== cases.length) {+ throw new Error(`Live roster ${rosterId} repeats a case`);+ }+ rosters.push({+ rosterId,+ rosterFile,+ configFile: relative(programRoot, configPath).split(sep).join("/"),+ model: String(roster.model ?? config.model ?? "unknown"),+ provider: String(config.provider ?? "codex"),+ driver: String(config.driver ?? "codex_app_server"),+ credentialName,+ cases,+ });+ }+ if (selected) {+ const found = new Set(+ rosters.flatMap((roster) => [roster.rosterId, roster.rosterFile]),+ );+ const missing = [...selected].filter((entry) => !found.has(entry));+ if (missing.length > 0) {+ throw new Error(`Unknown live roster selection: ${missing.join(", ")}`);+ }+ }+ if (rosters.length === 0) throw new Error("No live rosters were selected");++ const cells = rosters.flatMap((roster) =>+ roster.cases.map((caseId) => ({+ cellId: safeId(`${roster.rosterId}--${caseId}`, "cell ID"),+ rosterId: roster.rosterId,+ rosterFile: roster.rosterFile,+ caseId,+ model: roster.model,+ provider: roster.provider,+ driver: roster.driver,+ credentialName: roster.credentialName,+ })),+ );+ const shards = [[], []];+ cells.forEach((cell, index) => shards[index % shards.length].push(cell));+ if (shards.some((shard) => shard.length > 256)) {+ throw new Error(+ "Protocol eval catalog exceeds the two-shard GitHub matrix limit",+ );+ }+ return {+ schema: "paperclip.runner-protocol-eval.catalog/v1",+ campaignId,+ source,+ rosters,+ cells,+ matrices: shards.map((include) => ({ include })),+ maxParallel,+ maxParallelPerShard: Math.floor(maxParallel / 2),+ };+}++async function writeGithubOutput(entries) {+ const output = process.env.GITHUB_OUTPUT;+ if (!output) return;+ await appendFile(+ output,+ Object.entries(entries)+ .map(([key, value]) => `${key}=${value}\n`)+ .join(""),+ );+}++async function copyAttempt(source, destination) {+ const metadata = await lstat(source);+ if (!metadata.isDirectory() || metadata.isSymbolicLink()) {+ throw new Error(`Attempt source is not a real directory: ${source}`);+ }+ await mkdir(destination, { recursive: false, mode: 0o700 });+ for (const entry of await readdir(source, { withFileTypes: true })) {+ if (+ !entry.isFile() ||+ entry.isSymbolicLink() ||+ !ATTEMPT_FILES.has(entry.name)+ ) {+ throw new Error(`Unexpected attempt path ${join(source, entry.name)}`);+ }+ await cp(join(source, entry.name), join(destination, entry.name), {+ errorOnExist: true,+ });+ }+ for (const required of [+ "artifact.json",+ "score.json",+ "case.json",+ "config.json",+ ]) {+ await lstat(join(destination, required));+ }+}++async function findFiles(root, name) {+ const metadata = await lstat(root).catch(() => null);+ if (!metadata) return [];+ if (!metadata.isDirectory() || metadata.isSymbolicLink()) {+ throw new Error(`Download root must be a real directory: ${root}`);+ }+ const found = [];+ async function visit(directory) {+ for (const entry of await readdir(directory, { withFileTypes: true })) {+ const absolute = join(directory, entry.name);+ if (entry.isSymbolicLink())+ throw new Error(`Refusing downloaded symlink: ${absolute}`);+ if (entry.isDirectory()) await visit(absolute);+ else if (entry.isFile() && entry.name === name) found.push(absolute);+ }+ }+ await visit(root);+ return found.sort();+}++function safeUsage(usage) {+ const fields = [+ "agentTurns",+ "providerRequests",+ "inputTokens",+ "outputTokens",+ "cachedInputTokens",+ "reasoningTokens",+ "providerReportedCostNanodollars",+ "estimatedCostNanodollars",+ ];+ return Object.fromEntries(+ fields+ .filter((field) => Number.isFinite(usage?.[field]) && usage[field] >= 0)+ .map((field) => [field, usage[field]]),+ );+}++async function syntheticAttempt({ evalsRoot, runsOut, cell, campaignId }) {+ const programRoot = resolve(evalsRoot, "evals/paperclip-runner");+ const roster = await loadObject(+ resolve(programRoot, "rosters", cell.rosterFile),+ );+ const casePath = resolve(programRoot, "cases", `${cell.caseId}.json`);+ const configPath = inside(+ programRoot,+ resolve(programRoot, "rosters", String(roster.config)),+ `Config for ${cell.rosterId}`,+ );+ const [evalCase, config] = await Promise.all([+ loadObject(casePath),+ loadObject(configPath),+ ]);+ const attemptId = safeId(+ `${cell.cellId}-${campaignId}-missing`,+ "synthetic attempt ID",+ );+ const directory = resolve(runsOut, attemptId);+ await mkdir(directory, { recursive: false, mode: 0o700 });+ const createdAt = new Date().toISOString();+ const artifact = {+ schema: "paperclip-runner/eval-session-artifact/v1",+ attemptId,+ createdAt,+ requestedModel: config.model,+ provider: config.provider ?? "codex",+ driver: config.driver ?? "codex_app_server",+ providerVersion: config.opencodeVersion ?? null,+ usage: {},+ turn: { status: "failed" },+ snapshot: {+ createdAt,+ providerModel: {+ id: config.model,+ provider: config.modelProvider ?? config.provider ?? "unknown",+ },+ transcript: [],+ evidence: [],+ },+ devtools: { revisions: [] },+ infrastructureFailure: {+ class: "ci_cell_artifact_missing",+ category: "campaign_orchestration",+ retryable: false,+ },+ };+ const score = {+ schema: "paperclip-runner/eval-score/v1",+ attemptId,+ caseId: cell.caseId,+ disposition: "infrastructure_failure",+ passed: false,+ infrastructureErrors: [+ "The matrix cell did not retain a complete attempt artifact.",+ ],+ checks: [],+ };+ score.digest = `sha256:${createHash("sha256").update(JSON.stringify(score)).digest("hex")}`;+ await Promise.all([+ writeFile(join(directory, "artifact.json"), json(artifact), {+ mode: 0o600,+ }),+ writeFile(join(directory, "score.json"), json(score), { mode: 0o600 }),+ writeFile(join(directory, "case.json"), json(evalCase), { mode: 0o600 }),+ writeFile(join(directory, "config.json"), json(config), { mode: 0o600 }),+ ]);+ return attemptId;+}++export async function aggregateProtocolEvalCampaign({+ catalogPath,+ downloadsRoot,+ evalsRoot,+ runsOut,+ campaignOut,+ source,+}) {+ const catalog = await loadObject(catalogPath);+ if (catalog.schema !== "paperclip.runner-protocol-eval.catalog/v1") {+ throw new Error("Unsupported protocol eval catalog");+ }+ await rm(runsOut, { recursive: true, force: true });+ await mkdir(runsOut, { recursive: true });+ const retainedByCell = new Map();+ const expectedByCell = new Map(+ catalog.cells.map((cell) => [cell.cellId, cell]),+ );+ const copiedAttemptIds = new Set();+ for (const statusPath of await findFiles(downloadsRoot, "cell.json")) {+ const status = await loadObject(statusPath);+ const cellId = safeId(status.cellId, "recorded cell ID");+ if (retainedByCell.has(cellId))+ throw new Error(`Duplicate cell artifact ${cellId}`);+ const expected = expectedByCell.get(cellId);+ if (!expected)+ throw new Error(`Downloaded artifact names an unexpected cell ${cellId}`);+ if (+ status.caseId !== expected.caseId ||+ status.rosterFile !== expected.rosterFile+ ) {+ throw new Error(`Downloaded cell metadata drifted for ${cellId}`);+ }+ const attemptRoot = resolve(dirname(statusPath), "runs");+ const attemptIds = [];+ const attemptMetadata = await lstat(attemptRoot).catch(() => null);+ if (attemptMetadata?.isDirectory() && !attemptMetadata.isSymbolicLink()) {+ for (const entry of (+ await readdir(attemptRoot, { withFileTypes: true })+ ).sort((a, b) => a.name.localeCompare(b.name))) {+ if (!entry.isDirectory() || entry.isSymbolicLink()) {+ throw new Error(+ `Unexpected cell run path ${join(attemptRoot, entry.name)}`,+ );+ }+ const attemptId = safeId(entry.name, "attempt ID");+ if (copiedAttemptIds.has(attemptId))+ throw new Error(`Duplicate attempt artifact ${attemptId}`);+ const [score, artifact, config] = await Promise.all([+ loadObject(join(attemptRoot, entry.name, "score.json")),+ loadObject(join(attemptRoot, entry.name, "artifact.json")),+ loadObject(join(attemptRoot, entry.name, "config.json")),+ ]);+ if (+ score.attemptId !== attemptId ||+ artifact.attemptId !== attemptId ||+ score.caseId !== expected.caseId ||+ config.model !== expected.model ||+ (config.provider ?? "codex") !== expected.provider ||+ (config.driver ?? "codex_app_server") !== expected.driver+ ) {+ throw new Error(`Downloaded attempt identity drifted for ${cellId}`);+ }+ await copyAttempt(+ join(attemptRoot, entry.name),+ join(runsOut, attemptId),+ );+ copiedAttemptIds.add(attemptId);+ attemptIds.push(attemptId);+ }+ }+ retainedByCell.set(cellId, { status, attemptIds });+ }++ const results = [];+ for (const cell of catalog.cells) {+ const retained = retainedByCell.get(cell.cellId);+ const attemptIds = retained?.attemptIds?.length+ ? retained.attemptIds+ : [+ await syntheticAttempt({+ evalsRoot,+ runsOut,+ cell,+ campaignId: catalog.campaignId,+ }),+ ];+ const finalAttemptId = attemptIds.at(-1);+ const [score, artifact] = await Promise.all([+ loadObject(join(runsOut, finalAttemptId, "score.json")),+ loadObject(join(runsOut, finalAttemptId, "artifact.json")),+ ]);+ if (score.caseId !== cell.caseId || score.attemptId !== finalAttemptId) {+ throw new Error(`Final attempt identity drifted for ${cell.cellId}`);+ }+ results.push({+ cellId: cell.cellId,+ rosterId: cell.rosterId,+ caseId: cell.caseId,+ model: cell.model,+ provider: cell.provider,+ driver: cell.driver,+ attemptIds,+ finalAttemptId,+ disposition: score.disposition,+ passed: score.passed === true,+ usage: safeUsage(artifact.usage),+ cellExitCode: Number.isSafeInteger(retained?.status?.exitCode)+ ? retained.status.exitCode+ : null,+ });+ }+ const totals = {+ selected: results.length,+ passed: results.filter((result) => result.passed).length,+ behaviorFailures: results.filter(+ (result) => result.disposition === "behavior_failure",+ ).length,+ infrastructureFailures: results.filter(+ (result) => result.disposition === "infrastructure_failure",+ ).length,+ };+ const generatedAt = new Date().toISOString();+ const campaign = {+ schema: "paperclip.runner-protocol-eval.campaign/v1",+ campaignId: catalog.campaignId,+ generatedAt,+ source: {+ paperclip: source.paperclip,+ evals: source.evals,+ workflowRunUrl: source.workflowRunUrl,+ },+ complete: results.length === catalog.cells.length,+ allPassed: totals.passed === totals.selected,+ totals,+ rosters: catalog.rosters.map((roster) => ({+ rosterId: roster.rosterId,+ model: roster.model,+ provider: roster.provider,+ driver: roster.driver,+ selected: roster.cases.length,+ passed: results.filter(+ (result) => result.rosterId === roster.rosterId && result.passed,+ ).length,+ })),+ results,+ };+ await mkdir(dirname(campaignOut), { recursive: true });+ await writeFile(campaignOut, json(campaign), { mode: 0o600 });+ return campaign;+}++function publicArtifact(artifact) {+ const model = artifact.snapshot?.providerModel ?? {};+ const infrastructure = artifact.infrastructureFailure;+ const providerVersion =+ artifact.provider === "claude_managed" ||+ artifact.provider === "aws_agentcore"+ ? "remote profile redacted"+ : (artifact.providerVersion ?? null);+ return {+ schema: artifact.schema,+ attemptId: artifact.attemptId,+ createdAt: artifact.createdAt,+ requestedModel: artifact.requestedModel,+ provider: artifact.provider,+ driver: artifact.driver,+ providerVersion,+ retainedSession: false,+ retainedSessionStatus: "redacted from the public report",+ usage: safeUsage(artifact.usage),+ turn: { status: artifact.turn?.status ?? "failed" },+ snapshot: {+ createdAt: artifact.snapshot?.createdAt ?? artifact.createdAt,+ providerModel: {+ id: model.id ?? artifact.requestedModel,+ provider: model.provider ?? artifact.provider,+ },+ transcript: [],+ evidence: [],+ },+ devtools: { revisions: [] },+ ...(infrastructure && typeof infrastructure === "object"+ ? {+ infrastructureFailure: {+ class: infrastructure.class,+ category: infrastructure.category,+ retryable: infrastructure.retryable === true,+ },+ }+ : {}),+ };+}++function publicScore(score) {+ return {+ schema: score.schema,+ attemptId: score.attemptId,+ caseId: score.caseId,+ disposition: score.disposition,+ passed: score.passed === true,+ infrastructureErrors:+ score.disposition === "infrastructure_failure"+ ? [+ "Infrastructure failure; diagnostic details remain in the access-controlled artifact.",+ ]+ : [],+ checks: Array.isArray(score.checks)+ ? score.checks.map((check) => ({+ id: check.id,+ kind: check.kind,+ passed: check.passed === true,+ detail: check.passed === true ? "passed" : "failed",+ evidenceRefs: [],+ }))+ : [],+ digest: score.digest,+ };+}++function publicConfig(config) {+ return Object.fromEntries(+ [+ "schema",+ "id",+ "model",+ "provider",+ "driver",+ "opencodeVersion",+ "acpxAgent",+ "modelProvider",+ ]+ .filter((field) => config[field] !== undefined)+ .map((field) => [field, config[field]]),+ );+}++export async function sanitizeProtocolEvalRuns({ runsRoot, publicRunsRoot }) {+ await rm(publicRunsRoot, { recursive: true, force: true });+ await mkdir(publicRunsRoot, { recursive: true });+ const attemptIds = [];+ for (const entry of (await readdir(runsRoot, { withFileTypes: true })).sort(+ (a, b) => a.name.localeCompare(b.name),+ )) {+ if (!entry.isDirectory() || entry.isSymbolicLink()) {+ throw new Error(+ `Unexpected merged run path ${join(runsRoot, entry.name)}`,+ );+ }+ const attemptId = safeId(entry.name, "attempt ID");+ const source = join(runsRoot, attemptId);+ const [artifact, score, evalCase, config] = await Promise.all([+ loadObject(join(source, "artifact.json")),+ loadObject(join(source, "score.json")),+ loadObject(join(source, "case.json")),+ loadObject(join(source, "config.json")),+ ]);+ if (artifact.attemptId !== attemptId || score.attemptId !== attemptId) {+ throw new Error(`Attempt identity drifted in ${attemptId}`);+ }+ const destination = join(publicRunsRoot, attemptId);+ await mkdir(destination, { mode: 0o700 });+ await Promise.all([+ writeFile(+ join(destination, "artifact.json"),+ json(publicArtifact(artifact)),+ { mode: 0o600 },+ ),+ writeFile(join(destination, "score.json"), json(publicScore(score)), {+ mode: 0o600,+ }),+ writeFile(join(destination, "case.json"), json(evalCase), {+ mode: 0o600,+ }),+ writeFile(join(destination, "config.json"), json(publicConfig(config)), {+ mode: 0o600,+ }),+ ]);+ attemptIds.push(attemptId);+ }+ return attemptIds;+}++function argument(args, name, fallback) {+ const index = args.indexOf(name);+ return index < 0 ? fallback : args[index + 1];+}++async function main() {+ const [command, ...args] = process.argv.slice(2);+ if (command === "catalog") {+ const output = resolve(+ argument(args, "--output", "runner-protocol-eval-catalog.json"),+ );+ const catalog = await buildProtocolEvalCatalog({+ evalsRoot: resolve(argument(args, "--evals-root", ".paperclip-evals")),+ rosterSelection: argument(args, "--rosters", "all"),+ campaignId: argument(args, "--campaign-id", `local-${Date.now()}`),+ maxParallel: Number(argument(args, "--max-parallel", "100")),+ source: {+ paperclipSha: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA ?? null,+ evalsSha: process.env.PAPERCLIP_PROTOCOL_EVALS_SHA ?? null,+ },+ });+ await mkdir(dirname(output), { recursive: true });+ await writeFile(output, json(catalog), { mode: 0o600 });+ await writeGithubOutput({+ matrix_0: JSON.stringify(catalog.matrices[0]),+ matrix_1: JSON.stringify(catalog.matrices[1]),+ matrix_1_present: String(catalog.matrices[1].include.length > 0),+ max_parallel_per_shard: String(catalog.maxParallelPerShard),+ selected: String(catalog.cells.length),+ });+ console.log(+ json({+ output,+ selected: catalog.cells.length,+ rosters: catalog.rosters.length,+ }).trim(),+ );+ return;+ }+ if (command === "aggregate") {+ const campaign = await aggregateProtocolEvalCampaign({+ catalogPath: resolve(argument(args, "--catalog")),+ downloadsRoot: resolve(argument(args, "--downloads")),+ evalsRoot: resolve(argument(args, "--evals-root")),+ runsOut: resolve(argument(args, "--runs-out")),+ campaignOut: resolve(argument(args, "--campaign-out")),+ source: {+ paperclip: {+ sha: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA,+ ref: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_REF,+ },+ evals: {+ repository: "paperclipai/paperclip-evals",+ sha: process.env.PAPERCLIP_PROTOCOL_EVALS_SHA,+ },+ workflowRunUrl: process.env.PAPERCLIP_PROTOCOL_EVAL_WORKFLOW_URL,+ },+ });+ console.log(json(campaign.totals).trim());+ return;+ }+ if (command === "sanitize") {+ const attemptIds = await sanitizeProtocolEvalRuns({+ runsRoot: resolve(argument(args, "--runs-root")),+ publicRunsRoot: resolve(argument(args, "--output")),+ });+ console.log(json({ attempts: attemptIds.length }).trim());+ return;+ }+ throw new Error(+ "Usage: runner-protocol-eval-campaign.mjs <catalog|aggregate|sanitize> ...",+ );+}++if (+ process.argv[1] &&+ resolve(process.argv[1]) === resolve(import.meta.filename)+) {+ await main().catch((error) => {+ console.error(error instanceof Error ? error.message : String(error));+ process.exitCode = 1;+ });+}diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjsnew file mode 100644index 0000000000..474f89971b--- /dev/null+++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs@@ -0,0 +1,349 @@+import assert from "node:assert/strict";+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import test from "node:test";++import {+ aggregateProtocolEvalCampaign,+ buildProtocolEvalCatalog,+ credentialForConfig,+ sanitizeProtocolEvalRuns,+} from "./runner-protocol-eval-campaign.mjs";++const roots = [];+test.afterEach(async () => {+ await Promise.all(+ roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),+ );+});++async function fixture() {+ const root = await mkdtemp(join(tmpdir(), "runner-protocol-campaign-"));+ roots.push(root);+ const program = join(root, "evals/paperclip-runner");+ await Promise.all([+ mkdir(join(program, "rosters"), { recursive: true }),+ mkdir(join(program, "configs"), { recursive: true }),+ mkdir(join(program, "cases"), { recursive: true }),+ ]);+ const config = {+ schema: "paperclip-runner/eval-config/v1",+ id: "live-opencode-model",+ provider: "opencode",+ driver: "opencode_server",+ model: "openrouter/example/model",+ opencodeVersion: "1.18.17",+ };+ const evalCase = {+ schema: "paperclip-runner/eval-case/v1",+ id: "get-task-context",+ title: "Get context",+ description: "Synthetic public fixture",+ prompt: "Inspect the synthetic task.",+ fixture: "../fixtures/company.json",+ authority: { actorId: "agent-1", taskId: "task-1" },+ checks: [{ id: "context", kind: "semantic_operation" }],+ };+ const roster = {+ schema: "paperclip-runner/live-roster/v1",+ id: "protocol-live-opencode-model",+ model: config.model,+ config: "../configs/live-opencode-model.json",+ cases: [evalCase.id],+ };+ await Promise.all([+ writeFile(+ join(program, "configs/live-opencode-model.json"),+ JSON.stringify(config),+ ),+ writeFile(+ join(program, "cases/get-task-context.json"),+ JSON.stringify(evalCase),+ ),+ writeFile(+ join(program, "rosters/live-opencode-model.json"),+ JSON.stringify(roster),+ ),+ ]);+ return { root, program, config, evalCase, roster };+}++test("maps every qualified driver to one explicit credential boundary", () => {+ assert.equal(credentialForConfig({ provider: "codex" }), "OPENAI_API_KEY");+ assert.equal(+ credentialForConfig({ provider: "opencode" }),+ "OPENROUTER_API_KEY",+ );+ assert.equal(+ credentialForConfig({ provider: "acpx", acpxAgent: "claude" }),+ "ANTHROPIC_API_KEY",+ );+ assert.equal(+ credentialForConfig({ provider: "aws_agentcore" }),+ "AWS_AGENTCORE_OIDC",+ );+ assert.throws(+ () => credentialForConfig({ provider: "unknown" }),+ /credential policy/,+ );+});++test("catalogs roster plus case cells and emits bounded balanced shards", async () => {+ const { root } = await fixture();+ const catalog = await buildProtocolEvalCatalog({+ evalsRoot: root,+ campaignId: "gha-42-1",+ maxParallel: 80,+ });+ assert.equal(catalog.cells.length, 1);+ assert.equal(catalog.cells[0].credentialName, "OPENROUTER_API_KEY");+ assert.equal(catalog.maxParallelPerShard, 40);+ assert.equal(catalog.matrices[0].include.length, 1);+ assert.equal(catalog.matrices[1].include.length, 0);+ await assert.rejects(+ buildProtocolEvalCatalog({+ evalsRoot: root,+ campaignId: "gha-42-1",+ rosterSelection: "missing-roster",+ }),+ /Unknown live roster/,+ );+ await assert.rejects(+ buildProtocolEvalCatalog({+ evalsRoot: root,+ campaignId: "gha-42-1",+ maxParallel: 1,+ }),+ /from 2 through 100/,+ );+});++test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => {+ const { root, config, evalCase } = await fixture();+ const catalog = await buildProtocolEvalCatalog({+ evalsRoot: root,+ campaignId: "gha-42-1",+ });+ const catalogPath = join(root, "catalog.json");+ await writeFile(catalogPath, JSON.stringify(catalog));+ const campaign = await aggregateProtocolEvalCampaign({+ catalogPath,+ downloadsRoot: join(root, "missing-downloads"),+ evalsRoot: root,+ runsOut: join(root, "merged-runs"),+ campaignOut: join(root, "campaign.json"),+ source: {+ paperclip: { sha: "a".repeat(40), ref: "refs/heads/master" },+ evals: { repository: "paperclipai/paperclip-evals", sha: "b".repeat(40) },+ workflowRunUrl: "https://example.test/actions/runs/42",+ },+ });+ assert.deepEqual(campaign.totals, {+ selected: 1,+ passed: 0,+ behaviorFailures: 0,+ infrastructureFailures: 1,+ });+ assert.equal(campaign.results[0].disposition, "infrastructure_failure");+ assert.equal(campaign.complete, true);+ assert.equal(campaign.allPassed, false);+ const attempt = campaign.results[0].finalAttemptId;+ assert.deepEqual(+ JSON.parse(await readFile(join(root, "merged-runs", attempt, "case.json"))),+ evalCase,+ );+ assert.deepEqual(+ JSON.parse(+ await readFile(join(root, "merged-runs", attempt, "config.json")),+ ),+ config,+ );+});++test("rejects downloaded cells that were not declared by the immutable catalog", async () => {+ const { root } = await fixture();+ const catalog = await buildProtocolEvalCatalog({+ evalsRoot: root,+ campaignId: "gha-42-1",+ });+ const catalogPath = join(root, "catalog.json");+ const downloads = join(root, "downloads/unexpected");+ await mkdir(downloads, { recursive: true });+ await Promise.all([+ writeFile(catalogPath, JSON.stringify(catalog)),+ writeFile(+ join(downloads, "cell.json"),+ JSON.stringify({+ cellId: "protocol-live-opencode-model--undeclared",+ rosterFile: "live-opencode-model.json",+ caseId: "undeclared",+ exitCode: 1,+ }),+ ),+ ]);+ await assert.rejects(+ aggregateProtocolEvalCampaign({+ catalogPath,+ downloadsRoot: join(root, "downloads"),+ evalsRoot: root,+ runsOut: join(root, "merged-runs"),+ campaignOut: join(root, "campaign.json"),+ source: {},+ }),+ /unexpected cell/,+ );+});++test("public run projection removes provider sessions, traces, transcripts, evidence, and state", async () => {+ const { root, config, evalCase } = await fixture();+ const attemptId = "get-task-context-opencode-gha-42-1-attempt-01";+ const source = join(root, "raw-runs", attemptId);+ await mkdir(source, { recursive: true });+ await Promise.all([+ writeFile(+ join(source, "artifact.json"),+ JSON.stringify({+ schema: "paperclip-runner/eval-session-artifact/v1",+ attemptId,+ createdAt: "2026-09-05T00:00:00.000Z",+ requestedModel: config.model,+ provider: config.provider,+ driver: config.driver,+ providerVersion: config.opencodeVersion,+ providerSessionId: "private-session-id",+ trace: { secret: "private" },+ usage: {+ agentTurns: 1,+ inputTokens: 100,+ estimatedCostNanodollars: 5000,+ },+ turn: { status: "completed", turnId: "private-turn" },+ snapshot: {+ createdAt: "2026-09-05T00:00:00.000Z",+ providerModel: { id: config.model, provider: "openrouter" },+ transcript: [{ role: "assistant", text: "private transcript" }],+ evidence: [{ kind: "tool_call", data: { token: "private" } }],+ },+ issueThread: { messages: ["private"] },+ devtools: { revisions: [{ state: { private: true } }] },+ }),+ ),+ writeFile(+ join(source, "score.json"),+ JSON.stringify({+ schema: "paperclip-runner/eval-score/v1",+ attemptId,+ caseId: evalCase.id,+ disposition: "passed",+ passed: true,+ infrastructureErrors: [],+ checks: [+ {+ id: "context",+ kind: "semantic_operation",+ passed: true,+ detail: "private result",+ evidenceRefs: ["event-1"],+ },+ ],+ digest: "sha256:test",+ }),+ ),+ writeFile(join(source, "case.json"), JSON.stringify(evalCase)),+ writeFile(+ join(source, "config.json"),+ JSON.stringify({ ...config, managedProfile: { profileId: "private" } }),+ ),+ ]);+ await sanitizeProtocolEvalRuns({+ runsRoot: join(root, "raw-runs"),+ publicRunsRoot: join(root, "public-runs"),+ });+ const serialized = await readFile(+ join(root, "public-runs", attemptId, "artifact.json"),+ "utf8",+ );+ assert.doesNotMatch(+ serialized,+ /private-session|private transcript|private-turn|issueThread|trace/,+ );+ const artifact = JSON.parse(serialized);+ assert.deepEqual(artifact.snapshot.transcript, []);+ assert.deepEqual(artifact.snapshot.evidence, []);+ assert.deepEqual(artifact.devtools.revisions, []);+ const score = JSON.parse(+ await readFile(join(root, "public-runs", attemptId, "score.json"), "utf8"),+ );+ assert.deepEqual(score.checks[0], {+ id: "context",+ kind: "semantic_operation",+ passed: true,+ detail: "passed",+ evidenceRefs: [],+ });+ const publicConfig = await readFile(+ join(root, "public-runs", attemptId, "config.json"),+ "utf8",+ );+ assert.doesNotMatch(publicConfig, /managedProfile|private/);+});++test("public run projection redacts remote profile version identities", async () => {+ const { root, config, evalCase } = await fixture();+ const attemptId = "get-task-context-claude-managed-gha-42-1-attempt-01";+ const source = join(root, "raw-runs", attemptId);+ await mkdir(source, { recursive: true });+ await Promise.all([+ writeFile(+ join(source, "artifact.json"),+ JSON.stringify({+ schema: "paperclip-runner/eval-session-artifact/v1",+ attemptId,+ createdAt: "2026-09-05T00:00:00.000Z",+ requestedModel: "claude-sonnet-5",+ provider: "claude_managed",+ driver: "claude_managed_agents_api",+ providerVersion: "private-agent-version-id",+ usage: {},+ turn: { status: "completed" },+ snapshot: {+ createdAt: "2026-09-05T00:00:00.000Z",+ providerModel: { id: "claude-sonnet-5", provider: "anthropic" },+ },+ }),+ ),+ writeFile(+ join(source, "score.json"),+ JSON.stringify({+ schema: "paperclip-runner/eval-score/v1",+ attemptId,+ caseId: evalCase.id,+ disposition: "passed",+ passed: true,+ checks: [],+ }),+ ),+ writeFile(join(source, "case.json"), JSON.stringify(evalCase)),+ writeFile(+ join(source, "config.json"),+ JSON.stringify({+ ...config,+ id: "live-claude-managed",+ model: "claude-sonnet-5",+ provider: "claude_managed",+ driver: "claude_managed_agents_api",+ }),+ ),+ ]);+ await sanitizeProtocolEvalRuns({+ runsRoot: join(root, "raw-runs"),+ publicRunsRoot: join(root, "public-runs"),+ });+ const publicArtifact = await readFile(+ join(root, "public-runs", attemptId, "artifact.json"),+ "utf8",+ );+ assert.match(publicArtifact, /remote profile redacted/);+ assert.doesNotMatch(publicArtifact, /private-agent-version-id/);+});diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjsnew file mode 100644index 0000000000..5e9f990cb8--- /dev/null+++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs@@ -0,0 +1,109 @@+import assert from "node:assert/strict";+import { readFile } from "node:fs/promises";+import { resolve } from "node:path";+import test from "node:test";++const repositoryRoot = resolve(import.meta.dirname, "../../..");+const workflowPath = resolve(+ repositoryRoot,+ ".github/workflows/runner-protocol-live-evals.yml",+);++test("direct live eval workflow keeps paid execution behind stable actor authorization", async () => {+ const workflow = await readFile(workflowPath, "utf8");+ assert.match(workflow, /^\s{2}authorize:/mu);+ assert.match(workflow, /RUNNER_E2E_ALLOWED_ACTOR_IDS/u);+ assert.match(workflow, /github\.actor_id/u);+ assert.match(workflow, /github\.triggering_actor/u);+ assert.match(workflow, /refs\/heads\/\$DEFAULT_BRANCH/u);+ assert.match(workflow, /Reauthorize paid execution before provider access/u);+ assert.match(+ workflow,+ /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*Run one immutable direct protocol cell/u,+ );+ assert.doesNotMatch(+ workflow,+ /^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/mu,+ );+ const actions = [+ ...workflow.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gmu),+ ].map((match) => match[1]);+ assert.ok(actions.length > 0);+ for (const action of actions) assert.match(action, /^[^@]+@[0-9a-f]{40}$/u);+});++test("resolves both repositories immutably and bounds total matrix concurrency", async () => {+ const workflow = await readFile(workflowPath, "utf8");+ const authorize = workflow.slice(+ workflow.indexOf(" authorize:"),+ workflow.indexOf(" catalog:"),+ );+ assert.match(authorize, /repos\/\$REPOSITORY\/branches\/\$encoded_branch/u);+ assert.match(authorize, /\^\[0-9a-f\]\{40\}\$/u);+ assert.match(+ authorize,+ /repos\/paperclipai\/paperclip-evals\/commits\/\$EVALS_SHA/u,+ );+ assert.match(authorize, /COMMITPERCLIP_KEY/u);+ assert.match(+ authorize,+ /GH_TOKEN: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u,+ );+ assert.match(authorize, /test "\$resolved" = "\$EVALS_SHA"/u);+ const catalog = workflow.slice(+ workflow.indexOf(" catalog:"),+ workflow.indexOf(" build_runner:"),+ );+ assert.match(+ catalog,+ /ref: \$\{\{ needs\.authorize\.outputs\.evals_sha \}\}/u,+ );+ assert.match(catalog, /RUNNER_E2E_MAX_PARALLEL/u);+ assert.match(catalog, /max_parallel_per_shard/u);+ const privateCheckouts = [+ ...workflow.matchAll(+ /repository: paperclipai\/paperclip-evals[\s\S]*?persist-credentials: false/gmu,+ ),+ ];+ assert.equal(privateCheckouts.length, 3);+ for (const checkout of privateCheckouts) {+ assert.match(+ checkout[0],+ /token: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u,+ );+ }+ assert.match(workflow, /matrix_0/u);+ assert.match(workflow, /matrix_1/u);+});++test("publishes only the separately sanitized Evalbook through trusted OIDC code", async () => {+ const workflow = await readFile(workflowPath, "utf8");+ const report = workflow.slice(+ workflow.indexOf(" report:"),+ workflow.indexOf(" publish_history:"),+ );+ assert.match(+ report,+ /Render the access-controlled canonical Evalbook report/u,+ );+ assert.match(+ report,+ /--viewer-root runner-protocol-build\/extracted\/dist-issue-thread/u,+ );+ assert.match(report, /runner-protocol-eval-campaign\.mjs sanitize/u);+ assert.match(+ report,+ /Upload access-controlled canonical Evalbook and raw attempts/u,+ );+ assert.match(report, /Upload publisher-only sanitized Evalbook/u);++ const publisher = workflow.slice(workflow.indexOf(" publish_history:"));+ assert.match(publisher, /ref: \$\{\{ github\.sha \}\}/u);+ assert.match(publisher, /id-token: write/u);+ assert.match(publisher, /runner-protocol-eval-public-/u);+ assert.match(publisher, /publish-runner-protocol-eval-history\.mjs/u);+ assert.match(publisher, /runner-protocol-evals/u);+ assert.doesNotMatch(publisher, /(?:OPENAI|ANTHROPIC|OPENROUTER)_API_KEY/u);+ assert.doesNotMatch(publisher, /paperclipai\/paperclip-evals/u);+ assert.doesNotMatch(publisher, /downloaded-runner-protocol-evals/u);+});diff --git a/packages/paperclip-runner/scripts/stage-runner-binary.mjs b/packages/paperclip-runner/scripts/stage-runner-binary.mjsindex 14fcee5a92..d75f4bbb6d 100644--- a/packages/paperclip-runner/scripts/stage-runner-binary.mjs+++ b/packages/paperclip-runner/scripts/stage-runner-binary.mjs@@ -1,7 +1,11 @@+import { execFile } from "node:child_process";import { chmod, copyFile, mkdir } from "node:fs/promises";import path from "node:path";+import { promisify } from "node:util";import { fileURLToPath } from "node:url";+const execFileAsync = promisify(execFile);+const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");const executable = process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd";const source = path.join(packageRoot, "runner", "target", "release", executable);@@ -11,3 +15,10 @@ const destination = path.join(destinationDirectory, executable);await mkdir(destinationDirectory, { recursive: true });await copyFile(source, destination);if (process.platform !== "win32") await chmod(destination, 0o755);+// Rust's linker emits an ad-hoc Mach-O signature. Copying that executable to+// its package location preserves the bytes but can leave the kernel rejecting+// the new inode with SIGKILL. Re-sign the staged inode so local packaged-runner+// evals execute the same artifact that was just built.+if (process.platform === "darwin") {+ await execFileAsync("codesign", ["--force", "--sign", "-", destination]);+}diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.tsindex 947c22cb99..e2765f0307 100644--- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts+++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts@@ -5,6 +5,10 @@ import {evalSessionUsage,parseEvalSessionRequest,} from "./eval-session-contract.js";+import {+ boundedEvalSessionUsage,+ evalSessionProviderVersion,+} from "./eval-session.js";function request(overrides: Record<string, unknown> = {}): unknown {return {@@ -65,6 +69,43 @@ describe("eval-session request contract", () => {}))).toMatchObject({ provider: "acpx", acpxAgent: "claude" });});+ it("accepts null optional fields from the original Evalbook v1 producer", () => {+ const parsed = parseEvalSessionRequest(request({+ acpxAgent: null,+ agentCoreProfile: null,+ opencodeVersion: null,+ }));+ expect(parsed).toMatchObject({+ provider: "codex",+ driver: "codex_app_server",+ });+ expect(parsed).not.toHaveProperty("acpxAgent");+ expect(parsed).not.toHaveProperty("agentCoreProfile");+ expect(parsed).not.toHaveProperty("opencodeVersion");+ });++ it("attributes managed providers to their immutable deployed revisions", () => {+ expect(evalSessionProviderVersion(parseEvalSessionRequest(request({+ provider: "aws_agentcore",+ driver: "aws_agentcore_harness_api",+ model: "global.anthropic.claude-sonnet-4-6",+ agentCoreProfile: agentCoreProfile(),+ })))).toBe("aws-agentcore-harness-context-v2");+ expect(evalSessionProviderVersion(parseEvalSessionRequest(request({+ provider: "claude_managed",+ driver: "claude_managed_agents_api",+ model: "claude-sonnet-5",+ managedProfile: {+ profileId: "managed-qualified",+ anthropicAgentId: "agent-test",+ agentVersion: "17",+ environmentId: "environment-test",+ betaVersion: "managed-agents-2026-04-01",+ maxSessionListCostUsd: 1,+ },+ })))).toBe("17");+ });+it("rejects Pi and accepts both qualified remote provider profiles", () => {expect(() => parseEvalSessionRequest(request({provider: "acpx",@@ -143,6 +184,41 @@ describe("eval-session request contract", () => {});describe("eval-session usage", () => {+ it("retains durable failed turns even when their reported usage exceeds completed-turn limits", () => {+ const parsed = parseEvalSessionRequest(request());+ const snapshot = {+ usageLedger: [{+ receiptId: "receipt-failed",+ attemptId: "attempt-1",+ providerResponseId: "response-failed",+ turnId: "turn-failed",+ observedAt: "2026-09-05T00:00:00.000Z",+ providerCalls: 2,+ providerRequests: 2,+ inputTokens: 1_000,+ outputTokens: 100,+ cachedInputTokens: 0,+ reasoningTokens: 0,+ costNanodollars: 200_000_000,+ }],+ } as unknown as CapabilityLiveSessionSnapshot;+ const failedTurn = {+ turnId: "turn-failed",+ status: "failed" as const,+ assistantText: "",+ snapshot,+ };++ expect(boundedEvalSessionUsage(parsed, failedTurn)).toMatchObject({+ agentTurns: 2,+ providerReportedCostNanodollars: 200_000_000,+ });+ expect(() => boundedEvalSessionUsage(parsed, {+ ...failedTurn,+ status: "completed",+ })).toThrow("agent turn limit exceeded");+ });+it("deduplicates receipts and applies the versioned model price", () => {const receipt = {receiptId: "receipt-1",diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.ts b/packages/paperclip-runner/src/cli/eval-session-contract.tsindex 1caf1e04f0..9c48f987b9 100644--- a/packages/paperclip-runner/src/cli/eval-session-contract.ts+++ b/packages/paperclip-runner/src/cli/eval-session-contract.ts@@ -233,7 +233,10 @@ export function parseEvalSessionRequest(value: unknown): EvalSessionRequest {if (input.driver !== undefined && input.driver !== driver) {throw new Error("eval-session provider/driver mismatch");}- const acpxAgent = input.acpxAgent;+ // The original Evalbook v1 producer serialized absent provider-specific+ // options as JSON null. Preserve compatibility with those immutable request+ // artifacts while continuing to reject non-null values for the wrong lane.+ const acpxAgent = input.acpxAgent === null ? undefined : input.acpxAgent;if (acpxAgent === "pi") throw new Error("The Pi ACPX profile is not available");if (acpxAgent !== undefined &&@@ -245,16 +248,22 @@ export function parseEvalSessionRequest(value: unknown): EvalSessionRequest {if (provider !== "acpx" && acpxAgent !== undefined) {throw new Error("eval-session acpxAgent requires provider acpx");}+ const managedProfileInput = input.managedProfile === null+ ? undefined+ : input.managedProfile;+ const agentCoreProfileInput = input.agentCoreProfile === null+ ? undefined+ : input.agentCoreProfile;const managedProfile = provider === "claude_managed"- ? parseManagedProfile(input.managedProfile)+ ? parseManagedProfile(managedProfileInput): undefined;const agentCoreProfile = provider === "aws_agentcore"- ? parseAgentCoreProfile(input.agentCoreProfile)+ ? parseAgentCoreProfile(agentCoreProfileInput): undefined;- if (provider !== "claude_managed" && input.managedProfile !== undefined) {+ if (provider !== "claude_managed" && managedProfileInput !== undefined) {throw new Error("eval-session managedProfile requires provider claude_managed");}- if (provider !== "aws_agentcore" && input.agentCoreProfile !== undefined) {+ if (provider !== "aws_agentcore" && agentCoreProfileInput !== undefined) {throw new Error("eval-session agentCoreProfile requires provider aws_agentcore");}if (input.nativeResume !== undefined) {diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.tsindex f98e5ab1a8..609ec73090 100644--- a/packages/paperclip-runner/src/cli/eval-session.ts+++ b/packages/paperclip-runner/src/cli/eval-session.ts@@ -53,7 +53,9 @@ async function sha256(path: string): Promise<string> {return createHash("sha256").update(await readFile(path)).digest("hex");}-function providerVersion(request: EvalSessionRequest): string | null {+export function evalSessionProviderVersion(+ request: EvalSessionRequest,+): string | null {if (request.provider === "opencode") {const version = request.opencodeVersion ?? "1.18.17";if (version !== "1.18.17") {@@ -71,7 +73,7 @@ function providerVersion(request: EvalSessionRequest): string | null {return request.managedProfile!.agentVersion;}if (request.provider === "aws_agentcore") {- return request.agentCoreProfile!.harnessVersion;+ return request.agentCoreProfile!.qualificationRevision;}return null;}@@ -127,6 +129,32 @@ function usageIfAvailable(}}+export function boundedEvalSessionUsage(+ request: EvalSessionRequest,+ turn: CapabilityLiveTurnResult,+): EvalSessionUsage | null {+ if (turn.status !== "completed") {+ return usageIfAvailable(request, turn.snapshot);+ }+ const usage = evalSessionUsage(request.model, turn.snapshot);+ if (usage.agentTurns > request.limits.maxAgentTurns) {+ throw new Error("agent turn limit exceeded");+ }+ if (+ usage.estimatedCostNanodollars >+ request.limits.maxEstimatedCostNanodollars+ ) {+ throw new Error("estimated cost limit exceeded");+ }+ if (+ usage.providerReportedCostNanodollars >+ request.limits.maxEstimatedCostNanodollars+ ) {+ throw new Error("provider-reported cost limit exceeded");+ }+ return usage;+}+async function closeSession(session: CapabilityLiveSession | null,reason: string,@@ -160,11 +188,16 @@ export async function runEvalSessionCli(const requestedProvider = request.provider ?? "codex";const requestedDriver = request.driver ??expectedEvalSessionDriver(requestedProvider);- const requestedProviderVersion = providerVersion(request);+ const requestedProviderVersion = evalSessionProviderVersion(request);const service = options.serviceFactory?.(runnerdPath) ??new CapabilityLiveSessionService({transportOptions: {runnerBinary: runnerdPath,+ // The transport performs the provider-specific allowlisting. Supplying+ // the source environment here is still required: without it the+ // isolated Codex home has no credential source and runnerd receives no+ // executable PATH from the Evalbook CLI process.+ environment: process.env,onDiagnostic: (message) => {process.stderr.write(`[eval-session runnerd] ${message}\n`);},@@ -196,27 +229,11 @@ export async function runEvalSessionCli(} as unknown as CreateCapabilityLiveSessionInput;session = await service.create(createInput);turn = await session.sendMessage(request.prompt);- if (turn.status !== "completed") {- await session.completeAttempt("failed", `provider_turn_${turn.status}`);- throw new Error(`provider turn ended with status ${turn.status}`);- }- const usage = evalSessionUsage(request.model, turn.snapshot);- if (usage.agentTurns > request.limits.maxAgentTurns) {- throw new Error("agent turn limit exceeded");- }- if (- usage.estimatedCostNanodollars >- request.limits.maxEstimatedCostNanodollars- ) {- throw new Error("estimated cost limit exceeded");- }- if (- usage.providerReportedCostNanodollars >- request.limits.maxEstimatedCostNanodollars- ) {- throw new Error("provider-reported cost limit exceeded");- }- await session.completeAttempt("succeeded");+ const usage = boundedEvalSessionUsage(request, turn);+ await session.completeAttempt(+ turn.status === "completed" ? "succeeded" : "failed",+ turn.status === "completed" ? null : `provider_turn_${turn.status}`,+ );await closeSession(session, "eval session complete");snapshot = session.snapshot();@@ -259,7 +276,7 @@ export async function runEvalSessionCli(mode: "live",replaySource: "live",}),- usage,+ ...(usage === null ? {} : { usage }),timing: {startedAt,finishedAt: new Date().toISOString(),diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.tsindex b183da10a4..6d9a7e3e7e 100644--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts@@ -48,7 +48,10 @@ const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1";const maxFrameBytes = 1024 * 1024;const maxCommandBytes = maxFrameBytes - 4 * 1024;const maxCommands = 500;-const maxCommittedEventWindow = 64;+// A provider can emit several 100-event runner batches before the transport's+// polling turn regains the event loop. Match the transport's explicit deferred+// event bound so a valid burst is not compacted before it can be observed.+const maxCommittedEventWindow = 4_096;const maxStateBytes = 192 * 1024 * 1024;const authChallengeTtlMs = 5_000;const stableIdPattern = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/;diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.tsindex 2869830df4..2c4aca9107 100644--- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts+++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts@@ -1379,6 +1379,9 @@ describe("OpenCodeServerDriver", () => {"*": permissionMode,external_directory: "deny",});+ expect(config.provider.openrouter.models).toHaveProperty(+ "deepseek/deepseek-v4-flash-0731",+ );expect(config.permission.paperclip_finish).toBeUndefined();expect(config.permission["paperclip_*"]).toBe("allow");await session.close({ reason: "permission mode test complete" });@@ -1771,8 +1774,11 @@ describe("OpenCodeServerDriver", () => {workingDirectory: workspace,});expect(session.ids().providerSessionId).toBe("ses_fake_1");- expect(spawns).toHaveLength(2);- expect(commandLifecycle).toEqual(["before", "after", "before", "after"]);+ expect(spawns.length).toBeGreaterThanOrEqual(2);+ expect(spawns.length).toBeLessThanOrEqual(3);+ expect(commandLifecycle).toEqual(+ Array.from({ length: spawns.length }, () => ["before", "after"]).flat(),+ );await session.close({ reason: "test" });});diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.tsindex ba0d68dc7e..a764d7b605 100644--- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts+++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts@@ -1866,6 +1866,8 @@ async function startRuntime(input: {]);const instructionRoot =input.options.runtimeContext?.instructions.bundle.rootPath;+ const [modelProvider, ...modelIdParts] = input.options.model.split("/");+ const providerModelId = modelIdParts.join("/");const config = {$schema: "https://opencode.ai/config.json",model: input.options.model,@@ -1875,6 +1877,16 @@ async function startRuntime(input: {// system prompt; siblings remain available through the read-only root.instructions: [],plugin: [],+ // OpenCode's bundled models.dev snapshot can lag behind OpenRouter's live+ // catalog. Bind the already-qualified exact model slug into the built-in+ // provider instead of silently falling back or rejecting a newer model.+ provider: {+ [modelProvider!]: {+ models: {+ [providerModelId]: { name: providerModelId },+ },+ },+ },tools: {question: true,},diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.test.tsindex 1f62eb9232..f791426794 100644--- a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts+++ b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts@@ -86,7 +86,13 @@ vi.mock("../live/live-session.js", () => ({}));import { runnerWorkflowCase } from "./workflow-catalog.js";-import { executeLiveRunnerWorkflow } from "./live-workflow-executor.js";+import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";+import {+ advanceDelegationReturnMockState,+ executeLiveRunnerWorkflow,+ scorableLiveWorkflowCalls,+ unexpectedLiveWorkflowCalls,+} from "./live-workflow-executor.js";import {RUNNER_LIVE_CANDIDATE_SLOTS,type RunnerLiveScheduleEntry,@@ -124,6 +130,117 @@ describe("live workflow executor infrastructure failures", () => {vi.unstubAllEnvs();});+ it("allows read-only context support without permitting extra effects", () => {+ const calls = [+ "get_task_context",+ "get_task_history",+ "finish_task",+ "create_task",+ ];+ expect(unexpectedLiveWorkflowCalls(calls, ["finish_task"])).toEqual([+ "create_task",+ ]);+ expect(scorableLiveWorkflowCalls(calls, ["finish_task"])).toEqual([+ "finish_task",+ "create_task",+ ]);+ });++ it("keeps required read-only calls in trajectory scoring", () => {+ expect(+ scorableLiveWorkflowCalls(+ ["get_task_context", "get_task_history"],+ ["get_task_context"],+ ),+ ).toEqual(["get_task_context"]);+ });++ it("advances a delegated child through the authoritative return wake", async () => {+ const grants = ["delegation:tasks:create", "dependencies:write"];+ const adapter = new CapabilityMockControlPlaneAdapter({+ actors: [+ {+ id: "actor-1",+ companyId: "company-1",+ name: "Workflow eval actor",+ role: "engineer",+ status: "active",+ budgetId: "budget-actor-1",+ capabilityGrants: grants,+ },+ ],+ });+ await adapter.start();+ await adapter.openFixtureRun({+ identity: {+ runId: "delegation-parent",+ sessionId: "delegation-session",+ companyId: "company-1",+ issueId: "task-1",+ agentId: "actor-1",+ },+ backendKind: "runner",+ capabilities: grants,+ });+ const created = await adapter.applyCommand({+ runId: "delegation-parent",+ idempotencyKey: "create-child",+ command: {+ kind: "create_task",+ title: "Independent verification",+ description: "Verify the external boundary.",+ assigneeActorId: "actor-1",+ },+ });+ const childTaskId = created.entityRefs+ .find((entityRef) => entityRef.startsWith("task:"))!+ .slice("task:".length);+ await adapter.applyCommand({+ runId: "delegation-parent",+ idempotencyKey: "set-child-dependency",+ command: {+ kind: "set_dependencies",+ blockedByTaskIds: [childTaskId],+ },+ });++ const advanced = await advanceDelegationReturnMockState({+ mockState: adapter.serialize(),+ parentRunId: "delegation-parent",+ parentSessionId: "delegation-session",+ parentTaskId: "task-1",+ capabilities: grants,+ });+ expect(advanced).not.toBeNull();+ const restored = CapabilityMockControlPlaneAdapter.restore(+ advanced!.mockState,+ );+ expect(restored.snapshot().tasks).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ id: childTaskId, status: "done" }),+ expect.objectContaining({+ id: "task-1",+ status: "in_progress",+ checkoutRunId: advanced!.returnRunId,+ }),+ ]),+ );+ expect(restored.snapshot().blockers).toEqual([]);+ expect(restored.snapshot().wakes).toContainEqual(+ expect.objectContaining({+ taskId: "task-1",+ reason: "blockers_resolved",+ }),+ );+ await expect(+ restored.applyCommand({+ runId: advanced!.returnRunId,+ idempotencyKey: "finish-parent",+ command: { kind: "finish_task", summary: "Return completed." },+ }),+ ).resolves.toMatchObject({ disposition: "applied" });+ });+it("classifies shutdown failures as retryable infrastructure errors and redacts them", async () => {const leakedSecret = "sk-shutdown-secret-value";liveSessionMocks.shutdown.mockRejectedValueOnce(diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.tsindex 6f3b54fa95..7d81d5ef4f 100644--- a/packages/paperclip-runner/src/eval/live-workflow-executor.ts+++ b/packages/paperclip-runner/src/eval/live-workflow-executor.ts@@ -4,16 +4,24 @@ import { tmpdir } from "node:os";import { join } from "node:path";import type { CapabilityJsonValue } from "../mock-core/capability-control-plane-types.js";+import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";+import type {+ PrpStructuredRunResult,+ PrpTerminalState,+} from "../protocol/replay-contract.js";import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js";import {CapabilityLiveSessionService,InMemoryCapabilityLiveSessionStore,type CapabilityLiveSession,+ type CapabilityLiveSessionServiceOptions,type CapabilityLiveSessionSnapshot,+ type CapabilityLiveSessionStore,type CapabilityLiveTurnEvent,type CapabilityLiveTurnResult,} from "../live/live-session.js";import type { EvalObservation } from "./eval-scoring.js";+import { isCapabilitySemanticReadOperation } from "../semantic-tools/policy.js";import {RUNNER_WORKFLOW_OBSERVATION_SCHEMA,type RunnerWorkflowCheck,@@ -74,9 +82,261 @@ function continuationPrompt(evalCase: RunnerWorkflowEvalCase): string {if (evalCase.id === "steering-causality") {return "Steering update: use bullet points only. Apply only this formatting change and call finish_task exactly once without repeating prior work.";}+ if (evalCase.id === "delegation-return") {+ return "The plan is accepted. Create exactly one justified child, set the source task dependency to that child, then stop without calling finish_task. Wait for the authoritative child-completion result.";+ }return "Continue from the authoritative typed interaction result. Do not redo completed work.";}+function workflowResult(+ disposition: "done" | "blocked",+ summary: string,+ idempotencyKey: string,+): PrpStructuredRunResult {+ return {+ schema: "paperclip.run_result.v1",+ reportedWorkDisposition: disposition,+ summary,+ completionClaim: {+ contractRevision: "runner-workflow-eval-v1",+ objectiveSatisfied: disposition === "done",+ criteria: [],+ remainingWork:+ disposition === "done"+ ? []+ : [+ {+ description: "Wait for the delegated child to complete.",+ blocksCompletion: true,+ },+ ],+ },+ evidence: [],+ verification: [],+ attentionRequests: [],+ artifacts: [],+ ...(disposition === "done"+ ? {}+ : {+ blocker: {+ reasonCode: "delegated_child_in_progress",+ owner: { kind: "agent" as const, name: "Delegated child" },+ unblockAction: "Complete the delegated child task.",+ scope: "current_track" as const,+ },+ continuation: {+ kind: "delegated_issue" as const,+ summary:+ "Resume the source task after the delegated child completes.",+ idempotencyKey,+ },+ }),+ };+}++async function completeWorkflowRun(+ adapter: CapabilityMockControlPlaneAdapter,+ input: {+ runId: string;+ sessionId: string;+ source: string;+ disposition: "done" | "blocked";+ summary: string;+ },+): Promise<void> {+ const terminal: PrpTerminalState = {+ schema: "paperclip.prp.terminal.v1",+ turnTerminalState: "completed",+ runTerminalState: "succeeded",+ reportedWorkDisposition: input.disposition,+ };+ const current = adapter.snapshot().runs.find((run) => run.id === input.runId);+ if (current === undefined) {+ throw new Error(`workflow eval run ${input.runId} is missing`);+ }+ if (current.result !== null) return;+ if (+ !current.events.some(+ (event) => "eventType" in event && event.eventType === "run.terminal",+ )+ ) {+ await adapter.appendEvent({+ schema: "paperclip.prp.event.v1",+ sourceEventId: `${input.source}:terminal:1`,+ sourceSeq: 1,+ sourceInstanceId: input.source,+ sourceKind: "runner",+ runId: input.runId,+ normalizedSessionId: input.sessionId,+ turnId: `${input.source}-turn`,+ eventType: "run.terminal",+ schemaVersion: 1,+ priority: 0,+ emittedAt: new Date().toISOString(),+ payload: terminal,+ });+ }+ await adapter.completeRun({+ result: workflowResult(+ input.disposition,+ input.summary,+ `${input.source}-continuation`,+ ),+ terminal,+ });+}++export interface AdvancedDelegationReturnState {+ mockState: string;+ childTaskId: string;+ returnRunId: string;+}++/**+ * Advance the controlled mock authority through the child-return boundary.+ * This is orchestration evidence, not a model-authored semantic call: the+ * evaluated provider still has to author the child/dependency and final result.+ */+export async function advanceDelegationReturnMockState(input: {+ mockState: string;+ parentRunId: string;+ parentSessionId: string;+ parentTaskId: string;+ capabilities: readonly string[];+}): Promise<AdvancedDelegationReturnState | null> {+ const adapter = CapabilityMockControlPlaneAdapter.restore(input.mockState);+ if (adapter.snapshot().lifecycle !== "running") await adapter.start();+ const initial = adapter.snapshot();+ const children = initial.tasks.filter(+ (task) => task.parentId === input.parentTaskId,+ );+ if (children.length !== 1) return null;+ const child = children[0]!;+ if (+ !initial.blockers.some(+ (blocker) =>+ blocker.taskId === input.parentTaskId &&+ blocker.blockedByTaskId === child.id,+ )+ ) {+ return null;+ }++ await completeWorkflowRun(adapter, {+ runId: input.parentRunId,+ sessionId: input.parentSessionId,+ source: `${input.parentRunId}-delegation-wait`,+ disposition: "blocked",+ summary: "Source task is waiting on its delegated child.",+ });++ const childRunId = `${input.parentRunId}-child`;+ const childSessionId = `${input.parentSessionId}-child`;+ const actorId = child.assigneeActorId ?? initial.actors[0]!.id;+ await adapter.openFixtureRun({+ identity: {+ runId: childRunId,+ sessionId: childSessionId,+ companyId: child.companyId,+ issueId: child.id,+ agentId: actorId,+ },+ backendKind: "runner",+ sourceInstanceId: "runner-workflow-eval-child",+ capabilities: [...input.capabilities],+ });+ await adapter.applyCommand({+ runId: childRunId,+ idempotencyKey: `${childRunId}-finish`,+ command: {+ kind: "finish_task",+ taskId: child.id,+ summary: "Delegated verification completed.",+ },+ });+ await completeWorkflowRun(adapter, {+ runId: childRunId,+ sessionId: childSessionId,+ source: `${childRunId}-completion`,+ disposition: "done",+ summary: "Delegated verification completed.",+ });++ const returnRunId = `${input.parentRunId}-return`;+ await adapter.openFixtureRun({+ identity: {+ runId: returnRunId,+ sessionId: input.parentSessionId,+ companyId: initial.company.id,+ issueId: input.parentTaskId,+ agentId: initial.actors[0]!.id,+ },+ backendKind: "runner",+ sourceInstanceId: "runner-workflow-eval-return",+ capabilities: [...input.capabilities],+ wake: {+ reason: "blockers_resolved",+ payload: { completedTaskId: child.id },+ },+ });+ return { mockState: adapter.serialize(), childTaskId: child.id, returnRunId };+}++async function restoreAfterDelegatedChild(input: {+ session: CapabilityLiveSession;+ store: CapabilityLiveSessionStore;+ transportOptions: CapabilityLiveSessionServiceOptions["transportOptions"];+}): Promise<{+ service: CapabilityLiveSessionService;+ session: CapabilityLiveSession;+} | null> {+ const before = input.session.snapshot();+ const advanced = await advanceDelegationReturnMockState({+ mockState: before.mockState,+ parentRunId: before.authority.runId,+ parentSessionId: before.sessionId,+ parentTaskId: before.authority.taskId,+ capabilities: before.authority.capabilities,+ });+ if (advanced === null) return null;+ await input.session.suspend("workflow eval delegated child completed");+ const checkpoint = await input.store.load(before.sessionId);+ if (checkpoint === null) {+ throw new Error("workflow eval delegation checkpoint is missing");+ }+ const {+ providerRunBinding: _providerRunBinding,+ ...checkpointWithoutBinding+ } = checkpoint;+ const at = new Date().toISOString();+ const updated: CapabilityLiveSessionSnapshot = {+ ...checkpointWithoutBinding,+ revision: checkpoint.revision + 1,+ updatedAt: at,+ authority: {+ ...checkpoint.authority,+ runId: advanced.returnRunId,+ },+ mockState: advanced.mockState,+ stateHistory: [+ ...(checkpoint.stateHistory ?? []),+ {+ revision: JSON.parse(advanced.mockState).revision as number,+ at,+ turnId: null,+ operationId: "eval.delegated_child_completed",+ state: advanced.mockState,+ },+ ],+ };+ await input.store.save(updated);+ const service = new CapabilityLiveSessionService({+ store: input.store,+ transportOptions: input.transportOptions,+ });+ return { service, session: await service.restore(before.sessionId) };+}+function acpxAgent(candidate: RunnerLiveEvalCandidate,): "claude" | "codex" | undefined {@@ -142,6 +402,33 @@ function observedCalls(snapshot: CapabilityLiveSessionSnapshot): string[] {);}+export function unexpectedLiveWorkflowCalls(+ calls: readonly string[],+ expectedCalls: readonly string[],+): string[] {+ const expected = new Set(expectedCalls);+ return scorableLiveWorkflowCalls(calls, expectedCalls).filter(+ (operationId) => !expected.has(operationId),+ );+}++/**+ * Keeps required reads and every stateful operation in trajectory scoring while+ * ignoring optional read-only context gathering. The full call list remains in+ * the workflow evidence and metrics for auditability.+ */+export function scorableLiveWorkflowCalls(+ calls: readonly string[],+ expectedCalls: readonly string[],+): string[] {+ const expected = new Set(expectedCalls);+ return calls.filter(+ (operationId) =>+ expected.has(operationId) ||+ !isCapabilitySemanticReadOperation(operationId),+ );+}+function duplicateEffectSignals(snapshot: CapabilityLiveSessionSnapshot,): string[] {@@ -591,6 +878,24 @@ export async function executeLiveRunnerWorkflow(input: {),);await settleInteractions(input.evalCase, session, turns, withinBudget);+ if (input.evalCase.id === "delegation-return" && withinBudget()) {+ const restored = await restoreAfterDelegatedChild({+ session,+ store,+ transportOptions,+ });+ if (restored !== null) {+ service = restored.service;+ session = restored.session;+ subscribeToSession(session);+ turns.push(+ await session.sendMessage(+ "The delegated child completed successfully and the source task is unblocked. Use the authoritative child result and call finish_task exactly once without repeating the delegated work.",+ { allowMissingUsage: input.allowMissingUsage },+ ),+ );+ }+ }if (input.evalCase.id === "steering-causality" && withinBudget()) {turns.push(await session.sendMessage(continuationPrompt(input.evalCase), {@@ -660,9 +965,7 @@ export async function executeLiveRunnerWorkflow(input: {const missingCalls = expectedCalls.filter((operationId) => !calls.includes(operationId),);- const extraCalls = calls.filter(- (operationId) => !expectedCalls.includes(operationId),- );+ const extraCalls = unexpectedLiveWorkflowCalls(calls, expectedCalls);const duplicateSignals =snapshot === undefined ? [] : duplicateEffectSignals(snapshot);const pendingInteractions = session?.pendingInteractions().length ?? 0;@@ -712,7 +1015,7 @@ export async function executeLiveRunnerWorkflow(input: {provenance: { source: "live_model", behavior: input.evalCase.id },controlPlaneOwned: expectedCalls.length === 0,expectedCalls,- observedCalls: calls,+ observedCalls: scorableLiveWorkflowCalls(calls, expectedCalls),forbiddenCalls: [],finalState: {expected: expectedCalls.length === 0 ? "unchanged" : "mutated",diff --git a/packages/paperclip-runner/src/eval/live-workflow-matrix.ts b/packages/paperclip-runner/src/eval/live-workflow-matrix.tsindex c491bbe622..6ece8d18a4 100644--- a/packages/paperclip-runner/src/eval/live-workflow-matrix.ts+++ b/packages/paperclip-runner/src/eval/live-workflow-matrix.ts@@ -247,6 +247,65 @@ export interface RunnerLiveEvalSchedule {expectedExecutions: number;}+export interface RunnerLiveEvalSelection {+ candidateIds?: readonly string[];+ caseIds?: readonly string[];+ limit?: number;+}++/** Selects a stable paid subset without weakening validation of the full schedule. */+export function selectRunnerLiveEvalSchedule(+ schedule: RunnerLiveEvalSchedule,+ selection: RunnerLiveEvalSelection,+): RunnerLiveEvalSchedule {+ const candidateIds = new Set(selection.candidateIds ?? []);+ const caseIds = new Set(selection.caseIds ?? []);+ for (const id of candidateIds) {+ if (!schedule.candidates.some((candidate) => candidate.id === id)) {+ throw new Error(`unknown Runner live eval candidate: ${id}`);+ }+ }+ const scheduledCaseIds = new Set(+ schedule.entries.map((entry) => entry.caseId),+ );+ for (const id of caseIds) {+ if (!scheduledCaseIds.has(id as RunnerWorkflowEvalCase["id"])) {+ throw new Error(`unknown Runner live eval case: ${id}`);+ }+ }+ if (+ selection.limit !== undefined &&+ (!Number.isSafeInteger(selection.limit) || selection.limit <= 0)+ ) {+ throw new Error(+ "Runner live eval selection limit must be a positive integer",+ );+ }+ let entries = schedule.entries.filter(+ (entry) =>+ (candidateIds.size === 0 || candidateIds.has(entry.candidateId)) &&+ (caseIds.size === 0 || caseIds.has(entry.caseId)),+ );+ if (selection.limit !== undefined)+ entries = entries.slice(0, selection.limit);+ if (entries.length === 0) {+ throw new Error(+ "Runner live eval selection matched no scheduled executions",+ );+ }+ const selectedCandidateIds = new Set(+ entries.map((entry) => entry.candidateId),+ );+ return {+ ...schedule,+ candidates: schedule.candidates.filter((candidate) =>+ selectedCandidateIds.has(candidate.id),+ ),+ entries,+ expectedExecutions: entries.length,+ };+}+export function assertRunnerLiveCandidateManifest(): void {const slots = RUNNER_LIVE_CANDIDATE_SLOTS.map((slot) => slot.id);const candidates = RUNNER_LIVE_CANDIDATE_SLOTS.flatMap(diff --git a/packages/paperclip-runner/src/eval/workflow-evals.test.ts b/packages/paperclip-runner/src/eval/workflow-evals.test.tsindex aea24c13d3..b4e280dccb 100644--- a/packages/paperclip-runner/src/eval/workflow-evals.test.ts+++ b/packages/paperclip-runner/src/eval/workflow-evals.test.ts@@ -24,6 +24,7 @@ import {resolvedNightlyCandidates,runnerLiveRotationWeek,runnerLiveScheduleCoverage,+ selectRunnerLiveEvalSchedule,RunnerWorkflowInfrastructureError,} from "./live-workflow-matrix.js";import { unavailableLiveRunnerWorkflowObservation } from "./live-workflow-executor.js";@@ -306,6 +307,35 @@ describe("balanced live Runner workflow matrix", () => {);});+ it("selects a stable bounded paid subset and rejects invalid selectors", () => {+ const schedule = buildRunnerLiveEvalSchedule({+ seed: "subset-v1",+ rotationDay: 0,+ generatedAt: "2026-08-24T00:00:00.000Z",+ });+ const selected = selectRunnerLiveEvalSchedule(schedule, {+ candidateIds: ["codex-luna"],+ limit: 2,+ });+ expect(selected.expectedExecutions).toBe(2);+ expect(selected.entries).toHaveLength(2);+ expect(+ selected.entries.every((entry) => entry.candidateId === "codex-luna"),+ ).toBe(true);+ expect(selected.candidates.map((candidate) => candidate.id)).toEqual([+ "codex-luna",+ ]);+ expect(schedule.expectedExecutions).toBe(40);+ expect(() =>+ selectRunnerLiveEvalSchedule(schedule, {+ candidateIds: ["unknown-candidate"],+ }),+ ).toThrow("unknown Runner live eval candidate");+ expect(() => selectRunnerLiveEvalSchedule(schedule, { limit: 0 })).toThrow(+ "selection limit must be a positive integer",+ );+ });+it("retries only one retryable infrastructure failure", async () => {const schedule = buildRunnerLiveEvalSchedule({seed: "retry-v1",@@ -409,7 +439,7 @@ describe("workflow reports and stress traceability", () => {}});- it("renders safe JSON-derived Markdown, JUnit, and GitHub summaries", async () => {+ it("renders safe JSON-derived Evalbook, Markdown, JUnit, and GitHub summaries", async () => {const results = await runDeterministicRunnerWorkflowMatrix();const report = buildRunnerWorkflowEvalReport({source: "deterministic",diff --git a/packages/paperclip-runner/src/evals/model-pricing.test.ts b/packages/paperclip-runner/src/evals/model-pricing.test.tsindex 54c3a27c5d..b97495eb2e 100644--- a/packages/paperclip-runner/src/evals/model-pricing.test.ts+++ b/packages/paperclip-runner/src/evals/model-pricing.test.ts@@ -29,4 +29,21 @@ describe("model pricing", () => {}),).toThrow("model pricing unavailable");});++ it.each([+ ["openrouter/anthropic/claude-sonnet-5", 2, 0.2, 10],+ ["openrouter/qwen/qwen3.8-max-0902", 2, 0.25, 6],+ ["openrouter/google/gemini-3.8-flash", 0.75, 0.075, 3.75],+ ["openrouter/z-ai/glm-5.3", 1.4, 0.14, 4.4],+ ["openrouter/deepseek/deepseek-v4-flash-0731", 0.065, 0.016, 0.18],+ ["openrouter/openai/gpt-6-astra", 10, 1, 50],+ ])("pins the OpenRouter breadth price for %s", (model, input, cachedInput, output) => {+ expect(+ estimateModelCostNanodollars(model, {+ inputTokens: 1_000,+ cachedInputTokens: 400,+ outputTokens: 100,+ }).ratesUsdPerMillionTokens,+ ).toEqual({ input, cachedInput, output });+ });});diff --git a/packages/paperclip-runner/src/evals/model-pricing.ts b/packages/paperclip-runner/src/evals/model-pricing.tsindex 99cde385ed..8c374002a6 100644--- a/packages/paperclip-runner/src/evals/model-pricing.ts+++ b/packages/paperclip-runner/src/evals/model-pricing.ts@@ -1,4 +1,4 @@-export const MODEL_PRICING_VERSION = "provider-list-prices-2026-08-21" as const;+export const MODEL_PRICING_VERSION = "provider-list-prices-2026-09-05" as const;interface TokenRatesUsdPerMillion {input: number;@@ -18,6 +18,13 @@ const RATES: Readonly<Record<string, TokenRatesUsdPerMillion>> = Object.freeze({"claude-sonnet-5": { input: 3, cachedInput: 0.3, output: 15 },// Amazon Bedrock global cross-region list price for Claude Sonnet 4.6."global.anthropic.claude-sonnet-4-6": { input: 3, cachedInput: 0.3, output: 15 },+ // OpenRouter list-price snapshot used by the qualified OpenCode breadth lane.+ "openrouter/anthropic/claude-sonnet-5": { input: 2, cachedInput: 0.2, output: 10 },+ "openrouter/qwen/qwen3.8-max-0902": { input: 2, cachedInput: 0.25, output: 6 },+ "openrouter/google/gemini-3.8-flash": { input: 0.75, cachedInput: 0.075, output: 3.75 },+ "openrouter/z-ai/glm-5.3": { input: 1.4, cachedInput: 0.14, output: 4.4 },+ "openrouter/deepseek/deepseek-v4-flash-0731": { input: 0.065, cachedInput: 0.016, output: 0.18 },+ "openrouter/openai/gpt-6-astra": { input: 10, cachedInput: 1, output: 50 },});export interface EstimatedModelCost {diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.tsindex 9ed9cbaf04..b569868318 100644--- a/packages/paperclip-runner/src/live/live-session.test.ts+++ b/packages/paperclip-runner/src/live/live-session.test.ts@@ -87,6 +87,10 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {if (method === "initialize") return { user: { sessionId: this.state.providerSessionId } };if (method === "thread/start") {expect(Array.isArray(params.dynamicTools)).toBe(true);+ expect(params.completionContract).toEqual({+ revision: "paperclip-capability-live-v1",+ criterionIds: ["objective"],+ });return {model: "gpt-eval-test",modelProvider: "openai",@@ -633,10 +637,12 @@ describe("Capability live runnerd and Codex session", () => {it("persists the selected provider and passes it to the live runner transport", async () => {const state = providerState();const providers: Array<string | undefined> = [];+ const permissionModes: Array<string | undefined> = [];const baseFactory = fakeTransportFactory(state);const service = new CapabilityLiveSessionService({transportFactory: (options) => {providers.push(options.provider);+ permissionModes.push(options.opencodePermissionMode);return baseFactory(options);},});@@ -646,6 +652,7 @@ describe("Capability live runnerd and Codex session", () => {});expect(providers).toEqual(["opencode"]);+ expect(permissionModes).toEqual(["deny"]);expect(session.snapshot().config).toMatchObject({provider: "opencode",driver: "opencode_server",@@ -655,6 +662,45 @@ describe("Capability live runnerd and Codex session", () => {await service.shutdown(session.id);});+ it("attributes Claude Managed sessions to the pinned immutable Agent version", async () => {+ const state = providerState();+ const managedProfiles: Array<Record<string, unknown> | undefined> = [];+ const baseFactory = fakeTransportFactory(state);+ const service = new CapabilityLiveSessionService({+ transportFactory: (options) => {+ managedProfiles.push(options.managedProfile);+ return baseFactory(options);+ },+ });+ const session = await service.create({+ provider: "claude_managed",+ requestedModel: "claude-sonnet-5",+ managedProfile: {+ profileId: "managed-profile-1",+ anthropicAgentId: "agent-1",+ agentVersion: "17",+ environmentId: "environment-1",+ betaVersion: "managed-agents-2026-04-01",+ maxSessionListCostUsd: 0.5,+ },+ });++ expect(managedProfiles).toEqual([+ expect.objectContaining({+ agentVersion: "17",+ betaVersion: "managed-agents-2026-04-01",+ model: "claude-sonnet-5",+ }),+ ]);+ expect(session.snapshot().config).toMatchObject({+ provider: "claude_managed",+ driver: "claude_managed_agents_api",+ providerVersion: "17",+ requestedModel: "claude-sonnet-5",+ });+ await service.shutdown(session.id);+ });+it("opens lazy sessions with core and discovery gateways instead of optional schemas", async () => {const state = providerState();const claim = "discovery:agents:read";@@ -673,6 +719,74 @@ describe("Capability live runnerd and Codex session", () => {await service.shutdown(session.id);});+ it("keeps discovered invocations correlated to the provider gateway call", async () => {+ const state = providerState();+ const claim = "discovery:agents:read";+ const service = new CapabilityLiveSessionService({+ transportFactory: fakeTransportFactory(state),+ });+ const session = await service.create({+ runId: "run-live-lazy-invoke",+ sessionId: "session-live-lazy-invoke",+ toolExposure: "lazy",+ capabilities: [claim],+ explicitClaims: [claim],+ scenario: { id: "lazy-invoke", claims: [claim] },+ });+ let gatewayResult: Record<string, unknown> | null = null;+ state.onTurnStart = async () => {+ const transport = state.transports[0]!;+ const turnId = [...state.turns.keys()].at(-1)!;+ const discovery = await transport.invokeServerRequest({+ id: "request-discover",+ method: "item/tool/call",+ params: {+ threadId: state.threadId,+ turnId,+ callId: "call-discover",+ tool: "discover_capabilities",+ arguments: {+ query: "list company agents",+ namespace: "discovery",+ limit: 10,+ },+ },+ });+ expect(discovery.success).toBe(true);+ const invoked = await transport.invokeServerRequest({+ id: "request-invoke",+ method: "item/tool/call",+ params: {+ threadId: state.threadId,+ turnId,+ callId: "call-invoke",+ tool: "invoke_discovered_capability",+ arguments: { operationId: "list_agents", input: {} },+ },+ });+ gatewayResult = JSON.parse(+ String(+ (invoked.contentItems as Array<Record<string, unknown>>)[0]?.text,+ ),+ ) as Record<string, unknown>;+ };++ await session.sendMessage("Exercise the lazy discovery gateway.");++ expect(gatewayResult).toMatchObject({+ callId: "call-invoke",+ operationId: "invoke_discovered_capability",+ ok: true,+ });+ expect(+ session+ .snapshot()+ .evidence.filter((entry) => entry.kind === "tool_call")+ .map((entry) => entry.data.operationId),+ ).toContain("list_agents");+ await service.shutdown(session.id);+ });+it("returns typed mock results to the same multi-turn Codex thread without Paperclip network calls", async () => {const state = providerState();const fetchSpy = vi.spyOn(globalThis, "fetch");diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.tsindex dfd5cacefe..4f7b8c0f6b 100644--- a/packages/paperclip-runner/src/live/live-session.ts+++ b/packages/paperclip-runner/src/live/live-session.ts@@ -54,6 +54,10 @@ import {} from "./workspace-file-reference.js";const LIVE_SESSION_SCHEMA = "paperclip.capability.live-session.v1" as const;+const LIVE_COMPLETION_CONTRACT = Object.freeze({+ revision: "paperclip-capability-live-v1",+ criterionIds: ["objective"],+});const LIVE_BASE_INSTRUCTIONS = ["You are operating one mock Paperclip issue through typed semantic tools.","Use only the tools exposed in this thread; never call a Paperclip REST API.",@@ -922,7 +926,7 @@ export class CapabilityLiveSessionService {providerVersion: input.provider === "opencode"? "1.18.17": input.provider === "claude_managed"- ? input.managedProfile!.betaVersion+ ? input.managedProfile!.agentVersion: input.provider === "aws_agentcore"? input.agentCoreProfile!.qualificationRevision: input.provider === "acpx" ? acpxProfile!.acpxVersion : null,@@ -2269,6 +2273,15 @@ export class CapabilityLiveSession {const transportBundle = this.#transportFactory({...this.#transportOptions,provider,+ ...(provider === "opencode"+ ? {+ // Capability-live sessions expose only governed semantic tools and+ // use approvalPolicy=never. Keep OpenCode's ambient shell/file+ // tools fail-closed instead of brokering broader permissions.+ opencodePermissionMode:+ this.#transportOptions.opencodePermissionMode ?? "deny",+ }+ : {}),...(provider === "acpx" && this.#config.acpxAgent ? {acpxAgent: this.#config.acpxAgent,} : {}),@@ -2395,6 +2408,7 @@ export class CapabilityLiveSession {runtimeWorkspaceRoots: [this.#config.workingDirectory],approvalPolicy: "never",baseInstructions: LIVE_BASE_INSTRUCTIONS,+ completionContract: LIVE_COMPLETION_CONTRACT,dynamicTools: [...tools.map(dynamicToolSpec),...((this.#config.toolExposure ?? "eager") === "lazy" ? discoveryToolSpecs() : []),@@ -2554,7 +2568,15 @@ export class CapabilityLiveSession {this.#recordTerminalFact(this.#durableReplayTurnId(turnId), "completed");}await this.#persist();- return this.#codexToolResponse(result);+ // The durable provider bridge correlates the outer semantic result with+ // the exact dynamic tool Codex called. Keep the dispatched operation in+ // evidence, but preserve the discovery gateway identity on the response+ // envelope returned to runnerd.+ const providerResult =+ operationId === INVOKE_DISCOVERED_TOOL+ ? { ...result, operationId, callId }+ : result;+ return this.#codexToolResponse(providerResult);}#isDurableTerminalReplay(turnId: string, input: unknown): boolean {@@ -2579,7 +2601,7 @@ export class CapabilityLiveSession {return `${interruptedTurnId}:durable-duplicate-replay`;}- #codexToolResponse(result: CapabilitySemanticToolResult): Record<string, unknown> {+ #codexToolResponse(result: { readonly ok: boolean }): Record<string, unknown> {return {success: result.ok,contentItems: [{diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.tsindex b8ffc2d367..08815660d1 100644--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts@@ -59,6 +59,11 @@ import {withCodexCollaborationRuntimeInstructions,} from "./runnerd-codex-transport.js";+it("launches runnerd with its production durable outbox limits", () => {+ expect(runnerdLaunchProfileInternals.maxOutboxBytes).toBe(16 * 1024 * 1024);+ expect(runnerdLaunchProfileInternals.p0ReserveBytes).toBe(1024 * 1024);+});+it("replays the durable run attachment outcome and latest provider identity", () => {expect(runnerdRecoveryInternals.recoveredRunAttachment({@@ -1421,6 +1426,75 @@ it("runs the lab provider boundary through authenticated durable PRP", async ()});}, 30_000);+it("continues rehydrating events after the committed-event window slides", async () => {+ const stateDirectory = await mkdtemp(+ join(tmpdir(), "runnerd-sliding-event-window-"),+ );+ const bundle = createCapabilityRunnerdCodexTransport({+ runnerBinary: defaultCapabilityRunnerdBinary(),+ codexCommand: fakeCodex,+ codexArgs: fakeCodexArgs(stateDirectory, "--split-event-burst"),+ stateDirectory,+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },+ });+ bundle.transport.setServerRequestHandler(async () => ({+ success: true,+ contentItems: [+ {+ type: "inputText",+ text: JSON.stringify({ ok: true, result: { task: { id: "task-1" } } }),+ },+ ],+ }));+ try {+ await bundle.transport.request("initialize", {});+ await bundle.transport.request("thread/start", {+ cwd: tmpdir(),+ dynamicTools: [+ {+ name: "get_task_context",+ description: "Read the active task.",+ inputSchema: {+ type: "object",+ properties: {},+ additionalProperties: false,+ },+ },+ ],+ });+ await bundle.transport.request("turn/start", {+ input: [{ type: "text", text: "Emit a split event burst." }],+ });+ const notifications = bundle.transport+ .notifications()+ [Symbol.asyncIterator]();+ const methods: string[] = [];+ const deadline = Date.now() + 20_000;+ while (Date.now() < deadline) {+ const next = await Promise.race([+ notifications.next(),+ new Promise<never>((_, reject) =>+ setTimeout(+ () =>+ reject(new Error("sliding event window notification timeout")),+ 10_000,+ ),+ ),+ ]);+ if (!next.value) break;+ methods.push(next.value.method);+ if (next.value.method === "turn/completed") break;+ }+ expect(+ methods.filter((method) => method === "item/agentMessage/delta"),+ ).toHaveLength(144);+ expect(methods).toContain("turn/completed");+ } finally {+ await bundle.transport.close();+ await rm(stateDirectory, { recursive: true, force: true });+ }+}, 30_000);+it("binds an immediately failed durable turn before exposing its terminal", async () => {const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-fast-terminal-"),diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.tsindex fe384d7336..9698fb766a 100644--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts@@ -76,6 +76,8 @@ const MAX_NOTIFICATION_COUNT = 2_048;const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024;const RUNNER_CLIENT_VERSION = "0.3.0";const RUNNER_BOOTSTRAP_TICKET_TTL_MS = 60_000;+const RUNNERD_MAX_OUTBOX_BYTES = 16 * 1024 * 1024;+const RUNNERD_P0_RESERVE_BYTES = 1024 * 1024;function readLocalProcessStartedAt(pid: number): string | null {if (!Number.isInteger(pid) || pid <= 0) return null;@@ -1941,7 +1943,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {#handle: RunnerProcessHandle | null = null;#adoptedRunnerMonitor: NodeJS.Timeout | null = null;#pump: NodeJS.Timeout | null = null;- #eventIndex = 0;+ #eventSourceSeq = 0;+ #deferredTurnStartEvents: DurableRecoveryCommittedEvent[] = [];#threadId = "";#sessionId: string | null = null;#providerIdentity: Record<string, unknown> | null = null;@@ -2293,7 +2296,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {const previousRelease = this.#controlPlaneRelease;core.rotateRunIdentity(desired);- this.#eventIndex = 0;+ this.#eventSourceSeq = 0;+ this.#deferredTurnStartEvents = [];this.#durableTurnId = desired.turnId;this.#controlPlaneRelease = registration?.release ?? null;let previousReleased = false;@@ -3092,8 +3096,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {this.options.runnerStateDirectory ?? resolve(this.#root, "runner"),identity,ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS),- maxOutboxBytes: 256 * 1024,- p0ReserveBytes: 64 * 1024,+ maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,+ p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,maxRuntimeMs: 60 * 60 * 1_000,reconnectGraceMs: this.options.runnerReconnectGraceMs,lifecyclePolicy: this.options.lifecyclePolicy,@@ -3432,10 +3436,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {// If attachment completed, replay only its latest identity event into the// transport's in-memory evidence; session events are consumed internally// and are not duplicated onto the provider notification stream.- this.#eventIndex =+ this.#eventSourceSeq =runAttachment !== null && runAttachment.providerIdentityEventIndex >= 0- ? runAttachment.providerIdentityEventIndex- : committedEvents.length;+ ? committedEvents[runAttachment.providerIdentityEventIndex]!.sourceSeq -+ 1+ : core.store.state.ackedSourceSeq;const adoptedProviderIdentityIndex =latestProviderIdentityEventIndex(committedEvents);if (@@ -3494,8 +3499,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {this.options.runnerStateDirectory ?? resolve(this.#root, "runner"),identity,ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS),- maxOutboxBytes: 256 * 1024,- p0ReserveBytes: 64 * 1024,+ maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,+ p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,maxRuntimeMs: 60 * 60 * 1_000,reconnectGraceMs: this.options.runnerReconnectGraceMs,lifecyclePolicy: this.options.lifecyclePolicy,@@ -3781,8 +3786,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {#pumpEvents(): void {this.#flushPendingTraceRehydrations();const events = this.#core?.store.state.committedEvents ?? [];- while (this.#eventIndex < events.length) {- const event = events[this.#eventIndex]!;+ for (;;) {+ const deferredEvent =+ !this.#turnStartResponsePending || this.#expectedProviderTurnId !== null+ ? this.#deferredTurnStartEvents[0]+ : undefined;+ const event =+ deferredEvent ??+ events.find((candidate) => candidate.sourceSeq > this.#eventSourceSeq);+ if (event === undefined) return;+ const fromDeferredQueue = deferredEvent !== undefined;+ if (!fromDeferredQueue && event.sourceSeq !== this.#eventSourceSeq + 1) {+ throw new Error(+ `PRP provider event window advanced past source sequence ${this.#eventSourceSeq + 1}`,+ );+ }const eventPayload = record(event.envelope.payload).payload;const turnStartWhileCommandResultPending =this.#turnStartResponsePending &&@@ -3793,9 +3811,25 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {(notification) => notification.method === "turn/started",)));// The durable command result is the only correlation authority for a- // provider-assigned turn id. Leave an early start at the cursor until- // that exact expected identity is installed.- if (turnStartWhileCommandResultPending) return;+ // provider-assigned turn id. Copy an early start and its following+ // events out of the control plane's sliding window until that exact+ // expected identity is installed.+ if (+ !fromDeferredQueue &&+ (turnStartWhileCommandResultPending ||+ (this.#turnStartResponsePending &&+ this.#expectedProviderTurnId === null &&+ this.#deferredTurnStartEvents.length > 0))+ ) {+ if (this.#deferredTurnStartEvents.length >= 4_096) {+ throw new Error(+ "turn/start produced too many events before its durable command result",+ );+ }+ this.#eventSourceSeq = event.sourceSeq;+ this.#deferredTurnStartEvents.push(structuredClone(event));+ continue;+ }const terminalWhileTurnStartPending =this.#turnStartResponsePending &&([@@ -3808,8 +3842,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {unwrapRunnerdProviderNotifications(eventPayload).some((notification) => notification.method === "turn/completed",)));- if (terminalWhileTurnStartPending) return;- this.#eventIndex += 1;+ if (terminalWhileTurnStartPending) {+ if (!fromDeferredQueue) {+ this.#eventSourceSeq = event.sourceSeq;+ this.#deferredTurnStartEvents.push(structuredClone(event));+ }+ return;+ }+ // The control plane retains a sliding committed-event window. Track its+ // durable protocol cursor rather than an array index: once that array is+ // full, new events replace its prefix without increasing its length.+ if (fromDeferredQueue) this.#deferredTurnStartEvents.shift();+ else this.#eventSourceSeq = event.sourceSeq;if (event.eventType === "harness.ready" ||event.eventType === "session.started" ||@@ -4535,6 +4579,8 @@ export const runnerdLaunchProfileInternals = Object.freeze({acpxProviderPackageAuthority,acpxRunnerLaunchProfile,resolveBuildOwnedCliArtifact,+ maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,+ p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,});export const runnerdRecoveryInternals = Object.freeze({diff --git a/packages/paperclip-runner/src/semantic-tools/discovery.ts b/packages/paperclip-runner/src/semantic-tools/discovery.tsindex 0ff30e281f..c6adf41dcd 100644--- a/packages/paperclip-runner/src/semantic-tools/discovery.ts+++ b/packages/paperclip-runner/src/semantic-tools/discovery.ts@@ -17,6 +17,8 @@ export interface CapabilityDiscoveryResult {readonly truncated: boolean;}+const MAX_DISCOVERY_RESULTS = 10;+const NAMESPACE: Readonly<Record<CapabilitySemanticOperationId, string>> = Object.freeze({get_task_context: "active_task", get_task_history: "active_task",list_documents: "documents", read_document: "documents", list_document_revisions: "documents",@@ -47,7 +49,7 @@ export const CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS = Object.freeze([{type: "object", properties: {query: { type: "string", minLength: 1, maxLength: 500 },namespace: { type: "string" },- limit: { type: "integer", minimum: 1, maximum: 8 },+ limit: { type: "integer", minimum: 1, maximum: MAX_DISCOVERY_RESULTS },}, required: ["query"], additionalProperties: false,},outputSchema: {@@ -90,7 +92,10 @@ export function discoverCapabilityDefinitions(): CapabilityDiscoveryResult {const normalized = query.trim().toLowerCase();if (normalized.length === 0) throw new Error("discovery_query_empty");- const limit = Math.max(1, Math.min(options.limit ?? 5, 8));+ const limit = Math.max(+ 1,+ Math.min(options.limit ?? 5, MAX_DISCOVERY_RESULTS),+ );const tokens = normalized.split(/[^a-z0-9]+/).filter((token) => token.length > 1);const permitted = CAPABILITY_SEMANTIC_TOOL_CATALOG.filter((descriptor) => descriptor.exposure === "optional")diff --git a/packages/paperclip-runner/src/semantic-tools/policy.ts b/packages/paperclip-runner/src/semantic-tools/policy.tsindex 0b077eea8d..6bcff09bec 100644--- a/packages/paperclip-runner/src/semantic-tools/policy.ts+++ b/packages/paperclip-runner/src/semantic-tools/policy.ts@@ -8,7 +8,7 @@ import type {CapabilitySemanticToolDescriptor,} from "./types.js";-const READ_OPERATIONS = new Set<CapabilitySemanticOperationId>([+export const CAPABILITY_SEMANTIC_READ_OPERATION_IDS = ["get_task_context","get_task_history","list_documents",@@ -21,7 +21,15 @@ const READ_OPERATIONS = new Set<CapabilitySemanticOperationId>(["get_approval","get_approval_context","get_workspace_runtime",-]);+] as const satisfies readonly CapabilitySemanticOperationId[];++const READ_OPERATIONS = new Set<CapabilitySemanticOperationId>(+ CAPABILITY_SEMANTIC_READ_OPERATION_IDS,+);++export function isCapabilitySemanticReadOperation(operationId: string): boolean {+ return READ_OPERATIONS.has(operationId as CapabilitySemanticOperationId);+}export const DEFAULT_CAPABILITY_SCENARIO_POLICY: CapabilitySemanticScenarioPolicy = {id: "capability-default",diff --git a/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts b/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.tsindex 1d0fa1cb8c..5d7ca42dc1 100644--- a/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts+++ b/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";import type { CapabilityFixtureSeed } from "../mock-core/capability-control-plane-types.js";import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";import { CAPABILITY_SEMANTIC_TOOL_CATALOG } from "./catalog.js";+import { CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS } from "./discovery.js";import { CapabilitySemanticDispatcher } from "./dispatcher.js";import { createCapabilityProviderNeutralBinding } from "./provider-neutral.js";@@ -42,6 +43,13 @@ async function running(}describe("Capability semantic catalog and authorization", () => {+ it("accepts the conventional ten-result capability discovery limit", () => {+ expect(+ CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS[0].inputSchema.properties.limit+ .maximum,+ ).toBe(10);+ });+it("publishes a stable narrow catalog without credentials or control-plane-owned tools", () => {const names = CAPABILITY_SEMANTIC_TOOL_CATALOG.map((tool) => tool.operationId);expect(new Set(names).size).toBe(names.length);diff --git a/packages/paperclip-runner/vite.issue-thread.config.ts b/packages/paperclip-runner/vite.issue-thread.config.tsindex 17b33a483d..fe235b3172 100644--- a/packages/paperclip-runner/vite.issue-thread.config.ts+++ b/packages/paperclip-runner/vite.issue-thread.config.ts@@ -14,7 +14,6 @@ export default defineConfig({plugins: [react(), capabilityIssueThreadServerPlugin()],server: { host: "127.0.0.1" },preview: { host: "127.0.0.1" },- optimizeDeps: { esbuildOptions: { target: "esnext" } },build: {outDir: resolve(packageRoot, "dist-issue-thread"),emptyOutDir: true,diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjsindex d98ccbfcdd..aaf3179215 100644--- a/scripts/__tests__/release-verify-workflow.test.mjs+++ b/scripts/__tests__/release-verify-workflow.test.mjs@@ -248,6 +248,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {readWorkflow("runner-chaos-evals.yml"),readWorkflow("runner-full-stack-e2e.yml"),readWorkflow("e2e.yml"),+ readWorkflow("runner-protocol-live-evals.yml"),];for (const workflow of actionPinWorkflows) {@@ -286,6 +287,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {"e2e.yml","runner-full-stack-e2e.yml","runner-live-evals.yml",+ "runner-protocol-live-evals.yml",];const paidWorkflowNameSet = new Set(paidWorkflowNames);const providerSecretReference =@@ -322,7 +324,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {assert.match(block, /\n environment:\n name: runner-e2e-paid\n/);assert.match(block,- /\n steps:\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/,+ /\n steps:(?: &[A-Za-z0-9_-]+)?\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/,`${name} must reauthorize as the first provider-job step`,);const reauthorize = block.indexOf(" - name: Reauthorize");@@ -357,7 +359,11 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {);assert.doesNotMatch(historyPublisher, /^\s+cache: pnpm$/m);- for (const name of ["runner-full-stack-e2e.yml", "runner-live-evals.yml"]) {+ for (const name of [+ "runner-full-stack-e2e.yml",+ "runner-live-evals.yml",+ "runner-protocol-live-evals.yml",+ ]) {const workflow = readWorkflow(name);const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map((match) => match[1],diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.tsindex b875add0d3..03a68f6af8 100644--- a/tests/runner-e2e/workflow-security.test.ts+++ b/tests/runner-e2e/workflow-security.test.ts@@ -121,15 +121,18 @@ describe("public repository paid workflow security", () => {it("gates every provider-secret job with stable actor IDs", async () => {const workflows = await Promise.all(- ["runner-full-stack-e2e.yml", "runner-live-evals.yml", "e2e.yml"].map(- async (name) => ({- name,- contents: await readFile(- path.join(repositoryRoot, ".github/workflows", name),- "utf8",- ),- }),- ),+ [+ "runner-full-stack-e2e.yml",+ "runner-live-evals.yml",+ "runner-protocol-live-evals.yml",+ "e2e.yml",+ ].map(async (name) => ({+ name,+ contents: await readFile(+ path.join(repositoryRoot, ".github/workflows", name),+ "utf8",+ ),+ })),);for (const { name, contents } of workflows) {@@ -482,6 +485,7 @@ describe("public repository paid workflow security", () => {"e2e.yml","runner-full-stack-e2e.yml","runner-live-evals.yml",+ "runner-protocol-live-evals.yml",]);const names = (await readdir(workflowDirectory)).filter((name) =>/\.ya?ml$/.test(name),@@ -508,7 +512,11 @@ describe("public repository paid workflow security", () => {it("runs paid scheduled campaigns only on Sundays", async () => {const workflows = await Promise.all(- ["runner-full-stack-e2e.yml", "runner-live-evals.yml"].map((name) =>+ [+ "runner-full-stack-e2e.yml",+ "runner-live-evals.yml",+ "runner-protocol-live-evals.yml",+ ].map((name) =>readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),),);