Skip to content

The open-source app everyone uses to manage agents at work

feat(runner): restore direct live eval campaigns and reports (#12909)

Dottaaf8439a70b7a
diff --git a/.github/workflows/runner-live-evals.yml b/.github/workflows/runner-live-evals.yml
index 978bdca02e..9f797ab89d 100644
--- a/.github/workflows/runner-live-evals.yml
+++ b/.github/workflows/runner-live-evals.yml
@@ -4,6 +4,19 @@ on:
schedule:
- cron: "17 6 * * 0"
workflow_dispatch:
+ inputs:
+ candidate:
+ description: "Comma-separated live candidate IDs (for example codex-luna)"
+ type: string
+ required: false
+ case:
+ description: "Comma-separated workflow case IDs"
+ type: string
+ required: false
+ limit:
+ description: "Maximum executions after candidate/case filtering"
+ type: string
+ required: false
concurrency:
group: runner-live-evals-${{ github.ref }}
@@ -17,6 +30,8 @@ jobs:
timeout-minutes: 5
permissions:
contents: read
+ outputs:
+ eval_runner: ${{ steps.runner.outputs.runner }}
steps:
- name: Require default branch and allowlisted numeric actor IDs
env:
@@ -50,11 +65,30 @@ jobs:
fi
done
+ - name: Select paid eval runner
+ id: runner
+ env:
+ AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}
+ run: |
+ set -euo pipefail
+ github_runner='ubuntu-latest'
+ aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'
+
+ if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then
+ echo "runner=$aws_runner" >> "$GITHUB_OUTPUT"
+ echo '::notice title=Paid eval routing::Using an ephemeral RunsOn Fleet runner'
+ else
+ echo "runner=$github_runner" >> "$GITHUB_OUTPUT"
+ echo '::notice title=Paid eval routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner'
+ fi
+
live_matrix:
name: Balanced provider/model matrix
needs: authorize
if: github.event_name != 'schedule' || vars.RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true'
- runs-on: ubuntu-latest
+ # The authorize job selects one of two literal, reviewed runner labels;
+ # dispatch inputs and repository variables cannot inject an arbitrary label.
+ runs-on: ${{ needs.authorize.outputs.eval_runner }}
timeout-minutes: 180
permissions:
contents: read
@@ -81,6 +115,26 @@ jobs:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - name: Generate private eval-repository token
+ id: evals_token
+ env:
+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
+ GH_REPO: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ token="$(node .github/scripts/get-bot-token.mjs)"
+ echo "::add-mask::$token"
+ echo "value=$token" >> "$GITHUB_OUTPUT"
+
+ - name: Checkout canonical Evalbook reporter
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ repository: paperclipai/paperclip-evals
+ ref: 0c8dfea0ef71a73e909b59a5c0484554cbee199b
+ path: .paperclip-evals
+ token: ${{ steps.evals_token.outputs.value }}
+ persist-credentials: false
+
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
@@ -114,6 +168,10 @@ jobs:
PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }}
PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12"
PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-week-v1
+ PAPERCLIP_EVAL_CANDIDATE: ${{ inputs.candidate }}
+ PAPERCLIP_EVAL_CASE: ${{ inputs.case }}
+ PAPERCLIP_EVAL_LIMIT: ${{ inputs.limit }}
+ PAPERCLIP_EVALBOOK_PROGRAM: ${{ github.workspace }}/.paperclip-evals/evals/paperclip-runner/tools/eval_program.py
run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals
- name: Publish job summary
diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml
new file mode 100644
index 0000000000..9f6352b033
--- /dev/null
+++ b/.github/workflows/runner-protocol-live-evals.yml
@@ -0,0 +1,679 @@
+name: Runner Direct Live Protocol Evals
+
+on:
+ schedule:
+ - cron: "23 9 * * 0"
+ workflow_dispatch:
+ inputs:
+ target_branch:
+ description: "Branch in paperclipai/paperclip to evaluate; trusted orchestration still runs from master"
+ type: string
+ required: false
+ evals_sha:
+ description: "Exact 40-character paperclipai/paperclip-evals commit to execute"
+ type: string
+ required: false
+ rosters:
+ description: "Comma-separated live roster IDs/files, or all for the complete direct suite"
+ type: string
+ default: "all"
+ required: false
+ max_infrastructure_retries:
+ description: "Automatic retries only for explicitly retryable infrastructure failures (0-3)"
+ type: number
+ default: 1
+
+permissions:
+ contents: read
+
+concurrency:
+ group: runner-protocol-live-evals-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }}
+ cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}
+
+jobs:
+ authorize:
+ name: Authorize paid direct eval campaign
+ if: github.event_name != 'schedule' || vars.RUNNER_PROTOCOL_EVAL_NIGHTLY_ENABLED == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ permissions:
+ contents: read
+ outputs:
+ test_runner: ${{ steps.runner.outputs.runner }}
+ max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }}
+ max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }}
+ target_sha: ${{ steps.target.outputs.sha }}
+ target_ref: ${{ steps.target.outputs.ref }}
+ evals_sha: ${{ steps.evals.outputs.sha }}
+ steps:
+ - name: Require default branch and allowlisted numeric actor IDs
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPOSITORY: ${{ github.repository }}
+ REF: ${{ github.ref }}
+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ ACTOR: ${{ github.actor }}
+ ACTOR_ID: ${{ github.actor_id }}
+ TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+ ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
+ run: |
+ set -euo pipefail
+ if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then
+ echo "Paid direct Runner eval campaigns may run only from the default branch." >&2
+ exit 1
+ fi
+ if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then
+ echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2
+ exit 1
+ fi
+ triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
+ if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then
+ echo "GitHub actor identity contexts disagree; refusing the paid run." >&2
+ exit 1
+ fi
+ for candidate in "$triggering_actor_id" "$ACTOR_ID"; do
+ if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then
+ echo "The initiating GitHub account is not authorized to run paid Runner eval campaigns." >&2
+ exit 1
+ fi
+ done
+
+ - name: Resolve requested Paperclip branch to an immutable commit
+ id: target
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPOSITORY: ${{ github.repository }}
+ TARGET_BRANCH: ${{ inputs.target_branch || github.event.repository.default_branch }}
+ run: |
+ set -euo pipefail
+ if [ -z "$TARGET_BRANCH" ] || [[ "$TARGET_BRANCH" == refs/* ]]; then
+ echo "target_branch must name a branch in this repository without a refs/ prefix." >&2
+ exit 1
+ fi
+ encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')"
+ target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)"
+ [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]]
+ echo "sha=$target_sha" >> "$GITHUB_OUTPUT"
+ echo "ref=refs/heads/$TARGET_BRANCH" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Generate private eval-repository token
+ id: evals_token
+ env:
+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
+ GH_REPO: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ token="$(node .github/scripts/get-bot-token.mjs)"
+ echo "::add-mask::$token"
+ echo "value=$token" >> "$GITHUB_OUTPUT"
+
+ - name: Verify the private eval program is pinned to an exact commit
+ id: evals
+ env:
+ GH_TOKEN: ${{ steps.evals_token.outputs.value }}
+ EVALS_SHA: ${{ inputs.evals_sha || vars.RUNNER_PROTOCOL_EVALS_SHA }}
+ run: |
+ set -euo pipefail
+ if ! [[ "$EVALS_SHA" =~ ^[0-9a-f]{40}$ ]]; then
+ echo "evals_sha (or RUNNER_PROTOCOL_EVALS_SHA for schedules) must be an exact 40-character commit." >&2
+ exit 1
+ fi
+ resolved="$(gh api -X GET "repos/paperclipai/paperclip-evals/commits/$EVALS_SHA" --jq .sha)"
+ test "$resolved" = "$EVALS_SHA"
+ echo "sha=$resolved" >> "$GITHUB_OUTPUT"
+
+ - name: Validate retry envelope
+ env:
+ RETRIES: ${{ github.event_name == 'schedule' && 1 || inputs.max_infrastructure_retries }}
+ run: |
+ set -euo pipefail
+ [[ "$RETRIES" =~ ^[0-3]$ ]]
+
+ - name: Select paid test runner
+ id: runner
+ env:
+ AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}
+ run: |
+ set -euo pipefail
+ if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then
+ {
+ echo 'runner=runs-on/fleet=paperclip-public-pr-x64/env=public-ci'
+ echo 'max_parallel_default=100'
+ echo 'max_parallel_limit=100'
+ } >> "$GITHUB_OUTPUT"
+ else
+ {
+ echo 'runner=ubuntu-latest'
+ echo 'max_parallel_default=32'
+ echo 'max_parallel_limit=57'
+ } >> "$GITHUB_OUTPUT"
+ fi
+
+ catalog:
+ name: Pin and fan out the direct Evalbook roster
+ needs: authorize
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ outputs:
+ matrix_0: ${{ steps.catalog.outputs.matrix_0 }}
+ matrix_1: ${{ steps.catalog.outputs.matrix_1 }}
+ matrix_1_present: ${{ steps.catalog.outputs.matrix_1_present }}
+ max_parallel_per_shard: ${{ steps.catalog.outputs.max_parallel_per_shard }}
+ selected: ${{ steps.catalog.outputs.selected }}
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Generate private eval-repository token
+ id: evals_token
+ env:
+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
+ GH_REPO: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ token="$(node .github/scripts/get-bot-token.mjs)"
+ echo "::add-mask::$token"
+ echo "value=$token" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ repository: paperclipai/paperclip-evals
+ ref: ${{ needs.authorize.outputs.evals_sha }}
+ path: .paperclip-evals
+ token: ${{ steps.evals_token.outputs.value }}
+ persist-credentials: false
+
+ - name: Build the two bounded roster-plus-case matrices
+ id: catalog
+ env:
+ PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
+ PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }}
+ MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }}
+ MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }}
+ ROSTERS: ${{ inputs.rosters || 'all' }}
+ run: |
+ set -euo pipefail
+ if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -lt 2 ] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then
+ echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 2 through $MAX_PARALLEL_LIMIT for the two-shard direct suite." >&2
+ exit 1
+ fi
+ node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog \
+ --evals-root .paperclip-evals \
+ --rosters "$ROSTERS" \
+ --campaign-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
+ --max-parallel "$MAX_PARALLEL" \
+ --output runner-protocol-eval-catalog.json
+
+ - name: Upload immutable campaign catalog
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }}
+ path: runner-protocol-eval-catalog.json
+ retention-days: 30
+ if-no-files-found: error
+
+ build_runner:
+ name: Build portable direct-eval runner once
+ needs: [authorize, catalog]
+ runs-on: ${{ needs.authorize.outputs.test_runner }}
+ timeout-minutes: 30
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ needs.authorize.outputs.target_sha }}
+ persist-credentials: false
+
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+
+ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ env:
+ NPM_CONFIG_AUDIT: "false"
+ NPM_CONFIG_FUND: "false"
+ NPM_CONFIG_UPDATE_NOTIFIER: "false"
+ with:
+ version: 9.15.4
+
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+ cache: pnpm
+
+ - run: pnpm install --frozen-lockfile --ignore-scripts
+
+ - name: Build runner CLI, daemon, and canonical attempt viewer
+ run: |
+ set -euo pipefail
+ pnpm --filter @paperclipai/paperclip-runner build:typescript
+ pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
+ pnpm --filter @paperclipai/paperclip-runner build:issue-thread
+
+ - name: Package a portable provider runtime
+ run: |
+ set -euo pipefail
+ mkdir -p "$RUNNER_TEMP/runner-protocol-build/package" "$RUNNER_TEMP/runner-protocol-build/portable"
+ pnpm --dir packages/paperclip-runner pack \
+ --pack-destination "$RUNNER_TEMP/runner-protocol-build/package"
+ package="$(find "$RUNNER_TEMP/runner-protocol-build/package" -maxdepth 1 -type f -name '*.tgz' -print -quit)"
+ test -f "$package"
+ npm install --prefix "$RUNNER_TEMP/runner-protocol-build/portable" --omit=dev "$package"
+ cp "$package" "$RUNNER_TEMP/runner-protocol-build/paperclip-runner.tgz"
+ cp packages/paperclip-runner/runner/target/debug/paperclip-runnerd "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd"
+ cp -R packages/paperclip-runner/dist-issue-thread "$RUNNER_TEMP/runner-protocol-build/dist-issue-thread"
+ test -f "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js"
+ test -x "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd"
+ tar --create --gzip --file runner-protocol-build.tar.gz -C "$RUNNER_TEMP/runner-protocol-build" .
+ sha256sum runner-protocol-build.tar.gz > runner-protocol-build.tar.gz.sha256
+
+ - name: Upload immutable portable runner
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ runner-protocol-build.tar.gz
+ runner-protocol-build.tar.gz.sha256
+ retention-days: 1
+ compression-level: 0
+ if-no-files-found: error
+
+ eval_shard_0:
+ name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }}
+ needs: [authorize, catalog, build_runner]
+ runs-on: ${{ needs.authorize.outputs.test_runner }}
+ timeout-minutes: 18
+ permissions:
+ contents: read
+ id-token: write
+ environment:
+ name: runner-e2e-paid
+ strategy:
+ fail-fast: false
+ max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }}
+ matrix: ${{ fromJSON(needs.catalog.outputs.matrix_0) }}
+ steps: &direct_eval_steps
+ - name: Reauthorize paid execution before provider access
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REF: ${{ github.ref }}
+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ ACTOR: ${{ github.actor }}
+ ACTOR_ID: ${{ github.actor_id }}
+ TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+ ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
+ run: |
+ set -euo pipefail
+ test "$REF" = "refs/heads/$DEFAULT_BRANCH"
+ triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)"
+ test "$triggering_actor_id" = "$ACTOR_ID" || test "$TRIGGERING_ACTOR" != "$ACTOR"
+ for candidate in "$triggering_actor_id" "$ACTOR_ID"; do
+ jq -e --argjson candidate "$candidate" 'type == "array" and index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null
+ done
+
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Generate private eval-repository token
+ id: evals_token
+ env:
+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
+ GH_REPO: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ token="$(node .github/scripts/get-bot-token.mjs)"
+ echo "::add-mask::$token"
+ echo "value=$token" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ repository: paperclipai/paperclip-evals
+ ref: ${{ needs.authorize.outputs.evals_sha }}
+ path: .paperclip-evals
+ token: ${{ steps.evals_token.outputs.value }}
+ persist-credentials: false
+
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+
+ - name: Download portable runner
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+ path: runner-protocol-build
+
+ - name: Verify and extract portable runner
+ run: |
+ set -euo pipefail
+ cd runner-protocol-build
+ sha256sum --check runner-protocol-build.tar.gz.sha256
+ mkdir extracted
+ tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted
+ test -x extracted/paperclip-runnerd
+
+ - name: Prepare short-lived AgentCore web identity
+ if: matrix.credentialName == 'AWS_AGENTCORE_OIDC'
+ env:
+ AGENTCORE_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN }}
+ run: |
+ set -euo pipefail
+ test -n "$AGENTCORE_ROLE_ARN"
+ token="$(curl --fail --silent --show-error \
+ -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
+ "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" | jq -r .value)"
+ test -n "$token"
+ echo "::add-mask::$token"
+ token_file="$RUNNER_TEMP/runner-protocol-agentcore-token"
+ printf '%s' "$token" > "$token_file"
+ chmod 600 "$token_file"
+ {
+ echo "AWS_WEB_IDENTITY_TOKEN_FILE=$token_file"
+ echo "AWS_ROLE_ARN=$AGENTCORE_ROLE_ARN"
+ echo "AWS_ROLE_SESSION_NAME=runner-protocol-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ } >> "$GITHUB_ENV"
+
+ - name: Run one immutable direct protocol cell
+ id: direct_eval
+ env:
+ CELL_ID: ${{ matrix.cellId }}
+ ROSTER_FILE: ${{ matrix.rosterFile }}
+ CASE_ID: ${{ matrix.caseId }}
+ CREDENTIAL_NAME: ${{ matrix.credentialName }}
+ PROVIDER: ${{ matrix.provider }}
+ MAX_INFRASTRUCTURE_RETRIES: ${{ github.event_name == 'schedule' && 1 || inputs.max_infrastructure_retries }}
+ OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }}
+ ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }}
+ OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }}
+ PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID }}
+ PAPERCLIP_CLAUDE_MANAGED_AGENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_ID }}
+ PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION }}
+ PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID }}
+ AWS_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }}
+ AWS_DEFAULT_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }}
+ PAPERCLIP_AWS_AGENTCORE_PROFILE_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_PROFILE_ID }}
+ PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID }}
+ PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN }}
+ PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION }}
+ PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN }}
+ PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER }}
+ PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN }}
+ PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN }}
+ PAPERCLIP_AWS_AGENTCORE_MEMORY_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ID }}
+ PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN }}
+ PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET }}
+ PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX }}
+ PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN }}
+ PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION }}
+ run: |
+ set -euo pipefail
+ mkdir -p cell-output/runs
+ if [ "$CREDENTIAL_NAME" != "AWS_AGENTCORE_OIDC" ]; then
+ test -n "${!CREDENTIAL_NAME:-}"
+ fi
+ if [ "$PROVIDER" = "claude_managed" ]; then
+ test -n "$PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID"
+ test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_ID"
+ test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION"
+ test -n "$PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID"
+ fi
+ set +e
+ python3 .paperclip-evals/evals/paperclip-runner/tools/run_live_roster.py run \
+ --roster ".paperclip-evals/evals/paperclip-runner/rosters/$ROSTER_FILE" \
+ --case "$CASE_ID" \
+ --runner-cli runner-protocol-build/extracted/portable/node_modules/@paperclipai/paperclip-runner/dist/cli/eval-session.js \
+ --runner-package runner-protocol-build/extracted/paperclip-runner.tgz \
+ --runnerd runner-protocol-build/extracted/paperclip-runnerd \
+ --runs-root cell-output/runs \
+ --max-infrastructure-retries "$MAX_INFRASTRUCTURE_RETRIES" \
+ --run-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${CELL_ID}"
+ status=$?
+ set -e
+ CELL_EXIT_CODE="$status" node --input-type=module <<'NODE'
+ import { writeFileSync } from "node:fs";
+ writeFileSync("cell-output/cell.json", `${JSON.stringify({
+ schema: "paperclip.runner-protocol-eval.cell/v1",
+ cellId: process.env.CELL_ID,
+ rosterFile: process.env.ROSTER_FILE,
+ caseId: process.env.CASE_ID,
+ exitCode: Number(process.env.CELL_EXIT_CODE),
+ }, null, 2)}\n`, { mode: 0o600 });
+ NODE
+ exit "$status"
+
+ - name: Upload access-controlled cell attempt
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.cellId }}
+ path: cell-output/
+ retention-days: 30
+ if-no-files-found: error
+
+ eval_shard_1:
+ name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }}
+ if: needs.catalog.outputs.matrix_1_present == 'true'
+ needs: [authorize, catalog, build_runner]
+ runs-on: ${{ needs.authorize.outputs.test_runner }}
+ timeout-minutes: 18
+ permissions:
+ contents: read
+ id-token: write
+ environment:
+ name: runner-e2e-paid
+ strategy:
+ fail-fast: false
+ max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }}
+ matrix: ${{ fromJSON(needs.catalog.outputs.matrix_1) }}
+ steps: *direct_eval_steps
+
+ report:
+ name: Merge attempts and render canonical Evalbook
+ if: always() && !cancelled() && needs.catalog.result == 'success' && needs.build_runner.result == 'success'
+ needs: [authorize, catalog, build_runner, eval_shard_0, eval_shard_1]
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ permissions:
+ actions: read
+ contents: read
+ outputs:
+ public_report_ready: ${{ steps.public_report.outputs.ready }}
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Generate private eval-repository token
+ id: evals_token
+ env:
+ COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
+ GH_REPO: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ token="$(node .github/scripts/get-bot-token.mjs)"
+ echo "::add-mask::$token"
+ echo "value=$token" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ repository: paperclipai/paperclip-evals
+ ref: ${{ needs.authorize.outputs.evals_sha }}
+ path: .paperclip-evals
+ token: ${{ steps.evals_token.outputs.value }}
+ persist-credentials: false
+
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+
+ - name: Download immutable campaign catalog
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }}
+ path: runner-protocol-catalog
+
+ - name: Download portable runner and viewer
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+ path: runner-protocol-build
+
+ - name: Download every access-controlled cell
+ id: download_cells
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-*
+ path: downloaded-runner-protocol-evals
+ merge-multiple: false
+
+ - name: Retry cell download after artifact transport failure
+ if: steps.download_cells.outcome == 'failure'
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-*
+ path: downloaded-runner-protocol-evals
+ merge-multiple: false
+
+ - name: Materialize an empty download root when every cell failed early
+ run: mkdir -p downloaded-runner-protocol-evals
+
+ - name: Verify portable viewer
+ run: |
+ set -euo pipefail
+ cd runner-protocol-build
+ sha256sum --check runner-protocol-build.tar.gz.sha256
+ mkdir extracted
+ tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted
+ test -f extracted/dist-issue-thread/index.html
+
+ - name: Aggregate every expected cell, including missing infrastructure cells
+ env:
+ PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
+ PAPERCLIP_PROTOCOL_EVAL_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
+ PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }}
+ PAPERCLIP_PROTOCOL_EVAL_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs aggregate \
+ --catalog runner-protocol-catalog/runner-protocol-eval-catalog.json \
+ --downloads downloaded-runner-protocol-evals \
+ --evals-root .paperclip-evals \
+ --runs-out runner-protocol-merged/runs \
+ --campaign-out runner-protocol-merged/campaign.json
+
+ - name: Render the access-controlled canonical Evalbook report
+ run: |
+ python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \
+ --runs-root runner-protocol-merged/runs \
+ --output runner-protocol-merged/report \
+ --viewer-root runner-protocol-build/extracted/dist-issue-thread \
+ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \
+ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json
+ cp runner-protocol-merged/campaign.json runner-protocol-merged/report/campaign.json
+
+ - name: Render the same canonical grid from a public-safe evidence projection
+ run: |
+ node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs sanitize \
+ --runs-root runner-protocol-merged/runs \
+ --output runner-protocol-merged/public-runs
+ python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \
+ --runs-root runner-protocol-merged/public-runs \
+ --output runner-protocol-merged/public-report \
+ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \
+ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json
+ cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json
+
+ - name: Enforce the static public allowlist
+ id: public_report
+ run: |
+ node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report");'
+ echo "ready=true" >> "$GITHUB_OUTPUT"
+
+ - name: Add campaign result to the workflow summary
+ run: |
+ {
+ echo '## Runner direct live protocol evals'
+ echo
+ jq -r '"- Cells: \(.totals.passed)/\(.totals.selected) passed\n- Behavior failures: \(.totals.behaviorFailures)\n- Infrastructure failures: \(.totals.infrastructureFailures)\n- Paperclip: `\(.source.paperclip.sha)`\n- Evals: `\(.source.evals.sha)`"' runner-protocol-merged/campaign.json
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload access-controlled canonical Evalbook and raw attempts
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: runner-protocol-eval-report-${{ github.run_id }}-${{ github.run_attempt }}
+ path: runner-protocol-merged/
+ retention-days: 30
+ if-no-files-found: error
+
+ - name: Upload publisher-only sanitized Evalbook
+ if: steps.public_report.outputs.ready == 'true'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }}
+ path: runner-protocol-merged/public-report/
+ retention-days: 1
+ if-no-files-found: error
+
+ - name: Enforce complete green campaign
+ if: always()
+ run: jq -e '.complete == true and .allPassed == true' runner-protocol-merged/campaign.json >/dev/null
+
+ publish_history:
+ name: Publish immutable Evalbook and mutable campaign index
+ needs: [authorize, catalog, report]
+ if: always() && needs.report.outputs.public_report_ready == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ concurrency:
+ group: runner-protocol-eval-history-publish
+ cancel-in-progress: false
+ permissions:
+ contents: read
+ id-token: write
+ environment:
+ name: runner-e2e-history
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ # AWS credentials can execute only the publisher from the trusted workflow revision.
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+
+ - name: Download only the sanitized canonical Evalbook
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }}
+ path: runner-protocol-public-report
+
+ - name: Exchange GitHub OIDC identity for scoped AWS credentials
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
+ with:
+ role-to-assume: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_ROLE_ARN || vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }}
+ aws-region: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_REGION || vars.RUNNER_E2E_HISTORY_AWS_REGION }}
+
+ - name: Publish versioned report and refresh the root index
+ env:
+ PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report
+ RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET || vars.RUNNER_E2E_HISTORY_S3_BUCKET }}
+ RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX || 'runner-protocol-evals' }}
+ RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL || vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}
+ run: node packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs
diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md
index e0560b1dde..a628694615 100644
--- a/packages/paperclip-runner/README.md
+++ b/packages/paperclip-runner/README.md
@@ -190,7 +190,25 @@ Live console provider-backed routes are loopback-only and reject wildcard/LAN
binds. Browser mutations require same-origin Fetch Metadata, matching Origin,
and JSON content; see the protocol-server tutorial for direct `curl` examples.
-## Live, chaos, and AWS AgentCore operations
+## Direct live protocol qualification
+
+The canonical direct live protocol suite lives in the separate
+`paperclip-evals` repository under `evals/paperclip-runner/`. Its
+`live-mini.json` roster is the complete 35-case Codex qualification lane. Build
+this package's TypeScript output, release `paperclip-runnerd`, package tarball,
+and `dist-issue-thread` viewer, then use the roster runner documented in that
+repository. The package ships the required orchestration entry point as
+`paperclip-runner-eval-session` (`dist/cli/eval-session.js`). Evalbook owns the
+consistent HTML matrix and read-only attempt drill-down pages.
+
+The hosted full-campaign workflow, parallel matrix, credential boundaries,
+canonical report merge, and versioned S3 index are documented in
+[`docs/runner-protocol-live-evals.md`](docs/runner-protocol-live-evals.md).
+
+This direct protocol qualification is separate from the stress-derived Runner
+workflow schedule below and from the full-stack browser model E2E suite.
+
+## Stress-derived workflow, chaos, and AWS AgentCore operations
The deterministic workflow scorer and the chaos schedule do not require
provider credentials:
@@ -209,13 +227,22 @@ additional scheduling after the observed campaign total reaches that value:
```sh
PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD=12 \
+ PAPERCLIP_EVALS_ROOT=/path/to/paperclip-evals \
pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals
+
+# Run two scheduled native Codex executions only.
+PAPERCLIP_EVALS_ROOT=/path/to/paperclip-evals \
+ pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals -- \
+ --candidate codex-luna --limit 2
```
GitHub-hosted live campaigns additionally require the default branch, an
allowlisted numeric actor ID, the protected `runner-e2e-paid` environment, and
-an explicit repository variable before scheduled runs are enabled. Uploaded
-reports contain redacted observations and trace digests, not raw provider
+an explicit repository variable before scheduled runs are enabled. Manual
+dispatches accept the same candidate, case, and execution-limit selectors. The
+paid job uses the reviewed RunsOn Fleet label when `RUNNER_E2E_AWS_ENABLED=true`
+and otherwise stays on `ubuntu-latest`. Uploaded reports contain redacted
+observations and trace digests, not raw provider
frames, prompts, credentials, tool arguments, or hidden reasoning.
The AgentCore proof-of-concept uses an AWS CLI v2 profile to provision a
@@ -234,6 +261,21 @@ pnpm --filter @paperclipai/paperclip-runner smoke:capability:aws-agentcore
pnpm --filter @paperclipai/paperclip-runner aws-agentcore:destroy -- --yes
```
+To admit the hosted direct-eval workflow, provision with the account-local
+GitHub Actions OIDC provider and keep the default exact repository and protected
+environment binding:
+
+```sh
+pnpm --filter @paperclipai/paperclip-runner aws-agentcore:provision -- \
+ --aws-profile paperclip-dev \
+ --github-oidc-provider-arn arn:aws:iam::<account-id>:oidc-provider/token.actions.githubusercontent.com
+```
+
+This adds only `repo:paperclipai/paperclip:environment:runner-e2e-paid` as a
+web-identity subject on the scoped invocation role. The generated nonsecret
+profile records that role as both the local invocation role and the hosted
+execution role.
+
Provisioning can incur Bedrock, AgentCore Runtime/Memory, storage, and private
networking charges. Provisioning refuses to modify a colliding stack unless its
Paperclip ownership tags and template description match. A verified
@@ -256,8 +298,8 @@ recorded lab unless `--force` is also supplied.
| `check:clean-consumers` | Pack the runner and install its root, evals, and testing exports in a clean consumer. |
| `test:eval-slice` | Run the credential-free eval bundle, scoring, and behavior/fault slice. |
| `test:runner-workflow-evals` | Run the deterministic provider-neutral workflow matrix. |
-| `report:runner-workflow-evals` | Validate deterministic results and write local reports only when every scoreable result passes. |
-| `report:runner-live-evals` | Execute the paid forty-execution provider schedule with qualification and campaign-cost guards. |
+| `report:runner-workflow-evals` | Validate deterministic fail-closed results and write JSON, Markdown, JUnit, and GitHub-safe reports. |
+| `report:runner-live-evals` | Execute the paid provider schedule and render its immutable attempts with the canonical `paperclip-evals` HTML grid. |
| `report:runner-chaos-evals` | Write the credential-free eight-scenario chaos schedule. |
| `test:aws-agentcore-provisioning` | Validate the AgentCore template and wrapper safety contracts without provisioning. |
| `aws-agentcore:provision` / `probe` / `lab` / `destroy` | Manage the scoped AgentCore proof-of-concept lifecycle. |
diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md
new file mode 100644
index 0000000000..2cdc74774b
--- /dev/null
+++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md
@@ -0,0 +1,148 @@
+# Direct live Runner protocol evals
+
+This is the provider-backed, one-turn protocol qualification layer in
+`paperclipai/paperclip-evals/evals/paperclip-runner`. It is intentionally
+separate from both the browser full-stack model E2E and the stress-derived
+workflow schedule in `runner-workflow-evals.md`.
+
+The canonical unit of work is one live roster plus one authored case. A full
+campaign selects every `rosters/live-*.json` file at one immutable
+`paperclip-evals` commit. That includes the complete 35-case provider rosters
+and the smaller ACPX Codex control roster. The native resume reliability gate
+is not a normal one-turn roster: it requires its separately governed external
+resource campaign and remains opt-in.
+
+Managed-provider evidence identifies the immutable deployed provider artifact:
+Claude Managed uses its Agent version, while AgentCore uses its qualification
+revision. Transport API or beta versions remain separate protocol metadata and
+must not replace that runtime identity in a report.
+
+## Hosted campaign
+
+Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow
+from the default branch and provide:
+
+- `target_branch`: the Paperclip branch to build and test;
+- `evals_sha`: an exact 40-character commit from
+ `paperclipai/paperclip-evals`;
+- `rosters`: `all` for the entire direct suite, or a comma-separated diagnostic
+ subset;
+- `max_infrastructure_retries`: zero through three, applied only when an
+ attempt explicitly reports a retryable infrastructure failure.
+
+The authorization job resolves the Paperclip branch to a commit and verifies
+the supplied eval commit before any checkout. A short-lived bot token generated
+from `COMMITPERCLIP_KEY` authorizes each checkout of the private eval repository;
+the token is masked and is never forwarded to a provider process. The workflow
+uses the same numeric actor
+allowlist, protected `runner-e2e-paid` environment, RunsOn fleet selector, and
+`RUNNER_E2E_MAX_PARALLEL` ceiling as the full-stack E2E workflow. Two balanced
+GitHub matrices keep each matrix below GitHub's 256-job limit while keeping
+their combined concurrency at or below that shared ceiling. Runner TypeScript,
+the native daemon, provider dependencies, and the attempt viewer are built
+once and reused by every cell. Because the complete suite requires two matrix
+shards, this workflow accepts a shared concurrency ceiling from 2 through 100.
+
+`all` is intentionally literal. A disabled driver, missing remote profile, or
+unavailable provider is retained as an infrastructure result; it is not
+silently omitted. In particular, the ACPX Pi roster remains visible while Pi
+is disabled in the current Runner. Use a roster subset only for diagnosis, not
+to claim the full campaign is green.
+
+A provider turn that reaches a durable failed, interrupted, or otherwise
+non-completed terminal still produces an attempt artifact and is scored as a
+behavior result. It is an infrastructure failure only when the harness cannot
+produce usable evidence, such as an unavailable provider, invalid profile, or
+transport failure. Automatic retries therefore never rerun a measured behavior
+failure merely to improve its score.
+
+## Required protected configuration
+
+The paid jobs read only the credential selected for each roster:
+
+- `OPENAI_API_KEY` for native Codex and ACPX Codex;
+- `ANTHROPIC_API_KEY` for ACPX Claude and Claude Managed;
+- `OPENROUTER_API_KEY` for native OpenCode and ACPX Pi;
+- short-lived GitHub OIDC workload identity for AWS AgentCore.
+
+Claude Managed also requires the four nonsecret
+`PAPERCLIP_CLAUDE_MANAGED_*` profile variables. AgentCore requires the
+nonsecret `PAPERCLIP_AWS_AGENTCORE_*` profile variables, including
+`PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN` and the immutable
+`PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION`; the eval fails closed when
+that deployed revision differs from the pinned roster config. The workflow
+writes the GitHub OIDC token to a mode-`0600` file and never forwards long-lived
+AWS access keys.
+Provision the AgentCore stack with `--github-oidc-provider-arn` so that scoped
+role admits only the `paperclipai/paperclip` repository's protected
+`runner-e2e-paid` environment as its web-identity subject.
+Scheduled runs additionally require `RUNNER_PROTOCOL_EVAL_NIGHTLY_ENABLED=true`
+and the pinned `RUNNER_PROTOCOL_EVALS_SHA` repository variable.
+
+## Reports and history
+
+Each cell uploads its immutable run directory to an access-controlled Actions
+artifact. The trusted report job merges all expected cells, represents missing
+cell artifacts as infrastructure failures, and invokes the report program from
+the pinned eval commit. The full artifact contains the canonical Evalbook grid,
+read-only Runner issue-thread attempt pages, and raw immutable run records.
+
+Public publishing uses a separate projection and a separate trusted OIDC job.
+The projection retains model/config identity, status, usage totals, and check
+outcomes but removes provider session identifiers, transcripts, semantic-tool
+payloads, state revisions, traces, remote profile identities, and raw failure
+text. The same Evalbook `report` command renders that projection, so the public
+grid and test pages have the standard Evalbook layout. The publisher rejects
+scripts, remote resources, symlinks, unknown paths, broken links, raw session
+fields, and credential-shaped values.
+
+S3 publication is additive:
+
+```text
+runner-protocol-evals/
+ index.html
+ history.json
+ latest.json
+ latest-green.json
+ campaigns/
+ gha-<run-id>-<run-attempt>/
+ index.html
+ latest.html
+ inventory.html
+ tests/*.html
+ attempts/*.html
+ campaign.json
+ bundle-manifest.json
+```
+
+Campaign files use immutable cache headers and a digest manifest. Reusing a
+campaign ID with different bytes fails closed. Only the root history and
+pointer files are mutable, and the publisher never deletes objects. The root
+history retains at most 200 records, reserving one record for the latest green
+campaign when it would otherwise fall outside that window so its pointer stays
+valid.
+
+The publishing job uses dedicated `RUNNER_PROTOCOL_EVAL_HISTORY_*` variables
+when present and falls back to the existing Runner E2E history role, region,
+bucket, and public base URL. Its default top-level prefix is
+`runner-protocol-evals`, distinct from `runner-e2e`. The AWS role must allow
+additive writes and reads for that prefix.
+
+## Local publisher checks
+
+These tests make no provider or AWS calls:
+
+```sh
+pnpm --filter @paperclipai/paperclip-runner test:runner-protocol-eval-publish
+```
+
+To inspect the catalog without executing it, point the command at a local
+evals checkout:
+
+```sh
+pnpm --filter @paperclipai/paperclip-runner \
+ report:runner-protocol-eval:catalog -- \
+ --evals-root /path/to/paperclip-evals \
+ --campaign-id gha-1-1 \
+ --output /tmp/runner-protocol-eval-catalog.json
+```
diff --git a/packages/paperclip-runner/docs/runner-workflow-evals.md b/packages/paperclip-runner/docs/runner-workflow-evals.md
index 19b5357329..ceca8f0387 100644
--- a/packages/paperclip-runner/docs/runner-workflow-evals.md
+++ b/packages/paperclip-runner/docs/runner-workflow-evals.md
@@ -6,7 +6,9 @@ capability inventory, capability cases, and existing scoring/report readers.
The workspace-private `@paperclipai/paperclip-eval-kernel` package owns only
structural scenario-by-candidate orchestration. Runner-specific cases,
-observations, scoring, traceability, and report rendering remain package-local.
+observations, scoring, and traceability remain package-local. The HTML matrix is
+rendered by the canonical `paperclip-evals` report program so live results use
+the same grid and drill-down pages as the direct Runner eval suite.
## Lanes
@@ -14,9 +16,9 @@ observations, scoring, traceability, and report rendering remain package-local.
runs the credential-free PR gate over sanitized Codex, OpenCode, and ACPX
normalization fixtures.
- `pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals`
- validates the deterministic report and writes JSON, Markdown, JUnit, and
- GitHub-safe artifacts under `.paperclip-local/evals/workflows/` only when all
- scoreable fixture results pass. It makes no network requests.
+ validates the deterministic fail-closed fixture matrix and writes JSON,
+ Markdown, JUnit, and GitHub-safe artifacts under
+ `.paperclip-local/evals/workflows/`. It makes no network requests.
- `pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals` runs
the balanced forty-execution schedule against real provider sessions. Live
candidate failures are trend-only; missing credentials, qualification
@@ -30,10 +32,38 @@ qualification variable names, and budgets. Credentials remain in the
environment. `PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD` must be a positive finite
number and defaults to 12 USD for scheduled runs.
+Live executions export one immutable Evalbook attempt per workflow/candidate to
+`.paperclip-local/evals/workflows/evalbook-runs/`, then invoke
+`evals/paperclip-runner/tools/eval_program.py report` from a `paperclip-evals`
+checkout. That program writes the canonical matrix to
+`.paperclip-local/evals/workflows/index.html`, plus `latest.html`, test pages,
+and attempt pages. Set `PAPERCLIP_EVALBOOK_PROGRAM` to the program's absolute
+path or `PAPERCLIP_EVALS_ROOT` to its repository root. Conventional sibling
+worktree locations are discovered automatically. GitHub Actions checks out a
+pinned `paperclip-evals` revision, so every hosted run uses the same reviewed
+report implementation rather than a copied or package-local renderer.
+Filtered reports with only a few candidates expand their result columns to the
+available viewport, keeping the PASS, FAIL, and INFRA labels visible without a
+horizontal scroll. Full matrices retain the canonical scrollable grid and
+sticky test-name column.
+
+The exported artifact contains only the safe workflow observation and
+scorecard. Prompts, credentials, raw provider frames, tool arguments, and
+reasoning remain excluded. `evalbook-manifest.json` records the generator path
+and SHA-256 digest used for the render.
+
+Local and manual GitHub runs can bound paid execution with comma-separated
+`--candidate` and `--case` selectors plus `--limit`. For example,
+`report:runner-live-evals -- --candidate codex-luna --limit 2` executes only
+the first two Codex entries in that week's validated schedule. Subsets receive
+a distinct bundle identity and do not contaminate full-campaign trend history.
+
The hosted live workflow is default-branch-only and requires an allowlisted
numeric actor plus the protected `runner-e2e-paid` environment. Scheduled runs
also remain disabled until `RUNNER_LIVE_EVALS_NIGHTLY_ENABLED` is explicitly
-set to `true`.
+set to `true`. The paid job uses the full-stack workflow's literal runner
+selection: `RUNNER_E2E_AWS_ENABLED=true` routes it to the RunsOn Fleet, and any
+other value uses `ubuntu-latest`.
## Trace and reasoning safety
diff --git a/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml
index a7cbd72a1c..afa68ed17f 100644
--- a/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml
+++ b/packages/paperclip-runner/infra/aws-agentcore-paperclip.yaml
@@ -14,6 +14,19 @@ Parameters:
Type: String
Description: Stable IAM user or role ARN allowed to assume the Paperclip invocation role.
AllowedPattern: "^arn:aws(-[^:]+)?:iam::[0-9]{12}:(role|user)/.+$"
+ GitHubOidcProviderArn:
+ Type: String
+ Default: ""
+ Description: Optional account-local GitHub Actions OIDC provider for hosted Runner evals.
+ AllowedPattern: "^$|^arn:aws(-[^:]+)?:iam::[0-9]{12}:oidc-provider/token[.]actions[.]githubusercontent[.]com$"
+ GitHubRepository:
+ Type: String
+ Default: paperclipai/paperclip
+ AllowedPattern: "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"
+ GitHubEnvironment:
+ Type: String
+ Default: runner-e2e-paid
+ AllowedPattern: "^[A-Za-z0-9_.-]+$"
BedrockModelId:
Type: String
Default: global.anthropic.claude-sonnet-4-6
@@ -53,6 +66,7 @@ Parameters:
Conditions:
IsPrivate: !Equals [!Ref DeploymentMode, private]
+ HasGitHubOidcTrust: !Not [!Equals [!Ref GitHubOidcProviderArn, ""]]
Resources:
ContextEncryptionKey:
@@ -304,6 +318,17 @@ Resources:
Principal:
AWS: !Ref TrustedRunnerPrincipalArn
Action: sts:AssumeRole
+ - !If
+ - HasGitHubOidcTrust
+ - Effect: Allow
+ Principal:
+ Federated: !Ref GitHubOidcProviderArn
+ Action: sts:AssumeRoleWithWebIdentity
+ Condition:
+ StringEquals:
+ "token.actions.githubusercontent.com:aud": sts.amazonaws.com
+ "token.actions.githubusercontent.com:sub": !Sub repo:${GitHubRepository}:environment:${GitHubEnvironment}
+ - !Ref AWS::NoValue
Policies:
- PolicyName: InvokeAndOperatePinnedHarness
PolicyDocument:
diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json
index d69f6d9739..c6d02304fc 100644
--- a/packages/paperclip-runner/package.json
+++ b/packages/paperclip-runner/package.json
@@ -114,12 +114,15 @@
"test:capability-inventory": "node scripts/check-capability-inventory.test.mjs",
"test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts",
"test:eval-slice": "pnpm run ensure:eval-build-deps && vitest run src/eval",
- "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts",
+ "test:runner-workflow-evals": "pnpm run ensure:eval-build-deps && node --test scripts/render-runner-workflow-evalbook.test.mjs && vitest run src/eval/workflow-evals.test.ts src/eval/live-workflow-executor.test.ts",
+ "test:runner-protocol-eval-publish": "node --test scripts/runner-protocol-eval-campaign.test.mjs scripts/publish-runner-protocol-eval-history.test.mjs scripts/runner-protocol-eval-workflow-security.test.mjs",
"check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs",
"report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs",
"report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs",
"report:runner-workflow-evals": "pnpm run build:typescript && node scripts/run-runner-workflow-evals.mjs",
"report:runner-live-evals": "pnpm run build:typescript && pnpm run build:runner-binaries && node scripts/run-runner-live-eval-schedule.mjs --mode nightly --execute",
+ "report:runner-protocol-eval:catalog": "node scripts/runner-protocol-eval-campaign.mjs catalog",
+ "report:runner-protocol-eval:publish": "node scripts/publish-runner-protocol-eval-history.mjs",
"report:runner-chaos-evals": "pnpm run build:typescript && node scripts/run-runner-live-eval-schedule.mjs --mode chaos",
"check:conformance-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output",
"check:replay-parity": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core replay_fixture_parity",
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
index 6c77d15241..9bb1784012 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
@@ -31,6 +31,48 @@ fn send(value: Value) -> io::Result<()> {
stdout.flush()
}
+fn send_split_event_burst(state: &FakeState) -> io::Result<()> {
+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");
+ for index in 0..96 {
+ send(json!({
+ "method": "item/agentMessage/delta",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": turn_id,
+ "itemId": "split-burst-message",
+ "delta": format!("first-{index} "),
+ }
+ }))?;
+ }
+ send(json!({
+ "id": "split-burst-tool",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": turn_id,
+ "callId": "split-burst-semantic-call",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))
+}
+
+fn finish_split_event_burst(state: &FakeState) -> io::Result<()> {
+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");
+ for index in 0..48 {
+ send(json!({
+ "method": "item/agentMessage/delta",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": turn_id,
+ "itemId": "split-burst-message",
+ "delta": format!("second-{index} "),
+ }
+ }))?;
+ }
+ Ok(())
+}
+
fn load_state(path: &Path) -> FakeState {
fs::read(path)
.ok()
@@ -483,6 +525,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
.any(|value| value == "--opencode-proxy-runtime-question");
let emit_runtime_elicitation = args.iter().any(|value| value == "--runtime-elicitation");
let emit_structured_activity = args.iter().any(|value| value == "--structured-activity");
+ let emit_split_event_burst = args.iter().any(|value| value == "--split-event-burst");
let require_skill_instructions = args
.iter()
.any(|value| value == "--include-skill-instructions");
@@ -703,6 +746,15 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
finish_turn(&state_path, &mut state, "completed")?;
continue;
}
+ if message.get("method").is_none() && message.get("id") == Some(&json!("split-burst-tool"))
+ {
+ if message.pointer("/result/success") != Some(&json!(true)) {
+ return Err("split event burst semantic tool failed".into());
+ }
+ finish_split_event_burst(&state)?;
+ finish_turn(&state_path, &mut state, "completed")?;
+ continue;
+ }
if message.get("method").is_none() && message.get("id") == Some(&json!("tool-request-1")) {
if message.pointer("/result/success") == Some(&json!(false)) {
log_call(call_log.as_deref(), "tool-response:failure")?;
@@ -1110,6 +1162,8 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
} else if emit_structured_activity {
send_structured_activity(&state)?;
finish_turn(&state_path, &mut state, "completed")?;
+ } else if emit_split_event_burst {
+ send_split_event_burst(&state)?;
} else if emit_question {
send_question(&state)?;
} else if !hold_turn {
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs
index dcdfb5ed0e..0370c7bb90 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/managed_provider_backend.rs
@@ -140,7 +140,7 @@ impl ManagedProviderDescriptor {
fn version(&self) -> &str {
match self {
- Self::ClaudeManaged(config) => &config.beta_version,
+ Self::ClaudeManaged(config) => &config.agent_version,
Self::AwsAgentcore(config) => &config.qualification_revision,
}
}
@@ -2477,6 +2477,35 @@ mod tests {
);
}
+ #[test]
+ fn claude_runtime_identity_uses_the_pinned_agent_version() {
+ let descriptor = ManagedProviderDescriptor::ClaudeManaged(ClaudeManagedProviderConfig {
+ model: QUALIFIED_CLAUDE_MODEL.to_owned(),
+ profile_id: "profile-1".to_owned(),
+ anthropic_agent_id: "agent-1".to_owned(),
+ agent_version: "17".to_owned(),
+ environment_id: "environment-1".to_owned(),
+ beta_version: QUALIFIED_CLAUDE_BETA.to_owned(),
+ max_session_list_cost_usd: 1.0,
+ instructions: "Complete the supplied task.".to_owned(),
+ runtime_context: None,
+ });
+
+ assert_eq!(descriptor.version(), "17");
+ assert_eq!(
+ session_event_payload(
+ &descriptor,
+ &ProviderRuntimeIdentity::RemoteService {
+ service: "anthropic_managed_agents".to_owned(),
+ provider_session_id: "session-17".to_owned(),
+ process_id: None,
+ },
+ )
+ .pointer("/providerDescriptor/providerVersion"),
+ Some(&json!("17"))
+ );
+ }
+
#[test]
fn agentcore_usage_snapshot_accepts_only_bounded_reconciliation_states() {
let usage = json!({
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs
index 6ac380da5d..21eca672f7 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs
@@ -261,7 +261,7 @@ impl VerifiedProcessLaunch {
}
#[cfg(target_os = "macos")]
{
- let executable = materialize_executable(artifact)?;
+ let executable = materialize_bound_executable(artifact)?;
args.push(executable.path.to_string_lossy().into_owned());
temporary_executables.push(executable);
}
@@ -290,6 +290,7 @@ struct InheritedCommand {
#[cfg(target_os = "macos")]
struct TemporaryExecutable {
path: PathBuf,
+ cleanup_directory: Option<PathBuf>,
_file: File,
}
@@ -297,13 +298,16 @@ struct TemporaryExecutable {
impl Drop for TemporaryExecutable {
fn drop(&mut self) {
let _ = fs::remove_file(&self.path);
+ if let Some(directory) = &self.cleanup_directory {
+ let _ = fs::remove_dir(directory);
+ }
}
}
#[cfg(target_os = "macos")]
-fn materialize_executable(
+fn verified_executable_parent(
artifact: &VerifiedProcessArtifact,
-) -> Result<TemporaryExecutable, LocalRunnerError> {
+) -> Result<&Path, LocalRunnerError> {
let directory = artifact.display_path.parent().ok_or_else(|| {
LocalRunnerError::invalid(format!(
"verified process artifact {} has no parent directory",
@@ -318,10 +322,15 @@ fn materialize_executable(
directory.display()
)));
}
- let path = directory.join(format!(
- ".paperclip-verified-executable-{}",
- Uuid::new_v4().simple()
- ));
+ Ok(directory)
+}
+
+#[cfg(target_os = "macos")]
+fn materialize_executable_at(
+ artifact: &VerifiedProcessArtifact,
+ path: PathBuf,
+ cleanup_directory: Option<PathBuf>,
+) -> Result<TemporaryExecutable, LocalRunnerError> {
let mut writable = OpenOptions::new()
.read(true)
.write(true)
@@ -375,6 +384,7 @@ fn materialize_executable(
drop(writable);
Ok(TemporaryExecutable {
path: path.clone(),
+ cleanup_directory,
_file: file,
})
})();
@@ -384,6 +394,40 @@ fn materialize_executable(
result
}
+#[cfg(target_os = "macos")]
+fn materialize_executable(
+ artifact: &VerifiedProcessArtifact,
+) -> Result<TemporaryExecutable, LocalRunnerError> {
+ let directory = verified_executable_parent(artifact)?;
+ let path = directory.join(format!(
+ ".paperclip-verified-executable-{}",
+ Uuid::new_v4().simple()
+ ));
+ materialize_executable_at(artifact, path, None)
+}
+
+#[cfg(target_os = "macos")]
+fn materialize_bound_executable(
+ artifact: &VerifiedProcessArtifact,
+) -> Result<TemporaryExecutable, LocalRunnerError> {
+ let parent = verified_executable_parent(artifact)?;
+ let directory = parent.join(format!(
+ ".paperclip-verified-executable-{}",
+ Uuid::new_v4().simple()
+ ));
+ fs::create_dir(&directory).map_err(|error| snapshot_error(&artifact.display_path, error))?;
+ if let Err(error) = fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)) {
+ let _ = fs::remove_dir(&directory);
+ return Err(snapshot_error(&artifact.display_path, error));
+ }
+ let result =
+ materialize_executable_at(artifact, directory.join("launch"), Some(directory.clone()));
+ if result.is_err() {
+ let _ = fs::remove_dir(directory);
+ }
+ result
+}
+
#[cfg(any(target_os = "linux", target_os = "macos"))]
fn inherited_artifact(
artifact: &VerifiedProcessArtifact,
@@ -941,4 +985,50 @@ mod tests {
assert!(inherited.args[2].starts_with("/dev/fd/"));
fs::remove_dir_all(directory).unwrap();
}
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn executable_arguments_use_the_private_nested_launch_contract() {
+ let nonce = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_nanos();
+ let directory = std::env::temp_dir().join(format!(
+ "paperclip-executable-argument-launch-{}-{nonce}",
+ std::process::id()
+ ));
+ fs::create_dir_all(&directory).unwrap();
+ let program = verified_artifact(&directory.join("node"), b"node");
+ let executable = verified_artifact(&directory.join("opencode"), b"opencode");
+ let launch = VerifiedProcessLaunch::new(
+ program,
+ vec![VerifiedProcessArgument::ExecutableArtifact(executable)],
+ );
+
+ let inherited = launch.inherited_command().unwrap();
+ let command = PathBuf::from(&inherited.args[0]);
+ let private_directory = command.parent().unwrap().to_path_buf();
+ assert_eq!(command.file_name().unwrap(), "launch");
+ assert!(private_directory
+ .file_name()
+ .unwrap()
+ .to_string_lossy()
+ .starts_with(".paperclip-verified-executable-"));
+ assert_eq!(
+ fs::symlink_metadata(&private_directory)
+ .unwrap()
+ .permissions()
+ .mode()
+ & 0o777,
+ 0o700
+ );
+ assert_eq!(
+ fs::symlink_metadata(&command).unwrap().permissions().mode() & 0o777,
+ 0o500
+ );
+
+ drop(inherited);
+ assert!(!private_directory.exists());
+ fs::remove_dir_all(directory).unwrap();
+ }
}
diff --git a/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs
index eae5284f90..2bfdbe6a0b 100644
--- a/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs
+++ b/packages/paperclip-runner/scripts/aws-agentcore-provisioning.test.mjs
@@ -82,6 +82,10 @@ test("AgentCore template has closed development/private resources and explicit c
assert.match(source, /Sid: DecryptPinnedRuntimeContext[\s\S]*Action: kms:Decrypt[\s\S]*Resource: !GetAtt ContextEncryptionKey\.Arn/);
const invocationRole = source.slice(source.indexOf("RunnerInvocationRole:"), source.indexOf("PrivateVpc:"));
assert.doesNotMatch(invocationRole, /Action:\s+(?:-\s+)?(?:s3|kms):\*/);
+ assert.match(invocationRole, /Action: sts:AssumeRoleWithWebIdentity/);
+ assert.match(invocationRole, /token\.actions\.githubusercontent\.com:aud["']?: sts\.amazonaws\.com/);
+ assert.match(invocationRole, /token\.actions\.githubusercontent\.com:sub["']?: !Sub repo:\$\{GitHubRepository\}:environment:\$\{GitHubEnvironment\}/);
+ assert.doesNotMatch(invocationRole, /repo:\*|environment:\*/);
assert.match(source, /\$\{AgentHarness\.Arn\}\/harness-endpoint\/\$\{HarnessEndpointName\}/);
assert.match(source, /\$\{AgentHarness\.Arn\}\/runtime-endpoint\/\$\{HarnessEndpointName\}/);
assert.match(source, /BedrockMarketplaceProductId/);
@@ -136,10 +140,13 @@ test("AgentCore wrapper is valid shell and writes only nonsecret profile metadat
assert.match(source, /--query endpoint\.status/);
assert.match(source, /o\.endpoint\?\.arn/);
assert.match(source, /--marketplace-product-id/);
+ assert.match(source, /--github-oidc-provider-arn/);
+ assert.match(source, /GitHubOidcProviderArn=\$GITHUB_OIDC_PROVIDER_ARN/);
assert.match(source, /BedrockMarketplaceProductId=\$MARKETPLACE_PRODUCT_ID/);
assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=\$context_bucket/);
assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=\$context_prefix/);
assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=\$context_kms_key_arn/);
+ assert.match(generatedBlock, /PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN=\$role_arn/);
assert.match(source, /ContextPrefix=\$CONTEXT_PREFIX/);
assert.match(source, /s3 rm "s3:\/\/\$context_bucket\/\$context_prefix\/assets\/" --recursive/);
assert.ok(source.indexOf("delete-harness-endpoint") < source.lastIndexOf("cloudformation delete-stack"));
diff --git a/packages/paperclip-runner/scripts/aws-agentcore.sh b/packages/paperclip-runner/scripts/aws-agentcore.sh
index 26f4e493d8..c782a22e18 100644
--- a/packages/paperclip-runner/scripts/aws-agentcore.sh
+++ b/packages/paperclip-runner/scripts/aws-agentcore.sh
@@ -22,6 +22,9 @@ MODEL_ID="global.anthropic.claude-sonnet-4-6"
MARKETPLACE_PRODUCT_ID="prod-ffvjxvh4ltq64"
MARKETPLACE_PRODUCT_ID_EXPLICIT=false
TRUSTED_PRINCIPAL=""
+GITHUB_OIDC_PROVIDER_ARN="${PAPERCLIP_AWS_AGENTCORE_GITHUB_OIDC_PROVIDER_ARN:-}"
+GITHUB_REPOSITORY="paperclipai/paperclip"
+GITHUB_ENVIRONMENT="runner-e2e-paid"
CONTEXT_PREFIX=""
DRY_RUN=false
FORCE=false
@@ -38,6 +41,9 @@ usage() {
" --model MODEL_ID Bedrock-native model ID" \
" --marketplace-product-id ID exact AWS Marketplace product ID for the model" \
" --principal ARN stable IAM role/user trusted to assume runner role" \
+ " --github-oidc-provider-arn ARN optional account-local GitHub Actions OIDC provider" \
+ " --github-repository OWNER/REPO exact repository admitted by OIDC trust (default: $GITHUB_REPOSITORY)" \
+ " --github-environment NAME exact protected environment admitted by OIDC trust (default: $GITHUB_ENVIRONMENT)" \
" --context-prefix PFX S3 prefix dedicated to this qualified profile" \
" --dry-run validate and print changes without deployment" \
" --replace-failed-stack explicitly replace an owned ROLLBACK_COMPLETE stack" \
@@ -55,6 +61,9 @@ while [[ $# -gt 0 ]]; do
--model) MODEL_ID="$2"; shift 2 ;;
--marketplace-product-id) MARKETPLACE_PRODUCT_ID="$2"; MARKETPLACE_PRODUCT_ID_EXPLICIT=true; shift 2 ;;
--principal) TRUSTED_PRINCIPAL="$2"; shift 2 ;;
+ --github-oidc-provider-arn) GITHUB_OIDC_PROVIDER_ARN="$2"; shift 2 ;;
+ --github-repository) GITHUB_REPOSITORY="$2"; shift 2 ;;
+ --github-environment) GITHUB_ENVIRONMENT="$2"; shift 2 ;;
--context-prefix) CONTEXT_PREFIX="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
--replace-failed-stack) REPLACE_FAILED_STACK=true; shift ;;
@@ -93,6 +102,14 @@ if [[ ! "$MARKETPLACE_PRODUCT_ID" =~ ^prod-[a-z0-9]+$ ]]; then
printf 'Marketplace product ID must use the AWS prod-* form.\n' >&2
exit 2
fi
+if [[ -n "$GITHUB_OIDC_PROVIDER_ARN" && ! "$GITHUB_OIDC_PROVIDER_ARN" =~ ^arn:aws(-[^:]+)?:iam::[0-9]{12}:oidc-provider/token\.actions\.githubusercontent\.com$ ]]; then
+ printf 'GitHub OIDC provider must be the account-local token.actions.githubusercontent.com provider ARN.\n' >&2
+ exit 2
+fi
+if [[ ! "$GITHUB_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ || ! "$GITHUB_ENVIRONMENT" =~ ^[A-Za-z0-9_.-]+$ ]]; then
+ printf 'GitHub OIDC trust requires exact OWNER/REPO and environment names.\n' >&2
+ exit 2
+fi
if [[ -z "$CONTEXT_PREFIX" ]]; then CONTEXT_PREFIX="paperclip/agentcore/$STACK_NAME"; fi
if [[ ! "$CONTEXT_PREFIX" =~ ^[a-z0-9][a-z0-9/_-]{1,127}$ || "$CONTEXT_PREFIX" == /* || "$CONTEXT_PREFIX" == */ || "$CONTEXT_PREFIX" == *//* || "/$CONTEXT_PREFIX/" == */./* || "/$CONTEXT_PREFIX/" == */../* ]]; then
printf 'Context prefix must be a safe, relative S3 key prefix.\n' >&2
@@ -279,6 +296,14 @@ provision() {
caller_arn="$(printf '%s' "$identity" | json_field Arn)"
account_id="$(printf '%s' "$identity" | json_field Account)"
partition="$(printf '%s' "$caller_arn" | cut -d: -f2)"
+ if [[ -n "$GITHUB_OIDC_PROVIDER_ARN" ]]; then
+ local expected_github_oidc_provider_arn="arn:$partition:iam::$account_id:oidc-provider/token.actions.githubusercontent.com"
+ if [[ "$GITHUB_OIDC_PROVIDER_ARN" != "$expected_github_oidc_provider_arn" ]]; then
+ printf 'GitHub OIDC provider must belong to the current AWS account and partition.\n' >&2
+ exit 1
+ fi
+ aws_cli iam get-open-id-connect-provider --open-id-connect-provider-arn "$GITHUB_OIDC_PROVIDER_ARN" >/dev/null
+ fi
model_resource_arn="arn:$partition:bedrock:$AWS_REGION_NAME:$account_id:inference-profile/$MODEL_ID"
foundation_model_id="$(printf '%s' "$MODEL_ID" | sed -E 's/^(global|us|eu|apac)\.//')"
foundation_model_resource_arn="arn:$partition:bedrock:*::foundation-model/$foundation_model_id"
@@ -342,7 +367,11 @@ provision() {
"EnvironmentName=development" "DeploymentMode=$DEPLOYMENT_MODE" \
"HarnessEndpointName=$ENDPOINT_NAME" \
"ContextPrefix=$CONTEXT_PREFIX" \
- "TrustedRunnerPrincipalArn=$TRUSTED_PRINCIPAL" "BedrockModelId=$MODEL_ID" \
+ "TrustedRunnerPrincipalArn=$TRUSTED_PRINCIPAL" \
+ "GitHubOidcProviderArn=$GITHUB_OIDC_PROVIDER_ARN" \
+ "GitHubRepository=$GITHUB_REPOSITORY" \
+ "GitHubEnvironment=$GITHUB_ENVIRONMENT" \
+ "BedrockModelId=$MODEL_ID" \
"BedrockModelResourceArn=$model_resource_arn" \
"BedrockFoundationModelResourceArn=$foundation_model_resource_arn" \
"BedrockMarketplaceProductId=$MARKETPLACE_PRODUCT_ID" \
@@ -400,6 +429,7 @@ provision() {
"PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN=$memory_arn" \
"PAPERCLIP_AWS_AGENTCORE_MEMORY_ID=$memory_id" \
"PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN=$role_arn" \
+ "PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN=$role_arn" \
"PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET=$context_bucket" \
"PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX=$context_prefix" \
"PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN=$context_kms_key_arn" \
diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs
new file mode 100644
index 0000000000..b014d76686
--- /dev/null
+++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs
@@ -0,0 +1,547 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { execFile } from "node:child_process";
+import {
+ lstat,
+ mkdtemp,
+ mkdir,
+ readFile,
+ readdir,
+ stat,
+ writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { extname, join, relative, resolve, sep } from "node:path";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+const SAFE_CAMPAIGN = /^gha-[1-9][0-9]*-[1-9][0-9]*$/;
+const SAFE_REPORT_PATHS = [
+ /^(?:index|latest|inventory|real-server)\.html$/,
+ /^tests\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/,
+ /^attempts\/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.html$/,
+ /^campaign\.json$/,
+];
+const CREDENTIAL_PATTERNS = [
+ /\bAKIA[0-9A-Z]{16}\b/u,
+ /\bsk-[A-Za-z0-9_-]{20,}\b/u,
+ /\bBearer\s+[A-Za-z0-9._~-]{16,}\b/iu,
+ /["'](?:providerSessionId|sessionId)["']\s*:/u,
+];
+const ACTIVE_HTML_PATTERNS = [
+ /<script\b/iu,
+ /<iframe\b/iu,
+ /<object\b/iu,
+ /<embed\b/iu,
+ /<form\b/iu,
+ /\son[a-z]+\s*=/iu,
+ /javascript\s*:/iu,
+ /(?:src|href)\s*=\s*["'](?:https?:)?\/\//iu,
+];
+const MAX_HISTORY_CAMPAIGNS = 200;
+
+function json(value) {
+ return `${JSON.stringify(value, null, 2)}\n`;
+}
+
+function html(value) {
+ return String(value ?? "")
+ .replaceAll("&", "&amp;")
+ .replaceAll("<", "&lt;")
+ .replaceAll(">", "&gt;")
+ .replaceAll('"', "&quot;")
+ .replaceAll("'", "&#39;");
+}
+
+async function loadObject(path) {
+ const value = JSON.parse(await readFile(path, "utf8"));
+ if (value === null || Array.isArray(value) || typeof value !== "object") {
+ throw new Error(`Expected a JSON object: ${path}`);
+ }
+ return value;
+}
+
+export function validateProtocolEvalHistoryDestination({
+ bucket,
+ prefix,
+ publicBaseUrl,
+}) {
+ if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) {
+ throw new Error(
+ "RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET is not a valid bucket name",
+ );
+ }
+ const normalizedPrefix = String(prefix ?? "").replace(/^\/+|\/+$/g, "");
+ if (
+ !normalizedPrefix ||
+ normalizedPrefix
+ .split("/")
+ .some((segment) => !segment || segment === "." || segment === "..")
+ ) {
+ throw new Error(
+ "RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX must be a safe non-empty key prefix",
+ );
+ }
+ const url = new URL(publicBaseUrl);
+ if (
+ url.protocol !== "https:" ||
+ url.username ||
+ url.password ||
+ url.search ||
+ url.hash
+ ) {
+ throw new Error(
+ "RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL must be a credential-free HTTPS URL",
+ );
+ }
+ return {
+ bucket,
+ prefix: normalizedPrefix,
+ publicBaseUrl: url.href.replace(/\/$/, ""),
+ };
+}
+
+export function isPublicProtocolEvalPath(relativePath) {
+ if (
+ relativePath.includes("\\") ||
+ relativePath.startsWith("/") ||
+ relativePath
+ .split("/")
+ .some((segment) => !segment || segment === "." || segment === "..")
+ ) {
+ return false;
+ }
+ return SAFE_REPORT_PATHS.some((pattern) => pattern.test(relativePath));
+}
+
+async function relativeFiles(root, current = root) {
+ const entries = await readdir(current, { withFileTypes: true });
+ const files = [];
+ for (const entry of entries) {
+ const absolute = join(current, entry.name);
+ if (entry.isSymbolicLink())
+ throw new Error(`Refusing public report symlink ${absolute}`);
+ if (entry.isDirectory())
+ files.push(...(await relativeFiles(root, absolute)));
+ else if (entry.isFile())
+ files.push(relative(root, absolute).split(sep).join("/"));
+ else throw new Error(`Refusing unusual public report path ${absolute}`);
+ }
+ return files.sort();
+}
+
+function internalHtmlHrefs(content) {
+ return [...content.matchAll(/href\s*=\s*["']([^"']+)["']/giu)]
+ .map((match) => match[1])
+ .filter((href) => href && !href.startsWith("#"));
+}
+
+export async function validatePublicProtocolEvalReport(reportRoot) {
+ const root = resolve(reportRoot);
+ const files = await relativeFiles(root);
+ if (!files.includes("index.html") || !files.includes("campaign.json")) {
+ throw new Error(
+ "Public protocol eval report requires index.html and campaign.json",
+ );
+ }
+ for (const file of files) {
+ if (!isPublicProtocolEvalPath(file)) {
+ throw new Error(
+ `Refusing non-allowlisted public protocol eval path ${file}`,
+ );
+ }
+ const absolute = resolve(root, ...file.split("/"));
+ const metadata = await stat(absolute);
+ if (metadata.size === 0 || metadata.size > 12 * 1024 * 1024) {
+ throw new Error(
+ `Public protocol eval file exceeds its size boundary: ${file}`,
+ );
+ }
+ const content = await readFile(absolute, "utf8");
+ for (const pattern of CREDENTIAL_PATTERNS) {
+ if (pattern.test(content))
+ throw new Error(
+ `Public report contains credential/session material: ${file}`,
+ );
+ }
+ if (extname(file) !== ".html") continue;
+ for (const pattern of ACTIVE_HTML_PATTERNS) {
+ if (pattern.test(content))
+ throw new Error(
+ `Public report contains active or remote HTML: ${file}`,
+ );
+ }
+ for (const href of internalHtmlHrefs(content)) {
+ const clean = href.split("#", 1)[0].split("?", 1)[0];
+ const target = resolve(
+ root,
+ ...file.split("/").slice(0, -1),
+ ...clean.split("/"),
+ );
+ const rel = relative(root, target).split(sep).join("/");
+ if (
+ !isPublicProtocolEvalPath(rel) ||
+ !(await lstat(target).catch(() => null))?.isFile()
+ ) {
+ throw new Error(
+ `Public report contains a broken or unsafe link in ${file}: ${href}`,
+ );
+ }
+ }
+ }
+ const campaign = await loadObject(join(root, "campaign.json"));
+ if (
+ campaign.schema !== "paperclip.runner-protocol-eval.campaign/v1" ||
+ !SAFE_CAMPAIGN.test(String(campaign.campaignId ?? ""))
+ ) {
+ throw new Error("Public report campaign metadata is invalid");
+ }
+ return { files, campaign };
+}
+
+export async function createProtocolEvalBundleManifest(reportRoot, campaignId) {
+ if (!SAFE_CAMPAIGN.test(campaignId))
+ throw new Error("Unsafe protocol eval campaign ID");
+ const { files, campaign } =
+ await validatePublicProtocolEvalReport(reportRoot);
+ if (campaign.campaignId !== campaignId)
+ throw new Error("Report campaign ID does not match publication target");
+ const entries = await Promise.all(
+ files.map(async (file) => {
+ const absolute = resolve(reportRoot, ...file.split("/"));
+ const [content, metadata] = await Promise.all([
+ readFile(absolute),
+ stat(absolute),
+ ]);
+ return {
+ path: file,
+ sha256: createHash("sha256").update(content).digest("hex"),
+ bytes: metadata.size,
+ };
+ }),
+ );
+ return {
+ schema: "paperclip.runner-protocol-eval.bundle/v1",
+ campaignId,
+ bundleDigest: createHash("sha256")
+ .update(JSON.stringify(entries))
+ .digest("hex"),
+ files: entries,
+ };
+}
+
+export function emptyProtocolEvalHistory() {
+ return {
+ schema: "paperclip.runner-protocol-eval.history/v1",
+ updatedAt: new Date(0).toISOString(),
+ latestCampaignId: null,
+ latestGreenCampaignId: null,
+ campaigns: [],
+ };
+}
+
+export function protocolEvalHistoryRecord(campaign, publicRoot) {
+ return {
+ campaignId: campaign.campaignId,
+ generatedAt: campaign.generatedAt,
+ publicUrl: `${publicRoot}/campaigns/${encodeURIComponent(campaign.campaignId)}/`,
+ complete: campaign.complete === true,
+ allPassed: campaign.allPassed === true,
+ totals: campaign.totals,
+ rosters: campaign.rosters,
+ source: campaign.source,
+ };
+}
+
+export function mergeProtocolEvalHistory(history, record) {
+ if (history.schema !== "paperclip.runner-protocol-eval.history/v1") {
+ throw new Error("Unsupported protocol eval history schema");
+ }
+ const existing = history.campaigns.find(
+ (item) => item.campaignId === record.campaignId,
+ );
+ if (existing && JSON.stringify(existing) !== JSON.stringify(record)) {
+ throw new Error(
+ `Immutable campaign history changed for ${record.campaignId}`,
+ );
+ }
+ const campaigns = existing
+ ? history.campaigns
+ : [...history.campaigns, record];
+ campaigns.sort((left, right) =>
+ right.generatedAt.localeCompare(left.generatedAt),
+ );
+ const latest = campaigns[0] ?? null;
+ const latestGreen =
+ campaigns.find((campaign) => campaign.complete && campaign.allPassed) ??
+ null;
+ const retained = campaigns.slice(0, MAX_HISTORY_CAMPAIGNS);
+ if (
+ latestGreen &&
+ !retained.some(
+ (campaign) => campaign.campaignId === latestGreen.campaignId,
+ )
+ ) {
+ retained[retained.length - 1] = latestGreen;
+ }
+ return {
+ schema: history.schema,
+ updatedAt: new Date().toISOString(),
+ latestCampaignId: latest?.campaignId ?? null,
+ latestGreenCampaignId: latestGreen?.campaignId ?? null,
+ campaigns: retained,
+ };
+}
+
+export function buildProtocolEvalPointers(history) {
+ const byId = new Map(
+ history.campaigns.map((campaign) => [campaign.campaignId, campaign]),
+ );
+ const project = (id) => {
+ const campaign = id ? byId.get(id) : null;
+ return campaign
+ ? {
+ campaignId: campaign.campaignId,
+ generatedAt: campaign.generatedAt,
+ publicUrl: campaign.publicUrl,
+ paperclipSha: campaign.source?.paperclip?.sha ?? null,
+ evalsSha: campaign.source?.evals?.sha ?? null,
+ }
+ : null;
+ };
+ return {
+ latest: {
+ schema: "paperclip.runner-protocol-eval.pointer/v1",
+ updatedAt: history.updatedAt,
+ campaign: project(history.latestCampaignId),
+ },
+ latestGreen: {
+ schema: "paperclip.runner-protocol-eval.pointer/v1",
+ updatedAt: history.updatedAt,
+ campaign: project(history.latestGreenCampaignId),
+ },
+ };
+}
+
+function date(value) {
+ return new Intl.DateTimeFormat("en-US", {
+ dateStyle: "medium",
+ timeStyle: "short",
+ timeZone: "UTC",
+ }).format(new Date(value));
+}
+
+export function renderProtocolEvalHistoryIndex(history) {
+ const rows = history.campaigns.length
+ ? history.campaigns
+ .map((campaign) => {
+ const status =
+ campaign.complete && campaign.allPassed ? "passed" : "failed";
+ const rosters = campaign.rosters
+ .map(
+ (roster) =>
+ `${html(roster.model)} · ${roster.passed}/${roster.selected}`,
+ )
+ .join("<br>");
+ return `<tr><td><a href="${html(campaign.publicUrl)}"><code>${html(campaign.campaignId)}</code></a><small>${html(date(campaign.generatedAt))} UTC</small></td><td><span class="status ${status}">${status}</span></td><td><strong>${html(campaign.totals.passed)}/${html(campaign.totals.selected)}</strong><small>${html(campaign.totals.behaviorFailures)} behavior · ${html(campaign.totals.infrastructureFailures)} infrastructure</small></td><td>${rosters}</td><td><code>${html(campaign.source?.paperclip?.sha?.slice(0, 8) ?? "unknown")}</code><small>evals ${html(campaign.source?.evals?.sha?.slice(0, 8) ?? "unknown")}</small></td><td><a href="${html(campaign.publicUrl)}">Open Evalbook →</a></td></tr>`;
+ })
+ .join("")
+ : '<tr><td colspan="6" class="empty">No campaigns have been published yet.</td></tr>';
+ const latest = history.campaigns.find(
+ (campaign) => campaign.campaignId === history.latestCampaignId,
+ );
+ const latestGreen = history.campaigns.find(
+ (campaign) => campaign.campaignId === history.latestGreenCampaignId,
+ );
+ return `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><meta name="color-scheme" content="light dark"><title>Runner protocol eval campaigns · Paperclip</title>
+<style>:root{color-scheme:light dark;--bg:#fff;--fg:#172019;--muted:#667069;--line:#dfe3dc;--raised:#f8f9f6;--pass:#17603a;--pass-bg:#e5f3e9;--fail:#942f2b;--fail-bg:#f9e5e3} @media(prefers-color-scheme:dark){:root{--bg:#141413;--fg:#fafafa;--muted:#aaa;--line:#ffffff1f;--raised:#1c1c1b;--pass:#65d58c;--pass-bg:#22c55e1f;--fail:#ff7770;--fail-bg:#dc26262e}} *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.5 ui-sans-serif,system-ui,sans-serif}main{width:min(1560px,calc(100% - 48px));margin:48px auto 72px}h1{margin:0;font-size:clamp(34px,4vw,56px);line-height:1.05;letter-spacing:-.04em}p{max-width:760px;color:var(--muted);font-size:16px}a{color:inherit;text-underline-offset:3px}.pointers{display:flex;gap:10px;margin:28px 0 18px}.pointers a{padding:8px 11px;border:1px solid var(--line);border-radius:8px;background:var(--raised);text-decoration:none}.table{overflow:auto;border:1px solid var(--line);border-radius:12px}table{width:100%;border-collapse:collapse}th,td{padding:13px 14px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}th{background:var(--raised);color:var(--muted);font-size:10px;text-transform:uppercase;letter-spacing:.06em}tr:last-child td{border:0}small{display:block;margin-top:4px;color:var(--muted);font-size:10px}.status{display:inline-block;padding:3px 8px;border-radius:99px;font-size:10px;font-weight:750;text-transform:uppercase}.passed{color:var(--pass);background:var(--pass-bg)}.failed{color:var(--fail);background:var(--fail-bg)}.empty{padding:48px;text-align:center;color:var(--muted)}footer{margin-top:18px;color:var(--muted);font-size:11px}@media(max-width:700px){main{width:calc(100% - 28px);margin-top:28px}}</style></head>
+<body><main><div><small>Paperclip quality engineering</small><h1>Runner protocol eval campaigns</h1><p>Versioned direct live-runner Evalbook reports. Full provider transcripts, session identifiers, state, and raw tool evidence remain in access-controlled workflow artifacts.</p></div>
+<nav class="pointers">${latest ? `<a href="${html(latest.publicUrl)}">Latest · ${html(latest.campaignId)}</a>` : ""}${latestGreen ? `<a href="${html(latestGreen.publicUrl)}">Latest green · ${html(latestGreen.campaignId)}</a>` : ""}</nav>
+<div class="table"><table><thead><tr><th>Campaign</th><th>Status</th><th>Cells</th><th>Models / rosters</th><th>Source</th><th></th></tr></thead><tbody>${rows}</tbody></table></div><footer>Updated ${html(date(history.updatedAt))} UTC · Immutable campaign bundles · Canonical Evalbook layout with public-safe evidence projections</footer></main></body></html>`;
+}
+
+function awsObject(bucket, key) {
+ return `s3://${bucket}/${key}`;
+}
+
+async function objectExists(bucket, key) {
+ try {
+ await execFileAsync("aws", [
+ "s3api",
+ "head-object",
+ "--bucket",
+ bucket,
+ "--key",
+ key,
+ ]);
+ return true;
+ } catch (error) {
+ const detail = String(error?.stderr ?? error?.message ?? error);
+ if (/\b(?:404|Not Found|NoSuchKey)\b/iu.test(detail)) return false;
+ throw new Error(
+ `Unable to inspect protocol eval history object: ${detail.slice(0, 400)}`,
+ );
+ }
+}
+
+async function downloadJson(bucket, key, destination) {
+ if (!(await objectExists(bucket, key))) return null;
+ await execFileAsync("aws", [
+ "s3",
+ "cp",
+ awsObject(bucket, key),
+ destination,
+ "--only-show-errors",
+ ]);
+ return loadObject(destination);
+}
+
+async function uploadFile(bucket, key, file, cacheControl) {
+ const contentType =
+ extname(file) === ".html" ? "text/html; charset=utf-8" : "application/json";
+ await execFileAsync("aws", [
+ "s3",
+ "cp",
+ file,
+ awsObject(bucket, key),
+ "--only-show-errors",
+ "--content-type",
+ contentType,
+ "--cache-control",
+ cacheControl,
+ ]);
+}
+
+async function uploadImmutableReport(bucket, prefix, reportRoot) {
+ const cacheControl = "public,max-age=31536000,immutable";
+ await execFileAsync("aws", [
+ "s3",
+ "cp",
+ reportRoot,
+ awsObject(bucket, prefix),
+ "--recursive",
+ "--only-show-errors",
+ "--cache-control",
+ cacheControl,
+ "--exclude",
+ "*.json",
+ ]);
+ await uploadFile(
+ bucket,
+ `${prefix}/campaign.json`,
+ resolve(reportRoot, "campaign.json"),
+ cacheControl,
+ );
+}
+
+export async function publishProtocolEvalHistory({ reportRoot, destination }) {
+ const validatedDestination =
+ validateProtocolEvalHistoryDestination(destination);
+ const { campaign } = await validatePublicProtocolEvalReport(reportRoot);
+ const manifest = await createProtocolEvalBundleManifest(
+ reportRoot,
+ campaign.campaignId,
+ );
+ const temporary = await mkdtemp(
+ join(tmpdir(), "runner-protocol-eval-history-"),
+ );
+ const historyKey = `${validatedDestination.prefix}/history.json`;
+ const history = mergeProtocolEvalHistory(
+ (await downloadJson(
+ validatedDestination.bucket,
+ historyKey,
+ join(temporary, "history.json"),
+ )) ?? emptyProtocolEvalHistory(),
+ protocolEvalHistoryRecord(
+ campaign,
+ `${validatedDestination.publicBaseUrl}/${validatedDestination.prefix}`,
+ ),
+ );
+ const campaignPrefix = `${validatedDestination.prefix}/campaigns/${campaign.campaignId}`;
+ const manifestKey = `${campaignPrefix}/bundle-manifest.json`;
+ const existing = await downloadJson(
+ validatedDestination.bucket,
+ manifestKey,
+ join(temporary, "existing-manifest.json"),
+ );
+ if (existing && existing.bundleDigest !== manifest.bundleDigest) {
+ throw new Error(
+ `Immutable campaign ${campaign.campaignId} already exists with a different digest`,
+ );
+ }
+ if (!existing) {
+ await uploadImmutableReport(
+ validatedDestination.bucket,
+ campaignPrefix,
+ reportRoot,
+ );
+ const manifestFile = join(temporary, "bundle-manifest.json");
+ await writeFile(manifestFile, json(manifest));
+ await uploadFile(
+ validatedDestination.bucket,
+ manifestKey,
+ manifestFile,
+ "public,max-age=31536000,immutable",
+ );
+ }
+ const pointers = buildProtocolEvalPointers(history);
+ const mutable = {
+ "history.json": history,
+ "latest.json": pointers.latest,
+ "latest-green.json": pointers.latestGreen,
+ };
+ for (const [name, value] of Object.entries(mutable)) {
+ const file = join(temporary, name);
+ await writeFile(file, json(value));
+ await uploadFile(
+ validatedDestination.bucket,
+ `${validatedDestination.prefix}/${name}`,
+ file,
+ "no-cache",
+ );
+ }
+ const index = join(temporary, "index.html");
+ await writeFile(index, renderProtocolEvalHistoryIndex(history));
+ await uploadFile(
+ validatedDestination.bucket,
+ `${validatedDestination.prefix}/index.html`,
+ index,
+ "no-cache",
+ );
+ return {
+ campaignId: campaign.campaignId,
+ bundleDigest: manifest.bundleDigest,
+ historySize: history.campaigns.length,
+ };
+}
+
+async function main() {
+ const result = await publishProtocolEvalHistory({
+ reportRoot: resolve(
+ process.env.PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR ??
+ "runner-protocol-eval-public-report",
+ ),
+ destination: {
+ bucket: process.env.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET ?? "",
+ prefix:
+ process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX ??
+ "runner-protocol-evals",
+ publicBaseUrl:
+ process.env.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL ?? "",
+ },
+ });
+ console.log(
+ `Published immutable protocol eval campaign ${result.campaignId} (${result.bundleDigest}) and ${result.historySize} history record(s)`,
+ );
+}
+
+if (
+ process.argv[1] &&
+ resolve(process.argv[1]) === resolve(import.meta.filename)
+) {
+ await main().catch((error) => {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exitCode = 1;
+ });
+}
diff --git a/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs
new file mode 100644
index 0000000000..835144fdc5
--- /dev/null
+++ b/packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.test.mjs
@@ -0,0 +1,221 @@
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+import {
+ buildProtocolEvalPointers,
+ createProtocolEvalBundleManifest,
+ emptyProtocolEvalHistory,
+ isPublicProtocolEvalPath,
+ mergeProtocolEvalHistory,
+ protocolEvalHistoryRecord,
+ renderProtocolEvalHistoryIndex,
+ validateProtocolEvalHistoryDestination,
+ validatePublicProtocolEvalReport,
+} from "./publish-runner-protocol-eval-history.mjs";
+
+const roots = [];
+test.afterEach(async () => {
+ await Promise.all(
+ roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
+ );
+});
+
+function campaign(overrides = {}) {
+ return {
+ schema: "paperclip.runner-protocol-eval.campaign/v1",
+ campaignId: "gha-42-1",
+ generatedAt: "2026-09-05T00:00:00.000Z",
+ source: {
+ paperclip: { sha: "a".repeat(40), ref: "refs/heads/master" },
+ evals: { repository: "paperclipai/paperclip-evals", sha: "b".repeat(40) },
+ },
+ complete: true,
+ allPassed: true,
+ totals: {
+ selected: 35,
+ passed: 35,
+ behaviorFailures: 0,
+ infrastructureFailures: 0,
+ },
+ rosters: [
+ {
+ rosterId: "protocol-live-mini",
+ model: "gpt-5.4-mini",
+ provider: "codex",
+ driver: "codex_app_server",
+ selected: 35,
+ passed: 35,
+ },
+ ],
+ results: [],
+ ...overrides,
+ };
+}
+
+async function reportFixture() {
+ const root = await mkdtemp(join(tmpdir(), "runner-protocol-public-"));
+ roots.push(root);
+ await Promise.all([
+ mkdir(join(root, "tests"), { recursive: true }),
+ mkdir(join(root, "attempts"), { recursive: true }),
+ ]);
+ await Promise.all([
+ writeFile(join(root, "campaign.json"), JSON.stringify(campaign())),
+ writeFile(
+ join(root, "index.html"),
+ '<!doctype html><a href="tests/get-context.html">test</a><a href="attempts/attempt-01.html">attempt</a>',
+ ),
+ writeFile(
+ join(root, "tests/get-context.html"),
+ '<a href="../index.html">overview</a>',
+ ),
+ writeFile(
+ join(root, "attempts/attempt-01.html"),
+ '<a href="../index.html">overview</a>',
+ ),
+ ]);
+ return root;
+}
+
+test("accepts only credential-free HTTPS destinations and the dedicated prefix", () => {
+ assert.deepEqual(
+ validateProtocolEvalHistoryDestination({
+ bucket: "paperclip-public-reports",
+ prefix: "/runner-protocol-evals/",
+ publicBaseUrl: "https://reports.paperclip.ing/",
+ }),
+ {
+ bucket: "paperclip-public-reports",
+ prefix: "runner-protocol-evals",
+ publicBaseUrl: "https://reports.paperclip.ing",
+ },
+ );
+ assert.throws(
+ () =>
+ validateProtocolEvalHistoryDestination({
+ bucket: "bad",
+ prefix: "../evals",
+ publicBaseUrl: "http://example.test",
+ }),
+ /prefix|HTTPS/,
+ );
+});
+
+test("keeps publication to the canonical static Evalbook surface", () => {
+ for (const file of [
+ "index.html",
+ "latest.html",
+ "inventory.html",
+ "tests/get-context.html",
+ "attempts/attempt-01.html",
+ "campaign.json",
+ ]) {
+ assert.equal(isPublicProtocolEvalPath(file), true, file);
+ }
+ for (const file of [
+ "attempts/attempt-01/index.html",
+ "runs/attempt/artifact.json",
+ "provider-trace.log",
+ "../secret",
+ "assets/app.js",
+ ]) {
+ assert.equal(isPublicProtocolEvalPath(file), false, file);
+ }
+});
+
+test("validates inert linked reports and creates deterministic manifests", async () => {
+ const root = await reportFixture();
+ const result = await validatePublicProtocolEvalReport(root);
+ assert.equal(result.files.length, 4);
+ const first = await createProtocolEvalBundleManifest(root, "gha-42-1");
+ const second = await createProtocolEvalBundleManifest(root, "gha-42-1");
+ assert.deepEqual(second, first);
+ assert.match(first.bundleDigest, /^[0-9a-f]{64}$/);
+});
+
+test("rejects scripts, remote resources, raw sessions, secret-shaped values, and broken links", async () => {
+ for (const bad of [
+ "<script>alert(1)</script>",
+ '<a href="https://attacker.example">remote</a>',
+ '<pre>{"providerSessionId":"session"}</pre>',
+ `<pre>${"sk-" + "a".repeat(24)}</pre>`,
+ '<a href="missing.html">broken</a>',
+ ]) {
+ const root = await reportFixture();
+ await writeFile(join(root, "index.html"), bad);
+ await assert.rejects(validatePublicProtocolEvalReport(root));
+ }
+});
+
+test("retains immutable history and independent latest-green pointers", () => {
+ const green = protocolEvalHistoryRecord(
+ campaign(),
+ "https://reports.example/runner-protocol-evals",
+ );
+ const red = protocolEvalHistoryRecord(
+ campaign({
+ campaignId: "gha-43-1",
+ generatedAt: "2026-09-05T01:00:00.000Z",
+ allPassed: false,
+ totals: {
+ selected: 35,
+ passed: 34,
+ behaviorFailures: 1,
+ infrastructureFailures: 0,
+ },
+ rosters: [{ ...campaign().rosters[0], passed: 34 }],
+ }),
+ "https://reports.example/runner-protocol-evals",
+ );
+ let history = mergeProtocolEvalHistory(emptyProtocolEvalHistory(), green);
+ history = mergeProtocolEvalHistory(history, red);
+ assert.equal(history.latestCampaignId, "gha-43-1");
+ assert.equal(history.latestGreenCampaignId, "gha-42-1");
+ const pointers = buildProtocolEvalPointers(history);
+ assert.equal(pointers.latest.campaign.campaignId, "gha-43-1");
+ assert.equal(pointers.latestGreen.campaign.campaignId, "gha-42-1");
+ const index = renderProtocolEvalHistoryIndex(history);
+ assert.match(index, /Runner protocol eval campaigns/);
+ assert.match(index, /Open Evalbook/);
+ assert.match(index, /34\/35/);
+ assert.doesNotMatch(index, /private-session-id|sk-[A-Za-z0-9_-]{20,}/);
+ assert.throws(
+ () => mergeProtocolEvalHistory(history, { ...green, allPassed: false }),
+ /Immutable campaign history changed/,
+ );
+});
+
+test("retains the latest green pointer outside the 200 newest campaigns", () => {
+ const green = protocolEvalHistoryRecord(
+ campaign(),
+ "https://reports.example/runner-protocol-evals",
+ );
+ let history = mergeProtocolEvalHistory(emptyProtocolEvalHistory(), green);
+ for (let index = 0; index < 201; index += 1) {
+ history = mergeProtocolEvalHistory(
+ history,
+ protocolEvalHistoryRecord(
+ campaign({
+ campaignId: `gha-${index + 43}-1`,
+ generatedAt: new Date(
+ Date.UTC(2026, 8, 5, 1, 0, index),
+ ).toISOString(),
+ allPassed: false,
+ }),
+ "https://reports.example/runner-protocol-evals",
+ ),
+ );
+ }
+
+ assert.equal(history.campaigns.length, 200);
+ assert.equal(history.latestCampaignId, "gha-243-1");
+ assert.equal(history.latestGreenCampaignId, "gha-42-1");
+ assert.equal(history.campaigns.at(-1).campaignId, "gha-42-1");
+ assert.equal(
+ buildProtocolEvalPointers(history).latestGreen.campaign.campaignId,
+ "gha-42-1",
+ );
+});
diff --git a/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs
new file mode 100644
index 0000000000..9ebafc19a4
--- /dev/null
+++ b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.mjs
@@ -0,0 +1,352 @@
+import { createHash } from "node:crypto";
+import { spawn } from "node:child_process";
+import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+const EVAL_PROGRAM_RELATIVE_PATH =
+ "evals/paperclip-runner/tools/eval_program.py";
+
+function json(value) {
+ return `${JSON.stringify(value, null, 2)}\n`;
+}
+
+function sha256(value) {
+ return createHash("sha256").update(value).digest("hex");
+}
+
+function safeSegment(value) {
+ const segment = String(value)
+ .trim()
+ .replaceAll(/[^0-9A-Za-z._-]+/g, "-")
+ .replaceAll(/^-+|-+$/g, "");
+ if (!segment) throw new Error("Evalbook attempt identity is empty");
+ return segment;
+}
+
+function candidateDescriptor(report, candidateId) {
+ const descriptor =
+ report.bundle.providerVersions?.[candidateId] ?? candidateId;
+ const separator = descriptor.indexOf(":");
+ return separator < 0
+ ? { driver: "unknown driver", model: descriptor }
+ : {
+ driver: descriptor.slice(0, separator),
+ model: descriptor.slice(separator + 1),
+ };
+}
+
+function scoreChecks(result) {
+ const dimensionChecks = Object.values(result.scorecard.dimensions).map(
+ (dimension) => ({
+ id: dimension.dimension,
+ kind: "workflow_dimension",
+ passed: dimension.passed === true,
+ detail:
+ dimension.score === null
+ ? dimension.reasons.join("; ") || "not scored"
+ : `score ${dimension.score}${
+ dimension.reasons.length === 0
+ ? ""
+ : `; ${dimension.reasons.join("; ")}`
+ }`,
+ evidenceRefs: [],
+ }),
+ );
+ const observationChecks = [
+ ["lifecycle", result.observation.lifecycle?.checks ?? []],
+ ["continuation", result.observation.continuation?.checks ?? []],
+ ["presentation", result.observation.presentation?.checks ?? []],
+ ].flatMap(([group, checks]) =>
+ checks.map((check) => ({
+ id: `${group}.${check.id}`,
+ kind: `${group}_check`,
+ passed: check.passed === true,
+ detail: check.reason ?? (check.passed ? "passed" : "failed"),
+ evidenceRefs: [],
+ })),
+ );
+ return [...dimensionChecks, ...observationChecks];
+}
+
+function disposition(result) {
+ if (result.scorecard.overall.passed === true) return "passed";
+ if (
+ result.observation.classification === "infrastructure_failure" ||
+ result.observation.classification === "skipped"
+ ) {
+ return "infrastructure_failure";
+ }
+ return "behavior_failure";
+}
+
+function infrastructureErrors(result) {
+ if (disposition(result) !== "infrastructure_failure") return [];
+ return [
+ result.observation.failure?.message ??
+ `workflow execution was ${result.observation.classification}`,
+ ];
+}
+
+export function runnerWorkflowEvalbookAttempt({
+ report,
+ result,
+ caseDefinition,
+}) {
+ const { driver, model } = candidateDescriptor(report, result.candidateId);
+ const identity = {
+ generatedAt: report.generatedAt,
+ bundleId: report.bundle.id,
+ caseId: result.scenarioId,
+ candidateId: result.candidateId,
+ };
+ const timestamp = safeSegment(report.generatedAt);
+ const attemptId = `${timestamp}-${safeSegment(result.scenarioId)}-${safeSegment(
+ result.candidateId,
+ )}-${sha256(JSON.stringify(identity)).slice(0, 10)}`;
+ const checks = scoreChecks(result);
+ const costUsd = result.observation.metrics.costUsd;
+ const infrastructure =
+ disposition(result) === "infrastructure_failure"
+ ? result.observation.failure
+ : undefined;
+ const artifact = {
+ schema: "paperclip-runner/workflow-eval-artifact/v1",
+ attemptId,
+ createdAt: report.generatedAt,
+ requestedModel: model,
+ provider: result.observation.provider,
+ driver,
+ providerVersion:
+ report.bundle.runnerBuild ?? report.bundle.runnerVersion ?? "unknown",
+ providerSessionId: result.observation.base.trace.sessionId,
+ retainedSession: false,
+ retainedSessionStatus: "not retained by safe workflow eval projection",
+ usage: {
+ agentTurns: result.observation.metrics.attempts,
+ inputTokens: result.observation.metrics.totalTokens,
+ cachedInputTokens: 0,
+ outputTokens: 0,
+ reasoningTokens: 0,
+ ...(costUsd === undefined
+ ? {}
+ : { estimatedCostNanodollars: Math.round(costUsd * 1_000_000_000) }),
+ },
+ turn: {
+ status:
+ result.observation.classification === "completed"
+ ? "completed"
+ : "failed",
+ },
+ snapshot: {
+ createdAt: report.generatedAt,
+ providerModel: {
+ id: model,
+ provider: result.observation.provider,
+ },
+ transcript: [],
+ evidence: [],
+ },
+ devtools: { revisions: [] },
+ ...(infrastructure === undefined
+ ? {}
+ : {
+ infrastructureFailure: {
+ class: infrastructure.code,
+ category: infrastructure.category,
+ retryable: infrastructure.retryable,
+ },
+ }),
+ workflow: {
+ bundle: report.bundle,
+ observation: result.observation,
+ scorecard: result.scorecard,
+ },
+ };
+ const score = {
+ schema: "paperclip-runner/eval-score/v1",
+ attemptId,
+ caseId: result.scenarioId,
+ checks,
+ disposition: disposition(result),
+ infrastructureErrors: infrastructureErrors(result),
+ passed: result.scorecard.overall.passed === true,
+ digest: `sha256:${sha256(JSON.stringify({ identity, checks }))}`,
+ };
+ const evalCase = {
+ schema: "paperclip-runner/workflow-eval-case/v1",
+ id: result.scenarioId,
+ title: caseDefinition?.title ?? result.scenarioId,
+ description: caseDefinition
+ ? `Runner workflow case. Tags: ${caseDefinition.tags.join(", ")}.`
+ : "Runner workflow case.",
+ prompt:
+ "Redacted by the safe Runner workflow eval projection; inspect the authored workflow case for orchestration steps.",
+ fixture: "runner-workflow-live-harness",
+ authority: {
+ controlPlaneOwned: result.observation.base.controlPlaneOwned,
+ },
+ checks: Object.entries(caseDefinition?.assertions ?? {}).map(
+ ([id, expected]) => ({
+ id,
+ kind: "workflow_assertion",
+ expected,
+ }),
+ ),
+ ...(caseDefinition === undefined
+ ? {}
+ : { workflowDefinition: caseDefinition }),
+ };
+ const config = {
+ schema: "paperclip-runner/workflow-eval-config/v1",
+ id: result.candidateId,
+ model,
+ provider: result.observation.provider,
+ driver,
+ runnerVersion: report.bundle.runnerVersion,
+ runnerBuild: report.bundle.runnerBuild,
+ promptPolicyId: report.bundle.promptPolicyId,
+ };
+ return { attemptId, artifact, score, case: evalCase, config };
+}
+
+async function writeImmutable(path, value) {
+ const content = json(value);
+ try {
+ const existing = await readFile(path, "utf8");
+ if (existing !== content) {
+ throw new Error(`Immutable Evalbook record changed: ${path}`);
+ }
+ } catch (error) {
+ if (error?.code !== "ENOENT") throw error;
+ await writeFile(path, content, { flag: "wx", mode: 0o600 });
+ }
+}
+
+export async function writeRunnerWorkflowEvalbookAttempts({
+ report,
+ runsRoot,
+ caseForId,
+}) {
+ await mkdir(runsRoot, { recursive: true });
+ const attempts = [];
+ for (const result of report.results) {
+ const attempt = runnerWorkflowEvalbookAttempt({
+ report,
+ result,
+ caseDefinition: caseForId?.(result.scenarioId),
+ });
+ const directory = resolve(runsRoot, attempt.attemptId);
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ await Promise.all([
+ writeImmutable(resolve(directory, "artifact.json"), attempt.artifact),
+ writeImmutable(resolve(directory, "score.json"), attempt.score),
+ writeImmutable(resolve(directory, "case.json"), attempt.case),
+ writeImmutable(resolve(directory, "config.json"), attempt.config),
+ ]);
+ attempts.push(attempt.attemptId);
+ }
+ return attempts;
+}
+
+async function existingPath(paths) {
+ for (const path of paths.filter(Boolean)) {
+ try {
+ await access(path);
+ return resolve(path);
+ } catch {
+ // Continue through the explicit and conventional checkout locations.
+ }
+ }
+ return null;
+}
+
+export async function resolveCanonicalEvalProgram(packageRoot, environment) {
+ const fromRoot = environment.PAPERCLIP_EVALS_ROOT
+ ? resolve(environment.PAPERCLIP_EVALS_ROOT, EVAL_PROGRAM_RELATIVE_PATH)
+ : null;
+ const program = await existingPath([
+ environment.PAPERCLIP_EVALBOOK_PROGRAM,
+ fromRoot,
+ resolve(packageRoot, "../../.paperclip-evals", EVAL_PROGRAM_RELATIVE_PATH),
+ resolve(
+ packageRoot,
+ "../../../paperclip-evals",
+ EVAL_PROGRAM_RELATIVE_PATH,
+ ),
+ resolve(
+ packageRoot,
+ "../../../../paperclip-evals",
+ EVAL_PROGRAM_RELATIVE_PATH,
+ ),
+ ]);
+ if (program !== null) return program;
+ throw new Error(
+ `Canonical Evalbook generator not found. Set PAPERCLIP_EVALBOOK_PROGRAM to ${EVAL_PROGRAM_RELATIVE_PATH} in a paperclip-evals checkout.`,
+ );
+}
+
+async function run(command, args) {
+ await new Promise((accept, reject) => {
+ const child = spawn(command, args, { stdio: "inherit" });
+ child.once("error", reject);
+ child.once("exit", (code, signal) => {
+ if (code === 0) accept();
+ else
+ reject(
+ new Error(
+ `${command} exited ${code ?? `for signal ${signal ?? "unknown"}`}`,
+ ),
+ );
+ });
+ });
+}
+
+export async function renderRunnerWorkflowWithCanonicalEvalbook({
+ packageRoot,
+ outputDirectory,
+ report,
+ caseForId,
+ environment = process.env,
+}) {
+ const program = await resolveCanonicalEvalProgram(packageRoot, environment);
+ const runsRoot = resolve(outputDirectory, "evalbook-runs");
+ await rm(runsRoot, { recursive: true, force: true });
+ const attempts = await writeRunnerWorkflowEvalbookAttempts({
+ report,
+ runsRoot,
+ caseForId,
+ });
+ await Promise.all([
+ rm(resolve(outputDirectory, "attempts"), { recursive: true, force: true }),
+ rm(resolve(outputDirectory, "tests"), { recursive: true, force: true }),
+ rm(resolve(outputDirectory, "index.html"), { force: true }),
+ rm(resolve(outputDirectory, "latest.html"), { force: true }),
+ rm(resolve(outputDirectory, "live-report.html"), { force: true }),
+ rm(resolve(outputDirectory, "deterministic-report.html"), { force: true }),
+ ]);
+ await run(environment.PYTHON ?? "python3", [
+ program,
+ "report",
+ "--runs-root",
+ runsRoot,
+ "--output",
+ outputDirectory,
+ ]);
+ const programBytes = await readFile(program);
+ const manifest = {
+ schema: "paperclip.runner.workflow-evalbook-render.v1",
+ generatedAt: report.generatedAt,
+ generator: {
+ path: program,
+ sha256: `sha256:${sha256(programBytes)}`,
+ },
+ sourceReport: "live-report.json",
+ attempts,
+ index: resolve(outputDirectory, "index.html"),
+ };
+ await writeFile(
+ resolve(outputDirectory, "evalbook-manifest.json"),
+ json(manifest),
+ );
+ return manifest;
+}
diff --git a/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs
new file mode 100644
index 0000000000..54fe993633
--- /dev/null
+++ b/packages/paperclip-runner/scripts/render-runner-workflow-evalbook.test.mjs
@@ -0,0 +1,134 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { resolve } from "node:path";
+import test from "node:test";
+
+import {
+ runnerWorkflowEvalbookAttempt,
+ writeRunnerWorkflowEvalbookAttempts,
+} from "./render-runner-workflow-evalbook.mjs";
+
+function fixture(overrides = {}) {
+ const report = {
+ generatedAt: "2026-09-05T16:30:00.000Z",
+ bundle: {
+ id: "runner-live-v2-test",
+ runnerVersion: "1.2.3",
+ runnerBuild: "abc123",
+ promptPolicyId: "runner-live-workflow-v1",
+ providerVersions: {
+ "codex-luna": "codex_app_server:gpt-5.6-luna",
+ },
+ },
+ results: [],
+ };
+ const result = {
+ scenarioId: "verification-policy",
+ candidateId: "codex-luna",
+ observation: {
+ classification: "completed",
+ provider: "codex",
+ base: {
+ controlPlaneOwned: false,
+ trace: { sessionId: "eval-session-1" },
+ },
+ lifecycle: {
+ checks: [{ id: "terminal-authority", passed: true }],
+ },
+ continuation: { checks: [] },
+ presentation: { checks: [] },
+ metrics: {
+ attempts: 1,
+ totalTokens: 1234,
+ costUsd: 0.0025,
+ },
+ ...overrides.observation,
+ },
+ scorecard: {
+ dimensions: {
+ semantic_outcome: {
+ dimension: "semantic_outcome",
+ score: 1,
+ passed: true,
+ reasons: [],
+ },
+ },
+ overall: { score: 1, passed: true },
+ ...overrides.scorecard,
+ },
+ };
+ report.results.push(result);
+ return { report, result };
+}
+
+test("maps a workflow result to the canonical immutable-attempt inputs", () => {
+ const { report, result } = fixture();
+ const attempt = runnerWorkflowEvalbookAttempt({
+ report,
+ result,
+ caseDefinition: {
+ title: "Verify before completion",
+ tags: ["verification"],
+ },
+ });
+
+ assert.match(attempt.attemptId, /verification-policy-codex-luna/);
+ assert.equal(attempt.artifact.requestedModel, "gpt-5.6-luna");
+ assert.equal(attempt.artifact.driver, "codex_app_server");
+ assert.equal(attempt.artifact.usage.estimatedCostNanodollars, 2_500_000);
+ assert.equal(attempt.score.disposition, "passed");
+ assert.equal(attempt.config.id, "codex-luna");
+ assert.equal(attempt.case.title, "Verify before completion");
+ assert.match(attempt.case.prompt, /Redacted/);
+});
+
+test("preserves unscored infrastructure semantics for the grid", () => {
+ const { report, result } = fixture({
+ observation: {
+ classification: "infrastructure_failure",
+ failure: {
+ code: "provider_timeout",
+ category: "provider",
+ retryable: true,
+ message: "provider timed out",
+ },
+ },
+ scorecard: {
+ overall: { score: null, passed: null },
+ },
+ });
+ const attempt = runnerWorkflowEvalbookAttempt({ report, result });
+
+ assert.equal(attempt.score.disposition, "infrastructure_failure");
+ assert.deepEqual(attempt.score.infrastructureErrors, ["provider timed out"]);
+ assert.deepEqual(attempt.artifact.infrastructureFailure, {
+ class: "provider_timeout",
+ category: "provider",
+ retryable: true,
+ });
+});
+
+test("writes idempotent attempt records without raw provider content", async () => {
+ const root = await mkdtemp(resolve(tmpdir(), "runner-workflow-evalbook-"));
+ try {
+ const { report } = fixture();
+ const options = {
+ report,
+ runsRoot: root,
+ caseForId: () => ({ title: "Verify", tags: ["verification"] }),
+ };
+ const first = await writeRunnerWorkflowEvalbookAttempts(options);
+ const second = await writeRunnerWorkflowEvalbookAttempts(options);
+ assert.deepEqual(second, first);
+
+ const artifact = JSON.parse(
+ await readFile(resolve(root, first[0], "artifact.json"), "utf8"),
+ );
+ assert.deepEqual(artifact.snapshot.transcript, []);
+ assert.deepEqual(artifact.snapshot.evidence, []);
+ assert.equal(artifact.workflow.observation.provider, "codex");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs
index 19217335e3..e78643209f 100644
--- a/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs
+++ b/packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs
@@ -9,6 +9,8 @@ import {
} from "node:fs/promises";
import { resolve } from "node:path";
+import { renderRunnerWorkflowWithCanonicalEvalbook } from "./render-runner-workflow-evalbook.mjs";
+
const packageRoot = resolve(import.meta.dirname, "..");
const evals = await import(resolve(packageRoot, "dist/eval/index.js"));
const packageManifest = JSON.parse(
@@ -21,8 +23,7 @@ if (mode !== "nightly" && mode !== "chaos")
throw new Error(`unsupported workflow eval schedule mode: ${mode}`);
const now = process.env.PAPERCLIP_EVAL_GENERATED_AT ?? new Date().toISOString();
const rotationDay = Number(
- process.env.PAPERCLIP_EVAL_ROTATION_DAY ??
- evals.runnerLiveRotationWeek(now),
+ process.env.PAPERCLIP_EVAL_ROTATION_DAY ?? evals.runnerLiveRotationWeek(now),
);
const seed =
process.env.PAPERCLIP_EVAL_SCHEDULE_SEED ?? "runner-live-seven-week-v1";
@@ -35,6 +36,51 @@ const historyDirectory = resolve(
);
await mkdir(outputDirectory, { recursive: true });
+function selectorValues(flag, environmentName) {
+ const values = [];
+ for (let index = 0; index < process.argv.length; index += 1) {
+ if (process.argv[index] !== flag) continue;
+ const value = process.argv[index + 1];
+ if (!value || value.startsWith("--")) {
+ throw new Error(`${flag} requires a comma-separated value`);
+ }
+ values.push(value);
+ }
+ if (process.env[environmentName]) values.push(process.env[environmentName]);
+ return [
+ ...new Set(
+ values.flatMap((value) =>
+ value
+ .split(",")
+ .map((entry) => entry.trim())
+ .filter(Boolean),
+ ),
+ ),
+ ];
+}
+
+function selectionLimit() {
+ const index = process.argv.indexOf("--limit");
+ const raw =
+ index < 0 ? process.env.PAPERCLIP_EVAL_LIMIT : process.argv[index + 1];
+ if (raw === undefined || raw === "") return undefined;
+ const value = Number(raw);
+ if (!Number.isSafeInteger(value) || value <= 0) {
+ throw new Error("--limit must be a positive integer");
+ }
+ return value;
+}
+
+const selection = {
+ candidateIds: selectorValues("--candidate", "PAPERCLIP_EVAL_CANDIDATE"),
+ caseIds: selectorValues("--case", "PAPERCLIP_EVAL_CASE"),
+ limit: selectionLimit(),
+};
+const selectionActive =
+ selection.candidateIds.length > 0 ||
+ selection.caseIds.length > 0 ||
+ selection.limit !== undefined;
+
function safeBundleId(schedule) {
const runnerBuild =
process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version;
@@ -51,6 +97,13 @@ function safeBundleId(schedule) {
reasoningEffort,
}),
),
+ ...(selectionActive
+ ? {
+ selectedExecutions: schedule.entries.map(
+ (entry) => entry.executionId,
+ ),
+ }
+ : {}),
});
return `runner-live-v2-${createHash("sha256").update(identity).digest("hex").slice(0, 16)}`;
}
@@ -110,7 +163,7 @@ async function retainHistory(report) {
}
if (mode === "nightly") {
- const schedule = evals.buildRunnerLiveEvalSchedule({
+ const fullSchedule = evals.buildRunnerLiveEvalSchedule({
seed,
rotationDay,
generatedAt: now,
@@ -120,9 +173,12 @@ if (mode === "nightly") {
throw new Error(
`live schedule coverage failed: ${JSON.stringify(coverage)}`,
);
+ const schedule = selectionActive
+ ? evals.selectRunnerLiveEvalSchedule(fullSchedule, selection)
+ : fullSchedule;
await writeFile(
resolve(outputDirectory, "nightly-schedule.json"),
- `${JSON.stringify({ schedule, coverage }, null, 2)}\n`,
+ `${JSON.stringify({ schedule, coverage, selection: selectionActive ? selection : null }, null, 2)}\n`,
);
if (!execute) {
process.stdout.write(
@@ -230,9 +286,15 @@ if (mode === "nightly") {
`${evals.renderRunnerWorkflowGitHubSummary(report, alerts)}\n## Previous compatible bundle\n\n${comparison === null ? "No compatible prior bundle is available." : `Pass-rate delta: ${comparison.passRateDelta}; overall delta: ${comparison.overallDelta}.`}\n`,
),
]);
+ await renderRunnerWorkflowWithCanonicalEvalbook({
+ packageRoot,
+ outputDirectory,
+ report,
+ caseForId: evals.runnerWorkflowCase,
+ });
await retainHistory(report);
process.stdout.write(
- `Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s).\n`,
+ `Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s). Open ${resolve(outputDirectory, "index.html")}.\n`,
);
} else {
const payload = {
diff --git a/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs
index 80273bcd87..e52c720ad3 100644
--- a/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs
+++ b/packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs
@@ -3,30 +3,62 @@ import { resolve } from "node:path";
const packageRoot = resolve(import.meta.dirname, "..");
const evals = await import(resolve(packageRoot, "dist/eval/index.js"));
-const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));
-const traceability = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8"));
-const traceabilitySummary = evals.validateStressTraceabilityManifest(traceability);
-const results = await evals.runDeterministicRunnerWorkflowMatrix({ bundleId: "runner-workflows-deterministic-v1" });
+const packageManifest = JSON.parse(
+ await readFile(resolve(packageRoot, "package.json"), "utf8"),
+);
+const traceability = JSON.parse(
+ await readFile(
+ resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"),
+ "utf8",
+ ),
+);
+const traceabilitySummary =
+ evals.validateStressTraceabilityManifest(traceability);
+const results = await evals.runDeterministicRunnerWorkflowMatrix({
+ bundleId: "runner-workflows-deterministic-v1",
+});
const report = evals.buildRunnerWorkflowEvalReport({
source: "deterministic",
bundle: {
id: "runner-workflows-deterministic-v1",
runnerVersion: packageManifest.version,
promptPolicyId: "stress-sanitized-v1",
- providerVersions: { codex: "fixture-v1", opencode: "fixture-v1", acpx: "fixture-v1" },
+ providerVersions: {
+ codex: "fixture-v1",
+ opencode: "fixture-v1",
+ acpx: "fixture-v1",
+ },
},
results,
traceability: traceabilitySummary,
});
-if (report.aggregate.passed !== report.aggregate.scoreable) {
- throw new Error(`Runner workflow evals passed ${report.aggregate.passed}/${report.aggregate.scoreable}`);
-}
-const outputDirectory = resolve(packageRoot, ".paperclip-local/evals/workflows");
+const outputDirectory = resolve(
+ packageRoot,
+ ".paperclip-local/evals/workflows",
+);
await mkdir(outputDirectory, { recursive: true });
await Promise.all([
- writeFile(resolve(outputDirectory, "deterministic-report.json"), `${JSON.stringify(report, null, 2)}\n`),
- writeFile(resolve(outputDirectory, "deterministic-report.md"), evals.renderRunnerWorkflowMarkdown(report)),
- writeFile(resolve(outputDirectory, "deterministic-report.junit.xml"), evals.renderRunnerWorkflowJUnit(report)),
- writeFile(resolve(outputDirectory, "github-summary.md"), evals.renderRunnerWorkflowGitHubSummary(report)),
+ writeFile(
+ resolve(outputDirectory, "deterministic-report.json"),
+ `${JSON.stringify(report, null, 2)}\n`,
+ ),
+ writeFile(
+ resolve(outputDirectory, "deterministic-report.md"),
+ evals.renderRunnerWorkflowMarkdown(report),
+ ),
+ writeFile(
+ resolve(outputDirectory, "deterministic-report.junit.xml"),
+ evals.renderRunnerWorkflowJUnit(report),
+ ),
+ writeFile(
+ resolve(outputDirectory, "github-summary.md"),
+ evals.renderRunnerWorkflowGitHubSummary(report),
+ ),
]);
-process.stdout.write(`Runner workflow evals passed: ${report.aggregate.passed}/${report.aggregate.scoreable}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`);
+const outcome =
+ report.aggregate.passed === report.aggregate.scoreable
+ ? "all scoreable executions passed"
+ : `${report.aggregate.passed}/${report.aggregate.scoreable} scoreable executions passed; deterministic fixtures exercised fail-closed scoring`;
+process.stdout.write(
+ `Runner workflow eval report ready: ${outcome}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`,
+);
diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs
new file mode 100644
index 0000000000..7314b63748
--- /dev/null
+++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs
@@ -0,0 +1,698 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import {
+ appendFile,
+ cp,
+ lstat,
+ mkdir,
+ readFile,
+ readdir,
+ rm,
+ writeFile,
+} from "node:fs/promises";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
+
+const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/;
+const ATTEMPT_FILES = new Set([
+ "request.json",
+ "artifact.json",
+ "score.json",
+ "compiled-fixture.json",
+ "case.json",
+ "config.json",
+]);
+
+function json(value) {
+ return `${JSON.stringify(value, null, 2)}\n`;
+}
+
+async function loadObject(path) {
+ const value = JSON.parse(await readFile(path, "utf8"));
+ if (value === null || Array.isArray(value) || typeof value !== "object") {
+ throw new Error(`Expected a JSON object: ${path}`);
+ }
+ return value;
+}
+
+function safeId(value, label = "identifier") {
+ const result = String(value ?? "");
+ if (!SAFE_ID.test(result)) throw new Error(`Unsafe ${label}: ${result}`);
+ return result;
+}
+
+function inside(root, candidate, label) {
+ const rel = relative(resolve(root), resolve(candidate));
+ if (!rel || rel === ".." || rel.startsWith(`..${sep}`)) {
+ throw new Error(`${label} escapes its declared root`);
+ }
+ return resolve(candidate);
+}
+
+export function credentialForConfig(config) {
+ if (config.provider === "opencode") return "OPENROUTER_API_KEY";
+ if (config.provider === "claude_managed") return "ANTHROPIC_API_KEY";
+ if (config.provider === "aws_agentcore") return "AWS_AGENTCORE_OIDC";
+ if (config.provider === "codex" || config.provider === undefined) {
+ return "OPENAI_API_KEY";
+ }
+ if (config.provider === "acpx") {
+ if (config.acpxAgent === "pi") return "OPENROUTER_API_KEY";
+ if (config.acpxAgent === "claude") return "ANTHROPIC_API_KEY";
+ if (config.acpxAgent === "codex") return "OPENAI_API_KEY";
+ }
+ throw new Error(
+ `No credential policy for ${config.provider ?? "codex"}/${config.acpxAgent ?? "default"}`,
+ );
+}
+
+function parseRosterSelection(value) {
+ if (!value?.trim() || value.trim() === "all") return null;
+ const selected = value
+ .split(",")
+ .map((entry) => entry.trim())
+ .filter(Boolean);
+ if (selected.length === 0 || new Set(selected).size !== selected.length) {
+ throw new Error("Roster selection must contain unique comma-separated IDs");
+ }
+ return new Set(selected);
+}
+
+export async function buildProtocolEvalCatalog({
+ evalsRoot,
+ rosterSelection = "all",
+ campaignId,
+ source = {},
+ maxParallel = 100,
+}) {
+ safeId(campaignId, "campaign ID");
+ if (
+ !Number.isSafeInteger(maxParallel) ||
+ maxParallel < 2 ||
+ maxParallel > 100
+ ) {
+ throw new Error("maxParallel must be an integer from 2 through 100");
+ }
+ const programRoot = resolve(evalsRoot, "evals/paperclip-runner");
+ const rosterRoot = resolve(programRoot, "rosters");
+ const selected = parseRosterSelection(rosterSelection);
+ const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true }))
+ .filter(
+ (entry) =>
+ entry.isFile() &&
+ entry.name.startsWith("live-") &&
+ entry.name.endsWith(".json"),
+ )
+ .map((entry) => entry.name)
+ .sort();
+ const rosters = [];
+ for (const rosterFile of rosterFiles) {
+ const rosterPath = resolve(rosterRoot, rosterFile);
+ const roster = await loadObject(rosterPath);
+ const rosterId = safeId(roster.id, "roster ID");
+ if (selected && !selected.has(rosterId) && !selected.has(rosterFile)) {
+ continue;
+ }
+ if (roster.schema !== "paperclip-runner/live-roster/v1") {
+ throw new Error(`Unsupported live roster schema in ${rosterFile}`);
+ }
+ if (!Array.isArray(roster.cases) || roster.cases.length === 0) {
+ throw new Error(`Live roster ${rosterId} has no cases`);
+ }
+ const configPath = inside(
+ programRoot,
+ resolve(rosterRoot, String(roster.config ?? "")),
+ `Config for ${rosterId}`,
+ );
+ const config = await loadObject(configPath);
+ const credentialName = credentialForConfig(config);
+ const cases = roster.cases.map((caseId) => safeId(caseId, "case ID"));
+ if (new Set(cases).size !== cases.length) {
+ throw new Error(`Live roster ${rosterId} repeats a case`);
+ }
+ rosters.push({
+ rosterId,
+ rosterFile,
+ configFile: relative(programRoot, configPath).split(sep).join("/"),
+ model: String(roster.model ?? config.model ?? "unknown"),
+ provider: String(config.provider ?? "codex"),
+ driver: String(config.driver ?? "codex_app_server"),
+ credentialName,
+ cases,
+ });
+ }
+ if (selected) {
+ const found = new Set(
+ rosters.flatMap((roster) => [roster.rosterId, roster.rosterFile]),
+ );
+ const missing = [...selected].filter((entry) => !found.has(entry));
+ if (missing.length > 0) {
+ throw new Error(`Unknown live roster selection: ${missing.join(", ")}`);
+ }
+ }
+ if (rosters.length === 0) throw new Error("No live rosters were selected");
+
+ const cells = rosters.flatMap((roster) =>
+ roster.cases.map((caseId) => ({
+ cellId: safeId(`${roster.rosterId}--${caseId}`, "cell ID"),
+ rosterId: roster.rosterId,
+ rosterFile: roster.rosterFile,
+ caseId,
+ model: roster.model,
+ provider: roster.provider,
+ driver: roster.driver,
+ credentialName: roster.credentialName,
+ })),
+ );
+ const shards = [[], []];
+ cells.forEach((cell, index) => shards[index % shards.length].push(cell));
+ if (shards.some((shard) => shard.length > 256)) {
+ throw new Error(
+ "Protocol eval catalog exceeds the two-shard GitHub matrix limit",
+ );
+ }
+ return {
+ schema: "paperclip.runner-protocol-eval.catalog/v1",
+ campaignId,
+ source,
+ rosters,
+ cells,
+ matrices: shards.map((include) => ({ include })),
+ maxParallel,
+ maxParallelPerShard: Math.floor(maxParallel / 2),
+ };
+}
+
+async function writeGithubOutput(entries) {
+ const output = process.env.GITHUB_OUTPUT;
+ if (!output) return;
+ await appendFile(
+ output,
+ Object.entries(entries)
+ .map(([key, value]) => `${key}=${value}\n`)
+ .join(""),
+ );
+}
+
+async function copyAttempt(source, destination) {
+ const metadata = await lstat(source);
+ if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
+ throw new Error(`Attempt source is not a real directory: ${source}`);
+ }
+ await mkdir(destination, { recursive: false, mode: 0o700 });
+ for (const entry of await readdir(source, { withFileTypes: true })) {
+ if (
+ !entry.isFile() ||
+ entry.isSymbolicLink() ||
+ !ATTEMPT_FILES.has(entry.name)
+ ) {
+ throw new Error(`Unexpected attempt path ${join(source, entry.name)}`);
+ }
+ await cp(join(source, entry.name), join(destination, entry.name), {
+ errorOnExist: true,
+ });
+ }
+ for (const required of [
+ "artifact.json",
+ "score.json",
+ "case.json",
+ "config.json",
+ ]) {
+ await lstat(join(destination, required));
+ }
+}
+
+async function findFiles(root, name) {
+ const metadata = await lstat(root).catch(() => null);
+ if (!metadata) return [];
+ if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
+ throw new Error(`Download root must be a real directory: ${root}`);
+ }
+ const found = [];
+ async function visit(directory) {
+ for (const entry of await readdir(directory, { withFileTypes: true })) {
+ const absolute = join(directory, entry.name);
+ if (entry.isSymbolicLink())
+ throw new Error(`Refusing downloaded symlink: ${absolute}`);
+ if (entry.isDirectory()) await visit(absolute);
+ else if (entry.isFile() && entry.name === name) found.push(absolute);
+ }
+ }
+ await visit(root);
+ return found.sort();
+}
+
+function safeUsage(usage) {
+ const fields = [
+ "agentTurns",
+ "providerRequests",
+ "inputTokens",
+ "outputTokens",
+ "cachedInputTokens",
+ "reasoningTokens",
+ "providerReportedCostNanodollars",
+ "estimatedCostNanodollars",
+ ];
+ return Object.fromEntries(
+ fields
+ .filter((field) => Number.isFinite(usage?.[field]) && usage[field] >= 0)
+ .map((field) => [field, usage[field]]),
+ );
+}
+
+async function syntheticAttempt({ evalsRoot, runsOut, cell, campaignId }) {
+ const programRoot = resolve(evalsRoot, "evals/paperclip-runner");
+ const roster = await loadObject(
+ resolve(programRoot, "rosters", cell.rosterFile),
+ );
+ const casePath = resolve(programRoot, "cases", `${cell.caseId}.json`);
+ const configPath = inside(
+ programRoot,
+ resolve(programRoot, "rosters", String(roster.config)),
+ `Config for ${cell.rosterId}`,
+ );
+ const [evalCase, config] = await Promise.all([
+ loadObject(casePath),
+ loadObject(configPath),
+ ]);
+ const attemptId = safeId(
+ `${cell.cellId}-${campaignId}-missing`,
+ "synthetic attempt ID",
+ );
+ const directory = resolve(runsOut, attemptId);
+ await mkdir(directory, { recursive: false, mode: 0o700 });
+ const createdAt = new Date().toISOString();
+ const artifact = {
+ schema: "paperclip-runner/eval-session-artifact/v1",
+ attemptId,
+ createdAt,
+ requestedModel: config.model,
+ provider: config.provider ?? "codex",
+ driver: config.driver ?? "codex_app_server",
+ providerVersion: config.opencodeVersion ?? null,
+ usage: {},
+ turn: { status: "failed" },
+ snapshot: {
+ createdAt,
+ providerModel: {
+ id: config.model,
+ provider: config.modelProvider ?? config.provider ?? "unknown",
+ },
+ transcript: [],
+ evidence: [],
+ },
+ devtools: { revisions: [] },
+ infrastructureFailure: {
+ class: "ci_cell_artifact_missing",
+ category: "campaign_orchestration",
+ retryable: false,
+ },
+ };
+ const score = {
+ schema: "paperclip-runner/eval-score/v1",
+ attemptId,
+ caseId: cell.caseId,
+ disposition: "infrastructure_failure",
+ passed: false,
+ infrastructureErrors: [
+ "The matrix cell did not retain a complete attempt artifact.",
+ ],
+ checks: [],
+ };
+ score.digest = `sha256:${createHash("sha256").update(JSON.stringify(score)).digest("hex")}`;
+ await Promise.all([
+ writeFile(join(directory, "artifact.json"), json(artifact), {
+ mode: 0o600,
+ }),
+ writeFile(join(directory, "score.json"), json(score), { mode: 0o600 }),
+ writeFile(join(directory, "case.json"), json(evalCase), { mode: 0o600 }),
+ writeFile(join(directory, "config.json"), json(config), { mode: 0o600 }),
+ ]);
+ return attemptId;
+}
+
+export async function aggregateProtocolEvalCampaign({
+ catalogPath,
+ downloadsRoot,
+ evalsRoot,
+ runsOut,
+ campaignOut,
+ source,
+}) {
+ const catalog = await loadObject(catalogPath);
+ if (catalog.schema !== "paperclip.runner-protocol-eval.catalog/v1") {
+ throw new Error("Unsupported protocol eval catalog");
+ }
+ await rm(runsOut, { recursive: true, force: true });
+ await mkdir(runsOut, { recursive: true });
+ const retainedByCell = new Map();
+ const expectedByCell = new Map(
+ catalog.cells.map((cell) => [cell.cellId, cell]),
+ );
+ const copiedAttemptIds = new Set();
+ for (const statusPath of await findFiles(downloadsRoot, "cell.json")) {
+ const status = await loadObject(statusPath);
+ const cellId = safeId(status.cellId, "recorded cell ID");
+ if (retainedByCell.has(cellId))
+ throw new Error(`Duplicate cell artifact ${cellId}`);
+ const expected = expectedByCell.get(cellId);
+ if (!expected)
+ throw new Error(`Downloaded artifact names an unexpected cell ${cellId}`);
+ if (
+ status.caseId !== expected.caseId ||
+ status.rosterFile !== expected.rosterFile
+ ) {
+ throw new Error(`Downloaded cell metadata drifted for ${cellId}`);
+ }
+ const attemptRoot = resolve(dirname(statusPath), "runs");
+ const attemptIds = [];
+ const attemptMetadata = await lstat(attemptRoot).catch(() => null);
+ if (attemptMetadata?.isDirectory() && !attemptMetadata.isSymbolicLink()) {
+ for (const entry of (
+ await readdir(attemptRoot, { withFileTypes: true })
+ ).sort((a, b) => a.name.localeCompare(b.name))) {
+ if (!entry.isDirectory() || entry.isSymbolicLink()) {
+ throw new Error(
+ `Unexpected cell run path ${join(attemptRoot, entry.name)}`,
+ );
+ }
+ const attemptId = safeId(entry.name, "attempt ID");
+ if (copiedAttemptIds.has(attemptId))
+ throw new Error(`Duplicate attempt artifact ${attemptId}`);
+ const [score, artifact, config] = await Promise.all([
+ loadObject(join(attemptRoot, entry.name, "score.json")),
+ loadObject(join(attemptRoot, entry.name, "artifact.json")),
+ loadObject(join(attemptRoot, entry.name, "config.json")),
+ ]);
+ if (
+ score.attemptId !== attemptId ||
+ artifact.attemptId !== attemptId ||
+ score.caseId !== expected.caseId ||
+ config.model !== expected.model ||
+ (config.provider ?? "codex") !== expected.provider ||
+ (config.driver ?? "codex_app_server") !== expected.driver
+ ) {
+ throw new Error(`Downloaded attempt identity drifted for ${cellId}`);
+ }
+ await copyAttempt(
+ join(attemptRoot, entry.name),
+ join(runsOut, attemptId),
+ );
+ copiedAttemptIds.add(attemptId);
+ attemptIds.push(attemptId);
+ }
+ }
+ retainedByCell.set(cellId, { status, attemptIds });
+ }
+
+ const results = [];
+ for (const cell of catalog.cells) {
+ const retained = retainedByCell.get(cell.cellId);
+ const attemptIds = retained?.attemptIds?.length
+ ? retained.attemptIds
+ : [
+ await syntheticAttempt({
+ evalsRoot,
+ runsOut,
+ cell,
+ campaignId: catalog.campaignId,
+ }),
+ ];
+ const finalAttemptId = attemptIds.at(-1);
+ const [score, artifact] = await Promise.all([
+ loadObject(join(runsOut, finalAttemptId, "score.json")),
+ loadObject(join(runsOut, finalAttemptId, "artifact.json")),
+ ]);
+ if (score.caseId !== cell.caseId || score.attemptId !== finalAttemptId) {
+ throw new Error(`Final attempt identity drifted for ${cell.cellId}`);
+ }
+ results.push({
+ cellId: cell.cellId,
+ rosterId: cell.rosterId,
+ caseId: cell.caseId,
+ model: cell.model,
+ provider: cell.provider,
+ driver: cell.driver,
+ attemptIds,
+ finalAttemptId,
+ disposition: score.disposition,
+ passed: score.passed === true,
+ usage: safeUsage(artifact.usage),
+ cellExitCode: Number.isSafeInteger(retained?.status?.exitCode)
+ ? retained.status.exitCode
+ : null,
+ });
+ }
+ const totals = {
+ selected: results.length,
+ passed: results.filter((result) => result.passed).length,
+ behaviorFailures: results.filter(
+ (result) => result.disposition === "behavior_failure",
+ ).length,
+ infrastructureFailures: results.filter(
+ (result) => result.disposition === "infrastructure_failure",
+ ).length,
+ };
+ const generatedAt = new Date().toISOString();
+ const campaign = {
+ schema: "paperclip.runner-protocol-eval.campaign/v1",
+ campaignId: catalog.campaignId,
+ generatedAt,
+ source: {
+ paperclip: source.paperclip,
+ evals: source.evals,
+ workflowRunUrl: source.workflowRunUrl,
+ },
+ complete: results.length === catalog.cells.length,
+ allPassed: totals.passed === totals.selected,
+ totals,
+ rosters: catalog.rosters.map((roster) => ({
+ rosterId: roster.rosterId,
+ model: roster.model,
+ provider: roster.provider,
+ driver: roster.driver,
+ selected: roster.cases.length,
+ passed: results.filter(
+ (result) => result.rosterId === roster.rosterId && result.passed,
+ ).length,
+ })),
+ results,
+ };
+ await mkdir(dirname(campaignOut), { recursive: true });
+ await writeFile(campaignOut, json(campaign), { mode: 0o600 });
+ return campaign;
+}
+
+function publicArtifact(artifact) {
+ const model = artifact.snapshot?.providerModel ?? {};
+ const infrastructure = artifact.infrastructureFailure;
+ const providerVersion =
+ artifact.provider === "claude_managed" ||
+ artifact.provider === "aws_agentcore"
+ ? "remote profile redacted"
+ : (artifact.providerVersion ?? null);
+ return {
+ schema: artifact.schema,
+ attemptId: artifact.attemptId,
+ createdAt: artifact.createdAt,
+ requestedModel: artifact.requestedModel,
+ provider: artifact.provider,
+ driver: artifact.driver,
+ providerVersion,
+ retainedSession: false,
+ retainedSessionStatus: "redacted from the public report",
+ usage: safeUsage(artifact.usage),
+ turn: { status: artifact.turn?.status ?? "failed" },
+ snapshot: {
+ createdAt: artifact.snapshot?.createdAt ?? artifact.createdAt,
+ providerModel: {
+ id: model.id ?? artifact.requestedModel,
+ provider: model.provider ?? artifact.provider,
+ },
+ transcript: [],
+ evidence: [],
+ },
+ devtools: { revisions: [] },
+ ...(infrastructure && typeof infrastructure === "object"
+ ? {
+ infrastructureFailure: {
+ class: infrastructure.class,
+ category: infrastructure.category,
+ retryable: infrastructure.retryable === true,
+ },
+ }
+ : {}),
+ };
+}
+
+function publicScore(score) {
+ return {
+ schema: score.schema,
+ attemptId: score.attemptId,
+ caseId: score.caseId,
+ disposition: score.disposition,
+ passed: score.passed === true,
+ infrastructureErrors:
+ score.disposition === "infrastructure_failure"
+ ? [
+ "Infrastructure failure; diagnostic details remain in the access-controlled artifact.",
+ ]
+ : [],
+ checks: Array.isArray(score.checks)
+ ? score.checks.map((check) => ({
+ id: check.id,
+ kind: check.kind,
+ passed: check.passed === true,
+ detail: check.passed === true ? "passed" : "failed",
+ evidenceRefs: [],
+ }))
+ : [],
+ digest: score.digest,
+ };
+}
+
+function publicConfig(config) {
+ return Object.fromEntries(
+ [
+ "schema",
+ "id",
+ "model",
+ "provider",
+ "driver",
+ "opencodeVersion",
+ "acpxAgent",
+ "modelProvider",
+ ]
+ .filter((field) => config[field] !== undefined)
+ .map((field) => [field, config[field]]),
+ );
+}
+
+export async function sanitizeProtocolEvalRuns({ runsRoot, publicRunsRoot }) {
+ await rm(publicRunsRoot, { recursive: true, force: true });
+ await mkdir(publicRunsRoot, { recursive: true });
+ const attemptIds = [];
+ for (const entry of (await readdir(runsRoot, { withFileTypes: true })).sort(
+ (a, b) => a.name.localeCompare(b.name),
+ )) {
+ if (!entry.isDirectory() || entry.isSymbolicLink()) {
+ throw new Error(
+ `Unexpected merged run path ${join(runsRoot, entry.name)}`,
+ );
+ }
+ const attemptId = safeId(entry.name, "attempt ID");
+ const source = join(runsRoot, attemptId);
+ const [artifact, score, evalCase, config] = await Promise.all([
+ loadObject(join(source, "artifact.json")),
+ loadObject(join(source, "score.json")),
+ loadObject(join(source, "case.json")),
+ loadObject(join(source, "config.json")),
+ ]);
+ if (artifact.attemptId !== attemptId || score.attemptId !== attemptId) {
+ throw new Error(`Attempt identity drifted in ${attemptId}`);
+ }
+ const destination = join(publicRunsRoot, attemptId);
+ await mkdir(destination, { mode: 0o700 });
+ await Promise.all([
+ writeFile(
+ join(destination, "artifact.json"),
+ json(publicArtifact(artifact)),
+ { mode: 0o600 },
+ ),
+ writeFile(join(destination, "score.json"), json(publicScore(score)), {
+ mode: 0o600,
+ }),
+ writeFile(join(destination, "case.json"), json(evalCase), {
+ mode: 0o600,
+ }),
+ writeFile(join(destination, "config.json"), json(publicConfig(config)), {
+ mode: 0o600,
+ }),
+ ]);
+ attemptIds.push(attemptId);
+ }
+ return attemptIds;
+}
+
+function argument(args, name, fallback) {
+ const index = args.indexOf(name);
+ return index < 0 ? fallback : args[index + 1];
+}
+
+async function main() {
+ const [command, ...args] = process.argv.slice(2);
+ if (command === "catalog") {
+ const output = resolve(
+ argument(args, "--output", "runner-protocol-eval-catalog.json"),
+ );
+ const catalog = await buildProtocolEvalCatalog({
+ evalsRoot: resolve(argument(args, "--evals-root", ".paperclip-evals")),
+ rosterSelection: argument(args, "--rosters", "all"),
+ campaignId: argument(args, "--campaign-id", `local-${Date.now()}`),
+ maxParallel: Number(argument(args, "--max-parallel", "100")),
+ source: {
+ paperclipSha: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA ?? null,
+ evalsSha: process.env.PAPERCLIP_PROTOCOL_EVALS_SHA ?? null,
+ },
+ });
+ await mkdir(dirname(output), { recursive: true });
+ await writeFile(output, json(catalog), { mode: 0o600 });
+ await writeGithubOutput({
+ matrix_0: JSON.stringify(catalog.matrices[0]),
+ matrix_1: JSON.stringify(catalog.matrices[1]),
+ matrix_1_present: String(catalog.matrices[1].include.length > 0),
+ max_parallel_per_shard: String(catalog.maxParallelPerShard),
+ selected: String(catalog.cells.length),
+ });
+ console.log(
+ json({
+ output,
+ selected: catalog.cells.length,
+ rosters: catalog.rosters.length,
+ }).trim(),
+ );
+ return;
+ }
+ if (command === "aggregate") {
+ const campaign = await aggregateProtocolEvalCampaign({
+ catalogPath: resolve(argument(args, "--catalog")),
+ downloadsRoot: resolve(argument(args, "--downloads")),
+ evalsRoot: resolve(argument(args, "--evals-root")),
+ runsOut: resolve(argument(args, "--runs-out")),
+ campaignOut: resolve(argument(args, "--campaign-out")),
+ source: {
+ paperclip: {
+ sha: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA,
+ ref: process.env.PAPERCLIP_PROTOCOL_EVAL_SOURCE_REF,
+ },
+ evals: {
+ repository: "paperclipai/paperclip-evals",
+ sha: process.env.PAPERCLIP_PROTOCOL_EVALS_SHA,
+ },
+ workflowRunUrl: process.env.PAPERCLIP_PROTOCOL_EVAL_WORKFLOW_URL,
+ },
+ });
+ console.log(json(campaign.totals).trim());
+ return;
+ }
+ if (command === "sanitize") {
+ const attemptIds = await sanitizeProtocolEvalRuns({
+ runsRoot: resolve(argument(args, "--runs-root")),
+ publicRunsRoot: resolve(argument(args, "--output")),
+ });
+ console.log(json({ attempts: attemptIds.length }).trim());
+ return;
+ }
+ throw new Error(
+ "Usage: runner-protocol-eval-campaign.mjs <catalog|aggregate|sanitize> ...",
+ );
+}
+
+if (
+ process.argv[1] &&
+ resolve(process.argv[1]) === resolve(import.meta.filename)
+) {
+ await main().catch((error) => {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exitCode = 1;
+ });
+}
diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs
new file mode 100644
index 0000000000..474f89971b
--- /dev/null
+++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs
@@ -0,0 +1,349 @@
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+import {
+ aggregateProtocolEvalCampaign,
+ buildProtocolEvalCatalog,
+ credentialForConfig,
+ sanitizeProtocolEvalRuns,
+} from "./runner-protocol-eval-campaign.mjs";
+
+const roots = [];
+test.afterEach(async () => {
+ await Promise.all(
+ roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
+ );
+});
+
+async function fixture() {
+ const root = await mkdtemp(join(tmpdir(), "runner-protocol-campaign-"));
+ roots.push(root);
+ const program = join(root, "evals/paperclip-runner");
+ await Promise.all([
+ mkdir(join(program, "rosters"), { recursive: true }),
+ mkdir(join(program, "configs"), { recursive: true }),
+ mkdir(join(program, "cases"), { recursive: true }),
+ ]);
+ const config = {
+ schema: "paperclip-runner/eval-config/v1",
+ id: "live-opencode-model",
+ provider: "opencode",
+ driver: "opencode_server",
+ model: "openrouter/example/model",
+ opencodeVersion: "1.18.17",
+ };
+ const evalCase = {
+ schema: "paperclip-runner/eval-case/v1",
+ id: "get-task-context",
+ title: "Get context",
+ description: "Synthetic public fixture",
+ prompt: "Inspect the synthetic task.",
+ fixture: "../fixtures/company.json",
+ authority: { actorId: "agent-1", taskId: "task-1" },
+ checks: [{ id: "context", kind: "semantic_operation" }],
+ };
+ const roster = {
+ schema: "paperclip-runner/live-roster/v1",
+ id: "protocol-live-opencode-model",
+ model: config.model,
+ config: "../configs/live-opencode-model.json",
+ cases: [evalCase.id],
+ };
+ await Promise.all([
+ writeFile(
+ join(program, "configs/live-opencode-model.json"),
+ JSON.stringify(config),
+ ),
+ writeFile(
+ join(program, "cases/get-task-context.json"),
+ JSON.stringify(evalCase),
+ ),
+ writeFile(
+ join(program, "rosters/live-opencode-model.json"),
+ JSON.stringify(roster),
+ ),
+ ]);
+ return { root, program, config, evalCase, roster };
+}
+
+test("maps every qualified driver to one explicit credential boundary", () => {
+ assert.equal(credentialForConfig({ provider: "codex" }), "OPENAI_API_KEY");
+ assert.equal(
+ credentialForConfig({ provider: "opencode" }),
+ "OPENROUTER_API_KEY",
+ );
+ assert.equal(
+ credentialForConfig({ provider: "acpx", acpxAgent: "claude" }),
+ "ANTHROPIC_API_KEY",
+ );
+ assert.equal(
+ credentialForConfig({ provider: "aws_agentcore" }),
+ "AWS_AGENTCORE_OIDC",
+ );
+ assert.throws(
+ () => credentialForConfig({ provider: "unknown" }),
+ /credential policy/,
+ );
+});
+
+test("catalogs roster plus case cells and emits bounded balanced shards", async () => {
+ const { root } = await fixture();
+ const catalog = await buildProtocolEvalCatalog({
+ evalsRoot: root,
+ campaignId: "gha-42-1",
+ maxParallel: 80,
+ });
+ assert.equal(catalog.cells.length, 1);
+ assert.equal(catalog.cells[0].credentialName, "OPENROUTER_API_KEY");
+ assert.equal(catalog.maxParallelPerShard, 40);
+ assert.equal(catalog.matrices[0].include.length, 1);
+ assert.equal(catalog.matrices[1].include.length, 0);
+ await assert.rejects(
+ buildProtocolEvalCatalog({
+ evalsRoot: root,
+ campaignId: "gha-42-1",
+ rosterSelection: "missing-roster",
+ }),
+ /Unknown live roster/,
+ );
+ await assert.rejects(
+ buildProtocolEvalCatalog({
+ evalsRoot: root,
+ campaignId: "gha-42-1",
+ maxParallel: 1,
+ }),
+ /from 2 through 100/,
+ );
+});
+
+test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => {
+ const { root, config, evalCase } = await fixture();
+ const catalog = await buildProtocolEvalCatalog({
+ evalsRoot: root,
+ campaignId: "gha-42-1",
+ });
+ const catalogPath = join(root, "catalog.json");
+ await writeFile(catalogPath, JSON.stringify(catalog));
+ const campaign = await aggregateProtocolEvalCampaign({
+ catalogPath,
+ downloadsRoot: join(root, "missing-downloads"),
+ evalsRoot: root,
+ runsOut: join(root, "merged-runs"),
+ campaignOut: join(root, "campaign.json"),
+ source: {
+ paperclip: { sha: "a".repeat(40), ref: "refs/heads/master" },
+ evals: { repository: "paperclipai/paperclip-evals", sha: "b".repeat(40) },
+ workflowRunUrl: "https://example.test/actions/runs/42",
+ },
+ });
+ assert.deepEqual(campaign.totals, {
+ selected: 1,
+ passed: 0,
+ behaviorFailures: 0,
+ infrastructureFailures: 1,
+ });
+ assert.equal(campaign.results[0].disposition, "infrastructure_failure");
+ assert.equal(campaign.complete, true);
+ assert.equal(campaign.allPassed, false);
+ const attempt = campaign.results[0].finalAttemptId;
+ assert.deepEqual(
+ JSON.parse(await readFile(join(root, "merged-runs", attempt, "case.json"))),
+ evalCase,
+ );
+ assert.deepEqual(
+ JSON.parse(
+ await readFile(join(root, "merged-runs", attempt, "config.json")),
+ ),
+ config,
+ );
+});
+
+test("rejects downloaded cells that were not declared by the immutable catalog", async () => {
+ const { root } = await fixture();
+ const catalog = await buildProtocolEvalCatalog({
+ evalsRoot: root,
+ campaignId: "gha-42-1",
+ });
+ const catalogPath = join(root, "catalog.json");
+ const downloads = join(root, "downloads/unexpected");
+ await mkdir(downloads, { recursive: true });
+ await Promise.all([
+ writeFile(catalogPath, JSON.stringify(catalog)),
+ writeFile(
+ join(downloads, "cell.json"),
+ JSON.stringify({
+ cellId: "protocol-live-opencode-model--undeclared",
+ rosterFile: "live-opencode-model.json",
+ caseId: "undeclared",
+ exitCode: 1,
+ }),
+ ),
+ ]);
+ await assert.rejects(
+ aggregateProtocolEvalCampaign({
+ catalogPath,
+ downloadsRoot: join(root, "downloads"),
+ evalsRoot: root,
+ runsOut: join(root, "merged-runs"),
+ campaignOut: join(root, "campaign.json"),
+ source: {},
+ }),
+ /unexpected cell/,
+ );
+});
+
+test("public run projection removes provider sessions, traces, transcripts, evidence, and state", async () => {
+ const { root, config, evalCase } = await fixture();
+ const attemptId = "get-task-context-opencode-gha-42-1-attempt-01";
+ const source = join(root, "raw-runs", attemptId);
+ await mkdir(source, { recursive: true });
+ await Promise.all([
+ writeFile(
+ join(source, "artifact.json"),
+ JSON.stringify({
+ schema: "paperclip-runner/eval-session-artifact/v1",
+ attemptId,
+ createdAt: "2026-09-05T00:00:00.000Z",
+ requestedModel: config.model,
+ provider: config.provider,
+ driver: config.driver,
+ providerVersion: config.opencodeVersion,
+ providerSessionId: "private-session-id",
+ trace: { secret: "private" },
+ usage: {
+ agentTurns: 1,
+ inputTokens: 100,
+ estimatedCostNanodollars: 5000,
+ },
+ turn: { status: "completed", turnId: "private-turn" },
+ snapshot: {
+ createdAt: "2026-09-05T00:00:00.000Z",
+ providerModel: { id: config.model, provider: "openrouter" },
+ transcript: [{ role: "assistant", text: "private transcript" }],
+ evidence: [{ kind: "tool_call", data: { token: "private" } }],
+ },
+ issueThread: { messages: ["private"] },
+ devtools: { revisions: [{ state: { private: true } }] },
+ }),
+ ),
+ writeFile(
+ join(source, "score.json"),
+ JSON.stringify({
+ schema: "paperclip-runner/eval-score/v1",
+ attemptId,
+ caseId: evalCase.id,
+ disposition: "passed",
+ passed: true,
+ infrastructureErrors: [],
+ checks: [
+ {
+ id: "context",
+ kind: "semantic_operation",
+ passed: true,
+ detail: "private result",
+ evidenceRefs: ["event-1"],
+ },
+ ],
+ digest: "sha256:test",
+ }),
+ ),
+ writeFile(join(source, "case.json"), JSON.stringify(evalCase)),
+ writeFile(
+ join(source, "config.json"),
+ JSON.stringify({ ...config, managedProfile: { profileId: "private" } }),
+ ),
+ ]);
+ await sanitizeProtocolEvalRuns({
+ runsRoot: join(root, "raw-runs"),
+ publicRunsRoot: join(root, "public-runs"),
+ });
+ const serialized = await readFile(
+ join(root, "public-runs", attemptId, "artifact.json"),
+ "utf8",
+ );
+ assert.doesNotMatch(
+ serialized,
+ /private-session|private transcript|private-turn|issueThread|trace/,
+ );
+ const artifact = JSON.parse(serialized);
+ assert.deepEqual(artifact.snapshot.transcript, []);
+ assert.deepEqual(artifact.snapshot.evidence, []);
+ assert.deepEqual(artifact.devtools.revisions, []);
+ const score = JSON.parse(
+ await readFile(join(root, "public-runs", attemptId, "score.json"), "utf8"),
+ );
+ assert.deepEqual(score.checks[0], {
+ id: "context",
+ kind: "semantic_operation",
+ passed: true,
+ detail: "passed",
+ evidenceRefs: [],
+ });
+ const publicConfig = await readFile(
+ join(root, "public-runs", attemptId, "config.json"),
+ "utf8",
+ );
+ assert.doesNotMatch(publicConfig, /managedProfile|private/);
+});
+
+test("public run projection redacts remote profile version identities", async () => {
+ const { root, config, evalCase } = await fixture();
+ const attemptId = "get-task-context-claude-managed-gha-42-1-attempt-01";
+ const source = join(root, "raw-runs", attemptId);
+ await mkdir(source, { recursive: true });
+ await Promise.all([
+ writeFile(
+ join(source, "artifact.json"),
+ JSON.stringify({
+ schema: "paperclip-runner/eval-session-artifact/v1",
+ attemptId,
+ createdAt: "2026-09-05T00:00:00.000Z",
+ requestedModel: "claude-sonnet-5",
+ provider: "claude_managed",
+ driver: "claude_managed_agents_api",
+ providerVersion: "private-agent-version-id",
+ usage: {},
+ turn: { status: "completed" },
+ snapshot: {
+ createdAt: "2026-09-05T00:00:00.000Z",
+ providerModel: { id: "claude-sonnet-5", provider: "anthropic" },
+ },
+ }),
+ ),
+ writeFile(
+ join(source, "score.json"),
+ JSON.stringify({
+ schema: "paperclip-runner/eval-score/v1",
+ attemptId,
+ caseId: evalCase.id,
+ disposition: "passed",
+ passed: true,
+ checks: [],
+ }),
+ ),
+ writeFile(join(source, "case.json"), JSON.stringify(evalCase)),
+ writeFile(
+ join(source, "config.json"),
+ JSON.stringify({
+ ...config,
+ id: "live-claude-managed",
+ model: "claude-sonnet-5",
+ provider: "claude_managed",
+ driver: "claude_managed_agents_api",
+ }),
+ ),
+ ]);
+ await sanitizeProtocolEvalRuns({
+ runsRoot: join(root, "raw-runs"),
+ publicRunsRoot: join(root, "public-runs"),
+ });
+ const publicArtifact = await readFile(
+ join(root, "public-runs", attemptId, "artifact.json"),
+ "utf8",
+ );
+ assert.match(publicArtifact, /remote profile redacted/);
+ assert.doesNotMatch(publicArtifact, /private-agent-version-id/);
+});
diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs
new file mode 100644
index 0000000000..5e9f990cb8
--- /dev/null
+++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs
@@ -0,0 +1,109 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import test from "node:test";
+
+const repositoryRoot = resolve(import.meta.dirname, "../../..");
+const workflowPath = resolve(
+ repositoryRoot,
+ ".github/workflows/runner-protocol-live-evals.yml",
+);
+
+test("direct live eval workflow keeps paid execution behind stable actor authorization", async () => {
+ const workflow = await readFile(workflowPath, "utf8");
+ assert.match(workflow, /^\s{2}authorize:/mu);
+ assert.match(workflow, /RUNNER_E2E_ALLOWED_ACTOR_IDS/u);
+ assert.match(workflow, /github\.actor_id/u);
+ assert.match(workflow, /github\.triggering_actor/u);
+ assert.match(workflow, /refs\/heads\/\$DEFAULT_BRANCH/u);
+ assert.match(workflow, /Reauthorize paid execution before provider access/u);
+ assert.match(
+ workflow,
+ /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*Run one immutable direct protocol cell/u,
+ );
+ assert.doesNotMatch(
+ workflow,
+ /^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/mu,
+ );
+ const actions = [
+ ...workflow.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gmu),
+ ].map((match) => match[1]);
+ assert.ok(actions.length > 0);
+ for (const action of actions) assert.match(action, /^[^@]+@[0-9a-f]{40}$/u);
+});
+
+test("resolves both repositories immutably and bounds total matrix concurrency", async () => {
+ const workflow = await readFile(workflowPath, "utf8");
+ const authorize = workflow.slice(
+ workflow.indexOf(" authorize:"),
+ workflow.indexOf(" catalog:"),
+ );
+ assert.match(authorize, /repos\/\$REPOSITORY\/branches\/\$encoded_branch/u);
+ assert.match(authorize, /\^\[0-9a-f\]\{40\}\$/u);
+ assert.match(
+ authorize,
+ /repos\/paperclipai\/paperclip-evals\/commits\/\$EVALS_SHA/u,
+ );
+ assert.match(authorize, /COMMITPERCLIP_KEY/u);
+ assert.match(
+ authorize,
+ /GH_TOKEN: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u,
+ );
+ assert.match(authorize, /test "\$resolved" = "\$EVALS_SHA"/u);
+ const catalog = workflow.slice(
+ workflow.indexOf(" catalog:"),
+ workflow.indexOf(" build_runner:"),
+ );
+ assert.match(
+ catalog,
+ /ref: \$\{\{ needs\.authorize\.outputs\.evals_sha \}\}/u,
+ );
+ assert.match(catalog, /RUNNER_E2E_MAX_PARALLEL/u);
+ assert.match(catalog, /max_parallel_per_shard/u);
+ const privateCheckouts = [
+ ...workflow.matchAll(
+ /repository: paperclipai\/paperclip-evals[\s\S]*?persist-credentials: false/gmu,
+ ),
+ ];
+ assert.equal(privateCheckouts.length, 3);
+ for (const checkout of privateCheckouts) {
+ assert.match(
+ checkout[0],
+ /token: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u,
+ );
+ }
+ assert.match(workflow, /matrix_0/u);
+ assert.match(workflow, /matrix_1/u);
+});
+
+test("publishes only the separately sanitized Evalbook through trusted OIDC code", async () => {
+ const workflow = await readFile(workflowPath, "utf8");
+ const report = workflow.slice(
+ workflow.indexOf(" report:"),
+ workflow.indexOf(" publish_history:"),
+ );
+ assert.match(
+ report,
+ /Render the access-controlled canonical Evalbook report/u,
+ );
+ assert.match(
+ report,
+ /--viewer-root runner-protocol-build\/extracted\/dist-issue-thread/u,
+ );
+ assert.match(report, /runner-protocol-eval-campaign\.mjs sanitize/u);
+ assert.match(
+ report,
+ /Upload access-controlled canonical Evalbook and raw attempts/u,
+ );
+ assert.match(report, /Upload publisher-only sanitized Evalbook/u);
+
+ const publisher = workflow.slice(workflow.indexOf(" publish_history:"));
+ assert.match(publisher, /ref: \$\{\{ github\.sha \}\}/u);
+ assert.match(publisher, /id-token: write/u);
+ assert.match(publisher, /runner-protocol-eval-public-/u);
+ assert.match(publisher, /publish-runner-protocol-eval-history\.mjs/u);
+ assert.match(publisher, /runner-protocol-evals/u);
+ assert.doesNotMatch(publisher, /(?:OPENAI|ANTHROPIC|OPENROUTER)_API_KEY/u);
+ assert.doesNotMatch(publisher, /paperclipai\/paperclip-evals/u);
+ assert.doesNotMatch(publisher, /downloaded-runner-protocol-evals/u);
+});
diff --git a/packages/paperclip-runner/scripts/stage-runner-binary.mjs b/packages/paperclip-runner/scripts/stage-runner-binary.mjs
index 14fcee5a92..d75f4bbb6d 100644
--- a/packages/paperclip-runner/scripts/stage-runner-binary.mjs
+++ b/packages/paperclip-runner/scripts/stage-runner-binary.mjs
@@ -1,7 +1,11 @@
+import { execFile } from "node:child_process";
import { chmod, copyFile, mkdir } from "node:fs/promises";
import path from "node:path";
+import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
+const execFileAsync = promisify(execFile);
+
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const executable = process.platform === "win32" ? "paperclip-runnerd.exe" : "paperclip-runnerd";
const source = path.join(packageRoot, "runner", "target", "release", executable);
@@ -11,3 +15,10 @@ const destination = path.join(destinationDirectory, executable);
await mkdir(destinationDirectory, { recursive: true });
await copyFile(source, destination);
if (process.platform !== "win32") await chmod(destination, 0o755);
+// Rust's linker emits an ad-hoc Mach-O signature. Copying that executable to
+// its package location preserves the bytes but can leave the kernel rejecting
+// the new inode with SIGKILL. Re-sign the staged inode so local packaged-runner
+// evals execute the same artifact that was just built.
+if (process.platform === "darwin") {
+ await execFileAsync("codesign", ["--force", "--sign", "-", destination]);
+}
diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts
index 947c22cb99..e2765f0307 100644
--- a/packages/paperclip-runner/src/cli/eval-session-contract.test.ts
+++ b/packages/paperclip-runner/src/cli/eval-session-contract.test.ts
@@ -5,6 +5,10 @@ import {
evalSessionUsage,
parseEvalSessionRequest,
} from "./eval-session-contract.js";
+import {
+ boundedEvalSessionUsage,
+ evalSessionProviderVersion,
+} from "./eval-session.js";
function request(overrides: Record<string, unknown> = {}): unknown {
return {
@@ -65,6 +69,43 @@ describe("eval-session request contract", () => {
}))).toMatchObject({ provider: "acpx", acpxAgent: "claude" });
});
+ it("accepts null optional fields from the original Evalbook v1 producer", () => {
+ const parsed = parseEvalSessionRequest(request({
+ acpxAgent: null,
+ agentCoreProfile: null,
+ opencodeVersion: null,
+ }));
+ expect(parsed).toMatchObject({
+ provider: "codex",
+ driver: "codex_app_server",
+ });
+ expect(parsed).not.toHaveProperty("acpxAgent");
+ expect(parsed).not.toHaveProperty("agentCoreProfile");
+ expect(parsed).not.toHaveProperty("opencodeVersion");
+ });
+
+ it("attributes managed providers to their immutable deployed revisions", () => {
+ expect(evalSessionProviderVersion(parseEvalSessionRequest(request({
+ provider: "aws_agentcore",
+ driver: "aws_agentcore_harness_api",
+ model: "global.anthropic.claude-sonnet-4-6",
+ agentCoreProfile: agentCoreProfile(),
+ })))).toBe("aws-agentcore-harness-context-v2");
+ expect(evalSessionProviderVersion(parseEvalSessionRequest(request({
+ provider: "claude_managed",
+ driver: "claude_managed_agents_api",
+ model: "claude-sonnet-5",
+ managedProfile: {
+ profileId: "managed-qualified",
+ anthropicAgentId: "agent-test",
+ agentVersion: "17",
+ environmentId: "environment-test",
+ betaVersion: "managed-agents-2026-04-01",
+ maxSessionListCostUsd: 1,
+ },
+ })))).toBe("17");
+ });
+
it("rejects Pi and accepts both qualified remote provider profiles", () => {
expect(() => parseEvalSessionRequest(request({
provider: "acpx",
@@ -143,6 +184,41 @@ describe("eval-session request contract", () => {
});
describe("eval-session usage", () => {
+ it("retains durable failed turns even when their reported usage exceeds completed-turn limits", () => {
+ const parsed = parseEvalSessionRequest(request());
+ const snapshot = {
+ usageLedger: [{
+ receiptId: "receipt-failed",
+ attemptId: "attempt-1",
+ providerResponseId: "response-failed",
+ turnId: "turn-failed",
+ observedAt: "2026-09-05T00:00:00.000Z",
+ providerCalls: 2,
+ providerRequests: 2,
+ inputTokens: 1_000,
+ outputTokens: 100,
+ cachedInputTokens: 0,
+ reasoningTokens: 0,
+ costNanodollars: 200_000_000,
+ }],
+ } as unknown as CapabilityLiveSessionSnapshot;
+ const failedTurn = {
+ turnId: "turn-failed",
+ status: "failed" as const,
+ assistantText: "",
+ snapshot,
+ };
+
+ expect(boundedEvalSessionUsage(parsed, failedTurn)).toMatchObject({
+ agentTurns: 2,
+ providerReportedCostNanodollars: 200_000_000,
+ });
+ expect(() => boundedEvalSessionUsage(parsed, {
+ ...failedTurn,
+ status: "completed",
+ })).toThrow("agent turn limit exceeded");
+ });
+
it("deduplicates receipts and applies the versioned model price", () => {
const receipt = {
receiptId: "receipt-1",
diff --git a/packages/paperclip-runner/src/cli/eval-session-contract.ts b/packages/paperclip-runner/src/cli/eval-session-contract.ts
index 1caf1e04f0..9c48f987b9 100644
--- a/packages/paperclip-runner/src/cli/eval-session-contract.ts
+++ b/packages/paperclip-runner/src/cli/eval-session-contract.ts
@@ -233,7 +233,10 @@ export function parseEvalSessionRequest(value: unknown): EvalSessionRequest {
if (input.driver !== undefined && input.driver !== driver) {
throw new Error("eval-session provider/driver mismatch");
}
- const acpxAgent = input.acpxAgent;
+ // The original Evalbook v1 producer serialized absent provider-specific
+ // options as JSON null. Preserve compatibility with those immutable request
+ // artifacts while continuing to reject non-null values for the wrong lane.
+ const acpxAgent = input.acpxAgent === null ? undefined : input.acpxAgent;
if (acpxAgent === "pi") throw new Error("The Pi ACPX profile is not available");
if (
acpxAgent !== undefined &&
@@ -245,16 +248,22 @@ export function parseEvalSessionRequest(value: unknown): EvalSessionRequest {
if (provider !== "acpx" && acpxAgent !== undefined) {
throw new Error("eval-session acpxAgent requires provider acpx");
}
+ const managedProfileInput = input.managedProfile === null
+ ? undefined
+ : input.managedProfile;
+ const agentCoreProfileInput = input.agentCoreProfile === null
+ ? undefined
+ : input.agentCoreProfile;
const managedProfile = provider === "claude_managed"
- ? parseManagedProfile(input.managedProfile)
+ ? parseManagedProfile(managedProfileInput)
: undefined;
const agentCoreProfile = provider === "aws_agentcore"
- ? parseAgentCoreProfile(input.agentCoreProfile)
+ ? parseAgentCoreProfile(agentCoreProfileInput)
: undefined;
- if (provider !== "claude_managed" && input.managedProfile !== undefined) {
+ if (provider !== "claude_managed" && managedProfileInput !== undefined) {
throw new Error("eval-session managedProfile requires provider claude_managed");
}
- if (provider !== "aws_agentcore" && input.agentCoreProfile !== undefined) {
+ if (provider !== "aws_agentcore" && agentCoreProfileInput !== undefined) {
throw new Error("eval-session agentCoreProfile requires provider aws_agentcore");
}
if (input.nativeResume !== undefined) {
diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts
index f98e5ab1a8..609ec73090 100644
--- a/packages/paperclip-runner/src/cli/eval-session.ts
+++ b/packages/paperclip-runner/src/cli/eval-session.ts
@@ -53,7 +53,9 @@ async function sha256(path: string): Promise<string> {
return createHash("sha256").update(await readFile(path)).digest("hex");
}
-function providerVersion(request: EvalSessionRequest): string | null {
+export function evalSessionProviderVersion(
+ request: EvalSessionRequest,
+): string | null {
if (request.provider === "opencode") {
const version = request.opencodeVersion ?? "1.18.17";
if (version !== "1.18.17") {
@@ -71,7 +73,7 @@ function providerVersion(request: EvalSessionRequest): string | null {
return request.managedProfile!.agentVersion;
}
if (request.provider === "aws_agentcore") {
- return request.agentCoreProfile!.harnessVersion;
+ return request.agentCoreProfile!.qualificationRevision;
}
return null;
}
@@ -127,6 +129,32 @@ function usageIfAvailable(
}
}
+export function boundedEvalSessionUsage(
+ request: EvalSessionRequest,
+ turn: CapabilityLiveTurnResult,
+): EvalSessionUsage | null {
+ if (turn.status !== "completed") {
+ return usageIfAvailable(request, turn.snapshot);
+ }
+ const usage = evalSessionUsage(request.model, turn.snapshot);
+ if (usage.agentTurns > request.limits.maxAgentTurns) {
+ throw new Error("agent turn limit exceeded");
+ }
+ if (
+ usage.estimatedCostNanodollars >
+ request.limits.maxEstimatedCostNanodollars
+ ) {
+ throw new Error("estimated cost limit exceeded");
+ }
+ if (
+ usage.providerReportedCostNanodollars >
+ request.limits.maxEstimatedCostNanodollars
+ ) {
+ throw new Error("provider-reported cost limit exceeded");
+ }
+ return usage;
+}
+
async function closeSession(
session: CapabilityLiveSession | null,
reason: string,
@@ -160,11 +188,16 @@ export async function runEvalSessionCli(
const requestedProvider = request.provider ?? "codex";
const requestedDriver = request.driver ??
expectedEvalSessionDriver(requestedProvider);
- const requestedProviderVersion = providerVersion(request);
+ const requestedProviderVersion = evalSessionProviderVersion(request);
const service = options.serviceFactory?.(runnerdPath) ??
new CapabilityLiveSessionService({
transportOptions: {
runnerBinary: runnerdPath,
+ // The transport performs the provider-specific allowlisting. Supplying
+ // the source environment here is still required: without it the
+ // isolated Codex home has no credential source and runnerd receives no
+ // executable PATH from the Evalbook CLI process.
+ environment: process.env,
onDiagnostic: (message) => {
process.stderr.write(`[eval-session runnerd] ${message}\n`);
},
@@ -196,27 +229,11 @@ export async function runEvalSessionCli(
} as unknown as CreateCapabilityLiveSessionInput;
session = await service.create(createInput);
turn = await session.sendMessage(request.prompt);
- if (turn.status !== "completed") {
- await session.completeAttempt("failed", `provider_turn_${turn.status}`);
- throw new Error(`provider turn ended with status ${turn.status}`);
- }
- const usage = evalSessionUsage(request.model, turn.snapshot);
- if (usage.agentTurns > request.limits.maxAgentTurns) {
- throw new Error("agent turn limit exceeded");
- }
- if (
- usage.estimatedCostNanodollars >
- request.limits.maxEstimatedCostNanodollars
- ) {
- throw new Error("estimated cost limit exceeded");
- }
- if (
- usage.providerReportedCostNanodollars >
- request.limits.maxEstimatedCostNanodollars
- ) {
- throw new Error("provider-reported cost limit exceeded");
- }
- await session.completeAttempt("succeeded");
+ const usage = boundedEvalSessionUsage(request, turn);
+ await session.completeAttempt(
+ turn.status === "completed" ? "succeeded" : "failed",
+ turn.status === "completed" ? null : `provider_turn_${turn.status}`,
+ );
await closeSession(session, "eval session complete");
snapshot = session.snapshot();
@@ -259,7 +276,7 @@ export async function runEvalSessionCli(
mode: "live",
replaySource: "live",
}),
- usage,
+ ...(usage === null ? {} : { usage }),
timing: {
startedAt,
finishedAt: new Date().toISOString(),
diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
index b183da10a4..6d9a7e3e7e 100644
--- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
+++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
@@ -48,7 +48,10 @@ const coreStateSchema = "paperclip.runner.durable.control-plane-state.v1";
const maxFrameBytes = 1024 * 1024;
const maxCommandBytes = maxFrameBytes - 4 * 1024;
const maxCommands = 500;
-const maxCommittedEventWindow = 64;
+// A provider can emit several 100-event runner batches before the transport's
+// polling turn regains the event loop. Match the transport's explicit deferred
+// event bound so a valid burst is not compacted before it can be observed.
+const maxCommittedEventWindow = 4_096;
const maxStateBytes = 192 * 1024 * 1024;
const authChallengeTtlMs = 5_000;
const stableIdPattern = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/;
diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts
index 2869830df4..2c4aca9107 100644
--- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts
+++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts
@@ -1379,6 +1379,9 @@ describe("OpenCodeServerDriver", () => {
"*": permissionMode,
external_directory: "deny",
});
+ expect(config.provider.openrouter.models).toHaveProperty(
+ "deepseek/deepseek-v4-flash-0731",
+ );
expect(config.permission.paperclip_finish).toBeUndefined();
expect(config.permission["paperclip_*"]).toBe("allow");
await session.close({ reason: "permission mode test complete" });
@@ -1771,8 +1774,11 @@ describe("OpenCodeServerDriver", () => {
workingDirectory: workspace,
});
expect(session.ids().providerSessionId).toBe("ses_fake_1");
- expect(spawns).toHaveLength(2);
- expect(commandLifecycle).toEqual(["before", "after", "before", "after"]);
+ expect(spawns.length).toBeGreaterThanOrEqual(2);
+ expect(spawns.length).toBeLessThanOrEqual(3);
+ expect(commandLifecycle).toEqual(
+ Array.from({ length: spawns.length }, () => ["before", "after"]).flat(),
+ );
await session.close({ reason: "test" });
});
diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts
index ba0d68dc7e..a764d7b605 100644
--- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts
+++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts
@@ -1866,6 +1866,8 @@ async function startRuntime(input: {
]);
const instructionRoot =
input.options.runtimeContext?.instructions.bundle.rootPath;
+ const [modelProvider, ...modelIdParts] = input.options.model.split("/");
+ const providerModelId = modelIdParts.join("/");
const config = {
$schema: "https://opencode.ai/config.json",
model: input.options.model,
@@ -1875,6 +1877,16 @@ async function startRuntime(input: {
// system prompt; siblings remain available through the read-only root.
instructions: [],
plugin: [],
+ // OpenCode's bundled models.dev snapshot can lag behind OpenRouter's live
+ // catalog. Bind the already-qualified exact model slug into the built-in
+ // provider instead of silently falling back or rejecting a newer model.
+ provider: {
+ [modelProvider!]: {
+ models: {
+ [providerModelId]: { name: providerModelId },
+ },
+ },
+ },
tools: {
question: true,
},
diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts
index 1f62eb9232..f791426794 100644
--- a/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts
+++ b/packages/paperclip-runner/src/eval/live-workflow-executor.test.ts
@@ -86,7 +86,13 @@ vi.mock("../live/live-session.js", () => ({
}));
import { runnerWorkflowCase } from "./workflow-catalog.js";
-import { executeLiveRunnerWorkflow } from "./live-workflow-executor.js";
+import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";
+import {
+ advanceDelegationReturnMockState,
+ executeLiveRunnerWorkflow,
+ scorableLiveWorkflowCalls,
+ unexpectedLiveWorkflowCalls,
+} from "./live-workflow-executor.js";
import {
RUNNER_LIVE_CANDIDATE_SLOTS,
type RunnerLiveScheduleEntry,
@@ -124,6 +130,117 @@ describe("live workflow executor infrastructure failures", () => {
vi.unstubAllEnvs();
});
+ it("allows read-only context support without permitting extra effects", () => {
+ const calls = [
+ "get_task_context",
+ "get_task_history",
+ "finish_task",
+ "create_task",
+ ];
+ expect(unexpectedLiveWorkflowCalls(calls, ["finish_task"])).toEqual([
+ "create_task",
+ ]);
+ expect(scorableLiveWorkflowCalls(calls, ["finish_task"])).toEqual([
+ "finish_task",
+ "create_task",
+ ]);
+ });
+
+ it("keeps required read-only calls in trajectory scoring", () => {
+ expect(
+ scorableLiveWorkflowCalls(
+ ["get_task_context", "get_task_history"],
+ ["get_task_context"],
+ ),
+ ).toEqual(["get_task_context"]);
+ });
+
+ it("advances a delegated child through the authoritative return wake", async () => {
+ const grants = ["delegation:tasks:create", "dependencies:write"];
+ const adapter = new CapabilityMockControlPlaneAdapter({
+ actors: [
+ {
+ id: "actor-1",
+ companyId: "company-1",
+ name: "Workflow eval actor",
+ role: "engineer",
+ status: "active",
+ budgetId: "budget-actor-1",
+ capabilityGrants: grants,
+ },
+ ],
+ });
+ await adapter.start();
+ await adapter.openFixtureRun({
+ identity: {
+ runId: "delegation-parent",
+ sessionId: "delegation-session",
+ companyId: "company-1",
+ issueId: "task-1",
+ agentId: "actor-1",
+ },
+ backendKind: "runner",
+ capabilities: grants,
+ });
+ const created = await adapter.applyCommand({
+ runId: "delegation-parent",
+ idempotencyKey: "create-child",
+ command: {
+ kind: "create_task",
+ title: "Independent verification",
+ description: "Verify the external boundary.",
+ assigneeActorId: "actor-1",
+ },
+ });
+ const childTaskId = created.entityRefs
+ .find((entityRef) => entityRef.startsWith("task:"))!
+ .slice("task:".length);
+ await adapter.applyCommand({
+ runId: "delegation-parent",
+ idempotencyKey: "set-child-dependency",
+ command: {
+ kind: "set_dependencies",
+ blockedByTaskIds: [childTaskId],
+ },
+ });
+
+ const advanced = await advanceDelegationReturnMockState({
+ mockState: adapter.serialize(),
+ parentRunId: "delegation-parent",
+ parentSessionId: "delegation-session",
+ parentTaskId: "task-1",
+ capabilities: grants,
+ });
+ expect(advanced).not.toBeNull();
+ const restored = CapabilityMockControlPlaneAdapter.restore(
+ advanced!.mockState,
+ );
+ expect(restored.snapshot().tasks).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: childTaskId, status: "done" }),
+ expect.objectContaining({
+ id: "task-1",
+ status: "in_progress",
+ checkoutRunId: advanced!.returnRunId,
+ }),
+ ]),
+ );
+ expect(restored.snapshot().blockers).toEqual([]);
+ expect(restored.snapshot().wakes).toContainEqual(
+ expect.objectContaining({
+ taskId: "task-1",
+ reason: "blockers_resolved",
+ }),
+ );
+ await expect(
+ restored.applyCommand({
+ runId: advanced!.returnRunId,
+ idempotencyKey: "finish-parent",
+ command: { kind: "finish_task", summary: "Return completed." },
+ }),
+ ).resolves.toMatchObject({ disposition: "applied" });
+ });
+
it("classifies shutdown failures as retryable infrastructure errors and redacts them", async () => {
const leakedSecret = "sk-shutdown-secret-value";
liveSessionMocks.shutdown.mockRejectedValueOnce(
diff --git a/packages/paperclip-runner/src/eval/live-workflow-executor.ts b/packages/paperclip-runner/src/eval/live-workflow-executor.ts
index 6f3b54fa95..7d81d5ef4f 100644
--- a/packages/paperclip-runner/src/eval/live-workflow-executor.ts
+++ b/packages/paperclip-runner/src/eval/live-workflow-executor.ts
@@ -4,16 +4,24 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import type { CapabilityJsonValue } from "../mock-core/capability-control-plane-types.js";
+import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";
+import type {
+ PrpStructuredRunResult,
+ PrpTerminalState,
+} from "../protocol/replay-contract.js";
import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js";
import {
CapabilityLiveSessionService,
InMemoryCapabilityLiveSessionStore,
type CapabilityLiveSession,
+ type CapabilityLiveSessionServiceOptions,
type CapabilityLiveSessionSnapshot,
+ type CapabilityLiveSessionStore,
type CapabilityLiveTurnEvent,
type CapabilityLiveTurnResult,
} from "../live/live-session.js";
import type { EvalObservation } from "./eval-scoring.js";
+import { isCapabilitySemanticReadOperation } from "../semantic-tools/policy.js";
import {
RUNNER_WORKFLOW_OBSERVATION_SCHEMA,
type RunnerWorkflowCheck,
@@ -74,9 +82,261 @@ function continuationPrompt(evalCase: RunnerWorkflowEvalCase): string {
if (evalCase.id === "steering-causality") {
return "Steering update: use bullet points only. Apply only this formatting change and call finish_task exactly once without repeating prior work.";
}
+ if (evalCase.id === "delegation-return") {
+ return "The plan is accepted. Create exactly one justified child, set the source task dependency to that child, then stop without calling finish_task. Wait for the authoritative child-completion result.";
+ }
return "Continue from the authoritative typed interaction result. Do not redo completed work.";
}
+function workflowResult(
+ disposition: "done" | "blocked",
+ summary: string,
+ idempotencyKey: string,
+): PrpStructuredRunResult {
+ return {
+ schema: "paperclip.run_result.v1",
+ reportedWorkDisposition: disposition,
+ summary,
+ completionClaim: {
+ contractRevision: "runner-workflow-eval-v1",
+ objectiveSatisfied: disposition === "done",
+ criteria: [],
+ remainingWork:
+ disposition === "done"
+ ? []
+ : [
+ {
+ description: "Wait for the delegated child to complete.",
+ blocksCompletion: true,
+ },
+ ],
+ },
+ evidence: [],
+ verification: [],
+ attentionRequests: [],
+ artifacts: [],
+ ...(disposition === "done"
+ ? {}
+ : {
+ blocker: {
+ reasonCode: "delegated_child_in_progress",
+ owner: { kind: "agent" as const, name: "Delegated child" },
+ unblockAction: "Complete the delegated child task.",
+ scope: "current_track" as const,
+ },
+ continuation: {
+ kind: "delegated_issue" as const,
+ summary:
+ "Resume the source task after the delegated child completes.",
+ idempotencyKey,
+ },
+ }),
+ };
+}
+
+async function completeWorkflowRun(
+ adapter: CapabilityMockControlPlaneAdapter,
+ input: {
+ runId: string;
+ sessionId: string;
+ source: string;
+ disposition: "done" | "blocked";
+ summary: string;
+ },
+): Promise<void> {
+ const terminal: PrpTerminalState = {
+ schema: "paperclip.prp.terminal.v1",
+ turnTerminalState: "completed",
+ runTerminalState: "succeeded",
+ reportedWorkDisposition: input.disposition,
+ };
+ const current = adapter.snapshot().runs.find((run) => run.id === input.runId);
+ if (current === undefined) {
+ throw new Error(`workflow eval run ${input.runId} is missing`);
+ }
+ if (current.result !== null) return;
+ if (
+ !current.events.some(
+ (event) => "eventType" in event && event.eventType === "run.terminal",
+ )
+ ) {
+ await adapter.appendEvent({
+ schema: "paperclip.prp.event.v1",
+ sourceEventId: `${input.source}:terminal:1`,
+ sourceSeq: 1,
+ sourceInstanceId: input.source,
+ sourceKind: "runner",
+ runId: input.runId,
+ normalizedSessionId: input.sessionId,
+ turnId: `${input.source}-turn`,
+ eventType: "run.terminal",
+ schemaVersion: 1,
+ priority: 0,
+ emittedAt: new Date().toISOString(),
+ payload: terminal,
+ });
+ }
+ await adapter.completeRun({
+ result: workflowResult(
+ input.disposition,
+ input.summary,
+ `${input.source}-continuation`,
+ ),
+ terminal,
+ });
+}
+
+export interface AdvancedDelegationReturnState {
+ mockState: string;
+ childTaskId: string;
+ returnRunId: string;
+}
+
+/**
+ * Advance the controlled mock authority through the child-return boundary.
+ * This is orchestration evidence, not a model-authored semantic call: the
+ * evaluated provider still has to author the child/dependency and final result.
+ */
+export async function advanceDelegationReturnMockState(input: {
+ mockState: string;
+ parentRunId: string;
+ parentSessionId: string;
+ parentTaskId: string;
+ capabilities: readonly string[];
+}): Promise<AdvancedDelegationReturnState | null> {
+ const adapter = CapabilityMockControlPlaneAdapter.restore(input.mockState);
+ if (adapter.snapshot().lifecycle !== "running") await adapter.start();
+ const initial = adapter.snapshot();
+ const children = initial.tasks.filter(
+ (task) => task.parentId === input.parentTaskId,
+ );
+ if (children.length !== 1) return null;
+ const child = children[0]!;
+ if (
+ !initial.blockers.some(
+ (blocker) =>
+ blocker.taskId === input.parentTaskId &&
+ blocker.blockedByTaskId === child.id,
+ )
+ ) {
+ return null;
+ }
+
+ await completeWorkflowRun(adapter, {
+ runId: input.parentRunId,
+ sessionId: input.parentSessionId,
+ source: `${input.parentRunId}-delegation-wait`,
+ disposition: "blocked",
+ summary: "Source task is waiting on its delegated child.",
+ });
+
+ const childRunId = `${input.parentRunId}-child`;
+ const childSessionId = `${input.parentSessionId}-child`;
+ const actorId = child.assigneeActorId ?? initial.actors[0]!.id;
+ await adapter.openFixtureRun({
+ identity: {
+ runId: childRunId,
+ sessionId: childSessionId,
+ companyId: child.companyId,
+ issueId: child.id,
+ agentId: actorId,
+ },
+ backendKind: "runner",
+ sourceInstanceId: "runner-workflow-eval-child",
+ capabilities: [...input.capabilities],
+ });
+ await adapter.applyCommand({
+ runId: childRunId,
+ idempotencyKey: `${childRunId}-finish`,
+ command: {
+ kind: "finish_task",
+ taskId: child.id,
+ summary: "Delegated verification completed.",
+ },
+ });
+ await completeWorkflowRun(adapter, {
+ runId: childRunId,
+ sessionId: childSessionId,
+ source: `${childRunId}-completion`,
+ disposition: "done",
+ summary: "Delegated verification completed.",
+ });
+
+ const returnRunId = `${input.parentRunId}-return`;
+ await adapter.openFixtureRun({
+ identity: {
+ runId: returnRunId,
+ sessionId: input.parentSessionId,
+ companyId: initial.company.id,
+ issueId: input.parentTaskId,
+ agentId: initial.actors[0]!.id,
+ },
+ backendKind: "runner",
+ sourceInstanceId: "runner-workflow-eval-return",
+ capabilities: [...input.capabilities],
+ wake: {
+ reason: "blockers_resolved",
+ payload: { completedTaskId: child.id },
+ },
+ });
+ return { mockState: adapter.serialize(), childTaskId: child.id, returnRunId };
+}
+
+async function restoreAfterDelegatedChild(input: {
+ session: CapabilityLiveSession;
+ store: CapabilityLiveSessionStore;
+ transportOptions: CapabilityLiveSessionServiceOptions["transportOptions"];
+}): Promise<{
+ service: CapabilityLiveSessionService;
+ session: CapabilityLiveSession;
+} | null> {
+ const before = input.session.snapshot();
+ const advanced = await advanceDelegationReturnMockState({
+ mockState: before.mockState,
+ parentRunId: before.authority.runId,
+ parentSessionId: before.sessionId,
+ parentTaskId: before.authority.taskId,
+ capabilities: before.authority.capabilities,
+ });
+ if (advanced === null) return null;
+ await input.session.suspend("workflow eval delegated child completed");
+ const checkpoint = await input.store.load(before.sessionId);
+ if (checkpoint === null) {
+ throw new Error("workflow eval delegation checkpoint is missing");
+ }
+ const {
+ providerRunBinding: _providerRunBinding,
+ ...checkpointWithoutBinding
+ } = checkpoint;
+ const at = new Date().toISOString();
+ const updated: CapabilityLiveSessionSnapshot = {
+ ...checkpointWithoutBinding,
+ revision: checkpoint.revision + 1,
+ updatedAt: at,
+ authority: {
+ ...checkpoint.authority,
+ runId: advanced.returnRunId,
+ },
+ mockState: advanced.mockState,
+ stateHistory: [
+ ...(checkpoint.stateHistory ?? []),
+ {
+ revision: JSON.parse(advanced.mockState).revision as number,
+ at,
+ turnId: null,
+ operationId: "eval.delegated_child_completed",
+ state: advanced.mockState,
+ },
+ ],
+ };
+ await input.store.save(updated);
+ const service = new CapabilityLiveSessionService({
+ store: input.store,
+ transportOptions: input.transportOptions,
+ });
+ return { service, session: await service.restore(before.sessionId) };
+}
+
function acpxAgent(
candidate: RunnerLiveEvalCandidate,
): "claude" | "codex" | undefined {
@@ -142,6 +402,33 @@ function observedCalls(snapshot: CapabilityLiveSessionSnapshot): string[] {
);
}
+export function unexpectedLiveWorkflowCalls(
+ calls: readonly string[],
+ expectedCalls: readonly string[],
+): string[] {
+ const expected = new Set(expectedCalls);
+ return scorableLiveWorkflowCalls(calls, expectedCalls).filter(
+ (operationId) => !expected.has(operationId),
+ );
+}
+
+/**
+ * Keeps required reads and every stateful operation in trajectory scoring while
+ * ignoring optional read-only context gathering. The full call list remains in
+ * the workflow evidence and metrics for auditability.
+ */
+export function scorableLiveWorkflowCalls(
+ calls: readonly string[],
+ expectedCalls: readonly string[],
+): string[] {
+ const expected = new Set(expectedCalls);
+ return calls.filter(
+ (operationId) =>
+ expected.has(operationId) ||
+ !isCapabilitySemanticReadOperation(operationId),
+ );
+}
+
function duplicateEffectSignals(
snapshot: CapabilityLiveSessionSnapshot,
): string[] {
@@ -591,6 +878,24 @@ export async function executeLiveRunnerWorkflow(input: {
),
);
await settleInteractions(input.evalCase, session, turns, withinBudget);
+ if (input.evalCase.id === "delegation-return" && withinBudget()) {
+ const restored = await restoreAfterDelegatedChild({
+ session,
+ store,
+ transportOptions,
+ });
+ if (restored !== null) {
+ service = restored.service;
+ session = restored.session;
+ subscribeToSession(session);
+ turns.push(
+ await session.sendMessage(
+ "The delegated child completed successfully and the source task is unblocked. Use the authoritative child result and call finish_task exactly once without repeating the delegated work.",
+ { allowMissingUsage: input.allowMissingUsage },
+ ),
+ );
+ }
+ }
if (input.evalCase.id === "steering-causality" && withinBudget()) {
turns.push(
await session.sendMessage(continuationPrompt(input.evalCase), {
@@ -660,9 +965,7 @@ export async function executeLiveRunnerWorkflow(input: {
const missingCalls = expectedCalls.filter(
(operationId) => !calls.includes(operationId),
);
- const extraCalls = calls.filter(
- (operationId) => !expectedCalls.includes(operationId),
- );
+ const extraCalls = unexpectedLiveWorkflowCalls(calls, expectedCalls);
const duplicateSignals =
snapshot === undefined ? [] : duplicateEffectSignals(snapshot);
const pendingInteractions = session?.pendingInteractions().length ?? 0;
@@ -712,7 +1015,7 @@ export async function executeLiveRunnerWorkflow(input: {
provenance: { source: "live_model", behavior: input.evalCase.id },
controlPlaneOwned: expectedCalls.length === 0,
expectedCalls,
- observedCalls: calls,
+ observedCalls: scorableLiveWorkflowCalls(calls, expectedCalls),
forbiddenCalls: [],
finalState: {
expected: expectedCalls.length === 0 ? "unchanged" : "mutated",
diff --git a/packages/paperclip-runner/src/eval/live-workflow-matrix.ts b/packages/paperclip-runner/src/eval/live-workflow-matrix.ts
index c491bbe622..6ece8d18a4 100644
--- a/packages/paperclip-runner/src/eval/live-workflow-matrix.ts
+++ b/packages/paperclip-runner/src/eval/live-workflow-matrix.ts
@@ -247,6 +247,65 @@ export interface RunnerLiveEvalSchedule {
expectedExecutions: number;
}
+export interface RunnerLiveEvalSelection {
+ candidateIds?: readonly string[];
+ caseIds?: readonly string[];
+ limit?: number;
+}
+
+/** Selects a stable paid subset without weakening validation of the full schedule. */
+export function selectRunnerLiveEvalSchedule(
+ schedule: RunnerLiveEvalSchedule,
+ selection: RunnerLiveEvalSelection,
+): RunnerLiveEvalSchedule {
+ const candidateIds = new Set(selection.candidateIds ?? []);
+ const caseIds = new Set(selection.caseIds ?? []);
+ for (const id of candidateIds) {
+ if (!schedule.candidates.some((candidate) => candidate.id === id)) {
+ throw new Error(`unknown Runner live eval candidate: ${id}`);
+ }
+ }
+ const scheduledCaseIds = new Set(
+ schedule.entries.map((entry) => entry.caseId),
+ );
+ for (const id of caseIds) {
+ if (!scheduledCaseIds.has(id as RunnerWorkflowEvalCase["id"])) {
+ throw new Error(`unknown Runner live eval case: ${id}`);
+ }
+ }
+ if (
+ selection.limit !== undefined &&
+ (!Number.isSafeInteger(selection.limit) || selection.limit <= 0)
+ ) {
+ throw new Error(
+ "Runner live eval selection limit must be a positive integer",
+ );
+ }
+ let entries = schedule.entries.filter(
+ (entry) =>
+ (candidateIds.size === 0 || candidateIds.has(entry.candidateId)) &&
+ (caseIds.size === 0 || caseIds.has(entry.caseId)),
+ );
+ if (selection.limit !== undefined)
+ entries = entries.slice(0, selection.limit);
+ if (entries.length === 0) {
+ throw new Error(
+ "Runner live eval selection matched no scheduled executions",
+ );
+ }
+ const selectedCandidateIds = new Set(
+ entries.map((entry) => entry.candidateId),
+ );
+ return {
+ ...schedule,
+ candidates: schedule.candidates.filter((candidate) =>
+ selectedCandidateIds.has(candidate.id),
+ ),
+ entries,
+ expectedExecutions: entries.length,
+ };
+}
+
export function assertRunnerLiveCandidateManifest(): void {
const slots = RUNNER_LIVE_CANDIDATE_SLOTS.map((slot) => slot.id);
const candidates = RUNNER_LIVE_CANDIDATE_SLOTS.flatMap(
diff --git a/packages/paperclip-runner/src/eval/workflow-evals.test.ts b/packages/paperclip-runner/src/eval/workflow-evals.test.ts
index aea24c13d3..b4e280dccb 100644
--- a/packages/paperclip-runner/src/eval/workflow-evals.test.ts
+++ b/packages/paperclip-runner/src/eval/workflow-evals.test.ts
@@ -24,6 +24,7 @@ import {
resolvedNightlyCandidates,
runnerLiveRotationWeek,
runnerLiveScheduleCoverage,
+ selectRunnerLiveEvalSchedule,
RunnerWorkflowInfrastructureError,
} from "./live-workflow-matrix.js";
import { unavailableLiveRunnerWorkflowObservation } from "./live-workflow-executor.js";
@@ -306,6 +307,35 @@ describe("balanced live Runner workflow matrix", () => {
);
});
+ it("selects a stable bounded paid subset and rejects invalid selectors", () => {
+ const schedule = buildRunnerLiveEvalSchedule({
+ seed: "subset-v1",
+ rotationDay: 0,
+ generatedAt: "2026-08-24T00:00:00.000Z",
+ });
+ const selected = selectRunnerLiveEvalSchedule(schedule, {
+ candidateIds: ["codex-luna"],
+ limit: 2,
+ });
+ expect(selected.expectedExecutions).toBe(2);
+ expect(selected.entries).toHaveLength(2);
+ expect(
+ selected.entries.every((entry) => entry.candidateId === "codex-luna"),
+ ).toBe(true);
+ expect(selected.candidates.map((candidate) => candidate.id)).toEqual([
+ "codex-luna",
+ ]);
+ expect(schedule.expectedExecutions).toBe(40);
+ expect(() =>
+ selectRunnerLiveEvalSchedule(schedule, {
+ candidateIds: ["unknown-candidate"],
+ }),
+ ).toThrow("unknown Runner live eval candidate");
+ expect(() => selectRunnerLiveEvalSchedule(schedule, { limit: 0 })).toThrow(
+ "selection limit must be a positive integer",
+ );
+ });
+
it("retries only one retryable infrastructure failure", async () => {
const schedule = buildRunnerLiveEvalSchedule({
seed: "retry-v1",
@@ -409,7 +439,7 @@ describe("workflow reports and stress traceability", () => {
}
});
- it("renders safe JSON-derived Markdown, JUnit, and GitHub summaries", async () => {
+ it("renders safe JSON-derived Evalbook, Markdown, JUnit, and GitHub summaries", async () => {
const results = await runDeterministicRunnerWorkflowMatrix();
const report = buildRunnerWorkflowEvalReport({
source: "deterministic",
diff --git a/packages/paperclip-runner/src/evals/model-pricing.test.ts b/packages/paperclip-runner/src/evals/model-pricing.test.ts
index 54c3a27c5d..b97495eb2e 100644
--- a/packages/paperclip-runner/src/evals/model-pricing.test.ts
+++ b/packages/paperclip-runner/src/evals/model-pricing.test.ts
@@ -29,4 +29,21 @@ describe("model pricing", () => {
}),
).toThrow("model pricing unavailable");
});
+
+ it.each([
+ ["openrouter/anthropic/claude-sonnet-5", 2, 0.2, 10],
+ ["openrouter/qwen/qwen3.8-max-0902", 2, 0.25, 6],
+ ["openrouter/google/gemini-3.8-flash", 0.75, 0.075, 3.75],
+ ["openrouter/z-ai/glm-5.3", 1.4, 0.14, 4.4],
+ ["openrouter/deepseek/deepseek-v4-flash-0731", 0.065, 0.016, 0.18],
+ ["openrouter/openai/gpt-6-astra", 10, 1, 50],
+ ])("pins the OpenRouter breadth price for %s", (model, input, cachedInput, output) => {
+ expect(
+ estimateModelCostNanodollars(model, {
+ inputTokens: 1_000,
+ cachedInputTokens: 400,
+ outputTokens: 100,
+ }).ratesUsdPerMillionTokens,
+ ).toEqual({ input, cachedInput, output });
+ });
});
diff --git a/packages/paperclip-runner/src/evals/model-pricing.ts b/packages/paperclip-runner/src/evals/model-pricing.ts
index 99cde385ed..8c374002a6 100644
--- a/packages/paperclip-runner/src/evals/model-pricing.ts
+++ b/packages/paperclip-runner/src/evals/model-pricing.ts
@@ -1,4 +1,4 @@
-export const MODEL_PRICING_VERSION = "provider-list-prices-2026-08-21" as const;
+export const MODEL_PRICING_VERSION = "provider-list-prices-2026-09-05" as const;
interface TokenRatesUsdPerMillion {
input: number;
@@ -18,6 +18,13 @@ const RATES: Readonly<Record<string, TokenRatesUsdPerMillion>> = Object.freeze({
"claude-sonnet-5": { input: 3, cachedInput: 0.3, output: 15 },
// Amazon Bedrock global cross-region list price for Claude Sonnet 4.6.
"global.anthropic.claude-sonnet-4-6": { input: 3, cachedInput: 0.3, output: 15 },
+ // OpenRouter list-price snapshot used by the qualified OpenCode breadth lane.
+ "openrouter/anthropic/claude-sonnet-5": { input: 2, cachedInput: 0.2, output: 10 },
+ "openrouter/qwen/qwen3.8-max-0902": { input: 2, cachedInput: 0.25, output: 6 },
+ "openrouter/google/gemini-3.8-flash": { input: 0.75, cachedInput: 0.075, output: 3.75 },
+ "openrouter/z-ai/glm-5.3": { input: 1.4, cachedInput: 0.14, output: 4.4 },
+ "openrouter/deepseek/deepseek-v4-flash-0731": { input: 0.065, cachedInput: 0.016, output: 0.18 },
+ "openrouter/openai/gpt-6-astra": { input: 10, cachedInput: 1, output: 50 },
});
export interface EstimatedModelCost {
diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts
index 9ed9cbaf04..b569868318 100644
--- a/packages/paperclip-runner/src/live/live-session.test.ts
+++ b/packages/paperclip-runner/src/live/live-session.test.ts
@@ -87,6 +87,10 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
if (method === "initialize") return { user: { sessionId: this.state.providerSessionId } };
if (method === "thread/start") {
expect(Array.isArray(params.dynamicTools)).toBe(true);
+ expect(params.completionContract).toEqual({
+ revision: "paperclip-capability-live-v1",
+ criterionIds: ["objective"],
+ });
return {
model: "gpt-eval-test",
modelProvider: "openai",
@@ -633,10 +637,12 @@ describe("Capability live runnerd and Codex session", () => {
it("persists the selected provider and passes it to the live runner transport", async () => {
const state = providerState();
const providers: Array<string | undefined> = [];
+ const permissionModes: Array<string | undefined> = [];
const baseFactory = fakeTransportFactory(state);
const service = new CapabilityLiveSessionService({
transportFactory: (options) => {
providers.push(options.provider);
+ permissionModes.push(options.opencodePermissionMode);
return baseFactory(options);
},
});
@@ -646,6 +652,7 @@ describe("Capability live runnerd and Codex session", () => {
});
expect(providers).toEqual(["opencode"]);
+ expect(permissionModes).toEqual(["deny"]);
expect(session.snapshot().config).toMatchObject({
provider: "opencode",
driver: "opencode_server",
@@ -655,6 +662,45 @@ describe("Capability live runnerd and Codex session", () => {
await service.shutdown(session.id);
});
+ it("attributes Claude Managed sessions to the pinned immutable Agent version", async () => {
+ const state = providerState();
+ const managedProfiles: Array<Record<string, unknown> | undefined> = [];
+ const baseFactory = fakeTransportFactory(state);
+ const service = new CapabilityLiveSessionService({
+ transportFactory: (options) => {
+ managedProfiles.push(options.managedProfile);
+ return baseFactory(options);
+ },
+ });
+ const session = await service.create({
+ provider: "claude_managed",
+ requestedModel: "claude-sonnet-5",
+ managedProfile: {
+ profileId: "managed-profile-1",
+ anthropicAgentId: "agent-1",
+ agentVersion: "17",
+ environmentId: "environment-1",
+ betaVersion: "managed-agents-2026-04-01",
+ maxSessionListCostUsd: 0.5,
+ },
+ });
+
+ expect(managedProfiles).toEqual([
+ expect.objectContaining({
+ agentVersion: "17",
+ betaVersion: "managed-agents-2026-04-01",
+ model: "claude-sonnet-5",
+ }),
+ ]);
+ expect(session.snapshot().config).toMatchObject({
+ provider: "claude_managed",
+ driver: "claude_managed_agents_api",
+ providerVersion: "17",
+ requestedModel: "claude-sonnet-5",
+ });
+ await service.shutdown(session.id);
+ });
+
it("opens lazy sessions with core and discovery gateways instead of optional schemas", async () => {
const state = providerState();
const claim = "discovery:agents:read";
@@ -673,6 +719,74 @@ describe("Capability live runnerd and Codex session", () => {
await service.shutdown(session.id);
});
+ it("keeps discovered invocations correlated to the provider gateway call", async () => {
+ const state = providerState();
+ const claim = "discovery:agents:read";
+ const service = new CapabilityLiveSessionService({
+ transportFactory: fakeTransportFactory(state),
+ });
+ const session = await service.create({
+ runId: "run-live-lazy-invoke",
+ sessionId: "session-live-lazy-invoke",
+ toolExposure: "lazy",
+ capabilities: [claim],
+ explicitClaims: [claim],
+ scenario: { id: "lazy-invoke", claims: [claim] },
+ });
+ let gatewayResult: Record<string, unknown> | null = null;
+ state.onTurnStart = async () => {
+ const transport = state.transports[0]!;
+ const turnId = [...state.turns.keys()].at(-1)!;
+ const discovery = await transport.invokeServerRequest({
+ id: "request-discover",
+ method: "item/tool/call",
+ params: {
+ threadId: state.threadId,
+ turnId,
+ callId: "call-discover",
+ tool: "discover_capabilities",
+ arguments: {
+ query: "list company agents",
+ namespace: "discovery",
+ limit: 10,
+ },
+ },
+ });
+ expect(discovery.success).toBe(true);
+ const invoked = await transport.invokeServerRequest({
+ id: "request-invoke",
+ method: "item/tool/call",
+ params: {
+ threadId: state.threadId,
+ turnId,
+ callId: "call-invoke",
+ tool: "invoke_discovered_capability",
+ arguments: { operationId: "list_agents", input: {} },
+ },
+ });
+ gatewayResult = JSON.parse(
+ String(
+ (invoked.contentItems as Array<Record<string, unknown>>)[0]?.text,
+ ),
+ ) as Record<string, unknown>;
+ };
+
+ await session.sendMessage("Exercise the lazy discovery gateway.");
+
+ expect(gatewayResult).toMatchObject({
+ callId: "call-invoke",
+ operationId: "invoke_discovered_capability",
+ ok: true,
+ });
+ expect(
+ session
+ .snapshot()
+ .evidence.filter((entry) => entry.kind === "tool_call")
+ .map((entry) => entry.data.operationId),
+ ).toContain("list_agents");
+ await service.shutdown(session.id);
+ });
+
it("returns typed mock results to the same multi-turn Codex thread without Paperclip network calls", async () => {
const state = providerState();
const fetchSpy = vi.spyOn(globalThis, "fetch");
diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts
index dfd5cacefe..4f7b8c0f6b 100644
--- a/packages/paperclip-runner/src/live/live-session.ts
+++ b/packages/paperclip-runner/src/live/live-session.ts
@@ -54,6 +54,10 @@ import {
} from "./workspace-file-reference.js";
const LIVE_SESSION_SCHEMA = "paperclip.capability.live-session.v1" as const;
+const LIVE_COMPLETION_CONTRACT = Object.freeze({
+ revision: "paperclip-capability-live-v1",
+ criterionIds: ["objective"],
+});
const LIVE_BASE_INSTRUCTIONS = [
"You are operating one mock Paperclip issue through typed semantic tools.",
"Use only the tools exposed in this thread; never call a Paperclip REST API.",
@@ -922,7 +926,7 @@ export class CapabilityLiveSessionService {
providerVersion: input.provider === "opencode"
? "1.18.17"
: input.provider === "claude_managed"
- ? input.managedProfile!.betaVersion
+ ? input.managedProfile!.agentVersion
: input.provider === "aws_agentcore"
? input.agentCoreProfile!.qualificationRevision
: input.provider === "acpx" ? acpxProfile!.acpxVersion : null,
@@ -2269,6 +2273,15 @@ export class CapabilityLiveSession {
const transportBundle = this.#transportFactory({
...this.#transportOptions,
provider,
+ ...(provider === "opencode"
+ ? {
+ // Capability-live sessions expose only governed semantic tools and
+ // use approvalPolicy=never. Keep OpenCode's ambient shell/file
+ // tools fail-closed instead of brokering broader permissions.
+ opencodePermissionMode:
+ this.#transportOptions.opencodePermissionMode ?? "deny",
+ }
+ : {}),
...(provider === "acpx" && this.#config.acpxAgent ? {
acpxAgent: this.#config.acpxAgent,
} : {}),
@@ -2395,6 +2408,7 @@ export class CapabilityLiveSession {
runtimeWorkspaceRoots: [this.#config.workingDirectory],
approvalPolicy: "never",
baseInstructions: LIVE_BASE_INSTRUCTIONS,
+ completionContract: LIVE_COMPLETION_CONTRACT,
dynamicTools: [
...tools.map(dynamicToolSpec),
...((this.#config.toolExposure ?? "eager") === "lazy" ? discoveryToolSpecs() : []),
@@ -2554,7 +2568,15 @@ export class CapabilityLiveSession {
this.#recordTerminalFact(this.#durableReplayTurnId(turnId), "completed");
}
await this.#persist();
- return this.#codexToolResponse(result);
+ // The durable provider bridge correlates the outer semantic result with
+ // the exact dynamic tool Codex called. Keep the dispatched operation in
+ // evidence, but preserve the discovery gateway identity on the response
+ // envelope returned to runnerd.
+ const providerResult =
+ operationId === INVOKE_DISCOVERED_TOOL
+ ? { ...result, operationId, callId }
+ : result;
+ return this.#codexToolResponse(providerResult);
}
#isDurableTerminalReplay(turnId: string, input: unknown): boolean {
@@ -2579,7 +2601,7 @@ export class CapabilityLiveSession {
return `${interruptedTurnId}:durable-duplicate-replay`;
}
- #codexToolResponse(result: CapabilitySemanticToolResult): Record<string, unknown> {
+ #codexToolResponse(result: { readonly ok: boolean }): Record<string, unknown> {
return {
success: result.ok,
contentItems: [{
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
index b8ffc2d367..08815660d1 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
@@ -59,6 +59,11 @@ import {
withCodexCollaborationRuntimeInstructions,
} from "./runnerd-codex-transport.js";
+it("launches runnerd with its production durable outbox limits", () => {
+ expect(runnerdLaunchProfileInternals.maxOutboxBytes).toBe(16 * 1024 * 1024);
+ expect(runnerdLaunchProfileInternals.p0ReserveBytes).toBe(1024 * 1024);
+});
+
it("replays the durable run attachment outcome and latest provider identity", () => {
expect(
runnerdRecoveryInternals.recoveredRunAttachment({
@@ -1421,6 +1426,75 @@ it("runs the lab provider boundary through authenticated durable PRP", async ()
});
}, 30_000);
+it("continues rehydrating events after the committed-event window slides", async () => {
+ const stateDirectory = await mkdtemp(
+ join(tmpdir(), "runnerd-sliding-event-window-"),
+ );
+ const bundle = createCapabilityRunnerdCodexTransport({
+ runnerBinary: defaultCapabilityRunnerdBinary(),
+ codexCommand: fakeCodex,
+ codexArgs: fakeCodexArgs(stateDirectory, "--split-event-burst"),
+ stateDirectory,
+ lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 },
+ });
+ bundle.transport.setServerRequestHandler(async () => ({
+ success: true,
+ contentItems: [
+ {
+ type: "inputText",
+ text: JSON.stringify({ ok: true, result: { task: { id: "task-1" } } }),
+ },
+ ],
+ }));
+ try {
+ await bundle.transport.request("initialize", {});
+ await bundle.transport.request("thread/start", {
+ cwd: tmpdir(),
+ dynamicTools: [
+ {
+ name: "get_task_context",
+ description: "Read the active task.",
+ inputSchema: {
+ type: "object",
+ properties: {},
+ additionalProperties: false,
+ },
+ },
+ ],
+ });
+ await bundle.transport.request("turn/start", {
+ input: [{ type: "text", text: "Emit a split event burst." }],
+ });
+ const notifications = bundle.transport
+ .notifications()
+ [Symbol.asyncIterator]();
+ const methods: string[] = [];
+ const deadline = Date.now() + 20_000;
+ while (Date.now() < deadline) {
+ const next = await Promise.race([
+ notifications.next(),
+ new Promise<never>((_, reject) =>
+ setTimeout(
+ () =>
+ reject(new Error("sliding event window notification timeout")),
+ 10_000,
+ ),
+ ),
+ ]);
+ if (!next.value) break;
+ methods.push(next.value.method);
+ if (next.value.method === "turn/completed") break;
+ }
+ expect(
+ methods.filter((method) => method === "item/agentMessage/delta"),
+ ).toHaveLength(144);
+ expect(methods).toContain("turn/completed");
+ } finally {
+ await bundle.transport.close();
+ await rm(stateDirectory, { recursive: true, force: true });
+ }
+}, 30_000);
+
it("binds an immediately failed durable turn before exposing its terminal", async () => {
const stateDirectory = await mkdtemp(
join(tmpdir(), "runnerd-fast-terminal-"),
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
index fe384d7336..9698fb766a 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
@@ -76,6 +76,8 @@ const MAX_NOTIFICATION_COUNT = 2_048;
const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024;
const RUNNER_CLIENT_VERSION = "0.3.0";
const RUNNER_BOOTSTRAP_TICKET_TTL_MS = 60_000;
+const RUNNERD_MAX_OUTBOX_BYTES = 16 * 1024 * 1024;
+const RUNNERD_P0_RESERVE_BYTES = 1024 * 1024;
function readLocalProcessStartedAt(pid: number): string | null {
if (!Number.isInteger(pid) || pid <= 0) return null;
@@ -1941,7 +1943,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
#handle: RunnerProcessHandle | null = null;
#adoptedRunnerMonitor: NodeJS.Timeout | null = null;
#pump: NodeJS.Timeout | null = null;
- #eventIndex = 0;
+ #eventSourceSeq = 0;
+ #deferredTurnStartEvents: DurableRecoveryCommittedEvent[] = [];
#threadId = "";
#sessionId: string | null = null;
#providerIdentity: Record<string, unknown> | null = null;
@@ -2293,7 +2296,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
const previousRelease = this.#controlPlaneRelease;
core.rotateRunIdentity(desired);
- this.#eventIndex = 0;
+ this.#eventSourceSeq = 0;
+ this.#deferredTurnStartEvents = [];
this.#durableTurnId = desired.turnId;
this.#controlPlaneRelease = registration?.release ?? null;
let previousReleased = false;
@@ -3092,8 +3096,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.options.runnerStateDirectory ?? resolve(this.#root, "runner"),
identity,
ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS),
- maxOutboxBytes: 256 * 1024,
- p0ReserveBytes: 64 * 1024,
+ maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,
+ p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,
maxRuntimeMs: 60 * 60 * 1_000,
reconnectGraceMs: this.options.runnerReconnectGraceMs,
lifecyclePolicy: this.options.lifecyclePolicy,
@@ -3432,10 +3436,11 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
// If attachment completed, replay only its latest identity event into the
// transport's in-memory evidence; session events are consumed internally
// and are not duplicated onto the provider notification stream.
- this.#eventIndex =
+ this.#eventSourceSeq =
runAttachment !== null && runAttachment.providerIdentityEventIndex >= 0
- ? runAttachment.providerIdentityEventIndex
- : committedEvents.length;
+ ? committedEvents[runAttachment.providerIdentityEventIndex]!.sourceSeq -
+ 1
+ : core.store.state.ackedSourceSeq;
const adoptedProviderIdentityIndex =
latestProviderIdentityEventIndex(committedEvents);
if (
@@ -3494,8 +3499,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.options.runnerStateDirectory ?? resolve(this.#root, "runner"),
identity,
ticket: core.issueBootstrapTicket(RUNNER_BOOTSTRAP_TICKET_TTL_MS),
- maxOutboxBytes: 256 * 1024,
- p0ReserveBytes: 64 * 1024,
+ maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,
+ p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,
maxRuntimeMs: 60 * 60 * 1_000,
reconnectGraceMs: this.options.runnerReconnectGraceMs,
lifecyclePolicy: this.options.lifecyclePolicy,
@@ -3781,8 +3786,21 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
#pumpEvents(): void {
this.#flushPendingTraceRehydrations();
const events = this.#core?.store.state.committedEvents ?? [];
- while (this.#eventIndex < events.length) {
- const event = events[this.#eventIndex]!;
+ for (;;) {
+ const deferredEvent =
+ !this.#turnStartResponsePending || this.#expectedProviderTurnId !== null
+ ? this.#deferredTurnStartEvents[0]
+ : undefined;
+ const event =
+ deferredEvent ??
+ events.find((candidate) => candidate.sourceSeq > this.#eventSourceSeq);
+ if (event === undefined) return;
+ const fromDeferredQueue = deferredEvent !== undefined;
+ if (!fromDeferredQueue && event.sourceSeq !== this.#eventSourceSeq + 1) {
+ throw new Error(
+ `PRP provider event window advanced past source sequence ${this.#eventSourceSeq + 1}`,
+ );
+ }
const eventPayload = record(event.envelope.payload).payload;
const turnStartWhileCommandResultPending =
this.#turnStartResponsePending &&
@@ -3793,9 +3811,25 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
(notification) => notification.method === "turn/started",
)));
// The durable command result is the only correlation authority for a
- // provider-assigned turn id. Leave an early start at the cursor until
- // that exact expected identity is installed.
- if (turnStartWhileCommandResultPending) return;
+ // provider-assigned turn id. Copy an early start and its following
+ // events out of the control plane's sliding window until that exact
+ // expected identity is installed.
+ if (
+ !fromDeferredQueue &&
+ (turnStartWhileCommandResultPending ||
+ (this.#turnStartResponsePending &&
+ this.#expectedProviderTurnId === null &&
+ this.#deferredTurnStartEvents.length > 0))
+ ) {
+ if (this.#deferredTurnStartEvents.length >= 4_096) {
+ throw new Error(
+ "turn/start produced too many events before its durable command result",
+ );
+ }
+ this.#eventSourceSeq = event.sourceSeq;
+ this.#deferredTurnStartEvents.push(structuredClone(event));
+ continue;
+ }
const terminalWhileTurnStartPending =
this.#turnStartResponsePending &&
([
@@ -3808,8 +3842,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
unwrapRunnerdProviderNotifications(eventPayload).some(
(notification) => notification.method === "turn/completed",
)));
- if (terminalWhileTurnStartPending) return;
- this.#eventIndex += 1;
+ if (terminalWhileTurnStartPending) {
+ if (!fromDeferredQueue) {
+ this.#eventSourceSeq = event.sourceSeq;
+ this.#deferredTurnStartEvents.push(structuredClone(event));
+ }
+ return;
+ }
+ // The control plane retains a sliding committed-event window. Track its
+ // durable protocol cursor rather than an array index: once that array is
+ // full, new events replace its prefix without increasing its length.
+ if (fromDeferredQueue) this.#deferredTurnStartEvents.shift();
+ else this.#eventSourceSeq = event.sourceSeq;
if (
event.eventType === "harness.ready" ||
event.eventType === "session.started" ||
@@ -4535,6 +4579,8 @@ export const runnerdLaunchProfileInternals = Object.freeze({
acpxProviderPackageAuthority,
acpxRunnerLaunchProfile,
resolveBuildOwnedCliArtifact,
+ maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,
+ p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,
});
export const runnerdRecoveryInternals = Object.freeze({
diff --git a/packages/paperclip-runner/src/semantic-tools/discovery.ts b/packages/paperclip-runner/src/semantic-tools/discovery.ts
index 0ff30e281f..c6adf41dcd 100644
--- a/packages/paperclip-runner/src/semantic-tools/discovery.ts
+++ b/packages/paperclip-runner/src/semantic-tools/discovery.ts
@@ -17,6 +17,8 @@ export interface CapabilityDiscoveryResult {
readonly truncated: boolean;
}
+const MAX_DISCOVERY_RESULTS = 10;
+
const NAMESPACE: Readonly<Record<CapabilitySemanticOperationId, string>> = Object.freeze({
get_task_context: "active_task", get_task_history: "active_task",
list_documents: "documents", read_document: "documents", list_document_revisions: "documents",
@@ -47,7 +49,7 @@ export const CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS = Object.freeze([{
type: "object", properties: {
query: { type: "string", minLength: 1, maxLength: 500 },
namespace: { type: "string" },
- limit: { type: "integer", minimum: 1, maximum: 8 },
+ limit: { type: "integer", minimum: 1, maximum: MAX_DISCOVERY_RESULTS },
}, required: ["query"], additionalProperties: false,
},
outputSchema: {
@@ -90,7 +92,10 @@ export function discoverCapabilityDefinitions(
): CapabilityDiscoveryResult {
const normalized = query.trim().toLowerCase();
if (normalized.length === 0) throw new Error("discovery_query_empty");
- const limit = Math.max(1, Math.min(options.limit ?? 5, 8));
+ const limit = Math.max(
+ 1,
+ Math.min(options.limit ?? 5, MAX_DISCOVERY_RESULTS),
+ );
const tokens = normalized.split(/[^a-z0-9]+/).filter((token) => token.length > 1);
const permitted = CAPABILITY_SEMANTIC_TOOL_CATALOG
.filter((descriptor) => descriptor.exposure === "optional")
diff --git a/packages/paperclip-runner/src/semantic-tools/policy.ts b/packages/paperclip-runner/src/semantic-tools/policy.ts
index 0b077eea8d..6bcff09bec 100644
--- a/packages/paperclip-runner/src/semantic-tools/policy.ts
+++ b/packages/paperclip-runner/src/semantic-tools/policy.ts
@@ -8,7 +8,7 @@ import type {
CapabilitySemanticToolDescriptor,
} from "./types.js";
-const READ_OPERATIONS = new Set<CapabilitySemanticOperationId>([
+export const CAPABILITY_SEMANTIC_READ_OPERATION_IDS = [
"get_task_context",
"get_task_history",
"list_documents",
@@ -21,7 +21,15 @@ const READ_OPERATIONS = new Set<CapabilitySemanticOperationId>([
"get_approval",
"get_approval_context",
"get_workspace_runtime",
-]);
+] as const satisfies readonly CapabilitySemanticOperationId[];
+
+const READ_OPERATIONS = new Set<CapabilitySemanticOperationId>(
+ CAPABILITY_SEMANTIC_READ_OPERATION_IDS,
+);
+
+export function isCapabilitySemanticReadOperation(operationId: string): boolean {
+ return READ_OPERATIONS.has(operationId as CapabilitySemanticOperationId);
+}
export const DEFAULT_CAPABILITY_SCENARIO_POLICY: CapabilitySemanticScenarioPolicy = {
id: "capability-default",
diff --git a/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts b/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts
index 1d0fa1cb8c..5d7ca42dc1 100644
--- a/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts
+++ b/packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
import type { CapabilityFixtureSeed } from "../mock-core/capability-control-plane-types.js";
import { CapabilityMockControlPlaneAdapter } from "../mock-core/capability-mock-control-plane-adapter.js";
import { CAPABILITY_SEMANTIC_TOOL_CATALOG } from "./catalog.js";
+import { CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS } from "./discovery.js";
import { CapabilitySemanticDispatcher } from "./dispatcher.js";
import { createCapabilityProviderNeutralBinding } from "./provider-neutral.js";
@@ -42,6 +43,13 @@ async function running(
}
describe("Capability semantic catalog and authorization", () => {
+ it("accepts the conventional ten-result capability discovery limit", () => {
+ expect(
+ CAPABILITY_DISCOVERY_GATEWAY_DEFINITIONS[0].inputSchema.properties.limit
+ .maximum,
+ ).toBe(10);
+ });
+
it("publishes a stable narrow catalog without credentials or control-plane-owned tools", () => {
const names = CAPABILITY_SEMANTIC_TOOL_CATALOG.map((tool) => tool.operationId);
expect(new Set(names).size).toBe(names.length);
diff --git a/packages/paperclip-runner/vite.issue-thread.config.ts b/packages/paperclip-runner/vite.issue-thread.config.ts
index 17b33a483d..fe235b3172 100644
--- a/packages/paperclip-runner/vite.issue-thread.config.ts
+++ b/packages/paperclip-runner/vite.issue-thread.config.ts
@@ -14,7 +14,6 @@ export default defineConfig({
plugins: [react(), capabilityIssueThreadServerPlugin()],
server: { host: "127.0.0.1" },
preview: { host: "127.0.0.1" },
- optimizeDeps: { esbuildOptions: { target: "esnext" } },
build: {
outDir: resolve(packageRoot, "dist-issue-thread"),
emptyOutDir: true,
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index d98ccbfcdd..aaf3179215 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -248,6 +248,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
readWorkflow("runner-chaos-evals.yml"),
readWorkflow("runner-full-stack-e2e.yml"),
readWorkflow("e2e.yml"),
+ readWorkflow("runner-protocol-live-evals.yml"),
];
for (const workflow of actionPinWorkflows) {
@@ -286,6 +287,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
"e2e.yml",
"runner-full-stack-e2e.yml",
"runner-live-evals.yml",
+ "runner-protocol-live-evals.yml",
];
const paidWorkflowNameSet = new Set(paidWorkflowNames);
const providerSecretReference =
@@ -322,7 +324,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
assert.match(block, /\n environment:\n name: runner-e2e-paid\n/);
assert.match(
block,
- /\n steps:\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/,
+ /\n steps:(?: &[A-Za-z0-9_-]+)?\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/,
`${name} must reauthorize as the first provider-job step`,
);
const reauthorize = block.indexOf(" - name: Reauthorize");
@@ -357,7 +359,11 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
);
assert.doesNotMatch(historyPublisher, /^\s+cache: pnpm$/m);
- for (const name of ["runner-full-stack-e2e.yml", "runner-live-evals.yml"]) {
+ for (const name of [
+ "runner-full-stack-e2e.yml",
+ "runner-live-evals.yml",
+ "runner-protocol-live-evals.yml",
+ ]) {
const workflow = readWorkflow(name);
const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(
(match) => match[1],
diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts
index b875add0d3..03a68f6af8 100644
--- a/tests/runner-e2e/workflow-security.test.ts
+++ b/tests/runner-e2e/workflow-security.test.ts
@@ -121,15 +121,18 @@ describe("public repository paid workflow security", () => {
it("gates every provider-secret job with stable actor IDs", async () => {
const workflows = await Promise.all(
- ["runner-full-stack-e2e.yml", "runner-live-evals.yml", "e2e.yml"].map(
- async (name) => ({
- name,
- contents: await readFile(
- path.join(repositoryRoot, ".github/workflows", name),
- "utf8",
- ),
- }),
- ),
+ [
+ "runner-full-stack-e2e.yml",
+ "runner-live-evals.yml",
+ "runner-protocol-live-evals.yml",
+ "e2e.yml",
+ ].map(async (name) => ({
+ name,
+ contents: await readFile(
+ path.join(repositoryRoot, ".github/workflows", name),
+ "utf8",
+ ),
+ })),
);
for (const { name, contents } of workflows) {
@@ -482,6 +485,7 @@ describe("public repository paid workflow security", () => {
"e2e.yml",
"runner-full-stack-e2e.yml",
"runner-live-evals.yml",
+ "runner-protocol-live-evals.yml",
]);
const names = (await readdir(workflowDirectory)).filter((name) =>
/\.ya?ml$/.test(name),
@@ -508,7 +512,11 @@ describe("public repository paid workflow security", () => {
it("runs paid scheduled campaigns only on Sundays", async () => {
const workflows = await Promise.all(
- ["runner-full-stack-e2e.yml", "runner-live-evals.yml"].map((name) =>
+ [
+ "runner-full-stack-e2e.yml",
+ "runner-live-evals.yml",
+ "runner-protocol-live-evals.yml",
+ ].map((name) =>
readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),
),
);