Skip to content

The open-source app everyone uses to manage agents at work

Operationalize Product E2E accounting, tracing, and scheduling (#17009)

Dotta3bd709df8b83
diff --git a/.github/infra/runner-e2e-private-history.yml b/.github/infra/runner-e2e-private-history.yml
new file mode 100644
index 0000000000..0d837d2479
--- /dev/null
+++ b/.github/infra/runner-e2e-private-history.yml
@@ -0,0 +1,117 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: Private bounded Paperclip eval evidence and reports; no CDN reader
+Parameters:
+ BucketName:
+ Type: String
+ GitHubRepository:
+ Type: String
+ Default: paperclipai/paperclip
+ GitHubOidcProviderArn:
+ Type: String
+Resources:
+ PrivateHistoryBucket:
+ Type: AWS::S3::Bucket
+ DeletionPolicy: Retain
+ UpdateReplacePolicy: Retain
+ Properties:
+ BucketName: !Ref BucketName
+ PublicAccessBlockConfiguration:
+ BlockPublicAcls: true
+ IgnorePublicAcls: true
+ BlockPublicPolicy: true
+ RestrictPublicBuckets: true
+ OwnershipControls:
+ Rules:
+ - ObjectOwnership: BucketOwnerEnforced
+ VersioningConfiguration:
+ Status: Enabled
+ BucketEncryption:
+ ServerSideEncryptionConfiguration:
+ - ServerSideEncryptionByDefault:
+ SSEAlgorithm: AES256
+ LifecycleConfiguration:
+ Rules:
+ - Id: BoundedEvalRetentionV1
+ Status: Enabled
+ Prefix: runner-e2e-private/bounded-v1/
+ ExpirationInDays: 30
+ NoncurrentVersionExpiration:
+ NoncurrentDays: 1
+ AbortIncompleteMultipartUpload:
+ DaysAfterInitiation: 1
+ - Id: BoundedEvalDeleteMarkersV1
+ Status: Enabled
+ Prefix: runner-e2e-private/bounded-v1/
+ ExpiredObjectDeleteMarker: true
+ PrivateHistoryBucketPolicy:
+ Type: AWS::S3::BucketPolicy
+ Properties:
+ Bucket: !Ref PrivateHistoryBucket
+ PolicyDocument:
+ Version: '2012-10-17'
+ Statement:
+ - Sid: DenyInsecureTransport
+ Effect: Deny
+ Principal: '*'
+ Action: s3:*
+ Resource:
+ - !GetAtt PrivateHistoryBucket.Arn
+ - !Sub '${PrivateHistoryBucket.Arn}/*'
+ Condition:
+ Bool:
+ aws:SecureTransport: 'false'
+ - Sid: DenyUnconditionalWrites
+ Effect: Deny
+ Principal: '*'
+ Action: s3:PutObject
+ Resource: !Sub '${PrivateHistoryBucket.Arn}/runner-e2e-private/bounded-v1/*'
+ Condition:
+ 'Null':
+ s3:if-none-match: 'true'
+ PrivateHistoryRole:
+ Type: AWS::IAM::Role
+ Properties:
+ RoleName: paperclip-runner-e2e-private-history-github
+ MaxSessionDuration: 3600
+ AssumeRolePolicyDocument:
+ Version: '2012-10-17'
+ Statement:
+ - Effect: Allow
+ Principal:
+ Federated: !Ref GitHubOidcProviderArn
+ Action: sts:AssumeRoleWithWebIdentity
+ Condition:
+ StringEquals:
+ token.actions.githubusercontent.com:aud: sts.amazonaws.com
+ token.actions.githubusercontent.com:sub:
+ - !Sub 'repo:${GitHubRepository}:environment:runner-e2e-private-paid'
+ - !Sub 'repo:${GitHubRepository}:environment:runner-e2e-private-history'
+ Policies:
+ - PolicyName: private-eval-artifacts
+ PolicyDocument:
+ Version: '2012-10-17'
+ Statement:
+ - Sid: VerifyPrivateRetention
+ Effect: Allow
+ Action:
+ - s3:GetBucketVersioning
+ - s3:GetLifecycleConfiguration
+ - s3:GetBucketObjectLockConfiguration
+ - s3:GetReplicationConfiguration
+ - s3:GetBucketPublicAccessBlock
+ - s3:GetBucketOwnershipControls
+ - s3:GetBucketPolicy
+ Resource: !GetAtt PrivateHistoryBucket.Arn
+ - Sid: RetainAndReadPrivateArtifacts
+ Effect: Allow
+ Action:
+ - s3:GetObject
+ - s3:PutObject
+ Resource: !Sub '${PrivateHistoryBucket.Arn}/runner-e2e-private/bounded-v1/*'
+Outputs:
+ Bucket:
+ Value: !Ref PrivateHistoryBucket
+ RoleArn:
+ Value: !GetAtt PrivateHistoryRole.Arn
+ Region:
+ Value: !Ref AWS::Region
diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml
index c15a34085f..a04db993e7 100644
--- a/.github/workflows/pr-trusted.yml
+++ b/.github/workflows/pr-trusted.yml
@@ -119,7 +119,7 @@ jobs:
run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs
- name: Test release verify workflow wiring
- run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs
+ run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs
- name: Test standalone package build concurrency
run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs
diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml
index 65eebf32ed..99cdaf5232 100644
--- a/.github/workflows/runner-full-stack-e2e.yml
+++ b/.github/workflows/runner-full-stack-e2e.yml
@@ -1,10 +1,17 @@
name: Runner Full-Stack E2E
+run-name: Runner Full-Stack E2E / ${{ github.event.schedule || inputs.cadence || 'manual' }}
on:
schedule:
- - cron: "47 8 * * 0"
+ - cron: "47 8 * * *"
+ - cron: "47 9 * * 0"
workflow_dispatch:
inputs:
+ cadence:
+ description: "Bounded daily ($20) or weekly ($100) selection; manual keeps explicit selectors"
+ type: choice
+ options: [manual, daily, weekly]
+ default: manual
target_branch:
description: "Branch in paperclipai/paperclip to test; the trusted workflow still runs from master"
type: string
@@ -55,7 +62,7 @@ concurrency:
jobs:
authorize:
name: Authorize paid campaign
- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
@@ -115,14 +122,25 @@ jobs:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
TARGET_BRANCH: ${{ inputs.target_branch || github.event.repository.default_branch }}
+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ EVENT_NAME: ${{ github.event_name }}
+ EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}
run: |
set -euo pipefail
if [ -z "$TARGET_BRANCH" ] || [[ "$TARGET_BRANCH" == refs/* ]]; then
echo "target_branch must name a branch in this repository without a refs/ prefix." >&2
exit 1
fi
- encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')"
- target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)"
+ if [ "$EVENT_NAME" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then
+ if [ "$TARGET_BRANCH" != "$DEFAULT_BRANCH" ]; then
+ echo "Bounded campaigns require the workflow's default-branch commit." >&2
+ exit 1
+ fi
+ target_sha="$GITHUB_SHA"
+ else
+ encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')"
+ target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)"
+ fi
if ! [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "The requested repository branch did not resolve to a commit." >&2
exit 1
@@ -135,12 +153,23 @@ jobs:
id: runner
env:
AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}
+ EVENT_NAME: ${{ github.event_name }}
+ EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}
+ REPOSITORY_PRIVATE: ${{ github.event.repository.private }}
run: |
set -euo pipefail
github_runner='ubuntu-latest'
aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'
+ bounded=false
+ if [ "$EVENT_NAME" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then
+ if [ "$REPOSITORY_PRIVATE" != false ]; then
+ echo 'Bounded campaigns require a public repository and standard GitHub-hosted compute.' >&2
+ exit 1
+ fi
+ bounded=true
+ fi
- if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then
+ if [ "$AWS_PAID_RUNNER_ENABLED" = true ] && [ "$bounded" = false ]; then
{
echo "runner=$aws_runner"
echo "max_parallel_default=100"
@@ -155,7 +184,7 @@ jobs:
echo "max_parallel_limit=57"
echo "playwright_channel="
} >> "$GITHUB_OUTPUT"
- echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner'
+ echo '::notice title=Paid runner routing::Using a standard GitHub-hosted runner'
fi
target_lock:
@@ -200,6 +229,10 @@ jobs:
fi
echo "sha256=$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT"
+ - name: Bound scheduled lockfile upload
+ if: github.event_name == 'schedule' || (inputs.cadence != '' && inputs.cadence != 'manual')
+ run: node tests/runner-e2e/artifact-budget.mjs check lock pnpm-lock.yaml
+
- name: Upload resolved target lockfile
id: upload
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -212,13 +245,15 @@ jobs:
catalog:
name: Validate catalog and select cells
needs: [authorize, target_lock]
- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
outputs:
matrix: ${{ steps.catalog.outputs.matrix }}
+ operations_cadence: ${{ steps.catalog.outputs.operations_cadence }}
+ operating_overhead: ${{ steps.catalog.outputs.operating_overhead }}
needs_daytona: ${{ steps.catalog.outputs.needs_daytona }}
needs_runner_typescript: ${{ steps.catalog.outputs.needs_runner_typescript }}
needs_native_binaries: ${{ steps.catalog.outputs.needs_native_binaries }}
@@ -281,6 +316,13 @@ jobs:
id: catalog
env:
EVENT_NAME: ${{ github.event_name }}
+ EVAL_EVENT: ${{ github.event_name }}
+ EVAL_SCHEDULE: ${{ github.event.schedule }}
+ EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}
+ EVAL_OPERATIONS_ENABLED: ${{ vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED }}
+ EVAL_OPERATING_OVERHEAD_JSON: ${{ vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON }}
+ EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }}
+ TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
SELECT_ALL: ${{ inputs.all }}
SELECT_SUITE: ${{ inputs.suite }}
SELECT_GROUP: ${{ inputs.group }}
@@ -293,6 +335,31 @@ jobs:
REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}
run: |
set -euo pipefail
+ if [ "$EVAL_EVENT" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then
+ test "$TARGET_SHA" = "$GITHUB_SHA"
+ test -z "${SELECT_SUITE:-}${SELECT_GROUP:-}${SELECT_PROFILE:-}${SELECT_ENVIRONMENT:-}${SELECT_CASE:-}${SELECT_ID:-}"
+ catalog_json="$(node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-operations.ts matrix)"
+ cadence="$EVAL_CADENCE"
+ if [ "$EVAL_EVENT" = schedule ]; then
+ case "$EVAL_SCHEDULE" in
+ '47 8 * * *') cadence=daily ;;
+ '47 9 * * 0') cadence=weekly ;;
+ *) exit 1 ;;
+ esac
+ fi
+ {
+ echo "operations_cadence=$cadence"
+ echo "operating_overhead=$(jq -c '.overhead' <<< "$catalog_json")"
+ echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")"
+ echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")"
+ echo "needs_daytona=false"
+ echo "needs_runner_typescript=true"
+ echo "needs_native_binaries=true"
+ echo "needs_remote_provider_pack=false"
+ echo "max_parallel=1"
+ } >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
args=(--matrix-json)
add_values() {
local flag="$1"
@@ -509,7 +576,7 @@ jobs:
build_runner_artifacts:
name: Build reusable runner campaign artifacts
needs: [authorize, target_lock, catalog]
- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')
# Compile native binaries on the same reviewed image used to execute them,
# avoiding libc/architecture drift between GitHub-hosted and AWS lanes.
runs-on: ${{ needs.authorize.outputs.test_runner }}
@@ -618,6 +685,10 @@ jobs:
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
run: echo "name=runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
+ - name: Bound scheduled build upload
+ if: needs.catalog.outputs.operations_cadence != ''
+ run: node tests/runner-e2e/artifact-budget.mjs check build runner-e2e-build-bundle.tar.gz runner-e2e-build-bundle.tar.gz.sha256
+
- name: Upload immutable shared campaign outputs
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
@@ -633,7 +704,7 @@ jobs:
name: Build reusable remote provider pack
needs:
[authorize, target_lock, catalog, daytona_image, build_runner_artifacts]
- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
@@ -769,6 +840,7 @@ jobs:
test:
name: ${{ matrix.executionId }}
+ if: needs.catalog.outputs.operations_cadence == ''
needs:
[
authorize,
@@ -790,7 +862,7 @@ jobs:
fail-fast: false
max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }}
matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }}
- steps:
+ steps: &paid_test_steps
- name: Reauthorize paid execution before provider access
env:
GH_TOKEN: ${{ github.token }}
@@ -962,8 +1034,8 @@ jobs:
}
NODE
- - name: Install pinned legacy Claude CLI
- if: matrix.profileId == 'legacy-claude'
+ - name: Install pinned Claude CLI for setup verification
+ if: matrix.profileId == 'legacy-claude' || matrix.profileId == 'runner-acpx-claude'
run: |
npm install --global --omit=dev --ignore-scripts @anthropic-ai/[email protected]
launcher="$(npm root --global)/@anthropic-ai/claude-code/cli-wrapper.cjs"
@@ -1012,7 +1084,7 @@ jobs:
# This definition executes only from the authorized default-branch workflow.
# Provision host policy before credentials reach target-controlled tests.
- name: Provision Codex sandbox on the disposable trusted runner
- if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex' || matrix.profileId == 'runner-codex-subscription')
+ if: matrix.environmentId == 'local' && (needs.catalog.outputs.operations_cadence == 'weekly' || matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex' || matrix.profileId == 'runner-codex-subscription')
run: |
node --input-type=module <<'NODE'
import { execFileSync } from "node:child_process";
@@ -1052,7 +1124,60 @@ jobs:
timeout 120s docker pull "$oracle_image"
timeout 30s docker image inspect "$oracle_image" --format '{{.Id}}'
+ - name: Obtain scoped private history credentials
+ id: private_history_credentials
+ if: needs.catalog.outputs.operations_cadence != ''
+ uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
+ with:
+ role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
+ aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ output-env-credentials: false
+ output-credentials: true
+ disable-retry: true
+ action-timeout-s: 60
+
+ - name: Verify private evidence retention before paid execution
+ if: needs.catalog.outputs.operations_cadence != ''
+ env:
+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}
+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}
+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}
+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts verify evidence "$GITHUB_RUN_ID" bounded-runner-e2e-evidence
+
+ - name: Prepare bounded campaign before credentials
+ if: needs.catalog.outputs.operations_cadence != ''
+ env:
+ EVAL_EVENT: ${{ github.event_name }}
+ EVAL_SCHEDULE: ${{ github.event.schedule }}
+ EVAL_CADENCE: ${{ needs.catalog.outputs.operations_cadence }}
+ EVAL_OPERATIONS_ENABLED: ${{ vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED }}
+ EVAL_OPERATING_OVERHEAD_JSON: ${{ needs.catalog.outputs.operating_overhead }}
+ EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }}
+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: ${{ vars.RUNNER_E2E_BRAINTRUST_PROJECT_ID }}
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-operations.ts prepare
+
+ - name: Run bounded campaign and retain feedback review
+ if: needs.catalog.outputs.operations_cadence != ''
+ env:
+ BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: ${{ vars.RUNNER_E2E_BRAINTRUST_PROJECT_ID }}
+ PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
+ PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
+ PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}
+ run: |
+ set +e
+ args=(--max-parallel 1 --max-automatic-retries 0)
+ while IFS= read -r id; do args+=(--id "$id"); done < <(jq -r '.executionIds[]' "$PAPERCLIP_E2E_OPERATIONS_MANIFEST")
+ pnpm test:e2e:runner -- "${args[@]}"
+ run_status=$?
+ node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-finalization.ts
+ finalization_status=$?
+ test "$run_status" = 0 && test "$finalization_status" = 0
+
- name: Run paid cell
+ if: needs.catalog.outputs.operations_cadence == ''
env:
OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }}
ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }}
@@ -1063,13 +1188,19 @@ jobs:
PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }}
PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }}
+ PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}
run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}"
+ - name: Stage bounded campaign evidence
+ id: stage_evidence
+ if: always() && needs.catalog.outputs.operations_cadence != ''
+ run: node tests/runner-e2e/artifact-budget.mjs stage evidence tests/runner-e2e/results bounded-runner-e2e-evidence
+
- name: Upload access-controlled packaged cell evidence
- if: always()
+ if: always() && needs.catalog.outputs.operations_cadence == ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }}
@@ -1077,10 +1208,41 @@ jobs:
retention-days: 30
if-no-files-found: error
+ - name: Retain bounded campaign evidence in private AWS storage
+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_history_credentials.outcome == 'success'
+ env:
+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}
+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}
+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}
+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts upload evidence "$GITHUB_RUN_ID" bounded-runner-e2e-evidence
+
+ - name: Enforce complete bounded evidence retention
+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_evidence.outputs.complete != 'true'
+ run: exit 1
+
+ test_recurring:
+ name: ${{ matrix.executionId }}
+ if: needs.catalog.outputs.operations_cadence != '' && needs.authorize.outputs.target_sha == github.sha && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+ needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]
+ runs-on: ${{ needs.authorize.outputs.test_runner }}
+ timeout-minutes: ${{ matrix.timeoutMinutes }}
+ permissions:
+ contents: read
+ id-token: write
+ environment:
+ name: runner-e2e-private-paid
+ strategy:
+ fail-fast: false
+ max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }}
+ matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }}
+ steps: *paid_test_steps
+
report:
name: Merge and enforce campaign result
- if: always() && !cancelled() && needs.catalog.result == 'success'
- needs: [authorize, catalog, daytona_image, test]
+ if: always() && !cancelled() && needs.catalog.result == 'success' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+ needs: [authorize, catalog, daytona_image, test, test_recurring]
outputs:
history_source_ready: ${{ steps.history_source_ready.outputs.ready }}
runs-on: ubuntu-latest
@@ -1088,6 +1250,9 @@ jobs:
permissions:
actions: read
contents: read
+ id-token: write
+ environment:
+ name: runner-e2e-private-history
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
@@ -1107,15 +1272,8 @@ jobs:
with:
version: 9.15.4
- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
- with:
- node-version: 24
- cache: pnpm
-
- # Resolve from this trusted checkout only. Never use the target branch's
- # lockfile or install scripts in the publication job.
- - run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile
- - run: pnpm install --frozen-lockfile
+ - name: Install pinned reporting dependencies
+ run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-report-store"
- name: Resolve workflow job attempts
env:
@@ -1138,6 +1296,7 @@ jobs:
- name: Download cell evidence
id: download_evidence
+ if: needs.catalog.outputs.operations_cadence == ''
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
@@ -1146,15 +1305,37 @@ jobs:
merge-multiple: false
- name: Retry cell evidence download after transport failure
- if: steps.download_evidence.outcome == 'failure'
+ if: needs.catalog.outputs.operations_cadence == '' && steps.download_evidence.outcome == 'failure'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: runner-e2e-${{ github.run_id }}-*-*
path: downloaded-runner-e2e
merge-multiple: false
+ - name: Obtain scoped private history credentials
+ id: private_history_credentials
+ if: needs.catalog.outputs.operations_cadence != ''
+ uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
+ with:
+ role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
+ aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ output-env-credentials: false
+ output-credentials: true
+ disable-retry: true
+ action-timeout-s: 60
+
+ - name: Download private bounded campaign evidence
+ if: needs.catalog.outputs.operations_cadence != ''
+ env:
+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}
+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}
+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}
+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts download evidence "$GITHUB_RUN_ID" "downloaded-runner-e2e/runner-e2e-$GITHUB_RUN_ID-1-scheduled-campaign"
+
- name: Select latest workflow attempt per cell
- if: always()
+ if: always() && needs.catalog.outputs.operations_cadence == ''
env:
PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e
PAPERCLIP_RUNNER_E2E_SELECTED_ROOT: ${{ github.workspace }}/selected-runner-e2e
@@ -1164,6 +1345,25 @@ jobs:
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/select-rerun-artifacts.ts
+ - name: Preserve bounded campaign evidence and journal
+ if: always() && needs.catalog.outputs.operations_cadence != ''
+ env:
+ RUN_ID: ${{ github.run_id }}
+ run: |
+ set -euo pipefail
+ artifact="downloaded-runner-e2e/runner-e2e-$RUN_ID-1-scheduled-campaign"
+ directory="$GITHUB_WORKSPACE/selected-runner-e2e/runner-e2e-$RUN_ID-1-scheduled-campaign/scheduled"
+ echo "PAPERCLIP_E2E_OPERATIONS_LEDGER=$directory/ledger.jsonl" >> "$GITHUB_ENV"
+ test -d "$artifact"
+ mkdir -p selected-runner-e2e
+ cp -R "$artifact" selected-runner-e2e/
+ if [ -f "$directory/feedback-review.json" ]; then
+ echo "PAPERCLIP_E2E_FEEDBACK_REVIEW=$directory/feedback-review.json" >> "$GITHUB_ENV"
+ fi
+ if [ -f "$directory/experiment-receipt.json" ]; then
+ echo "PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT=$directory/experiment-receipt.json" >> "$GITHUB_ENV"
+ fi
+
- name: Collect blob reports
run: |
set -euo pipefail
@@ -1188,21 +1388,34 @@ jobs:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }}
- PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
+ PAPERCLIP_E2E_CAMPAIGN_ID: ${{ needs.catalog.outputs.operations_cadence != '' && format('gha-{0}-1-scheduled-campaign', github.run_id) || format('gha-{0}-{1}', github.run_id, github.run_attempt) }}
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}
- PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}
+ PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}
PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}
+ EVAL_OPERATIONS_CADENCE: ${{ needs.catalog.outputs.operations_cadence }}
run: |
set +e
- pnpm test:e2e:runner:report
+ if [ -n "$EVAL_OPERATIONS_CADENCE" ]; then
+ mkdir -p runner-e2e-merged-report
+ pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
+ else
+ pnpm test:e2e:runner:report
+ fi
report_status=$?
set -e
- cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY"
+ if [ -z "$EVAL_OPERATIONS_CADENCE" ]; then
+ cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY"
+ fi
exit "$report_status"
+ - name: Stage bounded merged report
+ id: stage_report
+ if: always() && needs.catalog.outputs.operations_cadence != ''
+ run: node tests/runner-e2e/artifact-budget.mjs stage report runner-e2e-merged-report bounded-runner-e2e-report
+
- name: Upload access-controlled merged report
- if: always()
+ if: always() && needs.catalog.outputs.operations_cadence == ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
@@ -1210,6 +1423,33 @@ jobs:
retention-days: 30
if-no-files-found: error
+ - name: Retain bounded report in private AWS storage
+ id: private_report
+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_history_credentials.outcome == 'success'
+ env:
+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}
+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}
+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}
+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
+ run: |
+ node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts upload report "$GITHUB_RUN_ID" bounded-runner-e2e-report
+ echo '{"version":1,"privateReportRetained":true}' > private-report-status.json
+ echo 'The recurring campaign report is retained in private AWS storage. See the eval operations runbook for authenticated retrieval.' >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Retain report availability for the schedule monitor
+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_report.outcome == 'success'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
+ path: private-report-status.json
+ retention-days: 30
+ if-no-files-found: error
+
+ - name: Enforce complete bounded report retention
+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_report.outputs.complete != 'true'
+ run: exit 1
+
- name: Verify normalized history source report
id: history_source_ready
if: always()
@@ -1225,7 +1465,7 @@ jobs:
publish_history:
name: Publish S3 history and Pages bundle with declared screenshots
needs: [authorize, catalog, report]
- if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true'
+ if: always() && needs.catalog.result == 'success' && needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
@@ -1257,12 +1497,10 @@ jobs:
with:
version: 9.15.4
- # Resolve the trusted publication checkout's lockfile without lifecycle
- # scripts, then install the exact result. This matches the report job's
- # frozen-install preparation while keeping target-controlled code out of
- # the AWS credentialed publisher.
- - run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile
- - run: pnpm install --frozen-lockfile
+ # Use a job-local store and disable lifecycle scripts throughout install.
+ # OIDC is available to every step, even before the AWS credential exchange.
+ - name: Install pinned publication dependencies
+ run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-publisher-store"
- name: Install publisher-only Chromium
run: pnpm exec playwright install --with-deps --only-shell chromium
diff --git a/doc/agent-observability.md b/doc/agent-observability.md
index 2932330f2d..2968ee6f50 100644
--- a/doc/agent-observability.md
+++ b/doc/agent-observability.md
@@ -28,9 +28,13 @@ requires the qualified executable version. Codex ACPX writes its isolated home.
The legacy ACPX adapter keeps export configuration in a private home per agent
and settings compatibility key, and shares conversation files with its stable managed home;
export changes do not change its logical conversation fingerprint.
-Codex's bundled Rust OTLP exporter percent-decodes header values, so launch
-material encodes them once to preserve literal percent sequences. Credentials
-remain literal values in the company secret; environment interpolation is not used.
+Codex receives authentication through `OTEL_EXPORTER_OTLP_LOGS_HEADERS` and
+`OTEL_EXPORTER_OTLP_TRACES_HEADERS` in its process environment. Its managed
+configuration and invocation profiles contain empty header maps, so exporting
+does not save the ingestion credential in those files. The bundled Rust OTLP
+exporter percent-decodes environment header values; launch material encodes
+them once to preserve literal percent sequences. Credentials remain literal
+values in the company secret; config-file environment interpolation is not used.
A caller-owned/preconfigured Codex home cannot be automatically configured in
this version: its files remain untouched and execution emits a diagnostic.
Other harnesses, personal Codex desktop sessions and hosted claude_managed are
@@ -69,8 +73,8 @@ entered secrets. Revisions prevent another administrator's edit being silently
overwritten. Credential values are excluded from task environment manifests,
run/config snapshots, argv, activity records, recovery configuration and query
cache data. Export credentials are necessarily accessible to the exporting
-harness itself. Claude Code supplies these headers through its process environment,
-which its tools can inherit or inspect. Codex can read its managed configuration.
+harness itself. Both Codex and Claude Code receive these headers through their
+process environment, which their tools can inherit or inspect.
This feature does not isolate export credentials from agent code. Use a credential
limited to telemetry ingestion for the selected destination; do not supply a broad
organization administration key. A user-operated collector can hold upstream
diff --git a/doc/evals.md b/doc/evals.md
index 985d358469..9d89aa297e 100644
--- a/doc/evals.md
+++ b/doc/evals.md
@@ -511,3 +511,8 @@ covers projectless task files, deferred repository acquisition, controller resta
and nested Git/dirty-file persistence on legacy local, native local and native
Daytona OpenCode. Its three configured cells use independent filesystem, public
API and download oracles; no live qualification is implied by fixture registration.
+
+The explicit-only [feedback persistence case](../tests/runner-e2e/README.md#feedback-persistence-explicit-only)
+checks real native complaint/suggestion submissions and retained bodies with
+run-attributed tool receipts. It is manual, uses the normal Product E2E report,
+and keeps synthetic feedback distinct from actionable product findings.
diff --git a/packages/adapter-utils/src/agent-observability.test.ts b/packages/adapter-utils/src/agent-observability.test.ts
index cb85c54eef..22fb580541 100644
--- a/packages/adapter-utils/src/agent-observability.test.ts
+++ b/packages/adapter-utils/src/agent-observability.test.ts
@@ -23,13 +23,23 @@ it("encodes headers losslessly and clears ambient exporter conflicts without cha
expect(observabilitySecretValues(launch)).toContain(input.headers.authorization);
expect(JSON.stringify(buildInvocationEnvForLogs({ ...launch.environment }))).not.toContain("a%2Cb");
});
-it("preserves literal percent sequences through the qualified Codex exporter's header decoder", () => {
+it("keeps Codex headers in the launch environment and preserves literal percent sequences", () => {
const original = "Bearer a,b=%26 c%";
const launch = buildProviderObservabilityLaunch({ ...input, kind: "codex", headers: { authorization: original } });
const exporter = launch.otelConfig!.exporter as { "otlp-http": { headers: Record<string, string> } };
- expect(decodeURIComponent(exporter["otlp-http"].headers.authorization)).toBe(original);
+ expect(exporter["otlp-http"].headers).toEqual({});
+ for (const key of ["OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"]) {
+ expect(launch.environment[key]).toBe(`authorization=${encodeURIComponent(original)}`);
+ }
+ expect(JSON.stringify(launch.otelConfig)).not.toContain(original);
+ expect(JSON.stringify(launch.otelConfig)).not.toContain(encodeURIComponent(original));
expect(observabilitySecretValues(launch)).toContain(original);
+ expect(observabilitySecretValues(launch)).toContain(encodeURIComponent(original));
expect(observabilitySecretValues(launch)).toContain(original.slice(7));
+ expect(JSON.stringify(buildInvocationEnvForLogs({ ...launch.environment }))).not.toContain("a%2Cb");
+ const disabled = buildProviderObservabilityLaunch({ ...input, kind: "codex", enabled: false });
+ expect(disabled.environment).not.toHaveProperty("OTEL_EXPORTER_OTLP_LOGS_HEADERS");
+ expect(disabled.environment).not.toHaveProperty("OTEL_EXPORTER_OTLP_TRACES_HEADERS");
});
it("replaces a compact explicit profile without leaving competing profile flags", () => {
const args = ["exec", "-preview", "-"];
@@ -52,6 +62,8 @@ it("creates invocation-owned profiles, preserves selected profile settings, and
const a = await prepareCodexObservabilityProfile({ home: root, runId: "run-a", sourceProfile: "review", launch });
const b = await prepareCodexObservabilityProfile({ home: root, runId: "run-b", launch });
const parsed = parse(await readFile(join(root, a.filename), "utf8"));
+ expect(JSON.stringify(parsed)).not.toContain(input.headers.authorization);
+ expect(JSON.stringify(parsed)).not.toContain(encodeURIComponent(input.headers.authorization));
expect(parsed.model).toBe("chosen-model");
expect(parsed.otel).toMatchObject({ log_agent_responses: true, tool_result: { max_bytes: 65536 } });
expect((await stat(join(root, a.filename))).mode & 0o777).toBe(0o600);
diff --git a/packages/adapter-utils/src/agent-observability.ts b/packages/adapter-utils/src/agent-observability.ts
index aab46b2445..051e7cb2c7 100644
--- a/packages/adapter-utils/src/agent-observability.ts
+++ b/packages/adapter-utils/src/agent-observability.ts
@@ -35,7 +35,10 @@ export function observabilitySecretValues(launch: ProviderObservabilityLaunch |
const headers = ((launch.otelConfig?.exporter as Record<string, unknown> | undefined)?.["otlp-http"] as { headers?: Record<string, string> } | undefined)?.headers;
const encoded = launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS;
const decoded = (value: string) => { try { return decodeURIComponent(value); } catch { return value; } };
- const values = [...Object.values(headers ?? {}).flatMap(value => [value, decoded(value)]), ...(encoded ? [encoded, ...encoded.split(",").map(item => decoded(item.slice(item.indexOf("=") + 1)))] : [])];
+ const values = [...Object.values(headers ?? {}).flatMap(value => [value, decoded(value)]), ...(encoded ? [encoded, ...encoded.split(",").flatMap(item => {
+ const value = item.slice(item.indexOf("=") + 1);
+ return [value, decoded(value)];
+ })] : [])];
return [...new Set(values.flatMap(value => [value, value.replace(/^Bearer /i, ""), JSON.stringify(value).slice(1, -1)]).filter(Boolean))];
}
@@ -50,10 +53,11 @@ export function buildProviderObservabilityLaunch(input: {
OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000", OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "3000",
};
if (kind === "codex") {
- // Codex passes these through opentelemetry-otlp 0.31's with_headers, which
- // percent-decodes values. Encode once so literal percent sequences survive.
- const headers = Object.fromEntries(Object.entries(input.headers ?? {}).map(([name, value]) => [name, encodeURIComponent(value)]));
- const exporter = (endpoint: string | undefined) => enabled ? { "otlp-http": { endpoint, protocol: "binary", headers } } : "none";
+ if (enabled) Object.assign(environment, {
+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: encodeObservabilityHeaders(input.headers ?? {}),
+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: encodeObservabilityHeaders(input.headers ?? {}),
+ });
+ const exporter = (endpoint: string | undefined) => enabled ? { "otlp-http": { endpoint, protocol: "binary", headers: {} } } : "none";
return { kind, compatibilityKey, environment, otelConfig: {
exporter: exporter(input.logsEndpoint), trace_exporter: exporter(input.tracesEndpoint), metrics_exporter: "none",
log_user_prompt: enabled && capture.userPrompts, log_agent_responses: enabled && capture.assistantResponses,
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs
index 8f4e4dc9c1..43edd3e019 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs
@@ -302,6 +302,8 @@ impl AcpxSidecarTransport {
"PAPERCLIP_OBSERVABILITY_CODEX_CONFIG",
"OTEL_EXPORTER_OTLP_LOGS_TIMEOUT",
"OTEL_EXPORTER_OTLP_TRACES_TIMEOUT",
+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",
+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
]);
}
let pi_keys = if agent == "pi" {
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
index 07e5cf8145..e5c77b954b 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
@@ -651,6 +651,25 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
let require_codex_home_auth = args
.iter()
.any(|value| value == "--require-codex-home-auth");
+ if args
+ .iter()
+ .any(|value| value == "--require-otel-environment")
+ {
+ for (name, expected) in [
+ (
+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",
+ "authorization=Bearer%20logs-fixture",
+ ),
+ (
+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
+ "authorization=Bearer%20traces-fixture",
+ ),
+ ] {
+ assert_eq!(std::env::var(name).as_deref(), Ok(expected));
+ }
+ assert!(std::env::var_os("OTEL_EXPORTER_OTLP_HEADERS").is_none());
+ assert!(std::env::var_os("OTEL_EXPORTER_OTLP_ENDPOINT").is_none());
+ }
let durable_turn_ids = args.iter().any(|value| value == "--durable-turn-ids");
let hold_first_durable_turn = args
.iter()
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
index b6f7675c5e..70aa673509 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
@@ -823,6 +823,8 @@ const GITHUB_CREDENTIAL_ENVIRONMENT_KEYS: &[&str] = &[
const CODEX_PROVIDER_ENVIRONMENT_KEYS: &[&str] = &[
"OTEL_EXPORTER_OTLP_LOGS_TIMEOUT",
"OTEL_EXPORTER_OTLP_TRACES_TIMEOUT",
+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",
+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
"PAPERCLIP_AI_PROVIDER_KEY",
"PAPERCLIP_AGENT_KEY_ID",
"PAPERCLIP_AGENT_PUBLIC_KEY",
diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
index 07c7912635..5d8ccc10ab 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
@@ -106,6 +106,19 @@ fn provider_receives_the_isolated_codex_auth_home() {
.env("PAPERCLIP_CODEX_AUTH_TEST_HOME", &directory)
.env("HOME", &directory)
.env("CODEX_HOME", &directory)
+ .env(
+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",
+ "authorization=Bearer%20logs-fixture",
+ )
+ .env(
+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
+ "authorization=Bearer%20traces-fixture",
+ )
+ .env(
+ "OTEL_EXPORTER_OTLP_HEADERS",
+ "authorization=unwanted-ambient",
+ )
+ .env("OTEL_EXPORTER_OTLP_ENDPOINT", "https://unwanted.example")
.status()
.expect("run isolated auth-home assertion");
assert!(status.success());
@@ -119,7 +132,10 @@ fn provider_receives_isolated_codex_auth_home_subprocess() {
else {
return;
};
- let config = provider_config(&directory, &["--require-codex-home-auth"]);
+ let config = provider_config(
+ &directory,
+ &["--require-codex-home-auth", "--require-otel-environment"],
+ );
let mut provider = CodexProvider::start(&config, None)
.expect("Codex provider should read auth from the isolated CODEX_HOME");
provider.shutdown().expect("stop fake Codex provider");
diff --git a/packages/paperclip-runner/scripts/build-inputs.test.mjs b/packages/paperclip-runner/scripts/build-inputs.test.mjs
index ce650a871d..7822de2cbb 100644
--- a/packages/paperclip-runner/scripts/build-inputs.test.mjs
+++ b/packages/paperclip-runner/scripts/build-inputs.test.mjs
@@ -491,26 +491,34 @@ test("two chains run by separate shells at the same time both record", { skip: p
await withRepo(async ({ repoRoot, packageRoot }) => {
const pendingDirectory = buildInputsPendingDirectory(packageRoot);
const marker = path.join(repoRoot, "release-the-chains");
- const records = () => (existsSync(pendingDirectory) ? readdirSync(pendingDirectory).sort() : []);
-
- const first = runChainInShell("sh", packageRoot, marker);
- await waitFor(() => records().length === 1, "the first chain's record");
- const second = runChainInShell("sh", packageRoot, marker);
- await waitFor(() => records().length === 2, "the second chain's record");
- const [firstId, secondId] = records().map((name) => name.replace(/\.json$/, ""));
- assert.notEqual(firstId, secondId);
- for (const id of [firstId, secondId]) {
- assert.match(id, /^\d+$/, "a chain's record is named after its shell's pid");
- assert.equal(readBuildInputsStamp(buildInputsPendingPath(packageRoot, id)).buildId, id);
+ const records = () => (existsSync(pendingDirectory) ? readdirSync(pendingDirectory).filter(name => name.endsWith(".json")).sort() : []);
+
+ const staged = path.join(pendingDirectory, "999999.json.999999.tmp");
+ const chains = [runChainInShell("sh", packageRoot, marker)];
+ let results;
+ try {
+ await waitFor(() => records().length === 1, "the first chain's record");
+ await writeFile(staged, "{");
+ chains.push(runChainInShell("sh", packageRoot, marker));
+ await waitFor(() => records().length === 2, "the second chain's record");
+ const [firstId, secondId] = records().map((name) => name.replace(/\.json$/, ""));
+ assert.notEqual(firstId, secondId);
+ for (const id of [firstId, secondId]) {
+ assert.match(id, /^\d+$/, "a chain's record is named after its shell's pid");
+ assert.equal(readBuildInputsStamp(buildInputsPendingPath(packageRoot, id)).buildId, id);
+ }
+ } finally {
+ await rm(staged, { force: true });
+ await writeFile(marker, "go\n");
+ results = await Promise.allSettled(chains);
}
- await writeFile(marker, "go\n");
- const results = await Promise.all([first, second]);
for (const result of results) {
- assert.match(result.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} begin: `));
- assert.match(result.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} recorded: `), "a chain finished without recording");
+ assert.equal(result.status, "fulfilled", result.reason?.message);
+ assert.match(result.value.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} begin: `));
+ assert.match(result.value.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} recorded: `), "a chain finished without recording");
}
- assert.deepEqual(records(), []);
+ assert.deepEqual(await readdir(pendingDirectory), []);
assert.equal(assessRunnerBuild({ repoRoot, packageRoot }).current, true);
// The same chain under bash, which may exec its last command: `--finish`
diff --git a/packages/paperclip-runner/src/drivers/agent-observability.test.ts b/packages/paperclip-runner/src/drivers/agent-observability.test.ts
index 4fdc2f9334..9e932db0ad 100644
--- a/packages/paperclip-runner/src/drivers/agent-observability.test.ts
+++ b/packages/paperclip-runner/src/drivers/agent-observability.test.ts
@@ -9,8 +9,12 @@ import { createSanitizedAcpxSpawnInput } from "./acpx/environment.js";
import { configuredEnvironmentProjection } from "../configured-environment.js";
const roots: string[] = [];
afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
-const launch: ProviderObservabilityLaunch = { kind: "codex", compatibilityKey: "revision", environment: { OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000" }, otelConfig: {
- exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: { authorization: "Bearer secret-canary" } } }, metrics_exporter: "none", log_agent_responses: true,
+const launch: ProviderObservabilityLaunch = { kind: "codex", compatibilityKey: "revision", environment: {
+ OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000",
+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20secret-canary",
+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: "authorization=Bearer%20secret-canary",
+}, otelConfig: {
+ exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: {} } }, metrics_exporter: "none", log_agent_responses: true,
} };
it("transports a valid encoded header set in both Codex exporters without rejecting the launch", () => {
const headers = Object.fromEntries([0, 1, 2, 3].map(index => [`x-custom-${index}`, encodeURIComponent("%".repeat(4000))]));
@@ -28,8 +32,19 @@ it("materializes the real native home with both governed MCP and protected expor
nativeMcp: { name: "paperclip-assigned", url: "https://paperclip.example/mcp", token: "mcp-secret" } });
const config = parse(await readFile(join(home, "config.toml"), "utf8"));
expect(config.otel).toEqual(launch.otelConfig); expect(config.mcp_servers).toHaveProperty("paperclip-assigned");
+ expect(JSON.stringify(config)).not.toContain("secret-canary");
expect((await stat(join(home, "config.toml"))).mode & 0o777).toBe(0o600);
});
+it("delivers Codex headers only through its provider environment and redacts diagnostics", () => {
+ const source = applyProviderObservabilityEnvironment({ KEEP: "task" }, launch, true);
+ const sanitized = createSanitizedAcpxSpawnInput(source, "codex").env;
+ expect(sanitized).toMatchObject(launch.environment);
+ expect(sanitized.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG).not.toContain("secret-canary");
+ expect(configuredEnvironmentProjection(source).PAPERCLIP_CONFIGURED_ENV_KEYS).toBe('["KEEP"]');
+ expect(createSanitizedAcpxSpawnInput(source, "pi").env).not.toHaveProperty("OTEL_EXPORTER_OTLP_LOGS_HEADERS");
+ const diagnostic = redactObservabilityDiagnostic("Bearer secret-canary Bearer%20secret-canary secret-canary", sanitized);
+ expect(diagnostic).toBe("[REDACTED] [REDACTED] [REDACTED]");
+});
it("passes only explicit Claude observability through its sanitizer and excludes task projection", () => {
const source = { OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20secret-canary", KEEP: "task" };
expect(createSanitizedAcpxSpawnInput(source, "claude").env).toMatchObject({ OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: source.OTEL_EXPORTER_OTLP_LOGS_HEADERS });
diff --git a/packages/paperclip-runner/src/drivers/agent-observability.ts b/packages/paperclip-runner/src/drivers/agent-observability.ts
index 37b11a1303..7ee8159df1 100644
--- a/packages/paperclip-runner/src/drivers/agent-observability.ts
+++ b/packages/paperclip-runner/src/drivers/agent-observability.ts
@@ -27,7 +27,7 @@ export function providerObservabilityEnvironment(source: NodeJS.ProcessEnv | und
if (!source || !["codex", "claude"].includes(agent)) return {};
const result: NodeJS.ProcessEnv = {};
const names: readonly string[] = agent === "claude" ? AGENT_OBSERVABILITY_ENV_KEYS.filter(key => key !== "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG")
- : ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"];
+ : ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"];
for (const key of names) if (source[key] !== undefined) {
// Codex carries two percent-encoded header maps. A valid 16-KiB company
// header set can exceed 64 KiB after encoding and duplication.
@@ -74,6 +74,7 @@ export function redactObservabilityDiagnostic(message: string, environment: Node
values.add(value);
for (const item of value.split(",")) {
const encoded = item.slice(item.indexOf("=") + 1);
+ values.add(encoded);
try { values.add(decodeURIComponent(encoded)); } catch { values.add(encoded); }
}
}
diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts
index 4184701c56..ddeef5fc80 100644
--- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts
+++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts
@@ -25,6 +25,46 @@ describe("Codex app-server transport limits", () => {
})).toEqual({ PAPERCLIP_AI_PROVIDER_KEY: "selected-provider-key", CODEX_HOME: "/isolated/connection" });
});
+ it.each([true, false])("passes configured OTLP authentication to the child only when present (%s)", async (enabled) => {
+ const headers = enabled ? {
+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20fixture-logs",
+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: "authorization=Bearer%20fixture-traces",
+ } : {};
+ const transport = new ProcessCodexAppServerTransport({
+ command: process.execPath,
+ args: ["-e", `
+ process.stdin.resume();
+ process.stdout.write(JSON.stringify({ method: "environment", params: {
+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS,
+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: process.env.OTEL_EXPORTER_OTLP_TRACES_HEADERS,
+ DATABASE_URL: process.env.DATABASE_URL,
+ OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+ } }) + "\\n");
+ `],
+ environment: createSanitizedCodexEnvironment({
+ ...headers, DATABASE_URL: "excluded-database", OPENAI_API_KEY: "excluded-provider-key",
+ }),
+ });
+ try {
+ const notification = await transport.notifications()[Symbol.asyncIterator]().next();
+ expect(notification.value).toEqual({ method: "environment", params: headers });
+ } finally {
+ await transport.close();
+ }
+ });
+
+ it("does not inherit ambient OTLP credentials without an explicit launch environment", () => {
+ vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "ambient-log-secret");
+ vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "ambient-trace-secret");
+ try {
+ const environment = createSanitizedCodexEnvironment();
+ expect(environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toBeUndefined();
+ expect(environment.OTEL_EXPORTER_OTLP_TRACES_HEADERS).toBeUndefined();
+ } finally {
+ vi.unstubAllEnvs();
+ }
+ });
+
it("passes only bounded controller-projected GitHub credentials", () => {
expect(
createSanitizedCodexEnvironment({
diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts
index 886e86c7a4..c54663b1b2 100644
--- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts
+++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts
@@ -251,7 +251,11 @@ export function createSanitizedCodexEnvironment(
if (key.includes("PROXY") && proxyContainsCredentials(value)) continue;
environment[key] = value;
}
- for (const key of ["PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY"]) {
+ // Secret-bearing launch material must come from the controller's explicit environment.
+ for (const key of [
+ "PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY",
+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS",
+ ]) {
if (explicit?.[key] !== undefined) environment[key] = explicit[key];
}
Object.assign(environment, githubCredentialEnvironment(source), configuredEnvironment(explicit));
diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts
index 30ed4b970a..a63abc777f 100644
--- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts
+++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts
@@ -18,7 +18,7 @@ import { execFile, type ChildProcess } from "node:child_process";
import { EventEmitter } from "node:events";
import { promisify } from "node:util";
import { getCACertificates } from "node:tls";
-import { join, resolve } from "node:path";
+import { basename, dirname, join, resolve } from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
@@ -1441,7 +1441,7 @@ describe("OpenCodeServerDriver", () => {
await session.close({ reason: "test" });
});
- it("uses the native system channel, selected skill tree, instruction root, and assigned MCP gateway", async () => {
+ it.each([false, true])("uses the native system channel, selected skill tree, instruction root, and assigned MCP gateway after startup retry=%s", async (retryStartup) => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-context-"));
const workspace = await mkdtemp(
@@ -1465,6 +1465,7 @@ describe("OpenCodeServerDriver", () => {
]);
const systemInstructions = `${PAPERCLIP_EXECUTION_PROMPT}\n\nRead sibling.md\n\nRead-only instruction sibling root: ${instructionRoot}`;
let submittedPrompt: Record<string, unknown> | null = null;
+ let startupFailurePending = retryStartup;
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
@@ -1479,6 +1480,10 @@ describe("OpenCodeServerDriver", () => {
PAPERCLIP_NATIVE_MCP_TOKEN: "x".repeat(40),
},
fetch: async (input, init) => {
+ if (startupFailurePending && String(input).endsWith("/session") && init?.method === "POST") {
+ startupFailurePending = false;
+ return new Response("Fixture startup failure", { status: 503 });
+ }
if (String(input).endsWith("/prompt_async"))
submittedPrompt = JSON.parse(String(init?.body));
return fetch(input, init);
@@ -1503,15 +1508,17 @@ describe("OpenCodeServerDriver", () => {
PAPERCLIP_EXECUTION_PROMPT,
);
const sessionRoot = join(root, "context");
- const isolatedHomes = (await readdir(sessionRoot)).filter((entry) =>
- entry.startsWith("home-"),
+ const runtimeEnvironment = JSON.parse(
+ await readFile(join(sessionRoot, "data", "fake-environment.json"), "utf8"),
);
- expect(isolatedHomes).toHaveLength(1);
+ const isolatedHome = runtimeEnvironment.home;
+ expect(startupFailurePending).toBe(false);
+ expect(dirname(isolatedHome)).toBe(sessionRoot);
+ expect(basename(isolatedHome)).toMatch(/^home-/);
await expect(
readFile(
join(
- sessionRoot,
- isolatedHomes[0]!,
+ isolatedHome,
".claude",
"skills",
"assigned",
diff --git a/scripts/__tests__/eval-schedule-monitor.test.mjs b/scripts/__tests__/eval-schedule-monitor.test.mjs
new file mode 100644
index 0000000000..ddf6db789e
--- /dev/null
+++ b/scripts/__tests__/eval-schedule-monitor.test.mjs
@@ -0,0 +1,196 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { acknowledgeEvalNotifications, expectedCampaign, inspectEvalSchedule, monitorTransition, runEvalMonitor, MONITORED_SCHEDULES } from "../eval-schedule-monitor.mjs";
+
+const now = Date.parse("2026-10-25T13:00:00Z");
+const enabledAt = "2026-10-24T00:00:00Z";
+const variables = { total_count: 3, variables: [
+ { name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED", value: "true" },
+ { name: "RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED", value: "true" },
+ { name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT", value: enabledAt },
+] };
+const workflow = { id: 17, path: ".github/workflows/runner-full-stack-e2e.yml", state: "active" };
+const run = (cadence, id, extra = {}) => ({ id, workflow_id: 17, run_attempt: 1, event: "schedule", status: "completed", conclusion: "success",
+ display_title: `Runner Full-Stack E2E / ${MONITORED_SCHEDULES[cadence]}`,
+ created_at: cadence === "daily" ? "2026-10-25T08:50:00Z" : "2026-10-25T09:50:00Z", ...extra });
+function fixture(overrides = {}) {
+ const calls = [];
+ const get = async endpoint => {
+ calls.push(endpoint);
+ if (endpoint.includes("/variables?")) return overrides.variables ?? structuredClone(variables);
+ if (endpoint.endsWith("runner-full-stack-e2e.yml")) return overrides.workflow ?? workflow;
+ if (endpoint.includes("/runs?")) {
+ const workflow_runs = overrides.runs ?? [run("daily", 101), run("weekly", 102)];
+ return { total_count: overrides.total_count ?? workflow_runs.length, workflow_runs };
+ }
+ if (endpoint.includes("/artifacts?")) {
+ const id = Number(endpoint.match(/\/runs\/(\d+)\//)[1]);
+ const artifacts = overrides.artifacts ?? [{ name: `runner-e2e-report-${id}-1`, expired: false, size_in_bytes: 100 }];
+ return { artifacts, total_count: artifacts.length };
+ }
+ throw new Error("Unexpected read");
+ };
+ return { get, calls };
+}
+
+test("UTC due windows honor grace, activation and the weekly boundary", () => {
+ assert.equal(expectedCampaign("daily", now, Date.parse(enabledAt)), "2026-10-25T08:47:00.000Z");
+ assert.equal(expectedCampaign("weekly", now, Date.parse(enabledAt)), "2026-10-25T09:47:00.000Z");
+ assert.equal(expectedCampaign("weekly", Date.parse("2026-10-25T10:00:00Z"), Date.parse(enabledAt)), undefined);
+ assert.equal(expectedCampaign("daily", now, now), undefined);
+ assert.equal(expectedCampaign("daily", Date.parse("2026-11-01T10:17:00Z"), 0), "2026-11-01T08:47:00.000Z");
+});
+
+test("disabled gates do not query or diagnose old paid campaigns", async () => {
+ const f = fixture({ variables: { total_count: 0, variables: [] } });
+ assert.equal((await inspectEvalSchedule({ ...f, now })).status, "disabled");
+ assert.equal(f.calls.length, 1);
+});
+
+for (const budgetGate of [undefined, "false"]) test(`an enabled legacy schedule with ${budgetGate ?? "missing"} budget gate alerts once`, async () => {
+ const rows = [{ name: "RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED", value: "true" }];
+ if (budgetGate !== undefined) rows.push({ name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED", value: budgetGate });
+ const f = fixture({ variables: { total_count: rows.length, variables: rows } });
+ const inspection = await inspectEvalSchedule({ ...f, now });
+ assert.equal(inspection.status, "configuration_incomplete");
+ assert.equal(inspection.reason, "legacy_schedule_enabled_without_budget_gate");
+ assert.equal(f.calls.length, 1);
+ const first = monitorTransition(undefined, inspection);
+ assert.equal(first.notifications.length, 1);
+ assert.equal(first.notifications[0].kind, "problem");
+ assert.deepEqual(monitorTransition(first, inspection).notifications, first.notifications);
+});
+
+test("an explicitly disabled legacy gate stays quiet even if the new budget gate is enabled", async () => {
+ const changed = structuredClone(variables); changed.variables[1].value = "false";
+ const f = fixture({ variables: changed });
+ const inspection = await inspectEvalSchedule({ ...f, now });
+ assert.equal(inspection.status, "disabled"); assert.equal(f.calls.length, 1);
+ assert.deepEqual(monitorTransition(undefined, inspection).notifications, []);
+});
+
+test("missing activation time and a disabled workflow stay distinct from missing runs", async () => {
+ const invalid = structuredClone(variables); invalid.variables[2].value = "not a date";
+ assert.equal((await inspectEvalSchedule({ ...fixture({ variables: invalid }), now })).status, "configuration_incomplete");
+ assert.equal((await inspectEvalSchedule({ ...fixture({ workflow: { ...workflow, state: "disabled_inactivity" } }), now })).status, "workflow_disabled");
+});
+
+test("requires one completed scheduled campaign and its unexpired report for each cadence", async () => {
+ const result = await inspectEvalSchedule({ ...fixture(), now });
+ assert.deepEqual(result.observations.map(row => row.status), ["healthy", "healthy"]);
+ assert.deepEqual(result.observations.map(row => row.runId), [101, 102]);
+ assert.deepEqual(monitorTransition(undefined, result).notifications, []);
+});
+
+for (const [name, overrides, status] of [
+ ["no campaign", { runs: [] }, "missing"],
+ ["manual run cannot substitute for schedule", { runs: [run("daily", 101, { event: "workflow_dispatch" })] }, "missing"],
+ ["failed campaign", { runs: [run("daily", 101, { conclusion: "failure" })] }, "failed"],
+ ["canceled campaign", { runs: [run("daily", 101, { conclusion: "cancelled" })] }, "failed"],
+ ["overdue queued campaign", { runs: [run("daily", 101, { status: "queued" })] }, "overdue"],
+ ["unexpected paid rerun", { runs: [run("daily", 101, { run_attempt: 2 })] }, "unexpected_retry"],
+ ["multiple matching runs", { runs: [run("daily", 101), run("daily", 103)] }, "ambiguous_runs"],
+ ["missing report", { artifacts: [] }, "missing_report"],
+ ["expired report", { artifacts: [{ name: "runner-e2e-report-101-1", expired: true, size_in_bytes: 100 }] }, "missing_report"],
+ ["empty report", { artifacts: [{ name: "runner-e2e-report-101-1", expired: false, size_in_bytes: 0 }] }, "missing_report"],
+]) test(name, async () => {
+ const result = await inspectEvalSchedule({ ...fixture(overrides), now });
+ assert.equal(result.observations[0].status, status);
+});
+
+test("new-day early run does not collide with the prior day during grace", async () => {
+ const runs = [run("daily", 100, { created_at: "2026-10-24T08:50:00Z" }), run("daily", 101)];
+ const result = await inspectEvalSchedule({ ...fixture({ runs }), now: Date.parse("2026-10-25T09:00:00Z") });
+ assert.equal(result.observations[0].runId, 100);
+ assert.equal(result.observations[0].status, "healthy");
+});
+
+test("running is quiet inside the runtime limit", async () => {
+ const result = await inspectEvalSchedule({ ...fixture({ runs: [run("daily", 101, { status: "in_progress", created_at: "2026-10-25T10:00:00Z" })] }), now });
+ assert.equal(result.observations[0].status, "running");
+});
+
+test("API errors and incomplete lists cannot imply a missing or successful run", async () => {
+ assert.equal((await inspectEvalSchedule({ ...fixture({ total_count: 101 }), now })).status, "api_unavailable");
+ assert.equal((await inspectEvalSchedule({ get: async () => { throw new Error("private upstream body"); }, now })).status, "api_unavailable");
+});
+
+test("alerts once per incident and recovery while keeping last observed run IDs through API failure", async () => {
+ const failed = await inspectEvalSchedule({ ...fixture({ runs: [run("daily", 101, { conclusion: "failure" }), run("weekly", 102)] }), now });
+ const first = monitorTransition(undefined, failed);
+ assert.equal(first.notifications.length, 1);
+ assert.deepEqual(monitorTransition(first, failed).notifications, first.notifications);
+ const outage = { observedAt: new Date(now + 1000).toISOString(), status: "api_unavailable", observations: [] };
+ const unavailable = monitorTransition({ ...first, notifications: [] }, outage);
+ assert.equal(unavailable.lastSuccessfulInspectionAt, first.lastSuccessfulInspectionAt);
+ assert.deepEqual(unavailable.lastInspectedRunIds, { daily: 101, weekly: 102 });
+ assert.equal(unavailable.notifications[0].status, "api_unavailable");
+ assert.deepEqual(monitorTransition(unavailable, outage).notifications, unavailable.notifications);
+ const resumed = monitorTransition({ ...unavailable, notifications: [] }, failed);
+ assert.deepEqual(resumed.notifications.map(row => [row.kind, row.cadence]), [["recovery", "monitor"]]);
+ const healthy = monitorTransition({ ...resumed, notifications: [] }, await inspectEvalSchedule({ ...fixture(), now }));
+ assert.deepEqual(healthy.notifications.map(row => [row.kind, row.cadence]), [["recovery", "daily"]]);
+});
+
+test("persists private state across invocations and refuses corrupt history", async () => {
+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-"));
+ try {
+ const file = path.join(directory, "state.json");
+ const first = await runEvalMonitor(file, fixture({ runs: [] }).get, now);
+ assert.equal(first.notifications.length, 2);
+ const repeated = await runEvalMonitor(file, fixture({ runs: [] }).get, now);
+ assert.deepEqual(repeated.notifications, first.notifications);
+ assert.deepEqual(await acknowledgeEvalNotifications(file, [first.notifications[0].id]), { acknowledged: 1, pending: 1 });
+ const afterPartialDelivery = await runEvalMonitor(file, fixture({ runs: [] }).get, now);
+ assert.deepEqual(afterPartialDelivery.notifications, first.notifications.slice(1));
+ assert.deepEqual(await acknowledgeEvalNotifications(file, first.notifications.map(row => row.id)), { acknowledged: 1, pending: 0 });
+ assert.deepEqual((await runEvalMonitor(file, fixture({ runs: [] }).get, now)).notifications, []);
+ assert.equal(JSON.parse(await readFile(file, "utf8")).repository, "paperclipai/paperclip");
+ await writeFile(file, "bad history");
+ await assert.rejects(runEvalMonitor(file, fixture().get, now));
+ assert.equal(await readFile(file, "utf8"), "bad history");
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+test("pending alerts survive later recovery and are acknowledged only after delivery", async () => {
+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-outbox-"));
+ try {
+ const file = path.join(directory, "state.json");
+ const lostOutput = await runEvalMonitor(file, fixture({ runs: [] }).get, now);
+ const recovered = await runEvalMonitor(file, fixture().get, now + 1000);
+ assert.deepEqual(recovered.notifications.slice(0, 2), lostOutput.notifications);
+ assert.deepEqual(recovered.notifications.map(row => row.kind), ["problem", "problem", "recovery", "recovery"]);
+ const unchanged = await runEvalMonitor(file, fixture().get, now + 2000);
+ assert.deepEqual(unchanged.notifications, recovered.notifications);
+ await assert.rejects(acknowledgeEvalNotifications(file, ["invalid"]));
+ assert.deepEqual(await acknowledgeEvalNotifications(file, recovered.notifications.map(row => row.id)), { acknowledged: 4, pending: 0 });
+ assert.deepEqual((await runEvalMonitor(file, fixture().get, now + 3000)).notifications, []);
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+test("rejects valid JSON with malformed history without querying or overwriting it", async () => {
+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-corrupt-"));
+ try {
+ const file = path.join(directory, "state.json");
+ await runEvalMonitor(file, fixture().get, now);
+ const valid = JSON.parse(await readFile(file, "utf8"));
+ for (const malformed of [null, false, 0, [], {}, { ...valid, signals: [] }, { ...valid, signals: { daily: null } },
+ { ...valid, notifications: null }, { ...valid, notifications: [{ id: "invalid" }] }]) {
+ const original = JSON.stringify(malformed);
+ await writeFile(file, original);
+ const f = fixture();
+ await assert.rejects(runEvalMonitor(file, f.get, now), /Invalid retained monitor state/);
+ assert.equal(f.calls.length, 0);
+ assert.equal(await readFile(file, "utf8"), original);
+ }
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+test("workflow titles carry the literal schedule used to distinguish daily and weekly runs", async () => {
+ const text = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8");
+ assert.ok(text.includes("run-name: Runner Full-Stack E2E / ${{ github.event.schedule || inputs.cadence || 'manual' }}"));
+ for (const cron of Object.values(MONITORED_SCHEDULES)) assert.ok(text.includes(`cron: "${cron}"`));
+});
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index 8ea57dadc7..69b7df521d 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -698,8 +698,22 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(
(match) => match[1],
);
- assert.equal(crons.length, 1, `${name} must have one schedule`);
- assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/);
+ if (name === "runner-full-stack-e2e.yml") {
+ assert.deepEqual(crons, ["47 8 * * *", "47 9 * * 0"]);
+ assert.ok(workflow.includes("vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'"));
+ assert.ok(workflow.includes("Prepare bounded campaign before credentials"));
+ assert.ok(workflow.includes("--max-parallel 1 --max-automatic-retries 0"));
+ const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:"));
+ assert.ok(catalog.includes("EVAL_OPERATING_OVERHEAD_JSON: ${{ vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON }}"));
+ assert.ok(catalog.includes("operating_overhead: ${{ steps.catalog.outputs.operating_overhead }}"));
+ const prepare = workflow.slice(workflow.indexOf(" - name: Prepare bounded campaign before credentials"), workflow.indexOf(" - name: Run bounded campaign and retain feedback review"));
+ assert.ok(prepare.includes("EVAL_OPERATING_OVERHEAD_JSON: ${{ needs.catalog.outputs.operating_overhead }}"));
+ assert.ok(prepare.includes("EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }}"));
+ assert.ok(!prepare.includes("vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON"));
+ } else {
+ assert.equal(crons.length, 1, `${name} must have one schedule`);
+ assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/);
+ }
}
const chaosWorkflow = actionPinWorkflows[2];
diff --git a/scripts/eval-schedule-monitor.mjs b/scripts/eval-schedule-monitor.mjs
new file mode 100644
index 0000000000..6bee0e1956
--- /dev/null
+++ b/scripts/eval-schedule-monitor.mjs
@@ -0,0 +1,176 @@
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+import { pathToFileURL } from "node:url";
+
+export const MONITORED_REPOSITORY = "paperclipai/paperclip";
+export const MONITORED_WORKFLOW = "runner-full-stack-e2e.yml";
+export const MONITORED_SCHEDULES = { daily: "47 8 * * *", weekly: "47 9 * * 0" };
+const hour = 3_600_000;
+const prefix = `repos/${MONITORED_REPOSITORY}/actions`;
+const numericId = value => Number.isSafeInteger(value) && value > 0;
+
+export function expectedCampaign(cadence, now, enabledAt) {
+ if (!Object.hasOwn(MONITORED_SCHEDULES, cadence) || !Number.isFinite(now) || !Number.isFinite(enabledAt)) {
+ throw new Error("Invalid monitor schedule input");
+ }
+ const due = new Date(now - 90 * 60_000);
+ due.setUTCHours(cadence === "daily" ? 8 : 9, 47, 0, 0);
+ if (due.getTime() > now - 90 * 60_000) due.setUTCDate(due.getUTCDate() - 1);
+ if (cadence === "weekly") due.setUTCDate(due.getUTCDate() - due.getUTCDay());
+ return due.getTime() >= enabledAt ? due.toISOString() : undefined;
+}
+
+function signal(cadence, status, dueAt, runId) {
+ return { cadence, status, ...(dueAt ? { dueAt } : {}), ...(runId ? {
+ runId, url: `https://github.com/${MONITORED_REPOSITORY}/actions/runs/${runId}`,
+ } : {}) };
+}
+
+export async function inspectEvalSchedule({ get, now = Date.now() }) {
+ const observations = [];
+ try {
+ const variables = await get(`${prefix}/variables?per_page=100`);
+ if (!Array.isArray(variables.variables) || !Number.isSafeInteger(variables.total_count)
+ || variables.total_count !== variables.variables.length) throw new Error("Incomplete variables");
+ const values = Object.fromEntries(variables.variables.map(row => [row.name, row.value]));
+ if (values.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED !== "true") {
+ return { observedAt: new Date(now).toISOString(), status: "disabled", observations: [] };
+ }
+ if (values.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED !== "true") {
+ return { observedAt: new Date(now).toISOString(), status: "configuration_incomplete",
+ reason: "legacy_schedule_enabled_without_budget_gate", observations: [] };
+ }
+ const enabledAt = Date.parse(values.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT ?? "");
+ if (!Number.isFinite(enabledAt) || enabledAt > now) {
+ return { observedAt: new Date(now).toISOString(), status: "configuration_incomplete", observations: [] };
+ }
+ const workflow = await get(`${prefix}/workflows/${MONITORED_WORKFLOW}`);
+ if (workflow.path !== `.github/workflows/${MONITORED_WORKFLOW}` || !numericId(workflow.id)) throw new Error("Wrong workflow");
+ if (workflow.state !== "active") return { observedAt: new Date(now).toISOString(), status: "workflow_disabled", observations: [] };
+ const expected = Object.keys(MONITORED_SCHEDULES).map(cadence => ({ cadence, dueAt: expectedCampaign(cadence, now, enabledAt) }));
+ const due = expected.filter(row => row.dueAt);
+ if (!due.length) return { observedAt: new Date(now).toISOString(), status: "waiting", observations: [] };
+ const since = due.map(row => row.dueAt).sort()[0];
+ const runs = await get(`${prefix}/workflows/${MONITORED_WORKFLOW}/runs?event=schedule&per_page=100&created=${encodeURIComponent(`>=${since}`)}`);
+ if (!Array.isArray(runs.workflow_runs) || !Number.isSafeInteger(runs.total_count)
+ || runs.total_count !== runs.workflow_runs.length) throw new Error("Incomplete runs");
+ for (const { cadence, dueAt } of expected) {
+ if (!dueAt) { observations.push(signal(cadence, "waiting")); continue; }
+ const matches = runs.workflow_runs.filter(run => run.event === "schedule" && run.workflow_id === workflow.id
+ && run.display_title === `Runner Full-Stack E2E / ${MONITORED_SCHEDULES[cadence]}`
+ && Date.parse(run.created_at) >= Date.parse(dueAt) && Date.parse(run.created_at) <= now
+ && Date.parse(run.created_at) < Date.parse(dueAt) + (cadence === "daily" ? 24 : 168) * hour);
+ if (!matches.length) { observations.push(signal(cadence, "missing", dueAt)); continue; }
+ if (matches.length !== 1) { observations.push(signal(cadence, "ambiguous_runs", dueAt)); continue; }
+ const run = matches[0];
+ if (!numericId(run.id) || !numericId(run.run_attempt) || typeof run.status !== "string") throw new Error("Invalid run identity");
+ if (run.run_attempt !== 1) { observations.push(signal(cadence, "unexpected_retry", dueAt, run.id)); continue; }
+ if (run.status !== "completed") {
+ observations.push(signal(cadence, now - Date.parse(run.created_at) > 4 * hour ? "overdue" : "running", dueAt, run.id));
+ continue;
+ }
+ if (run.conclusion !== "success") { observations.push(signal(cadence, "failed", dueAt, run.id)); continue; }
+ const artifacts = await get(`${prefix}/runs/${run.id}/artifacts?per_page=100`);
+ if (!Array.isArray(artifacts.artifacts) || !Number.isSafeInteger(artifacts.total_count)
+ || artifacts.total_count !== artifacts.artifacts.length) throw new Error("Incomplete artifacts");
+ const reports = artifacts.artifacts.filter(row => row.name === `runner-e2e-report-${run.id}-1`
+ && row.expired === false && Number.isSafeInteger(row.size_in_bytes) && row.size_in_bytes > 0);
+ observations.push(signal(cadence, reports.length === 1 ? "healthy" : "missing_report", dueAt, run.id));
+ }
+ return { observedAt: new Date(now).toISOString(), status: "observed", observations };
+ } catch {
+ return { observedAt: new Date(now).toISOString(), status: "api_unavailable", observations };
+ }
+}
+
+const quiet = new Set(["disabled", "waiting", "running", "healthy"]);
+const statuses = new Set([...quiet, "observed", "configuration_incomplete", "workflow_disabled", "api_unavailable",
+ "missing", "ambiguous_runs", "unexpected_retry", "overdue", "failed", "missing_report"]);
+const object = value => value !== null && typeof value === "object" && !Array.isArray(value);
+const timestamp = value => typeof value === "string" && Number.isFinite(Date.parse(value));
+const notificationId = value => typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value);
+const signalKey = row => !quiet.has(row.status) ? JSON.stringify([row.status, row.dueAt ?? null, row.runId ?? null]) : row.status;
+const validSignal = row => object(row) && ["monitor", "daily", "weekly"].includes(row.cadence) && statuses.has(row.status)
+ && (row.dueAt === undefined || timestamp(row.dueAt)) && (row.runId === undefined || numericId(row.runId))
+ && (row.runId === undefined ? row.url === undefined : row.url === `https://github.com/${MONITORED_REPOSITORY}/actions/runs/${row.runId}`);
+
+function validateState(previous) {
+ if (!object(previous) || previous.version !== 1 || previous.repository !== MONITORED_REPOSITORY
+ || previous.workflow !== MONITORED_WORKFLOW || !object(previous.signals)
+ || !timestamp(previous.checkedAt) || !(previous.lastSuccessfulInspectionAt === null || timestamp(previous.lastSuccessfulInspectionAt))
+ || !object(previous.lastInspectedRunIds) || Object.entries(previous.lastInspectedRunIds).some(([cadence, id]) => !["daily", "weekly"].includes(cadence) || !numericId(id))
+ || Object.entries(previous.signals).some(([cadence, row]) => !validSignal(row) || row.cadence !== cadence
+ || row.issue !== !quiet.has(row.status) || row.key !== signalKey(row))
+ || !Array.isArray(previous.notifications) || previous.notifications.some(row => !validSignal(row)
+ || !notificationId(row.id) || !["problem", "recovery"].includes(row.kind))
+ || new Set(previous.notifications.map(row => row.id)).size !== previous.notifications.length) {
+ throw new Error("Invalid retained monitor state; refusing to reset notification history");
+ }
+}
+
+export function monitorTransition(previous, inspection) {
+ if (previous !== undefined) validateState(previous);
+ const current = inspection.status === "observed" ? inspection.observations : [signal("monitor", inspection.status)];
+ const signals = {}, notifications = [];
+ const prior = previous?.signals ?? {};
+ for (const row of current) {
+ const issue = !quiet.has(row.status);
+ const key = signalKey(row);
+ signals[row.cadence] = { ...row, issue, key };
+ if (issue && prior[row.cadence]?.key !== key) notifications.push({ kind: "problem", ...row });
+ else if (!issue && prior[row.cadence]?.issue && row.status !== "disabled") notifications.push({ kind: "recovery", ...row });
+ }
+ if (inspection.status === "observed" && prior.monitor?.issue) notifications.push({ kind: "recovery", cadence: "monitor", status: "observed" });
+ const readable = !["api_unavailable", "configuration_incomplete"].includes(inspection.status);
+ return { version: 1, repository: MONITORED_REPOSITORY, workflow: MONITORED_WORKFLOW,
+ checkedAt: inspection.observedAt,
+ lastSuccessfulInspectionAt: readable ? inspection.observedAt : previous?.lastSuccessfulInspectionAt ?? null,
+ lastInspectedRunIds: { ...previous?.lastInspectedRunIds,
+ ...Object.fromEntries(inspection.observations.filter(row => row.runId).map(row => [row.cadence, row.runId])) },
+ signals: inspection.status === "observed" || quiet.has(inspection.status) ? signals : { ...prior, ...signals },
+ notifications: [...previous?.notifications ?? [], ...notifications.map(row => ({ id: randomUUID(), ...row }))] };
+}
+
+async function saveState(stateFile, next) {
+ await mkdir(path.dirname(stateFile), { recursive: true, mode: 0o700 });
+ await writeFile(`${stateFile}.pending`, JSON.stringify(next, null, 2) + "\n", { mode: 0o600 });
+ await rename(`${stateFile}.pending`, stateFile);
+}
+
+export async function acknowledgeEvalNotifications(stateFile, ids) {
+ if (!Array.isArray(ids) || !ids.length || ids.some(id => !notificationId(id))) throw new Error("Invalid notification acknowledgments");
+ const previous = JSON.parse(await readFile(stateFile, "utf8"));
+ validateState(previous);
+ const next = { ...previous, notifications: previous.notifications.filter(row => !ids.includes(row.id)) };
+ await saveState(stateFile, next);
+ return { acknowledged: previous.notifications.length - next.notifications.length, pending: next.notifications.length };
+}
+
+export async function runEvalMonitor(stateFile, get, now = Date.now()) {
+ let previous;
+ try { previous = JSON.parse(await readFile(stateFile, "utf8")); validateState(previous); }
+ catch (error) { if (error.code !== "ENOENT") throw error; }
+ const inspection = await inspectEvalSchedule({ get, now });
+ const next = monitorTransition(previous, inspection);
+ await saveState(stateFile, next);
+ return { ...inspection, lastSuccessfulInspectionAt: next.lastSuccessfulInspectionAt,
+ lastInspectedRunIds: next.lastInspectedRunIds, notifications: next.notifications };
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ if (process.argv.length !== 3 && !(process.argv[3] === "--ack" && process.argv.length >= 5)) {
+ throw new Error("Usage: node scripts/eval-schedule-monitor.mjs <private-state-file> [--ack <notification-id> ...]");
+ }
+ const execute = promisify(execFile);
+ const get = async endpoint => {
+ if (!endpoint.startsWith(`${prefix}/`)) throw new Error("Unapproved monitor endpoint");
+ const { stdout } = await execute("gh", ["api", "--method", "GET", endpoint], { timeout: 15_000, maxBuffer: 4 * 1024 * 1024 });
+ return JSON.parse(stdout);
+ };
+ console.log(JSON.stringify(process.argv[3] === "--ack"
+ ? await acknowledgeEvalNotifications(path.resolve(process.argv[2]), process.argv.slice(4))
+ : await runEvalMonitor(path.resolve(process.argv[2]), get), null, 2));
+}
diff --git a/server/src/__tests__/eval-feedback-recovery.integration.test.ts b/server/src/__tests__/eval-feedback-recovery.integration.test.ts
new file mode 100644
index 0000000000..873dc701af
--- /dev/null
+++ b/server/src/__tests__/eval-feedback-recovery.integration.test.ts
@@ -0,0 +1,87 @@
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { once } from "node:events";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { eq } from "drizzle-orm";
+import { agents, heartbeatRuns } from "@paperclipai/db";
+import { expect, it } from "vitest";
+import { createLocalAgentJwt } from "../agent-auth-jwt.js";
+import { startRunnerApiTestServer } from "./helpers/runner-api-server.js";
+import { recoverInterruptedFeedback, readRecoveredFeedback } from "../../../tests/runner-e2e/feedback-recovery.js";
+
+it("recovers real saved complaints and suggestions after their producer is killed", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-feedback-interruption-"));
+ const previousHome = process.env.PAPERCLIP_HOME, previousSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET;
+ process.env.PAPERCLIP_HOME = path.join(root, "home");
+ process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID();
+ let server: Awaited<ReturnType<typeof startRunnerApiTestServer>> | undefined;
+ let worker: ReturnType<typeof spawn> | undefined;
+ try {
+ server = await startRunnerApiTestServer({ deploymentMode: "local_trusted" });
+ const fixture = await server.fixture({ disableWakeOnDemand: true });
+ await server.db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null }).where(eq(heartbeatRuns.id, fixture.runId));
+ await server.db.update(agents).set({ adapterType: "codex_local" }).where(eq(agents.id, fixture.agentId));
+ const token = createLocalAgentJwt(fixture.agentId, fixture.companyId, "codex_local", fixture.runId)!;
+ const caseDirectory = path.join(root, "case");
+ const snapshots = path.join(caseDirectory, "snapshots");
+ const identity = { campaignId: randomUUID(), attemptId: randomUUID(), executionId: "synthetic.interruption.local" };
+ await mkdir(snapshots, { recursive: true });
+ await writeFile(path.join(snapshots, "fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId: fixture.companyId }));
+ worker = spawn(process.execPath, ["--input-type=module", "-e", `
+ for (const kind of ['complaint', 'suggestion']) {
+ const response = await fetch(process.env.FEEDBACK_URL, {
+ method: 'POST', headers: { 'content-type': 'application/json',
+ authorization: 'Bearer ' + process.env.FEEDBACK_TOKEN, 'X-Paperclip-Run-Id': process.env.FEEDBACK_RUN },
+ body: JSON.stringify({ kind, body: 'Synthetic interrupted producer ' + kind, idempotencyKey: kind }),
+ });
+ if (response.status !== 201) throw new Error('Synthetic feedback submission failed: ' + response.status);
+ await response.json();
+ }
+ process.send({ saved: 2 });
+ setInterval(() => {}, 1000);
+ `], { stdio: ["ignore", "pipe", "pipe", "ipc"], env: {
+ PATH: process.env.PATH, FEEDBACK_URL: `${server.apiUrl}/api/companies/${fixture.companyId}/agent-commentary`,
+ FEEDBACK_TOKEN: token, FEEDBACK_RUN: fixture.runId,
+ } });
+ const saved = await Promise.race([
+ once(worker, "message", { signal: AbortSignal.timeout(10_000) }),
+ once(worker, "exit").then(([code]) => { throw new Error(`Producer exited before saving feedback: ${code}`); }),
+ ]);
+ expect(saved[0]).toEqual({ saved: 2 });
+ const exited = once(worker, "exit");
+ worker.kill("SIGKILL");
+ expect((await exited)[1]).toBe("SIGKILL");
+ const apiUrl = server.apiUrl;
+ await recoverInterruptedFeedback({ identity, caseDirectory, secrets: [token], deadlineAt: Date.now() + 5_000,
+ api: { async get<T>(endpoint: string, options?: { timeout: number }): Promise<T> {
+ const response = await fetch(apiUrl + endpoint, { signal: AbortSignal.timeout(options?.timeout ?? 1_000) });
+ if (!response.ok) throw new Error(`Read failed: ${response.status}`);
+ return response.json() as Promise<T>;
+ } },
+ });
+ expect(await readRecoveredFeedback(caseDirectory, identity)).toMatchObject({
+ complaints: 1, suggestions: 1, bodiesCollected: 2, collection: "partial", invocationCollection: "partial",
+ invocations: { completed: 0, failed: 0, unknown: 0 },
+ });
+ await server.close(); server = undefined;
+ const evidenceText = await readFile(path.join(snapshots, "agent-feedback-recovered.json"), "utf8");
+ const evidence = JSON.parse(evidenceText);
+ expect(evidence.entries.map((entry: { body: string }) => entry.body).sort()).toEqual([
+ "Synthetic interrupted producer complaint", "Synthetic interrupted producer suggestion",
+ ]);
+ expect(evidence.entries.every((entry: { runId: string; agent: { id: string }; task: { id: string } }) =>
+ entry.runId === fixture.runId && entry.agent.id === fixture.agentId && entry.task.id === fixture.issueId)).toBe(true);
+ expect(evidenceText).not.toContain(token);
+ expect((await readRecoveredFeedback(caseDirectory, identity))?.review).toHaveLength(2);
+ } finally {
+ if (worker && worker.exitCode === null && worker.signalCode === null) {
+ const exited = once(worker, "exit"); worker.kill("SIGKILL"); await exited;
+ }
+ await server?.close();
+ if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;
+ if (previousSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; else process.env.PAPERCLIP_AGENT_JWT_SECRET = previousSecret;
+ await rm(root, { recursive: true, force: true });
+ }
+}, 90_000);
diff --git a/server/src/__tests__/fixtures/otlp-observability.proto b/server/src/__tests__/fixtures/otlp-observability.proto
index ed258755a2..98feb31205 100644
--- a/server/src/__tests__/fixtures/otlp-observability.proto
+++ b/server/src/__tests__/fixtures/otlp-observability.proto
@@ -13,7 +13,8 @@ message LogRecord { fixed64 time_unix_nano = 1; int32 severity_number = 2; strin
message ScopeLogs { Scope scope = 1; repeated LogRecord log_records = 2; }
message ResourceLogs { Resource resource = 1; repeated ScopeLogs scope_logs = 2; }
message ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1; }
-message Span { bytes trace_id = 1; bytes span_id = 2; bytes parent_span_id = 4; string name = 5; int32 kind = 6; fixed64 start_time_unix_nano = 7; fixed64 end_time_unix_nano = 8; repeated KeyValue attributes = 9; }
+message Status { string message = 2; int32 code = 3; }
+message Span { bytes trace_id = 1; bytes span_id = 2; bytes parent_span_id = 4; string name = 5; int32 kind = 6; fixed64 start_time_unix_nano = 7; fixed64 end_time_unix_nano = 8; repeated KeyValue attributes = 9; Status status = 15; }
message ScopeSpans { Scope scope = 1; repeated Span spans = 2; }
message ResourceSpans { Resource resource = 1; repeated ScopeSpans scope_spans = 2; }
message ExportTraceServiceRequest { repeated ResourceSpans resource_spans = 1; }
diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts
index fb08b142c6..f33983f14c 100644
--- a/server/src/services/native-runtime/native-session-executor.test.ts
+++ b/server/src/services/native-runtime/native-session-executor.test.ts
@@ -802,6 +802,9 @@ describe("remote runner process supervision", () => {
await writeFile(fixture, 'require("node:net").createServer().listen(0,"127.0.0.1",()=>process.stdout.write("ready\\n"));');
let child: ReturnType<typeof spawn> | undefined;
let probes = 0;
+ let healthyProbes = 0;
+ let releaseExitProbe!: () => void;
+ const exitProbeReady = new Promise<void>(resolve => { releaseExitProbe = resolve; });
let settled = false;
const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };
const execute = vi.fn(async (input: { command: string; args?: string[] }) => {
@@ -817,12 +820,15 @@ describe("remote runner process supervision", () => {
return ok;
}
if (label === "paperclip-runner-monitor" && ++probes === 1) return { ...ok, exitCode: null, timedOut: true };
- return await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => {
+ if (label === "paperclip-runner-monitor" && probes > 2) await exitProbeReady;
+ const observed = await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => {
execFile(input.command, input.args ?? [], { timeout: 2_000 }, (error, stdout, stderr) => resolve({
exitCode: error ? typeof error.code === "number" ? error.code : null : 0,
signal: null, timedOut: !!error?.killed, stdout, stderr,
}));
});
+ if (label === "paperclip-runner-monitor" && observed.exitCode === 0 && !observed.timedOut) healthyProbes++;
+ return observed;
});
const handle = createRemoteRunnerProcessLauncher({
target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: root },
@@ -831,17 +837,21 @@ describe("remote runner process supervision", () => {
})({ command: fixture, args: [], cwd: root, environment: {} });
void handle.completion.then(() => { settled = true; }, () => { settled = true; });
try {
- await vi.waitFor(() => expect(probes).toBeGreaterThanOrEqual(2), { timeout: 5_000 });
+ await vi.waitFor(() => expect(healthyProbes).toBe(1), { timeout: 5_000 });
expect(settled).toBe(false);
expect(handle.isExitConfirmed?.()).toBe(false);
expect(() => process.kill(child!.pid!, 0)).not.toThrow();
expect((await readFile(marker, "utf8")).split("\n")[1]).toBe(String(child!.pid));
expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);
+ const childStopped = new Promise<void>(resolve => { child!.once("exit", () => resolve()); });
child!.kill("SIGTERM");
+ await childStopped;
+ releaseExitProbe();
await handle.completion;
expect(handle.isExitConfirmed?.()).toBe(true);
} finally {
child?.kill("SIGKILL");
+ releaseExitProbe();
await handle.completion.catch(() => undefined);
await rm(root, { recursive: true, force: true });
}
diff --git a/tests/e2e/cases-read-only.spec.ts b/tests/e2e/cases-read-only.spec.ts
index 83f272b46b..fbd98223df 100644
--- a/tests/e2e/cases-read-only.spec.ts
+++ b/tests/e2e/cases-read-only.spec.ts
@@ -28,12 +28,9 @@ test.afterEach(async ({ request }, testInfo) => {
test("browses passive cases and rolled-up task documents without starting work or editing", async ({ page }, testInfo) => {
page.setDefaultTimeout(10_000);
- // Announcements can arrive after navigation and cover otherwise valid links.
- // Dismiss through the ordinary Board control, including after reloads.
- const dismissAnnouncement = page.getByRole("complementary", { name: "Paperclip announcements", exact: true })
- .getByRole("button", { name: "Dismiss announcement", exact: true });
- const dismissAnnouncements = async () => { await dismissAnnouncement.click(); };
- await page.addLocatorHandler(dismissAnnouncement, dismissAnnouncements);
+ await page.route("**/api/announcements/current", async (route) => {
+ await route.fulfill({ json: null });
+ });
const api = page.request;
const original = await json(await api.get("/api/instance/settings/experimental"));
const company = await json(await api.post("/api/companies", { data: { name: `Read-only cases ${randomUUID()}` } }));
@@ -189,13 +186,9 @@ test("browses passive cases and rolled-up task documents without starting work o
await expect(dialog.getByRole("button", { name: /^(edit|save|restore|delete)\b/i })).toHaveCount(0);
await expect(dialog.locator('textarea, [contenteditable="true"]')).toHaveCount(0);
await testInfo.attach("read-only-document-revisions", { body: await page.screenshot(), contentType: "image/png" });
- // The handler's own click moves focus. Keep it out of this focus-restoration
- // assertion, then resume ordinary dismissal before the next navigation.
- await page.removeLocatorHandler(dismissAnnouncement);
await page.keyboard.press("Escape");
await expect(dialog).toHaveCount(0);
await expect(openReceipt).toBeFocused();
- await page.addLocatorHandler(dismissAnnouncement, dismissAnnouncements);
const auditLink = casePage.getByRole("link", { name: "View in Audit", exact: true });
await expect(auditLink).toHaveAttribute("href", `/${company.issuePrefix}/activity?entityType=case&entityId=${receipt.id}`);
await auditLink.click();
diff --git a/tests/e2e/runner-e2e-dashboard.spec.ts b/tests/e2e/runner-e2e-dashboard.spec.ts
index e9e1d8e550..e9cc5c5237 100644
--- a/tests/e2e/runner-e2e-dashboard.spec.ts
+++ b/tests/e2e/runner-e2e-dashboard.spec.ts
@@ -363,3 +363,32 @@ test("case status dots summarize matching results and open failures", async ({ p
await page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }).click();
await expect(topLink).toHaveAttribute("data-case-status", "failed");
});
+
+test("keeps long feedback fingerprints inside a scrollable review table on phones", async ({ page }) => {
+ const { emptyFeedbackSummary } = await import("../runner-e2e/feedback.js");
+ const { feedbackReviewTemplate } = await import("../runner-e2e/feedback-dispositions.js");
+ const execution = runnerExecutionById(executionIds[0]);
+ const entry = resultFor(execution, "passed");
+ entry.result.feedback = { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete",
+ complaints: 2, bodiesCollected: 2,
+ review: ["a", "b"].map(value => ({ fingerprint: value.repeat(64), kind: "complaint", occurrences: 1, disposition: "needs_investigation" })),
+ };
+ const campaign = buildRunnerCampaign({ campaignId: "feedback-mobile", generatedAt: entry.result.finishedAt, expected: [execution.id], results: [entry.result] });
+ campaign.feedbackReview = { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "agent",
+ decisions: [{ fingerprint: "a".repeat(64), disposition: "actionable", issueNumber: 123 },
+ { fingerprint: "b".repeat(64), disposition: "duplicate", duplicateOf: "a".repeat(64) }],
+ };
+ await page.setViewportSize({ width: 390, height: 844 });
+ await openReport(page, renderRunnerE2EDashboard({ title: "Feedback review", generatedAt: campaign.generatedAt,
+ expected: campaign.expected, catalog: [execution], entries: [entry], campaign }));
+ await page.getByText("Campaign usage and spend", { exact: true }).click();
+ const decisions = page.getByRole("region", { name: "Feedback review decisions" });
+ await expect(decisions).toBeVisible();
+ expect(await decisions.evaluate(element => element.scrollWidth > element.clientWidth)).toBe(true);
+ expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);
+ await decisions.focus();
+ await page.keyboard.press("Tab");
+ const followup = decisions.getByRole("link", { name: "#123" });
+ await expect(followup).toBeFocused();
+ await expect(followup).toBeInViewport();
+});
diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md
index e1a34eee98..e499517506 100644
--- a/tests/runner-e2e/FIXTURES.md
+++ b/tests/runner-e2e/FIXTURES.md
@@ -1011,3 +1011,30 @@ The arithmetic journey records four delivered agent handoffs and one completed
terminal intent. The final case is `done` at handoff version 5, while the same
four provider work runs and unchanged workflow document revision finish the task.
The oracle rejects a missing or pending terminal intent.
+
+## Feedback persistence (explicit only)
+
+`--suite feedback-persistence` selects two local native Claude/Codex cells.
+Each creates one standard task through the browser, asks for two distinct
+synthetic feedback items, and requires the real `submit_complaint` and
+`submit_suggestion` tools. The five-minute case expects one task work run and
+disables automatic retries. Company and agent accounted-work budgets are $10.
+Setup probes and any background work must also pass through the separately
+budgeted gateway when using an operations manifest. Local compute is unmetered.
+Cost and duration forecasts remain unmeasured until a retained live attempt.
+The runner E2E maintainers own this manual case; `--all` excludes it.
+
+After collection and fixture teardown, the oracle reads the retained evidence
+file and requires exact nonce-bearing bodies, the original company/agent/task
+and observed run, plus one completed native receipt for each tool. Missing or
+partial collection, prose-only claims, foreign attribution, duplicate records,
+and failed submissions cannot pass. Full bodies stay in private evidence;
+public reports retain counts, fingerprints, and closed oracle checks. The usual
+result screenshot, trace, secret scanning and supervisor cleanup still apply.
+This case proves explicit submission and persistence, not spontaneous feedback
+quality or survival after an abrupt worker kill.
+
+```sh
+pnpm test:e2e:runner -- --list --suite feedback-persistence
+pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1
+```
diff --git a/tests/runner-e2e/OPERATIONS-COVERAGE.md b/tests/runner-e2e/OPERATIONS-COVERAGE.md
new file mode 100644
index 0000000000..da4e2efa6e
--- /dev/null
+++ b/tests/runner-e2e/OPERATIONS-COVERAGE.md
@@ -0,0 +1,232 @@
+# Eval coverage and operating costs
+
+Use this with [OPERATIONS.md](OPERATIONS.md). The bounded daily and weekly
+selections cover Product E2E only. They do not authorize or cap the separate
+Runner protocol workflow, manual matrix runs, or other live probes.
+
+## Initial cadence
+
+The runner E2E maintainers own these selections and their failure triage.
+Both profiles use the exact model and limits in the reviewed pricecard.
+
+| Journey | Failure detected | Feature-map association | Cadence |
+| --- | --- | --- | --- |
+| Native Claude message marker | A real task cannot complete and persist its expected answer through the metered route | `runs-adapters/runs` | Daily and weekly |
+| Native Claude complaint and suggestion | Completed tools fail to save attributed feedback or collection loses it | `activity/agent-commentary` | Daily and weekly |
+| Native Codex message marker | The corresponding Codex task/connection path fails | `runs-adapters/runs` | Weekly |
+| Native Codex complaint and suggestion | The corresponding Codex feedback/tool path fails | `activity/agent-commentary` | Weekly |
+
+These associations identify partial coverage. They do not qualify every run
+lifecycle, the Commentary browser page, continuation, or remote execution.
+The October 21 local pilot measured one successful sample per selected cell:
+$0.427624800 and 72.757 summed attempt seconds for the daily selection,
+$0.791539400 and 127.069 seconds for the weekly selection. These are historical
+gateway estimates from the Linux ARM64 container. They include model setup
+requests and attempt setup/teardown, but omit queue/build time and non-model
+charges. Earlier failed and interrupted attempts remain in campaign spending.
+
+`scheduled-measurements.json` retains the exact attempt, source revision,
+definition, model, price version, result hash and journal hashes for each sample.
+`forecast.json` includes this evidence and a snapshot hash. It separates the
+historical successful selection from samples matching the current definition,
+model, environment, runtime, route and pricing. A missing matching cell makes
+the compatible selection total unknown. The feedback definition changed after
+these four samples; its earlier results are not current qualification.
+
+Per-cell mean and median are descriptive. P95 stays unknown below 20 samples
+and then uses nearest rank; this is not a statistical confidence guarantee.
+Do not sum per-cell percentiles into a campaign percentile. Hosted cost/time
+and the selection's P95 remain unknown until separately measured. A passing
+sample does not predict failure frequency or account for infrastructure,
+storage, observability or subscriptions. Forecasts never reduce reservations
+or the approved budget limits.
+
+At the October 21, 2026 catalog audit, 47 suites declared 738 cells. `--all`
+included 179; the other 559 required explicit selection. The bounded weekly
+selection included four cells, and daily two. These are dated inventory counts,
+not live acceptance results or an enduring catalog-size contract. Inspect the
+current `pnpm test:e2e:runner -- --list` before choosing work. In particular,
+Everyday Workflows, provider connections, feedback persistence, interruption,
+and case handoffs must not be inferred from a passing `--all` campaign.
+
+Keep the broader rotating lane disabled until its exact cells have a metered
+route, complete retained attempts, a measured cost/time distribution, and a
+budget that includes failures. Prefer targeted continuation, approval, and
+cleanup cases that detect different failures over another model running the
+same marker. A release lane may broaden coverage only with its own reviewed
+manifest; the daily/weekly allowance does not authorize it.
+
+## Adding an eval or changing its cadence
+
+The change's fixture documentation and PR must declare:
+
+- A responsible maintainer/team and the concrete failure the case detects.
+- Feature-map capability IDs, or an explicit unmapped gap. State what the
+ journey does not exercise within a broader capability.
+- Exact suite, case, profiles, models, environment, and intended cadence.
+ State whether it is explicit-only and how it differs from existing coverage.
+- A deterministic alternative considered, and why paid behavior is necessary.
+- Attempt, request, output-token, concurrency, retry, and wall-time limits.
+ Include setup, judges, child/background work, and cleanup in the route audit.
+- Estimated cost/time before the first run, then all-attempt measurements with
+ sample count, retained failures, source/definition IDs, and uncertainty.
+ An unmeasured field is unknown, never zero.
+- Feedback collection/review, trace verification, missing-receipt handling,
+ and a failure/cleanup owner. Novel complaints are reviewed independently
+ of the product grade.
+
+Use the existing [Product E2E authoring skill](../../.agents/skills/add-product-e2e-eval/SKILL.md)
+or the separate [Runner authoring skill](../../.agents/skills/add-runner-eval/SKILL.md).
+Update the existing scheduled selection and forecast only after qualification;
+adding a catalog entry alone must not enable recurring spending.
+
+## Reading campaign time
+
+The report's summed attempt time uses every retained accounting receipt,
+including failed retries. A missing attempt makes the total unknown; the report
+keeps the observed subtotal and missing count. The observed attempt window runs
+from the earliest retained start to the latest retained finish. Parallel attempts
+overlap in that window, while their individual durations add to the sum. Gaps
+between attempts remain in the window. Carried-forward evidence can span older
+runs, so this window is not the current workflow's duration.
+
+Agent execution and Daytona lease time remain separate measurements. Attempt
+durations include the setup and cleanup inside their recorded boundaries.
+New browser-worker results retain a separate phase receipt for fixture setup,
+execution, feedback capture, cleanup and worker finalization. Repeated cleanup
+segments add together. Atomic checkpoints preserve completed phases if the
+worker is interrupted; the active phase has no inferred end time. Parent
+recovery checks identity, interval bounds and file size before retaining that
+partial receipt. Missing, malformed or unavailable receipts keep phase totals
+unknown, while closed segments remain visible as observed subtotals.
+
+These phases start inside the test callback, after browser/server startup, and
+end before result publication. Parent teardown/finalization, CI queue/build time
+and provider-billed runtime need their own receipts. Historical results and
+non-browser launchers without a worker receipt remain explicitly unmeasured.
+Test details show each selected attempt's receipt status and raw timing metadata.
+Do not infer missing measurements by subtracting summed agent time from the
+observed window. Refreshing a report computes these views from retained
+results without making provider calls or changing the original measurements.
+
+## Costs outside the model journal
+
+Keep a separate cost component for each resource, with its currency, service,
+resource/request identity, source window, allocation basis, and status. Join
+components to campaign/attempt IDs where evidence permits. Do not sum a model
+estimate with the invoice that later reconciles that same usage.
+
+| Component | Evidence and allocation | When evidence is missing |
+| --- | --- | --- |
+| Provider models and paid tools | Dedicated project/key usage and billing exports; match provider request IDs or the narrowest available billing window | Preserve estimates, unknown requests, held reservations and the reconciliation difference |
+| Daytona | Owned sandbox ID, actual resources, billable lifecycle and provider receipt; include failed creates, retries, stopped storage and delayed deletion | Retain the reservation or an explicitly partial estimate; a requested TTL does not prove deletion |
+| CI compute | Workflow/job IDs, runner type, actual billed runtime and invoice rates; include setup, builds and failed jobs | Mark CI cost unknown; a short agent turn does not imply a cheap CI job |
+| Storage and transfer | Artifact size, retention window and billed storage/egress | Report the unpriced component, including retained failure evidence |
+| Braintrust/gateway usage | Project usage export and its window; distinguish included allowance from incremental overage | Do not treat free-looking trace metrics as proof of zero provider charges |
+| Shared subscriptions | Show the existing Braintrust Pro $249/month separately under the October 21 operator policy; allocate newly incurred subscription/capacity charges from invoices with an explicit denominator | Keep new shared charges unallocated rather than assign an invented exact per-test cost |
+
+Retain both the service total and the allocation denominator so allocated
+amounts can be reproduced. A zero incremental usage charge can coexist with a
+nonzero shared subscription. Never add the entire subscription to every test.
+The approved daily/weekly budgets cover additional run costs, so they exclude
+the existing $249/month fee while including new usage and capacity charges.
+Scheduled admission now reserves the declared CI, storage, observability and
+subscription allowances inside the same $20/$100 journal limit. Missing or stale
+allowances stop scheduling before the AWS test runner starts. Missing receipts
+keep their reservations held and accounting incomplete. This is a local admission
+control, not a provider billing cap. Actual resource bounds, cost collection and
+reviewed allowances still need qualification before recurrence is enabled; see
+[the allowance contract](OPERATIONS.md#recurring-campaigns).
+
+Recurring evidence and reports use private S3 archives with file/byte/request
+boundaries, retained transfer receipts and verified access/expiration rules;
+see [private recurring history](OPERATIONS.md#private-recurring-history). The
+public workflow retains only a content-free availability marker for monitoring.
+Public history is a separate explicit publication step. Private retention still
+needs hosted OIDC qualification and actual storage/transfer billing evidence;
+local archive tests and an infrastructure template do not establish those facts.
+
+## Separate Runner Eval audit
+
+The October 21 audit inspected Paperclip source on master `8b873e0112` plus the
+eval operations stack, and the available `paperclip-evals` checkout at
+`0bae4e0947dc7959cae25b73d27533c096f23a14`. This is a source audit, not a
+new live run or proof of the workflow's configured private-repository revision.
+
+Runner reports already retain immutable attempts, retry-inclusive estimated
+cost and provider list cost as separate views, compatible-cohort comparisons,
+and missing-usage coverage. Its mock control plane is a different authority
+from the Product E2E server/database; Product feedback persistence cannot be
+claimed from a Runner tool assertion.
+
+The private `run_live_campaign.py` requires an explicit authorization envelope.
+The hosted `runner-protocol-live-evals.yml` instead fans out individual
+`run_live_roster.py` cells, supplies their direct provider credentials, and uses
+configured per-attempt limits. It does not use the Product E2E shared gateway
+journal, Braintrust attempt parent, indexing verifier, or feedback finalizer.
+The private campaign wrapper's aggregate authorization must not be assumed to
+protect that different hosted path.
+
+Before adding Runner recurrence, its owners must establish an aggregate
+allowance across fanout and retries, meter setup/provider bypasses, and qualify
+native tracing through its actual packaged runner. Preserve direct-provider
+qualification as its own route. Implement definition/roster changes in the
+maintained private repository, pin that reviewed revision in the existing
+workflow, and retain Runner's existing Evalbook renderer and scores. These
+gaps remain open; Product E2E qualification does not close them.
+
+## Missing campaign detection
+
+Workflow failure notifications cover runs that start. The independent observer
+is `scripts/eval-schedule-monitor.mjs`. Run it through a separate host with an
+authenticated GitHub CLI and one private state file:
+
+```sh
+node scripts/eval-schedule-monitor.mjs "$HOME/.config/paperclip-evals/schedule-monitor.json"
+```
+
+The observer uses only GitHub GET requests. It never starts or retries paid
+work and does not need provider credentials. It reads the two existing enable
+gates plus `RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT`, an ISO timestamp set
+when recurrence is enabled. Missing activation time fails visibly; occurrences
+before activation are not treated as missed. Workflow run titles carry the
+literal cron expression to distinguish daily and weekly runs, including Sunday.
+
+Only the legacy `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED` gate being off establishes
+disabled recurrence. If that gate is on while the operations gate is absent or
+off, the observer reports `configuration_incomplete` and retains one incident
+notification. Older deployed workflows use only the legacy gate, so a missing
+new gate must not imply that those paid schedules are disabled. Verify the
+deployed default-branch workflow and both variables before enabling recurrence.
+
+The observer allows 90 minutes for schedule delay. It checks the most recent due
+occurrence per cadence, marks a run overdue after four hours, and requires a
+successful terminal run plus a nonempty, unexpired merged-report artifact.
+The workflow's finalizer remains responsible for checking report content and
+accounting. Metadata inspection does not repeat that validation. Unexpected
+reruns, ambiguous matching runs, incomplete API lists and API failures remain
+visible. The observer reads at most 100 rows per collection and fails closed
+if a response is truncated.
+
+Retained state includes the last successful inspection and inspected run IDs.
+`notifications` is a durable outbox of problems and recovery transitions. Each
+entry has an ID. After confirmed delivery, the host acknowledges those IDs:
+
+```sh
+node scripts/eval-schedule-monitor.mjs "$HOME/.config/paperclip-evals/schedule-monitor.json" --ack <notification-id> ...
+```
+
+Until acknowledgment, later inspections return the same pending entries even
+if the problem has recovered. The host uses IDs to deduplicate delivery. This
+survives a process exit between saving state and printing output; it does not
+claim exactly-once delivery across an external notification service. After
+acknowledgment, an unchanged problem stays quiet. A new missed occurrence is a
+new incident. Disabled recurrence and unchanged healthy runs create no new
+alerts. Use one writer for both checks and acknowledgments; do not reset the
+state file to clear an incident. Malformed history fails before a query or
+write. The external host owns notification delivery and its own liveness.
+
+Schedule this observer outside the paid workflow after source and recurrence
+are ready. It should inspect at least daily, after both scheduled start times
+and their grace windows. Until that external trigger and notification delivery
+are configured and exercised, missed-run detection remains an activation gap.
diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md
new file mode 100644
index 0000000000..31b36ed856
--- /dev/null
+++ b/tests/runner-e2e/OPERATIONS.md
@@ -0,0 +1,707 @@
+# Bounded eval operations
+
+The request ledger and gateway are opt-in. The ordinary launcher still supports
+unmetered legacy campaigns. Do not infer a spending limit from a campaign ID.
+Enable the bounded path with `PAPERCLIP_E2E_OPERATIONS_MANIFEST=/absolute/file.json`.
+Keep this manifest, its journal, and request receipts private.
+
+The [coverage and operating-cost guide](OPERATIONS-COVERAGE.md) records cadence
+ownership, suite-growth requirements, non-model allocation, the separate Runner
+Eval audit, and remaining activation gaps.
+
+## Supported admission
+
+The initial path admits local `core-compatibility` and `feedback-persistence` cells for `runner-codex`,
+`runner-acpx-claude`, `legacy-codex`, and `legacy-claude`. Each cell must have one
+provider credential and an exact priced model. Other suites, remote environments,
+subscription profiles, hosted tools, multimodal inputs, premium service modes,
+and separate judges fail admission. This whitelist is implementation scope, not
+evidence that every admitted harness has passed live qualification.
+
+Before creating the journal or requesting credentials, manifest validation checks
+that every selected model's worst-case single-request reservation fits the initial
+campaign allowance after declared operating holds. Prices for unused models do
+not affect this selection check. Passing it does not promise the whole campaign
+will fit: later requests still use the journal's current commitments and limits.
+
+Bounded daily and weekly workflows check source-byte limits before upload: 16 MiB
+for the resolved lockfile, 1 GiB for the build bundle, and 256 MiB each for cell
+evidence and the merged report. Build artifacts expire after one day; the others
+expire after 30 days. These are source-byte and retention controls, not measured
+compressed storage charges. ZIP framing and platform billing need separate receipts.
+
+Evidence and reports are copied into fresh private staging directories. Up to
+4,096 directory entries are inspected. Discovery visits accounting and snapshot
+directories across attempts before archive and Playwright diagnostic directories.
+Accounting under `scheduled/` and JSON records have copy priority over logs,
+images and archives. Each retained file has a
+SHA-256 digest. The staging receipt reserves 8 MiB inside the total and identifies
+missing, changed, unsafe or oversized files and incomplete traversal. Originals
+are never deleted or rewritten. Hidden paths remain excluded. The expected regular
+`playwright-output/.last-run.json` bookkeeping file is outside the retained evidence
+scope and does not fail retention. Other hidden paths make the scope incomplete.
+Skipped paths and unvisited subtrees are named; overlong names use a bounded preview
+and full-path digest. Partial staging is uploaded before the workflow
+fails, so a storage limit cannot silently turn missing evidence into success.
+These upload limits do not cap runtime disk writes or Actions caches. Each service
+still needs billing evidence.
+
+### Private recurring history
+
+Daily and weekly evidence and merged reports are retained in a dedicated private
+S3 bucket. They are not uploaded as plaintext GitHub artifacts, sent to the public
+history publisher, or deployed to Pages. The report command's output also stays
+inside the private report. The public workflow retains only a fixed availability
+marker for the existing schedule monitor, with no report contents, storage address
+or object identity. Public-repository GitHub artifacts are readable by signed-in
+repository readers; authentication alone does not make those reports private.
+
+Deploy `.github/infra/runner-e2e-private-history.yml` with the existing GitHub OIDC
+provider and set `RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET`,
+`RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN` and
+`RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION`. The role trusts only this repository's
+`runner-e2e-private-paid` and `runner-e2e-private-history` environments. Create both
+with a deployment branch policy allowing only the default branch. Reserve them
+for this workflow: the shared `runner-e2e-paid` and `runner-e2e-history` environments
+also serve protocol evaluations that execute target-branch code and must not
+have access to the private archive role. Set the three archive variables in both
+dedicated environments. This workflow separately requires the default branch before
+paid access and reporting. Manual target-branch tests have no OIDC permission. A separate recurring
+job receives that permission only when the target SHA equals the default-branch
+workflow SHA. Both jobs reuse the same anchored setup and test steps. The role has configuration reads
+and GetObject/PutObject under `runner-e2e-private/bounded-v1/`; no deletion,
+policy changes or access to the public history bucket. AWS credentials are step
+outputs supplied only to private transfer commands, never exported to paid agents.
+
+Admission verifies retention before paid execution. Every transfer verifies all
+four public-access blocks, disabled object ACLs, no bucket-policy reader grants,
+versioning, the 30-day/current and one-day/noncurrent expiration rules, and absence
+of Object Lock and replication. Inspection failure stops the transfer. The private
+bucket is not a CDN origin and has no external reader grant.
+
+Evidence and report archives each contain at most 256 MiB of staged files and
+4,097 entries including the staging receipt, with an additional 8 MiB ceiling for
+ZIP framing. Archives use stored entries, without decompression. Extraction checks
+paths, entry types, duplicate names, directory size, payload totals and ZIP checksums
+before committing a fresh destination. Transfers use one conditional PutObject or
+one ranged GetObject. Automatic AWS retries and workflow reruns are refused. The
+bucket also rejects PutObject requests without an If-None-Match condition. A local
+transfer receipt cannot be overwritten by rerunning the command in the same directory.
+Each command retains admission and outcome metadata. After private storage passes
+inspection, completed and failed transfers each get one independent receipt-upload
+attempt without repeating the original transfer. Successful receipts include the
+archive hash and object version; lost responses remain failed or unknown. A failed
+privacy inspection prevents all uploads, and an unsuccessful receipt upload can
+leave only the local receipt. Its own upload remains pending in the snapshot,
+so the snapshot is not proof of complete billing. Operator downloads and STS
+credential exchanges also need separate cost attribution.
+
+For authenticated retrieval, set the bucket variable and AWS profile, then run:
+
+```sh
+node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts download report RUN_ID /tmp/eval-RUN_ID
+```
+
+Use `evidence` instead of `report` for the original staged accounting and diagnostics.
+The requested destination must not already exist. Keep the downloaded report private.
+To publish a selected campaign, review its retained normalized report and invoke the
+existing trusted history publisher explicitly with the public destination; no agent
+rerun is needed. Recurring execution never invokes that publication path.
+
+Reporting and publication install only the committed lockfile from the trusted
+workflow revision, with lifecycle scripts disabled and a job-local package store.
+They never resolve new dependency versions while the job has OIDC access. If
+that revision has an outdated lockfile, reporting fails before accessing the
+archive. Use the normal master lockfile maintenance process; do not bypass the
+frozen install in the credential-bearing job.
+
+### Public history retention
+
+Public history remains an explicit manual operation. The optional bounded publisher
+uses `<history-prefix>/bounded-v1/` when `PAPERCLIP_E2E_BOUNDED_HISTORY=true`; existing
+manual history otherwise retains its original prefix. Bounded publication is limited
+to the original CI workflow attempt and is not retried under the same allowance.
+
+Before any upload, the publisher validates all outgoing bundles: at most 64 MiB
+and 256 files for the immutable campaign including its manifest, 16 MiB and eight
+files for mutable history, and 64 MiB for the Pages bundle. History JSON reads
+are limited to 4 MiB. The active index keeps at most 64 campaigns from the last
+28 days and rebuilds latest/green pointers from those records. If publication
+stops, the static index can retain links that later expire.
+
+The bucket must have versioning enabled and exact prefix lifecycle rules for
+30-day current-version expiration, one-day noncurrent-version expiration without
+a retained-version floor, expired delete-marker removal and one-day incomplete
+multipart cleanup. The publisher rejects overlapping transitions, different
+expiration policies, Object Lock and replication. Access denied is a failure,
+not proof that a feature is absent. Its role needs `s3:GetBucketVersioning`,
+`s3:GetLifecycleConfiguration`, `s3:GetBucketObjectLockConfiguration` and
+`s3:GetReplicationConfiguration` on that bucket. Configuration is verified at
+each publication; it cannot guarantee that an operator will preserve it afterward.
+
+The publisher admits at most 280 S3 CLI commands over ten minutes, with a
+30-second command timeout and automatic AWS retries disabled. Uploads use
+single `PutObject` operations in STANDARD storage; immutable objects use
+`If-None-Match`. A private receipt is synced before each command and retained
+locally for the operator to retain privately. Unknown responses remain unknown. Source bytes, S3
+commands and expiration eligibility are not invoices or exact deletion times.
+Credential lookup, public downloads/CDN egress, Pages hosting and shared service
+charges require separate accounting; these limits do not cap public traffic.
+
+The launcher owns one loopback HTTP gateway per attempt. The real provider key
+stays in the parent launcher. Playwright receives an ephemeral token, and fixture
+setup saves that token through a shared AI connection with the existing custom
+Responses/Messages routing contract. The agent does not receive ambient provider
+keys. The server environment strips the gateway token and upstream credentials.
+
+Every upstream attempt reserves its maximum estimated charge in an fsynced
+journal before dispatch. Requests, retries, and cancelled calls retain distinct
+IDs. Unknown usage keeps the reservation held. One shared writer serializes
+admission across cells, so concurrent requests cannot each spend the same remaining
+allowance. Campaign deadline, request count, and in-flight count are also enforced.
+Private request receipts retain transport IDs, response IDs, usage and cache TTLs,
+route/cache/failover evidence, and final accounting status without request bodies
+or credentials. Raw provider responses remain only in the normal private evidence.
+
+## Manifest
+
+A manifest contains no credentials. This structural example uses fixture prices;
+replace them with independently verified model prices and context limits before
+running a real campaign. The launcher does not discover current prices for you.
+
+```json
+{
+ "version": 1,
+ "limits": {
+ "campaignId": "qualification-example",
+ "maxUsd": "100",
+ "deadlineAt": "2026-10-20T23:00:00Z",
+ "maxRequests": 200,
+ "maxConcurrent": 1
+ },
+ "ledgerPath": "/private/path/qualification/ledger.jsonl",
+ "executionIds": ["EXACT_SELECTED_EXECUTION_ID"],
+ "maxAttemptsPerExecution": 2,
+ "route": "direct",
+ "models": [{
+ "provider": "anthropic",
+ "maximumInputTokens": 1000000,
+ "maximumOutputTokens": 16384,
+ "timeoutMs": 180000,
+ "price": {
+ "provider": "anthropic",
+ "model": "EXACT_SELECTED_MODEL",
+ "version": "verified-pricecard-date",
+ "sourceUrl": "https://platform.claude.com/docs/en/about-claude/pricing",
+ "inputUsdPerMillion": "2",
+ "cachedReadUsdPerMillion": "0.2",
+ "cacheWrite5mUsdPerMillion": "2.5",
+ "cacheWrite1hUsdPerMillion": "4",
+ "outputUsdPerMillion": "10"
+ }
+ }]
+}
+```
+
+`executionIds` must equal the launcher's selected set, and the retry option must
+fit `maxAttemptsPerExecution`. Use `--list` to resolve the selection first.
+Operations records this attempt cap in the journal's immutable limits and fsyncs
+each attempt admission before starting its gateway. The count is per execution
+across the whole shared journal, including restarts, renamed output campaigns,
+and attempts that fail before making any provider request. Changing the cap or
+reusing an admitted attempt ID is refused.
+The maximum input value is the verified provider context ceiling, **not** a
+requested input cap. The gateway clamps output to `maximumOutputTokens`; it does
+not tokenize or truncate prompts. The conservative reservation prices the whole
+input ceiling at the highest cache-write/long-context rate, plus capped output.
+A small daily budget may therefore admit Claude but refuse Codex even when the
+expected charge is small. Do not reduce the context ceiling to bypass this guard.
+
+For modern OpenAI models, specify both `cacheWrite30mUsdPerMillion` and
+`cacheWrite12hUsdPerMillion`. Optional `longContext` contains `aboveInputTokens`
+and a complete `rates` object with the same token-rate fields. Above that total
+input threshold (including cache), the estimate uses those rates for the entire
+request. Reservations use the maximum rate across both tiers. Missing cache-write
+counters, ambiguous TTL splits for nonzero writes, and model mismatches remain
+unknown. Older OpenAI models without these usage fields need separate qualification.
+Write prices are total token prices, not surcharges. Sources:
+[OpenAI caching](https://developers.openai.com/api/docs/guides/prompt-caching),
+[OpenAI prices](https://developers.openai.com/api/docs/pricing), and
+[Claude prices](https://platform.claude.com/docs/en/about-claude/pricing).
+
+For `gpt-5.6-sol`, a response that omits the entire write-TTL breakdown can use
+the actual request's documented implicit cache policy: `prompt_cache_options.ttl`
+or its 30-minute default. This fallback requires a visible input, no explicit
+breakpoints, no stored prompt/conversation/response references, and no compaction.
+The private receipt records `cacheWriteAttribution` with the policy version and
+lifetime. These amounts remain estimates. Provider splits take precedence;
+partial or malformed splits are never repaired. `prompt_cache_retention: "24h"`
+does not imply a 12-hour write lifetime. The gateway does not change cache policy,
+and this fallback does not reprice historical requests or release old holds.
+
+For Braintrust, use `route: "braintrust"`, add the exact `projectId` UUID, and
+provide `BRAINTRUST_API_KEY` securely. Configure project providers named
+`OPENAI_API_KEY` and `ANTHROPIC_API_KEY`. Disable default provider fallback in the
+project. The request pins that endpoint and project, disables gateway response
+caching, and never automatically retries or requests fallback. A cache hit,
+failover header, or unexpected successful endpoint invalidates the receipt and
+holds its reservation. Provider prompt caching stays enabled and is metered.
+The response stream is passed through and cumulative usage is counted once.
+Gateway and native tracing are separate opt-ins; see the README tracing section.
+
+When both a bounded operations manifest and explicit Braintrust tracing are
+configured, the launcher starts a separate loopback OTLP relay for each attempt.
+The worker and its native providers receive an expiring ingest token. The
+Braintrust service key stays in the controller, even when model requests use
+Braintrust without native tracing. The relay accepts only trace/log exports to
+the configured project; neither request headers nor payload parent metadata can
+select another project or an API endpoint. JSON is canonicalized before export;
+protobuf payloads retain their original fields after bounded validation, which
+rejects ambiguous singular fields and routing values.
+
+Each relay allows at most 2,048 admitted exports, 8 MiB per request before and
+after decompression, 64 MiB of forwarded payloads, and four simultaneous exports.
+It also limits each attempt to 16,384 records and 256 MiB of expanded source
+JSON. This measurement includes the full decoded source plus each record in
+its own envelope with the shared resource and scope fields repeated. It counts
+span names, IDs, times, events, links and log bodies, not only routing attributes.
+Protobuf fields use the pinned OTLP v1.9 schema; unknown fields fail admission
+until the schema is updated. Existing valid payloads are forwarded unchanged.
+It closes at the campaign deadline or four hours after startup, whichever comes
+first. Admission consumes the allowance before upstream dispatch. Failures and
+client retries do not refund it; the relay itself never retries. A durable receipt
+must be written before dispatch and before acknowledging the receiver outcome.
+
+Version 2 private `<ledgerPath>.otlp/<attempt-hash>.jsonl` receipts record policy,
+admission time, wire/payload bytes, payload digest, dispatch/outcome, partial
+rejection counts, record counts, expanded source bytes, and closure totals.
+Version 1 receipts remain readable with unknown expanded size and record count.
+They contain no payload bodies, tokens or
+upstream error text. Receipt files are exclusive and cannot be reset on restart.
+An admission without an outcome is unresolved. Accepted ingestion does not prove
+indexing, and attempted forwarding bytes are not billed Braintrust storage.
+These receipts do not replace the observability overhead reservation or invoice
+reconciliation. After the worker relay closes, bounded controller finalization
+uses a separate one-request, 64 KiB relay with a 30-second deadline. Its exclusive
+expanded-source allowance is 1 MiB with at most 256 records. Its
+receipt is `<ledgerPath>.otlp/<attempt-hash>-finalization.jsonl`. This preserves
+the original trace and grade while accounting for the final controller export.
+This path currently supports the admitted local cells; remote transport remains
+unqualified.
+
+Scheduled experiment creation, grade insertion and summary retrieval share a
+60-second window, three-request cap, 4 MiB total payload allowance and eight-score
+cap. They run serially without automatic retry. Admission is persisted before
+dispatch in `<ledgerPath>.experiment-ingestion.jsonl`; uncertain responses retain
+their allowance and unresolved status. Reusing that file fails instead of
+resetting the budget. These limits cover the selected daily/weekly campaigns;
+the standalone experiment-import command does not use this transport budget.
+
+Private `ingestion-usage.json` binds all three receipt paths to the campaign's
+admitted attempts and ledger hash. Missing, malformed, unclosed or unresolved
+receipts leave the total unknown while preserving readable partial measurements.
+These are admitted transport payload bytes, not Braintrust processed-data units
+or an invoice. Vendor charges remain unreconciled. The current two-attempt daily
+maximum is 132.125 MiB and the four-attempt weekly maximum is 260.25 MiB,
+including finalization and the campaign's experiment payload allowance. These
+bounds do not include other projects, standalone imports or retained storage.
+The corresponding expanded-source ceilings are 514 MiB daily and 1,028 MiB
+weekly before the separate 4-MiB experiment allowance. These source measurements
+support a priced forecast; they are not Braintrust's processed-data meter or a
+vendor-enforced dollar limit. Gateway auto-logging is a separate input to that
+forecast. Retain vendor usage reports for reconciliation rather than equating
+transport or expanded source bytes with an invoice.
+
+## Recurring campaigns
+
+The existing `runner-full-stack-e2e.yml` owns both schedules: daily at 08:47 UTC
+and weekly on Sunday at 09:47 UTC. Scheduled runs require both
+`RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true` and
+`RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED=true`. Leave the operations gate unset
+until the workflow, billing coverage and infrastructure allowance are qualified.
+There is no separate scheduler. The protected `runner-e2e-private-paid` environment needs
+`BRAINTRUST_API_KEY`; repository variable `RUNNER_E2E_BRAINTRUST_PROJECT_ID` names
+the exact project UUID. The key is available only to the bounded execution and
+finalization step, after dependency installation and immutable builds.
+
+Native Claude gateway cases also require the pinned Claude CLI for the managed
+account's setup hello probe. The agent still runs through the pinned ACP runtime.
+The workflow installs and checks the CLI before providing credentials for both
+native and legacy Claude selections, including the aggregate scheduled selection.
+For a local campaign, verify that same CLI prerequisite before starting a paid
+attempt. Installing the ACP package alone does not satisfy this setup path.
+
+Bounded cases keep the gateway in the setup phase until the created agent's
+asynchronous lifecycle verification reaches ready. This wait uses the remaining
+attempt deadline, allows at most 45 seconds, and stops on an observed setup error
+or hold. The private `budgeted-setup.json` snapshot retains state transitions.
+Only then does the case begin agent-phase accounting and create its task.
+Older evidence that advanced after fixture creation may label hello probes as
+agent cost. Its campaign totals still include those requests; retain that phase
+limitation when comparing setup and task spending.
+
+Attempt wall time includes setup and teardown. Agent time is the sum of captured
+heartbeat-run windows, which can overlap. Missing runtime evidence, no captured
+runs or an invalid run timestamp leaves agent time unknown; the report never substitutes attempt
+wall time. A campaign total stays unknown if one of its retained attempts lacks
+that measurement. A measured zero remains zero. Missing or invalid Daytona lease
+windows likewise leave lease time and its runtime estimate unknown. Local attempts
+do not contribute Daytona lease time. Existing published numeric measurements are
+retained as historical evidence; this change does not reconstruct their provenance.
+
+Bounded daily/weekly campaigns require a public repository and use the existing
+standard `ubuntu-latest` runner for both compilation and execution. They do not
+follow the manual campaign's AWS runner switch. GitHub charges no standard
+hosted compute minutes for public repositories; retain runner/job evidence and
+recheck the [billing policy](https://docs.github.com/en/billing/concepts/product-billing/github-actions)
+when reviewing the allowance. This does not establish zero artifact, cache,
+external history, observability or subscription cost. Manual selections keep
+the existing AWS/GitHub runner choice.
+
+Daily selects native Claude's marker and feedback-persistence cases with a shared
+$20 allowance. Weekly adds native Codex's two corresponding cases with a
+shared $100 allowance. Model requests and declared operating overhead share
+this limit. Each campaign runs sequentially in one job with one
+journal, one attempt per execution, zero automatic retries, a 30-minute request
+deadline and a finite request count. Codex's worst-case request reservation exceeds
+$20, so it is absent from daily selection. Its context ceiling is unchanged.
+The $20/$100 limits cover additional costs caused by each run. Show the existing
+Braintrust Pro subscription separately at $249/month (operator policy confirmed
+October 21, 2026); do not allocate it into these run limits. New processing,
+scoring, retention or capacity charges still count toward the run allowance.
+The monthly run allowance is at most $1,120 (31 daily plus five weekly runs),
+or $1,369 including the separately reported existing subscription. These are
+maximum allowances, not forecasts of actual spending.
+This is an admission limit with retained reservations, not a reconciled invoice
+total or a provider-enforced billing limit.
+
+Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to a reviewed
+allowance with `version: 1`, an evidence `reference`, the actual `runner`
+(`github-hosted` for bounded schedules), and `verifiedAt`/`expiresAt` timestamps no more
+than 31 days apart. Its `components` array must contain exactly `ci_compute`,
+`artifact_storage`, `observability`, and `shared_subscription`. Each component
+needs a USD-string `maxUsd` and its own evidence `reference`. References are
+stable audit IDs or paths, not credentials or arbitrary prose. No default
+amounts are supplied. The supporting audit must include rates, resource/time
+bounds, included allowances, allocation denominators, and the controls that
+keep actual resource use within those bounds. Zero requires evidence of an
+included or zero incremental charge; setting a zero allowance is not a receipt.
+Under the current policy, `shared_subscription` may reserve zero for the
+existing Braintrust subscription with a reference to that policy and current
+plan evidence. Any newly incurred subscription or capacity charge needs its own
+incremental allocation; the existing fee must remain visible separately.
+
+The catalog checks that this allowance matches the selected runner, covers at
+least the next four hours, and leaves room for a full conservative model-request
+reservation. This happens before the build and test runners start. The workflow passes
+the exact validated allowance to preparation; it does not read a potentially
+changed repository variable again. Preparation rechecks it and retains it in the
+manifest and forecast. Setup, builds, failed jobs, publication and retained
+artifacts must all fit the declared components, not just the agent's runtime.
+
+The journal initializes `overhead:<component>` reservations under
+`campaign-overhead` from its immutable limits before admitting model requests.
+These are allocation holds, not dispatched requests or completed resource
+lifecycles. They do not occupy model concurrency, but they count toward the
+finite charge-record limit. Resuming cannot change or drop them. Until matched
+cost observations arrive, the report shows them as unknown and keeps their
+allowance held; even a zero allowance does not claim a known zero charge.
+The existing finalizer reports incomplete accounting while these receipts are
+missing. Do not dismiss that finding to enable recurrence.
+
+Reconcile a component through its existing request ID with provenance-bound
+`infrastructure` or `invoice` observations. Partial observations keep the
+remaining reservation held; a final nonconflicting observation can replace it.
+Never mark a receipt final before its resource and billing window are complete.
+An external bill above the allowance remains visible and stops further model
+admission. The journal cannot stop AWS, storage or subscription billing itself;
+provider resource limits and a verified receipt collector remain activation
+requirements. No operating-cost allowance has been configured or qualified by
+the presence of this code.
+
+Collect retained GitHub usage after the workflow completes, including failed or
+cancelled workflows, without dispatching or retrying anything:
+
+```sh
+node --import tsx tests/runner-e2e/operating-receipts.ts github RUN_ID /private/path/github-usage.json
+```
+
+The collector uses the existing `gh` authentication and only reads
+`paperclipai/paperclip`. It verifies the Product E2E workflow, first attempt,
+revision and repository, reads every returned job/artifact page up to 1,000
+records per inventory, and rejects changed or incomplete inventories. It retains
+job identities, summed job seconds, run-start delay, artifact sizes and expiration
+dates. It omits arbitrary job names, artifact names and upstream responses.
+Existing output files cannot be replaced. This is a current inventory, not an
+invoice: deleted artifacts are absent, compressed bytes are not billed
+byte-hours, job duration is not VM lifetime, and present repository visibility
+does not establish visibility throughout an earlier run. Caches and external
+S3 history remain separate. A visible public-hosted zero compute rate never
+automatically releases an overhead reservation.
+
+Import a reviewed cost receipt and its exact source evidence into an existing
+campaign's declared overhead reservation:
+
+```sh
+node --import tsx tests/runner-e2e/operating-receipts.ts import /private/path/ledger.jsonl /private/path/receipt.json /private/path/source-evidence.json
+```
+
+The receipt contains `version: 2`, `costBasis: "component_cumulative"`,
+`campaignId`, `allowanceReference`, `component`,
+the source file's `evidenceSha256`, `windowStartedAt`, `windowEndedAt`, `coverage`
+(`partial` or `complete`), `source` (`infrastructure` or `invoice`), `provider`,
+`classification`, and an explicit USD-string `costUsd` when known. Estimated and
+allocated charges also require a stable `pricingVersion`. `reconciled` requires
+an invoice source. An unknown amount has no `costUsd` and always remains partial.
+The observation window must have ended; completeness is the reviewer's assertion
+that this component's entire resource and billing window is represented. A
+checksum proves which file was used, not that the invoice or allocation is true.
+Shared allocations must record their numerator, denominator, window and excluded
+costs in that source file. Do not import a whole shared invoice as one campaign's
+charge without an explicit allocation method.
+
+Each amount is the cumulative total for the entire component, including all its
+providers and separate charges through `windowEndedAt`. Later receipts must keep
+the same `windowStartedAt` and cannot shorten the window or make completed
+coverage partial again. Combine separate billing windows in the retained source
+evidence before importing their total; two separate $0.80 charges require a
+cumulative $1.60 receipt. Version 1 receipts lacked this declaration and are
+rejected on import. Existing journal records are preserved; the importer also
+refuses to extend a component whose prior receipts lack cumulative evidence.
+It also checks the ledger's precedence rules before import: a larger cumulative
+estimate cannot be hidden by an earlier, smaller invoice. Such imports fail
+without changing retained evidence or the journal; reconcile the entire expanded
+window with evidence the ledger can account for before retrying.
+
+The importer validates campaign/allowance identity and source hash, retains both
+files privately under `ledger.jsonl.overhead/`, and syncs them before appending
+the observation. Files are written and synced under unique temporary names, then
+published atomically without replacing existing evidence. A partial write is
+removed without occupying the final path, so retry needs no manual repair.
+Retries sync matching existing files too. Reimporting an identical
+receipt adds no journal entry. A partial amount
+keeps the rest of the allowance held. Complete invoices can supersede estimates;
+conflicting complete invoices remain visible and block admission. This command
+does not open a new spending window, change limits, dispatch tests or rewrite
+previous records. Regenerate the campaign report from the same ledger afterward;
+an old report snapshot is not rewritten by an offline receipt import.
+
+The checked pricecard expires on November 19, 2026. Admission stops when the
+pricecard expires, the catalog model changes, the project is missing, or the
+preflight already exists. Review current official prices and context limits
+before updating `scheduled-prices.json`. `forecast.json` records the selected
+IDs, provenance-linked pilot measurements, compatibility gaps, uncertainty,
+reservations and price expiry. The historical daily/weekly successful model
+subtotals are $0.427624800/$0.791539400; they are not campaign bills. Setup model
+calls and attempt setup/teardown are included, while CI setup/build/queue and
+non-model charges are excluded. Current compatible totals stay unknown if any
+selected cell lacks a matching measurement. Hosted timing is still unmeasured.
+Per-cell medians describe only the retained samples; P95 requires at least 20
+matching samples and does not establish reliability. See
+[measurement provenance and limitations](OPERATIONS-COVERAGE.md#initial-cadence).
+Daily coverage is limited to the two Claude journeys. The feedback definition
+has changed since its passing pilot. Broader rotation and Daytona remain
+unqualified and are not silently included by `--all`.
+
+Dispatch `cadence: daily` or `weekly` from the default branch to qualify the same
+path manually. Bounded dispatch rejects custom target revisions and selectors.
+Bounded authorization pins the commit from the workflow event; a later merge
+does not replace that revision or invalidate a queued campaign.
+Workflow attempts after the first cannot start paid work with a fresh journal.
+Rerunning reporting can reuse the original attempt-one evidence. Recover the
+retained ledger for a deliberately resumed campaign; do not reset it or launch a
+new campaign merely to hide a failure. The existing `manual` mode remains an
+explicit, unbounded qualification path and does not inherit these allowances.
+
+After execution, deterministic review dismisses only exact synthetic feedback
+bodies corroborated by all feedback-persistence graders. Novel feedback remains
+`needs_investigation`; no extra model call is made. Missing accounting, unresolved
+receipts, feedback collection/submission failures, novel feedback, export failure
+and incomplete experiment import or indexing fail the job with bounded diagnostic codes.
+Successful unchanged runs produce artifacts without posting messages. GitHub's
+workflow failure notifications remain the operator's notification channel.
+Missing/non-starting schedules require an external workflow monitor; this change
+does not claim that a workflow can detect its own failure to start.
+
+The launcher starts the public Playwright JavaScript entry point directly with
+Node and sends SIGINT for cancellation and watchdog cleanup. The package-manager
+wrapper can exit before Playwright completes its shutdown, so it is excluded.
+Playwright handles that signal by stopping workers and closing its web server;
+SIGTERM can exit before web-server cleanup. The web-server supervisor still
+receives SIGTERM and owns its server's shutdown. Both paths retain the existing
+bounded grace period and forced cleanup for surviving owned processes.
+When a worker exits without a result, the launcher also recovers its original
+trace identity from the private fixture snapshots. The supervisor can finalize
+that trace with the failed outcome. A mismatched project, execution or company
+is rejected. Recovered identity does not prove native export or complete phases.
+
+If a worker exits before writing its result, the server supervisor spends at
+most five seconds collecting feedback from that attempt's saved company before
+stopping the isolated server. Recovery snapshots remain private and are kept
+separate from the worker's original evidence. The failed result includes any
+recovered counts and review fingerprints; active runs or missing reads remain
+partial or unavailable. Each atomic recovery checkpoint carries its attempt
+identity, so a later write timeout cannot hide already saved feedback when the
+final receipt is absent. Completed results are never replaced. The launcher
+retains the private database whenever a worker result is missing, including when
+the supervisor was killed before it could capture feedback. This retention is
+reported as unconfirmed cleanup and requires inspecting the retained state.
+Authenticated public-MCP fixtures do not use the local operator recovery path.
+
+Original grades are imported into Braintrust without rerunning agents. The
+access-controlled artifact includes the journal, forecast, feedback dispositions,
+experiment receipt, indexing receipt and findings. The trusted report job rebuilds the dashboard
+and retains its evidence-bound experiment link. Export acceptance and native
+indexing are separate. A bounded, read-only Braintrust query verifies the attempt
+root, finalization, native API activity and any expected feedback tools. It also
+compares indexed experiment rows with original grades, revisions and attempt IDs.
+The verifier tries at most three rounds within 45 seconds, retains missing/query
+failures, and never reruns an agent. Native record counts measure export volume,
+not distinct provider requests or charges. Codex exports can be high volume and
+Braintrust can double-count native/log observations; use the deduplicated gateway
+ledger as the estimate authority. Private indexing receipts contain only counts,
+identities and verification status, without trace contents or credentials.
+
+## Inspection and limits
+
+Retained deterministic grades can be imported into a private Braintrust experiment
+without invoking agents or judges. Prepare a JSON array of absolute paths to the
+original `result.json` for **every** retained attempt in `normalized-results.json`.
+The original files supply each attempt's actual source and definition hash;
+the selected result's revision must not be assigned to older attempts.
+
+```sh
+PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID=PROJECT_UUID \
+node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/braintrust-experiment-import.ts \
+ /private/campaign/normalized-results.json /private/campaign/original-result-paths.json \
+ /private/campaign/experiment-plan.json --dry-run
+```
+
+Inspect the plan. To import it, provide `BRAINTRUST_API_KEY` securely, remove
+`--dry-run`, and use a separate `experiment-receipt.json` output path. The importer
+requires explicit accounting identities and source/definition hashes; older
+incomplete provenance fails closed. It retains original failed and successful
+grades, cleanup outcomes, feedback counts, source revisions and trace identities.
+Carried-forward CI attempts retain their original campaign IDs and must match
+the report's retained receipts. Request costs attach only when the attempt's
+campaign also matches the journal's campaign, even if attempt IDs coincide.
+It does not upload prompts, feedback bodies, matcher details or raw errors.
+
+The evidence digest determines the private experiment name. Identical evidence
+reuses that experiment and stable row IDs after an interrupted insert. Changed
+evidence creates a different experiment and requires a separate receipt path.
+The importer validates an existing receipt before retrying and never lowers its
+confirmed accepted-row count. The receipt checkpoints accepted batches;
+HTTP acceptance still requires independent indexing/UI verification. Retain failed
+receipts. A retry must use the same original evidence, not newly run agents.
+Set `PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT` to the receipt path when rebuilding the
+campaign report to retain the experiment link. Its evidence hash must match the
+report's selected source/definition identities, attempts, request-ledger checkpoint
+and feedback review. Refreshing only
+the report rendering preserves the link; changed evidence requires a new import.
+
+Request-ledger amounts remain metadata with their journal hash and classifications.
+They are not additive cost metrics, and missing rows remain unknown rather than
+zero. The campaign ledger may include setup probes outside the imported tests.
+Do not add experiment metadata amounts to native or gateway trace costs. Native
+trace rollups are not invoice reconciliation and can omit setup or duplicate logs.
+API contracts: [create experiment](https://www.braintrust.dev/docs/api-reference/experiments/create-experiment)
+and [insert retained events](https://www.braintrust.dev/docs/api-reference/experiments/insert-experiment-events).
+
+- `operations-preflight.json` preserves the exact manifest in the campaign folder.
+- `ledger.jsonl` is the append-only authority for request admission and observations.
+- `ledger.jsonl.requests/` contains private per-request checkpoints.
+- `ledger.jsonl.summary.json` preserves known, reserved/held, conflicting, and
+ unknown amounts on normal finalization, including failed campaign workers.
+- `campaign.json` and the dashboard preserve retained attempts and show missing
+ selected results after worker failure. Their request-ledger section shows the
+ shared ledger's known subtotal, classifications, additional held reservations,
+ unresolved requests and per-attempt totals. It can include earlier campaigns
+ sharing that budget. It is a separate view of model spending: do not add it to
+ heartbeat totals. The journal hash and record count identify the checkpoint.
+- If the journal cannot be read or validated, the report records accounting as
+ unavailable with no numeric totals. Retained results and reports are still
+ written, and the launcher/report command exits unsuccessfully. Recover the
+ original journal before resuming paid work; reporting does not repair or reset it.
+- To regenerate a merged report with the same ledger, pass
+ `PAPERCLIP_E2E_OPERATIONS_LEDGER=/absolute/ledger.jsonl` to `report.ts`.
+ Dashboard refresh and public publication preserve and validate its closed
+ numeric projection. Raw request receipts and the journal remain private.
+
+All model amounts from usage times a pricecard are **estimated**, not billed or
+reconciled. This path does not yet price CI compute, storage, subscriptions,
+Daytona, separate judges, or gateway overhead. Unknown observations and these
+coverage gaps must be included in operational reporting. Provider billing can
+lag and discounts can differ; a local list-price cap is not a provider-side cap.
+
+Resuming the same journal requires exactly the original limits. A dead process
+can leave a lock and held reservations; inspect the owner and upstream receipts
+before recovering it. Never delete a live writer's lock or reset a ledger merely
+to regain budget. Older journals without an attempt cap remain readable for
+reporting and reconciliation. Operations refuses an implicit upgrade because request receipts cannot reconstruct
+zero-request attempts. Use the explicit audited migration below only after
+reviewing all retained campaign attempts. Do not replace or reset an active pilot
+ledger to evade this check.
+
+No recurring schedule is enabled by this integration. Qualification of actual
+agent streams, tools, continuation, cancellation, native traces, and remote routes
+is required before treating those routes as operationally proven. Forecasts,
+provider reconciliation, and the campaign feedback disposition workflow are
+separate remaining integration work.
+
+### Setup and agent request attribution
+
+Each admitted attempt creates a private phase checkpoint under
+`<ledgerPath>.attempts/`. The launcher passes its path only to the test worker;
+the Paperclip server does not inherit it. Before fixture setup finishes,
+requests are recorded as `setup`, including connection checks and hello probes.
+The worker atomically advances the checkpoint to `agent` immediately after
+fixture creation and before task execution. Cleanup does not reset this phase,
+so late agent requests retain their task attribution.
+
+The gateway reads and validates the checkpoint for each request before reserving
+spend. That admission fixes the request's phase even if the phase changes while
+its response streams. Missing, malformed, or foreign-attempt checkpoints stop
+new dispatches rather than guessing a phase. Request records remain in the
+fsynced journal after the worker exits. This boundary applies to the admitted
+local core cells; it does not establish coverage for separate judges, paid tools,
+child-agent attribution, or remote routes. Historical journals retain their
+original attribution and are not retroactively rewritten.
+
+### Auditing a legacy journal before adding an attempt cap
+
+A journal created before durable attempt admission can acquire one attempt policy
+through `ledger-attempt-policy.ts`. Stop its launcher and gateway first. Review
+all retained campaigns, including startup failures with no model request. Save
+the review and its source references as a private evidence file. Each prior
+attempt needs its original execution and accounting attempt ID; request receipts
+alone are insufficient. If that history is incomplete, do not resume paid work.
+
+Create a private JSON audit with these fields:
+
+- `version`: `1`.
+- `ledgerPath` and `evidencePath`: absolute private file paths.
+- `previousJournalHash`: the current journal's final record hash.
+- `evidenceSha256`: SHA-256 of the reviewed evidence file.
+- `maxAttemptsPerExecution`: the finite cap to adopt, including past attempts.
+- `priorAttempts`: an array of `{ "executionId": "...", "attemptId": "..." }`.
+
+Run `node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/ledger-attempt-policy.ts /absolute/private-audit.json`.
+The command checks evidence and journal hashes, takes the normal exclusive writer
+lock, and appends one fsynced policy record. It refuses unfinished requests,
+omitted recorded request identities, duplicate attempts, or an existing attempt
+policy. The imported count can already exceed the cap; then no new attempt is
+admitted for that execution. The command cannot prove the completeness of the
+operator's review of zero-request attempts. Retain the evidence for inspection.
+
+All original journal bytes, request counts, charges, held reservations, campaign
+identity, dollar limit, concurrency limit and deadline remain intact. Subsequent
+launch manifests must use the adopted cap. The migration cannot extend an expired
+deadline, change an existing cap, or replenish an allowance. An interrupted append
+remains an accounting failure and must be recovered before more paid work.
+
+The explicit local `feedback-persistence` native Claude/Codex cells also accept
+an operations manifest. They use the same metered connection, prices, campaign
+journal, setup boundary, and durable attempt allowance as core compatibility.
+Collection and the independent saved-evidence oracle require no additional
+model call. The synthetic items need review as test data, not product incidents.
diff --git a/tests/runner-e2e/OTLP-LICENSE.txt b/tests/runner-e2e/OTLP-LICENSE.txt
new file mode 100644
index 0000000000..261eeb9e9f
--- /dev/null
+++ b/tests/runner-e2e/OTLP-LICENSE.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/tests/runner-e2e/OTLP-NOTICE.md b/tests/runner-e2e/OTLP-NOTICE.md
new file mode 100644
index 0000000000..6ae059f624
--- /dev/null
+++ b/tests/runner-e2e/OTLP-NOTICE.md
@@ -0,0 +1,11 @@
+# OTLP field definitions
+
+The field definitions in `eval-otlp-schema.ts` are derived from
+[OpenTelemetry Protocol v1.9.0](https://github.com/open-telemetry/opentelemetry-proto/tree/v1.9.0).
+
+Copyright The OpenTelemetry Authors.
+Licensed under the [Apache License, Version 2.0](./OTLP-LICENSE.txt).
+
+The source comments and package declarations were removed to embed these field
+definitions in the eval harness. The harness adds a combined response message.
+Accounting behavior and limits are documented in [OPERATIONS.md](./OPERATIONS.md).
diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md
index df516223c8..5f199a2fad 100644
--- a/tests/runner-e2e/README.md
+++ b/tests/runner-e2e/README.md
@@ -1054,9 +1054,32 @@ timings, token and cost accounting, and evidence links. The campaign header
aggregates input, output, and cached tokens, provider-reported LLM spend,
Daytona list-price runtime estimates, and pricing coverage. Missing provider
usage is labeled `unavailable` or `unpriced`; it is never presented as zero
-cost. The CI report job stages the same portable site at
+cost. A known subtotal from a partially priced run stays in campaign spending,
+with a partial-amount label and incomplete billing coverage. The CI report job stages the same portable site at
`normalized/index.html` inside the access-controlled merged report artifact.
+Campaign billing retains a separate numeric receipt for each attempt, including
+failed retries. The selected result still controls the grade and result card.
+Copies of the same attempt do not add another charge. The supervisor stamps a
+campaign/attempt identity; CI cells share `PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID`
+while retaining their separate artifact directories. Report regeneration keeps
+these receipts and their totals. A later cleanup update replaces an earlier
+observation of the same attempt; contradictory costs at the same timestamp are
+rejected instead of silently choosing one.
+
+The report shows missing attempt receipts and artifacts excluded from another
+campaign. Legacy CI evidence can be grouped by workflow run URL. Unscoped local
+artifacts cannot establish campaign membership, so only the selected result is
+used. Older normalized bundles without receipts are labeled `selected_results`;
+rendering them again cannot recover lost retry costs.
+
+Billing reads decimal `costUsdExact` receipts and separates model estimates
+from provider-reported amounts. Explicit zero usage differs from a missing or
+unfinished receipt. Coverage is limited to captured usage. It does not certify
+setup probes, subscription allocations, external tools, CI, storage, or provider
+invoice reconciliation. These require separate receipts. Do not use the current
+numeric report subtotals as an enforced spending limit.
+
The trusted publisher discovers display-only entries for selected execution IDs
absent from its local catalog, so branch-only suites remain visible in the
dashboard, filters, gallery, and summary image. It validates execution identity
@@ -2748,3 +2771,173 @@ The arithmetic journey records four delivered agent handoffs and one completed
terminal intent. The final case is `done` at handoff version 5, while the same
four provider work runs and unchanged workflow document revision finish the task.
The oracle rejects a missing or pending terminal intent.
+
+### Complaints and suggestions
+
+Each ordinary live attempt collects company-wide saved feedback after retiring
+runs and before fixture teardown, even when cancellation or grading fails. This
+includes feedback from child and background runs. Full paginated bodies and
+native tool invocation outcomes are checkpointed in the access-controlled
+`snapshots/agent-feedback.json` artifact. Collection has a 30-second deadline;
+missing detail pages, truncated run lists, active work, and unavailable APIs
+remain explicit partial/unavailable observations. Legacy shell helpers do not
+have complete native invocation coverage. An empty saved-feedback list does not
+prove that a tool was available, and a completed tool receipt does not by itself
+prove a saved submission.
+
+Results and retained-attempt receipts contain only counts, closed statuses, and
+body fingerprints. Campaign reports group identical fingerprints across attempts
+into distinct items awaiting review. Feedback presence does not change product
+grades. Inspect full bodies in private evidence and record a disposition before
+considering the review complete; collection itself performs no model call or
+semantic triage. Historical artifacts without feedback fields are labeled as
+uncaptured. A killed worker that never reaches finalization is also uncaptured,
+not evidence of zero feedback.
+
+Create a private review template from the merged `normalized-results.json`:
+
+```sh
+pnpm exec tsx tests/runner-e2e/feedback-dispositions.ts /private/report/normalized-results.json /private/feedback-review.json
+```
+
+Inspect the full bodies in the retained attempt snapshots. Set `reviewedAt` to
+an ISO timestamp with an explicit timezone, such as `2026-10-20T01:00:00Z`,
+and `reviewedBy` to `human` or `agent`. Timestamp comments and free-form dates
+are rejected so private notes cannot enter public report metadata. Assign each fingerprint
+one disposition: `actionable`, `duplicate`, `already_fixed`, `unsupported`, or
+`needs_investigation`. A duplicate must name a non-duplicate fingerprint in the
+same campaign via `duplicateOf`. Optional `issueNumber` links a public
+`paperclipai/paperclip` issue; keep private rationale outside this file.
+
+Import a completed review into an existing bundle without rerunning agents:
+
+```sh
+pnpm test:e2e:runner:dashboard -- /private/report --feedback-review /private/feedback-review.json
+```
+
+Alternatively, rebuild the same campaign and selection with
+`PAPERCLIP_E2E_FEEDBACK_REVIEW=/private/feedback-review.json`. Keep its original
+campaign ID. The importer requires exactly one decision per collected group
+and rejects a review if retained attempts or feedback coverage changed. Blank
+templates cannot count as reviewed. Dashboard regeneration and publication
+preserve the review. Regeneration refreshes the review status in an existing
+`summary.md` while preserving its other measurements and links. An empty review
+can record that no saved items were found; it never closes collection gaps,
+failed submissions, or changes product grades.
+This workflow makes no additional model call. Review performed by a separate
+paid model must be budgeted and recorded as `feedback_review` work separately.
+
+### Cost-ledger integration contract
+
+`cost-ledger.ts` provides the request-level journal used by the eval operations
+integration. It is a library; the existing launcher is not yet universally routed
+through it, and merely setting a campaign ID does not enforce a spending cap.
+Every paid dispatch must await a durable reservation before contacting a provider.
+The caller supplies an explicit campaign ceiling, deadline, request count, and
+concurrency. Monetary decisions use integer nanodollars. Model list prices remain
+estimates with a pinned version; they do not become provider invoices.
+
+The journal retains setup/agent/judge/tool/infrastructure attribution separately
+from heartbeat IDs. Multiple sources for the same request are retained but only
+one cost contributes to the total: reconciled, reported, estimated, then allocated.
+Missing receipts and cancellations retain their reservation. Contradictory final
+receipts stop new admissions. An observed overshoot is retained in full and stops
+further work; local estimated limits are not provider-enforced dollar limits.
+A request bound must account for input/context, output limits, cache-write pricing,
+and any paid tools before claiming a bounded overshoot.
+
+A single writer serializes admissions and fsyncs each journal record. Read-only
+inspection survives process loss; truncated/tampered records fail closed. A stale
+`.lock` requires inspection of the recorded owner PID before operator recovery;
+never remove a live writer's lock. Resuming a campaign cannot silently change its
+limits. The ledger rejects unreviewed fields and contains no prompts, responses,
+headers, or credentials. Cross-source request joins must use a verified request
+identity; ambiguous provider-request joins are rejected rather than double counted.
+
+### Opt-in Braintrust traces
+
+Set `PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID` to the destination project UUID and
+provide `BRAINTRUST_API_KEY` securely to the launcher. An ambient key alone does
+not enable export. The launcher includes this key in secret scans and removes
+Braintrust variables from the server's inherited environment. Fixture setup
+saves the company's native observability configuration through the public,
+revision-fenced API using a non-Playwright sensitive request, before credential,
+agent, and setup-probe creation. The onboarding flow installs it immediately
+after company creation. The existing company setting controls native capture.
+
+Each ordinary `runner.spec.ts` attempt records orchestration spans for setup,
+execution, feedback, and cleanup, and exports protobuf over HTTPS with a ten
+second deadline and no redirect/retry. `snapshots/braintrust-trace.json` checkpoints
+the identifiers/configuration and final export receipt. Normalized results retain
+the receipt, including failed exports. Export failure does not rewrite the product
+verdict; operational qualification must separately require accepted and verified
+traces. A killed worker can leave a pending receipt, not a claimed successful export.
+
+`accepted` means the OTLP receiver accepted the batch; it does not establish that
+Braintrust indexed it or that native agent spans arrived. Search the project for
+`paperclip.eval.attempt_id` and its trace ID, then verify native records using the
+company ID and `paperclip.eval.run_ids`. The disposable company uses the custom
+OTLP destination at `https://api.braintrust.dev/otel` with a private `x-bt-parent`
+header containing the exported attempt parent. Braintrust attaches independent
+native trace roots beneath that attempt; native children keep their own hierarchy.
+This uses the existing company observability API without changing production
+trace parentage or enabling export on other companies.
+
+The parent encoder uses Braintrust's V4 project-log wire format with JSON identity
+fields (version 4, project-log type 2, zero compressed fields). Its fixture was
+round-tripped through the public `SpanComponentsV4.fromStr` export in
+`braintrust/util` 3.38.0. The implementation is specified in the
+[official SDK source](https://github.com/braintrustdata/braintrust-sdk-javascript/blob/403ae94fb3f1916a101d8d84de4c4871e4ae3bec/js/util/span_identifier_v4.ts).
+A model-free live probe verified the indexed child's `root_span_id` and
+`span_parents` in Braintrust; a native-agent run must separately qualify the path.
+The parent contains identifiers only and is installed through the sensitive
+credential-header path, outside browser evidence.
+
+Attempt spans include suite ID and definition hash, profile, case, configured
+provider/model, source revision (or explicit `unknown`), and the runner-spec grader
+path. The source revision identifies that grader's implementation. Worker exports
+are marked `paperclip.eval.outcome_stage=worker`. After evidence packaging,
+gateway shutdown and outer cleanup, the supervisor updates the same attempt root
+with its final outcome and adds a finalization span. This preserves failures first
+discovered after the worker returned. The result retains a separate finalization
+export receipt; old or killed attempts can explicitly lack one. Export failure
+remains separate from the product grade. Orchestration spans omit model usage and
+cost metrics. Gateway model logging remains a separate observation lane;
+its model costs must not be added to native costs without a verified request join.
+The separate provider connection suite and standalone judges do not yet use this
+attempt exporter.
+
+### Bounded local gateway pilot
+
+[OPERATIONS.md](./OPERATIONS.md) documents the opt-in operations manifest, exact
+selection admission, durable request reservations, local Responses/Messages proxy,
+and Braintrust routing. Current support is deliberately limited to local core
+Codex/Claude cells. The document distinguishes implemented accounting controls
+from live qualification and lists remaining billing/reporting coverage gaps.
+
+## Feedback persistence (explicit only)
+
+`--suite feedback-persistence` selects two local native Claude/Codex cells.
+Each creates one standard task through the browser, asks for two distinct
+synthetic feedback items, and requires the real `submit_complaint` and
+`submit_suggestion` tools. The five-minute case expects one task work run and
+disables automatic retries. Company and agent accounted-work budgets are $10.
+Setup probes and any background work must also pass through the separately
+budgeted gateway when using an operations manifest. Local compute is unmetered.
+Cost and duration forecasts remain unmeasured until a retained live attempt.
+The runner E2E maintainers own this manual case; `--all` excludes it.
+
+After collection and fixture teardown, the oracle reads the retained evidence
+file and requires exact nonce-bearing bodies, the original company/agent/task
+and observed run, plus one completed native receipt for each tool. Missing or
+partial collection, prose-only claims, foreign attribution, duplicate records,
+and failed submissions cannot pass. Full bodies stay in private evidence;
+public reports retain counts, fingerprints, and closed oracle checks. The usual
+result screenshot, trace, secret scanning and supervisor cleanup still apply.
+This case proves explicit submission and persistence, not spontaneous feedback
+quality or survival after an abrupt worker kill.
+
+```sh
+pnpm test:e2e:runner -- --list --suite feedback-persistence
+pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1
+```
diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md
index 093f206613..2a84a315e5 100644
--- a/tests/runner-e2e/SECURITY.md
+++ b/tests/runner-e2e/SECURITY.md
@@ -423,6 +423,66 @@ Two passive seed cases support browser list assertions; screenshots remain on
the exact primary case/task routes and document dialogs preserve that case URL.
The public screenshot allowlist is unchanged.
+## Eval feedback capture
+
+`snapshots/agent-feedback.json` retains company-wide complaint/suggestion bodies
+and attributed invocation outcomes in the access-controlled Actions artifact.
+It passes the existing credential scanner and is excluded from public history
+like other snapshots. Normalized results and campaign attempt receipts contain
+only numeric counts, closed collection statuses, and SHA-256 fingerprints for
+review grouping. Public reports never copy feedback bodies, previews, tool
+arguments, or free-form collection errors. Collecting feedback performs no new
+model call and does not forward it to an external service.
+
+Feedback capture retains at most 1,000 full entries and 8 MiB of serialized
+entries/invocations per attempt. Event inspection stops at 10,000 events or
+16 MiB of serialized event data. Limits produce partial collection and a closed
+`evidence_limit_reached` diagnostic; omitted evidence is never reported as absent.
+Checkpoints are considered every ten details, at page boundaries, and at stage
+completion; geometric data growth bounds repeated writes. Reads stop before the
+overall deadline, reserving up to five seconds (20% of shorter windows) to persist
+the collected prefix. Persistence shares the overall collection deadline and
+receives an abort signal. The fixture writes an abortable temporary
+snapshot before replacing the last complete checkpoint, so a stalled write does
+not overwrite earlier evidence with a partially written JSON file.
+Review counts and fingerprints advance only after a checkpoint succeeds. A later
+write failure or expired deadline retains the last saved bodies and marks coverage
+incomplete.
+
+Braintrust export requires an explicit eval project UUID. Native capture includes
+prompts, responses, and tool content only after that opt-in configures the isolated
+company's observability destination. The ingestion credential crosses the public API via
+Node fetch, outside Playwright tracing, and is included in launcher evidence
+redaction. Orchestration spans contain identifiers, phase timing, outcome, and
+feedback counts; they never contain provider credentials or feedback bodies.
+
+With a bounded operations manifest, the ingestion credential is an expiring
+per-attempt relay token. The organization-wide Braintrust key stays in the
+controller. Only authenticated POSTs to the relay's trace/log ingest paths are
+allowed, with a fixed HTTPS upstream and redirects disabled. Header and payload
+`braintrust.parent` values must reference the selected project. Duplicate protobuf
+singular/oneof fields are refused, and JSON is canonicalized to remove duplicate
+key ambiguity. The token cannot call model, read, delete, or other API endpoints;
+it can still write records within the selected project until its finite request,
+byte or time allowance is exhausted. It is not an attempt-level row permission.
+
+The worker environment removes upstream provider/Braintrust credentials before
+inserting the model and OTLP relay tokens. Ambient relay URLs are discarded, and
+the fixture server receives the selected endpoint through company settings.
+Both tokens participate in evidence redaction. Private OTLP receipts retain
+metadata only, and persistence errors stop further admission. The relay closes
+with the model gateway before controller finalization. The controller's direct
+finalization export and tracing without an operations manifest retain the
+existing direct credential path. Remote relay publication is not enabled here.
+
+The bounded operations manifest is credential-free and private. Its parent-owned
+loopback gateway uses an ephemeral per-attempt token. Upstream credentials stay
+in the launcher, are included in secret scans, and are removed from the child
+provider environment. Only the selected Responses or Messages endpoint is
+forwarded to a fixed provider or Braintrust HTTPS origin; request headers do not
+select arbitrary destinations. Gateway response caching and fallback are refused.
+Request journal and usage receipts contain no prompt/response bodies or keys.
+The manifest, journal, and receipts are not public report projections.
## Configured repository and conversion proof
diff --git a/tests/runner-e2e/api.ts b/tests/runner-e2e/api.ts
index 398691d1e2..5032c3e56f 100644
--- a/tests/runner-e2e/api.ts
+++ b/tests/runner-e2e/api.ts
@@ -58,15 +58,27 @@ export class RunnerApi {
* never serialized into trace/blob evidence before Paperclip encrypts it.
*/
async postSensitive<T>(path: string, data: unknown): Promise<T> {
- const response = await fetch(new URL(path, this.baseURL), {
- method: "POST",
- headers: { "content-type": "application/json", ...this.sessionOptions().headers },
- body: JSON.stringify(data),
- });
+ return this.sensitive<T>("POST", path, data);
+ }
+
+ async putSensitive<T>(path: string, data: unknown): Promise<T> {
+ return this.sensitive<T>("PUT", path, data);
+ }
+
+ private async sensitive<T>(method: "POST" | "PUT", path: string, data: unknown): Promise<T> {
+ const url = new URL(path, this.baseURL);
+ if (url.origin !== this.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Sensitive request must use the fixture API");
+ let response: Response;
+ try {
+ response = await fetch(url, {
+ method, redirect: "error", signal: AbortSignal.timeout(15_000),
+ headers: { "content-type": "application/json", ...this.sessionOptions().headers },
+ body: JSON.stringify(data),
+ });
+ } catch { throw new Error(`Sensitive ${method} request failed; details withheld`); }
if (!response.ok) {
- throw new Error(
- `Sensitive POST ${path} returned ${response.status}; response body withheld`,
- );
+ await response.body?.cancel();
+ throw new Error(`Sensitive ${method} ${url.pathname} returned ${response.status}; response body withheld`);
}
return response.json() as Promise<T>;
}
diff --git a/tests/runner-e2e/artifact-budget.mjs b/tests/runner-e2e/artifact-budget.mjs
new file mode 100644
index 0000000000..cf84bbcb8a
--- /dev/null
+++ b/tests/runner-e2e/artifact-budget.mjs
@@ -0,0 +1,163 @@
+import { constants } from "node:fs";
+import { createHash } from "node:crypto";
+import { lstat, mkdir, open, opendir, realpath, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+export const ARTIFACT_BOUNDS = Object.freeze({
+ lock: 16 * 1024 * 1024,
+ build: 1024 * 1024 * 1024,
+ evidence: 256 * 1024 * 1024,
+ report: 256 * 1024 * 1024,
+ receipt: 8 * 1024 * 1024,
+ entries: 4096,
+});
+
+export async function checkArtifactFiles(kind, files) {
+ if (!["lock", "build"].includes(kind) || !files.length) throw new Error("Invalid artifact size check");
+ let bytes = 0;
+ for (const file of files) {
+ const info = await lstat(file);
+ if (!info.isFile() || info.isSymbolicLink() || info.size === 0) throw new Error("Invalid artifact input file");
+ bytes += info.size;
+ if (!Number.isSafeInteger(bytes) || bytes > ARTIFACT_BOUNDS[kind]) throw new Error(`${kind} artifact exceeds its source-byte allowance`);
+ }
+ return { kind, files: files.length, sourceBytes: bytes, limitBytes: ARTIFACT_BOUNDS[kind] };
+}
+
+function priority(relative) {
+ if (relative === "scheduled" || relative.startsWith("scheduled/")) return 0;
+ if (/\.(json|jsonl)$/.test(relative)) return 1;
+ if (/\.(md|html|xml|txt|log)$/.test(relative)) return 2;
+ return 3;
+}
+
+export async function stageBoundedArtifact(kind, source, destination, bounds = ARTIFACT_BOUNDS) {
+ if (!["evidence", "report"].includes(kind) || !Number.isSafeInteger(bounds[kind])
+ || !Number.isSafeInteger(bounds.receipt) || bounds[kind] <= bounds.receipt
+ || !Number.isSafeInteger(bounds.entries) || bounds.entries < 1 || bounds.entries > ARTIFACT_BOUNDS.entries) {
+ throw new Error("Invalid artifact staging allowance");
+ }
+ const root = path.resolve(source), out = path.resolve(destination);
+ if (out === root || out.startsWith(`${root}${path.sep}`) || root.startsWith(`${out}${path.sep}`)) {
+ throw new Error("Artifact staging paths must be separate");
+ }
+ await mkdir(out, { mode: 0o700 });
+ const receipt = { version: 1, kind, sourceComplete: true, inventoryComplete: true,
+ scope: "Artifact evidence; regular playwright-output/.last-run.json bookkeeping is excluded",
+ sourceBytes: 0, retainedBytes: 0, limitBytes: bounds[kind], entryLimit: bounds.entries,
+ findings: [], files: [] };
+ const candidates = [];
+ let visited = 0;
+ const finding = code => { receipt.sourceComplete = false; if (!receipt.findings.includes(code)) receipt.findings.push(code); };
+ const skipped = (relative, reason, subtree = false) => {
+ const shortened = relative.length > 512 || Buffer.byteLength(JSON.stringify(relative)) > 1024;
+ receipt.files.push({ path: shortened ? `${relative.slice(0, 160)}…${relative.slice(-80)}` : relative,
+ ...(shortened ? { pathTruncated: true, pathSha256: createHash("sha256").update(relative).digest("hex") } : {}),
+ ...(subtree ? { scope: "subtree" } : {}), retained: false, reason });
+ };
+ const scan = async () => {
+ const pending = [{ absolute: root, relative: ".", depth: 0, order: 0 }];
+ while (pending.length) {
+ pending.sort((a, b) => a.order - b.order || a.depth - b.depth || a.relative.localeCompare(b.relative));
+ const directory = pending.shift();
+ if (visited >= bounds.entries || directory.depth > 24) {
+ const reason = visited >= bounds.entries ? "entry_limit" : "directory_depth_limit";
+ receipt.inventoryComplete = false; finding(reason); skipped(directory.relative, reason, true); continue;
+ }
+ try {
+ const handle = await opendir(directory.absolute);
+ for await (const entry of handle) {
+ if (visited >= bounds.entries) {
+ receipt.inventoryComplete = false; finding("entry_limit"); skipped(directory.relative, "entry_limit", true); break;
+ }
+ visited++;
+ const absolute = path.join(directory.absolute, entry.name), relative = path.relative(root, absolute).split(path.sep).join("/");
+ if (relative.length > 512 || Buffer.byteLength(JSON.stringify(relative)) > 1024) {
+ receipt.inventoryComplete = false; finding("path_length_limit"); skipped(relative, "path_length_limit", entry.isDirectory()); continue;
+ }
+ if (entry.isFile() && /(?:^|\/)playwright-output\/\.last-run\.json$/.test(relative)) {
+ skipped(relative, "playwright_bookkeeping_excluded"); continue;
+ }
+ if (entry.name.startsWith(".")) {
+ receipt.inventoryComplete = false; finding("hidden_path_excluded"); skipped(relative, "hidden_path_excluded", entry.isDirectory()); continue;
+ }
+ if (entry.isDirectory()) {
+ const diagnostic = /(?:^|\/)(?:blob-report|html-report|playwright-output)(?:\/|$)/.test(relative);
+ pending.push({ absolute, relative, depth: directory.depth + 1,
+ order: diagnostic ? 3 : relative === "scheduled" || relative.startsWith("scheduled/") ? 0 : 1 });
+ } else if (entry.isFile()) candidates.push({ absolute, relative });
+ else { finding("non_regular_file"); skipped(relative, "non_regular_file"); }
+ }
+ } catch { receipt.inventoryComplete = false; finding("inventory_failed"); skipped(directory.relative, "inventory_failed", true); }
+ }
+ };
+ try {
+ const metadata = await lstat(root);
+ if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw new Error("Invalid evidence root");
+ await scan();
+ } catch { receipt.inventoryComplete = false; finding("inventory_failed"); skipped(".", "inventory_failed", true); }
+ const canonicalRoot = await realpath(root).catch(() => root);
+ candidates.sort((a, b) => priority(a.relative) - priority(b.relative) || a.relative.localeCompare(b.relative));
+ for (const { absolute, relative } of candidates) {
+ let input, output;
+ const target = path.join(out, relative);
+ const row = { path: relative, retained: false };
+ receipt.files.push(row);
+ try {
+ if (relative === "artifact-budget-receipt.json") throw new Error("reserved_path");
+ const canonical = await realpath(absolute);
+ if (!canonical.startsWith(`${canonicalRoot}${path.sep}`)) throw new Error("source_escaped");
+ input = await open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW);
+ const before = await input.stat();
+ if (!before.isFile()) throw new Error("non_regular_file");
+ row.sourceBytes = before.size;
+ receipt.sourceBytes += before.size;
+ if (!Number.isSafeInteger(receipt.sourceBytes)) throw new Error("source_size_overflow");
+ if (receipt.retainedBytes + before.size > bounds[kind] - bounds.receipt) throw new Error("byte_limit");
+ await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
+ output = await open(target, "wx", 0o600);
+ const buffer = Buffer.alloc(64 * 1024), hash = createHash("sha256");
+ let copied = 0;
+ while (copied < before.size) {
+ const read = await input.read(buffer, 0, Math.min(buffer.length, before.size - copied), copied);
+ if (!read.bytesRead) throw new Error("source_changed");
+ hash.update(buffer.subarray(0, read.bytesRead));
+ let written = 0;
+ while (written < read.bytesRead) {
+ const part = await output.write(buffer, written, read.bytesRead - written, copied + written);
+ if (!part.bytesWritten) throw new Error("copy_failed");
+ written += part.bytesWritten;
+ }
+ copied += read.bytesRead;
+ }
+ const after = await input.stat();
+ if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("source_changed");
+ await output.sync();
+ row.retained = true; row.sha256 = hash.digest("hex");
+ receipt.retainedBytes += copied;
+ } catch (error) {
+ const code = ["reserved_path", "source_escaped", "non_regular_file", "source_size_overflow", "byte_limit", "source_changed"].includes(error.message) ? error.message : "copy_failed";
+ row.reason = code; finding(code);
+ } finally {
+ await input?.close(); await output?.close();
+ if (!row.retained) await rm(target, { force: true });
+ }
+ }
+ const serialized = `${JSON.stringify(receipt, null, 2)}\n`;
+ if (Buffer.byteLength(serialized) > bounds.receipt) throw new Error("Artifact receipt exceeds reserved allowance");
+ await writeFile(path.join(out, "artifact-budget-receipt.json"), serialized, { mode: 0o600, flag: "wx" });
+ return { ...receipt, stagedBytes: receipt.retainedBytes + Buffer.byteLength(serialized) };
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [command, kind, ...args] = process.argv.slice(2);
+ if (command === "check") console.log(JSON.stringify(await checkArtifactFiles(kind, args)));
+ else if (command === "stage" && args.length === 2) {
+ const receipt = await stageBoundedArtifact(kind, args[0], args[1]);
+ if (process.env.GITHUB_OUTPUT) await writeFile(process.env.GITHUB_OUTPUT, `complete=${receipt.sourceComplete}\n`, { flag: "a" });
+ const summary = `Eval ${kind}: retained ${receipt.retainedBytes} source bytes; staged ${receipt.stagedBytes} bytes; complete=${receipt.sourceComplete}. Findings: ${receipt.findings.join(", ") || "none"}.`;
+ if (process.env.GITHUB_STEP_SUMMARY) await writeFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`, { flag: "a" });
+ console.log(summary);
+ } else throw new Error("Usage: artifact-budget.mjs check <lock|build> <files...> | stage <evidence|report> <source> <destination>");
+}
diff --git a/tests/runner-e2e/artifact-budget.test.mjs b/tests/runner-e2e/artifact-budget.test.mjs
new file mode 100644
index 0000000000..06493ca2a6
--- /dev/null
+++ b/tests/runner-e2e/artifact-budget.test.mjs
@@ -0,0 +1,120 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdtemp, mkdir, readFile, rm, stat, symlink, truncate, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { test } from "node:test";
+import { ARTIFACT_BOUNDS, checkArtifactFiles, stageBoundedArtifact } from "./artifact-budget.mjs";
+
+const bounds = { ...ARTIFACT_BOUNDS, evidence: 12_288, receipt: 8192, entries: 32 };
+async function fixture(run) {
+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-artifacts-"));
+ try { const source = path.join(root, "source"); await mkdir(source); await run({ root, source, out: path.join(root, "out") }); }
+ finally { await rm(root, { recursive: true, force: true }); }
+}
+
+test("retains exact accounting and feedback before oversized diagnostics and keeps the source intact", async () => fixture(async ({ source, out }) => {
+ await mkdir(path.join(source, "scheduled"));
+ await writeFile(path.join(source, "scheduled/ledger.jsonl"), '{"reserved":"4.16"}\n');
+ await writeFile(path.join(source, "feedback.json"), '{"complaint":"retained"}\n');
+ await writeFile(path.join(source, "trace.zip"), Buffer.alloc(5000, 1));
+ const result = await stageBoundedArtifact("evidence", source, out, bounds);
+ assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, true);
+ assert.deepEqual(result.findings, ["byte_limit"]);
+ assert.equal(await readFile(path.join(out, "scheduled/ledger.jsonl"), "utf8"), '{"reserved":"4.16"}\n');
+ assert.equal(await readFile(path.join(out, "feedback.json"), "utf8"), '{"complaint":"retained"}\n');
+ assert.equal((await stat(path.join(source, "trace.zip"))).size, 5000);
+ await assert.rejects(stat(path.join(out, "trace.zip")), { code: "ENOENT" });
+ const receipt = JSON.parse(await readFile(path.join(out, "artifact-budget-receipt.json"), "utf8"));
+ assert.equal(receipt.files.find(row => row.path === "trace.zip").reason, "byte_limit");
+ assert.match(receipt.files.find(row => row.path === "feedback.json").sha256, /^[a-f0-9]{64}$/);
+ assert.ok(result.stagedBytes <= bounds.evidence);
+}));
+
+test("keeps complete evidence unchanged at the exact payload boundary and refuses destination reuse", async () => fixture(async ({ source, out }) => {
+ const content = Buffer.alloc(bounds.evidence - bounds.receipt, 42);
+ await writeFile(path.join(source, "result.json"), content);
+ const result = await stageBoundedArtifact("evidence", source, out, bounds);
+ assert.equal(result.sourceComplete, true); assert.equal(result.retainedBytes, content.length);
+ assert.deepEqual(await readFile(path.join(out, "result.json")), content);
+ await assert.rejects(stageBoundedArtifact("evidence", source, out, bounds), { code: "EEXIST" });
+}));
+
+test("refuses symlinks and records missing evidence without reading outside the source", async () => fixture(async ({ root, source, out }) => {
+ await writeFile(path.join(root, "private"), "secret outside source");
+ await symlink(path.join(root, "private"), path.join(source, "result.json"));
+ const result = await stageBoundedArtifact("evidence", source, out, bounds);
+ assert.equal(result.sourceComplete, false); assert.deepEqual(result.findings, ["non_regular_file"]);
+ assert.equal(result.retainedBytes, 0);
+ const missing = await stageBoundedArtifact("evidence", path.join(root, "missing"), path.join(root, "missing-out"), bounds);
+ assert.equal(missing.inventoryComplete, false); assert.deepEqual(missing.findings, ["inventory_failed"]);
+}));
+
+test("bounds entry inventory and exposes partial traversal", async () => fixture(async ({ source, out }) => {
+ for (let i = 0; i < 5; i++) await writeFile(path.join(source, `${i}.json`), "{}");
+ const result = await stageBoundedArtifact("evidence", source, out, { ...bounds, entries: 3 });
+ assert.equal(result.inventoryComplete, false); assert.equal(result.sourceComplete, false);
+ assert.equal(result.files.filter(row => row.retained).length, 3); assert.deepEqual(result.findings, ["entry_limit"]);
+ assert.deepEqual(result.files.find(row => row.reason === "entry_limit"), { path: ".", scope: "subtree", retained: false, reason: "entry_limit" });
+}));
+
+test("keeps hidden files excluded from upload and exposes that incomplete scope", async () => fixture(async ({ source, out }) => {
+ await mkdir(path.join(source, ".private"));
+ await writeFile(path.join(source, ".private/token"), "fixture");
+ const result = await stageBoundedArtifact("evidence", source, out, bounds);
+ assert.equal(result.inventoryComplete, false); assert.equal(result.sourceComplete, false);
+ assert.deepEqual(result.findings, ["hidden_path_excluded"]);
+ assert.deepEqual(result.files, [{ path: ".private", scope: "subtree", retained: false, reason: "hidden_path_excluded" }]);
+ await assert.rejects(stat(path.join(out, ".private")), { code: "ENOENT" });
+}));
+
+test("excludes ordinary packaged Playwright bookkeeping without failing passing evidence", async () => fixture(async ({ source, out }) => {
+ const folder = "core-compatibility/runner-codex/local/message-marker/attempt-1";
+ await mkdir(path.join(source, folder, "playwright-output"), { recursive: true });
+ await writeFile(path.join(source, folder, "playwright-output/.last-run.json"), '{"status":"passed","failedTests":[]}');
+ await writeFile(path.join(source, folder, "result.json"), '{"status":"passed"}');
+ const result = await stageBoundedArtifact("evidence", source, out, bounds);
+ assert.equal(result.sourceComplete, true); assert.equal(result.inventoryComplete, true);
+ assert.deepEqual(result.findings, []);
+ assert.equal(result.files.find(row => row.path.endsWith(".last-run.json")).reason, "playwright_bookkeeping_excluded");
+ assert.equal(await readFile(path.join(out, folder, "result.json"), "utf8"), '{"status":"passed"}');
+}));
+
+test("discovers snapshots in all attempts before archive directories can consume the entry limit", async () => fixture(async ({ source, out }) => {
+ for (const attempt of ["attempt-a", "attempt-b"]) {
+ await mkdir(path.join(source, attempt, "blob-report"), { recursive: true });
+ await mkdir(path.join(source, attempt, "snapshots"));
+ await writeFile(path.join(source, attempt, "snapshots/feedback.json"), '{"saved":true}');
+ for (let i = 0; i < 20; i++) await writeFile(path.join(source, attempt, `blob-report/${i}.zip`), "archive");
+ }
+ const result = await stageBoundedArtifact("evidence", source, out, { ...bounds, entries: 12 });
+ assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, false);
+ for (const attempt of ["attempt-a", "attempt-b"]) {
+ assert.equal(await readFile(path.join(out, attempt, "snapshots/feedback.json"), "utf8"), '{"saved":true}');
+ assert.ok(result.files.some(row => row.path === `${attempt}/blob-report` && row.scope === "subtree" && row.reason === "entry_limit"));
+ }
+}));
+
+test("identifies an overlong skipped subtree with a bounded preview and full path digest", async () => fixture(async ({ source, out }) => {
+ const segments = Array.from({ length: 6 }, (_, index) => `${index}-${"x".repeat(90)}`);
+ await mkdir(path.join(source, ...segments), { recursive: true });
+ await writeFile(path.join(source, ...segments, "feedback.json"), "{}");
+ const result = await stageBoundedArtifact("evidence", source, out, bounds);
+ const omitted = result.files.find(row => row.reason === "path_length_limit");
+ assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, false);
+ assert.equal(omitted.scope, "subtree"); assert.equal(omitted.pathTruncated, true);
+ assert.match(omitted.pathSha256, /^[a-f0-9]{64}$/); assert.ok(omitted.path.startsWith(segments[0]));
+ assert.ok(omitted.path.endsWith(segments[5].slice(-80)));
+ assert.equal(omitted.pathSha256, createHash("sha256").update(segments.join("/")).digest("hex"));
+ assert.ok(result.stagedBytes <= bounds.evidence);
+}));
+
+test("checks immutable build and lock source totals before upload without reading their contents", async () => fixture(async ({ source }) => {
+ const a = path.join(source, "a"), b = path.join(source, "b");
+ await writeFile(a, "a"); await writeFile(b, "b");
+ await truncate(a, ARTIFACT_BOUNDS.lock - 1);
+ assert.equal((await checkArtifactFiles("lock", [a, b])).sourceBytes, ARTIFACT_BOUNDS.lock);
+ await truncate(a, ARTIFACT_BOUNDS.lock);
+ await assert.rejects(checkArtifactFiles("lock", [a, b]), /exceeds/);
+ await assert.rejects(checkArtifactFiles("other", [a]), /Invalid/);
+}));
diff --git a/tests/runner-e2e/attempt-accounting.test.ts b/tests/runner-e2e/attempt-accounting.test.ts
new file mode 100644
index 0000000000..e7335879ee
--- /dev/null
+++ b/tests/runner-e2e/attempt-accounting.test.ts
@@ -0,0 +1,87 @@
+import { describe, expect, it } from "vitest";
+import { buildCampaignAccounting, isSelectedCarryForward, retainedCampaignAttempts } from "./attempt-accounting.js";
+import { buildRunnerCampaign, campaignHistoryRecord } from "./history.js";
+import type { RunnerE2EResult } from "./types.js";
+
+function result(attempt: number, cost: number, overrides: Partial<RunnerE2EResult> = {}): RunnerE2EResult {
+ return {
+ schema: "paperclip.runner-e2e.result/v2",
+ accountingIdentity: { campaignId: "campaign", attemptId: `attempt-${attempt}` },
+ executionId: "core-compatibility.runner-codex.local.message-marker", suiteId: "core-compatibility",
+ attempt, status: attempt === 1 ? "failed" : "passed", profileId: "runner-codex",
+ environmentId: "local", caseId: "message-marker", provider: "codex", model: "fixture", runtimeMode: "native",
+ startedAt: `2026-10-20T00:0${attempt}:00Z`, finishedAt: `2026-10-20T00:0${attempt}:01Z`, durationMs: 1000,
+ runIds: [`run-${attempt}`], usage: { inputTokens: 100, outputTokens: 10, costUsd: cost }, cleanup: "passed",
+ ...overrides,
+ };
+}
+
+describe("retained attempt accounting", () => {
+ it("adds failed and successful attempts exactly once without changing the chosen verdict", () => {
+ const failed = result(1, 0.01);
+ const passed = result(2, 0.02);
+ const campaign = buildRunnerCampaign({ campaignId: "campaign", generatedAt: passed.finishedAt,
+ expected: [passed.executionId], results: [passed], attempts: [failed, passed, structuredClone(passed)] });
+ expect(campaign).toMatchObject({ passed: 1, failed: 0, selected: 1,
+ accounting: { scope: "retained_attempts", missingAttempts: 0, attempts: [{ attempt: 1 }, { attempt: 2 }] },
+ billing: { testCount: 1, testsWithCompleteBilling: 1, reportedLlmCostUsd: 0.03,
+ llm: { runCount: 2, inputTokens: 200, outputTokens: 20 } } });
+ expect(campaign.results[0]!.billing!.reportedCostUsd).toBe(0.02);
+ expect(campaign.suites[0]!.billing.reportedLlmCostUsd).toBe(0.03);
+ expect(campaignHistoryRecord(campaign, "https://example.test").billing.reportedLlmCostUsd).toBe(0.03);
+ });
+
+ it("retains known costs from interrupted attempts and marks missing attempts", () => {
+ const interrupted = result(2, 0.02, { status: "failed", cleanup: "failed" });
+ const campaign = buildRunnerCampaign({ campaignId: "campaign", generatedAt: interrupted.finishedAt,
+ expected: [interrupted.executionId], results: [interrupted], attempts: [interrupted] });
+ expect(campaign.accounting!.missingAttempts).toBe(1);
+ expect(campaign.billing.reportedLlmCostUsd).toBe(0.02);
+ expect(campaign.billing.testsWithCompleteBilling).toBe(0);
+ const missing = result(0, 0, { runIds: [], usage: null });
+ expect(buildCampaignAccounting({ results: [missing], attempts: [] })).toMatchObject({ missingAttempts: 1, attempts: [] });
+ });
+
+ it("labels legacy selected-only bundles without inventing receipts for retries", () => {
+ const selected = result(2, 0.02, { accountingIdentity: undefined });
+ expect(buildCampaignAccounting({ results: [selected] })).toMatchObject({
+ scope: "selected_results", missingAttempts: 1, attempts: [{ attempt: 2 }],
+ });
+ });
+
+ it("excludes other campaigns and ambiguous local artifacts", () => {
+ const selected = result(2, 0.02);
+ const other = result(1, 0.9, { accountingIdentity: { campaignId: "other", attemptId: "attempt-1" } });
+ const first = result(1, 0.01);
+ expect(retainedCampaignAttempts([selected], [first, selected, other])).toEqual({ attempts: [first, selected], excludedArtifacts: 1 });
+ const legacy = { ...selected, accountingIdentity: undefined };
+ expect(retainedCampaignAttempts([legacy], [legacy, { ...legacy }])).toEqual({ attempts: [legacy], excludedArtifacts: 1 });
+ });
+
+ it("uses the final supervisor receipt but rejects contradictory same-version costs", () => {
+ const selected = result(1, 0.02);
+ const early = { ...selected, finishedAt: selected.startedAt, usage: { inputTokens: 20, costUsd: 0.01 } };
+ expect(buildCampaignAccounting({ results: [selected], attempts: [selected, early] }).attempts).toMatchObject([
+ { billing: { reportedCostUsd: 0.02 } },
+ ]);
+ expect(() => buildCampaignAccounting({ results: [selected], attempts: [selected,
+ { ...selected, usage: { inputTokens: 20, costUsd: 0.1 } }] })).toThrow("Conflicting accounting receipts");
+ });
+});
+
+it("requires exact selector provenance before admitting a carried workflow result", () => {
+ const retained = result(1, 0.01, { accountingIdentity: { campaignId: "gha-123-1", attemptId: "a" },
+ source: { sha: "source", ref: "master", workflowRunUrl: "https://github.com/paperclipai/paperclip/actions/runs/123" } });
+ const artifactName = `runner-e2e-123-1-${retained.executionId}`;
+ const input = { requestedCampaignId: "gha-123-2", result: retained,
+ relativeResultPath: `${artifactName}/gha-123-1-${retained.executionId}/attempt-1/result.json`,
+ selection: { version: 1, workflowRunId: "123", workflowRunAttempt: 2, sourceSha: retained.source!.sha,
+ sourceRef: retained.source!.ref, workflowRunUrl: retained.source!.workflowRunUrl,
+ selections: [{ executionId: retained.executionId, workflowAttempt: 1, artifactName }] } };
+ expect(isSelectedCarryForward(input)).toBe(true);
+ expect(isSelectedCarryForward({ ...input, selection: undefined })).toBe(false);
+ expect(isSelectedCarryForward({ ...input, requestedCampaignId: "gha-999-2" })).toBe(false);
+ expect(isSelectedCarryForward({ ...input, relativeResultPath: `unselected/${input.relativeResultPath}` })).toBe(false);
+ expect(isSelectedCarryForward({ ...input, result: { ...retained, source: { ...retained.source!, sha: "other" } } })).toBe(false);
+ expect(isSelectedCarryForward({ ...input, result: { ...retained, accountingIdentity: { campaignId: "gha-123-3", attemptId: "a" } } })).toBe(false);
+});
diff --git a/tests/runner-e2e/attempt-accounting.ts b/tests/runner-e2e/attempt-accounting.ts
new file mode 100644
index 0000000000..3fea6442d7
--- /dev/null
+++ b/tests/runner-e2e/attempt-accounting.ts
@@ -0,0 +1,99 @@
+import { createHash } from "node:crypto";
+import { summarizeExecutionBilling } from "./billing.js";
+import type { RunnerE2EAttemptReceipt, RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js";
+
+export function isSelectedCarryForward(input: {
+ requestedCampaignId: string; result: RunnerE2EResult; relativeResultPath: string; selection: unknown;
+}): boolean {
+ const s = input.selection as Record<string, unknown> | null;
+ if (!s || s.version !== 1 || typeof s.workflowRunId !== "string" || !/^[0-9]+$/.test(s.workflowRunId)
+ || !Number.isSafeInteger(s.workflowRunAttempt) || Number(s.workflowRunAttempt) < 1
+ || input.requestedCampaignId !== `gha-${s.workflowRunId}-${s.workflowRunAttempt}` || !Array.isArray(s.selections)) return false;
+ const source = input.result.source;
+ if (!source || !source.sha || !source.ref || !source.workflowRunUrl
+ || source.sha !== s.sourceSha || source.ref !== s.sourceRef || source.workflowRunUrl !== s.workflowRunUrl) return false;
+ if (input.relativeResultPath.split("/").some(part => !part || part === "." || part === "..")) return false;
+ return s.selections.some(value => {
+ if (!value || typeof value !== "object") return false;
+ const selected = value as Record<string, unknown>;
+ const attempt = selected.workflowAttempt;
+ const campaign = `gha-${s.workflowRunId}-${attempt}`;
+ const artifact = `runner-e2e-${s.workflowRunId}-${attempt}-${input.result.executionId}`;
+ return selected.executionId === input.result.executionId && Number.isSafeInteger(attempt)
+ && Number(attempt) >= 1 && Number(attempt) < Number(s.workflowRunAttempt)
+ && selected.artifactName === artifact && input.result.accountingIdentity?.campaignId === campaign
+ && input.relativeResultPath.startsWith(`${artifact}/${campaign}-${input.result.executionId}/`);
+ });
+}
+
+function scope(result: RunnerE2EResult): string | undefined {
+ return result.accountingIdentity?.campaignId ?? result.source?.workflowRunUrl ?? undefined;
+}
+
+export function retainedCampaignAttempts(
+ selected: readonly RunnerE2EResult[],
+ candidates: readonly RunnerE2EResult[],
+): { attempts: RunnerE2EResult[]; excludedArtifacts: number } {
+ const attempts = candidates.filter(candidate => selected.some(result =>
+ result.executionId === candidate.executionId &&
+ (scope(result) ? scope(result) === scope(candidate) : result === candidate),
+ ));
+ return { attempts, excludedArtifacts: candidates.length - attempts.length };
+}
+
+function receipt(result: RunnerE2EResult): RunnerE2EAttemptReceipt {
+ const identity = [scope(result) ?? null, result.executionId,
+ result.accountingIdentity?.attemptId ?? [result.attempt, result.startedAt]];
+ return {
+ id: createHash("sha256").update(JSON.stringify(identity)).digest("hex"),
+ executionId: result.executionId,
+ attempt: result.attempt,
+ startedAt: result.startedAt,
+ finishedAt: result.finishedAt,
+ durationMs: result.durationMs,
+ ...(result.workerTiming ? { workerTiming: result.workerTiming } : {}),
+ status: result.status,
+ cleanup: result.cleanup,
+ billing: summarizeExecutionBilling(result),
+ ...(result.feedback ? { feedback: result.feedback } : {}),
+ ...(result.tracing ? { tracing: result.tracing } : {}),
+ };
+}
+
+export function buildCampaignAccounting(input: {
+ results: readonly RunnerE2EResult[];
+ attempts?: readonly RunnerE2EResult[];
+ excludedArtifacts?: number;
+}): RunnerE2ECampaignAccounting {
+ const receipts = new Map<string, RunnerE2EAttemptReceipt>();
+ for (const result of input.attempts ?? input.results) {
+ if (result.attempt === 0) continue; // Missing result placeholders are not free attempts.
+ const next = receipt(result);
+ const previous = receipts.get(next.id);
+ if (previous && previous.finishedAt === next.finishedAt &&
+ JSON.stringify(previous.billing) !== JSON.stringify(next.billing)) {
+ throw new Error(`Conflicting accounting receipts for ${result.executionId} attempt ${result.attempt}`);
+ }
+ if (previous && previous.finishedAt === next.finishedAt && previous.workerTiming && next.workerTiming &&
+ JSON.stringify(previous.workerTiming) !== JSON.stringify(next.workerTiming)) {
+ throw new Error(`Conflicting timing receipts for ${result.executionId} attempt ${result.attempt}`);
+ }
+ if (previous && previous.finishedAt === next.finishedAt && previous.workerTiming && !next.workerTiming) next.workerTiming = previous.workerTiming;
+ if (!previous || Date.parse(next.finishedAt) >= Date.parse(previous.finishedAt)) receipts.set(next.id, next);
+ }
+ const attempts = [...receipts.values()].sort((a, b) =>
+ a.executionId.localeCompare(b.executionId) || a.attempt - b.attempt || a.id.localeCompare(b.id));
+ let missingAttempts = 0;
+ for (const result of input.results) {
+ const retained = attempts.filter(attempt => attempt.executionId === result.executionId);
+ const latest = Math.max(result.attempt, ...retained.map(attempt => attempt.attempt), 1);
+ missingAttempts += Math.max(0, latest - new Set(retained.map(attempt => attempt.attempt)).size);
+ }
+ return {
+ version: 1,
+ scope: input.attempts ? "retained_attempts" : "selected_results",
+ attempts,
+ missingAttempts,
+ excludedArtifacts: input.excludedArtifacts ?? 0,
+ };
+}
diff --git a/tests/runner-e2e/attempt-phase.ts b/tests/runner-e2e/attempt-phase.ts
new file mode 100644
index 0000000000..1c8204e4c6
--- /dev/null
+++ b/tests/runner-e2e/attempt-phase.ts
@@ -0,0 +1,34 @@
+import { randomUUID } from "node:crypto";
+import { readFile, rename, rm, writeFile } from "node:fs/promises";
+import type { EvalChargeIdentity } from "./cost-ledger.js";
+
+type Identity = Pick<EvalChargeIdentity, "executionId" | "attemptId">;
+interface Checkpoint extends Identity { version: 1; phase: "setup" | "agent" }
+
+export async function createEvalAttemptPhase(file: string, identity: Identity) {
+ await writeFile(file, JSON.stringify({ version: 1, ...identity, phase: "setup" }), { mode: 0o600, flag: "wx" });
+}
+
+export async function readEvalAttemptPhase(file: string, expected: Identity): Promise<EvalChargeIdentity> {
+ const text = await readFile(file, "utf8");
+ if (text.length > 4096) throw new Error("Invalid eval phase checkpoint");
+ const value = JSON.parse(text) as Checkpoint;
+ if (!value || typeof value !== "object" || Array.isArray(value)
+ || Object.keys(value).some(key => !["version", "executionId", "attemptId", "phase"].includes(key))
+ || value.version !== 1 || value.executionId !== expected.executionId || value.attemptId !== expected.attemptId
+ || !["setup", "agent"].includes(value.phase)) throw new Error("Invalid eval phase checkpoint");
+ return { ...expected, phase: value.phase };
+}
+
+export async function beginEvalAgentPhase(executionId: string, env: NodeJS.ProcessEnv = process.env) {
+ const file = env.PAPERCLIP_E2E_GATEWAY_PHASE_PATH;
+ if (!file && !env.PAPERCLIP_E2E_GATEWAY_BASE_URL) return;
+ if (!file || !env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID) throw new Error("Missing eval phase checkpoint");
+ const identity = await readEvalAttemptPhase(file, { executionId, attemptId: env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID });
+ if (identity.phase === "agent") return;
+ const temporary = `${file}.${randomUUID()}.tmp`;
+ try {
+ await writeFile(temporary, JSON.stringify({ version: 1, ...identity, phase: "agent" }), { mode: 0o600, flag: "wx" });
+ await rename(temporary, file);
+ } finally { await rm(temporary, { force: true }); }
+}
diff --git a/tests/runner-e2e/billing.test.ts b/tests/runner-e2e/billing.test.ts
index f16e2d9a60..48352815c0 100644
--- a/tests/runner-e2e/billing.test.ts
+++ b/tests/runner-e2e/billing.test.ts
@@ -6,6 +6,8 @@ import {
summarizeExecutionBilling,
} from "./billing.js";
import type { RunnerE2EResult } from "./types.js";
+import { durationLabel, llmCostLabel } from "./report-format.js";
+import { validateRetainedRunnerResult } from "./result-validation.js";
function result(overrides: Partial<RunnerE2EResult> = {}): RunnerE2EResult {
return {
@@ -29,6 +31,80 @@ function result(overrides: Partial<RunnerE2EResult> = {}): RunnerE2EResult {
}
describe("runner E2E billing summaries", () => {
+ it("keeps missing agent time separate from attempt duration and measured zero", () => {
+ const interrupted = result({ status: "failed", durationMs: 108_850 });
+ expect(summarizeExecutionBilling(interrupted).runtime.agentRunDurationMs).toBeNull();
+ const noCapturedRuns = result({ ...interrupted, runIds: [],
+ runtimeUsage: buildRuntimeUsage({ environmentId: "local", runs: [] }) });
+ expect(noCapturedRuns.runtimeUsage?.agentRunDurationMs).toBeNull();
+ expect(() => validateRetainedRunnerResult(noCapturedRuns)).not.toThrow();
+ const measured = result({ runtimeUsage: buildRuntimeUsage({ environmentId: "local", runs: [
+ { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:04Z" },
+ { startedAt: "2026-10-21T00:00:01Z", finishedAt: "2026-10-21T00:00:05Z" },
+ ] }) });
+ expect(aggregateCampaignBilling([measured]).agentRunDurationMs).toBe(8000);
+ expect(aggregateCampaignBilling([measured, interrupted]).agentRunDurationMs).toBeNull();
+ expect(aggregateCampaignBilling([measured, noCapturedRuns]).agentRunDurationMs).toBeNull();
+ expect(durationLabel(aggregateCampaignBilling([measured, interrupted]).agentRunDurationMs)).toBe("Unknown");
+ const zero = buildRuntimeUsage({ environmentId: "local", runs: [
+ { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:00Z" },
+ ] });
+ expect(zero.agentRunDurationMs).toBe(0);
+ expect(durationLabel(zero.agentRunDurationMs)).toBe("0ms");
+ });
+
+ it.each([undefined, "not-a-date", "2026-10-20T23:59:59Z"])("retains unknown time for a missing or invalid end timestamp: %s", finishedAt => {
+ const runtimeUsage = buildRuntimeUsage({ environmentId: "daytona", runs: [
+ { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:04Z" },
+ { startedAt: "2026-10-21T00:00:00Z", finishedAt },
+ ], leases: [{ acquiredAt: "2026-10-21T00:00:00Z", releasedAt: finishedAt,
+ metadata: { cpu: 4, memory: 4, disk: 10 } }] });
+ expect(runtimeUsage).toMatchObject({ agentRunDurationMs: null, leaseDurationMs: null, costStatus: "unavailable" });
+ expect(runtimeUsage.estimatedListCostUsd).toBeUndefined();
+ const retained = result({ environmentId: "daytona", runtimeUsage });
+ expect(() => validateRetainedRunnerResult(retained)).not.toThrow();
+ expect(aggregateCampaignBilling([retained, result()]).leaseDurationMs).toBeNull();
+ expect(aggregateCampaignBilling([result()]).leaseDurationMs).toBe(0);
+ });
+
+ it("reads precise provider amounts without relabeling estimates or unpriced usage", () => {
+ const precise = summarizeExecutionBilling(result({ usage: {
+ inputTokens: 0, outputTokens: 0, cachedInputTokens: 5,
+ costUsdExact: "0.000000123", costStatus: "reported", accountingReceiptReady: true,
+ } }));
+ expect(precise.llm).toMatchObject({ reportedCostUsd: 0.000000123, runsWithReportedCost: 1, runsWithTokenUsage: 1 });
+ const estimated = summarizeExecutionBilling(result({ usage: {
+ inputTokens: 100, outputTokens: 10, costUsdExact: "0.012345678", costStatus: "estimated",
+ } }));
+ expect(estimated.llm).toMatchObject({ reportedCostUsd: 0, estimatedCostUsd: 0.012345678,
+ runsWithReportedCost: 0, runsWithEstimatedCost: 1, costStatus: "estimated" });
+ expect(estimated.observedAndEstimatedCostUsd).toBe(0.012345678);
+ const unpriced = summarizeExecutionBilling(result({ usage: {
+ inputTokens: 100, outputTokens: 10, costUsd: 0, costStatus: "unpriced",
+ } }));
+ expect(unpriced.llm).toMatchObject({ runsWithReportedCost: 0, costStatus: "unpriced" });
+ });
+
+ it("distinguishes explicit zero usage from a partial or missing receipt", () => {
+ const zero = summarizeExecutionBilling(result({ usage: { inputTokens: 0, outputTokens: 0, costUsdExact: "0", accountingReceiptReady: true } }));
+ expect(zero.complete).toBe(true);
+ const partial = summarizeExecutionBilling(result({ usage: { inputTokens: 100, outputTokens: 10, costUsd: 0.01, accountingReceiptReady: false } }));
+ expect(partial.reportedCostUsd).toBe(0.01);
+ expect(partial.complete).toBe(false);
+ expect(summarizeExecutionBilling(result({ usage: null })).complete).toBe(false);
+ });
+ it("retains a known OpenCode retry subtotal while the missing turn price stays incomplete", () => {
+ const partial = result({ usage: { inputTokens: 30, outputTokens: 6, costUsd: 0.5, costStatus: "unpriced", accountingReceiptReady: true } });
+ const billing = summarizeExecutionBilling(partial);
+ expect(billing.llm).toMatchObject({ reportedCostUsd: 0.5, runsWithReportedCost: 1, runsWithPartialReportedCost: 1, costStatus: "partial" });
+ expect(billing.observedAndEstimatedCostUsd).toBe(0.5);
+ expect(billing.complete).toBe(false);
+ expect(llmCostLabel(billing.reportedCostUsd, billing.llm)).toBe("$0.5000 (partial amount)");
+ const campaign = aggregateCampaignBilling([partial, result({ executionId: "other", usage: { inputTokens: 1, outputTokens: 1, costUsd: 0.2 } })]);
+ expect(campaign).toMatchObject({ reportedLlmCostUsd: 0.7, observedAndEstimatedCostUsd: 0.7, testsWithCompleteBilling: 1,
+ llm: { runsWithReportedCost: 2, runsWithPartialReportedCost: 1, costStatus: "partial" } });
+ expect(llmCostLabel(campaign.reportedLlmCostUsd, campaign.llm)).toBe("$0.7000 (partial amount)");
+ });
it("counts completion judge reservations and preserves unknown spend after interruption", () => {
const pending = { ...reserveCompletionQuality({ sourceId: "chat", marker: "x", worker: { id: "task", status: "done", completedAt: "2026-09-01" }, documents: [{ id: "doc", issueId: "task", body: "result" }], comments: [{ id: "reply", issueId: "chat", authorAgentId: "agent", createdAt: "2026-09-02", body: "ready" }], runs: [] }, 0.5), name: "completion", expectedPass: true };
const unknown = summarizeExecutionBilling(result({ completionQuality: [pending] }));
diff --git a/tests/runner-e2e/billing.ts b/tests/runner-e2e/billing.ts
index 4681696a7e..35f70e8356 100644
--- a/tests/runner-e2e/billing.ts
+++ b/tests/runner-e2e/billing.ts
@@ -32,21 +32,30 @@ export interface CampaignBillingSummary {
assistant?: import("./types.js").RunnerE2EAggregateBillingSummary["assistant"];
judge?: import("./types.js").RunnerE2EJudgeBillingSummary;
testCount: number;
- agentRunDurationMs: number;
- leaseDurationMs: number;
+ agentRunDurationMs: number | null;
+ leaseDurationMs: number | null;
llm: RunnerE2EBillingSummary["llm"];
reportedLlmCostUsd: number;
+ estimatedLlmCostUsd?: number;
estimatedRuntimeCostUsd: number;
observedAndEstimatedCostUsd: number | null;
testsWithCompleteBilling: number;
}
/** Render observed subtotals without presenting absent measurements as zero. */
-export function billingCoverageLabel(value: string, covered: number, total: number): string {
+export function billingCoverageLabel(value: string, covered: number, total: number, incompleteAmount = false): string {
if (!Number.isInteger(covered) || !Number.isInteger(total) || covered <= 0 || total <= 0 || covered > total) {
return "Unavailable";
}
- return covered < total ? `${value} (partial: ${covered}/${total} runs)` : value;
+ return covered < total ? `${value} (partial: ${covered}/${total} runs)` : incompleteAmount ? `${value} (partial amount)` : value;
+}
+
+function costCoverageStatus(runs: number, tokens: number, reported: number, estimated: number, partial = 0): RunnerE2EBillingSummary["llm"]["costStatus"] {
+ if (partial > 0) return "partial";
+ if (runs > 0 && reported === runs) return "reported";
+ if (runs > 0 && reported + estimated === runs) return "estimated";
+ if (reported + estimated > 0) return "partial";
+ return tokens > 0 ? "unpriced" : "unavailable";
}
function record(value: unknown): Record<string, unknown> {
@@ -92,24 +101,25 @@ function usageMeasurement(usage: Record<string, unknown>) {
}
function usageCostUsd(usage: Record<string, unknown>) {
+ if (usage.billingType === "subscription") return undefined;
const measurement = usageMeasurement(usage);
+ const exact = typeof usage.costUsdExact === "string" && /^\d+(?:\.\d+)?$/.test(usage.costUsdExact)
+ ? finiteNumber(Number(usage.costUsdExact)) : undefined;
const direct =
- firstNumber(usage, [
- "cacheAdjustedCostUsd",
- "costUsd",
- "providerCostUsd",
- ]) ??
+ firstNumber(usage, ["cacheAdjustedCostUsd"]) ?? exact ??
+ firstNumber(usage, ["costUsd", "providerCostUsd"]) ??
firstNumber(measurement, [
"cacheAdjustedCostUsd",
"costUsd",
"providerCostUsd",
]);
- if (direct !== undefined) return direct;
+ if (direct !== undefined) return usage.costStatus === "unpriced" && direct === 0 ? undefined : direct;
const cost = record(usage.cost);
const currency =
typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
if (currency !== "USD") return undefined;
- return firstNumber(cost, ["amount", "total"]);
+ const amount = firstNumber(cost, ["amount", "total"]);
+ return usage.costStatus === "unpriced" && amount === 0 ? undefined : amount;
}
function usageEntries(
@@ -144,9 +154,13 @@ function durationBetween(
) {
const started = startedAt ? new Date(startedAt).getTime() : Number.NaN;
const finished = finishedAt ? new Date(finishedAt).getTime() : Number.NaN;
- return Number.isFinite(started) && Number.isFinite(finished)
- ? Math.max(0, finished - started)
- : 0;
+ return Number.isFinite(started) && Number.isFinite(finished) && finished >= started
+ ? finished - started
+ : null;
+}
+
+function sumDurations(values: readonly (number | null)[]): number | null {
+ return values.some(value => value === null) ? null : values.reduce<number>((total, value) => total + value!, 0);
}
function resourceValue(
@@ -162,10 +176,9 @@ export function buildRuntimeUsage(input: {
leases?: readonly RuntimeLeaseUsageInput[];
fallbackFinishedAt?: string | Date;
}): RunnerE2ERuntimeUsage {
- const agentRunDurationMs = input.runs.reduce(
- (total, run) => total + durationBetween(run.startedAt, run.finishedAt),
- 0,
- );
+ const agentRunDurationMs = input.runs.length > 0
+ ? sumDurations(input.runs.map(run => durationBetween(run.startedAt, run.finishedAt)))
+ : null;
if (input.environmentId === "local") {
return {
provider: "local",
@@ -179,6 +192,7 @@ export function buildRuntimeUsage(input: {
const leases = input.leases ?? [];
let leaseDurationMs = 0;
+ let leaseDurationsComplete = leases.length > 0;
let estimatedListCostUsd = 0;
let resourcesComplete = leases.length > 0;
const resourceRows: Array<{ cpu: number; memory: number; disk: number }> = [];
@@ -186,6 +200,11 @@ export function buildRuntimeUsage(input: {
const finishedAt =
lease.releasedAt ?? input.fallbackFinishedAt ?? lease.updatedAt ?? null;
const durationMs = durationBetween(lease.acquiredAt, finishedAt);
+ if (durationMs === null) {
+ leaseDurationsComplete = false;
+ resourcesComplete = false;
+ continue;
+ }
leaseDurationMs += durationMs;
const cpu = resourceValue(lease.metadata, "cpu");
const memory = resourceValue(lease.metadata, "memory");
@@ -209,7 +228,7 @@ export function buildRuntimeUsage(input: {
return {
provider: "daytona",
agentRunDurationMs,
- leaseDurationMs: leases.length > 0 ? leaseDurationMs : null,
+ leaseDurationMs: leaseDurationsComplete ? leaseDurationMs : null,
leaseCount: leases.length,
...(commonResource("cpu") === undefined
? {}
@@ -243,7 +262,7 @@ export function fallbackRuntimeUsage(
if (result.runtimeUsage) return result.runtimeUsage;
return {
provider: result.environmentId,
- agentRunDurationMs: result.durationMs,
+ agentRunDurationMs: null,
leaseDurationMs: null,
leaseCount: 0,
costStatus:
@@ -269,7 +288,11 @@ export function summarizeExecutionBilling(
let cachedInputTokens = 0;
let runsWithTokenUsage = 0;
let runsWithReportedCost = 0;
+ let runsWithEstimatedCost = 0;
+ let runsWithPartialReportedCost = 0;
let reportedCostUsd = 0;
+ let estimatedCostUsd = 0;
+ let incompleteReceipt = false;
for (const usage of entries) {
if (!usage) continue;
const measurement = usageMeasurement(usage);
@@ -290,22 +313,24 @@ export function summarizeExecutionBilling(
inputTokens += input;
outputTokens += output;
cachedInputTokens += cached;
- if (input > 0 || output > 0 || cached > 0) runsWithTokenUsage += 1;
+ const completeTokens = firstNumber(measurement, ["inputTokens", "input", "promptTokens"]) !== undefined &&
+ firstNumber(measurement, ["outputTokens", "output", "completionTokens"]) !== undefined && usage.accountingReceiptReady !== false;
+ if (completeTokens) runsWithTokenUsage += 1;
+ if (usage.accountingReceiptReady === false) incompleteReceipt = true;
const costUsd = usageCostUsd(usage);
- if (costUsd !== undefined && (costUsd > 0 || input > 0 || output > 0)) {
- runsWithReportedCost += 1;
- reportedCostUsd += costUsd;
+ if (costUsd !== undefined) {
+ if (usage.costStatus === "estimated") {
+ runsWithEstimatedCost += 1;
+ estimatedCostUsd += costUsd;
+ } else {
+ runsWithReportedCost += 1;
+ if (usage.costStatus === "unpriced") runsWithPartialReportedCost += 1;
+ reportedCostUsd += costUsd;
+ }
}
}
const runCount = Math.max(requestedRunCount, entries.length);
- const costStatus =
- runsWithReportedCost === runCount
- ? "reported"
- : runsWithReportedCost > 0
- ? "partial"
- : runsWithTokenUsage > 0
- ? "unpriced"
- : "unavailable";
+ const costStatus = costCoverageStatus(runCount, runsWithTokenUsage, runsWithReportedCost, runsWithEstimatedCost, runsWithPartialReportedCost);
const runtime = { ...fallbackRuntimeUsage(result) };
if (browserOnly) runtime.agentRunDurationMs = 0;
const estimatedRuntimeCostUsd = runtime.estimatedListCostUsd ?? 0;
@@ -315,12 +340,13 @@ export function summarizeExecutionBilling(
// The operations receipt covers setup and agent requests for ONE owning attempt.
// Descendants reuse the fixture context without copying its receipt or charges.
const operations = result.productJourney?.operations;
- const modelSubtotal = operations ? operations.modelCostUsd : reportedCostUsd;
+ const modelSubtotal = operations ? operations.modelCostUsd : reportedCostUsd + estimatedCostUsd;
const complete =
+ !incompleteReceipt && runsWithPartialReportedCost === 0 &&
(!quality || quality.estimatedCostUsd !== null) &&
(!operations || operations.modelCostUsd !== null) &&
runsWithTokenUsage === runCount &&
- runsWithReportedCost === runCount &&
+ runsWithReportedCost + runsWithEstimatedCost === runCount &&
runtime.costStatus !== "unavailable";
return {
...(result.publicMcp ? { assistant: result.publicMcp } : {}),
@@ -328,11 +354,14 @@ export function summarizeExecutionBilling(
runCount,
runsWithTokenUsage,
runsWithReportedCost,
+ runsWithEstimatedCost,
+ runsWithPartialReportedCost,
inputTokens,
outputTokens,
cachedInputTokens,
totalTokens: inputTokens + cachedInputTokens + outputTokens,
reportedCostUsd,
+ estimatedCostUsd,
costStatus,
},
runtime,
@@ -347,7 +376,16 @@ export function summarizeExecutionBilling(
export function aggregateCampaignBilling(
results: readonly RunnerE2EResult[],
): CampaignBillingSummary {
- const summaries = results.map(summarizeExecutionBilling);
+ return aggregateBillingSummaries(results.map(result => ({
+ executionId: result.executionId, billing: summarizeExecutionBilling(result),
+ })));
+}
+
+export function aggregateBillingSummaries(
+ entries: readonly { executionId: string; billing: RunnerE2EBillingSummary }[],
+): CampaignBillingSummary {
+ const summaries = entries.map(entry => entry.billing);
+ const executionIds = [...new Set(entries.map(entry => entry.executionId))];
const runCount = summaries.reduce(
(total, summary) => total + summary.llm.runCount,
0,
@@ -364,6 +402,9 @@ export function aggregateCampaignBilling(
(total, summary) => total + summary.reportedCostUsd,
0,
);
+ const runsWithEstimatedCost = summaries.reduce((sum, summary) => sum + (summary.llm.runsWithEstimatedCost ?? 0), 0);
+ const runsWithPartialReportedCost = summaries.reduce((sum, summary) => sum + (summary.llm.runsWithPartialReportedCost ?? 0), 0);
+ const estimatedLlmCostUsd = summaries.reduce((sum, summary) => sum + (summary.llm.estimatedCostUsd ?? 0), 0);
const estimatedRuntimeCostUsd = summaries.reduce(
(total, summary) => total + summary.estimatedRuntimeCostUsd,
0,
@@ -386,19 +427,16 @@ export function aggregateCampaignBilling(
reservedCostUsd: judges.reduce((n, q) => n + q.reservedCostUsd, 0),
attemptsWithUnknownUsage: judges.filter(q => q.estimatedCostUsd === null).length,
} } : {}),
- testCount: results.length,
- agentRunDurationMs: summaries.reduce(
- (total, summary) => total + summary.runtime.agentRunDurationMs,
- 0,
- ),
- leaseDurationMs: summaries.reduce(
- (total, summary) => total + (summary.runtime.leaseDurationMs ?? 0),
- 0,
- ),
+ testCount: executionIds.length,
+ agentRunDurationMs: sumDurations(summaries.map(summary => summary.runtime.agentRunDurationMs)),
+ leaseDurationMs: sumDurations(summaries.filter(summary => summary.runtime.provider === "daytona")
+ .map(summary => summary.runtime.leaseDurationMs)),
llm: {
runCount,
runsWithTokenUsage,
runsWithReportedCost,
+ runsWithEstimatedCost,
+ runsWithPartialReportedCost,
inputTokens: summaries.reduce(
(total, summary) => total + summary.llm.inputTokens,
0,
@@ -416,19 +454,14 @@ export function aggregateCampaignBilling(
0,
),
reportedCostUsd: reportedLlmCostUsd,
- costStatus:
- runsWithReportedCost === runCount
- ? "reported"
- : runsWithReportedCost > 0
- ? "partial"
- : runsWithTokenUsage > 0
- ? "unpriced"
- : "unavailable",
+ estimatedCostUsd: estimatedLlmCostUsd,
+ costStatus: costCoverageStatus(runCount, runsWithTokenUsage, runsWithReportedCost, runsWithEstimatedCost, runsWithPartialReportedCost),
},
reportedLlmCostUsd,
+ estimatedLlmCostUsd,
estimatedRuntimeCostUsd,
observedAndEstimatedCostUsd: summaries.some(s => s.observedAndEstimatedCostUsd === null) ? null : summaries.reduce((total, summary) => total + (summary.observedAndEstimatedCostUsd ?? 0), 0),
- testsWithCompleteBilling: summaries.filter((summary) => summary.complete)
- .length,
+ testsWithCompleteBilling: executionIds.filter(id =>
+ entries.filter(entry => entry.executionId === id).every(entry => entry.billing.complete)).length,
};
}
diff --git a/tests/runner-e2e/braintrust-experiment-import.ts b/tests/runner-e2e/braintrust-experiment-import.ts
new file mode 100644
index 0000000000..eef8f3598c
--- /dev/null
+++ b/tests/runner-e2e/braintrust-experiment-import.ts
@@ -0,0 +1,45 @@
+import { readFile, rename, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { evalBraintrustConfig } from "./braintrust.js";
+import { buildBraintrustExperiment, importBraintrustExperiment } from "./braintrust-experiment.js";
+
+const [campaignFile, originalPathsFile, outputFile, mode] = process.argv.slice(2);
+if (!campaignFile || !originalPathsFile || !outputFile || ![undefined, "--dry-run"].includes(mode)) {
+ throw new Error("Usage: braintrust-experiment-import.ts <normalized-results.json> <original-result-paths.json> <receipt.json> [--dry-run]");
+}
+let totalBytes = 0;
+const readJson = async (file: string) => {
+ const bytes = await readFile(file);
+ totalBytes += bytes.length;
+ if (bytes.length > 8 * 1024 * 1024 || totalBytes > 64 * 1024 * 1024) throw new Error("Experiment input exceeds its file or campaign size limit");
+ return JSON.parse(bytes.toString("utf8"));
+};
+const paths = await readJson(originalPathsFile);
+if (!Array.isArray(paths) || paths.length === 0 || paths.length > 10_000
+ || paths.some(file => typeof file !== "string" || !path.isAbsolute(file))) throw new Error("Expected explicit absolute original result paths");
+if ([campaignFile, originalPathsFile, ...paths].some(file => path.resolve(file) === path.resolve(outputFile))) {
+ throw new Error("Experiment output must not replace input evidence");
+}
+const campaign = await readJson(campaignFile), originals = [];
+for (const file of paths) originals.push(await readJson(file));
+const projectId = process.env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "";
+const plan = buildBraintrustExperiment(projectId, campaign, originals);
+const persist = async (value: unknown) => {
+ const temporary = `${outputFile}.${randomUUID()}.tmp`;
+ await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx", mode: 0o600 });
+ await rename(temporary, outputFile);
+};
+if (mode === "--dry-run") {
+ await persist(plan);
+ console.log(`Prepared ${plan.events.length} retained rows; no network requests made.`);
+} else {
+ const config = evalBraintrustConfig(process.env);
+ if (!config) throw new Error("Braintrust project and ingestion credential required");
+ const previous = await readJson(outputFile).catch(error => {
+ if (error?.code === "ENOENT") return undefined;
+ throw error;
+ });
+ const receipt = await importBraintrustExperiment(config, campaign, originals, persist, fetch, previous);
+ console.log(`Braintrust accepted ${receipt.acceptedRows} retained rows in experiment ${receipt.experimentId}; indexed verification remains required.`);
+}
diff --git a/tests/runner-e2e/braintrust-experiment.test.ts b/tests/runner-e2e/braintrust-experiment.test.ts
new file mode 100644
index 0000000000..41c61fc5b2
--- /dev/null
+++ b/tests/runner-e2e/braintrust-experiment.test.ts
@@ -0,0 +1,328 @@
+import { expect, it, vi } from "vitest";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { buildBraintrustExperiment, importBraintrustExperiment, renderBraintrustExperiment, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";
+import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
+import { buildRunnerCampaign } from "./history.js";
+import { runnerMatrix } from "./catalog.js";
+import { emptyFeedbackSummary } from "./feedback.js";
+import { openExperimentIngestion } from "./experiment-ingestion.js";
+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";
+
+const projectId = "11111111-1111-4111-8111-111111111111";
+const experimentId = "22222222-2222-4222-8222-222222222222";
+function fixture() {
+ const cell = runnerMatrix.find(cell => cell.id === "core-compatibility.runner-codex.local.message-marker")!;
+ const first: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id,
+ suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash,
+ accountingIdentity: { campaignId: "retained-pilot", attemptId: "first" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null },
+ attempt: 1, status: "failed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id,
+ environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,
+ startedAt: "2026-10-20T00:00:00.000Z", finishedAt: "2026-10-20T00:00:01.000Z", durationMs: 1000,
+ error: "private-canary-error", usage: { privateCanary: "private-canary-usage" }, feedback: emptyFeedbackSummary() };
+ const second: RunnerE2EResult = { ...structuredClone(first), source: { ...first.source!, sha: "b".repeat(40) },
+ accountingIdentity: { campaignId: "retained-pilot", attemptId: "second" }, attempt: 2, status: "passed", error: undefined,
+ startedAt: "2026-10-20T00:01:00.000Z", finishedAt: "2026-10-20T00:01:01.000Z",
+ tracing: { projectId, traceId: "c".repeat(32), spanId: "d".repeat(16), status: "accepted", verification: "unverified", nativeConfiguration: "configured" } };
+ const originals = [first, second];
+ const campaign = buildRunnerCampaign({ campaignId: "retained-pilot", generatedAt: second.finishedAt,
+ expected: [cell.id], results: [second], attempts: originals });
+ return { campaign, originals };
+}
+
+it("imports all attempts with their own source and original grades, excluding private evidence and duplicate cost metrics", () => {
+ const { campaign, originals } = fixture();
+ const plan = buildBraintrustExperiment(projectId, campaign, originals);
+ expect(plan.events).toHaveLength(2);
+ expect(plan.events.map(row => row.scores.product_pass).sort()).toEqual([0, 1]);
+ expect(plan.events.map(row => row.input.source_revision).sort()).toEqual(["a".repeat(40), "b".repeat(40)]);
+ expect(JSON.stringify(plan)).not.toMatch(/private-canary|estimated_cost|prompt_tokens/);
+ expect(plan.events.every(row => row.metadata.request_ledger === null)).toBe(true);
+ expect(plan.events.find(row => row.metadata.attempt_id === "second")!.metadata.trace).toMatchObject({ root_span_id: "c".repeat(32) });
+ expect(buildBraintrustExperiment(projectId, { ...campaign, generatedAt: "2026-10-21T00:00:00Z" }, [...originals].reverse())).toEqual(plan);
+ expect(buildBraintrustExperiment(projectId, campaign, [...originals, originals[0]!])).toEqual(plan);
+});
+
+it("imports retained grades through the scheduled transport allowance without changing the evidence", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "bounded-experiment-import-"));
+ const { campaign, originals } = fixture(), before = structuredClone({ campaign, originals });
+ const plan = buildBraintrustExperiment(projectId, campaign, originals), receiptPath = path.join(root, "ingestion.jsonl");
+ const meter = await openExperimentIngestion({ projectId, campaignId: campaign.campaignId, receiptPath, deadlineAt: Date.now() + 60_000,
+ transport: async (url, init) => String(url).endsWith("/experiment")
+ ? Response.json({ ...JSON.parse(String(init!.body)), id: experimentId })
+ : String(url).endsWith("/insert") ? Response.json({ row_ids: plan.events.map(row => row.id) }) : Response.json({}) });
+ try {
+ expect(await importBraintrustExperiment({ projectId, apiKey: "private-fixture" }, campaign, originals, async () => {}, meter.transport))
+ .toMatchObject({ status: "accepted", acceptedRows: 2, expectedRows: 2 });
+ await meter.close();
+ const rows = (await readFile(receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ expect(rows.at(-1)).toMatchObject({ admittedRequests: 3, admittedScores: 4, unresolvedRequests: 0 });
+ expect({ campaign, originals }).toEqual(before);
+ } finally { await meter.close(); await rm(root, { recursive: true, force: true }); }
+});
+
+it.each(["missing", "foreign", "modified-grade", "ambiguous-source", "missing-source", "foreign-trace", "missing-hash"])("rejects %s evidence before network requests", async (kind) => {
+ const { campaign, originals } = fixture();
+ if (kind === "missing") originals.pop();
+ if (kind === "foreign") originals[0]!.accountingIdentity!.campaignId = "other";
+ if (kind === "modified-grade") originals[0]!.status = "passed";
+ if (kind === "ambiguous-source") originals.push({ ...originals[0]!, source: { ...originals[0]!.source!, sha: "c".repeat(40) } });
+ if (kind === "missing-source") originals[0]!.source = undefined;
+ if (kind === "foreign-trace") originals[1]!.tracing!.projectId = experimentId;
+ if (kind === "missing-hash") originals[0]!.suiteDefinitionHash = undefined;
+ const transport = vi.fn<typeof fetch>();
+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, transport)).rejects.toThrow();
+ expect(transport).not.toHaveBeenCalled();
+});
+
+it("changes experiment identity for changed evidence while keeping unknown accounting unknown", () => {
+ const { campaign, originals } = fixture();
+ const before = buildBraintrustExperiment(projectId, campaign, originals);
+ campaign.accounting!.missingAttempts = 1;
+ const after = buildBraintrustExperiment(projectId, campaign, originals);
+ expect(after.payloadHash).not.toBe(before.payloadHash);
+ expect(after.metadata.request_ledger).toBeNull();
+ expect(after.metadata.missing_attempts).toBe(1);
+});
+
+it("imports carried-forward receipts with their original campaign, revision, and grades", () => {
+ const { campaign, originals } = fixture();
+ const report = { ...campaign, campaignId: "later-ci-attempt" };
+ const before = structuredClone(originals);
+ const plan = buildBraintrustExperiment(projectId, report, originals);
+ expect(plan.metadata.campaign_id).toBe("later-ci-attempt");
+ expect(plan.events.every(row => row.metadata.campaign_id === "retained-pilot")).toBe(true);
+ expect(plan.events.map(row => row.input.source_revision).sort()).toEqual(["a".repeat(40), "b".repeat(40)]);
+ expect(plan.events.map(row => row.scores.product_pass).sort()).toEqual([0, 1]);
+ expect(plan.events.map(row => row.id)).toEqual(buildBraintrustExperiment(projectId, campaign, originals).events.map(row => row.id));
+ expect(originals).toEqual(before);
+ const wrong = structuredClone(originals);
+ wrong[0]!.accountingIdentity!.campaignId = report.campaignId;
+ expect(() => buildBraintrustExperiment(projectId, report, wrong)).toThrow();
+});
+
+it("imports original grades when report evidence validation changes the selected outcome", () => {
+ const { campaign, originals } = fixture();
+ const adjusted = structuredClone(campaign);
+ adjusted.results[0]!.status = "failed";
+ adjusted.results[0]!.error = "Missing final-state.png";
+ const plan = buildBraintrustExperiment(projectId, adjusted, originals);
+ expect(plan.events.find(row => row.metadata.attempt_id === "second")!.scores.product_pass).toBe(1);
+ expect(plan).toEqual(buildBraintrustExperiment(projectId, campaign, originals));
+ expect(adjusted.results[0]!.status).toBe("failed");
+ const changed = structuredClone(originals);
+ changed[1]!.status = "failed";
+ expect(() => buildBraintrustExperiment(projectId, adjusted, changed)).toThrow();
+});
+
+it("accepts current catalog model names including Cursor qualification parameters", () => {
+ for (const model of new Set(runnerMatrix.map(cell => cell.profile.model))) {
+ const { campaign, originals } = fixture();
+ const changed = originals.map(original => ({ ...original, model }));
+ const selected = { ...campaign, results: campaign.results.map(result => ({ ...result, model })) };
+ const plan = buildBraintrustExperiment(projectId, selected, changed);
+ expect(plan.events.every(row => row.input.model === model)).toBe(true);
+ }
+});
+
+it.each(["", "model\nprivate", "model\u0000private", "model<private>", "a".repeat(257)])("rejects malformed model names %j", model => {
+ const { campaign, originals } = fixture();
+ expect(() => buildBraintrustExperiment(projectId, campaign, originals.map(result => ({ ...result, model })))).toThrow();
+});
+
+it("keeps precise ledger amounts and reservations separate from trace metrics and unobserved attempts", () => {
+ const { campaign, originals } = fixture();
+ campaign.operations = { version: 1, scope: "shared_gateway_ledger", ledgerCampaignId: campaign.campaignId,
+ journalHash: "c".repeat(64), records: 5, deadlineAt: "2026-10-21T01:00:00Z", limitUsd: "100.000000000",
+ knownUsd: "0.000000001", heldUsd: "4.000000000", committedUsd: "4.000000001", requests: 1,
+ unresolvedRequests: 1, conflictingRequests: 0, inflight: 0,
+ breakdown: { estimated: "0.000000001", reported: "0.000000000", reconciled: "0.000000000", allocated: "0.000000000" },
+ attempts: [{ executionId: originals[0]!.executionId, attemptId: "first", requests: 1, unresolvedRequests: 1,
+ knownUsd: "0.000000001", heldUsd: "4.000000000" }] };
+ const plan = buildBraintrustExperiment(projectId, campaign, originals);
+ expect(plan.metadata.request_ledger).toMatchObject({ known_usd: "0.000000001", additional_held_usd: "4.000000000" });
+ expect(plan.events.find(row => row.metadata.attempt_id === "first")!.metadata.request_ledger).toMatchObject({ unresolved_requests: 1 });
+ expect(plan.events.find(row => row.metadata.attempt_id === "second")!.metadata.request_ledger).toBeNull();
+ expect(plan.events.every(row => Object.keys(row.metrics).sort().join() === "end,start")).toBe(true);
+ const carried = buildBraintrustExperiment(projectId, {
+ ...campaign, campaignId: "later-ci-attempt",
+ operations: { ...campaign.operations, ledgerCampaignId: "later-ci-attempt" },
+ }, originals);
+ expect(carried.metadata.request_ledger).toMatchObject({ requests: 1 });
+ expect(carried.events.every(row => row.metadata.request_ledger === null)).toBe(true);
+});
+
+function receiver(campaign: ReturnType<typeof fixture>["campaign"], originals: RunnerE2EResult[]) {
+ const plan = buildBraintrustExperiment(projectId, campaign, originals);
+ return vi.fn<typeof fetch>(async (url, init) => {
+ expect(String(url).startsWith("https://api.braintrust.dev/v1/")).toBe(true);
+ expect(init?.redirect).toBe("error");
+ if (String(url).endsWith("/summarize")) return Response.json({ experiment_name: plan.name,
+ project_url: "https://www.braintrust.dev/app/Example/p/Evals", experiment_url: `https://www.braintrust.dev/app/Example/p/Evals/experiments/${plan.name}` });
+ const sent = JSON.parse(String(init?.body));
+ if (String(url).endsWith("/experiment")) {
+ expect(sent).toMatchObject({ public: false, ensure_new: false, project_id: projectId, name: plan.name });
+ return Response.json({ ...sent, id: experimentId });
+ }
+ return Response.json({ row_ids: sent.events.map((row: { id: string }) => row.id) });
+ });
+}
+
+it("retries identical evidence using the same private experiment and stable row IDs without invoking agents", async () => {
+ const { campaign, originals } = fixture();
+ const transport = receiver(campaign, originals), receipts: BraintrustExperimentReceipt[] = [];
+ for (let i = 0; i < 2; i++) await importBraintrustExperiment({ projectId, apiKey: "private-key" }, campaign, originals,
+ async receipt => { receipts.push(receipt); }, transport);
+ expect(receipts.map(receipt => receipt.status)).toEqual(["created", "accepted", "accepted", "created", "accepted", "accepted"]);
+ expect(receipts[2]).toEqual(receipts[5]);
+ expect(receipts[1]).toMatchObject({ acceptedRows: 2, expectedRows: 2, verification: "unverified" });
+ expect(transport.mock.calls[0]![1]!.body).toBe(transport.mock.calls[3]![1]!.body);
+ expect(transport.mock.calls[1]![1]!.body).toBe(transport.mock.calls[4]![1]!.body);
+});
+
+it("preserves an evidence-bound experiment link through report regeneration and rejects stale or unsafe links", async () => {
+ const { campaign, originals } = fixture();
+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals));
+ campaign.braintrustExperiment = receipt;
+ const root = await mkdtemp(path.join(tmpdir(), "experiment-report-"));
+ try {
+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));
+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });
+ expect(JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")).braintrustExperiment).toEqual(receipt);
+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain(receipt.experimentUrl);
+ expect(() => renderBraintrustExperiment({ ...campaign, braintrustExperiment: { ...receipt, experimentUrl: "https://untrusted.example/app/secret" } })).toThrow();
+ expect(() => renderBraintrustExperiment({ ...campaign, braintrustExperiment: { ...receipt, privateNote: "private-canary" } as never })).toThrow();
+ campaign.accounting!.missingAttempts++;
+ expect(() => renderBraintrustExperiment(campaign)).toThrow();
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
+
+it("rebuilds a report with a missing cell and its saved experiment without rerunning retained attempts", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "experiment-missing-report-"));
+ const repositoryRoot = path.resolve(import.meta.dirname, "../..");
+ const original = fixture().originals[1]!;
+ const missing = runnerMatrix.find(cell => cell.id !== original.executionId)!;
+ const input = path.join(root, "input"), attempt = path.join(input, "attempt");
+ const receiptPath = path.join(root, "experiment.json");
+ async function report(output: string, savedReceipt?: string) {
+ const result = await promisify(execFile)(process.execPath,
+ [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {
+ cwd: repositoryRoot,
+ env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: input, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
+ PAPERCLIP_E2E_CAMPAIGN_ID: "retained-pilot", PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([original.executionId, missing.id]),
+ PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT: savedReceipt },
+ }).catch(error => error as { code: number; stderr: string });
+ // Missing coverage still fails the campaign, but must not prevent rendering it.
+ expect(result).toMatchObject({ code: 1 });
+ expect(result.stderr).not.toContain("Invalid retained experiment evidence");
+ return JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")) as RunnerE2ECampaign;
+ }
+ try {
+ await mkdir(attempt, { recursive: true });
+ const raw = JSON.stringify(original);
+ await writeFile(path.join(attempt, "result.json"), raw);
+ await writeFile(path.join(attempt, "final-state.png"), "fixture-png");
+ await writeFile(path.join(attempt, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] }));
+ const before = await report(path.join(root, "before"));
+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "fixture-key" }, before, [original], async () => {}, receiver(before, [original]));
+ await writeFile(receiptPath, JSON.stringify(receipt));
+ const after = await report(path.join(root, "after"), receiptPath);
+ expect(after.results.find(result => result.attempt === 0)!.startedAt).not.toBe(before.results.find(result => result.attempt === 0)!.startedAt);
+ expect(after.braintrustExperiment).toEqual(receipt);
+ expect(after.accounting).toEqual(before.accounting);
+ expect(await readFile(path.join(root, "after", "index.html"), "utf8")).toContain(receipt.experimentUrl);
+ expect(await readFile(path.join(attempt, "result.json"), "utf8")).toBe(raw);
+ const changed = structuredClone(after);
+ changed.results.find(result => result.attempt > 0)!.startedAt = "2026-10-20T00:00:30.000Z";
+ expect(() => renderBraintrustExperiment(changed)).toThrow();
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
+
+it("checkpoints accepted batches before a later insert fails", async () => {
+ const base = fixture().originals[0]!;
+ const originals = Array.from({ length: 101 }, (_, i) => ({ ...base, attempt: i + 1,
+ accountingIdentity: { campaignId: "retained-pilot", attemptId: `attempt-${i}` } }));
+ const campaign = buildRunnerCampaign({ campaignId: "retained-pilot", generatedAt: base.finishedAt,
+ expected: [base.executionId], results: [originals[100]!], attempts: originals });
+ const normal = receiver(campaign, originals), receipts: BraintrustExperimentReceipt[] = [];
+ let calls = 0;
+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals,
+ async receipt => { receipts.push(receipt); }, async (url, init) => {
+ if (++calls === 3) throw new Error("private-network-error");
+ return normal(url, init);
+ })).rejects.toThrow("Remote details withheld");
+ expect(receipts).toHaveLength(2);
+ expect(receipts[1]).toMatchObject({ status: "created", acceptedRows: 100, expectedRows: 101 });
+ const previous = structuredClone(receipts[1]!);
+ calls = 0;
+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals,
+ async receipt => { receipts.push(receipt); }, async (url, init) => {
+ if (++calls === 2) throw new Error("retry insert failed");
+ return normal(url, init);
+ }, previous)).rejects.toThrow("Remote details withheld");
+ expect(receipts).toHaveLength(2);
+ expect(receipts.at(-1)).toEqual(previous);
+ await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals,
+ async receipt => { receipts.push(receipt); }, normal, previous);
+ expect(receipts.slice(2).map(receipt => receipt.acceptedRows)).toEqual([100, 101, 101]);
+ expect(receipts.at(-1)!.status).toBe("accepted");
+});
+
+it.each(["source", "definition"])("rejects a saved link and original import after selected %s provenance changes", async kind => {
+ const { campaign, originals } = fixture();
+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals));
+ const changed = structuredClone(campaign);
+ changed.braintrustExperiment = receipt;
+ if (kind === "source") changed.results[0]!.source!.sha = "e".repeat(40);
+ else changed.results[0]!.suiteDefinitionHash = "f".repeat(64);
+ expect(() => renderBraintrustExperiment(changed)).toThrow();
+ expect(() => buildBraintrustExperiment(projectId, changed, originals)).toThrow();
+ expect(buildBraintrustExperiment(projectId, campaign, originals).events.map(row => row.input.source_revision).sort())
+ .toEqual(["a".repeat(40), "b".repeat(40)]);
+});
+
+it("propagates a failed receipt save after successful summary lookup while retaining the accepted checkpoint", async () => {
+ const { campaign, originals } = fixture(), receipts: BraintrustExperimentReceipt[] = [];
+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async receipt => {
+ if (receipt.experimentUrl) throw new Error("disk write failed");
+ receipts.push(receipt);
+ }, receiver(campaign, originals))).rejects.toThrow("import incomplete");
+ expect(receipts.at(-1)).toMatchObject({ status: "accepted", acceptedRows: 2 });
+ expect(receipts.at(-1)!.experimentUrl).toBeUndefined();
+});
+
+it("rejects conflicting retry receipts before overwriting saved evidence", async () => {
+ const { campaign, originals } = fixture();
+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals));
+ const persist = vi.fn(), transport = vi.fn<typeof fetch>();
+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, persist, transport,
+ { ...receipt, projectId: experimentId })).rejects.toThrow();
+ expect(persist).not.toHaveBeenCalled();
+ expect(transport).not.toHaveBeenCalled();
+});
+
+it.each(["public", "wrong-project", "wrong-hash", "partial-insert", "receiver-error", "oversized"])("retains safe incomplete status for %s", async kind => {
+ const { campaign, originals } = fixture(), receipts: BraintrustExperimentReceipt[] = [];
+ const normal = receiver(campaign, originals);
+ const transport: typeof fetch = async (url, init) => {
+ const response = await normal(url, init), value = await response.json() as Record<string, unknown>;
+ if (String(url).endsWith("/experiment")) {
+ if (kind === "public") value.public = true;
+ if (kind === "wrong-project") value.project_id = experimentId;
+ if (kind === "wrong-hash") value.metadata = { payload_hash: "a".repeat(64) };
+ if (kind === "oversized") return new Response("x".repeat(1_048_577));
+ } else {
+ if (kind === "partial-insert") value.row_ids = [];
+ if (kind === "receiver-error") return new Response("private-upstream-failure", { status: 503 });
+ }
+ return Response.json(value);
+ };
+ await expect(importBraintrustExperiment({ projectId, apiKey: "private-key" }, campaign, originals,
+ async receipt => { receipts.push(receipt); }, transport)).rejects.toThrow("Remote details withheld");
+ expect(receipts.some(receipt => receipt.status === "accepted")).toBe(false);
+ expect(JSON.stringify(receipts)).not.toMatch(/private-key|private-upstream/);
+ if (["partial-insert", "receiver-error"].includes(kind)) expect(receipts[0]).toMatchObject({ status: "created", acceptedRows: 0 });
+});
diff --git a/tests/runner-e2e/braintrust-experiment.ts b/tests/runner-e2e/braintrust-experiment.ts
new file mode 100644
index 0000000000..4b2859e17d
--- /dev/null
+++ b/tests/runner-e2e/braintrust-experiment.ts
@@ -0,0 +1,207 @@
+import { createHash } from "node:crypto";
+import { buildCampaignAccounting } from "./attempt-accounting.js";
+import { exportBraintrustAttemptParent, type EvalBraintrustConfig } from "./braintrust.js";
+import { validateCampaignFeedbackReview } from "./feedback-dispositions.js";
+import { validateEvalOperationsReport } from "./operations-report.js";
+import { html, isPublicReportTimestamp } from "./report-format.js";
+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";
+
+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+const identifier = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/.test(value);
+const modelName = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/@=,\[\]-]{0,255}$/.test(value);
+const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0;
+const invalid = (): never => { throw new Error("Invalid retained experiment evidence"); };
+
+function attemptProjection(result: RunnerE2EResult, projectId: string) {
+ if (result.schema !== "paperclip.runner-e2e.result/v2"
+ || ![result.executionId, result.suiteId, result.profileId, result.caseId, result.provider,
+ result.accountingIdentity?.campaignId, result.accountingIdentity?.attemptId].every(identifier)
+ || !modelName(result.model)
+ || !/^[0-9a-f]{64}$/.test(result.suiteDefinitionHash ?? "") || !/^[0-9a-f]{40}$/.test(result.source?.sha ?? "")
+ || !count(result.attempt) || result.attempt < 1 || !count(result.durationMs)
+ || !["passed", "failed"].includes(result.status) || !["passed", "failed", "not_started"].includes(result.cleanup)
+ || !["native", "legacy"].includes(result.runtimeMode) || !["local", "daytona"].includes(result.environmentId)
+ || !isPublicReportTimestamp(result.startedAt) || !isPublicReportTimestamp(result.finishedAt)
+ || Date.parse(result.finishedAt) < Date.parse(result.startedAt)) invalid();
+ const feedback = result.feedback;
+ if (feedback && (![feedback.complaints, feedback.suggestions, feedback.invocations?.failed, feedback.invocations?.unknown].every(count)
+ || !["complete", "partial", "unavailable", "not_started"].includes(feedback.collection)
+ || !["complete", "partial", "unavailable", "not_started"].includes(feedback.invocationCollection))) invalid();
+ const trace = result.tracing;
+ if (trace) {
+ exportBraintrustAttemptParent(trace);
+ if (trace.projectId !== projectId || !["pending", "accepted", "failed"].includes(trace.status)) invalid();
+ }
+ return {
+ input: { execution_id: result.executionId, suite_id: result.suiteId, profile_id: result.profileId,
+ case_id: result.caseId, provider: result.provider, model: result.model, environment: result.environmentId,
+ runtime: result.runtimeMode, source_revision: result.source!.sha, definition_hash: result.suiteDefinitionHash },
+ output: { product: result.status, cleanup: result.cleanup },
+ scores: { product_pass: Number(result.status === "passed"), cleanup_pass: Number(result.cleanup === "passed") },
+ metrics: { start: Date.parse(result.startedAt) / 1000, end: Date.parse(result.finishedAt) / 1000 },
+ metadata: { campaign_id: result.accountingIdentity!.campaignId, attempt_id: result.accountingIdentity!.attemptId,
+ attempt_number: result.attempt, wall_duration_ms: result.durationMs, grader: "tests/runner-e2e/runner.spec.ts",
+ feedback: feedback ? { collection: feedback.collection, invocation_collection: feedback.invocationCollection,
+ complaints: feedback.complaints, suggestions: feedback.suggestions,
+ failed_submissions: feedback.invocations.failed, unresolved_submissions: feedback.invocations.unknown } : null,
+ trace: trace ? { project_id: trace.projectId, root_span_id: trace.traceId, span_id: trace.spanId,
+ export_status: trace.status, indexed_verification: "unverified" } : null },
+ span_attributes: { name: result.executionId, type: "eval" },
+ };
+}
+
+export function buildBraintrustExperiment(projectId: string, campaign: RunnerE2ECampaign, originals: readonly RunnerE2EResult[]) {
+ if (!uuid.test(projectId) || campaign.schema !== "paperclip.runner-e2e.campaign/v2" || !identifier(campaign.campaignId)
+ || campaign.accounting?.scope !== "retained_attempts" || !Array.isArray(campaign.accounting.attempts)
+ || !count(campaign.accounting.missingAttempts) || !Array.isArray(campaign.expected)
+ || campaign.expected.length === 0 || !campaign.expected.every(identifier)
+ || new Set(campaign.expected).size !== campaign.expected.length || !Array.isArray(originals)
+ || originals.length > 10_000 || campaign.accounting.attempts.length === 0) invalid();
+ if (campaign.operations) validateEvalOperationsReport(campaign.operations);
+ if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview);
+ const accounting = campaign.accounting!;
+ const operations = campaign.operations?.status !== "unavailable" ? campaign.operations : undefined;
+ if (operations && operations.ledgerCampaignId !== campaign.campaignId) invalid();
+ const candidates = new Map<string, { original: RunnerE2EResult; projection: ReturnType<typeof attemptProjection> }>();
+ for (const original of originals) {
+ const projection = attemptProjection(original, projectId);
+ if (!campaign.expected.includes(original.executionId)) invalid();
+ const receipt = buildCampaignAccounting({ results: [original] }).attempts[0]!;
+ const retained = accounting.attempts.find(attempt => attempt.id === receipt.id);
+ if (!retained || JSON.stringify(retained) !== JSON.stringify(receipt)) invalid();
+ const previous = candidates.get(receipt.id);
+ if (previous && digest(previous.projection) !== digest(projection)) invalid();
+ candidates.set(receipt.id, { original, projection });
+ }
+ if (candidates.size !== accounting.attempts.length) invalid();
+ for (const selected of campaign.results.filter(result => result.attempt > 0)) {
+ const id = buildCampaignAccounting({ results: [selected] }).attempts[0]!.id;
+ const original = candidates.get(id);
+ const selectedProjection = attemptProjection(selected, projectId);
+ if (!original || digest(original.projection.input) !== digest(selectedProjection.input)
+ || digest(original.projection.metrics) !== digest(selectedProjection.metrics)
+ || digest(original.projection.metadata) !== digest(selectedProjection.metadata)) invalid();
+ }
+ const events = [...candidates.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([id, { original, projection }]) => {
+ const cost = operations && original.accountingIdentity!.campaignId === operations.ledgerCampaignId
+ ? operations.attempts.find(row => row.executionId === original.executionId && row.attemptId === original.accountingIdentity!.attemptId)
+ : undefined;
+ return { ...projection, id, span_id: id, root_span_id: id,
+ metadata: { ...projection.metadata, request_ledger: cost ? {
+ known_usd: cost.knownUsd, additional_held_usd: cost.heldUsd, requests: cost.requests,
+ unresolved_requests: cost.unresolvedRequests, journal_hash: operations!.journalHash,
+ } : null } };
+ });
+ const metadata = { importer: "paperclip-retained-attempts-v1", campaign_id: campaign.campaignId,
+ expected_execution_ids: [...campaign.expected].sort(), retained_attempts: events.length,
+ missing_attempts: accounting.missingAttempts,
+ request_ledger: operations ? { journal_hash: operations.journalHash, known_usd: operations.knownUsd,
+ additional_held_usd: operations.heldUsd, committed_usd: operations.committedUsd,
+ requests: operations.requests, unresolved_requests: operations.unresolvedRequests, breakdown: operations.breakdown } : null,
+ feedback_review: campaign.feedbackReview ?? null,
+ accounting_note: "Ledger amounts are separate observations, not additive trace metrics. Held reservations are not charges. Requests outside the gateway and infrastructure require separate accounting." };
+ const payloadHash = digest({ projectId, metadata, events });
+ return { projectId, payloadHash, name: `${campaign.campaignId}-${payloadHash}`, metadata: { ...metadata, payload_hash: payloadHash }, events };
+}
+
+export interface BraintrustExperimentReceipt {
+ version: 1; projectId: string; experimentId: string; name: string; payloadHash: string;
+ status: "created" | "accepted"; acceptedRows: number; expectedRows: number; verification: "unverified";
+ evidenceHash: string; projectUrl?: string; experimentUrl?: string;
+}
+
+export function experimentEvidenceHash(campaign: RunnerE2ECampaign) {
+ return digest({ campaignId: campaign.campaignId, expected: [...campaign.expected].sort(),
+ selected: campaign.results.map(result => ({ executionId: result.executionId, accountingIdentity: result.accountingIdentity,
+ attempt: result.attempt, startedAt: result.attempt === 0 ? null : result.startedAt, sourceRevision: result.source?.sha ?? null,
+ suiteDefinitionHash: result.suiteDefinitionHash ?? null, suiteId: result.suiteId, profileId: result.profileId,
+ caseId: result.caseId, provider: result.provider, model: result.model, environmentId: result.environmentId,
+ runtimeMode: result.runtimeMode })).sort((a, b) => digest(a).localeCompare(digest(b))),
+ accounting: campaign.accounting, operations: campaign.operations ?? null, feedbackReview: campaign.feedbackReview ?? null });
+}
+
+function safeBraintrustUrl(value: unknown): value is string {
+ if (typeof value !== "string") return false;
+ try {
+ const url = new URL(value);
+ return url.protocol === "https:" && ["braintrust.dev", "www.braintrust.dev"].includes(url.host)
+ && !url.username && !url.password && !url.search && !url.hash && url.pathname.startsWith("/app/");
+ } catch { return false; }
+}
+
+export function validateBraintrustExperimentReceipt(campaign: RunnerE2ECampaign, receipt: BraintrustExperimentReceipt): void {
+ if (!receipt || Object.keys(receipt).some(key => !["version", "projectId", "experimentId", "name", "payloadHash", "status", "acceptedRows", "expectedRows", "verification", "evidenceHash", "projectUrl", "experimentUrl"].includes(key))
+ || receipt.version !== 1 || !uuid.test(receipt.projectId) || !uuid.test(receipt.experimentId)
+ || receipt.evidenceHash !== experimentEvidenceHash(campaign) || !/^[a-f0-9]{64}$/.test(receipt.payloadHash)
+ || receipt.name !== `${campaign.campaignId}-${receipt.payloadHash}` || !["created", "accepted"].includes(receipt.status)
+ || !count(receipt.acceptedRows) || receipt.expectedRows !== campaign.accounting?.attempts.length
+ || receipt.acceptedRows > receipt.expectedRows || receipt.verification !== "unverified"
+ || receipt.status === "accepted" && receipt.acceptedRows !== receipt.expectedRows
+ || receipt.experimentUrl !== undefined && !safeBraintrustUrl(receipt.experimentUrl)
+ || receipt.projectUrl !== undefined && !safeBraintrustUrl(receipt.projectUrl)) invalid();
+}
+
+export function renderBraintrustExperiment(campaign: RunnerE2ECampaign): string {
+ const receipt = campaign.braintrustExperiment;
+ if (!receipt) return "";
+ validateBraintrustExperimentReceipt(campaign, receipt);
+ return `<section aria-label="Braintrust experiment"><h2>Braintrust experiment</h2><p>${receipt.experimentUrl
+ ? `<a href="${html(receipt.experimentUrl)}">Open retained attempt scores in Braintrust</a>`
+ : `Experiment ${html(receipt.experimentId)}`}. ${receipt.acceptedRows}/${receipt.expectedRows} rows accepted.
+ Original product and cleanup grades are separate. Indexing requires independent verification; no agents were rerun for this import.</p></section>`;
+}
+
+async function request(config: EvalBraintrustConfig, endpoint: string, payload: unknown, transport: typeof fetch) {
+ const response = await transport(`https://api.braintrust.dev/v1/${endpoint}`, {
+ method: payload === undefined ? "GET" : "POST", redirect: "error", signal: AbortSignal.timeout(15_000),
+ headers: { authorization: `Bearer ${config.apiKey}`, "content-type": "application/json" },
+ ...(payload === undefined ? {} : { body: JSON.stringify(payload) }),
+ });
+ const reader = response.body?.getReader(), chunks: Uint8Array[] = []; let size = 0;
+ try {
+ if (!response.ok) throw new Error("rejected");
+ while (reader) {
+ const next = await reader.read(); if (next.done) break;
+ size += next.value.length; if (size > 1_048_576) throw new Error("oversized");
+ chunks.push(next.value);
+ }
+ return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;
+ } finally { await reader?.cancel().catch(() => {}); }
+}
+
+export async function importBraintrustExperiment(config: EvalBraintrustConfig, campaign: RunnerE2ECampaign,
+ originals: readonly RunnerE2EResult[], persist: (receipt: BraintrustExperimentReceipt) => Promise<void>, transport: typeof fetch = fetch,
+ previous?: BraintrustExperimentReceipt) {
+ const plan = buildBraintrustExperiment(config.projectId, campaign, originals);
+ if (!config.apiKey?.trim() || /[\r\n]/.test(config.apiKey)) throw new Error("Missing Braintrust ingestion credential");
+ if (previous) {
+ validateBraintrustExperimentReceipt(campaign, previous);
+ if (previous.projectId !== plan.projectId || previous.payloadHash !== plan.payloadHash) invalid();
+ }
+ try {
+ const created = await request(config, "experiment", { project_id: plan.projectId, name: plan.name,
+ public: false, ensure_new: false, metadata: plan.metadata }, transport);
+ if (!uuid.test(String(created?.id)) || created.project_id !== plan.projectId || created.name !== plan.name || created.public !== false
+ || (created.metadata as Record<string, unknown> | undefined)?.payload_hash !== plan.payloadHash) throw new Error("conflicting_experiment");
+ if (previous && previous.experimentId !== created.id) throw new Error("conflicting_experiment");
+ const receipt: BraintrustExperimentReceipt = previous ? { ...previous } : { version: 1, projectId: plan.projectId, experimentId: String(created.id),
+ name: plan.name, payloadHash: plan.payloadHash, evidenceHash: experimentEvidenceHash(campaign),
+ status: "created", acceptedRows: 0, expectedRows: plan.events.length, verification: "unverified" };
+ if (!previous) await persist({ ...receipt });
+ for (let offset = 0; offset < plan.events.length; offset += 100) {
+ const events = plan.events.slice(offset, offset + 100);
+ const inserted = await request(config, `experiment/${receipt.experimentId}/insert`, { events }, transport);
+ if (!Array.isArray(inserted?.row_ids) || JSON.stringify([...inserted.row_ids].sort()) !== JSON.stringify(events.map(event => event.id).sort())) throw new Error("missing_rows");
+ receipt.acceptedRows = Math.max(receipt.acceptedRows, offset + events.length);
+ if (receipt.acceptedRows === receipt.expectedRows) receipt.status = "accepted";
+ await persist({ ...receipt });
+ }
+ const summary = await request(config, `experiment/${receipt.experimentId}/summarize`, undefined, transport).catch(() => undefined);
+ if (summary?.experiment_name === plan.name && safeBraintrustUrl(summary.project_url) && safeBraintrustUrl(summary.experiment_url)) {
+ receipt.projectUrl = summary.project_url; receipt.experimentUrl = summary.experiment_url;
+ await persist({ ...receipt });
+ }
+ return receipt;
+ } catch { throw new Error("Braintrust experiment import incomplete; retain the receipt and retry identical evidence. Remote details withheld."); }
+}
diff --git a/tests/runner-e2e/braintrust-indexing.test.ts b/tests/runner-e2e/braintrust-indexing.test.ts
new file mode 100644
index 0000000000..65087433ba
--- /dev/null
+++ b/tests/runner-e2e/braintrust-indexing.test.ts
@@ -0,0 +1,113 @@
+import { expect, it, vi } from "vitest";
+import { buildBraintrustExperiment, experimentEvidenceHash, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";
+import { braintrustTraceIndexQuery, verifyBraintrustIndexing } from "./braintrust-indexing.js";
+import { buildRunnerCampaign } from "./history.js";
+import { runnerMatrix } from "./catalog.js";
+import { emptyFeedbackSummary } from "./feedback.js";
+import type { RunnerE2EResult } from "./types.js";
+
+const projectId = "11111111-1111-4111-8111-111111111111";
+const config = { projectId, apiKey: "private-key-canary" };
+const aggregate = { records: 2500, roots: 1, finalizations: 1, native_api_records: 2, complaints: 1, suggestions: 1 };
+
+function fixture(profile = "runner-codex") {
+ const cell = runnerMatrix.find(row => row.id === `feedback-persistence.${profile}.local.complaint-and-suggestion`)!;
+ const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id,
+ suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash,
+ accountingIdentity: { campaignId: "indexed-pilot", attemptId: "first" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null },
+ attempt: 1, status: "passed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id,
+ environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,
+ startedAt: "2026-10-20T00:00:00.000Z", finishedAt: "2026-10-20T00:00:01.000Z", durationMs: 1000,
+ feedback: { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete", complaints: 1, suggestions: 1 },
+ tracing: { projectId, traceId: "a".repeat(32), spanId: "b".repeat(16), status: "accepted", verification: "unverified", nativeConfiguration: "configured",
+ finalization: { spanId: "c".repeat(16), status: "accepted", outcome: "passed", verification: "unverified" } } };
+ const originals = [result];
+ const campaign = buildRunnerCampaign({ campaignId: "indexed-pilot", generatedAt: result.finishedAt, expected: [cell.id], results: originals, attempts: originals });
+ const plan = buildBraintrustExperiment(projectId, campaign, originals);
+ const experiment: BraintrustExperimentReceipt = { version: 1, projectId, experimentId: "22222222-2222-4222-8222-222222222222", name: plan.name,
+ payloadHash: plan.payloadHash, evidenceHash: experimentEvidenceHash(campaign), status: "accepted", acceptedRows: 1, expectedRows: 1, verification: "unverified" };
+ return { originals, campaign, plan, experiment };
+}
+
+it.each(["runner-codex", "runner-acpx-claude"])("verifies %s native records and original experiment grades without fetching content", async profile => {
+ const { originals, campaign, plan, experiment } = fixture(profile);
+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [aggregate] })).mockResolvedValueOnce(Response.json({ data: plan.events }));
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport);
+ expect(receipt.status).toBe("verified");
+ expect(receipt.traces[0]).toMatchObject({ records: 2500, nativeApiRecords: 2, complaints: 1 });
+ expect(receipt.experiment.indexedRows).toBe(1);
+ expect(transport).toHaveBeenCalledTimes(2);
+ const sql = JSON.parse(String(transport.mock.calls[0]![1]!.body)).query;
+ expect(sql).toContain(`root_span_id = '${"a".repeat(32)}'`);
+ expect(sql).toContain(profile === "runner-codex" ? "codex.api_request" : "claude_code.llm_request");
+ expect(sql).not.toMatch(/arguments|prompt|response|tool_input|tool_result_size/);
+ expect(JSON.stringify(receipt)).not.toContain(config.apiKey);
+});
+
+it.each(["roots", "finalizations", "native_api_records", "complaints", "suggestions"])("does not pass a trace missing %s", async missing => {
+ const { originals, campaign, plan, experiment } = fixture();
+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [{ ...aggregate, [missing]: 0 }] })).mockResolvedValueOnce(Response.json({ data: plan.events }));
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });
+ expect(receipt.status).toBe("incomplete");
+ expect(receipt.experiment.status).toBe("verified");
+});
+
+it("retries delayed indexing without re-querying already verified records", async () => {
+ const { originals, campaign, plan, experiment } = fixture();
+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [{ ...aggregate, roots: 0 }] }))
+ .mockResolvedValueOnce(Response.json({ data: plan.events })).mockResolvedValueOnce(Response.json({ data: [aggregate] }));
+ const pause = vi.fn(async () => {});
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { pause });
+ expect(receipt.status).toBe("verified");
+ expect(transport).toHaveBeenCalledTimes(3);
+ expect(pause).toHaveBeenCalledTimes(1);
+});
+
+it("treats an empty aggregate as missing indexing, without inventing native activity", async () => {
+ const { originals, campaign, plan, experiment } = fixture();
+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [{ records: 0, roots: null,
+ finalizations: null, native_api_records: null, complaints: null, suggestions: null }] })).mockResolvedValueOnce(Response.json({ data: plan.events }));
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });
+ expect(receipt.status).toBe("incomplete");
+ expect(receipt.traces[0]).toMatchObject({ status: "incomplete", records: 0, nativeApiRecords: 0 });
+});
+
+it.each(["missing", "extra", "duplicate", "grade-changed", "wrong-revision", "wrong-attempt"])("rejects %s indexed experiment evidence", async fault => {
+ const { originals, campaign, plan, experiment } = fixture();
+ const rows = structuredClone(plan.events);
+ if (fault === "missing") rows.pop();
+ if (fault === "extra") rows.push({ ...rows[0]!, id: "other" });
+ if (fault === "duplicate") rows.push(rows[0]!);
+ if (fault === "grade-changed") rows[0]!.scores.product_pass = 0;
+ if (fault === "wrong-revision") rows[0]!.input.source_revision = "b".repeat(40);
+ if (fault === "wrong-attempt") rows[0]!.metadata.attempt_id = "other";
+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [aggregate] })).mockResolvedValueOnce(Response.json({ data: rows }));
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });
+ expect(receipt.status).toBe("incomplete");
+ expect(receipt.traces[0]!.status).toBe("verified");
+});
+
+it.each(["rejected", "malformed", "oversized", "invalid-counts"])("retains %s query failures without raw error content", async fault => {
+ const { originals, campaign, experiment } = fixture();
+ const transport = vi.fn<typeof fetch>().mockImplementation(async () => fault === "rejected" ? new Response("private-response-canary", { status: 401 })
+ : fault === "malformed" ? new Response("private-response-canary")
+ : fault === "oversized" ? new Response("private-response-canary".repeat(100_000))
+ : Response.json({ data: [{ ...aggregate, records: -1 }] }));
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });
+ expect(receipt.status).toBe("unavailable");
+ expect(JSON.stringify(receipt)).not.toMatch(/private-response-canary|private-key-canary/);
+});
+
+it("honors cancellation before making a query", async () => {
+ const { originals, campaign, experiment } = fixture();
+ const transport = vi.fn<typeof fetch>();
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { signal: AbortSignal.abort() });
+ expect(receipt.status).toBe("unavailable");
+ expect(transport).not.toHaveBeenCalled();
+});
+
+it("rejects foreign or malformed trace identities before constructing a query", () => {
+ const { originals } = fixture();
+ originals[0]!.tracing!.traceId = "injected'";
+ expect(() => braintrustTraceIndexQuery(projectId, originals[0]!)).toThrow();
+});
diff --git a/tests/runner-e2e/braintrust-indexing.ts b/tests/runner-e2e/braintrust-indexing.ts
new file mode 100644
index 0000000000..8ca5d9f11b
--- /dev/null
+++ b/tests/runner-e2e/braintrust-indexing.ts
@@ -0,0 +1,118 @@
+import { setTimeout as delay } from "node:timers/promises";
+import { buildBraintrustExperiment, experimentEvidenceHash, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";
+import { exportBraintrustAttemptParent, type EvalBraintrustConfig } from "./braintrust.js";
+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";
+
+type IndexStatus = "verified" | "incomplete" | "unavailable";
+export interface BraintrustIndexReceipt {
+ version: 1; projectId: string; evidenceHash: string; checkedAt: string;
+ status: IndexStatus;
+ traces: Array<{ attemptId: string; traceId: string | null; status: IndexStatus;
+ records?: number; roots?: number; finalizations?: number; nativeApiRecords?: number;
+ complaints?: number; suggestions?: number }>;
+ experiment: { id: string | null; status: IndexStatus; expectedRows: number; indexedRows?: number };
+}
+
+async function query(config: EvalBraintrustConfig, sql: string, transport: typeof fetch, signal: AbortSignal): Promise<Record<string, unknown>[]> {
+ const response = await transport("https://api.braintrust.dev/btql", {
+ method: "POST", redirect: "error", signal: AbortSignal.any([signal, AbortSignal.timeout(10_000)]),
+ headers: { authorization: `Bearer ${config.apiKey}`, "content-type": "application/json" },
+ body: JSON.stringify({ query: sql, fmt: "json" }),
+ });
+ const reader = response.body?.getReader(), chunks: Uint8Array[] = []; let size = 0;
+ try {
+ if (!response.ok) throw new Error("query_rejected");
+ while (reader) {
+ const next = await reader.read(); if (next.done) break;
+ size += next.value.length; if (size > 1_048_576) throw new Error("query_oversized");
+ chunks.push(next.value);
+ }
+ const payload = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+ if (!payload || !Array.isArray(payload.data) || payload.data.some((row: unknown) => !row || typeof row !== "object" || Array.isArray(row))) throw new Error("invalid_query_result");
+ return payload.data;
+ } finally { await reader?.cancel().catch(() => {}); }
+}
+
+export function braintrustTraceIndexQuery(projectId: string, result: RunnerE2EResult): string {
+ const trace = result.tracing;
+ if (!trace || trace.projectId !== projectId || !/^[0-9a-f]{16}$/.test(trace.finalization?.spanId ?? "")) throw new Error("Missing trace identity");
+ exportBraintrustAttemptParent(trace);
+ if (!(result.profileId === "runner-codex" && result.provider === "codex")
+ && !(result.profileId === "runner-acpx-claude" && result.provider === "acpx")) throw new Error("Unsupported native index contract");
+ const nativeRequest = result.profileId === "runner-codex" ? `metadata."event.name" = 'codex.api_request'`
+ : `span_attributes.name = 'claude_code.llm_request' AND span_attributes.type = 'llm'`;
+ const tool = (name: string) => result.profileId === "runner-codex"
+ ? `metadata."event.name" = 'codex.tool_result' AND metadata.tool_name = '${name}' AND metadata.success = 'true'`
+ : `span_attributes.type = 'tool' AND span_attributes.name = 'mcp__paperclip__${name}'`;
+ return `SELECT COUNT(*) AS records,
+ SUM(CASE WHEN span_id = '${trace.spanId}' AND span_attributes.name = 'paperclip.eval.attempt' THEN 1 ELSE 0 END) AS roots,
+ SUM(CASE WHEN span_id = '${trace.finalization!.spanId}' AND span_attributes.name = 'paperclip.eval.finalization' THEN 1 ELSE 0 END) AS finalizations,
+ SUM(CASE WHEN ${nativeRequest} THEN 1 ELSE 0 END) AS native_api_records,
+ SUM(CASE WHEN ${tool("submit_complaint")} THEN 1 ELSE 0 END) AS complaints,
+ SUM(CASE WHEN ${tool("submit_suggestion")} THEN 1 ELSE 0 END) AS suggestions
+ FROM project_logs('${projectId}') WHERE root_span_id = '${trace.traceId}'`;
+}
+
+function counts(rows: Record<string, unknown>[]) {
+ if (rows.length !== 1) throw new Error("Invalid trace aggregate");
+ const row = rows[0]!;
+ const fields = ["records", "roots", "finalizations", "native_api_records", "complaints", "suggestions"];
+ if (row.records === 0) for (const key of fields.slice(1)) if (row[key] === null) row[key] = 0;
+ if (fields.some(key => !Number.isSafeInteger(row[key]) || Number(row[key]) < 0)) throw new Error("Invalid trace counts");
+ return { records: Number(row.records), roots: Number(row.roots), finalizations: Number(row.finalizations),
+ nativeApiRecords: Number(row.native_api_records), complaints: Number(row.complaints), suggestions: Number(row.suggestions) };
+}
+
+export async function verifyBraintrustIndexing(config: EvalBraintrustConfig, campaign: RunnerE2ECampaign,
+ originals: readonly RunnerE2EResult[], experiment: BraintrustExperimentReceipt | undefined,
+ transport: typeof fetch = fetch, options: { rounds?: number; pause?: () => Promise<void>; signal?: AbortSignal } = {}): Promise<BraintrustIndexReceipt> {
+ const plan = buildBraintrustExperiment(config.projectId, campaign, originals);
+ const receipt: BraintrustIndexReceipt = { version: 1, projectId: config.projectId,
+ evidenceHash: experimentEvidenceHash(campaign), checkedAt: new Date().toISOString(), status: "incomplete",
+ traces: originals.map(result => ({ attemptId: result.accountingIdentity!.attemptId,
+ traceId: result.tracing?.traceId ?? null, status: "incomplete" })),
+ experiment: { id: experiment?.experimentId ?? null, status: "incomplete", expectedRows: plan.events.length } };
+ if (!config.apiKey?.trim() || /[\r\n]/.test(config.apiKey)) throw new Error("Missing Braintrust query credential");
+ const rounds = options.rounds ?? 3;
+ if (!Number.isInteger(rounds) || rounds < 1 || rounds > 3) throw new Error("Invalid index verification limit");
+ const signal = AbortSignal.any([AbortSignal.timeout(45_000), ...(options.signal ? [options.signal] : [])]);
+ const pause = options.pause ?? (() => delay(2_000, undefined, { signal }));
+ for (let round = 0; round < rounds && !signal.aborted; round++) {
+ for (let index = 0; index < originals.length && !signal.aborted; index++) {
+ const result = originals[index]!, row = receipt.traces[index]!;
+ if (row.status === "verified") continue;
+ try {
+ const values = counts(await query(config, braintrustTraceIndexQuery(config.projectId, result), transport, signal));
+ Object.assign(row, values);
+ row.status = values.roots > 0 && values.finalizations > 0 && values.nativeApiRecords > 0
+ && (result.feedback?.complaints ?? 0) <= values.complaints
+ && (result.feedback?.suggestions ?? 0) <= values.suggestions ? "verified" : "incomplete";
+ } catch { row.status = "unavailable"; }
+ }
+ if (!signal.aborted && receipt.experiment.status !== "verified" && experiment?.status === "accepted"
+ && experiment.projectId === config.projectId && /^[0-9a-f-]{36}$/.test(experiment.experimentId)
+ && experiment.payloadHash === plan.payloadHash && experiment.evidenceHash === receipt.evidenceHash) {
+ try {
+ const rows = await query(config, `SELECT id, input, output, scores, metadata FROM experiment('${experiment.experimentId}') LIMIT ${plan.events.length + 1}`, transport, signal);
+ receipt.experiment.indexedRows = rows.length;
+ const equal = (left: unknown, right: unknown): boolean => {
+ if (left === right) return true;
+ if (!left || !right || typeof left !== "object" || typeof right !== "object" || Array.isArray(left) || Array.isArray(right)) return false;
+ const a = left as Record<string, unknown>, b = right as Record<string, unknown>;
+ return Object.keys(a).length === Object.keys(b).length && Object.keys(a).every(key => equal(a[key], b[key]));
+ };
+ receipt.experiment.status = rows.length === plan.events.length && new Set(rows.map(row => row.id)).size === rows.length
+ && plan.events.every(expected => rows.some(row => row.id === expected.id && equal(row.input, expected.input)
+ && equal(row.output, expected.output) && equal(row.scores, expected.scores)
+ && (row.metadata as Record<string, unknown> | undefined)?.attempt_id === expected.metadata.attempt_id)) ? "verified" : "incomplete";
+ } catch { receipt.experiment.status = "unavailable"; }
+ }
+ if (receipt.experiment.status === "verified" && receipt.traces.every(row => row.status === "verified")) break;
+ if (round + 1 < rounds && !signal.aborted) { try { await pause(); } catch { break; } }
+ }
+ receipt.checkedAt = new Date().toISOString();
+ const statuses = [...receipt.traces.map(row => row.status), receipt.experiment.status];
+ receipt.status = statuses.every(status => status === "verified") ? "verified"
+ : statuses.some(status => status === "unavailable") || signal.aborted ? "unavailable" : "incomplete";
+ return receipt;
+}
diff --git a/tests/runner-e2e/braintrust.test.ts b/tests/runner-e2e/braintrust.test.ts
new file mode 100644
index 0000000000..118621d3ff
--- /dev/null
+++ b/tests/runner-e2e/braintrust.test.ts
@@ -0,0 +1,213 @@
+import { readFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { afterEach, expect, it, vi } from "vitest";
+import { EvalAttemptTrace, evalBraintrustConfig, exportBraintrustAttemptParent, finalizeEvalAttemptTrace } from "./braintrust.js";
+import { RunnerApi } from "./api.js";
+import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js";
+import { setupLiveFixtures } from "./live-fixtures.js";
+import { runnerMatrix } from "./catalog.js";
+import { buildProviderObservabilityLaunch } from "../../packages/adapter-utils/src/agent-observability.js";
+const require = createRequire(new URL("../../server/package.json", import.meta.url));
+const protobuf = require("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js");
+const projectId = "11111111-1111-4111-8111-111111111111", companyId = "22222222-2222-4222-8222-222222222222";
+const config = { projectId, apiKey: "private-ingestion-fixture" };
+const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "hello.codex.local", provider: "codex", model: "fixture-model", environment: "local" };
+const provenance = { suiteId: "core-compatibility", suiteDefinitionHash: "a".repeat(64), profileId: "runner-codex", caseId: "message-marker", sourceRevision: "b".repeat(40) };
+const input = { failed: false, runIds: ["run-1"], cleanup: "passed", complaints: 1, suggestions: 2 };
+afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
+
+it("requires explicit export routing and keeps ingestion credentials out of server inheritance", () => {
+ expect(evalBraintrustConfig({ BRAINTRUST_API_KEY: config.apiKey })).toBeUndefined();
+ expect(() => evalBraintrustConfig({ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: projectId })).toThrow("API key");
+ expect(() => evalBraintrustConfig({ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "bad", BRAINTRUST_API_KEY: config.apiKey })).toThrow("UUID");
+ expect(buildPaperclipServerEnvironment({ BRAINTRUST_API_KEY: config.apiKey, BRAINTRUST_OTHER_KEY: "private", KEEP: "yes" })).toEqual({ KEEP: "yes" });
+});
+
+it("accepts only explicit loopback relay destinations and strips inherited relay endpoints", async () => {
+ const endpointBaseUrl = "http://127.0.0.1:43123/otel";
+ const env = { PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: projectId, BRAINTRUST_API_KEY: "attempt-token", PAPERCLIP_E2E_OTLP_RELAY_URL: endpointBaseUrl };
+ expect(evalBraintrustConfig(env)).toEqual({ projectId, apiKey: "attempt-token", endpointBaseUrl });
+ expect(buildPaperclipServerEnvironment(env).PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined();
+ expect(buildRunnerE2EProcessEnvironment(env, []).PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined();
+ for (const invalid of ["https://foreign.invalid/otel", "http://127.0.0.1:43123/otel?key=secret", "http://localhost:43123/otel", "http://127.0.0.1:99999/otel"]) {
+ expect(() => evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: invalid })).toThrow();
+ }
+ const trace = new EvalAttemptTrace(evalBraintrustConfig(env)!, identity);
+ const putSensitive = vi.fn(async () => ({ settings: { enabled: true, credentialsConfigured: true, revision: projectId,
+ destination: { provider: "otlp", endpointBaseUrl } } }));
+ await trace.configureCompany({ putSensitive } as unknown as RunnerApi, companyId);
+ expect(putSensitive).toHaveBeenCalledWith(`/api/companies/${companyId}/observability`, expect.objectContaining({
+ destination: { provider: "otlp", endpointBaseUrl }, credentials: { mode: "replace", headers: [
+ { name: "authorization", value: "Bearer attempt-token" }, { name: "x-bt-parent", value: exportBraintrustAttemptParent(trace.snapshot()) },
+ ] },
+ }));
+});
+
+it("exports the SDK-verified project-log parent format and refuses invalid identifiers", () => {
+ const receipt = { projectId, spanId: "1234567890abcdef", traceId: "abcdef0123456789abcdef0123456789" };
+ expect(exportBraintrustAttemptParent(receipt)).toBe("BAIAeyJvYmplY3RfaWQiOiIxMTExMTExMS0xMTExLTQxMTEtODExMS0xMTExMTExMTExMTEiLCJyb3dfaWQiOiIxMjM0NTY3ODkwYWJjZGVmIiwic3Bhbl9pZCI6IjEyMzQ1Njc4OTBhYmNkZWYiLCJyb290X3NwYW5faWQiOiJhYmNkZWYwMTIzNDU2Nzg5YWJjZGVmMDEyMzQ1Njc4OSJ9");
+ for (const invalid of [{ projectId: "project" }, { spanId: "0".repeat(16) }, { traceId: "0".repeat(32) }, { spanId: "1234567890abcdeg" }]) {
+ expect(() => exportBraintrustAttemptParent({ ...receipt, ...invalid })).toThrow("parent identity");
+ }
+});
+
+it.each(["codex", "claude"] as const)("preserves the attempt parent through the production %s launch encoder", kind => {
+ const trace = new EvalAttemptTrace(config, identity);
+ const parent = exportBraintrustAttemptParent(trace.snapshot());
+ const launch = buildProviderObservabilityLaunch({
+ kind, companyId, agentId: projectId, revision: projectId, compatibilityKey: "fixture", enabled: true,
+ capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true },
+ tracesEndpoint: "https://api.braintrust.dev/otel/v1/traces", logsEndpoint: "https://api.braintrust.dev/otel/v1/logs",
+ headers: { authorization: `Bearer ${config.apiKey}`, "x-bt-parent": parent },
+ });
+ if (kind === "codex") {
+ const exporter = launch.otelConfig!.trace_exporter as { "otlp-http": { headers: Record<string, string> } };
+ expect(exporter["otlp-http"].headers).toEqual({});
+ expect(JSON.stringify(launch.otelConfig)).not.toContain(config.apiKey);
+ }
+ for (const name of ["OTEL_EXPORTER_OTLP_TRACES_HEADERS", "OTEL_EXPORTER_OTLP_LOGS_HEADERS"]) {
+ const encoded = launch.environment[name].split(",").find(header => header.startsWith("x-bt-parent="))!;
+ expect(decodeURIComponent(encoded.slice("x-bt-parent=".length))).toBe(parent);
+ }
+ expect(JSON.stringify(trace.snapshot())).not.toContain(config.apiKey);
+});
+
+it("rejects invalid provenance while retaining an explicitly unknown source revision", () => {
+ expect(() => new EvalAttemptTrace(config, identity, { ...provenance, suiteDefinitionHash: "unknown" })).toThrow("provenance");
+ expect(() => new EvalAttemptTrace(config, identity, { ...provenance, sourceRevision: "master" })).toThrow("provenance");
+ expect(() => new EvalAttemptTrace(config, identity, { ...provenance, sourceRevision: null })).not.toThrow();
+});
+
+it("sends real protobuf with parented phases, exact project routing, and no duplicate model charges", async () => {
+ const root = protobuf.parse(await readFile(new URL("../../server/src/__tests__/fixtures/otlp-observability.proto", import.meta.url), "utf8")).root;
+ const trace = new EvalAttemptTrace(config, identity, provenance);
+ trace.phase("setup"); trace.phase("execution"); trace.phase("feedback"); trace.phase("cleanup");
+ let captured: RequestInit | undefined;
+ const transport = vi.fn<typeof fetch>(async (_url, init) => { captured = init; return new Response(null, { status: 200 }); });
+ expect(await trace.export(input, transport)).toMatchObject({ status: "accepted", verification: "unverified", nativeConfiguration: "not_started" });
+ expect(captured).toMatchObject({ method: "POST", redirect: "error", headers: { authorization: `Bearer ${config.apiKey}`, "x-bt-parent": `project_id:${projectId}`, "content-type": "application/x-protobuf" } });
+ const bytes = captured!.body as Buffer;
+ expect(bytes.includes(config.apiKey)).toBe(false);
+ const decoded = root.lookupType("paperclip.test.otlp.ExportTraceServiceRequest").decode(bytes).toJSON();
+ const spans = decoded.resourceSpans[0].scopeSpans[0].spans;
+ expect(spans.map((span: { name: string }) => span.name)).toEqual(["paperclip.eval.attempt", "paperclip.eval.setup", "paperclip.eval.execution", "paperclip.eval.feedback", "paperclip.eval.cleanup"]);
+ expect(spans[0].parentSpanId).toBeUndefined();
+ for (const span of spans.slice(1)) {
+ expect(span.traceId).toBe(spans[0].traceId);
+ expect(span.parentSpanId).toBe(spans[0].spanId);
+ }
+ expect(JSON.stringify(decoded)).toContain("paperclip.eval.attempt_id");
+ const attributes = Object.fromEntries(spans[0].attributes.map((attribute: { key: string; value: { stringValue: string } }) => [attribute.key, attribute.value.stringValue]));
+ expect(attributes).toMatchObject({
+ "paperclip.eval.suite_id": provenance.suiteId, "paperclip.eval.suite_definition_hash": provenance.suiteDefinitionHash,
+ "paperclip.eval.profile_id": provenance.profileId, "paperclip.eval.case_id": provenance.caseId,
+ "paperclip.eval.source_revision": provenance.sourceRevision, "paperclip.eval.grader": "tests/runner-e2e/runner.spec.ts",
+ });
+ expect(JSON.stringify(decoded)).not.toMatch(/gen_ai\.usage|cost_usd|private-ingestion/);
+ expect(transport).toHaveBeenCalledTimes(1);
+});
+
+it.each([
+ [() => new Response("private-provider-error", { status: 403 }), "receiver_rejected"],
+ [() => Response.json({ partialSuccess: { rejectedSpans: "1" } }), "partial_rejection"],
+ [() => Response.json({ partialSuccess: { rejectedSpans: -1 } }), "export_failed"],
+ [() => new Response("invalid", { status: 200, headers: { "content-type": "text/html" } }), "export_failed"],
+ [() => new Response(Buffer.alloc(65_537)), "export_failed"],
+])("retains export failure without raw receiver diagnostics", async (response, errorCode) => {
+ const trace = new EvalAttemptTrace(config, identity);
+ expect(await trace.export(input, async () => response())).toMatchObject({ status: "failed", verification: "unverified", errorCode });
+ expect(JSON.stringify(trace.snapshot())).not.toMatch(/private-provider-error|private-ingestion/);
+});
+
+it("configures native export through a revision-fenced sensitive PUT before any credential or agent setup", async () => {
+ const order: string[] = [];
+ const trace = new EvalAttemptTrace(config, identity);
+ const api = {
+ async get() { return [{ id: "local", driver: "local" }]; },
+ async putSensitive(path: string, data: unknown) {
+ order.push("observability");
+ expect(path).toBe(`/api/companies/${companyId}/observability`);
+ expect(data).toMatchObject({ expectedRevision: null, enabled: true,
+ credentials: { mode: "replace", headers: [{ name: "authorization", value: `Bearer ${config.apiKey}` }, { name: "x-bt-parent", value: exportBraintrustAttemptParent(trace.snapshot()) }] },
+ destination: { provider: "otlp", endpointBaseUrl: "https://api.braintrust.dev/otel" } });
+ return { settings: { enabled: true, credentialsConfigured: true, revision: projectId, destination: { provider: "otlp", endpointBaseUrl: "https://api.braintrust.dev/otel" } } };
+ },
+ async postSensitive() { order.push("secret"); return { id: "secret" }; },
+ async post(path: string) { order.push(path === "/api/companies" ? "company" : "agent"); return { id: companyId, name: "fixture" }; },
+ } as unknown as RunnerApi;
+ const execution = runnerMatrix.find(cell => cell.profile.id === "runner-codex" && cell.environment.id === "local" && cell.task.flow === "single_turn")!;
+ expect(execution).toBeDefined();
+ const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { OPENAI_API_KEY: "private-provider" }, onCompanyCreated: id => trace.configureCompany(api, id) });
+ await fixtures.teardown();
+ expect(order.slice(0, 3)).toEqual(["company", "observability", "secret"]);
+ expect(trace.snapshot()).toMatchObject({ nativeConfiguration: "configured", companyId, revision: projectId });
+});
+
+it.each([
+ { provider: "braintrust", projectId },
+ { provider: "otlp", endpointBaseUrl: "https://elsewhere.invalid/otel" },
+])("refuses a saved native destination that cannot carry the attempt parent", async destination => {
+ const trace = new EvalAttemptTrace(config, identity);
+ const api = { putSensitive: async () => ({ settings: { enabled: true, credentialsConfigured: true, revision: projectId, destination } }) } as unknown as RunnerApi;
+ await expect(trace.configureCompany(api, companyId)).rejects.toThrow("configuration failed");
+ expect(trace.snapshot().nativeConfiguration).toBe("failed");
+});
+
+it("never sends sensitive PUT bodies through Playwright and refuses redirects or foreign destinations", async () => {
+ vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "43123");
+ const playwright = { put: vi.fn() };
+ const api = new RunnerApi(playwright as never);
+ api.setBrowserSession("session=private");
+ const transport = vi.fn().mockResolvedValue(Response.json({ ok: true })); vi.stubGlobal("fetch", transport);
+ await api.putSensitive("/api/companies/fixture/observability", { apiKey: config.apiKey });
+ expect(playwright.put).not.toHaveBeenCalled();
+ expect(transport).toHaveBeenCalledWith(new URL("http://127.0.0.1:43123/api/companies/fixture/observability"), expect.objectContaining({ method: "PUT", redirect: "error", signal: expect.any(AbortSignal), headers: expect.objectContaining({ Cookie: "session=private" }) }));
+ await expect(api.putSensitive("https://other.invalid/api/collect", { apiKey: config.apiKey })).rejects.toThrow("fixture API");
+ transport.mockRejectedValue(new Error(config.apiKey));
+ await expect(api.putSensitive("/api/fixture", {})).rejects.toThrow("details withheld");
+});
+
+it.each([
+ { status: "failed", cleanup: "passed", failureClass: "permanent_infrastructure" },
+ { status: "failed", cleanup: "failed", failureClass: "cleanup_failure" },
+] as const)("updates a passing worker trace with the supervisor's $failureClass outcome", async outcome => {
+ const root = protobuf.parse(await readFile(new URL("../../server/src/__tests__/fixtures/otlp-observability.proto", import.meta.url), "utf8")).root;
+ const trace = new EvalAttemptTrace(config, identity, provenance);
+ const original = await trace.export(input, async () => new Response(null, { status: 200 }));
+ const saved = structuredClone(original);
+ let captured: RequestInit | undefined;
+ const finalization = await finalizeEvalAttemptTrace(config, { receipt: original, identity, provenance,
+ result: { startedAt: new Date(Date.now() - 10_000).toISOString(), ...outcome, runIds: [companyId] },
+ }, async (_url, init) => { captured = init; return new Response(null, { status: 200 }); });
+ expect(finalization).toMatchObject({ status: "accepted", outcome: "failed", verification: "unverified" });
+ expect(original).toEqual(saved);
+ const decoded = root.lookupType("paperclip.test.otlp.ExportTraceServiceRequest").decode(captured!.body as Buffer).toJSON();
+ const spans = decoded.resourceSpans[0].scopeSpans[0].spans;
+ expect(spans.map((span: { name: string }) => span.name)).toEqual(["paperclip.eval.attempt", "paperclip.eval.finalization"]);
+ expect(Buffer.from(spans[0].traceId, "base64").toString("hex")).toBe(original.traceId);
+ expect(Buffer.from(spans[0].spanId, "base64").toString("hex")).toBe(original.spanId);
+ expect(spans[1].parentSpanId).toBe(spans[0].spanId);
+ expect(spans.every((span: { status: { code: number } }) => span.status.code === 2)).toBe(true);
+ const attributes = Object.fromEntries(spans[0].attributes.map((attribute: { key: string; value: { stringValue: string } }) => [attribute.key, attribute.value.stringValue]));
+ expect(attributes).toMatchObject({ "paperclip.eval.outcome_stage": "supervisor", "paperclip.eval.outcome": "failed",
+ "paperclip.eval.failure_class": outcome.failureClass, "paperclip.eval.source_revision": provenance.sourceRevision });
+ expect(JSON.stringify(decoded)).not.toMatch(/private-ingestion|gen_ai\.usage|cost_usd/);
+});
+
+it("retains finalization export failures without changing the recorded product result", async () => {
+ const trace = new EvalAttemptTrace(config, identity, provenance);
+ const result = { startedAt: new Date().toISOString(), status: "passed" as const, cleanup: "passed" as const };
+ const saved = structuredClone(result);
+ const receipt = await finalizeEvalAttemptTrace(config, { receipt: trace.snapshot(), identity, provenance, result }, async () => new Response("private-error", { status: 503 }));
+ expect(receipt).toMatchObject({ status: "failed", outcome: "passed", errorCode: "receiver_rejected", httpStatus: 503 });
+ expect(result).toEqual(saved);
+ expect(JSON.stringify(receipt)).not.toMatch(/private-error|private-ingestion/);
+ const transport = vi.fn<typeof fetch>();
+ expect(await finalizeEvalAttemptTrace(config, { receipt: { ...trace.snapshot(), projectId: companyId }, identity, provenance, result }, transport))
+ .toMatchObject({ status: "failed", errorCode: "invalid_finalization_context" });
+ expect(transport).not.toHaveBeenCalled();
+ expect(await finalizeEvalAttemptTrace(config, { receipt: trace.snapshot(), identity, provenance,
+ result: { ...result, failureClass: "private-target-output" as never },
+ }, transport)).toMatchObject({ status: "failed", errorCode: "invalid_finalization_context" });
+ expect(transport).not.toHaveBeenCalled();
+});
diff --git a/tests/runner-e2e/braintrust.ts b/tests/runner-e2e/braintrust.ts
new file mode 100644
index 0000000000..adb5d94e4b
--- /dev/null
+++ b/tests/runner-e2e/braintrust.ts
@@ -0,0 +1,218 @@
+import { randomBytes } from "node:crypto";
+import { createRequire } from "node:module";
+import type { RunnerApi } from "./api.js";
+import type { FailureClass, RunnerE2EResult } from "./types.js";
+
+export const EVAL_OBSERVABILITY_SECRET_NAMES = ["BRAINTRUST_API_KEY"] as const;
+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+export interface EvalBraintrustConfig { projectId: string; apiKey: string; endpointBaseUrl?: string }
+function evalOtlpEndpoint(config: EvalBraintrustConfig) {
+ if (config.endpointBaseUrl !== undefined && !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(config.endpointBaseUrl)) {
+ throw new Error("Eval OTLP relay must use the parent-owned loopback endpoint");
+ }
+ if (config.endpointBaseUrl && Number(new URL(config.endpointBaseUrl).port) > 65535) throw new Error("Invalid eval OTLP relay port");
+ return config.endpointBaseUrl ?? "https://api.braintrust.dev/otel";
+}
+export function evalBraintrustConfig(env: NodeJS.ProcessEnv): EvalBraintrustConfig | undefined {
+ const projectId = env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID;
+ if (!projectId) return undefined;
+ if (!uuid.test(projectId) || !env.BRAINTRUST_API_KEY?.trim() || /[\r\n]/.test(env.BRAINTRUST_API_KEY)) {
+ throw new Error("Eval Braintrust export requires an explicit project UUID and API key");
+ }
+ const config = { projectId, apiKey: env.BRAINTRUST_API_KEY.trim(),
+ ...(env.PAPERCLIP_E2E_OTLP_RELAY_URL ? { endpointBaseUrl: env.PAPERCLIP_E2E_OTLP_RELAY_URL } : {}) };
+ evalOtlpEndpoint(config);
+ return config;
+}
+
+type Attributes = Record<string, string | number | boolean | string[]>;
+type Phase = "setup" | "execution" | "feedback" | "cleanup" | "finalization";
+interface PhaseSpan { name: Phase; spanId: string; start: number; end?: number; failed?: boolean }
+export interface EvalTraceIdentity { campaignId: string; attemptId: string; executionId: string; provider: string; model: string; environment: string }
+export interface EvalTraceProvenance {
+ suiteId: string; suiteDefinitionHash: string; profileId: string; caseId: string; sourceRevision: string | null;
+}
+export interface EvalTraceReceipt {
+ projectId: string; traceId: string; spanId: string;
+ status: "pending" | "accepted" | "failed";
+ verification: "unverified";
+ nativeConfiguration: "not_started" | "configured" | "failed";
+ companyId?: string; revision?: string; errorCode?: string; httpStatus?: number;
+ finalization?: EvalFinalizationReceipt;
+}
+export interface EvalFinalizationReceipt {
+ spanId: string; status: "accepted" | "failed"; outcome: "passed" | "failed"; verification: "unverified";
+ errorCode?: string; httpStatus?: number;
+}
+const require = createRequire(new URL("../../server/package.json", import.meta.url));
+const time = (ms: number): [number, number] => [Math.floor(ms / 1000), (ms % 1000) * 1_000_000];
+
+export function exportBraintrustAttemptParent(receipt: Pick<EvalTraceReceipt, "projectId" | "spanId" | "traceId">): string {
+ if (!uuid.test(receipt.projectId) || !/^[0-9a-f]{16}$/.test(receipt.spanId) || !/^[0-9a-f]{32}$/.test(receipt.traceId)
+ || /^0+$/.test(receipt.spanId) || /^0+$/.test(receipt.traceId)) throw new Error("Invalid eval trace parent identity");
+ return Buffer.concat([Buffer.from([4, 2, 0]), Buffer.from(JSON.stringify({
+ object_id: receipt.projectId, row_id: receipt.spanId, span_id: receipt.spanId, root_span_id: receipt.traceId,
+ }))]).toString("base64");
+}
+
+export class EvalAttemptTrace {
+ private readonly startedAt = Date.now();
+ private readonly phases: PhaseSpan[] = [];
+ private active?: PhaseSpan;
+ private readonly receipt: EvalTraceReceipt;
+ private readonly endpointBaseUrl: string;
+ constructor(private readonly config: EvalBraintrustConfig, private readonly identity: EvalTraceIdentity, private readonly provenance?: EvalTraceProvenance,
+ context?: { receipt: EvalTraceReceipt; startedAtMs: number }) {
+ this.endpointBaseUrl = evalOtlpEndpoint(config);
+ if (Object.values(identity).some(value => typeof value !== "string" || !value || value.length > 512 || /[\r\n]/.test(value))) throw new Error("Invalid eval trace identity");
+ if (provenance && (Object.values(provenance).some(value => value !== null && (typeof value !== "string" || !value || value.length > 512 || /[\r\n]/.test(value)))
+ || !/^[0-9a-f]{64}$/.test(provenance.suiteDefinitionHash)
+ || provenance.sourceRevision !== null && !/^[0-9a-f]{40}$/.test(provenance.sourceRevision))) throw new Error("Invalid eval trace provenance");
+ this.receipt = { projectId: config.projectId, traceId: randomBytes(16).toString("hex"), spanId: randomBytes(8).toString("hex"),
+ status: "pending", verification: "unverified", nativeConfiguration: "not_started" };
+ if (context) {
+ const { receipt, startedAtMs } = context;
+ exportBraintrustAttemptParent(receipt);
+ if (receipt.projectId !== config.projectId || !Number.isFinite(startedAtMs) || startedAtMs < 0 || startedAtMs > Date.now()
+ || !["not_started", "configured", "failed"].includes(receipt.nativeConfiguration)
+ || receipt.companyId !== undefined && !uuid.test(receipt.companyId)
+ || receipt.revision !== undefined && !uuid.test(receipt.revision)) throw new Error("Invalid eval finalization context");
+ this.startedAt = startedAtMs;
+ Object.assign(this.receipt, { traceId: receipt.traceId, spanId: receipt.spanId, nativeConfiguration: receipt.nativeConfiguration,
+ ...(receipt.companyId ? { companyId: receipt.companyId } : {}), ...(receipt.revision ? { revision: receipt.revision } : {}) });
+ }
+ }
+ snapshot(): EvalTraceReceipt { return { ...this.receipt }; }
+ phase(name: Phase, previousFailed = false) {
+ this.endPhase(previousFailed);
+ this.active = { name, spanId: randomBytes(8).toString("hex"), start: Date.now() };
+ this.phases.push(this.active);
+ return this.active.spanId;
+ }
+ endPhase(failed = false) {
+ if (this.active) { this.active.end = Date.now(); this.active.failed = failed; this.active = undefined; }
+ }
+ async configureCompany(api: Pick<RunnerApi, "putSensitive">, companyId: string) {
+ if (!uuid.test(companyId)) throw new Error("Invalid eval trace company identity");
+ this.receipt.companyId = companyId;
+ try {
+ const saved = await api.putSensitive<{ settings: { revision: string; enabled: boolean; credentialsConfigured: boolean; destination: { provider: string; endpointBaseUrl?: string } } }>(
+ `/api/companies/${companyId}/observability`, {
+ expectedRevision: null, enabled: true,
+ destination: { provider: "otlp", endpointBaseUrl: this.endpointBaseUrl },
+ credentials: { mode: "replace", headers: [
+ { name: "authorization", value: `Bearer ${this.config.apiKey}` },
+ { name: "x-bt-parent", value: exportBraintrustAttemptParent(this.receipt) },
+ ] },
+ capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true },
+ });
+ if (!saved.settings?.enabled || !saved.settings.credentialsConfigured || !uuid.test(saved.settings.revision)
+ || saved.settings.destination.provider !== "otlp" || saved.settings.destination.endpointBaseUrl !== this.endpointBaseUrl) throw new Error("Unexpected observability setting");
+ this.receipt.revision = saved.settings.revision;
+ this.receipt.nativeConfiguration = "configured";
+ } catch {
+ this.receipt.nativeConfiguration = "failed";
+ throw new Error("Eval native observability configuration failed; response withheld");
+ }
+ }
+ async export(input: { failed: boolean; runIds: string[]; cleanup: string; complaints: number; suggestions: number;
+ outcomeStage?: "worker" | "supervisor"; failureClass?: FailureClass }, transport: typeof fetch = fetch): Promise<EvalTraceReceipt> {
+ this.endPhase(input.failed);
+ const endedAt = Date.now();
+ const { resourceFromAttributes } = require("@opentelemetry/resources") as { resourceFromAttributes(attributes: Attributes): unknown };
+ const { ProtobufTraceSerializer: serializer } = require("@opentelemetry/otlp-transformer") as {
+ ProtobufTraceSerializer: { serializeRequest(spans: unknown[]): Uint8Array | undefined; deserializeResponse(bytes: Uint8Array): unknown };
+ };
+ const attributes: Attributes = {
+ "paperclip.eval.campaign_id": this.identity.campaignId, "paperclip.eval.attempt_id": this.identity.attemptId,
+ "paperclip.eval.execution_id": this.identity.executionId, "paperclip.eval.provider": this.identity.provider,
+ "paperclip.eval.model": this.identity.model, "paperclip.eval.environment": this.identity.environment,
+ "paperclip.eval.outcome_stage": input.outcomeStage ?? "worker", "paperclip.eval.outcome": input.failed ? "failed" : "passed",
+ ...(input.failureClass ? { "paperclip.eval.failure_class": input.failureClass } : {}),
+ "paperclip.eval.cleanup": input.cleanup, "paperclip.eval.run_ids": input.runIds,
+ "paperclip.eval.feedback.complaints": input.complaints, "paperclip.eval.feedback.suggestions": input.suggestions,
+ ...(this.provenance ? {
+ "paperclip.eval.suite_id": this.provenance.suiteId, "paperclip.eval.suite_definition_hash": this.provenance.suiteDefinitionHash,
+ "paperclip.eval.profile_id": this.provenance.profileId, "paperclip.eval.case_id": this.provenance.caseId,
+ "paperclip.eval.source_revision": this.provenance.sourceRevision ?? "unknown",
+ "paperclip.eval.grader": "tests/runner-e2e/runner.spec.ts",
+ } : {}),
+ ...(this.receipt.companyId ? { "paperclip.company.id": this.receipt.companyId } : {}),
+ };
+ const resource = resourceFromAttributes({ "service.name": "paperclip-evals" });
+ const instrumentationScope = { name: "paperclip.eval-operations", version: "1" };
+ const rootContext = { traceId: this.receipt.traceId, spanId: this.receipt.spanId, traceFlags: 1 };
+ const span = (name: string, spanId: string, start: number, end: number, failed: boolean, parent = false) => ({
+ name, kind: 0, spanContext: () => ({ ...rootContext, spanId }), ...(parent ? { parentSpanContext: rootContext } : {}),
+ startTime: time(start), endTime: time(end), duration: time(end - start), status: { code: failed ? 2 : 1 }, attributes,
+ links: [], events: [], ended: true, resource, instrumentationScope,
+ droppedAttributesCount: 0, droppedEventsCount: 0, droppedLinksCount: 0,
+ });
+ try {
+ const body = serializer.serializeRequest([
+ span("paperclip.eval.attempt", this.receipt.spanId, this.startedAt, endedAt, input.failed),
+ ...this.phases.map(phase => span(`paperclip.eval.${phase.name}`, phase.spanId, phase.start, phase.end!, phase.failed ?? false, true)),
+ ]);
+ if (!body) throw new Error("serialization");
+ const response = await transport(`${this.endpointBaseUrl}/v1/traces`, {
+ method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000),
+ headers: { authorization: `Bearer ${this.config.apiKey}`, "x-bt-parent": `project_id:${this.config.projectId}`,
+ "content-type": "application/x-protobuf", accept: "application/x-protobuf, application/json" },
+ body: Buffer.from(body),
+ });
+ this.receipt.httpStatus = response.status;
+ if (!response.ok) { this.receipt.errorCode = "receiver_rejected"; await response.body?.cancel(); throw new Error("rejected"); }
+ const contentType = response.headers.get("content-type")?.split(";")[0].trim();
+ if (contentType && !["application/json", "application/x-protobuf"].includes(contentType)) { await response.body?.cancel(); throw new Error("invalid_response"); }
+ const reader = response.body?.getReader();
+ const chunks: Uint8Array[] = []; let length = 0;
+ try {
+ while (reader) {
+ const next = await reader.read(); if (next.done) break;
+ length += next.value.length; if (length > 65_536) throw new Error("response_too_large");
+ chunks.push(next.value);
+ }
+ } finally { await reader?.cancel().catch(() => {}); }
+ if (length) {
+ const bytes = Buffer.concat(chunks);
+ const decoded = contentType === "application/json" ? JSON.parse(bytes.toString("utf8"))
+ : contentType === "application/x-protobuf" ? serializer.deserializeResponse(bytes) : null;
+ if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_response");
+ const partial = (decoded as { partialSuccess?: { rejectedSpans?: unknown } }).partialSuccess;
+ if (partial !== undefined && (!partial || typeof partial !== "object" || Array.isArray(partial))) throw new Error("invalid_response");
+ const rejected = partial?.rejectedSpans ?? 0;
+ if (!/^[0-9]+$/.test(String(rejected)) || !Number.isSafeInteger(Number(rejected))) throw new Error("invalid_response");
+ if (Number(rejected) !== 0) { this.receipt.errorCode = "partial_rejection"; throw new Error("partial_rejection"); }
+ }
+ this.receipt.status = "accepted";
+ } catch {
+ this.receipt.status = "failed";
+ this.receipt.errorCode ??= "export_failed";
+ }
+ return this.snapshot();
+ }
+}
+
+export async function finalizeEvalAttemptTrace(config: EvalBraintrustConfig, input: {
+ receipt: EvalTraceReceipt; identity: EvalTraceIdentity; provenance: EvalTraceProvenance;
+ result: Pick<RunnerE2EResult, "startedAt" | "status" | "cleanup" | "runIds" | "feedback" | "failureClass">;
+}, transport: typeof fetch = fetch): Promise<EvalFinalizationReceipt> {
+ const failed = input.result.status !== "passed" || input.result.cleanup !== "passed";
+ const receipt: EvalFinalizationReceipt = { spanId: randomBytes(8).toString("hex"), status: "failed", outcome: failed ? "failed" : "passed", verification: "unverified" };
+ try {
+ if (!["passed", "failed"].includes(input.result.status) || !["not_started", "passed", "failed"].includes(input.result.cleanup)
+ || input.result.failureClass !== undefined && !["candidate_failure", "provider_variance", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"].includes(input.result.failureClass)
+ || input.result.runIds !== undefined && (!Array.isArray(input.result.runIds) || input.result.runIds.some(id => !uuid.test(id)))
+ || [input.result.feedback?.complaints ?? 0, input.result.feedback?.suggestions ?? 0].some(count => !Number.isSafeInteger(count) || count < 0)) throw new Error("Invalid eval finalization result");
+ const trace = new EvalAttemptTrace(config, input.identity, input.provenance, { receipt: input.receipt, startedAtMs: Date.parse(input.result.startedAt) });
+ receipt.spanId = trace.phase("finalization");
+ const exported = await trace.export({ failed, outcomeStage: "supervisor", failureClass: input.result.failureClass,
+ runIds: input.result.runIds ?? [], cleanup: input.result.cleanup,
+ complaints: input.result.feedback?.complaints ?? 0, suggestions: input.result.feedback?.suggestions ?? 0,
+ }, transport);
+ receipt.status = exported.status === "accepted" ? "accepted" : "failed";
+ if (exported.httpStatus !== undefined) receipt.httpStatus = exported.httpStatus;
+ if (exported.errorCode) receipt.errorCode = exported.errorCode;
+ } catch { receipt.errorCode = "invalid_finalization_context"; }
+ return receipt;
+}
diff --git a/tests/runner-e2e/budgeted-setup.test.ts b/tests/runner-e2e/budgeted-setup.test.ts
new file mode 100644
index 0000000000..f7ddaf2c5b
--- /dev/null
+++ b/tests/runner-e2e/budgeted-setup.test.ts
@@ -0,0 +1,75 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it, vi } from "vitest";
+import type { RunnerApi } from "./api.js";
+import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js";
+import { finishBudgetedEvalSetup, type EvalSetupObservation } from "./budgeted-setup.js";
+
+const identity = { executionId: "fixture-execution", attemptId: "fixture-attempt" };
+const ready = { id: "agent", companyId: "company", status: "idle", lifecycleState: "ready", lifecycleError: null };
+async function fixture() {
+ const directory = await mkdtemp(path.join(tmpdir(), "budgeted-setup-"));
+ const file = path.join(directory, "phase.json");
+ await createEvalAttemptPhase(file, identity);
+ return { directory, file, env: { PAPERCLIP_E2E_GATEWAY_BASE_URL: "http://127.0.0.1:12345",
+ PAPERCLIP_E2E_GATEWAY_PHASE_PATH: file, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: identity.attemptId } };
+}
+
+it("keeps asynchronous hello probes in setup until the same agent is ready", async () => {
+ const f = await fixture();
+ const get = vi.fn().mockResolvedValueOnce({ ...ready, status: "paused", lifecycleState: "verifying", adapterConfig: { private: true } }).mockResolvedValue(ready);
+ let saved: EvalSetupObservation[] = [];
+ try {
+ await finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",
+ deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi,
+ observe: async observations => {
+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
+ saved = structuredClone(observations);
+ } }, f.env);
+ expect(saved.map(row => row.lifecycleState)).toEqual(["verifying", "ready"]);
+ expect(saved[0]).not.toHaveProperty("adapterConfig");
+ expect(get).toHaveBeenCalledTimes(2);
+ expect(get.mock.calls.every(([url, options]) => url === "/api/agents/agent" && options.timeout > 0 && options.timeout <= 5_000)).toBe(true);
+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("agent");
+ } finally { await rm(f.directory, { recursive: true, force: true }); }
+});
+
+it.each([
+ [{ ...ready, lifecycleError: "harness_command_unavailable" }, "setup failed"],
+ [{ ...ready, lifecycleHolds: ["approval"] }, "setup is held"],
+ [{ ...ready, companyId: "other" }, "scope changed"],
+ [{ ...ready, lifecycleState: "terminated" }, "setup stopped"],
+] as const)("preserves setup attribution and the failed state when readiness is rejected: %s", async (state, reason) => {
+ const f = await fixture(), observe = vi.fn(async () => {}), get = vi.fn().mockResolvedValue(state);
+ try {
+ await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",
+ deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi, observe }, f.env)).rejects.toThrow(reason);
+ expect(get).toHaveBeenCalledOnce();
+ expect(observe).toHaveBeenCalledOnce();
+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
+ } finally { await rm(f.directory, { recursive: true, force: true }); }
+});
+
+it("does not start a readiness request after the attempt deadline", async () => {
+ const f = await fixture(), get = vi.fn();
+ try {
+ await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",
+ deadlineAt: Date.now() - 1, api: { get } as unknown as RunnerApi, observe: async () => {} }, f.env)).rejects.toThrow("Timed out");
+ expect(get).not.toHaveBeenCalled();
+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
+ } finally { await rm(f.directory, { recursive: true, force: true }); }
+});
+
+it("does not advance when saving readiness crosses the deadline", async () => {
+ const f = await fixture(), get = vi.fn().mockResolvedValue(ready);
+ const deadlineAt = Date.now() + 5_000;
+ const now = vi.spyOn(Date, "now");
+ try {
+ await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",
+ deadlineAt, api: { get } as unknown as RunnerApi,
+ observe: async () => { now.mockReturnValue(deadlineAt); } }, f.env)).rejects.toThrow("Timed out saving");
+ expect(get).toHaveBeenCalledOnce();
+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
+ } finally { now.mockRestore(); await rm(f.directory, { recursive: true, force: true }); }
+});
diff --git a/tests/runner-e2e/budgeted-setup.ts b/tests/runner-e2e/budgeted-setup.ts
new file mode 100644
index 0000000000..42dc216ea5
--- /dev/null
+++ b/tests/runner-e2e/budgeted-setup.ts
@@ -0,0 +1,44 @@
+import type { Agent } from "../../packages/shared/src/types/agent.js";
+import { pollUntil, type RunnerApi } from "./api.js";
+import { beginEvalAgentPhase } from "./attempt-phase.js";
+
+type SetupState = Pick<Agent, "id" | "companyId" | "status" | "lifecycleState" | "lifecycleError"> & { lifecycleHolds?: string[] };
+export type EvalSetupObservation = SetupState & { observedAt: string };
+
+export async function finishBudgetedEvalSetup(input: {
+ executionId: string;
+ agentId: string;
+ companyId: string;
+ deadlineAt: number;
+ api: Pick<RunnerApi, "get">;
+ observe: (observations: EvalSetupObservation[]) => Promise<void>;
+}, env: NodeJS.ProcessEnv = process.env) {
+ if (env.PAPERCLIP_E2E_GATEWAY_BASE_URL) {
+ const deadlineAt = Math.min(input.deadlineAt, Date.now() + 45_000);
+ const observations: EvalSetupObservation[] = [];
+ let previous: string | undefined;
+ const scoped = (agent: SetupState) => agent.id === input.agentId && agent.companyId === input.companyId;
+ await pollUntil({ label: "budgeted eval agent setup", deadlineAt, intervalMs: 500,
+ load: async () => {
+ const agent = await input.api.get<SetupState>(`/api/agents/${input.agentId}`, { timeout: Math.max(1, Math.min(5_000, deadlineAt - Date.now())) });
+ const state: SetupState = { id: agent.id, companyId: agent.companyId, status: agent.status,
+ lifecycleState: agent.lifecycleState, lifecycleError: agent.lifecycleError, lifecycleHolds: agent.lifecycleHolds };
+ const current = JSON.stringify(state);
+ if (current !== previous) {
+ observations.push({ ...state, observedAt: new Date().toISOString() });
+ await input.observe(observations);
+ previous = current;
+ }
+ return state;
+ },
+ accept: agent => scoped(agent) && agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status)
+ && !agent.lifecycleError && !agent.lifecycleHolds?.length,
+ reject: agent => !scoped(agent) ? "Agent scope changed"
+ : agent.lifecycleError ? `Agent setup failed: ${agent.lifecycleError}`
+ : agent.lifecycleHolds?.length ? "Agent setup is held"
+ : ["paused", "rejected", "terminated"].includes(agent.lifecycleState ?? "") ? `Agent setup stopped: ${agent.lifecycleState}` : undefined,
+ });
+ if (Date.now() >= deadlineAt) throw new Error("Timed out saving budgeted eval agent readiness");
+ }
+ await beginEvalAgentPhase(input.executionId, env);
+}
diff --git a/tests/runner-e2e/campaign-timing.test.ts b/tests/runner-e2e/campaign-timing.test.ts
new file mode 100644
index 0000000000..cf7409be76
--- /dev/null
+++ b/tests/runner-e2e/campaign-timing.test.ts
@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { buildCampaignAccounting } from "./attempt-accounting.js";
+import { campaignAttemptTimeLabel, summarizeCampaignTiming } from "./campaign-timing.js";
+import type { RunnerE2EResult } from "./types.js";
+
+function result(executionId: string, startMs: number, durationMs: number, attempt = 1): RunnerE2EResult {
+ return {
+ schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "timing-fixture",
+ accountingIdentity: { campaignId: "timing-campaign", attemptId: `${executionId}-${attempt}` },
+ attempt, status: "passed", profileId: "runner-codex", environmentId: "local",
+ caseId: executionId, provider: "codex", model: "fixture-model", runtimeMode: "native",
+ startedAt: new Date(startMs).toISOString(), finishedAt: new Date(startMs + durationMs).toISOString(),
+ durationMs, cleanup: "passed",
+ };
+}
+
+describe("campaign timing evidence", () => {
+ it("separates overlapping attempt work from the observed window and retains a failed retry", () => {
+ const failed = { ...result("a", 0, 30_000), status: "failed" as const };
+ const selected = [result("a", 30_000, 120_000, 2), result("b", 60_000, 60_000)];
+ const accounting = buildCampaignAccounting({ results: selected, attempts: [failed, ...selected] });
+ expect(summarizeCampaignTiming(selected, accounting)).toEqual({
+ retainedAttempts: 3, missingAttempts: 0, subtotalMs: 210_000, totalMs: 210_000, observedWindowMs: 150_000,
+ });
+ });
+
+ it("keeps a missing retry unknown while exposing the retained subtotal", () => {
+ const timing = summarizeCampaignTiming([result("a", 60_000, 30_000, 2)]);
+ expect(timing).toEqual({ retainedAttempts: 1, missingAttempts: 1, subtotalMs: 30_000, totalMs: null, observedWindowMs: 30_000 });
+ expect(campaignAttemptTimeLabel(timing)).toBe("Unknown (30s retained subtotal)");
+ const missing = summarizeCampaignTiming([{ ...result("a", 0, 0, 0), status: "failed" }]);
+ expect(missing.totalMs).toBeNull();
+ expect(missing.observedWindowMs).toBeNull();
+ expect(campaignAttemptTimeLabel(missing)).toBe("Unknown");
+ });
+
+ it("does not derive an observed window from reversed retained timestamps", () => {
+ const timing = summarizeCampaignTiming([{ ...result("a", 1000, 1000), finishedAt: new Date(0).toISOString() }]);
+ expect(timing.totalMs).toBe(1000);
+ expect(timing.observedWindowMs).toBeNull();
+ });
+});
diff --git a/tests/runner-e2e/campaign-timing.ts b/tests/runner-e2e/campaign-timing.ts
new file mode 100644
index 0000000000..0958dede05
--- /dev/null
+++ b/tests/runner-e2e/campaign-timing.ts
@@ -0,0 +1,83 @@
+import { buildCampaignAccounting } from "./attempt-accounting.js";
+import { durationLabel, html } from "./report-format.js";
+import { WORKER_TIMING_PHASES } from "./worker-timing.js";
+import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js";
+
+export const campaignTimingScope = "Attempt durations include setup and cleanup captured inside each attempt. The observed window spans the earliest retained start to the latest retained finish, including gaps and overlap. Carried-forward evidence can span older runs, so this window is not the current workflow's duration. Neither measure is CI queue/setup time or provider-billed runtime.";
+
+export function summarizeCampaignTiming(
+ results: readonly RunnerE2EResult[],
+ retained?: RunnerE2ECampaignAccounting,
+) {
+ const accounting = retained ?? buildCampaignAccounting({ results });
+ const attempts = accounting.attempts;
+ const subtotal = attempts.reduce((total, attempt) => total + attempt.durationMs, 0);
+ const subtotalMs = Number.isFinite(subtotal) ? subtotal : null;
+ const starts = attempts.map(attempt => Date.parse(attempt.startedAt));
+ const finishes = attempts.map(attempt => Date.parse(attempt.finishedAt));
+ const validWindows = attempts.length > 0 && starts.every((start, index) =>
+ Number.isFinite(start) && Number.isFinite(finishes[index]) && finishes[index]! >= start);
+ const observedWindowMs = validWindows ? Math.max(...finishes) - Math.min(...starts) : null;
+ const complete = attempts.length > 0 && accounting.missingAttempts === 0 && subtotalMs !== null;
+ return {
+ retainedAttempts: attempts.length,
+ missingAttempts: accounting.missingAttempts,
+ subtotalMs,
+ totalMs: complete ? subtotalMs : null,
+ observedWindowMs,
+ };
+}
+
+export function campaignAttemptTimeLabel(timing: ReturnType<typeof summarizeCampaignTiming>) {
+ if (timing.totalMs !== null) return durationLabel(timing.totalMs);
+ return timing.retainedAttempts > 0 && timing.subtotalMs !== null
+ ? `Unknown (${durationLabel(timing.subtotalMs)} retained subtotal)`
+ : "Unknown";
+}
+
+export function campaignTimingText(timing: ReturnType<typeof summarizeCampaignTiming>) {
+ return [
+ `Summed attempt time: ${campaignAttemptTimeLabel(timing)}; ${timing.retainedAttempts} retained attempts; ${timing.missingAttempts} missing attempts.`,
+ `Observed attempt window: ${durationLabel(timing.observedWindowMs)}.`,
+ campaignTimingScope,
+ ];
+}
+
+export function summarizeWorkerPhaseTiming(results: readonly RunnerE2EResult[], retained?: RunnerE2ECampaignAccounting) {
+ const accounting = retained ?? buildCampaignAccounting({ results });
+ const receipts = accounting.attempts.map(attempt => attempt.workerTiming);
+ const completeAttempts = receipts.filter(receipt => receipt?.status === "complete").length;
+ const partialAttempts = receipts.filter(receipt => receipt?.status === "partial").length;
+ const complete = receipts.length > 0 && completeAttempts === receipts.length && accounting.missingAttempts === 0;
+ return {
+ retainedAttempts: receipts.length, completeAttempts, partialAttempts, missingAttempts: accounting.missingAttempts,
+ phases: WORKER_TIMING_PHASES.map(phase => {
+ const segments = receipts.flatMap(receipt => receipt && receipt.status !== "unavailable"
+ ? receipt.segments.filter(segment => segment.phase === phase && segment.finishedAt !== null) : []);
+ const subtotalMs = segments.reduce((total, segment) => total + Date.parse(segment.finishedAt!) - Date.parse(segment.startedAt), 0);
+ return { phase, totalMs: complete ? subtotalMs : null, subtotalMs, observedSegments: segments.length };
+ }),
+ };
+}
+
+export const workerTimingScope = "Worker phases cover the test callback through finalization before result publication. Browser/server startup, parent cleanup, CI queue/build time and provider-billed runtime are outside these measurements. An interrupted phase has no inferred end time.";
+
+function workerPhaseLabel(phase: ReturnType<typeof summarizeWorkerPhaseTiming>["phases"][number]) {
+ return phase.totalMs !== null ? durationLabel(phase.totalMs)
+ : phase.observedSegments ? `Unknown (${durationLabel(phase.subtotalMs)} observed)` : "Unknown";
+}
+
+export function workerPhaseTimingText(timing: ReturnType<typeof summarizeWorkerPhaseTiming>) {
+ return [
+ `Worker phase coverage: ${timing.completeAttempts}/${timing.retainedAttempts} complete receipts; ${timing.partialAttempts} partial; ${timing.missingAttempts} missing attempts.`,
+ ...timing.phases.map(phase => `Worker ${phase.phase} time: ${workerPhaseLabel(phase)}.`),
+ workerTimingScope,
+ ];
+}
+
+export function renderWorkerPhaseTiming(timing: ReturnType<typeof summarizeWorkerPhaseTiming>) {
+ return `<section class="billing-overview" aria-label="Worker phase timing">
+ ${timing.phases.map(phase => `<div class="billing-metric"><strong>${html(workerPhaseLabel(phase))}</strong><span>Worker ${phase.phase} time</span></div>`).join("")}
+ <p class="billing-note">${timing.completeAttempts}/${timing.retainedAttempts} complete worker receipts; ${timing.partialAttempts} partial; ${timing.missingAttempts} missing attempts. ${html(workerTimingScope)}</p>
+ </section>`;
+}
diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts
index b06ccb8b1f..8fe1d5957f 100644
--- a/tests/runner-e2e/catalog.test.ts
+++ b/tests/runner-e2e/catalog.test.ts
@@ -219,10 +219,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
- 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 4, 116, 2,
+ 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2,
]);
- expect(validateRunnerCatalog()).toHaveLength(761);
- expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(761);
+ expect(validateRunnerCatalog()).toHaveLength(763);
+ expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(763);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(48);
diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts
index 8f33bc2ad3..c6bc7b4d29 100644
--- a/tests/runner-e2e/catalog.ts
+++ b/tests/runner-e2e/catalog.ts
@@ -1,3 +1,4 @@
+import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js";
import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js";
import { finalizationRecoveryTask, finalizationRecoveryDigest } from "./finalization-recovery.js";
import { necessaryQuestionTask, necessaryQuestionDigest } from "./necessary-question.js";
@@ -1664,6 +1665,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
},
...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []),
+ {
+ id: FEEDBACK_PERSISTENCE_SUITE, label: "Feedback Persistence", manualOnly: true,
+ description: "Real complaint and suggestion submissions, durable bodies, and retained tool receipts.",
+ groups: ["native"],
+ profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)),
+ environments: [localEnvironment], tasks: [feedbackPersistenceTask(runnerTasks.find(task => task.id === "message-marker")!)], expectedMatrixSize: 2,
+ definitionMetadata: { version: 1, owner: "runner-e2e-maintainers", scheduling: "explicit-only",
+ sourceDigest: feedbackPersistenceDigest(runnerTasks.find(task => task.id === "message-marker")!), expectedWorkRuns: 1, automaticRetryPolicy: "single_attempt",
+ budgetCents: 1_000, forecast: "unmeasured; gateway manifest required for bounded qualification",
+ grading: "retained-exact-bodies-and-run-attributed-completed-tools", feedback: "synthetic-disposable-items" },
+ },
{
id: "core-compatibility",
label: "Core Runner Compatibility",
diff --git a/tests/runner-e2e/cleanup-verification.ts b/tests/runner-e2e/cleanup-verification.ts
index d21191e99f..2e935b226b 100644
--- a/tests/runner-e2e/cleanup-verification.ts
+++ b/tests/runner-e2e/cleanup-verification.ts
@@ -22,7 +22,7 @@ export function mustPreserveRecoveryState(input: {
resourceAdmissionStarted?: boolean;
results: readonly { cleanup: string; synthetic?: boolean }[];
}) {
- return input.processCleanupFailed || input.results.some(result => result.cleanup === "failed")
+ return input.processCleanupFailed || input.results.some(result => result.cleanup === "failed" || result.synthetic)
|| (input.resourceAdmissionStarted === true && (input.results.length === 0 || input.results.some(result => result.synthetic)));
}
diff --git a/tests/runner-e2e/codex-ci-sandbox.test.ts b/tests/runner-e2e/codex-ci-sandbox.test.ts
index 0594234598..94b043ed20 100644
--- a/tests/runner-e2e/codex-ci-sandbox.test.ts
+++ b/tests/runner-e2e/codex-ci-sandbox.test.ts
@@ -34,7 +34,7 @@ describe("Codex CI sandbox trust boundary", () => {
/- name: Provision Codex sandbox on the disposable trusted runner\n\s+if: ([^\n]+)/,
)?.[1];
expect(condition).toMatch(
- /^matrix\.environmentId == 'local' && \(matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/,
+ /^matrix\.environmentId == 'local' && \(needs\.catalog\.outputs\.operations_cadence == 'weekly' \|\| matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/,
);
const provisionedProfiles = new Set(
[...condition!.matchAll(/matrix\.profileId == '([^']+)'/g)].map((match) => match[1]),
@@ -77,7 +77,7 @@ describe("Codex CI sandbox trust boundary", () => {
const setup = workflow.indexOf(
"- name: Provision Codex sandbox on the disposable trusted runner",
);
- const paid = workflow.indexOf("- name: Run paid cell");
+ const paid = workflow.indexOf("- name: Run bounded campaign and retain feedback review");
expect(setup).toBeGreaterThan(
workflow.indexOf(
"- name: Reauthorize paid execution before provider access",
diff --git a/tests/runner-e2e/cost-ledger.test.ts b/tests/runner-e2e/cost-ledger.test.ts
new file mode 100644
index 0000000000..c4c0346775
--- /dev/null
+++ b/tests/runner-e2e/cost-ledger.test.ts
@@ -0,0 +1,197 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalCostObservation, type EvalSpendLimits } from "./cost-ledger.js";
+
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+const limits = (overrides: Partial<EvalSpendLimits> = {}): EvalSpendLimits => ({ campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2, ...overrides });
+const identity = { executionId: "codex.local.fixture", attemptId: "attempt-1", phase: "setup" as const };
+const observation = (overrides: Partial<EvalCostObservation> = {}): EvalCostObservation => ({ observationId: "receipt", requestId: "request", source: "provider", provider: "openai", model: "fixture", providerRequestId: "provider-request", currency: "USD", route: "direct", classification: "reported", costUsd: "0.000000123", final: true, ...overrides });
+async function fixture(overrides: Partial<EvalSpendLimits> = {}) {
+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-cost-ledger-")); roots.push(root);
+ const file = path.join(root, "costs.jsonl");
+ return { file, ledger: await openEvalCostLedger(file, limits(overrides)) };
+}
+
+it("persists a reservation before dispatch and counts setup spending with nanodollar precision", async () => {
+ const { file, ledger } = await fixture();
+ try {
+ await ledger.reserve("request", identity, "0.1");
+ const beforeDispatch = (await readEvalCostLedger(file)).state.snapshot();
+ expect(beforeDispatch).toMatchObject({ committedUsd: "0.100000000", heldUsd: "0.100000000", unknownRequests: 1 });
+ await ledger.observe(observation());
+ await ledger.finish("request", "succeeded");
+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.000000123", committedUsd: "0.000000123", heldUsd: "0.000000000",
+ requests: [{ phase: "setup", receiptComplete: true }] });
+ } finally { await ledger.close(); }
+ expect((await readEvalCostLedger(file)).state.snapshot().knownUsd).toBe("0.000000123");
+});
+
+it("serializes competing admissions so concurrent requests cannot oversubscribe a cap", async () => {
+ const { ledger } = await fixture();
+ try {
+ const admitted = await Promise.allSettled([ledger.reserve("one", identity, "0.6"), ledger.reserve("two", identity, "0.6")]);
+ expect(admitted.map(value => value.status)).toEqual(["fulfilled", "rejected"]);
+ expect(ledger.snapshot().committedUsd).toBe("0.600000000");
+ } finally { await ledger.close(); }
+});
+
+it("durably caps attempts including zero-request failures and rejects unadmitted requests", async () => {
+ const cap = limits({ maxAttemptsPerExecution: 2 });
+ const { file, ledger } = await fixture({ maxAttemptsPerExecution: 2 });
+ await expect(ledger.reserve("request", identity, "0.1")).rejects.toThrow("no admitted attempt");
+ await ledger.admitAttempt(identity.executionId, "attempt-1");
+ await ledger.close();
+ const resumed = await openEvalCostLedger(file, cap);
+ try {
+ await expect(resumed.admitAttempt(identity.executionId, "attempt-1")).rejects.toThrow("already admitted");
+ const competing = await Promise.allSettled([
+ resumed.admitAttempt(identity.executionId, "attempt-2"),
+ resumed.admitAttempt(identity.executionId, "attempt-3"),
+ ]);
+ expect(competing.map(result => result.status)).toEqual(["fulfilled", "rejected"]);
+ await resumed.reserve("request", { ...identity, attemptId: "attempt-2" }, "0.1");
+ await resumed.admitAttempt("another.cell", "another-attempt");
+ await expect(resumed.reserve("wrong-cell", { ...identity, attemptId: "another-attempt" }, "0.1")).rejects.toThrow("no admitted attempt");
+ expect((await readEvalCostLedger(file)).state.snapshot().admittedAttempts).toHaveLength(3);
+ } finally { await resumed.close(); }
+ await expect(openEvalCostLedger(file, limits({ maxAttemptsPerExecution: 3 }))).rejects.toThrow("limits changed");
+});
+
+it("holds unknown/cancelled charges after restart and counts failed retries independently", async () => {
+ const { file, ledger } = await fixture();
+ await ledger.reserve("request", identity, "0.5");
+ await ledger.finish("request", "cancelled");
+ await ledger.close();
+ const resumed = await openEvalCostLedger(file, limits());
+ try {
+ expect(resumed.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 1, committedUsd: "0.500000000" });
+ await resumed.reserve("retry", { ...identity, attemptId: "attempt-2" }, "0.5");
+ await resumed.observe(observation({ observationId: "retry-cost", requestId: "retry", providerRequestId: "provider-retry", costUsd: "0.2" }));
+ await resumed.finish("retry", "failed");
+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.500000000", committedUsd: "0.700000000" });
+ await resumed.observe(observation({ costUsd: "0.3" }));
+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.500000000", heldUsd: "0.000000000" });
+ } finally { await resumed.close(); }
+});
+
+it("deduplicates provider/gateway/native observations and preserves invoice adjustments", async () => {
+ const { ledger } = await fixture();
+ try {
+ await ledger.reserve("request", identity, "1");
+ await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" }));
+ await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" }));
+ await ledger.observe(observation({ observationId: "native", source: "native_otlp", costUsd: "0.2" }));
+ await ledger.observe(observation({ observationId: "invoice", source: "invoice", classification: "reconciled", costUsd: "0.15" }));
+ await ledger.finish("request", "succeeded");
+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.150000000", breakdown: { estimated: "0.000000000", reported: "0.000000000", reconciled: "0.150000000" } });
+ await expect(ledger.observe(observation({ costUsd: "0.7" }))).rejects.toThrow("identity conflict");
+ } finally { await ledger.close(); }
+});
+
+it("retains partial costs and contradictory receipts without freeing their reservations", async () => {
+ const { ledger } = await fixture();
+ try {
+ await ledger.reserve("request", identity, "0.5");
+ await ledger.observe(observation({ final: false, costUsd: "0.2" }));
+ await ledger.finish("request", "failed");
+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.300000000", unknownRequests: 1 });
+ await ledger.observe(observation({ observationId: "final1", costUsd: "0.3" }));
+ await ledger.observe(observation({ observationId: "final2", costUsd: "0.4" }));
+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.400000000", heldUsd: "0.100000000", conflictingRequests: 1 });
+ await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("receipts conflict");
+ } finally { await ledger.close(); }
+});
+
+it("rejects ambiguous request joins and stops on measured overshoot", async () => {
+ const { ledger } = await fixture();
+ try {
+ await ledger.reserve("request", identity, "0.1");
+ await ledger.reserve("another", identity, "0.1");
+ await ledger.observe(observation({ costUsd: "1.1" }));
+ await expect(ledger.observe(observation({ observationId: "duplicate-request", requestId: "another" }))).rejects.toThrow("another reservation");
+ expect(ledger.snapshot().overBudget).toBe(true);
+ await ledger.finish("request", "succeeded");
+ await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("spending cap");
+ } finally { await ledger.close(); }
+});
+
+it("enforces deadlines, request count, concurrency, and explicit pricing/usage invariants", async () => {
+ const expired = await fixture({ deadlineAt: "2000-01-01T00:00:00Z" });
+ await expect(expired.ledger.reserve("request", identity, "0.1")).rejects.toThrow("deadline");
+ await expired.ledger.close();
+ const { ledger } = await fixture({ maxRequests: 2, maxConcurrent: 1 });
+ try {
+ await ledger.reserve("request", identity, "0.1");
+ await expect(ledger.reserve("two", identity, "0.1")).rejects.toThrow("concurrency");
+ await expect(ledger.observe(observation({ classification: "estimated" }))).rejects.toThrow("pricing provenance");
+ await expect(ledger.observe(observation({ tokens: { output: 1, reasoning: 2 } }))).rejects.toThrow("token usage");
+ await ledger.finish("request", "failed");
+ await ledger.reserve("two", identity, "0.1");
+ await ledger.finish("two", "failed");
+ await expect(ledger.reserve("three", identity, "0.1")).rejects.toThrow("request cap");
+ } finally { await ledger.close(); }
+});
+
+it("fails closed on concurrent writers, changed limits, incomplete writes, and tampering", async () => {
+ const { file, ledger } = await fixture();
+ await expect(openEvalCostLedger(file, limits())).rejects.toThrow("writer lock");
+ await ledger.reserve("request", identity, "0.1");
+ await ledger.close();
+ await expect(openEvalCostLedger(file, limits({ maxUsd: "2" }))).rejects.toThrow("limits changed");
+ const contents = await readFile(file, "utf8");
+ await writeFile(file, contents.slice(0, -1));
+ await expect(readEvalCostLedger(file)).rejects.toThrow("incomplete");
+ await writeFile(file, contents.replace('"maxUsd":"1"', '"maxUsd":"2"'));
+ await expect(readEvalCostLedger(file)).rejects.toThrow("integrity");
+});
+
+it("rejects accidental raw payload fields and accepts equivalent receipt key order", async () => {
+ const { file, ledger } = await fixture();
+ try {
+ await ledger.reserve("request", identity, "0.1");
+ await expect(ledger.observe({ ...observation(), rawResponse: "private response" } as EvalCostObservation)).rejects.toThrow("Unknown ledger field");
+ await ledger.observe(observation());
+ await ledger.observe(Object.fromEntries(Object.entries(observation()).reverse()) as unknown as EvalCostObservation);
+ expect(await readFile(file, "utf8")).not.toContain("private response");
+ } finally { await ledger.close(); }
+});
+
+it("does not let a terminal estimate erase a stronger partial provider charge", async () => {
+ const { ledger } = await fixture();
+ try {
+ await ledger.reserve("request", identity, "0.8");
+ await ledger.observe(observation({ observationId: "provider-partial", final: false, costUsd: "0.6" }));
+ await ledger.observe(observation({ observationId: "gateway-final", source: "gateway", classification: "estimated", pricingVersion: "fixture", costUsd: "0.1" }));
+ await ledger.finish("request", "succeeded");
+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.600000000", heldUsd: "0.200000000", unknownRequests: 1, breakdown: { reported: "0.600000000", estimated: "0.000000000" } });
+ await expect(ledger.reserve("next", identity, "0.3")).rejects.toThrow("spending cap");
+ await ledger.observe(observation({ observationId: "provider-final", costUsd: "0.65" }));
+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.650000000", heldUsd: "0.000000000", unknownRequests: 0 });
+ } finally { await ledger.close(); }
+});
+
+it("replays admissions when the deadline crosses between validation and journal writing", async () => {
+ const admittedAt = Date.parse("2098-12-31T23:59:59.999Z");
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(admittedAt);
+ const { file, ledger } = await fixture();
+ const validate = EvalSpendState.prototype.validate;
+ const validation = vi.spyOn(EvalSpendState.prototype, "validate").mockImplementation(function (this: EvalSpendState, event, now) {
+ validate.call(this, event, now);
+ vi.setSystemTime(admittedAt + 2);
+ });
+ try {
+ await ledger.reserve("request", identity, "0.1");
+ validation.mockRestore();
+ expect((await readEvalCostLedger(file)).state.snapshot().committedUsd).toBe("0.100000000");
+ const records = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ expect(records[1].at).toBe(new Date(admittedAt).toISOString());
+ } finally {
+ validation.mockRestore();
+ vi.useRealTimers();
+ await ledger.close();
+ }
+});
diff --git a/tests/runner-e2e/cost-ledger.ts b/tests/runner-e2e/cost-ledger.ts
new file mode 100644
index 0000000000..c132a6257f
--- /dev/null
+++ b/tests/runner-e2e/cost-ledger.ts
@@ -0,0 +1,293 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import { mkdir, open, readFile, rm, type FileHandle } from "node:fs/promises";
+import path from "node:path";
+import { usdToUnits } from "../../packages/shared/src/money.js";
+import { validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";
+
+export interface EvalSpendLimits {
+ campaignId: string;
+ maxUsd: string;
+ deadlineAt: string;
+ maxRequests: number;
+ maxConcurrent: number;
+ maxAttemptsPerExecution?: number;
+ overhead?: EvalOperatingOverhead;
+}
+export interface EvalChargeIdentity {
+ executionId: string;
+ attemptId: string;
+ phase: "setup" | "agent" | "judge" | "feedback_review" | "tool" | "infrastructure";
+ runId?: string;
+}
+export interface EvalLegacyAttemptAudit {
+ previousJournalHash: string;
+ evidenceSha256: string;
+ maxAttemptsPerExecution: number;
+ priorAttempts: Array<{ executionId: string; attemptId: string }>;
+}
+export interface EvalCostObservation {
+ observationId: string;
+ requestId: string;
+ source: "provider" | "gateway" | "native_otlp" | "heartbeat" | "infrastructure" | "invoice";
+ provider: string;
+ model?: string;
+ providerRequestId?: string;
+ currency: "USD";
+ route: "direct" | "braintrust" | "other_gateway" | "subscription" | "infrastructure";
+ classification: "reported" | "estimated" | "allocated" | "reconciled" | "unknown";
+ costUsd?: string;
+ pricingVersion?: string;
+ tokens?: { input?: number; cachedRead?: number; cachedWrite?: number; output?: number; reasoning?: number };
+ final: boolean;
+}
+
+type Event =
+ | { type: "campaign"; limits: EvalSpendLimits }
+ | ({ type: "attempt_policy" } & EvalLegacyAttemptAudit)
+ | { type: "attempt"; executionId: string; attemptId: string }
+ | { type: "reserve"; requestId: string; identity: EvalChargeIdentity; maxUsd: string }
+ | { type: "observe"; observation: EvalCostObservation }
+ | { type: "finish"; requestId: string; outcome: "succeeded" | "failed" | "cancelled" };
+interface Line { seq: number; at: string; previous: string; event: Event; hash: string }
+interface RequestState { identity: EvalChargeIdentity; reserved: bigint; observations: EvalCostObservation[]; finished: boolean }
+const usd = (units: bigint) => `${units / 1_000_000_000n}.${String(units % 1_000_000_000n).padStart(9, "0")}`;
+export const validEvalLedgerId = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value);
+const validId = validEvalLedgerId;
+const amount = (value: string) => {
+ if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(value)) throw new Error("Invalid ledger USD amount");
+ return usdToUnits(value);
+};
+const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value, (_key, item) =>
+ item && typeof item === "object" && !Array.isArray(item)
+ ? Object.fromEntries(Object.keys(item).sort().map(key => [key, item[key]])) : item)).digest("hex");
+const fields = (value: object, allowed: string[]) => {
+ if (Object.keys(value).some(key => !allowed.includes(key))) throw new Error("Unknown ledger field; refusing unreviewed data");
+};
+const ranks = { unknown: 0, allocated: 1, estimated: 2, reported: 3, reconciled: 4 };
+
+export class EvalAdmissionError extends Error {}
+
+export class EvalSpendState {
+ readonly limits: EvalSpendLimits;
+ readonly requests = new Map<string, RequestState>();
+ readonly observations = new Map<string, EvalCostObservation>();
+ readonly attempts = new Map<string, string>();
+ private providerRequests = new Map<string, string>();
+ constructor(limits: EvalSpendLimits) {
+ this.limits = structuredClone(limits);
+ fields(limits, ["campaignId", "maxUsd", "deadlineAt", "maxRequests", "maxConcurrent", "maxAttemptsPerExecution", "overhead"]);
+ if (!validId(limits.campaignId) || amount(limits.maxUsd) <= 0n || !Number.isFinite(Date.parse(limits.deadlineAt))
+ || !Number.isSafeInteger(limits.maxRequests) || limits.maxRequests < 1
+ || !Number.isSafeInteger(limits.maxConcurrent) || limits.maxConcurrent < 1
+ || limits.maxAttemptsPerExecution !== undefined && (!Number.isSafeInteger(limits.maxAttemptsPerExecution) || limits.maxAttemptsPerExecution < 1)) throw new Error("Invalid eval spend limits");
+ if (this.limits.overhead !== undefined) {
+ validateOperatingOverhead(this.limits.overhead, this.limits.maxUsd);
+ if (this.limits.maxAttemptsPerExecution === undefined || this.limits.maxRequests <= this.limits.overhead.components.length) {
+ throw new Error("Operating overhead requires bounded attempts and room for model requests");
+ }
+ for (const component of this.limits.overhead.components) this.requests.set(`overhead:${component.component}`, {
+ identity: { executionId: "campaign-overhead", attemptId: component.component, phase: "infrastructure" },
+ reserved: amount(component.maxUsd), observations: [], finished: true,
+ });
+ }
+ }
+ private authoritative(request: RequestState) {
+ const priced = request.observations.filter(value => value.costUsd !== undefined);
+ const bestRank = Math.max(-1, ...priced.map(value => ranks[value.classification]));
+ const strongest = priced.filter(value => ranks[value.classification] === bestRank);
+ const finalized = strongest.filter(value => value.final);
+ const best = finalized.length ? finalized : strongest;
+ if (!best.length) return { cost: null, classification: "unknown" as const, conflict: false, resolved: false };
+ const amounts = new Set(best.map(value => amount(value.costUsd!).toString()));
+ const cost = best.reduce((maximum, value) => amount(value.costUsd!) > maximum ? amount(value.costUsd!) : maximum, 0n);
+ return { cost, classification: best[0]!.classification, conflict: finalized.length > 0 && amounts.size !== 1,
+ resolved: finalized.length > 0 && amounts.size === 1 };
+ }
+ snapshot() {
+ let known = 0n, held = 0n, committed = 0n, inflight = 0, unknown = 0, conflicts = 0;
+ const breakdown: Record<"reported" | "estimated" | "allocated" | "reconciled", bigint> = { reported: 0n, estimated: 0n, allocated: 0n, reconciled: 0n };
+ const requests = [...this.requests].map(([requestId, request]) => {
+ const receipt = this.authoritative(request);
+ if (!request.finished) inflight++;
+ if (receipt.conflict) conflicts++;
+ if (!receipt.resolved) unknown++;
+ const cost = receipt.cost ?? 0n;
+ known += cost;
+ if (receipt.classification !== "unknown") breakdown[receipt.classification] += cost;
+ const commitment = !request.finished || !receipt.resolved
+ ? (request.reserved > cost ? request.reserved : cost) : cost;
+ committed += commitment;
+ held += commitment - cost;
+ return { requestId, ...request.identity, reservedUsd: usd(request.reserved), costUsd: receipt.cost === null ? null : usd(cost),
+ classification: receipt.classification, conflict: receipt.conflict, receiptComplete: receipt.resolved, finished: request.finished };
+ });
+ return { campaignId: this.limits.campaignId, limitUsd: usd(amount(this.limits.maxUsd)), knownUsd: usd(known), heldUsd: usd(held), committedUsd: usd(committed),
+ overBudget: committed > amount(this.limits.maxUsd), unknownRequests: unknown, conflictingRequests: conflicts, inflight,
+ breakdown: Object.fromEntries(Object.entries(breakdown).map(([key, value]) => [key, usd(value)])), requests,
+ admittedAttempts: [...this.attempts].map(([attemptId, executionId]) => ({ executionId, attemptId })) };
+ }
+ validate(event: Event, now = Date.now()) {
+ if ((event.type === "attempt" || event.type === "reserve") && this.limits.overhead
+ && (now < Date.parse(this.limits.overhead.verifiedAt) || now >= Date.parse(this.limits.overhead.expiresAt))) {
+ throw new EvalAdmissionError("Operating overhead allowance needs review before paid admission");
+ }
+ if (event.type === "campaign") throw new Error("Duplicate campaign event");
+ if (event.type === "attempt_policy") {
+ fields(event, ["type", "previousJournalHash", "evidenceSha256", "maxAttemptsPerExecution", "priorAttempts"]);
+ if (this.limits.maxAttemptsPerExecution !== undefined || this.attempts.size) throw new Error("Eval attempt policy already established");
+ if (!/^[0-9a-f]{64}$/.test(event.previousJournalHash) || !/^[0-9a-f]{64}$/.test(event.evidenceSha256)
+ || !Number.isSafeInteger(event.maxAttemptsPerExecution) || event.maxAttemptsPerExecution < 1
+ || !Array.isArray(event.priorAttempts)) throw new Error("Invalid legacy attempt audit");
+ const audited = new Map<string, string>();
+ for (const attempt of event.priorAttempts) {
+ fields(attempt, ["executionId", "attemptId"]);
+ if (!validId(attempt.executionId) || !validId(attempt.attemptId) || audited.has(attempt.attemptId)) throw new Error("Invalid legacy attempt identity");
+ audited.set(attempt.attemptId, attempt.executionId);
+ }
+ for (const request of this.requests.values()) {
+ if (!request.finished) throw new Error("Legacy attempt audit requires stopped requests");
+ if (audited.get(request.identity.attemptId) !== request.identity.executionId) throw new Error("Legacy attempt audit omits a recorded request");
+ }
+ } else if (event.type === "attempt") {
+ fields(event, ["type", "executionId", "attemptId"]);
+ if (!validId(event.executionId) || !validId(event.attemptId) || this.limits.maxAttemptsPerExecution === undefined) throw new Error("Invalid attempt admission");
+ if (this.attempts.has(event.attemptId)) throw new EvalAdmissionError("Eval attempt already admitted");
+ if (now >= Date.parse(this.limits.deadlineAt)) throw new EvalAdmissionError("Eval campaign deadline reached");
+ if ([...this.attempts.values()].filter(id => id === event.executionId).length >= this.limits.maxAttemptsPerExecution) throw new EvalAdmissionError("Eval campaign attempt cap reached");
+ } else if (event.type === "reserve") {
+ fields(event.identity, ["executionId", "attemptId", "phase", "runId"]);
+ if (!validId(event.requestId) || !validId(event.identity.executionId) || !validId(event.identity.attemptId)
+ || event.identity.runId !== undefined && !validId(event.identity.runId)
+ || !["setup", "agent", "judge", "feedback_review", "tool", "infrastructure"].includes(event.identity.phase)) throw new Error("Invalid request identity");
+ if (this.limits.maxAttemptsPerExecution !== undefined && this.attempts.get(event.identity.attemptId) !== event.identity.executionId) throw new EvalAdmissionError("Eval request has no admitted attempt");
+ if (this.requests.has(event.requestId)) throw new Error("Request already reserved");
+ if (amount(event.maxUsd) <= 0n) throw new Error("Request reservation must be positive");
+ const current = this.snapshot();
+ if (now >= Date.parse(this.limits.deadlineAt)) throw new EvalAdmissionError("Eval campaign deadline reached");
+ if (this.requests.size >= this.limits.maxRequests) throw new EvalAdmissionError("Eval campaign request cap reached");
+ if (current.inflight >= this.limits.maxConcurrent) throw new EvalAdmissionError("Eval campaign concurrency cap reached");
+ if (current.conflictingRequests) throw new EvalAdmissionError("Eval cost receipts conflict; admission stopped");
+ if (amount(current.committedUsd) + amount(event.maxUsd) > amount(this.limits.maxUsd)) throw new EvalAdmissionError("Eval campaign spending cap reached");
+ } else if (event.type === "observe") {
+ const o = event.observation;
+ fields(o, ["observationId", "requestId", "source", "provider", "model", "providerRequestId", "currency", "route", "classification", "costUsd", "pricingVersion", "tokens", "final"]);
+ if (o.tokens) fields(o.tokens, ["input", "cachedRead", "cachedWrite", "output", "reasoning"]);
+ if (!this.requests.has(o.requestId) || !validId(o.observationId) || !validId(o.provider)
+ || o.model !== undefined && !validId(o.model) || o.providerRequestId !== undefined && !validId(o.providerRequestId)
+ || o.currency !== "USD" || !Object.hasOwn(ranks, o.classification) || typeof o.final !== "boolean"
+ || !["provider", "gateway", "native_otlp", "heartbeat", "infrastructure", "invoice"].includes(o.source)
+ || !["direct", "braintrust", "other_gateway", "subscription", "infrastructure"].includes(o.route)) throw new Error("Invalid cost observation");
+ if (o.classification === "unknown" ? o.costUsd !== undefined : o.costUsd === undefined) throw new Error("Cost classification requires an explicit amount or unknown");
+ if (o.costUsd !== undefined) amount(o.costUsd);
+ if ((o.classification === "estimated" || o.classification === "allocated") && !validId(o.pricingVersion)) throw new Error("Estimate requires pricing provenance");
+ if (o.tokens && (Object.values(o.tokens).some(value => !Number.isSafeInteger(value) || Number(value) < 0)
+ || o.tokens.reasoning !== undefined && o.tokens.output !== undefined && o.tokens.reasoning > o.tokens.output)) throw new Error("Invalid normalized token usage");
+ const previous = this.observations.get(o.observationId);
+ if (previous && digest(previous) !== digest(o)) throw new Error("Observation identity conflict");
+ if (o.providerRequestId) {
+ const request = this.providerRequests.get(`${o.provider}:${o.providerRequestId}`);
+ if (request && request !== o.requestId) throw new Error("Provider request belongs to another reservation; reconcile identities before import");
+ }
+ } else if (event.type === "finish") {
+ if (!this.requests.has(event.requestId) || !["succeeded", "failed", "cancelled"].includes(event.outcome)) throw new Error("Invalid request completion");
+ } else throw new Error("Unknown ledger event");
+ }
+ apply(event: Event) {
+ if (event.type === "attempt_policy") {
+ this.limits.maxAttemptsPerExecution = event.maxAttemptsPerExecution;
+ for (const attempt of event.priorAttempts) this.attempts.set(attempt.attemptId, attempt.executionId);
+ } else if (event.type === "attempt") this.attempts.set(event.attemptId, event.executionId);
+ else if (event.type === "reserve") this.requests.set(event.requestId, { identity: event.identity, reserved: amount(event.maxUsd), observations: [], finished: false });
+ else if (event.type === "observe") {
+ const o = event.observation;
+ if (this.observations.has(o.observationId)) return;
+ this.observations.set(o.observationId, o);
+ this.requests.get(o.requestId)!.observations.push(o);
+ if (o.providerRequestId) this.providerRequests.set(`${o.provider}:${o.providerRequestId}`, o.requestId);
+ } else if (event.type === "finish") this.requests.get(event.requestId)!.finished = true;
+ }
+}
+
+export async function readEvalCostLedger(file: string) {
+ const raw = await readFile(file, "utf8");
+ if (!raw.endsWith("\n")) throw new Error("Eval ledger has an incomplete durable record");
+ const lines = raw.trimEnd().split("\n").map(value => JSON.parse(value) as Line);
+ let previous = "", state: EvalSpendState | undefined;
+ for (const [index, line] of lines.entries()) {
+ const { hash, ...body } = line;
+ if (line.seq !== index || line.previous !== previous || digest(body) !== hash || !Number.isFinite(Date.parse(line.at))) throw new Error("Eval ledger integrity check failed");
+ if (index === 0) {
+ if (line.event.type !== "campaign") throw new Error("Eval ledger lacks campaign limits");
+ state = new EvalSpendState(line.event.limits);
+ } else {
+ if (line.event.type === "attempt_policy" && line.event.previousJournalHash !== previous) throw new Error("Legacy attempt audit checkpoint changed");
+ state!.validate(line.event, Date.parse(line.at));
+ state!.apply(line.event);
+ }
+ previous = hash;
+ }
+ return { state: state!, sequence: lines.length, previous };
+}
+
+export async function openEvalCostLedger(file: string, limits: EvalSpendLimits) {
+ await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
+ const lockPath = `${file}.lock`;
+ let lock: FileHandle;
+ try { lock = await open(lockPath, "wx", 0o600); }
+ catch { throw new Error("Eval ledger writer lock unavailable; inspect prior writer before recovery"); }
+ let handle: FileHandle | undefined;
+ try {
+ await lock.writeFile(`${process.pid}\n`);
+ await lock.sync();
+ let state = new EvalSpendState(limits), sequence = 0, previous = "";
+ try {
+ const loaded = await readEvalCostLedger(file);
+ if (digest(loaded.state.limits) !== digest(limits)) throw new Error("Eval campaign limits changed during resume");
+ ({ state, sequence, previous } = loaded);
+ } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
+ handle = await open(file, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
+ const write = async (event: Event, at = Date.now()) => {
+ const body = { seq: sequence, at: new Date(at).toISOString(), previous, event };
+ const hash = digest(body);
+ await handle!.appendFile(`${JSON.stringify({ ...body, hash })}\n`);
+ await handle!.sync();
+ previous = hash;
+ sequence++;
+ };
+ if (!sequence) await write({ type: "campaign", limits });
+ let queue = Promise.resolve(), closed = false, poisoned = false;
+ const append = (event: Event) => {
+ if (closed) return Promise.reject(new Error("Eval ledger writer unavailable"));
+ const operation = queue.then(async () => {
+ if (poisoned) throw new Error("Eval ledger writer unavailable");
+ const admittedAt = Date.now();
+ if (event.type === "attempt_policy" && event.previousJournalHash !== previous) throw new Error("Legacy attempt audit checkpoint changed");
+ state.validate(event, admittedAt);
+ try { await write(event, admittedAt); } catch (error) { poisoned = true; throw error; }
+ state.apply(event);
+ });
+ queue = operation.catch(() => {});
+ return operation;
+ };
+ return {
+ adoptLegacyAttemptPolicy: (audit: EvalLegacyAttemptAudit) => append({ ...structuredClone(audit), type: "attempt_policy" }),
+ admitAttempt: (executionId: string, attemptId: string) => append({ type: "attempt", executionId, attemptId }),
+ reserve: (requestId: string, identity: EvalChargeIdentity, maxUsd: string) => append({ type: "reserve", requestId, identity: structuredClone(identity), maxUsd }),
+ observe: (observation: EvalCostObservation) => append({ type: "observe", observation: structuredClone(observation) }),
+ finish: (requestId: string, outcome: "succeeded" | "failed" | "cancelled") => append({ type: "finish", requestId, outcome }),
+ snapshot: () => state.snapshot(),
+ async close() {
+ if (closed) return;
+ closed = true;
+ await queue;
+ try { await handle!.close(); } finally { await lock.close(); await rm(lockPath); }
+ },
+ };
+ } catch (error) {
+ await handle?.close();
+ await lock.close();
+ await rm(lockPath);
+ throw error;
+ }
+}
diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts
index 96976b8b9b..2913bd9645 100644
--- a/tests/runner-e2e/dashboard-regenerate.ts
+++ b/tests/runner-e2e/dashboard-regenerate.ts
@@ -2,6 +2,7 @@ import path from "node:path";
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import { runnerMatrix } from "./catalog.js";
import { renderRunnerE2EDashboard } from "./dashboard.js";
+import { campaignFeedbackReviewText } from "./feedback-dispositions.js";
import {
buildRunnerCampaign,
canonicalExecutionId,
@@ -25,6 +26,10 @@ interface PublishedCampaign {
generatedAt: string;
expected: string[];
results: PublishedResult[];
+ accounting?: RunnerE2ECampaign["accounting"];
+ operations?: RunnerE2ECampaign["operations"];
+ feedbackReview?: RunnerE2ECampaign["feedbackReview"];
+ braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"];
}
async function relativeFiles(root: string, current = root): Promise<string[]> {
@@ -57,6 +62,7 @@ export async function regenerateRunnerDashboard(input: {
historyFile?: string | null;
outputDirectory?: string;
evidenceHrefPrefix?: string;
+ feedbackReviewFile?: string;
}) {
const bundle = path.resolve(input.bundle);
const outputDirectory = path.resolve(input.outputDirectory ?? bundle);
@@ -93,6 +99,12 @@ export async function regenerateRunnerDashboard(input: {
generatedAt: normalized.generatedAt,
expected,
results,
+ accounting: normalized.accounting,
+ operations: normalized.operations,
+ braintrustExperiment: normalized.braintrustExperiment,
+ feedbackReview: input.feedbackReviewFile
+ ? JSON.parse(await readFile(input.feedbackReviewFile, "utf8"))
+ : normalized.feedbackReview,
});
// A rendering refresh must not inherit the renderer's checkout or CI event.
const retainedSource = results.find((result) =>
@@ -168,10 +180,21 @@ export async function regenerateRunnerDashboard(input: {
evidenceErrors: normalized.results[index]?.evidenceErrors ?? [],
})),
};
+ const summary = await readFile(path.join(bundle, "summary.md"), "utf8").then(text => text.replace(
+ /^(Feedback across retained attempts:.*?); [0-9]+ distinct items awaiting review;/m, "$1;",
+ )).catch((error: NodeJS.ErrnoException) => {
+ if (error.code === "ENOENT") return undefined;
+ throw error;
+ });
+ const reviewStatus = /^(?:[0-9]+ distinct feedback items await review\.|Feedback reviewed by (?:human|agent) at ).*$/m;
+ const refreshedSummary = summary === undefined ? undefined : reviewStatus.test(summary)
+ ? summary.replace(reviewStatus, () => campaignFeedbackReviewText(campaign))
+ : `${summary.trimEnd()}\n\n## Campaign feedback review\n\n${campaignFeedbackReviewText(campaign)}\n`;
await mkdir(outputDirectory, { recursive: true });
await Promise.all([
writeFile(path.join(outputDirectory, "index.html"), dashboard, "utf8"),
writeFile(path.join(outputDirectory, "dashboard.html"), dashboard, "utf8"),
+ ...(refreshedSummary === undefined ? [] : [writeFile(path.join(outputDirectory, "summary.md"), refreshedSummary, "utf8")]),
writeFile(
path.join(outputDirectory, "normalized-results.json"),
`${JSON.stringify(upgraded, null, 2)}\n`,
@@ -192,7 +215,7 @@ async function main() {
);
if (!bundleArgument) {
throw new Error(
- "Usage: pnpm test:e2e:runner:dashboard -- <published-bundle-directory> [--history <history.json>]",
+ "Usage: pnpm test:e2e:runner:dashboard -- <published-bundle-directory> [--history <history.json>] [--feedback-review <review.json>]",
);
}
const historyIndex = arguments_.indexOf("--history");
@@ -200,7 +223,10 @@ async function main() {
historyIndex >= 0 ? arguments_[historyIndex + 1] : undefined;
if (historyIndex >= 0 && !historyFile)
throw new Error("--history requires a path");
- await regenerateRunnerDashboard({ bundle: bundleArgument, historyFile });
+ const reviewIndex = arguments_.indexOf("--feedback-review");
+ const feedbackReviewFile = reviewIndex >= 0 ? arguments_[reviewIndex + 1] : undefined;
+ if (reviewIndex >= 0 && !feedbackReviewFile) throw new Error("--feedback-review requires a path");
+ await regenerateRunnerDashboard({ bundle: bundleArgument, historyFile, feedbackReviewFile });
}
if (
diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts
index c949f6a07b..b8cb4ac92b 100644
--- a/tests/runner-e2e/dashboard.ts
+++ b/tests/runner-e2e/dashboard.ts
@@ -1,3 +1,4 @@
+import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, summarizeCampaignTiming, summarizeWorkerPhaseTiming } from "./campaign-timing.js";
import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js";
import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js";
import { reportBrowserScript } from "./report-browser.js";
@@ -8,6 +9,10 @@ import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js";
import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js";
import { STARTUP_REUSE_LAYERS, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js";
import { renderFirstTaskDetails } from "./first-task-report.js";
+import { summarizeCampaignFeedback } from "./feedback.js";
+import { renderCampaignFeedbackReview } from "./feedback-dispositions.js";
+import { renderEvalOperationsReport } from "./operations-report.js";
+import { renderBraintrustExperiment } from "./braintrust-experiment.js";
import {
discoverReportCatalog,
type ReportExecution,
@@ -20,6 +25,7 @@ import type {
} from "./types.js";
import {
aggregateCampaignBilling,
+ billingCoverageLabel,
summarizeExecutionBilling,
} from "./billing.js";
@@ -269,6 +275,7 @@ function renderCase(
? `<div class="billing-strip" aria-label="Billing for ${html(execution.id)}">
<div><span>Tokens</span><strong>${html(tokenLabel(billing.llm.inputTokens, billing.llm))} in · ${html(tokenLabel(billing.llm.outputTokens, billing.llm))} out</strong><small>${html(tokenLabel(billing.llm.cachedInputTokens, billing.llm))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered</small></div>
<div><span>LLM spend</span><strong>${html(llmCostLabel(billing.reportedCostUsd, billing.llm))}</strong><small>${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced</small></div>
+ ${(billing.llm.runsWithEstimatedCost ?? 0) > 0 ? `<div><span>LLM estimate</span><strong>${html(billingCoverageLabel(usdLabel(billing.llm.estimatedCostUsd ?? 0), billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount))}</strong><small>Estimated from captured usage</small></div>` : ""}
<div><span>Execution</span><strong>${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}</strong><small>${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}</small></div>
${billing.assistant ? `<div><span>External assistant</span><strong>${html(usdLabel(billing.assistant.estimatedCostUsd))} est.</strong><small>${html(billing.assistant.model)} · ${billing.assistant.requests} requests · ${html(tokenLabel(billing.assistant.inputTokens + billing.assistant.cachedInputTokens))} in / ${html(tokenLabel(billing.assistant.outputTokens))} out</small></div>` : ""}
</div>`
@@ -301,6 +308,9 @@ function renderCase(
<summary><span>Test details</span><span class="case-detail-summary">${matcherResults.length ? `${passedMatchers}/${matcherResults.length} checks${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded"}${entry ? ` · ${html(durationLabel(entry.result.durationMs))}` : ""}</span></summary>
${passed ? outcome : ""}
${billingStrip}
+ ${entry ? `<p class="detail">Worker phase timing: ${entry.result.workerTiming ? html(entry.result.workerTiming.status === "unavailable" ? `unavailable (${entry.result.workerTiming.reason})` : entry.result.workerTiming.status) : "not captured"}. ${entry.result.workerTiming?.status === "partial" ? "Only completed phases have observed durations." : ""}</p>` : ""}
+ ${entry?.result.feedback ? `<p class="detail">Feedback: ${entry.result.feedback.complaints} complaints · ${entry.result.feedback.suggestions} suggestions · ${entry.result.feedback.review.length} distinct items; see campaign feedback review. Saved-feedback collection: ${html(entry.result.feedback.collection)}; invocation coverage: ${html(entry.result.feedback.invocationCollection)}. Tool receipts: ${entry.result.feedback.invocations.failed} failed · ${entry.result.feedback.invocations.unknown} unresolved. Full bodies: access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.</p>` : entry ? `<p class="detail">Feedback was not captured for this attempt.</p>` : ""}
+ ${entry?.result.tracing ? `<p class="detail">Braintrust worker export: ${html(entry.result.tracing.status)}; native configuration: ${html(entry.result.tracing.nativeConfiguration)}. ${entry.result.tracing.finalization ? `Supervisor outcome: ${html(entry.result.tracing.finalization.outcome)}; finalization export: ${html(entry.result.tracing.finalization.status)}.` : "Supervisor finalization was not captured."} Trace ${html(entry.result.tracing.traceId)} in project ${html(entry.result.tracing.projectId)}. Indexing and native-span correlation require independent verification.</p>` : ""}
${links}
${renderFirstTaskDetails(entry?.result)}
<code class="execution-id">${html(execution.id)}</code>
@@ -324,7 +334,7 @@ function renderCase(
${latencyRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Event</th><th>Timing verdict</th><th>Possible latency</th><th>Target</th><th>Resolution</th></tr></thead><tbody>${latencyRows}</tbody></table></div>` : ""}
${startupReuseRows ? `<div class="matcher-wrap"><table class="matchers"><caption>Startup reuse evidence</caption><thead><tr><th>Run</th><th>Warmth</th><th>Environment/lease</th><th>Workspace</th><th>Runner</th><th>Provider process</th><th>Provider session</th><th>Provider launches</th></tr></thead><tbody>${startupReuseRows}</tbody></table></div>` : ""}
${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Result</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}
- ${entry ? `<details class="usage"><summary>Usage and billing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}</pre></details>` : ""}
+ ${entry ? `<details class="usage"><summary>Usage, billing and timing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null, workerTiming: entry.result.workerTiming ?? null }, null, 2))}</pre></details>` : ""}
</details>
</article>`;
}
@@ -342,7 +352,7 @@ function renderSuiteResults(input: {
? `<div class="suite-summary" aria-label="${html(suite.label)} current campaign summary">
<div><span>Pass rate</span><strong>${summary.selected > 0 ? ((summary.passed / summary.selected) * 100).toFixed(1) : "0.0"}%</strong><small>${summary.passed}/${summary.selected} passed${summary.incomplete ? ` · ${summary.incomplete} incomplete` : ""}</small></div>
<div><span>Tokens</span><strong>${html(tokenLabel(summary.billing.llm.totalTokens, summary.billing.llm))}</strong><small>${html(tokenLabel(summary.billing.llm.inputTokens, summary.billing.llm))} input · ${html(tokenLabel(summary.billing.llm.outputTokens, summary.billing.llm))} output</small></div>
- <div><span>Known spend</span><strong>${html(summary.billing.llm.runsWithReportedCost > 0 || summary.billing.estimatedRuntimeCostUsd > 0 || summary.billing.judge ? `${usdLabel(summary.billing.observedAndEstimatedCostUsd)}${summary.billing.testsWithCompleteBilling < summary.billing.testCount ? " (partial)" : ""}` : "Unavailable")}</strong><small>reported LLM + runtime${summary.billing.judge ? " + judge" : ""} estimate</small></div>
+ <div><span>Known spend</span><strong>${html(summary.billing.llm.runsWithReportedCost > 0 || (summary.billing.llm.runsWithEstimatedCost ?? 0) > 0 || summary.billing.estimatedRuntimeCostUsd > 0 || summary.billing.judge || summary.billing.assistant ? `${usdLabel(summary.billing.observedAndEstimatedCostUsd)}${summary.billing.testsWithCompleteBilling < summary.billing.testCount ? " (partial)" : ""}` : "Unavailable")}</strong><small>reported LLM + model/runtime estimates${summary.billing.judge ? " + judge" : ""} estimate</small></div>
<div><span>Agent time</span><strong>${html(durationLabel(summary.billing.agentRunDurationMs))}</strong><small>${html(durationLabel(summary.billing.leaseDurationMs))} lease</small></div>
<div><span>Execution</span><strong>${summary.executed}/${summary.selected}</strong><small>${summary.retries} retries · cleanup ${summary.cleanupPassed ? "passed" : "failed"}</small></div>
</div>`
@@ -386,15 +396,13 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
!entry.valid && isIncompleteFirstTaskResult(entry.result, entry.errors),
).length;
const failed = input.expected.length - passed - incomplete;
- const totalDuration = selectedEntries.reduce(
- (total, entry) => total + entry.result.durationMs,
- 0,
- );
+ const timing = summarizeCampaignTiming(selectedEntries.map(entry => entry.result), input.campaign?.accounting);
const screenshotCount = selectedEntries.reduce(
(total, entry) => total + resolveScreenshots(entry).length,
0,
);
- const campaignBilling = aggregateCampaignBilling(
+ const campaignFeedback = summarizeCampaignFeedback(input.campaign?.accounting?.attempts ?? input.entries.map(entry => entry.result));
+ const campaignBilling = input.campaign?.billing ?? aggregateCampaignBilling(
selectedEntries.map((entry) => entry.result),
);
const suites = [
@@ -834,7 +842,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
<div class="metric"><strong>${passed}/${input.expected.length}</strong><span>Passed</span></div>
<div class="metric"><strong>${failed}</strong><span>Failed</span></div>
${incomplete ? `<div class="metric"><strong>${incomplete}</strong><span>Incomplete</span></div>` : ""}
- <div class="metric"><strong>${html(durationLabel(totalDuration))}</strong><span>Test time</span></div>
+ <div class="metric"><strong>${html(campaignAttemptTimeLabel(timing))}</strong><span>Summed attempt time</span></div>
</div>
<button class="gallery-launch" type="button" data-gallery-open ${screenshotCount === 0 ? "disabled" : ""}>${screenshotCount === 0 ? "Visual evidence · workflow artifact only" : `View gallery · ${screenshotCount}`}</button>
</div>
@@ -842,18 +850,23 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", false)}
<details class="campaign-billing-details"><summary>Campaign usage and spend</summary>
<section class="billing-overview" aria-label="Campaign billing summary">
+ <div class="billing-metric"><strong>${campaignFeedback.complaints} / ${campaignFeedback.suggestions}</strong><span>Complaints / suggestions · retained attempts</span><small>${campaignFeedback.review.length} distinct items · ${campaignFeedback.incomplete} attempts with incomplete collection · ${campaignFeedback.failedSubmissions} failed submissions · ${campaignFeedback.unknownSubmissions} unresolved submissions</small></div>
+ ${input.campaign?.accounting ? `<div class="billing-metric"><strong>${input.campaign.accounting.attempts.length}</strong><span>${input.campaign.accounting.scope === "retained_attempts" ? "Retained attempts" : "Selected results only"}</span><small>${input.campaign.accounting.missingAttempts} missing attempts · ${input.campaign.accounting.excludedArtifacts} out-of-scope artifacts</small></div>` : ""}
<div class="billing-metric"><strong>${html(tokenLabel(campaignBilling.llm.inputTokens, campaignBilling.llm))}</strong><span>Input tokens</span></div>
<div class="billing-metric"><strong>${html(tokenLabel(campaignBilling.llm.outputTokens, campaignBilling.llm))}</strong><span>Output tokens</span></div>
<div class="billing-metric"><strong>${html(tokenLabel(campaignBilling.llm.cachedInputTokens, campaignBilling.llm))}</strong><span>Cached tokens</span></div>
<div class="billing-metric"><strong>${html(llmCostLabel(campaignBilling.reportedLlmCostUsd, campaignBilling.llm))}</strong><span>LLM reported subtotal</span></div>
+ ${(campaignBilling.llm.runsWithEstimatedCost ?? 0) > 0 ? `<div class="billing-metric"><strong>${html(billingCoverageLabel(usdLabel(campaignBilling.estimatedLlmCostUsd ?? 0), campaignBilling.llm.runsWithEstimatedCost ?? 0, campaignBilling.llm.runCount))}</strong><span>LLM estimated subtotal</span></div>` : ""}
<div class="billing-metric"><strong>${html(usdLabel(campaignBilling.estimatedRuntimeCostUsd))}</strong><span>Daytona list estimate</span></div>
${campaignBilling.judge ? `<div class="billing-metric"><strong>${html(usdLabel(campaignBilling.judge.estimatedCostUsd))}</strong><span>Judge estimate · ${campaignBilling.judge.attempts} attempts · ${campaignBilling.judge.attemptsWithUnknownUsage} unknown usage</span><small>${html(tokenLabel(campaignBilling.judge.inputTokens))} in / ${html(tokenLabel(campaignBilling.judge.outputTokens))} out · $${campaignBilling.judge.reservedCostUsd.toFixed(6)} reserved</small></div>` : ""}
${campaignBilling.assistant ? `<div class="billing-metric"><strong>${html(usdLabel(campaignBilling.assistant.estimatedCostUsd))}</strong><span>External assistant estimate · ${campaignBilling.assistant.requests} requests</span><small>${html(tokenLabel(campaignBilling.assistant.inputTokens + campaignBilling.assistant.cachedInputTokens))} in / ${html(tokenLabel(campaignBilling.assistant.outputTokens))} out</small></div>` : ""}
+ <div class="billing-metric"><strong>${html(durationLabel(timing.observedWindowMs))}</strong><span>Observed attempt window</span><small>${timing.retainedAttempts} retained attempts · ${timing.missingAttempts} missing attempts</small></div>
+ <p class="billing-note">${html(campaignTimingScope)}</p>
<div class="billing-metric"><strong>${html(durationLabel(campaignBilling.agentRunDurationMs))}</strong><span>Agent execution time</span></div>
<div class="billing-metric"><strong>${html(durationLabel(campaignBilling.leaseDurationMs))}</strong><span>Daytona lease time</span></div>
<div class="billing-metric"><strong>${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}</strong><span>Runs provider-priced</span></div>
- <p class="billing-note">Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.</p>
- </section></details>
+ <p class="billing-note">Campaign costs include retained attempts, including failed retries. Result cards show the selected attempt. Provider-reported amounts and model estimates are separate; unpriced or unavailable runs are excluded, never counted as free. Coverage describes captured usage; unobserved setup, subscriptions, tools, CI and storage require additional accounting. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.</p>
+ </section>${renderWorkerPhaseTiming(summarizeWorkerPhaseTiming(selectedEntries.map(entry => entry.result), input.campaign?.accounting))}${input.campaign?.operations ? renderEvalOperationsReport(input.campaign.operations) : ""}${input.campaign ? renderCampaignFeedbackReview(input.campaign) + renderBraintrustExperiment(input.campaign) : ""}</details>
</section>
<div class="results-heading" id="results"><h2 data-results-title>All results</h2><span data-results-description></span></div>
<div data-feature-coverage>${renderJourneyFeatureCoverage(catalog, entryById)}</div>
diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts
index 4ca6a41d37..8d3daf45be 100644
--- a/tests/runner-e2e/daytona-image-content.ts
+++ b/tests/runner-e2e/daytona-image-content.ts
@@ -58,8 +58,12 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [
"packages/paperclip-runner/test/fixtures/copilot-profile-v47-identity.json",
"packages/paperclip-runner/test/fixtures/copilot-profile-v48-identity.json",
"packages/paperclip-runner/test/fixtures/copilot-profile-v49-identity.json",
+ "packages/paperclip-runner/test/fixtures/copilot-profile-v50-identity.json",
+ "packages/paperclip-runner/test/fixtures/copilot-profile-v51-identity.json",
+ "packages/paperclip-runner/test/fixtures/copilot-profile-v52-identity.json",
"packages/paperclip-runner/test/fixtures/copilot-profile-v54-identity.json",
"packages/paperclip-runner/test/fixtures/copilot-profile-v55-identity.json",
+ "packages/paperclip-runner/test/fixtures/copilot-profile-v57-identity.json",
"packages/paperclip-runner/test/fixtures/copilot-profile-v58-identity.json",
"packages/paperclip-runner/scripts/materialize-cursor-distribution.mjs",
"packages/paperclip-runner/scripts/cursor-runtime-patch.mjs",
diff --git a/tests/runner-e2e/eval-finalization-relay.test.ts b/tests/runner-e2e/eval-finalization-relay.test.ts
new file mode 100644
index 0000000000..d2453eb78e
--- /dev/null
+++ b/tests/runner-e2e/eval-finalization-relay.test.ts
@@ -0,0 +1,40 @@
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it, vi } from "vitest";
+import { EVAL_FINALIZATION_LIMITS, finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js";
+
+it("meters finalization after the worker has closed without changing the original trace or failed outcome", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-finalization-ingestion-"));
+ try {
+ const projectId = "11111111-1111-4111-8111-111111111111";
+ const input = { receipt: { projectId, traceId: "a".repeat(32), spanId: "b".repeat(16), status: "pending" as const,
+ verification: "unverified" as const, nativeConfiguration: "configured" as const },
+ identity: { campaignId: "campaign", attemptId: "attempt", executionId: "fixture.local", provider: "fixture", model: "fixture", environment: "local" },
+ provenance: { suiteId: "fixture", suiteDefinitionHash: "c".repeat(64), profileId: "fixture", caseId: "case", sourceRevision: "d".repeat(40) },
+ result: { startedAt: new Date(Date.now() - 60_000).toISOString(), status: "failed" as const, cleanup: "failed" as const } };
+ const before = structuredClone(input);
+ let sentBytes = 0;
+ const upstream = vi.fn<typeof fetch>(async (url, init) => {
+ expect(String(url)).toBe("https://api.braintrust.dev/otel/v1/traces");
+ expect(init!.headers).toMatchObject({ authorization: "Bearer upstream-private" });
+ const body = Buffer.from(init!.body as Uint8Array); sentBytes = body.length;
+ expect(body.toString()).toContain("paperclip.eval.finalization");
+ expect(body.includes(Buffer.from(input.receipt.traceId, "hex"))).toBe(true);
+ return new Response(null, { status: 200, headers: { "content-type": "application/x-protobuf" } });
+ });
+ const file = path.join(root, "finalization.jsonl");
+ const result = await finalizeEvalAttemptThroughRelay({ projectId, apiKey: "upstream-private" }, input, file, upstream);
+ expect(result).toMatchObject({ status: "accepted", outcome: "failed" });
+ expect(input).toEqual(before);
+ const raw = await readFile(file, "utf8"), records = raw.trim().split("\n").map(line => JSON.parse(line));
+ expect(records[0]).toMatchObject({ campaignId: "campaign", attemptId: "attempt", limits: EVAL_FINALIZATION_LIMITS });
+ expect(records.at(-1)).toMatchObject({ type: "closed", admittedRequests: 1, admittedBytes: sentBytes, acceptedRequests: 1 });
+ expect(raw).not.toContain("upstream-private");
+ expect((await stat(file)).mode & 0o777).toBe(0o600);
+ expect(await finalizeEvalAttemptThroughRelay({ projectId, apiKey: "upstream-private" }, input, file, upstream))
+ .toMatchObject({ status: "failed", errorCode: "finalization_ingestion_unavailable" });
+ expect(upstream).toHaveBeenCalledTimes(1);
+ expect(await readFile(file, "utf8")).toBe(raw);
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
diff --git a/tests/runner-e2e/eval-finalization-relay.ts b/tests/runner-e2e/eval-finalization-relay.ts
new file mode 100644
index 0000000000..db0c2119f8
--- /dev/null
+++ b/tests/runner-e2e/eval-finalization-relay.ts
@@ -0,0 +1,24 @@
+import { randomBytes } from "node:crypto";
+import { finalizeEvalAttemptTrace, type EvalBraintrustConfig, type EvalFinalizationReceipt } from "./braintrust.js";
+import { startEvalOtlpRelay } from "./eval-otlp-relay.js";
+
+export const EVAL_FINALIZATION_LIMITS = { maxRequests: 1, maxRequestBytes: 64 * 1024,
+ maxTotalBytes: 64 * 1024, maxConcurrent: 1, maxExpandedBytes: 1024 * 1024, maxRecords: 256 } as const;
+
+export async function finalizeEvalAttemptThroughRelay(config: EvalBraintrustConfig,
+ input: Parameters<typeof finalizeEvalAttemptTrace>[1], receiptPath: string, transport?: typeof fetch): Promise<EvalFinalizationReceipt> {
+ let relay: Awaited<ReturnType<typeof startEvalOtlpRelay>> | undefined;
+ let receipt: EvalFinalizationReceipt = { spanId: randomBytes(8).toString("hex"), status: "failed",
+ outcome: input.result.status === "passed" && input.result.cleanup === "passed" ? "passed" : "failed", verification: "unverified" };
+ try {
+ relay = await startEvalOtlpRelay({ projectId: config.projectId, apiKey: config.apiKey,
+ campaignId: input.identity.campaignId, attemptId: input.identity.attemptId, receiptPath,
+ deadlineAt: Date.now() + 30_000, limits: EVAL_FINALIZATION_LIMITS, transport });
+ receipt = await finalizeEvalAttemptTrace({ ...config, apiKey: relay.token, endpointBaseUrl: relay.baseUrl }, input);
+ } catch { receipt.errorCode = "finalization_ingestion_unavailable"; }
+ finally {
+ try { await relay?.close(); }
+ catch { receipt.status = "failed"; receipt.errorCode = "finalization_ingestion_persistence_failed"; }
+ }
+ return receipt;
+}
diff --git a/tests/runner-e2e/eval-gateway.test.ts b/tests/runner-e2e/eval-gateway.test.ts
new file mode 100644
index 0000000000..4fa02239ca
--- /dev/null
+++ b/tests/runner-e2e/eval-gateway.test.ts
@@ -0,0 +1,315 @@
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { connect } from "node:net";
+import { afterEach, expect, it, vi } from "vitest";
+import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js";
+import { beginEvalAgentPhase, createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js";
+import { evalRequestReservation, EvalUsageStream, forwardEvalRequest, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js";
+const directories: string[] = [], closes: Array<() => Promise<void>> = [];
+afterEach(async () => { for (const close of closes.splice(0).reverse()) await close(); for (const dir of directories.splice(0)) await rm(dir, { recursive: true, force: true }); });
+const identity = { executionId: "hello", attemptId: "attempt", phase: "agent" as const };
+const route: EvalGatewayRoute = { provider: "openai", mode: "braintrust", apiKey: "private-upstream-fixture", projectId: "11111111-1111-4111-8111-111111111111", endpointName: "OPENAI_API_KEY",
+ maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 2000,
+ price: { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.invalid/prices", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" } };
+const usage = { object: "response", status: "completed", id: "provider-response", model: "fixture", usage: { input_tokens: 10, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 5, output_tokens_details: { reasoning_tokens: 3 } } };
+const response = () => Response.json(usage, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY", "x-bt-cached": "MISS", "x-bt-request-id": "gateway-request", "x-bt-span-id": "gateway-span" } });
+async function fixture(maxUsd = "1", maxConcurrent = 2) {
+ const dir = await mkdtemp(path.join(tmpdir(), "eval-forward-test-")); directories.push(dir);
+ const file = path.join(dir, "ledger.jsonl"), deadlineAt = Date.now() + 30_000;
+ const ledger = await openEvalCostLedger(file, { campaignId: "campaign", maxUsd, deadlineAt: new Date(deadlineAt).toISOString(), maxRequests: 10, maxConcurrent });
+ closes.push(() => ledger.close());
+ const receipts: EvalGatewayReceipt[] = [];
+ return { ledger, file, deadlineAt, identity, route, receipts, payload: { model: "fixture", input: "hello", max_output_tokens: 500 },
+ persist: async (receipt: EvalGatewayReceipt) => { receipts.push(structuredClone(receipt)); }, onHeaders: vi.fn(), onChunk: vi.fn(async () => {}) };
+}
+it("fsyncs reservation before dispatch, pins gateway routing, and replaces reservation with one estimate", async () => {
+ const f = await fixture();
+ let captured: RequestInit | undefined;
+ let reservedBeforeDispatch = false;
+ const transport = vi.fn<typeof fetch>(async (_url, init) => { captured = init; reservedBeforeDispatch = (await readEvalCostLedger(f.file)).state.snapshot().inflight === 1; return response(); });
+ const result = await forwardEvalRequest({ ...f, transport });
+ expect(reservedBeforeDispatch).toBe(true);
+ expect(transport.mock.calls[0]![0]).toBe("https://gateway.braintrust.dev/responses");
+ expect(captured).toMatchObject({ redirect: "error", headers: { authorization: `Bearer ${route.apiKey}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", "x-bt-project-id": route.projectId, "x-bt-endpoint-name": route.endpointName, "x-bt-parent": `project_id:${route.projectId}` } });
+ expect(JSON.parse(captured!.body as string)).toMatchObject({ max_output_tokens: 100, service_tier: "default" });
+ expect(result).toMatchObject({ outcome: "succeeded", costUsd: "0.000068000", final: true, usageComplete: true, gatewaySpanId: "gateway-span" });
+ expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000068000", heldUsd: "0.000000000", inflight: 0 });
+ expect(await readFile(f.file, "utf8")).not.toContain(route.apiKey);
+});
+it("stops at the campaign cap before any upstream request", async () => {
+ const f = await fixture("0.002"), transport = vi.fn<typeof fetch>();
+ expect(evalRequestReservation(route)).toBe("0.003000000");
+ await expect(forwardEvalRequest({ ...f, transport })).rejects.toThrow("spending cap");
+ expect(transport).not.toHaveBeenCalled();
+});
+it.each(["direct", "braintrust"] as const)("retains request-policy cost provenance without changing the %s cache policy", async mode => {
+ const f = await fixture();
+ const pricedRoute = { ...route, mode, price: { ...route.price, model: "gpt-5.6-sol", inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4",
+ outputUsdPerMillion: "20", cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10" } };
+ const payload = { ...f.payload, model: pricedRoute.price.model, prompt_cache_retention: "24h", stream: true };
+ const body = { ...usage, model: pricedRoute.price.model, usage: { ...usage.usage, input_tokens_details: { cache_write_tokens: 6, cached_tokens: 2 } } };
+ const transport = vi.fn<typeof fetch>(async () => new Response(`data: ${JSON.stringify({ type: "response.completed", response: body })}\n\n`, {
+ headers: { "content-type": "text/event-stream", "x-bt-used-endpoint": "OPENAI_API_KEY", "x-bt-cached": "MISS" },
+ }));
+ const receipt = await forwardEvalRequest({ ...f, route: pricedRoute, payload, transport });
+ expect(JSON.parse(transport.mock.calls[0]![1]!.body as string)).toEqual({ ...payload, max_output_tokens: 100, service_tier: "default" });
+ expect(receipt).toMatchObject({ outcome: "succeeded", final: true, costUsd: "0.000138800", usage: { cacheWrite30m: 6, cacheWrite12h: 0,
+ cacheWriteAttribution: { source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl: "30m" } } });
+ expect(f.receipts.at(-1)).toEqual(receipt);
+ expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000138800", heldUsd: "0.000000000", inflight: 0 });
+});
+it("retains cancelled and provider-error requests as unknown, including repeated retries", async () => {
+ const f = await fixture();
+ const first = await forwardEvalRequest({ ...f, transport: async () => { throw new Error(route.apiKey); } });
+ const second = await forwardEvalRequest({ ...f, transport: async () => Response.json({ error: "private-provider-message" }, { status: 429 }) });
+ expect(first).toMatchObject({ outcome: "failed", errorCode: "request_failed", costUsd: null });
+ expect(second).toMatchObject({ outcome: "failed", errorCode: "upstream_rejected", costUsd: null });
+ expect(first.requestId).not.toBe(second.requestId);
+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.006000000", knownUsd: "0.000000000", unknownRequests: 2 });
+ expect(JSON.stringify(f.receipts)).not.toMatch(/private-upstream|private-provider-message/);
+});
+it.each([
+ { model: "other" }, { model: "fixture", background: true }, { model: "fixture", service_tier: "priority" },
+ { model: "fixture", tools: [{ type: "web_search" }] }, { model: "fixture", input: [{ type: "input_image", image_url: "https://image.invalid" }] },
+ { model: "fixture", speed: "fast" }, { model: "fixture", inference_geo: "us" },
+])("rejects unpriced request shapes before admission", async payload => {
+ const f = await fixture(), transport = vi.fn<typeof fetch>();
+ await expect(forwardEvalRequest({ ...f, payload, transport })).rejects.toThrow();
+ expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0);
+});
+it("does not dispatch when the private pre-request receipt cannot be persisted", async () => {
+ const f = await fixture(), transport = vi.fn<typeof fetch>();
+ await expect(forwardEvalRequest({ ...f, persist: async () => { throw new Error("disk unavailable"); }, transport })).rejects.toThrow("disk unavailable");
+ expect(transport).not.toHaveBeenCalled();
+ expect(f.ledger.snapshot()).toMatchObject({ unknownRequests: 1, heldUsd: "0.003000000" });
+});
+it("parses actual fragmented SSE without summing cumulative counters", async () => {
+ const f = await fixture();
+ const wire = `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "hello 👋" })}\r\n\r\ndata: ${JSON.stringify({ type: "response.completed", response: usage })}\r\n\r\ndata: [DONE]\r\n\r\n`;
+ const bytes = Buffer.from(wire);
+ const body = new ReadableStream<Uint8Array>({ start(controller) { for (let i = 0; i < bytes.length; i += 3) controller.enqueue(bytes.subarray(i, i + 3)); controller.close(); } });
+ const result = await forwardEvalRequest({ ...f, transport: async () => new Response(body, { headers: { "content-type": "text/event-stream", "x-bt-used-endpoint": "OPENAI_API_KEY" } }) });
+ expect(result).toMatchObject({ costUsd: "0.000068000", final: true });
+ expect(f.onChunk.mock.calls.length).toBeGreaterThan(1);
+ const truncated = new EvalUsageStream(() => {});
+ expect(() => truncated.push(Buffer.from('data: {"type":"response.completed"}'), true)).toThrow("incomplete_usage_stream");
+});
+it.each([{ "x-bt-cached": "HIT" }, { "x-bt-used-endpoint": "FOREIGN" }, { "x-bt-failover-to": "fixture/FOREIGN" }])("rejects cached/failover/foreign route evidence without releasing the reservation", async headers => {
+ const f = await fixture();
+ const result = await forwardEvalRequest({ ...f, transport: async () => Response.json(usage, { headers: Object.fromEntries(Object.entries({ "x-bt-used-endpoint": "OPENAI_API_KEY", ...headers }).filter((pair): pair is [string, string] => pair[1] !== undefined)) }) });
+ expect(result).toMatchObject({ outcome: "failed", errorCode: "gateway_route_not_qualified", costUsd: null });
+ expect(f.onChunk).not.toHaveBeenCalled();
+ expect(f.ledger.snapshot().heldUsd).toBe("0.003000000");
+});
+it("uses the loopback gateway through real HTTP while withholding upstream credentials from the client", async () => {
+ const f = await fixture();
+ const transport = vi.fn<typeof fetch>(async () => response());
+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());
+ const denied = await fetch(`${gateway.baseUrl}/v1/responses`, { method: "POST", body: "{}" });
+ expect(denied.status).toBe(401); expect(transport).not.toHaveBeenCalled();
+ const accepted = await fetch(`${gateway.baseUrl}/v1/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}`, "content-type": "application/json" }, body: JSON.stringify(f.payload) });
+ expect(accepted.status).toBe(200); expect(await accepted.json()).toEqual(usage);
+ expect(gateway.token).not.toBe(route.apiKey);
+ expect(accepted.headers.get("x-paperclip-eval-request-id")).toBeTruthy();
+ expect([...accepted.headers.values()].join(" ")).not.toContain(route.apiKey);
+});
+it.each([
+ { status: 400, type: "text/plain", body: "private fixture rejection" },
+ { status: 503, type: "text/html", body: "<html>private fixture rejection</html>" },
+ { status: 400, type: "application/json", body: "invalid JSON fixture rejection" },
+])("finishes non-JSON upstream errors without truncating the client response: $status $type", async error => {
+ const f = await fixture();
+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(new Response(error.body, { status: error.status, headers: { "content-type": error.type } }))
+ .mockImplementation(async () => response());
+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());
+ const request = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });
+ const rejected = await request();
+ expect(rejected.status).toBe(error.status);
+ expect(await rejected.text()).toBe(error.body);
+ expect(f.receipts.at(-1)).toMatchObject({ outcome: "failed", errorCode: "upstream_rejected", costUsd: null, usageComplete: false });
+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });
+ expect(JSON.stringify(f.receipts)).not.toContain(error.body);
+ expect(await (await request()).json()).toEqual(usage);
+ expect(transport).toHaveBeenCalledTimes(2);
+});
+it("still fails successful responses with invalid usage JSON and holds their reservation", async () => {
+ const f = await fixture();
+ const result = await forwardEvalRequest({ ...f, transport: async () => new Response("invalid successful response", {
+ headers: { "content-type": "application/json", "x-bt-used-endpoint": "OPENAI_API_KEY" },
+ }) });
+ expect(result).toMatchObject({ outcome: "failed", errorCode: "request_failed", costUsd: null });
+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });
+});
+
+it("reserves the highest cache-write and long-context rates before dispatch", () => {
+ expect(evalRequestReservation({ ...route, price: { ...route.price, cacheWrite12hUsdPerMillion: "5",
+ longContext: { aboveInputTokens: 100, rates: { inputUsdPerMillion: "4", cachedReadUsdPerMillion: "2", cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10", outputUsdPerMillion: "15" } } } })).toBe("0.011500000");
+ expect(() => evalRequestReservation({ ...route, price: { ...route.price, cacheWrite12hUsdPerMillion: undefined } })).toThrow("both provider cache-write");
+});
+it("propagates cancellation to an in-flight upstream request and retains its reservation", async () => {
+ const f = await fixture(), controller = new AbortController();
+ const transport = vi.fn<typeof fetch>(async (_url, init) => {
+ controller.abort();
+ init!.signal!.throwIfAborted();
+ throw new Error("unreachable");
+ });
+ const receipt = await forwardEvalRequest({ ...f, signal: controller.signal, transport });
+ expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled", final: false, costUsd: null });
+ expect(transport).toHaveBeenCalledOnce();
+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });
+});
+it("does not release a reservation for an otherwise final response missing cache-write usage", async () => {
+ const f = await fixture(), body = structuredClone(usage);
+ delete (body.usage.input_tokens_details as any).cache_write_tokens;
+ const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json(body, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } }) });
+ expect(receipt).toMatchObject({ outcome: "succeeded", usageComplete: false, final: true, costUsd: null });
+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1 });
+});
+it.each([{ version: "prices with spaces" }, { model: "model with spaces" }])("rejects invalid ledger price identities before reserving or dispatching", async invalid => {
+ const f = await fixture(), transport = vi.fn<typeof fetch>();
+ const price = { ...route.price, ...invalid };
+ await expect(forwardEvalRequest({ ...f, route: { ...route, price }, payload: { model: price.model }, transport })).rejects.toThrow("price identity");
+ expect(transport).not.toHaveBeenCalled();
+ expect(f.ledger.snapshot().requests).toHaveLength(0);
+});
+it("retains invalid reasoning counters privately and finishes an unknown charge", async () => {
+ const f = await fixture(), body = structuredClone(usage);
+ body.usage.output_tokens_details.reasoning_tokens = 6;
+ const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json(body, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } }) });
+ expect(receipt).toMatchObject({ outcome: "succeeded", errorCode: "invalid_usage", usageComplete: false, final: true, costUsd: null,
+ usage: { tokens: { output: 5, reasoning: 6 } } });
+ expect(f.receipts.at(-1)).toEqual(receipt);
+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });
+ const journal = await readFile(f.file, "utf8");
+ expect(journal).not.toContain('"reasoning":6');
+ expect(journal).toContain('"type":"finish"');
+});
+it("retains malformed provider identifiers without losing the final receipt or ledger finish", async () => {
+ const f = await fixture();
+ const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json({ ...usage, model: "unexpected model name" },
+ { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY", "x-request-id": "request with spaces" } }) });
+ expect(receipt).toMatchObject({ providerRequestId: "request with spaces", usage: { model: "unexpected model name" }, final: true, costUsd: null });
+ expect(f.receipts.at(-1)).toEqual(receipt);
+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });
+});
+it("admits later HTTP requests after temporary concurrency exhaustion", async () => {
+ const f = await fixture("1", 1);
+ let admitted!: () => void, release!: () => void;
+ const started = new Promise<void>(resolve => { admitted = resolve; });
+ const blocked = new Promise<void>(resolve => { release = resolve; });
+ const transport = vi.fn<typeof fetch>(async () => { admitted(); await blocked; return response(); });
+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());
+ const request = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });
+ const first = request(); await started;
+ try {
+ const denied = await request();
+ expect(denied.status).toBe(429); expect(transport).toHaveBeenCalledOnce();
+ } finally { release(); }
+ expect(await (await first).json()).toEqual(usage);
+ const later = await request();
+ expect(later.status).toBe(200); expect(await later.json()).toEqual(usage);
+ expect(transport).toHaveBeenCalledTimes(2);
+ expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 0 });
+});
+it("rejects malformed, interrupted and unqualified uploads without disabling later valid requests", async () => {
+ const f = await fixture(), transport = vi.fn<typeof fetch>(async () => response());
+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());
+ const headers = { authorization: `Bearer ${gateway.token}` };
+ for (const body of ["{", JSON.stringify({ model: "other" }), JSON.stringify({ model: "fixture", tools: [{ type: "web_search" }] })]) {
+ const rejected = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers, body });
+ expect(rejected.status).toBe(400); await rejected.text();
+ }
+ await new Promise<void>((resolve, reject) => {
+ const socket = connect(Number(new URL(gateway.baseUrl).port), "127.0.0.1");
+ socket.on("error", reject); socket.on("close", () => resolve()); socket.resume();
+ socket.on("connect", () => socket.end(`POST /responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ${gateway.token}\r\nContent-Length: 100\r\n\r\n{`));
+ });
+ expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0);
+ const valid = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers, body: JSON.stringify(f.payload) });
+ expect(valid.status).toBe(200); expect(await valid.json()).toEqual(usage);
+ expect(transport).toHaveBeenCalledOnce(); expect(f.ledger.snapshot().requests).toHaveLength(1);
+});
+it("still disables the HTTP gateway after a receipt persistence failure", async () => {
+ const f = await fixture(), transport = vi.fn<typeof fetch>(async () => response());
+ const gateway = await startEvalGateway({ ...f, transport, persist: async () => { throw new Error("storage failure"); } }); closes.push(() => gateway.close());
+ for (let i = 0; i < 2; i++) {
+ const denied = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });
+ expect(denied.status).toBe(503); await denied.text();
+ }
+ expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(1);
+});
+it("applies the request deadline while waiting for downstream backpressure", async () => {
+ const f = await fixture();
+ const receipt = await forwardEvalRequest({ ...f, route: { ...f.route, timeoutMs: 50 }, transport: async () => response(),
+ onChunk: async (_chunk, signal) => {
+ signal.throwIfAborted();
+ await new Promise<void>((_resolve, reject) => signal.addEventListener("abort", () => reject(signal.reason), { once: true }));
+ },
+ });
+ expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled" });
+ expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 1, heldUsd: "0.003000000" });
+});
+it("forwards Claude's Messages beta query without admitting unrelated endpoints or queries", async () => {
+ const f = await fixture();
+ const anthropicRoute: EvalGatewayRoute = { ...route, provider: "anthropic", mode: "direct", price: { provider: "anthropic", model: "fixture",
+ version: "fixture", sourceUrl: "https://provider.invalid/prices", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10",
+ cacheWrite5mUsdPerMillion: "2.5", cacheWrite1hUsdPerMillion: "4" } };
+ const body = { type: "message", model: "fixture", usage: { input_tokens: 2, output_tokens: 3, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } };
+ const transport = vi.fn<typeof fetch>(async () => Response.json(body));
+ const gateway = await startEvalGateway({ ...f, route: anthropicRoute, transport }); closes.push(() => gateway.close());
+ const request = (pathname: string) => fetch(gateway.baseUrl + pathname, { method: "POST", headers: { "x-api-key": gateway.token, "anthropic-beta": "fixture-beta" }, body: JSON.stringify({ model: "fixture", messages: [] }) });
+ expect((await request("/v1/messages?beta=true&extra=true")).status).toBe(404);
+ const accepted = await request("/v1/messages?beta=true");
+ expect(accepted.status).toBe(200); expect(await accepted.json()).toEqual(body);
+ expect(transport).toHaveBeenCalledOnce();
+ expect(transport.mock.calls[0]![0]).toBe("https://api.anthropic.com/v1/messages?beta=true");
+ expect(transport.mock.calls[0]![1]!.headers).toMatchObject({ "anthropic-beta": "fixture-beta" });
+ expect(f.ledger.snapshot()).toMatchObject({ unknownRequests: 0, inflight: 0 });
+});
+
+it("attributes setup at admission, preserves it during execution, and blocks missing phase evidence", async () => {
+ const f = await fixture();
+ const phasePath = path.join(path.dirname(f.file), "phase.json");
+ await createEvalAttemptPhase(phasePath, identity);
+ const env = { PAPERCLIP_E2E_GATEWAY_PHASE_PATH: phasePath, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: identity.attemptId };
+ let started!: () => void, release!: () => void, calls = 0;
+ const firstStarted = new Promise<void>(resolve => { started = resolve; });
+ const continuation = new Promise<void>(resolve => { release = resolve; });
+ const gateway = await startEvalGateway({ ...f, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity),
+ transport: async () => {
+ calls++;
+ if (calls === 1) { started(); await continuation; }
+ return Response.json({ ...usage, id: `provider-${calls}` }, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } });
+ },
+ });
+ closes.push(() => gateway.close());
+ const send = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });
+ const first = send();
+ await firstStarted;
+ await beginEvalAgentPhase(identity.executionId, env);
+ release();
+ expect((await first).status).toBe(200);
+ expect((await send()).status).toBe(200);
+ expect(f.ledger.snapshot().requests.map(request => request.phase)).toEqual(["setup", "agent"]);
+ await writeFile(phasePath, JSON.stringify({ version: 1, ...identity, attemptId: "foreign-attempt" }));
+ expect((await send()).status).toBe(503);
+ expect(calls).toBe(2);
+ expect((await readEvalCostLedger(f.file)).state.snapshot().requests.map(request => request.phase)).toEqual(["setup", "agent"]);
+});
+
+it("refuses missing or unreviewed attribution checkpoints before request dispatch", async () => {
+ const f = await fixture(), transport = vi.fn<typeof fetch>();
+ const phasePath = path.join(path.dirname(f.file), "missing-phase.json");
+ const gateway = await startEvalGateway({ ...f, transport, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity) });
+ closes.push(() => gateway.close());
+ const response = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });
+ expect(response.status).toBe(503);
+ expect(transport).not.toHaveBeenCalled();
+ expect(f.ledger.snapshot().requests).toEqual([]);
+ await expect(beginEvalAgentPhase(identity.executionId, { PAPERCLIP_E2E_GATEWAY_BASE_URL: gateway.baseUrl })).rejects.toThrow("Missing eval phase");
+ await createEvalAttemptPhase(phasePath, identity);
+ await expect(createEvalAttemptPhase(phasePath, identity)).rejects.toThrow();
+ await writeFile(phasePath, JSON.stringify({ version: 1, ...identity, providerResponse: "private" }));
+ await expect(readEvalAttemptPhase(phasePath, identity)).rejects.toThrow("Invalid eval phase");
+});
diff --git a/tests/runner-e2e/eval-gateway.ts b/tests/runner-e2e/eval-gateway.ts
new file mode 100644
index 0000000000..7252890866
--- /dev/null
+++ b/tests/runner-e2e/eval-gateway.ts
@@ -0,0 +1,256 @@
+import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
+import { once } from "node:events";
+import { usdToUnits } from "../../packages/shared/src/money.js";
+import { estimateProviderCost, ProviderUsageAccumulator, type EvalModelPrice, type ProviderUsageReceipt } from "./provider-usage.js";
+import { EvalAdmissionError, validEvalLedgerId, type EvalChargeIdentity, type openEvalCostLedger } from "./cost-ledger.js";
+
+type Ledger = Awaited<ReturnType<typeof openEvalCostLedger>>;
+export interface EvalGatewayRoute {
+ provider: "openai" | "anthropic";
+ mode: "direct" | "braintrust";
+ apiKey: string;
+ projectId?: string;
+ endpointName?: string;
+ price: EvalModelPrice;
+ maximumInputTokens: number;
+ maximumOutputTokens: number;
+ timeoutMs: number;
+}
+export interface EvalGatewayReceipt {
+ requestId: string; status?: number; route: "direct" | "braintrust";
+ gatewayRequestId?: string; gatewaySpanId?: string; providerRequestId?: string;
+ endpoint?: string; cache?: string; failoverFrom?: string; failoverTo?: string; errorOrigin?: string;
+ outcome: "succeeded" | "failed" | "cancelled"; errorCode?: string;
+ usageComplete: boolean; final: boolean; costUsd: string | null;
+ usage?: ProviderUsageReceipt;
+}
+const money = (n: bigint) => `${n / 1_000_000_000n}.${String(n % 1_000_000_000n).padStart(9, "0")}`;
+const safeHeader = (value: string | null) => value && /^[A-Za-z0-9_.:/ -]{1,512}$/.test(value) ? value : undefined;
+const row = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+class EvalRequestError extends Error {}
+
+export function evalRequestReservation(route: EvalGatewayRoute): string {
+ if (![route.maximumInputTokens, route.maximumOutputTokens, route.timeoutMs].every(n => Number.isSafeInteger(n) && n > 0)
+ || route.timeoutMs > 300_000 || route.provider !== route.price.provider || !validEvalLedgerId(route.price.version) || !validEvalLedgerId(route.price.model)) throw new Error("Invalid eval gateway limits or price identity");
+ const parse = (rate: string) => {
+ if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(rate)) throw new Error("Invalid eval gateway price");
+ return usdToUnits(rate);
+ };
+ if (route.price.longContext && (!Number.isSafeInteger(route.price.longContext.aboveInputTokens) || route.price.longContext.aboveInputTokens < 1)) throw new Error("Invalid long-context threshold");
+ const tiers = [route.price, ...(route.price.longContext ? [route.price.longContext.rates] : [])];
+ let maximumInputRate = 0n, maximumOutputRate = 0n;
+ for (const tier of tiers) {
+ const writes = route.provider === "openai" ? [tier.cacheWrite30mUsdPerMillion, tier.cacheWrite12hUsdPerMillion] : [tier.cacheWrite5mUsdPerMillion, tier.cacheWrite1hUsdPerMillion];
+ if (writes.some(rate => rate === undefined)) throw new Error("Reservation requires both provider cache-write prices");
+ for (const rate of [tier.inputUsdPerMillion, tier.cachedReadUsdPerMillion, ...writes]) {
+ const units = parse(rate!); if (units > maximumInputRate) maximumInputRate = units;
+ }
+ const output = parse(tier.outputUsdPerMillion); if (output > maximumOutputRate) maximumOutputRate = output;
+ }
+ const numerator = BigInt(route.maximumInputTokens) * maximumInputRate + BigInt(route.maximumOutputTokens) * maximumOutputRate;
+ return money((numerator + 999_999n) / 1_000_000n);
+}
+
+function boundedPayload(value: unknown, route: EvalGatewayRoute) {
+ const body = row(value);
+ if (body.model !== route.price.model) throw new EvalRequestError("unqualified_model");
+ if (body.background === true || body.mcp_servers !== undefined || body.service_tier !== undefined && !["auto", "default", "standard_only"].includes(String(body.service_tier))) throw new EvalRequestError("unqualified_billing_mode");
+ if (body.speed !== undefined && body.speed !== "standard" || body.inference_geo !== undefined) throw new EvalRequestError("unqualified_billing_mode");
+ if (body.tools !== undefined && (!Array.isArray(body.tools) || body.tools.some(tool => {
+ const type = row(tool).type;
+ return route.provider === "openai" ? !["function", "custom"].includes(String(type)) : type !== undefined && type !== "custom";
+ }))) throw new EvalRequestError("unqualified_hosted_tool");
+ const inspect = (item: unknown): void => {
+ if (Array.isArray(item)) { item.forEach(inspect); return; }
+ if (!item || typeof item !== "object") return;
+ const r = row(item);
+ if (["input_image", "input_audio", "input_video", "image", "audio", "document", "video", "input_file"].includes(String(r.type))) throw new EvalRequestError("unqualified_multimodal_input");
+ Object.values(r).forEach(inspect);
+ };
+ inspect(body);
+ const field = route.provider === "openai" ? "max_output_tokens" : "max_tokens";
+ if (body[field] !== undefined && (!Number.isSafeInteger(body[field]) || Number(body[field]) < 1)) throw new EvalRequestError("invalid_output_limit");
+ return { ...body, [field]: Math.min(Number(body[field] ?? route.maximumOutputTokens), route.maximumOutputTokens),
+ service_tier: route.provider === "openai" ? "default" : "standard_only" };
+}
+
+export class EvalUsageStream {
+ private readonly decoder = new TextDecoder();
+ private pending = "";
+ constructor(private readonly observe: (value: unknown) => void) {}
+ push(bytes: Uint8Array, final = false) {
+ this.pending += this.decoder.decode(bytes, { stream: !final });
+ if (this.pending.length > 4 * 1024 * 1024) throw new Error("usage_frame_too_large");
+ for (;;) {
+ const match = /\r?\n\r?\n/.exec(this.pending);
+ if (!match) break;
+ const frame = this.pending.slice(0, match.index); this.pending = this.pending.slice(match.index + match[0].length);
+ const data = frame.split(/\r?\n/).filter(line => line.startsWith("data:")).map(line => line.slice(5).trimStart()).join("\n");
+ if (data && data !== "[DONE]") this.observe(JSON.parse(data));
+ }
+ if (final && this.pending.trim()) throw new Error("incomplete_usage_stream");
+ }
+}
+
+export async function forwardEvalRequest(input: {
+ ledger: Ledger; identity: EvalChargeIdentity; route: EvalGatewayRoute; payload: unknown;
+ deadlineAt: number; signal?: AbortSignal; anthropicBeta?: string; anthropicBetaQuery?: boolean;
+ onHeaders: (status: number, headers: Record<string, string>) => void;
+ onChunk: (chunk: Uint8Array, signal: AbortSignal) => Promise<void>;
+ persist: (receipt: EvalGatewayReceipt) => Promise<void>;
+ transport?: typeof fetch;
+}): Promise<EvalGatewayReceipt> {
+ const { route, ledger } = input;
+ let body: ReturnType<typeof boundedPayload>;
+ try { body = boundedPayload(input.payload, route); }
+ catch (error) { throw error instanceof EvalRequestError ? error : new EvalRequestError("invalid_request_payload"); }
+ const reservation = evalRequestReservation(route);
+ if (!route.apiKey || /[\r\n]/.test(route.apiKey)) throw new Error("missing_gateway_credential");
+ if (route.mode === "braintrust" && (!/^[0-9a-f-]{36}$/i.test(route.projectId ?? "") || !/^[A-Za-z0-9_-]+$/.test(route.endpointName ?? ""))) throw new Error("invalid_project_provider");
+ if (input.signal?.aborted) throw new EvalRequestError("request_cancelled");
+ if (Date.now() >= input.deadlineAt) throw new EvalAdmissionError("request_deadline_reached");
+ const requestId = randomUUID();
+ const receipt: EvalGatewayReceipt = { requestId, route: route.mode, outcome: "failed", usageComplete: false, final: false, costUsd: null };
+ const usage = new ProviderUsageAccumulator(route.provider, body);
+ await ledger.reserve(requestId, input.identity, reservation);
+ const abort = new AbortController();
+ const signal = AbortSignal.any([abort.signal, AbortSignal.timeout(Math.max(1, Math.min(route.timeoutMs, input.deadlineAt - Date.now()))), ...(input.signal ? [input.signal] : [])]);
+ let response: Response | undefined;
+ try {
+ await input.persist(receipt);
+ const headers: Record<string, string> = { "content-type": "application/json", "accept-encoding": "identity" };
+ if (route.mode === "braintrust") Object.assign(headers, {
+ authorization: `Bearer ${route.apiKey}`, "x-bt-project-id": route.projectId!, "x-bt-endpoint-name": route.endpointName!,
+ "x-bt-parent": `project_id:${route.projectId}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store",
+ });
+ else if (route.provider === "openai") headers.authorization = `Bearer ${route.apiKey}`;
+ else headers["x-api-key"] = route.apiKey;
+ if (route.provider === "anthropic") {
+ headers["anthropic-version"] = "2023-06-01";
+ if (input.anthropicBeta && /^[A-Za-z0-9, _-]{1,2048}$/.test(input.anthropicBeta)) headers["anthropic-beta"] = input.anthropicBeta;
+ }
+ const base = route.mode === "braintrust" ? "https://gateway.braintrust.dev" : route.provider === "openai" ? "https://api.openai.com/v1" : "https://api.anthropic.com/v1";
+ const endpoint = route.provider === "openai" ? "/responses" : input.anthropicBetaQuery ? "/messages?beta=true" : "/messages";
+ response = await (input.transport ?? fetch)(base + endpoint, { method: "POST", headers, body: JSON.stringify(body), redirect: "error", signal });
+ receipt.status = response.status;
+ receipt.gatewayRequestId = safeHeader(response.headers.get("x-bt-request-id"));
+ receipt.gatewaySpanId = safeHeader(response.headers.get("x-bt-span-id"));
+ receipt.providerRequestId = safeHeader(response.headers.get("x-request-id") ?? response.headers.get("request-id"));
+ receipt.endpoint = safeHeader(response.headers.get("x-bt-used-endpoint"));
+ receipt.cache = safeHeader(response.headers.get("x-bt-cached"));
+ receipt.failoverFrom = safeHeader(response.headers.get("x-bt-failover-from"));
+ receipt.failoverTo = safeHeader(response.headers.get("x-bt-failover-to"));
+ receipt.errorOrigin = safeHeader(response.headers.get("x-bt-error-origin"));
+ await input.persist(receipt);
+ if (route.mode === "braintrust" && (receipt.cache === "HIT" || receipt.failoverFrom || receipt.failoverTo
+ || response.ok && receipt.endpoint !== route.endpointName)) throw new Error("gateway_route_not_qualified");
+ const contentType = response.headers.get("content-type") ?? "application/json";
+ input.onHeaders(response.status, { "content-type": contentType, "cache-control": "no-store", "x-paperclip-eval-request-id": requestId });
+ const reader = response.body?.getReader();
+ const stream = contentType.startsWith("text/event-stream") ? new EvalUsageStream(value => usage.observe(value)) : undefined;
+ const chunks: Uint8Array[] = []; let size = 0;
+ try {
+ while (reader) {
+ const next = await reader.read(); if (next.done) break;
+ size += next.value.length; if (size > 32 * 1024 * 1024) throw new Error("response_too_large");
+ if (stream) stream.push(next.value); else chunks.push(next.value);
+ await input.onChunk(next.value, signal);
+ }
+ if (stream) stream.push(new Uint8Array(), true);
+ else if (size) {
+ try { usage.observe(JSON.parse(Buffer.concat(chunks).toString("utf8"))); }
+ catch (error) { if (response.ok) throw error; }
+ }
+ } finally { await reader?.cancel().catch(() => {}); }
+ receipt.outcome = response.ok ? "succeeded" : "failed";
+ if (!response.ok) receipt.errorCode = "upstream_rejected";
+ } catch (error) {
+ receipt.outcome = signal.aborted ? "cancelled" : "failed";
+ const code = error instanceof Error ? error.message : "";
+ receipt.errorCode = ["gateway_route_not_qualified", "response_too_large", "usage_frame_too_large", "incomplete_usage_stream"].includes(code) ? code : signal.aborted ? "request_cancelled" : "request_failed";
+ } finally {
+ abort.abort();
+ await response?.body?.cancel().catch(() => {});
+ const observed = usage.receipt();
+ receipt.usage = observed;
+ receipt.usageComplete = observed.complete;
+ receipt.final = observed.final;
+ receipt.costUsd = estimateProviderCost(observed, route.price);
+ const tokens = { ...observed.tokens };
+ if (tokens.reasoning !== undefined && tokens.output !== undefined && tokens.reasoning > tokens.output) {
+ delete tokens.reasoning;
+ receipt.errorCode ??= "invalid_usage";
+ }
+ await input.persist(receipt);
+ await ledger.observe({ observationId: `${requestId}:usage`, requestId, source: route.mode === "braintrust" ? "gateway" : "provider",
+ provider: route.provider,
+ ...(validEvalLedgerId(observed.model) ? { model: observed.model } : {}),
+ ...(validEvalLedgerId(receipt.providerRequestId) ? { providerRequestId: receipt.providerRequestId } : {}),
+ currency: "USD", route: route.mode, classification: receipt.costUsd === null ? "unknown" : "estimated",
+ ...(receipt.costUsd !== null ? { costUsd: receipt.costUsd, pricingVersion: route.price.version } : {}), tokens, final: observed.final });
+ await ledger.finish(requestId, receipt.outcome);
+ }
+ return receipt;
+}
+
+async function requestJson(req: IncomingMessage) {
+ const chunks: Buffer[] = []; let size = 0;
+ for await (const chunk of req) {
+ size += chunk.length; if (size > 4 * 1024 * 1024) throw new Error("request_too_large");
+ chunks.push(chunk);
+ }
+ return JSON.parse(Buffer.concat(chunks).toString("utf8"));
+}
+
+export async function startEvalGateway(input: Omit<Parameters<typeof forwardEvalRequest>[0], "payload" | "onHeaders" | "onChunk" | "signal" | "anthropicBeta" | "anthropicBetaQuery"> & {
+ resolveIdentity?: () => Promise<EvalChargeIdentity>;
+}) {
+ const token = randomBytes(32).toString("hex"), pending = new Set<Promise<void>>(), controllers = new Set<AbortController>();
+ let closed = false, poisoned = false;
+ const respond = (res: ServerResponse, status: number, code: string) => {
+ if (res.destroyed) return;
+ if (!res.headersSent) { res.writeHead(status, { "content-type": "application/json" }); res.end(JSON.stringify({ error: { type: "eval_gateway", message: code } })); }
+ else res.destroy();
+ };
+ const server = createServer((req, res) => {
+ const operation = (async () => {
+ const auth = String(req.headers.authorization ?? "").replace(/^Bearer /, "") || String(req.headers["x-api-key"] ?? "");
+ if (Buffer.byteLength(auth) !== Buffer.byteLength(token) || !timingSafeEqual(Buffer.from(auth), Buffer.from(token))) { respond(res, 401, "unauthorized"); return; }
+ const endpoint = input.route.provider === "openai" ? "responses" : "messages";
+ if (closed || poisoned) { respond(res, 503, "gateway_unavailable"); return; }
+ const paths = [`/${endpoint}`, `/v1/${endpoint}`];
+ if (input.route.provider === "anthropic") paths.push("/messages?beta=true", "/v1/messages?beta=true");
+ if (req.method !== "POST" || !paths.includes(req.url ?? "")) { respond(res, 404, "unqualified_endpoint"); return; }
+ const abort = new AbortController(); controllers.add(abort);
+ const onClose = () => { if (!res.writableEnded) abort.abort(); }; res.on("close", onClose);
+ try {
+ let payload: unknown;
+ try { payload = await requestJson(req); }
+ catch { respond(res, 400, "invalid_request_body"); return; }
+ const identity = input.resolveIdentity ? await input.resolveIdentity() : input.identity;
+ if (identity.executionId !== input.identity.executionId || identity.attemptId !== input.identity.attemptId) throw new Error("Invalid eval request attribution");
+ const receipt = await forwardEvalRequest({ ...input, identity, payload, signal: abort.signal,
+ anthropicBeta: typeof req.headers["anthropic-beta"] === "string" ? req.headers["anthropic-beta"] : undefined,
+ anthropicBetaQuery: input.route.provider === "anthropic" && req.url?.endsWith("?beta=true"),
+ onHeaders: (status, headers) => { res.writeHead(status, headers); },
+ onChunk: async (chunk, signal) => { if (!res.write(chunk)) await once(res, "drain", { signal }); },
+ });
+ if (receipt.outcome === "succeeded" || receipt.errorCode === "upstream_rejected") res.end();
+ else respond(res, 502, receipt.errorCode ?? "request_failed");
+ } catch (error) {
+ if (error instanceof EvalRequestError) { respond(res, 400, "invalid_request_payload"); return; }
+ if (error instanceof EvalAdmissionError) { respond(res, 429, "eval_admission_denied"); return; }
+ poisoned = true; respond(res, 503, "eval_admission_or_persistence_failed");
+ } finally { controllers.delete(abort); res.off("close", onClose); }
+ })();
+ pending.add(operation); void operation.then(() => pending.delete(operation), () => { poisoned = true; respond(res, 503, "gateway_unavailable"); pending.delete(operation); });
+ });
+ server.requestTimeout = 15_000; server.headersTimeout = 10_000;
+ server.listen(0, "127.0.0.1"); await once(server, "listening");
+ const address = server.address(); if (!address || typeof address === "string") throw new Error("Eval gateway did not bind loopback");
+ return { baseUrl: `http://127.0.0.1:${address.port}`, token,
+ async close() { closed = true; controllers.forEach(controller => controller.abort()); server.closeAllConnections();
+ await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve())); await Promise.allSettled([...pending]); },
+ };
+}
diff --git a/tests/runner-e2e/eval-otlp-payload.ts b/tests/runner-e2e/eval-otlp-payload.ts
new file mode 100644
index 0000000000..345734f73e
--- /dev/null
+++ b/tests/runner-e2e/eval-otlp-payload.ts
@@ -0,0 +1,127 @@
+import { evalOtlpSchema as root, protobuf } from "./eval-otlp-schema.js";
+
+export type EvalOtlpSignal = "traces" | "logs";
+export const isEvalProjectId = (value: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value);
+
+export function validateEvalOtlpParent(value: string, projectId: string): string {
+ if (!isEvalProjectId(projectId)) throw new Error("invalid_otlp_project");
+ if (value === `project_id:${projectId}`) return value;
+ if (value.length > 2048 || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) throw new Error("invalid_otlp_parent");
+ const bytes = Buffer.from(value, "base64");
+ if (bytes.toString("base64") !== value || !bytes.subarray(0, 3).equals(Buffer.from([4, 2, 0]))) throw new Error("invalid_otlp_parent");
+ const data = JSON.parse(bytes.subarray(3).toString("utf8"));
+ if (!data || Array.isArray(data) || Object.keys(data).sort().join(",") !== "object_id,root_span_id,row_id,span_id"
+ || data.object_id !== projectId || data.row_id !== data.span_id
+ || !/^[0-9a-f]{16}$/.test(data.span_id) || /^0+$/.test(data.span_id)
+ || !/^[0-9a-f]{32}$/.test(data.root_span_id) || /^0+$/.test(data.root_span_id)) throw new Error("invalid_otlp_parent");
+ return value;
+}
+
+function validateWire(body: Buffer, type: import("../../server/node_modules/protobufjs/index.js").Type) {
+ const reader = protobuf.Reader.create(body);
+ let fields = 0;
+ const inspect = (message: typeof type, end: number, depth: number): void => {
+ if (depth > 64) throw new Error("otlp_structure_limit");
+ const seen = new Set<number | string>();
+ while (reader.pos < end) {
+ if (++fields > 200_000) throw new Error("otlp_structure_limit");
+ const tag = reader.uint32(), id = tag >>> 3, wire = tag & 7, field = message.fieldsById[id];
+ if (!id || ![0, 1, 2, 5].includes(wire)) throw new Error("invalid_otlp_wire");
+ if (field) {
+ if (!field.repeated && seen.has(id) || field.partOf && seen.has(field.partOf.name)) throw new Error("ambiguous_otlp_wire");
+ seen.add(id); if (field.partOf) seen.add(field.partOf.name);
+ const nested = field.resolvedType instanceof protobuf.Type ? field.resolvedType : undefined;
+ const expected = nested || ["string", "bytes"].includes(field.type) ? 2
+ : ["double", "fixed64", "sfixed64"].includes(field.type) ? 1
+ : ["float", "fixed32", "sfixed32"].includes(field.type) ? 5 : 0;
+ if (wire !== expected) throw new Error("invalid_otlp_wire");
+ if (nested) {
+ const length = reader.uint32(), next = reader.pos + length;
+ if (next > end) throw new Error("invalid_otlp_wire");
+ inspect(nested, next, depth + 1);
+ } else reader.skipType(wire);
+ } else throw new Error("unqualified_otlp_field");
+ if (reader.pos > end) throw new Error("invalid_otlp_wire");
+ }
+ };
+ inspect(type, body.length, 0);
+}
+
+export function validateEvalOtlpPayload(body: Buffer, contentType: string, signal: EvalOtlpSignal, projectId: string) {
+ const type = root.lookupType(signal === "traces" ? "TracesData" : "LogsData");
+ if (contentType !== "application/json") validateWire(body, type);
+ const decoded: unknown = contentType === "application/json" ? JSON.parse(body.toString("utf8")) : type.decode(body).toJSON();
+ if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_otlp_payload");
+ let visited = 0;
+ const inspect = (value: unknown, depth: number): void => {
+ if (++visited > 200_000 || depth > 64) throw new Error("otlp_structure_limit");
+ if (Array.isArray(value)) { value.forEach(item => inspect(item, depth + 1)); return; }
+ if (!value || typeof value !== "object") return;
+ const row = value as Record<string, unknown>;
+ if (row.key === "braintrust.parent") {
+ const parent = row.value as { stringValue?: unknown; string_value?: unknown } | undefined;
+ if (!parent || Object.keys(parent).length !== 1) throw new Error("invalid_otlp_parent");
+ const text = parent.stringValue ?? parent.string_value;
+ if (typeof text !== "string") throw new Error("invalid_otlp_parent");
+ validateEvalOtlpParent(text, projectId);
+ }
+ if (Object.hasOwn(row, "braintrust.parent")) {
+ if (typeof row["braintrust.parent"] !== "string") throw new Error("invalid_otlp_parent");
+ validateEvalOtlpParent(row["braintrust.parent"], projectId);
+ }
+ Object.values(row).forEach(item => inspect(item, depth + 1));
+ };
+ inspect(decoded, 0);
+ const canonical = Buffer.from(JSON.stringify(decoded));
+ const sizing = expandedEvalOtlpSize(decoded as Record<string, unknown>, signal);
+ return { body: contentType === "application/json" ? canonical : body, ...sizing };
+}
+
+function expandedEvalOtlpSize(decoded: Record<string, unknown>, signal: EvalOtlpSignal) {
+ const bytes = (value: unknown) => Buffer.byteLength(JSON.stringify(value));
+ const object = (value: unknown): Record<string, unknown> => {
+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid_otlp_record");
+ return value as Record<string, unknown>;
+ };
+ const take = (row: Record<string, unknown>, camel: string, snake = camel) => {
+ if (camel !== snake && Object.hasOwn(row, camel) && Object.hasOwn(row, snake)) throw new Error("ambiguous_otlp_collection");
+ const value = Object.hasOwn(row, camel) ? row[camel] : Object.hasOwn(row, snake) ? row[snake] : [];
+ if (!Array.isArray(value)) throw new Error("invalid_otlp_collection");
+ const rest = Object.fromEntries(Object.entries(row).filter(([key]) => key !== camel && key !== snake));
+ return { rows: value, rest };
+ };
+ const traces = signal === "traces";
+ const resources = take(decoded, traces ? "resourceSpans" : "resourceLogs", traces ? "resource_spans" : "resource_logs");
+ let recordCount = 0, expandedBytes = bytes(decoded);
+ for (const resource of resources.rows) {
+ const scopes = take(object(resource), traces ? "scopeSpans" : "scopeLogs", traces ? "scope_spans" : "scope_logs");
+ for (const scope of scopes.rows) {
+ const records = traces ? take(object(scope), "spans") : take(object(scope), "logRecords", "log_records");
+ const envelopeBytes = bytes({ source: resources.rest, resource: scopes.rest, scope: records.rest, record: {} }) - 2;
+ for (const record of records.rows) {
+ recordCount++;
+ expandedBytes += envelopeBytes + bytes(object(record));
+ }
+ }
+ }
+ return { recordCount, expandedBytes };
+}
+
+export function evalOtlpRejected(body: Buffer, contentType: string, signal: EvalOtlpSignal): number {
+ if (!body.length) return 0;
+ const decoded = contentType === "application/json" ? JSON.parse(body.toString("utf8"))
+ : root.lookupType("Response").decode(body).toJSON();
+ if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_otlp_response");
+ const partial = decoded.partialSuccess;
+ if (partial !== undefined && (!partial || typeof partial !== "object" || Array.isArray(partial))) throw new Error("invalid_otlp_response");
+ const rejected = partial?.[signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"] ?? partial?.rejected ?? 0;
+ if (!/^[0-9]+$/.test(String(rejected)) || !Number.isSafeInteger(Number(rejected))) throw new Error("invalid_otlp_response");
+ return Number(rejected);
+}
+
+export function evalOtlpResponse(rejected: number, contentType: string, signal: EvalOtlpSignal): Buffer {
+ if (contentType === "application/json") return Buffer.from(JSON.stringify(rejected
+ ? { partialSuccess: { [signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"]: String(rejected) } } : {}));
+ const type = root.lookupType("Response");
+ return Buffer.from(type.encode(type.fromObject(rejected ? { partialSuccess: { rejected: String(rejected) } } : {})).finish());
+}
diff --git a/tests/runner-e2e/eval-otlp-relay.test.ts b/tests/runner-e2e/eval-otlp-relay.test.ts
new file mode 100644
index 0000000000..d241934242
--- /dev/null
+++ b/tests/runner-e2e/eval-otlp-relay.test.ts
@@ -0,0 +1,258 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { mkdtemp, open, readFile, rm, stat } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createServer } from "node:http";
+import { once } from "node:events";
+import { gzipSync } from "node:zlib";
+import { setTimeout as delay } from "node:timers/promises";
+import { EVAL_OTLP_LIMITS, startEvalOtlpRelay } from "./eval-otlp-relay.js";
+import { EvalAttemptTrace, exportBraintrustAttemptParent } from "./braintrust.js";
+import { evalOtlpRejected, evalOtlpResponse } from "./eval-otlp-payload.js";
+import { evalOtlpSchema } from "./eval-otlp-schema.js";
+
+const projectId = "11111111-1111-4111-8111-111111111111", apiKey = "upstream-private-fixture";
+const protobuf = createRequire(new URL("../../server/package.json", import.meta.url))("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js");
+const message = (field: number, ...parts: Uint8Array[]) => Buffer.from(protobuf.Writer.create().uint32(field * 8 + 2).bytes(Buffer.concat(parts)).finish());
+const roots: string[] = [], closers: Array<() => Promise<void>> = [];
+afterEach(async () => { await Promise.allSettled(closers.splice(0).map(close => close()));
+ await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); vi.restoreAllMocks(); });
+async function fixture(overrides: Partial<Parameters<typeof startEvalOtlpRelay>[0]> = {}) {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-otlp-")); roots.push(root);
+ const policy = { projectId, apiKey, campaignId: "campaign", attemptId: "attempt", receiptPath: path.join(root, "receipts.jsonl"),
+ deadlineAt: Date.now() + 60_000, ...overrides };
+ const relay = await startEvalOtlpRelay(policy); closers.push(relay.close);
+ const send = (body = "{}", headers: Record<string, string> = {}, endpoint = "/v1/traces") => fetch(relay.baseUrl + endpoint, {
+ method: "POST", headers: { authorization: `Bearer ${relay.token}`, "content-type": "application/json", ...headers }, body,
+ });
+ const records = async () => (await readFile(policy.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ return { relay, policy, send, records };
+}
+
+it("forwards real attempt protobuf over HTTP using only the controller credential and retains body-free receipts", async () => {
+ const received: Array<{ authorization?: string; parent?: string; body: Buffer }> = [];
+ const server = createServer(async (req, res) => {
+ const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk));
+ received.push({ authorization: req.headers.authorization, parent: req.headers["x-bt-parent"] as string, body: Buffer.concat(chunks) });
+ res.writeHead(200, { "content-type": "application/x-protobuf" }); res.end();
+ });
+ server.listen(0, "127.0.0.1"); await once(server, "listening");
+ closers.push(async () => { server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve())); });
+ const address = server.address(); if (!address || typeof address === "string") throw new Error("fixture_bind_failed");
+ const upstream = vi.fn<typeof fetch>((url, init) => {
+ expect(String(url)).toBe("https://api.braintrust.dev/otel/v1/traces");
+ expect(init?.redirect).toBe("error");
+ return fetch(`http://127.0.0.1:${address.port}`, init);
+ });
+ const f = await fixture({ transport: upstream });
+ const trace = new EvalAttemptTrace({ projectId, apiKey: f.relay.token, endpointBaseUrl: f.relay.baseUrl },
+ { campaignId: "campaign", attemptId: "attempt", executionId: "fixture.local", provider: "fixture", model: "fixture", environment: "local" });
+ trace.phase("setup");
+ expect(await trace.export({ failed: false, runIds: [], cleanup: "passed", complaints: 0, suggestions: 0 })).toMatchObject({ status: "accepted" });
+ expect(received).toHaveLength(1);
+ expect(received[0]).toMatchObject({ authorization: `Bearer ${apiKey}`, parent: `project_id:${projectId}` });
+ expect(received[0]!.body.toString()).toContain("paperclip.eval.attempt");
+ expect(received[0]!.body.toString()).not.toContain(f.relay.token);
+ await f.relay.close();
+ const records = await f.records();
+ expect(records.map(row => row.type)).toEqual(["policy", "admitted", "outcome", "closed"]);
+ expect(records[2]).toMatchObject({ outcome: "accepted", dispatched: true, httpStatus: 200, rejectedRecords: 0 });
+ expect(records[3]).toMatchObject({ admittedRequests: 1, admittedBytes: received[0]!.body.length,
+ attemptedForwardBytes: received[0]!.body.length, acceptedRequests: 1, incompleteRequests: 0 });
+ expect(JSON.stringify(records)).not.toMatch(/upstream-private-fixture|paperclip.eval.attempt/);
+ expect((await stat(f.policy.receiptPath)).mode & 0o777).toBe(0o600);
+ await expect(startEvalOtlpRelay(f.policy)).rejects.toMatchObject({ code: "EEXIST" });
+});
+
+it("refuses token, endpoint, header and payload routing bypasses before contacting Braintrust", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));
+ const f = await fixture({ transport: upstream });
+ expect((await f.send("{}", { authorization: `Bearer ${apiKey}` })).status).toBe(401);
+ expect((await f.send("{}", {}, "/../api/project")).status).toBe(404);
+ expect((await f.send("{}", {}, "/v1/traces?target=other")).status).toBe(404);
+ expect((await f.send("{}", { "x-bt-parent": "project_name:another" })).status).toBe(400);
+ for (const attributes of [
+ { resourceSpans: [{ resource: { attributes: [{ key: "braintrust.parent", value: { stringValue: "project_id:22222222-2222-4222-8222-222222222222" } }] } }] },
+ { resourceSpans: [{ scopeSpans: [{ spans: [{ attributes: [{ key: "braintrust.parent", value: { stringValue: "experiment_id:private" } }] }] }] }] },
+ ]) expect((await f.send(JSON.stringify(attributes))).status).toBe(400);
+ expect(upstream).not.toHaveBeenCalled();
+ const parent = exportBraintrustAttemptParent({ projectId, spanId: "1234567890abcdef", traceId: "abcdef0123456789abcdef0123456789" });
+ expect((await f.send("{}", { "x-bt-parent": parent })).status).toBe(200);
+ expect(upstream.mock.calls[0]?.[1]?.headers).toMatchObject({ "x-bt-parent": parent });
+});
+
+it("counts decompressed payload bytes and refuses a compressed body above its allowance", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));
+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRequestBytes: 1024 } });
+ const body = JSON.stringify({ resourceLogs: [{ resource: { attributes: [{ key: "test", value: { stringValue: "private-content-canary" } }] } }] });
+ const request = (bytes: Buffer) => fetch(f.relay.baseUrl + "/v1/logs", { method: "POST", body: Buffer.from(bytes),
+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/json", "content-encoding": "gzip" } });
+ expect((await request(gzipSync(body))).status).toBe(200);
+ expect((await request(gzipSync(JSON.stringify({ payload: "x".repeat(2048) })))).status).toBe(400);
+ expect(upstream).toHaveBeenCalledTimes(1);
+ expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array).toString()).toBe(body);
+ const records = await f.records();
+ expect(records[1]).toMatchObject({ signal: "logs", bodyBytes: Buffer.byteLength(body), wireBytes: gzipSync(body).length });
+ expect(JSON.stringify(records)).not.toContain("private-content-canary");
+});
+
+it("rejects protobuf routing metadata and ambiguous singular fields before forwarding", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));
+ const f = await fixture({ transport: upstream });
+ const attribute = message(1, Buffer.from("braintrust.parent"));
+ const foreign = message(2, message(1, Buffer.from("project_id:22222222-2222-4222-8222-222222222222")));
+ const local = message(2, message(1, Buffer.from(`project_id:${projectId}`)));
+ const resource = message(1, message(1, attribute, foreign));
+ const bodies = [
+ message(1, resource),
+ message(1, resource, message(1)),
+ message(1, message(1, message(1, attribute, foreign, local))),
+ message(1, message(2, message(2, message(9, attribute, foreign)))),
+ message(1, message(2, message(1, message(3, attribute, foreign)))),
+ ];
+ for (const body of bodies) {
+ const response = await fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body: Buffer.from(body),
+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/x-protobuf" } });
+ expect(response.status).toBe(400);
+ }
+ expect(upstream).not.toHaveBeenCalled();
+});
+
+it("canonicalizes duplicate JSON keys and refuses conflicting parent value aliases", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));
+ const f = await fixture({ transport: upstream });
+ const foreign = "project_id:22222222-2222-4222-8222-222222222222";
+ const body = `{"resourceSpans":[{"resource":{"attributes":[{"key":"braintrust.parent","value":{"stringValue":"${foreign}","stringValue":"project_id:${projectId}"}}]}}]}`;
+ expect((await f.send(body)).status).toBe(200);
+ expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array).toString()).not.toContain(foreign);
+ const ambiguous = JSON.stringify({ resourceSpans: [{ resource: { attributes: [{ key: "braintrust.parent",
+ value: { stringValue: `project_id:${projectId}`, string_value: foreign } }] } }] });
+ expect((await f.send(ambiguous)).status).toBe(400);
+ expect(upstream).toHaveBeenCalledTimes(1);
+});
+
+it("does not dispatch when durable admission fails", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));
+ const f = await fixture({ transport: upstream });
+ const handle = await open(path.join(path.dirname(f.policy.receiptPath), "fixture"), "w");
+ const sync = vi.spyOn(Object.getPrototypeOf(handle), "sync").mockRejectedValueOnce(new Error("fixture_disk_failure"));
+ try {
+ expect((await f.send()).status).toBe(503);
+ expect((await f.send()).status).toBe(503);
+ expect(upstream).not.toHaveBeenCalled();
+ await expect(f.relay.close()).rejects.toThrow("fixture_disk_failure");
+ } finally { sync.mockRestore(); await handle.close(); }
+});
+
+it("bounds simultaneous exports without retrying the refused request", async () => {
+ let entered!: () => void, release!: () => void;
+ const active = new Promise<void>(resolve => { entered = resolve; });
+ const wait = new Promise<void>(resolve => { release = resolve; });
+ const upstream = vi.fn<typeof fetch>(async () => { entered(); await wait; return new Response(null, { status: 200 }); });
+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxConcurrent: 1 } });
+ const first = f.send(); await active;
+ try { expect((await f.send()).status).toBe(429); } finally { release(); }
+ expect((await first).status).toBe(200);
+ expect(upstream).toHaveBeenCalledTimes(1);
+});
+
+it.each(["count", "bytes"])("retains failed attempts against the %s allowance without retrying or resetting", async kind => {
+ const upstream = vi.fn<typeof fetch>(async () => new Response("private-vendor-error", { status: 503 }));
+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS,
+ ...(kind === "count" ? { maxRequests: 1 } : { maxTotalBytes: 3 }) } });
+ const first = await f.send(); expect(first.status).toBe(502); expect(await first.text()).not.toContain("private-vendor-error");
+ expect((await f.send()).status).toBe(429);
+ expect((await f.send()).status).toBe(429);
+ expect(upstream).toHaveBeenCalledTimes(1);
+ await f.relay.close();
+ const records = await f.records();
+ expect(records.filter(row => row.type === "quota_exhausted")).toHaveLength(1);
+ expect(records.find(row => row.type === "outcome")).toMatchObject({ outcome: "rejected", httpStatus: 503 });
+ expect(records.at(-1)).toMatchObject({ admittedRequests: 1, admittedBytes: 2, incompleteRequests: 1 });
+});
+
+it.each(["traces", "logs"] as const)("preserves partial %s rejection without returning upstream diagnostic text", async signal => {
+ const upstream = vi.fn<typeof fetch>(async () => Response.json({ partialSuccess: {
+ [signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"]: "2", errorMessage: "private-upstream-diagnostic" } }));
+ const f = await fixture({ transport: upstream });
+ const response = await f.send("{}", {}, `/v1/${signal}`);
+ expect(response.status).toBe(200);
+ const text = await response.text(); expect(text).not.toContain("private-upstream-diagnostic");
+ expect(evalOtlpRejected(Buffer.from(text), "application/json", signal)).toBe(2);
+ const protobuf = evalOtlpResponse(2, "application/x-protobuf", signal);
+ expect(evalOtlpRejected(protobuf, "application/x-protobuf", signal)).toBe(2);
+ expect((await f.records()).find(row => row.type === "outcome")).toMatchObject({ outcome: "partial", rejectedRecords: 2 });
+});
+
+it("aborts an in-flight export on close and retains its uncertain outcome", async () => {
+ let started!: () => void, aborted = false;
+ const active = new Promise<void>(resolve => { started = resolve; });
+ const upstream = vi.fn<typeof fetch>(async (_url, init) => {
+ started(); await new Promise<void>((_resolve, reject) => init!.signal!.addEventListener("abort", () => { aborted = true; reject(new Error("fixture_abort")); }, { once: true }));
+ return new Response(null, { status: 200 });
+ });
+ const f = await fixture({ transport: upstream });
+ const request = f.send().catch(() => undefined); await active;
+ await f.relay.close(); await request;
+ expect(aborted).toBe(true);
+ expect((await f.records()).find(row => row.type === "outcome")).toMatchObject({ outcome: "uncertain", dispatched: true });
+ await f.relay.close();
+ expect(upstream).toHaveBeenCalledTimes(1);
+});
+
+it("retires the listening endpoint at its deadline without a supervisor close", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));
+ const f = await fixture({ transport: upstream, deadlineAt: Date.now() + 120 });
+ await delay(180);
+ await expect(f.send()).rejects.toThrow();
+ await f.relay.close();
+ expect(upstream).not.toHaveBeenCalled();
+ expect((await f.records()).at(-1)).toMatchObject({ type: "closed", admittedRequests: 0 });
+});
+
+it.each(["json", "protobuf"])("meters repeated resource/scope data and full record fields in %s before dispatch", async encoding => {
+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));
+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxExpandedBytes: 40_000 } });
+ const payload = { resourceSpans: [{ resource: { attributes: [{ key: "shared", value: { stringValue: "r".repeat(3000) } }] },
+ scopeSpans: [{ scope: { name: "s".repeat(1000), version: "fixture" }, spans: Array.from({ length: 4 }, () => ({
+ name: "n".repeat(1000), traceId: "AQEBAQEBAQEBAQEBAQEBAQ==", spanId: "AgICAgICAgI=", flags: 1,
+ startTimeUnixNano: "1234567890123456789", endTimeUnixNano: "1234567890123456790",
+ events: [{ name: "e".repeat(1000), timeUnixNano: "1234567890123456789" }],
+ links: [{ traceId: "AwMDAwMDAwMDAwMDAwMDAw==", spanId: "BAQEBAQEBAQ=", flags: 1 }],
+ })) }] }] };
+ const type = evalOtlpSchema.lookupType("TracesData");
+ const body = encoding === "json" ? Buffer.from(JSON.stringify(payload)) : Buffer.from(type.encode(type.fromObject(payload)).finish());
+ const send = () => fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body: Buffer.from(body),
+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": encoding === "json" ? "application/json" : "application/x-protobuf" } });
+ expect(body.length).toBeLessThan(20_000);
+ expect((await send()).status).toBe(200);
+ expect((await send()).status).toBe(429);
+ expect(upstream).toHaveBeenCalledTimes(1);
+ expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array)).toEqual(body);
+ await f.relay.close();
+ const records = await f.records(), admitted = records.find(row => row.type === "admitted");
+ expect(admitted.recordCount).toBe(4);
+ expect(admitted.expandedBytes).toBeGreaterThan(35_000);
+ expect(records.at(-1)).toMatchObject({ admittedRecords: 4, admittedExpandedBytes: admitted.expandedBytes });
+});
+
+it("accepts thousands of small native records while bounding their total count", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));
+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRecords: 3000 } });
+ const body = JSON.stringify({ resource_logs: [{ scope_logs: [{ log_records: Array.from({ length: 2533 }, () => ({ body: { stringValue: "event" } })) }] }] });
+ expect((await f.send(body, {}, "/v1/logs")).status).toBe(200);
+ expect((await f.send(body, {}, "/v1/logs")).status).toBe(429);
+ expect(upstream).toHaveBeenCalledTimes(1);
+ expect((await f.records()).find(row => row.type === "admitted").recordCount).toBe(2533);
+});
+
+it("rejects unknown protobuf fields instead of excluding forwarded data from sizing", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));
+ const f = await fixture({ transport: upstream });
+ const body = message(1, message(2, message(2, message(100, Buffer.from("unknown-content")))));
+ const response = await fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body,
+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/x-protobuf" } });
+ expect(response.status).toBe(400);
+ expect(upstream).not.toHaveBeenCalled();
+});
diff --git a/tests/runner-e2e/eval-otlp-relay.ts b/tests/runner-e2e/eval-otlp-relay.ts
new file mode 100644
index 0000000000..36e61d9dde
--- /dev/null
+++ b/tests/runner-e2e/eval-otlp-relay.ts
@@ -0,0 +1,139 @@
+import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
+import { once } from "node:events";
+import { mkdir, open } from "node:fs/promises";
+import { createServer, type ServerResponse } from "node:http";
+import path from "node:path";
+import { gunzipSync } from "node:zlib";
+import { evalOtlpRejected, evalOtlpResponse, isEvalProjectId, validateEvalOtlpParent, validateEvalOtlpPayload, type EvalOtlpSignal } from "./eval-otlp-payload.js";
+
+export const EVAL_OTLP_LIMITS = { maxRequests: 2048, maxRequestBytes: 8 * 1024 * 1024,
+ maxTotalBytes: 64 * 1024 * 1024, maxConcurrent: 4,
+ maxExpandedBytes: 256 * 1024 * 1024, maxRecords: 16_384 } as const;
+
+export async function startEvalOtlpRelay(input: {
+ projectId: string; apiKey: string; campaignId: string; attemptId: string; receiptPath: string; deadlineAt: number;
+ limits?: { [K in keyof typeof EVAL_OTLP_LIMITS]: number }; transport?: typeof fetch;
+}) {
+ const limits = input.limits ?? EVAL_OTLP_LIMITS;
+ if (!isEvalProjectId(input.projectId) || !input.apiKey.trim() || /[\r\n]/.test(input.apiKey)
+ || ![input.campaignId, input.attemptId].every(value => /^[A-Za-z0-9_.:-]{1,512}$/.test(value))
+ || !path.isAbsolute(input.receiptPath) || !Number.isFinite(input.deadlineAt)
+ || input.deadlineAt <= Date.now() || input.deadlineAt > Date.now() + 4 * 3_600_000
+ || Object.keys(limits).sort().join(",") !== Object.keys(EVAL_OTLP_LIMITS).sort().join(",")
+ || Object.entries(EVAL_OTLP_LIMITS).some(([key, maximum]) => !Number.isSafeInteger(limits[key as keyof typeof limits])
+ || limits[key as keyof typeof limits] < 1 || limits[key as keyof typeof limits] > maximum)) throw new Error("Invalid eval OTLP relay policy");
+ await mkdir(path.dirname(input.receiptPath), { recursive: true, mode: 0o700 });
+ const file = await open(input.receiptPath, "ax", 0o600);
+ let writes = Promise.resolve(), poisoned = false, closed = false, quotaRecorded = false;
+ const record = (value: object) => {
+ writes = writes.then(async () => { await file.writeFile(JSON.stringify(value) + "\n"); await file.sync(); })
+ .catch(error => { poisoned = true; throw error; });
+ return writes;
+ };
+ try { await record({ type: "policy", version: 2, sizing: "otlp-v1.9-expanded-source-json", projectId: input.projectId, campaignId: input.campaignId,
+ attemptId: input.attemptId, limits, deadlineAt: new Date(input.deadlineAt).toISOString(), openedAt: new Date().toISOString() }); }
+ catch (error) { await file.close(); throw error; }
+ const token = randomBytes(32).toString("hex");
+ const pending = new Set<Promise<void>>(), controllers = new Set<AbortController>();
+ const usage = { admittedRequests: 0, admittedWireBytes: 0, admittedBytes: 0, admittedExpandedBytes: 0,
+ admittedRecords: 0, attemptedForwardBytes: 0, acceptedRequests: 0, incompleteRequests: 0 };
+ const deny = (response: ServerResponse, status: number, code: string) => {
+ if (response.destroyed) return;
+ response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" });
+ response.end(JSON.stringify({ error: { type: "eval_otlp_relay", message: code } }));
+ };
+ const server = createServer((request, response) => {
+ const operation = (async () => {
+ const auth = typeof request.headers.authorization === "string" ? request.headers.authorization.replace(/^Bearer /, "") : "";
+ if (Buffer.byteLength(auth) !== Buffer.byteLength(token) || !timingSafeEqual(Buffer.from(auth), Buffer.from(token))) return deny(response, 401, "unauthorized");
+ if (closed || poisoned || Date.now() >= input.deadlineAt) return deny(response, 503, "relay_unavailable");
+ const signal: EvalOtlpSignal | undefined = request.url === "/otel/v1/traces" ? "traces" : request.url === "/otel/v1/logs" ? "logs" : undefined;
+ if (request.method !== "POST" || !signal) return deny(response, 404, "unqualified_endpoint");
+ if (pending.size >= limits.maxConcurrent) return deny(response, 429, "concurrency_limit");
+ const contentType = request.headers["content-type"]?.split(";")[0].trim();
+ const encoding = request.headers["content-encoding"];
+ if (!["application/json", "application/x-protobuf"].includes(contentType ?? "") || encoding && encoding !== "gzip" && encoding !== "identity") return deny(response, 415, "unqualified_encoding");
+ let body: Buffer, parent: string, wireBytes = 0, expandedBytes = 0, recordCount = 0;
+ try {
+ parent = validateEvalOtlpParent(String(request.headers["x-bt-parent"] ?? `project_id:${input.projectId}`), input.projectId);
+ const chunks: Buffer[] = [];
+ for await (const chunk of request) {
+ wireBytes += chunk.length;
+ if (wireBytes > limits.maxRequestBytes) throw new Error("request_limit");
+ chunks.push(Buffer.from(chunk));
+ }
+ body = Buffer.concat(chunks);
+ if (encoding === "gzip") body = gunzipSync(body, { maxOutputLength: limits.maxRequestBytes });
+ const validated = validateEvalOtlpPayload(body, contentType!, signal, input.projectId);
+ ({ body, expandedBytes, recordCount } = validated);
+ if (body.length > limits.maxRequestBytes) throw new Error("request_limit");
+ } catch { return deny(response, 400, "invalid_or_oversized_payload"); }
+ if (closed || poisoned || Date.now() >= input.deadlineAt) return deny(response, 503, "relay_unavailable");
+ if (usage.admittedRequests >= limits.maxRequests || usage.admittedBytes + body.length > limits.maxTotalBytes
+ || usage.admittedExpandedBytes + expandedBytes > limits.maxExpandedBytes
+ || usage.admittedRecords + recordCount > limits.maxRecords) {
+ if (!quotaRecorded) { quotaRecorded = true; await record({ type: "quota_exhausted", at: new Date().toISOString(), ...usage }); }
+ return deny(response, 429, "ingestion_allowance_exhausted");
+ }
+ const id = randomUUID();
+ usage.admittedRequests++; usage.admittedWireBytes += wireBytes; usage.admittedBytes += body.length;
+ usage.admittedExpandedBytes += expandedBytes; usage.admittedRecords += recordCount;
+ await record({ type: "admitted", id, signal, at: new Date().toISOString(), wireBytes, bodyBytes: body.length,
+ expandedBytes, recordCount,
+ payloadSha256: createHash("sha256").update(body).digest("hex") });
+ const abort = new AbortController(); controllers.add(abort);
+ const disconnected = () => { if (!response.writableEnded) abort.abort(); };
+ response.on("close", disconnected);
+ let status: number | undefined, outcome = "uncertain", rejected: number | undefined, dispatched = false;
+ try {
+ if (closed || poisoned || response.destroyed || Date.now() >= input.deadlineAt) throw new Error("relay_unavailable");
+ dispatched = true; usage.attemptedForwardBytes += body.length;
+ const upstream = await (input.transport ?? fetch)(`https://api.braintrust.dev/otel/v1/${signal}`, {
+ method: "POST", redirect: "error", signal: AbortSignal.any([abort.signal,
+ AbortSignal.timeout(Math.max(1, Math.min(10_000, input.deadlineAt - Date.now())))]),
+ headers: { authorization: `Bearer ${input.apiKey}`, "x-bt-parent": parent, "content-type": contentType!,
+ "accept-encoding": "identity", accept: "application/x-protobuf, application/json" }, body: Buffer.from(body),
+ });
+ status = upstream.status;
+ if (!upstream.ok) { await upstream.body?.cancel(); outcome = "rejected"; throw new Error("receiver_rejected"); }
+ const type = upstream.headers.get("content-type")?.split(";")[0].trim() ?? contentType!;
+ if (!["application/json", "application/x-protobuf"].includes(type)) { await upstream.body?.cancel(); throw new Error("invalid_response"); }
+ const reader = upstream.body?.getReader(), chunks: Uint8Array[] = []; let bytes = 0;
+ try {
+ while (reader) {
+ const next = await reader.read(); if (next.done) break;
+ bytes += next.value.length; if (bytes > 65_536) throw new Error("response_limit");
+ chunks.push(next.value);
+ }
+ } finally { await reader?.cancel().catch(() => {}); }
+ rejected = evalOtlpRejected(Buffer.concat(chunks), type, signal);
+ outcome = rejected ? "partial" : "accepted";
+ } catch { if (!dispatched) outcome = "not_dispatched"; }
+ finally { controllers.delete(abort); response.off("close", disconnected); }
+ if (outcome === "accepted") usage.acceptedRequests++; else usage.incompleteRequests++;
+ await record({ type: "outcome", id, at: new Date().toISOString(), outcome, dispatched,
+ ...(status === undefined ? {} : { httpStatus: status }), ...(rejected === undefined ? {} : { rejectedRecords: rejected }) });
+ if (response.destroyed) return;
+ if (outcome !== "accepted" && outcome !== "partial") return deny(response, 502, "export_unconfirmed");
+ response.writeHead(200, { "content-type": contentType!, "cache-control": "no-store" });
+ response.end(evalOtlpResponse(rejected!, contentType!, signal));
+ })();
+ pending.add(operation);
+ void operation.then(() => pending.delete(operation), () => { poisoned = true; pending.delete(operation); deny(response, 503, "receipt_persistence_failed"); });
+ });
+ server.requestTimeout = 15_000; server.headersTimeout = 10_000; server.maxConnections = limits.maxConcurrent + 8;
+ try { server.listen(0, "127.0.0.1"); await once(server, "listening"); }
+ catch (error) { await file.close(); throw error; }
+ const address = server.address();
+ if (!address || typeof address === "string") { await file.close(); throw new Error("Eval OTLP relay did not bind loopback"); }
+ let closing: Promise<void> | undefined;
+ const close = () => closing ??= (async () => {
+ closed = true; clearTimeout(timer); controllers.forEach(controller => controller.abort()); server.closeAllConnections();
+ await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
+ await Promise.allSettled([...pending]);
+ try { await record({ type: "closed", at: new Date().toISOString(), ...usage, poisoned }); }
+ finally { await file.close(); }
+ })();
+ const timer = setTimeout(() => { void close().catch(() => {}); }, Math.max(1, input.deadlineAt - Date.now())); timer.unref();
+ return { baseUrl: `http://127.0.0.1:${address.port}/otel`, token, receiptPath: input.receiptPath, close };
+}
diff --git a/tests/runner-e2e/eval-otlp-schema.ts b/tests/runner-e2e/eval-otlp-schema.ts
new file mode 100644
index 0000000000..a277cbd7f1
--- /dev/null
+++ b/tests/runner-e2e/eval-otlp-schema.ts
@@ -0,0 +1,170 @@
+import { createRequire } from "node:module";
+const require = createRequire(new URL("../../server/package.json", import.meta.url));
+export const protobuf = require("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js");
+export const evalOtlpSchema = protobuf.parse(`syntax = "proto3";
+message AnyValue {
+oneof value {
+string string_value = 1;
+bool bool_value = 2;
+int64 int_value = 3;
+double double_value = 4;
+ArrayValue array_value = 5;
+KeyValueList kvlist_value = 6;
+bytes bytes_value = 7;
+}
+}
+message ArrayValue {
+repeated AnyValue values = 1;
+}
+message KeyValueList {
+repeated KeyValue values = 1;
+}
+message KeyValue {
+string key = 1;
+AnyValue value = 2;
+}
+message InstrumentationScope {
+string name = 1;
+string version = 2;
+repeated KeyValue attributes = 3;
+uint32 dropped_attributes_count = 4;
+}
+message EntityRef {
+string schema_url = 1;
+string type = 2;
+repeated string id_keys = 3;
+repeated string description_keys = 4;
+}
+message Resource {
+repeated KeyValue attributes = 1;
+uint32 dropped_attributes_count = 2;
+repeated EntityRef entity_refs = 3;
+}
+message TracesData {
+repeated ResourceSpans resource_spans = 1;
+}
+message ResourceSpans {
+Resource resource = 1;
+repeated ScopeSpans scope_spans = 2;
+string schema_url = 3;
+}
+message ScopeSpans {
+InstrumentationScope scope = 1;
+repeated Span spans = 2;
+string schema_url = 3;
+}
+message Span {
+bytes trace_id = 1;
+bytes span_id = 2;
+string trace_state = 3;
+bytes parent_span_id = 4;
+fixed32 flags = 16;
+string name = 5;
+enum SpanKind {
+SPAN_KIND_UNSPECIFIED = 0;
+SPAN_KIND_INTERNAL = 1;
+SPAN_KIND_SERVER = 2;
+SPAN_KIND_CLIENT = 3;
+SPAN_KIND_PRODUCER = 4;
+SPAN_KIND_CONSUMER = 5;
+}
+SpanKind kind = 6;
+fixed64 start_time_unix_nano = 7;
+fixed64 end_time_unix_nano = 8;
+repeated KeyValue attributes = 9;
+uint32 dropped_attributes_count = 10;
+message Event {
+fixed64 time_unix_nano = 1;
+string name = 2;
+repeated KeyValue attributes = 3;
+uint32 dropped_attributes_count = 4;
+}
+repeated Event events = 11;
+uint32 dropped_events_count = 12;
+message Link {
+bytes trace_id = 1;
+bytes span_id = 2;
+string trace_state = 3;
+repeated KeyValue attributes = 4;
+uint32 dropped_attributes_count = 5;
+fixed32 flags = 6;
+}
+repeated Link links = 13;
+uint32 dropped_links_count = 14;
+Status status = 15;
+}
+message Status {
+string message = 2;
+enum StatusCode {
+STATUS_CODE_UNSET = 0;
+STATUS_CODE_OK = 1;
+STATUS_CODE_ERROR = 2;
+};
+StatusCode code = 3;
+}
+enum SpanFlags {
+SPAN_FLAGS_DO_NOT_USE = 0;
+SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF;
+SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100;
+SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200;
+}
+message LogsData {
+repeated ResourceLogs resource_logs = 1;
+}
+message ResourceLogs {
+Resource resource = 1;
+repeated ScopeLogs scope_logs = 2;
+string schema_url = 3;
+}
+message ScopeLogs {
+InstrumentationScope scope = 1;
+repeated LogRecord log_records = 2;
+string schema_url = 3;
+}
+enum SeverityNumber {
+SEVERITY_NUMBER_UNSPECIFIED = 0;
+SEVERITY_NUMBER_TRACE = 1;
+SEVERITY_NUMBER_TRACE2 = 2;
+SEVERITY_NUMBER_TRACE3 = 3;
+SEVERITY_NUMBER_TRACE4 = 4;
+SEVERITY_NUMBER_DEBUG = 5;
+SEVERITY_NUMBER_DEBUG2 = 6;
+SEVERITY_NUMBER_DEBUG3 = 7;
+SEVERITY_NUMBER_DEBUG4 = 8;
+SEVERITY_NUMBER_INFO = 9;
+SEVERITY_NUMBER_INFO2 = 10;
+SEVERITY_NUMBER_INFO3 = 11;
+SEVERITY_NUMBER_INFO4 = 12;
+SEVERITY_NUMBER_WARN = 13;
+SEVERITY_NUMBER_WARN2 = 14;
+SEVERITY_NUMBER_WARN3 = 15;
+SEVERITY_NUMBER_WARN4 = 16;
+SEVERITY_NUMBER_ERROR = 17;
+SEVERITY_NUMBER_ERROR2 = 18;
+SEVERITY_NUMBER_ERROR3 = 19;
+SEVERITY_NUMBER_ERROR4 = 20;
+SEVERITY_NUMBER_FATAL = 21;
+SEVERITY_NUMBER_FATAL2 = 22;
+SEVERITY_NUMBER_FATAL3 = 23;
+SEVERITY_NUMBER_FATAL4 = 24;
+}
+enum LogRecordFlags {
+LOG_RECORD_FLAGS_DO_NOT_USE = 0;
+LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 0x000000FF;
+}
+message LogRecord {
+fixed64 time_unix_nano = 1;
+fixed64 observed_time_unix_nano = 11;
+SeverityNumber severity_number = 2;
+string severity_text = 3;
+AnyValue body = 5;
+repeated KeyValue attributes = 6;
+uint32 dropped_attributes_count = 7;
+fixed32 flags = 8;
+bytes trace_id = 9;
+bytes span_id = 10;
+string event_name = 12;
+}
+message PartialSuccess { int64 rejected = 1; }
+message Response { PartialSuccess partial_success = 1; }
+`).root.resolveAll();
diff --git a/tests/runner-e2e/experiment-ingestion.test.ts b/tests/runner-e2e/experiment-ingestion.test.ts
new file mode 100644
index 0000000000..984bcb91a7
--- /dev/null
+++ b/tests/runner-e2e/experiment-ingestion.test.ts
@@ -0,0 +1,108 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { EVAL_EXPERIMENT_LIMITS, openExperimentIngestion } from "./experiment-ingestion.js";
+
+const projectId = "11111111-1111-4111-8111-111111111111", experimentId = "22222222-2222-4222-8222-222222222222";
+const roots: string[] = [], meters: Array<Awaited<ReturnType<typeof openExperimentIngestion>>> = [];
+afterEach(async () => { await Promise.allSettled(meters.splice(0).map(meter => meter.close()));
+ await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+async function fixture(transport: typeof fetch = async () => Response.json({ ok: true })) {
+ const root = await mkdtemp(path.join(tmpdir(), "experiment-ingestion-")); roots.push(root);
+ const policy = { projectId, campaignId: "campaign", deadlineAt: Date.now() + 60_000, receiptPath: path.join(root, "receipts.jsonl"), transport };
+ const meter = await openExperimentIngestion(policy); meters.push(meter);
+ return { ...meter, policy, records: async () => (await readFile(policy.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)),
+ send: (payload: unknown, endpoint = "experiment") => meter.transport(`https://api.braintrust.dev/v1/${endpoint}`, {
+ method: "POST", body: JSON.stringify(payload), headers: { authorization: "Bearer private-secret" },
+ }) };
+}
+
+it("durably meters payloads and scores before dispatch while retaining body-free private evidence", async () => {
+ const upstream = vi.fn<typeof fetch>(async (_url, init) => {
+ expect((await f.records()).at(-1)).toMatchObject({ type: "admitted", bodyBytes: init?.body === undefined ? 0 : Buffer.byteLength(String(init.body)) });
+ expect(init!.redirect).toBe("error");
+ return Response.json({ ok: true });
+ });
+ const f = await fixture(upstream);
+ const create = { project_id: projectId, name: "private-payload-canary" };
+ const insert = { events: [{ scores: { product_pass: 0, cleanup_pass: 0 }, input: "private-input-canary" }] };
+ await f.send(create); await f.send(insert, `experiment/${experimentId}/insert`);
+ await f.transport(`https://api.braintrust.dev/v1/experiment/${experimentId}/summarize`, { method: "GET" });
+ await f.close();
+ const records = await f.records();
+ expect(records.at(-1)).toMatchObject({ type: "closed", admittedRequests: 3, admittedScores: 2,
+ admittedBytes: Buffer.byteLength(JSON.stringify(create)) + Buffer.byteLength(JSON.stringify(insert)), unresolvedRequests: 0 });
+ const raw = await readFile(f.policy.receiptPath, "utf8");
+ expect(raw).not.toMatch(/private-secret|private-payload-canary|private-input-canary/);
+ expect((await stat(f.policy.receiptPath)).mode & 0o777).toBe(0o600);
+ await expect(openExperimentIngestion(f.policy)).rejects.toMatchObject({ code: "EEXIST" });
+ expect(await readFile(f.policy.receiptPath, "utf8")).toBe(raw);
+});
+
+it("keeps failed forwarding charged to the finite allowance and records an unresolved outcome", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => { throw new Error("private-upstream-error"); });
+ const f = await fixture(upstream);
+ for (let i = 0; i < EVAL_EXPERIMENT_LIMITS.maxRequests; i++) {
+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed");
+ }
+ await expect(f.send({ project_id: projectId })).rejects.toThrow("allowance exhausted");
+ expect(upstream).toHaveBeenCalledTimes(3);
+ await f.close();
+ expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 3, responses: 0, unresolvedRequests: 3 });
+});
+
+it("retains an unresolved allowance when response headers arrive but the body disconnects", async () => {
+ const f = await fixture(async () => new Response(new ReadableStream({ pull(controller) {
+ controller.error(new Error("private-disconnect-error"));
+ } }), { status: 200 }));
+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed");
+ await f.close();
+ const rows = await f.records();
+ expect(rows[2]).toMatchObject({ type: "outcome", outcome: "unconfirmed" });
+ expect(rows.at(-1)).toMatchObject({ admittedRequests: 1, responses: 0, unresolvedRequests: 1 });
+ expect(JSON.stringify(rows)).not.toContain("private-disconnect-error");
+});
+
+it("keeps a pending response body unresolved until the whole response is read", async () => {
+ let finish!: () => void;
+ const f = await fixture(async () => new Response(new ReadableStream({ start(controller) {
+ controller.enqueue(new TextEncoder().encode('{"ok":'));
+ finish = () => { controller.enqueue(new TextEncoder().encode('true}')); controller.close(); };
+ } })));
+ const pending = f.send({ project_id: projectId });
+ await vi.waitFor(() => expect(finish).toBeTypeOf("function"));
+ expect((await f.records()).at(-1)).toMatchObject({ type: "admitted" });
+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable");
+ finish();
+ expect(await (await pending).json()).toEqual({ ok: true });
+ await f.close();
+ expect((await f.records()).at(-1)).toMatchObject({ responses: 1, unresolvedRequests: 0 });
+});
+
+it("refuses excess bytes, scores, foreign project and unqualified endpoints before dispatch", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));
+ const f = await fixture(upstream);
+ await expect(f.send({ project_id: experimentId })).rejects.toThrow("project mismatch");
+ await expect(f.send({ project_id: projectId, data: "x".repeat(EVAL_EXPERIMENT_LIMITS.maxTotalBytes) })).rejects.toThrow("byte limit");
+ await expect(f.send({ events: Array.from({ length: 9 }, () => ({ scores: { pass: 1 } })) }, `experiment/${experimentId}/insert`))
+ .rejects.toThrow("allowance exhausted");
+ await expect(f.send({}, "project")).rejects.toThrow("Unqualified");
+ expect(upstream).not.toHaveBeenCalled();
+ await f.close();
+ expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 0, admittedBytes: 0, admittedScores: 0 });
+});
+
+it("refuses concurrent requests and expired windows without resetting reservations", async () => {
+ let release!: () => void;
+ const upstream = vi.fn<typeof fetch>(async () => { await new Promise<void>(resolve => { release = resolve; }); return Response.json({}); });
+ const f = await fixture(upstream);
+ const pending = f.send({ project_id: projectId });
+ await vi.waitFor(() => expect(upstream).toHaveBeenCalledOnce());
+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable");
+ release(); await pending;
+ const now = vi.spyOn(Date, "now").mockReturnValue(f.policy.deadlineAt);
+ try { await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); }
+ finally { now.mockRestore(); }
+ expect(upstream).toHaveBeenCalledOnce();
+});
diff --git a/tests/runner-e2e/experiment-ingestion.ts b/tests/runner-e2e/experiment-ingestion.ts
new file mode 100644
index 0000000000..7cb1c102bb
--- /dev/null
+++ b/tests/runner-e2e/experiment-ingestion.ts
@@ -0,0 +1,91 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, open } from "node:fs/promises";
+import path from "node:path";
+import { isEvalProjectId } from "./eval-otlp-payload.js";
+
+export const EVAL_EXPERIMENT_LIMITS = { maxRequests: 3, maxTotalBytes: 4 * 1024 * 1024, maxScores: 8 } as const;
+
+export async function openExperimentIngestion(input: {
+ receiptPath: string; campaignId: string; projectId: string; deadlineAt: number; transport?: typeof fetch;
+}) {
+ if (!path.isAbsolute(input.receiptPath) || !/^[A-Za-z0-9_.:-]{1,512}$/.test(input.campaignId)
+ || !isEvalProjectId(input.projectId) || !Number.isFinite(input.deadlineAt)
+ || input.deadlineAt <= Date.now() || input.deadlineAt > Date.now() + 60_000) throw new Error("Invalid experiment ingestion policy");
+ await mkdir(path.dirname(input.receiptPath), { recursive: true, mode: 0o700 });
+ const file = await open(input.receiptPath, "ax", 0o600);
+ let writes = Promise.resolve(), closed = false, poisoned = false, active = false;
+ let admittedRequests = 0, admittedBytes = 0, admittedScores = 0, responses = 0;
+ const record = (value: object) => {
+ writes = writes.then(async () => { await file.writeFile(JSON.stringify(value) + "\n"); await file.sync(); })
+ .catch(error => { poisoned = true; throw error; });
+ return writes;
+ };
+ try { await record({ type: "policy", version: 1, kind: "experiment", projectId: input.projectId, campaignId: input.campaignId,
+ limits: EVAL_EXPERIMENT_LIMITS, openedAt: new Date().toISOString(), deadlineAt: new Date(input.deadlineAt).toISOString() }); }
+ catch (error) { await file.close(); throw error; }
+ const transport: typeof fetch = async (target, init) => {
+ if (closed || poisoned || active || Date.now() >= input.deadlineAt) throw new Error("Experiment ingestion unavailable");
+ const url = String(target), method = init?.method;
+ const create = url === "https://api.braintrust.dev/v1/experiment" && method === "POST";
+ const insert = /^https:\/\/api\.braintrust\.dev\/v1\/experiment\/[0-9a-f-]{36}\/insert$/.test(url) && method === "POST";
+ const summarize = /^https:\/\/api\.braintrust\.dev\/v1\/experiment\/[0-9a-f-]{36}\/summarize$/.test(url) && method === "GET";
+ if (!create && !insert && !summarize || summarize && init?.body !== undefined
+ || !summarize && typeof init?.body !== "string") throw new Error("Unqualified experiment ingestion request");
+ const body = summarize ? "" : init!.body as string;
+ const bytes = Buffer.byteLength(body);
+ if (bytes > EVAL_EXPERIMENT_LIMITS.maxTotalBytes) throw new Error("Experiment ingestion byte limit");
+ let scores = 0;
+ if (body) {
+ const payload = JSON.parse(body);
+ if (create && payload.project_id !== input.projectId) throw new Error("Experiment ingestion project mismatch");
+ if (insert) {
+ if (!Array.isArray(payload.events)) throw new Error("Invalid experiment ingestion events");
+ scores = payload.events.reduce((sum: number, event: { scores?: object }) => sum + Object.keys(event.scores ?? {}).length, 0);
+ }
+ }
+ if (admittedRequests >= EVAL_EXPERIMENT_LIMITS.maxRequests || admittedBytes + bytes > EVAL_EXPERIMENT_LIMITS.maxTotalBytes
+ || admittedScores + scores > EVAL_EXPERIMENT_LIMITS.maxScores) throw new Error("Experiment ingestion allowance exhausted");
+ active = true;
+ const id = randomUUID();
+ admittedRequests++; admittedBytes += bytes; admittedScores += scores;
+ try {
+ await record({ type: "admitted", id, at: new Date().toISOString(), operation: create ? "create" : insert ? "insert" : "summarize",
+ bodyBytes: bytes, scores, payloadSha256: createHash("sha256").update(body).digest("hex") });
+ if (closed || poisoned || Date.now() >= input.deadlineAt) throw new Error("Experiment ingestion unavailable");
+ let response: Response;
+ try {
+ const upstream = await (input.transport ?? fetch)(target, { ...init, redirect: "error", signal: AbortSignal.any([
+ AbortSignal.timeout(Math.max(1, Math.min(15_000, input.deadlineAt - Date.now()))), ...(init?.signal ? [init.signal] : []),
+ ]) });
+ const reader = upstream.body?.getReader(), chunks: Uint8Array[] = [];
+ let responseBytes = 0;
+ try {
+ while (reader) {
+ const next = await reader.read();
+ if (next.done) break;
+ responseBytes += next.value.length;
+ if (responseBytes > 1_048_576) throw new Error("Oversized experiment response");
+ chunks.push(next.value);
+ }
+ response = new Response(upstream.body === null ? null : Buffer.concat(chunks), { status: upstream.status, headers: upstream.headers });
+ } finally { await reader?.cancel().catch(() => {}); }
+ } catch {
+ await record({ type: "outcome", id, at: new Date().toISOString(), outcome: "unconfirmed" });
+ throw new Error("Experiment ingestion response unconfirmed");
+ }
+ try { await record({ type: "outcome", id, at: new Date().toISOString(), outcome: "response", httpStatus: response.status }); }
+ catch (error) { await response.body?.cancel().catch(() => {}); throw error; }
+ responses++;
+ return response;
+ } finally { active = false; }
+ };
+ let closing: Promise<void> | undefined;
+ return { transport, close: () => closing ??= (async () => {
+ closed = true;
+ try {
+ if (active) throw new Error("Experiment ingestion is still in flight");
+ await record({ type: "closed", at: new Date().toISOString(), admittedRequests, admittedBytes, admittedScores,
+ responses, unresolvedRequests: admittedRequests - responses });
+ } finally { await file.close(); }
+ })() };
+}
diff --git a/tests/runner-e2e/feedback-dispositions.test.ts b/tests/runner-e2e/feedback-dispositions.test.ts
new file mode 100644
index 0000000000..743ac95fb8
--- /dev/null
+++ b/tests/runner-e2e/feedback-dispositions.test.ts
@@ -0,0 +1,129 @@
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { runnerMatrix } from "./catalog.js";
+import { emptyFeedbackSummary } from "./feedback.js";
+import { buildRunnerCampaign } from "./history.js";
+import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
+import { campaignFeedbackEvidence, campaignFeedbackReviewText, feedbackReviewTemplate, validateCampaignFeedbackReview, type CampaignFeedbackReview } from "./feedback-dispositions.js";
+import type { RunnerE2EResult } from "./types.js";
+
+function fixture() {
+ const cell = runnerMatrix[0]!;
+ const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id, suiteId: cell.suite.id,
+ suiteDefinitionHash: cell.suiteDefinitionHash, attempt: 1, status: "failed", profileId: cell.profile.id,
+ environmentId: cell.environment.id, caseId: cell.task.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,
+ runIds: [], startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed",
+ feedback: { ...emptyFeedbackSummary(), collection: "partial", invocationCollection: "complete", complaints: 2, suggestions: 1, bodiesCollected: 3,
+ review: ["a", "b", "c"].map(value => ({ fingerprint: value.repeat(64), kind: value === "c" ? "suggestion" : "complaint", occurrences: 1, disposition: "needs_investigation" })) } };
+ return buildRunnerCampaign({ campaignId: "feedback-campaign", generatedAt: result.finishedAt, expected: [cell.id], results: [result] });
+}
+function reviewed(campaign = fixture()): CampaignFeedbackReview {
+ return { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "agent",
+ decisions: [
+ { fingerprint: "a".repeat(64), disposition: "actionable", issueNumber: 123 },
+ { fingerprint: "b".repeat(64), disposition: "duplicate", duplicateOf: "a".repeat(64) },
+ { fingerprint: "c".repeat(64), disposition: "unsupported" },
+ ] };
+}
+
+it("keeps an unreviewed template invalid until a reviewer records the decision metadata", () => {
+ const campaign = fixture();
+ expect(() => validateCampaignFeedbackReview(campaign, feedbackReviewTemplate(campaign) as never)).toThrow("Invalid or stale");
+ expect(campaignFeedbackReviewText(campaign)).toContain("3 distinct feedback items await review");
+ validateCampaignFeedbackReview(campaign, reviewed(campaign));
+});
+
+it("preserves dispositions on regeneration without changing failed grades or incomplete collection", async () => {
+ const campaign = fixture(); campaign.feedbackReview = reviewed(campaign);
+ const root = await mkdtemp(path.join(tmpdir(), "feedback-review-"));
+ try {
+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify({ ...campaign, feedbackReview: undefined }));
+ const retainedHeader = "# Existing campaign\n\nProvider-reported LLM cost: $0.123456\n\n";
+ await writeFile(path.join(root, "summary.md"), `${retainedHeader}${campaignFeedbackReviewText({ ...campaign, feedbackReview: undefined })}\n`);
+ const feedbackReviewFile = path.join(root, "review.json");
+ await writeFile(feedbackReviewFile, JSON.stringify(campaign.feedbackReview));
+ await regenerateRunnerDashboard({ bundle: root, historyFile: null, feedbackReviewFile });
+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });
+ const retained = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));
+ expect(retained.feedbackReview).toEqual(campaign.feedbackReview);
+ expect(retained.results[0].status).toBe("failed");
+ expect(retained.accounting.attempts[0].feedback.collection).toBe("partial");
+ const html = await readFile(path.join(root, "dashboard.html"), "utf8");
+ expect(html).toContain("1 actionable; 1 duplicate; 0 already fixed; 1 unsupported; 0 needs investigation");
+ expect(html).toContain("1 attempts have incomplete collection");
+ expect(html).toContain("https://github.com/paperclipai/paperclip/issues/123");
+ const summary = await readFile(path.join(root, "summary.md"), "utf8");
+ expect(summary).toBe(`${retainedHeader}${campaignFeedbackReviewText(campaign)}\n`);
+ expect(summary).not.toContain("await review");
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
+
+it("rejects stale reviews when attempts or collection evidence change, but permits a render refresh", () => {
+ const campaign = fixture(), review = reviewed(campaign);
+ campaign.generatedAt = "2026-10-21T00:00:00Z";
+ validateCampaignFeedbackReview(campaign, review);
+ const changed = structuredClone(campaign);
+ changed.accounting!.attempts[0]!.feedback!.collection = "complete";
+ expect(() => validateCampaignFeedbackReview(changed, review)).toThrow("Invalid or stale");
+ changed.accounting = structuredClone(campaign.accounting);
+ changed.accounting!.missingAttempts++;
+ expect(() => validateCampaignFeedbackReview(changed, review)).toThrow("Invalid or stale");
+ expect(campaignFeedbackEvidence({ ...campaign, expected: [...campaign.expected].reverse() })).toBe(review.evidenceHash);
+});
+
+it("requires exactly one closed decision per group and refuses raw notes and duplicate cycles", () => {
+ const campaign = fixture(), review = reviewed(campaign);
+ for (const changed of [
+ { ...review, rationale: "private notes" },
+ { ...review, decisions: review.decisions.slice(1) },
+ { ...review, decisions: [...review.decisions, review.decisions[0]!] },
+ { ...review, decisions: review.decisions.map(item => ({ ...item, rawBody: "private body" })) },
+ { ...review, decisions: review.decisions.map(item => item.fingerprint === "a".repeat(64) ? { ...item, disposition: "duplicate", duplicateOf: "b".repeat(64) } : item) },
+ ]) expect(() => validateCampaignFeedbackReview(campaign, changed as never)).toThrow("Invalid or stale");
+});
+
+it.each(["2026-10-20T01:00:00Z", "2026-10-20T01:00:00.936739+00:00", "2026-10-19T20:00:00-05:00"])("accepts explicit ISO review timestamps: %s", reviewedAt => {
+ const campaign = fixture();
+ validateCampaignFeedbackReview(campaign, { ...reviewed(campaign), reviewedAt });
+});
+
+it("rejects private timestamp notes before a review can alter the public report", async () => {
+ const campaign = fixture();
+ const reviewedAt = "Tue, 20 Oct 2026 01:00:00 GMT (Private rationale: customer account was suspended)";
+ expect(Number.isFinite(Date.parse(reviewedAt))).toBe(true);
+ const root = await mkdtemp(path.join(tmpdir(), "feedback-private-timestamp-"));
+ try {
+ const original = JSON.stringify(campaign);
+ await writeFile(path.join(root, "normalized-results.json"), original);
+ const feedbackReviewFile = path.join(root, "review.json");
+ await writeFile(feedbackReviewFile, JSON.stringify({ ...reviewed(campaign), reviewedAt }));
+ await expect(regenerateRunnerDashboard({ bundle: root, historyFile: null, feedbackReviewFile })).rejects.toThrow("Invalid or stale");
+ expect(await readFile(path.join(root, "normalized-results.json"), "utf8")).toBe(original);
+ for (const invalid of ["2026-10-20T01:00:00Z\n", "2026-10-20", "2026-10-20T24:00:00Z"]) {
+ expect(() => validateCampaignFeedbackReview(campaign, { ...reviewed(campaign), reviewedAt: invalid })).toThrow("Invalid or stale");
+ }
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
+
+it("records an explicit empty review without implying that uncaptured feedback was absent", () => {
+ const campaign = fixture(); campaign.accounting!.attempts[0]!.feedback = emptyFeedbackSummary();
+ campaign.feedbackReview = { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "human", decisions: [] };
+ validateCampaignFeedbackReview(campaign, campaign.feedbackReview);
+ expect(campaignFeedbackReviewText(campaign)).toContain("1 attempts have incomplete collection");
+});
+
+it("refreshes an older summary that kept the awaiting-review count inline", async () => {
+ const campaign = fixture(); campaign.feedbackReview = reviewed(campaign);
+ const root = await mkdtemp(path.join(tmpdir(), "feedback-legacy-summary-"));
+ try {
+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));
+ await writeFile(path.join(root, "summary.md"), "# Existing campaign\n\nFeedback across retained attempts: 2 complaints; 1 suggestions; 3 distinct items awaiting review; 1 attempts with incomplete collection.\n");
+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });
+ const summary = await readFile(path.join(root, "summary.md"), "utf8");
+ expect(summary).toContain("2 complaints; 1 suggestions; 1 attempts with incomplete collection.");
+ expect(summary).toContain(campaignFeedbackReviewText(campaign));
+ expect(summary).not.toContain("awaiting review");
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
diff --git a/tests/runner-e2e/feedback-dispositions.ts b/tests/runner-e2e/feedback-dispositions.ts
new file mode 100644
index 0000000000..243a79a70d
--- /dev/null
+++ b/tests/runner-e2e/feedback-dispositions.ts
@@ -0,0 +1,95 @@
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+import { summarizeCampaignFeedback } from "./feedback.js";
+import { html, isPublicReportTimestamp } from "./report-format.js";
+import type { RunnerE2ECampaign } from "./types.js";
+
+const dispositions = ["actionable", "duplicate", "already_fixed", "unsupported", "needs_investigation"] as const;
+export interface CampaignFeedbackReview {
+ version: 1;
+ campaignId: string;
+ evidenceHash: string;
+ reviewedAt: string;
+ reviewedBy: "human" | "agent";
+ decisions: Array<{
+ fingerprint: string;
+ disposition: typeof dispositions[number];
+ duplicateOf?: string;
+ issueNumber?: number;
+ }>;
+}
+
+export function campaignFeedbackEvidence(campaign: RunnerE2ECampaign) {
+ if (!campaign.accounting) throw new Error("Feedback review requires retained attempt accounting");
+ const attempts = campaign.accounting.attempts.map(attempt => ({
+ id: attempt.id, executionId: attempt.executionId, attempt: attempt.attempt,
+ feedback: attempt.feedback ? {
+ ...attempt.feedback,
+ review: [...attempt.feedback.review].sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)),
+ } : null,
+ })).sort((a, b) => a.id.localeCompare(b.id));
+ const evidence = { campaignId: campaign.campaignId, expected: [...campaign.expected].sort(),
+ scope: campaign.accounting.scope, missingAttempts: campaign.accounting.missingAttempts, attempts };
+ const canonical = (_key: string, value: unknown) => value && typeof value === "object" && !Array.isArray(value)
+ ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) : value;
+ return createHash("sha256").update(JSON.stringify(evidence, canonical)).digest("hex");
+}
+
+export function feedbackReviewTemplate(campaign: RunnerE2ECampaign) {
+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);
+ return { version: 1, campaignId: campaign.campaignId, evidenceHash: campaignFeedbackEvidence(campaign),
+ reviewedAt: null, reviewedBy: null,
+ decisions: feedback.review.map(({ fingerprint }) => ({ fingerprint, disposition: "needs_investigation" })) };
+}
+
+export function validateCampaignFeedbackReview(campaign: RunnerE2ECampaign, review: CampaignFeedbackReview): void {
+ const fail = (): never => { throw new Error("Invalid or stale campaign feedback review"); };
+ const fields = (value: unknown, allowed: string[]) => {
+ if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(key => !allowed.includes(key))) fail();
+ };
+ fields(review, ["version", "campaignId", "evidenceHash", "reviewedAt", "reviewedBy", "decisions"]);
+ if (review.version !== 1 || review.campaignId !== campaign.campaignId || review.evidenceHash !== campaignFeedbackEvidence(campaign)
+ || !isPublicReportTimestamp(review.reviewedAt)
+ || !["human", "agent"].includes(review.reviewedBy) || !Array.isArray(review.decisions)) fail();
+ const groups = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results).review;
+ const expected = new Set(groups.map(group => group.fingerprint)), seen = new Set<string>();
+ for (const decision of review.decisions) {
+ fields(decision, ["fingerprint", "disposition", "duplicateOf", "issueNumber"]);
+ if (!expected.has(decision.fingerprint) || seen.has(decision.fingerprint) || !dispositions.includes(decision.disposition)) fail();
+ seen.add(decision.fingerprint);
+ if (decision.issueNumber !== undefined && (!Number.isSafeInteger(decision.issueNumber) || decision.issueNumber < 1)) fail();
+ if (decision.disposition === "duplicate") {
+ if (!decision.duplicateOf || decision.duplicateOf === decision.fingerprint || !expected.has(decision.duplicateOf)
+ || review.decisions.find(item => item.fingerprint === decision.duplicateOf)?.disposition === "duplicate") fail();
+ } else if (decision.duplicateOf !== undefined) fail();
+ }
+ if (seen.size !== expected.size) fail();
+}
+
+export function campaignFeedbackReviewText(campaign: RunnerE2ECampaign): string {
+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);
+ const review = campaign.feedbackReview;
+ if (!review) return `${feedback.review.length} distinct feedback items await review. Collection gaps and failed submissions remain separate findings.`;
+ validateCampaignFeedbackReview(campaign, review);
+ const count = (disposition: typeof dispositions[number]) => review.decisions.filter(item => item.disposition === disposition).length;
+ return `Feedback reviewed by ${review.reviewedBy} at ${review.reviewedAt}: ${dispositions.map(value => `${count(value)} ${value.replaceAll("_", " ")}`).join("; ")}. ${feedback.incomplete} attempts have incomplete collection; review does not close these gaps or change product grades.`;
+}
+
+export function renderCampaignFeedbackReview(campaign: RunnerE2ECampaign): string {
+ const summary = campaignFeedbackReviewText(campaign);
+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);
+ const rows = feedback.review.map(group => {
+ const decision = campaign.feedbackReview?.decisions.find(item => item.fingerprint === group.fingerprint);
+ return `<tr><td>${html(group.kind)}</td><td><code>${html(group.fingerprint)}</code></td><td>${group.occurrences}</td><td>${html(decision?.disposition ?? "unreviewed")}</td><td>${decision?.duplicateOf ? html(decision.duplicateOf) : ""}${decision?.issueNumber ? `<a href="https://github.com/paperclipai/paperclip/issues/${decision.issueNumber}">#${decision.issueNumber}</a>` : ""}</td></tr>`;
+ }).join("");
+ return `<section aria-label="Campaign feedback review"><h2>Campaign feedback review</h2><p>${html(summary)}</p><p>Full bodies and review rationale stay in private evidence. Exact duplicate bodies are grouped once; related causes can be linked by a reviewed duplicate disposition.</p>${rows ? `<div class="matcher-wrap" tabindex="0" role="region" aria-label="Feedback review decisions"><table><thead><tr><th>Kind</th><th>Fingerprint</th><th>Occurrences</th><th>Disposition</th><th>Follow-up</th></tr></thead><tbody>${rows}</tbody></table></div>` : ""}</section>`;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [campaignFile, outputFile] = process.argv.slice(2);
+ if (!campaignFile || !outputFile) throw new Error("Usage: feedback-dispositions.ts <normalized-results.json> <private-review.json>");
+ const campaign = JSON.parse(await readFile(campaignFile, "utf8")) as RunnerE2ECampaign;
+ await writeFile(outputFile, `${JSON.stringify(feedbackReviewTemplate(campaign), null, 2)}\n`, { mode: 0o600, flag: "wx" });
+ console.log("Created an unreviewed template. Inspect private feedback bodies, fill dispositions and review metadata, then import with PAPERCLIP_E2E_FEEDBACK_REVIEW.");
+}
diff --git a/tests/runner-e2e/feedback-persistence.test.ts b/tests/runner-e2e/feedback-persistence.test.ts
new file mode 100644
index 0000000000..ca178d36a7
--- /dev/null
+++ b/tests/runner-e2e/feedback-persistence.test.ts
@@ -0,0 +1,68 @@
+import { describe, expect, it } from "vitest";
+import { expectedFixtureFeedback, feedbackPersistenceDigest, gradeFeedbackPersistence } from "./feedback-persistence.js";
+import { runnerMatrix, runnerTasks, suiteDefinitionHash } from "./catalog.js";
+import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
+
+function fixture() {
+ const nonce = "fixture-nonce", bodies = expectedFixtureFeedback(nonce);
+ return { nonce, companyId: "company", agentId: "agent", taskId: "task", runIds: ["run"],
+ evidence: { version: 1, companyId: "company", summary: { collection: "complete", invocationCollection: "complete" },
+ entries: (["complaint", "suggestion"] as const).map(kind => ({ id: kind, kind, body: bodies[kind],
+ agent: { id: "agent" }, task: { id: "task" }, runId: "run" })),
+ invocations: (["complaint", "suggestion"] as const).map(kind => ({ callId: `${kind}-call`, kind, runId: "run", outcome: "completed" })),
+ },
+ };
+}
+
+describe("feedback persistence oracle", () => {
+ it("changes the suite hash when the inherited prompt or matchers change", () => {
+ const base = runnerTasks.find(task => task.id === "message-marker")!;
+ const suite = runnerMatrix.find(execution => execution.suite.id === "feedback-persistence")!.suite;
+ const baseline = suiteDefinitionHash(suite);
+ expect(suite.definitionMetadata?.sourceDigest).toBe(feedbackPersistenceDigest(base));
+ for (const changed of [
+ { ...base, buildPrompt: () => "A different completion contract" },
+ { ...base, buildMatchers: () => [] },
+ ]) {
+ expect(suiteDefinitionHash({ ...suite, definitionMetadata: {
+ ...suite.definitionMetadata, sourceDigest: feedbackPersistenceDigest(changed),
+ } })).not.toBe(baseline);
+ }
+ });
+ it("accepts independently retained bodies with same-run real tool receipts", () => {
+ expect(gradeFeedbackPersistence(fixture()).every(check => check.passed)).toBe(true);
+ });
+ it.each([
+ ["missing saved evidence", (f: ReturnType<typeof fixture>) => { f.evidence.entries = []; }],
+ ["prose without tool receipts", (f: ReturnType<typeof fixture>) => { f.evidence.invocations = []; }],
+ ["wrong saved body", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.body = "submitted successfully"; }],
+ ["foreign company", (f: ReturnType<typeof fixture>) => { f.evidence.companyId = "other"; }],
+ ["foreign task", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.task.id = "other"; }],
+ ["foreign agent", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.agent.id = "other"; }],
+ ["foreign run", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.runId = "other"; }],
+ ["unrelated successful receipt", (f: ReturnType<typeof fixture>) => { f.evidence.invocations[0]!.runId = "other"; }],
+ ["failed tool", (f: ReturnType<typeof fixture>) => { f.evidence.invocations[0]!.outcome = "failed"; }],
+ ["partial collection", (f: ReturnType<typeof fixture>) => { f.evidence.summary.collection = "partial"; }],
+ ["partial invocation scan", (f: ReturnType<typeof fixture>) => { f.evidence.summary.invocationCollection = "partial"; }],
+ ["duplicate submission", (f: ReturnType<typeof fixture>) => { f.evidence.invocations.push(f.evidence.invocations[0]!); }],
+ ["duplicate saved row", (f: ReturnType<typeof fixture>) => { f.evidence.entries.push(f.evidence.entries[0]!); }],
+ ] as const)("rejects %s", (_name, mutate) => {
+ const f = fixture(); mutate(f);
+ expect(gradeFeedbackPersistence(f).some(check => !check.passed)).toBe(true);
+ });
+ it("fails closed on unavailable or malformed retained evidence", () => {
+ for (const evidence of [undefined, null, {}, { entries: [null], invocations: [null] }]) {
+ expect(gradeFeedbackPersistence({ ...fixture(), evidence }).some(check => !check.passed)).toBe(true);
+ }
+ });
+ it("registers only two explicit local native cases with no automatic retry", () => {
+ const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", "feedback-persistence"]));
+ expect(selected.map(execution => execution.id)).toEqual([
+ "feedback-persistence.runner-codex.local.complaint-and-suggestion",
+ "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion",
+ ]);
+ expect(selected.every(execution => execution.task.automaticRetryPolicy === "single_attempt")).toBe(true);
+ expect(runnerMatrix).toEqual(expect.arrayContaining(selected));
+ expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "feedback-persistence")).toBe(false);
+ });
+});
diff --git a/tests/runner-e2e/feedback-persistence.ts b/tests/runner-e2e/feedback-persistence.ts
new file mode 100644
index 0000000000..fd4faf1ee7
--- /dev/null
+++ b/tests/runner-e2e/feedback-persistence.ts
@@ -0,0 +1,78 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import type { RunnerTaskFixture } from "./types.js";
+
+export const FEEDBACK_PERSISTENCE_SUITE = "feedback-persistence";
+const feedbackPersistenceSourceDigest = createHash("sha256")
+ .update(readFileSync(new URL(import.meta.url)))
+ .update(readFileSync(new URL("./catalog.ts", import.meta.url)))
+ .update(readFileSync(new URL("./feedback.ts", import.meta.url)))
+ .update(readFileSync(new URL("./runner.spec.ts", import.meta.url)))
+ .digest("hex");
+
+export function feedbackPersistenceDigest(base: RunnerTaskFixture) {
+ return createHash("sha256").update(feedbackPersistenceSourceDigest)
+ .update(JSON.stringify(base, (_key, value) => typeof value === "function" ? value.toString() : value))
+ .digest("hex");
+}
+
+export function expectedFixtureFeedback(nonce: string) {
+ return {
+ complaint: `Synthetic eval complaint ${nonce}: the fixture export omitted its receipt. This is test data, not a real product incident.`,
+ suggestion: `Synthetic eval suggestion ${nonce}: add a keyboard shortcut to the fixture preview. This is a separate test idea, not a real feature request.`,
+ };
+}
+
+export function feedbackPersistenceTask(base: RunnerTaskFixture): RunnerTaskFixture {
+ return {
+ ...base, id: "complaint-and-suggestion", label: "Retain submitted feedback", automaticRetryPolicy: "single_attempt",
+ attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 },
+ buildPrompt(nonce) {
+ const bodies = expectedFixtureFeedback(nonce);
+ return [
+ "This disposable acceptance task verifies two different feedback tools. Before completing the task, discover the feedback tools and submit exactly one complaint and one suggestion through their real tool-call channel.",
+ `Call submit_complaint with body ${JSON.stringify(bodies.complaint)} and idempotencyKey ${JSON.stringify(`complaint-${nonce}`)}.`,
+ `Call submit_suggestion with body ${JSON.stringify(bodies.suggestion)} and idempotencyKey ${JSON.stringify(`suggestion-${nonce}`)}.`,
+ "Wait for both saved acknowledgments. Do not substitute prose, a task comment, direct API calls, or files for these two tool submissions. These are distinct synthetic items; do not submit additional feedback.",
+ base.buildPrompt(nonce),
+ ].join("\n");
+ },
+ };
+}
+
+const row = (value: unknown): Record<string, unknown> =>
+ value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+
+export function gradeFeedbackPersistence(input: {
+ evidence: unknown; nonce: string; companyId: string | undefined; agentId: string | undefined;
+ taskId: string | undefined; runIds: string[];
+}) {
+ const evidence = row(input.evidence), summary = row(evidence.summary);
+ const entries = Array.isArray(evidence.entries) ? evidence.entries.map(row) : [];
+ const invocations = Array.isArray(evidence.invocations) ? evidence.invocations.map(row) : [];
+ const expected = expectedFixtureFeedback(input.nonce);
+ const checks: Array<{ id: string; passed: boolean; detail: string }> = [];
+ const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail });
+ check("complete-retained-evidence", evidence.version === 1 && Boolean(input.companyId)
+ && evidence.companyId === input.companyId && summary.collection === "complete" && summary.invocationCollection === "complete",
+ "Both collection passes completed and the saved evidence belongs to this company.");
+ check("one-of-each", entries.length === 2 && new Set(entries.map(entry => entry.id)).size === 2
+ && entries.every(entry => typeof entry.id === "string" && entry.id.length > 0),
+ "Exactly two distinct saved feedback records remain after teardown.");
+ for (const kind of ["complaint", "suggestion"] as const) {
+ const saved = entries.filter(entry => entry.kind === kind);
+ const entry = saved[0] ?? {};
+ const calls = invocations.filter(call => call.kind === kind);
+ check(`${kind}-body-and-attribution`, saved.length === 1 && entry.body === expected[kind]
+ && Boolean(input.agentId) && row(entry.agent).id === input.agentId && Boolean(input.taskId)
+ && row(entry.task).id === input.taskId && typeof entry.runId === "string" && input.runIds.includes(entry.runId),
+ "The independently read saved body has the exact nonce, agent, task, and observed run.");
+ check(`${kind}-completed-tool`, calls.length === 1 && calls[0]!.outcome === "completed"
+ && typeof calls[0]!.callId === "string" && Boolean(calls[0]!.callId)
+ && typeof entry.runId === "string" && calls[0]!.runId === entry.runId,
+ "A real completed tool receipt from the same run corroborates the saved feedback.");
+ }
+ check("only-expected-submissions", invocations.length === 2 && new Set(invocations.map(call => call.callId)).size === 2,
+ "There are no duplicate or extra feedback submissions.");
+ return checks;
+}
diff --git a/tests/runner-e2e/feedback-recovery.test.ts b/tests/runner-e2e/feedback-recovery.test.ts
new file mode 100644
index 0000000000..21e23ea200
--- /dev/null
+++ b/tests/runner-e2e/feedback-recovery.test.ts
@@ -0,0 +1,111 @@
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { afterEach, expect, it, vi } from "vitest";
+import { recoverInterruptedFeedback, readRecoveredFeedback } from "./feedback-recovery.js";
+import { mustPreserveRecoveryState } from "./cleanup-verification.js";
+import { isHistoricalBundlePathAllowed } from "./history-publish.js";
+import type { RunnerApi } from "./api.js";
+
+vi.mock("node:fs/promises", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("node:fs/promises")>();
+ return { ...actual, writeFile: vi.fn(actual.writeFile) };
+});
+
+const roots: string[] = [];
+afterEach(async () => { vi.mocked(writeFile).mockReset(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution" };
+const companyId = "88c1939b-2d9f-4bb3-8ebc-8a7a85efeeac";
+const api = (get: (url: string) => unknown): Pick<RunnerApi, "get"> => ({ async get<T>(url: string) { return get(url) as T; } });
+
+async function fixture() {
+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-feedback-recovery-")); roots.push(root);
+ await mkdir(path.join(root, "snapshots"));
+ await writeFile(path.join(root, "snapshots/fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId }));
+ return root;
+}
+
+it("retains saved feedback after worker loss while active-run coverage stays partial", async () => {
+ const root = await fixture();
+ const entry = { id: "saved", kind: "complaint", runId: "run", body: "Synthetic lost-worker complaint. private-canary-value" };
+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000,
+ secrets: ["private-canary-value"], api: api(url => {
+ if (url.includes("agent-commentary?")) return { items: [entry], total: 1 };
+ if (url.endsWith("agent-commentary/saved")) return entry;
+ if (url.endsWith("heartbeat-runs?limit=1000")) return [{ id: "run" }];
+ if (url.endsWith("heartbeat-runs/run")) return { id: "run", runtimeMode: "native", status: "running" };
+ return [];
+ }),
+ });
+ const summary = await readRecoveredFeedback(root, identity);
+ expect(summary).toMatchObject({ collection: "partial", invocationCollection: "partial", complaints: 1, bodiesCollected: 1 });
+ expect(summary?.review).toHaveLength(1);
+ const saved = await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8");
+ expect(saved).toContain("Synthetic lost-worker complaint.");
+ expect(saved).not.toContain("private-canary-value");
+ expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/agent-feedback-recovered.json")).toBe(false);
+ expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/feedback-recovery.json")).toBe(false);
+ await writeFile(path.join(root, "snapshots/agent-feedback.json"), saved);
+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [],
+ api: api(() => { throw new Error("controller stopped after the worker checkpoint"); }),
+ });
+ expect(await readRecoveredFeedback(root, identity)).toMatchObject({ complaints: 1, bodiesCollected: 1, collection: "partial" });
+ expect(await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8")).toContain("Synthetic lost-worker complaint.");
+ expect(await readRecoveredFeedback(root, { ...identity, attemptId: "different" })).toBeUndefined();
+ const receiptPath = path.join(root, "snapshots/feedback-recovery.json");
+ const receipt = JSON.parse(await readFile(receiptPath, "utf8"));
+ receipt.summary.collection = "complete";
+ await writeFile(receiptPath, JSON.stringify(receipt));
+ expect(await readRecoveredFeedback(root, identity)).toBeUndefined();
+});
+
+it("preserves completed attempt evidence without making another API request", async () => {
+ const root = await fixture();
+ await writeFile(path.join(root, "result.json"), "retained original");
+ let calls = 0;
+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [],
+ api: api(() => { calls++; throw new Error("must not read"); }),
+ });
+ expect(calls).toBe(0);
+ expect(await readFile(path.join(root, "result.json"), "utf8")).toBe("retained original");
+ expect(await readRecoveredFeedback(root, identity)).toBeUndefined();
+});
+
+it("reports the earlier atomic checkpoint when a later write exhausts the recovery deadline", async () => {
+ const root = await fixture();
+ const actual = await vi.importActual<typeof import("node:fs/promises")>("node:fs/promises");
+ let checkpointWrites = 0;
+ vi.mocked(writeFile).mockImplementation(async (file, data, options) => {
+ if (String(file).endsWith("agent-feedback-recovered.json.pending") && ++checkpointWrites === 3) {
+ const signal = typeof options === "object" && options ? options.signal : undefined;
+ if (!signal?.aborted) await new Promise(resolve => signal!.addEventListener("abort", resolve, { once: true }));
+ signal!.throwIfAborted();
+ }
+ return actual.writeFile(file, data, options);
+ });
+ const entry = { id: "saved", kind: "suggestion", runId: "run", body: "Keep this earlier durable checkpoint." };
+ await expect(recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 1000,
+ secrets: [], api: api(url => {
+ if (url.includes("agent-commentary?")) return { items: [entry], total: 1 };
+ if (url.endsWith("agent-commentary/saved")) return entry;
+ return [];
+ }),
+ })).rejects.toThrow();
+ expect(checkpointWrites).toBe(3);
+ expect(await readRecoveredFeedback(root, identity)).toMatchObject({ suggestions: 1, bodiesCollected: 1,
+ collection: "partial", invocationCollection: "unavailable" });
+ expect(await readRecoveredFeedback(root, { ...identity, attemptId: "different" })).toBeUndefined();
+ await expect(readFile(path.join(root, "snapshots/feedback-recovery.json"))).rejects.toMatchObject({ code: "ENOENT" });
+ const saved = JSON.parse(await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8"));
+ expect(saved.entries[0].body).toBe(entry.body);
+});
+
+it("reports unavailable recovery without fabricating an empty successful collection", async () => {
+ const root = await fixture();
+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [],
+ api: api(() => { throw new Error("controller unavailable"); }),
+ });
+ expect(await readRecoveredFeedback(root, identity)).toMatchObject({ collection: "unavailable", invocationCollection: "unavailable" });
+ expect(mustPreserveRecoveryState({ processCleanupFailed: false, results: [{ cleanup: "passed", synthetic: true }] })).toBe(true);
+ expect(mustPreserveRecoveryState({ processCleanupFailed: false, results: [{ cleanup: "passed", synthetic: false }] })).toBe(false);
+});
diff --git a/tests/runner-e2e/feedback-recovery.ts b/tests/runner-e2e/feedback-recovery.ts
new file mode 100644
index 0000000000..0d8c571fd9
--- /dev/null
+++ b/tests/runner-e2e/feedback-recovery.ts
@@ -0,0 +1,101 @@
+import { access, readFile, rename, writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { RunnerApi } from "./api.js";
+import { collectAttemptFeedback, emptyFeedbackSummary, type FeedbackEvidence, type FeedbackSummary } from "./feedback.js";
+import { assertSecretFree, sanitizeJson } from "./redaction.js";
+
+export interface FeedbackRecoveryIdentity {
+ campaignId: string;
+ attemptId: string;
+ executionId: string;
+}
+
+const exists = (file: string) => access(file).then(() => true, () => false);
+
+export async function recoverInterruptedFeedback(input: {
+ identity: FeedbackRecoveryIdentity;
+ caseDirectory: string;
+ api: Pick<RunnerApi, "get">;
+ secrets: readonly string[];
+ deadlineAt: number;
+}) {
+ const deadline = AbortSignal.timeout(Math.max(1, input.deadlineAt - Date.now()));
+ if (await exists(path.join(input.caseDirectory, "result.json"))) return;
+ const snapshots = path.join(input.caseDirectory, "snapshots");
+ let fixture: { executionId?: string; companyId?: string };
+ try { fixture = JSON.parse(await readFile(path.join(snapshots, "fixtures.json"), { encoding: "utf8", signal: deadline })); }
+ catch { return; }
+ if (fixture.executionId !== input.identity.executionId || !/^[a-f0-9-]{36}$/i.test(fixture.companyId ?? "")) return;
+ const companyId = fixture.companyId!;
+ let summary = emptyFeedbackSummary();
+ let checkpoint: FeedbackEvidence | undefined;
+ const recoveryReceipt = (summary: FeedbackSummary) => ({ version: 1, ...input.identity, companyId,
+ reason: "worker_exited_without_result", summary });
+ const save = async (name: string, value: unknown, signal: AbortSignal = deadline) => {
+ const text = JSON.stringify(sanitizeJson(value, input.secrets), null, 2) + "\n";
+ assertSecretFree(text, input.secrets, name);
+ await writeFile(path.join(snapshots, `${name}.pending`), text, { mode: 0o600, signal });
+ signal.throwIfAborted();
+ await rename(path.join(snapshots, `${name}.pending`), path.join(snapshots, name));
+ };
+ const saveCheckpoint = (evidence: FeedbackEvidence, signal: AbortSignal = deadline) =>
+ save("agent-feedback-recovered.json", { ...evidence, recovery: recoveryReceipt({ ...evidence.summary,
+ collection: evidence.summary.bodiesCollected ? "partial" : "unavailable",
+ invocationCollection: evidence.invocations.length ? "partial" : "unavailable" }) }, signal);
+ try {
+ const original = JSON.parse(await readFile(path.join(snapshots, "agent-feedback.json"), { encoding: "utf8", signal: deadline }));
+ if (original.version === 1 && original.companyId === companyId && original.summary?.version === 1
+ && Array.isArray(original.entries) && Array.isArray(original.invocations)
+ && original.summary.bodiesCollected === original.entries.length) {
+ await saveCheckpoint(original);
+ checkpoint = original;
+ summary = structuredClone(original.summary);
+ }
+ } catch {}
+ try {
+ const collected = await collectAttemptFeedback({ api: input.api, companyId, deadlineAt: input.deadlineAt,
+ persist: async (evidence, signal) => {
+ if (checkpoint && (!checkpoint.entries.every(entry => evidence.entries.some(next => next.id === entry.id))
+ || !checkpoint.invocations.every(call => evidence.invocations.some(next => next.runId === call.runId && next.callId === call.callId)))) return;
+ await saveCheckpoint(evidence, signal);
+ checkpoint = structuredClone(evidence);
+ summary = structuredClone(evidence.summary);
+ },
+ });
+ summary.collection = summary.bodiesCollected ? "partial" : collected.collection;
+ summary.invocationCollection = summary.invocations.completed + summary.invocations.failed + summary.invocations.unknown
+ ? "partial" : collected.invocationCollection;
+ } catch {
+ summary.collection = summary.bodiesCollected ? "partial" : "unavailable";
+ summary.invocationCollection = "unavailable";
+ }
+ await save("feedback-recovery.json", recoveryReceipt(summary));
+}
+
+export async function readRecoveredFeedback(caseDirectory: string, identity: FeedbackRecoveryIdentity): Promise<FeedbackSummary | undefined> {
+ try {
+ const snapshots = path.join(caseDirectory, "snapshots");
+ const evidence = JSON.parse(await readFile(path.join(snapshots, "agent-feedback-recovered.json"), "utf8"));
+ let receipt = evidence.recovery;
+ try { receipt = JSON.parse(await readFile(path.join(snapshots, "feedback-recovery.json"), "utf8")); }
+ catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") return; }
+ if (receipt?.version !== 1 || evidence.version !== 1 || receipt.reason !== "worker_exited_without_result"
+ || !receipt.companyId || receipt.companyId !== evidence.companyId
+ || Object.entries(identity).some(([key, value]) => receipt[key] !== value)) return;
+ const summary = evidence.summary as FeedbackSummary;
+ const statuses = ["complete", "partial", "unavailable", "not_started"];
+ if (summary?.version !== 1 || !Array.isArray(evidence.entries) || !Array.isArray(evidence.invocations)
+ || summary.bodiesCollected !== evidence.entries.length || !Array.isArray(summary.review)
+ || ![summary.complaints, summary.suggestions, summary.bodiesCollected, summary.runsInspected,
+ summary.invocations?.completed, summary.invocations?.failed, summary.invocations?.unknown]
+ .every(value => Number.isSafeInteger(value) && value >= 0)
+ || !["observed", "unknown"].includes(summary.toolAvailability)
+ || !["collection", "invocationCollection"].every(key => {
+ const original = summary[key as "collection" | "invocationCollection"];
+ const recovered = receipt.summary?.[key];
+ return statuses.includes(original) && statuses.includes(recovered)
+ && (recovered === original || recovered === "partial" || recovered === "unavailable");
+ })) return;
+ return { ...summary, collection: receipt.summary.collection, invocationCollection: receipt.summary.invocationCollection };
+ } catch { return; }
+}
diff --git a/tests/runner-e2e/feedback-shutdown-process.test.ts b/tests/runner-e2e/feedback-shutdown-process.test.ts
new file mode 100644
index 0000000000..ffd6119416
--- /dev/null
+++ b/tests/runner-e2e/feedback-shutdown-process.test.ts
@@ -0,0 +1,113 @@
+import { spawn } from "node:child_process";
+import { createRequire } from "node:module";
+import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { expect, it } from "vitest";
+import { readRecoveredFeedback } from "./feedback-recovery.js";
+import { runnerE2EWebServerCommand, runnerE2EPlaywrightInvocation, runnerE2EWebServerGracefulShutdown, runnerE2EPlaywrightShutdownSignal } from "./web-server-command.js";
+import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";
+
+it.skipIf(process.platform === "win32")("retains feedback before shutdown through the real Playwright launch chain", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "eval feedback 'shutdown-"));
+ const repository = path.resolve(import.meta.dirname, "../..");
+ const caseDirectory = path.join(root, "case");
+ const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution" };
+ const companyId = "88c1939b-2d9f-4bb3-8ebc-8a7a85efeeac";
+ let child: ReturnType<typeof spawn> | undefined;
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ let supervisorPid: number | undefined;
+ let owner: ReturnType<typeof createProcessTreeOwner> | undefined;
+ try {
+ await mkdir(path.join(root, "tests/runner-e2e"), { recursive: true });
+ await mkdir(path.join(caseDirectory, "snapshots"), { recursive: true });
+ await symlink(path.join(repository, "cli"), path.join(root, "cli"));
+ await writeFile(path.join(root, "package.json"), JSON.stringify({ type: "module" }));
+ await writeFile(path.join(caseDirectory, "snapshots/fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId }));
+ await writeFile(path.join(root, "tests/runner-e2e/server.ts"), `
+ import { createServer } from 'node:http';
+ import { writeFile } from 'node:fs/promises';
+ import { recoverInterruptedFeedback } from ${JSON.stringify(pathToFileURL(path.join(repository, "tests/runner-e2e/feedback-recovery.ts")).href)};
+ const entries = ['complaint', 'suggestion'].map(kind => ({ id: kind, kind, runId: 'run', body: 'Saved ' + kind + ' private-canary-value' }));
+ const server = createServer((req, res) => {
+ setTimeout(() => {
+ res.setHeader('content-type', 'application/json');
+ res.end(JSON.stringify(req.url.includes('agent-commentary?') ? { items: entries, total: 2 }
+ : req.url.includes('agent-commentary/') ? entries.find(entry => req.url.endsWith('/' + entry.id))
+ : req.url.endsWith('/heartbeat-runs/run') ? { id: 'run', runtimeMode: 'native', status: 'running' }
+ : req.url.includes('heartbeat-runs?') ? [{ id: 'run' }] : []));
+ }, 100);
+ });
+ let closing;
+ process.on('SIGTERM', () => closing ??= (async () => {
+ await recoverInterruptedFeedback({ identity: ${JSON.stringify(identity)}, caseDirectory: ${JSON.stringify(caseDirectory)},
+ deadlineAt: Date.now() + 5000, secrets: ['private-canary-value'], api: {
+ async get(endpoint, options) {
+ return (await fetch('http://127.0.0.1:' + server.address().port + endpoint,
+ { signal: AbortSignal.timeout(options?.timeout ?? 1000) })).json();
+ }
+ } });
+ server.close();
+ server.closeAllConnections();
+ })());
+ server.listen(0, '127.0.0.1', async () => {
+ await writeFile(${JSON.stringify(path.join(root, "ready.json"))}, JSON.stringify({ pid: process.pid }));
+ console.log('SUPERVISOR_READY');
+ });
+ `);
+ const testModule = createRequire(import.meta.url).resolve("@playwright/test");
+ const configPath = path.join(root, "playwright.config.cjs");
+ await writeFile(configPath, `module.exports = {
+ testDir: ${JSON.stringify(root)}, testMatch: 'interrupted.spec.cjs', workers: 1, reporter: 'line', timeout: 30000,
+ webServer: { command: ${JSON.stringify(runnerE2EWebServerCommand(root))},
+ wait: { stdout: /SUPERVISOR_READY/ }, timeout: 10000,
+ gracefulShutdown: ${JSON.stringify(runnerE2EWebServerGracefulShutdown)} }
+ };`);
+ await writeFile(path.join(root, "interrupted.spec.cjs"), `
+ const { test } = require(${JSON.stringify(testModule)});
+ const { writeFile } = require('node:fs/promises');
+ test('interrupted worker', async () => {
+ await writeFile(${JSON.stringify(path.join(root, "worker-ready"))}, 'ready');
+ await new Promise(() => {});
+ });
+ `);
+ const invocation = runnerE2EPlaywrightInvocation(repository, ["test", "--config", configPath]);
+ child = spawn(invocation.command, invocation.args, { cwd: repository, detached: true, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, CI: "1" } });
+ owner = createProcessTreeOwner(child);
+ let output = "";
+ child.stdout!.on("data", chunk => { output += chunk; });
+ child.stderr!.on("data", chunk => { output += chunk; });
+ const closed = new Promise<void>((resolve, reject) => {
+ child!.once("error", reject);
+ child!.once("close", () => resolve());
+ });
+ const deadline = Date.now() + 10_000;
+ let workerReady = false;
+ while ((!supervisorPid || !workerReady) && Date.now() < deadline) {
+ const ready = await readFile(path.join(root, "ready.json"), "utf8").catch(() => null);
+ if (ready) supervisorPid = JSON.parse(ready).pid;
+ workerReady = await readFile(path.join(root, "worker-ready"), "utf8").then(() => true, () => false);
+ if (!supervisorPid || !workerReady) await new Promise(resolve => setTimeout(resolve, 25));
+ }
+ expect(supervisorPid, output).toBeTypeOf("number");
+ expect(workerReady, output).toBe(true);
+ await stopOwnedProcessTree(child, owner, 10_000, 2_000, runnerE2EPlaywrightShutdownSignal);
+ await Promise.race([closed, new Promise<never>((_, reject) => {
+ timer = setTimeout(() => reject(new Error(`Shutdown did not finish: ${output}`)), 10_000);
+ })]);
+ expect(await readRecoveredFeedback(caseDirectory, identity), output).toMatchObject({
+ collection: "partial", invocationCollection: "partial", complaints: 1, suggestions: 1, bodiesCollected: 2,
+ });
+ const evidence = await readFile(path.join(caseDirectory, "snapshots/agent-feedback-recovered.json"), "utf8");
+ expect(evidence).toContain("Saved complaint");
+ expect(evidence).not.toContain("private-canary-value");
+ expect(() => process.kill(supervisorPid!, 0)).toThrow();
+ } finally {
+ owner?.stopObserving();
+ if (timer) clearTimeout(timer);
+ if (owner) await owner.signal("SIGKILL").catch(() => {});
+ if (child?.pid && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
+ await rm(root, { recursive: true, force: true });
+ }
+}, 35_000);
diff --git a/tests/runner-e2e/feedback.test.ts b/tests/runner-e2e/feedback.test.ts
new file mode 100644
index 0000000000..6cb8ead6bb
--- /dev/null
+++ b/tests/runner-e2e/feedback.test.ts
@@ -0,0 +1,319 @@
+import { expect, it, vi } from "vitest";
+import type { AgentCommentaryDetail } from "../../packages/shared/src/validators/agent-commentary.js";
+import type { RunnerApi } from "./api.js";
+import { collectAttemptFeedback, emptyFeedbackSummary, feedbackReview, observeFeedbackInvocations, summarizeCampaignFeedback, FEEDBACK_COLLECTION_LIMITS, type FeedbackEvidence } from "./feedback.js";
+import { runCleanupWithObservers } from "./cleanup-verification.js";
+import { isHistoricalBundlePathAllowed } from "./history-publish.js";
+import { canonicalProviderEventsFromOpenCodePart } from "../../packages/paperclip-runner/src/provider-events.js";
+
+function entry(id: string, body = "Please preserve the original tool error."): AgentCommentaryDetail {
+ return { id, kind: "complaint", preview: body.slice(0, 280), body,
+ createdAt: "2026-10-20T00:00:00.000Z", agent: { id: "agent", name: "Fixture", icon: null }, runId: "run", task: null };
+}
+const api = (read: (path: string) => unknown | Promise<unknown>) => ({ get: async <T>(path: string) => await read(path) as T }) as Pick<RunnerApi, "get">;
+const event = (seq: number, eventType: string, payload: unknown, runId = "run") => ({ seq, eventType,
+ payload: { prpEvent: { runId, sourceKind: "runner", eventType, payload } } });
+
+it("collects full paginated company feedback, including feedback outside the selected task, and checkpoints it", async () => {
+ const entries = Array.from({ length: 101 }, (_, i) => ({ ...entry(String(i), "a".repeat(300)), runId: `child-${i}` }));
+ const checkpoints: FeedbackEvidence[] = [];
+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { checkpoints.push(structuredClone(value)); },
+ api: api(path => {
+ if (path.includes("agent-commentary?")) {
+ const offset = Number(new URL(path, "http://fixture").searchParams.get("offset"));
+ return { items: entries.slice(offset, offset + 100).map(({ body: _body, ...rest }) => rest), total: 101 };
+ }
+ if (path.includes("agent-commentary/")) return entries[Number(path.split("/").at(-1))];
+ return [];
+ }),
+ });
+ expect(summary).toMatchObject({ collection: "complete", complaints: 101, bodiesCollected: 101, toolAvailability: "unknown" });
+ expect(summary.review).toEqual([{ fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), occurrences: 101, kind: "complaint", disposition: "needs_investigation" }]);
+ expect(checkpoints.at(-1)!.entries[100]!.body).toHaveLength(300);
+ expect(checkpoints.some(value => value.entries.length === 10)).toBe(true);
+ expect(JSON.stringify(summary)).not.toContain("aaa");
+});
+
+it("keeps saved-feedback absence separate from unavailable reads and unknown tool availability", async () => {
+ const empty = await collectAttemptFeedback({ companyId: "company", persist: async () => {},
+ api: api(path => path.includes("agent-commentary") ? { items: [], total: 0 } : []),
+ });
+ expect(empty).toMatchObject({ collection: "complete", complaints: 0, suggestions: 0, toolAvailability: "unknown" });
+ const unavailable = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(() => { throw new Error("unavailable"); }) });
+ expect(unavailable).toMatchObject({ collection: "unavailable", invocationCollection: "unavailable", complaints: 0 });
+});
+
+it("retains partial bodies and failed submissions without inventing persistence from a completed tool", async () => {
+ let retained: FeedbackEvidence | undefined;
+ const events = [
+ event(1, "tool.execution.started", { name: "submit_suggestion", executionId: "failed" }),
+ event(2, "tool.execution.completed", { name: "submit_suggestion", executionId: "failed", status: "failed" }),
+ event(3, "tool.execution.started", { name: "submit_complaint", executionId: "done" }),
+ event(4, "tool.execution.completed", { name: "submit_complaint", executionId: "done", status: "completed" }),
+ event(5, "tool.execution.started", { name: "submit_suggestion", executionId: "interrupted" }),
+ ];
+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); },
+ api: api(path => {
+ if (path.includes("agent-commentary?")) return { items: [entry("1"), entry("2")], total: 2 };
+ if (path.endsWith("agent-commentary/1")) return entry("1");
+ if (path.endsWith("agent-commentary/2")) throw new Error("body unavailable");
+ if (path.includes("/events?")) return events;
+ if (path === "/api/heartbeat-runs/run") return { id: "run", runtimeMode: "native", status: "failed" };
+ return [{ id: "run", status: "failed" }];
+ }),
+ });
+ expect(summary).toMatchObject({ collection: "partial", complaints: 2, bodiesCollected: 1, invocationCollection: "complete",
+ invocations: { completed: 1, failed: 1, unknown: 1 } });
+ expect(retained!.diagnostics).toContain("feedback_detail_failed");
+});
+
+it("binds native invocations to their run, ignores prose, and recognizes compatibility receipts", () => {
+ const events = [
+ event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "call", name: "paperclip.submit_complaint" } }),
+ event(2, "item.completed", { kind: "tool_result", item: { type: "tool_result", id: "call", status: "completed", isError: true } }),
+ event(3, "tool.execution.started", { name: "submit_suggestion", executionId: "foreign" }, "foreign-run"),
+ event(4, "item.completed", { item: { type: "agentMessage", text: "I used submit_complaint" } }),
+ ];
+ expect(observeFeedbackInvocations("run", events)).toEqual([{ runId: "run", callId: "call", kind: "complaint", outcome: "failed" }]);
+});
+
+it.each(["paperclip_", "paperclip__", "mcp__paperclip__"])("maps namespaced feedback receipts through native events: %s", prefix => {
+ const events = [
+ event(1, "tool.execution.started", { executionId: "failed", name: `${prefix}submit_complaint` }),
+ event(2, "tool.execution.completed", { executionId: "failed", name: `${prefix}submit_complaint`, status: "failed" }),
+ event(3, "item.started", { kind: "tool_call", item: { id: "done", name: `${prefix}submit_suggestion` } }),
+ event(4, "item.completed", { kind: "tool_result", item: { id: "done", status: "completed" } }),
+ event(5, "tool.execution.started", { executionId: "interrupted", name: `${prefix}submit_suggestion` }),
+ event(6, "tool.execution.started", { executionId: "unrelated", name: "other_submit_complaint" }),
+ ];
+ expect(observeFeedbackInvocations("run", events)).toEqual([
+ { runId: "run", callId: "failed", kind: "complaint", outcome: "failed" },
+ { runId: "run", callId: "done", kind: "suggestion", outcome: "completed" },
+ { runId: "run", callId: "interrupted", kind: "suggestion", outcome: "unknown" },
+ ]);
+});
+
+it.each([{ runtimeMode: "legacy", status: "succeeded" }, { runtimeMode: "native", status: "running" }])("does not claim complete invocation coverage for legacy helpers or still-active work: %o", async run => {
+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(path => {
+ if (path.includes("agent-commentary")) return { items: [], total: 0 };
+ if (path.includes("/events?")) return [];
+ if (path === "/api/heartbeat-runs/run") return { id: "run", ...run };
+ return [{ id: "run", status: run.status }];
+ }) });
+ expect(summary.invocationCollection).toBe("partial");
+});
+
+it("captures feedback after cancellation failure but before destructive teardown, and still tears down if persistence fails", async () => {
+ const calls: string[] = [];
+ const result = await runCleanupWithObservers({
+ retireRuns: async () => { throw new Error("cancel failed"); }, assertions: [],
+ teardown: async () => {
+ try {
+ await collectAttemptFeedback({ companyId: "company", api: api(() => []), persist: async () => { calls.push("capture"); throw new Error("disk full"); } });
+ } finally { calls.push("teardown"); }
+ },
+ });
+ expect(calls).toEqual(["capture", "teardown"]);
+ expect(result.errors).toHaveLength(2);
+});
+
+it("bounds collection by a deadline and retains content-free diagnostics", async () => {
+ let requests = 0, last: FeedbackEvidence | undefined;
+ const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 0,
+ persist: async value => { last = structuredClone(value); }, api: api(() => { requests++; throw new Error("secret response"); }),
+ });
+ expect(requests).toBe(0);
+ expect(summary.collection).toBe("unavailable");
+ expect(last).toBeUndefined();
+});
+
+it("preserves the read prefix when slow detail requests exhaust the read budget mid-page", async () => {
+ let now = 0, detailReads = 0;
+ const clock = vi.spyOn(Date, "now").mockImplementation(() => now);
+ const checkpoints: FeedbackEvidence[] = [];
+ const slowApi = { get: async (url: string, options: { timeout: number }) => {
+ if (url.includes("agent-commentary?")) return { items: Array.from({ length: 100 }, (_, index) => entry(String(index))), total: 100 };
+ if (!url.includes("agent-commentary/")) throw new Error("No read should start after the reserved cutoff");
+ detailReads++;
+ now += Math.min(400, options.timeout);
+ if (options.timeout < 400) throw new Error("request timeout");
+ return entry(url.split("/").at(-1)!);
+ } } as Pick<RunnerApi, "get">;
+ try {
+ const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000, api: slowApi,
+ persist: async evidence => { checkpoints.push(structuredClone(evidence)); now += 20; },
+ });
+ expect(summary).toMatchObject({ collection: "partial", invocationCollection: "unavailable" });
+ expect(summary.bodiesCollected).toBeGreaterThan(10);
+ expect(summary.bodiesCollected).toBeLessThan(100);
+ expect(checkpoints[1].entries).toHaveLength(10);
+ expect(checkpoints.at(-1)!.entries).toHaveLength(summary.bodiesCollected);
+ expect(checkpoints.at(-1)!.entries.every(value => value.body === entry("fixture").body)).toBe(true);
+ expect(checkpoints.at(-1)!.diagnostics).toContain("deadline_reached");
+ expect(detailReads).toBe(summary.bodiesCollected + 1);
+ expect(now).toBeLessThan(30_000);
+ } finally { clock.mockRestore(); }
+});
+
+it("groups identical feedback for one review without conflating kinds", () => {
+ expect(feedbackReview([entry("1", " slow request "), entry("2", "slow request"), { ...entry("3", "slow request"), kind: "suggestion" }]))
+ .toMatchObject([{ kind: "complaint", occurrences: 2 }, { kind: "suggestion", occurrences: 1 }]);
+});
+
+it("groups feedback across retries without omitting incomplete collection and keeps bodies out of public history", () => {
+ const feedback = { ...emptyFeedbackSummary(), complaints: 1, review: feedbackReview([entry("1")]) };
+ const aggregate = summarizeCampaignFeedback([{ feedback }, { feedback }, {}]);
+ expect(aggregate).toMatchObject({ complaints: 2, incomplete: 3, review: [{ occurrences: 2 }] });
+ expect(feedback.review[0]!.occurrences).toBe(1);
+ expect(JSON.stringify(aggregate)).not.toContain(entry("1").body);
+ expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/agent-feedback.json")).toBe(false);
+});
+
+it("uses run details for native coverage when the real list projection omits runtimeMode", async () => {
+ const reads: string[] = [];
+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(path => {
+ reads.push(path);
+ if (path.includes("agent-commentary")) return { items: [], total: 0 };
+ if (path.includes("/events?")) return [];
+ if (path === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" };
+ return [{ id: "run", status: "succeeded" }];
+ }) });
+ expect(reads).toContain("/api/heartbeat-runs/run");
+ expect(summary).toMatchObject({ collection: "complete", invocationCollection: "complete", runsInspected: 1 });
+});
+it("retains partial evidence if the run-detail lookup fails", async () => {
+ let retained: FeedbackEvidence | undefined;
+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, api: api(path => {
+ if (path.includes("agent-commentary")) return { items: [], total: 0 };
+ if (path.includes("/events?")) return [event(1, "tool.execution.started", { name: "submit_complaint", executionId: "call" })];
+ if (path === "/api/heartbeat-runs/run") throw new Error("unavailable");
+ return [{ id: "run", status: "succeeded" }];
+ }) });
+ expect(summary).toMatchObject({ collection: "partial", invocationCollection: "partial", invocations: { unknown: 1 }, runsInspected: 1 });
+ expect(retained!.diagnostics).toContain("run_detail_failed");
+});
+
+it("retains OpenCode feedback whose first observed update is running or terminal", () => {
+ const updates = [
+ { id: "running", status: "running", tool: "paperclip_submit_suggestion" },
+ { id: "failed", status: "error", tool: "paperclip_submit_complaint" },
+ { id: "completed", status: "completed", tool: "paperclip_submit_suggestion" },
+ ].flatMap(update => canonicalProviderEventsFromOpenCodePart({
+ id: update.id, messageID: "message", type: "tool", callID: update.id, tool: update.tool,
+ state: { status: update.status, input: {}, output: "fixture result", error: "fixture failure", time: { start: 1, end: 2 } },
+ }));
+ expect(updates.map(update => update.eventType)).toEqual(["tool.execution.progressed", "tool.execution.completed", "tool.execution.completed"]);
+ expect(observeFeedbackInvocations("run", updates.map((update, index) => event(index, update.eventType, update.payload)))).toEqual([
+ { runId: "run", callId: "running", kind: "suggestion", outcome: "unknown" },
+ { runId: "run", callId: "failed", kind: "complaint", outcome: "failed" },
+ { runId: "run", callId: "completed", kind: "suggestion", outcome: "completed" },
+ ]);
+});
+
+it("bounds retained bytes and checkpoint writes for maximum-size feedback bodies", async () => {
+ const body = "x".repeat(524_288);
+ let detailReads = 0, writtenBytes = 0, writes = 0, retained: FeedbackEvidence | undefined;
+ const summary = await collectAttemptFeedback({ companyId: "company",
+ persist: async evidence => { writes++; writtenBytes += Buffer.byteLength(JSON.stringify(evidence, null, 2)); retained = structuredClone(evidence); },
+ api: api(url => {
+ if (url.includes("agent-commentary?")) return { items: Array.from({ length: 100 }, (_, index) => entry(String(index), "preview")), total: 10_000 };
+ if (url.includes("agent-commentary/")) { detailReads++; return entry(url.split("/").at(-1)!, body); }
+ return [];
+ }),
+ });
+ expect(detailReads).toBeLessThanOrEqual(16);
+ expect(writes).toBeLessThanOrEqual(4);
+ expect(writtenBytes).toBeLessThan(3 * FEEDBACK_COLLECTION_LIMITS.retainedBytes);
+ expect(Buffer.byteLength(JSON.stringify(retained!.entries))).toBeLessThanOrEqual(FEEDBACK_COLLECTION_LIMITS.retainedBytes);
+ expect(summary.collection).toBe("partial");
+ expect(retained!.diagnostics).toContain("evidence_limit_reached");
+ expect(retained!.entries.every(item => item.body === body)).toBe(true);
+});
+
+it("caps entry count and stops requesting further feedback pages", async () => {
+ let pages = 0, retained: FeedbackEvidence | undefined;
+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async evidence => { retained = structuredClone(evidence); },
+ api: api(url => {
+ if (url.includes("agent-commentary?")) {
+ const offset = Number(new URL(url, "http://fixture").searchParams.get("offset")); pages++;
+ return { items: Array.from({ length: 100 }, (_, index) => entry(String(offset + index))), total: 2000 };
+ }
+ if (url.includes("agent-commentary/")) return entry(url.split("/").at(-1)!);
+ return [];
+ }),
+ });
+ expect(pages).toBe(10);
+ expect(summary.bodiesCollected).toBe(FEEDBACK_COLLECTION_LIMITS.entries);
+ expect(summary.collection).toBe("partial");
+ expect(retained!.diagnostics).toContain("evidence_limit_reached");
+});
+
+it("aborts stalled evidence persistence at the collection deadline", async () => {
+ let signal: AbortSignal | undefined, reads = 0;
+ await expect(collectAttemptFeedback({ companyId: "company", deadlineAt: Date.now() + 50,
+ persist: async (_evidence, currentSignal) => { signal = currentSignal; await new Promise<void>(() => {}); },
+ api: api(() => { reads++; return []; }),
+ })).rejects.toThrow("Feedback persistence deadline reached");
+ expect(signal?.aborted).toBe(true);
+ expect(reads).toBe(0);
+});
+
+it("acknowledges only successfully persisted feedback when a later checkpoint fails", async () => {
+ let retained: FeedbackEvidence | undefined, acknowledged = emptyFeedbackSummary();
+ const entries = Array.from({ length: 11 }, (_, index) => entry(String(index)));
+ await expect(collectAttemptFeedback({ companyId: "company",
+ onCheckpoint: summary => { acknowledged = summary; },
+ persist: async evidence => {
+ if (evidence.entries.length > 10) throw new Error("disk full");
+ retained = structuredClone(evidence);
+ },
+ api: api(url => url.includes("agent-commentary?") ? { items: entries, total: entries.length }
+ : entries[Number(url.split("/").at(-1))]),
+ })).rejects.toThrow("disk full");
+ expect(retained!.entries).toHaveLength(10);
+ expect(acknowledged).toEqual(retained!.summary);
+ expect(acknowledged.bodiesCollected).toBe(10);
+ expect(acknowledged.review).toMatchObject([{ occurrences: 10 }]);
+});
+
+it("returns saved review counts when the deadline expires before a later checkpoint", async () => {
+ let now = 0, retained: FeedbackEvidence | undefined;
+ const clock = vi.spyOn(Date, "now").mockImplementation(() => now);
+ const entries = Array.from({ length: 11 }, (_, index) => entry(String(index)));
+ try {
+ const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000,
+ persist: async evidence => { retained = structuredClone(evidence); },
+ api: api(url => {
+ if (url.includes("agent-commentary?")) return { items: entries, total: entries.length };
+ const index = Number(url.split("/").at(-1));
+ if (index === 10) now = 30_001;
+ return entries[index];
+ }),
+ });
+ expect(retained!.entries).toHaveLength(10);
+ expect(summary).toMatchObject({ bodiesCollected: 10, collection: "partial", invocationCollection: "unavailable" });
+ expect(summary.review).toEqual(retained!.summary.review);
+ } finally { clock.mockRestore(); }
+});
+
+it("bounds raw event inspection while retaining the observed feedback receipt prefix", async () => {
+ let pages = 0, retained: FeedbackEvidence | undefined;
+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async evidence => { retained = structuredClone(evidence); },
+ api: api(url => {
+ if (url.includes("agent-commentary?")) return { items: [], total: 0 };
+ if (url.includes("/events?")) {
+ pages++;
+ return Array.from({ length: 100 }, (_, index) => event(index + 1, "tool.execution.started", {
+ executionId: `call-${index}`, name: "submit_complaint", input: "x".repeat(200_000),
+ }));
+ }
+ if (url === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" };
+ return [{ id: "run", status: "succeeded" }];
+ }),
+ });
+ expect(pages).toBe(1);
+ expect(summary.invocationCollection).toBe("partial");
+ expect(summary.invocations.unknown).toBeGreaterThan(0);
+ expect(summary.invocations.unknown).toBeLessThan(100);
+ expect(retained!.diagnostics).toContain("evidence_limit_reached");
+});
diff --git a/tests/runner-e2e/feedback.ts b/tests/runner-e2e/feedback.ts
new file mode 100644
index 0000000000..7a24b21cb6
--- /dev/null
+++ b/tests/runner-e2e/feedback.ts
@@ -0,0 +1,268 @@
+import { createHash } from "node:crypto";
+import type { AgentCommentaryDetail, AgentCommentaryPage } from "../../packages/shared/src/validators/agent-commentary.js";
+import type { RunnerApi } from "./api.js";
+
+type Row = Record<string, unknown>;
+const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
+export type FeedbackCollectionStatus = "complete" | "partial" | "unavailable" | "not_started";
+export interface FeedbackSummary {
+ version: 1;
+ collection: FeedbackCollectionStatus;
+ invocationCollection: FeedbackCollectionStatus;
+ toolAvailability: "observed" | "unknown";
+ complaints: number;
+ suggestions: number;
+ bodiesCollected: number;
+ runsInspected: number;
+ invocations: { completed: number; failed: number; unknown: number };
+ review: Array<{ fingerprint: string; kind: "complaint" | "suggestion"; occurrences: number; disposition: "needs_investigation" }>;
+}
+interface FeedbackInvocation {
+ runId: string;
+ callId: string;
+ kind: "complaint" | "suggestion";
+ outcome: "completed" | "failed" | "unknown";
+}
+export interface FeedbackEvidence {
+ version: 1;
+ companyId: string;
+ entries: AgentCommentaryDetail[];
+ invocations: FeedbackInvocation[];
+ diagnostics: Array<"feedback_read_failed" | "feedback_detail_failed" | "feedback_pagination_incomplete" | "run_read_failed" | "run_detail_failed" | "run_limit_reached" | "events_read_failed" | "deadline_reached" | "evidence_limit_reached">;
+ summary: FeedbackSummary;
+}
+
+export const FEEDBACK_COLLECTION_LIMITS = {
+ entries: 1000, retainedBytes: 8 * 1024 * 1024, eventBytes: 16 * 1024 * 1024, events: 10_000,
+} as const;
+
+export function emptyFeedbackSummary(): FeedbackSummary {
+ return { version: 1, collection: "not_started", invocationCollection: "not_started", toolAvailability: "unknown",
+ complaints: 0, suggestions: 0, bodiesCollected: 0, runsInspected: 0,
+ invocations: { completed: 0, failed: 0, unknown: 0 }, review: [] };
+}
+
+function feedbackKind(name: unknown): FeedbackInvocation["kind"] | undefined {
+ if (typeof name !== "string") return undefined;
+ const canonicalName = name.replace(/^(?:mcp__paperclip__|paperclip__|paperclip_)/, "");
+ if (/(?:^|[.\/])submit_complaint$/.test(canonicalName)) return "complaint";
+ if (/(?:^|[.\/])submit_suggestion$/.test(canonicalName)) return "suggestion";
+ return undefined;
+}
+
+export function observeFeedbackInvocations(runId: string, events: readonly Row[]): FeedbackInvocation[] {
+ const calls = new Map<string, FeedbackInvocation>();
+ for (const event of events) {
+ const envelope = row(row(event.payload).prpEvent);
+ if (envelope.runId !== runId || envelope.sourceKind !== "runner" || envelope.eventType !== event.eventType) continue;
+ const payload = row(envelope.payload), item = row(payload.item);
+ const execution = typeof event.eventType === "string" && event.eventType.startsWith("tool.execution.");
+ const callId = execution ? payload.executionId : item.id;
+ if (typeof callId !== "string" || !callId) continue;
+ const kind = feedbackKind(execution ? payload.name : item.name);
+ const started = event.eventType === "tool.execution.started" || event.eventType === "item.started" && (payload.kind === "tool_call" || item.type === "tool_call");
+ const completed = event.eventType === "tool.execution.completed" || event.eventType === "item.completed" && (payload.kind === "tool_result" || item.type === "tool_result");
+ if ((execution || started || completed) && kind) calls.set(callId, calls.get(callId) ?? { runId, callId, kind, outcome: "unknown" });
+ const call = calls.get(callId);
+ if (!completed || !call || kind && kind !== call.kind) continue;
+ const status = execution ? payload.status : item.status;
+ if (payload.isError === true || item.isError === true || ["failed", "error", "cancelled"].includes(String(status))) call.outcome = "failed";
+ else if (status === "completed") call.outcome = "completed";
+ }
+ return [...calls.values()];
+}
+
+export function feedbackReview(entries: readonly AgentCommentaryDetail[]): FeedbackSummary["review"] {
+ const groups = new Map<string, FeedbackSummary["review"][number]>();
+ for (const entry of entries) {
+ const fingerprint = createHash("sha256").update(`${entry.kind}\n${entry.body.trim().replace(/\s+/g, " ")}`).digest("hex");
+ const group = groups.get(fingerprint);
+ if (group) group.occurrences++;
+ else groups.set(fingerprint, { fingerprint, kind: entry.kind, occurrences: 1, disposition: "needs_investigation" });
+ }
+ return [...groups.values()];
+}
+
+export function summarizeCampaignFeedback(attempts: readonly { feedback?: FeedbackSummary }[]) {
+ const review = new Map<string, FeedbackSummary["review"][number]>();
+ let complaints = 0, suggestions = 0, incomplete = 0, failedSubmissions = 0, unknownSubmissions = 0;
+ for (const { feedback } of attempts) {
+ if (!feedback || feedback.collection !== "complete" || feedback.invocationCollection !== "complete") incomplete++;
+ if (!feedback) continue;
+ complaints += feedback.complaints;
+ suggestions += feedback.suggestions;
+ failedSubmissions += feedback.invocations.failed;
+ unknownSubmissions += feedback.invocations.unknown;
+ for (const group of feedback.review) {
+ const previous = review.get(group.fingerprint);
+ if (previous) previous.occurrences += group.occurrences;
+ else review.set(group.fingerprint, { ...group });
+ }
+ }
+ return { complaints, suggestions, incomplete, failedSubmissions, unknownSubmissions, review: [...review.values()] };
+}
+
+export async function collectAttemptFeedback(input: {
+ api: Pick<RunnerApi, "get">;
+ companyId: string;
+ persist: (evidence: FeedbackEvidence, signal: AbortSignal) => Promise<void>;
+ onCheckpoint?: (summary: FeedbackSummary) => void;
+ deadlineAt?: number;
+}): Promise<FeedbackSummary> {
+ const deadlineAt = input.deadlineAt ?? Date.now() + 30_000;
+ const readDeadlineAt = deadlineAt - Math.min(5_000, Math.max(0, Math.floor((deadlineAt - Date.now()) / 5)));
+ const summary = emptyFeedbackSummary();
+ let checkpointSummary = emptyFeedbackSummary();
+ const retainedSummary = () => ({ ...structuredClone(checkpointSummary),
+ ...(summary.collection !== "complete" ? { collection: summary.collection } : {}),
+ ...(summary.invocationCollection !== "complete" ? { invocationCollection: summary.invocationCollection } : {}),
+ });
+ const evidence: FeedbackEvidence = { version: 1, companyId: input.companyId, entries: [], invocations: [], diagnostics: [], summary };
+ let retainedBytes = 0, checkpointBytes = 0, eventBytes = 0, eventCount = 0;
+ const diagnose = (code: FeedbackEvidence["diagnostics"][number]) => { if (!evidence.diagnostics.includes(code)) evidence.diagnostics.push(code); };
+ const get = <T>(url: string) => {
+ const remaining = readDeadlineAt - Date.now();
+ if (remaining <= 0) { diagnose("deadline_reached"); throw new Error("Feedback collection deadline reached"); }
+ return input.api.get<T>(url, { timeout: Math.min(5_000, remaining) });
+ };
+ const persist = async (force = false) => {
+ summary.bodiesCollected = evidence.entries.length;
+ summary.invocations = { completed: 0, failed: 0, unknown: 0 };
+ for (const invocation of evidence.invocations) summary.invocations[invocation.outcome]++;
+ summary.toolAvailability = evidence.invocations.length ? "observed" : "unknown";
+ if (Date.now() >= deadlineAt) { diagnose("deadline_reached"); return false; }
+ if (!force && (!retainedBytes || checkpointBytes > 0 && retainedBytes < Math.max(1024 * 1024, checkpointBytes * 2))) return true;
+ summary.review = feedbackReview(evidence.entries);
+ const remaining = deadlineAt - Date.now();
+ if (remaining <= 0) { diagnose("deadline_reached"); return false; }
+ const abort = new AbortController();
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ new Promise<never>((_resolve, reject) => { timer = setTimeout(() => {
+ abort.abort(); reject(new Error("Feedback persistence deadline reached"));
+ }, remaining); }),
+ input.persist(evidence, abort.signal),
+ ]);
+ if (Date.now() >= deadlineAt) { abort.abort(); diagnose("deadline_reached"); return false; }
+ checkpointBytes = retainedBytes;
+ checkpointSummary = structuredClone(summary);
+ input.onCheckpoint?.(structuredClone(checkpointSummary));
+ } finally { clearTimeout(timer); }
+ return true;
+ };
+ if (!await persist(true)) {
+ summary.collection = "unavailable";
+ summary.invocationCollection = "unavailable";
+ return summary;
+ }
+ const seen = new Set<string>();
+ let expectedTotal: number | undefined;
+ let detailsFailed = false;
+ feedbackPages: for (let offset = 0; offset < FEEDBACK_COLLECTION_LIMITS.entries; offset += 100) {
+ let page: AgentCommentaryPage;
+ try {
+ page = await get<AgentCommentaryPage>(`/api/companies/${input.companyId}/agent-commentary?limit=100&offset=${offset}`);
+ if (!Array.isArray(page.items) || !Number.isSafeInteger(page.total) || page.total < 0 || page.items.length > 100) throw new Error("Invalid feedback page");
+ } catch {
+ summary.collection = seen.size ? "partial" : "unavailable";
+ diagnose("feedback_read_failed");
+ break;
+ }
+ expectedTotal ??= page.total;
+ summary.collection = "partial";
+ for (const entry of page.items) {
+ if (Date.now() >= readDeadlineAt) { diagnose("deadline_reached"); break feedbackPages; }
+ if (seen.has(entry.id)) continue;
+ seen.add(entry.id);
+ if (entry.kind === "complaint") summary.complaints++;
+ else if (entry.kind === "suggestion") summary.suggestions++;
+ else { detailsFailed = true; continue; }
+ try {
+ const detail = await get<AgentCommentaryDetail>(`/api/companies/${input.companyId}/agent-commentary/${encodeURIComponent(entry.id)}`);
+ if (detail.id !== entry.id || detail.kind !== entry.kind || detail.runId !== entry.runId || typeof detail.body !== "string") throw new Error("Invalid feedback detail");
+ const bytes = Buffer.byteLength(JSON.stringify(detail)) + 1;
+ if (retainedBytes + bytes + 4 > FEEDBACK_COLLECTION_LIMITS.retainedBytes) { diagnose("evidence_limit_reached"); break feedbackPages; }
+ evidence.entries.push(detail);
+ retainedBytes += bytes;
+ } catch {
+ detailsFailed = true;
+ diagnose("feedback_detail_failed");
+ }
+ if (evidence.entries.length % 10 === 0 && !await persist()) break feedbackPages;
+ }
+ if (!await persist()) break;
+ if (offset + page.items.length >= page.total || page.items.length < 100) {
+ if (seen.size === page.total && expectedTotal === page.total && !detailsFailed) summary.collection = "complete";
+ else diagnose("feedback_pagination_incomplete");
+ break;
+ }
+ if (offset + 100 >= FEEDBACK_COLLECTION_LIMITS.entries) diagnose("evidence_limit_reached");
+ }
+ if (!await persist(true)) summary.collection = evidence.entries.length ? "partial" : "unavailable";
+ let runs: Array<{ id: string; status?: string }>;
+ try {
+ runs = await get(`/api/companies/${input.companyId}/heartbeat-runs?limit=1000`);
+ if (!Array.isArray(runs) || runs.length > 1000 || runs.some(run => typeof run.id !== "string")) throw new Error("Invalid runs");
+ } catch {
+ summary.invocationCollection = "unavailable";
+ diagnose("run_read_failed");
+ await persist(true);
+ return retainedSummary();
+ }
+ summary.invocationCollection = runs.length < 1000 ? "complete" : "partial";
+ if (runs.length >= 1000) diagnose("run_limit_reached");
+ for (const listed of runs) {
+ let run: { id: string; status?: string; runtimeMode?: string } = listed;
+ try {
+ const detail = await get<typeof run>(`/api/heartbeat-runs/${encodeURIComponent(listed.id)}`);
+ if (detail.id !== listed.id) throw new Error("Invalid run detail");
+ run = detail;
+ } catch {
+ summary.invocationCollection = "partial";
+ summary.collection = summary.collection === "complete" ? "partial" : summary.collection;
+ diagnose("run_detail_failed");
+ }
+ if (!["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(run.status ?? "")) {
+ summary.collection = summary.collection === "complete" ? "partial" : summary.collection;
+ summary.invocationCollection = "partial";
+ }
+ if (run.runtimeMode !== "native") summary.invocationCollection = "partial";
+ const events: Row[] = [];
+ let afterSeq = 0;
+ try {
+ for (let pageNumber = 0; pageNumber < 100; pageNumber++) {
+ const page = await get<Row[]>(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=100`);
+ if (!Array.isArray(page) || page.length > 100) throw new Error("Invalid events");
+ for (const event of page) {
+ if (!Number.isSafeInteger(event.seq) || Number(event.seq) <= afterSeq) throw new Error("Invalid event sequence");
+ const bytes = Buffer.byteLength(JSON.stringify(event));
+ if (eventCount >= FEEDBACK_COLLECTION_LIMITS.events || eventBytes + bytes > FEEDBACK_COLLECTION_LIMITS.eventBytes) {
+ diagnose("evidence_limit_reached"); throw new Error("Feedback event limit reached");
+ }
+ eventCount++; eventBytes += bytes;
+ afterSeq = Number(event.seq);
+ events.push(event);
+ }
+ if (page.length < 100) break;
+ if (pageNumber === 99) { diagnose("evidence_limit_reached"); throw new Error("Event limit reached"); }
+ }
+ summary.runsInspected++;
+ } catch {
+ summary.invocationCollection = "partial";
+ diagnose("events_read_failed");
+ }
+ for (const invocation of observeFeedbackInvocations(run.id, events)) {
+ const bytes = Buffer.byteLength(JSON.stringify(invocation)) + 1;
+ if (retainedBytes + bytes + 4 > FEEDBACK_COLLECTION_LIMITS.retainedBytes) { diagnose("evidence_limit_reached"); break; }
+ retainedBytes += bytes;
+ evidence.invocations.push(invocation);
+ }
+ if (!await persist() || evidence.diagnostics.includes("evidence_limit_reached")) {
+ summary.invocationCollection = "partial";
+ break;
+ }
+ }
+ if (!await persist(true)) summary.invocationCollection = "partial";
+ return retainedSummary();
+}
diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts
index 11c16a0792..f131ee9131 100644
--- a/tests/runner-e2e/first-task-flow.ts
+++ b/tests/runner-e2e/first-task-flow.ts
@@ -276,6 +276,7 @@ export async function setupFirstTaskFixtures(input: {
daytonaImage?: string;
deadlineAt?: number;
observe: (fixtures: LiveFixtureValues) => void;
+ onCompanyCreated?: (companyId: string) => Promise<void>;
}): Promise<FirstTaskFixtureValues> {
const { page, api, execution, nonce } = input;
await api.patch("/api/instance/settings/experimental", {
@@ -296,6 +297,7 @@ export async function setupFirstTaskFixtures(input: {
const companies = await api.get<Row[]>("/api/companies");
const company = companies.find((c) => c.name === `First task ${nonce}`);
if (!company) throw new Error("Onboarding fixture company missing");
+ await input.onCompanyCreated?.(company.id);
const fixtures = await provisionFirstTaskFixtures({
api,
execution,
diff --git a/tests/runner-e2e/github-operating-usage.test.ts b/tests/runner-e2e/github-operating-usage.test.ts
new file mode 100644
index 0000000000..dac23f7ba3
--- /dev/null
+++ b/tests/runner-e2e/github-operating-usage.test.ts
@@ -0,0 +1,64 @@
+import { expect, it } from "vitest";
+import { readGitHubOperatingUsage } from "./github-operating-usage.js";
+
+const now = Date.parse("2026-10-21T12:00:00Z"), sha = "a".repeat(40);
+const run = { id: 123, repository: { id: 456, full_name: "paperclipai/paperclip" },
+ path: ".github/workflows/runner-full-stack-e2e.yml", run_attempt: 1, status: "completed", conclusion: "failure", head_sha: sha,
+ created_at: "2026-10-21T10:00:00Z", run_started_at: "2026-10-21T10:01:00Z", updated_at: "2026-10-21T10:10:00Z" };
+const job = (id = 1) => ({ id, run_id: 123, run_attempt: 1, status: "completed", conclusion: "success", runner_id: 1,
+ runner_name: "GitHub Actions 1", runner_group_name: "GitHub Actions", labels: ["ubuntu-latest"],
+ started_at: "2026-10-21T10:01:00Z", completed_at: "2026-10-21T10:02:00Z", name: "unretained arbitrary user text" });
+const artifact = { id: 9, workflow_run: { id: 123, head_sha: sha }, size_in_bytes: 1024, expired: false,
+ created_at: "2026-10-21T10:03:00Z", expires_at: "2026-11-20T10:03:00Z", name: "unretained arbitrary artifact text" };
+function api(overrides: Record<string, any> = {}) {
+ const reads: string[] = [];
+ return { reads, read: async (endpoint: string) => {
+ reads.push(endpoint);
+ if (endpoint === "repos/paperclipai/paperclip") return { id: 456, full_name: "paperclipai/paperclip", private: false, ...overrides.repo };
+ if (endpoint.endsWith("/runs/123")) return { ...run, ...overrides.run, ...(reads.filter(row => row.endsWith("/runs/123")).length === 2 ? overrides.finalRun : {}) };
+ if (endpoint.includes("/jobs?")) {
+ const jobs = overrides.jobs ?? [job()];
+ const page = Number(new URL(`https://api.github.com/${endpoint}`).searchParams.get("page"));
+ return { total_count: jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100), ...overrides.jobPage };
+ }
+ if (endpoint.includes("/artifacts?")) return { total_count: 1, artifacts: [artifact], ...overrides.artifactPage };
+ throw new Error("Unexpected read");
+ } };
+}
+
+it("collects failed-run usage without equating job time, current rates or inventory with billed cost", async () => {
+ const source = api();
+ const usage = await readGitHubOperatingUsage("123", source.read, now);
+ expect(usage).toMatchObject({ runAttempt: 1, sourceSha: sha, queueSeconds: 60, summedJobSeconds: 60,
+ allStartedJobsStandardHosted: true, artifacts: { retainedBytes: 1024, lifetimeCoverage: "incomplete" },
+ billing: { ciComputeUsd: null, artifactStorageUsd: null, publicStandardComputeRateUsdPerMinute: "0" } });
+ expect(JSON.stringify(usage)).not.toContain("unretained");
+ expect(source.reads).toHaveLength(5);
+});
+
+it("paginates all jobs and preserves nonstandard compute as unpriced", async () => {
+ const jobs = Array.from({ length: 101 }, (_, index) => job(index + 1));
+ jobs[100]!.runner_group_name = "custom-fleet";
+ const source = api({ jobs });
+ expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 6060,
+ allStartedJobsStandardHosted: false, billing: { publicStandardComputeRateUsdPerMinute: null } });
+ expect(source.reads.some(endpoint => endpoint.includes("page=2"))).toBe(true);
+ expect((await readGitHubOperatingUsage("123", api({ repo: { private: true } }).read, now)).billing.publicStandardComputeRateUsdPerMinute).toBeNull();
+});
+
+it("rejects changing, truncated, duplicated, misattributed, rerun and uncompleted evidence", async () => {
+ for (const override of [{ run: { run_attempt: 2 } }, { run: { status: "in_progress" } }, { run: { path: "other.yml" } },
+ { finalRun: { head_sha: "b".repeat(40) } }, { jobPage: { total_count: 2 } }, { jobs: [job(), job()] },
+ { jobs: [{ ...job(), run_id: 999 }] }, { jobs: [{ ...job(), completed_at: null }] }, { jobPage: { total_count: 1001 } },
+ { artifactPage: { artifacts: [{ ...artifact, workflow_run: { id: 999, head_sha: sha } }] } },
+ { artifactPage: { artifacts: [{ ...artifact, size_in_bytes: -1 }] } }]) {
+ await expect(readGitHubOperatingUsage("123", api(override).read, now)).rejects.toThrow();
+ }
+});
+
+it("accounts skipped jobs without requiring a fictitious VM and keeps expired artifact identity", async () => {
+ const source = api({ jobs: [{ ...job(), conclusion: "skipped", runner_id: 0, started_at: null, completed_at: null }],
+ artifactPage: { artifacts: [{ ...artifact, expired: true }] } });
+ expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 0,
+ jobs: [{ runner: "not_started" }], artifacts: { retainedBytes: 0, inventory: [{ id: 9, expired: true }] } });
+});
diff --git a/tests/runner-e2e/github-operating-usage.ts b/tests/runner-e2e/github-operating-usage.ts
new file mode 100644
index 0000000000..e55ead9a50
--- /dev/null
+++ b/tests/runner-e2e/github-operating-usage.ts
@@ -0,0 +1,100 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const repository = "paperclipai/paperclip";
+type Json = Record<string, any>;
+type ReadGitHub = (endpoint: string) => Promise<Json>;
+const integer = (value: unknown) => Number.isSafeInteger(value) && Number(value) >= 0;
+const timestamp = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value));
+
+async function inventory(read: ReadGitHub, endpoint: string, field: string) {
+ let total: number | undefined;
+ const rows: Json[] = [], ids = new Set<number>();
+ for (let page = 1; page <= 10; page++) {
+ const response = await read(`${endpoint}${endpoint.includes("?") ? "&" : "?"}per_page=100&page=${page}`);
+ if (!integer(response.total_count) || response.total_count > 1_000 || total !== undefined && total !== response.total_count
+ || !Array.isArray(response[field]) || response[field].length > 100) throw new Error("Incomplete or changing GitHub inventory");
+ total = response.total_count;
+ for (const row of response[field]) {
+ if (!row || !integer(row.id) || row.id < 1 || ids.has(row.id)) throw new Error("Invalid or duplicate GitHub inventory row");
+ ids.add(row.id); rows.push(row);
+ }
+ if (rows.length === total) return rows;
+ if (rows.length > total! || response[field].length < 100) throw new Error("Incomplete GitHub inventory");
+ }
+ throw new Error("GitHub inventory exceeds collection bound");
+}
+
+export async function readGitHubOperatingUsage(runId: string, read: ReadGitHub, now = Date.now()) {
+ if (!/^[1-9][0-9]{0,15}$/.test(runId) || !Number.isSafeInteger(Number(runId)) || !Number.isFinite(now)) throw new Error("Invalid workflow run identity");
+ const prefix = `repos/${repository}`;
+ const repo = await read(prefix);
+ const run = await read(`${prefix}/actions/runs/${runId}`);
+ const validRun = (value: Json) => value.id === Number(runId) && value.repository?.id === repo.id
+ && value.repository?.full_name === repository && value.path === ".github/workflows/runner-full-stack-e2e.yml"
+ && value.run_attempt === 1 && value.status === "completed" && /^[a-f0-9]{40}$/.test(value.head_sha ?? "")
+ && timestamp(value.created_at) && timestamp(value.run_started_at) && timestamp(value.updated_at)
+ && Date.parse(value.created_at) <= Date.parse(value.run_started_at)
+ && Date.parse(value.run_started_at) <= Date.parse(value.updated_at) && Date.parse(value.updated_at) <= now;
+ if (!integer(repo.id) || repo.id < 1 || repo.full_name !== repository || typeof repo.private !== "boolean" || !validRun(run)) {
+ throw new Error("Operating usage requires an identified, completed first-attempt Product E2E workflow");
+ }
+ const rawJobs = await inventory(read, `${prefix}/actions/runs/${runId}/attempts/1/jobs`, "jobs");
+ if (!rawJobs.length) throw new Error("Missing workflow jobs");
+ const jobs = rawJobs.map(job => {
+ if (job.run_id !== run.id || job.run_attempt !== 1 || job.status !== "completed" || !integer(job.runner_id)
+ || !Array.isArray(job.labels) || job.labels.some((label: unknown) => typeof label !== "string")) throw new Error("Invalid workflow job identity");
+ const skipped = job.conclusion === "skipped" && job.runner_id === 0;
+ if (!skipped && (!timestamp(job.started_at) || !timestamp(job.completed_at)
+ || Date.parse(job.started_at) < Date.parse(run.created_at) || Date.parse(job.completed_at) > now
+ || Date.parse(job.completed_at) < Date.parse(job.started_at))) throw new Error("Incomplete workflow job timing");
+ const standard = job.runner_group_name === "GitHub Actions" && /^GitHub Actions [0-9]+$/.test(job.runner_name ?? "")
+ && job.labels.length === 1 && job.labels[0] === "ubuntu-latest";
+ return { id: job.id as number, runner: skipped ? "not_started" : standard ? "standard_github_hosted" : "other_or_unknown",
+ elapsedSeconds: skipped ? 0 : (Date.parse(job.completed_at) - Date.parse(job.started_at)) / 1_000 };
+ });
+ const rawArtifacts = await inventory(read, `${prefix}/actions/runs/${runId}/artifacts`, "artifacts");
+ const artifacts = rawArtifacts.map(artifact => {
+ if (artifact.workflow_run?.id !== run.id || artifact.workflow_run?.head_sha !== run.head_sha || !integer(artifact.size_in_bytes)
+ || typeof artifact.expired !== "boolean" || !timestamp(artifact.created_at) || !timestamp(artifact.expires_at)
+ || Date.parse(artifact.created_at) < Date.parse(run.created_at) || Date.parse(artifact.created_at) > now
+ || Date.parse(artifact.expires_at) < Date.parse(artifact.created_at)) throw new Error("Invalid workflow artifact identity or retention");
+ return { id: artifact.id as number, sizeBytes: artifact.size_in_bytes as number,
+ createdAt: artifact.created_at as string, expiresAt: artifact.expires_at as string, expired: artifact.expired as boolean };
+ });
+ const finalRun = await read(`${prefix}/actions/runs/${runId}`);
+ if (!validRun(finalRun) || ["id", "head_sha", "run_attempt", "status", "updated_at", "conclusion"].some(key => finalRun[key] !== run[key])) {
+ throw new Error("Workflow changed during operating usage collection");
+ }
+ const retainedBytes = artifacts.filter(row => !row.expired).reduce((sum, row) => sum + row.sizeBytes, 0);
+ if (!Number.isSafeInteger(retainedBytes)) throw new Error("Artifact inventory byte count overflow");
+ return {
+ version: 1, source: "github_actions", repository, repositoryId: repo.id as number,
+ observedAt: new Date(now).toISOString(), repositoryCurrentlyPublic: !repo.private,
+ runId: run.id as number, runAttempt: 1, sourceSha: run.head_sha as string,
+ workflowUrl: `https://github.com/${repository}/actions/runs/${runId}`,
+ startedAt: run.run_started_at as string, completedSnapshotAt: run.updated_at as string,
+ queueSeconds: (Date.parse(run.run_started_at) - Date.parse(run.created_at)) / 1_000,
+ summedJobSeconds: jobs.reduce((sum, row) => sum + row.elapsedSeconds, 0),
+ allStartedJobsStandardHosted: jobs.every(row => row.runner !== "other_or_unknown"), jobs,
+ artifacts: { inventory: artifacts, retainedBytes, lifetimeCoverage: "incomplete" },
+ billing: { ciComputeUsd: null, artifactStorageUsd: null, classification: "unknown",
+ publicStandardComputeRateUsdPerMinute: !repo.private && jobs.every(row => row.runner !== "other_or_unknown") ? "0" : null,
+ rateSource: "https://docs.github.com/en/billing/concepts/product-billing/github-actions",
+ limitations: ["Current repository visibility does not prove visibility throughout the billing window.",
+ "Job elapsed time is not VM billed lifetime; queue and summed job time are separate.",
+ "Artifacts already deleted are absent; returned compressed bytes and expiry are an inventory, not billed byte-hours.",
+ "Caches, logs, external S3 history, model calls, observability and shared subscriptions require separate evidence."] },
+ };
+}
+
+export async function collectGitHubOperatingUsage(runId: string) {
+ const run = promisify(execFile);
+ return readGitHubOperatingUsage(runId, async endpoint => {
+ try {
+ const result = await run("gh", ["api", "--hostname", "github.com", "--method", "GET", endpoint],
+ { timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: "utf8" });
+ return JSON.parse(result.stdout) as Json;
+ } catch { throw new Error("GitHub operating usage read failed; no retry or paid dispatch was attempted"); }
+ });
+}
diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts
index c52a430276..2ea3cd91a4 100644
--- a/tests/runner-e2e/harness-env.ts
+++ b/tests/runner-e2e/harness-env.ts
@@ -11,6 +11,9 @@ const AMBIENT_PAPERCLIP_CREDENTIAL_KEYS = [
"PAPERCLIP_API_KEY",
"PAPERCLIP_AGENT_API_KEY",
"PAPERCLIP_TASK_BRIDGE_TOKEN",
+ "PAPERCLIP_E2E_GATEWAY_TOKEN",
+ "PAPERCLIP_E2E_GATEWAY_PHASE_PATH",
+ "PAPERCLIP_E2E_OTLP_RELAY_URL",
"PAPERCLIP_SETUP_TOKEN",
"PAPERCLIP_SECRETS_MASTER_KEY",
"PAPERCLIP_SECRETS_MASTER_KEY_FILE",
@@ -28,7 +31,25 @@ const AMBIENT_EXTERNAL_STATE_KEYS = [
"PAPERCLIP_STORAGE_S3_PREFIX",
"PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE",
] as const;
-const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/;
+const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|BRAINTRUST|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/;
+
+export function applyEvalGatewayEnvironment(env: NodeJS.ProcessEnv, credential: string, gateway: {
+ token: string; baseUrl: string; phasePath: string; observability?: { token: string; baseUrl: string };
+}) {
+ for (const name of Object.keys(env)) {
+ if (CREDENTIAL_NAMES.includes(name as typeof CREDENTIAL_NAMES[number])
+ || PROVIDER_SECRET_KEY.test(name) && /KEY|TOKEN|SECRET|PASSWORD/.test(name)) delete env[name];
+ }
+ delete env.PAPERCLIP_E2E_OTLP_RELAY_URL;
+ env[credential] = gateway.token;
+ env.PAPERCLIP_E2E_GATEWAY_BASE_URL = gateway.baseUrl;
+ env.PAPERCLIP_E2E_GATEWAY_TOKEN = gateway.token;
+ env.PAPERCLIP_E2E_GATEWAY_PHASE_PATH = gateway.phasePath;
+ if (gateway.observability) {
+ env.BRAINTRUST_API_KEY = gateway.observability.token;
+ env.PAPERCLIP_E2E_OTLP_RELAY_URL = gateway.observability.baseUrl;
+ }
+}
export function runnerE2EServerControlPaths(temporaryRoot: string) {
const controlDirectory = path.join(temporaryRoot, "control");
@@ -110,6 +131,8 @@ export function buildRunnerE2EProcessEnvironment(
const installed = usesInstalledCli(source);
if (installed) assertInstalledCliSelection(source, executions);
const result = { ...source };
+ delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH;
+ delete result.PAPERCLIP_E2E_OTLP_RELAY_URL;
// Announcements are unrelated to the scenarios and obscure screenshot evidence.
result.PAPERCLIP_ANNOUNCEMENTS_ENABLED = "false";
delete result.OPENCODE_ALLOW_ALL_MODELS;
diff --git a/tests/runner-e2e/history-budget.test.ts b/tests/runner-e2e/history-budget.test.ts
new file mode 100644
index 0000000000..3b6b1aa190
--- /dev/null
+++ b/tests/runner-e2e/history-budget.test.ts
@@ -0,0 +1,170 @@
+import { mkdir, mkdtemp, readFile, rm, truncate, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { BoundedHistoryPublisher, checkHistoryPayload, HISTORY_LIMITS, validateBoundedHistoryRetention } from "./history-budget.js";
+import { createBundleManifest, uploadImmutableBundle } from "./history-publish.js";
+import { buildRunnerCampaign, campaignHistoryRecord, emptyRunnerHistory, mergeRunnerHistory } from "./history.js";
+
+const roots: string[] = [];
+async function temporary() {
+ const root = await mkdtemp(path.join(os.tmpdir(), "history-budget-"));
+ roots.push(root);
+ return root;
+}
+afterEach(async () => {
+ vi.restoreAllMocks();
+ await Promise.all(roots.splice(0).map(root => rm(root, { force: true, recursive: true })));
+});
+
+function lifecycle() {
+ return { Rules: [
+ { Status: "Enabled", Filter: { Prefix: "" }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 7 } },
+ { Status: "Enabled", Filter: { Prefix: "runner-e2e/bounded-v1/" }, Expiration: { Days: 30 },
+ NoncurrentVersionExpiration: { NoncurrentDays: 1 }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } },
+ { Status: "Enabled", Filter: { Prefix: "runner-e2e/bounded-v1/" }, Expiration: { ExpiredObjectDeleteMarker: true } },
+ ] };
+}
+function providerError(code: string) { return Object.assign(new Error("provider failure"), { stderr: `An error occurred (${code})` }); }
+
+describe("bounded public history", () => {
+ it("rejects excess bytes, count, invalid sizes and duplicate paths", () => {
+ expect(checkHistoryPayload([{ path: "a", bytes: 8 }], 8, 1, "report")).toEqual({ files: 1, bytes: 8 });
+ for (const files of [[{ path: "a", bytes: 9 }], [{ path: "a", bytes: -1 }],
+ [{ path: "a", bytes: 0 }, { path: "b", bytes: 0 }], [{ path: "a", bytes: 0 }, { path: "a", bytes: 0 }]]) {
+ expect(() => checkHistoryPayload(files, 8, 1, "report")).toThrow();
+ }
+ });
+
+ it("rejects oversized real bundle files before hashing them", async () => {
+ const root = await temporary();
+ await writeFile(path.join(root, "index.html"), "");
+ await truncate(path.join(root, "index.html"), HISTORY_LIMITS.campaignBytes + 1);
+ await expect(createBundleManifest(root, "oversized", false, new Set(), true)).rejects.toThrow("publication boundary");
+ });
+
+ it("uploads an allowlisted uppercase PNG with the same raster content type", async () => {
+ const root = await temporary(); const bundle = path.join(root, "bundle");
+ const screenshot = "evidence/execution/attempt-1/shot.PNG";
+ await mkdir(path.dirname(path.join(bundle, screenshot)), { recursive: true });
+ await writeFile(path.join(bundle, screenshot), Buffer.from("89504e470d0a1a0a0000000d494844520000000100000001", "hex"));
+ const manifest = await createBundleManifest(bundle, "uppercase", false, new Set([screenshot]), true);
+ const runner = vi.fn(async () => ({ stdout: "{}" }));
+ await uploadImmutableBundle("bucket", "campaigns/uppercase", bundle, manifest,
+ new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner));
+ expect(runner).toHaveBeenCalledWith(expect.arrayContaining(["--content-type", "image/png", "--key", `campaigns/uppercase/${screenshot}`]), expect.any(Number));
+ });
+
+ it("requires expiration for both current and noncurrent versions in the isolated prefix", () => {
+ expect(() => validateBoundedHistoryRetention(lifecycle(), "runner-e2e/bounded-v1")).not.toThrow();
+ expect(() => validateBoundedHistoryRetention(lifecycle(), "runner-e2e")).toThrow("isolated");
+ for (const mutate of [
+ (config: ReturnType<typeof lifecycle>) => { config.Rules[1]!.Status = "Disabled"; },
+ (config: ReturnType<typeof lifecycle>) => { config.Rules[1]!.NoncurrentVersionExpiration = undefined; },
+ (config: ReturnType<typeof lifecycle>) => { Object.assign(config.Rules[1]!.NoncurrentVersionExpiration!, { NewerNoncurrentVersions: 10 }); },
+ (config: ReturnType<typeof lifecycle>) => { Object.assign(config.Rules[1]!.Filter, { Tag: { Key: "optional", Value: "true" } }); },
+ (config: ReturnType<typeof lifecycle>) => { config.Rules.pop(); },
+ ]) {
+ const config = lifecycle(); mutate(config);
+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("expiry");
+ }
+ });
+
+ it("rejects overlapping transitions or earlier expiration without affecting unrelated prefixes", () => {
+ const config = lifecycle();
+ Object.assign(config.Rules[0]!, { Transitions: [{ Days: 1, StorageClass: "GLACIER" }] });
+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("overlapping");
+ config.Rules[0]!.Filter.Prefix = "storybook/";
+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).not.toThrow();
+ Object.assign(config.Rules[2]!.Expiration!, { Days: 1 });
+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("overlapping");
+ });
+
+ it("rebuilds pointers after removing old, future and excess history records", () => {
+ const now = Date.parse("2026-10-21T12:00:00Z");
+ const record = (id: string, age: number) => campaignHistoryRecord(buildRunnerCampaign({ campaignId: id,
+ generatedAt: new Date(now - age * 86_400_000).toISOString(), expected: [], results: [] }), "https://example.test/bounded-v1");
+ const old = { ...record("expired-green", 29), complete: true, failed: 0 };
+ const current = { ...emptyRunnerHistory(), campaigns: [old, record("future", -2), record("yesterday", 1), record("earlier", 2)] };
+ const result = mergeRunnerHistory(current, record("today", 0), { now, maxAgeMs: 28 * 86_400_000, maxCampaigns: 2 });
+ expect(result.campaigns.map(row => row.campaignId)).toEqual(["today", "yesterday"]);
+ expect(result.latestCampaignId).toBe("today");
+ expect(result.latestGreenCampaignId).not.toBe("expired-green");
+ expect(current.campaigns).toHaveLength(4);
+ });
+
+ it("distinguishes absent lock/replication from denied inspection", async () => {
+ const root = await temporary();
+ const runner = vi.fn(async (args: string[]) => {
+ if (args[1] === "get-bucket-versioning") return { stdout: '{"Status":"Enabled"}' };
+ if (args[1] === "get-bucket-lifecycle-configuration") return { stdout: JSON.stringify(lifecycle()) };
+ throw providerError(args[1] === "get-object-lock-configuration" ? "ObjectLockConfigurationNotFoundError" : "ReplicationConfigurationNotFoundError");
+ });
+ await new BoundedHistoryPublisher(path.join(root, "allowed.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1");
+ runner.mockImplementation(async () => { throw providerError("AccessDenied"); });
+ await expect(new BoundedHistoryPublisher(path.join(root, "denied.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1")).rejects.toThrow();
+ });
+
+ it.each(["get-object-lock-configuration", "get-bucket-replication"])("rejects configured %s", async blocked => {
+ const root = await temporary();
+ const runner = async (args: string[]) => {
+ if (args[1] === "get-bucket-versioning") return { stdout: '{"Status":"Enabled"}' };
+ if (args[1] === "get-bucket-lifecycle-configuration") return { stdout: JSON.stringify(lifecycle()) };
+ if (args[1] === blocked) return { stdout: "{}" };
+ throw providerError("ObjectLockConfigurationNotFoundError");
+ };
+ await expect(new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1")).rejects.toThrow("cannot prove");
+ });
+
+ it("bounds history reads and preserves permission errors", async () => {
+ const root = await temporary(); const destination = path.join(root, "download.json");
+ const runner = vi.fn(async (args: string[]) => {
+ expect(args).toContain(`bytes=0-${HISTORY_LIMITS.jsonBytes}`);
+ await writeFile(destination, ""); await truncate(destination, HISTORY_LIMITS.jsonBytes + 1);
+ return { stdout: "{}" };
+ });
+ const publisher = new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner);
+ await expect(publisher.downloadJson("bucket", "history.json", destination)).rejects.toThrow("read boundary");
+ runner.mockImplementation(async () => { throw providerError("NoSuchKey"); });
+ await expect(publisher.downloadJson("bucket", "missing", destination)).resolves.toBeNull();
+ runner.mockImplementation(async () => { throw providerError("AccessDenied"); });
+ await expect(publisher.downloadJson("bucket", "denied", destination)).rejects.toThrow();
+ });
+
+ it("retains an attempted upload before dispatch and preserves unknown failures", async () => {
+ const root = await temporary(); const receipt = path.join(root, "receipt.json"); const source = path.join(root, "source.json");
+ await writeFile(source, "12345");
+ const runner = vi.fn(async (args: string[], timeout: number) => {
+ expect(args).toContain("--if-none-match"); expect(args).toContain("STANDARD"); expect(timeout).toBeLessThanOrEqual(30_000);
+ expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ attemptedSourceBytes: 5, requests: [{ status: "unknown" }] });
+ throw new Error("lost response");
+ });
+ const publisher = new BoundedHistoryPublisher(receipt, runner);
+ await expect(publisher.upload("bucket", "campaign/a", source, "application/json", "no-cache", true)).rejects.toThrow("lost response");
+ expect(runner).toHaveBeenCalledTimes(1);
+ expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ billedUsd: null, attemptedSourceBytes: 5,
+ requests: [{ operation: "put-object", status: "failed_or_unknown", sourceBytes: 5 }] });
+ });
+
+ it("stops before another command when the request or elapsed-time boundary is reached", async () => {
+ const root = await temporary(); const runner = vi.fn(async () => ({ stdout: "{}" }));
+ const publisher = new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner);
+ for (let i = 0; i < HISTORY_LIMITS.requests; i++) await publisher.run(["s3api", "head-object"]);
+ await expect(publisher.run(["s3api", "head-object"])).rejects.toThrow("allowance");
+ expect(runner).toHaveBeenCalledTimes(HISTORY_LIMITS.requests);
+ const timed = new BoundedHistoryPublisher(path.join(root, "timed.json"), runner);
+ vi.spyOn(Date, "now").mockReturnValue(Date.now() + HISTORY_LIMITS.deadlineMs + 1);
+ await expect(timed.run(["s3api", "head-object"])).rejects.toThrow("allowance");
+ expect(runner).toHaveBeenCalledTimes(HISTORY_LIMITS.requests);
+ });
+
+ it("retains the object identity and provider version separately from billed cost", async () => {
+ const root = await temporary(); const receipt = path.join(root, "receipt.json"); const source = path.join(root, "source.json");
+ await writeFile(source, "{}");
+ const publisher = new BoundedHistoryPublisher(receipt, async () => ({ stdout: '{"VersionId":"version-1","ETag":"etag-1"}' }));
+ await publisher.upload("bucket", "campaign/result.json", source, "application/json", "no-cache", true);
+ expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ billedUsd: null, requests: [{
+ operation: "put-object", status: "accepted", bucket: "bucket", key: "campaign/result.json", versionId: "version-1", etag: "etag-1", sourceBytes: 2,
+ }] });
+ });
+});
diff --git a/tests/runner-e2e/history-budget.ts b/tests/runner-e2e/history-budget.ts
new file mode 100644
index 0000000000..b7ffdf89d1
--- /dev/null
+++ b/tests/runner-e2e/history-budget.ts
@@ -0,0 +1,177 @@
+import { execFile } from "node:child_process";
+import { open, readFile, rename, stat } from "node:fs/promises";
+import { promisify } from "node:util";
+
+export const HISTORY_LIMITS = Object.freeze({
+ campaignBytes: 64 * 1024 * 1024,
+ campaignFiles: 256,
+ mutableBytes: 16 * 1024 * 1024,
+ pagesBytes: 64 * 1024 * 1024,
+ jsonBytes: 4 * 1024 * 1024,
+ indexDays: 28,
+ indexCampaigns: 64,
+ requests: 280,
+ deadlineMs: 10 * 60_000,
+});
+
+export function checkHistoryPayload(
+ files: readonly { path: string; bytes: number }[],
+ maxBytes: number,
+ maxFiles: number,
+ label: string,
+) {
+ let bytes = 0;
+ const paths = new Set<string>();
+ for (const file of files) {
+ if (paths.has(file.path) || !Number.isSafeInteger(file.bytes) || file.bytes < 0) {
+ throw new Error(`${label} contains invalid or duplicate file metadata`);
+ }
+ paths.add(file.path);
+ bytes += file.bytes;
+ }
+ if (!Number.isSafeInteger(bytes) || bytes > maxBytes || files.length > maxFiles) {
+ throw new Error(`${label} exceeds the publication boundary (${files.length} files, ${bytes} bytes)`);
+ }
+ return { files: files.length, bytes };
+}
+
+interface LifecycleRule {
+ Status?: string;
+ Prefix?: string;
+ Filter?: { Prefix?: string; And?: { Prefix?: string } };
+ Expiration?: { Days?: number; Date?: string; ExpiredObjectDeleteMarker?: boolean };
+ NoncurrentVersionExpiration?: { NoncurrentDays?: number; NewerNoncurrentVersions?: number };
+ AbortIncompleteMultipartUpload?: { DaysAfterInitiation?: number };
+ Transitions?: unknown[];
+ NoncurrentVersionTransitions?: unknown[];
+}
+
+export function validateBoundedHistoryRetention(configuration: unknown, prefix: string) {
+ const rules = (configuration as { Rules?: LifecycleRule[] })?.Rules;
+ if (!Array.isArray(rules) || !prefix.endsWith("/bounded-v1")) {
+ throw new Error("Bounded history requires an isolated bounded-v1 prefix and lifecycle rules");
+ }
+ const scoped = rules.filter(rule => rule.Status === "Enabled" &&
+ rule.Filter?.Prefix === `${prefix}/` && Object.keys(rule.Filter).length === 1);
+ const retention = scoped.some(rule => rule.Expiration?.Days === 30 &&
+ rule.NoncurrentVersionExpiration?.NoncurrentDays === 1 &&
+ rule.NoncurrentVersionExpiration?.NewerNoncurrentVersions === undefined &&
+ rule.AbortIncompleteMultipartUpload?.DaysAfterInitiation === 1);
+ const markers = scoped.some(rule => rule.Expiration?.ExpiredObjectDeleteMarker === true);
+ if (!retention || !markers) {
+ throw new Error("Bounded history needs 30-day current, 1-day noncurrent, delete-marker and multipart expiry");
+ }
+ for (const rule of rules.filter(rule => rule.Status === "Enabled")) {
+ const filterPrefix = rule.Filter?.Prefix ?? rule.Filter?.And?.Prefix ?? rule.Prefix ?? "";
+ if (!`${prefix}/`.startsWith(filterPrefix) && !filterPrefix.startsWith(`${prefix}/`)) continue;
+ if (rule.Transitions?.length || rule.NoncurrentVersionTransitions?.length ||
+ (rule.Expiration && !scoped.includes(rule))) {
+ throw new Error("An overlapping lifecycle rule changes bounded history retention or storage class");
+ }
+ if (rule.Expiration?.Days !== undefined && rule.Expiration.Days !== 30 || rule.Expiration?.Date) {
+ throw new Error("An overlapping lifecycle rule changes bounded history expiration");
+ }
+ }
+}
+
+type AwsRunner = (args: string[], timeout: number) => Promise<{ stdout: string }>;
+const execFileAsync = promisify(execFile);
+const defaultRunner: AwsRunner = async (args, timeout) => execFileAsync("aws", args, {
+ timeout, maxBuffer: 1024 * 1024,
+ env: { ...process.env, AWS_MAX_ATTEMPTS: "1", AWS_RETRY_MODE: "standard", AWS_PAGER: "" },
+});
+
+function errorDetail(error: unknown) {
+ return String((error as { stderr?: string })?.stderr ?? (error instanceof Error ? error.message : error));
+}
+
+export class BoundedHistoryPublisher {
+ private readonly startedAt = Date.now();
+ private readonly observations: Array<{ operation: string; status: string; bucket?: string; key?: string; sourceBytes?: number; versionId?: string; etag?: string }> = [];
+
+ constructor(private readonly receipt: string, private readonly runner: AwsRunner = defaultRunner) {}
+
+ async run(args: string[], sourceBytes?: number) {
+ const remaining = HISTORY_LIMITS.deadlineMs - (Date.now() - this.startedAt);
+ if (remaining <= 0 || this.observations.length >= HISTORY_LIMITS.requests) {
+ throw new Error("Bounded history publisher exhausted its time or request allowance");
+ }
+ const argument = (name: string) => args.includes(name) ? args[args.indexOf(name) + 1] : undefined;
+ const observation: typeof this.observations[number] = { operation: args[1]!, status: "unknown",
+ bucket: argument("--bucket"), key: argument("--key"), ...(sourceBytes === undefined ? {} : { sourceBytes }) };
+ this.observations.push(observation);
+ await this.save();
+ try {
+ const result = await this.runner([...args, "--cli-connect-timeout", "10", "--cli-read-timeout", "30"], Math.min(30_000, remaining));
+ observation.status = "accepted";
+ if (observation.operation === "put-object") {
+ const response = JSON.parse(result.stdout) as { VersionId?: unknown; ETag?: unknown };
+ if (typeof response.VersionId === "string") observation.versionId = response.VersionId.slice(0, 1024);
+ if (typeof response.ETag === "string") observation.etag = response.ETag.slice(0, 1024);
+ }
+ return result;
+ } catch (error) {
+ observation.status = /\((?:NoSuchKey|404|NotFound|ObjectLockConfigurationNotFoundError|ReplicationConfigurationNotFoundError)\)/.test(errorDetail(error))
+ ? "absent" : "failed_or_unknown";
+ throw error;
+ } finally {
+ await this.save();
+ }
+ }
+
+ async save() {
+ const file = await open(`${this.receipt}.tmp`, "w", 0o600);
+ try {
+ await file.writeFile(`${JSON.stringify({
+ schema: "paperclip.runner-e2e.history-publication/v1",
+ startedAt: new Date(this.startedAt).toISOString(), observedAt: new Date().toISOString(),
+ workflowRunId: process.env.GITHUB_RUN_ID ?? null, workflowAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
+ limits: HISTORY_LIMITS, billedUsd: null, requests: this.observations,
+ attemptedSourceBytes: this.observations.reduce((sum, entry) => sum + (entry.sourceBytes ?? 0), 0),
+ }, null, 2)}\n`);
+ await file.sync();
+ } finally {
+ await file.close();
+ }
+ await rename(`${this.receipt}.tmp`, this.receipt);
+ }
+
+ async verifyRetention(bucket: string, prefix: string) {
+ const config = async (operation: string) => JSON.parse((await this.run(["s3api", operation, "--bucket", bucket])).stdout);
+ if ((await config("get-bucket-versioning")).Status !== "Enabled") {
+ throw new Error("Bounded history requires versioning-enabled storage");
+ }
+ validateBoundedHistoryRetention(await config("get-bucket-lifecycle-configuration"), prefix);
+ for (const [operation, absence] of [
+ ["get-object-lock-configuration", "ObjectLockConfigurationNotFoundError"],
+ ["get-bucket-replication", "ReplicationConfigurationNotFoundError"],
+ ]) {
+ try {
+ await config(operation!);
+ } catch (error) {
+ if (errorDetail(error).includes(`(${absence})`)) continue;
+ throw error;
+ }
+ throw new Error(`Bounded history cannot prove expiration with ${operation} configured`);
+ }
+ }
+
+ async downloadJson<T>(bucket: string, key: string, destination: string): Promise<T | null> {
+ try {
+ await this.run(["s3api", "get-object", "--bucket", bucket, "--key", key,
+ "--range", `bytes=0-${HISTORY_LIMITS.jsonBytes}`, destination]);
+ } catch (error) {
+ if (/\((?:NoSuchKey|404|NotFound)\)/.test(errorDetail(error))) return null;
+ throw error;
+ }
+ if ((await stat(destination)).size > HISTORY_LIMITS.jsonBytes) throw new Error("History JSON exceeds the read boundary");
+ return JSON.parse(await readFile(destination, "utf8")) as T;
+ }
+
+ async upload(bucket: string, key: string, file: string, contentType: string, cacheControl: string, immutable = false) {
+ const bytes = (await stat(file)).size;
+ return this.run(["s3api", "put-object", "--bucket", bucket, "--key", key, "--body", file,
+ "--content-length", String(bytes), "--content-type", contentType, "--cache-control", cacheControl,
+ "--storage-class", "STANDARD", ...(immutable ? ["--if-none-match", "*"] : [])], bytes);
+ }
+}
diff --git a/tests/runner-e2e/history-index.ts b/tests/runner-e2e/history-index.ts
index 5c72cb9dde..2845cc834f 100644
--- a/tests/runner-e2e/history-index.ts
+++ b/tests/runner-e2e/history-index.ts
@@ -25,7 +25,8 @@ function usd(value: number | null) {
return `$${value.toFixed(value < 0.01 ? 6 : 2)}`;
}
-function duration(durationMs: number) {
+function duration(durationMs: number | null) {
+ if (durationMs === null) return "Unknown";
const seconds = Math.round(durationMs / 1_000);
if (seconds < 60) return `${seconds}s`;
const minutes = Math.floor(seconds / 60);
diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts
index 80159a5f0a..a39dee6ab7 100644
--- a/tests/runner-e2e/history-publish.ts
+++ b/tests/runner-e2e/history-publish.ts
@@ -1,4 +1,6 @@
-import { validateRetainedRunnerResult } from "./result-validation.js";
+import { validateCampaignAccounting, validateRetainedRunnerResult } from "./result-validation.js";
+import { validateEvalOperationsReport } from "./operations-report.js";
+import { validateCampaignFeedbackReview } from "./feedback-dispositions.js";
import { createHash } from "node:crypto";
import { execFile } from "node:child_process";
import path from "node:path";
@@ -22,6 +24,7 @@ import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
import { renderRunnerHistoryIndex } from "./history-index.js";
import { validateHistoryDestination } from "./history-destination.js";
import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js";
+import { BoundedHistoryPublisher, checkHistoryPayload, HISTORY_LIMITS } from "./history-budget.js";
import {
campaignHistoryRecord,
emptyRunnerHistory,
@@ -277,10 +280,12 @@ async function relativeFiles(
current = root,
allowPublicSummary = false,
publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,
+ boundary?: { entries: number },
): Promise<string[]> {
const entries = await readdir(current, { withFileTypes: true });
const files: string[] = [];
for (const entry of entries) {
+ if (boundary && ++boundary.entries > 1024) throw new Error("Public bundle exceeds the directory-entry boundary");
const absolute = path.join(current, entry.name);
if (entry.isSymbolicLink()) {
throw new Error(`Refusing to publish symbolic link ${entry.name}`);
@@ -292,6 +297,7 @@ async function relativeFiles(
absolute,
allowPublicSummary,
publicScreenshots,
+ boundary,
)),
);
} else if (entry.isFile()) {
@@ -319,11 +325,17 @@ export async function createBundleManifest(
campaignId: string,
allowPublicSummary = false,
publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,
+ bounded = false,
): Promise<BundleManifest> {
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(campaignId)) {
throw new Error("Campaign ID is unsafe for immutable object storage");
}
- const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots);
+ const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots, bounded ? { entries: 0 } : undefined);
+ if (bounded) {
+ checkHistoryPayload(await Promise.all(retainedPaths.map(async file => ({
+ path: file, bytes: (await stat(path.join(root, file))).size,
+ }))), HISTORY_LIMITS.campaignBytes, HISTORY_LIMITS.campaignFiles, "Public campaign");
+ }
const retained = new Set(retainedPaths);
const missingScreenshots = [...publicScreenshots].filter((relative) => !retained.has(relative));
if (missingScreenshots.length > 0) {
@@ -431,7 +443,9 @@ async function downloadJson<T>(
bucket: string,
key: string,
destination: string,
+ bounded?: BoundedHistoryPublisher,
) {
+ if (bounded) return bounded.downloadJson<T>(bucket, key, destination);
if (!(await objectExists(bucket, key))) return null;
await execFileAsync("aws", [
"s3",
@@ -472,11 +486,24 @@ async function uploadFile(
]);
}
-async function uploadImmutableBundle(
+export async function uploadImmutableBundle(
bucket: string,
key: string,
directory: string,
+ manifest?: BundleManifest,
+ bounded?: BoundedHistoryPublisher,
) {
+ if (bounded && manifest) {
+ for (const file of manifest.files) {
+ const extension = path.extname(file.path).toLowerCase();
+ const contentType = ({ ".png": "image/png", ".json": "application/json", ".html": "text/html; charset=utf-8",
+ ".woff2": "font/woff2", ".xml": "application/xml", ".md": "text/plain; charset=utf-8" } as Record<string, string>)[extension];
+ if (!contentType) throw new Error(`Missing public content type for ${file.path}`);
+ await bounded.upload(bucket, `${key}/${file.path}`, path.join(directory, file.path), contentType,
+ "public,max-age=2592000,immutable", true);
+ }
+ return;
+ }
const common = [
"--recursive",
"--only-show-errors",
@@ -524,10 +551,19 @@ async function main() {
prefix: process.env.RUNNER_E2E_HISTORY_PREFIX ?? "runner-e2e",
publicBaseUrl: process.env.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL ?? "",
});
+ const bounded = process.env.PAPERCLIP_E2E_BOUNDED_HISTORY === "true";
+ if (bounded && process.env.GITHUB_ACTIONS === "true" && process.env.GITHUB_RUN_ATTEMPT !== "1") {
+ throw new Error("Bounded history publication requires the original workflow attempt; retain failed publication evidence");
+ }
+ const publisher = bounded ? new BoundedHistoryPublisher(path.join(path.dirname(reportRoot), "history-publication-receipt.json")) : undefined;
+ if (publisher) await publisher.verifyRetention(bucket, destination.prefix);
const campaign = JSON.parse(
await readFile(path.join(reportRoot, "normalized-results.json"), "utf8"),
) as RunnerE2ECampaign;
for (const result of campaign.results ?? []) validateRetainedRunnerResult(result);
+ if (campaign.accounting) validateCampaignAccounting(campaign.accounting);
+ if (campaign.operations) validateEvalOperationsReport(campaign.operations);
+ if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview);
assertPublicCampaign(campaign);
if (campaign.schema !== "paperclip.runner-e2e.campaign/v2") {
throw new Error("Historical publishing requires a v2 normalized campaign");
@@ -542,6 +578,7 @@ async function main() {
bucket,
historyKey,
path.join(temporary, "current-history.json"),
+ publisher,
)) ?? emptyRunnerHistory();
const history = mergeRunnerHistory(
current,
@@ -549,7 +586,11 @@ async function main() {
campaign,
`${destination.publicBaseUrl}/${destination.prefix}`,
),
+ bounded ? { now: Date.now(), maxAgeMs: HISTORY_LIMITS.indexDays * 86_400_000, maxCampaigns: HISTORY_LIMITS.indexCampaigns } : undefined,
);
+ if (bounded && !history.campaigns.some(entry => entry.campaignId === campaign.campaignId)) {
+ throw new Error("Bounded campaign is outside the retained history window");
+ }
const publicScreenshots = publicScreenshotPaths(campaign);
// Campaign bundles are immutable and must not capture a mutable history
@@ -581,6 +622,7 @@ async function main() {
campaign.campaignId,
true,
publicScreenshots,
+ bounded,
);
const campaignPrefix = `${destination.prefix}/campaigns/${campaign.campaignId}`;
const manifestKey = `${campaignPrefix}/bundle-manifest.json`;
@@ -588,6 +630,7 @@ async function main() {
bucket,
manifestKey,
path.join(temporary, "existing-manifest.json"),
+ publisher,
);
if (
existingManifest &&
@@ -597,17 +640,8 @@ async function main() {
`Immutable campaign ${campaign.campaignId} already exists with a different digest`,
);
}
- if (!existingManifest) {
- await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot);
- const manifestFile = path.join(temporary, "bundle-manifest.json");
- await writeFile(manifestFile, json(manifest), "utf8");
- await uploadJson(
- bucket,
- manifestKey,
- manifestFile,
- "public,max-age=31536000,immutable",
- );
- }
+ const manifestFile = path.join(temporary, "bundle-manifest.json");
+ await writeFile(manifestFile, json(manifest), "utf8");
// Pages uses the same declared-screenshot boundary in a sibling stage. The
// downloaded access-controlled report stays intact for this whole job.
@@ -630,11 +664,12 @@ async function main() {
writeFile(latestGreenFile, json(pointers.latestGreen), "utf8"),
]);
await regenerateRunnerDashboard({ bundle: pagesRoot, historyFile });
- await createBundleManifest(
+ const pagesManifest = await createBundleManifest(
pagesRoot,
campaign.campaignId,
false,
publicScreenshots,
+ bounded,
);
const landingDirectory = path.join(temporary, "landing");
await regenerateRunnerDashboard({
@@ -648,6 +683,37 @@ async function main() {
renderRunnerHistoryIndex(history),
"utf8",
);
+ if (publisher) {
+ const mutableFiles = [
+ { key: historyKey, file: historyFile, contentType: "application/json" },
+ { key: `${destination.prefix}/latest.json`, file: latestFile, contentType: "application/json" },
+ { key: `${destination.prefix}/latest-green.json`, file: latestGreenFile, contentType: "application/json" },
+ ...["index.html", "dashboard.html", "normalized-results.json"].map(file => ({ key: `${destination.prefix}/${file}`,
+ file: path.join(landingDirectory, file), contentType: file.endsWith("json") ? "application/json" : "text/html; charset=utf-8" })),
+ ...["favicon-32x32.png", "InterVariable.woff2"].map(file => ({ key: `${destination.prefix}/assets/${file}`,
+ file: path.join(s3ReportRoot, "assets", file), contentType: file.endsWith("png") ? "image/png" : "font/woff2" })),
+ ];
+ const metadata = async (file: string) => ({ path: file, bytes: (await stat(file)).size });
+ const campaignSize = checkHistoryPayload([...manifest.files, await metadata(manifestFile)],
+ HISTORY_LIMITS.campaignBytes, HISTORY_LIMITS.campaignFiles, "Public campaign with manifest");
+ const mutableMetadata = await Promise.all(mutableFiles.map(file => metadata(file.file)));
+ const mutableSize = checkHistoryPayload(mutableMetadata, HISTORY_LIMITS.mutableBytes, 8, "Mutable history");
+ const pointerMetadata = await Promise.all([historyFile, latestFile, latestGreenFile].map(metadata));
+ checkHistoryPayload(pointerMetadata, HISTORY_LIMITS.jsonBytes, 3, "History JSON");
+ const pagesSize = checkHistoryPayload([...pagesManifest.files, ...pointerMetadata],
+ HISTORY_LIMITS.pagesBytes, HISTORY_LIMITS.campaignFiles + 3, "Pages bundle");
+ if (!existingManifest) {
+ await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot, manifest, publisher);
+ await publisher.upload(bucket, manifestKey, manifestFile, "application/json", "public,max-age=2592000,immutable", true);
+ }
+ for (const file of mutableFiles) await publisher.upload(bucket, file.key, file.file, file.contentType, "no-cache");
+ console.log(JSON.stringify({ campaign: campaignSize, mutable: mutableSize, pages: pagesSize, billedUsd: null }));
+ return;
+ }
+ if (!existingManifest) {
+ await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot);
+ await uploadJson(bucket, manifestKey, manifestFile, "public,max-age=31536000,immutable");
+ }
await Promise.all([
uploadJson(bucket, historyKey, historyFile, "no-cache"),
uploadJson(
diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts
index c60278769d..c717441e32 100644
--- a/tests/runner-e2e/history.test.ts
+++ b/tests/runner-e2e/history.test.ts
@@ -69,6 +69,41 @@ function result(execution: MatrixExecution, status: "passed" | "failed") {
}
describe("runner E2E campaign history", () => {
+ it("retains the supervisor trace outcome independently of the worker export through regeneration", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-finalized-trace-"));
+ temporaryDirectories.push(root);
+ const execution = runnerMatrix[0]!;
+ const attempt: RunnerE2EResult = { ...result(execution, "failed"), failureClass: "permanent_infrastructure",
+ tracing: { projectId: "11111111-1111-4111-8111-111111111111", traceId: "a".repeat(32), spanId: "b".repeat(16),
+ status: "accepted", verification: "unverified", nativeConfiguration: "configured",
+ finalization: { spanId: "c".repeat(16), status: "accepted", outcome: "failed", verification: "unverified" } },
+ };
+ const campaign = buildRunnerCampaign({ campaignId: "finalized-trace", generatedAt: attempt.finishedAt,
+ expected: [execution.id], results: [attempt], attempts: [attempt] });
+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));
+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });
+ const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));
+ expect(rebuilt.results[0].tracing).toEqual(attempt.tracing);
+ expect(rebuilt.accounting.attempts[0].tracing).toEqual(attempt.tracing);
+ expect(rebuilt.failed).toBe(1);
+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("Supervisor outcome: failed; finalization export: accepted");
+ });
+
+ it("preserves all-attempt costs when regenerating a retained report", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-attempt-costs-"));
+ temporaryDirectories.push(root);
+ const execution = runnerMatrix[0]!;
+ const first = result(execution, "failed");
+ const last = { ...result(execution, "passed"), attempt: 2, usage: { inputTokens: 200, outputTokens: 50, costUsd: 0.02 } };
+ const campaign = buildRunnerCampaign({ campaignId: "retries", generatedAt: last.finishedAt,
+ expected: [execution.id], results: [last], attempts: [first, last] });
+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));
+ await regenerateRunnerDashboard({ bundle: root });
+ const regenerated = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));
+ expect(regenerated.accounting).toEqual(campaign.accounting);
+ expect(regenerated.billing.reportedLlmCostUsd).toBe(0.03);
+ expect(regenerated.results[0].billing.reportedCostUsd).toBe(0.02);
+ });
it("does not let empty legacy provenance hide a later measured source", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "runner-legacy-source-"));
temporaryDirectories.push(root);
diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.ts
index 90476f4ee8..822bd81b7e 100644
--- a/tests/runner-e2e/history.ts
+++ b/tests/runner-e2e/history.ts
@@ -1,7 +1,11 @@
import { isIncompleteFirstTaskResult } from "./case-outcome.js";
+import { buildCampaignAccounting } from "./attempt-accounting.js";
+import { validateEvalOperationsReport } from "./operations-report.js";
+import { validateCampaignFeedbackReview } from "./feedback-dispositions.js";
+import { validateBraintrustExperimentReceipt } from "./braintrust-experiment.js";
import { classifyLatency } from "./latency-timing.js";
import { summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js";
-import { validateRetainedRunnerResult } from "./result-validation.js";
+import { validateCampaignAccounting, validateRetainedRunnerResult } from "./result-validation.js";
import {
discoverReportCatalog,
parseReportExecutionId,
@@ -9,12 +13,13 @@ import {
} from "./report-catalog.js";
import { runnerMatrix, runnerSuites } from "./catalog.js";
import {
- aggregateCampaignBilling,
+ aggregateBillingSummaries,
summarizeExecutionBilling,
} from "./billing.js";
import { resolveRunnerE2ESource } from "./source.js";
import type {
RunnerE2ECampaign,
+ RunnerE2ECampaignAccounting,
RunnerE2EHistoryCampaign,
RunnerE2EHistoryIndex,
RunnerE2EResult,
@@ -71,6 +76,12 @@ export function buildRunnerCampaign(input: {
generatedAt: string;
expected: readonly string[];
results: readonly RunnerE2EResult[];
+ attempts?: readonly RunnerE2EResult[];
+ accounting?: RunnerE2ECampaignAccounting;
+ operations?: RunnerE2ECampaign["operations"];
+ feedbackReview?: RunnerE2ECampaign["feedbackReview"];
+ braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"];
+ excludedArtifacts?: number;
eventName?: string | null;
catalog?: readonly ReportExecution[];
}): RunnerE2ECampaign {
@@ -79,6 +90,28 @@ export function buildRunnerCampaign(input: {
...upgradeRunnerResult(result),
billing: result.billing ?? summarizeExecutionBilling(result),
}));
+ const accounting = input.accounting ?? buildCampaignAccounting({
+ results,
+ attempts: input.attempts?.map(upgradeRunnerResult),
+ excludedArtifacts: input.excludedArtifacts,
+ });
+ validateCampaignAccounting(accounting);
+ if (input.operations) validateEvalOperationsReport(input.operations);
+ const billingFor = (selectedResults: readonly RunnerE2EResult[]) => {
+ const ids = new Set(selectedResults.map(result => result.executionId));
+ const receipts = accounting.attempts.filter(attempt => ids.has(attempt.executionId));
+ const billing = aggregateBillingSummaries([
+ ...receipts,
+ ...selectedResults.filter(result => !receipts.some(attempt => attempt.executionId === result.executionId))
+ .map(result => ({ executionId: result.executionId, billing: summarizeExecutionBilling(result) })),
+ ]);
+ billing.testsWithCompleteBilling = selectedResults.filter(result => {
+ const retained = receipts.filter(attempt => attempt.executionId === result.executionId);
+ return retained.length > 0 && retained.every(attempt => attempt.billing.complete) &&
+ new Set(retained.map(attempt => attempt.attempt)).size >= Math.max(result.attempt, ...retained.map(attempt => attempt.attempt));
+ }).length;
+ return billing;
+ };
const knownCatalog = input.catalog ?? runnerMatrix;
const catalog = discoverReportCatalog({
catalog: knownCatalog,
@@ -150,7 +183,7 @@ export function buildRunnerCampaign(input: {
(total, result) => total + result.durationMs,
0,
),
- billing: aggregateCampaignBilling(suiteResults),
+ billing: billingFor(suiteResults),
};
})
.filter((suite): suite is NonNullable<typeof suite> => Boolean(suite));
@@ -175,7 +208,7 @@ export function buildRunnerCampaign(input: {
),
).values(),
];
- return {
+ const campaign: RunnerE2ECampaign = {
schema: "paperclip.runner-e2e.campaign/v2",
campaignId: input.campaignId,
generatedAt: input.generatedAt,
@@ -195,10 +228,17 @@ export function buildRunnerCampaign(input: {
),
cleanupPassed: results.every((result) => result.cleanup === "passed"),
rankingSnapshots,
- billing: aggregateCampaignBilling(results),
+ billing: billingFor(results),
+ accounting,
+ ...(input.operations ? { operations: input.operations } : {}),
+ ...(input.feedbackReview ? { feedbackReview: input.feedbackReview } : {}),
+ ...(input.braintrustExperiment ? { braintrustExperiment: input.braintrustExperiment } : {}),
suites,
results,
};
+ if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview);
+ if (campaign.braintrustExperiment) validateBraintrustExperimentReceipt(campaign, campaign.braintrustExperiment);
+ return campaign;
}
export function campaignHistoryRecord(
@@ -253,6 +293,7 @@ export function emptyRunnerHistory(): RunnerE2EHistoryIndex {
export function mergeRunnerHistory(
current: RunnerE2EHistoryIndex | null | undefined,
campaign: RunnerE2EHistoryCampaign,
+ retention?: { now: number; maxAgeMs: number; maxCampaigns: number },
): RunnerE2EHistoryIndex {
const base =
current?.schema === "paperclip.runner-e2e.history/v1"
@@ -263,10 +304,14 @@ export function mergeRunnerHistory(
...base.campaigns.filter(
(candidate) => candidate.campaignId !== campaign.campaignId,
),
- ].sort(
+ ].filter(candidate => !retention || (
+ Number.isFinite(Date.parse(candidate.generatedAt)) &&
+ Date.parse(candidate.generatedAt) > retention.now - retention.maxAgeMs &&
+ Date.parse(candidate.generatedAt) <= retention.now + 60_000
+ )).sort(
(left, right) =>
Date.parse(right.generatedAt) - Date.parse(left.generatedAt),
- );
+ ).slice(0, retention?.maxCampaigns);
const latestBySuite: Record<string, string> = {};
const latestGreenBySuite: Record<string, string> = {};
for (const candidate of campaigns) {
@@ -279,7 +324,7 @@ export function mergeRunnerHistory(
}
return {
schema: "paperclip.runner-e2e.history/v1",
- updatedAt: new Date().toISOString(),
+ updatedAt: new Date(retention?.now ?? Date.now()).toISOString(),
latestCampaignId: campaigns[0]?.campaignId ?? null,
latestGreenCampaignId:
campaigns.find(
diff --git a/tests/runner-e2e/ingestion-usage.test.ts b/tests/runner-e2e/ingestion-usage.test.ts
new file mode 100644
index 0000000000..fdb327f454
--- /dev/null
+++ b/tests/runner-e2e/ingestion-usage.test.ts
@@ -0,0 +1,108 @@
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { afterEach, expect, it } from "vitest";
+import { openEvalCostLedger } from "./cost-ledger.js";
+import { startEvalOtlpRelay } from "./eval-otlp-relay.js";
+import { EVAL_FINALIZATION_LIMITS } from "./eval-finalization-relay.js";
+import { openExperimentIngestion } from "./experiment-ingestion.js";
+import { collectScheduledIngestion } from "./ingestion-usage.js";
+
+const projectId = "11111111-1111-4111-8111-111111111111";
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+async function fixture() {
+ const root = await mkdtemp(path.join(tmpdir(), "scheduled-ingestion-")); roots.push(root);
+ const input = { ledgerPath: path.join(root, "ledger.jsonl"), campaignId: "campaign", projectId, executionIds: ["fixture.local"] };
+ const ledger = await openEvalCostLedger(input.ledgerPath, { campaignId: input.campaignId, maxUsd: "1",
+ deadlineAt: new Date(Date.now() + 60_000).toISOString(), maxRequests: 1, maxConcurrent: 1, maxAttemptsPerExecution: 1 });
+ try { await ledger.admitAttempt(input.executionIds[0]!, "attempt"); } finally { await ledger.close(); }
+ const name = createHash("sha256").update("attempt").digest("hex"), files: string[] = [];
+ for (const kind of ["native", "finalization"] as const) {
+ const receiptPath = path.join(`${input.ledgerPath}.otlp`, `${name}${kind === "finalization" ? "-finalization" : ""}.jsonl`);
+ files.push(receiptPath);
+ const relay = await startEvalOtlpRelay({ ...input, attemptId: "attempt", apiKey: "private-fixture", receiptPath,
+ deadlineAt: Date.now() + 60_000, ...(kind === "finalization" ? { limits: EVAL_FINALIZATION_LIMITS } : {}),
+ transport: async () => Response.json({}) });
+ try {
+ expect((await fetch(relay.baseUrl + "/v1/traces", { method: "POST", body: "{}",
+ headers: { authorization: `Bearer ${relay.token}`, "content-type": "application/json" } })).status).toBe(200);
+ } finally { await relay.close(); }
+ }
+ const experiment = await openExperimentIngestion({ ...input, receiptPath: `${input.ledgerPath}.experiment-ingestion.jsonl`,
+ deadlineAt: Date.now() + 60_000, transport: async () => Response.json({}) });
+ const body = JSON.stringify({ project_id: projectId });
+ try { await experiment.transport("https://api.braintrust.dev/v1/experiment", { method: "POST", body }); }
+ finally { await experiment.close(); }
+ return { input, files, expectedBytes: 4 + Buffer.byteLength(body) };
+}
+
+it("reconciles real durable receipts for all three transport paths without inventing vendor billing", async () => {
+ const f = await fixture(), usage = await collectScheduledIngestion(f.input);
+ expect(usage).toMatchObject({ status: "complete", observedAdmittedPayloadBytes: f.expectedBytes,
+ totalAdmittedPayloadBytes: f.expectedBytes, billedCostUsd: null, billingStatus: "unreconciled", missingExecutions: [] });
+ expect(usage.receipts.map(row => row.kind)).toEqual(["native", "finalization", "experiment"]);
+ expect(usage.receipts[0]).toMatchObject({ admittedExpandedSourceBytes: 2, admittedRecords: 0 });
+ expect(usage.receipts.every(row => "receiptSha256" in row && /^[a-f0-9]{64}$/.test(row.receiptSha256!))).toBe(true);
+ expect(usage.ledgerHash).toMatch(/^[a-f0-9]{64}$/);
+ expect(JSON.stringify(usage)).not.toContain("private-fixture");
+});
+
+it.each(["missing", "unclosed", "unresolved", "foreign-project", "bad-total", "bad-expanded-total", "missing-record-count", "duplicate-outcome"])(
+ "retains other measured traffic and an unknown total for a %s receipt", async reason => {
+ const f = await fixture(), file = f.files[0]!;
+ if (reason === "missing") await rm(file);
+ else {
+ const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ if (reason === "unclosed") rows.pop();
+ if (reason === "unresolved") rows[2].outcome = "uncertain";
+ if (reason === "foreign-project") rows[0].projectId = "22222222-2222-4222-8222-222222222222";
+ if (reason === "bad-total") rows.at(-1).admittedBytes = 0;
+ if (reason === "bad-expanded-total") rows.at(-1).admittedExpandedBytes = 0;
+ if (reason === "missing-record-count") delete rows[1].recordCount;
+ if (reason === "duplicate-outcome") rows.splice(3, 0, rows[2]);
+ await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n");
+ }
+ const usage = await collectScheduledIngestion(f.input);
+ expect(usage).toMatchObject({ status: "incomplete", totalAdmittedPayloadBytes: null, billedCostUsd: null });
+ expect(usage.observedAdmittedPayloadBytes).toBeGreaterThan(0);
+ expect(usage.receipts[0]!.status).not.toBe("complete");
+ expect(usage.receipts.slice(1).every(row => row.status === "complete")).toBe(true);
+ });
+
+it("binds the expected receipt set to admitted attempts and retains unstarted selections", async () => {
+ const f = await fixture();
+ expect(await collectScheduledIngestion({ ...f.input, executionIds: [...f.input.executionIds, "missing.local"] }))
+ .toMatchObject({ status: "incomplete", missingExecutions: ["missing.local"], totalAdmittedPayloadBytes: null });
+ await expect(collectScheduledIngestion({ ...f.input, executionIds: [] })).rejects.toThrow("selection");
+ await expect(collectScheduledIngestion({ ...f.input, executionIds: ["foreign.local"] })).rejects.toThrow("Foreign ingestion attempt");
+ await expect(collectScheduledIngestion({ ...f.input, campaignId: "foreign" })).rejects.toThrow("campaign mismatch");
+});
+
+it("keeps version 1 transport receipts readable without inventing expanded data measurements", async () => {
+ const f = await fixture(), file = f.files[0]!;
+ const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ rows[0].version = 1; delete rows[0].sizing;
+ delete rows[0].limits.maxExpandedBytes; delete rows[0].limits.maxRecords;
+ for (const row of rows) {
+ delete row.expandedBytes; delete row.recordCount;
+ delete row.admittedExpandedBytes; delete row.admittedRecords;
+ }
+ await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n");
+ const usage = await collectScheduledIngestion(f.input);
+ expect(usage).toMatchObject({ status: "complete", billedCostUsd: null });
+ expect(usage.receipts[0]).toMatchObject({ admittedPayloadBytes: 2, admittedExpandedSourceBytes: null, admittedRecords: null });
+});
+
+it.each(["outcome", "closed"])("retains the verified prefix and digest when the final %s line is truncated", async kind => {
+ const f = await fixture(), file = f.files[0]!;
+ const lines = (await readFile(file, "utf8")).trim().split("\n");
+ const prefix = lines.slice(0, kind === "outcome" ? 2 : 3);
+ const truncated = prefix.join("\n") + '\n{"type":"' + kind;
+ await writeFile(file, truncated);
+ const usage = await collectScheduledIngestion(f.input);
+ expect(usage).toMatchObject({ status: "incomplete", observedAdmittedPayloadBytes: f.expectedBytes, totalAdmittedPayloadBytes: null });
+ expect(usage.receipts[0]).toMatchObject({ status: "incomplete", admittedRequests: 1, admittedPayloadBytes: 2,
+ unresolvedRequests: kind === "outcome" ? 1 : 0, receiptSha256: createHash("sha256").update(truncated).digest("hex") });
+});
diff --git a/tests/runner-e2e/ingestion-usage.ts b/tests/runner-e2e/ingestion-usage.ts
new file mode 100644
index 0000000000..f5c925844c
--- /dev/null
+++ b/tests/runner-e2e/ingestion-usage.ts
@@ -0,0 +1,98 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { readEvalCostLedger, validEvalLedgerId } from "./cost-ledger.js";
+import { EVAL_OTLP_LIMITS } from "./eval-otlp-relay.js";
+import { EVAL_FINALIZATION_LIMITS } from "./eval-finalization-relay.js";
+import { EVAL_EXPERIMENT_LIMITS } from "./experiment-ingestion.js";
+
+type Kind = "native" | "finalization" | "experiment";
+interface ReceiptScope { kind: Kind; file: string; campaignId: string; projectId: string; attemptId?: string }
+async function readReceipt(scope: ReceiptScope) {
+ const bytes = await readFile(scope.file);
+ if (bytes.length > 4 * 1024 * 1024) throw new Error("Oversized ingestion receipt");
+ const rows: Array<Record<string, any>> = [];
+ let malformed = false;
+ for (const line of bytes.toString("utf8").trim().split("\n")) {
+ try { rows.push(JSON.parse(line)); }
+ catch { malformed = true; break; }
+ }
+ const policy = rows[0], closed = rows.at(-1);
+ const maximum = scope.kind === "native" ? EVAL_OTLP_LIMITS : scope.kind === "finalization" ? EVAL_FINALIZATION_LIMITS : EVAL_EXPERIMENT_LIMITS;
+ const expanded = scope.kind !== "experiment" && policy?.version === 2;
+ if (policy?.type !== "policy" || !(policy.version === 1 || expanded && policy.sizing === "otlp-v1.9-expanded-source-json")
+ || policy.campaignId !== scope.campaignId || policy.projectId !== scope.projectId
+ || (scope.kind === "experiment" ? policy?.kind !== "experiment" : policy?.attemptId !== scope.attemptId)) {
+ throw new Error("Invalid ingestion receipt identity");
+ }
+ if (!policy.limits || Object.entries(maximum).filter(([key]) => expanded || !["maxExpandedBytes", "maxRecords"].includes(key))
+ .some(([key, max]) => !Number.isSafeInteger(policy.limits[key])
+ || policy.limits[key] < 1 || policy.limits[key] > max)) throw new Error("Invalid ingestion receipt limits");
+ const admissions = new Map<string, { bodyBytes: number; scores: number }>();
+ const outcomes = new Map<string, string>();
+ let payloadBytes = 0, scores = 0, expandedBytes = 0, recordCount = 0;
+ const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0;
+ for (let index = 1; index < rows.length; index++) {
+ const row = rows[index];
+ if (row.type === "admitted") {
+ const scoreCount = scope.kind === "experiment" ? row.scores : 0;
+ if (typeof row.id !== "string" || admissions.has(row.id) || !count(row.bodyBytes) || !count(scoreCount)
+ || scope.kind !== "experiment" && row.bodyBytes > policy.limits.maxRequestBytes
+ || expanded && (!count(row.expandedBytes) || !count(row.recordCount))
+ || !/^[0-9a-f]{64}$/.test(row.payloadSha256)) throw new Error("Invalid ingestion admission");
+ admissions.set(row.id, { bodyBytes: row.bodyBytes, scores: scoreCount });
+ payloadBytes += row.bodyBytes; scores += scoreCount;
+ if (expanded) { expandedBytes += row.expandedBytes; recordCount += row.recordCount; }
+ } else if (row.type === "outcome") {
+ const allowed = scope.kind === "experiment" ? ["response", "unconfirmed"] : ["accepted", "partial", "rejected", "uncertain", "not_dispatched"];
+ if (!admissions.has(row.id) || outcomes.has(row.id) || !allowed.includes(row.outcome)
+ || ["response", "accepted", "partial", "rejected"].includes(row.outcome)
+ && (!Number.isInteger(row.httpStatus) || row.httpStatus < 100 || row.httpStatus > 599)) throw new Error("Invalid ingestion outcome");
+ outcomes.set(row.id, row.outcome);
+ } else if (row.type === "closed") {
+ if (index !== rows.length - 1 || row.admittedRequests !== admissions.size || row.admittedBytes !== payloadBytes
+ || expanded && (row.admittedExpandedBytes !== expandedBytes || row.admittedRecords !== recordCount)
+ || scope.kind === "experiment" && row.admittedScores !== scores || row.poisoned) throw new Error("Invalid ingestion closure");
+ } else if (row.type !== "quota_exhausted" || scope.kind === "experiment") throw new Error("Unexpected ingestion record");
+ }
+ if (admissions.size > policy.limits.maxRequests || payloadBytes > policy.limits.maxTotalBytes
+ || expanded && (expandedBytes > policy.limits.maxExpandedBytes || recordCount > policy.limits.maxRecords)
+ || scope.kind === "experiment" && scores > policy.limits.maxScores) throw new Error("Ingestion allowance exceeded");
+ const unresolved = [...admissions.keys()].filter(id => !outcomes.has(id) || ["uncertain", "unconfirmed"].includes(outcomes.get(id)!)).length;
+ return { kind: scope.kind, ...(scope.attemptId ? { attemptId: scope.attemptId } : {}),
+ receiptSha256: createHash("sha256").update(bytes).digest("hex"),
+ status: !malformed && closed?.type === "closed" && unresolved === 0 ? "complete" : "incomplete",
+ admittedRequests: admissions.size, admittedPayloadBytes: payloadBytes, admittedScores: scope.kind === "experiment" ? scores : null,
+ admittedExpandedSourceBytes: expanded ? expandedBytes : null, admittedRecords: expanded ? recordCount : null,
+ unresolvedRequests: unresolved };
+}
+
+export async function collectScheduledIngestion(input: {
+ ledgerPath: string; campaignId: string; projectId: string; executionIds: string[];
+}) {
+ if (!input.executionIds.length || new Set(input.executionIds).size !== input.executionIds.length
+ || !input.executionIds.every(validEvalLedgerId)) throw new Error("Invalid ingestion execution selection");
+ const ledger = await readEvalCostLedger(input.ledgerPath);
+ if (ledger.state.limits.campaignId !== input.campaignId) throw new Error("Ingestion ledger campaign mismatch");
+ const attempts = ledger.state.snapshot().admittedAttempts;
+ if (attempts.some(row => !input.executionIds.includes(row.executionId))) throw new Error("Foreign ingestion attempt");
+ const missingExecutions = input.executionIds.filter(id => !attempts.some(row => row.executionId === id));
+ const scopes: ReceiptScope[] = attempts.flatMap(row => {
+ const name = createHash("sha256").update(row.attemptId).digest("hex");
+ return (["native", "finalization"] as const).map(kind => ({ kind, attemptId: row.attemptId,
+ campaignId: input.campaignId, projectId: input.projectId,
+ file: path.join(`${input.ledgerPath}.otlp`, `${name}${kind === "finalization" ? "-finalization" : ""}.jsonl`) }));
+ });
+ scopes.push({ kind: "experiment", file: `${input.ledgerPath}.experiment-ingestion.jsonl`, campaignId: input.campaignId, projectId: input.projectId });
+ const receipts = await Promise.all(scopes.map(async scope => {
+ try { return await readReceipt(scope); }
+ catch { return { kind: scope.kind, ...(scope.attemptId ? { attemptId: scope.attemptId } : {}), status: "unavailable" }; }
+ }));
+ const complete = missingExecutions.length === 0 && receipts.every(row => row.status === "complete");
+ return { version: 1, campaignId: input.campaignId, projectId: input.projectId, ledgerHash: ledger.previous,
+ status: complete ? "complete" : "incomplete", missingExecutions, receipts,
+ observedAdmittedPayloadBytes: receipts.reduce((sum, row) => sum + ("admittedPayloadBytes" in row ? row.admittedPayloadBytes : 0), 0),
+ totalAdmittedPayloadBytes: complete ? receipts.reduce((sum, row) => sum + ("admittedPayloadBytes" in row ? row.admittedPayloadBytes : 0), 0) : null,
+ billedCostUsd: null, billingStatus: "unreconciled",
+ scope: "Bounded attempt relay, controller finalization and retained experiment requests; not vendor processed bytes or provider model charges" };
+}
diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.ts
index b1ab4577a8..83770cd98a 100644
--- a/tests/runner-e2e/installed-cli.test.ts
+++ b/tests/runner-e2e/installed-cli.test.ts
@@ -108,10 +108,9 @@ it("uses the direct public Playwright JS bin without bin-shim Node injection", a
const f = await fixture(); const cli = join(f.root,"node_modules/@playwright/test/cli.js");
await mkdir(join(f.root,"node_modules/@playwright/test"),{recursive:true});
await writeFile(cli,"console.log(JSON.stringify({nodePath:process.env.NODE_PATH??null,nodeOptions:process.env.NODE_OPTIONS??null,args:process.argv.slice(2)}))");
- const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"],true);
+ const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"]);
expect(invocation.command).toBe(process.execPath);
expect(JSON.parse(execFileSync(invocation.command,invocation.args,{env:{},timeout:5000,encoding:"utf8"}))).toEqual({nodePath:null,nodeOptions:null,args:["--version"]});
- expect(runnerE2EPlaywrightInvocation(f.root,["--version"],false)).toEqual({command:"pnpm",args:["exec","playwright","--version"]});
for (const key of ["NODE_PATH","NODE_OPTIONS"]) expect(()=>assertInstalledCliSelection({...f.env,[key]:""},[cells[0]!])).toThrow("ambient Node injection");
});
diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts
index 092ce425f9..30c489ad92 100644
--- a/tests/runner-e2e/journey-features.ts
+++ b/tests/runner-e2e/journey-features.ts
@@ -34,6 +34,9 @@ associate(["local-session-integrity", "extended-harnesses", "grok-qualification"
associate(["question-resume"], {
"question-answer-resume": ["questions-and-approvals/questions", "questions-and-approvals/continuation"],
});
+associate(["feedback-persistence"], {
+ "complaint-and-suggestion": ["activity/agent-commentary"],
+});
associate(["continuation"], {
"answer-updates-scope": ["questions-and-approvals/questions", "questions-and-approvals/continuation"],
"clarification-not-approval": ["questions-and-approvals/questions", "questions-and-approvals/confirmations"],
diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts
index 1d0971ab97..934daa1dc8 100644
--- a/tests/runner-e2e/launch.ts
+++ b/tests/runner-e2e/launch.ts
@@ -1,9 +1,15 @@
-import { runnerE2EPlaywrightInvocation } from "./web-server-command.js";
+import { recoverWorkerTiming } from "./worker-timing.js";
+import { openEvalOperations, type EvalOperations } from "./operations.js";
+import { readRecoveredFeedback } from "./feedback-recovery.js";
+import { readInterruptedAttemptTrace } from "./trace-recovery.js";
+import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js";
+import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js";
+import { runnerE2EPlaywrightInvocation, runnerE2EPlaywrightShutdownSignal } from "./web-server-command.js";
import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js";
import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";
import { createRunnerE2ETemporaryRoot } from "./server-config.js";
-import { randomBytes } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
import { spawn } from "node:child_process";
import { createWriteStream } from "node:fs";
@@ -30,7 +36,9 @@ import { classifyFailure } from "./failure-classifier.js";
import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js";
import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js";
import { buildRunnerCampaign } from "./history.js";
+import { readEvalOperationsReport } from "./operations-report.js";
import {
+ applyEvalGatewayEnvironment,
buildRunnerE2EProcessEnvironment,
resolvePaperclipRemoteRunnerBinaryForHarness,
resolvePaperclipRunnerBinaryForHarness,
@@ -305,7 +313,7 @@ async function runProcess(
const cleanupFinished = new Promise<number>(resolve => { cleanupSettled = () => resolve(1); });
const stopChildTree = (_diagnostic?: ProcessTreeDiagnostic) => {
if (boundedCleanup) return;
- boundedCleanup = stopOwnedProcessTree(child, processOwner)
+ boundedCleanup = stopOwnedProcessTree(child, processOwner, 45_000, 5_000, runnerE2EPlaywrightShutdownSignal)
.then(() => null, error => error instanceof Error ? error.message : String(error))
.finally(cleanupSettled);
activeProcessCleanup.set(child.pid!, boundedCleanup);
@@ -467,8 +475,12 @@ async function runAttempt(input: {
attempt: number;
campaignId: string;
options: ReturnType<typeof parseRunnerSelectors>;
+ operations?: EvalOperations;
}): Promise<RunnerE2EResult[]> {
const { executions, attempt, campaignId, options } = input;
+ const accountingCampaignId = input.operations?.manifest.limits.campaignId ?? process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? campaignId;
+ const accountingAttemptId = randomBytes(16).toString("hex");
+ const traceConfig = evalBraintrustConfig({ ...process.env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined });
const execution = executions[0];
if (!execution) throw new Error("A runner E2E cell must contain a task case");
if (
@@ -494,6 +506,7 @@ async function runAttempt(input: {
let attemptSecrets: string[] = [];
let processCleanupFailed = false;
let startupReceiptPath: string | undefined;
+ let gateway: Awaited<ReturnType<EvalOperations["start"]>> | undefined;
const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = [];
try {
const paperclipHome = path.join(temporaryRoot, "paperclip-home");
@@ -523,8 +536,11 @@ async function runAttempt(input: {
const decisionSigningSecret = secret(48);
const toolActionSigningSecret = secret(48);
const betterAuthSecret = secret(48);
+ gateway = await input.operations?.start(execution, accountingAttemptId, attempt);
const credentialSources = {
- ...Object.fromEntries(CREDENTIAL_NAMES.map((name) => [name, process.env[name]])),
+ PAPERCLIP_E2E_GATEWAY_TOKEN: gateway?.token,
+ PAPERCLIP_E2E_OTLP_RELAY_TOKEN: gateway?.observability?.token,
+ ...Object.fromEntries([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => [name, process.env[name]])),
PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret,
PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret,
PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,
@@ -545,6 +561,8 @@ async function runAttempt(input: {
executions.map((candidate) => candidate.id),
),
PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt),
+ PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: accountingCampaignId,
+ PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: accountingAttemptId,
PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY: options.installedStartupOnly ? "1" : undefined,
PAPERCLIP_RUNNER_E2E_PUBLIC_MCP: executions.some(candidate => candidate.task.flow === "public_mcp") ? "1" : "0",
PAPERCLIP_RUNNER_E2E_PORT: String(port),
@@ -575,6 +593,9 @@ async function runAttempt(input: {
PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,
BETTER_AUTH_SECRET: betterAuthSecret,
};
+ if (gateway) {
+ applyEvalGatewayEnvironment(childEnv, execution.profile.credential, gateway);
+ }
// The database URLs are stripped here and again at the Playwright web-server
// boundary so a developer's shell can never redirect this paid test.
delete childEnv.DATABASE_URL;
@@ -600,7 +621,7 @@ async function runAttempt(input: {
) +
5 * 60_000;
const processResult = await runProcess(
- runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs, Boolean(installedCli)),
+ runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs),
childEnv,
watchdog,
path.join(privateDir, "playwright.log"),
@@ -653,6 +674,16 @@ async function runAttempt(input: {
processFailureClass,
);
const result = await readResult(resultPath, fallback);
+ if (!result.workerTiming) result.workerTiming = await recoverWorkerTiming(path.join(path.dirname(resultPath), "snapshots", "worker-timing.json"),
+ { executionId: candidate.id, attempt, startedAt: result.startedAt, finishedAt: result.finishedAt });
+ if (result === fallback) {
+ result.feedback = await readRecoveredFeedback(path.dirname(resultPath), {
+ campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,
+ });
+ if (traceConfig) result.tracing = await readInterruptedAttemptTrace(path.dirname(resultPath), {
+ projectId: traceConfig.projectId, executionId: candidate.id,
+ });
+ }
// Keep cleanup authority before evidence copying/publication can fail.
rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback, status: result.status });
return enforceResultProcessIntegrity(result, processResult);
@@ -712,7 +743,10 @@ async function runAttempt(input: {
}
}
for (const [index, candidate] of executions.entries()) {
- let result = results[index];
+ let result = {
+ ...results[index],
+ accountingIdentity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId },
+ };
if (
isolationError &&
result.failureClass !== "cleanup_failure" &&
@@ -836,6 +870,8 @@ async function runAttempt(input: {
}
return [...publishedResults];
} finally {
+ let gatewayCleanupError: unknown;
+ try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; }
if (!processCleanupFailed) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline);
let cleanupError: unknown;
const resourceAdmissionStarted = await access(path.join(temporaryRoot, "artifacts-private", "resource-admission-started"))
@@ -877,6 +913,7 @@ async function runAttempt(input: {
`Refusing to remove unexpected temporary path ${temporaryRoot}`,
);
}
+ if (gatewayCleanupError) cleanupError = new AggregateError([cleanupError, gatewayCleanupError].filter(Boolean), "Eval gateway cleanup failed");
if (startupReceiptPath) {
const receipt = JSON.parse(await readFile(startupReceiptPath, "utf8"));
receipt.status = cleanupError ? "installed_launch_chain_cleanup_failed" : "installed_launch_chain_passed";
@@ -909,6 +946,26 @@ async function runAttempt(input: {
console.error(message);
if (options.installedStartupOnly) throw new Error(message);
}
+ for (const result of publishedResults) {
+ const resultPath = publishedResultPaths.get(result.executionId);
+ const candidate = executions.find(candidate => candidate.id === result.executionId);
+ if (!traceConfig || !result.tracing || !resultPath || !candidate) continue;
+ const finalizationInput = {
+ receipt: result.tracing,
+ identity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,
+ provider: candidate.profile.provider, model: candidate.profile.model, environment: candidate.environment.id },
+ provenance: { suiteId: candidate.suite.id, suiteDefinitionHash: candidate.suiteDefinitionHash, profileId: candidate.profile.id,
+ caseId: candidate.task.id, sourceRevision: resolveRunnerE2ESource().sha },
+ result,
+ };
+ result.tracing.finalization = input.operations
+ ? await finalizeEvalAttemptThroughRelay(traceConfig, finalizationInput,
+ path.join(`${input.operations.manifest.ledgerPath}.otlp`, `${createHash("sha256").update(accountingAttemptId).digest("hex")}-finalization.jsonl`))
+ : await finalizeEvalAttemptTrace(traceConfig, finalizationInput);
+ const safeResult = `${JSON.stringify(sanitizeJson(result, attemptSecrets), null, 2)}\n`;
+ assertSecretFree(safeResult, attemptSecrets, resultPath);
+ await writeFile(resultPath, safeResult, "utf8");
+ }
}
}
@@ -932,6 +989,8 @@ async function runExecutionWithRetry(input: {
execution: MatrixExecution;
campaignId: string;
options: ReturnType<typeof parseRunnerSelectors>;
+ onAttempt: (result: RunnerE2EResult) => void;
+ operations?: EvalOperations;
}): Promise<RunnerE2EResult> {
const { execution, campaignId, options } = input;
return executeWithAutomaticRetry({
@@ -942,8 +1001,9 @@ async function runExecutionWithRetry(input: {
`Retrying ${execution.id} in a fresh isolated harness after ${result.failureClass!.replaceAll("_", " ")}`,
),
runAttempt: async attempt => {
- const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options });
+ const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options, operations: input.operations });
if (!result) throw new Error(`No result produced for ${execution.id} attempt ${attempt}`);
+ input.onAttempt(result);
return result;
},
});
@@ -968,7 +1028,9 @@ async function runWithConcurrency<T, R>(
}
},
);
- await Promise.all(workers);
+ const settled = await Promise.allSettled(workers);
+ const failures = settled.filter((result): result is PromiseRejectedResult => result.status === "rejected");
+ if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Eval campaign workers failed");
return results;
}
@@ -1011,6 +1073,7 @@ async function main() {
return;
}
+ if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST && executions.some(execution => execution.task.flow === "provider_connection")) throw new Error("Provider connection campaign has no budgeted routing qualification");
if (executions.some(execution => execution.task.flow === "provider_connection")) {
const { runConnectionCampaign } = await import("./connection-launch.js");
await runConnectionCampaign({ executions, catalog: runnerMatrix, configFile: options.connectionConfig, repositoryRoot });
@@ -1041,7 +1104,7 @@ async function main() {
await loadLocalEnvironment(process.env);
assertRemoteNativeEvidencePrerequisites(executions, process.env);
- const missingCredentials = [
+ const missingCredentials = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST ? [] : [
...new Set(
executions.flatMap((execution) => execution.requiredCredentials),
),
@@ -1084,21 +1147,35 @@ async function main() {
console.log(
`Running ${executions.length} isolated execution(s) with max parallelism ${requestedParallelism}`,
);
- const finalResults = await runWithConcurrency(
- executions,
- requestedParallelism,
- (execution) => runExecutionWithRetry({ execution, campaignId, options }),
- );
+ const attempts: RunnerE2EResult[] = [];
+ const operations = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST
+ ? await openEvalOperations(process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, executions, options.maxAutomaticRetries, process.env) : undefined;
+ let finalResults: RunnerE2EResult[] = [];
+ let campaignFailed = false;
+ try {
+ if (operations) await writeFile(path.join(summaryDir, "operations-preflight.json"), `${JSON.stringify(operations.manifest, null, 2)}\n`, { mode: 0o600 });
+ finalResults = await runWithConcurrency(executions, operations ? Math.min(requestedParallelism, operations.manifest.limits.maxConcurrent) : requestedParallelism,
+ execution => runExecutionWithRetry({ execution, campaignId, options, operations, onAttempt: result => attempts.push(result) }));
+ } catch {
+ campaignFailed = true;
+ finalResults = [...new Map(attempts.map(result => [result.executionId, result])).values()];
+ } finally {
+ try { await operations?.close(); } catch { campaignFailed = true; }
+ }
const generatedAt = new Date().toISOString();
+ const operationsReport = operations ? await readEvalOperationsReport(operations.manifest.ledgerPath) : undefined;
+ if (operationsReport?.status === "unavailable") campaignFailed = true;
const campaign = buildRunnerCampaign({
campaignId,
generatedAt,
expected: executions.map((execution) => execution.id),
results: finalResults,
+ attempts,
+ ...(operationsReport ? { operations: operationsReport } : {}),
});
const campaignSecrets = normalizedSecrets(
- CREDENTIAL_NAMES.map((name) => process.env[name]),
+ [...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => process.env[name]),
);
const safeCampaign = sanitizeJson(
campaign,
@@ -1138,7 +1215,8 @@ async function main() {
console.log(
`Campaign ${campaignId}: ${safeCampaign.passed}/${safeCampaign.selected} passed`,
);
- if (safeCampaign.failed > 0) process.exitCode = 1;
+ if (campaignFailed) console.error("Campaign finalization incomplete; inspect retained attempt artifacts and the operations ledger.");
+ if (campaignFailed || safeCampaign.failed > 0) process.exitCode = 1;
}
await main().catch((error) => {
diff --git a/tests/runner-e2e/ledger-attempt-policy.test.ts b/tests/runner-e2e/ledger-attempt-policy.test.ts
new file mode 100644
index 0000000000..138398236c
--- /dev/null
+++ b/tests/runner-e2e/ledger-attempt-policy.test.ts
@@ -0,0 +1,120 @@
+import { afterEach, expect, it } from "vitest";
+import { createHash } from "node:crypto";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js";
+import { adoptLegacyEvalAttemptPolicy, type LegacyAttemptAuditFile } from "./ledger-attempt-policy.js";
+
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+const limits: EvalSpendLimits = { campaignId: "legacy-pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 3, maxConcurrent: 1 };
+const identity = { executionId: "claude.local.fixture", attemptId: "old-request-attempt", phase: "agent" as const };
+const hash = (value: string) => createHash("sha256").update(value).digest("hex");
+
+async function fixture(overrides: Partial<EvalSpendLimits> = {}) {
+ const root = await mkdtemp(path.join(os.tmpdir(), "legacy-eval-attempts-")); roots.push(root);
+ const file = path.join(root, "ledger.jsonl");
+ const originalLimits = { ...limits, ...overrides };
+ const ledger = await openEvalCostLedger(file, originalLimits);
+ await ledger.reserve("cancelled-request", identity, "0.6");
+ await ledger.observe({ observationId: "partial", requestId: "cancelled-request", source: "provider", provider: "anthropic",
+ currency: "USD", route: "direct", classification: "reported", costUsd: "0.2", final: false });
+ await ledger.finish("cancelled-request", "cancelled");
+ await ledger.close();
+ const evidencePath = path.join(root, "reviewed-attempts.md");
+ const evidence = "Reviewed retained campaigns, including the earlier zero-request startup failure.";
+ await writeFile(evidencePath, evidence);
+ const loaded = await readEvalCostLedger(file);
+ const audit: LegacyAttemptAuditFile = { version: 1, ledgerPath: file, evidencePath, evidenceSha256: hash(evidence),
+ previousJournalHash: loaded.previous, maxAttemptsPerExecution: 3,
+ priorAttempts: [identity, { executionId: identity.executionId, attemptId: "old-zero-request-attempt" }]
+ .map(({ executionId, attemptId }) => ({ executionId, attemptId })) };
+ return { file, audit, originalLimits, before: loaded.state.snapshot(), prefix: await readFile(file, "utf8") };
+}
+
+it("appends an audited policy without rewriting charges, held reservations, or original limits", async () => {
+ const { file, audit, before, prefix, originalLimits } = await fixture();
+ const migrated = await adoptLegacyEvalAttemptPolicy(audit);
+ expect((await readFile(file, "utf8")).startsWith(prefix)).toBe(true);
+ expect(migrated.limits).toEqual({ ...originalLimits, maxAttemptsPerExecution: 3 });
+ expect({ ...migrated.accounting, admittedAttempts: [] }).toEqual(before);
+ expect(migrated.accounting.admittedAttempts).toEqual(audit.priorAttempts);
+ const resumed = await openEvalCostLedger(file, migrated.limits);
+ try {
+ await resumed.admitAttempt(identity.executionId, "new-attempt");
+ await expect(resumed.admitAttempt(identity.executionId, "extra-attempt")).rejects.toThrow("attempt cap");
+ await expect(resumed.reserve("new-request", { ...identity, attemptId: "new-attempt" }, "0.5")).rejects.toThrow("spending cap");
+ await resumed.reserve("new-request", { ...identity, attemptId: "new-attempt" }, "0.4");
+ expect(resumed.snapshot().committedUsd).toBe("1.000000000");
+ } finally { await resumed.close(); }
+ const replayed = await readEvalCostLedger(file);
+ expect(replayed.state.snapshot().admittedAttempts).toHaveLength(3);
+ expect(replayed.state.snapshot().knownUsd).toBe("0.200000000");
+ await expect(openEvalCostLedger(file, originalLimits)).rejects.toThrow("limits changed");
+ await expect(openEvalCostLedger(file, { ...migrated.limits, maxUsd: "2" })).rejects.toThrow("limits changed");
+});
+
+it("requires all recorded request identities and rejects duplicate or foreign attempts", async () => {
+ const { file, audit, prefix } = await fixture();
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [] })).rejects.toThrow("omits a recorded request");
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [audit.priorAttempts[0]!, audit.priorAttempts[0]!] })).rejects.toThrow("Invalid legacy attempt identity");
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [{ ...audit.priorAttempts[0]!, executionId: "other-cell" }] })).rejects.toThrow("omits a recorded request");
+ expect(await readFile(file, "utf8")).toBe(prefix);
+});
+
+it("rejects changed evidence and changed journal checkpoints without appending", async () => {
+ const { file, audit, prefix, originalLimits } = await fixture();
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, evidenceSha256: "a".repeat(64) })).rejects.toThrow("evidence changed");
+ expect(await readFile(file, "utf8")).toBe(prefix);
+ const writer = await openEvalCostLedger(file, originalLimits);
+ try {
+ await writer.observe({ observationId: "late", requestId: "cancelled-request", source: "invoice", provider: "anthropic",
+ currency: "USD", route: "direct", classification: "reconciled", costUsd: "0.1", final: true });
+ await expect(writer.adoptLegacyAttemptPolicy({ previousJournalHash: audit.previousJournalHash, evidenceSha256: audit.evidenceSha256,
+ maxAttemptsPerExecution: 3, priorAttempts: audit.priorAttempts })).rejects.toThrow("checkpoint changed");
+ } finally { await writer.close(); }
+ await expect(adoptLegacyEvalAttemptPolicy(audit)).rejects.toThrow("checkpoint changed");
+ expect((await readEvalCostLedger(file)).state.limits.maxAttemptsPerExecution).toBeUndefined();
+});
+
+it("refuses concurrent writers and unfinished requests", async () => {
+ const { file, audit, originalLimits } = await fixture();
+ const writer = await openEvalCostLedger(file, originalLimits);
+ try {
+ await expect(adoptLegacyEvalAttemptPolicy(audit)).rejects.toThrow("writer lock");
+ await writer.reserve("inflight", identity, "0.1");
+ } finally { await writer.close(); }
+ const changed = await readEvalCostLedger(file);
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, previousJournalHash: changed.previous })).rejects.toThrow("stopped requests");
+ expect((await readEvalCostLedger(file)).state.snapshot().inflight).toBe(1);
+});
+
+it("does not change established caps or admit extra attempts after an over-cap history", async () => {
+ const { file, audit } = await fixture();
+ const migrated = await adoptLegacyEvalAttemptPolicy({ ...audit, maxAttemptsPerExecution: 1 });
+ const writer = await openEvalCostLedger(file, migrated.limits);
+ try { await expect(writer.admitAttempt(identity.executionId, "extra")).rejects.toThrow("attempt cap"); }
+ finally { await writer.close(); }
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, previousJournalHash: migrated.journalHash, maxAttemptsPerExecution: 10 })).rejects.toThrow("already established");
+});
+
+it("preserves an expired campaign deadline during a historical audit", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "expired-eval-attempts-")); roots.push(root);
+ const file = path.join(root, "ledger.jsonl"), evidencePath = path.join(root, "audit.md");
+ const writer = await openEvalCostLedger(file, { ...limits, deadlineAt: "2000-01-01T00:00:00Z" });
+ await writer.close(); await writeFile(evidencePath, "No launched attempts.");
+ const original = await readEvalCostLedger(file);
+ const migrated = await adoptLegacyEvalAttemptPolicy({ version: 1, ledgerPath: file, evidencePath, evidenceSha256: hash("No launched attempts."),
+ previousJournalHash: original.previous, maxAttemptsPerExecution: 1, priorAttempts: [] });
+ const resumed = await openEvalCostLedger(file, migrated.limits);
+ try { await expect(resumed.admitAttempt("cell", "attempt")).rejects.toThrow("deadline"); }
+ finally { await resumed.close(); }
+});
+
+it("rejects extra policy fields instead of accepting limit changes or private notes", async () => {
+ const { file, audit, prefix } = await fixture();
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, maxUsd: "100" } as LegacyAttemptAuditFile)).rejects.toThrow("Invalid legacy attempt audit file");
+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [{ ...audit.priorAttempts[0]!, note: "private" }] } as unknown as LegacyAttemptAuditFile)).rejects.toThrow("Unknown ledger field");
+ expect(await readFile(file, "utf8")).toBe(prefix);
+});
diff --git a/tests/runner-e2e/ledger-attempt-policy.ts b/tests/runner-e2e/ledger-attempt-policy.ts
new file mode 100644
index 0000000000..3bcbc28676
--- /dev/null
+++ b/tests/runner-e2e/ledger-attempt-policy.ts
@@ -0,0 +1,40 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { openEvalCostLedger, readEvalCostLedger, type EvalLegacyAttemptAudit } from "./cost-ledger.js";
+
+export interface LegacyAttemptAuditFile extends EvalLegacyAttemptAudit {
+ version: 1;
+ ledgerPath: string;
+ evidencePath: string;
+}
+
+export async function adoptLegacyEvalAttemptPolicy(audit: LegacyAttemptAuditFile) {
+ const allowed = ["version", "ledgerPath", "evidencePath", "previousJournalHash", "evidenceSha256", "maxAttemptsPerExecution", "priorAttempts"];
+ if (!audit || typeof audit !== "object" || Array.isArray(audit) || Object.keys(audit).some(key => !allowed.includes(key))
+ || audit.version !== 1 || typeof audit.ledgerPath !== "string" || !path.isAbsolute(audit.ledgerPath)
+ || typeof audit.evidencePath !== "string" || !path.isAbsolute(audit.evidencePath)) throw new Error("Invalid legacy attempt audit file");
+ const evidence = await readFile(audit.evidencePath);
+ if (createHash("sha256").update(evidence).digest("hex") !== audit.evidenceSha256) throw new Error("Legacy attempt audit evidence changed");
+ const original = await readEvalCostLedger(audit.ledgerPath);
+ if (original.previous !== audit.previousJournalHash) throw new Error("Legacy attempt audit checkpoint changed");
+ const ledger = await openEvalCostLedger(audit.ledgerPath, original.state.limits);
+ try {
+ await ledger.adoptLegacyAttemptPolicy({ previousJournalHash: audit.previousJournalHash, evidenceSha256: audit.evidenceSha256,
+ maxAttemptsPerExecution: audit.maxAttemptsPerExecution, priorAttempts: audit.priorAttempts });
+ } finally { await ledger.close(); }
+ const updated = await readEvalCostLedger(audit.ledgerPath);
+ return { journalHash: updated.previous, records: updated.sequence, limits: updated.state.limits, accounting: updated.state.snapshot() };
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ try {
+ if (process.argv.length !== 3) throw new Error("Usage: ledger-attempt-policy.ts /absolute/private-audit.json");
+ const result = await adoptLegacyEvalAttemptPolicy(JSON.parse(await readFile(process.argv[2]!, "utf8")));
+ process.stdout.write(`Legacy attempt policy saved at journal record ${result.records}. Existing costs and limits are preserved.\n`);
+ } catch {
+ process.stderr.write("Legacy attempt audit failed. Inspect the private audit and original journal before retrying.\n");
+ process.exitCode = 1;
+ }
+}
diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts
index 5cc7ff5172..cc861c6232 100644
--- a/tests/runner-e2e/live-fixtures.ts
+++ b/tests/runner-e2e/live-fixtures.ts
@@ -1,3 +1,4 @@
+import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js";
import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js";
import path from "node:path";
@@ -44,11 +45,7 @@ interface AgentRecord {
}
interface ManagedAccountFixture {
connectionId: string;
- binding: {
- provider: "openai" | "anthropic" | "github" | "openrouter";
- method: "api_key";
- mode: "responsible_user";
- };
+ binding: AiConnectionBinding;
}
interface ProjectRecord {
@@ -143,6 +140,7 @@ export async function setupLiveFixtures(input: {
workspacePath: string;
credentials: Partial<Record<CredentialName, string>>;
daytonaImage?: string;
+ onCompanyCreated?: (companyId: string) => Promise<void>;
}): Promise<LiveFixtureValues> {
const { api, execution } = input;
const receiptSmoke = execution.suite.id === "case-handoffs" && execution.task.id === caseReceiptTask.id;
@@ -166,15 +164,17 @@ export async function setupLiveFixtures(input: {
registry.register<CompanyRecord>({
id: "company",
async setup() {
- return api.post<CompanyRecord>("/api/companies", {
+ const company = await api.post<CompanyRecord>("/api/companies", {
name: `Runner E2E ${execution.id} ${input.executionNonce}`,
description: "Ephemeral paid full-stack runner acceptance fixture",
budgetMonthlyCents: receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS
: ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id)
|| (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS
: execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS
- : ["stock-harness", "task-workspaces", "case-handoffs"].includes(execution.suite.id) ? 1_000 : 0,
+ : ["stock-harness", "task-workspaces", "case-handoffs", "feedback-persistence"].includes(execution.suite.id) ? 1_000 : 0,
});
+ await input.onCompanyCreated?.(company.id);
+ return company;
},
async teardown() {
// The isolated instance/database is removed by the launcher. Do not call
@@ -275,7 +275,12 @@ export async function setupLiveFixtures(input: {
const managedHiring = isManagedHiringCase(execution.suite.id, execution.task.id);
const savedCopilot = execution.profile.qualificationCandidate === "copilot";
- if (managedHiring || savedCopilot) {
+ const gatewayUrl = process.env.PAPERCLIP_E2E_GATEWAY_BASE_URL;
+ const gatewayToken = process.env.PAPERCLIP_E2E_GATEWAY_TOKEN;
+ const budgetedGateway = Boolean(gatewayUrl || gatewayToken);
+ if (budgetedGateway && (!gatewayUrl || !gatewayToken || !/^http:\/\/127\.0\.0\.1:[0-9]+$/.test(gatewayUrl)
+ || execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id))) throw new Error("Unqualified fixture gateway route");
+ if (managedHiring || savedCopilot || budgetedGateway) {
registry.register<ManagedAccountFixture>({
id: "ai-connection",
dependencies: ["company", ...(savedCopilot ? ["environment"] : [])],
@@ -288,22 +293,26 @@ export async function setupLiveFixtures(input: {
if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`);
const apiKey = input.credentials[key];
if (!apiKey) throw new Error(`Missing credential ${key}`);
- const account = await api.postSensitive<{ connectionId: string }>(
+ const account = await api.postSensitive<{ connectionId: string; grantId: string }>(
`/api/companies/${company.id}/ai-connections`,
{
provider,
method: "api_key",
name: `Runner E2E account ${input.executionNonce}`,
- ownership: "personal",
+ ownership: budgetedGateway ? "shared" : "personal",
+ ...(budgetedGateway ? { routing: { kind: "gateway", protocol: provider === "anthropic" ? "messages" : "responses",
+ baseUrl: provider === "anthropic" ? gatewayUrl : `${gatewayUrl}/v1`, auth: "bearer", models: [{ id: execution.profile.model }] } } : {}),
...(savedCopilot ? { environmentId: value<EnvironmentRecord>(resolved, "environment").id } : {}),
- apiKey,
+ apiKey: budgetedGateway ? gatewayToken : apiKey,
agentIds: [],
- allAgents: false,
+ allAgents: budgetedGateway,
},
);
return {
connectionId: account.connectionId,
- binding: { provider, method: "api_key", mode: "responsible_user" },
+ binding: budgetedGateway
+ ? { provider, method: "api_key", mode: "shared", connectionId: account.connectionId, grantId: account.grantId }
+ : { provider, method: "api_key", mode: "responsible_user" },
};
},
});
@@ -316,7 +325,7 @@ export async function setupLiveFixtures(input: {
"secrets",
"environment",
...((grokSubscription || codexSubscription) ? ["subscription-login"] : []),
- ...(managedHiring || savedCopilot ? ["ai-connection"] : []),
+ ...(managedHiring || savedCopilot || budgetedGateway ? ["ai-connection"] : []),
],
async setup(resolved) {
const company = value<CompanyRecord>(resolved, "company");
@@ -356,7 +365,7 @@ export async function setupLiveFixtures(input: {
setupProbe.defaultRunner = selected.defaultRunner;
}
}
- if (["stock-harness", "native-connection-guidance"].includes(execution.suite.id)
+ if (["stock-harness", "native-connection-guidance", "feedback-persistence"].includes(execution.suite.id)
|| (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) {
agent.budgetMonthlyCents = 1_000;
}
@@ -369,7 +378,7 @@ export async function setupLiveFixtures(input: {
heartbeat: { ...((runtimeConfig.heartbeat ?? {}) as Record<string, unknown>),
enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 1, maxDailyCostCents: CASE_RECEIPT_BUDGET_CENTS } };
}
- if (managedHiring || savedCopilot) {
+ if (managedHiring || savedCopilot || budgetedGateway) {
const account = value<ManagedAccountFixture>(resolved, "ai-connection");
const config = agent.adapterConfig as Record<string, unknown>;
delete config.env;
diff --git a/tests/runner-e2e/operating-overhead.test.ts b/tests/runner-e2e/operating-overhead.test.ts
new file mode 100644
index 0000000000..c837450da0
--- /dev/null
+++ b/tests/runner-e2e/operating-overhead.test.ts
@@ -0,0 +1,119 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js";
+import { EVAL_OVERHEAD_COMPONENTS, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";
+import { readEvalOperationsReport, renderEvalOperationsReport } from "./operations-report.js";
+
+const now = Date.parse("2026-10-21T00:00:00Z");
+const allowance = (): EvalOperatingOverhead => ({ version: 1, reference: "fixture-only", runner: "aws-fleet",
+ verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",
+ components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) });
+const currentAllowance = (time = Date.now()): EvalOperatingOverhead => ({ ...allowance(),
+ verifiedAt: new Date(time - 86_400_000).toISOString(), expiresAt: new Date(time + 30 * 86_400_000).toISOString() });
+const runner = "ubuntu-latest";
+const modelIdentity = { executionId: "claude.local.fixture", attemptId: "attempt-1", phase: "agent" as const };
+
+it("keeps operating overhead inside the same cap through model completion and a journal restart", async () => {
+ const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-"));
+ const file = path.join(directory, "ledger.jsonl");
+ const limits: EvalSpendLimits = { campaignId: "fixture", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z",
+ maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() };
+ try {
+ const ledger = await openEvalCostLedger(file, limits);
+ try {
+ expect((await readEvalCostLedger(file)).state.snapshot()).toMatchObject({ knownUsd: "0.000000000", heldUsd: "4.000000000", inflight: 0, unknownRequests: 4 });
+ await ledger.admitAttempt(modelIdentity.executionId, modelIdentity.attemptId);
+ await expect(ledger.reserve("too-large", modelIdentity, "17")).rejects.toThrow("spending cap");
+ await ledger.reserve("model", modelIdentity, "16");
+ await ledger.observe({ observationId: "model-receipt", requestId: "model", source: "provider", provider: "anthropic",
+ currency: "USD", route: "direct", classification: "reported", costUsd: "1", final: true });
+ await ledger.finish("model", "succeeded");
+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "1.000000000", heldUsd: "4.000000000", committedUsd: "5.000000000" });
+ } finally { await ledger.close(); }
+ const changed = structuredClone(limits); changed.overhead!.components[0]!.maxUsd = "0";
+ await expect(openEvalCostLedger(file, changed)).rejects.toThrow("limits changed");
+ const resumed = await openEvalCostLedger(file, limits);
+ try {
+ await resumed.finish("overhead:ci_compute", "cancelled");
+ expect(resumed.snapshot().heldUsd).toBe("4.000000000");
+ await resumed.observe({ observationId: "ci-partial", requestId: "overhead:ci_compute", source: "infrastructure", provider: "aws",
+ currency: "USD", route: "infrastructure", classification: "estimated", pricingVersion: "fixture", costUsd: "0.2", final: false });
+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "1.200000000", heldUsd: "3.800000000", committedUsd: "5.000000000", unknownRequests: 4 });
+ await resumed.observe({ observationId: "ci-invoice", requestId: "overhead:ci_compute", source: "invoice", provider: "aws",
+ currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "0.5", final: true });
+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "1.500000000", heldUsd: "3.000000000", committedUsd: "4.500000000", unknownRequests: 3 });
+ } finally { await resumed.close(); }
+ const report = await readEvalOperationsReport(file);
+ expect(report).toMatchObject({ requests: 5, unresolvedRequests: 3, knownUsd: "1.500000000", heldUsd: "3.000000000" });
+ expect(renderEvalOperationsReport(report)).toContain("campaign-overhead");
+ expect(renderEvalOperationsReport(report)).toContain("Held reservations are not charges");
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it("stops further model admission when a later infrastructure receipt exceeds the shared cap", async () => {
+ const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-overrun-"));
+ try {
+ const ledger = await openEvalCostLedger(path.join(directory, "ledger.jsonl"), { campaignId: "overrun", maxUsd: "20",
+ deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() });
+ try {
+ await ledger.admitAttempt(modelIdentity.executionId, modelIdentity.attemptId);
+ await ledger.observe({ observationId: "late-invoice", requestId: "overhead:ci_compute", source: "invoice", provider: "aws",
+ currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "21", final: true });
+ expect(ledger.snapshot()).toMatchObject({ overBudget: true, committedUsd: "24.000000000" });
+ await expect(ledger.reserve("next", modelIdentity, "0.1")).rejects.toThrow("spending cap");
+ } finally { await ledger.close(); }
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it("requires each cost category once and rejects unknown policy fields or missing provenance", () => {
+ expect(validateOperatingOverhead(allowance(), "20")).toBe(4_000_000_000n);
+ expect(() => validateOperatingOverhead({ ...allowance(), components: allowance().components.slice(1) }, "20")).toThrow("Incomplete");
+ const duplicate = allowance(); duplicate.components[0] = duplicate.components[1]!;
+ expect(() => validateOperatingOverhead(duplicate, "20")).toThrow("component");
+ expect(() => validateOperatingOverhead({ ...allowance(), apiKey: "never-allowed" } as never, "20")).toThrow("fields");
+ const missing = allowance(); missing.components[0]!.reference = "";
+ expect(() => validateOperatingOverhead(missing, "20")).toThrow("component");
+ expect(() => validateOperatingOverhead(null as never, "20")).toThrow("fields");
+ expect(() => validateOperatingOverhead({ ...allowance(), expiresAt: "2027-10-20T00:00:00Z" }, "20")).toThrow("Incomplete");
+ expect(() => validateOperatingOverhead(allowance(), "4")).toThrow("no model allowance");
+});
+
+it("requires fresh reviewed allowances tied to the actual CI runner before scheduled work", () => {
+ const hosted = { ...allowance(), runner: "github-hosted" as const };
+ const raw = JSON.stringify(hosted);
+ expect(scheduledOperatingOverhead(raw, runner, "20", now)).toEqual(hosted);
+ expect(() => scheduledOperatingOverhead(undefined, runner, "20", now)).toThrow("Missing reviewed");
+ expect(() => scheduledOperatingOverhead("not json", runner, "20", now)).toThrow("JSON");
+ expect(() => scheduledOperatingOverhead(JSON.stringify(allowance()), runner, "20", now)).toThrow("runner");
+ expect(() => scheduledOperatingOverhead(raw, "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", "20", now)).toThrow("runner");
+ expect(() => scheduledOperatingOverhead(raw, "unrecognized", "20", now)).toThrow("runner");
+ expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-10-19T23:59:59Z"))).toThrow("needs review");
+ expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-11-18T20:00:00Z"))).toThrow("needs review");
+});
+
+it("does not turn a zero allowance into an observed zero charge", async () => {
+ const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-zero-"));
+ const overhead = allowance(); overhead.components[0]!.maxUsd = "0";
+ try {
+ const ledger = await openEvalCostLedger(path.join(directory, "ledger.jsonl"), { campaignId: "zero", maxUsd: "20",
+ deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead });
+ try {
+ expect(ledger.snapshot().requests.find(row => row.requestId === "overhead:ci_compute"))
+ .toMatchObject({ costUsd: null, classification: "unknown", receiptComplete: false, reservedUsd: "0.000000000" });
+ } finally { await ledger.close(); }
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it("stops new admission after allowance expiry while preserving late invoice reconciliation", () => {
+ const state = new EvalSpendState({ campaignId: "expired", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z",
+ maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: allowance() });
+ const admitted = { type: "attempt" as const, executionId: modelIdentity.executionId, attemptId: modelIdentity.attemptId };
+ state.validate(admitted, now); state.apply(admitted);
+ const expiredAt = Date.parse(allowance().expiresAt);
+ expect(() => state.validate({ ...admitted, attemptId: "next" }, expiredAt)).toThrow("needs review");
+ expect(() => state.validate({ type: "reserve", requestId: "model", identity: modelIdentity, maxUsd: "1" }, expiredAt)).toThrow("needs review");
+ expect(() => state.validate({ type: "observe", observation: { observationId: "late-invoice", requestId: "overhead:ci_compute",
+ source: "invoice", provider: "aws", currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "0.5", final: true } }, expiredAt)).not.toThrow();
+});
diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.ts
new file mode 100644
index 0000000000..08396df0ef
--- /dev/null
+++ b/tests/runner-e2e/operating-overhead.ts
@@ -0,0 +1,53 @@
+import { usdToUnits } from "../../packages/shared/src/money.js";
+
+export const EVAL_OVERHEAD_COMPONENTS = ["ci_compute", "artifact_storage", "observability", "shared_subscription"] as const;
+export interface EvalOperatingOverhead {
+ version: 1;
+ reference: string;
+ runner: "github-hosted" | "aws-fleet";
+ verifiedAt: string;
+ expiresAt: string;
+ components: Array<{ component: typeof EVAL_OVERHEAD_COMPONENTS[number]; maxUsd: string; reference: string }>;
+}
+
+const object = (value: unknown, allowed: string[]) => {
+ if (!value || typeof value !== "object" || Array.isArray(value)
+ || Object.keys(value).some(key => !allowed.includes(key))) throw new Error("Invalid operating overhead fields");
+};
+const reference = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value);
+
+export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd: string): bigint {
+ object(value, ["version", "reference", "runner", "verifiedAt", "expiresAt", "components"]);
+ if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet"].includes(value.runner)
+ || typeof value.verifiedAt !== "string" || typeof value.expiresAt !== "string"
+ || !Number.isFinite(Date.parse(value.verifiedAt)) || !Number.isFinite(Date.parse(value.expiresAt))
+ || Date.parse(value.expiresAt) <= Date.parse(value.verifiedAt)
+ || Date.parse(value.expiresAt) - Date.parse(value.verifiedAt) > 31 * 86_400_000
+ || !Array.isArray(value.components) || value.components.length !== EVAL_OVERHEAD_COMPONENTS.length) {
+ throw new Error("Incomplete operating overhead allowance");
+ }
+ const seen = new Set<string>();
+ let reserved = 0n;
+ for (const component of value.components) {
+ object(component, ["component", "maxUsd", "reference"]);
+ if (!EVAL_OVERHEAD_COMPONENTS.includes(component.component) || seen.has(component.component)
+ || !reference(component.reference) || typeof component.maxUsd !== "string"
+ || !/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(component.maxUsd)) throw new Error("Invalid operating overhead component");
+ seen.add(component.component);
+ reserved += usdToUnits(component.maxUsd);
+ }
+ if (reserved >= usdToUnits(totalUsd)) throw new Error("Operating overhead leaves no model allowance");
+ return reserved;
+}
+
+export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number): EvalOperatingOverhead {
+ if (!raw || raw.length > 16_384) throw new Error("Missing reviewed operating overhead allowance");
+ let value: EvalOperatingOverhead;
+ try { value = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }
+ validateOperatingOverhead(value, totalUsd);
+ if (runner !== "ubuntu-latest" || value.runner !== "github-hosted") throw new Error("Bounded campaigns require the standard GitHub-hosted runner allowance");
+ if (!Number.isFinite(now) || now < Date.parse(value.verifiedAt) || now + 4 * 3_600_000 >= Date.parse(value.expiresAt)) {
+ throw new Error("Operating overhead allowance needs review before paid admission");
+ }
+ return value;
+}
diff --git a/tests/runner-e2e/operating-receipts.test.ts b/tests/runner-e2e/operating-receipts.test.ts
new file mode 100644
index 0000000000..4ce448949f
--- /dev/null
+++ b/tests/runner-e2e/operating-receipts.test.ts
@@ -0,0 +1,216 @@
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
+import * as fs from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it, vi } from "vitest";
+import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js";
+import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js";
+import { importOperatingReceipt, operatingReceiptObservation, type EvalOperatingReceipt } from "./operating-receipts.js";
+
+vi.mock("node:fs/promises", async (importOriginal) => {
+ const actual = await importOriginal<typeof fs>();
+ return { ...actual, open: vi.fn(actual.open) };
+});
+
+const evidence = Buffer.from("fixture-only infrastructure receipt");
+const overhead: EvalOperatingOverhead = { version: 1, runner: "github-hosted", reference: "fixture-allowance",
+ verifiedAt: "2026-10-01T00:00:00Z", expiresAt: "2026-11-01T00:00:00Z",
+ components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) };
+const receipt = (fields: Partial<EvalOperatingReceipt> = {}): EvalOperatingReceipt => ({ version: 2, costBasis: "component_cumulative", campaignId: "fixture",
+ allowanceReference: overhead.reference, component: "artifact_storage", evidenceSha256: createHash("sha256").update(evidence).digest("hex"),
+ windowStartedAt: "2026-10-01T00:00:00Z", windowEndedAt: "2026-10-02T00:00:00Z", coverage: "partial",
+ source: "infrastructure", provider: "fixture", classification: "estimated", costUsd: "0.2", pricingVersion: "fixture-pricing", ...fields });
+const now = Date.parse("2026-10-03T00:00:00Z");
+
+it("binds operating observations to the campaign, allowance, evidence and closed observation window", () => {
+ expect(operatingReceiptObservation(receipt(), "fixture", overhead, evidence, now)).toMatchObject({ requestId: "overhead:artifact_storage", final: false });
+ for (const changed of [receipt({ campaignId: "other" }), receipt({ allowanceReference: "other" }), receipt({ evidenceSha256: "0".repeat(64) }),
+ receipt({ windowEndedAt: "2026-10-04T00:00:00Z" }), receipt({ windowStartedAt: "2026-10-03T00:00:00Z" }),
+ receipt({ component: "model" as never }), receipt({ version: 1 as never }), receipt({ costBasis: undefined }),
+ receipt({ costBasis: "window_charge" as never }), { ...receipt(), apiKey: "disallowed" }]) {
+ expect(() => operatingReceiptObservation(changed, "fixture", overhead, evidence, now)).toThrow();
+ }
+ expect(() => operatingReceiptObservation(receipt(), "fixture", overhead, Buffer.from("different"), now)).toThrow();
+});
+
+it("does not upgrade missing or partial evidence to an invoice or completed receipt", () => {
+ for (const changed of [receipt({ classification: "reconciled" }), receipt({ classification: "unknown" }),
+ receipt({ classification: "unknown", costUsd: undefined, coverage: "complete" }), receipt({ pricingVersion: undefined }), receipt({ costUsd: "-1" })]) {
+ expect(() => operatingReceiptObservation(changed, "fixture", overhead, evidence, now)).toThrow();
+ }
+ expect(operatingReceiptObservation(receipt({ classification: "unknown", costUsd: undefined }), "fixture", overhead, evidence, now))
+ .toMatchObject({ classification: "unknown", final: false });
+ expect(operatingReceiptObservation(receipt({ component: "shared_subscription", classification: "allocated", coverage: "complete" }), "fixture", overhead, evidence, now))
+ .toMatchObject({ classification: "allocated", route: "subscription", final: true });
+});
+
+async function fixture(maxUsd = "20") {
+ const directory = await mkdtemp(path.join(tmpdir(), "operating-receipts-"));
+ const ledgerPath = path.join(directory, "ledger.jsonl");
+ const ledger = await openEvalCostLedger(ledgerPath, { campaignId: "fixture", maxUsd, deadlineAt: "2099-01-01T00:00:00Z",
+ maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead });
+ await ledger.close();
+ return { directory, ledgerPath };
+}
+
+it("retains private source evidence, imports idempotently and keeps partial holds until invoice reconciliation", async () => {
+ const { directory, ledgerPath } = await fixture();
+ try {
+ expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000", unknownRequests: 4 });
+ const before = await readFile(ledgerPath, "utf8");
+ await importOperatingReceipt(ledgerPath, Object.fromEntries(Object.entries(receipt()).reverse()) as unknown as EvalOperatingReceipt, evidence);
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ const retained = path.join(`${ledgerPath}.overhead`, `${receipt().evidenceSha256}.evidence`);
+ expect(await readFile(retained)).toEqual(evidence);
+ expect((await stat(retained)).mode & 0o777).toBe(0o600);
+ expect(await readdir(`${ledgerPath}.overhead`)).toHaveLength(2);
+ expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.3", coverage: "complete" }), evidence))
+ .toMatchObject({ knownUsd: "0.300000000", heldUsd: "3.000000000", unknownRequests: 3 });
+ expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.4", coverage: "complete" }), evidence))
+ .toMatchObject({ knownUsd: "0.400000000", heldUsd: "3.600000000", conflictingRequests: 1 });
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it("fails before ledger mutation when retained evidence is corrupt or another writer owns the journal", async () => {
+ const { directory, ledgerPath } = await fixture();
+ try {
+ await importOperatingReceipt(ledgerPath, receipt(), evidence);
+ const before = await readFile(ledgerPath, "utf8");
+ await writeFile(path.join(`${ledgerPath}.overhead`, `${receipt().evidenceSha256}.evidence`), "corrupted");
+ await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "0.5" }), evidence)).rejects.toThrow("conflicts");
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits);
+ try { await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("writer lock"); }
+ finally { await writer.close(); }
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it("rejects separate partial windows and accounts for their cumulative total before admitting more work", async () => {
+ const { directory, ledgerPath } = await fixture("4.5");
+ try {
+ await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8" }), evidence);
+ const before = await readFile(ledgerPath, "utf8");
+ await expect(importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8",
+ windowStartedAt: "2026-10-02T00:00:00Z", windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).rejects.toThrow("cumulative component window");
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "1.6",
+ windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).toMatchObject({ knownUsd: "1.600000000", heldUsd: "3.000000000", committedUsd: "4.600000000", overBudget: true });
+ const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits);
+ try {
+ await writer.admitAttempt("fixture-cell", "next");
+ await expect(writer.reserve("next", { executionId: "fixture-cell", attemptId: "next", phase: "agent" }, "0.01"))
+ .rejects.toThrow("spending cap reached");
+ } finally { await writer.close(); }
+ await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "0.9" }), evidence)).rejects.toThrow("cumulative component window");
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it.each(["partial", "complete"] as const)("rejects a larger cumulative estimate hidden by an earlier %s invoice", async coverage => {
+ const { directory, ledgerPath } = await fixture("4.5");
+ try {
+ await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8", coverage }), evidence);
+ const before = await readFile(ledgerPath, "utf8"), retained = await readdir(`${ledgerPath}.overhead`);
+ await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "1.6", coverage,
+ windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).rejects.toThrow("would understate the cumulative total");
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ expect(await readdir(`${ledgerPath}.overhead`)).toEqual(retained);
+ const updated = await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "1.6", coverage,
+ windowEndedAt: "2026-10-03T00:00:00Z" }), evidence);
+ expect(updated).toMatchObject({ knownUsd: "1.600000000", committedUsd: "4.600000000", overBudget: true });
+ const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits);
+ try {
+ await writer.admitAttempt("fixture-cell", "next");
+ await expect(writer.reserve("next", { executionId: "fixture-cell", attemptId: "next", phase: "agent" }, "0.01"))
+ .rejects.toThrow(coverage === "complete" ? "receipts conflict" : "spending cap reached");
+ } finally { await writer.close(); }
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it.each(["evidence", "receipt.json"])("retries must persist the existing %s file before recording its charge", async suffix => {
+ const { directory, ledgerPath } = await fixture();
+ const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual<typeof fs>("node:fs/promises")).open;
+ const value = receipt(), canonical = Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)));
+ const observation = operatingReceiptObservation(value, "fixture", overhead, evidence, now);
+ await mkdir(`${ledgerPath}.overhead`, { mode: 0o700 });
+ await writeFile(path.join(`${ledgerPath}.overhead`, suffix === "evidence" ? `${value.evidenceSha256}.evidence`
+ : `${observation.observationId.split(":")[1]}.receipt.json`), suffix === "evidence" ? evidence : `${JSON.stringify(canonical, null, 2)}\n`, { mode: 0o600 });
+ let syncs = 0;
+ const opened = vi.mocked(fs.open).mockImplementation(async (...args) => {
+ const handle = await originalOpen(...args);
+ if (String(args[0]).endsWith(`.${suffix}`)) {
+ const originalSync = handle.sync.bind(handle);
+ vi.spyOn(handle, "sync").mockImplementation(async () => {
+ if (++syncs <= 2) throw new Error("fixture file sync failure");
+ await originalSync();
+ });
+ }
+ return handle;
+ });
+ try {
+ for (let attempt = 0; attempt < 2; attempt++) {
+ await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("file sync failure");
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ }
+ expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000" });
+ expect(syncs).toBe(3);
+ } finally {
+ opened.mockImplementation(originalOpen);
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
+it.each(["evidence", "receipt.json"])("recovers an incomplete %s write without publishing corrupt evidence or requiring manual deletion", async suffix => {
+ const { directory, ledgerPath } = await fixture();
+ const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual<typeof fs>("node:fs/promises")).open;
+ let injected = false;
+ const opened = vi.mocked(fs.open).mockImplementation(async (...args) => {
+ const handle = await originalOpen(...args);
+ if (!injected && String(args[0]).includes(`.${suffix}.`) && String(args[0]).endsWith(".tmp")) {
+ const originalWrite = handle.writeFile.bind(handle);
+ vi.spyOn(handle, "writeFile").mockImplementation(async () => {
+ injected = true;
+ await originalWrite("partial");
+ throw new Error("fixture partial write");
+ });
+ }
+ return handle;
+ });
+ try {
+ await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("partial write");
+ expect(injected).toBe(true);
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ expect((await readdir(`${ledgerPath}.overhead`)).some(name => name.endsWith(`.${suffix}`) || name.endsWith(".tmp"))).toBe(false);
+ expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000" });
+ expect(await readdir(`${ledgerPath}.overhead`)).toHaveLength(2);
+ } finally {
+ opened.mockImplementation(originalOpen);
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
+it("retains the hold when the first evidence directory cannot be persisted in its parent", async () => {
+ const { directory, ledgerPath } = await fixture();
+ const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual<typeof fs>("node:fs/promises")).open;
+ let parentSyncs = 0;
+ const opened = vi.mocked(fs.open).mockImplementation(async (...args) => {
+ const handle = await originalOpen(...args);
+ if (args[0] === directory && args[1] === "r") {
+ vi.spyOn(handle, "sync").mockImplementation(async () => {
+ parentSyncs++;
+ throw new Error("fixture parent directory sync failure");
+ });
+ }
+ return handle;
+ });
+ try {
+ await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("parent directory sync failure");
+ expect(parentSyncs).toBe(1);
+ expect(await readFile(ledgerPath, "utf8")).toBe(before);
+ expect((await readEvalCostLedger(ledgerPath)).state.snapshot()).toMatchObject({ knownUsd: "0.000000000", heldUsd: "4.000000000" });
+ } finally {
+ opened.mockImplementation(originalOpen);
+ await rm(directory, { recursive: true, force: true });
+ }
+});
diff --git a/tests/runner-e2e/operating-receipts.ts b/tests/runner-e2e/operating-receipts.ts
new file mode 100644
index 0000000000..8806292b0c
--- /dev/null
+++ b/tests/runner-e2e/operating-receipts.ts
@@ -0,0 +1,136 @@
+import { createHash, randomUUID } from "node:crypto";
+import { link, mkdir, open, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { usdToUnits } from "../../packages/shared/src/money.js";
+import { openEvalCostLedger, readEvalCostLedger, validEvalLedgerId, type EvalCostObservation } from "./cost-ledger.js";
+import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js";
+import { collectGitHubOperatingUsage } from "./github-operating-usage.js";
+
+export interface EvalOperatingReceipt {
+ version: 2;
+ costBasis: "component_cumulative";
+ campaignId: string;
+ allowanceReference: string;
+ component: typeof EVAL_OVERHEAD_COMPONENTS[number];
+ evidenceSha256: string;
+ windowStartedAt: string;
+ windowEndedAt: string;
+ coverage: "complete" | "partial";
+ source: "infrastructure" | "invoice";
+ provider: string;
+ classification: EvalCostObservation["classification"];
+ costUsd?: string;
+ pricingVersion?: string;
+}
+const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");
+const canonical = (receipt: EvalOperatingReceipt) => Object.fromEntries(Object.entries(receipt).sort(([left], [right]) => left.localeCompare(right)));
+
+export function operatingReceiptObservation(receipt: EvalOperatingReceipt, campaignId: string, overhead: EvalOperatingOverhead,
+ evidence: Buffer, now = Date.now()): EvalCostObservation {
+ if (!receipt || typeof receipt !== "object" || Array.isArray(receipt) || Object.keys(receipt).some(key => ![
+ "version", "costBasis", "campaignId", "allowanceReference", "component", "evidenceSha256", "windowStartedAt", "windowEndedAt",
+ "coverage", "source", "provider", "classification", "costUsd", "pricingVersion",
+ ].includes(key))) throw new Error("Invalid operating receipt fields");
+ if (receipt.version !== 2 || receipt.costBasis !== "component_cumulative"
+ || receipt.campaignId !== campaignId || receipt.allowanceReference !== overhead.reference
+ || !EVAL_OVERHEAD_COMPONENTS.includes(receipt.component) || !validEvalLedgerId(receipt.provider)
+ || !["infrastructure", "invoice"].includes(receipt.source) || !["complete", "partial"].includes(receipt.coverage)
+ || receipt.evidenceSha256 !== hash(evidence) || !/^[a-f0-9]{64}$/.test(receipt.evidenceSha256)
+ || !Number.isFinite(now) || typeof receipt.windowStartedAt !== "string" || typeof receipt.windowEndedAt !== "string"
+ || !Number.isFinite(Date.parse(receipt.windowStartedAt)) || !Number.isFinite(Date.parse(receipt.windowEndedAt))
+ || Date.parse(receipt.windowStartedAt) > Date.parse(receipt.windowEndedAt) || Date.parse(receipt.windowEndedAt) > now) {
+ throw new Error("Operating receipt scope, evidence or window mismatch");
+ }
+ if (!["reported", "estimated", "allocated", "reconciled", "unknown"].includes(receipt.classification)
+ || receipt.classification === "reconciled" && receipt.source !== "invoice"
+ || receipt.classification === "unknown" && (receipt.costUsd !== undefined || receipt.coverage === "complete")
+ || receipt.classification !== "unknown" && (typeof receipt.costUsd !== "string" || !/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(receipt.costUsd))
+ || ["estimated", "allocated"].includes(receipt.classification) && !validEvalLedgerId(receipt.pricingVersion)
+ || receipt.pricingVersion !== undefined && !validEvalLedgerId(receipt.pricingVersion)) throw new Error("Invalid operating cost classification or provenance");
+ return { observationId: `operating-receipt:${hash(JSON.stringify(canonical(receipt)))}`, requestId: `overhead:${receipt.component}`,
+ source: receipt.source, provider: receipt.provider, currency: "USD",
+ route: receipt.component === "shared_subscription" ? "subscription" : "infrastructure",
+ classification: receipt.classification, ...(receipt.costUsd === undefined ? {} : { costUsd: receipt.costUsd }),
+ ...(receipt.pricingVersion === undefined ? {} : { pricingVersion: receipt.pricingVersion }), final: receipt.coverage === "complete" };
+}
+
+async function retain(file: string, bytes: Buffer) {
+ const temporary = `${file}.${randomUUID()}.tmp`;
+ const handle = await open(temporary, "wx", 0o600);
+ try {
+ try { await handle.writeFile(bytes); await handle.sync(); } finally { await handle.close(); }
+ // Hard-link publication is atomic and cannot replace previously retained evidence.
+ try { await link(temporary, file); }
+ catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+ const existing = await open(file, "r+");
+ try {
+ if (!(await existing.readFile()).equals(bytes)) throw new Error("Retained operating evidence conflicts");
+ await existing.sync();
+ } finally { await existing.close(); }
+ }
+ } finally { await rm(temporary, { force: true }); }
+}
+
+export async function importOperatingReceipt(ledgerPath: string, receipt: EvalOperatingReceipt, evidence: Buffer) {
+ if (evidence.length < 1 || evidence.length > 8 * 1024 * 1024) throw new Error("Operating evidence must be between one byte and 8 MiB");
+ const loaded = await readEvalCostLedger(ledgerPath);
+ if (!loaded.state.limits.overhead) throw new Error("Campaign has no declared operating overhead");
+ const observation = operatingReceiptObservation(receipt, loaded.state.limits.campaignId, loaded.state.limits.overhead, evidence);
+ const ledger = await openEvalCostLedger(ledgerPath, loaded.state.limits);
+ try {
+ const directory = `${ledgerPath}.overhead`;
+ const current = (await readEvalCostLedger(ledgerPath)).state;
+ const alreadyImported = current.observations.has(observation.observationId);
+ if (!alreadyImported) {
+ for (const previous of current.observations.values()) {
+ if (previous.requestId !== observation.requestId) continue;
+ if (!/^operating-receipt:[a-f0-9]{64}$/.test(previous.observationId)) {
+ throw new Error("Operating receipt lacks cumulative window evidence");
+ }
+ const prior = JSON.parse(await readFile(path.join(directory, `${previous.observationId.split(":")[1]}.receipt.json`), "utf8")) as EvalOperatingReceipt;
+ if (previous.observationId !== `operating-receipt:${hash(JSON.stringify(canonical(prior)))}`
+ || prior.version !== 2 || prior.costBasis !== "component_cumulative") {
+ throw new Error("Retained operating receipt lacks verified cumulative window evidence");
+ }
+ if (Date.parse(receipt.windowStartedAt) !== Date.parse(prior.windowStartedAt)
+ || Date.parse(receipt.windowEndedAt) < Date.parse(prior.windowEndedAt)
+ || prior.coverage === "complete" && receipt.coverage !== "complete") {
+ throw new Error("Operating receipts must cover one cumulative component window; combine separate charges before importing");
+ }
+ }
+ // Preview the ledger's own precedence rules before retaining or appending a receipt.
+ current.validate({ type: "observe", observation });
+ current.apply({ type: "observe", observation });
+ const counted = current.snapshot().requests.find(request => request.requestId === observation.requestId)?.costUsd;
+ if (observation.costUsd !== undefined && (counted == null || usdToUnits(counted) < usdToUnits(observation.costUsd))) {
+ throw new Error("Operating receipt would understate the cumulative total; reconcile the entire component window before importing");
+ }
+ }
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ await retain(path.join(directory, `${receipt.evidenceSha256}.evidence`), evidence);
+ await retain(path.join(directory, `${observation.observationId.split(":")[1]}.receipt.json`), Buffer.from(`${JSON.stringify(canonical(receipt), null, 2)}\n`));
+ for (const durableDirectory of [directory, path.dirname(directory)]) {
+ const directoryHandle = await open(durableDirectory, "r");
+ try { await directoryHandle.sync(); } finally { await directoryHandle.close(); }
+ }
+ if (!alreadyImported) await ledger.observe(observation);
+ return ledger.snapshot();
+ } finally { await ledger.close(); }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [command, first, second, third, ...rest] = process.argv.slice(2);
+ if (command === "github" && first && second && !third) {
+ const usage = await collectGitHubOperatingUsage(first);
+ await retain(path.resolve(second), Buffer.from(`${JSON.stringify(usage, null, 2)}\n`));
+ console.log(JSON.stringify({ source: usage.workflowUrl, summedJobSeconds: usage.summedJobSeconds,
+ retainedArtifactBytes: usage.artifacts.retainedBytes, billing: "unknown", output: path.resolve(second) }));
+ } else if (command === "import" && first && second && third && !rest.length) {
+ const receipt = JSON.parse(await readFile(second, "utf8")) as EvalOperatingReceipt;
+ const result = await importOperatingReceipt(path.resolve(first), receipt, await readFile(third));
+ console.log(JSON.stringify({ campaignId: result.campaignId, knownUsd: result.knownUsd, heldUsd: result.heldUsd,
+ committedUsd: result.committedUsd, unknownRequests: result.unknownRequests, conflictingRequests: result.conflictingRequests }));
+ } else throw new Error("Usage: operating-receipts.ts github <run-id> <private-output> | import <ledger> <receipt> <evidence>");
+}
diff --git a/tests/runner-e2e/operations-report.test.ts b/tests/runner-e2e/operations-report.test.ts
new file mode 100644
index 0000000000..a310ddd35e
--- /dev/null
+++ b/tests/runner-e2e/operations-report.test.ts
@@ -0,0 +1,93 @@
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { openEvalCostLedger } from "./cost-ledger.js";
+import { readEvalOperationsReport, renderEvalOperationsReport, validateEvalOperationsReport } from "./operations-report.js";
+import { buildRunnerCampaign } from "./history.js";
+import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
+
+it("reports shared setup/retry spending once, preserves unknown holds and omits private provider receipts", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-"));
+ const file = path.join(root, "ledger.jsonl");
+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 });
+ try {
+ await ledger.reserve("setup", { executionId: "cell", attemptId: "first", phase: "setup" }, "0.2");
+ await ledger.observe({ observationId: "gateway", requestId: "setup", provider: "fixture", providerRequestId: "private-provider-id", source: "gateway",
+ currency: "USD", route: "direct", classification: "estimated", pricingVersion: "fixture", costUsd: "0.05", final: true });
+ await ledger.observe({ observationId: "native", requestId: "setup", provider: "fixture", providerRequestId: "private-provider-id", source: "native_otlp",
+ currency: "USD", route: "direct", classification: "reported", costUsd: "0.05", final: true });
+ await ledger.finish("setup", "failed");
+ await ledger.reserve("retry", { executionId: "cell", attemptId: "retry", phase: "agent" }, "0.2");
+ await ledger.finish("retry", "cancelled");
+ const report = await readEvalOperationsReport(file);
+ validateEvalOperationsReport(report);
+ if (report.status === "unavailable") throw new Error("Expected readable fixture ledger");
+ expect(report).toMatchObject({ knownUsd: "0.050000000", heldUsd: "0.200000000", committedUsd: "0.250000000", requests: 2,
+ unresolvedRequests: 1, inflight: 0, breakdown: { reported: "0.050000000", estimated: "0.000000000" },
+ attempts: [{ attemptId: "first", requests: 1, knownUsd: "0.050000000" }, { attemptId: "retry", unresolvedRequests: 1, heldUsd: "0.200000000" }] });
+ expect(JSON.stringify(report)).not.toContain("private-provider-id");
+ const campaign = buildRunnerCampaign({ campaignId: "retry-campaign", generatedAt: new Date().toISOString(), expected: [], results: [], operations: report });
+ expect(campaign.operations).toEqual(report);
+ expect(campaign.billing.reportedLlmCostUsd).toBe(0);
+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));
+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });
+ const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));
+ expect(rebuilt.operations).toEqual(report);
+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("$0.200000000 additional held reservations");
+ expect(renderEvalOperationsReport(report)).toContain("Do not add its model subtotal to heartbeat totals");
+ expect(renderEvalOperationsReport(report)).toContain("Unrecorded infrastructure, storage, subscriptions");
+ expect(() => validateEvalOperationsReport({ ...report, secret: "forbidden" } as never)).toThrow("Invalid retained");
+ expect(() => validateEvalOperationsReport({ ...report, knownUsd: "0.060000000" })).toThrow("Invalid retained");
+ expect(() => validateEvalOperationsReport({ ...report, attempts: [] })).toThrow("Invalid retained");
+ expect(() => validateEvalOperationsReport({ ...report, deadlineAt: "Tue, 20 Oct 2026 01:00:00 GMT (private note)" })).toThrow("Invalid retained");
+ } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); }
+});
+
+it("publishes the parsed deadline without private text accepted by the journal's date parser", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl");
+ const deadlineAt = "Thu, 01 Jan 2099 00:00:00 GMT (private budget rationale)";
+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt, maxRequests: 10, maxConcurrent: 2 });
+ try {
+ const report = await readEvalOperationsReport(file);
+ expect(report).toMatchObject({ deadlineAt: "2099-01-01T00:00:00.000Z" });
+ expect(JSON.stringify(report)).not.toContain("private budget rationale");
+ expect(await readFile(file, "utf8")).toContain(deadlineAt);
+ } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); }
+});
+
+it("preserves reports with explicit unknown accounting after a partial journal write", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl");
+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 });
+ await ledger.reserve("request", { executionId: "cell", attemptId: "attempt", phase: "setup" }, "0.2");
+ await ledger.close();
+ try {
+ await writeFile(file, '{"partial-record":', { flag: "a" });
+ const damaged = await readFile(file, "utf8");
+ const report = await readEvalOperationsReport(file);
+ expect(report).toEqual({ version: 1, scope: "shared_gateway_ledger", status: "unavailable" });
+ validateEvalOperationsReport(report);
+ expect(renderEvalOperationsReport(report)).toContain("Request amounts, counts, and held reservations are unknown");
+ expect(renderEvalOperationsReport(report)).not.toContain("$0");
+ expect(() => validateEvalOperationsReport({ ...report, knownUsd: "0.000000000" } as never)).toThrow("Invalid retained");
+ const campaign = buildRunnerCampaign({ campaignId: "damaged-ledger", generatedAt: new Date().toISOString(), expected: [], results: [], operations: report });
+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));
+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });
+ const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));
+ expect(rebuilt.operations).toEqual(report);
+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("Request ledger accounting unavailable");
+ expect(await readFile(file, "utf8")).toBe(damaged);
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
+
+it("keeps an in-flight final receipt reserved until the forwarding request finishes", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl");
+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 });
+ try {
+ await ledger.reserve("request", { executionId: "cell", attemptId: "attempt", phase: "agent" }, "0.2");
+ await ledger.observe({ observationId: "usage", requestId: "request", provider: "fixture", source: "provider", currency: "USD", route: "direct", classification: "reported", costUsd: "0.05", final: true });
+ const report = await readEvalOperationsReport(file);
+ validateEvalOperationsReport(report);
+ expect(report).toMatchObject({ inflight: 1, knownUsd: "0.050000000", heldUsd: "0.150000000", committedUsd: "0.200000000", unresolvedRequests: 0 });
+ } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); }
+});
diff --git a/tests/runner-e2e/operations-report.ts b/tests/runner-e2e/operations-report.ts
new file mode 100644
index 0000000000..97d8a3df77
--- /dev/null
+++ b/tests/runner-e2e/operations-report.ts
@@ -0,0 +1,114 @@
+import { readEvalCostLedger, validEvalLedgerId } from "./cost-ledger.js";
+import { usdToUnits } from "../../packages/shared/src/money.js";
+import { html, isPublicReportTimestamp } from "./report-format.js";
+
+const money = (n: bigint) => `${n / 1_000_000_000n}.${String(n % 1_000_000_000n).padStart(9, "0")}`;
+interface EvalOperationsAvailableReport {
+ version: 1;
+ scope: "shared_gateway_ledger";
+ status?: never;
+ ledgerCampaignId: string;
+ journalHash: string;
+ records: number;
+ deadlineAt: string;
+ limitUsd: string;
+ knownUsd: string;
+ heldUsd: string;
+ committedUsd: string;
+ requests: number;
+ unresolvedRequests: number;
+ conflictingRequests: number;
+ inflight: number;
+ breakdown: { reported: string; estimated: string; allocated: string; reconciled: string };
+ attempts: Array<{ executionId: string; attemptId: string; requests: number; unresolvedRequests: number; knownUsd: string; heldUsd: string }>;
+}
+export type EvalOperationsReport = EvalOperationsAvailableReport | {
+ version: 1; scope: "shared_gateway_ledger"; status: "unavailable";
+};
+
+export function validateEvalOperationsReport(report: EvalOperationsReport): void {
+ const fail = (): never => { throw new Error("Invalid retained eval operations report"); };
+ const fields = (value: unknown, allowed: string[]) => {
+ if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(key => !allowed.includes(key))) fail();
+ };
+ if (report?.status === "unavailable") {
+ fields(report, ["version", "scope", "status"]);
+ if (report.version !== 1 || report.scope !== "shared_gateway_ledger") fail();
+ return;
+ }
+ const dollars = (value: unknown): bigint => typeof value === "string" && /^(?:0|[1-9][0-9]*)\.[0-9]{9}$/.test(value) ? usdToUnits(value) : fail();
+ const count = (value: unknown) => { if (!Number.isSafeInteger(value) || Number(value) < 0) fail(); };
+ fields(report, ["version", "scope", "ledgerCampaignId", "journalHash", "records", "deadlineAt", "limitUsd", "knownUsd", "heldUsd", "committedUsd", "requests", "unresolvedRequests", "conflictingRequests", "inflight", "breakdown", "attempts"]);
+ if (report.version !== 1 || report.scope !== "shared_gateway_ledger" || !validEvalLedgerId(report.ledgerCampaignId)
+ || !/^[a-f0-9]{64}$/.test(report.journalHash) || !isPublicReportTimestamp(report.deadlineAt) || !Array.isArray(report.attempts)) fail();
+ [report.records, report.requests, report.unresolvedRequests, report.conflictingRequests, report.inflight].forEach(count);
+ if (report.records < 1 || report.unresolvedRequests > report.requests || report.conflictingRequests > report.unresolvedRequests || report.inflight > report.requests) fail();
+ dollars(report.limitUsd);
+ const known = dollars(report.knownUsd), held = dollars(report.heldUsd);
+ if (known + held !== dollars(report.committedUsd)) fail();
+ fields(report.breakdown, ["reported", "estimated", "allocated", "reconciled"]);
+ if (["reported", "estimated", "allocated", "reconciled"].reduce((sum, key) => sum + dollars(report.breakdown[key as keyof typeof report.breakdown]), 0n) !== known) fail();
+ let requests = 0, unresolved = 0, rowKnown = 0n, rowHeld = 0n;
+ const seen = new Set<string>();
+ for (const attempt of report.attempts) {
+ fields(attempt, ["executionId", "attemptId", "requests", "unresolvedRequests", "knownUsd", "heldUsd"]);
+ if (!validEvalLedgerId(attempt.executionId) || !validEvalLedgerId(attempt.attemptId)) fail();
+ const key = JSON.stringify([attempt.executionId, attempt.attemptId]); if (seen.has(key)) fail(); seen.add(key);
+ count(attempt.requests); count(attempt.unresolvedRequests);
+ if (attempt.requests < 1 || attempt.unresolvedRequests > attempt.requests) fail();
+ requests += attempt.requests; unresolved += attempt.unresolvedRequests;
+ rowKnown += dollars(attempt.knownUsd); rowHeld += dollars(attempt.heldUsd);
+ }
+ if (requests !== report.requests || unresolved !== report.unresolvedRequests || rowKnown !== known || rowHeld !== held) fail();
+}
+
+export async function readEvalOperationsReport(file: string): Promise<EvalOperationsReport> {
+ try { return await readAvailableEvalOperationsReport(file); }
+ catch { return { version: 1, scope: "shared_gateway_ledger", status: "unavailable" }; }
+}
+
+async function readAvailableEvalOperationsReport(file: string): Promise<EvalOperationsAvailableReport> {
+ const loaded = await readEvalCostLedger(file), snapshot = loaded.state.snapshot();
+ const groups = new Map<string, EvalOperationsAvailableReport["attempts"][number]>();
+ for (const request of snapshot.requests) {
+ const key = JSON.stringify([request.executionId, request.attemptId]);
+ const group = groups.get(key) ?? { executionId: request.executionId, attemptId: request.attemptId,
+ requests: 0, unresolvedRequests: 0, knownUsd: money(0n), heldUsd: money(0n) };
+ const cost = usdToUnits(request.costUsd ?? "0"), reserved = usdToUnits(request.reservedUsd);
+ const held = (!request.finished || !request.receiptComplete) && reserved > cost ? reserved - cost : 0n;
+ group.requests++; if (!request.receiptComplete) group.unresolvedRequests++;
+ group.knownUsd = money(usdToUnits(group.knownUsd) + cost);
+ group.heldUsd = money(usdToUnits(group.heldUsd) + held);
+ groups.set(key, group);
+ }
+ const report: EvalOperationsAvailableReport = { version: 1, scope: "shared_gateway_ledger", ledgerCampaignId: snapshot.campaignId,
+ journalHash: loaded.previous, records: loaded.sequence, deadlineAt: new Date(loaded.state.limits.deadlineAt).toISOString(),
+ limitUsd: snapshot.limitUsd, knownUsd: snapshot.knownUsd, heldUsd: snapshot.heldUsd, committedUsd: snapshot.committedUsd,
+ requests: snapshot.requests.length, unresolvedRequests: snapshot.unknownRequests, conflictingRequests: snapshot.conflictingRequests,
+ inflight: snapshot.inflight, breakdown: snapshot.breakdown as EvalOperationsAvailableReport["breakdown"], attempts: [...groups.values()] };
+ validateEvalOperationsReport(report);
+ return report;
+}
+
+export function operationsReportText(report: EvalOperationsReport): string[] {
+ if (report.status === "unavailable") return [
+ "Request ledger accounting unavailable: the journal could not be read or validated. Request amounts, counts, and held reservations are unknown.",
+ "Retained test results are preserved. Recover and validate the original journal before resuming paid work; do not reset the allowance or treat missing amounts as zero.",
+ ];
+ return [
+ `Shared request ledger ${report.ledgerCampaignId}: $${report.knownUsd} known subtotal; $${report.heldUsd} additional held reservations; $${report.committedUsd} committed against $${report.limitUsd}.`,
+ `Charges: $${report.breakdown.reconciled} reconciled; $${report.breakdown.reported} reported; $${report.breakdown.estimated} estimated; $${report.breakdown.allocated} allocated.`,
+ `${report.requests} recorded requests or overhead reservations; ${report.unresolvedRequests} unresolved; ${report.conflictingRequests} conflicting; ${report.inflight} in flight. Deadline ${report.deadlineAt}.`,
+ "This shared ledger includes earlier attempts and setup probes routed through the gateway, plus any declared campaign-overhead reservations. Do not add its model subtotal to heartbeat totals. Held reservations are not charges. Unrecorded infrastructure, storage, subscriptions and requests outside the gateway still need accounting; zero recorded requests does not prove zero cost.",
+ `Journal checkpoint: ${report.records} records; SHA-256 ${report.journalHash}.`,
+ ];
+}
+
+export function renderEvalOperationsReport(report: EvalOperationsReport): string {
+ validateEvalOperationsReport(report);
+ if (report.status === "unavailable") return `<section aria-label="Request ledger accounting"><h2>Request ledger accounting unavailable</h2>${operationsReportText(report).map(line => `<p>${html(line)}</p>`).join("")}</section>`;
+ return `<section aria-label="Request ledger accounting"><h2>Request ledger accounting</h2>${operationsReportText(report).map(line => `<p>${html(line)}</p>`).join("")}
+ <table><thead><tr><th>Execution</th><th>Attempt or component</th><th>Requests / reservations</th><th>Known subtotal</th><th>Additional held</th><th>Unresolved</th></tr></thead><tbody>
+ ${report.attempts.map(attempt => `<tr><td>${html(attempt.executionId)}</td><td>${html(attempt.attemptId)}</td><td>${attempt.requests}</td><td>$${html(attempt.knownUsd)}</td><td>$${html(attempt.heldUsd)}</td><td>${attempt.unresolvedRequests}</td></tr>`).join("")}
+ </tbody></table></section>`;
+}
diff --git a/tests/runner-e2e/operations.test.ts b/tests/runner-e2e/operations.test.ts
new file mode 100644
index 0000000000..7193837a01
--- /dev/null
+++ b/tests/runner-e2e/operations.test.ts
@@ -0,0 +1,173 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { tmpdir } from "node:os";
+import { runnerMatrix } from "./catalog.js";
+import { validateEvalOperationsManifest, openEvalOperations, type EvalOperationsManifest } from "./operations.js";
+import { setupLiveFixtures } from "./live-fixtures.js";
+import { createAiConnectionSchema, aiConnectionBindingSchema } from "../../packages/shared/src/ai-connections.js";
+import type { RunnerApi } from "./api.js";
+import { beginEvalAgentPhase, readEvalAttemptPhase } from "./attempt-phase.js";
+import { applyEvalGatewayEnvironment, buildPaperclipServerEnvironment } from "./harness-env.js";
+const cell = runnerMatrix.find(e => e.suite.id === "core-compatibility" && e.profile.id === "runner-codex" && e.environment.id === "local")!;
+function manifest(): EvalOperationsManifest {
+ return { version: 1, limits: { campaignId: "pilot", maxUsd: "100", deadlineAt: new Date(Date.now() + 60_000).toISOString(), maxRequests: 20, maxConcurrent: 1 },
+ ledgerPath: "/tmp/fixture-not-opened.jsonl", executionIds: [cell.id], maxAttemptsPerExecution: 2, route: "direct",
+ models: [{ provider: "openai", maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 1000,
+ price: { provider: "openai", model: cell.profile.model, version: "fixture-only", sourceUrl: "https://provider.invalid/fixture", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" } }] };
+}
+afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
+it("admits only the exact bounded local selection and explicitly priced models", () => {
+ expect(validateEvalOperationsManifest(manifest(), [cell], 1)).toMatchObject({ route: "direct" });
+ expect(() => validateEvalOperationsManifest({ ...manifest(), executionIds: [] }, [cell], 1)).toThrow("selection");
+ expect(() => validateEvalOperationsManifest(manifest(), [cell], 2)).toThrow("limits");
+ const remote = runnerMatrix.find(e => e.suite.id === cell.suite.id && e.profile.id === cell.profile.id && e.environment.id === "daytona")!;
+ expect(() => validateEvalOperationsManifest({ ...manifest(), executionIds: [remote.id] }, [remote], 1)).toThrow("Unqualified");
+ const changed = manifest(); changed.models[0]!.price.model = "other";
+ expect(() => validateEvalOperationsManifest(changed, [cell], 1)).toThrow("Missing exact model");
+});
+it("rejects credentials in the manifest and unknown pricing/billing policy fields", () => {
+ expect(() => validateEvalOperationsManifest({ ...manifest(), apiKey: "private" } as never, [cell], 0)).toThrow("fields");
+ const changed = manifest(); Object.assign(changed.models[0]!, { apiKey: "private" });
+ expect(() => validateEvalOperationsManifest(changed, [cell], 0)).toThrow("fields");
+ expect(() => validateEvalOperationsManifest({ ...manifest(), route: "braintrust" }, [cell], 0)).toThrow("project UUID");
+});
+
+it("rejects an unaffordable selected request before opening a ledger or checking credentials", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-unaffordable-preflight-"));
+ try {
+ const value = manifest(); value.limits.maxUsd = "0.002999999"; value.ledgerPath = path.join(root, "ledger.jsonl");
+ const file = path.join(root, "manifest.json"); await writeFile(file, JSON.stringify(value));
+ await expect(openEvalOperations(file, [cell], 0, {})).rejects.toThrow("Insufficient initial model allowance");
+ await expect(access(value.ledgerPath)).rejects.toMatchObject({ code: "ENOENT" });
+ await expect(access(`${value.ledgerPath}.requests`)).rejects.toMatchObject({ code: "ENOENT" });
+ value.limits.maxUsd = "0.003";
+ expect(validateEvalOperationsManifest(value, [cell], 0)).toBe(value);
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
+
+it("subtracts declared operating holds and checks only models used by the selection", () => {
+ const value = manifest(); value.limits.maxUsd = "0.007"; value.limits.maxAttemptsPerExecution = 2;
+ value.limits.overhead = { version: 1, reference: "fixture", runner: "github-hosted",
+ verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",
+ components: (["ci_compute", "artifact_storage", "observability", "shared_subscription"] as const).map(component => ({
+ component, maxUsd: "0.001", reference: "fixture" })) };
+ value.models.push({ ...structuredClone(value.models[0]!), maximumInputTokens: 1_000_000,
+ price: { ...value.models[0]!.price, model: "unused-model" } });
+ expect(validateEvalOperationsManifest(value, [cell], 0)).toBe(value);
+ value.limits.maxUsd = "0.006999999";
+ expect(() => validateEvalOperationsManifest(value, [cell], 0)).toThrow("Insufficient initial model allowance");
+});
+it("retains a campaign summary and refuses changing its budget on restart", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-ops-test-"));
+ try {
+ const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl") }, file = path.join(root, "manifest.json");
+ await writeFile(file, JSON.stringify(value));
+ const operations = await openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" });
+ const endpoint = await operations.start(cell, "attempt", 1);
+ expect(endpoint.baseUrl).toMatch(/^http:\/\/127\.0\.0\.1:/);
+ expect(await readEvalAttemptPhase(endpoint.phasePath, { executionId: cell.id, attemptId: "attempt" })).toMatchObject({ phase: "setup" });
+ await beginEvalAgentPhase(cell.id, { PAPERCLIP_E2E_GATEWAY_PHASE_PATH: endpoint.phasePath, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: "attempt" });
+ expect(await readEvalAttemptPhase(endpoint.phasePath, { executionId: cell.id, attemptId: "attempt" })).toMatchObject({ phase: "agent" });
+ await endpoint.close(); await operations.close();
+ expect(JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8"))).toMatchObject({ limitUsd: "100.000000000", knownUsd: "0.000000000", requests: [] });
+ const resumed = await openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" });
+ const retry = await resumed.start(cell, "retry-after-restart", 1);
+ await retry.close();
+ await expect(resumed.start(cell, "third-attempt", 1)).rejects.toThrow("attempt cap");
+ await resumed.close();
+ const summary = JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8"));
+ expect(summary.admittedAttempts).toEqual([{ executionId: cell.id, attemptId: "attempt" }, { executionId: cell.id, attemptId: "retry-after-restart" }]);
+ await writeFile(file, JSON.stringify({ ...value, maxAttemptsPerExecution: 3 }));
+ await expect(openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" })).rejects.toThrow("limits changed");
+ await writeFile(file, JSON.stringify({ ...value, limits: { ...value.limits, maxUsd: "101" } }));
+ await expect(openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" })).rejects.toThrow("limits changed");
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
+
+it("keeps both upstream keys on the controller and owns the trace relay lifetime", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-ops-relay-"));
+ const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl") }, file = path.join(root, "manifest.json");
+ const env = { OPENAI_API_KEY: "provider-private", BRAINTRUST_API_KEY: "braintrust-private", BRAINTRUST_OTHER_KEY: "other-private",
+ GITHUB_SHA: "a".repeat(40), PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "11111111-1111-4111-8111-111111111111",
+ PAPERCLIP_E2E_OTLP_RELAY_URL: "https://unowned.invalid/otel" };
+ let operations: Awaited<ReturnType<typeof openEvalOperations>> | undefined;
+ try {
+ await writeFile(file, JSON.stringify(value));
+ operations = await openEvalOperations(file, [cell], 0, env);
+ const endpoint = await operations.start(cell, "with-observability", 1);
+ const worker: NodeJS.ProcessEnv = { ...env };
+ applyEvalGatewayEnvironment(worker, cell.profile.credential, endpoint);
+ expect(worker).toMatchObject({ OPENAI_API_KEY: endpoint.token, BRAINTRUST_API_KEY: endpoint.observability!.token,
+ PAPERCLIP_E2E_OTLP_RELAY_URL: endpoint.observability!.baseUrl, GITHUB_SHA: env.GITHUB_SHA });
+ expect(JSON.stringify(worker)).not.toMatch(/provider-private|braintrust-private|other-private/);
+ expect(buildPaperclipServerEnvironment(worker).BRAINTRUST_API_KEY).toBeUndefined();
+ expect((await fetch(endpoint.observability!.baseUrl + "/v1/traces", { method: "POST" })).status).toBe(401);
+ await operations.close(); operations = undefined;
+ await expect(fetch(endpoint.baseUrl)).rejects.toThrow();
+ await expect(fetch(endpoint.observability!.baseUrl)).rejects.toThrow();
+ const receipts = (await readFile(endpoint.observability!.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ expect(receipts.map(row => row.type)).toEqual(["policy", "closed"]);
+ expect(receipts[0]).toMatchObject({ campaignId: "pilot", attemptId: "with-observability", projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID });
+ const withoutTracing: NodeJS.ProcessEnv = { ...env };
+ applyEvalGatewayEnvironment(withoutTracing, cell.profile.credential, { ...endpoint, observability: undefined });
+ expect(withoutTracing.BRAINTRUST_API_KEY).toBeUndefined();
+ expect(withoutTracing.PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined();
+ } finally { await operations?.close(); await rm(root, { recursive: true, force: true }); }
+});
+
+it("rejects mismatched trace projects before opening the journal and retains failed relay admission", async () => {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-ops-relay-failure-"));
+ const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl"), route: "braintrust" as const,
+ projectId: "11111111-1111-4111-8111-111111111111", maxAttemptsPerExecution: 1 }, file = path.join(root, "manifest.json");
+ const env = { BRAINTRUST_API_KEY: "braintrust-private", PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "22222222-2222-4222-8222-222222222222" };
+ let operations: Awaited<ReturnType<typeof openEvalOperations>> | undefined;
+ try {
+ await writeFile(file, JSON.stringify(value));
+ await expect(openEvalOperations(file, [cell], 0, env)).rejects.toThrow("projects differ");
+ await expect(access(value.ledgerPath)).rejects.toMatchObject({ code: "ENOENT" });
+ await mkdir(`${value.ledgerPath}.otlp`);
+ const receipt = path.join(`${value.ledgerPath}.otlp`, `${createHash("sha256").update("collision").digest("hex")}.jsonl`);
+ await writeFile(receipt, "retained-receipt");
+ operations = await openEvalOperations(file, [cell], 0, { ...env, PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: value.projectId });
+ await expect(operations.start(cell, "collision", 1)).rejects.toMatchObject({ code: "EEXIST" });
+ await expect(operations.start(cell, "replacement", 1)).rejects.toThrow("attempt cap");
+ await operations.close(); operations = undefined;
+ expect(await readFile(receipt, "utf8")).toBe("retained-receipt");
+ expect(JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8")).admittedAttempts)
+ .toEqual([{ executionId: cell.id, attemptId: "collision" }]);
+ } finally { await operations?.close(); await rm(root, { recursive: true, force: true }); }
+});
+it.each([
+ cell.id,
+ "feedback-persistence.runner-codex.local.complaint-and-suggestion",
+ "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion",
+])("routes %s through a shared connection without ambient provider bindings", async executionId => {
+ const selected = runnerMatrix.find(execution => execution.id === executionId)!;
+ const anthropic = selected.profile.credential === "ANTHROPIC_API_KEY";
+ const admitted = manifest(); admitted.executionIds = [selected.id];
+ admitted.models[0]!.provider = anthropic ? "anthropic" : "openai";
+ admitted.models[0]!.price.provider = admitted.models[0]!.provider;
+ admitted.models[0]!.price.model = selected.profile.model;
+ if (anthropic) Object.assign(admitted.models[0]!.price, { cacheWrite5mUsdPerMillion: "2", cacheWrite1hUsdPerMillion: "2" });
+ expect(validateEvalOperationsManifest(admitted, [selected], 0)).toBe(admitted);
+ vi.stubEnv("PAPERCLIP_E2E_GATEWAY_BASE_URL", "http://127.0.0.1:43123");
+ vi.stubEnv("PAPERCLIP_E2E_GATEWAY_TOKEN", "ephemeral-fixture-token");
+ const id = "11111111-1111-4111-8111-111111111111", grantId = "22222222-2222-4222-8222-222222222222";
+ let connection: unknown, agent: Record<string, any> | undefined, company: Record<string, unknown> | undefined;
+ const api = {
+ async get() { return [{ id, driver: "local" }]; },
+ async postSensitive(url: string, body: unknown) { if (url.endsWith("/ai-connections")) { connection = body; return { connectionId: id, grantId }; } return { id }; },
+ async post(url: string, body: Record<string, unknown>) { if (url.endsWith("/agents")) agent = body; if (url === "/api/companies") company = body; return { id, name: "fixture" }; },
+ } as unknown as RunnerApi;
+ const fixtures = await setupLiveFixtures({ api, execution: selected, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { [selected.profile.credential]: "ephemeral-fixture-token" } });
+ await fixtures.teardown();
+ expect(createAiConnectionSchema.parse(connection)).toMatchObject({ ownership: "shared", apiKey: "ephemeral-fixture-token", routing: { kind: "gateway", protocol: anthropic ? "messages" : "responses", baseUrl: `http://127.0.0.1:43123${anthropic ? "" : "/v1"}`, auth: "bearer" } });
+ expect(aiConnectionBindingSchema.parse(agent!.runtimeConfig.aiConnection)).toEqual({ provider: anthropic ? "anthropic" : "openai", method: "api_key", mode: "shared", connectionId: id, grantId });
+ expect(agent!.adapterConfig.env).toBeUndefined();
+ if (selected.suite.id === "feedback-persistence") {
+ expect(company!.budgetMonthlyCents).toBe(1_000);
+ expect(agent!.budgetMonthlyCents).toBe(1_000);
+ }
+});
diff --git a/tests/runner-e2e/operations.ts b/tests/runner-e2e/operations.ts
new file mode 100644
index 0000000000..2ab81eb267
--- /dev/null
+++ b/tests/runner-e2e/operations.ts
@@ -0,0 +1,123 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js";
+import { EvalSpendState, openEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js";
+import { evalRequestReservation, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js";
+import type { MatrixExecution } from "./types.js";
+import { evalBraintrustConfig } from "./braintrust.js";
+import { startEvalOtlpRelay } from "./eval-otlp-relay.js";
+import { usdToUnits } from "../../packages/shared/src/money.js";
+
+export interface EvalOperationsManifest {
+ version: 1;
+ limits: EvalSpendLimits;
+ ledgerPath: string;
+ executionIds: string[];
+ maxAttemptsPerExecution: number;
+ route: "direct" | "braintrust";
+ projectId?: string;
+ models: Array<Omit<EvalGatewayRoute, "apiKey" | "projectId" | "endpointName" | "mode">>;
+}
+const fields = (value: object, names: string[]) => {
+ if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(name => !names.includes(name))) throw new Error("Invalid eval operations manifest fields");
+};
+export function validateEvalOperationsManifest(value: EvalOperationsManifest, executions: readonly MatrixExecution[], maxRetries: number) {
+ fields(value, ["version", "limits", "ledgerPath", "executionIds", "maxAttemptsPerExecution", "route", "projectId", "models"]);
+ const initial = new EvalSpendState(value.limits).snapshot();
+ const initialModelAllowance = usdToUnits(value.limits.maxUsd) - usdToUnits(initial.committedUsd);
+ if (value.version !== 1 || !["direct", "braintrust"].includes(value.route) || !path.isAbsolute(value.ledgerPath)
+ || !Array.isArray(value.executionIds) || new Set(value.executionIds).size !== value.executionIds.length
+ || JSON.stringify([...value.executionIds].sort()) !== JSON.stringify(executions.map(e => e.id).sort())
+ || !Number.isSafeInteger(value.maxAttemptsPerExecution) || value.maxAttemptsPerExecution < 1
+ || value.limits.maxAttemptsPerExecution !== undefined && value.limits.maxAttemptsPerExecution !== value.maxAttemptsPerExecution
+ || maxRetries + 1 > value.maxAttemptsPerExecution || !Array.isArray(value.models) || value.models.length === 0) throw new Error("Eval operations selection or limits mismatch");
+ if (value.route === "braintrust" && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value.projectId ?? "")) throw new Error("Braintrust gateway requires an explicit project UUID");
+ const models = new Map<string, bigint>();
+ for (const model of value.models) {
+ fields(model, ["provider", "price", "maximumInputTokens", "maximumOutputTokens", "timeoutMs"]);
+ const rateFields = ["inputUsdPerMillion", "cachedReadUsdPerMillion", "outputUsdPerMillion", "cacheWrite5mUsdPerMillion", "cacheWrite1hUsdPerMillion", "cacheWrite30mUsdPerMillion", "cacheWrite12hUsdPerMillion"];
+ fields(model.price, ["provider", "model", "version", "sourceUrl", "longContext", ...rateFields]);
+ if (model.price.longContext) { fields(model.price.longContext, ["aboveInputTokens", "rates"]); fields(model.price.longContext.rates, rateFields); }
+ const url = new URL(model.price.sourceUrl);
+ if (!["openai", "anthropic"].includes(model.provider) || url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("Invalid model price provenance");
+ const key = `${model.provider}:${model.price.model}`;
+ if (models.has(key)) throw new Error("Duplicate eval model price");
+ models.set(key, usdToUnits(evalRequestReservation({ ...model, mode: value.route, apiKey: "validation" })));
+ }
+ for (const execution of executions) {
+ if (execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id)
+ || !["runner-codex", "runner-acpx-claude", "legacy-codex", "legacy-claude"].includes(execution.profile.id)
+ || execution.requiredCredentials.length !== 1) throw new Error(`Unqualified budgeted eval route: ${execution.id}`);
+ const provider = execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai";
+ if (!models.has(`${provider}:${execution.profile.model}`)) throw new Error(`Missing exact model pricing: ${execution.id}`);
+ if (models.get(`${provider}:${execution.profile.model}`)! > initialModelAllowance) {
+ throw new Error(`Insufficient initial model allowance for selected eval: ${execution.id}`);
+ }
+ }
+ return value;
+}
+
+export async function openEvalOperations(file: string, executions: readonly MatrixExecution[], maxRetries: number, env: NodeJS.ProcessEnv) {
+ const manifest = validateEvalOperationsManifest(JSON.parse(await readFile(file, "utf8")), executions, maxRetries);
+ if (Date.now() >= Date.parse(manifest.limits.deadlineAt)) throw new Error("Eval campaign deadline reached");
+ if (manifest.limits.overhead && (Date.now() < Date.parse(manifest.limits.overhead.verifiedAt)
+ || Date.parse(manifest.limits.deadlineAt) >= Date.parse(manifest.limits.overhead.expiresAt))) {
+ throw new Error("Operating overhead allowance does not cover the campaign window");
+ }
+ const required = manifest.route === "braintrust" ? ["BRAINTRUST_API_KEY"] : executions.map(execution => execution.profile.credential);
+ if (required.some(name => !env[name]?.trim())) throw new Error("Missing eval gateway upstream credential");
+ const observability = evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined });
+ if (observability && manifest.route === "braintrust" && observability.projectId !== manifest.projectId) throw new Error("Eval gateway and OTLP projects differ");
+ const receiptDirectory = `${manifest.ledgerPath}.requests`;
+ await mkdir(receiptDirectory, { recursive: true, mode: 0o700 });
+ const phaseDirectory = `${manifest.ledgerPath}.attempts`;
+ await mkdir(phaseDirectory, { recursive: true, mode: 0o700 });
+ const ledger = await openEvalCostLedger(manifest.ledgerPath, { ...manifest.limits, maxAttemptsPerExecution: manifest.maxAttemptsPerExecution });
+ const active = new Set<{ close(): Promise<void> }>();
+ const atomicJson = async (file: string, value: unknown) => {
+ await writeFile(`${file}.tmp`, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
+ await rename(`${file}.tmp`, file);
+ };
+ return {
+ manifest,
+ async start(execution: MatrixExecution, attemptId: string, attempt: number) {
+ if (!Number.isSafeInteger(attempt) || attempt < 1 || attempt > manifest.maxAttemptsPerExecution || Date.now() >= Date.parse(manifest.limits.deadlineAt)) throw new Error("Eval operations attempt/deadline limit reached");
+ if (!manifest.executionIds.includes(execution.id)) throw new Error("Eval operation is outside the admitted selection");
+ const provider = execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai";
+ const model = manifest.models.find(m => m.provider === provider && m.price.model === execution.profile.model)!;
+ const route: EvalGatewayRoute = { ...model, mode: manifest.route, projectId: manifest.projectId,
+ endpointName: provider === "openai" ? "OPENAI_API_KEY" : "ANTHROPIC_API_KEY",
+ apiKey: env[manifest.route === "braintrust" ? "BRAINTRUST_API_KEY" : execution.profile.credential] ?? "" };
+ if (!route.apiKey) throw new Error("Missing eval gateway upstream credential");
+ await ledger.admitAttempt(execution.id, attemptId);
+ const identity = { executionId: execution.id, attemptId, phase: "setup" as const };
+ const phasePath = path.join(phaseDirectory, `${createHash("sha256").update(attemptId).digest("hex")}.json`);
+ await createEvalAttemptPhase(phasePath, { executionId: execution.id, attemptId });
+ const gateway = await startEvalGateway({ ledger, route,
+ identity, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity), deadlineAt: Date.parse(manifest.limits.deadlineAt),
+ persist: (receipt: EvalGatewayReceipt) => atomicJson(path.join(receiptDirectory, `${receipt.requestId}.json`), receipt),
+ });
+ let relay: Awaited<ReturnType<typeof startEvalOtlpRelay>> | undefined;
+ try {
+ if (observability) relay = await startEvalOtlpRelay({ projectId: observability.projectId, apiKey: observability.apiKey,
+ campaignId: manifest.limits.campaignId, attemptId, deadlineAt: Math.min(Date.parse(manifest.limits.deadlineAt), Date.now() + 4 * 3_600_000),
+ receiptPath: path.join(`${manifest.ledgerPath}.otlp`, `${createHash("sha256").update(attemptId).digest("hex")}.jsonl`) });
+ } catch (error) { await gateway.close(); throw error; }
+ let closing: Promise<void> | undefined;
+ const endpoint = { ...gateway, phasePath, observability: relay, close: () => closing ??= (async () => {
+ const results = await Promise.allSettled([gateway.close(), relay?.close()]);
+ active.delete(endpoint);
+ if (results.some(result => result.status === "rejected")) throw new Error("Eval gateway or OTLP relay cleanup failed");
+ })() };
+ active.add(endpoint);
+ return endpoint;
+ },
+ async close() {
+ const results = await Promise.allSettled([...active].map(gateway => gateway.close())); active.clear();
+ try { await atomicJson(`${manifest.ledgerPath}.summary.json`, ledger.snapshot()); } finally { await ledger.close(); }
+ if (results.some(result => result.status === "rejected")) throw new Error("Eval gateway cleanup failed");
+ },
+ };
+}
+export type EvalOperations = Awaited<ReturnType<typeof openEvalOperations>>;
diff --git a/tests/runner-e2e/paid-runner-routing.test.mjs b/tests/runner-e2e/paid-runner-routing.test.mjs
new file mode 100644
index 0000000000..669ade6481
--- /dev/null
+++ b/tests/runner-e2e/paid-runner-routing.test.mjs
@@ -0,0 +1,51 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { expect, it } from "vitest";
+
+const workflow = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8");
+const step = workflow.split(" - name: Select paid test runner\n")[1]?.split("\n target_lock:")[0];
+if (!step?.includes(" run: |\n")) throw new Error("Missing actual workflow runner selection");
+const script = step.split(" run: |\n")[1].split("\n").map(line => line.replace(/^ /, "")).join("\n");
+
+async function select(overrides) {
+ const root = await mkdtemp(path.join(tmpdir(), "eval-runner-routing-"));
+ try {
+ const output = path.join(root, "outputs");
+ const result = spawnSync("bash", ["-c", script], { encoding: "utf8", timeout: 5000, env: {
+ PATH: process.env.PATH, GITHUB_OUTPUT: output, AWS_PAID_RUNNER_ENABLED: "true",
+ EVENT_NAME: "workflow_dispatch", EVAL_CADENCE: "manual", REPOSITORY_PRIVATE: "false", ...overrides,
+ } });
+ const saved = await readFile(output, "utf8").catch(error => { if (error.code === "ENOENT") return ""; throw error; });
+ return { ...result, outputs: Object.fromEntries(saved.trim().split("\n").filter(Boolean).map(line => {
+ const separator = line.indexOf("="); return [line.slice(0, separator), line.slice(separator + 1)];
+ })) };
+ } finally { await rm(root, { recursive: true, force: true }); }
+}
+
+it.each([
+ { EVENT_NAME: "schedule" },
+ { EVAL_CADENCE: "daily" },
+ { EVAL_CADENCE: "weekly" },
+])("routes bounded $EVENT_NAME/$EVAL_CADENCE campaigns to standard hosted compute despite the AWS switch", async trigger => {
+ const result = await select(trigger);
+ expect(result.status).toBe(0);
+ expect(result.outputs).toMatchObject({ runner: "ubuntu-latest", playwright_channel: "" });
+});
+
+it.each(["true", "", "unknown"])("refuses bounded campaigns when repository privacy is %s", async REPOSITORY_PRIVATE => {
+ const result = await select({ EVAL_CADENCE: "daily", REPOSITORY_PRIVATE });
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toContain("require a public repository");
+ expect(result.outputs).toEqual({});
+});
+
+it("preserves the explicit manual AWS and GitHub options", async () => {
+ const aws = await select({ REPOSITORY_PRIVATE: "true" });
+ expect(aws.status).toBe(0);
+ expect(aws.outputs).toMatchObject({ runner: "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", playwright_channel: "chrome" });
+ const hosted = await select({ AWS_PAID_RUNNER_ENABLED: "false" });
+ expect(hosted.status).toBe(0);
+ expect(hosted.outputs).toMatchObject({ runner: "ubuntu-latest", playwright_channel: "" });
+});
diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts
index 599d8c6fc2..bae4834456 100644
--- a/tests/runner-e2e/pi-controls-evidence.test.ts
+++ b/tests/runner-e2e/pi-controls-evidence.test.ts
@@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => {
expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true);
expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]);
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false);
- expect(validateRunnerCatalog()).toHaveLength(761);
+ expect(validateRunnerCatalog()).toHaveLength(763);
for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow();
});
diff --git a/tests/runner-e2e/private-history-archive.py b/tests/runner-e2e/private-history-archive.py
new file mode 100644
index 0000000000..02c3271234
--- /dev/null
+++ b/tests/runner-e2e/private-history-archive.py
@@ -0,0 +1,148 @@
+import json
+import os
+from pathlib import Path, PurePosixPath
+import shutil
+import stat
+import struct
+import sys
+import tempfile
+import zipfile
+
+
+MAX_BYTES = 256 * 1024 * 1024
+MAX_ARCHIVE_BYTES = MAX_BYTES + 8 * 1024 * 1024
+MAX_FILES = 4097
+
+
+def safe_path(name):
+ parts = name.split("/")
+ if (not name or len(name) > 512 or "\\" in name or ":" in name
+ or any(ord(c) < 32 or ord(c) == 127 for c in name)
+ or any(p in ("", ".", "..") for p in parts)):
+ raise ValueError("Unsafe private archive path")
+ return PurePosixPath(name)
+
+
+def pack(source, destination):
+ source, destination = Path(source), Path(destination)
+ if source.is_symlink() or not source.is_dir():
+ raise ValueError("Private archive source must be a directory")
+ root = source.resolve()
+ if destination.resolve().is_relative_to(root):
+ raise ValueError("Private archive must be outside its source")
+ files = []
+ names_seen = set()
+ total = 0
+ visited = 0
+ for directory, directories, names in os.walk(source, followlinks=False):
+ visited += len(directories) + len(names)
+ if visited > MAX_FILES:
+ raise ValueError("Private archive exceeds entry allowance")
+ for name in directories:
+ entry = Path(directory) / name
+ safe_path(entry.relative_to(source).as_posix())
+ if entry.is_symlink():
+ raise ValueError("Private archive cannot contain symlinks")
+ for name in names:
+ entry = Path(directory) / name
+ relative = entry.relative_to(source).as_posix()
+ safe_path(relative)
+ if relative.casefold() in names_seen:
+ raise ValueError("Private archive has colliding paths")
+ names_seen.add(relative.casefold())
+ metadata = entry.lstat()
+ if not stat.S_ISREG(metadata.st_mode) or not entry.resolve().is_relative_to(root):
+ raise ValueError("Private archive requires regular files")
+ total += metadata.st_size
+ if total > MAX_BYTES:
+ raise ValueError("Private archive exceeds byte allowance")
+ files.append((entry, relative, metadata))
+ if not files:
+ raise ValueError("Private archive is empty")
+ output = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ try:
+ with os.fdopen(output, "wb") as stream, zipfile.ZipFile(stream, "w", zipfile.ZIP_STORED, allowZip64=False) as archive:
+ for entry, relative, before in sorted(files):
+ with os.fdopen(os.open(entry, os.O_RDONLY | os.O_NOFOLLOW), "rb") as data:
+ observed = os.fstat(data.fileno())
+ if (observed.st_ino, observed.st_size, observed.st_mtime_ns) != (before.st_ino, before.st_size, before.st_mtime_ns):
+ raise ValueError("Private archive source changed")
+ info = zipfile.ZipInfo(relative)
+ info.external_attr = (stat.S_IFREG | 0o600) << 16
+ copied = 0
+ with archive.open(info, "w") as target:
+ while chunk := data.read(64 * 1024):
+ copied += len(chunk)
+ if copied > before.st_size:
+ raise ValueError("Private archive source grew")
+ target.write(chunk)
+ after = os.fstat(data.fileno())
+ if copied != before.st_size or (after.st_size, after.st_mtime_ns, after.st_ctime_ns) != (before.st_size, before.st_mtime_ns, before.st_ctime_ns):
+ raise ValueError("Private archive source changed")
+ if destination.stat().st_size > MAX_ARCHIVE_BYTES:
+ raise ValueError("Private archive exceeds transfer allowance")
+ except BaseException:
+ destination.unlink(missing_ok=True)
+ raise
+ return {"files": len(files), "sourceBytes": total, "archiveBytes": destination.stat().st_size}
+
+
+def unpack(source, destination):
+ source, destination = Path(source), Path(destination)
+ if source.is_symlink() or not source.is_file() or source.stat().st_size > MAX_ARCHIVE_BYTES:
+ raise ValueError("Invalid private archive file")
+ if destination.exists() or destination.is_symlink():
+ raise ValueError("Private archive destination already exists")
+ with source.open("rb") as data:
+ if source.stat().st_size < 22:
+ raise ValueError("Private archive is truncated")
+ data.seek(-22, os.SEEK_END)
+ signature, disk, start_disk, disk_entries, entries, central_bytes, central_offset, comment_bytes = struct.unpack("<4s4H2LH", data.read(22))
+ if (signature != b"PK\x05\x06" or disk or start_disk or comment_bytes
+ or entries != disk_entries or not 0 < entries <= MAX_FILES
+ or central_bytes > 8 * 1024 * 1024
+ or central_offset + central_bytes != source.stat().st_size - 22):
+ raise ValueError("Private archive directory exceeds its boundary")
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ temporary = Path(tempfile.mkdtemp(prefix=".private-eval-", dir=destination.parent))
+ try:
+ with zipfile.ZipFile(source) as archive:
+ entries = archive.infolist()
+ if not entries or len(entries) != disk_entries or len(entries) > MAX_FILES:
+ raise ValueError("Private archive exceeds entry allowance")
+ paths = set()
+ total = 0
+ for entry in entries:
+ relative = safe_path(entry.filename)
+ mode = entry.external_attr >> 16
+ if (entry.compress_type != zipfile.ZIP_STORED or entry.flag_bits & 1
+ or stat.S_IFMT(mode) not in (0, stat.S_IFREG)
+ or str(relative).casefold() in paths):
+ raise ValueError("Private archive has unsupported or duplicate entries")
+ paths.add(str(relative).casefold())
+ total += entry.file_size
+ if total > MAX_BYTES:
+ raise ValueError("Private archive exceeds byte allowance")
+ for entry in entries:
+ target = temporary.joinpath(*safe_path(entry.filename).parts)
+ target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+ copied = 0
+ with archive.open(entry) as data, os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as output:
+ while chunk := data.read(64 * 1024):
+ copied += len(chunk)
+ if copied > entry.file_size:
+ raise ValueError("Private archive entry exceeded its declared size")
+ output.write(chunk)
+ if copied != entry.file_size:
+ raise ValueError("Private archive entry was truncated")
+ os.rename(temporary, destination)
+ except BaseException:
+ shutil.rmtree(temporary)
+ raise
+ return {"files": len(entries), "sourceBytes": total}
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 4 or sys.argv[1] not in ("pack", "unpack"):
+ raise SystemExit("Usage: private-history-archive.py pack|unpack SOURCE DESTINATION")
+ print(json.dumps((pack if sys.argv[1] == "pack" else unpack)(sys.argv[2], sys.argv[3])))
diff --git a/tests/runner-e2e/private-history.test.ts b/tests/runner-e2e/private-history.test.ts
new file mode 100644
index 0000000000..9af459a01b
--- /dev/null
+++ b/tests/runner-e2e/private-history.test.ts
@@ -0,0 +1,162 @@
+import { execFile } from "node:child_process";
+import { copyFile, lstat, mkdir, mkdtemp, readFile, rm, symlink, truncate, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { afterEach, expect, it, vi } from "vitest";
+import { BoundedHistoryPublisher } from "./history-budget.js";
+import { archivePrivateEvidence, PRIVATE_HISTORY_PREFIX, privateArtifactKey, retainPrivateTransfer, transferPrivateArtifact, verifyPrivateHistory } from "./private-history.js";
+
+const execute = promisify(execFile);
+const roots: string[] = [];
+async function temporary() {
+ const root = await mkdtemp(path.join(os.tmpdir(), "private-history-test-")); roots.push(root); return root;
+}
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+
+function configurations() {
+ return {
+ "get-bucket-versioning": { Status: "Enabled" },
+ "get-bucket-lifecycle-configuration": { Rules: [
+ { Status: "Enabled", Filter: { Prefix: `${PRIVATE_HISTORY_PREFIX}/` }, Expiration: { Days: 30 },
+ NoncurrentVersionExpiration: { NoncurrentDays: 1 }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } },
+ { Status: "Enabled", Filter: { Prefix: `${PRIVATE_HISTORY_PREFIX}/` }, Expiration: { ExpiredObjectDeleteMarker: true } },
+ ] },
+ "get-public-access-block": { PublicAccessBlockConfiguration: { BlockPublicAcls: true, IgnorePublicAcls: true, BlockPublicPolicy: true, RestrictPublicBuckets: true } },
+ "get-bucket-ownership-controls": { OwnershipControls: { Rules: [{ ObjectOwnership: "BucketOwnerEnforced" }] } },
+ "get-bucket-policy": { Policy: JSON.stringify({ Statement: [{ Effect: "Deny" }] }) },
+ };
+}
+function configurationRunner(config: Record<string, unknown>) {
+ return async (args: string[]) => {
+ if (args[1] === "get-object-lock-configuration" || args[1] === "get-bucket-replication") {
+ throw Object.assign(new Error("Not configured"), { stderr: `(${args[1] === "get-object-lock-configuration" ? "ObjectLockConfigurationNotFoundError" : "ReplicationConfigurationNotFoundError"})` });
+ }
+ if (!(args[1]! in config)) throw new Error(`Unexpected AWS operation ${args[1]}`);
+ return { stdout: JSON.stringify(config[args[1]!]) };
+ };
+}
+
+it("round trips binary evidence and a ledger without exposing unrelated files", async () => {
+ const root = await temporary(), source = path.join(root, "source"), archive = path.join(root, "evidence.zip"), destination = path.join(root, "restored");
+ await mkdir(path.join(source, "scheduled"), { recursive: true });
+ await writeFile(path.join(source, "scheduled/ledger.jsonl"), '{"cost":"unknown"}\n');
+ await writeFile(path.join(source, "binary.png"), Buffer.from([0, 255, 17, 0]));
+ await writeFile(path.join(root, "outside"), "private outside material");
+ expect(await archivePrivateEvidence("pack", source, archive)).toMatchObject({ files: 2 });
+ expect(await archivePrivateEvidence("unpack", archive, destination)).toMatchObject({ files: 2 });
+ expect(await readFile(path.join(destination, "binary.png"))).toEqual(Buffer.from([0, 255, 17, 0]));
+ expect(await readFile(path.join(destination, "scheduled/ledger.jsonl"), "utf8")).toBe('{"cost":"unknown"}\n');
+ expect((await lstat(path.join(destination, "binary.png"))).mode & 0o777).toBe(0o600);
+ await expect(lstat(path.join(destination, "outside"))).rejects.toThrow();
+ await expect(archivePrivateEvidence("unpack", archive, destination)).rejects.toThrow("already exists");
+});
+
+it("rejects symlink sources and oversized source files before producing an archive", async () => {
+ const root = await temporary(), source = path.join(root, "source"), archive = path.join(root, "evidence.zip");
+ await mkdir(source); await writeFile(path.join(root, "outside"), "outside");
+ await symlink(path.join(root, "outside"), path.join(source, "link"));
+ await expect(archivePrivateEvidence("pack", source, archive)).rejects.toThrow("regular files");
+ await rm(path.join(source, "link")); await writeFile(path.join(source, "huge"), "");
+ await truncate(path.join(source, "huge"), 256 * 1024 * 1024 + 1);
+ await expect(archivePrivateEvidence("pack", source, archive)).rejects.toThrow("byte allowance");
+ await expect(lstat(archive)).rejects.toThrow();
+});
+
+it.each(["traversal", "absolute", "symlink", "compressed", "duplicate", "case_collision", "comment", "many_entries", "truncated"])("rejects %s archives without retaining a partial extraction", async kind => {
+ const root = await temporary(), archive = path.join(root, "bad.zip"), destination = path.join(root, "restored");
+ await execute("python3", ["-c", `
+import sys,zipfile,stat
+p,kind=sys.argv[1:]
+with zipfile.ZipFile(p,'w') as z:
+ name={'traversal':'../outside','absolute':'/tmp/outside'}.get(kind,'safe.txt')
+ info=zipfile.ZipInfo(name)
+ if kind=='symlink': info.external_attr=(stat.S_IFLNK|0o777)<<16
+ if kind=='compressed': info.compress_type=zipfile.ZIP_DEFLATED
+ z.writestr(info,'data')
+ if kind=='duplicate': z.writestr(name,'other')
+ if kind=='case_collision': z.writestr('SAFE.txt','other')
+ if kind=='comment': z.comment=b'extra'
+ if kind=='many_entries':
+ for n in range(4097): z.writestr(str(n),'')
+if kind=='truncated':
+ with open(p,'r+b') as f: f.truncate(12)
+`, archive, kind]);
+ await expect(archivePrivateEvidence("unpack", archive, destination)).rejects.toThrow();
+ await expect(lstat(destination)).rejects.toThrow();
+ await expect(lstat(path.join(root, "outside"))).rejects.toThrow();
+});
+
+it("fails closed on public access, ACLs, a reader grant, or unavailable inspection", async () => {
+ const root = await temporary();
+ await verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, "allowed.json"), configurationRunner(configurations())), "private-bucket");
+ for (const operation of ["get-public-access-block", "get-bucket-ownership-controls", "get-bucket-policy"]) {
+ const config: Record<string, unknown> = configurations();
+ config[operation] = operation === "get-bucket-policy" ? { Policy: '{"Statement":[{"Effect":"Allow"}]}' } : {};
+ const runner = vi.fn(configurationRunner(config));
+ await expect(verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, `${operation}.json`), runner), "private-bucket")).rejects.toThrow();
+ expect(runner.mock.calls.some(([args]) => args[1] === "put-object")).toBe(false);
+ }
+ await expect(verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, "denied.json"), async () => { throw new Error("AccessDenied"); }), "private-bucket")).rejects.toThrow("AccessDenied");
+});
+
+it("uses immutable private keys and restores the uploaded archive through the actual transfer path", async () => {
+ const root = await temporary(), source = path.join(root, "source"), stored = path.join(root, "stored.zip"), destination = path.join(root, "restored");
+ await mkdir(source); await writeFile(path.join(source, "ledger.jsonl"), "retained attempt\n");
+ const config = configurationRunner(configurations());
+ const runner = vi.fn(async (args: string[]) => {
+ if (args[1] === "put-object") {
+ expect(args).toEqual(expect.arrayContaining(["--if-none-match", "*", "--cache-control", "private, no-store"]));
+ expect(args[args.indexOf("--key") + 1]).toBe(privateArtifactKey("evidence", "12345"));
+ await copyFile(args[args.indexOf("--body") + 1]!, stored);
+ return { stdout: '{"VersionId":"retained-version","ETag":"tag"}' };
+ }
+ if (args[1] === "get-object") {
+ expect(args).toContain(`bytes=0-${264 * 1024 * 1024}`);
+ await copyFile(stored, args[args.indexOf("--cli-connect-timeout") - 1]!);
+ return { stdout: '{"VersionId":"retained-version"}' };
+ }
+ return config(args);
+ });
+ const upload = new BoundedHistoryPublisher(path.join(root, "upload.json"), runner);
+ expect(await transferPrivateArtifact({ operation: "upload", kind: "evidence", runId: "12345", directory: source, bucket: "private-bucket", publisher: upload }))
+ .toMatchObject({ private: true, versionId: "retained-version", files: 1, sha256: expect.stringMatching(/^[0-9a-f]{64}$/) });
+ await transferPrivateArtifact({ operation: "download", kind: "evidence", runId: "12345", directory: destination, bucket: "private-bucket", publisher: new BoundedHistoryPublisher(path.join(root, "download.json"), runner) });
+ expect(await readFile(path.join(destination, "ledger.jsonl"), "utf8")).toBe("retained attempt\n");
+ expect(runner).toHaveBeenCalledTimes(16);
+});
+
+it("rejects unsafe run identities and other artifact kinds", () => {
+ for (const id of ["../123", "0", "1/2", "123-2"]) expect(() => privateArtifactKey("report", id)).toThrow();
+ expect(() => privateArtifactKey("public", "123")).toThrow();
+});
+
+it.each([false, true])("retains an unknown transfer without retrying it when receipt upload fails=%s", async receiptFails => {
+ const root = await temporary(), source = path.join(root, "source"), receipt = path.join(root, "transfer.json");
+ await mkdir(source); await writeFile(path.join(source, "ledger.jsonl"), "original evidence\n");
+ const config = configurationRunner(configurations());
+ const runner = vi.fn(async (args: string[]) => {
+ if (args[1] !== "put-object") return config(args);
+ await copyFile(args[args.indexOf("--body") + 1]!, path.join(root, "accepted.zip"));
+ throw new Error("Response lost after server accepted archive");
+ });
+ const retain = vi.fn(async (args: string[]) => {
+ expect(args[args.indexOf("--key") + 1]).toBe(`${privateArtifactKey("evidence", "12345")}.upload-receipt.json`);
+ const snapshot = JSON.parse(await readFile(args[args.indexOf("--body") + 1]!, "utf8"));
+ expect(snapshot.transfer.status).toBe("failed_or_unknown");
+ expect(snapshot.requests.at(-1)).toMatchObject({ operation: "put-object", status: "failed_or_unknown" });
+ expect(snapshot.billedUsd).toBeNull();
+ expect(snapshot.receiptUpload.status).toBe("pending_at_snapshot");
+ if (receiptFails) throw new Error("Receipt response also lost");
+ return { stdout: '{"VersionId":"receipt-version"}' };
+ });
+ await expect(retainPrivateTransfer({ operation: "upload", kind: "evidence", runId: "12345", directory: source,
+ bucket: "private-bucket", publisher: new BoundedHistoryPublisher(receipt, runner) }, receipt,
+ () => new BoundedHistoryPublisher(`${receipt}.retention.json`, retain)))
+ .rejects.toThrow(receiptFails ? "receipt retention failed" : "Response lost");
+ expect(runner.mock.calls.filter(([args]) => args[1] === "put-object")).toHaveLength(1);
+ expect(retain).toHaveBeenCalledTimes(1);
+ expect((await lstat(`${receipt}.transfer.json`)).mode & 0o777).toBe(0o600);
+ expect(JSON.parse(await readFile(`${receipt}.retention.json`, "utf8")).requests[0].status)
+ .toBe(receiptFails ? "failed_or_unknown" : "accepted");
+});
diff --git a/tests/runner-e2e/private-history.ts b/tests/runner-e2e/private-history.ts
new file mode 100644
index 0000000000..85224bf997
--- /dev/null
+++ b/tests/runner-e2e/private-history.ts
@@ -0,0 +1,142 @@
+import { execFile } from "node:child_process";
+import { createReadStream } from "node:fs";
+import { createHash } from "node:crypto";
+import { lstat, mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { pathToFileURL } from "node:url";
+import { BoundedHistoryPublisher } from "./history-budget.js";
+
+const execFileAsync = promisify(execFile);
+export const PRIVATE_HISTORY_PREFIX = "runner-e2e-private/bounded-v1";
+export const PRIVATE_ARCHIVE_BYTES = 264 * 1024 * 1024;
+
+export function privateArtifactKey(kind: string, runId: string) {
+ if (!["evidence", "report"].includes(kind) || !/^[1-9][0-9]{0,19}$/.test(runId)) {
+ throw new Error("Private history needs an evidence/report kind and numeric workflow run ID");
+ }
+ return `${PRIVATE_HISTORY_PREFIX}/campaigns/gha-${runId}-1/${kind}.zip`;
+}
+
+export async function archivePrivateEvidence(operation: "pack" | "unpack", source: string, destination: string) {
+ const result = await execFileAsync("python3", [path.join(import.meta.dirname, "private-history-archive.py"), operation, source, destination], {
+ timeout: 60_000, maxBuffer: 64 * 1024,
+ });
+ return JSON.parse(result.stdout) as { files: number; sourceBytes: number; archiveBytes?: number };
+}
+
+export async function verifyPrivateHistory(publisher: BoundedHistoryPublisher, bucket: string) {
+ if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) throw new Error("Invalid private history bucket");
+ await publisher.verifyRetention(bucket, PRIVATE_HISTORY_PREFIX);
+ const configuration = async (operation: string) => JSON.parse((await publisher.run(["s3api", operation, "--bucket", bucket])).stdout);
+ const block = (await configuration("get-public-access-block")).PublicAccessBlockConfiguration;
+ if (!["BlockPublicAcls", "IgnorePublicAcls", "BlockPublicPolicy", "RestrictPublicBuckets"].every(key => block?.[key] === true)) {
+ throw new Error("Private history requires all bucket public-access blocks");
+ }
+ const ownership = (await configuration("get-bucket-ownership-controls")).OwnershipControls?.Rules;
+ if (!Array.isArray(ownership) || ownership.length !== 1 || ownership[0]?.ObjectOwnership !== "BucketOwnerEnforced") {
+ throw new Error("Private history requires disabled object ACLs");
+ }
+ const policy = JSON.parse((await configuration("get-bucket-policy")).Policy);
+ if (!Array.isArray(policy.Statement) || policy.Statement.some((statement: { Effect?: string }) => statement.Effect !== "Deny")) {
+ throw new Error("Private history bucket policy must not grant access to external readers or a CDN");
+ }
+}
+
+type TransferOptions = {
+ operation: "upload" | "download";
+ kind: "evidence" | "report";
+ runId: string;
+ directory: string;
+ bucket: string;
+ publisher: BoundedHistoryPublisher;
+};
+
+export async function transferPrivateArtifact(options: TransferOptions) {
+ privateArtifactKey(options.kind, options.runId);
+ await verifyPrivateHistory(options.publisher, options.bucket);
+ return transferVerifiedArtifact(options);
+}
+
+async function transferVerifiedArtifact(options: TransferOptions) {
+ const key = privateArtifactKey(options.kind, options.runId);
+ const temporary = await mkdtemp(path.join(os.tmpdir(), "paperclip-private-history-"));
+ const archive = path.join(temporary, "archive.zip");
+ try {
+ if (options.operation === "upload") {
+ const metadata = await archivePrivateEvidence("pack", options.directory, archive);
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(archive)) hash.update(chunk);
+ const result = await options.publisher.upload(options.bucket, key, archive, "application/zip", "private, no-store", true);
+ const response = JSON.parse(result.stdout) as { VersionId?: string };
+ if (!response.VersionId) throw new Error("Private archive upload did not return a version identity");
+ return { private: true, ...metadata, uri: `s3://${options.bucket}/${key}`, versionId: response.VersionId, sha256: hash.digest("hex") };
+ }
+ await options.publisher.run(["s3api", "get-object", "--bucket", options.bucket, "--key", key,
+ "--range", `bytes=0-${PRIVATE_ARCHIVE_BYTES}`, archive]);
+ if ((await lstat(archive)).size > PRIVATE_ARCHIVE_BYTES) throw new Error("Private archive exceeds transfer allowance");
+ const metadata = await archivePrivateEvidence("unpack", archive, options.directory);
+ return { ...metadata, uri: `s3://${options.bucket}/${key}` };
+ } finally {
+ await rm(temporary, { recursive: true, force: true });
+ }
+}
+
+export async function retainPrivateTransfer(options: TransferOptions, receipt: string, receiptPublisher?: () => BoundedHistoryPublisher) {
+ const key = privateArtifactKey(options.kind, options.runId);
+ await verifyPrivateHistory(options.publisher, options.bucket);
+ let result: Awaited<ReturnType<typeof transferVerifiedArtifact>> | undefined;
+ let failure: unknown;
+ try {
+ result = await transferVerifiedArtifact(options);
+ } catch (error) {
+ failure = error;
+ }
+ if (receiptPublisher) {
+ await options.publisher.save();
+ const snapshot = JSON.stringify({ ...JSON.parse(await readFile(receipt, "utf8")),
+ transfer: result ? { status: "complete", ...result } : { status: "failed_or_unknown" },
+ receiptUpload: { status: "pending_at_snapshot", maxBytes: 1024 * 1024 } });
+ if (Buffer.byteLength(snapshot) > 1024 * 1024) throw new Error("Private transfer receipt exceeds allowance");
+ await writeFile(`${receipt}.transfer.json`, snapshot, { mode: 0o600, flag: "wx" });
+ try {
+ await receiptPublisher().upload(options.bucket, `${key}.${options.operation}-receipt.json`, `${receipt}.transfer.json`, "application/json", "private, no-store", true);
+ } catch (retentionError) {
+ throw new AggregateError(failure ? [failure, retentionError] : [retentionError], "Private transfer receipt retention failed; local receipts remain authoritative");
+ }
+ }
+ if (failure) throw failure;
+ return result!;
+}
+
+async function main() {
+ const [operation, kind, runId, directory] = process.argv.slice(2);
+ if (!["verify", "upload", "download"].includes(operation ?? "") || !kind || !runId || !directory) {
+ throw new Error("Usage: private-history.ts verify|upload|download evidence|report RUN_ID DIRECTORY");
+ }
+ privateArtifactKey(kind, runId);
+ if (process.env.GITHUB_ACTIONS === "true" && (process.env.GITHUB_RUN_ATTEMPT !== "1" || process.env.GITHUB_RUN_ID !== runId)) {
+ throw new Error("Private bounded history accepts only the initial attempt of this workflow run");
+ }
+ const bucket = process.env.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET ?? "";
+ const receipt = path.resolve(`private-history-${operation}-${kind}-${runId}.json`);
+ await (await open(receipt, "wx", 0o600)).close();
+ const publisher = new BoundedHistoryPublisher(receipt);
+ if (operation === "verify") {
+ await verifyPrivateHistory(publisher, bucket);
+ console.log("Private retention and access checks passed");
+ return;
+ }
+ const receiptPublisher = operation === "upload" || process.env.GITHUB_ACTIONS === "true"
+ ? () => new BoundedHistoryPublisher(`${receipt}.retention.json`) : undefined;
+ const result = await retainPrivateTransfer({ operation: operation as "upload" | "download", kind: kind as "evidence" | "report", runId, directory, bucket, publisher }, receipt, receiptPublisher);
+ console.log(JSON.stringify(process.env.GITHUB_ACTIONS === "true" ? { version: 1, privateTransferCompleted: true } : result));
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ main().catch(error => {
+ console.error(process.env.GITHUB_ACTIONS === "true" ? "Private history transfer failed; inspect private retained evidence." : error instanceof Error ? error.message : String(error));
+ process.exitCode = 1;
+ });
+}
diff --git a/tests/runner-e2e/process-tree-owner.ts b/tests/runner-e2e/process-tree-owner.ts
index 86b44902bc..3dcae17791 100644
--- a/tests/runner-e2e/process-tree-owner.ts
+++ b/tests/runner-e2e/process-tree-owner.ts
@@ -177,13 +177,13 @@ export function createProcessTreeOwner(root: ChildProcess, options: {
}
let graceComplete = false;
let directGraceDelivered = false;
- async function signalGracefully() {
+ async function signalGracefully(signal: "SIGTERM" | "SIGINT" = "SIGTERM") {
if (graceComplete) return;
await observe();
// Choose and deliver from one validated table. ESRCH grants no ownership
// coverage: the next grace poll can select that branch's surviving owner.
const selected = new Set([...gracefulRoots()].filter(pid => !covered.has(pid)));
- const delivered = signalSnapshot("SIGTERM", selected);
+ const delivered = signalSnapshot(signal, selected);
for (const pid of delivered) {
if (pid === root.pid) directGraceDelivered = true;
covered.add(pid);
@@ -220,9 +220,10 @@ export async function stopKnownDirectChild(child: ChildProcess, phase: { gracefu
}
export async function incompleteTreeFallback(error: unknown, child: ChildProcess,
- phase: { gracefulSent: boolean; forcedDeadline?: number }, deadline: number, forcedMs: number) {
+ phase: { gracefulSent: boolean; forcedDeadline?: number }, deadline: number, forcedMs: number,
+ signal: "SIGTERM" | "SIGINT" = "SIGTERM") {
let fallbackError: unknown;
- try { await stopKnownDirectChild(child, phase, deadline, forcedMs); }
+ try { await stopKnownDirectChild(child, phase, deadline, forcedMs, signal); }
catch (failure) { fallbackError = failure; }
throw new Error(`Owned tree cleanup incomplete: ${String(error)}${fallbackError ? `; ${String(fallbackError)}` : ""}`);
}
@@ -234,13 +235,14 @@ export async function stopOwnedProcessTree(
owner: ReturnType<typeof createProcessTreeOwner>,
gracefulMs = 45_000,
forcedMs = 5_000,
+ gracefulSignal: "SIGTERM" | "SIGINT" = "SIGTERM",
): Promise<void> {
const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
const deadline = Date.now() + gracefulMs;
const phase: { gracefulSent: boolean; forcedDeadline?: number } = { gracefulSent: false };
try {
if (process.platform === "win32") {
- await stopKnownDirectChild(child, phase, deadline, forcedMs);
+ await stopKnownDirectChild(child, phase, deadline, forcedMs, gracefulSignal);
return;
}
const stopped = async () => {
@@ -248,7 +250,7 @@ export async function stopOwnedProcessTree(
return (child.exitCode !== null || child.signalCode !== null) && owner.liveGroups().length === 0;
};
do {
- await owner.signalGracefully();
+ await owner.signalGracefully(gracefulSignal);
phase.gracefulSent ||= owner.directGraceDelivered();
if (await stopped()) return;
await wait(50);
@@ -262,6 +264,6 @@ export async function stopOwnedProcessTree(
throw new Error("Observed launcher descendants remained after bounded cleanup");
} catch (error) {
phase.gracefulSent ||= owner.directGraceDelivered();
- await incompleteTreeFallback(error, child, phase, deadline, forcedMs);
+ await incompleteTreeFallback(error, child, phase, deadline, forcedMs, gracefulSignal);
}
}
diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts
index 82762c533f..5e90e3c2e2 100644
--- a/tests/runner-e2e/product-journeys/contracts.test.ts
+++ b/tests/runner-e2e/product-journeys/contracts.test.ts
@@ -143,6 +143,8 @@ describe("deployment journey contracts", () => {
onboarding.runIds=["run-1"]; onboarding.usage={runs:[{runId:"run-1",usage:{costUsd:0.5,inputTokens:10,outputTokens:10}}]};
onboarding.productJourney!.operations={campaignId:"campaign",attemptId:"attempt",modelCostUsd:0.7,setupCostUsd:0.2,requestCount:2,feedback:{collection:"complete",complaints:0,suggestions:1},traceIds:[]};
expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeCloseTo(0.7);
+ onboarding.usage={runs:[{runId:"run-1",usage:{costUsdExact:"0.5",costStatus:"estimated",inputTokens:10,outputTokens:10}}]};
+ expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeCloseTo(0.7);
expect(()=>validateJourneyCollection({journeyId:"cloud-onboarding",runIds:["run"],usage:{runs:[{runId:"run",usage:{apiKey:"not-retainable"} as any}]}})).toThrow("invalid_collection_receipt");
onboarding.productJourney!.operations.modelCostUsd=null;
expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeNull();
diff --git a/tests/runner-e2e/provider-usage.test.ts b/tests/runner-e2e/provider-usage.test.ts
new file mode 100644
index 0000000000..b651c9e8a8
--- /dev/null
+++ b/tests/runner-e2e/provider-usage.test.ts
@@ -0,0 +1,113 @@
+import { expect, it } from "vitest";
+import { estimateProviderCost, normalizeProviderUsage, ProviderUsageAccumulator, type EvalModelPrice } from "./provider-usage.js";
+const price: EvalModelPrice = { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.test/prices", inputUsdPerMillion: "2.5", cachedReadUsdPerMillion: "0.25", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" };
+
+it("excludes cached reads from OpenAI input and treats reasoning as a subset of output", () => {
+ const usage = normalizeProviderUsage("openai", { id: "response", model: "fixture", usage: { input_tokens: 100, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 40 }, output_tokens: 20, output_tokens_details: { reasoning_tokens: 15 } } });
+ expect(usage).toMatchObject({ complete: true, tokens: { input: 60, cachedRead: 40, cachedWrite: 0, output: 20, reasoning: 15 } });
+ expect(estimateProviderCost(usage, price)).toBe("0.000360000");
+});
+it("never turns missing usage or incompatible model/pricing into a free charge", () => {
+ const absent = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, output_tokens: 0 } });
+ expect(absent.complete).toBe(false);
+ expect(estimateProviderCost(absent, price)).toBeNull();
+ const zero = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 0 } });
+ expect(estimateProviderCost(zero, price)).toBe("0.000000000");
+ expect(estimateProviderCost(zero, { ...price, model: "other" })).toBeNull();
+});
+it("does not add Anthropic cumulative output counters and prices cache writes by TTL", () => {
+ const stream = new ProviderUsageAccumulator("anthropic");
+ stream.observe({ type: "message_start", message: { id: "message", model: "fixture", usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 20, cache_creation_input_tokens: 30, cache_creation: { ephemeral_5m_input_tokens: 20, ephemeral_1h_input_tokens: 10 } } } });
+ stream.observe({ type: "message_delta", usage: { output_tokens: 5 } });
+ stream.observe({ type: "message_delta", usage: { output_tokens: 8 } });
+ expect(stream.receipt().final).toBe(false);
+ stream.observe({ type: "message_stop" });
+ expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 10, cachedRead: 20, cachedWrite: 30, output: 8 } });
+ expect(estimateProviderCost(stream.receipt(), { ...price, provider: "anthropic", cacheWrite5mUsdPerMillion: "3.125", cacheWrite1hUsdPerMillion: "5" })).toBe("0.000222500");
+});
+it("retains missing cache TTL and malformed token subsets as unknown", () => {
+ const write = normalizeProviderUsage("anthropic", { model: "fixture", usage: { input_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 4, output_tokens: 1 } });
+ expect(estimateProviderCost(write, { ...price, provider: "anthropic" })).toBeNull();
+ expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 1 } }).complete).toBe(false);
+ expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1, output_tokens_details: { reasoning_tokens: 2 } } }).complete).toBe(false);
+});
+it("uses one terminal Responses usage receipt regardless of streamed text or repeat notifications", () => {
+ const stream = new ProviderUsageAccumulator("openai");
+ stream.observe({ type: "response.output_text.delta", delta: "not a usage receipt" });
+ expect(stream.receipt().complete).toBe(false);
+ const terminal = { type: "response.completed", response: { id: "response", model: "fixture", usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1 } } };
+ stream.observe(terminal); stream.observe(terminal);
+ expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 1, output: 1 } });
+ expect(estimateProviderCost(stream.receipt(), price)).toBe("0.000012500");
+});
+
+const modernPrice: EvalModelPrice = { ...price, inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4", outputUsdPerMillion: "20",
+ cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10",
+ longContext: { aboveInputTokens: 272_000, rates: { inputUsdPerMillion: "8", cachedReadUsdPerMillion: "0.8", outputUsdPerMillion: "30", cacheWrite30mUsdPerMillion: "10", cacheWrite12hUsdPerMillion: "20" } } };
+const modernUsage = (total = 12_000, read = 4000, shortWrite = 3000, longWrite = 4000) => ({ model: "fixture", usage: {
+ input_tokens: total, input_tokens_details: { cached_tokens: read, cache_write_tokens: shortWrite + longWrite,
+ cache_write_tokens_details: { ttl_30m_tokens: shortWrite, ttl_12h_tokens: longWrite } }, output_tokens: 100,
+} });
+it("separates modern OpenAI ordinary input and mixed-TTL writes without adding the base rate twice", () => {
+ const receipt = normalizeProviderUsage("openai", modernUsage());
+ expect(receipt).toMatchObject({ complete: true, tokens: { input: 1000, cachedRead: 4000, cachedWrite: 7000 }, cacheWrite30m: 3000, cacheWrite12h: 4000 });
+ expect(estimateProviderCost(receipt, modernPrice)).toBe("0.062600000");
+});
+it("applies long-context pricing to the entire request using total input including cache", () => {
+ expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_000, 270_000, 0, 1000)), modernPrice)).toBe("0.124000000");
+ expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_001, 270_000, 0, 1000)), modernPrice)).toBe("0.247008000");
+});
+it("keeps missing modern write counters or missing/inconsistent TTL splits unknown", () => {
+ const missing = modernUsage(); delete (missing.usage.input_tokens_details as any).cache_write_tokens;
+ expect(normalizeProviderUsage("openai", missing).complete).toBe(false);
+ expect(estimateProviderCost(normalizeProviderUsage("openai", missing), modernPrice)).toBeNull();
+ const noTtl = modernUsage(); delete (noTtl.usage.input_tokens_details as any).cache_write_tokens_details;
+ expect(estimateProviderCost(normalizeProviderUsage("openai", noTtl), modernPrice)).toBeNull();
+ const mismatch = modernUsage(); mismatch.usage.input_tokens_details.cache_write_tokens_details.ttl_12h_tokens++;
+ expect(normalizeProviderUsage("openai", mismatch).complete).toBe(false);
+});
+
+const policyPrice = { ...modernPrice, model: "gpt-5.6-sol" };
+function requestPolicyReceipt(request: Record<string, unknown> = {}, details: Record<string, unknown> = {}) {
+ const stream = new ProviderUsageAccumulator("openai", { model: policyPrice.model, input: "hello", ...request });
+ stream.observe({ type: "response.completed", response: { model: policyPrice.model, usage: {
+ input_tokens: 12_000, input_tokens_details: { cached_tokens: 4000, cache_write_tokens: 7000, ...details }, output_tokens: 100,
+ } } });
+ return stream.receipt();
+}
+it.each([
+ [{}, "30m", "0.042600000"],
+ [{ prompt_cache_retention: "24h" }, "30m", "0.042600000"],
+ [{ prompt_cache_options: { mode: "implicit", ttl: "12h" } }, "12h", "0.077600000"],
+] as const)("prices omitted write TTLs from a documented visible request policy: %j", (request, ttl, cost) => {
+ const receipt = requestPolicyReceipt(request);
+ expect(receipt).toMatchObject({ final: true, complete: true, cacheWriteAttribution: {
+ source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl,
+ } });
+ expect(estimateProviderCost(receipt, policyPrice)).toBe(cost);
+});
+it.each([
+ { model: "unqualified-model" }, { previous_response_id: "stored-response" }, { conversation: "stored-conversation" },
+ { prompt: { id: "stored-prompt" } }, { context_management: [] },
+ { input: [{ type: "item_reference", id: "stored-item" }] },
+ { input: [{ type: "compaction", encrypted_content: "opaque" }] },
+ { input: [{ role: "user", content: [{ type: "input_text", text: "hello", prompt_cache_breakpoint: { mode: "explicit", ttl: "12h" } }] }] },
+ { prompt_cache_options: { mode: "explicit" } }, { prompt_cache_options: { ttl: "unknown" } },
+ { prompt_cache_options: { ttl: null } }, { prompt_cache_options: null },
+ { prompt_cache_options: { future_option: true } },
+])("keeps hidden, explicit, or unqualified cache policies unknown: %j", request => {
+ const receipt = requestPolicyReceipt(request);
+ expect(receipt.cacheWriteAttribution).toBeUndefined();
+ expect(estimateProviderCost(receipt, policyPrice)).toBeNull();
+});
+it("prefers provider TTL splits and never repairs malformed or partial provider counters", () => {
+ const reported = requestPolicyReceipt({}, { cache_write_tokens_details: { ttl_30m_tokens: 3000, ttl_12h_tokens: 4000 } });
+ expect(reported.cacheWriteAttribution).toBeUndefined();
+ expect(estimateProviderCost(reported, policyPrice)).toBe("0.062600000");
+ for (const details of [{ cache_write_tokens: undefined }, { cache_write_tokens_details: null },
+ { cache_write_tokens_details: { ttl_30m_tokens: 7000 } }, { cache_write_tokens_details: { ttl_30m_tokens: 7000, ttl_12h_tokens: 1 } }]) {
+ const receipt = requestPolicyReceipt({}, details);
+ expect(receipt.cacheWriteAttribution).toBeUndefined();
+ expect(estimateProviderCost(receipt, policyPrice)).toBeNull();
+ }
+});
diff --git a/tests/runner-e2e/provider-usage.ts b/tests/runner-e2e/provider-usage.ts
new file mode 100644
index 0000000000..bb3453a761
--- /dev/null
+++ b/tests/runner-e2e/provider-usage.ts
@@ -0,0 +1,154 @@
+import { usdToUnits } from "../../packages/shared/src/money.js";
+import type { EvalCostObservation } from "./cost-ledger.js";
+
+type Tokens = NonNullable<EvalCostObservation["tokens"]>;
+type Row = Record<string, unknown>;
+const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
+const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
+export interface ProviderUsageReceipt {
+ provider: "openai" | "anthropic";
+ tokens: Tokens;
+ complete: boolean;
+ providerResponseId?: string;
+ model?: string;
+ cacheWrite5m?: number;
+ cacheWrite1h?: number;
+ cacheWrite30m?: number;
+ cacheWrite12h?: number;
+ cacheWriteAttribution?: { source: "request_policy"; version: "openai-prompt-cache-2026-10-20"; ttl: "30m" | "12h" };
+}
+
+function openAiRequestCachePolicy(payload: unknown): { model: string; ttl: "30m" | "12h" } | undefined {
+ const body = row(payload);
+ if (body.model !== "gpt-5.6-sol" || body.previous_response_id !== undefined || body.conversation !== undefined || body.prompt !== undefined
+ || body.context_management !== undefined || !(typeof body.input === "string" || Array.isArray(body.input))) return;
+ const options = row(body.prompt_cache_options);
+ if (body.prompt_cache_options !== undefined && body.prompt_cache_options !== options) return;
+ if (Object.keys(options).some(key => !["mode", "ttl"].includes(key)) || options.mode !== undefined && options.mode !== "implicit") return;
+ const ttl = options.ttl === undefined ? "30m" : options.ttl;
+ if (ttl !== "30m" && ttl !== "12h") return;
+ const hasUnqualifiedInput = (value: unknown): boolean => {
+ if (Array.isArray(value)) return value.some(hasUnqualifiedInput);
+ if (!value || typeof value !== "object") return false;
+ const item = row(value);
+ return "prompt_cache_breakpoint" in item || item.type === "compaction" || item.type === "item_reference"
+ || Object.values(item).some(hasUnqualifiedInput);
+ };
+ if (hasUnqualifiedInput(body.input)) return;
+ return { model: body.model, ttl };
+}
+
+export function normalizeProviderUsage(provider: "openai" | "anthropic", payload: unknown): ProviderUsageReceipt {
+ const body = row(payload), usage = row(body.usage);
+ const output = count(usage.output_tokens);
+ let tokens: Tokens;
+ let cacheWrite5m: number | undefined, cacheWrite1h: number | undefined;
+ let cacheWrite30m: number | undefined, cacheWrite12h: number | undefined;
+ if (provider === "openai") {
+ const totalInput = count(usage.input_tokens);
+ const details = row(usage.input_tokens_details);
+ const cachedRead = count(details.cached_tokens);
+ let cachedWrite = count(details.cache_write_tokens);
+ const writes = row(details.cache_write_tokens_details);
+ cacheWrite30m = count(writes.ttl_30m_tokens);
+ cacheWrite12h = count(writes.ttl_12h_tokens);
+ if (cachedWrite !== undefined && cacheWrite30m !== undefined && cacheWrite12h !== undefined && cacheWrite30m + cacheWrite12h !== cachedWrite) cachedWrite = undefined;
+ const reasoning = count(row(usage.output_tokens_details).reasoning_tokens);
+ tokens = { input: totalInput !== undefined && cachedRead !== undefined && cachedWrite !== undefined && totalInput >= cachedRead + cachedWrite ? totalInput - cachedRead - cachedWrite : undefined,
+ cachedRead, cachedWrite, output, reasoning };
+ } else {
+ const cache = row(usage.cache_creation);
+ cacheWrite5m = count(cache.ephemeral_5m_input_tokens);
+ cacheWrite1h = count(cache.ephemeral_1h_input_tokens);
+ tokens = { input: count(usage.input_tokens), cachedRead: count(usage.cache_read_input_tokens), cachedWrite: count(usage.cache_creation_input_tokens), output };
+ if (tokens.cachedWrite !== undefined && cacheWrite5m !== undefined && cacheWrite1h !== undefined && cacheWrite5m + cacheWrite1h !== tokens.cachedWrite) tokens.cachedWrite = undefined;
+ }
+ const complete = tokens.input !== undefined && tokens.cachedRead !== undefined && tokens.cachedWrite !== undefined && output !== undefined
+ && !(tokens.reasoning !== undefined && tokens.reasoning > output);
+ return { provider, tokens: Object.fromEntries(Object.entries(tokens).filter(([, value]) => value !== undefined)), complete,
+ ...(typeof body.id === "string" ? { providerResponseId: body.id } : {}),
+ ...(typeof body.model === "string" ? { model: body.model } : {}),
+ ...(cacheWrite5m !== undefined ? { cacheWrite5m } : {}), ...(cacheWrite1h !== undefined ? { cacheWrite1h } : {}),
+ ...(cacheWrite30m !== undefined ? { cacheWrite30m } : {}), ...(cacheWrite12h !== undefined ? { cacheWrite12h } : {}) };
+}
+
+export class ProviderUsageAccumulator {
+ private body: Row = {};
+ private usage: Row = {};
+ private final = false;
+ private readonly cachePolicy: ReturnType<typeof openAiRequestCachePolicy>;
+ constructor(readonly provider: "openai" | "anthropic", request?: unknown) {
+ this.cachePolicy = provider === "openai" ? openAiRequestCachePolicy(request) : undefined;
+ }
+ observe(event: unknown) {
+ const value = row(event);
+ if (this.provider === "openai") {
+ const response = row(value.response);
+ if (["response.completed", "response.incomplete", "response.failed"].includes(String(value.type))) {
+ this.body = response;
+ this.usage = row(response.usage);
+ this.final = true;
+ } else if (value.object === "response") {
+ this.body = value;
+ this.usage = row(value.usage);
+ this.final = ["completed", "incomplete", "failed"].includes(String(value.status));
+ }
+ } else {
+ if (value.type === "message_start") {
+ this.body = row(value.message);
+ this.usage = { ...this.usage, ...row(this.body.usage) };
+ } else if (value.type === "message_delta") this.usage = { ...this.usage, ...row(value.usage) };
+ else if (value.type === "message_stop") this.final = true;
+ else if (value.type === "message") { this.body = value; this.usage = row(value.usage); this.final = true; }
+ }
+ }
+ receipt() {
+ const receipt = normalizeProviderUsage(this.provider, { ...this.body, usage: this.usage });
+ const details = row(this.usage.input_tokens_details);
+ if (this.cachePolicy && receipt.model === this.cachePolicy.model && receipt.complete && receipt.tokens.cachedWrite! > 0
+ && !("cache_write_tokens_details" in details)) {
+ receipt.cacheWrite30m = this.cachePolicy.ttl === "30m" ? receipt.tokens.cachedWrite : 0;
+ receipt.cacheWrite12h = this.cachePolicy.ttl === "12h" ? receipt.tokens.cachedWrite : 0;
+ receipt.cacheWriteAttribution = { source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl: this.cachePolicy.ttl };
+ }
+ return { ...receipt, final: this.final };
+ }
+}
+
+export interface EvalTokenRates {
+ inputUsdPerMillion: string;
+ cachedReadUsdPerMillion: string;
+ outputUsdPerMillion: string;
+ cacheWrite5mUsdPerMillion?: string;
+ cacheWrite1hUsdPerMillion?: string;
+ cacheWrite30mUsdPerMillion?: string;
+ cacheWrite12hUsdPerMillion?: string;
+}
+export interface EvalModelPrice extends EvalTokenRates {
+ provider: "openai" | "anthropic";
+ model: string;
+ version: string;
+ sourceUrl: string;
+ longContext?: { aboveInputTokens: number; rates: EvalTokenRates };
+}
+
+export function estimateProviderCost(receipt: ProviderUsageReceipt, price: EvalModelPrice): string | null {
+ if (!receipt.complete || receipt.model !== price.model || receipt.provider !== price.provider) return null;
+ const { tokens } = receipt;
+ const totalInput = tokens.input! + tokens.cachedRead! + tokens.cachedWrite!;
+ if (!Number.isSafeInteger(totalInput)) return null;
+ const rates = price.longContext && totalInput > price.longContext.aboveInputTokens ? price.longContext.rates : price;
+ const priced = (count: number, rate: string) => {
+ if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(rate)) throw new Error("Invalid nonnegative model price");
+ return BigInt(count) * usdToUnits(rate);
+ };
+ let numerator = priced(tokens.input!, rates.inputUsdPerMillion) + priced(tokens.cachedRead!, rates.cachedReadUsdPerMillion) + priced(tokens.output!, rates.outputUsdPerMillion);
+ if (tokens.cachedWrite! > 0) {
+ const counts = receipt.provider === "openai" ? [receipt.cacheWrite30m, receipt.cacheWrite12h] : [receipt.cacheWrite5m, receipt.cacheWrite1h];
+ const writeRates = receipt.provider === "openai" ? [rates.cacheWrite30mUsdPerMillion, rates.cacheWrite12hUsdPerMillion] : [rates.cacheWrite5mUsdPerMillion, rates.cacheWrite1hUsdPerMillion];
+ if (counts[0] === undefined || counts[1] === undefined || counts[0] + counts[1] !== tokens.cachedWrite || writeRates[0] === undefined || writeRates[1] === undefined) return null;
+ numerator += priced(counts[0], writeRates[0]) + priced(counts[1], writeRates[1]);
+ }
+ const units = (numerator + 999_999n) / 1_000_000n;
+ return `${units / 1_000_000_000n}.${String(units % 1_000_000_000n).padStart(9, "0")}`;
+}
diff --git a/tests/runner-e2e/report-format.ts b/tests/runner-e2e/report-format.ts
index 4b50371ed5..7237d45697 100644
--- a/tests/runner-e2e/report-format.ts
+++ b/tests/runner-e2e/report-format.ts
@@ -1,6 +1,12 @@
import type { RunnerE2EBillingSummary } from "./types.js";
import { billingCoverageLabel } from "./billing.js";
+export function isPublicReportTimestamp(value: unknown): value is string {
+ return typeof value === "string" && value === value.trim()
+ && /^\d{4}-\d{2}-\d{2}T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d{1,9})?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/.test(value)
+ && Number.isFinite(Date.parse(value));
+}
+
export function html(value: unknown) {
return String(value ?? "")
.replaceAll("&", "&amp;")
@@ -10,7 +16,8 @@ export function html(value: unknown) {
.replaceAll("'", "&#39;");
}
-export function durationLabel(durationMs: number) {
+export function durationLabel(durationMs: number | null) {
+ if (durationMs === null) return "Unknown";
if (durationMs < 1_000) return `${durationMs}ms`;
const seconds = Math.round(durationMs / 1_000);
if (seconds < 60) return `${seconds}s`;
@@ -23,7 +30,7 @@ export function tokenLabel(value: number, llm?: RunnerE2EBillingSummary["llm"])
}
export function llmCostLabel(value: number, llm: RunnerE2EBillingSummary["llm"]) {
- return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount);
+ return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount, (llm.runsWithPartialReportedCost ?? 0) > 0);
}
export function usdLabel(value: number | null) {
diff --git a/tests/runner-e2e/report-playwright.config.ts b/tests/runner-e2e/report-playwright.config.ts
new file mode 100644
index 0000000000..390ee13208
--- /dev/null
+++ b/tests/runner-e2e/report-playwright.config.ts
@@ -0,0 +1,7 @@
+import { defineConfig } from "@playwright/test";
+
+export default defineConfig({
+ testDir: "../e2e", testMatch: "runner-e2e-dashboard.spec.ts", workers: 1, retries: 0,
+ timeout: 30_000, outputDir: "../../test-results/eval-report-navigation",
+ use: { browserName: "chromium", headless: true }, reporter: "list",
+});
diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts
index 2677ec7d3b..f3c516fbd8 100644
--- a/tests/runner-e2e/report.test.ts
+++ b/tests/runner-e2e/report.test.ts
@@ -1,3 +1,4 @@
+import { createWorkerTiming } from "./worker-timing.js";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
@@ -20,6 +21,50 @@ afterEach(async () => {
});
describe("runner E2E report aggregation", () => {
+ it("publishes retained passing results but exits unsuccessfully when the request journal is damaged", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-damaged-ledger-report-"));
+ cleanupDirectories.push(root);
+ const executionId = "core-compatibility.runner-codex.local.message-marker";
+ const directory = path.join(root, "attempt-1");
+ await mkdir(directory);
+ const result: RunnerE2EResult = {
+ schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "core-compatibility",
+ attempt: 1, status: "passed", profileId: "runner-codex", environmentId: "local",
+ caseId: "message-marker", provider: "codex", model: "fixture-model", runtimeMode: "native",
+ startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed",
+ };
+ const phaseStart = Date.parse(result.startedAt);
+ const timing = await createWorkerTiming(path.join(directory, "worker-timing.json"), result, phaseStart);
+ await timing.enter("execution", phaseStart + 200);
+ await timing.enter("cleanup", phaseStart + 800);
+ await timing.enter("finalization", phaseStart + 900);
+ result.workerTiming = await timing.finish(phaseStart + 1000);
+ const original = JSON.stringify(result);
+ await writeFile(path.join(directory, "result.json"), original);
+ await writeFile(path.join(directory, "final-state.png"), "fake-png");
+ await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] }));
+ const journal = path.join(root, "journal.jsonl"), output = path.join(root, "merged");
+ await writeFile(journal, '{"damaged-journal":');
+ await expect(execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {
+ cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
+ PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), PAPERCLIP_E2E_OPERATIONS_LEDGER: journal },
+ })).rejects.toMatchObject({ code: 1 });
+ const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8"));
+ expect(normalized).toMatchObject({ passed: 1, failed: 0, operations: { status: "unavailable" } });
+ expect(normalized.results[0]).toMatchObject({ executionId, status: "passed" });
+ expect(normalized.accounting.attempts[0].workerTiming).toEqual(result.workerTiming);
+ expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("Worker setup time: 200ms.");
+ expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("<strong>200ms</strong><span>Worker setup time</span>");
+ expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("Worker phase timing: complete.");
+ expect(normalized.billing.agentRunDurationMs).toBeNull();
+ expect(await readFile(path.join(output, "index.html"), "utf8"))
+ .toContain("<strong>Unknown</strong><span>Agent execution time</span>");
+ for (const file of ["index.html", "summary.md"]) {
+ expect(await readFile(path.join(output, file), "utf8")).toContain("Request ledger accounting unavailable");
+ }
+ expect(await readFile(path.join(directory, "result.json"), "utf8")).toBe(original);
+ });
+
it("corrects overconfident timing verdicts without changing retained bounds or original files", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "runner-latency-report-"));
cleanupDirectories.push(root);
@@ -331,13 +376,14 @@ describe("runner E2E report aggregation", () => {
},
});
expect(normalized.billing).toMatchObject({
- reportedLlmCostUsd: 0.0125,
+ reportedLlmCostUsd: 0.025,
llm: {
- inputTokens: 1_250,
- outputTokens: 75,
- runsWithReportedCost: 1,
+ inputTokens: 2_500,
+ outputTokens: 150,
+ runsWithReportedCost: 2,
},
});
+ expect(normalized.accounting.attempts).toHaveLength(2);
expect(normalized.results[0]).toMatchObject({
attempt: 2,
evidenceValid: true,
@@ -353,6 +399,10 @@ describe("runner E2E report aggregation", () => {
"utf8",
);
expect(dashboard).toContain("Runner Full-Stack E2E");
+ expect(dashboard).toMatch(/\$0\.0250*<\/strong><span>LLM reported subtotal/);
+ expect(dashboard).toContain("<strong>2s</strong><span>Summed attempt time</span>");
+ expect(dashboard).toContain("Carried-forward evidence can span older runs");
+ expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("Summed attempt time: 2s; 2 retained attempts; 0 missing attempts.");
expect(dashboard).toContain(executionId);
expect(dashboard).toContain("case-passed");
expect(dashboard).not.toContain(
@@ -668,3 +718,45 @@ describe("runner E2E report aggregation", () => {
expect(junit).not.toContain("<?xml-stylesheet");
});
});
+
+it("retains selector-approved successes and their spending when only failed workflow jobs rerun", async () => {
+ const { selectRerunArtifacts } = await import("./select-rerun-artifacts.js");
+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-partial-rerun-"));
+ cleanupDirectories.push(root);
+ const artifactRoot = path.join(root, "downloaded"), selectedRoot = path.join(root, "selected");
+ const executions = ["core-compatibility.runner-codex.local.message-marker", "core-compatibility.runner-claude.local.message-marker"];
+ const source = { sha: "a".repeat(40), ref: "master", workflowRunUrl: "https://github.com/paperclipai/paperclip/actions/runs/12345" };
+ for (const [index, executionId] of executions.entries()) {
+ const workflowAttempt = index + 1;
+ const artifactName = `runner-e2e-12345-${workflowAttempt}-${executionId}`;
+ const directory = path.join(artifactRoot, artifactName, `gha-12345-${workflowAttempt}-${executionId}`, "attempt-1");
+ await mkdir(directory, { recursive: true });
+ const result: RunnerE2EResult = {
+ schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "core-compatibility", source,
+ accountingIdentity: { campaignId: `gha-12345-${workflowAttempt}`, attemptId: `attempt-${index}` },
+ attempt: 1, status: "passed", profileId: index ? "runner-claude" : "runner-codex", environmentId: "local",
+ caseId: "message-marker", provider: index ? "claude" : "codex", model: "fixture", runtimeMode: "native",
+ startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed",
+ usage: { inputTokens: 100, outputTokens: 10, costUsd: 0.01 * workflowAttempt },
+ };
+ await writeFile(path.join(directory, "result.json"), JSON.stringify(result));
+ await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] }));
+ await writeFile(path.join(directory, "final-state.png"), "fixture-png");
+ }
+ await selectRerunArtifacts({ artifactRoot, selectedRoot, expectedExecutionIds: executions, workflowRunId: "12345", workflowRunAttempt: 2,
+ sourceSha: source.sha, sourceRef: source.ref, workflowRunUrl: source.workflowRunUrl,
+ jobs: { attempts: [{ run_attempt: 1, run_started_at: "2026-10-20T00:00:00Z" }, { run_attempt: 2, run_started_at: "2026-10-20T01:00:00Z" }],
+ jobs: [{ name: executions[0]!, run_attempt: 1, started_at: "2026-10-20T00:01:00Z" },
+ { name: executions[1]!, run_attempt: 1, started_at: "2026-10-20T00:01:00Z" },
+ { name: executions[1]!, run_attempt: 2, started_at: "2026-10-20T01:01:00Z" }] },
+ });
+ const output = path.join(root, "report");
+ await execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {
+ cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_E2E_CAMPAIGN_ID: "gha-12345-2", PAPERCLIP_RUNNER_E2E_REPORT_ROOT: selectedRoot,
+ PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify(executions) },
+ });
+ const campaign = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8"));
+ expect(campaign).toMatchObject({ passed: 2, failed: 0, billing: { reportedLlmCostUsd: 0.03 }, accounting: { excludedArtifacts: 0 } });
+ expect(campaign.accounting.attempts).toHaveLength(2);
+ expect(campaign.results[0].accountingIdentity.campaignId).toBe("gha-12345-1");
+});
diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts
index 81103993b2..a8a9664c3a 100644
--- a/tests/runner-e2e/report.ts
+++ b/tests/runner-e2e/report.ts
@@ -1,3 +1,4 @@
+import { campaignTimingText, summarizeCampaignTiming, summarizeWorkerPhaseTiming, workerPhaseTimingText } from "./campaign-timing.js";
import { stageDashboardBrandAssets } from "./report-assets.js";
import { discoverReportCatalog } from "./report-catalog.js";
import path from "node:path";
@@ -12,6 +13,10 @@ import {
} from "node:fs/promises";
import { runnerMatrix } from "./catalog.js";
import { billingCoverageLabel, summarizeExecutionBilling } from "./billing.js";
+import { isSelectedCarryForward, retainedCampaignAttempts } from "./attempt-accounting.js";
+import { summarizeCampaignFeedback } from "./feedback.js";
+import { campaignFeedbackReviewText } from "./feedback-dispositions.js";
+import { operationsReportText, readEvalOperationsReport } from "./operations-report.js";
import { renderRunnerE2EDashboard } from "./dashboard.js";
import {
buildRunnerCampaign,
@@ -195,6 +200,12 @@ async function main() {
const resultFiles = (await walk(root)).filter(
(file) => path.basename(file) === "result.json",
);
+ const requestedCampaignId = process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??
+ (process.env.GITHUB_RUN_ID ? `gha-${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT ?? "1"}` : undefined);
+ const selection = await readFile(path.join(root, "selected-artifacts.json"), "utf8")
+ .then(value => JSON.parse(value) as unknown)
+ .catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; });
+ let excludedArtifacts = 0;
const candidates = new Map<string, AggregatedResult[]>();
for (const resultFile of resultFiles) {
const parsed = JSON.parse(
@@ -206,6 +217,13 @@ async function main() {
)
continue;
const result = upgradeRunnerResult(parsed);
+ if (requestedCampaignId && result.accountingIdentity &&
+ result.accountingIdentity.campaignId !== requestedCampaignId &&
+ !isSelectedCarryForward({ requestedCampaignId, result, selection,
+ relativeResultPath: path.relative(root, resultFile).split(path.sep).join("/") })) {
+ excludedArtifacts += 1;
+ continue;
+ }
const directory = path.dirname(resultFile);
const evidence = await readFile(
path.join(directory, "evidence-manifest.json"),
@@ -314,17 +332,24 @@ async function main() {
billing: summarizeExecutionBilling(entry.result),
}));
const generatedAt = new Date().toISOString();
+ const retained = retainedCampaignAttempts(
+ selected.map(entry => entry.result),
+ [...candidates.values()].flatMap(entries => entries.map(entry => entry.result)),
+ );
const campaign = buildRunnerCampaign({
campaignId:
- process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??
- (process.env.GITHUB_RUN_ID
- ? `gha-${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT ?? "1"}`
- : `report-${generatedAt.replace(/[:.]/g, "-")}`),
+ requestedCampaignId ?? `report-${generatedAt.replace(/[:.]/g, "-")}`,
generatedAt,
expected,
results: resolvedResults,
+ attempts: retained.attempts,
+ excludedArtifacts: excludedArtifacts + retained.excludedArtifacts,
+ ...(process.env.PAPERCLIP_E2E_OPERATIONS_LEDGER ? { operations: await readEvalOperationsReport(process.env.PAPERCLIP_E2E_OPERATIONS_LEDGER) } : {}),
+ ...(process.env.PAPERCLIP_E2E_FEEDBACK_REVIEW ? { feedbackReview: JSON.parse(await readFile(process.env.PAPERCLIP_E2E_FEEDBACK_REVIEW, "utf8")) } : {}),
+ ...(process.env.PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT ? { braintrustExperiment: JSON.parse(await readFile(process.env.PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT, "utf8")) } : {}),
});
const billing = campaign.billing;
+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);
const normalized = {
...campaign,
results: selected.map((entry, index) => ({
@@ -386,7 +411,15 @@ async function main() {
: []),
`Tokens: ${billingCoverageLabel(`${billing.llm.inputTokens} input / ${billing.llm.outputTokens} output / ${billing.llm.cachedInputTokens} cached`, billing.llm.runsWithTokenUsage, billing.llm.runCount)}`,
"",
- `Provider-reported LLM cost: ${billingCoverageLabel(`$${billing.reportedLlmCostUsd.toFixed(6)}`, billing.llm.runsWithReportedCost, billing.llm.runCount)}`,
+ `Provider-reported LLM cost: ${billingCoverageLabel(`$${billing.reportedLlmCostUsd.toFixed(6)}`, billing.llm.runsWithReportedCost, billing.llm.runCount, (billing.llm.runsWithPartialReportedCost ?? 0) > 0)}`,
+ `Estimated LLM cost: ${billingCoverageLabel(`$${(billing.estimatedLlmCostUsd ?? 0).toFixed(6)}`, billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount)}`,
+ `Accounting: ${campaign.accounting!.attempts.length} retained attempts; ${campaign.accounting!.missingAttempts} missing attempts; ${campaign.accounting!.excludedArtifacts} out-of-scope artifacts excluded. Costs cover captured usage, not unobserved setup or external charges.`,
+ ...campaignTimingText(summarizeCampaignTiming(campaign.results, campaign.accounting)),
+ ...workerPhaseTimingText(summarizeWorkerPhaseTiming(campaign.results, campaign.accounting)),
+ ...(campaign.operations ? operationsReportText(campaign.operations) : []),
+ "",
+ `Feedback across retained attempts: ${feedback.complaints} complaints; ${feedback.suggestions} suggestions; ${feedback.incomplete} attempts with incomplete collection. Tool submissions: ${feedback.failedSubmissions} failed, ${feedback.unknownSubmissions} unresolved. Full bodies are in access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.`,
+ campaignFeedbackReviewText(campaign),
"",
`Estimated Daytona list-price runtime cost: $${billing.estimatedRuntimeCostUsd.toFixed(6)}`,
...(billing.judge ? [`Estimated judge cost: ${billing.judge.estimatedCostUsd === null ? "unknown" : `$${billing.judge.estimatedCostUsd.toFixed(6)}`}; ${billing.judge.attempts} attempts; ${billing.judge.attemptsWithUnknownUsage} with unknown usage; $${billing.judge.reservedCostUsd.toFixed(6)} reserved`] : []),
@@ -401,7 +434,7 @@ async function main() {
const cell = publicCampaignUrl
? `[${resolved.executionId}](${publicCampaignUrl}#execution-${encodeURIComponent(resolved.executionId)})`
: resolved.executionId;
- return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${billingCoverageLabel(`${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens}`, cellBilling.llm.runsWithTokenUsage, cellBilling.llm.runCount)} | ${billingCoverageLabel(`$${cellBilling.reportedCostUsd.toFixed(6)}`, cellBilling.llm.runsWithReportedCost, cellBilling.llm.runCount)} | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;
+ return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${billingCoverageLabel(`${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens}`, cellBilling.llm.runsWithTokenUsage, cellBilling.llm.runCount)} | ${billingCoverageLabel(`$${cellBilling.reportedCostUsd.toFixed(6)}`, cellBilling.llm.runsWithReportedCost, cellBilling.llm.runCount, (cellBilling.llm.runsWithPartialReportedCost ?? 0) > 0)} | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;
}),
"",
];
@@ -419,7 +452,7 @@ async function main() {
const junit = `<?xml version="1.0" encoding="UTF-8"?><testsuite name="Runner Full-Stack E2E" tests="${selected.length}" failures="${failures}">${cases}</testsuite>\n`;
await writeFile(path.join(output, "junit.xml"), junit);
console.log(summaryLines.join("\n"));
- if (failures > 0 || (dependable && (!dependable.complete || dependable.unclassifiedResults.length > 0))) process.exitCode = 1;
+ if (failures > 0 || campaign.operations?.status === "unavailable" || (dependable && (!dependable.complete || dependable.unclassifiedResults.length > 0))) process.exitCode = 1;
}
await main().catch((error) => {
diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts
index eb06745b10..460855184f 100644
--- a/tests/runner-e2e/result-validation.ts
+++ b/tests/runner-e2e/result-validation.ts
@@ -1,6 +1,7 @@
+import { validateWorkerTiming } from "./worker-timing.js";
import { connectionCheckpoints, connectionEvidencePasses, connectionRunSignals } from "./connection-evidence.js";
import { STARTUP_REUSE_LAYERS, STARTUP_REUSE_REASONS } from "../../packages/adapter-utils/src/startup-reuse.js";
-import type { RunnerE2EResult } from "./types.js";
+import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js";
import { DEPENDABLE_CELLS } from "./dependable-qualification.js";
type Rule = (value: unknown, at: string) => void;
@@ -117,7 +118,7 @@ const relativeFile: Rule = (value, at) => {
};
const runtime = shape({
provider: oneOf("local", "daytona"),
- agentRunDurationMs: number,
+ agentRunDurationMs: nullable(number),
leaseDurationMs: nullable(number),
leaseCount: integer,
cpuCores: optional(number),
@@ -138,12 +139,14 @@ const llm = shape({
runCount: integer,
runsWithTokenUsage: integer,
runsWithReportedCost: integer,
+ runsWithEstimatedCost: optional(integer),
inputTokens: number,
outputTokens: number,
cachedInputTokens: number,
totalTokens: number,
reportedCostUsd: number,
- costStatus: oneOf("reported", "partial", "unpriced", "unavailable"),
+ estimatedCostUsd: optional(number),
+ costStatus: oneOf("reported", "estimated", "partial", "unpriced", "unavailable"),
});
const billing = shape({
judge: optional(shape({ inputTokens: nullable(number), outputTokens: nullable(number), estimatedCostUsd: nullable(number), reservedCostUsd: number })),
@@ -155,6 +158,43 @@ const billing = shape({
observedAndEstimatedCostUsd: nullable(number),
complete: boolean,
});
+const tracing = shape({
+ projectId: string, traceId: string, spanId: string, status: oneOf("pending", "accepted", "failed"),
+ verification: oneOf("unverified"), nativeConfiguration: oneOf("not_started", "configured", "failed"),
+ companyId: optional(string), revision: optional(string), errorCode: optional(string), httpStatus: optional(number),
+ finalization: optional(shape({ spanId: string, status: oneOf("accepted", "failed"), outcome: oneOf("passed", "failed"),
+ verification: oneOf("unverified"), errorCode: optional(string), httpStatus: optional(number) })),
+});
+const feedback = shape({
+ version: (value, at) => { if (value !== 1) invalid(at); },
+ collection: oneOf("complete", "partial", "unavailable", "not_started"),
+ invocationCollection: oneOf("complete", "partial", "unavailable", "not_started"),
+ toolAvailability: oneOf("observed", "unknown"),
+ complaints: integer, suggestions: integer, bodiesCollected: integer, runsInspected: integer,
+ invocations: shape({ completed: integer, failed: integer, unknown: integer }),
+ review: array(shape({ fingerprint: (value, at) => { if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) invalid(at); },
+ kind: oneOf("complaint", "suggestion"), occurrences: integer, disposition: oneOf("needs_investigation") })),
+});
+
+export function validateCampaignAccounting(value: RunnerE2ECampaignAccounting) {
+ shape({
+ version: (version, at) => { if (version !== 1) invalid(at); },
+ scope: oneOf("retained_attempts", "selected_results"),
+ missingAttempts: integer,
+ excludedArtifacts: integer,
+ attempts: array(shape({
+ id: string, executionId: string, attempt: integer,
+ startedAt: date, finishedAt: date, durationMs: number,
+ status: oneOf("passed", "failed"), cleanup: oneOf("not_started", "passed", "failed"), billing, feedback: optional(feedback), tracing: optional(tracing),
+ })),
+ })(value, "accounting");
+ if (new Set(value.attempts.map(attempt => attempt.id)).size !== value.attempts.length) invalid("accounting.attempts duplicate id");
+ for (const attempt of value.attempts) {
+ if (attempt.workerTiming !== undefined) {
+ try { validateWorkerTiming(attempt.workerTiming, attempt); } catch { invalid("accounting.attempts.workerTiming"); }
+ }
+ }
+}
const matcher: Rule = (value, at) => {
object(value, at);
const item = value as Record<string, unknown>;
@@ -182,6 +222,7 @@ const matcher: Rule = (value, at) => {
rule(value, at);
};
const fields = {
+ accountingIdentity: optional(shape({ campaignId: string, attemptId: string })),
publicMcp: optional(assistant),
schema: oneOf(
"paperclip.runner-e2e.result/v1",
@@ -268,11 +309,13 @@ const fields = {
startedAt: date,
finishedAt: date,
durationMs: number,
+ workerTiming: optional(object),
// Provider usage is intentionally opaque. Billing reads only finite numbers;
// the dashboard shows opaque diagnostics through escaped JSON.
usage: optional(nullable(object)),
runtimeUsage: optional(runtime),
billing: optional(billing),
+ feedback: optional(feedback), tracing: optional(tracing),
matcherResults: optional(
array(shape({ matcher, passed: boolean, detail: string })),
),
@@ -344,6 +387,9 @@ export function validateRetainedRunnerResult(
): asserts value is RunnerE2EResult {
result(value, "result");
const checked = value as RunnerE2EResult;
+ if (checked.workerTiming !== undefined) {
+ try { validateWorkerTiming(checked.workerTiming, checked); } catch { invalid("result.workerTiming"); }
+ }
if (checked.suiteId === "product-journeys" && checked.status === "passed") {
const evidence = checked.productJourney;
if (!evidence) return invalid("result.productJourney");
diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs
index d2a1f01848..ee7e6dfd1f 100644
--- a/tests/runner-e2e/run-unit.mjs
+++ b/tests/runner-e2e/run-unit.mjs
@@ -20,7 +20,7 @@ delete env.NODE_OPTIONS;
for (const [name, command] of [
["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]],
- ["Node tests", ["--test", "tests/runner-e2e/native-completion-checks.test.mjs",
+ ["Node tests", ["--test", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs",
"tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"]],
]) {
const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" });
diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts
index b4b0824653..420d8c97cf 100644
--- a/tests/runner-e2e/runner.spec.ts
+++ b/tests/runner-e2e/runner.spec.ts
@@ -1,3 +1,6 @@
+import { createWorkerTiming } from "./worker-timing.js";
+import { FEEDBACK_PERSISTENCE_SUITE, gradeFeedbackPersistence } from "./feedback-persistence.js";
+import { EvalAttemptTrace, evalBraintrustConfig, EVAL_OBSERVABILITY_SECRET_NAMES } from "./braintrust.js";
import { captureFollowupStopFailureApiState } from "./followup-stop-evidence.js";
import { runFollowupStopResumeFlow } from "./followup-stop-resume-flow.js";
import { runFinalizationRecoveryFlow } from "./finalization-recovery-flow.js";
@@ -58,6 +61,8 @@ import path from "node:path";
import { expect, test, type Page } from "@playwright/test";
import { RunnerApi, pollUntil } from "./api.js";
import { buildRuntimeUsage, summarizeExecutionBilling } from "./billing.js";
+import { collectAttemptFeedback, emptyFeedbackSummary } from "./feedback.js";
+import { finishBudgetedEvalSetup } from "./budgeted-setup.js";
import { establishPublicMcpSession, runPublicMcpFlow } from "./public-mcp-flow.js";
import { assistantUsage } from "./public-mcp-model.js";
import { runnerExecutionById } from "./catalog.js";
@@ -359,7 +364,9 @@ async function writeSanitizedJson(
name: string,
value: unknown,
secrets: readonly string[],
+ signal?: AbortSignal,
) {
+ signal?.throwIfAborted();
const raw = `${JSON.stringify(value)}\n`;
// Public API payloads may contain harmless provider-shaped fixtures or
// opaque generated tokens. Reject exact campaign credentials before writing,
@@ -368,8 +375,9 @@ async function writeSanitizedJson(
const safe = `${JSON.stringify(sanitizeJson(value, secrets), null, 2)}\n`;
const leak = findSecretLeakInJsonValues(JSON.parse(safe), secrets);
if (leak) throw new Error(`Secret leak in ${name}: ${leak}`);
+ signal?.throwIfAborted();
await mkdir(directory, { recursive: true });
- await writeFile(path.join(directory, name), safe, "utf8");
+ await writeFile(path.join(directory, name), safe, { encoding: "utf8", signal });
}
async function submitTaskRevision(page: Page, body: string): Promise<number> {
@@ -593,6 +601,7 @@ for (const execution of executions) {
}
const startedAtMs = Date.now();
const startedAt = new Date(startedAtMs).toISOString();
+ const workerTiming = await createWorkerTiming(path.join(snapshotsDir, "worker-timing.json"), { executionId: execution.id, attempt }, startedAtMs);
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
const marker = execution.task.buildVisibleMarker(nonce);
const title = execution.task.buildTitle(nonce);
@@ -609,7 +618,20 @@ for (const execution of executions) {
verifyNativeInstructionPreflight(process.env[NATIVE_INSTRUCTION_PREFLIGHT_ENV]);
}
const credentials = credentialValues();
- const secrets = normalizedSecrets(Object.values(credentials));
+ const secrets = normalizedSecrets([...Object.values(credentials), ...EVAL_OBSERVABILITY_SECRET_NAMES.map(name => process.env[name]), process.env.PAPERCLIP_E2E_GATEWAY_TOKEN]);
+ const traceConfig = evalBraintrustConfig(process.env);
+ const trace = traceConfig ? new EvalAttemptTrace(traceConfig, {
+ campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? "",
+ attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ?? "",
+ executionId: execution.id, provider: execution.profile.provider, model: execution.profile.model, environment: execution.environment.id,
+ }, {
+ suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash,
+ profileId: execution.profile.id, caseId: execution.task.id, sourceRevision: resolveRunnerE2ESource().sha,
+ }) : undefined;
+ const saveTrace = async () => { if (trace) await writeSanitizedJson(snapshotsDir, "braintrust-trace.json", trace.snapshot(), secrets); };
+ const onCompanyCreated = async (id: string) => {
+ if (trace) { try { await trace.configureCompany(api, id); } finally { await saveTrace(); } }
+ };
const api = new RunnerApi(request);
const companyRunFlow = execution.suite.id === "task-titles" || ["plan_task_guidance", "blocker_guidance", "continuation_accounting", "continuation", "context_integrity", "repository_skills", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "pi_controls", "copilot_protection", "cursor_native", "native_active_stop", "native_provider_loss", "public_mcp", "case_handoff"].includes(execution.task.flow);
const publicMcpUsage = execution.task.flow === "public_mcp" ? assistantUsage(execution.profile.provider === "claude" ? "anthropic" : "openai", execution.profile.model) : undefined;
@@ -626,6 +648,7 @@ for (const execution of executions) {
let issue: IssueRecord | undefined;
let workflowCase: { id: string; identifier?: string } | undefined;
let selectedRuns: RunRecord[] = [];
+ let feedback = emptyFeedbackSummary();
let runtimeLeases: EnvironmentLeaseRecord[] = [];
let matcherResults: MatcherResult[] = [];
let downloadedResponseProof: Awaited<ReturnType<typeof readResponseProof>> | undefined;
@@ -891,6 +914,8 @@ for (const execution of executions) {
});
try {
+ trace?.phase("setup");
+ await saveTrace();
if (execution.task.flow === "public_mcp") publicMcpUserId = await establishPublicMcpSession(api, page, secrets);
if (execution.suite.id === "stock-harness") {
const receipt = verifyStockHarnessPreflight(process.env[STOCK_PREFLIGHT_ENV]);
@@ -920,7 +945,7 @@ for (const execution of executions) {
await setupGitStreamingWorkspace(workspacePath);
}
fixtures = execution.task.flow === "first_task"
- ? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, deadlineAt: startedAtMs + deadlineMs, observe: value => { fixtures = value; } })
+ ? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, deadlineAt: startedAtMs + deadlineMs, observe: value => { fixtures = value; }, onCompanyCreated })
: await setupLiveFixtures({
api,
execution,
@@ -928,7 +953,13 @@ for (const execution of executions) {
workspacePath,
credentials,
daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE,
+ onCompanyCreated,
});
+ await finishBudgetedEvalSetup({ executionId: execution.id, agentId: fixtures.agent.id,
+ companyId: fixtures.company.id, deadlineAt: startedAtMs + deadlineMs, api,
+ observe: observations => writeSanitizedJson(snapshotsDir, "budgeted-setup.json", { observations }, secrets) });
+ await workerTiming.enter("execution");
+ trace?.phase("execution");
if (["native-completion", NATIVE_INSTRUCTION_SUITE].includes(execution.suite.id)) {
const receipt = await captureNativeDefault({ api, agentId: fixtures.agent.id, companyId: fixtures.company.id });
@@ -3078,6 +3109,8 @@ for (const execution of executions) {
}
}
} finally {
+ await workerTiming.enter("cleanup");
+ trace?.phase("cleanup", Boolean(primaryError));
await responseTiming?.stop();
await reviewProvider?.close();
try {
@@ -3136,7 +3169,29 @@ for (const execution of executions) {
}
},
assertions: beforeEnvironmentTeardownAssertions,
- teardown: () => fixtures!.teardown(),
+ teardown: async () => {
+ await workerTiming.enter("feedback");
+ trace?.phase("feedback");
+ try {
+ feedback = await collectAttemptFeedback({ api, companyId: fixtures!.company.id,
+ onCheckpoint: summary => { feedback = summary; },
+ persist: async (evidence, signal) => {
+ await writeSanitizedJson(snapshotsDir, "agent-feedback.pending.json", evidence, secrets, signal);
+ signal.throwIfAborted();
+ await rename(path.join(snapshotsDir, "agent-feedback.pending.json"), path.join(snapshotsDir, "agent-feedback.json"));
+ signal.throwIfAborted();
+ },
+ });
+ } catch {
+ feedback.collection = feedback.bodiesCollected ? "partial" : "unavailable";
+ feedback.invocationCollection = "unavailable";
+ networkDiagnostics.push({ feedbackCaptureError: "feedback_evidence_write_failed" });
+ } finally {
+ await workerTiming.enter("cleanup");
+ trace?.phase("cleanup", !["complete", "not_started"].includes(feedback.collection));
+ await fixtures!.teardown();
+ }
+ },
});
matcherResults.push(...verification.checks.map(check => ({
matcher: { kind: "json_path" as const, path: `remoteCleanup.${check.id}`, expected: true }, passed: check.passed, detail: check.detail,
@@ -3202,6 +3257,20 @@ for (const execution of executions) {
}
}
+ await workerTiming.enter("finalization");
+ if (execution.suite.id === FEEDBACK_PERSISTENCE_SUITE) {
+ let evidence: unknown;
+ try { evidence = JSON.parse(await readFile(path.join(snapshotsDir, "agent-feedback.json"), "utf8")); } catch { evidence = undefined; }
+ const checks = gradeFeedbackPersistence({ evidence, nonce, companyId: fixtures?.company.id,
+ agentId: fixtures?.agent.id, taskId: issue?.id, runIds: selectedRuns.map(run => run.id) });
+ matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const,
+ path: `feedbackPersistence.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })));
+ if (checks.some(check => !check.passed)) primaryError ??= new Error("Feedback persistence matcher failed");
+ }
+
+ const traceReceipt = await trace?.export({ failed: Boolean(primaryError), runIds: selectedRuns.map(run => run.id),
+ cleanup, complaints: feedback.complaints, suggestions: feedback.suggestions });
+ await saveTrace();
const finishedAtMs = Date.now();
const runtimeUsage = fixtureRuntimeCostCoverage(buildRuntimeUsage({
environmentId: execution.environment.id,
@@ -3211,6 +3280,12 @@ for (const execution of executions) {
}), fixtures ?? {});
const resultWithoutBilling: RunnerE2EResult = {
schema: "paperclip.runner-e2e.result/v2",
+ ...(process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID && process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ? {
+ accountingIdentity: {
+ campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID,
+ attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID,
+ },
+ } : {}),
executionId: execution.id,
suiteId: execution.suite.id,
suiteDefinitionHash: execution.suiteDefinitionHash,
@@ -3248,6 +3323,7 @@ for (const execution of executions) {
startedAt,
finishedAt: new Date(finishedAtMs).toISOString(),
durationMs: finishedAtMs - startedAtMs,
+ workerTiming: await workerTiming.finish(finishedAtMs),
usage:
selectedRuns.length === 1
? (selectedRuns[0]?.usageJson ?? null)
@@ -3258,6 +3334,8 @@ for (const execution of executions) {
})),
},
runtimeUsage,
+ feedback,
+ ...(traceReceipt ? { tracing: traceReceipt } : {}),
matcherResults,
screenshots,
cleanup,
diff --git a/tests/runner-e2e/scheduled-finalization.test.ts b/tests/runner-e2e/scheduled-finalization.test.ts
new file mode 100644
index 0000000000..8b90c9fd37
--- /dev/null
+++ b/tests/runner-e2e/scheduled-finalization.test.ts
@@ -0,0 +1,54 @@
+import { expect, it } from "vitest";
+import { buildRunnerCampaign } from "./history.js";
+import { runnerMatrix } from "./catalog.js";
+import { emptyFeedbackSummary, feedbackReview, type FeedbackEvidence } from "./feedback.js";
+import { expectedFixtureFeedback } from "./feedback-persistence.js";
+import { reviewScheduledFeedback, scheduledCampaignFindings } from "./scheduled-finalization.js";
+import type { RunnerE2EResult } from "./types.js";
+
+function fixture() {
+ const cell = runnerMatrix.find(row => row.id === "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion")!;
+ const nonce = "abcdef123-1", bodies = expectedFixtureFeedback(nonce);
+ const entries = ["complaint", "suggestion"].map(kind => ({ kind, body: bodies[kind as keyof typeof bodies] })) as FeedbackEvidence["entries"];
+ const feedback = { version: 1, companyId: "company", entries, invocations: [], diagnostics: [], summary: emptyFeedbackSummary() } as FeedbackEvidence;
+ const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id,
+ suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash,
+ accountingIdentity: { campaignId: "scheduled-fixture", attemptId: "one" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null },
+ attempt: 1, status: "passed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id,
+ environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,
+ startedAt: "2026-10-21T00:00:00.000Z", finishedAt: "2026-10-21T00:00:01.000Z", durationMs: 1000,
+ feedback: { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete", complaints: 1, suggestions: 1,
+ bodiesCollected: 2, review: feedbackReview(entries) },
+ matcherResults: [{ matcher: { kind: "message_exact", expected: `PAPERCLIP_E2E_OK_${nonce}` }, passed: true, detail: "matched" },
+ ...["complete-retained-evidence", "one-of-each", "complaint-body-and-attribution", "complaint-completed-tool",
+ "suggestion-body-and-attribution", "suggestion-completed-tool", "only-expected-submissions"].map(id => ({
+ matcher: { kind: "json_path" as const, path: `feedbackPersistence.${id}`, expected: true }, passed: true, detail: "matched" }))] };
+ const campaign = buildRunnerCampaign({ campaignId: "scheduled-fixture", generatedAt: result.finishedAt,
+ expected: [cell.id], results: [result], attempts: [result] });
+ return { result, feedback, campaign };
+}
+
+it("reviews independently retained exact synthetic bodies without a model call", () => {
+ const { result, feedback, campaign } = fixture();
+ const review = reviewScheduledFeedback(campaign, [{ result, feedback }], result.finishedAt);
+ expect(review.decisions.map(row => row.disposition)).toEqual(["unsupported", "unsupported"]);
+ expect(JSON.stringify(review)).not.toContain("Synthetic eval");
+});
+
+it.each(["body-changed", "wrong-suite", "missing-evidence", "failed-grade", "partial-collection"])("leaves %s feedback for investigation", reason => {
+ const { result, feedback, campaign } = fixture();
+ if (reason === "body-changed") feedback.entries[0]!.body = "A real product problem";
+ if (reason === "wrong-suite") result.suiteId = "core-compatibility";
+ if (reason === "failed-grade") result.matcherResults![2]!.passed = false;
+ if (reason === "partial-collection") result.feedback!.collection = "partial";
+ const review = reviewScheduledFeedback(campaign, [{ result, feedback: reason === "missing-evidence" ? null : feedback }], result.finishedAt);
+ expect(review.decisions.every(row => row.disposition === "needs_investigation")).toBe(true);
+});
+
+it("keeps incomplete accounting and trace export visible independently of a passing product grade", () => {
+ const { result, feedback, campaign } = fixture();
+ campaign.feedbackReview = reviewScheduledFeedback(campaign, [{ result, feedback }], result.finishedAt);
+ expect(scheduledCampaignFindings(campaign)).toEqual(["accounting_unavailable", "trace_export_incomplete"]);
+ campaign.feedbackReview.decisions[0]!.disposition = "needs_investigation";
+ expect(scheduledCampaignFindings(campaign)).toContain("feedback_needs_review");
+});
diff --git a/tests/runner-e2e/scheduled-finalization.ts b/tests/runner-e2e/scheduled-finalization.ts
new file mode 100644
index 0000000000..b9653b1772
--- /dev/null
+++ b/tests/runner-e2e/scheduled-finalization.ts
@@ -0,0 +1,117 @@
+import { readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { evalBraintrustConfig } from "./braintrust.js";
+import { openExperimentIngestion } from "./experiment-ingestion.js";
+import { collectScheduledIngestion } from "./ingestion-usage.js";
+import { importBraintrustExperiment, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";
+import { verifyBraintrustIndexing } from "./braintrust-indexing.js";
+import { feedbackReviewTemplate, validateCampaignFeedbackReview, type CampaignFeedbackReview } from "./feedback-dispositions.js";
+import { expectedFixtureFeedback } from "./feedback-persistence.js";
+import { feedbackReview, summarizeCampaignFeedback, type FeedbackEvidence } from "./feedback.js";
+import type { EvalOperationsManifest } from "./operations.js";
+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";
+
+export function reviewScheduledFeedback(campaign: RunnerE2ECampaign,
+ evidence: Array<{ result: RunnerE2EResult; feedback: FeedbackEvidence | null }>, now: string): CampaignFeedbackReview {
+ const synthetic = new Set<string>();
+ for (const { result, feedback } of evidence) {
+ if (result.suiteId !== "feedback-persistence" || !feedback || result.feedback?.collection !== "complete"
+ || result.feedback.invocationCollection !== "complete") continue;
+ const marker = result.matcherResults?.find(row => row.matcher.kind === "message_exact")?.matcher;
+ const nonce = marker?.kind === "message_exact" ? /^PAPERCLIP_E2E_OK_([a-z0-9]+-[1-9][0-9]*)$/.exec(marker.expected)?.[1] : undefined;
+ if (!nonce) continue;
+ const checks = ["complete-retained-evidence", "one-of-each", "complaint-body-and-attribution", "complaint-completed-tool",
+ "suggestion-body-and-attribution", "suggestion-completed-tool", "only-expected-submissions"];
+ if (!checks.every(id => result.matcherResults?.some(row => row.matcher.kind === "json_path"
+ && row.matcher.path === `feedbackPersistence.${id}` && row.passed))) continue;
+ const bodies = expectedFixtureFeedback(nonce);
+ if (feedback.entries.length !== 2 || !["complaint", "suggestion"].every(kind => feedback.entries.filter(entry =>
+ entry.kind === kind && entry.body === bodies[kind as keyof typeof bodies]).length === 1)) continue;
+ for (const group of feedbackReview(feedback.entries)) synthetic.add(group.fingerprint);
+ }
+ const template = feedbackReviewTemplate(campaign);
+ const review: CampaignFeedbackReview = { ...template, version: 1, reviewedAt: now, reviewedBy: "agent",
+ decisions: template.decisions.map(row => ({ fingerprint: row.fingerprint,
+ disposition: synthetic.has(row.fingerprint) ? "unsupported" : "needs_investigation" })) };
+ validateCampaignFeedbackReview(campaign, review);
+ return review;
+}
+
+export function scheduledCampaignFindings(campaign: RunnerE2ECampaign): string[] {
+ const findings: string[] = [];
+ if (campaign.failed || campaign.results.length !== campaign.expected.length || campaign.accounting?.missingAttempts) findings.push("product_or_missing_attempt");
+ if (campaign.accounting?.attempts.some(row => row.cleanup !== "passed")) findings.push("cleanup_failure");
+ const operations = campaign.operations;
+ if (!operations || operations.status === "unavailable") findings.push("accounting_unavailable");
+ else if (operations.unresolvedRequests || operations.conflictingRequests || operations.inflight) findings.push("accounting_incomplete");
+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);
+ if (feedback.incomplete || feedback.failedSubmissions || feedback.unknownSubmissions) findings.push("feedback_collection_incomplete");
+ if (!campaign.feedbackReview || campaign.feedbackReview.decisions.some(row => ["actionable", "needs_investigation"].includes(row.disposition))) findings.push("feedback_needs_review");
+ if (campaign.results.some(row => row.tracing?.status !== "accepted" || row.tracing.nativeConfiguration !== "configured"
+ || row.tracing.finalization?.status !== "accepted")) findings.push("trace_export_incomplete");
+ return findings;
+}
+
+export async function finalizeScheduledCampaign(env: NodeJS.ProcessEnv) {
+ if (!env.PAPERCLIP_E2E_OPERATIONS_MANIFEST || !env.GITHUB_WORKSPACE) throw new Error("Missing scheduled campaign manifest");
+ const manifest = JSON.parse(await readFile(env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, "utf8")) as EvalOperationsManifest;
+ const directory = path.dirname(env.PAPERCLIP_E2E_OPERATIONS_MANIFEST);
+ const root = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results", manifest.limits.campaignId);
+ const campaign = JSON.parse(await readFile(path.join(root, "campaign.json"), "utf8")) as RunnerE2ECampaign;
+ const originals: RunnerE2EResult[] = [], evidence: Array<{ result: RunnerE2EResult; feedback: FeedbackEvidence | null }> = [];
+ for (const selected of campaign.results) {
+ if (selected.attempt < 1) continue;
+ if (!manifest.executionIds.includes(selected.executionId) || selected.attempt !== 1) throw new Error("Unexpected scheduled attempt");
+ const resultPath = path.join(root, ...selected.executionId.split("."), "attempt-1");
+ const result = JSON.parse(await readFile(path.join(resultPath, "result.json"), "utf8")) as RunnerE2EResult;
+ originals.push(result);
+ let feedback: FeedbackEvidence | null = null;
+ try { feedback = JSON.parse(await readFile(path.join(resultPath, "snapshots/agent-feedback.json"), "utf8")); } catch {}
+ evidence.push({ result, feedback });
+ }
+ campaign.feedbackReview = reviewScheduledFeedback(campaign, evidence, new Date().toISOString());
+ await writeFile(path.join(directory, "feedback-review.json"), `${JSON.stringify(campaign.feedbackReview, null, 2)}\n`, { mode: 0o600 });
+ const findings = scheduledCampaignFindings(campaign);
+ const config = evalBraintrustConfig(env);
+ let experiment: BraintrustExperimentReceipt | undefined;
+ let ingestion: Awaited<ReturnType<typeof openExperimentIngestion>> | undefined;
+ try {
+ if (!config) throw new Error("Missing Braintrust export configuration");
+ if (config.projectId !== manifest.projectId || campaign.campaignId !== manifest.limits.campaignId) throw new Error("Scheduled ingestion identity mismatch");
+ ingestion = await openExperimentIngestion({ projectId: config.projectId, campaignId: manifest.limits.campaignId,
+ receiptPath: `${manifest.ledgerPath}.experiment-ingestion.jsonl`, deadlineAt: Date.now() + 60_000 });
+ await importBraintrustExperiment(config, campaign, originals, async receipt => {
+ await writeFile(path.join(directory, "experiment-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
+ experiment = receipt;
+ }, ingestion.transport);
+ } catch { findings.push("experiment_import_incomplete"); }
+ finally {
+ try { await ingestion?.close(); }
+ catch { findings.push("experiment_ingestion_accounting_incomplete"); }
+ }
+ try {
+ if (!config) throw new Error("Missing Braintrust ingestion configuration");
+ const usage = await collectScheduledIngestion({ ledgerPath: manifest.ledgerPath, campaignId: manifest.limits.campaignId,
+ projectId: config.projectId, executionIds: manifest.executionIds });
+ await writeFile(path.join(directory, "ingestion-usage.json"), `${JSON.stringify(usage, null, 2)}\n`, { mode: 0o600 });
+ if (usage.status !== "complete") findings.push("ingestion_accounting_incomplete");
+ } catch { findings.push("ingestion_accounting_unavailable"); }
+ let indexing = "unavailable";
+ try {
+ if (!config) throw new Error("Missing Braintrust query configuration");
+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment);
+ indexing = receipt.status;
+ await writeFile(path.join(directory, "indexing-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
+ } catch {}
+ if (indexing !== "verified") findings.push("trace_or_experiment_indexing_incomplete");
+ await writeFile(path.join(directory, "findings.json"), `${JSON.stringify({ version: 1, findings,
+ nativeIndexVerification: indexing, invoiceReconciliation: "pending",
+ feedbackReviewMethod: "Only exact synthetic fixture bodies with all corroborating graders are automatically dismissed; all other feedback requires review" }, null, 2)}\n`, { mode: 0o600 });
+ for (const finding of findings) console.error(`::error title=Eval operations::${finding}`);
+ return findings;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ if ((await finalizeScheduledCampaign(process.env)).length) process.exitCode = 1;
+}
diff --git a/tests/runner-e2e/scheduled-forecast.test.ts b/tests/runner-e2e/scheduled-forecast.test.ts
new file mode 100644
index 0000000000..daf5b007a1
--- /dev/null
+++ b/tests/runner-e2e/scheduled-forecast.test.ts
@@ -0,0 +1,91 @@
+import { expect, it } from "vitest";
+import { runnerMatrix } from "./catalog.js";
+import { scheduledForecast } from "./scheduled-forecast.js";
+import { scheduledSelection } from "./scheduled-operations.js";
+import measurements from "./scheduled-measurements.json" with { type: "json" };
+import prices from "./scheduled-prices.json" with { type: "json" };
+import type { EvalOperationsManifest } from "./operations.js";
+
+const models = prices.models as EvalOperationsManifest["models"];
+const selected = (cadence: "daily" | "weekly") => runnerMatrix.filter(cell => scheduledSelection(cadence).includes(cell.id));
+
+it.each([
+ ["daily", "0.427624800", 72757], ["weekly", "0.791539400", 127069],
+] as const)("retains exact historical %s measurements without inventing hosted or tail estimates", (cadence, modelUsd, summedAttemptMs) => {
+ const forecast = scheduledForecast(selected(cadence), models);
+ expect(forecast.historicalSuccessfulSelection).toEqual({ modelUsd, summedAttemptMs });
+ expect(forecast.measurementSnapshotSha256).toMatch(/^[0-9a-f]{64}$/);
+ expect(forecast).toMatchObject({ hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null, costStatus: "estimated" });
+ for (const cell of forecast.cells) {
+ expect(cell.historicalSuccessful.samples).toBe(1);
+ expect(cell.historicalSuccessful.modelUsd.p95).toBeNull();
+ expect(cell.historicalSuccessful.attemptMs.p95).toBeNull();
+ expect(cell.samples[0]!.resultSha256).toMatch(/^[0-9a-f]{64}$/);
+ }
+});
+
+it("does not treat the older feedback definition as a current measurement", () => {
+ const forecast = scheduledForecast(selected("daily"), models);
+ const feedback = forecast.cells.find(cell => cell.executionId.startsWith("feedback-"))!;
+ expect(feedback.samples[0]!.mismatches).toContain("definition");
+ expect(feedback.compatibleSuccessful.samples).toBe(0);
+ expect(forecast.compatibleSuccessfulSelection).toEqual({ modelUsd: null, summedAttemptMs: null });
+});
+
+it.each(["definitionHash", "model", "environmentId", "runtimeMode", "route", "priceVersion"] as const)("excludes incompatible %s samples without dropping their evidence", field => {
+ const snapshot = structuredClone(measurements);
+ const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!;
+ const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!;
+ sample[field] = field === "definitionHash" ? "a".repeat(64) : "different";
+ const forecast = scheduledForecast([cell], models, snapshot);
+ expect(forecast.cells[0]!.compatibleSuccessful.samples).toBe(0);
+ expect(forecast.cells[0]!.samples).toHaveLength(1);
+ expect(forecast.compatibleSuccessfulSelection.modelUsd).toBeNull();
+});
+
+it("keeps missing cells and empty selections unknown", () => {
+ const snapshot = structuredClone(measurements);
+ snapshot.samples = [];
+ expect(scheduledForecast(selected("daily"), models, snapshot).historicalSuccessfulSelection)
+ .toEqual({ modelUsd: null, summedAttemptMs: null });
+ expect(scheduledForecast([], models).compatibleSuccessfulSelection)
+ .toEqual({ modelUsd: null, summedAttemptMs: null });
+});
+
+it("computes distributions within one cohort and does not sum P95s", () => {
+ const snapshot = structuredClone(measurements);
+ const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!;
+ const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!;
+ snapshot.samples = Array.from({ length: 20 }, (_, index) => ({ ...sample, attemptId: `attempt-${index}`,
+ knownModelUsd: String(index + 1), attemptDurationMs: (index + 1) * 1000 }));
+ const forecast = scheduledForecast([cell], models, snapshot);
+ expect(forecast.cells[0]!.compatibleSuccessful).toEqual({ samples: 20,
+ modelUsd: { mean: "10.500000000", median: "10.500000000", p95: "19.000000000" },
+ attemptMs: { mean: 10500, median: 10500, p95: 19000 } });
+ expect(forecast.selectionP95).toBeNull();
+ snapshot.samples.push({ ...sample, attemptId: "other-cohort", definitionHash: "a".repeat(64) });
+ const mixed = scheduledForecast([cell], models, snapshot);
+ expect(mixed.cells[0]!.historicalCohorts).toHaveLength(2);
+ expect(mixed.historicalSuccessfulSelection.modelUsd).toBeNull();
+ expect(mixed.cells[0]!.compatibleSuccessful.samples).toBe(20);
+});
+
+it("rejects duplicate artifacts and invalid measurement provenance", () => {
+ const snapshot = structuredClone(measurements);
+ snapshot.samples.push(snapshot.samples[0]!);
+ expect(() => scheduledForecast(selected("daily"), models, snapshot)).toThrow("Duplicate forecast attempt");
+ snapshot.samples.pop();
+ snapshot.samples[0]!.resultSha256 = "missing";
+ expect(() => scheduledForecast(selected("daily"), models, snapshot)).toThrow("provenance");
+});
+
+it("preserves nanodollar costs and genuine measured zero", () => {
+ const snapshot = structuredClone(measurements);
+ const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!;
+ const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!;
+ snapshot.samples = [{ ...sample, knownModelUsd: "0.000000001", attemptDurationMs: 0 }];
+ expect(scheduledForecast([cell], models, snapshot).compatibleSuccessfulSelection)
+ .toEqual({ modelUsd: "0.000000001", summedAttemptMs: 0 });
+ snapshot.samples[0]!.knownModelUsd = "0";
+ expect(scheduledForecast([cell], models, snapshot).compatibleSuccessfulSelection.modelUsd).toBe("0.000000000");
+});
diff --git a/tests/runner-e2e/scheduled-forecast.ts b/tests/runner-e2e/scheduled-forecast.ts
new file mode 100644
index 0000000000..7cc1acf23f
--- /dev/null
+++ b/tests/runner-e2e/scheduled-forecast.ts
@@ -0,0 +1,92 @@
+import { createHash } from "node:crypto";
+import { centsToUsd, unitsToCents, usdToUnits } from "../../packages/shared/src/money.js";
+import measurements from "./scheduled-measurements.json" with { type: "json" };
+import type { MatrixExecution } from "./types.js";
+import type { EvalOperationsManifest } from "./operations.js";
+
+type Sample = (typeof measurements.samples)[number];
+const cohortKey = (sample: Sample) => JSON.stringify([sample.definitionHash, sample.model, sample.environmentId,
+ sample.runtimeMode, sample.route, sample.priceVersion]);
+const usd = (units: bigint) => centsToUsd(unitsToCents(units));
+const average = (values: bigint[]) => values.length
+ ? (values.reduce((sum, value) => sum + value, 0n) + BigInt(Math.floor(values.length / 2))) / BigInt(values.length) : null;
+
+function statistics(values: bigint[]) {
+ const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
+ const middle = Math.floor(sorted.length / 2);
+ return {
+ count: sorted.length,
+ mean: average(sorted),
+ median: sorted.length ? average(sorted.length % 2 ? [sorted[middle]!] : [sorted[middle - 1]!, sorted[middle]!]) : null,
+ p95: sorted.length >= 20 ? sorted[Math.ceil(sorted.length * 0.95) - 1]! : null,
+ };
+}
+
+function summarize(samples: Sample[]) {
+ const cost = statistics(samples.map(sample => usdToUnits(sample.knownModelUsd)));
+ const time = statistics(samples.map(sample => BigInt(sample.attemptDurationMs)));
+ return {
+ samples: cost.count,
+ modelUsd: { mean: cost.mean === null ? null : usd(cost.mean), median: cost.median === null ? null : usd(cost.median),
+ p95: cost.p95 === null ? null : usd(cost.p95) },
+ attemptMs: { mean: time.mean === null ? null : Number(time.mean), median: time.median === null ? null : Number(time.median),
+ p95: time.p95 === null ? null : Number(time.p95) },
+ };
+}
+
+function total(rows: ReturnType<typeof summarize>[]) {
+ return {
+ modelUsd: rows.length && rows.every(row => row.modelUsd.mean !== null)
+ ? usd(rows.reduce((sum, row) => sum + usdToUnits(row.modelUsd.mean!), 0n)) : null,
+ summedAttemptMs: rows.length && rows.every(row => row.attemptMs.mean !== null)
+ ? rows.reduce((sum, row) => sum + row.attemptMs.mean!, 0) : null,
+ };
+}
+
+export function scheduledForecast(
+ executions: MatrixExecution[], models: EvalOperationsManifest["models"], snapshot = measurements,
+) {
+ const seen = new Set<string>();
+ for (const sample of snapshot.samples) {
+ if (seen.has(sample.attemptId)) throw new Error("Duplicate forecast attempt");
+ seen.add(sample.attemptId);
+ if (!Number.isSafeInteger(sample.attemptDurationMs) || sample.attemptDurationMs < 0 || usdToUnits(sample.knownModelUsd) < 0n
+ || !/^[0-9a-f]{40}$/.test(sample.sourceRevision)
+ || [sample.definitionHash, sample.resultSha256, sample.ledgerSha256, sample.ledgerFinalHash].some(hash => !/^[0-9a-f]{64}$/.test(hash))) {
+ throw new Error("Invalid forecast measurement or provenance");
+ }
+ }
+ if (new Set(executions.map(execution => execution.id)).size !== executions.length) throw new Error("Duplicate forecast execution");
+ const cells = executions.map(execution => {
+ const retained = snapshot.samples.filter(sample => sample.executionId === execution.id);
+ const cohorts = [...new Set(retained.map(cohortKey))].map(key => ({
+ key, ...summarize(retained.filter(sample => cohortKey(sample) === key)),
+ }));
+ const samples = retained.map(sample => {
+ const mismatches = [
+ sample.definitionHash === execution.suiteDefinitionHash ? null : "definition",
+ sample.model === execution.profile.model ? null : "model",
+ sample.environmentId === execution.environment.id ? null : "environment",
+ sample.runtimeMode === execution.profile.expectedRuntimeMode ? null : "runtime",
+ sample.route === "braintrust" ? null : "route",
+ models.some(model => model.price.model === sample.model && model.price.version === sample.priceVersion) ? null : "pricing",
+ ].filter(value => value !== null);
+ return { ...sample, mismatches };
+ });
+ return { executionId: execution.id, currentDefinitionHash: execution.suiteDefinitionHash,
+ historicalSuccessful: summarize(cohorts.length === 1 ? retained : []), historicalCohorts: cohorts,
+ compatibleSuccessful: summarize(samples.filter(sample => sample.mismatches.length === 0)), samples };
+ });
+ return {
+ measurementSnapshotSha256: createHash("sha256").update(JSON.stringify(snapshot)).digest("hex"),
+ measuredHost: snapshot.host, costStatus: snapshot.costStatus, sampleScope: snapshot.sampleScope,
+ historicalSuccessfulSelection: total(cells.map(cell => cell.historicalSuccessful)),
+ compatibleSuccessfulSelection: total(cells.map(cell => cell.compatibleSuccessful)),
+ hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null,
+ cells, limitations: [...snapshot.limitations,
+ "Compatibility compares definition, model, environment, runtime, route and price version; source revisions remain explicit",
+ "Per-cell P95 uses nearest rank only with at least 20 samples; it is descriptive, not a confidence bound",
+ "Summed attempt time is serial attempt work, not campaign wall time, summed agent time or billed runner time",
+ "Historical successful samples do not estimate failure frequency; every failure remains in campaign accounting"],
+ };
+}
diff --git a/tests/runner-e2e/scheduled-measurements.json b/tests/runner-e2e/scheduled-measurements.json
new file mode 100644
index 0000000000..afd9b43aaf
--- /dev/null
+++ b/tests/runner-e2e/scheduled-measurements.json
@@ -0,0 +1,84 @@
+{
+ "version": 1,
+ "recordedAt": "2026-10-21T18:13:01.055Z",
+ "costStatus": "estimated",
+ "sampleScope": "successful local native qualification attempts only",
+ "host": "linux-arm64-local-container",
+ "samples": [
+ {
+ "executionId": "feedback-persistence.runner-codex.local.complaint-and-suggestion",
+ "attemptId": "96850eb44e6a9f9856fcf24a40fb4903",
+ "sourceRevision": "276b8ff966c177b8cb34af12479915d9c8f2c9a2",
+ "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92",
+ "knownModelUsd": "0.189173400",
+ "attemptDurationMs": 26770,
+ "model": "gpt-5.6-sol",
+ "environmentId": "local",
+ "runtimeMode": "native",
+ "route": "braintrust",
+ "priceVersion": "openai-standard-2026-10-20",
+ "measuredAt": "2026-10-21T16:32:44.767Z",
+ "resultSha256": "e1748f875784065f2d40ebd7cc83cb9c3f84b1ce8dd05629482a97e818608188",
+ "ledgerSha256": "b5f42a64e8a72c5ae23c4dbf33fda1c561a7e2833d499e86b63627166144f1ef",
+ "ledgerFinalHash": "f06b687f8063926b77c0378946f1d6a9bfb718e1d8b9ca7d6e2aa344187be2dc"
+ },
+ {
+ "executionId": "core-compatibility.runner-acpx-claude.local.message-marker",
+ "attemptId": "8da10d462119f3cf6bb3a4669c43b6b5",
+ "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c",
+ "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa",
+ "knownModelUsd": "0.123679200",
+ "attemptDurationMs": 34495,
+ "model": "claude-sonnet-5",
+ "environmentId": "local",
+ "runtimeMode": "native",
+ "route": "braintrust",
+ "priceVersion": "anthropic-standard-2026-10-20",
+ "measuredAt": "2026-10-21T16:54:26.516Z",
+ "resultSha256": "89aac65b9a703d812a5a3c83c51732031e9b6fcadc6bac3c14a86c419f169d7f",
+ "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756",
+ "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e"
+ },
+ {
+ "executionId": "core-compatibility.runner-codex.local.message-marker",
+ "attemptId": "03720091468148fe7a0d2b1fa68d3966",
+ "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c",
+ "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa",
+ "knownModelUsd": "0.174741200",
+ "attemptDurationMs": 27542,
+ "model": "gpt-5.6-sol",
+ "environmentId": "local",
+ "runtimeMode": "native",
+ "route": "braintrust",
+ "priceVersion": "openai-standard-2026-10-20",
+ "measuredAt": "2026-10-21T16:53:20.764Z",
+ "resultSha256": "188a0655a7aebe37ec21c582e6b4afef1633210fa8f93e1049b9db0078657c4d",
+ "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756",
+ "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e"
+ },
+ {
+ "executionId": "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion",
+ "attemptId": "3849bc83e59ccddfa964120053d3428c",
+ "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c",
+ "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92",
+ "knownModelUsd": "0.303945600",
+ "attemptDurationMs": 38262,
+ "model": "claude-sonnet-5",
+ "environmentId": "local",
+ "runtimeMode": "native",
+ "route": "braintrust",
+ "priceVersion": "anthropic-standard-2026-10-20",
+ "measuredAt": "2026-10-21T16:52:21.240Z",
+ "resultSha256": "c086648b3dfc67a6d6ee04bb66a1914b3db69f9d40457198e2ce53c3a752ac24",
+ "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756",
+ "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e"
+ }
+ ],
+ "limitations": [
+ "One successful sample per cell; no measured slow-tail distribution or predicted failure rate",
+ "Gateway estimates include setup requests; historical phase labels may be inaccurate",
+ "Attempt time includes setup and teardown; it excludes queue, dependency installation and compilation",
+ "No hosted timing qualification or complete infrastructure, storage and subscription allocation",
+ "No change to the conservative per-request reservations or approved budgets"
+ ]
+}
diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.ts
new file mode 100644
index 0000000000..72ebe6d4d1
--- /dev/null
+++ b/tests/runner-e2e/scheduled-operations.test.ts
@@ -0,0 +1,77 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { prepareScheduledCampaign, scheduledCadence, scheduledManifest, scheduledMatrix, scheduledSelection, validateScheduledBudget } from "./scheduled-operations.js";
+import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
+import { runnerMatrix } from "./catalog.js";
+import { evalRequestReservation } from "./eval-gateway.js";
+import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js";
+
+const overhead: EvalOperatingOverhead = { version: 1, reference: "fixture-only", runner: "github-hosted",
+ verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",
+ components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) };
+
+const input = { cadence: "daily" as const, campaignId: "scheduled-fixture", ledgerPath: "/tmp/scheduled-fixture/ledger.jsonl",
+ projectId: "11111111-1111-4111-8111-111111111111", now: Date.parse("2026-10-21T00:00:00Z"), runAttempt: "1", overhead };
+
+it.each(["daily", "weekly"] as const)("runs the exact %s selection through one shared budget and catalog job", cadence => {
+ const manifest = scheduledManifest({ ...input, cadence });
+ const selected = selectRunnerExecutions(parseRunnerSelectors(manifest.executionIds.flatMap(id => ["--id", id])), runnerMatrix);
+ expect(selected.map(row => row.id).sort()).toEqual([...scheduledSelection(cadence)].sort());
+ expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxConcurrent: 1, maxAttemptsPerExecution: 1 });
+ expect(Date.parse(manifest.limits.deadlineAt) - input.now).toBe(30 * 60_000);
+ expect(scheduledMatrix(cadence).include).toHaveLength(1);
+ expect(manifest.executionIds).toHaveLength(cadence === "daily" ? 2 : 4);
+ expect(manifest.models.every(model => Number(evalRequestReservation({ ...model, mode: "braintrust", apiKey: "fixture" })) <= Number(manifest.limits.maxUsd))).toBe(true);
+});
+
+it("does not reduce Codex's verified context ceiling to fit the daily allowance", () => {
+ const codex = scheduledManifest({ ...input, cadence: "weekly" }).models.find(row => row.provider === "openai")!;
+ expect(codex.maximumInputTokens).toBe(1_050_000);
+ expect(Number(evalRequestReservation({ ...codex, mode: "braintrust", apiKey: "fixture" }))).toBeGreaterThan(20);
+ expect(scheduledManifest(input).models.map(model => model.provider)).toEqual(["anthropic"]);
+});
+
+it.each(["2", "0", "", "01"])("refuses workflow attempt %s before creating a fresh allowance", runAttempt => {
+ expect(() => scheduledManifest({ ...input, runAttempt })).toThrow("fresh allowance");
+});
+
+it.each(["2026-10-19T23:59:59Z", "2026-11-19T00:00:00Z"])("rejects pricing outside its reviewed window at %s", time => {
+ expect(() => scheduledManifest({ ...input, now: Date.parse(time) })).toThrow("pricing needs review");
+});
+
+it("rejects unknown triggers and maps the reviewed schedule expressions", () => {
+ expect(scheduledCadence("schedule", "47 8 * * *")).toBe("daily");
+ expect(scheduledCadence("schedule", "47 9 * * 0")).toBe("weekly");
+ expect(scheduledCadence("workflow_dispatch", "", "manual")).toBeUndefined();
+ expect(() => scheduledCadence("schedule", "* * * * *")).toThrow();
+ expect(() => scheduledCadence("pull_request", "", "daily")).toThrow();
+});
+
+it("fails closed before paid work if disabled, missing project identity, or preflight is repeated", async () => {
+ const directory = await mkdtemp(path.join(tmpdir(), "eval-schedule-"));
+ const env = { EVAL_EVENT: "schedule", EVAL_SCHEDULE: "47 8 * * *", EVAL_OPERATIONS_ENABLED: "true",
+ GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory,
+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(overhead), EVAL_TEST_RUNNER: "ubuntu-latest",
+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId };
+ try {
+ await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATIONS_ENABLED: "false" }, input.now)).rejects.toThrow("not enabled");
+ await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATING_OVERHEAD_JSON: "" }, input.now)).rejects.toThrow("Missing reviewed");
+ await expect(prepareScheduledCampaign({ ...env, PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "" }, input.now)).rejects.toThrow("UUID");
+ const prepared = await prepareScheduledCampaign(env, input.now);
+ expect(prepared.manifest.limits.campaignId).toBe(prepared.campaignId);
+ const forecast = JSON.parse(await readFile(path.join(path.dirname(prepared.manifestPath), "forecast.json"), "utf8"));
+ expect(forecast.measurements.historicalSuccessfulSelection).toEqual({ modelUsd: "0.427624800", summedAttemptMs: 72757 });
+ expect(forecast.measurements.hostedAttemptMs).toBeNull();
+ expect(forecast.maxUsd).toBe("20");
+ await expect(prepareScheduledCampaign(env, input.now)).rejects.toThrow("EEXIST");
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
+
+it("rejects an overhead allowance that cannot leave room for a full selected request before emitting the matrix", () => {
+ const expensive = structuredClone(overhead);
+ expensive.components[0]!.maxUsd = "13";
+ expect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance");
+ expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") });
+});
diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.ts
new file mode 100644
index 0000000000..84bb950e08
--- /dev/null
+++ b/tests/runner-e2e/scheduled-operations.ts
@@ -0,0 +1,121 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { runnerMatrix } from "./catalog.js";
+import { validateEvalOperationsManifest, type EvalOperationsManifest } from "./operations.js";
+import { evalRequestReservation } from "./eval-gateway.js";
+import prices from "./scheduled-prices.json" with { type: "json" };
+import { scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";
+import { usdToUnits } from "../../packages/shared/src/money.js";
+import { scheduledForecast } from "./scheduled-forecast.js";
+
+export type EvalCadence = "daily" | "weekly";
+export const EVAL_SCHEDULES = { daily: "47 8 * * *", weekly: "47 9 * * 0" } as const;
+const budget = (cadence: EvalCadence) => cadence === "daily" ? "20" : "100";
+const marker = (profile: string) => `core-compatibility.${profile}.local.message-marker`;
+const feedback = (profile: string) => `feedback-persistence.${profile}.local.complaint-and-suggestion`;
+
+export function scheduledSelection(cadence: EvalCadence) {
+ if (!["daily", "weekly"].includes(cadence)) throw new Error("Unknown eval cadence");
+ return [marker("runner-acpx-claude"), feedback("runner-acpx-claude"),
+ ...(cadence === "weekly" ? [marker("runner-codex"), feedback("runner-codex")] : [])];
+}
+
+export function scheduledCadence(event: string, schedule: string, requested = "manual"): EvalCadence | undefined {
+ if (event === "schedule") {
+ const found = Object.entries(EVAL_SCHEDULES).find(([, expression]) => expression === schedule);
+ if (!found) throw new Error("Unrecognized paid eval schedule");
+ return found[0] as EvalCadence;
+ }
+ if (event !== "workflow_dispatch" || !["manual", "daily", "weekly"].includes(requested)) throw new Error("Unrecognized eval trigger");
+ return requested === "manual" ? undefined : requested as EvalCadence;
+}
+
+export function validateScheduledBudget(input: { cadence: EvalCadence; now: number; overhead: EvalOperatingOverhead }) {
+ if (!Number.isFinite(input.now) || input.now < Date.parse(prices.verifiedAt) || input.now >= Date.parse(prices.expiresAt)) {
+ throw new Error("Scheduled eval pricing needs review before paid admission");
+ }
+ const executionIds = scheduledSelection(input.cadence);
+ const executions = executionIds.map(id => {
+ const execution = runnerMatrix.find(row => row.id === id);
+ if (!execution) throw new Error(`Scheduled execution is absent: ${id}`);
+ return execution;
+ });
+ const models = prices.models.filter(model => executions.some(execution => execution.profile.model === model.price.model)) as EvalOperationsManifest["models"];
+ if (executions.some(execution => !models.some(model => model.price.model === execution.profile.model
+ && model.provider === (execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai")))) {
+ throw new Error("Missing exact scheduled model pricing");
+ }
+ const reserved = validateOperatingOverhead(input.overhead, budget(input.cadence));
+ if (input.now < Date.parse(input.overhead.verifiedAt) || input.now + 30 * 60_000 >= Date.parse(input.overhead.expiresAt)) {
+ throw new Error("Operating overhead allowance does not cover the campaign window");
+ }
+ if (models.some(model => reserved + usdToUnits(evalRequestReservation({ ...model, apiKey: "forecast", mode: "braintrust" })) > usdToUnits(budget(input.cadence)))) {
+ throw new Error("Operating overhead leaves insufficient allowance for the selected model");
+ }
+ return { executionIds, executions, models };
+}
+
+export function scheduledManifest(input: {
+ cadence: EvalCadence; campaignId: string; ledgerPath: string; projectId: string; now: number; runAttempt: string;
+ overhead: EvalOperatingOverhead;
+}): EvalOperationsManifest {
+ if (input.runAttempt !== "1") throw new Error("Paid campaign reruns require the retained journal; a fresh allowance is refused");
+ const { executionIds, executions, models } = validateScheduledBudget(input);
+ const manifest: EvalOperationsManifest = {
+ version: 1, limits: { campaignId: input.campaignId, maxUsd: budget(input.cadence), overhead: input.overhead,
+ deadlineAt: new Date(input.now + 30 * 60_000).toISOString(), maxRequests: input.cadence === "daily" ? 60 : 120,
+ maxConcurrent: 1, maxAttemptsPerExecution: 1 },
+ ledgerPath: input.ledgerPath, executionIds, maxAttemptsPerExecution: 1, route: "braintrust", projectId: input.projectId,
+ models,
+ };
+ validateEvalOperationsManifest(manifest, executions, 0);
+ return manifest;
+}
+
+export function scheduledMatrix(cadence: EvalCadence) {
+ return { include: [{ executionId: "scheduled-campaign", suiteId: "core-compatibility", profileId: "runner-acpx-claude",
+ credentialName: "", environmentId: "local", caseId: "scheduled-campaign", timeoutMinutes: 60, needsDaytona: false }],
+ needsDaytona: false, executionIds: scheduledSelection(cadence) };
+}
+
+export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Date.now()) {
+ const cadence = scheduledCadence(env.EVAL_EVENT ?? "", env.EVAL_SCHEDULE ?? "", env.EVAL_CADENCE);
+ if (!cadence || env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled");
+ if (!/^[1-9][0-9]*$/.test(env.GITHUB_RUN_ID ?? "") || !env.GITHUB_WORKSPACE) throw new Error("Missing workflow campaign identity");
+ const campaignId = `gha-${env.GITHUB_RUN_ID}-1-scheduled-campaign`;
+ const directory = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results/scheduled");
+ const manifest = scheduledManifest({ cadence, campaignId, ledgerPath: path.join(directory, "ledger.jsonl"),
+ projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "", now, runAttempt: env.GITHUB_RUN_ATTEMPT ?? "",
+ overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now) });
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ const manifestPath = path.join(directory, "manifest.json");
+ await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" });
+ const forecast = { cadence, executionIds: manifest.executionIds, maxUsd: manifest.limits.maxUsd,
+ overhead: manifest.limits.overhead, budgetScope: "model requests plus declared operating overhead reservations",
+ measurements: scheduledForecast(runnerMatrix.filter(execution => manifest.executionIds.includes(execution.id)), manifest.models),
+ basis: "Retained local pilot measurements; successful samples and matching configurations are reported separately",
+ uncertainty: "Incomplete provider receipts and no invoice reconciliation; estimates are not a guarantee",
+ maximumRequestReservations: manifest.models.map(model => ({ model: model.price.model,
+ usd: evalRequestReservation({ ...model, apiKey: "forecast", mode: "braintrust" }) })),
+ maximumMonthlyTotalAllowanceUsd: "1120", pricingExpiresAt: prices.expiresAt };
+ await writeFile(path.join(directory, "forecast.json"), `${JSON.stringify(forecast, null, 2)}\n`, { mode: 0o600, flag: "wx" });
+ return { manifest, manifestPath, campaignId, cadence };
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const command = process.argv[2];
+ if (command === "matrix") {
+ const cadence = scheduledCadence(process.env.EVAL_EVENT ?? "", process.env.EVAL_SCHEDULE ?? "", process.env.EVAL_CADENCE);
+ if (!cadence || process.env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled");
+ const now = Date.now();
+ const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now);
+ validateScheduledBudget({ cadence, overhead, now });
+ console.log(JSON.stringify({ ...scheduledMatrix(cadence), overhead }));
+ } else if (command === "prepare") {
+ const prepared = await prepareScheduledCampaign(process.env);
+ if (!process.env.GITHUB_ENV) throw new Error("Missing workflow environment file");
+ const prior = await readFile(process.env.GITHUB_ENV, "utf8");
+ await writeFile(process.env.GITHUB_ENV, `${prior}PAPERCLIP_E2E_OPERATIONS_MANIFEST=${prepared.manifestPath}\nPAPERCLIP_E2E_CAMPAIGN_ID=${prepared.campaignId}\nPAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID=${prepared.campaignId}\n`);
+ } else throw new Error("Usage: scheduled-operations.ts <matrix|prepare>");
+}
diff --git a/tests/runner-e2e/scheduled-prices.json b/tests/runner-e2e/scheduled-prices.json
new file mode 100644
index 0000000000..c1d437be77
--- /dev/null
+++ b/tests/runner-e2e/scheduled-prices.json
@@ -0,0 +1,51 @@
+{
+ "version": 1,
+ "verifiedAt": "2026-10-20T00:00:00Z",
+ "expiresAt": "2026-11-19T00:00:00Z",
+ "models": [
+ {
+ "provider": "openai",
+ "maximumInputTokens": 1050000,
+ "maximumOutputTokens": 16384,
+ "timeoutMs": 180000,
+ "price": {
+ "provider": "openai",
+ "model": "gpt-5.6-sol",
+ "version": "openai-standard-2026-10-20",
+ "sourceUrl": "https://developers.openai.com/api/docs/pricing",
+ "inputUsdPerMillion": "4",
+ "cachedReadUsdPerMillion": "0.4",
+ "outputUsdPerMillion": "20",
+ "cacheWrite30mUsdPerMillion": "5",
+ "cacheWrite12hUsdPerMillion": "10",
+ "longContext": {
+ "aboveInputTokens": 272000,
+ "rates": {
+ "inputUsdPerMillion": "8",
+ "cachedReadUsdPerMillion": "0.8",
+ "outputUsdPerMillion": "30",
+ "cacheWrite30mUsdPerMillion": "10",
+ "cacheWrite12hUsdPerMillion": "20"
+ }
+ }
+ }
+ },
+ {
+ "provider": "anthropic",
+ "maximumInputTokens": 1000000,
+ "maximumOutputTokens": 16384,
+ "timeoutMs": 180000,
+ "price": {
+ "provider": "anthropic",
+ "model": "claude-sonnet-5",
+ "version": "anthropic-standard-2026-10-20",
+ "sourceUrl": "https://platform.claude.com/docs/en/models/sonnet-5/overview",
+ "inputUsdPerMillion": "2",
+ "cachedReadUsdPerMillion": "0.2",
+ "outputUsdPerMillion": "10",
+ "cacheWrite5mUsdPerMillion": "2.5",
+ "cacheWrite1hUsdPerMillion": "4"
+ }
+ }
+ ]
+}
diff --git a/tests/runner-e2e/select-rerun-artifacts.ts b/tests/runner-e2e/select-rerun-artifacts.ts
index 3745881532..9dab414180 100644
--- a/tests/runner-e2e/select-rerun-artifacts.ts
+++ b/tests/runner-e2e/select-rerun-artifacts.ts
@@ -1,4 +1,4 @@
-import { cp, mkdir, readFile, readdir } from "node:fs/promises";
+import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import type { RunnerE2EResult } from "./types.js";
@@ -307,6 +307,11 @@ export async function selectRerunArtifacts(input: SelectRerunArtifactsInput) {
});
selections.push({ executionId, workflowAttempt, artifactName });
}
+ await mkdir(input.selectedRoot, { recursive: true });
+ await writeFile(path.join(input.selectedRoot, "selected-artifacts.json"), JSON.stringify({
+ version: 1, workflowRunId: runId, workflowRunAttempt: currentAttempt,
+ sourceSha: input.sourceSha, sourceRef: input.sourceRef, workflowRunUrl: input.workflowRunUrl, selections,
+ }));
return selections;
}
diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts
index 05a82316c1..8956ac3f5a 100644
--- a/tests/runner-e2e/server.ts
+++ b/tests/runner-e2e/server.ts
@@ -1,5 +1,9 @@
import { parseFinalizationCrashBinding, readFinalizationCrashBoundary, type FinalizationCrashBinding } from "./finalization-crash-boundary.js";
import { parseRestartRunnerIdentity, type RestartRunnerIdentity } from "./process-tree-owner.js";
+import { recoverInterruptedFeedback } from "./feedback-recovery.js";
+import { normalizedSecrets } from "./redaction.js";
+import { CREDENTIAL_NAMES } from "./types.js";
+import { EVAL_OBSERVABILITY_SECRET_NAMES } from "./braintrust.js";
import { usesInstalledCli, verifyInstalledCli } from "./installed-cli.js";
import { runnerMatrix } from "./catalog.js";
import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server-stop.js";
@@ -117,6 +121,43 @@ let child: ChildProcess | null = null;
let unexpectedChildFailure: Error | null = null;
let shutdownSignal: NodeJS.Signals | null = null;
let activeRestartRequestId: string | null = null;
+let feedbackRecovery: Promise<void> | undefined;
+
+function recoverFeedbackBeforeShutdown() {
+ return feedbackRecovery ??= (async () => {
+ const privateDir = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR;
+ const campaignId = process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID;
+ const attemptId = process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID;
+ if (!privateDir || !campaignId || !attemptId || process.env.PAPERCLIP_RUNNER_E2E_PUBLIC_MCP === "1") return;
+ const deadlineAt = Date.now() + 5_000;
+ const secrets = normalizedSecrets([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES,
+ "PAPERCLIP_E2E_GATEWAY_TOKEN", "PAPERCLIP_AGENT_JWT_SECRET", "PAPERCLIP_DECISION_SIGNING_SECRET",
+ "PAPERCLIP_TOOL_ACTION_SIGNING_SECRET", "BETTER_AUTH_SECRET"].map(name => process.env[name]));
+ for (const executionId of executionIds) {
+ if (Date.now() >= deadlineAt) break;
+ const execution = runnerMatrix.find(row => row.id === executionId);
+ if (!execution) continue;
+ try {
+ await recoverInterruptedFeedback({ identity: { campaignId, attemptId, executionId },
+ caseDirectory: path.join(privateDir, "cases", execution.task.id), secrets, deadlineAt,
+ api: { async get<T>(endpoint: string, options?: { timeout: number }): Promise<T> {
+ const base = `http://127.0.0.1:${port}`;
+ const url = new URL(endpoint, base);
+ if (url.origin !== base || !url.pathname.startsWith("/api/")) throw new Error("Invalid feedback recovery endpoint");
+ const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(Math.max(1, Math.min(options?.timeout ?? 1_000, deadlineAt - Date.now()))) });
+ if (!response.ok) { await response.body?.cancel(); throw new Error("Feedback recovery read failed"); }
+ return response.json() as Promise<T>;
+ } },
+ });
+ } catch { appendLog("Feedback recovery checkpoint unavailable; retain private recovery state.\n"); }
+ }
+ })();
+}
+
+async function stopWithFeedback(candidate: ChildProcess, signal: NodeJS.Signals) {
+ await recoverFeedbackBeforeShutdown();
+ await stopServer(candidate, signal);
+}
function appendLog(message: string) {
process.stderr.write(message);
@@ -186,7 +227,7 @@ async function startServer() {
// that race create an unowned replacement server.
if (shutdownSignal) {
// The main/error path awaits this same promise and reports any failure.
- void stopServer(candidate, shutdownSignal).catch(() => {});
+ void stopWithFeedback(candidate, shutdownSignal).catch(() => {});
}
await stopServer.watch(candidate);
return candidate;
@@ -344,9 +385,7 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
if (shutdownSignal) return;
shutdownSignal = signal;
if (!child) return;
- // Begin immediately, including during a health/restart wait. The final
- // cleanup joins this promise instead of sending a second graceful signal.
- void stopServer(child, signal).catch(() => {});
+ void stopWithFeedback(child, signal).catch(() => {});
});
}
@@ -374,6 +413,7 @@ async function supervise() {
await delay(200);
}
+ await recoverFeedbackBeforeShutdown();
await stopServer.stopAll(shutdownSignal ?? "SIGTERM");
}
@@ -394,6 +434,7 @@ try {
}
}
try {
+ await recoverFeedbackBeforeShutdown();
await stopServer.stopAll();
} catch (stopError) {
appendLog(`Failed to stop Paperclip after supervisor failure: ${stopError instanceof Error ? stopError.message : String(stopError)}\n`);
diff --git a/tests/runner-e2e/trace-recovery.test.ts b/tests/runner-e2e/trace-recovery.test.ts
new file mode 100644
index 0000000000..da24f6f824
--- /dev/null
+++ b/tests/runner-e2e/trace-recovery.test.ts
@@ -0,0 +1,46 @@
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { expect, it, vi } from "vitest";
+import { EvalAttemptTrace, finalizeEvalAttemptTrace } from "./braintrust.js";
+import { readInterruptedAttemptTrace } from "./trace-recovery.js";
+
+it("finalizes a lost worker's original trace while rejecting unrelated or malformed snapshots", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-trace-recovery-"));
+ const projectId = "11111111-1111-4111-8111-111111111111";
+ const companyId = "22222222-2222-4222-8222-222222222222";
+ const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution", provider: "acpx", model: "claude", environment: "local" };
+ const config = { projectId, apiKey: "synthetic-canary" };
+ const trace = new EvalAttemptTrace(config, identity).snapshot();
+ const saved = { ...trace, companyId, nativeConfiguration: "configured", privateCanary: "must-not-copy", finalization: { status: "accepted" } };
+ const fixture = { companyId, executionId: identity.executionId };
+ const snapshot = path.join(root, "snapshots/braintrust-trace.json");
+ try {
+ await mkdir(path.join(root, "snapshots"));
+ await writeFile(path.join(root, "snapshots/fixtures.json"), JSON.stringify(fixture));
+ await writeFile(snapshot, JSON.stringify(saved));
+ const recovered = await readInterruptedAttemptTrace(root, { projectId, executionId: identity.executionId });
+ expect(recovered).toMatchObject({ traceId: trace.traceId, spanId: trace.spanId, status: "pending", verification: "unverified" });
+ expect(recovered).not.toHaveProperty("privateCanary");
+ expect(recovered).not.toHaveProperty("finalization");
+ const transport = vi.fn<typeof fetch>(async (_url, init) => {
+ const body = init!.body as Buffer;
+ expect(body.includes(Buffer.from(trace.traceId, "hex"))).toBe(true);
+ expect(body.includes(Buffer.from(trace.spanId, "hex"))).toBe(true);
+ expect(body.toString()).not.toContain("must-not-copy");
+ return new Response(null, { status: 200 });
+ });
+ expect(await finalizeEvalAttemptTrace(config, { receipt: recovered!, identity,
+ provenance: { suiteId: "suite", suiteDefinitionHash: "a".repeat(64), profileId: "profile", caseId: "case", sourceRevision: "b".repeat(40) },
+ result: { startedAt: new Date(Date.now() - 1000).toISOString(), status: "failed", cleanup: "failed", failureClass: "cleanup_failure" },
+ }, transport)).toMatchObject({ status: "accepted", outcome: "failed", verification: "unverified" });
+ expect(transport).toHaveBeenCalledOnce();
+ expect(JSON.parse(await readFile(snapshot, "utf8"))).toEqual(saved);
+ expect(await readInterruptedAttemptTrace(root, { projectId: companyId, executionId: identity.executionId })).toBeUndefined();
+ expect(await readInterruptedAttemptTrace(root, { projectId, executionId: "other" })).toBeUndefined();
+ for (const delta of [{ companyId: projectId }, { spanId: "0".repeat(16) }, { traceId: "invalid" }, { revision: "invalid" }, { nativeConfiguration: "unknown" }]) {
+ await writeFile(snapshot, JSON.stringify({ ...saved, ...delta }));
+ expect(await readInterruptedAttemptTrace(root, { projectId, executionId: identity.executionId })).toBeUndefined();
+ }
+ } finally { await rm(root, { recursive: true, force: true }); }
+});
diff --git a/tests/runner-e2e/trace-recovery.ts b/tests/runner-e2e/trace-recovery.ts
new file mode 100644
index 0000000000..2f9feef9ee
--- /dev/null
+++ b/tests/runner-e2e/trace-recovery.ts
@@ -0,0 +1,23 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { exportBraintrustAttemptParent, type EvalTraceReceipt } from "./braintrust.js";
+
+export async function readInterruptedAttemptTrace(caseDirectory: string, expected: {
+ projectId: string; executionId: string;
+}): Promise<EvalTraceReceipt | undefined> {
+ try {
+ const snapshots = path.join(caseDirectory, "snapshots");
+ const [fixture, trace] = await Promise.all(["fixtures.json", "braintrust-trace.json"].map(async name =>
+ JSON.parse(await readFile(path.join(snapshots, name), "utf8"))));
+ const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+ if (fixture.executionId !== expected.executionId || !uuid.test(fixture.companyId)
+ || trace.projectId !== expected.projectId || trace.companyId !== fixture.companyId
+ || trace.revision !== undefined && !uuid.test(trace.revision)
+ || !["pending", "accepted", "failed"].includes(trace.status)
+ || !["not_started", "configured", "failed"].includes(trace.nativeConfiguration)) return;
+ exportBraintrustAttemptParent(trace);
+ return { projectId: trace.projectId, traceId: trace.traceId, spanId: trace.spanId,
+ companyId: trace.companyId, ...(trace.revision ? { revision: trace.revision } : {}),
+ status: trace.status, nativeConfiguration: trace.nativeConfiguration, verification: "unverified" };
+ } catch { return; }
+}
diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts
index df280261e5..f54348ef35 100644
--- a/tests/runner-e2e/types.ts
+++ b/tests/runner-e2e/types.ts
@@ -244,7 +244,7 @@ export type RunnerE2ECostStatus =
export interface RunnerE2ERuntimeUsage {
provider: RunnerEnvironmentId;
/** Sum of the selected Paperclip heartbeat-run spans. */
- agentRunDurationMs: number;
+ agentRunDurationMs: number | null;
/** Sum of provider lease windows when the environment exposes leases. */
leaseDurationMs: number | null;
leaseCount: number;
@@ -267,15 +267,17 @@ export interface RunnerE2EBillingSummary {
runCount: number;
runsWithTokenUsage: number;
runsWithReportedCost: number;
+ runsWithEstimatedCost?: number;
+ runsWithPartialReportedCost?: number;
inputTokens: number;
outputTokens: number;
cachedInputTokens: number;
totalTokens: number;
reportedCostUsd: number;
- costStatus: Exclude<RunnerE2ECostStatus, "estimated" | "not_metered">;
+ estimatedCostUsd?: number;
+ costStatus: Exclude<RunnerE2ECostStatus, "not_metered">;
};
runtime: RunnerE2ERuntimeUsage;
- /** Provider-reported model spend only; never includes unknown/unpriced runs. */
reportedCostUsd: number;
/** Public-list-price estimate for metered execution infrastructure. */
estimatedRuntimeCostUsd: number;
@@ -288,6 +290,7 @@ export interface RunnerE2EBillingSummary {
export interface RunnerE2EResult {
schema: "paperclip.runner-e2e.result/v1" | "paperclip.runner-e2e.result/v2";
+ accountingIdentity?: { campaignId: string; attemptId: string };
executionId: string;
suiteId?: string;
suiteDefinitionHash?: string;
@@ -338,9 +341,12 @@ export interface RunnerE2EResult {
startedAt: string;
finishedAt: string;
durationMs: number;
+ workerTiming?: import("./worker-timing.js").WorkerTiming;
usage?: Record<string, unknown> | null;
runtimeUsage?: RunnerE2ERuntimeUsage;
billing?: RunnerE2EBillingSummary;
+ feedback?: import("./feedback.js").FeedbackSummary;
+ tracing?: import("./braintrust.js").EvalTraceReceipt;
matcherResults?: Array<{
matcher: Matcher;
passed: boolean;
@@ -393,15 +399,39 @@ export interface RunnerE2EAggregateBillingSummary {
assistant?: { requests: number; inputTokens: number; outputTokens: number; cachedInputTokens: number; estimatedCostUsd: number };
judge?: RunnerE2EJudgeBillingSummary;
testCount: number;
- agentRunDurationMs: number;
- leaseDurationMs: number;
+ agentRunDurationMs: number | null;
+ leaseDurationMs: number | null;
llm: RunnerE2EBillingSummary["llm"];
reportedLlmCostUsd: number;
+ estimatedLlmCostUsd?: number;
estimatedRuntimeCostUsd: number;
observedAndEstimatedCostUsd: number | null;
testsWithCompleteBilling: number;
}
+export interface RunnerE2EAttemptReceipt {
+ id: string;
+ executionId: string;
+ attempt: number;
+ startedAt: string;
+ finishedAt: string;
+ durationMs: number;
+ workerTiming?: import("./worker-timing.js").WorkerTiming;
+ status: RunnerE2EResult["status"];
+ cleanup: RunnerE2EResult["cleanup"];
+ billing: RunnerE2EBillingSummary;
+ feedback?: import("./feedback.js").FeedbackSummary;
+ tracing?: import("./braintrust.js").EvalTraceReceipt;
+}
+
+export interface RunnerE2ECampaignAccounting {
+ version: 1;
+ scope: "retained_attempts" | "selected_results";
+ attempts: RunnerE2EAttemptReceipt[];
+ missingAttempts: number;
+ excludedArtifacts: number;
+}
+
export interface RunnerE2ECampaign {
schema: "paperclip.runner-e2e.campaign/v2";
campaignId: string;
@@ -427,6 +457,10 @@ export interface RunnerE2ECampaign {
sourceUrl: string;
}>;
billing: RunnerE2EAggregateBillingSummary;
+ accounting?: RunnerE2ECampaignAccounting;
+ operations?: import("./operations-report.js").EvalOperationsReport;
+ feedbackReview?: import("./feedback-dispositions.js").CampaignFeedbackReview;
+ braintrustExperiment?: import("./braintrust-experiment.js").BraintrustExperimentReceipt;
suites: RunnerE2ESuiteSummary[];
results: RunnerE2EResult[];
}
diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts
index 7d0dcdbc0f..448f2df923 100644
--- a/tests/runner-e2e/vitest.config.ts
+++ b/tests/runner-e2e/vitest.config.ts
@@ -6,7 +6,7 @@ export default defineConfig({
name: "runner-e2e-support",
include: ["**/*.test.ts", "**/*.test.mjs"],
// These use node:test and run separately in test:e2e:runner:unit.
- exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs"],
+ exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs"],
testTimeout: 30_000,
},
});
diff --git a/tests/runner-e2e/web-server-command.ts b/tests/runner-e2e/web-server-command.ts
index 7735119d24..df8425695a 100644
--- a/tests/runner-e2e/web-server-command.ts
+++ b/tests/runner-e2e/web-server-command.ts
@@ -1,5 +1,7 @@
import path from "node:path";
+export const runnerE2EPlaywrightShutdownSignal = "SIGINT" as const;
+
// Let the restart-aware supervisor reap Paperclip and finish its log streams.
// Its stop path allows 30 seconds for SIGTERM and 5 seconds for SIGKILL.
export const runnerE2EWebServerGracefulShutdown = {
@@ -22,11 +24,6 @@ export function runnerE2ETypeScriptProcessArgs(repositoryRoot: string, entry: st
return ["--import", path.join(repositoryRoot, "cli/node_modules/tsx/dist/loader.mjs"), entry, ...args];
}
-// pnpm's generated playwright bin shim adds NODE_PATH, even when the caller
-// rejected ambient injection. The installed lane invokes the public JS bin
-// with the current Node directly, keeping the WebServer environment closed.
-export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[], installed: boolean) {
- return installed
- ? { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] }
- : { command: "pnpm", args: ["exec", "playwright", ...args] };
+export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[]) {
+ return { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] };
}
diff --git a/tests/runner-e2e/worker-timing.test.ts b/tests/runner-e2e/worker-timing.test.ts
new file mode 100644
index 0000000000..2a188ea582
--- /dev/null
+++ b/tests/runner-e2e/worker-timing.test.ts
@@ -0,0 +1,127 @@
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import { mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import os from "node:os";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { afterEach, describe, expect, it } from "vitest";
+import { buildCampaignAccounting } from "./attempt-accounting.js";
+import { summarizeWorkerPhaseTiming, workerPhaseTimingText } from "./campaign-timing.js";
+import { validateRetainedRunnerResult } from "./result-validation.js";
+import { createWorkerTiming, recoverWorkerTiming, validateWorkerTiming } from "./worker-timing.js";
+import type { RunnerE2EResult } from "./types.js";
+
+const temporary: string[] = [];
+afterEach(async () => { await Promise.all(temporary.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });
+async function fixture() {
+ const dir = await mkdtemp(path.join(os.tmpdir(), "worker-timing-")); temporary.push(dir);
+ const result: RunnerE2EResult = {
+ schema: "paperclip.runner-e2e.result/v2", executionId: "timing-case", suiteId: "timing-fixture", attempt: 1,
+ accountingIdentity: { campaignId: "timing-campaign", attemptId: "timing-attempt" },
+ status: "passed", profileId: "runner-codex", environmentId: "local", caseId: "marker", provider: "codex",
+ model: "fixture-model", runtimeMode: "native", startedAt: new Date(1000).toISOString(),
+ finishedAt: new Date(1500).toISOString(), durationMs: 500, cleanup: "passed",
+ };
+ return { file: path.join(dir, "snapshots", "worker-timing.json"), dir, result };
+}
+
+async function complete(file: string, result: RunnerE2EResult) {
+ const recorder = await createWorkerTiming(file, result, 1000);
+ await recorder.enter("execution", 1100);
+ await recorder.enter("cleanup", 1300);
+ await recorder.enter("feedback", 1350);
+ await recorder.enter("cleanup", 1430);
+ await recorder.enter("finalization", 1470);
+ return recorder.finish(1500);
+}
+
+describe("worker phase timing", () => {
+ it("retains and validates all phase boundaries, including both cleanup segments", async () => {
+ const { file, result } = await fixture(); result.workerTiming = await complete(file, result);
+ validateRetainedRunnerResult(result);
+ expect(await recoverWorkerTiming(file, result)).toEqual(result.workerTiming);
+ const accounting = buildCampaignAccounting({ results: [result], attempts: [result] });
+ expect(accounting.attempts[0]?.workerTiming).toEqual(result.workerTiming);
+ const summary = summarizeWorkerPhaseTiming([result], accounting);
+ expect(summary).toMatchObject({ completeAttempts: 1, partialAttempts: 0, missingAttempts: 0 });
+ expect(summary.phases.map(phase => [phase.phase, phase.totalMs])).toEqual([
+ ["setup", 100], ["execution", 200], ["feedback", 80], ["cleanup", 90], ["finalization", 30],
+ ]);
+ expect(workerPhaseTimingText(summary)).toContain("Worker cleanup time: 90ms.");
+ });
+
+ it("recovers only completed segments after a real worker process is killed", async () => {
+ const { file, result } = await fixture();
+ const loader = createRequire(new URL("../../cli/package.json", import.meta.url)).resolve("tsx");
+ const module = pathToFileURL(path.join(import.meta.dirname, "worker-timing.ts")).href;
+ const program = `import { createWorkerTiming } from ${JSON.stringify(module)};
+ const recorder = await createWorkerTiming(${JSON.stringify(file)}, { executionId: "timing-case", attempt: 1 }, 1000);
+ await recorder.enter("execution", 1200); process.stdout.write("checkpoint-ready"); setInterval(() => {}, 1000);`;
+ const child = spawn(process.execPath, ["--import", loader, "--input-type=module", "-e", program], { stdio: ["ignore", "pipe", "pipe"] });
+ const closed = once(child, "close");
+ try {
+ await new Promise<void>((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error("Worker checkpoint was not written")), 10_000);
+ child.once("error", error => { clearTimeout(timer); reject(error); });
+ child.stdout!.once("data", chunk => {
+ clearTimeout(timer); String(chunk).includes("checkpoint-ready") ? resolve() : reject(new Error("Unexpected worker output"));
+ });
+ child.once("exit", () => { clearTimeout(timer); reject(new Error("Worker exited before interruption")); });
+ });
+ child.kill("SIGKILL"); await closed;
+ result.status = "failed"; result.workerTiming = await recoverWorkerTiming(file, result);
+ expect(result.workerTiming).toMatchObject({ status: "partial", finishedAt: null,
+ segments: [{ phase: "setup", finishedAt: new Date(1200).toISOString() }, { phase: "execution", finishedAt: null }] });
+ validateRetainedRunnerResult(result);
+ const summary = summarizeWorkerPhaseTiming([result]);
+ expect(summary).toMatchObject({ completeAttempts: 0, partialAttempts: 1 });
+ expect(summary.phases.every(phase => phase.totalMs === null)).toBe(true);
+ expect(workerPhaseTimingText(summary)).toContain("Worker setup time: Unknown (200ms observed).");
+ expect(workerPhaseTimingText(summary)).toContain("Worker execution time: Unknown.");
+ } finally { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); await closed; }
+ });
+
+ it("rejects wrong identities, overlapping segments and out-of-window timestamps before reporting", async () => {
+ const { file, result } = await fixture(); const receipt = await complete(file, result);
+ if (receipt.status === "unavailable") throw new Error("Missing fixture timing");
+ const other = structuredClone(receipt); other.executionId = "another-case";
+ const overlap = structuredClone(receipt); overlap.segments[1]!.startedAt = new Date(1050).toISOString();
+ const future = structuredClone(receipt); future.finishedAt = new Date(1600).toISOString(); future.segments.at(-1)!.finishedAt = future.finishedAt;
+ for (const invalid of [other, overlap, future, { ...receipt, extra: "unapproved field" }]) {
+ expect(() => validateWorkerTiming(invalid, result)).toThrow("Invalid worker timing");
+ expect(() => validateRetainedRunnerResult({ ...result, workerTiming: invalid })).toThrow("result.workerTiming");
+ }
+ });
+
+ it("bounds recovery reads and refuses symlinks or mismatched attempt files", async () => {
+ const { file, dir, result } = await fixture(); await complete(file, result);
+ expect(await recoverWorkerTiming(file, { ...result, attempt: 2 })).toMatchObject({ status: "unavailable", reason: "invalid_evidence" });
+ const link = path.join(dir, "linked.json"); await symlink(file, link);
+ expect(await recoverWorkerTiming(link, result)).toMatchObject({ status: "unavailable", reason: "invalid_evidence" });
+ await writeFile(file, " ".repeat(8193));
+ expect(await recoverWorkerTiming(file, result)).toMatchObject({ status: "unavailable", reason: "invalid_evidence" });
+ expect(await recoverWorkerTiming(path.join(dir, "missing.json"), result)).toBeUndefined();
+ });
+
+ it("keeps timing failures from interrupting cleanup or overwriting earlier evidence", async () => {
+ const { file, result } = await fixture(); await complete(file, result); const original = await readFile(file, "utf8");
+ const recorder = await createWorkerTiming(file, result, 1000);
+ await recorder.enter("cleanup", 1100);
+ expect(await recorder.finish(1500)).toMatchObject({ status: "unavailable", reason: "write_failed" });
+ expect(await readFile(file, "utf8")).toBe(original);
+ const clockFile = file + ".clock"; const clock = await createWorkerTiming(clockFile, result, 1000);
+ await clock.enter("execution", 900);
+ expect(await clock.finish(1500)).toMatchObject({ status: "unavailable", reason: "clock_regression" });
+ });
+
+ it("preserves a timing observation through duplicate legacy copies and rejects contradictions", async () => {
+ const { file, result } = await fixture(); const measured = { ...result, workerTiming: await complete(file, result) };
+ for (const attempts of [[measured, result], [result, measured]]) {
+ expect(buildCampaignAccounting({ results: [measured], attempts }).attempts[0]?.workerTiming).toEqual(measured.workerTiming);
+ }
+ expect(() => buildCampaignAccounting({ results: [measured], attempts: [measured, { ...measured,
+ workerTiming: { version: 1, scope: "worker", status: "unavailable", reason: "write_failed" },
+ }] })).toThrow("Conflicting timing receipts");
+ });
+});
diff --git a/tests/runner-e2e/worker-timing.ts b/tests/runner-e2e/worker-timing.ts
new file mode 100644
index 0000000000..8bbe9abaec
--- /dev/null
+++ b/tests/runner-e2e/worker-timing.ts
@@ -0,0 +1,148 @@
+import { randomUUID } from "node:crypto";
+import { constants } from "node:fs";
+import { mkdir, open, rename, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+export const WORKER_TIMING_PHASES = ["setup", "execution", "feedback", "cleanup", "finalization"] as const;
+export type WorkerTimingPhase = typeof WORKER_TIMING_PHASES[number];
+type TimingContext = { executionId: string; attempt: number; startedAt: string; finishedAt: string };
+export type WorkerTiming = {
+ version: 1;
+ scope: "worker";
+ status: "complete" | "partial";
+ executionId: string;
+ attempt: number;
+ startedAt: string;
+ finishedAt: string | null;
+ segments: Array<{ phase: WorkerTimingPhase; startedAt: string; finishedAt: string | null }>;
+} | {
+ version: 1;
+ scope: "worker";
+ status: "unavailable";
+ reason: "write_failed" | "invalid_evidence" | "clock_regression";
+};
+
+function invalid(): never { throw new Error("Invalid worker timing receipt"); }
+function shape(value: unknown, keys: string[]): Record<string, unknown> {
+ if (!value || typeof value !== "object" || Array.isArray(value)) invalid();
+ const record = value as Record<string, unknown>;
+ if (Object.keys(record).length !== keys.length || keys.some(key => !Object.hasOwn(record, key))) invalid();
+ return record;
+}
+function timestamp(value: unknown) {
+ if (typeof value !== "string") invalid();
+ const ms = Date.parse(value);
+ if (!Number.isFinite(ms) || new Date(ms).toISOString() !== value) invalid();
+ return ms;
+}
+
+export function validateWorkerTiming(value: unknown, context: TimingContext): asserts value is WorkerTiming {
+ if (!value || typeof value !== "object" || Array.isArray(value)) invalid();
+ const status = (value as Record<string, unknown>).status;
+ const record = shape(value, status === "unavailable"
+ ? ["version", "scope", "status", "reason"]
+ : ["version", "scope", "status", "executionId", "attempt", "startedAt", "finishedAt", "segments"]);
+ if (record.version !== 1 || record.scope !== "worker") invalid();
+ if (status === "unavailable") {
+ if (typeof record.reason !== "string" || !["write_failed", "invalid_evidence", "clock_regression"].includes(record.reason)) invalid();
+ return;
+ }
+ if (status !== "complete" && status !== "partial" || record.executionId !== context.executionId || record.attempt !== context.attempt
+ || typeof record.executionId !== "string" || record.executionId.length < 1 || record.executionId.length > 512
+ || !Number.isSafeInteger(record.attempt) || Number(record.attempt) < 1) invalid();
+ const start = timestamp(record.startedAt), lower = Date.parse(context.startedAt), upper = Date.parse(context.finishedAt);
+ if (!Number.isFinite(lower) || !Number.isFinite(upper) || start < lower || start > upper) invalid();
+ if (!Array.isArray(record.segments) || record.segments.length < 1 || record.segments.length > 16) invalid();
+ let cursor = start;
+ record.segments.forEach((raw, index) => {
+ const segment = shape(raw, ["phase", "startedAt", "finishedAt"]);
+ if (!WORKER_TIMING_PHASES.includes(segment.phase as WorkerTimingPhase) || index === 0 && segment.phase !== "setup") invalid();
+ if (timestamp(segment.startedAt) !== cursor || cursor > upper) invalid();
+ if (segment.finishedAt === null) {
+ if (status !== "partial" || index !== (record.segments as unknown[]).length - 1) invalid();
+ } else {
+ const end = timestamp(segment.finishedAt);
+ if (end < cursor || end > upper || status === "partial" && index === (record.segments as unknown[]).length - 1) invalid();
+ cursor = end;
+ }
+ });
+ if (status === "partial" ? record.finishedAt !== null : timestamp(record.finishedAt) !== cursor) invalid();
+}
+
+function unavailable(reason: Extract<WorkerTiming, { status: "unavailable" }>["reason"]): WorkerTiming {
+ return { version: 1, scope: "worker", status: "unavailable", reason };
+}
+
+export async function createWorkerTiming(
+ file: string,
+ identity: Pick<TimingContext, "executionId" | "attempt">,
+ startedAtMs: number,
+) {
+ let receipt: WorkerTiming = {
+ version: 1, scope: "worker", status: "partial", executionId: identity.executionId, attempt: identity.attempt,
+ startedAt: new Date(startedAtMs).toISOString(), finishedAt: null,
+ segments: [{ phase: "setup", startedAt: new Date(startedAtMs).toISOString(), finishedAt: null }],
+ };
+ async function persist(initial = false) {
+ if (receipt.status === "unavailable") return;
+ const temporary = `${file}.${randomUUID()}.pending`;
+ try {
+ const data = JSON.stringify(receipt);
+ if (initial) {
+ await mkdir(path.dirname(file), { recursive: true });
+ await writeFile(file, data, { mode: 0o600, flag: "wx" });
+ }
+ else {
+ await writeFile(temporary, data, { mode: 0o600, flag: "wx" });
+ await rename(temporary, file);
+ }
+ } catch { receipt = unavailable("write_failed"); }
+ finally { await rm(temporary, { force: true }).catch(() => undefined); }
+ }
+ function closeCurrent(at: number) {
+ if (receipt.status === "unavailable" || receipt.status === "complete") return false;
+ const active = receipt.segments.at(-1)!;
+ if (!Number.isSafeInteger(at) || !Number.isFinite(new Date(at).getTime()) || at < Date.parse(active.startedAt)) {
+ receipt = unavailable("clock_regression");
+ return false;
+ }
+ active.finishedAt = new Date(at).toISOString();
+ return true;
+ }
+ await persist(true);
+ return {
+ async enter(phase: WorkerTimingPhase, at = Date.now()) {
+ if (!closeCurrent(at) || receipt.status === "unavailable") return;
+ if (receipt.segments.length >= 16) { receipt = unavailable("invalid_evidence"); return; }
+ receipt.segments.push({ phase, startedAt: new Date(at).toISOString(), finishedAt: null });
+ await persist();
+ },
+ async finish(at = Date.now()): Promise<WorkerTiming> {
+ if (closeCurrent(at) && receipt.status !== "unavailable") {
+ receipt.status = "complete";
+ receipt.finishedAt = new Date(at).toISOString();
+ await persist();
+ }
+ return structuredClone(receipt);
+ },
+ };
+}
+
+export async function recoverWorkerTiming(file: string, context: TimingContext): Promise<WorkerTiming | undefined> {
+ try {
+ if (typeof constants.O_NOFOLLOW !== "number" || typeof constants.O_NONBLOCK !== "number") return unavailable("invalid_evidence");
+ const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+ try {
+ const stat = await handle.stat();
+ if (!stat.isFile() || stat.size > 8192) return unavailable("invalid_evidence");
+ const buffer = Buffer.alloc(8193);
+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+ if (bytesRead > 8192) return unavailable("invalid_evidence");
+ const receipt: unknown = JSON.parse(buffer.subarray(0, bytesRead).toString("utf8"));
+ validateWorkerTiming(receipt, context);
+ return receipt;
+ } finally { await handle.close(); }
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code === "ENOENT" ? undefined : unavailable("invalid_evidence");
+ }
+}
diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts
index b89cc7c3b1..e6c8b1c16e 100644
--- a/tests/runner-e2e/workflow-security.test.ts
+++ b/tests/runner-e2e/workflow-security.test.ts
@@ -93,7 +93,7 @@ describe("public repository paid workflow security", () => {
const workflows = [
{
name: "runner-full-stack-e2e.yml",
- expectedCachedSetupNodeSteps: 4,
+ expectedCachedSetupNodeSteps: 3,
},
{
name: "pr-trusted.yml",
@@ -311,7 +311,7 @@ describe("public repository paid workflow security", () => {
const everydayOraclePreparation = paidJob.indexOf(
"- name: Prepare pinned Python artifact oracle image",
);
- const paidExecution = paidJob.indexOf("- name: Run paid cell");
+ const paidExecution = paidJob.indexOf("- name: Run bounded campaign and retain feedback review");
expect(paidInstall).toBeGreaterThan(0);
expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall);
expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation);
@@ -588,7 +588,7 @@ describe("public repository paid workflow security", () => {
}
});
- it("runs paid scheduled campaigns only on Sundays", async () => {
+ it("uses bounded daily and weekly Product schedules while preserving Runner schedules", async () => {
const workflows = await Promise.all(
[
"runner-full-stack-e2e.yml",
@@ -598,12 +598,20 @@ describe("public repository paid workflow security", () => {
readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),
),
);
- for (const workflow of workflows) {
- const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(
- (match) => match[1]!,
- );
- expect(crons).toHaveLength(1);
- expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);
+ for (const [index, workflow] of workflows.entries()) {
+ const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(match => match[1]!);
+ if (index === 0) {
+ expect(crons).toEqual(["47 8 * * *", "47 9 * * 0"]);
+ expect(workflow).toContain("vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'");
+ expect(workflow).toContain('test "$TARGET_SHA" = "$GITHUB_SHA"');
+ expect(workflow).toContain('echo "max_parallel=1"');
+ expect(workflow).toContain("--max-automatic-retries 0");
+ const paid = workflow.split("- name: Run bounded campaign and retain feedback review")[1]!.split("- name: Run paid cell")[0]!;
+ expect([...paid.matchAll(/secrets\.([A-Z_]+)/g)].map(match => match[1])).toEqual(["BRAINTRUST_API_KEY"]);
+ } else {
+ expect(crons).toHaveLength(1);
+ expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);
+ }
expect(workflow).toContain("workflow_dispatch:");
}
});
@@ -720,18 +728,18 @@ describe("public repository paid workflow security", () => {
);
});
- it("resolves each trusted reporting lockfile before frozen install and AWS credentials", async () => {
+ it("keeps reporting jobs on the committed lockfile without registry re-resolution", async () => {
const workflow = await readFile(
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8",
);
for (const jobName of ["report", "publish_history"]) {
const job = workflow.split(`\n ${jobName}:`)[1]!.split(/\n [a-z_]+:/u)[0]!;
- const resolve = job.indexOf("pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile");
- const install = job.indexOf("pnpm install --frozen-lockfile");
+ const install = job.indexOf("pnpm install --frozen-lockfile --ignore-scripts");
expect(job).toContain("ref: ${{ github.sha }}");
expect(job).not.toContain("resolved-target-lockfile");
- expect(resolve, jobName).toBeGreaterThan(-1);
- expect(install, jobName).toBeGreaterThan(resolve);
+ expect(job, jobName).not.toContain("--no-frozen-lockfile");
+ expect(job, jobName).not.toContain("--lockfile-only");
+ expect(install, jobName).toBeGreaterThan(job.indexOf("ref: ${{ github.sha }}"));
const credentials = job.indexOf("aws-actions/configure-aws-credentials@");
if (credentials >= 0) expect(install).toBeLessThan(credentials);
}
@@ -767,11 +775,12 @@ describe("public repository paid workflow security", () => {
"PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e",
);
expect(report).toContain(
- "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}",
- );
- expect(report).toContain(
- "PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}",
+ "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}",
);
+ const historyPrefix = "${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}";
+ expect(report).toContain(`PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`);
+ expect(publisher).toContain(`RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`);
+ expect(publisher).toContain("needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'");
expect(
report.indexOf("Select latest workflow attempt per cell"),
).toBeLessThan(report.indexOf("Collect blob reports"));
@@ -789,6 +798,64 @@ describe("public repository paid workflow security", () => {
);
});
+ it("keeps recurring reports private while preserving a content-free liveness receipt", async () => {
+ const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
+ const paid = workflow.split("\n test:")[1]!.split("\n report:")[0]!;
+ const manual = paid.split("\n test_recurring:")[0]!;
+ const recurring = paid.split("\n test_recurring:")[1]!;
+ expect(manual).not.toContain("id-token: write");
+ expect(manual).toContain("if: needs.catalog.outputs.operations_cadence == ''");
+ expect(recurring).toContain("id-token: write");
+ expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha");
+ expect(recurring).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)");
+ expect(recurring).toContain("steps: *paid_test_steps");
+ expect(manual).toContain("steps: &paid_test_steps");
+ const report = workflow.split("\n report:")[1]!.split("\n publish_history:")[0]!;
+ const step = (source: string, name: string) => source.split(` - name: ${name}\n`)[1]!.split(/\n - /u)[0]!;
+ for (const [job, name] of [[paid, "Upload access-controlled packaged cell evidence"], [report, "Upload access-controlled merged report"]]) {
+ expect(step(job!, name!)).toContain("if: always() && needs.catalog.outputs.operations_cadence == ''");
+ }
+ expect(step(report, "Download cell evidence")).toContain("if: needs.catalog.outputs.operations_cadence == ''");
+ expect(step(paid, "Run bounded campaign and retain feedback review")).not.toContain("AWS_");
+ for (const job of [paid, report]) {
+ const credentials = step(job, "Obtain scoped private history credentials");
+ expect(credentials).toContain("output-env-credentials: false");
+ expect(credentials).toContain("disable-retry: true");
+ expect(credentials).toContain("RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN");
+ }
+ expect(paid.indexOf("Verify private evidence retention before paid execution")).toBeLessThan(paid.indexOf("Run bounded campaign and retain feedback review"));
+ expect(report).toContain("report > runner-e2e-merged-report/report.log 2>&1");
+ const marker = step(report, "Retain report availability for the schedule monitor");
+ expect(marker).toContain("steps.private_report.outcome == 'success'");
+ expect(marker).toContain("path: private-report-status.json");
+ expect(report).toContain('echo \'{"version":1,"privateReportRetained":true}\' > private-report-status.json');
+ expect(marker).not.toMatch(/BUCKET|ROLE_ARN|private-report-reference/);
+ });
+
+ it("isolates private archive trust from workflows that execute target-branch code", async () => {
+ const workflows = path.join(repositoryRoot, ".github/workflows");
+ const source = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8");
+ const trust = await readFile(path.join(repositoryRoot, ".github/infra/runner-e2e-private-history.yml"), "utf8");
+ const environments = ["runner-e2e-private-paid", "runner-e2e-private-history"];
+ const subjects = [...trust.matchAll(/!Sub 'repo:\$\{GitHubRepository\}:environment:([^']+)'/gu)].map(match => match[1]);
+ expect(subjects).toEqual(environments);
+ const recurring = source.split("\n test_recurring:")[1]!.split("\n report:")[0]!;
+ const report = source.split("\n report:")[1]!.split("\n publish_history:")[0]!;
+ for (const [index, job] of [recurring, report].entries()) {
+ expect(job).toContain(`name: ${environments[index]}`);
+ expect(job).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)");
+ }
+ expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha");
+ for (const filename of await readdir(workflows)) {
+ if (!/\.ya?ml$/u.test(filename) || filename === "runner-full-stack-e2e.yml") continue;
+ const other = await readFile(path.join(workflows, filename), "utf8");
+ for (const environment of environments) expect(other, filename).not.toContain(environment);
+ }
+ const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8");
+ expect(protocol).toContain("name: runner-e2e-paid");
+ expect(protocol).toContain("needs.authorize.outputs.target_sha");
+ });
+
it("uses environment-scoped OIDC for a no-delete history publisher", async () => {
const workflow = await readFile(
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
@@ -827,4 +894,26 @@ describe("public repository paid workflow security", () => {
workflow.indexOf("pnpm test:e2e:runner:history:publish"),
).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@"));
});
+
+ it("does not restore dependency caches or run install scripts in OIDC-authorized jobs", async () => {
+ const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
+ const jobs = workflow.split(/\n(?= [a-z_]+:)/u);
+ const paidSteps = workflow.split("steps: &paid_test_steps")[1]!.split("\n test_recurring:")[0]!;
+ const oidcJobs = jobs.filter(job => job.includes("id-token: write"));
+ expect(oidcJobs.length).toBeGreaterThanOrEqual(3);
+ for (const job of oidcJobs) {
+ const steps = job + (job.includes("steps: *paid_test_steps") ? paidSteps : "");
+ expect(steps, job.split("\n")[0]).not.toMatch(/cache:\s*(?:pnpm|npm|yarn)\b/u);
+ expect(steps).not.toMatch(/actions\/cache(?:\/restore)?@/u);
+ for (const install of steps.matchAll(/\bpnpm install[^\n]*/gu)) {
+ expect(install[0]).toContain("--ignore-scripts");
+ }
+ }
+ for (const name of ["report", "publish_history"]) {
+ const job = jobs.find(value => value.startsWith(` ${name}:`))!;
+ const installs = [...job.matchAll(/\bpnpm install[^\n]*/gu)];
+ expect(installs).toHaveLength(1);
+ for (const install of installs) expect(install[0]).toContain('--store-dir "${{ runner.temp }}/eval-');
+ }
+ });
});
diff --git a/tests/runner-e2e/workflow-target.test.ts b/tests/runner-e2e/workflow-target.test.ts
new file mode 100644
index 0000000000..88aa5b6cb5
--- /dev/null
+++ b/tests/runner-e2e/workflow-target.test.ts
@@ -0,0 +1,49 @@
+import { spawnSync } from "node:child_process";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+
+const queuedSha = "a".repeat(40);
+const advancedSha = "b".repeat(40);
+
+async function resolveTarget(event: string, cadence: string, branch = "master") {
+ const workflow = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8");
+ const step = workflow.split(" - name: Resolve requested repository branch to an immutable commit\n")[1]!.split("\n - name:")[0]!;
+ const script = step.split(" run: |\n")[1]!.replace(/^ {10}/gm, "");
+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-workflow-target-"));
+ try {
+ const output = path.join(directory, "output");
+ const result = spawnSync("bash", ["-c", `gh() { printf '%s\\n' "$MOCK_BRANCH_SHA"; }\n${script}`], {
+ encoding: "utf8", env: { ...process.env, EVENT_NAME: event, EVAL_CADENCE: cadence,
+ TARGET_BRANCH: branch, DEFAULT_BRANCH: "master", GITHUB_SHA: queuedSha,
+ MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip" },
+ });
+ return { status: result.status, stderr: result.stderr,
+ output: await readFile(output, "utf8").catch(() => "") };
+ } finally { await rm(directory, { recursive: true, force: true }); }
+}
+
+describe("paid workflow target authorization", () => {
+ it.each([["schedule", "manual"], ["workflow_dispatch", "daily"], ["workflow_dispatch", "weekly"]])(
+ "pins %s/%s to the queued revision when the branch advances", async (event, cadence) => {
+ const result = await resolveTarget(event, cadence);
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.output).toContain(`sha=${queuedSha}\n`);
+ expect(result.output).not.toContain(advancedSha);
+ });
+
+ it("rejects custom bounded targets before emitting a commit", async () => {
+ const result = await resolveTarget("workflow_dispatch", "daily", "development");
+ expect(result.status).toBe(1);
+ expect(result.output).toBe("");
+ expect(result.stderr).toContain("default-branch commit");
+ });
+
+ it("still resolves explicit manual targets at authorization time", async () => {
+ const result = await resolveTarget("workflow_dispatch", "manual", "development");
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.output).toContain(`sha=${advancedSha}\n`);
+ expect(result.output).toContain("ref=refs/heads/development\n");
+ });
+});
diff --git a/ui/src/pages/Cases.test.tsx b/ui/src/pages/Cases.test.tsx
index c952f558d1..613314a8cd 100644
--- a/ui/src/pages/Cases.test.tsx
+++ b/ui/src/pages/Cases.test.tsx
@@ -278,6 +278,7 @@ describe("Read-only cases list", () => {
})));
expect(container.querySelector('[data-testid="location"]')?.textContent).toBe("/PAP/cases?sort=title&dir=asc");
await escape();
+ await waitFor(() => expect(document.activeElement).toBe(labelledButton("Sort")));
await click(labelledButton("Group"));
const type = [...document.body.querySelectorAll('[aria-label="Group cases"] button')]
.find(element => element.textContent === "Type")!;