The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Operationalize Product E2E accounting, tracing, and scheduling (#17009)
Dotta3bd709df8b83
diff --git a/.github/infra/runner-e2e-private-history.yml b/.github/infra/runner-e2e-private-history.ymlnew file mode 100644index 0000000000..0d837d2479--- /dev/null+++ b/.github/infra/runner-e2e-private-history.yml@@ -0,0 +1,117 @@+AWSTemplateFormatVersion: '2010-09-09'+Description: Private bounded Paperclip eval evidence and reports; no CDN reader+Parameters:+ BucketName:+ Type: String+ GitHubRepository:+ Type: String+ Default: paperclipai/paperclip+ GitHubOidcProviderArn:+ Type: String+Resources:+ PrivateHistoryBucket:+ Type: AWS::S3::Bucket+ DeletionPolicy: Retain+ UpdateReplacePolicy: Retain+ Properties:+ BucketName: !Ref BucketName+ PublicAccessBlockConfiguration:+ BlockPublicAcls: true+ IgnorePublicAcls: true+ BlockPublicPolicy: true+ RestrictPublicBuckets: true+ OwnershipControls:+ Rules:+ - ObjectOwnership: BucketOwnerEnforced+ VersioningConfiguration:+ Status: Enabled+ BucketEncryption:+ ServerSideEncryptionConfiguration:+ - ServerSideEncryptionByDefault:+ SSEAlgorithm: AES256+ LifecycleConfiguration:+ Rules:+ - Id: BoundedEvalRetentionV1+ Status: Enabled+ Prefix: runner-e2e-private/bounded-v1/+ ExpirationInDays: 30+ NoncurrentVersionExpiration:+ NoncurrentDays: 1+ AbortIncompleteMultipartUpload:+ DaysAfterInitiation: 1+ - Id: BoundedEvalDeleteMarkersV1+ Status: Enabled+ Prefix: runner-e2e-private/bounded-v1/+ ExpiredObjectDeleteMarker: true+ PrivateHistoryBucketPolicy:+ Type: AWS::S3::BucketPolicy+ Properties:+ Bucket: !Ref PrivateHistoryBucket+ PolicyDocument:+ Version: '2012-10-17'+ Statement:+ - Sid: DenyInsecureTransport+ Effect: Deny+ Principal: '*'+ Action: s3:*+ Resource:+ - !GetAtt PrivateHistoryBucket.Arn+ - !Sub '${PrivateHistoryBucket.Arn}/*'+ Condition:+ Bool:+ aws:SecureTransport: 'false'+ - Sid: DenyUnconditionalWrites+ Effect: Deny+ Principal: '*'+ Action: s3:PutObject+ Resource: !Sub '${PrivateHistoryBucket.Arn}/runner-e2e-private/bounded-v1/*'+ Condition:+ 'Null':+ s3:if-none-match: 'true'+ PrivateHistoryRole:+ Type: AWS::IAM::Role+ Properties:+ RoleName: paperclip-runner-e2e-private-history-github+ MaxSessionDuration: 3600+ AssumeRolePolicyDocument:+ Version: '2012-10-17'+ Statement:+ - Effect: Allow+ Principal:+ Federated: !Ref GitHubOidcProviderArn+ Action: sts:AssumeRoleWithWebIdentity+ Condition:+ StringEquals:+ token.actions.githubusercontent.com:aud: sts.amazonaws.com+ token.actions.githubusercontent.com:sub:+ - !Sub 'repo:${GitHubRepository}:environment:runner-e2e-private-paid'+ - !Sub 'repo:${GitHubRepository}:environment:runner-e2e-private-history'+ Policies:+ - PolicyName: private-eval-artifacts+ PolicyDocument:+ Version: '2012-10-17'+ Statement:+ - Sid: VerifyPrivateRetention+ Effect: Allow+ Action:+ - s3:GetBucketVersioning+ - s3:GetLifecycleConfiguration+ - s3:GetBucketObjectLockConfiguration+ - s3:GetReplicationConfiguration+ - s3:GetBucketPublicAccessBlock+ - s3:GetBucketOwnershipControls+ - s3:GetBucketPolicy+ Resource: !GetAtt PrivateHistoryBucket.Arn+ - Sid: RetainAndReadPrivateArtifacts+ Effect: Allow+ Action:+ - s3:GetObject+ - s3:PutObject+ Resource: !Sub '${PrivateHistoryBucket.Arn}/runner-e2e-private/bounded-v1/*'+Outputs:+ Bucket:+ Value: !Ref PrivateHistoryBucket+ RoleArn:+ Value: !GetAtt PrivateHistoryRole.Arn+ Region:+ Value: !Ref AWS::Regiondiff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.ymlindex c15a34085f..a04db993e7 100644--- a/.github/workflows/pr-trusted.yml+++ b/.github/workflows/pr-trusted.yml@@ -119,7 +119,7 @@ jobs:run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs- name: Test release verify workflow wiring- run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs+ run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs- name: Test standalone package build concurrencyrun: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjsdiff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.ymlindex 65eebf32ed..99cdaf5232 100644--- a/.github/workflows/runner-full-stack-e2e.yml+++ b/.github/workflows/runner-full-stack-e2e.yml@@ -1,10 +1,17 @@name: Runner Full-Stack E2E+run-name: Runner Full-Stack E2E / ${{ github.event.schedule || inputs.cadence || 'manual' }}on:schedule:- - cron: "47 8 * * 0"+ - cron: "47 8 * * *"+ - cron: "47 9 * * 0"workflow_dispatch:inputs:+ cadence:+ description: "Bounded daily ($20) or weekly ($100) selection; manual keeps explicit selectors"+ type: choice+ options: [manual, daily, weekly]+ default: manualtarget_branch:description: "Branch in paperclipai/paperclip to test; the trusted workflow still runs from master"type: string@@ -55,7 +62,7 @@ concurrency:jobs:authorize:name: Authorize paid campaign- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')runs-on: ubuntu-latesttimeout-minutes: 5permissions:@@ -115,14 +122,25 @@ jobs:GH_TOKEN: ${{ github.token }}REPOSITORY: ${{ github.repository }}TARGET_BRANCH: ${{ inputs.target_branch || github.event.repository.default_branch }}+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}+ EVENT_NAME: ${{ github.event_name }}+ EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}run: |set -euo pipefailif [ -z "$TARGET_BRANCH" ] || [[ "$TARGET_BRANCH" == refs/* ]]; thenecho "target_branch must name a branch in this repository without a refs/ prefix." >&2exit 1fi- encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')"- target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)"+ if [ "$EVENT_NAME" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then+ if [ "$TARGET_BRANCH" != "$DEFAULT_BRANCH" ]; then+ echo "Bounded campaigns require the workflow's default-branch commit." >&2+ exit 1+ fi+ target_sha="$GITHUB_SHA"+ else+ encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')"+ target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)"+ fiif ! [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]]; thenecho "The requested repository branch did not resolve to a commit." >&2exit 1@@ -135,12 +153,23 @@ jobs:id: runnerenv:AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}+ EVENT_NAME: ${{ github.event_name }}+ EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}+ REPOSITORY_PRIVATE: ${{ github.event.repository.private }}run: |set -euo pipefailgithub_runner='ubuntu-latest'aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'+ bounded=false+ if [ "$EVENT_NAME" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then+ if [ "$REPOSITORY_PRIVATE" != false ]; then+ echo 'Bounded campaigns require a public repository and standard GitHub-hosted compute.' >&2+ exit 1+ fi+ bounded=true+ fi- if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then+ if [ "$AWS_PAID_RUNNER_ENABLED" = true ] && [ "$bounded" = false ]; then{echo "runner=$aws_runner"echo "max_parallel_default=100"@@ -155,7 +184,7 @@ jobs:echo "max_parallel_limit=57"echo "playwright_channel="} >> "$GITHUB_OUTPUT"- echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner'+ echo '::notice title=Paid runner routing::Using a standard GitHub-hosted runner'fitarget_lock:@@ -200,6 +229,10 @@ jobs:fiecho "sha256=$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT"+ - name: Bound scheduled lockfile upload+ if: github.event_name == 'schedule' || (inputs.cadence != '' && inputs.cadence != 'manual')+ run: node tests/runner-e2e/artifact-budget.mjs check lock pnpm-lock.yaml+- name: Upload resolved target lockfileid: uploaduses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7@@ -212,13 +245,15 @@ jobs:catalog:name: Validate catalog and select cellsneeds: [authorize, target_lock]- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')runs-on: ubuntu-latesttimeout-minutes: 10permissions:contents: readoutputs:matrix: ${{ steps.catalog.outputs.matrix }}+ operations_cadence: ${{ steps.catalog.outputs.operations_cadence }}+ operating_overhead: ${{ steps.catalog.outputs.operating_overhead }}needs_daytona: ${{ steps.catalog.outputs.needs_daytona }}needs_runner_typescript: ${{ steps.catalog.outputs.needs_runner_typescript }}needs_native_binaries: ${{ steps.catalog.outputs.needs_native_binaries }}@@ -281,6 +316,13 @@ jobs:id: catalogenv:EVENT_NAME: ${{ github.event_name }}+ EVAL_EVENT: ${{ github.event_name }}+ EVAL_SCHEDULE: ${{ github.event.schedule }}+ EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}+ EVAL_OPERATIONS_ENABLED: ${{ vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED }}+ EVAL_OPERATING_OVERHEAD_JSON: ${{ vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON }}+ EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }}+ TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}SELECT_ALL: ${{ inputs.all }}SELECT_SUITE: ${{ inputs.suite }}SELECT_GROUP: ${{ inputs.group }}@@ -293,6 +335,31 @@ jobs:REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}run: |set -euo pipefail+ if [ "$EVAL_EVENT" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then+ test "$TARGET_SHA" = "$GITHUB_SHA"+ test -z "${SELECT_SUITE:-}${SELECT_GROUP:-}${SELECT_PROFILE:-}${SELECT_ENVIRONMENT:-}${SELECT_CASE:-}${SELECT_ID:-}"+ catalog_json="$(node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-operations.ts matrix)"+ cadence="$EVAL_CADENCE"+ if [ "$EVAL_EVENT" = schedule ]; then+ case "$EVAL_SCHEDULE" in+ '47 8 * * *') cadence=daily ;;+ '47 9 * * 0') cadence=weekly ;;+ *) exit 1 ;;+ esac+ fi+ {+ echo "operations_cadence=$cadence"+ echo "operating_overhead=$(jq -c '.overhead' <<< "$catalog_json")"+ echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")"+ echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")"+ echo "needs_daytona=false"+ echo "needs_runner_typescript=true"+ echo "needs_native_binaries=true"+ echo "needs_remote_provider_pack=false"+ echo "max_parallel=1"+ } >> "$GITHUB_OUTPUT"+ exit 0+ fiargs=(--matrix-json)add_values() {local flag="$1"@@ -509,7 +576,7 @@ jobs:build_runner_artifacts:name: Build reusable runner campaign artifactsneeds: [authorize, target_lock, catalog]- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')# Compile native binaries on the same reviewed image used to execute them,# avoiding libc/architecture drift between GitHub-hosted and AWS lanes.runs-on: ${{ needs.authorize.outputs.test_runner }}@@ -618,6 +685,10 @@ jobs:TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}run: echo "name=runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"+ - name: Bound scheduled build upload+ if: needs.catalog.outputs.operations_cadence != ''+ run: node tests/runner-e2e/artifact-budget.mjs check build runner-e2e-build-bundle.tar.gz runner-e2e-build-bundle.tar.gz.sha256+- name: Upload immutable shared campaign outputsuses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7with:@@ -633,7 +704,7 @@ jobs:name: Build reusable remote provider packneeds:[authorize, target_lock, catalog, daytona_image, build_runner_artifacts]- if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'+ if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')runs-on: ubuntu-latesttimeout-minutes: 15permissions:@@ -769,6 +840,7 @@ jobs:test:name: ${{ matrix.executionId }}+ if: needs.catalog.outputs.operations_cadence == ''needs:[authorize,@@ -790,7 +862,7 @@ jobs:fail-fast: falsemax-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }}matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }}- steps:+ steps: &paid_test_steps- name: Reauthorize paid execution before provider accessenv:GH_TOKEN: ${{ github.token }}@@ -962,8 +1034,8 @@ jobs:}NODE- - name: Install pinned legacy Claude CLI- if: matrix.profileId == 'legacy-claude'+ - name: Install pinned Claude CLI for setup verification+ if: matrix.profileId == 'legacy-claude' || matrix.profileId == 'runner-acpx-claude'run: |npm install --global --omit=dev --ignore-scripts @anthropic-ai/[email protected]launcher="$(npm root --global)/@anthropic-ai/claude-code/cli-wrapper.cjs"@@ -1012,7 +1084,7 @@ jobs:# This definition executes only from the authorized default-branch workflow.# Provision host policy before credentials reach target-controlled tests.- name: Provision Codex sandbox on the disposable trusted runner- if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex' || matrix.profileId == 'runner-codex-subscription')+ if: matrix.environmentId == 'local' && (needs.catalog.outputs.operations_cadence == 'weekly' || matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex' || matrix.profileId == 'runner-codex-subscription')run: |node --input-type=module <<'NODE'import { execFileSync } from "node:child_process";@@ -1052,7 +1124,60 @@ jobs:timeout 120s docker pull "$oracle_image"timeout 30s docker image inspect "$oracle_image" --format '{{.Id}}'+ - name: Obtain scoped private history credentials+ id: private_history_credentials+ if: needs.catalog.outputs.operations_cadence != ''+ uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6+ with:+ role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}+ aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}+ output-env-credentials: false+ output-credentials: true+ disable-retry: true+ action-timeout-s: 60++ - name: Verify private evidence retention before paid execution+ if: needs.catalog.outputs.operations_cadence != ''+ env:+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts verify evidence "$GITHUB_RUN_ID" bounded-runner-e2e-evidence++ - name: Prepare bounded campaign before credentials+ if: needs.catalog.outputs.operations_cadence != ''+ env:+ EVAL_EVENT: ${{ github.event_name }}+ EVAL_SCHEDULE: ${{ github.event.schedule }}+ EVAL_CADENCE: ${{ needs.catalog.outputs.operations_cadence }}+ EVAL_OPERATIONS_ENABLED: ${{ vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED }}+ EVAL_OPERATING_OVERHEAD_JSON: ${{ needs.catalog.outputs.operating_overhead }}+ EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }}+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: ${{ vars.RUNNER_E2E_BRAINTRUST_PROJECT_ID }}+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-operations.ts prepare++ - name: Run bounded campaign and retain feedback review+ if: needs.catalog.outputs.operations_cadence != ''+ env:+ BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: ${{ vars.RUNNER_E2E_BRAINTRUST_PROJECT_ID }}+ PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}+ PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}+ PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}+ run: |+ set +e+ args=(--max-parallel 1 --max-automatic-retries 0)+ while IFS= read -r id; do args+=(--id "$id"); done < <(jq -r '.executionIds[]' "$PAPERCLIP_E2E_OPERATIONS_MANIFEST")+ pnpm test:e2e:runner -- "${args[@]}"+ run_status=$?+ node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-finalization.ts+ finalization_status=$?+ test "$run_status" = 0 && test "$finalization_status" = 0+- name: Run paid cell+ if: needs.catalog.outputs.operations_cadence == ''env:OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }}ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }}@@ -1063,13 +1188,19 @@ jobs:PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }}PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-packPAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }}+ PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}"+ - name: Stage bounded campaign evidence+ id: stage_evidence+ if: always() && needs.catalog.outputs.operations_cadence != ''+ run: node tests/runner-e2e/artifact-budget.mjs stage evidence tests/runner-e2e/results bounded-runner-e2e-evidence+- name: Upload access-controlled packaged cell evidence- if: always()+ if: always() && needs.catalog.outputs.operations_cadence == ''uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7with:name: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }}@@ -1077,10 +1208,41 @@ jobs:retention-days: 30if-no-files-found: error+ - name: Retain bounded campaign evidence in private AWS storage+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_history_credentials.outcome == 'success'+ env:+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts upload evidence "$GITHUB_RUN_ID" bounded-runner-e2e-evidence++ - name: Enforce complete bounded evidence retention+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_evidence.outputs.complete != 'true'+ run: exit 1++ test_recurring:+ name: ${{ matrix.executionId }}+ if: needs.catalog.outputs.operations_cadence != '' && needs.authorize.outputs.target_sha == github.sha && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)+ needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack]+ runs-on: ${{ needs.authorize.outputs.test_runner }}+ timeout-minutes: ${{ matrix.timeoutMinutes }}+ permissions:+ contents: read+ id-token: write+ environment:+ name: runner-e2e-private-paid+ strategy:+ fail-fast: false+ max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }}+ matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }}+ steps: *paid_test_steps+report:name: Merge and enforce campaign result- if: always() && !cancelled() && needs.catalog.result == 'success'- needs: [authorize, catalog, daytona_image, test]+ if: always() && !cancelled() && needs.catalog.result == 'success' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)+ needs: [authorize, catalog, daytona_image, test, test_recurring]outputs:history_source_ready: ${{ steps.history_source_ready.outputs.ready }}runs-on: ubuntu-latest@@ -1088,6 +1250,9 @@ jobs:permissions:actions: readcontents: read+ id-token: write+ environment:+ name: runner-e2e-private-historysteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7with:@@ -1107,15 +1272,8 @@ jobs:with:version: 9.15.4- - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7- with:- node-version: 24- cache: pnpm-- # Resolve from this trusted checkout only. Never use the target branch's- # lockfile or install scripts in the publication job.- - run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile- - run: pnpm install --frozen-lockfile+ - name: Install pinned reporting dependencies+ run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-report-store"- name: Resolve workflow job attemptsenv:@@ -1138,6 +1296,7 @@ jobs:- name: Download cell evidenceid: download_evidence+ if: needs.catalog.outputs.operations_cadence == ''continue-on-error: trueuses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8with:@@ -1146,15 +1305,37 @@ jobs:merge-multiple: false- name: Retry cell evidence download after transport failure- if: steps.download_evidence.outcome == 'failure'+ if: needs.catalog.outputs.operations_cadence == '' && steps.download_evidence.outcome == 'failure'uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8with:pattern: runner-e2e-${{ github.run_id }}-*-*path: downloaded-runner-e2emerge-multiple: false+ - name: Obtain scoped private history credentials+ id: private_history_credentials+ if: needs.catalog.outputs.operations_cadence != ''+ uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6+ with:+ role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}+ aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}+ output-env-credentials: false+ output-credentials: true+ disable-retry: true+ action-timeout-s: 60++ - name: Download private bounded campaign evidence+ if: needs.catalog.outputs.operations_cadence != ''+ env:+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts download evidence "$GITHUB_RUN_ID" "downloaded-runner-e2e/runner-e2e-$GITHUB_RUN_ID-1-scheduled-campaign"+- name: Select latest workflow attempt per cell- if: always()+ if: always() && needs.catalog.outputs.operations_cadence == ''env:PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT: ${{ github.workspace }}/downloaded-runner-e2ePAPERCLIP_RUNNER_E2E_SELECTED_ROOT: ${{ github.workspace }}/selected-runner-e2e@@ -1164,6 +1345,25 @@ jobs:PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/select-rerun-artifacts.ts+ - name: Preserve bounded campaign evidence and journal+ if: always() && needs.catalog.outputs.operations_cadence != ''+ env:+ RUN_ID: ${{ github.run_id }}+ run: |+ set -euo pipefail+ artifact="downloaded-runner-e2e/runner-e2e-$RUN_ID-1-scheduled-campaign"+ directory="$GITHUB_WORKSPACE/selected-runner-e2e/runner-e2e-$RUN_ID-1-scheduled-campaign/scheduled"+ echo "PAPERCLIP_E2E_OPERATIONS_LEDGER=$directory/ledger.jsonl" >> "$GITHUB_ENV"+ test -d "$artifact"+ mkdir -p selected-runner-e2e+ cp -R "$artifact" selected-runner-e2e/+ if [ -f "$directory/feedback-review.json" ]; then+ echo "PAPERCLIP_E2E_FEEDBACK_REVIEW=$directory/feedback-review.json" >> "$GITHUB_ENV"+ fi+ if [ -f "$directory/experiment-receipt.json" ]; then+ echo "PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT=$directory/experiment-receipt.json" >> "$GITHUB_ENV"+ fi+- name: Collect blob reportsrun: |set -euo pipefail@@ -1188,21 +1388,34 @@ jobs:PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2ePAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalizedPAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }}- PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}+ PAPERCLIP_E2E_CAMPAIGN_ID: ${{ needs.catalog.outputs.operations_cadence != '' && format('gha-{0}-1-scheduled-campaign', github.run_id) || format('gha-{0}-{1}', github.run_id, github.run_attempt) }}PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}- PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}+ PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}+ EVAL_OPERATIONS_CADENCE: ${{ needs.catalog.outputs.operations_cadence }}run: |set +e- pnpm test:e2e:runner:report+ if [ -n "$EVAL_OPERATIONS_CADENCE" ]; then+ mkdir -p runner-e2e-merged-report+ pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1+ else+ pnpm test:e2e:runner:report+ fireport_status=$?set -e- cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY"+ if [ -z "$EVAL_OPERATIONS_CADENCE" ]; then+ cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY"+ fiexit "$report_status"+ - name: Stage bounded merged report+ id: stage_report+ if: always() && needs.catalog.outputs.operations_cadence != ''+ run: node tests/runner-e2e/artifact-budget.mjs stage report runner-e2e-merged-report bounded-runner-e2e-report+- name: Upload access-controlled merged report- if: always()+ if: always() && needs.catalog.outputs.operations_cadence == ''uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7with:name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}@@ -1210,6 +1423,33 @@ jobs:retention-days: 30if-no-files-found: error+ - name: Retain bounded report in private AWS storage+ id: private_report+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_history_credentials.outcome == 'success'+ env:+ AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }}+ AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }}+ AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }}+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}+ run: |+ node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts upload report "$GITHUB_RUN_ID" bounded-runner-e2e-report+ echo '{"version":1,"privateReportRetained":true}' > private-report-status.json+ echo 'The recurring campaign report is retained in private AWS storage. See the eval operations runbook for authenticated retrieval.' >> "$GITHUB_STEP_SUMMARY"++ - name: Retain report availability for the schedule monitor+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_report.outcome == 'success'+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7+ with:+ name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}+ path: private-report-status.json+ retention-days: 30+ if-no-files-found: error++ - name: Enforce complete bounded report retention+ if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_report.outputs.complete != 'true'+ run: exit 1+- name: Verify normalized history source reportid: history_source_readyif: always()@@ -1225,7 +1465,7 @@ jobs:publish_history:name: Publish S3 history and Pages bundle with declared screenshotsneeds: [authorize, catalog, report]- if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true'+ if: always() && needs.catalog.result == 'success' && needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'runs-on: ubuntu-latesttimeout-minutes: 15outputs:@@ -1257,12 +1497,10 @@ jobs:with:version: 9.15.4- # Resolve the trusted publication checkout's lockfile without lifecycle- # scripts, then install the exact result. This matches the report job's- # frozen-install preparation while keeping target-controlled code out of- # the AWS credentialed publisher.- - run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile- - run: pnpm install --frozen-lockfile+ # Use a job-local store and disable lifecycle scripts throughout install.+ # OIDC is available to every step, even before the AWS credential exchange.+ - name: Install pinned publication dependencies+ run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-publisher-store"- name: Install publisher-only Chromiumrun: pnpm exec playwright install --with-deps --only-shell chromiumdiff --git a/doc/agent-observability.md b/doc/agent-observability.mdindex 2932330f2d..2968ee6f50 100644--- a/doc/agent-observability.md+++ b/doc/agent-observability.md@@ -28,9 +28,13 @@ requires the qualified executable version. Codex ACPX writes its isolated home.The legacy ACPX adapter keeps export configuration in a private home per agentand settings compatibility key, and shares conversation files with its stable managed home;export changes do not change its logical conversation fingerprint.-Codex's bundled Rust OTLP exporter percent-decodes header values, so launch-material encodes them once to preserve literal percent sequences. Credentials-remain literal values in the company secret; environment interpolation is not used.+Codex receives authentication through `OTEL_EXPORTER_OTLP_LOGS_HEADERS` and+`OTEL_EXPORTER_OTLP_TRACES_HEADERS` in its process environment. Its managed+configuration and invocation profiles contain empty header maps, so exporting+does not save the ingestion credential in those files. The bundled Rust OTLP+exporter percent-decodes environment header values; launch material encodes+them once to preserve literal percent sequences. Credentials remain literal+values in the company secret; config-file environment interpolation is not used.A caller-owned/preconfigured Codex home cannot be automatically configured inthis version: its files remain untouched and execution emits a diagnostic.Other harnesses, personal Codex desktop sessions and hosted claude_managed are@@ -69,8 +73,8 @@ entered secrets. Revisions prevent another administrator's edit being silentlyoverwritten. Credential values are excluded from task environment manifests,run/config snapshots, argv, activity records, recovery configuration and querycache data. Export credentials are necessarily accessible to the exporting-harness itself. Claude Code supplies these headers through its process environment,-which its tools can inherit or inspect. Codex can read its managed configuration.+harness itself. Both Codex and Claude Code receive these headers through their+process environment, which their tools can inherit or inspect.This feature does not isolate export credentials from agent code. Use a credentiallimited to telemetry ingestion for the selected destination; do not supply a broadorganization administration key. A user-operated collector can hold upstreamdiff --git a/doc/evals.md b/doc/evals.mdindex 985d358469..9d89aa297e 100644--- a/doc/evals.md+++ b/doc/evals.md@@ -511,3 +511,8 @@ covers projectless task files, deferred repository acquisition, controller restaand nested Git/dirty-file persistence on legacy local, native local and nativeDaytona OpenCode. Its three configured cells use independent filesystem, publicAPI and download oracles; no live qualification is implied by fixture registration.++The explicit-only [feedback persistence case](../tests/runner-e2e/README.md#feedback-persistence-explicit-only)+checks real native complaint/suggestion submissions and retained bodies with+run-attributed tool receipts. It is manual, uses the normal Product E2E report,+and keeps synthetic feedback distinct from actionable product findings.diff --git a/packages/adapter-utils/src/agent-observability.test.ts b/packages/adapter-utils/src/agent-observability.test.tsindex cb85c54eef..22fb580541 100644--- a/packages/adapter-utils/src/agent-observability.test.ts+++ b/packages/adapter-utils/src/agent-observability.test.ts@@ -23,13 +23,23 @@ it("encodes headers losslessly and clears ambient exporter conflicts without chaexpect(observabilitySecretValues(launch)).toContain(input.headers.authorization);expect(JSON.stringify(buildInvocationEnvForLogs({ ...launch.environment }))).not.toContain("a%2Cb");});-it("preserves literal percent sequences through the qualified Codex exporter's header decoder", () => {+it("keeps Codex headers in the launch environment and preserves literal percent sequences", () => {const original = "Bearer a,b=%26 c%";const launch = buildProviderObservabilityLaunch({ ...input, kind: "codex", headers: { authorization: original } });const exporter = launch.otelConfig!.exporter as { "otlp-http": { headers: Record<string, string> } };- expect(decodeURIComponent(exporter["otlp-http"].headers.authorization)).toBe(original);+ expect(exporter["otlp-http"].headers).toEqual({});+ for (const key of ["OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"]) {+ expect(launch.environment[key]).toBe(`authorization=${encodeURIComponent(original)}`);+ }+ expect(JSON.stringify(launch.otelConfig)).not.toContain(original);+ expect(JSON.stringify(launch.otelConfig)).not.toContain(encodeURIComponent(original));expect(observabilitySecretValues(launch)).toContain(original);+ expect(observabilitySecretValues(launch)).toContain(encodeURIComponent(original));expect(observabilitySecretValues(launch)).toContain(original.slice(7));+ expect(JSON.stringify(buildInvocationEnvForLogs({ ...launch.environment }))).not.toContain("a%2Cb");+ const disabled = buildProviderObservabilityLaunch({ ...input, kind: "codex", enabled: false });+ expect(disabled.environment).not.toHaveProperty("OTEL_EXPORTER_OTLP_LOGS_HEADERS");+ expect(disabled.environment).not.toHaveProperty("OTEL_EXPORTER_OTLP_TRACES_HEADERS");});it("replaces a compact explicit profile without leaving competing profile flags", () => {const args = ["exec", "-preview", "-"];@@ -52,6 +62,8 @@ it("creates invocation-owned profiles, preserves selected profile settings, andconst a = await prepareCodexObservabilityProfile({ home: root, runId: "run-a", sourceProfile: "review", launch });const b = await prepareCodexObservabilityProfile({ home: root, runId: "run-b", launch });const parsed = parse(await readFile(join(root, a.filename), "utf8"));+ expect(JSON.stringify(parsed)).not.toContain(input.headers.authorization);+ expect(JSON.stringify(parsed)).not.toContain(encodeURIComponent(input.headers.authorization));expect(parsed.model).toBe("chosen-model");expect(parsed.otel).toMatchObject({ log_agent_responses: true, tool_result: { max_bytes: 65536 } });expect((await stat(join(root, a.filename))).mode & 0o777).toBe(0o600);diff --git a/packages/adapter-utils/src/agent-observability.ts b/packages/adapter-utils/src/agent-observability.tsindex aab46b2445..051e7cb2c7 100644--- a/packages/adapter-utils/src/agent-observability.ts+++ b/packages/adapter-utils/src/agent-observability.ts@@ -35,7 +35,10 @@ export function observabilitySecretValues(launch: ProviderObservabilityLaunch |const headers = ((launch.otelConfig?.exporter as Record<string, unknown> | undefined)?.["otlp-http"] as { headers?: Record<string, string> } | undefined)?.headers;const encoded = launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS;const decoded = (value: string) => { try { return decodeURIComponent(value); } catch { return value; } };- const values = [...Object.values(headers ?? {}).flatMap(value => [value, decoded(value)]), ...(encoded ? [encoded, ...encoded.split(",").map(item => decoded(item.slice(item.indexOf("=") + 1)))] : [])];+ const values = [...Object.values(headers ?? {}).flatMap(value => [value, decoded(value)]), ...(encoded ? [encoded, ...encoded.split(",").flatMap(item => {+ const value = item.slice(item.indexOf("=") + 1);+ return [value, decoded(value)];+ })] : [])];return [...new Set(values.flatMap(value => [value, value.replace(/^Bearer /i, ""), JSON.stringify(value).slice(1, -1)]).filter(Boolean))];}@@ -50,10 +53,11 @@ export function buildProviderObservabilityLaunch(input: {OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000", OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "3000",};if (kind === "codex") {- // Codex passes these through opentelemetry-otlp 0.31's with_headers, which- // percent-decodes values. Encode once so literal percent sequences survive.- const headers = Object.fromEntries(Object.entries(input.headers ?? {}).map(([name, value]) => [name, encodeURIComponent(value)]));- const exporter = (endpoint: string | undefined) => enabled ? { "otlp-http": { endpoint, protocol: "binary", headers } } : "none";+ if (enabled) Object.assign(environment, {+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: encodeObservabilityHeaders(input.headers ?? {}),+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: encodeObservabilityHeaders(input.headers ?? {}),+ });+ const exporter = (endpoint: string | undefined) => enabled ? { "otlp-http": { endpoint, protocol: "binary", headers: {} } } : "none";return { kind, compatibilityKey, environment, otelConfig: {exporter: exporter(input.logsEndpoint), trace_exporter: exporter(input.tracesEndpoint), metrics_exporter: "none",log_user_prompt: enabled && capture.userPrompts, log_agent_responses: enabled && capture.assistantResponses,diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rsindex 8f4e4dc9c1..43edd3e019 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs@@ -302,6 +302,8 @@ impl AcpxSidecarTransport {"PAPERCLIP_OBSERVABILITY_CODEX_CONFIG","OTEL_EXPORTER_OTLP_LOGS_TIMEOUT","OTEL_EXPORTER_OTLP_TRACES_TIMEOUT",+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS",]);}let pi_keys = if agent == "pi" {diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rsindex 07e5cf8145..e5c77b954b 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs@@ -651,6 +651,25 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {let require_codex_home_auth = args.iter().any(|value| value == "--require-codex-home-auth");+ if args+ .iter()+ .any(|value| value == "--require-otel-environment")+ {+ for (name, expected) in [+ (+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",+ "authorization=Bearer%20logs-fixture",+ ),+ (+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS",+ "authorization=Bearer%20traces-fixture",+ ),+ ] {+ assert_eq!(std::env::var(name).as_deref(), Ok(expected));+ }+ assert!(std::env::var_os("OTEL_EXPORTER_OTLP_HEADERS").is_none());+ assert!(std::env::var_os("OTEL_EXPORTER_OTLP_ENDPOINT").is_none());+ }let durable_turn_ids = args.iter().any(|value| value == "--durable-turn-ids");let hold_first_durable_turn = args.iter()diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rsindex b6f7675c5e..70aa673509 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs@@ -823,6 +823,8 @@ const GITHUB_CREDENTIAL_ENVIRONMENT_KEYS: &[&str] = &[const CODEX_PROVIDER_ENVIRONMENT_KEYS: &[&str] = &["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT","OTEL_EXPORTER_OTLP_TRACES_TIMEOUT",+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS","PAPERCLIP_AI_PROVIDER_KEY","PAPERCLIP_AGENT_KEY_ID","PAPERCLIP_AGENT_PUBLIC_KEY",diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rsindex 07c7912635..5d8ccc10ab 100644--- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs@@ -106,6 +106,19 @@ fn provider_receives_the_isolated_codex_auth_home() {.env("PAPERCLIP_CODEX_AUTH_TEST_HOME", &directory).env("HOME", &directory).env("CODEX_HOME", &directory)+ .env(+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS",+ "authorization=Bearer%20logs-fixture",+ )+ .env(+ "OTEL_EXPORTER_OTLP_TRACES_HEADERS",+ "authorization=Bearer%20traces-fixture",+ )+ .env(+ "OTEL_EXPORTER_OTLP_HEADERS",+ "authorization=unwanted-ambient",+ )+ .env("OTEL_EXPORTER_OTLP_ENDPOINT", "https://unwanted.example").status().expect("run isolated auth-home assertion");assert!(status.success());@@ -119,7 +132,10 @@ fn provider_receives_isolated_codex_auth_home_subprocess() {else {return;};- let config = provider_config(&directory, &["--require-codex-home-auth"]);+ let config = provider_config(+ &directory,+ &["--require-codex-home-auth", "--require-otel-environment"],+ );let mut provider = CodexProvider::start(&config, None).expect("Codex provider should read auth from the isolated CODEX_HOME");provider.shutdown().expect("stop fake Codex provider");diff --git a/packages/paperclip-runner/scripts/build-inputs.test.mjs b/packages/paperclip-runner/scripts/build-inputs.test.mjsindex ce650a871d..7822de2cbb 100644--- a/packages/paperclip-runner/scripts/build-inputs.test.mjs+++ b/packages/paperclip-runner/scripts/build-inputs.test.mjs@@ -491,26 +491,34 @@ test("two chains run by separate shells at the same time both record", { skip: pawait withRepo(async ({ repoRoot, packageRoot }) => {const pendingDirectory = buildInputsPendingDirectory(packageRoot);const marker = path.join(repoRoot, "release-the-chains");- const records = () => (existsSync(pendingDirectory) ? readdirSync(pendingDirectory).sort() : []);-- const first = runChainInShell("sh", packageRoot, marker);- await waitFor(() => records().length === 1, "the first chain's record");- const second = runChainInShell("sh", packageRoot, marker);- await waitFor(() => records().length === 2, "the second chain's record");- const [firstId, secondId] = records().map((name) => name.replace(/\.json$/, ""));- assert.notEqual(firstId, secondId);- for (const id of [firstId, secondId]) {- assert.match(id, /^\d+$/, "a chain's record is named after its shell's pid");- assert.equal(readBuildInputsStamp(buildInputsPendingPath(packageRoot, id)).buildId, id);+ const records = () => (existsSync(pendingDirectory) ? readdirSync(pendingDirectory).filter(name => name.endsWith(".json")).sort() : []);++ const staged = path.join(pendingDirectory, "999999.json.999999.tmp");+ const chains = [runChainInShell("sh", packageRoot, marker)];+ let results;+ try {+ await waitFor(() => records().length === 1, "the first chain's record");+ await writeFile(staged, "{");+ chains.push(runChainInShell("sh", packageRoot, marker));+ await waitFor(() => records().length === 2, "the second chain's record");+ const [firstId, secondId] = records().map((name) => name.replace(/\.json$/, ""));+ assert.notEqual(firstId, secondId);+ for (const id of [firstId, secondId]) {+ assert.match(id, /^\d+$/, "a chain's record is named after its shell's pid");+ assert.equal(readBuildInputsStamp(buildInputsPendingPath(packageRoot, id)).buildId, id);+ }+ } finally {+ await rm(staged, { force: true });+ await writeFile(marker, "go\n");+ results = await Promise.allSettled(chains);}- await writeFile(marker, "go\n");- const results = await Promise.all([first, second]);for (const result of results) {- assert.match(result.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} begin: `));- assert.match(result.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} recorded: `), "a chain finished without recording");+ assert.equal(result.status, "fulfilled", result.reason?.message);+ assert.match(result.value.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} begin: `));+ assert.match(result.value.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} recorded: `), "a chain finished without recording");}- assert.deepEqual(records(), []);+ assert.deepEqual(await readdir(pendingDirectory), []);assert.equal(assessRunnerBuild({ repoRoot, packageRoot }).current, true);// The same chain under bash, which may exec its last command: `--finish`diff --git a/packages/paperclip-runner/src/drivers/agent-observability.test.ts b/packages/paperclip-runner/src/drivers/agent-observability.test.tsindex 4fdc2f9334..9e932db0ad 100644--- a/packages/paperclip-runner/src/drivers/agent-observability.test.ts+++ b/packages/paperclip-runner/src/drivers/agent-observability.test.ts@@ -9,8 +9,12 @@ import { createSanitizedAcpxSpawnInput } from "./acpx/environment.js";import { configuredEnvironmentProjection } from "../configured-environment.js";const roots: string[] = [];afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });-const launch: ProviderObservabilityLaunch = { kind: "codex", compatibilityKey: "revision", environment: { OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000" }, otelConfig: {- exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: { authorization: "Bearer secret-canary" } } }, metrics_exporter: "none", log_agent_responses: true,+const launch: ProviderObservabilityLaunch = { kind: "codex", compatibilityKey: "revision", environment: {+ OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000",+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20secret-canary",+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: "authorization=Bearer%20secret-canary",+}, otelConfig: {+ exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: {} } }, metrics_exporter: "none", log_agent_responses: true,} };it("transports a valid encoded header set in both Codex exporters without rejecting the launch", () => {const headers = Object.fromEntries([0, 1, 2, 3].map(index => [`x-custom-${index}`, encodeURIComponent("%".repeat(4000))]));@@ -28,8 +32,19 @@ it("materializes the real native home with both governed MCP and protected expornativeMcp: { name: "paperclip-assigned", url: "https://paperclip.example/mcp", token: "mcp-secret" } });const config = parse(await readFile(join(home, "config.toml"), "utf8"));expect(config.otel).toEqual(launch.otelConfig); expect(config.mcp_servers).toHaveProperty("paperclip-assigned");+ expect(JSON.stringify(config)).not.toContain("secret-canary");expect((await stat(join(home, "config.toml"))).mode & 0o777).toBe(0o600);});+it("delivers Codex headers only through its provider environment and redacts diagnostics", () => {+ const source = applyProviderObservabilityEnvironment({ KEEP: "task" }, launch, true);+ const sanitized = createSanitizedAcpxSpawnInput(source, "codex").env;+ expect(sanitized).toMatchObject(launch.environment);+ expect(sanitized.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG).not.toContain("secret-canary");+ expect(configuredEnvironmentProjection(source).PAPERCLIP_CONFIGURED_ENV_KEYS).toBe('["KEEP"]');+ expect(createSanitizedAcpxSpawnInput(source, "pi").env).not.toHaveProperty("OTEL_EXPORTER_OTLP_LOGS_HEADERS");+ const diagnostic = redactObservabilityDiagnostic("Bearer secret-canary Bearer%20secret-canary secret-canary", sanitized);+ expect(diagnostic).toBe("[REDACTED] [REDACTED] [REDACTED]");+});it("passes only explicit Claude observability through its sanitizer and excludes task projection", () => {const source = { OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20secret-canary", KEEP: "task" };expect(createSanitizedAcpxSpawnInput(source, "claude").env).toMatchObject({ OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: source.OTEL_EXPORTER_OTLP_LOGS_HEADERS });diff --git a/packages/paperclip-runner/src/drivers/agent-observability.ts b/packages/paperclip-runner/src/drivers/agent-observability.tsindex 37b11a1303..7ee8159df1 100644--- a/packages/paperclip-runner/src/drivers/agent-observability.ts+++ b/packages/paperclip-runner/src/drivers/agent-observability.ts@@ -27,7 +27,7 @@ export function providerObservabilityEnvironment(source: NodeJS.ProcessEnv | undif (!source || !["codex", "claude"].includes(agent)) return {};const result: NodeJS.ProcessEnv = {};const names: readonly string[] = agent === "claude" ? AGENT_OBSERVABILITY_ENV_KEYS.filter(key => key !== "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG")- : ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"];+ : ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"];for (const key of names) if (source[key] !== undefined) {// Codex carries two percent-encoded header maps. A valid 16-KiB company// header set can exceed 64 KiB after encoding and duplication.@@ -74,6 +74,7 @@ export function redactObservabilityDiagnostic(message: string, environment: Nodevalues.add(value);for (const item of value.split(",")) {const encoded = item.slice(item.indexOf("=") + 1);+ values.add(encoded);try { values.add(decodeURIComponent(encoded)); } catch { values.add(encoded); }}}diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.tsindex 4184701c56..ddeef5fc80 100644--- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts+++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts@@ -25,6 +25,46 @@ describe("Codex app-server transport limits", () => {})).toEqual({ PAPERCLIP_AI_PROVIDER_KEY: "selected-provider-key", CODEX_HOME: "/isolated/connection" });});+ it.each([true, false])("passes configured OTLP authentication to the child only when present (%s)", async (enabled) => {+ const headers = enabled ? {+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20fixture-logs",+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: "authorization=Bearer%20fixture-traces",+ } : {};+ const transport = new ProcessCodexAppServerTransport({+ command: process.execPath,+ args: ["-e", `+ process.stdin.resume();+ process.stdout.write(JSON.stringify({ method: "environment", params: {+ OTEL_EXPORTER_OTLP_LOGS_HEADERS: process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS,+ OTEL_EXPORTER_OTLP_TRACES_HEADERS: process.env.OTEL_EXPORTER_OTLP_TRACES_HEADERS,+ DATABASE_URL: process.env.DATABASE_URL,+ OPENAI_API_KEY: process.env.OPENAI_API_KEY,+ } }) + "\\n");+ `],+ environment: createSanitizedCodexEnvironment({+ ...headers, DATABASE_URL: "excluded-database", OPENAI_API_KEY: "excluded-provider-key",+ }),+ });+ try {+ const notification = await transport.notifications()[Symbol.asyncIterator]().next();+ expect(notification.value).toEqual({ method: "environment", params: headers });+ } finally {+ await transport.close();+ }+ });++ it("does not inherit ambient OTLP credentials without an explicit launch environment", () => {+ vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "ambient-log-secret");+ vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "ambient-trace-secret");+ try {+ const environment = createSanitizedCodexEnvironment();+ expect(environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toBeUndefined();+ expect(environment.OTEL_EXPORTER_OTLP_TRACES_HEADERS).toBeUndefined();+ } finally {+ vi.unstubAllEnvs();+ }+ });+it("passes only bounded controller-projected GitHub credentials", () => {expect(createSanitizedCodexEnvironment({diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.tsindex 886e86c7a4..c54663b1b2 100644--- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts+++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts@@ -251,7 +251,11 @@ export function createSanitizedCodexEnvironment(if (key.includes("PROXY") && proxyContainsCredentials(value)) continue;environment[key] = value;}- for (const key of ["PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY"]) {+ // Secret-bearing launch material must come from the controller's explicit environment.+ for (const key of [+ "PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY",+ "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS",+ ]) {if (explicit?.[key] !== undefined) environment[key] = explicit[key];}Object.assign(environment, githubCredentialEnvironment(source), configuredEnvironment(explicit));diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.tsindex 30ed4b970a..a63abc777f 100644--- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts+++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts@@ -18,7 +18,7 @@ import { execFile, type ChildProcess } from "node:child_process";import { EventEmitter } from "node:events";import { promisify } from "node:util";import { getCACertificates } from "node:tls";-import { join, resolve } from "node:path";+import { basename, dirname, join, resolve } from "node:path";import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";import {@@ -1441,7 +1441,7 @@ describe("OpenCodeServerDriver", () => {await session.close({ reason: "test" });});- it("uses the native system channel, selected skill tree, instruction root, and assigned MCP gateway", async () => {+ it.each([false, true])("uses the native system channel, selected skill tree, instruction root, and assigned MCP gateway after startup retry=%s", async (retryStartup) => {await chmod(fixture, 0o755);const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-context-"));const workspace = await mkdtemp(@@ -1465,6 +1465,7 @@ describe("OpenCodeServerDriver", () => {]);const systemInstructions = `${PAPERCLIP_EXECUTION_PROMPT}\n\nRead sibling.md\n\nRead-only instruction sibling root: ${instructionRoot}`;let submittedPrompt: Record<string, unknown> | null = null;+ let startupFailurePending = retryStartup;const driver = new OpenCodeServerDriver({model: "openrouter/deepseek/deepseek-v4-flash-0731",runtimeDirectory: root,@@ -1479,6 +1480,10 @@ describe("OpenCodeServerDriver", () => {PAPERCLIP_NATIVE_MCP_TOKEN: "x".repeat(40),},fetch: async (input, init) => {+ if (startupFailurePending && String(input).endsWith("/session") && init?.method === "POST") {+ startupFailurePending = false;+ return new Response("Fixture startup failure", { status: 503 });+ }if (String(input).endsWith("/prompt_async"))submittedPrompt = JSON.parse(String(init?.body));return fetch(input, init);@@ -1503,15 +1508,17 @@ describe("OpenCodeServerDriver", () => {PAPERCLIP_EXECUTION_PROMPT,);const sessionRoot = join(root, "context");- const isolatedHomes = (await readdir(sessionRoot)).filter((entry) =>- entry.startsWith("home-"),+ const runtimeEnvironment = JSON.parse(+ await readFile(join(sessionRoot, "data", "fake-environment.json"), "utf8"),);- expect(isolatedHomes).toHaveLength(1);+ const isolatedHome = runtimeEnvironment.home;+ expect(startupFailurePending).toBe(false);+ expect(dirname(isolatedHome)).toBe(sessionRoot);+ expect(basename(isolatedHome)).toMatch(/^home-/);await expect(readFile(join(- sessionRoot,- isolatedHomes[0]!,+ isolatedHome,".claude","skills","assigned",diff --git a/scripts/__tests__/eval-schedule-monitor.test.mjs b/scripts/__tests__/eval-schedule-monitor.test.mjsnew file mode 100644index 0000000000..ddf6db789e--- /dev/null+++ b/scripts/__tests__/eval-schedule-monitor.test.mjs@@ -0,0 +1,196 @@+import assert from "node:assert/strict";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import test from "node:test";+import { acknowledgeEvalNotifications, expectedCampaign, inspectEvalSchedule, monitorTransition, runEvalMonitor, MONITORED_SCHEDULES } from "../eval-schedule-monitor.mjs";++const now = Date.parse("2026-10-25T13:00:00Z");+const enabledAt = "2026-10-24T00:00:00Z";+const variables = { total_count: 3, variables: [+ { name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED", value: "true" },+ { name: "RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED", value: "true" },+ { name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT", value: enabledAt },+] };+const workflow = { id: 17, path: ".github/workflows/runner-full-stack-e2e.yml", state: "active" };+const run = (cadence, id, extra = {}) => ({ id, workflow_id: 17, run_attempt: 1, event: "schedule", status: "completed", conclusion: "success",+ display_title: `Runner Full-Stack E2E / ${MONITORED_SCHEDULES[cadence]}`,+ created_at: cadence === "daily" ? "2026-10-25T08:50:00Z" : "2026-10-25T09:50:00Z", ...extra });+function fixture(overrides = {}) {+ const calls = [];+ const get = async endpoint => {+ calls.push(endpoint);+ if (endpoint.includes("/variables?")) return overrides.variables ?? structuredClone(variables);+ if (endpoint.endsWith("runner-full-stack-e2e.yml")) return overrides.workflow ?? workflow;+ if (endpoint.includes("/runs?")) {+ const workflow_runs = overrides.runs ?? [run("daily", 101), run("weekly", 102)];+ return { total_count: overrides.total_count ?? workflow_runs.length, workflow_runs };+ }+ if (endpoint.includes("/artifacts?")) {+ const id = Number(endpoint.match(/\/runs\/(\d+)\//)[1]);+ const artifacts = overrides.artifacts ?? [{ name: `runner-e2e-report-${id}-1`, expired: false, size_in_bytes: 100 }];+ return { artifacts, total_count: artifacts.length };+ }+ throw new Error("Unexpected read");+ };+ return { get, calls };+}++test("UTC due windows honor grace, activation and the weekly boundary", () => {+ assert.equal(expectedCampaign("daily", now, Date.parse(enabledAt)), "2026-10-25T08:47:00.000Z");+ assert.equal(expectedCampaign("weekly", now, Date.parse(enabledAt)), "2026-10-25T09:47:00.000Z");+ assert.equal(expectedCampaign("weekly", Date.parse("2026-10-25T10:00:00Z"), Date.parse(enabledAt)), undefined);+ assert.equal(expectedCampaign("daily", now, now), undefined);+ assert.equal(expectedCampaign("daily", Date.parse("2026-11-01T10:17:00Z"), 0), "2026-11-01T08:47:00.000Z");+});++test("disabled gates do not query or diagnose old paid campaigns", async () => {+ const f = fixture({ variables: { total_count: 0, variables: [] } });+ assert.equal((await inspectEvalSchedule({ ...f, now })).status, "disabled");+ assert.equal(f.calls.length, 1);+});++for (const budgetGate of [undefined, "false"]) test(`an enabled legacy schedule with ${budgetGate ?? "missing"} budget gate alerts once`, async () => {+ const rows = [{ name: "RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED", value: "true" }];+ if (budgetGate !== undefined) rows.push({ name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED", value: budgetGate });+ const f = fixture({ variables: { total_count: rows.length, variables: rows } });+ const inspection = await inspectEvalSchedule({ ...f, now });+ assert.equal(inspection.status, "configuration_incomplete");+ assert.equal(inspection.reason, "legacy_schedule_enabled_without_budget_gate");+ assert.equal(f.calls.length, 1);+ const first = monitorTransition(undefined, inspection);+ assert.equal(first.notifications.length, 1);+ assert.equal(first.notifications[0].kind, "problem");+ assert.deepEqual(monitorTransition(first, inspection).notifications, first.notifications);+});++test("an explicitly disabled legacy gate stays quiet even if the new budget gate is enabled", async () => {+ const changed = structuredClone(variables); changed.variables[1].value = "false";+ const f = fixture({ variables: changed });+ const inspection = await inspectEvalSchedule({ ...f, now });+ assert.equal(inspection.status, "disabled"); assert.equal(f.calls.length, 1);+ assert.deepEqual(monitorTransition(undefined, inspection).notifications, []);+});++test("missing activation time and a disabled workflow stay distinct from missing runs", async () => {+ const invalid = structuredClone(variables); invalid.variables[2].value = "not a date";+ assert.equal((await inspectEvalSchedule({ ...fixture({ variables: invalid }), now })).status, "configuration_incomplete");+ assert.equal((await inspectEvalSchedule({ ...fixture({ workflow: { ...workflow, state: "disabled_inactivity" } }), now })).status, "workflow_disabled");+});++test("requires one completed scheduled campaign and its unexpired report for each cadence", async () => {+ const result = await inspectEvalSchedule({ ...fixture(), now });+ assert.deepEqual(result.observations.map(row => row.status), ["healthy", "healthy"]);+ assert.deepEqual(result.observations.map(row => row.runId), [101, 102]);+ assert.deepEqual(monitorTransition(undefined, result).notifications, []);+});++for (const [name, overrides, status] of [+ ["no campaign", { runs: [] }, "missing"],+ ["manual run cannot substitute for schedule", { runs: [run("daily", 101, { event: "workflow_dispatch" })] }, "missing"],+ ["failed campaign", { runs: [run("daily", 101, { conclusion: "failure" })] }, "failed"],+ ["canceled campaign", { runs: [run("daily", 101, { conclusion: "cancelled" })] }, "failed"],+ ["overdue queued campaign", { runs: [run("daily", 101, { status: "queued" })] }, "overdue"],+ ["unexpected paid rerun", { runs: [run("daily", 101, { run_attempt: 2 })] }, "unexpected_retry"],+ ["multiple matching runs", { runs: [run("daily", 101), run("daily", 103)] }, "ambiguous_runs"],+ ["missing report", { artifacts: [] }, "missing_report"],+ ["expired report", { artifacts: [{ name: "runner-e2e-report-101-1", expired: true, size_in_bytes: 100 }] }, "missing_report"],+ ["empty report", { artifacts: [{ name: "runner-e2e-report-101-1", expired: false, size_in_bytes: 0 }] }, "missing_report"],+]) test(name, async () => {+ const result = await inspectEvalSchedule({ ...fixture(overrides), now });+ assert.equal(result.observations[0].status, status);+});++test("new-day early run does not collide with the prior day during grace", async () => {+ const runs = [run("daily", 100, { created_at: "2026-10-24T08:50:00Z" }), run("daily", 101)];+ const result = await inspectEvalSchedule({ ...fixture({ runs }), now: Date.parse("2026-10-25T09:00:00Z") });+ assert.equal(result.observations[0].runId, 100);+ assert.equal(result.observations[0].status, "healthy");+});++test("running is quiet inside the runtime limit", async () => {+ const result = await inspectEvalSchedule({ ...fixture({ runs: [run("daily", 101, { status: "in_progress", created_at: "2026-10-25T10:00:00Z" })] }), now });+ assert.equal(result.observations[0].status, "running");+});++test("API errors and incomplete lists cannot imply a missing or successful run", async () => {+ assert.equal((await inspectEvalSchedule({ ...fixture({ total_count: 101 }), now })).status, "api_unavailable");+ assert.equal((await inspectEvalSchedule({ get: async () => { throw new Error("private upstream body"); }, now })).status, "api_unavailable");+});++test("alerts once per incident and recovery while keeping last observed run IDs through API failure", async () => {+ const failed = await inspectEvalSchedule({ ...fixture({ runs: [run("daily", 101, { conclusion: "failure" }), run("weekly", 102)] }), now });+ const first = monitorTransition(undefined, failed);+ assert.equal(first.notifications.length, 1);+ assert.deepEqual(monitorTransition(first, failed).notifications, first.notifications);+ const outage = { observedAt: new Date(now + 1000).toISOString(), status: "api_unavailable", observations: [] };+ const unavailable = monitorTransition({ ...first, notifications: [] }, outage);+ assert.equal(unavailable.lastSuccessfulInspectionAt, first.lastSuccessfulInspectionAt);+ assert.deepEqual(unavailable.lastInspectedRunIds, { daily: 101, weekly: 102 });+ assert.equal(unavailable.notifications[0].status, "api_unavailable");+ assert.deepEqual(monitorTransition(unavailable, outage).notifications, unavailable.notifications);+ const resumed = monitorTransition({ ...unavailable, notifications: [] }, failed);+ assert.deepEqual(resumed.notifications.map(row => [row.kind, row.cadence]), [["recovery", "monitor"]]);+ const healthy = monitorTransition({ ...resumed, notifications: [] }, await inspectEvalSchedule({ ...fixture(), now }));+ assert.deepEqual(healthy.notifications.map(row => [row.kind, row.cadence]), [["recovery", "daily"]]);+});++test("persists private state across invocations and refuses corrupt history", async () => {+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-"));+ try {+ const file = path.join(directory, "state.json");+ const first = await runEvalMonitor(file, fixture({ runs: [] }).get, now);+ assert.equal(first.notifications.length, 2);+ const repeated = await runEvalMonitor(file, fixture({ runs: [] }).get, now);+ assert.deepEqual(repeated.notifications, first.notifications);+ assert.deepEqual(await acknowledgeEvalNotifications(file, [first.notifications[0].id]), { acknowledged: 1, pending: 1 });+ const afterPartialDelivery = await runEvalMonitor(file, fixture({ runs: [] }).get, now);+ assert.deepEqual(afterPartialDelivery.notifications, first.notifications.slice(1));+ assert.deepEqual(await acknowledgeEvalNotifications(file, first.notifications.map(row => row.id)), { acknowledged: 1, pending: 0 });+ assert.deepEqual((await runEvalMonitor(file, fixture({ runs: [] }).get, now)).notifications, []);+ assert.equal(JSON.parse(await readFile(file, "utf8")).repository, "paperclipai/paperclip");+ await writeFile(file, "bad history");+ await assert.rejects(runEvalMonitor(file, fixture().get, now));+ assert.equal(await readFile(file, "utf8"), "bad history");+ } finally { await rm(directory, { recursive: true, force: true }); }+});++test("pending alerts survive later recovery and are acknowledged only after delivery", async () => {+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-outbox-"));+ try {+ const file = path.join(directory, "state.json");+ const lostOutput = await runEvalMonitor(file, fixture({ runs: [] }).get, now);+ const recovered = await runEvalMonitor(file, fixture().get, now + 1000);+ assert.deepEqual(recovered.notifications.slice(0, 2), lostOutput.notifications);+ assert.deepEqual(recovered.notifications.map(row => row.kind), ["problem", "problem", "recovery", "recovery"]);+ const unchanged = await runEvalMonitor(file, fixture().get, now + 2000);+ assert.deepEqual(unchanged.notifications, recovered.notifications);+ await assert.rejects(acknowledgeEvalNotifications(file, ["invalid"]));+ assert.deepEqual(await acknowledgeEvalNotifications(file, recovered.notifications.map(row => row.id)), { acknowledged: 4, pending: 0 });+ assert.deepEqual((await runEvalMonitor(file, fixture().get, now + 3000)).notifications, []);+ } finally { await rm(directory, { recursive: true, force: true }); }+});++test("rejects valid JSON with malformed history without querying or overwriting it", async () => {+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-corrupt-"));+ try {+ const file = path.join(directory, "state.json");+ await runEvalMonitor(file, fixture().get, now);+ const valid = JSON.parse(await readFile(file, "utf8"));+ for (const malformed of [null, false, 0, [], {}, { ...valid, signals: [] }, { ...valid, signals: { daily: null } },+ { ...valid, notifications: null }, { ...valid, notifications: [{ id: "invalid" }] }]) {+ const original = JSON.stringify(malformed);+ await writeFile(file, original);+ const f = fixture();+ await assert.rejects(runEvalMonitor(file, f.get, now), /Invalid retained monitor state/);+ assert.equal(f.calls.length, 0);+ assert.equal(await readFile(file, "utf8"), original);+ }+ } finally { await rm(directory, { recursive: true, force: true }); }+});++test("workflow titles carry the literal schedule used to distinguish daily and weekly runs", async () => {+ const text = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8");+ assert.ok(text.includes("run-name: Runner Full-Stack E2E / ${{ github.event.schedule || inputs.cadence || 'manual' }}"));+ for (const cron of Object.values(MONITORED_SCHEDULES)) assert.ok(text.includes(`cron: "${cron}"`));+});diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjsindex 8ea57dadc7..69b7df521d 100644--- a/scripts/__tests__/release-verify-workflow.test.mjs+++ b/scripts/__tests__/release-verify-workflow.test.mjs@@ -698,8 +698,22 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map((match) => match[1],);- assert.equal(crons.length, 1, `${name} must have one schedule`);- assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/);+ if (name === "runner-full-stack-e2e.yml") {+ assert.deepEqual(crons, ["47 8 * * *", "47 9 * * 0"]);+ assert.ok(workflow.includes("vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'"));+ assert.ok(workflow.includes("Prepare bounded campaign before credentials"));+ assert.ok(workflow.includes("--max-parallel 1 --max-automatic-retries 0"));+ const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:"));+ assert.ok(catalog.includes("EVAL_OPERATING_OVERHEAD_JSON: ${{ vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON }}"));+ assert.ok(catalog.includes("operating_overhead: ${{ steps.catalog.outputs.operating_overhead }}"));+ const prepare = workflow.slice(workflow.indexOf(" - name: Prepare bounded campaign before credentials"), workflow.indexOf(" - name: Run bounded campaign and retain feedback review"));+ assert.ok(prepare.includes("EVAL_OPERATING_OVERHEAD_JSON: ${{ needs.catalog.outputs.operating_overhead }}"));+ assert.ok(prepare.includes("EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }}"));+ assert.ok(!prepare.includes("vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON"));+ } else {+ assert.equal(crons.length, 1, `${name} must have one schedule`);+ assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/);+ }}const chaosWorkflow = actionPinWorkflows[2];diff --git a/scripts/eval-schedule-monitor.mjs b/scripts/eval-schedule-monitor.mjsnew file mode 100644index 0000000000..6bee0e1956--- /dev/null+++ b/scripts/eval-schedule-monitor.mjs@@ -0,0 +1,176 @@+import { execFile } from "node:child_process";+import { randomUUID } from "node:crypto";+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";+import path from "node:path";+import { promisify } from "node:util";+import { pathToFileURL } from "node:url";++export const MONITORED_REPOSITORY = "paperclipai/paperclip";+export const MONITORED_WORKFLOW = "runner-full-stack-e2e.yml";+export const MONITORED_SCHEDULES = { daily: "47 8 * * *", weekly: "47 9 * * 0" };+const hour = 3_600_000;+const prefix = `repos/${MONITORED_REPOSITORY}/actions`;+const numericId = value => Number.isSafeInteger(value) && value > 0;++export function expectedCampaign(cadence, now, enabledAt) {+ if (!Object.hasOwn(MONITORED_SCHEDULES, cadence) || !Number.isFinite(now) || !Number.isFinite(enabledAt)) {+ throw new Error("Invalid monitor schedule input");+ }+ const due = new Date(now - 90 * 60_000);+ due.setUTCHours(cadence === "daily" ? 8 : 9, 47, 0, 0);+ if (due.getTime() > now - 90 * 60_000) due.setUTCDate(due.getUTCDate() - 1);+ if (cadence === "weekly") due.setUTCDate(due.getUTCDate() - due.getUTCDay());+ return due.getTime() >= enabledAt ? due.toISOString() : undefined;+}++function signal(cadence, status, dueAt, runId) {+ return { cadence, status, ...(dueAt ? { dueAt } : {}), ...(runId ? {+ runId, url: `https://github.com/${MONITORED_REPOSITORY}/actions/runs/${runId}`,+ } : {}) };+}++export async function inspectEvalSchedule({ get, now = Date.now() }) {+ const observations = [];+ try {+ const variables = await get(`${prefix}/variables?per_page=100`);+ if (!Array.isArray(variables.variables) || !Number.isSafeInteger(variables.total_count)+ || variables.total_count !== variables.variables.length) throw new Error("Incomplete variables");+ const values = Object.fromEntries(variables.variables.map(row => [row.name, row.value]));+ if (values.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED !== "true") {+ return { observedAt: new Date(now).toISOString(), status: "disabled", observations: [] };+ }+ if (values.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED !== "true") {+ return { observedAt: new Date(now).toISOString(), status: "configuration_incomplete",+ reason: "legacy_schedule_enabled_without_budget_gate", observations: [] };+ }+ const enabledAt = Date.parse(values.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT ?? "");+ if (!Number.isFinite(enabledAt) || enabledAt > now) {+ return { observedAt: new Date(now).toISOString(), status: "configuration_incomplete", observations: [] };+ }+ const workflow = await get(`${prefix}/workflows/${MONITORED_WORKFLOW}`);+ if (workflow.path !== `.github/workflows/${MONITORED_WORKFLOW}` || !numericId(workflow.id)) throw new Error("Wrong workflow");+ if (workflow.state !== "active") return { observedAt: new Date(now).toISOString(), status: "workflow_disabled", observations: [] };+ const expected = Object.keys(MONITORED_SCHEDULES).map(cadence => ({ cadence, dueAt: expectedCampaign(cadence, now, enabledAt) }));+ const due = expected.filter(row => row.dueAt);+ if (!due.length) return { observedAt: new Date(now).toISOString(), status: "waiting", observations: [] };+ const since = due.map(row => row.dueAt).sort()[0];+ const runs = await get(`${prefix}/workflows/${MONITORED_WORKFLOW}/runs?event=schedule&per_page=100&created=${encodeURIComponent(`>=${since}`)}`);+ if (!Array.isArray(runs.workflow_runs) || !Number.isSafeInteger(runs.total_count)+ || runs.total_count !== runs.workflow_runs.length) throw new Error("Incomplete runs");+ for (const { cadence, dueAt } of expected) {+ if (!dueAt) { observations.push(signal(cadence, "waiting")); continue; }+ const matches = runs.workflow_runs.filter(run => run.event === "schedule" && run.workflow_id === workflow.id+ && run.display_title === `Runner Full-Stack E2E / ${MONITORED_SCHEDULES[cadence]}`+ && Date.parse(run.created_at) >= Date.parse(dueAt) && Date.parse(run.created_at) <= now+ && Date.parse(run.created_at) < Date.parse(dueAt) + (cadence === "daily" ? 24 : 168) * hour);+ if (!matches.length) { observations.push(signal(cadence, "missing", dueAt)); continue; }+ if (matches.length !== 1) { observations.push(signal(cadence, "ambiguous_runs", dueAt)); continue; }+ const run = matches[0];+ if (!numericId(run.id) || !numericId(run.run_attempt) || typeof run.status !== "string") throw new Error("Invalid run identity");+ if (run.run_attempt !== 1) { observations.push(signal(cadence, "unexpected_retry", dueAt, run.id)); continue; }+ if (run.status !== "completed") {+ observations.push(signal(cadence, now - Date.parse(run.created_at) > 4 * hour ? "overdue" : "running", dueAt, run.id));+ continue;+ }+ if (run.conclusion !== "success") { observations.push(signal(cadence, "failed", dueAt, run.id)); continue; }+ const artifacts = await get(`${prefix}/runs/${run.id}/artifacts?per_page=100`);+ if (!Array.isArray(artifacts.artifacts) || !Number.isSafeInteger(artifacts.total_count)+ || artifacts.total_count !== artifacts.artifacts.length) throw new Error("Incomplete artifacts");+ const reports = artifacts.artifacts.filter(row => row.name === `runner-e2e-report-${run.id}-1`+ && row.expired === false && Number.isSafeInteger(row.size_in_bytes) && row.size_in_bytes > 0);+ observations.push(signal(cadence, reports.length === 1 ? "healthy" : "missing_report", dueAt, run.id));+ }+ return { observedAt: new Date(now).toISOString(), status: "observed", observations };+ } catch {+ return { observedAt: new Date(now).toISOString(), status: "api_unavailable", observations };+ }+}++const quiet = new Set(["disabled", "waiting", "running", "healthy"]);+const statuses = new Set([...quiet, "observed", "configuration_incomplete", "workflow_disabled", "api_unavailable",+ "missing", "ambiguous_runs", "unexpected_retry", "overdue", "failed", "missing_report"]);+const object = value => value !== null && typeof value === "object" && !Array.isArray(value);+const timestamp = value => typeof value === "string" && Number.isFinite(Date.parse(value));+const notificationId = value => typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value);+const signalKey = row => !quiet.has(row.status) ? JSON.stringify([row.status, row.dueAt ?? null, row.runId ?? null]) : row.status;+const validSignal = row => object(row) && ["monitor", "daily", "weekly"].includes(row.cadence) && statuses.has(row.status)+ && (row.dueAt === undefined || timestamp(row.dueAt)) && (row.runId === undefined || numericId(row.runId))+ && (row.runId === undefined ? row.url === undefined : row.url === `https://github.com/${MONITORED_REPOSITORY}/actions/runs/${row.runId}`);++function validateState(previous) {+ if (!object(previous) || previous.version !== 1 || previous.repository !== MONITORED_REPOSITORY+ || previous.workflow !== MONITORED_WORKFLOW || !object(previous.signals)+ || !timestamp(previous.checkedAt) || !(previous.lastSuccessfulInspectionAt === null || timestamp(previous.lastSuccessfulInspectionAt))+ || !object(previous.lastInspectedRunIds) || Object.entries(previous.lastInspectedRunIds).some(([cadence, id]) => !["daily", "weekly"].includes(cadence) || !numericId(id))+ || Object.entries(previous.signals).some(([cadence, row]) => !validSignal(row) || row.cadence !== cadence+ || row.issue !== !quiet.has(row.status) || row.key !== signalKey(row))+ || !Array.isArray(previous.notifications) || previous.notifications.some(row => !validSignal(row)+ || !notificationId(row.id) || !["problem", "recovery"].includes(row.kind))+ || new Set(previous.notifications.map(row => row.id)).size !== previous.notifications.length) {+ throw new Error("Invalid retained monitor state; refusing to reset notification history");+ }+}++export function monitorTransition(previous, inspection) {+ if (previous !== undefined) validateState(previous);+ const current = inspection.status === "observed" ? inspection.observations : [signal("monitor", inspection.status)];+ const signals = {}, notifications = [];+ const prior = previous?.signals ?? {};+ for (const row of current) {+ const issue = !quiet.has(row.status);+ const key = signalKey(row);+ signals[row.cadence] = { ...row, issue, key };+ if (issue && prior[row.cadence]?.key !== key) notifications.push({ kind: "problem", ...row });+ else if (!issue && prior[row.cadence]?.issue && row.status !== "disabled") notifications.push({ kind: "recovery", ...row });+ }+ if (inspection.status === "observed" && prior.monitor?.issue) notifications.push({ kind: "recovery", cadence: "monitor", status: "observed" });+ const readable = !["api_unavailable", "configuration_incomplete"].includes(inspection.status);+ return { version: 1, repository: MONITORED_REPOSITORY, workflow: MONITORED_WORKFLOW,+ checkedAt: inspection.observedAt,+ lastSuccessfulInspectionAt: readable ? inspection.observedAt : previous?.lastSuccessfulInspectionAt ?? null,+ lastInspectedRunIds: { ...previous?.lastInspectedRunIds,+ ...Object.fromEntries(inspection.observations.filter(row => row.runId).map(row => [row.cadence, row.runId])) },+ signals: inspection.status === "observed" || quiet.has(inspection.status) ? signals : { ...prior, ...signals },+ notifications: [...previous?.notifications ?? [], ...notifications.map(row => ({ id: randomUUID(), ...row }))] };+}++async function saveState(stateFile, next) {+ await mkdir(path.dirname(stateFile), { recursive: true, mode: 0o700 });+ await writeFile(`${stateFile}.pending`, JSON.stringify(next, null, 2) + "\n", { mode: 0o600 });+ await rename(`${stateFile}.pending`, stateFile);+}++export async function acknowledgeEvalNotifications(stateFile, ids) {+ if (!Array.isArray(ids) || !ids.length || ids.some(id => !notificationId(id))) throw new Error("Invalid notification acknowledgments");+ const previous = JSON.parse(await readFile(stateFile, "utf8"));+ validateState(previous);+ const next = { ...previous, notifications: previous.notifications.filter(row => !ids.includes(row.id)) };+ await saveState(stateFile, next);+ return { acknowledged: previous.notifications.length - next.notifications.length, pending: next.notifications.length };+}++export async function runEvalMonitor(stateFile, get, now = Date.now()) {+ let previous;+ try { previous = JSON.parse(await readFile(stateFile, "utf8")); validateState(previous); }+ catch (error) { if (error.code !== "ENOENT") throw error; }+ const inspection = await inspectEvalSchedule({ get, now });+ const next = monitorTransition(previous, inspection);+ await saveState(stateFile, next);+ return { ...inspection, lastSuccessfulInspectionAt: next.lastSuccessfulInspectionAt,+ lastInspectedRunIds: next.lastInspectedRunIds, notifications: next.notifications };+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ if (process.argv.length !== 3 && !(process.argv[3] === "--ack" && process.argv.length >= 5)) {+ throw new Error("Usage: node scripts/eval-schedule-monitor.mjs <private-state-file> [--ack <notification-id> ...]");+ }+ const execute = promisify(execFile);+ const get = async endpoint => {+ if (!endpoint.startsWith(`${prefix}/`)) throw new Error("Unapproved monitor endpoint");+ const { stdout } = await execute("gh", ["api", "--method", "GET", endpoint], { timeout: 15_000, maxBuffer: 4 * 1024 * 1024 });+ return JSON.parse(stdout);+ };+ console.log(JSON.stringify(process.argv[3] === "--ack"+ ? await acknowledgeEvalNotifications(path.resolve(process.argv[2]), process.argv.slice(4))+ : await runEvalMonitor(path.resolve(process.argv[2]), get), null, 2));+}diff --git a/server/src/__tests__/eval-feedback-recovery.integration.test.ts b/server/src/__tests__/eval-feedback-recovery.integration.test.tsnew file mode 100644index 0000000000..873dc701af--- /dev/null+++ b/server/src/__tests__/eval-feedback-recovery.integration.test.ts@@ -0,0 +1,87 @@+import { spawn } from "node:child_process";+import { randomUUID } from "node:crypto";+import { once } from "node:events";+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { eq } from "drizzle-orm";+import { agents, heartbeatRuns } from "@paperclipai/db";+import { expect, it } from "vitest";+import { createLocalAgentJwt } from "../agent-auth-jwt.js";+import { startRunnerApiTestServer } from "./helpers/runner-api-server.js";+import { recoverInterruptedFeedback, readRecoveredFeedback } from "../../../tests/runner-e2e/feedback-recovery.js";++it("recovers real saved complaints and suggestions after their producer is killed", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-feedback-interruption-"));+ const previousHome = process.env.PAPERCLIP_HOME, previousSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET;+ process.env.PAPERCLIP_HOME = path.join(root, "home");+ process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID();+ let server: Awaited<ReturnType<typeof startRunnerApiTestServer>> | undefined;+ let worker: ReturnType<typeof spawn> | undefined;+ try {+ server = await startRunnerApiTestServer({ deploymentMode: "local_trusted" });+ const fixture = await server.fixture({ disableWakeOnDemand: true });+ await server.db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null }).where(eq(heartbeatRuns.id, fixture.runId));+ await server.db.update(agents).set({ adapterType: "codex_local" }).where(eq(agents.id, fixture.agentId));+ const token = createLocalAgentJwt(fixture.agentId, fixture.companyId, "codex_local", fixture.runId)!;+ const caseDirectory = path.join(root, "case");+ const snapshots = path.join(caseDirectory, "snapshots");+ const identity = { campaignId: randomUUID(), attemptId: randomUUID(), executionId: "synthetic.interruption.local" };+ await mkdir(snapshots, { recursive: true });+ await writeFile(path.join(snapshots, "fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId: fixture.companyId }));+ worker = spawn(process.execPath, ["--input-type=module", "-e", `+ for (const kind of ['complaint', 'suggestion']) {+ const response = await fetch(process.env.FEEDBACK_URL, {+ method: 'POST', headers: { 'content-type': 'application/json',+ authorization: 'Bearer ' + process.env.FEEDBACK_TOKEN, 'X-Paperclip-Run-Id': process.env.FEEDBACK_RUN },+ body: JSON.stringify({ kind, body: 'Synthetic interrupted producer ' + kind, idempotencyKey: kind }),+ });+ if (response.status !== 201) throw new Error('Synthetic feedback submission failed: ' + response.status);+ await response.json();+ }+ process.send({ saved: 2 });+ setInterval(() => {}, 1000);+ `], { stdio: ["ignore", "pipe", "pipe", "ipc"], env: {+ PATH: process.env.PATH, FEEDBACK_URL: `${server.apiUrl}/api/companies/${fixture.companyId}/agent-commentary`,+ FEEDBACK_TOKEN: token, FEEDBACK_RUN: fixture.runId,+ } });+ const saved = await Promise.race([+ once(worker, "message", { signal: AbortSignal.timeout(10_000) }),+ once(worker, "exit").then(([code]) => { throw new Error(`Producer exited before saving feedback: ${code}`); }),+ ]);+ expect(saved[0]).toEqual({ saved: 2 });+ const exited = once(worker, "exit");+ worker.kill("SIGKILL");+ expect((await exited)[1]).toBe("SIGKILL");+ const apiUrl = server.apiUrl;+ await recoverInterruptedFeedback({ identity, caseDirectory, secrets: [token], deadlineAt: Date.now() + 5_000,+ api: { async get<T>(endpoint: string, options?: { timeout: number }): Promise<T> {+ const response = await fetch(apiUrl + endpoint, { signal: AbortSignal.timeout(options?.timeout ?? 1_000) });+ if (!response.ok) throw new Error(`Read failed: ${response.status}`);+ return response.json() as Promise<T>;+ } },+ });+ expect(await readRecoveredFeedback(caseDirectory, identity)).toMatchObject({+ complaints: 1, suggestions: 1, bodiesCollected: 2, collection: "partial", invocationCollection: "partial",+ invocations: { completed: 0, failed: 0, unknown: 0 },+ });+ await server.close(); server = undefined;+ const evidenceText = await readFile(path.join(snapshots, "agent-feedback-recovered.json"), "utf8");+ const evidence = JSON.parse(evidenceText);+ expect(evidence.entries.map((entry: { body: string }) => entry.body).sort()).toEqual([+ "Synthetic interrupted producer complaint", "Synthetic interrupted producer suggestion",+ ]);+ expect(evidence.entries.every((entry: { runId: string; agent: { id: string }; task: { id: string } }) =>+ entry.runId === fixture.runId && entry.agent.id === fixture.agentId && entry.task.id === fixture.issueId)).toBe(true);+ expect(evidenceText).not.toContain(token);+ expect((await readRecoveredFeedback(caseDirectory, identity))?.review).toHaveLength(2);+ } finally {+ if (worker && worker.exitCode === null && worker.signalCode === null) {+ const exited = once(worker, "exit"); worker.kill("SIGKILL"); await exited;+ }+ await server?.close();+ if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;+ if (previousSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; else process.env.PAPERCLIP_AGENT_JWT_SECRET = previousSecret;+ await rm(root, { recursive: true, force: true });+ }+}, 90_000);diff --git a/server/src/__tests__/fixtures/otlp-observability.proto b/server/src/__tests__/fixtures/otlp-observability.protoindex ed258755a2..98feb31205 100644--- a/server/src/__tests__/fixtures/otlp-observability.proto+++ b/server/src/__tests__/fixtures/otlp-observability.proto@@ -13,7 +13,8 @@ message LogRecord { fixed64 time_unix_nano = 1; int32 severity_number = 2; strinmessage ScopeLogs { Scope scope = 1; repeated LogRecord log_records = 2; }message ResourceLogs { Resource resource = 1; repeated ScopeLogs scope_logs = 2; }message ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1; }-message Span { bytes trace_id = 1; bytes span_id = 2; bytes parent_span_id = 4; string name = 5; int32 kind = 6; fixed64 start_time_unix_nano = 7; fixed64 end_time_unix_nano = 8; repeated KeyValue attributes = 9; }+message Status { string message = 2; int32 code = 3; }+message Span { bytes trace_id = 1; bytes span_id = 2; bytes parent_span_id = 4; string name = 5; int32 kind = 6; fixed64 start_time_unix_nano = 7; fixed64 end_time_unix_nano = 8; repeated KeyValue attributes = 9; Status status = 15; }message ScopeSpans { Scope scope = 1; repeated Span spans = 2; }message ResourceSpans { Resource resource = 1; repeated ScopeSpans scope_spans = 2; }message ExportTraceServiceRequest { repeated ResourceSpans resource_spans = 1; }diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.tsindex fb08b142c6..f33983f14c 100644--- a/server/src/services/native-runtime/native-session-executor.test.ts+++ b/server/src/services/native-runtime/native-session-executor.test.ts@@ -802,6 +802,9 @@ describe("remote runner process supervision", () => {await writeFile(fixture, 'require("node:net").createServer().listen(0,"127.0.0.1",()=>process.stdout.write("ready\\n"));');let child: ReturnType<typeof spawn> | undefined;let probes = 0;+ let healthyProbes = 0;+ let releaseExitProbe!: () => void;+ const exitProbeReady = new Promise<void>(resolve => { releaseExitProbe = resolve; });let settled = false;const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" };const execute = vi.fn(async (input: { command: string; args?: string[] }) => {@@ -817,12 +820,15 @@ describe("remote runner process supervision", () => {return ok;}if (label === "paperclip-runner-monitor" && ++probes === 1) return { ...ok, exitCode: null, timedOut: true };- return await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => {+ if (label === "paperclip-runner-monitor" && probes > 2) await exitProbeReady;+ const observed = await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => {execFile(input.command, input.args ?? [], { timeout: 2_000 }, (error, stdout, stderr) => resolve({exitCode: error ? typeof error.code === "number" ? error.code : null : 0,signal: null, timedOut: !!error?.killed, stdout, stderr,}));});+ if (label === "paperclip-runner-monitor" && observed.exitCode === 0 && !observed.timedOut) healthyProbes++;+ return observed;});const handle = createRemoteRunnerProcessLauncher({target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: root },@@ -831,17 +837,21 @@ describe("remote runner process supervision", () => {})({ command: fixture, args: [], cwd: root, environment: {} });void handle.completion.then(() => { settled = true; }, () => { settled = true; });try {- await vi.waitFor(() => expect(probes).toBeGreaterThanOrEqual(2), { timeout: 5_000 });+ await vi.waitFor(() => expect(healthyProbes).toBe(1), { timeout: 5_000 });expect(settled).toBe(false);expect(handle.isExitConfirmed?.()).toBe(false);expect(() => process.kill(child!.pid!, 0)).not.toThrow();expect((await readFile(marker, "utf8")).split("\n")[1]).toBe(String(child!.pid));expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1);+ const childStopped = new Promise<void>(resolve => { child!.once("exit", () => resolve()); });child!.kill("SIGTERM");+ await childStopped;+ releaseExitProbe();await handle.completion;expect(handle.isExitConfirmed?.()).toBe(true);} finally {child?.kill("SIGKILL");+ releaseExitProbe();await handle.completion.catch(() => undefined);await rm(root, { recursive: true, force: true });}diff --git a/tests/e2e/cases-read-only.spec.ts b/tests/e2e/cases-read-only.spec.tsindex 83f272b46b..fbd98223df 100644--- a/tests/e2e/cases-read-only.spec.ts+++ b/tests/e2e/cases-read-only.spec.ts@@ -28,12 +28,9 @@ test.afterEach(async ({ request }, testInfo) => {test("browses passive cases and rolled-up task documents without starting work or editing", async ({ page }, testInfo) => {page.setDefaultTimeout(10_000);- // Announcements can arrive after navigation and cover otherwise valid links.- // Dismiss through the ordinary Board control, including after reloads.- const dismissAnnouncement = page.getByRole("complementary", { name: "Paperclip announcements", exact: true })- .getByRole("button", { name: "Dismiss announcement", exact: true });- const dismissAnnouncements = async () => { await dismissAnnouncement.click(); };- await page.addLocatorHandler(dismissAnnouncement, dismissAnnouncements);+ await page.route("**/api/announcements/current", async (route) => {+ await route.fulfill({ json: null });+ });const api = page.request;const original = await json(await api.get("/api/instance/settings/experimental"));const company = await json(await api.post("/api/companies", { data: { name: `Read-only cases ${randomUUID()}` } }));@@ -189,13 +186,9 @@ test("browses passive cases and rolled-up task documents without starting work oawait expect(dialog.getByRole("button", { name: /^(edit|save|restore|delete)\b/i })).toHaveCount(0);await expect(dialog.locator('textarea, [contenteditable="true"]')).toHaveCount(0);await testInfo.attach("read-only-document-revisions", { body: await page.screenshot(), contentType: "image/png" });- // The handler's own click moves focus. Keep it out of this focus-restoration- // assertion, then resume ordinary dismissal before the next navigation.- await page.removeLocatorHandler(dismissAnnouncement);await page.keyboard.press("Escape");await expect(dialog).toHaveCount(0);await expect(openReceipt).toBeFocused();- await page.addLocatorHandler(dismissAnnouncement, dismissAnnouncements);const auditLink = casePage.getByRole("link", { name: "View in Audit", exact: true });await expect(auditLink).toHaveAttribute("href", `/${company.issuePrefix}/activity?entityType=case&entityId=${receipt.id}`);await auditLink.click();diff --git a/tests/e2e/runner-e2e-dashboard.spec.ts b/tests/e2e/runner-e2e-dashboard.spec.tsindex e9e1d8e550..e9cc5c5237 100644--- a/tests/e2e/runner-e2e-dashboard.spec.ts+++ b/tests/e2e/runner-e2e-dashboard.spec.ts@@ -363,3 +363,32 @@ test("case status dots summarize matching results and open failures", async ({ pawait page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }).click();await expect(topLink).toHaveAttribute("data-case-status", "failed");});++test("keeps long feedback fingerprints inside a scrollable review table on phones", async ({ page }) => {+ const { emptyFeedbackSummary } = await import("../runner-e2e/feedback.js");+ const { feedbackReviewTemplate } = await import("../runner-e2e/feedback-dispositions.js");+ const execution = runnerExecutionById(executionIds[0]);+ const entry = resultFor(execution, "passed");+ entry.result.feedback = { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete",+ complaints: 2, bodiesCollected: 2,+ review: ["a", "b"].map(value => ({ fingerprint: value.repeat(64), kind: "complaint", occurrences: 1, disposition: "needs_investigation" })),+ };+ const campaign = buildRunnerCampaign({ campaignId: "feedback-mobile", generatedAt: entry.result.finishedAt, expected: [execution.id], results: [entry.result] });+ campaign.feedbackReview = { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "agent",+ decisions: [{ fingerprint: "a".repeat(64), disposition: "actionable", issueNumber: 123 },+ { fingerprint: "b".repeat(64), disposition: "duplicate", duplicateOf: "a".repeat(64) }],+ };+ await page.setViewportSize({ width: 390, height: 844 });+ await openReport(page, renderRunnerE2EDashboard({ title: "Feedback review", generatedAt: campaign.generatedAt,+ expected: campaign.expected, catalog: [execution], entries: [entry], campaign }));+ await page.getByText("Campaign usage and spend", { exact: true }).click();+ const decisions = page.getByRole("region", { name: "Feedback review decisions" });+ await expect(decisions).toBeVisible();+ expect(await decisions.evaluate(element => element.scrollWidth > element.clientWidth)).toBe(true);+ expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);+ await decisions.focus();+ await page.keyboard.press("Tab");+ const followup = decisions.getByRole("link", { name: "#123" });+ await expect(followup).toBeFocused();+ await expect(followup).toBeInViewport();+});diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.mdindex e1a34eee98..e499517506 100644--- a/tests/runner-e2e/FIXTURES.md+++ b/tests/runner-e2e/FIXTURES.md@@ -1011,3 +1011,30 @@ The arithmetic journey records four delivered agent handoffs and one completedterminal intent. The final case is `done` at handoff version 5, while the samefour provider work runs and unchanged workflow document revision finish the task.The oracle rejects a missing or pending terminal intent.++## Feedback persistence (explicit only)++`--suite feedback-persistence` selects two local native Claude/Codex cells.+Each creates one standard task through the browser, asks for two distinct+synthetic feedback items, and requires the real `submit_complaint` and+`submit_suggestion` tools. The five-minute case expects one task work run and+disables automatic retries. Company and agent accounted-work budgets are $10.+Setup probes and any background work must also pass through the separately+budgeted gateway when using an operations manifest. Local compute is unmetered.+Cost and duration forecasts remain unmeasured until a retained live attempt.+The runner E2E maintainers own this manual case; `--all` excludes it.++After collection and fixture teardown, the oracle reads the retained evidence+file and requires exact nonce-bearing bodies, the original company/agent/task+and observed run, plus one completed native receipt for each tool. Missing or+partial collection, prose-only claims, foreign attribution, duplicate records,+and failed submissions cannot pass. Full bodies stay in private evidence;+public reports retain counts, fingerprints, and closed oracle checks. The usual+result screenshot, trace, secret scanning and supervisor cleanup still apply.+This case proves explicit submission and persistence, not spontaneous feedback+quality or survival after an abrupt worker kill.++```sh+pnpm test:e2e:runner -- --list --suite feedback-persistence+pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1+```diff --git a/tests/runner-e2e/OPERATIONS-COVERAGE.md b/tests/runner-e2e/OPERATIONS-COVERAGE.mdnew file mode 100644index 0000000000..da4e2efa6e--- /dev/null+++ b/tests/runner-e2e/OPERATIONS-COVERAGE.md@@ -0,0 +1,232 @@+# Eval coverage and operating costs++Use this with [OPERATIONS.md](OPERATIONS.md). The bounded daily and weekly+selections cover Product E2E only. They do not authorize or cap the separate+Runner protocol workflow, manual matrix runs, or other live probes.++## Initial cadence++The runner E2E maintainers own these selections and their failure triage.+Both profiles use the exact model and limits in the reviewed pricecard.++| Journey | Failure detected | Feature-map association | Cadence |+| --- | --- | --- | --- |+| Native Claude message marker | A real task cannot complete and persist its expected answer through the metered route | `runs-adapters/runs` | Daily and weekly |+| Native Claude complaint and suggestion | Completed tools fail to save attributed feedback or collection loses it | `activity/agent-commentary` | Daily and weekly |+| Native Codex message marker | The corresponding Codex task/connection path fails | `runs-adapters/runs` | Weekly |+| Native Codex complaint and suggestion | The corresponding Codex feedback/tool path fails | `activity/agent-commentary` | Weekly |++These associations identify partial coverage. They do not qualify every run+lifecycle, the Commentary browser page, continuation, or remote execution.+The October 21 local pilot measured one successful sample per selected cell:+$0.427624800 and 72.757 summed attempt seconds for the daily selection,+$0.791539400 and 127.069 seconds for the weekly selection. These are historical+gateway estimates from the Linux ARM64 container. They include model setup+requests and attempt setup/teardown, but omit queue/build time and non-model+charges. Earlier failed and interrupted attempts remain in campaign spending.++`scheduled-measurements.json` retains the exact attempt, source revision,+definition, model, price version, result hash and journal hashes for each sample.+`forecast.json` includes this evidence and a snapshot hash. It separates the+historical successful selection from samples matching the current definition,+model, environment, runtime, route and pricing. A missing matching cell makes+the compatible selection total unknown. The feedback definition changed after+these four samples; its earlier results are not current qualification.++Per-cell mean and median are descriptive. P95 stays unknown below 20 samples+and then uses nearest rank; this is not a statistical confidence guarantee.+Do not sum per-cell percentiles into a campaign percentile. Hosted cost/time+and the selection's P95 remain unknown until separately measured. A passing+sample does not predict failure frequency or account for infrastructure,+storage, observability or subscriptions. Forecasts never reduce reservations+or the approved budget limits.++At the October 21, 2026 catalog audit, 47 suites declared 738 cells. `--all`+included 179; the other 559 required explicit selection. The bounded weekly+selection included four cells, and daily two. These are dated inventory counts,+not live acceptance results or an enduring catalog-size contract. Inspect the+current `pnpm test:e2e:runner -- --list` before choosing work. In particular,+Everyday Workflows, provider connections, feedback persistence, interruption,+and case handoffs must not be inferred from a passing `--all` campaign.++Keep the broader rotating lane disabled until its exact cells have a metered+route, complete retained attempts, a measured cost/time distribution, and a+budget that includes failures. Prefer targeted continuation, approval, and+cleanup cases that detect different failures over another model running the+same marker. A release lane may broaden coverage only with its own reviewed+manifest; the daily/weekly allowance does not authorize it.++## Adding an eval or changing its cadence++The change's fixture documentation and PR must declare:++- A responsible maintainer/team and the concrete failure the case detects.+- Feature-map capability IDs, or an explicit unmapped gap. State what the+ journey does not exercise within a broader capability.+- Exact suite, case, profiles, models, environment, and intended cadence.+ State whether it is explicit-only and how it differs from existing coverage.+- A deterministic alternative considered, and why paid behavior is necessary.+- Attempt, request, output-token, concurrency, retry, and wall-time limits.+ Include setup, judges, child/background work, and cleanup in the route audit.+- Estimated cost/time before the first run, then all-attempt measurements with+ sample count, retained failures, source/definition IDs, and uncertainty.+ An unmeasured field is unknown, never zero.+- Feedback collection/review, trace verification, missing-receipt handling,+ and a failure/cleanup owner. Novel complaints are reviewed independently+ of the product grade.++Use the existing [Product E2E authoring skill](../../.agents/skills/add-product-e2e-eval/SKILL.md)+or the separate [Runner authoring skill](../../.agents/skills/add-runner-eval/SKILL.md).+Update the existing scheduled selection and forecast only after qualification;+adding a catalog entry alone must not enable recurring spending.++## Reading campaign time++The report's summed attempt time uses every retained accounting receipt,+including failed retries. A missing attempt makes the total unknown; the report+keeps the observed subtotal and missing count. The observed attempt window runs+from the earliest retained start to the latest retained finish. Parallel attempts+overlap in that window, while their individual durations add to the sum. Gaps+between attempts remain in the window. Carried-forward evidence can span older+runs, so this window is not the current workflow's duration.++Agent execution and Daytona lease time remain separate measurements. Attempt+durations include the setup and cleanup inside their recorded boundaries.+New browser-worker results retain a separate phase receipt for fixture setup,+execution, feedback capture, cleanup and worker finalization. Repeated cleanup+segments add together. Atomic checkpoints preserve completed phases if the+worker is interrupted; the active phase has no inferred end time. Parent+recovery checks identity, interval bounds and file size before retaining that+partial receipt. Missing, malformed or unavailable receipts keep phase totals+unknown, while closed segments remain visible as observed subtotals.++These phases start inside the test callback, after browser/server startup, and+end before result publication. Parent teardown/finalization, CI queue/build time+and provider-billed runtime need their own receipts. Historical results and+non-browser launchers without a worker receipt remain explicitly unmeasured.+Test details show each selected attempt's receipt status and raw timing metadata.+Do not infer missing measurements by subtracting summed agent time from the+observed window. Refreshing a report computes these views from retained+results without making provider calls or changing the original measurements.++## Costs outside the model journal++Keep a separate cost component for each resource, with its currency, service,+resource/request identity, source window, allocation basis, and status. Join+components to campaign/attempt IDs where evidence permits. Do not sum a model+estimate with the invoice that later reconciles that same usage.++| Component | Evidence and allocation | When evidence is missing |+| --- | --- | --- |+| Provider models and paid tools | Dedicated project/key usage and billing exports; match provider request IDs or the narrowest available billing window | Preserve estimates, unknown requests, held reservations and the reconciliation difference |+| Daytona | Owned sandbox ID, actual resources, billable lifecycle and provider receipt; include failed creates, retries, stopped storage and delayed deletion | Retain the reservation or an explicitly partial estimate; a requested TTL does not prove deletion |+| CI compute | Workflow/job IDs, runner type, actual billed runtime and invoice rates; include setup, builds and failed jobs | Mark CI cost unknown; a short agent turn does not imply a cheap CI job |+| Storage and transfer | Artifact size, retention window and billed storage/egress | Report the unpriced component, including retained failure evidence |+| Braintrust/gateway usage | Project usage export and its window; distinguish included allowance from incremental overage | Do not treat free-looking trace metrics as proof of zero provider charges |+| Shared subscriptions | Show the existing Braintrust Pro $249/month separately under the October 21 operator policy; allocate newly incurred subscription/capacity charges from invoices with an explicit denominator | Keep new shared charges unallocated rather than assign an invented exact per-test cost |++Retain both the service total and the allocation denominator so allocated+amounts can be reproduced. A zero incremental usage charge can coexist with a+nonzero shared subscription. Never add the entire subscription to every test.+The approved daily/weekly budgets cover additional run costs, so they exclude+the existing $249/month fee while including new usage and capacity charges.+Scheduled admission now reserves the declared CI, storage, observability and+subscription allowances inside the same $20/$100 journal limit. Missing or stale+allowances stop scheduling before the AWS test runner starts. Missing receipts+keep their reservations held and accounting incomplete. This is a local admission+control, not a provider billing cap. Actual resource bounds, cost collection and+reviewed allowances still need qualification before recurrence is enabled; see+[the allowance contract](OPERATIONS.md#recurring-campaigns).++Recurring evidence and reports use private S3 archives with file/byte/request+boundaries, retained transfer receipts and verified access/expiration rules;+see [private recurring history](OPERATIONS.md#private-recurring-history). The+public workflow retains only a content-free availability marker for monitoring.+Public history is a separate explicit publication step. Private retention still+needs hosted OIDC qualification and actual storage/transfer billing evidence;+local archive tests and an infrastructure template do not establish those facts.++## Separate Runner Eval audit++The October 21 audit inspected Paperclip source on master `8b873e0112` plus the+eval operations stack, and the available `paperclip-evals` checkout at+`0bae4e0947dc7959cae25b73d27533c096f23a14`. This is a source audit, not a+new live run or proof of the workflow's configured private-repository revision.++Runner reports already retain immutable attempts, retry-inclusive estimated+cost and provider list cost as separate views, compatible-cohort comparisons,+and missing-usage coverage. Its mock control plane is a different authority+from the Product E2E server/database; Product feedback persistence cannot be+claimed from a Runner tool assertion.++The private `run_live_campaign.py` requires an explicit authorization envelope.+The hosted `runner-protocol-live-evals.yml` instead fans out individual+`run_live_roster.py` cells, supplies their direct provider credentials, and uses+configured per-attempt limits. It does not use the Product E2E shared gateway+journal, Braintrust attempt parent, indexing verifier, or feedback finalizer.+The private campaign wrapper's aggregate authorization must not be assumed to+protect that different hosted path.++Before adding Runner recurrence, its owners must establish an aggregate+allowance across fanout and retries, meter setup/provider bypasses, and qualify+native tracing through its actual packaged runner. Preserve direct-provider+qualification as its own route. Implement definition/roster changes in the+maintained private repository, pin that reviewed revision in the existing+workflow, and retain Runner's existing Evalbook renderer and scores. These+gaps remain open; Product E2E qualification does not close them.++## Missing campaign detection++Workflow failure notifications cover runs that start. The independent observer+is `scripts/eval-schedule-monitor.mjs`. Run it through a separate host with an+authenticated GitHub CLI and one private state file:++```sh+node scripts/eval-schedule-monitor.mjs "$HOME/.config/paperclip-evals/schedule-monitor.json"+```++The observer uses only GitHub GET requests. It never starts or retries paid+work and does not need provider credentials. It reads the two existing enable+gates plus `RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT`, an ISO timestamp set+when recurrence is enabled. Missing activation time fails visibly; occurrences+before activation are not treated as missed. Workflow run titles carry the+literal cron expression to distinguish daily and weekly runs, including Sunday.++Only the legacy `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED` gate being off establishes+disabled recurrence. If that gate is on while the operations gate is absent or+off, the observer reports `configuration_incomplete` and retains one incident+notification. Older deployed workflows use only the legacy gate, so a missing+new gate must not imply that those paid schedules are disabled. Verify the+deployed default-branch workflow and both variables before enabling recurrence.++The observer allows 90 minutes for schedule delay. It checks the most recent due+occurrence per cadence, marks a run overdue after four hours, and requires a+successful terminal run plus a nonempty, unexpired merged-report artifact.+The workflow's finalizer remains responsible for checking report content and+accounting. Metadata inspection does not repeat that validation. Unexpected+reruns, ambiguous matching runs, incomplete API lists and API failures remain+visible. The observer reads at most 100 rows per collection and fails closed+if a response is truncated.++Retained state includes the last successful inspection and inspected run IDs.+`notifications` is a durable outbox of problems and recovery transitions. Each+entry has an ID. After confirmed delivery, the host acknowledges those IDs:++```sh+node scripts/eval-schedule-monitor.mjs "$HOME/.config/paperclip-evals/schedule-monitor.json" --ack <notification-id> ...+```++Until acknowledgment, later inspections return the same pending entries even+if the problem has recovered. The host uses IDs to deduplicate delivery. This+survives a process exit between saving state and printing output; it does not+claim exactly-once delivery across an external notification service. After+acknowledgment, an unchanged problem stays quiet. A new missed occurrence is a+new incident. Disabled recurrence and unchanged healthy runs create no new+alerts. Use one writer for both checks and acknowledgments; do not reset the+state file to clear an incident. Malformed history fails before a query or+write. The external host owns notification delivery and its own liveness.++Schedule this observer outside the paid workflow after source and recurrence+are ready. It should inspect at least daily, after both scheduled start times+and their grace windows. Until that external trigger and notification delivery+are configured and exercised, missed-run detection remains an activation gap.diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.mdnew file mode 100644index 0000000000..31b36ed856--- /dev/null+++ b/tests/runner-e2e/OPERATIONS.md@@ -0,0 +1,707 @@+# Bounded eval operations++The request ledger and gateway are opt-in. The ordinary launcher still supports+unmetered legacy campaigns. Do not infer a spending limit from a campaign ID.+Enable the bounded path with `PAPERCLIP_E2E_OPERATIONS_MANIFEST=/absolute/file.json`.+Keep this manifest, its journal, and request receipts private.++The [coverage and operating-cost guide](OPERATIONS-COVERAGE.md) records cadence+ownership, suite-growth requirements, non-model allocation, the separate Runner+Eval audit, and remaining activation gaps.++## Supported admission++The initial path admits local `core-compatibility` and `feedback-persistence` cells for `runner-codex`,+`runner-acpx-claude`, `legacy-codex`, and `legacy-claude`. Each cell must have one+provider credential and an exact priced model. Other suites, remote environments,+subscription profiles, hosted tools, multimodal inputs, premium service modes,+and separate judges fail admission. This whitelist is implementation scope, not+evidence that every admitted harness has passed live qualification.++Before creating the journal or requesting credentials, manifest validation checks+that every selected model's worst-case single-request reservation fits the initial+campaign allowance after declared operating holds. Prices for unused models do+not affect this selection check. Passing it does not promise the whole campaign+will fit: later requests still use the journal's current commitments and limits.++Bounded daily and weekly workflows check source-byte limits before upload: 16 MiB+for the resolved lockfile, 1 GiB for the build bundle, and 256 MiB each for cell+evidence and the merged report. Build artifacts expire after one day; the others+expire after 30 days. These are source-byte and retention controls, not measured+compressed storage charges. ZIP framing and platform billing need separate receipts.++Evidence and reports are copied into fresh private staging directories. Up to+4,096 directory entries are inspected. Discovery visits accounting and snapshot+directories across attempts before archive and Playwright diagnostic directories.+Accounting under `scheduled/` and JSON records have copy priority over logs,+images and archives. Each retained file has a+SHA-256 digest. The staging receipt reserves 8 MiB inside the total and identifies+missing, changed, unsafe or oversized files and incomplete traversal. Originals+are never deleted or rewritten. Hidden paths remain excluded. The expected regular+`playwright-output/.last-run.json` bookkeeping file is outside the retained evidence+scope and does not fail retention. Other hidden paths make the scope incomplete.+Skipped paths and unvisited subtrees are named; overlong names use a bounded preview+and full-path digest. Partial staging is uploaded before the workflow+fails, so a storage limit cannot silently turn missing evidence into success.+These upload limits do not cap runtime disk writes or Actions caches. Each service+still needs billing evidence.++### Private recurring history++Daily and weekly evidence and merged reports are retained in a dedicated private+S3 bucket. They are not uploaded as plaintext GitHub artifacts, sent to the public+history publisher, or deployed to Pages. The report command's output also stays+inside the private report. The public workflow retains only a fixed availability+marker for the existing schedule monitor, with no report contents, storage address+or object identity. Public-repository GitHub artifacts are readable by signed-in+repository readers; authentication alone does not make those reports private.++Deploy `.github/infra/runner-e2e-private-history.yml` with the existing GitHub OIDC+provider and set `RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET`,+`RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN` and+`RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION`. The role trusts only this repository's+`runner-e2e-private-paid` and `runner-e2e-private-history` environments. Create both+with a deployment branch policy allowing only the default branch. Reserve them+for this workflow: the shared `runner-e2e-paid` and `runner-e2e-history` environments+also serve protocol evaluations that execute target-branch code and must not+have access to the private archive role. Set the three archive variables in both+dedicated environments. This workflow separately requires the default branch before+paid access and reporting. Manual target-branch tests have no OIDC permission. A separate recurring+job receives that permission only when the target SHA equals the default-branch+workflow SHA. Both jobs reuse the same anchored setup and test steps. The role has configuration reads+and GetObject/PutObject under `runner-e2e-private/bounded-v1/`; no deletion,+policy changes or access to the public history bucket. AWS credentials are step+outputs supplied only to private transfer commands, never exported to paid agents.++Admission verifies retention before paid execution. Every transfer verifies all+four public-access blocks, disabled object ACLs, no bucket-policy reader grants,+versioning, the 30-day/current and one-day/noncurrent expiration rules, and absence+of Object Lock and replication. Inspection failure stops the transfer. The private+bucket is not a CDN origin and has no external reader grant.++Evidence and report archives each contain at most 256 MiB of staged files and+4,097 entries including the staging receipt, with an additional 8 MiB ceiling for+ZIP framing. Archives use stored entries, without decompression. Extraction checks+paths, entry types, duplicate names, directory size, payload totals and ZIP checksums+before committing a fresh destination. Transfers use one conditional PutObject or+one ranged GetObject. Automatic AWS retries and workflow reruns are refused. The+bucket also rejects PutObject requests without an If-None-Match condition. A local+transfer receipt cannot be overwritten by rerunning the command in the same directory.+Each command retains admission and outcome metadata. After private storage passes+inspection, completed and failed transfers each get one independent receipt-upload+attempt without repeating the original transfer. Successful receipts include the+archive hash and object version; lost responses remain failed or unknown. A failed+privacy inspection prevents all uploads, and an unsuccessful receipt upload can+leave only the local receipt. Its own upload remains pending in the snapshot,+so the snapshot is not proof of complete billing. Operator downloads and STS+credential exchanges also need separate cost attribution.++For authenticated retrieval, set the bucket variable and AWS profile, then run:++```sh+node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts download report RUN_ID /tmp/eval-RUN_ID+```++Use `evidence` instead of `report` for the original staged accounting and diagnostics.+The requested destination must not already exist. Keep the downloaded report private.+To publish a selected campaign, review its retained normalized report and invoke the+existing trusted history publisher explicitly with the public destination; no agent+rerun is needed. Recurring execution never invokes that publication path.++Reporting and publication install only the committed lockfile from the trusted+workflow revision, with lifecycle scripts disabled and a job-local package store.+They never resolve new dependency versions while the job has OIDC access. If+that revision has an outdated lockfile, reporting fails before accessing the+archive. Use the normal master lockfile maintenance process; do not bypass the+frozen install in the credential-bearing job.++### Public history retention++Public history remains an explicit manual operation. The optional bounded publisher+uses `<history-prefix>/bounded-v1/` when `PAPERCLIP_E2E_BOUNDED_HISTORY=true`; existing+manual history otherwise retains its original prefix. Bounded publication is limited+to the original CI workflow attempt and is not retried under the same allowance.++Before any upload, the publisher validates all outgoing bundles: at most 64 MiB+and 256 files for the immutable campaign including its manifest, 16 MiB and eight+files for mutable history, and 64 MiB for the Pages bundle. History JSON reads+are limited to 4 MiB. The active index keeps at most 64 campaigns from the last+28 days and rebuilds latest/green pointers from those records. If publication+stops, the static index can retain links that later expire.++The bucket must have versioning enabled and exact prefix lifecycle rules for+30-day current-version expiration, one-day noncurrent-version expiration without+a retained-version floor, expired delete-marker removal and one-day incomplete+multipart cleanup. The publisher rejects overlapping transitions, different+expiration policies, Object Lock and replication. Access denied is a failure,+not proof that a feature is absent. Its role needs `s3:GetBucketVersioning`,+`s3:GetLifecycleConfiguration`, `s3:GetBucketObjectLockConfiguration` and+`s3:GetReplicationConfiguration` on that bucket. Configuration is verified at+each publication; it cannot guarantee that an operator will preserve it afterward.++The publisher admits at most 280 S3 CLI commands over ten minutes, with a+30-second command timeout and automatic AWS retries disabled. Uploads use+single `PutObject` operations in STANDARD storage; immutable objects use+`If-None-Match`. A private receipt is synced before each command and retained+locally for the operator to retain privately. Unknown responses remain unknown. Source bytes, S3+commands and expiration eligibility are not invoices or exact deletion times.+Credential lookup, public downloads/CDN egress, Pages hosting and shared service+charges require separate accounting; these limits do not cap public traffic.++The launcher owns one loopback HTTP gateway per attempt. The real provider key+stays in the parent launcher. Playwright receives an ephemeral token, and fixture+setup saves that token through a shared AI connection with the existing custom+Responses/Messages routing contract. The agent does not receive ambient provider+keys. The server environment strips the gateway token and upstream credentials.++Every upstream attempt reserves its maximum estimated charge in an fsynced+journal before dispatch. Requests, retries, and cancelled calls retain distinct+IDs. Unknown usage keeps the reservation held. One shared writer serializes+admission across cells, so concurrent requests cannot each spend the same remaining+allowance. Campaign deadline, request count, and in-flight count are also enforced.+Private request receipts retain transport IDs, response IDs, usage and cache TTLs,+route/cache/failover evidence, and final accounting status without request bodies+or credentials. Raw provider responses remain only in the normal private evidence.++## Manifest++A manifest contains no credentials. This structural example uses fixture prices;+replace them with independently verified model prices and context limits before+running a real campaign. The launcher does not discover current prices for you.++```json+{+ "version": 1,+ "limits": {+ "campaignId": "qualification-example",+ "maxUsd": "100",+ "deadlineAt": "2026-10-20T23:00:00Z",+ "maxRequests": 200,+ "maxConcurrent": 1+ },+ "ledgerPath": "/private/path/qualification/ledger.jsonl",+ "executionIds": ["EXACT_SELECTED_EXECUTION_ID"],+ "maxAttemptsPerExecution": 2,+ "route": "direct",+ "models": [{+ "provider": "anthropic",+ "maximumInputTokens": 1000000,+ "maximumOutputTokens": 16384,+ "timeoutMs": 180000,+ "price": {+ "provider": "anthropic",+ "model": "EXACT_SELECTED_MODEL",+ "version": "verified-pricecard-date",+ "sourceUrl": "https://platform.claude.com/docs/en/about-claude/pricing",+ "inputUsdPerMillion": "2",+ "cachedReadUsdPerMillion": "0.2",+ "cacheWrite5mUsdPerMillion": "2.5",+ "cacheWrite1hUsdPerMillion": "4",+ "outputUsdPerMillion": "10"+ }+ }]+}+```++`executionIds` must equal the launcher's selected set, and the retry option must+fit `maxAttemptsPerExecution`. Use `--list` to resolve the selection first.+Operations records this attempt cap in the journal's immutable limits and fsyncs+each attempt admission before starting its gateway. The count is per execution+across the whole shared journal, including restarts, renamed output campaigns,+and attempts that fail before making any provider request. Changing the cap or+reusing an admitted attempt ID is refused.+The maximum input value is the verified provider context ceiling, **not** a+requested input cap. The gateway clamps output to `maximumOutputTokens`; it does+not tokenize or truncate prompts. The conservative reservation prices the whole+input ceiling at the highest cache-write/long-context rate, plus capped output.+A small daily budget may therefore admit Claude but refuse Codex even when the+expected charge is small. Do not reduce the context ceiling to bypass this guard.++For modern OpenAI models, specify both `cacheWrite30mUsdPerMillion` and+`cacheWrite12hUsdPerMillion`. Optional `longContext` contains `aboveInputTokens`+and a complete `rates` object with the same token-rate fields. Above that total+input threshold (including cache), the estimate uses those rates for the entire+request. Reservations use the maximum rate across both tiers. Missing cache-write+counters, ambiguous TTL splits for nonzero writes, and model mismatches remain+unknown. Older OpenAI models without these usage fields need separate qualification.+Write prices are total token prices, not surcharges. Sources:+[OpenAI caching](https://developers.openai.com/api/docs/guides/prompt-caching),+[OpenAI prices](https://developers.openai.com/api/docs/pricing), and+[Claude prices](https://platform.claude.com/docs/en/about-claude/pricing).++For `gpt-5.6-sol`, a response that omits the entire write-TTL breakdown can use+the actual request's documented implicit cache policy: `prompt_cache_options.ttl`+or its 30-minute default. This fallback requires a visible input, no explicit+breakpoints, no stored prompt/conversation/response references, and no compaction.+The private receipt records `cacheWriteAttribution` with the policy version and+lifetime. These amounts remain estimates. Provider splits take precedence;+partial or malformed splits are never repaired. `prompt_cache_retention: "24h"`+does not imply a 12-hour write lifetime. The gateway does not change cache policy,+and this fallback does not reprice historical requests or release old holds.++For Braintrust, use `route: "braintrust"`, add the exact `projectId` UUID, and+provide `BRAINTRUST_API_KEY` securely. Configure project providers named+`OPENAI_API_KEY` and `ANTHROPIC_API_KEY`. Disable default provider fallback in the+project. The request pins that endpoint and project, disables gateway response+caching, and never automatically retries or requests fallback. A cache hit,+failover header, or unexpected successful endpoint invalidates the receipt and+holds its reservation. Provider prompt caching stays enabled and is metered.+The response stream is passed through and cumulative usage is counted once.+Gateway and native tracing are separate opt-ins; see the README tracing section.++When both a bounded operations manifest and explicit Braintrust tracing are+configured, the launcher starts a separate loopback OTLP relay for each attempt.+The worker and its native providers receive an expiring ingest token. The+Braintrust service key stays in the controller, even when model requests use+Braintrust without native tracing. The relay accepts only trace/log exports to+the configured project; neither request headers nor payload parent metadata can+select another project or an API endpoint. JSON is canonicalized before export;+protobuf payloads retain their original fields after bounded validation, which+rejects ambiguous singular fields and routing values.++Each relay allows at most 2,048 admitted exports, 8 MiB per request before and+after decompression, 64 MiB of forwarded payloads, and four simultaneous exports.+It also limits each attempt to 16,384 records and 256 MiB of expanded source+JSON. This measurement includes the full decoded source plus each record in+its own envelope with the shared resource and scope fields repeated. It counts+span names, IDs, times, events, links and log bodies, not only routing attributes.+Protobuf fields use the pinned OTLP v1.9 schema; unknown fields fail admission+until the schema is updated. Existing valid payloads are forwarded unchanged.+It closes at the campaign deadline or four hours after startup, whichever comes+first. Admission consumes the allowance before upstream dispatch. Failures and+client retries do not refund it; the relay itself never retries. A durable receipt+must be written before dispatch and before acknowledging the receiver outcome.++Version 2 private `<ledgerPath>.otlp/<attempt-hash>.jsonl` receipts record policy,+admission time, wire/payload bytes, payload digest, dispatch/outcome, partial+rejection counts, record counts, expanded source bytes, and closure totals.+Version 1 receipts remain readable with unknown expanded size and record count.+They contain no payload bodies, tokens or+upstream error text. Receipt files are exclusive and cannot be reset on restart.+An admission without an outcome is unresolved. Accepted ingestion does not prove+indexing, and attempted forwarding bytes are not billed Braintrust storage.+These receipts do not replace the observability overhead reservation or invoice+reconciliation. After the worker relay closes, bounded controller finalization+uses a separate one-request, 64 KiB relay with a 30-second deadline. Its exclusive+expanded-source allowance is 1 MiB with at most 256 records. Its+receipt is `<ledgerPath>.otlp/<attempt-hash>-finalization.jsonl`. This preserves+the original trace and grade while accounting for the final controller export.+This path currently supports the admitted local cells; remote transport remains+unqualified.++Scheduled experiment creation, grade insertion and summary retrieval share a+60-second window, three-request cap, 4 MiB total payload allowance and eight-score+cap. They run serially without automatic retry. Admission is persisted before+dispatch in `<ledgerPath>.experiment-ingestion.jsonl`; uncertain responses retain+their allowance and unresolved status. Reusing that file fails instead of+resetting the budget. These limits cover the selected daily/weekly campaigns;+the standalone experiment-import command does not use this transport budget.++Private `ingestion-usage.json` binds all three receipt paths to the campaign's+admitted attempts and ledger hash. Missing, malformed, unclosed or unresolved+receipts leave the total unknown while preserving readable partial measurements.+These are admitted transport payload bytes, not Braintrust processed-data units+or an invoice. Vendor charges remain unreconciled. The current two-attempt daily+maximum is 132.125 MiB and the four-attempt weekly maximum is 260.25 MiB,+including finalization and the campaign's experiment payload allowance. These+bounds do not include other projects, standalone imports or retained storage.+The corresponding expanded-source ceilings are 514 MiB daily and 1,028 MiB+weekly before the separate 4-MiB experiment allowance. These source measurements+support a priced forecast; they are not Braintrust's processed-data meter or a+vendor-enforced dollar limit. Gateway auto-logging is a separate input to that+forecast. Retain vendor usage reports for reconciliation rather than equating+transport or expanded source bytes with an invoice.++## Recurring campaigns++The existing `runner-full-stack-e2e.yml` owns both schedules: daily at 08:47 UTC+and weekly on Sunday at 09:47 UTC. Scheduled runs require both+`RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true` and+`RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED=true`. Leave the operations gate unset+until the workflow, billing coverage and infrastructure allowance are qualified.+There is no separate scheduler. The protected `runner-e2e-private-paid` environment needs+`BRAINTRUST_API_KEY`; repository variable `RUNNER_E2E_BRAINTRUST_PROJECT_ID` names+the exact project UUID. The key is available only to the bounded execution and+finalization step, after dependency installation and immutable builds.++Native Claude gateway cases also require the pinned Claude CLI for the managed+account's setup hello probe. The agent still runs through the pinned ACP runtime.+The workflow installs and checks the CLI before providing credentials for both+native and legacy Claude selections, including the aggregate scheduled selection.+For a local campaign, verify that same CLI prerequisite before starting a paid+attempt. Installing the ACP package alone does not satisfy this setup path.++Bounded cases keep the gateway in the setup phase until the created agent's+asynchronous lifecycle verification reaches ready. This wait uses the remaining+attempt deadline, allows at most 45 seconds, and stops on an observed setup error+or hold. The private `budgeted-setup.json` snapshot retains state transitions.+Only then does the case begin agent-phase accounting and create its task.+Older evidence that advanced after fixture creation may label hello probes as+agent cost. Its campaign totals still include those requests; retain that phase+limitation when comparing setup and task spending.++Attempt wall time includes setup and teardown. Agent time is the sum of captured+heartbeat-run windows, which can overlap. Missing runtime evidence, no captured+runs or an invalid run timestamp leaves agent time unknown; the report never substitutes attempt+wall time. A campaign total stays unknown if one of its retained attempts lacks+that measurement. A measured zero remains zero. Missing or invalid Daytona lease+windows likewise leave lease time and its runtime estimate unknown. Local attempts+do not contribute Daytona lease time. Existing published numeric measurements are+retained as historical evidence; this change does not reconstruct their provenance.++Bounded daily/weekly campaigns require a public repository and use the existing+standard `ubuntu-latest` runner for both compilation and execution. They do not+follow the manual campaign's AWS runner switch. GitHub charges no standard+hosted compute minutes for public repositories; retain runner/job evidence and+recheck the [billing policy](https://docs.github.com/en/billing/concepts/product-billing/github-actions)+when reviewing the allowance. This does not establish zero artifact, cache,+external history, observability or subscription cost. Manual selections keep+the existing AWS/GitHub runner choice.++Daily selects native Claude's marker and feedback-persistence cases with a shared+$20 allowance. Weekly adds native Codex's two corresponding cases with a+shared $100 allowance. Model requests and declared operating overhead share+this limit. Each campaign runs sequentially in one job with one+journal, one attempt per execution, zero automatic retries, a 30-minute request+deadline and a finite request count. Codex's worst-case request reservation exceeds+$20, so it is absent from daily selection. Its context ceiling is unchanged.+The $20/$100 limits cover additional costs caused by each run. Show the existing+Braintrust Pro subscription separately at $249/month (operator policy confirmed+October 21, 2026); do not allocate it into these run limits. New processing,+scoring, retention or capacity charges still count toward the run allowance.+The monthly run allowance is at most $1,120 (31 daily plus five weekly runs),+or $1,369 including the separately reported existing subscription. These are+maximum allowances, not forecasts of actual spending.+This is an admission limit with retained reservations, not a reconciled invoice+total or a provider-enforced billing limit.++Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to a reviewed+allowance with `version: 1`, an evidence `reference`, the actual `runner`+(`github-hosted` for bounded schedules), and `verifiedAt`/`expiresAt` timestamps no more+than 31 days apart. Its `components` array must contain exactly `ci_compute`,+`artifact_storage`, `observability`, and `shared_subscription`. Each component+needs a USD-string `maxUsd` and its own evidence `reference`. References are+stable audit IDs or paths, not credentials or arbitrary prose. No default+amounts are supplied. The supporting audit must include rates, resource/time+bounds, included allowances, allocation denominators, and the controls that+keep actual resource use within those bounds. Zero requires evidence of an+included or zero incremental charge; setting a zero allowance is not a receipt.+Under the current policy, `shared_subscription` may reserve zero for the+existing Braintrust subscription with a reference to that policy and current+plan evidence. Any newly incurred subscription or capacity charge needs its own+incremental allocation; the existing fee must remain visible separately.++The catalog checks that this allowance matches the selected runner, covers at+least the next four hours, and leaves room for a full conservative model-request+reservation. This happens before the build and test runners start. The workflow passes+the exact validated allowance to preparation; it does not read a potentially+changed repository variable again. Preparation rechecks it and retains it in the+manifest and forecast. Setup, builds, failed jobs, publication and retained+artifacts must all fit the declared components, not just the agent's runtime.++The journal initializes `overhead:<component>` reservations under+`campaign-overhead` from its immutable limits before admitting model requests.+These are allocation holds, not dispatched requests or completed resource+lifecycles. They do not occupy model concurrency, but they count toward the+finite charge-record limit. Resuming cannot change or drop them. Until matched+cost observations arrive, the report shows them as unknown and keeps their+allowance held; even a zero allowance does not claim a known zero charge.+The existing finalizer reports incomplete accounting while these receipts are+missing. Do not dismiss that finding to enable recurrence.++Reconcile a component through its existing request ID with provenance-bound+`infrastructure` or `invoice` observations. Partial observations keep the+remaining reservation held; a final nonconflicting observation can replace it.+Never mark a receipt final before its resource and billing window are complete.+An external bill above the allowance remains visible and stops further model+admission. The journal cannot stop AWS, storage or subscription billing itself;+provider resource limits and a verified receipt collector remain activation+requirements. No operating-cost allowance has been configured or qualified by+the presence of this code.++Collect retained GitHub usage after the workflow completes, including failed or+cancelled workflows, without dispatching or retrying anything:++```sh+node --import tsx tests/runner-e2e/operating-receipts.ts github RUN_ID /private/path/github-usage.json+```++The collector uses the existing `gh` authentication and only reads+`paperclipai/paperclip`. It verifies the Product E2E workflow, first attempt,+revision and repository, reads every returned job/artifact page up to 1,000+records per inventory, and rejects changed or incomplete inventories. It retains+job identities, summed job seconds, run-start delay, artifact sizes and expiration+dates. It omits arbitrary job names, artifact names and upstream responses.+Existing output files cannot be replaced. This is a current inventory, not an+invoice: deleted artifacts are absent, compressed bytes are not billed+byte-hours, job duration is not VM lifetime, and present repository visibility+does not establish visibility throughout an earlier run. Caches and external+S3 history remain separate. A visible public-hosted zero compute rate never+automatically releases an overhead reservation.++Import a reviewed cost receipt and its exact source evidence into an existing+campaign's declared overhead reservation:++```sh+node --import tsx tests/runner-e2e/operating-receipts.ts import /private/path/ledger.jsonl /private/path/receipt.json /private/path/source-evidence.json+```++The receipt contains `version: 2`, `costBasis: "component_cumulative"`,+`campaignId`, `allowanceReference`, `component`,+the source file's `evidenceSha256`, `windowStartedAt`, `windowEndedAt`, `coverage`+(`partial` or `complete`), `source` (`infrastructure` or `invoice`), `provider`,+`classification`, and an explicit USD-string `costUsd` when known. Estimated and+allocated charges also require a stable `pricingVersion`. `reconciled` requires+an invoice source. An unknown amount has no `costUsd` and always remains partial.+The observation window must have ended; completeness is the reviewer's assertion+that this component's entire resource and billing window is represented. A+checksum proves which file was used, not that the invoice or allocation is true.+Shared allocations must record their numerator, denominator, window and excluded+costs in that source file. Do not import a whole shared invoice as one campaign's+charge without an explicit allocation method.++Each amount is the cumulative total for the entire component, including all its+providers and separate charges through `windowEndedAt`. Later receipts must keep+the same `windowStartedAt` and cannot shorten the window or make completed+coverage partial again. Combine separate billing windows in the retained source+evidence before importing their total; two separate $0.80 charges require a+cumulative $1.60 receipt. Version 1 receipts lacked this declaration and are+rejected on import. Existing journal records are preserved; the importer also+refuses to extend a component whose prior receipts lack cumulative evidence.+It also checks the ledger's precedence rules before import: a larger cumulative+estimate cannot be hidden by an earlier, smaller invoice. Such imports fail+without changing retained evidence or the journal; reconcile the entire expanded+window with evidence the ledger can account for before retrying.++The importer validates campaign/allowance identity and source hash, retains both+files privately under `ledger.jsonl.overhead/`, and syncs them before appending+the observation. Files are written and synced under unique temporary names, then+published atomically without replacing existing evidence. A partial write is+removed without occupying the final path, so retry needs no manual repair.+Retries sync matching existing files too. Reimporting an identical+receipt adds no journal entry. A partial amount+keeps the rest of the allowance held. Complete invoices can supersede estimates;+conflicting complete invoices remain visible and block admission. This command+does not open a new spending window, change limits, dispatch tests or rewrite+previous records. Regenerate the campaign report from the same ledger afterward;+an old report snapshot is not rewritten by an offline receipt import.++The checked pricecard expires on November 19, 2026. Admission stops when the+pricecard expires, the catalog model changes, the project is missing, or the+preflight already exists. Review current official prices and context limits+before updating `scheduled-prices.json`. `forecast.json` records the selected+IDs, provenance-linked pilot measurements, compatibility gaps, uncertainty,+reservations and price expiry. The historical daily/weekly successful model+subtotals are $0.427624800/$0.791539400; they are not campaign bills. Setup model+calls and attempt setup/teardown are included, while CI setup/build/queue and+non-model charges are excluded. Current compatible totals stay unknown if any+selected cell lacks a matching measurement. Hosted timing is still unmeasured.+Per-cell medians describe only the retained samples; P95 requires at least 20+matching samples and does not establish reliability. See+[measurement provenance and limitations](OPERATIONS-COVERAGE.md#initial-cadence).+Daily coverage is limited to the two Claude journeys. The feedback definition+has changed since its passing pilot. Broader rotation and Daytona remain+unqualified and are not silently included by `--all`.++Dispatch `cadence: daily` or `weekly` from the default branch to qualify the same+path manually. Bounded dispatch rejects custom target revisions and selectors.+Bounded authorization pins the commit from the workflow event; a later merge+does not replace that revision or invalidate a queued campaign.+Workflow attempts after the first cannot start paid work with a fresh journal.+Rerunning reporting can reuse the original attempt-one evidence. Recover the+retained ledger for a deliberately resumed campaign; do not reset it or launch a+new campaign merely to hide a failure. The existing `manual` mode remains an+explicit, unbounded qualification path and does not inherit these allowances.++After execution, deterministic review dismisses only exact synthetic feedback+bodies corroborated by all feedback-persistence graders. Novel feedback remains+`needs_investigation`; no extra model call is made. Missing accounting, unresolved+receipts, feedback collection/submission failures, novel feedback, export failure+and incomplete experiment import or indexing fail the job with bounded diagnostic codes.+Successful unchanged runs produce artifacts without posting messages. GitHub's+workflow failure notifications remain the operator's notification channel.+Missing/non-starting schedules require an external workflow monitor; this change+does not claim that a workflow can detect its own failure to start.++The launcher starts the public Playwright JavaScript entry point directly with+Node and sends SIGINT for cancellation and watchdog cleanup. The package-manager+wrapper can exit before Playwright completes its shutdown, so it is excluded.+Playwright handles that signal by stopping workers and closing its web server;+SIGTERM can exit before web-server cleanup. The web-server supervisor still+receives SIGTERM and owns its server's shutdown. Both paths retain the existing+bounded grace period and forced cleanup for surviving owned processes.+When a worker exits without a result, the launcher also recovers its original+trace identity from the private fixture snapshots. The supervisor can finalize+that trace with the failed outcome. A mismatched project, execution or company+is rejected. Recovered identity does not prove native export or complete phases.++If a worker exits before writing its result, the server supervisor spends at+most five seconds collecting feedback from that attempt's saved company before+stopping the isolated server. Recovery snapshots remain private and are kept+separate from the worker's original evidence. The failed result includes any+recovered counts and review fingerprints; active runs or missing reads remain+partial or unavailable. Each atomic recovery checkpoint carries its attempt+identity, so a later write timeout cannot hide already saved feedback when the+final receipt is absent. Completed results are never replaced. The launcher+retains the private database whenever a worker result is missing, including when+the supervisor was killed before it could capture feedback. This retention is+reported as unconfirmed cleanup and requires inspecting the retained state.+Authenticated public-MCP fixtures do not use the local operator recovery path.++Original grades are imported into Braintrust without rerunning agents. The+access-controlled artifact includes the journal, forecast, feedback dispositions,+experiment receipt, indexing receipt and findings. The trusted report job rebuilds the dashboard+and retains its evidence-bound experiment link. Export acceptance and native+indexing are separate. A bounded, read-only Braintrust query verifies the attempt+root, finalization, native API activity and any expected feedback tools. It also+compares indexed experiment rows with original grades, revisions and attempt IDs.+The verifier tries at most three rounds within 45 seconds, retains missing/query+failures, and never reruns an agent. Native record counts measure export volume,+not distinct provider requests or charges. Codex exports can be high volume and+Braintrust can double-count native/log observations; use the deduplicated gateway+ledger as the estimate authority. Private indexing receipts contain only counts,+identities and verification status, without trace contents or credentials.++## Inspection and limits++Retained deterministic grades can be imported into a private Braintrust experiment+without invoking agents or judges. Prepare a JSON array of absolute paths to the+original `result.json` for **every** retained attempt in `normalized-results.json`.+The original files supply each attempt's actual source and definition hash;+the selected result's revision must not be assigned to older attempts.++```sh+PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID=PROJECT_UUID \+node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/braintrust-experiment-import.ts \+ /private/campaign/normalized-results.json /private/campaign/original-result-paths.json \+ /private/campaign/experiment-plan.json --dry-run+```++Inspect the plan. To import it, provide `BRAINTRUST_API_KEY` securely, remove+`--dry-run`, and use a separate `experiment-receipt.json` output path. The importer+requires explicit accounting identities and source/definition hashes; older+incomplete provenance fails closed. It retains original failed and successful+grades, cleanup outcomes, feedback counts, source revisions and trace identities.+Carried-forward CI attempts retain their original campaign IDs and must match+the report's retained receipts. Request costs attach only when the attempt's+campaign also matches the journal's campaign, even if attempt IDs coincide.+It does not upload prompts, feedback bodies, matcher details or raw errors.++The evidence digest determines the private experiment name. Identical evidence+reuses that experiment and stable row IDs after an interrupted insert. Changed+evidence creates a different experiment and requires a separate receipt path.+The importer validates an existing receipt before retrying and never lowers its+confirmed accepted-row count. The receipt checkpoints accepted batches;+HTTP acceptance still requires independent indexing/UI verification. Retain failed+receipts. A retry must use the same original evidence, not newly run agents.+Set `PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT` to the receipt path when rebuilding the+campaign report to retain the experiment link. Its evidence hash must match the+report's selected source/definition identities, attempts, request-ledger checkpoint+and feedback review. Refreshing only+the report rendering preserves the link; changed evidence requires a new import.++Request-ledger amounts remain metadata with their journal hash and classifications.+They are not additive cost metrics, and missing rows remain unknown rather than+zero. The campaign ledger may include setup probes outside the imported tests.+Do not add experiment metadata amounts to native or gateway trace costs. Native+trace rollups are not invoice reconciliation and can omit setup or duplicate logs.+API contracts: [create experiment](https://www.braintrust.dev/docs/api-reference/experiments/create-experiment)+and [insert retained events](https://www.braintrust.dev/docs/api-reference/experiments/insert-experiment-events).++- `operations-preflight.json` preserves the exact manifest in the campaign folder.+- `ledger.jsonl` is the append-only authority for request admission and observations.+- `ledger.jsonl.requests/` contains private per-request checkpoints.+- `ledger.jsonl.summary.json` preserves known, reserved/held, conflicting, and+ unknown amounts on normal finalization, including failed campaign workers.+- `campaign.json` and the dashboard preserve retained attempts and show missing+ selected results after worker failure. Their request-ledger section shows the+ shared ledger's known subtotal, classifications, additional held reservations,+ unresolved requests and per-attempt totals. It can include earlier campaigns+ sharing that budget. It is a separate view of model spending: do not add it to+ heartbeat totals. The journal hash and record count identify the checkpoint.+- If the journal cannot be read or validated, the report records accounting as+ unavailable with no numeric totals. Retained results and reports are still+ written, and the launcher/report command exits unsuccessfully. Recover the+ original journal before resuming paid work; reporting does not repair or reset it.+- To regenerate a merged report with the same ledger, pass+ `PAPERCLIP_E2E_OPERATIONS_LEDGER=/absolute/ledger.jsonl` to `report.ts`.+ Dashboard refresh and public publication preserve and validate its closed+ numeric projection. Raw request receipts and the journal remain private.++All model amounts from usage times a pricecard are **estimated**, not billed or+reconciled. This path does not yet price CI compute, storage, subscriptions,+Daytona, separate judges, or gateway overhead. Unknown observations and these+coverage gaps must be included in operational reporting. Provider billing can+lag and discounts can differ; a local list-price cap is not a provider-side cap.++Resuming the same journal requires exactly the original limits. A dead process+can leave a lock and held reservations; inspect the owner and upstream receipts+before recovering it. Never delete a live writer's lock or reset a ledger merely+to regain budget. Older journals without an attempt cap remain readable for+reporting and reconciliation. Operations refuses an implicit upgrade because request receipts cannot reconstruct+zero-request attempts. Use the explicit audited migration below only after+reviewing all retained campaign attempts. Do not replace or reset an active pilot+ledger to evade this check.++No recurring schedule is enabled by this integration. Qualification of actual+agent streams, tools, continuation, cancellation, native traces, and remote routes+is required before treating those routes as operationally proven. Forecasts,+provider reconciliation, and the campaign feedback disposition workflow are+separate remaining integration work.++### Setup and agent request attribution++Each admitted attempt creates a private phase checkpoint under+`<ledgerPath>.attempts/`. The launcher passes its path only to the test worker;+the Paperclip server does not inherit it. Before fixture setup finishes,+requests are recorded as `setup`, including connection checks and hello probes.+The worker atomically advances the checkpoint to `agent` immediately after+fixture creation and before task execution. Cleanup does not reset this phase,+so late agent requests retain their task attribution.++The gateway reads and validates the checkpoint for each request before reserving+spend. That admission fixes the request's phase even if the phase changes while+its response streams. Missing, malformed, or foreign-attempt checkpoints stop+new dispatches rather than guessing a phase. Request records remain in the+fsynced journal after the worker exits. This boundary applies to the admitted+local core cells; it does not establish coverage for separate judges, paid tools,+child-agent attribution, or remote routes. Historical journals retain their+original attribution and are not retroactively rewritten.++### Auditing a legacy journal before adding an attempt cap++A journal created before durable attempt admission can acquire one attempt policy+through `ledger-attempt-policy.ts`. Stop its launcher and gateway first. Review+all retained campaigns, including startup failures with no model request. Save+the review and its source references as a private evidence file. Each prior+attempt needs its original execution and accounting attempt ID; request receipts+alone are insufficient. If that history is incomplete, do not resume paid work.++Create a private JSON audit with these fields:++- `version`: `1`.+- `ledgerPath` and `evidencePath`: absolute private file paths.+- `previousJournalHash`: the current journal's final record hash.+- `evidenceSha256`: SHA-256 of the reviewed evidence file.+- `maxAttemptsPerExecution`: the finite cap to adopt, including past attempts.+- `priorAttempts`: an array of `{ "executionId": "...", "attemptId": "..." }`.++Run `node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/ledger-attempt-policy.ts /absolute/private-audit.json`.+The command checks evidence and journal hashes, takes the normal exclusive writer+lock, and appends one fsynced policy record. It refuses unfinished requests,+omitted recorded request identities, duplicate attempts, or an existing attempt+policy. The imported count can already exceed the cap; then no new attempt is+admitted for that execution. The command cannot prove the completeness of the+operator's review of zero-request attempts. Retain the evidence for inspection.++All original journal bytes, request counts, charges, held reservations, campaign+identity, dollar limit, concurrency limit and deadline remain intact. Subsequent+launch manifests must use the adopted cap. The migration cannot extend an expired+deadline, change an existing cap, or replenish an allowance. An interrupted append+remains an accounting failure and must be recovered before more paid work.++The explicit local `feedback-persistence` native Claude/Codex cells also accept+an operations manifest. They use the same metered connection, prices, campaign+journal, setup boundary, and durable attempt allowance as core compatibility.+Collection and the independent saved-evidence oracle require no additional+model call. The synthetic items need review as test data, not product incidents.diff --git a/tests/runner-e2e/OTLP-LICENSE.txt b/tests/runner-e2e/OTLP-LICENSE.txtnew file mode 100644index 0000000000..261eeb9e9f--- /dev/null+++ b/tests/runner-e2e/OTLP-LICENSE.txt@@ -0,0 +1,201 @@+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.diff --git a/tests/runner-e2e/OTLP-NOTICE.md b/tests/runner-e2e/OTLP-NOTICE.mdnew file mode 100644index 0000000000..6ae059f624--- /dev/null+++ b/tests/runner-e2e/OTLP-NOTICE.md@@ -0,0 +1,11 @@+# OTLP field definitions++The field definitions in `eval-otlp-schema.ts` are derived from+[OpenTelemetry Protocol v1.9.0](https://github.com/open-telemetry/opentelemetry-proto/tree/v1.9.0).++Copyright The OpenTelemetry Authors.+Licensed under the [Apache License, Version 2.0](./OTLP-LICENSE.txt).++The source comments and package declarations were removed to embed these field+definitions in the eval harness. The harness adds a combined response message.+Accounting behavior and limits are documented in [OPERATIONS.md](./OPERATIONS.md).diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.mdindex df516223c8..5f199a2fad 100644--- a/tests/runner-e2e/README.md+++ b/tests/runner-e2e/README.md@@ -1054,9 +1054,32 @@ timings, token and cost accounting, and evidence links. The campaign headeraggregates input, output, and cached tokens, provider-reported LLM spend,Daytona list-price runtime estimates, and pricing coverage. Missing providerusage is labeled `unavailable` or `unpriced`; it is never presented as zero-cost. The CI report job stages the same portable site at+cost. A known subtotal from a partially priced run stays in campaign spending,+with a partial-amount label and incomplete billing coverage. The CI report job stages the same portable site at`normalized/index.html` inside the access-controlled merged report artifact.+Campaign billing retains a separate numeric receipt for each attempt, including+failed retries. The selected result still controls the grade and result card.+Copies of the same attempt do not add another charge. The supervisor stamps a+campaign/attempt identity; CI cells share `PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID`+while retaining their separate artifact directories. Report regeneration keeps+these receipts and their totals. A later cleanup update replaces an earlier+observation of the same attempt; contradictory costs at the same timestamp are+rejected instead of silently choosing one.++The report shows missing attempt receipts and artifacts excluded from another+campaign. Legacy CI evidence can be grouped by workflow run URL. Unscoped local+artifacts cannot establish campaign membership, so only the selected result is+used. Older normalized bundles without receipts are labeled `selected_results`;+rendering them again cannot recover lost retry costs.++Billing reads decimal `costUsdExact` receipts and separates model estimates+from provider-reported amounts. Explicit zero usage differs from a missing or+unfinished receipt. Coverage is limited to captured usage. It does not certify+setup probes, subscription allocations, external tools, CI, storage, or provider+invoice reconciliation. These require separate receipts. Do not use the current+numeric report subtotals as an enforced spending limit.+The trusted publisher discovers display-only entries for selected execution IDsabsent from its local catalog, so branch-only suites remain visible in thedashboard, filters, gallery, and summary image. It validates execution identity@@ -2748,3 +2771,173 @@ The arithmetic journey records four delivered agent handoffs and one completedterminal intent. The final case is `done` at handoff version 5, while the samefour provider work runs and unchanged workflow document revision finish the task.The oracle rejects a missing or pending terminal intent.++### Complaints and suggestions++Each ordinary live attempt collects company-wide saved feedback after retiring+runs and before fixture teardown, even when cancellation or grading fails. This+includes feedback from child and background runs. Full paginated bodies and+native tool invocation outcomes are checkpointed in the access-controlled+`snapshots/agent-feedback.json` artifact. Collection has a 30-second deadline;+missing detail pages, truncated run lists, active work, and unavailable APIs+remain explicit partial/unavailable observations. Legacy shell helpers do not+have complete native invocation coverage. An empty saved-feedback list does not+prove that a tool was available, and a completed tool receipt does not by itself+prove a saved submission.++Results and retained-attempt receipts contain only counts, closed statuses, and+body fingerprints. Campaign reports group identical fingerprints across attempts+into distinct items awaiting review. Feedback presence does not change product+grades. Inspect full bodies in private evidence and record a disposition before+considering the review complete; collection itself performs no model call or+semantic triage. Historical artifacts without feedback fields are labeled as+uncaptured. A killed worker that never reaches finalization is also uncaptured,+not evidence of zero feedback.++Create a private review template from the merged `normalized-results.json`:++```sh+pnpm exec tsx tests/runner-e2e/feedback-dispositions.ts /private/report/normalized-results.json /private/feedback-review.json+```++Inspect the full bodies in the retained attempt snapshots. Set `reviewedAt` to+an ISO timestamp with an explicit timezone, such as `2026-10-20T01:00:00Z`,+and `reviewedBy` to `human` or `agent`. Timestamp comments and free-form dates+are rejected so private notes cannot enter public report metadata. Assign each fingerprint+one disposition: `actionable`, `duplicate`, `already_fixed`, `unsupported`, or+`needs_investigation`. A duplicate must name a non-duplicate fingerprint in the+same campaign via `duplicateOf`. Optional `issueNumber` links a public+`paperclipai/paperclip` issue; keep private rationale outside this file.++Import a completed review into an existing bundle without rerunning agents:++```sh+pnpm test:e2e:runner:dashboard -- /private/report --feedback-review /private/feedback-review.json+```++Alternatively, rebuild the same campaign and selection with+`PAPERCLIP_E2E_FEEDBACK_REVIEW=/private/feedback-review.json`. Keep its original+campaign ID. The importer requires exactly one decision per collected group+and rejects a review if retained attempts or feedback coverage changed. Blank+templates cannot count as reviewed. Dashboard regeneration and publication+preserve the review. Regeneration refreshes the review status in an existing+`summary.md` while preserving its other measurements and links. An empty review+can record that no saved items were found; it never closes collection gaps,+failed submissions, or changes product grades.+This workflow makes no additional model call. Review performed by a separate+paid model must be budgeted and recorded as `feedback_review` work separately.++### Cost-ledger integration contract++`cost-ledger.ts` provides the request-level journal used by the eval operations+integration. It is a library; the existing launcher is not yet universally routed+through it, and merely setting a campaign ID does not enforce a spending cap.+Every paid dispatch must await a durable reservation before contacting a provider.+The caller supplies an explicit campaign ceiling, deadline, request count, and+concurrency. Monetary decisions use integer nanodollars. Model list prices remain+estimates with a pinned version; they do not become provider invoices.++The journal retains setup/agent/judge/tool/infrastructure attribution separately+from heartbeat IDs. Multiple sources for the same request are retained but only+one cost contributes to the total: reconciled, reported, estimated, then allocated.+Missing receipts and cancellations retain their reservation. Contradictory final+receipts stop new admissions. An observed overshoot is retained in full and stops+further work; local estimated limits are not provider-enforced dollar limits.+A request bound must account for input/context, output limits, cache-write pricing,+and any paid tools before claiming a bounded overshoot.++A single writer serializes admissions and fsyncs each journal record. Read-only+inspection survives process loss; truncated/tampered records fail closed. A stale+`.lock` requires inspection of the recorded owner PID before operator recovery;+never remove a live writer's lock. Resuming a campaign cannot silently change its+limits. The ledger rejects unreviewed fields and contains no prompts, responses,+headers, or credentials. Cross-source request joins must use a verified request+identity; ambiguous provider-request joins are rejected rather than double counted.++### Opt-in Braintrust traces++Set `PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID` to the destination project UUID and+provide `BRAINTRUST_API_KEY` securely to the launcher. An ambient key alone does+not enable export. The launcher includes this key in secret scans and removes+Braintrust variables from the server's inherited environment. Fixture setup+saves the company's native observability configuration through the public,+revision-fenced API using a non-Playwright sensitive request, before credential,+agent, and setup-probe creation. The onboarding flow installs it immediately+after company creation. The existing company setting controls native capture.++Each ordinary `runner.spec.ts` attempt records orchestration spans for setup,+execution, feedback, and cleanup, and exports protobuf over HTTPS with a ten+second deadline and no redirect/retry. `snapshots/braintrust-trace.json` checkpoints+the identifiers/configuration and final export receipt. Normalized results retain+the receipt, including failed exports. Export failure does not rewrite the product+verdict; operational qualification must separately require accepted and verified+traces. A killed worker can leave a pending receipt, not a claimed successful export.++`accepted` means the OTLP receiver accepted the batch; it does not establish that+Braintrust indexed it or that native agent spans arrived. Search the project for+`paperclip.eval.attempt_id` and its trace ID, then verify native records using the+company ID and `paperclip.eval.run_ids`. The disposable company uses the custom+OTLP destination at `https://api.braintrust.dev/otel` with a private `x-bt-parent`+header containing the exported attempt parent. Braintrust attaches independent+native trace roots beneath that attempt; native children keep their own hierarchy.+This uses the existing company observability API without changing production+trace parentage or enabling export on other companies.++The parent encoder uses Braintrust's V4 project-log wire format with JSON identity+fields (version 4, project-log type 2, zero compressed fields). Its fixture was+round-tripped through the public `SpanComponentsV4.fromStr` export in+`braintrust/util` 3.38.0. The implementation is specified in the+[official SDK source](https://github.com/braintrustdata/braintrust-sdk-javascript/blob/403ae94fb3f1916a101d8d84de4c4871e4ae3bec/js/util/span_identifier_v4.ts).+A model-free live probe verified the indexed child's `root_span_id` and+`span_parents` in Braintrust; a native-agent run must separately qualify the path.+The parent contains identifiers only and is installed through the sensitive+credential-header path, outside browser evidence.++Attempt spans include suite ID and definition hash, profile, case, configured+provider/model, source revision (or explicit `unknown`), and the runner-spec grader+path. The source revision identifies that grader's implementation. Worker exports+are marked `paperclip.eval.outcome_stage=worker`. After evidence packaging,+gateway shutdown and outer cleanup, the supervisor updates the same attempt root+with its final outcome and adds a finalization span. This preserves failures first+discovered after the worker returned. The result retains a separate finalization+export receipt; old or killed attempts can explicitly lack one. Export failure+remains separate from the product grade. Orchestration spans omit model usage and+cost metrics. Gateway model logging remains a separate observation lane;+its model costs must not be added to native costs without a verified request join.+The separate provider connection suite and standalone judges do not yet use this+attempt exporter.++### Bounded local gateway pilot++[OPERATIONS.md](./OPERATIONS.md) documents the opt-in operations manifest, exact+selection admission, durable request reservations, local Responses/Messages proxy,+and Braintrust routing. Current support is deliberately limited to local core+Codex/Claude cells. The document distinguishes implemented accounting controls+from live qualification and lists remaining billing/reporting coverage gaps.++## Feedback persistence (explicit only)++`--suite feedback-persistence` selects two local native Claude/Codex cells.+Each creates one standard task through the browser, asks for two distinct+synthetic feedback items, and requires the real `submit_complaint` and+`submit_suggestion` tools. The five-minute case expects one task work run and+disables automatic retries. Company and agent accounted-work budgets are $10.+Setup probes and any background work must also pass through the separately+budgeted gateway when using an operations manifest. Local compute is unmetered.+Cost and duration forecasts remain unmeasured until a retained live attempt.+The runner E2E maintainers own this manual case; `--all` excludes it.++After collection and fixture teardown, the oracle reads the retained evidence+file and requires exact nonce-bearing bodies, the original company/agent/task+and observed run, plus one completed native receipt for each tool. Missing or+partial collection, prose-only claims, foreign attribution, duplicate records,+and failed submissions cannot pass. Full bodies stay in private evidence;+public reports retain counts, fingerprints, and closed oracle checks. The usual+result screenshot, trace, secret scanning and supervisor cleanup still apply.+This case proves explicit submission and persistence, not spontaneous feedback+quality or survival after an abrupt worker kill.++```sh+pnpm test:e2e:runner -- --list --suite feedback-persistence+pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1+```diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.mdindex 093f206613..2a84a315e5 100644--- a/tests/runner-e2e/SECURITY.md+++ b/tests/runner-e2e/SECURITY.md@@ -423,6 +423,66 @@ Two passive seed cases support browser list assertions; screenshots remain onthe exact primary case/task routes and document dialogs preserve that case URL.The public screenshot allowlist is unchanged.+## Eval feedback capture++`snapshots/agent-feedback.json` retains company-wide complaint/suggestion bodies+and attributed invocation outcomes in the access-controlled Actions artifact.+It passes the existing credential scanner and is excluded from public history+like other snapshots. Normalized results and campaign attempt receipts contain+only numeric counts, closed collection statuses, and SHA-256 fingerprints for+review grouping. Public reports never copy feedback bodies, previews, tool+arguments, or free-form collection errors. Collecting feedback performs no new+model call and does not forward it to an external service.++Feedback capture retains at most 1,000 full entries and 8 MiB of serialized+entries/invocations per attempt. Event inspection stops at 10,000 events or+16 MiB of serialized event data. Limits produce partial collection and a closed+`evidence_limit_reached` diagnostic; omitted evidence is never reported as absent.+Checkpoints are considered every ten details, at page boundaries, and at stage+completion; geometric data growth bounds repeated writes. Reads stop before the+overall deadline, reserving up to five seconds (20% of shorter windows) to persist+the collected prefix. Persistence shares the overall collection deadline and+receives an abort signal. The fixture writes an abortable temporary+snapshot before replacing the last complete checkpoint, so a stalled write does+not overwrite earlier evidence with a partially written JSON file.+Review counts and fingerprints advance only after a checkpoint succeeds. A later+write failure or expired deadline retains the last saved bodies and marks coverage+incomplete.++Braintrust export requires an explicit eval project UUID. Native capture includes+prompts, responses, and tool content only after that opt-in configures the isolated+company's observability destination. The ingestion credential crosses the public API via+Node fetch, outside Playwright tracing, and is included in launcher evidence+redaction. Orchestration spans contain identifiers, phase timing, outcome, and+feedback counts; they never contain provider credentials or feedback bodies.++With a bounded operations manifest, the ingestion credential is an expiring+per-attempt relay token. The organization-wide Braintrust key stays in the+controller. Only authenticated POSTs to the relay's trace/log ingest paths are+allowed, with a fixed HTTPS upstream and redirects disabled. Header and payload+`braintrust.parent` values must reference the selected project. Duplicate protobuf+singular/oneof fields are refused, and JSON is canonicalized to remove duplicate+key ambiguity. The token cannot call model, read, delete, or other API endpoints;+it can still write records within the selected project until its finite request,+byte or time allowance is exhausted. It is not an attempt-level row permission.++The worker environment removes upstream provider/Braintrust credentials before+inserting the model and OTLP relay tokens. Ambient relay URLs are discarded, and+the fixture server receives the selected endpoint through company settings.+Both tokens participate in evidence redaction. Private OTLP receipts retain+metadata only, and persistence errors stop further admission. The relay closes+with the model gateway before controller finalization. The controller's direct+finalization export and tracing without an operations manifest retain the+existing direct credential path. Remote relay publication is not enabled here.++The bounded operations manifest is credential-free and private. Its parent-owned+loopback gateway uses an ephemeral per-attempt token. Upstream credentials stay+in the launcher, are included in secret scans, and are removed from the child+provider environment. Only the selected Responses or Messages endpoint is+forwarded to a fixed provider or Braintrust HTTPS origin; request headers do not+select arbitrary destinations. Gateway response caching and fallback are refused.+Request journal and usage receipts contain no prompt/response bodies or keys.+The manifest, journal, and receipts are not public report projections.## Configured repository and conversion proofdiff --git a/tests/runner-e2e/api.ts b/tests/runner-e2e/api.tsindex 398691d1e2..5032c3e56f 100644--- a/tests/runner-e2e/api.ts+++ b/tests/runner-e2e/api.ts@@ -58,15 +58,27 @@ export class RunnerApi {* never serialized into trace/blob evidence before Paperclip encrypts it.*/async postSensitive<T>(path: string, data: unknown): Promise<T> {- const response = await fetch(new URL(path, this.baseURL), {- method: "POST",- headers: { "content-type": "application/json", ...this.sessionOptions().headers },- body: JSON.stringify(data),- });+ return this.sensitive<T>("POST", path, data);+ }++ async putSensitive<T>(path: string, data: unknown): Promise<T> {+ return this.sensitive<T>("PUT", path, data);+ }++ private async sensitive<T>(method: "POST" | "PUT", path: string, data: unknown): Promise<T> {+ const url = new URL(path, this.baseURL);+ if (url.origin !== this.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Sensitive request must use the fixture API");+ let response: Response;+ try {+ response = await fetch(url, {+ method, redirect: "error", signal: AbortSignal.timeout(15_000),+ headers: { "content-type": "application/json", ...this.sessionOptions().headers },+ body: JSON.stringify(data),+ });+ } catch { throw new Error(`Sensitive ${method} request failed; details withheld`); }if (!response.ok) {- throw new Error(- `Sensitive POST ${path} returned ${response.status}; response body withheld`,- );+ await response.body?.cancel();+ throw new Error(`Sensitive ${method} ${url.pathname} returned ${response.status}; response body withheld`);}return response.json() as Promise<T>;}diff --git a/tests/runner-e2e/artifact-budget.mjs b/tests/runner-e2e/artifact-budget.mjsnew file mode 100644index 0000000000..cf84bbcb8a--- /dev/null+++ b/tests/runner-e2e/artifact-budget.mjs@@ -0,0 +1,163 @@+import { constants } from "node:fs";+import { createHash } from "node:crypto";+import { lstat, mkdir, open, opendir, realpath, rm, writeFile } from "node:fs/promises";+import path from "node:path";+import { pathToFileURL } from "node:url";++export const ARTIFACT_BOUNDS = Object.freeze({+ lock: 16 * 1024 * 1024,+ build: 1024 * 1024 * 1024,+ evidence: 256 * 1024 * 1024,+ report: 256 * 1024 * 1024,+ receipt: 8 * 1024 * 1024,+ entries: 4096,+});++export async function checkArtifactFiles(kind, files) {+ if (!["lock", "build"].includes(kind) || !files.length) throw new Error("Invalid artifact size check");+ let bytes = 0;+ for (const file of files) {+ const info = await lstat(file);+ if (!info.isFile() || info.isSymbolicLink() || info.size === 0) throw new Error("Invalid artifact input file");+ bytes += info.size;+ if (!Number.isSafeInteger(bytes) || bytes > ARTIFACT_BOUNDS[kind]) throw new Error(`${kind} artifact exceeds its source-byte allowance`);+ }+ return { kind, files: files.length, sourceBytes: bytes, limitBytes: ARTIFACT_BOUNDS[kind] };+}++function priority(relative) {+ if (relative === "scheduled" || relative.startsWith("scheduled/")) return 0;+ if (/\.(json|jsonl)$/.test(relative)) return 1;+ if (/\.(md|html|xml|txt|log)$/.test(relative)) return 2;+ return 3;+}++export async function stageBoundedArtifact(kind, source, destination, bounds = ARTIFACT_BOUNDS) {+ if (!["evidence", "report"].includes(kind) || !Number.isSafeInteger(bounds[kind])+ || !Number.isSafeInteger(bounds.receipt) || bounds[kind] <= bounds.receipt+ || !Number.isSafeInteger(bounds.entries) || bounds.entries < 1 || bounds.entries > ARTIFACT_BOUNDS.entries) {+ throw new Error("Invalid artifact staging allowance");+ }+ const root = path.resolve(source), out = path.resolve(destination);+ if (out === root || out.startsWith(`${root}${path.sep}`) || root.startsWith(`${out}${path.sep}`)) {+ throw new Error("Artifact staging paths must be separate");+ }+ await mkdir(out, { mode: 0o700 });+ const receipt = { version: 1, kind, sourceComplete: true, inventoryComplete: true,+ scope: "Artifact evidence; regular playwright-output/.last-run.json bookkeeping is excluded",+ sourceBytes: 0, retainedBytes: 0, limitBytes: bounds[kind], entryLimit: bounds.entries,+ findings: [], files: [] };+ const candidates = [];+ let visited = 0;+ const finding = code => { receipt.sourceComplete = false; if (!receipt.findings.includes(code)) receipt.findings.push(code); };+ const skipped = (relative, reason, subtree = false) => {+ const shortened = relative.length > 512 || Buffer.byteLength(JSON.stringify(relative)) > 1024;+ receipt.files.push({ path: shortened ? `${relative.slice(0, 160)}…${relative.slice(-80)}` : relative,+ ...(shortened ? { pathTruncated: true, pathSha256: createHash("sha256").update(relative).digest("hex") } : {}),+ ...(subtree ? { scope: "subtree" } : {}), retained: false, reason });+ };+ const scan = async () => {+ const pending = [{ absolute: root, relative: ".", depth: 0, order: 0 }];+ while (pending.length) {+ pending.sort((a, b) => a.order - b.order || a.depth - b.depth || a.relative.localeCompare(b.relative));+ const directory = pending.shift();+ if (visited >= bounds.entries || directory.depth > 24) {+ const reason = visited >= bounds.entries ? "entry_limit" : "directory_depth_limit";+ receipt.inventoryComplete = false; finding(reason); skipped(directory.relative, reason, true); continue;+ }+ try {+ const handle = await opendir(directory.absolute);+ for await (const entry of handle) {+ if (visited >= bounds.entries) {+ receipt.inventoryComplete = false; finding("entry_limit"); skipped(directory.relative, "entry_limit", true); break;+ }+ visited++;+ const absolute = path.join(directory.absolute, entry.name), relative = path.relative(root, absolute).split(path.sep).join("/");+ if (relative.length > 512 || Buffer.byteLength(JSON.stringify(relative)) > 1024) {+ receipt.inventoryComplete = false; finding("path_length_limit"); skipped(relative, "path_length_limit", entry.isDirectory()); continue;+ }+ if (entry.isFile() && /(?:^|\/)playwright-output\/\.last-run\.json$/.test(relative)) {+ skipped(relative, "playwright_bookkeeping_excluded"); continue;+ }+ if (entry.name.startsWith(".")) {+ receipt.inventoryComplete = false; finding("hidden_path_excluded"); skipped(relative, "hidden_path_excluded", entry.isDirectory()); continue;+ }+ if (entry.isDirectory()) {+ const diagnostic = /(?:^|\/)(?:blob-report|html-report|playwright-output)(?:\/|$)/.test(relative);+ pending.push({ absolute, relative, depth: directory.depth + 1,+ order: diagnostic ? 3 : relative === "scheduled" || relative.startsWith("scheduled/") ? 0 : 1 });+ } else if (entry.isFile()) candidates.push({ absolute, relative });+ else { finding("non_regular_file"); skipped(relative, "non_regular_file"); }+ }+ } catch { receipt.inventoryComplete = false; finding("inventory_failed"); skipped(directory.relative, "inventory_failed", true); }+ }+ };+ try {+ const metadata = await lstat(root);+ if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw new Error("Invalid evidence root");+ await scan();+ } catch { receipt.inventoryComplete = false; finding("inventory_failed"); skipped(".", "inventory_failed", true); }+ const canonicalRoot = await realpath(root).catch(() => root);+ candidates.sort((a, b) => priority(a.relative) - priority(b.relative) || a.relative.localeCompare(b.relative));+ for (const { absolute, relative } of candidates) {+ let input, output;+ const target = path.join(out, relative);+ const row = { path: relative, retained: false };+ receipt.files.push(row);+ try {+ if (relative === "artifact-budget-receipt.json") throw new Error("reserved_path");+ const canonical = await realpath(absolute);+ if (!canonical.startsWith(`${canonicalRoot}${path.sep}`)) throw new Error("source_escaped");+ input = await open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW);+ const before = await input.stat();+ if (!before.isFile()) throw new Error("non_regular_file");+ row.sourceBytes = before.size;+ receipt.sourceBytes += before.size;+ if (!Number.isSafeInteger(receipt.sourceBytes)) throw new Error("source_size_overflow");+ if (receipt.retainedBytes + before.size > bounds[kind] - bounds.receipt) throw new Error("byte_limit");+ await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });+ output = await open(target, "wx", 0o600);+ const buffer = Buffer.alloc(64 * 1024), hash = createHash("sha256");+ let copied = 0;+ while (copied < before.size) {+ const read = await input.read(buffer, 0, Math.min(buffer.length, before.size - copied), copied);+ if (!read.bytesRead) throw new Error("source_changed");+ hash.update(buffer.subarray(0, read.bytesRead));+ let written = 0;+ while (written < read.bytesRead) {+ const part = await output.write(buffer, written, read.bytesRead - written, copied + written);+ if (!part.bytesWritten) throw new Error("copy_failed");+ written += part.bytesWritten;+ }+ copied += read.bytesRead;+ }+ const after = await input.stat();+ if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("source_changed");+ await output.sync();+ row.retained = true; row.sha256 = hash.digest("hex");+ receipt.retainedBytes += copied;+ } catch (error) {+ const code = ["reserved_path", "source_escaped", "non_regular_file", "source_size_overflow", "byte_limit", "source_changed"].includes(error.message) ? error.message : "copy_failed";+ row.reason = code; finding(code);+ } finally {+ await input?.close(); await output?.close();+ if (!row.retained) await rm(target, { force: true });+ }+ }+ const serialized = `${JSON.stringify(receipt, null, 2)}\n`;+ if (Buffer.byteLength(serialized) > bounds.receipt) throw new Error("Artifact receipt exceeds reserved allowance");+ await writeFile(path.join(out, "artifact-budget-receipt.json"), serialized, { mode: 0o600, flag: "wx" });+ return { ...receipt, stagedBytes: receipt.retainedBytes + Buffer.byteLength(serialized) };+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const [command, kind, ...args] = process.argv.slice(2);+ if (command === "check") console.log(JSON.stringify(await checkArtifactFiles(kind, args)));+ else if (command === "stage" && args.length === 2) {+ const receipt = await stageBoundedArtifact(kind, args[0], args[1]);+ if (process.env.GITHUB_OUTPUT) await writeFile(process.env.GITHUB_OUTPUT, `complete=${receipt.sourceComplete}\n`, { flag: "a" });+ const summary = `Eval ${kind}: retained ${receipt.retainedBytes} source bytes; staged ${receipt.stagedBytes} bytes; complete=${receipt.sourceComplete}. Findings: ${receipt.findings.join(", ") || "none"}.`;+ if (process.env.GITHUB_STEP_SUMMARY) await writeFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`, { flag: "a" });+ console.log(summary);+ } else throw new Error("Usage: artifact-budget.mjs check <lock|build> <files...> | stage <evidence|report> <source> <destination>");+}diff --git a/tests/runner-e2e/artifact-budget.test.mjs b/tests/runner-e2e/artifact-budget.test.mjsnew file mode 100644index 0000000000..06493ca2a6--- /dev/null+++ b/tests/runner-e2e/artifact-budget.test.mjs@@ -0,0 +1,120 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { mkdtemp, mkdir, readFile, rm, stat, symlink, truncate, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { test } from "node:test";+import { ARTIFACT_BOUNDS, checkArtifactFiles, stageBoundedArtifact } from "./artifact-budget.mjs";++const bounds = { ...ARTIFACT_BOUNDS, evidence: 12_288, receipt: 8192, entries: 32 };+async function fixture(run) {+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-artifacts-"));+ try { const source = path.join(root, "source"); await mkdir(source); await run({ root, source, out: path.join(root, "out") }); }+ finally { await rm(root, { recursive: true, force: true }); }+}++test("retains exact accounting and feedback before oversized diagnostics and keeps the source intact", async () => fixture(async ({ source, out }) => {+ await mkdir(path.join(source, "scheduled"));+ await writeFile(path.join(source, "scheduled/ledger.jsonl"), '{"reserved":"4.16"}\n');+ await writeFile(path.join(source, "feedback.json"), '{"complaint":"retained"}\n');+ await writeFile(path.join(source, "trace.zip"), Buffer.alloc(5000, 1));+ const result = await stageBoundedArtifact("evidence", source, out, bounds);+ assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, true);+ assert.deepEqual(result.findings, ["byte_limit"]);+ assert.equal(await readFile(path.join(out, "scheduled/ledger.jsonl"), "utf8"), '{"reserved":"4.16"}\n');+ assert.equal(await readFile(path.join(out, "feedback.json"), "utf8"), '{"complaint":"retained"}\n');+ assert.equal((await stat(path.join(source, "trace.zip"))).size, 5000);+ await assert.rejects(stat(path.join(out, "trace.zip")), { code: "ENOENT" });+ const receipt = JSON.parse(await readFile(path.join(out, "artifact-budget-receipt.json"), "utf8"));+ assert.equal(receipt.files.find(row => row.path === "trace.zip").reason, "byte_limit");+ assert.match(receipt.files.find(row => row.path === "feedback.json").sha256, /^[a-f0-9]{64}$/);+ assert.ok(result.stagedBytes <= bounds.evidence);+}));++test("keeps complete evidence unchanged at the exact payload boundary and refuses destination reuse", async () => fixture(async ({ source, out }) => {+ const content = Buffer.alloc(bounds.evidence - bounds.receipt, 42);+ await writeFile(path.join(source, "result.json"), content);+ const result = await stageBoundedArtifact("evidence", source, out, bounds);+ assert.equal(result.sourceComplete, true); assert.equal(result.retainedBytes, content.length);+ assert.deepEqual(await readFile(path.join(out, "result.json")), content);+ await assert.rejects(stageBoundedArtifact("evidence", source, out, bounds), { code: "EEXIST" });+}));++test("refuses symlinks and records missing evidence without reading outside the source", async () => fixture(async ({ root, source, out }) => {+ await writeFile(path.join(root, "private"), "secret outside source");+ await symlink(path.join(root, "private"), path.join(source, "result.json"));+ const result = await stageBoundedArtifact("evidence", source, out, bounds);+ assert.equal(result.sourceComplete, false); assert.deepEqual(result.findings, ["non_regular_file"]);+ assert.equal(result.retainedBytes, 0);+ const missing = await stageBoundedArtifact("evidence", path.join(root, "missing"), path.join(root, "missing-out"), bounds);+ assert.equal(missing.inventoryComplete, false); assert.deepEqual(missing.findings, ["inventory_failed"]);+}));++test("bounds entry inventory and exposes partial traversal", async () => fixture(async ({ source, out }) => {+ for (let i = 0; i < 5; i++) await writeFile(path.join(source, `${i}.json`), "{}");+ const result = await stageBoundedArtifact("evidence", source, out, { ...bounds, entries: 3 });+ assert.equal(result.inventoryComplete, false); assert.equal(result.sourceComplete, false);+ assert.equal(result.files.filter(row => row.retained).length, 3); assert.deepEqual(result.findings, ["entry_limit"]);+ assert.deepEqual(result.files.find(row => row.reason === "entry_limit"), { path: ".", scope: "subtree", retained: false, reason: "entry_limit" });+}));++test("keeps hidden files excluded from upload and exposes that incomplete scope", async () => fixture(async ({ source, out }) => {+ await mkdir(path.join(source, ".private"));+ await writeFile(path.join(source, ".private/token"), "fixture");+ const result = await stageBoundedArtifact("evidence", source, out, bounds);+ assert.equal(result.inventoryComplete, false); assert.equal(result.sourceComplete, false);+ assert.deepEqual(result.findings, ["hidden_path_excluded"]);+ assert.deepEqual(result.files, [{ path: ".private", scope: "subtree", retained: false, reason: "hidden_path_excluded" }]);+ await assert.rejects(stat(path.join(out, ".private")), { code: "ENOENT" });+}));++test("excludes ordinary packaged Playwright bookkeeping without failing passing evidence", async () => fixture(async ({ source, out }) => {+ const folder = "core-compatibility/runner-codex/local/message-marker/attempt-1";+ await mkdir(path.join(source, folder, "playwright-output"), { recursive: true });+ await writeFile(path.join(source, folder, "playwright-output/.last-run.json"), '{"status":"passed","failedTests":[]}');+ await writeFile(path.join(source, folder, "result.json"), '{"status":"passed"}');+ const result = await stageBoundedArtifact("evidence", source, out, bounds);+ assert.equal(result.sourceComplete, true); assert.equal(result.inventoryComplete, true);+ assert.deepEqual(result.findings, []);+ assert.equal(result.files.find(row => row.path.endsWith(".last-run.json")).reason, "playwright_bookkeeping_excluded");+ assert.equal(await readFile(path.join(out, folder, "result.json"), "utf8"), '{"status":"passed"}');+}));++test("discovers snapshots in all attempts before archive directories can consume the entry limit", async () => fixture(async ({ source, out }) => {+ for (const attempt of ["attempt-a", "attempt-b"]) {+ await mkdir(path.join(source, attempt, "blob-report"), { recursive: true });+ await mkdir(path.join(source, attempt, "snapshots"));+ await writeFile(path.join(source, attempt, "snapshots/feedback.json"), '{"saved":true}');+ for (let i = 0; i < 20; i++) await writeFile(path.join(source, attempt, `blob-report/${i}.zip`), "archive");+ }+ const result = await stageBoundedArtifact("evidence", source, out, { ...bounds, entries: 12 });+ assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, false);+ for (const attempt of ["attempt-a", "attempt-b"]) {+ assert.equal(await readFile(path.join(out, attempt, "snapshots/feedback.json"), "utf8"), '{"saved":true}');+ assert.ok(result.files.some(row => row.path === `${attempt}/blob-report` && row.scope === "subtree" && row.reason === "entry_limit"));+ }+}));++test("identifies an overlong skipped subtree with a bounded preview and full path digest", async () => fixture(async ({ source, out }) => {+ const segments = Array.from({ length: 6 }, (_, index) => `${index}-${"x".repeat(90)}`);+ await mkdir(path.join(source, ...segments), { recursive: true });+ await writeFile(path.join(source, ...segments, "feedback.json"), "{}");+ const result = await stageBoundedArtifact("evidence", source, out, bounds);+ const omitted = result.files.find(row => row.reason === "path_length_limit");+ assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, false);+ assert.equal(omitted.scope, "subtree"); assert.equal(omitted.pathTruncated, true);+ assert.match(omitted.pathSha256, /^[a-f0-9]{64}$/); assert.ok(omitted.path.startsWith(segments[0]));+ assert.ok(omitted.path.endsWith(segments[5].slice(-80)));+ assert.equal(omitted.pathSha256, createHash("sha256").update(segments.join("/")).digest("hex"));+ assert.ok(result.stagedBytes <= bounds.evidence);+}));++test("checks immutable build and lock source totals before upload without reading their contents", async () => fixture(async ({ source }) => {+ const a = path.join(source, "a"), b = path.join(source, "b");+ await writeFile(a, "a"); await writeFile(b, "b");+ await truncate(a, ARTIFACT_BOUNDS.lock - 1);+ assert.equal((await checkArtifactFiles("lock", [a, b])).sourceBytes, ARTIFACT_BOUNDS.lock);+ await truncate(a, ARTIFACT_BOUNDS.lock);+ await assert.rejects(checkArtifactFiles("lock", [a, b]), /exceeds/);+ await assert.rejects(checkArtifactFiles("other", [a]), /Invalid/);+}));diff --git a/tests/runner-e2e/attempt-accounting.test.ts b/tests/runner-e2e/attempt-accounting.test.tsnew file mode 100644index 0000000000..e7335879ee--- /dev/null+++ b/tests/runner-e2e/attempt-accounting.test.ts@@ -0,0 +1,87 @@+import { describe, expect, it } from "vitest";+import { buildCampaignAccounting, isSelectedCarryForward, retainedCampaignAttempts } from "./attempt-accounting.js";+import { buildRunnerCampaign, campaignHistoryRecord } from "./history.js";+import type { RunnerE2EResult } from "./types.js";++function result(attempt: number, cost: number, overrides: Partial<RunnerE2EResult> = {}): RunnerE2EResult {+ return {+ schema: "paperclip.runner-e2e.result/v2",+ accountingIdentity: { campaignId: "campaign", attemptId: `attempt-${attempt}` },+ executionId: "core-compatibility.runner-codex.local.message-marker", suiteId: "core-compatibility",+ attempt, status: attempt === 1 ? "failed" : "passed", profileId: "runner-codex",+ environmentId: "local", caseId: "message-marker", provider: "codex", model: "fixture", runtimeMode: "native",+ startedAt: `2026-10-20T00:0${attempt}:00Z`, finishedAt: `2026-10-20T00:0${attempt}:01Z`, durationMs: 1000,+ runIds: [`run-${attempt}`], usage: { inputTokens: 100, outputTokens: 10, costUsd: cost }, cleanup: "passed",+ ...overrides,+ };+}++describe("retained attempt accounting", () => {+ it("adds failed and successful attempts exactly once without changing the chosen verdict", () => {+ const failed = result(1, 0.01);+ const passed = result(2, 0.02);+ const campaign = buildRunnerCampaign({ campaignId: "campaign", generatedAt: passed.finishedAt,+ expected: [passed.executionId], results: [passed], attempts: [failed, passed, structuredClone(passed)] });+ expect(campaign).toMatchObject({ passed: 1, failed: 0, selected: 1,+ accounting: { scope: "retained_attempts", missingAttempts: 0, attempts: [{ attempt: 1 }, { attempt: 2 }] },+ billing: { testCount: 1, testsWithCompleteBilling: 1, reportedLlmCostUsd: 0.03,+ llm: { runCount: 2, inputTokens: 200, outputTokens: 20 } } });+ expect(campaign.results[0]!.billing!.reportedCostUsd).toBe(0.02);+ expect(campaign.suites[0]!.billing.reportedLlmCostUsd).toBe(0.03);+ expect(campaignHistoryRecord(campaign, "https://example.test").billing.reportedLlmCostUsd).toBe(0.03);+ });++ it("retains known costs from interrupted attempts and marks missing attempts", () => {+ const interrupted = result(2, 0.02, { status: "failed", cleanup: "failed" });+ const campaign = buildRunnerCampaign({ campaignId: "campaign", generatedAt: interrupted.finishedAt,+ expected: [interrupted.executionId], results: [interrupted], attempts: [interrupted] });+ expect(campaign.accounting!.missingAttempts).toBe(1);+ expect(campaign.billing.reportedLlmCostUsd).toBe(0.02);+ expect(campaign.billing.testsWithCompleteBilling).toBe(0);+ const missing = result(0, 0, { runIds: [], usage: null });+ expect(buildCampaignAccounting({ results: [missing], attempts: [] })).toMatchObject({ missingAttempts: 1, attempts: [] });+ });++ it("labels legacy selected-only bundles without inventing receipts for retries", () => {+ const selected = result(2, 0.02, { accountingIdentity: undefined });+ expect(buildCampaignAccounting({ results: [selected] })).toMatchObject({+ scope: "selected_results", missingAttempts: 1, attempts: [{ attempt: 2 }],+ });+ });++ it("excludes other campaigns and ambiguous local artifacts", () => {+ const selected = result(2, 0.02);+ const other = result(1, 0.9, { accountingIdentity: { campaignId: "other", attemptId: "attempt-1" } });+ const first = result(1, 0.01);+ expect(retainedCampaignAttempts([selected], [first, selected, other])).toEqual({ attempts: [first, selected], excludedArtifacts: 1 });+ const legacy = { ...selected, accountingIdentity: undefined };+ expect(retainedCampaignAttempts([legacy], [legacy, { ...legacy }])).toEqual({ attempts: [legacy], excludedArtifacts: 1 });+ });++ it("uses the final supervisor receipt but rejects contradictory same-version costs", () => {+ const selected = result(1, 0.02);+ const early = { ...selected, finishedAt: selected.startedAt, usage: { inputTokens: 20, costUsd: 0.01 } };+ expect(buildCampaignAccounting({ results: [selected], attempts: [selected, early] }).attempts).toMatchObject([+ { billing: { reportedCostUsd: 0.02 } },+ ]);+ expect(() => buildCampaignAccounting({ results: [selected], attempts: [selected,+ { ...selected, usage: { inputTokens: 20, costUsd: 0.1 } }] })).toThrow("Conflicting accounting receipts");+ });+});++it("requires exact selector provenance before admitting a carried workflow result", () => {+ const retained = result(1, 0.01, { accountingIdentity: { campaignId: "gha-123-1", attemptId: "a" },+ source: { sha: "source", ref: "master", workflowRunUrl: "https://github.com/paperclipai/paperclip/actions/runs/123" } });+ const artifactName = `runner-e2e-123-1-${retained.executionId}`;+ const input = { requestedCampaignId: "gha-123-2", result: retained,+ relativeResultPath: `${artifactName}/gha-123-1-${retained.executionId}/attempt-1/result.json`,+ selection: { version: 1, workflowRunId: "123", workflowRunAttempt: 2, sourceSha: retained.source!.sha,+ sourceRef: retained.source!.ref, workflowRunUrl: retained.source!.workflowRunUrl,+ selections: [{ executionId: retained.executionId, workflowAttempt: 1, artifactName }] } };+ expect(isSelectedCarryForward(input)).toBe(true);+ expect(isSelectedCarryForward({ ...input, selection: undefined })).toBe(false);+ expect(isSelectedCarryForward({ ...input, requestedCampaignId: "gha-999-2" })).toBe(false);+ expect(isSelectedCarryForward({ ...input, relativeResultPath: `unselected/${input.relativeResultPath}` })).toBe(false);+ expect(isSelectedCarryForward({ ...input, result: { ...retained, source: { ...retained.source!, sha: "other" } } })).toBe(false);+ expect(isSelectedCarryForward({ ...input, result: { ...retained, accountingIdentity: { campaignId: "gha-123-3", attemptId: "a" } } })).toBe(false);+});diff --git a/tests/runner-e2e/attempt-accounting.ts b/tests/runner-e2e/attempt-accounting.tsnew file mode 100644index 0000000000..3fea6442d7--- /dev/null+++ b/tests/runner-e2e/attempt-accounting.ts@@ -0,0 +1,99 @@+import { createHash } from "node:crypto";+import { summarizeExecutionBilling } from "./billing.js";+import type { RunnerE2EAttemptReceipt, RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js";++export function isSelectedCarryForward(input: {+ requestedCampaignId: string; result: RunnerE2EResult; relativeResultPath: string; selection: unknown;+}): boolean {+ const s = input.selection as Record<string, unknown> | null;+ if (!s || s.version !== 1 || typeof s.workflowRunId !== "string" || !/^[0-9]+$/.test(s.workflowRunId)+ || !Number.isSafeInteger(s.workflowRunAttempt) || Number(s.workflowRunAttempt) < 1+ || input.requestedCampaignId !== `gha-${s.workflowRunId}-${s.workflowRunAttempt}` || !Array.isArray(s.selections)) return false;+ const source = input.result.source;+ if (!source || !source.sha || !source.ref || !source.workflowRunUrl+ || source.sha !== s.sourceSha || source.ref !== s.sourceRef || source.workflowRunUrl !== s.workflowRunUrl) return false;+ if (input.relativeResultPath.split("/").some(part => !part || part === "." || part === "..")) return false;+ return s.selections.some(value => {+ if (!value || typeof value !== "object") return false;+ const selected = value as Record<string, unknown>;+ const attempt = selected.workflowAttempt;+ const campaign = `gha-${s.workflowRunId}-${attempt}`;+ const artifact = `runner-e2e-${s.workflowRunId}-${attempt}-${input.result.executionId}`;+ return selected.executionId === input.result.executionId && Number.isSafeInteger(attempt)+ && Number(attempt) >= 1 && Number(attempt) < Number(s.workflowRunAttempt)+ && selected.artifactName === artifact && input.result.accountingIdentity?.campaignId === campaign+ && input.relativeResultPath.startsWith(`${artifact}/${campaign}-${input.result.executionId}/`);+ });+}++function scope(result: RunnerE2EResult): string | undefined {+ return result.accountingIdentity?.campaignId ?? result.source?.workflowRunUrl ?? undefined;+}++export function retainedCampaignAttempts(+ selected: readonly RunnerE2EResult[],+ candidates: readonly RunnerE2EResult[],+): { attempts: RunnerE2EResult[]; excludedArtifacts: number } {+ const attempts = candidates.filter(candidate => selected.some(result =>+ result.executionId === candidate.executionId &&+ (scope(result) ? scope(result) === scope(candidate) : result === candidate),+ ));+ return { attempts, excludedArtifacts: candidates.length - attempts.length };+}++function receipt(result: RunnerE2EResult): RunnerE2EAttemptReceipt {+ const identity = [scope(result) ?? null, result.executionId,+ result.accountingIdentity?.attemptId ?? [result.attempt, result.startedAt]];+ return {+ id: createHash("sha256").update(JSON.stringify(identity)).digest("hex"),+ executionId: result.executionId,+ attempt: result.attempt,+ startedAt: result.startedAt,+ finishedAt: result.finishedAt,+ durationMs: result.durationMs,+ ...(result.workerTiming ? { workerTiming: result.workerTiming } : {}),+ status: result.status,+ cleanup: result.cleanup,+ billing: summarizeExecutionBilling(result),+ ...(result.feedback ? { feedback: result.feedback } : {}),+ ...(result.tracing ? { tracing: result.tracing } : {}),+ };+}++export function buildCampaignAccounting(input: {+ results: readonly RunnerE2EResult[];+ attempts?: readonly RunnerE2EResult[];+ excludedArtifacts?: number;+}): RunnerE2ECampaignAccounting {+ const receipts = new Map<string, RunnerE2EAttemptReceipt>();+ for (const result of input.attempts ?? input.results) {+ if (result.attempt === 0) continue; // Missing result placeholders are not free attempts.+ const next = receipt(result);+ const previous = receipts.get(next.id);+ if (previous && previous.finishedAt === next.finishedAt &&+ JSON.stringify(previous.billing) !== JSON.stringify(next.billing)) {+ throw new Error(`Conflicting accounting receipts for ${result.executionId} attempt ${result.attempt}`);+ }+ if (previous && previous.finishedAt === next.finishedAt && previous.workerTiming && next.workerTiming &&+ JSON.stringify(previous.workerTiming) !== JSON.stringify(next.workerTiming)) {+ throw new Error(`Conflicting timing receipts for ${result.executionId} attempt ${result.attempt}`);+ }+ if (previous && previous.finishedAt === next.finishedAt && previous.workerTiming && !next.workerTiming) next.workerTiming = previous.workerTiming;+ if (!previous || Date.parse(next.finishedAt) >= Date.parse(previous.finishedAt)) receipts.set(next.id, next);+ }+ const attempts = [...receipts.values()].sort((a, b) =>+ a.executionId.localeCompare(b.executionId) || a.attempt - b.attempt || a.id.localeCompare(b.id));+ let missingAttempts = 0;+ for (const result of input.results) {+ const retained = attempts.filter(attempt => attempt.executionId === result.executionId);+ const latest = Math.max(result.attempt, ...retained.map(attempt => attempt.attempt), 1);+ missingAttempts += Math.max(0, latest - new Set(retained.map(attempt => attempt.attempt)).size);+ }+ return {+ version: 1,+ scope: input.attempts ? "retained_attempts" : "selected_results",+ attempts,+ missingAttempts,+ excludedArtifacts: input.excludedArtifacts ?? 0,+ };+}diff --git a/tests/runner-e2e/attempt-phase.ts b/tests/runner-e2e/attempt-phase.tsnew file mode 100644index 0000000000..1c8204e4c6--- /dev/null+++ b/tests/runner-e2e/attempt-phase.ts@@ -0,0 +1,34 @@+import { randomUUID } from "node:crypto";+import { readFile, rename, rm, writeFile } from "node:fs/promises";+import type { EvalChargeIdentity } from "./cost-ledger.js";++type Identity = Pick<EvalChargeIdentity, "executionId" | "attemptId">;+interface Checkpoint extends Identity { version: 1; phase: "setup" | "agent" }++export async function createEvalAttemptPhase(file: string, identity: Identity) {+ await writeFile(file, JSON.stringify({ version: 1, ...identity, phase: "setup" }), { mode: 0o600, flag: "wx" });+}++export async function readEvalAttemptPhase(file: string, expected: Identity): Promise<EvalChargeIdentity> {+ const text = await readFile(file, "utf8");+ if (text.length > 4096) throw new Error("Invalid eval phase checkpoint");+ const value = JSON.parse(text) as Checkpoint;+ if (!value || typeof value !== "object" || Array.isArray(value)+ || Object.keys(value).some(key => !["version", "executionId", "attemptId", "phase"].includes(key))+ || value.version !== 1 || value.executionId !== expected.executionId || value.attemptId !== expected.attemptId+ || !["setup", "agent"].includes(value.phase)) throw new Error("Invalid eval phase checkpoint");+ return { ...expected, phase: value.phase };+}++export async function beginEvalAgentPhase(executionId: string, env: NodeJS.ProcessEnv = process.env) {+ const file = env.PAPERCLIP_E2E_GATEWAY_PHASE_PATH;+ if (!file && !env.PAPERCLIP_E2E_GATEWAY_BASE_URL) return;+ if (!file || !env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID) throw new Error("Missing eval phase checkpoint");+ const identity = await readEvalAttemptPhase(file, { executionId, attemptId: env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID });+ if (identity.phase === "agent") return;+ const temporary = `${file}.${randomUUID()}.tmp`;+ try {+ await writeFile(temporary, JSON.stringify({ version: 1, ...identity, phase: "agent" }), { mode: 0o600, flag: "wx" });+ await rename(temporary, file);+ } finally { await rm(temporary, { force: true }); }+}diff --git a/tests/runner-e2e/billing.test.ts b/tests/runner-e2e/billing.test.tsindex f16e2d9a60..48352815c0 100644--- a/tests/runner-e2e/billing.test.ts+++ b/tests/runner-e2e/billing.test.ts@@ -6,6 +6,8 @@ import {summarizeExecutionBilling,} from "./billing.js";import type { RunnerE2EResult } from "./types.js";+import { durationLabel, llmCostLabel } from "./report-format.js";+import { validateRetainedRunnerResult } from "./result-validation.js";function result(overrides: Partial<RunnerE2EResult> = {}): RunnerE2EResult {return {@@ -29,6 +31,80 @@ function result(overrides: Partial<RunnerE2EResult> = {}): RunnerE2EResult {}describe("runner E2E billing summaries", () => {+ it("keeps missing agent time separate from attempt duration and measured zero", () => {+ const interrupted = result({ status: "failed", durationMs: 108_850 });+ expect(summarizeExecutionBilling(interrupted).runtime.agentRunDurationMs).toBeNull();+ const noCapturedRuns = result({ ...interrupted, runIds: [],+ runtimeUsage: buildRuntimeUsage({ environmentId: "local", runs: [] }) });+ expect(noCapturedRuns.runtimeUsage?.agentRunDurationMs).toBeNull();+ expect(() => validateRetainedRunnerResult(noCapturedRuns)).not.toThrow();+ const measured = result({ runtimeUsage: buildRuntimeUsage({ environmentId: "local", runs: [+ { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:04Z" },+ { startedAt: "2026-10-21T00:00:01Z", finishedAt: "2026-10-21T00:00:05Z" },+ ] }) });+ expect(aggregateCampaignBilling([measured]).agentRunDurationMs).toBe(8000);+ expect(aggregateCampaignBilling([measured, interrupted]).agentRunDurationMs).toBeNull();+ expect(aggregateCampaignBilling([measured, noCapturedRuns]).agentRunDurationMs).toBeNull();+ expect(durationLabel(aggregateCampaignBilling([measured, interrupted]).agentRunDurationMs)).toBe("Unknown");+ const zero = buildRuntimeUsage({ environmentId: "local", runs: [+ { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:00Z" },+ ] });+ expect(zero.agentRunDurationMs).toBe(0);+ expect(durationLabel(zero.agentRunDurationMs)).toBe("0ms");+ });++ it.each([undefined, "not-a-date", "2026-10-20T23:59:59Z"])("retains unknown time for a missing or invalid end timestamp: %s", finishedAt => {+ const runtimeUsage = buildRuntimeUsage({ environmentId: "daytona", runs: [+ { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:04Z" },+ { startedAt: "2026-10-21T00:00:00Z", finishedAt },+ ], leases: [{ acquiredAt: "2026-10-21T00:00:00Z", releasedAt: finishedAt,+ metadata: { cpu: 4, memory: 4, disk: 10 } }] });+ expect(runtimeUsage).toMatchObject({ agentRunDurationMs: null, leaseDurationMs: null, costStatus: "unavailable" });+ expect(runtimeUsage.estimatedListCostUsd).toBeUndefined();+ const retained = result({ environmentId: "daytona", runtimeUsage });+ expect(() => validateRetainedRunnerResult(retained)).not.toThrow();+ expect(aggregateCampaignBilling([retained, result()]).leaseDurationMs).toBeNull();+ expect(aggregateCampaignBilling([result()]).leaseDurationMs).toBe(0);+ });++ it("reads precise provider amounts without relabeling estimates or unpriced usage", () => {+ const precise = summarizeExecutionBilling(result({ usage: {+ inputTokens: 0, outputTokens: 0, cachedInputTokens: 5,+ costUsdExact: "0.000000123", costStatus: "reported", accountingReceiptReady: true,+ } }));+ expect(precise.llm).toMatchObject({ reportedCostUsd: 0.000000123, runsWithReportedCost: 1, runsWithTokenUsage: 1 });+ const estimated = summarizeExecutionBilling(result({ usage: {+ inputTokens: 100, outputTokens: 10, costUsdExact: "0.012345678", costStatus: "estimated",+ } }));+ expect(estimated.llm).toMatchObject({ reportedCostUsd: 0, estimatedCostUsd: 0.012345678,+ runsWithReportedCost: 0, runsWithEstimatedCost: 1, costStatus: "estimated" });+ expect(estimated.observedAndEstimatedCostUsd).toBe(0.012345678);+ const unpriced = summarizeExecutionBilling(result({ usage: {+ inputTokens: 100, outputTokens: 10, costUsd: 0, costStatus: "unpriced",+ } }));+ expect(unpriced.llm).toMatchObject({ runsWithReportedCost: 0, costStatus: "unpriced" });+ });++ it("distinguishes explicit zero usage from a partial or missing receipt", () => {+ const zero = summarizeExecutionBilling(result({ usage: { inputTokens: 0, outputTokens: 0, costUsdExact: "0", accountingReceiptReady: true } }));+ expect(zero.complete).toBe(true);+ const partial = summarizeExecutionBilling(result({ usage: { inputTokens: 100, outputTokens: 10, costUsd: 0.01, accountingReceiptReady: false } }));+ expect(partial.reportedCostUsd).toBe(0.01);+ expect(partial.complete).toBe(false);+ expect(summarizeExecutionBilling(result({ usage: null })).complete).toBe(false);+ });+ it("retains a known OpenCode retry subtotal while the missing turn price stays incomplete", () => {+ const partial = result({ usage: { inputTokens: 30, outputTokens: 6, costUsd: 0.5, costStatus: "unpriced", accountingReceiptReady: true } });+ const billing = summarizeExecutionBilling(partial);+ expect(billing.llm).toMatchObject({ reportedCostUsd: 0.5, runsWithReportedCost: 1, runsWithPartialReportedCost: 1, costStatus: "partial" });+ expect(billing.observedAndEstimatedCostUsd).toBe(0.5);+ expect(billing.complete).toBe(false);+ expect(llmCostLabel(billing.reportedCostUsd, billing.llm)).toBe("$0.5000 (partial amount)");+ const campaign = aggregateCampaignBilling([partial, result({ executionId: "other", usage: { inputTokens: 1, outputTokens: 1, costUsd: 0.2 } })]);+ expect(campaign).toMatchObject({ reportedLlmCostUsd: 0.7, observedAndEstimatedCostUsd: 0.7, testsWithCompleteBilling: 1,+ llm: { runsWithReportedCost: 2, runsWithPartialReportedCost: 1, costStatus: "partial" } });+ expect(llmCostLabel(campaign.reportedLlmCostUsd, campaign.llm)).toBe("$0.7000 (partial amount)");+ });it("counts completion judge reservations and preserves unknown spend after interruption", () => {const pending = { ...reserveCompletionQuality({ sourceId: "chat", marker: "x", worker: { id: "task", status: "done", completedAt: "2026-09-01" }, documents: [{ id: "doc", issueId: "task", body: "result" }], comments: [{ id: "reply", issueId: "chat", authorAgentId: "agent", createdAt: "2026-09-02", body: "ready" }], runs: [] }, 0.5), name: "completion", expectedPass: true };const unknown = summarizeExecutionBilling(result({ completionQuality: [pending] }));diff --git a/tests/runner-e2e/billing.ts b/tests/runner-e2e/billing.tsindex 4681696a7e..35f70e8356 100644--- a/tests/runner-e2e/billing.ts+++ b/tests/runner-e2e/billing.ts@@ -32,21 +32,30 @@ export interface CampaignBillingSummary {assistant?: import("./types.js").RunnerE2EAggregateBillingSummary["assistant"];judge?: import("./types.js").RunnerE2EJudgeBillingSummary;testCount: number;- agentRunDurationMs: number;- leaseDurationMs: number;+ agentRunDurationMs: number | null;+ leaseDurationMs: number | null;llm: RunnerE2EBillingSummary["llm"];reportedLlmCostUsd: number;+ estimatedLlmCostUsd?: number;estimatedRuntimeCostUsd: number;observedAndEstimatedCostUsd: number | null;testsWithCompleteBilling: number;}/** Render observed subtotals without presenting absent measurements as zero. */-export function billingCoverageLabel(value: string, covered: number, total: number): string {+export function billingCoverageLabel(value: string, covered: number, total: number, incompleteAmount = false): string {if (!Number.isInteger(covered) || !Number.isInteger(total) || covered <= 0 || total <= 0 || covered > total) {return "Unavailable";}- return covered < total ? `${value} (partial: ${covered}/${total} runs)` : value;+ return covered < total ? `${value} (partial: ${covered}/${total} runs)` : incompleteAmount ? `${value} (partial amount)` : value;+}++function costCoverageStatus(runs: number, tokens: number, reported: number, estimated: number, partial = 0): RunnerE2EBillingSummary["llm"]["costStatus"] {+ if (partial > 0) return "partial";+ if (runs > 0 && reported === runs) return "reported";+ if (runs > 0 && reported + estimated === runs) return "estimated";+ if (reported + estimated > 0) return "partial";+ return tokens > 0 ? "unpriced" : "unavailable";}function record(value: unknown): Record<string, unknown> {@@ -92,24 +101,25 @@ function usageMeasurement(usage: Record<string, unknown>) {}function usageCostUsd(usage: Record<string, unknown>) {+ if (usage.billingType === "subscription") return undefined;const measurement = usageMeasurement(usage);+ const exact = typeof usage.costUsdExact === "string" && /^\d+(?:\.\d+)?$/.test(usage.costUsdExact)+ ? finiteNumber(Number(usage.costUsdExact)) : undefined;const direct =- firstNumber(usage, [- "cacheAdjustedCostUsd",- "costUsd",- "providerCostUsd",- ]) ??+ firstNumber(usage, ["cacheAdjustedCostUsd"]) ?? exact ??+ firstNumber(usage, ["costUsd", "providerCostUsd"]) ??firstNumber(measurement, ["cacheAdjustedCostUsd","costUsd","providerCostUsd",]);- if (direct !== undefined) return direct;+ if (direct !== undefined) return usage.costStatus === "unpriced" && direct === 0 ? undefined : direct;const cost = record(usage.cost);const currency =typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";if (currency !== "USD") return undefined;- return firstNumber(cost, ["amount", "total"]);+ const amount = firstNumber(cost, ["amount", "total"]);+ return usage.costStatus === "unpriced" && amount === 0 ? undefined : amount;}function usageEntries(@@ -144,9 +154,13 @@ function durationBetween() {const started = startedAt ? new Date(startedAt).getTime() : Number.NaN;const finished = finishedAt ? new Date(finishedAt).getTime() : Number.NaN;- return Number.isFinite(started) && Number.isFinite(finished)- ? Math.max(0, finished - started)- : 0;+ return Number.isFinite(started) && Number.isFinite(finished) && finished >= started+ ? finished - started+ : null;+}++function sumDurations(values: readonly (number | null)[]): number | null {+ return values.some(value => value === null) ? null : values.reduce<number>((total, value) => total + value!, 0);}function resourceValue(@@ -162,10 +176,9 @@ export function buildRuntimeUsage(input: {leases?: readonly RuntimeLeaseUsageInput[];fallbackFinishedAt?: string | Date;}): RunnerE2ERuntimeUsage {- const agentRunDurationMs = input.runs.reduce(- (total, run) => total + durationBetween(run.startedAt, run.finishedAt),- 0,- );+ const agentRunDurationMs = input.runs.length > 0+ ? sumDurations(input.runs.map(run => durationBetween(run.startedAt, run.finishedAt)))+ : null;if (input.environmentId === "local") {return {provider: "local",@@ -179,6 +192,7 @@ export function buildRuntimeUsage(input: {const leases = input.leases ?? [];let leaseDurationMs = 0;+ let leaseDurationsComplete = leases.length > 0;let estimatedListCostUsd = 0;let resourcesComplete = leases.length > 0;const resourceRows: Array<{ cpu: number; memory: number; disk: number }> = [];@@ -186,6 +200,11 @@ export function buildRuntimeUsage(input: {const finishedAt =lease.releasedAt ?? input.fallbackFinishedAt ?? lease.updatedAt ?? null;const durationMs = durationBetween(lease.acquiredAt, finishedAt);+ if (durationMs === null) {+ leaseDurationsComplete = false;+ resourcesComplete = false;+ continue;+ }leaseDurationMs += durationMs;const cpu = resourceValue(lease.metadata, "cpu");const memory = resourceValue(lease.metadata, "memory");@@ -209,7 +228,7 @@ export function buildRuntimeUsage(input: {return {provider: "daytona",agentRunDurationMs,- leaseDurationMs: leases.length > 0 ? leaseDurationMs : null,+ leaseDurationMs: leaseDurationsComplete ? leaseDurationMs : null,leaseCount: leases.length,...(commonResource("cpu") === undefined? {}@@ -243,7 +262,7 @@ export function fallbackRuntimeUsage(if (result.runtimeUsage) return result.runtimeUsage;return {provider: result.environmentId,- agentRunDurationMs: result.durationMs,+ agentRunDurationMs: null,leaseDurationMs: null,leaseCount: 0,costStatus:@@ -269,7 +288,11 @@ export function summarizeExecutionBilling(let cachedInputTokens = 0;let runsWithTokenUsage = 0;let runsWithReportedCost = 0;+ let runsWithEstimatedCost = 0;+ let runsWithPartialReportedCost = 0;let reportedCostUsd = 0;+ let estimatedCostUsd = 0;+ let incompleteReceipt = false;for (const usage of entries) {if (!usage) continue;const measurement = usageMeasurement(usage);@@ -290,22 +313,24 @@ export function summarizeExecutionBilling(inputTokens += input;outputTokens += output;cachedInputTokens += cached;- if (input > 0 || output > 0 || cached > 0) runsWithTokenUsage += 1;+ const completeTokens = firstNumber(measurement, ["inputTokens", "input", "promptTokens"]) !== undefined &&+ firstNumber(measurement, ["outputTokens", "output", "completionTokens"]) !== undefined && usage.accountingReceiptReady !== false;+ if (completeTokens) runsWithTokenUsage += 1;+ if (usage.accountingReceiptReady === false) incompleteReceipt = true;const costUsd = usageCostUsd(usage);- if (costUsd !== undefined && (costUsd > 0 || input > 0 || output > 0)) {- runsWithReportedCost += 1;- reportedCostUsd += costUsd;+ if (costUsd !== undefined) {+ if (usage.costStatus === "estimated") {+ runsWithEstimatedCost += 1;+ estimatedCostUsd += costUsd;+ } else {+ runsWithReportedCost += 1;+ if (usage.costStatus === "unpriced") runsWithPartialReportedCost += 1;+ reportedCostUsd += costUsd;+ }}}const runCount = Math.max(requestedRunCount, entries.length);- const costStatus =- runsWithReportedCost === runCount- ? "reported"- : runsWithReportedCost > 0- ? "partial"- : runsWithTokenUsage > 0- ? "unpriced"- : "unavailable";+ const costStatus = costCoverageStatus(runCount, runsWithTokenUsage, runsWithReportedCost, runsWithEstimatedCost, runsWithPartialReportedCost);const runtime = { ...fallbackRuntimeUsage(result) };if (browserOnly) runtime.agentRunDurationMs = 0;const estimatedRuntimeCostUsd = runtime.estimatedListCostUsd ?? 0;@@ -315,12 +340,13 @@ export function summarizeExecutionBilling(// The operations receipt covers setup and agent requests for ONE owning attempt.// Descendants reuse the fixture context without copying its receipt or charges.const operations = result.productJourney?.operations;- const modelSubtotal = operations ? operations.modelCostUsd : reportedCostUsd;+ const modelSubtotal = operations ? operations.modelCostUsd : reportedCostUsd + estimatedCostUsd;const complete =+ !incompleteReceipt && runsWithPartialReportedCost === 0 &&(!quality || quality.estimatedCostUsd !== null) &&(!operations || operations.modelCostUsd !== null) &&runsWithTokenUsage === runCount &&- runsWithReportedCost === runCount &&+ runsWithReportedCost + runsWithEstimatedCost === runCount &&runtime.costStatus !== "unavailable";return {...(result.publicMcp ? { assistant: result.publicMcp } : {}),@@ -328,11 +354,14 @@ export function summarizeExecutionBilling(runCount,runsWithTokenUsage,runsWithReportedCost,+ runsWithEstimatedCost,+ runsWithPartialReportedCost,inputTokens,outputTokens,cachedInputTokens,totalTokens: inputTokens + cachedInputTokens + outputTokens,reportedCostUsd,+ estimatedCostUsd,costStatus,},runtime,@@ -347,7 +376,16 @@ export function summarizeExecutionBilling(export function aggregateCampaignBilling(results: readonly RunnerE2EResult[],): CampaignBillingSummary {- const summaries = results.map(summarizeExecutionBilling);+ return aggregateBillingSummaries(results.map(result => ({+ executionId: result.executionId, billing: summarizeExecutionBilling(result),+ })));+}++export function aggregateBillingSummaries(+ entries: readonly { executionId: string; billing: RunnerE2EBillingSummary }[],+): CampaignBillingSummary {+ const summaries = entries.map(entry => entry.billing);+ const executionIds = [...new Set(entries.map(entry => entry.executionId))];const runCount = summaries.reduce((total, summary) => total + summary.llm.runCount,0,@@ -364,6 +402,9 @@ export function aggregateCampaignBilling((total, summary) => total + summary.reportedCostUsd,0,);+ const runsWithEstimatedCost = summaries.reduce((sum, summary) => sum + (summary.llm.runsWithEstimatedCost ?? 0), 0);+ const runsWithPartialReportedCost = summaries.reduce((sum, summary) => sum + (summary.llm.runsWithPartialReportedCost ?? 0), 0);+ const estimatedLlmCostUsd = summaries.reduce((sum, summary) => sum + (summary.llm.estimatedCostUsd ?? 0), 0);const estimatedRuntimeCostUsd = summaries.reduce((total, summary) => total + summary.estimatedRuntimeCostUsd,0,@@ -386,19 +427,16 @@ export function aggregateCampaignBilling(reservedCostUsd: judges.reduce((n, q) => n + q.reservedCostUsd, 0),attemptsWithUnknownUsage: judges.filter(q => q.estimatedCostUsd === null).length,} } : {}),- testCount: results.length,- agentRunDurationMs: summaries.reduce(- (total, summary) => total + summary.runtime.agentRunDurationMs,- 0,- ),- leaseDurationMs: summaries.reduce(- (total, summary) => total + (summary.runtime.leaseDurationMs ?? 0),- 0,- ),+ testCount: executionIds.length,+ agentRunDurationMs: sumDurations(summaries.map(summary => summary.runtime.agentRunDurationMs)),+ leaseDurationMs: sumDurations(summaries.filter(summary => summary.runtime.provider === "daytona")+ .map(summary => summary.runtime.leaseDurationMs)),llm: {runCount,runsWithTokenUsage,runsWithReportedCost,+ runsWithEstimatedCost,+ runsWithPartialReportedCost,inputTokens: summaries.reduce((total, summary) => total + summary.llm.inputTokens,0,@@ -416,19 +454,14 @@ export function aggregateCampaignBilling(0,),reportedCostUsd: reportedLlmCostUsd,- costStatus:- runsWithReportedCost === runCount- ? "reported"- : runsWithReportedCost > 0- ? "partial"- : runsWithTokenUsage > 0- ? "unpriced"- : "unavailable",+ estimatedCostUsd: estimatedLlmCostUsd,+ costStatus: costCoverageStatus(runCount, runsWithTokenUsage, runsWithReportedCost, runsWithEstimatedCost, runsWithPartialReportedCost),},reportedLlmCostUsd,+ estimatedLlmCostUsd,estimatedRuntimeCostUsd,observedAndEstimatedCostUsd: summaries.some(s => s.observedAndEstimatedCostUsd === null) ? null : summaries.reduce((total, summary) => total + (summary.observedAndEstimatedCostUsd ?? 0), 0),- testsWithCompleteBilling: summaries.filter((summary) => summary.complete)- .length,+ testsWithCompleteBilling: executionIds.filter(id =>+ entries.filter(entry => entry.executionId === id).every(entry => entry.billing.complete)).length,};}diff --git a/tests/runner-e2e/braintrust-experiment-import.ts b/tests/runner-e2e/braintrust-experiment-import.tsnew file mode 100644index 0000000000..eef8f3598c--- /dev/null+++ b/tests/runner-e2e/braintrust-experiment-import.ts@@ -0,0 +1,45 @@+import { readFile, rename, writeFile } from "node:fs/promises";+import path from "node:path";+import { randomUUID } from "node:crypto";+import { evalBraintrustConfig } from "./braintrust.js";+import { buildBraintrustExperiment, importBraintrustExperiment } from "./braintrust-experiment.js";++const [campaignFile, originalPathsFile, outputFile, mode] = process.argv.slice(2);+if (!campaignFile || !originalPathsFile || !outputFile || ![undefined, "--dry-run"].includes(mode)) {+ throw new Error("Usage: braintrust-experiment-import.ts <normalized-results.json> <original-result-paths.json> <receipt.json> [--dry-run]");+}+let totalBytes = 0;+const readJson = async (file: string) => {+ const bytes = await readFile(file);+ totalBytes += bytes.length;+ if (bytes.length > 8 * 1024 * 1024 || totalBytes > 64 * 1024 * 1024) throw new Error("Experiment input exceeds its file or campaign size limit");+ return JSON.parse(bytes.toString("utf8"));+};+const paths = await readJson(originalPathsFile);+if (!Array.isArray(paths) || paths.length === 0 || paths.length > 10_000+ || paths.some(file => typeof file !== "string" || !path.isAbsolute(file))) throw new Error("Expected explicit absolute original result paths");+if ([campaignFile, originalPathsFile, ...paths].some(file => path.resolve(file) === path.resolve(outputFile))) {+ throw new Error("Experiment output must not replace input evidence");+}+const campaign = await readJson(campaignFile), originals = [];+for (const file of paths) originals.push(await readJson(file));+const projectId = process.env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "";+const plan = buildBraintrustExperiment(projectId, campaign, originals);+const persist = async (value: unknown) => {+ const temporary = `${outputFile}.${randomUUID()}.tmp`;+ await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx", mode: 0o600 });+ await rename(temporary, outputFile);+};+if (mode === "--dry-run") {+ await persist(plan);+ console.log(`Prepared ${plan.events.length} retained rows; no network requests made.`);+} else {+ const config = evalBraintrustConfig(process.env);+ if (!config) throw new Error("Braintrust project and ingestion credential required");+ const previous = await readJson(outputFile).catch(error => {+ if (error?.code === "ENOENT") return undefined;+ throw error;+ });+ const receipt = await importBraintrustExperiment(config, campaign, originals, persist, fetch, previous);+ console.log(`Braintrust accepted ${receipt.acceptedRows} retained rows in experiment ${receipt.experimentId}; indexed verification remains required.`);+}diff --git a/tests/runner-e2e/braintrust-experiment.test.ts b/tests/runner-e2e/braintrust-experiment.test.tsnew file mode 100644index 0000000000..41c61fc5b2--- /dev/null+++ b/tests/runner-e2e/braintrust-experiment.test.ts@@ -0,0 +1,328 @@+import { expect, it, vi } from "vitest";+import { execFile } from "node:child_process";+import { promisify } from "node:util";+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { buildBraintrustExperiment, importBraintrustExperiment, renderBraintrustExperiment, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";+import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";+import { buildRunnerCampaign } from "./history.js";+import { runnerMatrix } from "./catalog.js";+import { emptyFeedbackSummary } from "./feedback.js";+import { openExperimentIngestion } from "./experiment-ingestion.js";+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";++const projectId = "11111111-1111-4111-8111-111111111111";+const experimentId = "22222222-2222-4222-8222-222222222222";+function fixture() {+ const cell = runnerMatrix.find(cell => cell.id === "core-compatibility.runner-codex.local.message-marker")!;+ const first: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id,+ suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash,+ accountingIdentity: { campaignId: "retained-pilot", attemptId: "first" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null },+ attempt: 1, status: "failed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id,+ environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,+ startedAt: "2026-10-20T00:00:00.000Z", finishedAt: "2026-10-20T00:00:01.000Z", durationMs: 1000,+ error: "private-canary-error", usage: { privateCanary: "private-canary-usage" }, feedback: emptyFeedbackSummary() };+ const second: RunnerE2EResult = { ...structuredClone(first), source: { ...first.source!, sha: "b".repeat(40) },+ accountingIdentity: { campaignId: "retained-pilot", attemptId: "second" }, attempt: 2, status: "passed", error: undefined,+ startedAt: "2026-10-20T00:01:00.000Z", finishedAt: "2026-10-20T00:01:01.000Z",+ tracing: { projectId, traceId: "c".repeat(32), spanId: "d".repeat(16), status: "accepted", verification: "unverified", nativeConfiguration: "configured" } };+ const originals = [first, second];+ const campaign = buildRunnerCampaign({ campaignId: "retained-pilot", generatedAt: second.finishedAt,+ expected: [cell.id], results: [second], attempts: originals });+ return { campaign, originals };+}++it("imports all attempts with their own source and original grades, excluding private evidence and duplicate cost metrics", () => {+ const { campaign, originals } = fixture();+ const plan = buildBraintrustExperiment(projectId, campaign, originals);+ expect(plan.events).toHaveLength(2);+ expect(plan.events.map(row => row.scores.product_pass).sort()).toEqual([0, 1]);+ expect(plan.events.map(row => row.input.source_revision).sort()).toEqual(["a".repeat(40), "b".repeat(40)]);+ expect(JSON.stringify(plan)).not.toMatch(/private-canary|estimated_cost|prompt_tokens/);+ expect(plan.events.every(row => row.metadata.request_ledger === null)).toBe(true);+ expect(plan.events.find(row => row.metadata.attempt_id === "second")!.metadata.trace).toMatchObject({ root_span_id: "c".repeat(32) });+ expect(buildBraintrustExperiment(projectId, { ...campaign, generatedAt: "2026-10-21T00:00:00Z" }, [...originals].reverse())).toEqual(plan);+ expect(buildBraintrustExperiment(projectId, campaign, [...originals, originals[0]!])).toEqual(plan);+});++it("imports retained grades through the scheduled transport allowance without changing the evidence", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "bounded-experiment-import-"));+ const { campaign, originals } = fixture(), before = structuredClone({ campaign, originals });+ const plan = buildBraintrustExperiment(projectId, campaign, originals), receiptPath = path.join(root, "ingestion.jsonl");+ const meter = await openExperimentIngestion({ projectId, campaignId: campaign.campaignId, receiptPath, deadlineAt: Date.now() + 60_000,+ transport: async (url, init) => String(url).endsWith("/experiment")+ ? Response.json({ ...JSON.parse(String(init!.body)), id: experimentId })+ : String(url).endsWith("/insert") ? Response.json({ row_ids: plan.events.map(row => row.id) }) : Response.json({}) });+ try {+ expect(await importBraintrustExperiment({ projectId, apiKey: "private-fixture" }, campaign, originals, async () => {}, meter.transport))+ .toMatchObject({ status: "accepted", acceptedRows: 2, expectedRows: 2 });+ await meter.close();+ const rows = (await readFile(receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line));+ expect(rows.at(-1)).toMatchObject({ admittedRequests: 3, admittedScores: 4, unresolvedRequests: 0 });+ expect({ campaign, originals }).toEqual(before);+ } finally { await meter.close(); await rm(root, { recursive: true, force: true }); }+});++it.each(["missing", "foreign", "modified-grade", "ambiguous-source", "missing-source", "foreign-trace", "missing-hash"])("rejects %s evidence before network requests", async (kind) => {+ const { campaign, originals } = fixture();+ if (kind === "missing") originals.pop();+ if (kind === "foreign") originals[0]!.accountingIdentity!.campaignId = "other";+ if (kind === "modified-grade") originals[0]!.status = "passed";+ if (kind === "ambiguous-source") originals.push({ ...originals[0]!, source: { ...originals[0]!.source!, sha: "c".repeat(40) } });+ if (kind === "missing-source") originals[0]!.source = undefined;+ if (kind === "foreign-trace") originals[1]!.tracing!.projectId = experimentId;+ if (kind === "missing-hash") originals[0]!.suiteDefinitionHash = undefined;+ const transport = vi.fn<typeof fetch>();+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, transport)).rejects.toThrow();+ expect(transport).not.toHaveBeenCalled();+});++it("changes experiment identity for changed evidence while keeping unknown accounting unknown", () => {+ const { campaign, originals } = fixture();+ const before = buildBraintrustExperiment(projectId, campaign, originals);+ campaign.accounting!.missingAttempts = 1;+ const after = buildBraintrustExperiment(projectId, campaign, originals);+ expect(after.payloadHash).not.toBe(before.payloadHash);+ expect(after.metadata.request_ledger).toBeNull();+ expect(after.metadata.missing_attempts).toBe(1);+});++it("imports carried-forward receipts with their original campaign, revision, and grades", () => {+ const { campaign, originals } = fixture();+ const report = { ...campaign, campaignId: "later-ci-attempt" };+ const before = structuredClone(originals);+ const plan = buildBraintrustExperiment(projectId, report, originals);+ expect(plan.metadata.campaign_id).toBe("later-ci-attempt");+ expect(plan.events.every(row => row.metadata.campaign_id === "retained-pilot")).toBe(true);+ expect(plan.events.map(row => row.input.source_revision).sort()).toEqual(["a".repeat(40), "b".repeat(40)]);+ expect(plan.events.map(row => row.scores.product_pass).sort()).toEqual([0, 1]);+ expect(plan.events.map(row => row.id)).toEqual(buildBraintrustExperiment(projectId, campaign, originals).events.map(row => row.id));+ expect(originals).toEqual(before);+ const wrong = structuredClone(originals);+ wrong[0]!.accountingIdentity!.campaignId = report.campaignId;+ expect(() => buildBraintrustExperiment(projectId, report, wrong)).toThrow();+});++it("imports original grades when report evidence validation changes the selected outcome", () => {+ const { campaign, originals } = fixture();+ const adjusted = structuredClone(campaign);+ adjusted.results[0]!.status = "failed";+ adjusted.results[0]!.error = "Missing final-state.png";+ const plan = buildBraintrustExperiment(projectId, adjusted, originals);+ expect(plan.events.find(row => row.metadata.attempt_id === "second")!.scores.product_pass).toBe(1);+ expect(plan).toEqual(buildBraintrustExperiment(projectId, campaign, originals));+ expect(adjusted.results[0]!.status).toBe("failed");+ const changed = structuredClone(originals);+ changed[1]!.status = "failed";+ expect(() => buildBraintrustExperiment(projectId, adjusted, changed)).toThrow();+});++it("accepts current catalog model names including Cursor qualification parameters", () => {+ for (const model of new Set(runnerMatrix.map(cell => cell.profile.model))) {+ const { campaign, originals } = fixture();+ const changed = originals.map(original => ({ ...original, model }));+ const selected = { ...campaign, results: campaign.results.map(result => ({ ...result, model })) };+ const plan = buildBraintrustExperiment(projectId, selected, changed);+ expect(plan.events.every(row => row.input.model === model)).toBe(true);+ }+});++it.each(["", "model\nprivate", "model\u0000private", "model<private>", "a".repeat(257)])("rejects malformed model names %j", model => {+ const { campaign, originals } = fixture();+ expect(() => buildBraintrustExperiment(projectId, campaign, originals.map(result => ({ ...result, model })))).toThrow();+});++it("keeps precise ledger amounts and reservations separate from trace metrics and unobserved attempts", () => {+ const { campaign, originals } = fixture();+ campaign.operations = { version: 1, scope: "shared_gateway_ledger", ledgerCampaignId: campaign.campaignId,+ journalHash: "c".repeat(64), records: 5, deadlineAt: "2026-10-21T01:00:00Z", limitUsd: "100.000000000",+ knownUsd: "0.000000001", heldUsd: "4.000000000", committedUsd: "4.000000001", requests: 1,+ unresolvedRequests: 1, conflictingRequests: 0, inflight: 0,+ breakdown: { estimated: "0.000000001", reported: "0.000000000", reconciled: "0.000000000", allocated: "0.000000000" },+ attempts: [{ executionId: originals[0]!.executionId, attemptId: "first", requests: 1, unresolvedRequests: 1,+ knownUsd: "0.000000001", heldUsd: "4.000000000" }] };+ const plan = buildBraintrustExperiment(projectId, campaign, originals);+ expect(plan.metadata.request_ledger).toMatchObject({ known_usd: "0.000000001", additional_held_usd: "4.000000000" });+ expect(plan.events.find(row => row.metadata.attempt_id === "first")!.metadata.request_ledger).toMatchObject({ unresolved_requests: 1 });+ expect(plan.events.find(row => row.metadata.attempt_id === "second")!.metadata.request_ledger).toBeNull();+ expect(plan.events.every(row => Object.keys(row.metrics).sort().join() === "end,start")).toBe(true);+ const carried = buildBraintrustExperiment(projectId, {+ ...campaign, campaignId: "later-ci-attempt",+ operations: { ...campaign.operations, ledgerCampaignId: "later-ci-attempt" },+ }, originals);+ expect(carried.metadata.request_ledger).toMatchObject({ requests: 1 });+ expect(carried.events.every(row => row.metadata.request_ledger === null)).toBe(true);+});++function receiver(campaign: ReturnType<typeof fixture>["campaign"], originals: RunnerE2EResult[]) {+ const plan = buildBraintrustExperiment(projectId, campaign, originals);+ return vi.fn<typeof fetch>(async (url, init) => {+ expect(String(url).startsWith("https://api.braintrust.dev/v1/")).toBe(true);+ expect(init?.redirect).toBe("error");+ if (String(url).endsWith("/summarize")) return Response.json({ experiment_name: plan.name,+ project_url: "https://www.braintrust.dev/app/Example/p/Evals", experiment_url: `https://www.braintrust.dev/app/Example/p/Evals/experiments/${plan.name}` });+ const sent = JSON.parse(String(init?.body));+ if (String(url).endsWith("/experiment")) {+ expect(sent).toMatchObject({ public: false, ensure_new: false, project_id: projectId, name: plan.name });+ return Response.json({ ...sent, id: experimentId });+ }+ return Response.json({ row_ids: sent.events.map((row: { id: string }) => row.id) });+ });+}++it("retries identical evidence using the same private experiment and stable row IDs without invoking agents", async () => {+ const { campaign, originals } = fixture();+ const transport = receiver(campaign, originals), receipts: BraintrustExperimentReceipt[] = [];+ for (let i = 0; i < 2; i++) await importBraintrustExperiment({ projectId, apiKey: "private-key" }, campaign, originals,+ async receipt => { receipts.push(receipt); }, transport);+ expect(receipts.map(receipt => receipt.status)).toEqual(["created", "accepted", "accepted", "created", "accepted", "accepted"]);+ expect(receipts[2]).toEqual(receipts[5]);+ expect(receipts[1]).toMatchObject({ acceptedRows: 2, expectedRows: 2, verification: "unverified" });+ expect(transport.mock.calls[0]![1]!.body).toBe(transport.mock.calls[3]![1]!.body);+ expect(transport.mock.calls[1]![1]!.body).toBe(transport.mock.calls[4]![1]!.body);+});++it("preserves an evidence-bound experiment link through report regeneration and rejects stale or unsafe links", async () => {+ const { campaign, originals } = fixture();+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals));+ campaign.braintrustExperiment = receipt;+ const root = await mkdtemp(path.join(tmpdir(), "experiment-report-"));+ try {+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });+ expect(JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")).braintrustExperiment).toEqual(receipt);+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain(receipt.experimentUrl);+ expect(() => renderBraintrustExperiment({ ...campaign, braintrustExperiment: { ...receipt, experimentUrl: "https://untrusted.example/app/secret" } })).toThrow();+ expect(() => renderBraintrustExperiment({ ...campaign, braintrustExperiment: { ...receipt, privateNote: "private-canary" } as never })).toThrow();+ campaign.accounting!.missingAttempts++;+ expect(() => renderBraintrustExperiment(campaign)).toThrow();+ } finally { await rm(root, { recursive: true, force: true }); }+});++it("rebuilds a report with a missing cell and its saved experiment without rerunning retained attempts", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "experiment-missing-report-"));+ const repositoryRoot = path.resolve(import.meta.dirname, "../..");+ const original = fixture().originals[1]!;+ const missing = runnerMatrix.find(cell => cell.id !== original.executionId)!;+ const input = path.join(root, "input"), attempt = path.join(input, "attempt");+ const receiptPath = path.join(root, "experiment.json");+ async function report(output: string, savedReceipt?: string) {+ const result = await promisify(execFile)(process.execPath,+ [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {+ cwd: repositoryRoot,+ env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: input, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,+ PAPERCLIP_E2E_CAMPAIGN_ID: "retained-pilot", PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([original.executionId, missing.id]),+ PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT: savedReceipt },+ }).catch(error => error as { code: number; stderr: string });+ // Missing coverage still fails the campaign, but must not prevent rendering it.+ expect(result).toMatchObject({ code: 1 });+ expect(result.stderr).not.toContain("Invalid retained experiment evidence");+ return JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")) as RunnerE2ECampaign;+ }+ try {+ await mkdir(attempt, { recursive: true });+ const raw = JSON.stringify(original);+ await writeFile(path.join(attempt, "result.json"), raw);+ await writeFile(path.join(attempt, "final-state.png"), "fixture-png");+ await writeFile(path.join(attempt, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] }));+ const before = await report(path.join(root, "before"));+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "fixture-key" }, before, [original], async () => {}, receiver(before, [original]));+ await writeFile(receiptPath, JSON.stringify(receipt));+ const after = await report(path.join(root, "after"), receiptPath);+ expect(after.results.find(result => result.attempt === 0)!.startedAt).not.toBe(before.results.find(result => result.attempt === 0)!.startedAt);+ expect(after.braintrustExperiment).toEqual(receipt);+ expect(after.accounting).toEqual(before.accounting);+ expect(await readFile(path.join(root, "after", "index.html"), "utf8")).toContain(receipt.experimentUrl);+ expect(await readFile(path.join(attempt, "result.json"), "utf8")).toBe(raw);+ const changed = structuredClone(after);+ changed.results.find(result => result.attempt > 0)!.startedAt = "2026-10-20T00:00:30.000Z";+ expect(() => renderBraintrustExperiment(changed)).toThrow();+ } finally { await rm(root, { recursive: true, force: true }); }+});++it("checkpoints accepted batches before a later insert fails", async () => {+ const base = fixture().originals[0]!;+ const originals = Array.from({ length: 101 }, (_, i) => ({ ...base, attempt: i + 1,+ accountingIdentity: { campaignId: "retained-pilot", attemptId: `attempt-${i}` } }));+ const campaign = buildRunnerCampaign({ campaignId: "retained-pilot", generatedAt: base.finishedAt,+ expected: [base.executionId], results: [originals[100]!], attempts: originals });+ const normal = receiver(campaign, originals), receipts: BraintrustExperimentReceipt[] = [];+ let calls = 0;+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals,+ async receipt => { receipts.push(receipt); }, async (url, init) => {+ if (++calls === 3) throw new Error("private-network-error");+ return normal(url, init);+ })).rejects.toThrow("Remote details withheld");+ expect(receipts).toHaveLength(2);+ expect(receipts[1]).toMatchObject({ status: "created", acceptedRows: 100, expectedRows: 101 });+ const previous = structuredClone(receipts[1]!);+ calls = 0;+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals,+ async receipt => { receipts.push(receipt); }, async (url, init) => {+ if (++calls === 2) throw new Error("retry insert failed");+ return normal(url, init);+ }, previous)).rejects.toThrow("Remote details withheld");+ expect(receipts).toHaveLength(2);+ expect(receipts.at(-1)).toEqual(previous);+ await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals,+ async receipt => { receipts.push(receipt); }, normal, previous);+ expect(receipts.slice(2).map(receipt => receipt.acceptedRows)).toEqual([100, 101, 101]);+ expect(receipts.at(-1)!.status).toBe("accepted");+});++it.each(["source", "definition"])("rejects a saved link and original import after selected %s provenance changes", async kind => {+ const { campaign, originals } = fixture();+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals));+ const changed = structuredClone(campaign);+ changed.braintrustExperiment = receipt;+ if (kind === "source") changed.results[0]!.source!.sha = "e".repeat(40);+ else changed.results[0]!.suiteDefinitionHash = "f".repeat(64);+ expect(() => renderBraintrustExperiment(changed)).toThrow();+ expect(() => buildBraintrustExperiment(projectId, changed, originals)).toThrow();+ expect(buildBraintrustExperiment(projectId, campaign, originals).events.map(row => row.input.source_revision).sort())+ .toEqual(["a".repeat(40), "b".repeat(40)]);+});++it("propagates a failed receipt save after successful summary lookup while retaining the accepted checkpoint", async () => {+ const { campaign, originals } = fixture(), receipts: BraintrustExperimentReceipt[] = [];+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async receipt => {+ if (receipt.experimentUrl) throw new Error("disk write failed");+ receipts.push(receipt);+ }, receiver(campaign, originals))).rejects.toThrow("import incomplete");+ expect(receipts.at(-1)).toMatchObject({ status: "accepted", acceptedRows: 2 });+ expect(receipts.at(-1)!.experimentUrl).toBeUndefined();+});++it("rejects conflicting retry receipts before overwriting saved evidence", async () => {+ const { campaign, originals } = fixture();+ const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals));+ const persist = vi.fn(), transport = vi.fn<typeof fetch>();+ await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, persist, transport,+ { ...receipt, projectId: experimentId })).rejects.toThrow();+ expect(persist).not.toHaveBeenCalled();+ expect(transport).not.toHaveBeenCalled();+});++it.each(["public", "wrong-project", "wrong-hash", "partial-insert", "receiver-error", "oversized"])("retains safe incomplete status for %s", async kind => {+ const { campaign, originals } = fixture(), receipts: BraintrustExperimentReceipt[] = [];+ const normal = receiver(campaign, originals);+ const transport: typeof fetch = async (url, init) => {+ const response = await normal(url, init), value = await response.json() as Record<string, unknown>;+ if (String(url).endsWith("/experiment")) {+ if (kind === "public") value.public = true;+ if (kind === "wrong-project") value.project_id = experimentId;+ if (kind === "wrong-hash") value.metadata = { payload_hash: "a".repeat(64) };+ if (kind === "oversized") return new Response("x".repeat(1_048_577));+ } else {+ if (kind === "partial-insert") value.row_ids = [];+ if (kind === "receiver-error") return new Response("private-upstream-failure", { status: 503 });+ }+ return Response.json(value);+ };+ await expect(importBraintrustExperiment({ projectId, apiKey: "private-key" }, campaign, originals,+ async receipt => { receipts.push(receipt); }, transport)).rejects.toThrow("Remote details withheld");+ expect(receipts.some(receipt => receipt.status === "accepted")).toBe(false);+ expect(JSON.stringify(receipts)).not.toMatch(/private-key|private-upstream/);+ if (["partial-insert", "receiver-error"].includes(kind)) expect(receipts[0]).toMatchObject({ status: "created", acceptedRows: 0 });+});diff --git a/tests/runner-e2e/braintrust-experiment.ts b/tests/runner-e2e/braintrust-experiment.tsnew file mode 100644index 0000000000..4b2859e17d--- /dev/null+++ b/tests/runner-e2e/braintrust-experiment.ts@@ -0,0 +1,207 @@+import { createHash } from "node:crypto";+import { buildCampaignAccounting } from "./attempt-accounting.js";+import { exportBraintrustAttemptParent, type EvalBraintrustConfig } from "./braintrust.js";+import { validateCampaignFeedbackReview } from "./feedback-dispositions.js";+import { validateEvalOperationsReport } from "./operations-report.js";+import { html, isPublicReportTimestamp } from "./report-format.js";+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";++const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");+const identifier = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/.test(value);+const modelName = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/@=,\[\]-]{0,255}$/.test(value);+const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0;+const invalid = (): never => { throw new Error("Invalid retained experiment evidence"); };++function attemptProjection(result: RunnerE2EResult, projectId: string) {+ if (result.schema !== "paperclip.runner-e2e.result/v2"+ || ![result.executionId, result.suiteId, result.profileId, result.caseId, result.provider,+ result.accountingIdentity?.campaignId, result.accountingIdentity?.attemptId].every(identifier)+ || !modelName(result.model)+ || !/^[0-9a-f]{64}$/.test(result.suiteDefinitionHash ?? "") || !/^[0-9a-f]{40}$/.test(result.source?.sha ?? "")+ || !count(result.attempt) || result.attempt < 1 || !count(result.durationMs)+ || !["passed", "failed"].includes(result.status) || !["passed", "failed", "not_started"].includes(result.cleanup)+ || !["native", "legacy"].includes(result.runtimeMode) || !["local", "daytona"].includes(result.environmentId)+ || !isPublicReportTimestamp(result.startedAt) || !isPublicReportTimestamp(result.finishedAt)+ || Date.parse(result.finishedAt) < Date.parse(result.startedAt)) invalid();+ const feedback = result.feedback;+ if (feedback && (![feedback.complaints, feedback.suggestions, feedback.invocations?.failed, feedback.invocations?.unknown].every(count)+ || !["complete", "partial", "unavailable", "not_started"].includes(feedback.collection)+ || !["complete", "partial", "unavailable", "not_started"].includes(feedback.invocationCollection))) invalid();+ const trace = result.tracing;+ if (trace) {+ exportBraintrustAttemptParent(trace);+ if (trace.projectId !== projectId || !["pending", "accepted", "failed"].includes(trace.status)) invalid();+ }+ return {+ input: { execution_id: result.executionId, suite_id: result.suiteId, profile_id: result.profileId,+ case_id: result.caseId, provider: result.provider, model: result.model, environment: result.environmentId,+ runtime: result.runtimeMode, source_revision: result.source!.sha, definition_hash: result.suiteDefinitionHash },+ output: { product: result.status, cleanup: result.cleanup },+ scores: { product_pass: Number(result.status === "passed"), cleanup_pass: Number(result.cleanup === "passed") },+ metrics: { start: Date.parse(result.startedAt) / 1000, end: Date.parse(result.finishedAt) / 1000 },+ metadata: { campaign_id: result.accountingIdentity!.campaignId, attempt_id: result.accountingIdentity!.attemptId,+ attempt_number: result.attempt, wall_duration_ms: result.durationMs, grader: "tests/runner-e2e/runner.spec.ts",+ feedback: feedback ? { collection: feedback.collection, invocation_collection: feedback.invocationCollection,+ complaints: feedback.complaints, suggestions: feedback.suggestions,+ failed_submissions: feedback.invocations.failed, unresolved_submissions: feedback.invocations.unknown } : null,+ trace: trace ? { project_id: trace.projectId, root_span_id: trace.traceId, span_id: trace.spanId,+ export_status: trace.status, indexed_verification: "unverified" } : null },+ span_attributes: { name: result.executionId, type: "eval" },+ };+}++export function buildBraintrustExperiment(projectId: string, campaign: RunnerE2ECampaign, originals: readonly RunnerE2EResult[]) {+ if (!uuid.test(projectId) || campaign.schema !== "paperclip.runner-e2e.campaign/v2" || !identifier(campaign.campaignId)+ || campaign.accounting?.scope !== "retained_attempts" || !Array.isArray(campaign.accounting.attempts)+ || !count(campaign.accounting.missingAttempts) || !Array.isArray(campaign.expected)+ || campaign.expected.length === 0 || !campaign.expected.every(identifier)+ || new Set(campaign.expected).size !== campaign.expected.length || !Array.isArray(originals)+ || originals.length > 10_000 || campaign.accounting.attempts.length === 0) invalid();+ if (campaign.operations) validateEvalOperationsReport(campaign.operations);+ if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview);+ const accounting = campaign.accounting!;+ const operations = campaign.operations?.status !== "unavailable" ? campaign.operations : undefined;+ if (operations && operations.ledgerCampaignId !== campaign.campaignId) invalid();+ const candidates = new Map<string, { original: RunnerE2EResult; projection: ReturnType<typeof attemptProjection> }>();+ for (const original of originals) {+ const projection = attemptProjection(original, projectId);+ if (!campaign.expected.includes(original.executionId)) invalid();+ const receipt = buildCampaignAccounting({ results: [original] }).attempts[0]!;+ const retained = accounting.attempts.find(attempt => attempt.id === receipt.id);+ if (!retained || JSON.stringify(retained) !== JSON.stringify(receipt)) invalid();+ const previous = candidates.get(receipt.id);+ if (previous && digest(previous.projection) !== digest(projection)) invalid();+ candidates.set(receipt.id, { original, projection });+ }+ if (candidates.size !== accounting.attempts.length) invalid();+ for (const selected of campaign.results.filter(result => result.attempt > 0)) {+ const id = buildCampaignAccounting({ results: [selected] }).attempts[0]!.id;+ const original = candidates.get(id);+ const selectedProjection = attemptProjection(selected, projectId);+ if (!original || digest(original.projection.input) !== digest(selectedProjection.input)+ || digest(original.projection.metrics) !== digest(selectedProjection.metrics)+ || digest(original.projection.metadata) !== digest(selectedProjection.metadata)) invalid();+ }+ const events = [...candidates.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([id, { original, projection }]) => {+ const cost = operations && original.accountingIdentity!.campaignId === operations.ledgerCampaignId+ ? operations.attempts.find(row => row.executionId === original.executionId && row.attemptId === original.accountingIdentity!.attemptId)+ : undefined;+ return { ...projection, id, span_id: id, root_span_id: id,+ metadata: { ...projection.metadata, request_ledger: cost ? {+ known_usd: cost.knownUsd, additional_held_usd: cost.heldUsd, requests: cost.requests,+ unresolved_requests: cost.unresolvedRequests, journal_hash: operations!.journalHash,+ } : null } };+ });+ const metadata = { importer: "paperclip-retained-attempts-v1", campaign_id: campaign.campaignId,+ expected_execution_ids: [...campaign.expected].sort(), retained_attempts: events.length,+ missing_attempts: accounting.missingAttempts,+ request_ledger: operations ? { journal_hash: operations.journalHash, known_usd: operations.knownUsd,+ additional_held_usd: operations.heldUsd, committed_usd: operations.committedUsd,+ requests: operations.requests, unresolved_requests: operations.unresolvedRequests, breakdown: operations.breakdown } : null,+ feedback_review: campaign.feedbackReview ?? null,+ accounting_note: "Ledger amounts are separate observations, not additive trace metrics. Held reservations are not charges. Requests outside the gateway and infrastructure require separate accounting." };+ const payloadHash = digest({ projectId, metadata, events });+ return { projectId, payloadHash, name: `${campaign.campaignId}-${payloadHash}`, metadata: { ...metadata, payload_hash: payloadHash }, events };+}++export interface BraintrustExperimentReceipt {+ version: 1; projectId: string; experimentId: string; name: string; payloadHash: string;+ status: "created" | "accepted"; acceptedRows: number; expectedRows: number; verification: "unverified";+ evidenceHash: string; projectUrl?: string; experimentUrl?: string;+}++export function experimentEvidenceHash(campaign: RunnerE2ECampaign) {+ return digest({ campaignId: campaign.campaignId, expected: [...campaign.expected].sort(),+ selected: campaign.results.map(result => ({ executionId: result.executionId, accountingIdentity: result.accountingIdentity,+ attempt: result.attempt, startedAt: result.attempt === 0 ? null : result.startedAt, sourceRevision: result.source?.sha ?? null,+ suiteDefinitionHash: result.suiteDefinitionHash ?? null, suiteId: result.suiteId, profileId: result.profileId,+ caseId: result.caseId, provider: result.provider, model: result.model, environmentId: result.environmentId,+ runtimeMode: result.runtimeMode })).sort((a, b) => digest(a).localeCompare(digest(b))),+ accounting: campaign.accounting, operations: campaign.operations ?? null, feedbackReview: campaign.feedbackReview ?? null });+}++function safeBraintrustUrl(value: unknown): value is string {+ if (typeof value !== "string") return false;+ try {+ const url = new URL(value);+ return url.protocol === "https:" && ["braintrust.dev", "www.braintrust.dev"].includes(url.host)+ && !url.username && !url.password && !url.search && !url.hash && url.pathname.startsWith("/app/");+ } catch { return false; }+}++export function validateBraintrustExperimentReceipt(campaign: RunnerE2ECampaign, receipt: BraintrustExperimentReceipt): void {+ if (!receipt || Object.keys(receipt).some(key => !["version", "projectId", "experimentId", "name", "payloadHash", "status", "acceptedRows", "expectedRows", "verification", "evidenceHash", "projectUrl", "experimentUrl"].includes(key))+ || receipt.version !== 1 || !uuid.test(receipt.projectId) || !uuid.test(receipt.experimentId)+ || receipt.evidenceHash !== experimentEvidenceHash(campaign) || !/^[a-f0-9]{64}$/.test(receipt.payloadHash)+ || receipt.name !== `${campaign.campaignId}-${receipt.payloadHash}` || !["created", "accepted"].includes(receipt.status)+ || !count(receipt.acceptedRows) || receipt.expectedRows !== campaign.accounting?.attempts.length+ || receipt.acceptedRows > receipt.expectedRows || receipt.verification !== "unverified"+ || receipt.status === "accepted" && receipt.acceptedRows !== receipt.expectedRows+ || receipt.experimentUrl !== undefined && !safeBraintrustUrl(receipt.experimentUrl)+ || receipt.projectUrl !== undefined && !safeBraintrustUrl(receipt.projectUrl)) invalid();+}++export function renderBraintrustExperiment(campaign: RunnerE2ECampaign): string {+ const receipt = campaign.braintrustExperiment;+ if (!receipt) return "";+ validateBraintrustExperimentReceipt(campaign, receipt);+ return `<section aria-label="Braintrust experiment"><h2>Braintrust experiment</h2><p>${receipt.experimentUrl+ ? `<a href="${html(receipt.experimentUrl)}">Open retained attempt scores in Braintrust</a>`+ : `Experiment ${html(receipt.experimentId)}`}. ${receipt.acceptedRows}/${receipt.expectedRows} rows accepted.+ Original product and cleanup grades are separate. Indexing requires independent verification; no agents were rerun for this import.</p></section>`;+}++async function request(config: EvalBraintrustConfig, endpoint: string, payload: unknown, transport: typeof fetch) {+ const response = await transport(`https://api.braintrust.dev/v1/${endpoint}`, {+ method: payload === undefined ? "GET" : "POST", redirect: "error", signal: AbortSignal.timeout(15_000),+ headers: { authorization: `Bearer ${config.apiKey}`, "content-type": "application/json" },+ ...(payload === undefined ? {} : { body: JSON.stringify(payload) }),+ });+ const reader = response.body?.getReader(), chunks: Uint8Array[] = []; let size = 0;+ try {+ if (!response.ok) throw new Error("rejected");+ while (reader) {+ const next = await reader.read(); if (next.done) break;+ size += next.value.length; if (size > 1_048_576) throw new Error("oversized");+ chunks.push(next.value);+ }+ return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record<string, unknown>;+ } finally { await reader?.cancel().catch(() => {}); }+}++export async function importBraintrustExperiment(config: EvalBraintrustConfig, campaign: RunnerE2ECampaign,+ originals: readonly RunnerE2EResult[], persist: (receipt: BraintrustExperimentReceipt) => Promise<void>, transport: typeof fetch = fetch,+ previous?: BraintrustExperimentReceipt) {+ const plan = buildBraintrustExperiment(config.projectId, campaign, originals);+ if (!config.apiKey?.trim() || /[\r\n]/.test(config.apiKey)) throw new Error("Missing Braintrust ingestion credential");+ if (previous) {+ validateBraintrustExperimentReceipt(campaign, previous);+ if (previous.projectId !== plan.projectId || previous.payloadHash !== plan.payloadHash) invalid();+ }+ try {+ const created = await request(config, "experiment", { project_id: plan.projectId, name: plan.name,+ public: false, ensure_new: false, metadata: plan.metadata }, transport);+ if (!uuid.test(String(created?.id)) || created.project_id !== plan.projectId || created.name !== plan.name || created.public !== false+ || (created.metadata as Record<string, unknown> | undefined)?.payload_hash !== plan.payloadHash) throw new Error("conflicting_experiment");+ if (previous && previous.experimentId !== created.id) throw new Error("conflicting_experiment");+ const receipt: BraintrustExperimentReceipt = previous ? { ...previous } : { version: 1, projectId: plan.projectId, experimentId: String(created.id),+ name: plan.name, payloadHash: plan.payloadHash, evidenceHash: experimentEvidenceHash(campaign),+ status: "created", acceptedRows: 0, expectedRows: plan.events.length, verification: "unverified" };+ if (!previous) await persist({ ...receipt });+ for (let offset = 0; offset < plan.events.length; offset += 100) {+ const events = plan.events.slice(offset, offset + 100);+ const inserted = await request(config, `experiment/${receipt.experimentId}/insert`, { events }, transport);+ if (!Array.isArray(inserted?.row_ids) || JSON.stringify([...inserted.row_ids].sort()) !== JSON.stringify(events.map(event => event.id).sort())) throw new Error("missing_rows");+ receipt.acceptedRows = Math.max(receipt.acceptedRows, offset + events.length);+ if (receipt.acceptedRows === receipt.expectedRows) receipt.status = "accepted";+ await persist({ ...receipt });+ }+ const summary = await request(config, `experiment/${receipt.experimentId}/summarize`, undefined, transport).catch(() => undefined);+ if (summary?.experiment_name === plan.name && safeBraintrustUrl(summary.project_url) && safeBraintrustUrl(summary.experiment_url)) {+ receipt.projectUrl = summary.project_url; receipt.experimentUrl = summary.experiment_url;+ await persist({ ...receipt });+ }+ return receipt;+ } catch { throw new Error("Braintrust experiment import incomplete; retain the receipt and retry identical evidence. Remote details withheld."); }+}diff --git a/tests/runner-e2e/braintrust-indexing.test.ts b/tests/runner-e2e/braintrust-indexing.test.tsnew file mode 100644index 0000000000..65087433ba--- /dev/null+++ b/tests/runner-e2e/braintrust-indexing.test.ts@@ -0,0 +1,113 @@+import { expect, it, vi } from "vitest";+import { buildBraintrustExperiment, experimentEvidenceHash, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";+import { braintrustTraceIndexQuery, verifyBraintrustIndexing } from "./braintrust-indexing.js";+import { buildRunnerCampaign } from "./history.js";+import { runnerMatrix } from "./catalog.js";+import { emptyFeedbackSummary } from "./feedback.js";+import type { RunnerE2EResult } from "./types.js";++const projectId = "11111111-1111-4111-8111-111111111111";+const config = { projectId, apiKey: "private-key-canary" };+const aggregate = { records: 2500, roots: 1, finalizations: 1, native_api_records: 2, complaints: 1, suggestions: 1 };++function fixture(profile = "runner-codex") {+ const cell = runnerMatrix.find(row => row.id === `feedback-persistence.${profile}.local.complaint-and-suggestion`)!;+ const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id,+ suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash,+ accountingIdentity: { campaignId: "indexed-pilot", attemptId: "first" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null },+ attempt: 1, status: "passed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id,+ environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,+ startedAt: "2026-10-20T00:00:00.000Z", finishedAt: "2026-10-20T00:00:01.000Z", durationMs: 1000,+ feedback: { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete", complaints: 1, suggestions: 1 },+ tracing: { projectId, traceId: "a".repeat(32), spanId: "b".repeat(16), status: "accepted", verification: "unverified", nativeConfiguration: "configured",+ finalization: { spanId: "c".repeat(16), status: "accepted", outcome: "passed", verification: "unverified" } } };+ const originals = [result];+ const campaign = buildRunnerCampaign({ campaignId: "indexed-pilot", generatedAt: result.finishedAt, expected: [cell.id], results: originals, attempts: originals });+ const plan = buildBraintrustExperiment(projectId, campaign, originals);+ const experiment: BraintrustExperimentReceipt = { version: 1, projectId, experimentId: "22222222-2222-4222-8222-222222222222", name: plan.name,+ payloadHash: plan.payloadHash, evidenceHash: experimentEvidenceHash(campaign), status: "accepted", acceptedRows: 1, expectedRows: 1, verification: "unverified" };+ return { originals, campaign, plan, experiment };+}++it.each(["runner-codex", "runner-acpx-claude"])("verifies %s native records and original experiment grades without fetching content", async profile => {+ const { originals, campaign, plan, experiment } = fixture(profile);+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [aggregate] })).mockResolvedValueOnce(Response.json({ data: plan.events }));+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport);+ expect(receipt.status).toBe("verified");+ expect(receipt.traces[0]).toMatchObject({ records: 2500, nativeApiRecords: 2, complaints: 1 });+ expect(receipt.experiment.indexedRows).toBe(1);+ expect(transport).toHaveBeenCalledTimes(2);+ const sql = JSON.parse(String(transport.mock.calls[0]![1]!.body)).query;+ expect(sql).toContain(`root_span_id = '${"a".repeat(32)}'`);+ expect(sql).toContain(profile === "runner-codex" ? "codex.api_request" : "claude_code.llm_request");+ expect(sql).not.toMatch(/arguments|prompt|response|tool_input|tool_result_size/);+ expect(JSON.stringify(receipt)).not.toContain(config.apiKey);+});++it.each(["roots", "finalizations", "native_api_records", "complaints", "suggestions"])("does not pass a trace missing %s", async missing => {+ const { originals, campaign, plan, experiment } = fixture();+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [{ ...aggregate, [missing]: 0 }] })).mockResolvedValueOnce(Response.json({ data: plan.events }));+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });+ expect(receipt.status).toBe("incomplete");+ expect(receipt.experiment.status).toBe("verified");+});++it("retries delayed indexing without re-querying already verified records", async () => {+ const { originals, campaign, plan, experiment } = fixture();+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [{ ...aggregate, roots: 0 }] }))+ .mockResolvedValueOnce(Response.json({ data: plan.events })).mockResolvedValueOnce(Response.json({ data: [aggregate] }));+ const pause = vi.fn(async () => {});+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { pause });+ expect(receipt.status).toBe("verified");+ expect(transport).toHaveBeenCalledTimes(3);+ expect(pause).toHaveBeenCalledTimes(1);+});++it("treats an empty aggregate as missing indexing, without inventing native activity", async () => {+ const { originals, campaign, plan, experiment } = fixture();+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [{ records: 0, roots: null,+ finalizations: null, native_api_records: null, complaints: null, suggestions: null }] })).mockResolvedValueOnce(Response.json({ data: plan.events }));+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });+ expect(receipt.status).toBe("incomplete");+ expect(receipt.traces[0]).toMatchObject({ status: "incomplete", records: 0, nativeApiRecords: 0 });+});++it.each(["missing", "extra", "duplicate", "grade-changed", "wrong-revision", "wrong-attempt"])("rejects %s indexed experiment evidence", async fault => {+ const { originals, campaign, plan, experiment } = fixture();+ const rows = structuredClone(plan.events);+ if (fault === "missing") rows.pop();+ if (fault === "extra") rows.push({ ...rows[0]!, id: "other" });+ if (fault === "duplicate") rows.push(rows[0]!);+ if (fault === "grade-changed") rows[0]!.scores.product_pass = 0;+ if (fault === "wrong-revision") rows[0]!.input.source_revision = "b".repeat(40);+ if (fault === "wrong-attempt") rows[0]!.metadata.attempt_id = "other";+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(Response.json({ data: [aggregate] })).mockResolvedValueOnce(Response.json({ data: rows }));+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });+ expect(receipt.status).toBe("incomplete");+ expect(receipt.traces[0]!.status).toBe("verified");+});++it.each(["rejected", "malformed", "oversized", "invalid-counts"])("retains %s query failures without raw error content", async fault => {+ const { originals, campaign, experiment } = fixture();+ const transport = vi.fn<typeof fetch>().mockImplementation(async () => fault === "rejected" ? new Response("private-response-canary", { status: 401 })+ : fault === "malformed" ? new Response("private-response-canary")+ : fault === "oversized" ? new Response("private-response-canary".repeat(100_000))+ : Response.json({ data: [{ ...aggregate, records: -1 }] }));+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 });+ expect(receipt.status).toBe("unavailable");+ expect(JSON.stringify(receipt)).not.toMatch(/private-response-canary|private-key-canary/);+});++it("honors cancellation before making a query", async () => {+ const { originals, campaign, experiment } = fixture();+ const transport = vi.fn<typeof fetch>();+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { signal: AbortSignal.abort() });+ expect(receipt.status).toBe("unavailable");+ expect(transport).not.toHaveBeenCalled();+});++it("rejects foreign or malformed trace identities before constructing a query", () => {+ const { originals } = fixture();+ originals[0]!.tracing!.traceId = "injected'";+ expect(() => braintrustTraceIndexQuery(projectId, originals[0]!)).toThrow();+});diff --git a/tests/runner-e2e/braintrust-indexing.ts b/tests/runner-e2e/braintrust-indexing.tsnew file mode 100644index 0000000000..8ca5d9f11b--- /dev/null+++ b/tests/runner-e2e/braintrust-indexing.ts@@ -0,0 +1,118 @@+import { setTimeout as delay } from "node:timers/promises";+import { buildBraintrustExperiment, experimentEvidenceHash, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";+import { exportBraintrustAttemptParent, type EvalBraintrustConfig } from "./braintrust.js";+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";++type IndexStatus = "verified" | "incomplete" | "unavailable";+export interface BraintrustIndexReceipt {+ version: 1; projectId: string; evidenceHash: string; checkedAt: string;+ status: IndexStatus;+ traces: Array<{ attemptId: string; traceId: string | null; status: IndexStatus;+ records?: number; roots?: number; finalizations?: number; nativeApiRecords?: number;+ complaints?: number; suggestions?: number }>;+ experiment: { id: string | null; status: IndexStatus; expectedRows: number; indexedRows?: number };+}++async function query(config: EvalBraintrustConfig, sql: string, transport: typeof fetch, signal: AbortSignal): Promise<Record<string, unknown>[]> {+ const response = await transport("https://api.braintrust.dev/btql", {+ method: "POST", redirect: "error", signal: AbortSignal.any([signal, AbortSignal.timeout(10_000)]),+ headers: { authorization: `Bearer ${config.apiKey}`, "content-type": "application/json" },+ body: JSON.stringify({ query: sql, fmt: "json" }),+ });+ const reader = response.body?.getReader(), chunks: Uint8Array[] = []; let size = 0;+ try {+ if (!response.ok) throw new Error("query_rejected");+ while (reader) {+ const next = await reader.read(); if (next.done) break;+ size += next.value.length; if (size > 1_048_576) throw new Error("query_oversized");+ chunks.push(next.value);+ }+ const payload = JSON.parse(Buffer.concat(chunks).toString("utf8"));+ if (!payload || !Array.isArray(payload.data) || payload.data.some((row: unknown) => !row || typeof row !== "object" || Array.isArray(row))) throw new Error("invalid_query_result");+ return payload.data;+ } finally { await reader?.cancel().catch(() => {}); }+}++export function braintrustTraceIndexQuery(projectId: string, result: RunnerE2EResult): string {+ const trace = result.tracing;+ if (!trace || trace.projectId !== projectId || !/^[0-9a-f]{16}$/.test(trace.finalization?.spanId ?? "")) throw new Error("Missing trace identity");+ exportBraintrustAttemptParent(trace);+ if (!(result.profileId === "runner-codex" && result.provider === "codex")+ && !(result.profileId === "runner-acpx-claude" && result.provider === "acpx")) throw new Error("Unsupported native index contract");+ const nativeRequest = result.profileId === "runner-codex" ? `metadata."event.name" = 'codex.api_request'`+ : `span_attributes.name = 'claude_code.llm_request' AND span_attributes.type = 'llm'`;+ const tool = (name: string) => result.profileId === "runner-codex"+ ? `metadata."event.name" = 'codex.tool_result' AND metadata.tool_name = '${name}' AND metadata.success = 'true'`+ : `span_attributes.type = 'tool' AND span_attributes.name = 'mcp__paperclip__${name}'`;+ return `SELECT COUNT(*) AS records,+ SUM(CASE WHEN span_id = '${trace.spanId}' AND span_attributes.name = 'paperclip.eval.attempt' THEN 1 ELSE 0 END) AS roots,+ SUM(CASE WHEN span_id = '${trace.finalization!.spanId}' AND span_attributes.name = 'paperclip.eval.finalization' THEN 1 ELSE 0 END) AS finalizations,+ SUM(CASE WHEN ${nativeRequest} THEN 1 ELSE 0 END) AS native_api_records,+ SUM(CASE WHEN ${tool("submit_complaint")} THEN 1 ELSE 0 END) AS complaints,+ SUM(CASE WHEN ${tool("submit_suggestion")} THEN 1 ELSE 0 END) AS suggestions+ FROM project_logs('${projectId}') WHERE root_span_id = '${trace.traceId}'`;+}++function counts(rows: Record<string, unknown>[]) {+ if (rows.length !== 1) throw new Error("Invalid trace aggregate");+ const row = rows[0]!;+ const fields = ["records", "roots", "finalizations", "native_api_records", "complaints", "suggestions"];+ if (row.records === 0) for (const key of fields.slice(1)) if (row[key] === null) row[key] = 0;+ if (fields.some(key => !Number.isSafeInteger(row[key]) || Number(row[key]) < 0)) throw new Error("Invalid trace counts");+ return { records: Number(row.records), roots: Number(row.roots), finalizations: Number(row.finalizations),+ nativeApiRecords: Number(row.native_api_records), complaints: Number(row.complaints), suggestions: Number(row.suggestions) };+}++export async function verifyBraintrustIndexing(config: EvalBraintrustConfig, campaign: RunnerE2ECampaign,+ originals: readonly RunnerE2EResult[], experiment: BraintrustExperimentReceipt | undefined,+ transport: typeof fetch = fetch, options: { rounds?: number; pause?: () => Promise<void>; signal?: AbortSignal } = {}): Promise<BraintrustIndexReceipt> {+ const plan = buildBraintrustExperiment(config.projectId, campaign, originals);+ const receipt: BraintrustIndexReceipt = { version: 1, projectId: config.projectId,+ evidenceHash: experimentEvidenceHash(campaign), checkedAt: new Date().toISOString(), status: "incomplete",+ traces: originals.map(result => ({ attemptId: result.accountingIdentity!.attemptId,+ traceId: result.tracing?.traceId ?? null, status: "incomplete" })),+ experiment: { id: experiment?.experimentId ?? null, status: "incomplete", expectedRows: plan.events.length } };+ if (!config.apiKey?.trim() || /[\r\n]/.test(config.apiKey)) throw new Error("Missing Braintrust query credential");+ const rounds = options.rounds ?? 3;+ if (!Number.isInteger(rounds) || rounds < 1 || rounds > 3) throw new Error("Invalid index verification limit");+ const signal = AbortSignal.any([AbortSignal.timeout(45_000), ...(options.signal ? [options.signal] : [])]);+ const pause = options.pause ?? (() => delay(2_000, undefined, { signal }));+ for (let round = 0; round < rounds && !signal.aborted; round++) {+ for (let index = 0; index < originals.length && !signal.aborted; index++) {+ const result = originals[index]!, row = receipt.traces[index]!;+ if (row.status === "verified") continue;+ try {+ const values = counts(await query(config, braintrustTraceIndexQuery(config.projectId, result), transport, signal));+ Object.assign(row, values);+ row.status = values.roots > 0 && values.finalizations > 0 && values.nativeApiRecords > 0+ && (result.feedback?.complaints ?? 0) <= values.complaints+ && (result.feedback?.suggestions ?? 0) <= values.suggestions ? "verified" : "incomplete";+ } catch { row.status = "unavailable"; }+ }+ if (!signal.aborted && receipt.experiment.status !== "verified" && experiment?.status === "accepted"+ && experiment.projectId === config.projectId && /^[0-9a-f-]{36}$/.test(experiment.experimentId)+ && experiment.payloadHash === plan.payloadHash && experiment.evidenceHash === receipt.evidenceHash) {+ try {+ const rows = await query(config, `SELECT id, input, output, scores, metadata FROM experiment('${experiment.experimentId}') LIMIT ${plan.events.length + 1}`, transport, signal);+ receipt.experiment.indexedRows = rows.length;+ const equal = (left: unknown, right: unknown): boolean => {+ if (left === right) return true;+ if (!left || !right || typeof left !== "object" || typeof right !== "object" || Array.isArray(left) || Array.isArray(right)) return false;+ const a = left as Record<string, unknown>, b = right as Record<string, unknown>;+ return Object.keys(a).length === Object.keys(b).length && Object.keys(a).every(key => equal(a[key], b[key]));+ };+ receipt.experiment.status = rows.length === plan.events.length && new Set(rows.map(row => row.id)).size === rows.length+ && plan.events.every(expected => rows.some(row => row.id === expected.id && equal(row.input, expected.input)+ && equal(row.output, expected.output) && equal(row.scores, expected.scores)+ && (row.metadata as Record<string, unknown> | undefined)?.attempt_id === expected.metadata.attempt_id)) ? "verified" : "incomplete";+ } catch { receipt.experiment.status = "unavailable"; }+ }+ if (receipt.experiment.status === "verified" && receipt.traces.every(row => row.status === "verified")) break;+ if (round + 1 < rounds && !signal.aborted) { try { await pause(); } catch { break; } }+ }+ receipt.checkedAt = new Date().toISOString();+ const statuses = [...receipt.traces.map(row => row.status), receipt.experiment.status];+ receipt.status = statuses.every(status => status === "verified") ? "verified"+ : statuses.some(status => status === "unavailable") || signal.aborted ? "unavailable" : "incomplete";+ return receipt;+}diff --git a/tests/runner-e2e/braintrust.test.ts b/tests/runner-e2e/braintrust.test.tsnew file mode 100644index 0000000000..118621d3ff--- /dev/null+++ b/tests/runner-e2e/braintrust.test.ts@@ -0,0 +1,213 @@+import { readFile } from "node:fs/promises";+import { createRequire } from "node:module";+import { afterEach, expect, it, vi } from "vitest";+import { EvalAttemptTrace, evalBraintrustConfig, exportBraintrustAttemptParent, finalizeEvalAttemptTrace } from "./braintrust.js";+import { RunnerApi } from "./api.js";+import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js";+import { setupLiveFixtures } from "./live-fixtures.js";+import { runnerMatrix } from "./catalog.js";+import { buildProviderObservabilityLaunch } from "../../packages/adapter-utils/src/agent-observability.js";+const require = createRequire(new URL("../../server/package.json", import.meta.url));+const protobuf = require("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js");+const projectId = "11111111-1111-4111-8111-111111111111", companyId = "22222222-2222-4222-8222-222222222222";+const config = { projectId, apiKey: "private-ingestion-fixture" };+const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "hello.codex.local", provider: "codex", model: "fixture-model", environment: "local" };+const provenance = { suiteId: "core-compatibility", suiteDefinitionHash: "a".repeat(64), profileId: "runner-codex", caseId: "message-marker", sourceRevision: "b".repeat(40) };+const input = { failed: false, runIds: ["run-1"], cleanup: "passed", complaints: 1, suggestions: 2 };+afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); });++it("requires explicit export routing and keeps ingestion credentials out of server inheritance", () => {+ expect(evalBraintrustConfig({ BRAINTRUST_API_KEY: config.apiKey })).toBeUndefined();+ expect(() => evalBraintrustConfig({ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: projectId })).toThrow("API key");+ expect(() => evalBraintrustConfig({ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "bad", BRAINTRUST_API_KEY: config.apiKey })).toThrow("UUID");+ expect(buildPaperclipServerEnvironment({ BRAINTRUST_API_KEY: config.apiKey, BRAINTRUST_OTHER_KEY: "private", KEEP: "yes" })).toEqual({ KEEP: "yes" });+});++it("accepts only explicit loopback relay destinations and strips inherited relay endpoints", async () => {+ const endpointBaseUrl = "http://127.0.0.1:43123/otel";+ const env = { PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: projectId, BRAINTRUST_API_KEY: "attempt-token", PAPERCLIP_E2E_OTLP_RELAY_URL: endpointBaseUrl };+ expect(evalBraintrustConfig(env)).toEqual({ projectId, apiKey: "attempt-token", endpointBaseUrl });+ expect(buildPaperclipServerEnvironment(env).PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined();+ expect(buildRunnerE2EProcessEnvironment(env, []).PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined();+ for (const invalid of ["https://foreign.invalid/otel", "http://127.0.0.1:43123/otel?key=secret", "http://localhost:43123/otel", "http://127.0.0.1:99999/otel"]) {+ expect(() => evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: invalid })).toThrow();+ }+ const trace = new EvalAttemptTrace(evalBraintrustConfig(env)!, identity);+ const putSensitive = vi.fn(async () => ({ settings: { enabled: true, credentialsConfigured: true, revision: projectId,+ destination: { provider: "otlp", endpointBaseUrl } } }));+ await trace.configureCompany({ putSensitive } as unknown as RunnerApi, companyId);+ expect(putSensitive).toHaveBeenCalledWith(`/api/companies/${companyId}/observability`, expect.objectContaining({+ destination: { provider: "otlp", endpointBaseUrl }, credentials: { mode: "replace", headers: [+ { name: "authorization", value: "Bearer attempt-token" }, { name: "x-bt-parent", value: exportBraintrustAttemptParent(trace.snapshot()) },+ ] },+ }));+});++it("exports the SDK-verified project-log parent format and refuses invalid identifiers", () => {+ const receipt = { projectId, spanId: "1234567890abcdef", traceId: "abcdef0123456789abcdef0123456789" };+ expect(exportBraintrustAttemptParent(receipt)).toBe("BAIAeyJvYmplY3RfaWQiOiIxMTExMTExMS0xMTExLTQxMTEtODExMS0xMTExMTExMTExMTEiLCJyb3dfaWQiOiIxMjM0NTY3ODkwYWJjZGVmIiwic3Bhbl9pZCI6IjEyMzQ1Njc4OTBhYmNkZWYiLCJyb290X3NwYW5faWQiOiJhYmNkZWYwMTIzNDU2Nzg5YWJjZGVmMDEyMzQ1Njc4OSJ9");+ for (const invalid of [{ projectId: "project" }, { spanId: "0".repeat(16) }, { traceId: "0".repeat(32) }, { spanId: "1234567890abcdeg" }]) {+ expect(() => exportBraintrustAttemptParent({ ...receipt, ...invalid })).toThrow("parent identity");+ }+});++it.each(["codex", "claude"] as const)("preserves the attempt parent through the production %s launch encoder", kind => {+ const trace = new EvalAttemptTrace(config, identity);+ const parent = exportBraintrustAttemptParent(trace.snapshot());+ const launch = buildProviderObservabilityLaunch({+ kind, companyId, agentId: projectId, revision: projectId, compatibilityKey: "fixture", enabled: true,+ capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true },+ tracesEndpoint: "https://api.braintrust.dev/otel/v1/traces", logsEndpoint: "https://api.braintrust.dev/otel/v1/logs",+ headers: { authorization: `Bearer ${config.apiKey}`, "x-bt-parent": parent },+ });+ if (kind === "codex") {+ const exporter = launch.otelConfig!.trace_exporter as { "otlp-http": { headers: Record<string, string> } };+ expect(exporter["otlp-http"].headers).toEqual({});+ expect(JSON.stringify(launch.otelConfig)).not.toContain(config.apiKey);+ }+ for (const name of ["OTEL_EXPORTER_OTLP_TRACES_HEADERS", "OTEL_EXPORTER_OTLP_LOGS_HEADERS"]) {+ const encoded = launch.environment[name].split(",").find(header => header.startsWith("x-bt-parent="))!;+ expect(decodeURIComponent(encoded.slice("x-bt-parent=".length))).toBe(parent);+ }+ expect(JSON.stringify(trace.snapshot())).not.toContain(config.apiKey);+});++it("rejects invalid provenance while retaining an explicitly unknown source revision", () => {+ expect(() => new EvalAttemptTrace(config, identity, { ...provenance, suiteDefinitionHash: "unknown" })).toThrow("provenance");+ expect(() => new EvalAttemptTrace(config, identity, { ...provenance, sourceRevision: "master" })).toThrow("provenance");+ expect(() => new EvalAttemptTrace(config, identity, { ...provenance, sourceRevision: null })).not.toThrow();+});++it("sends real protobuf with parented phases, exact project routing, and no duplicate model charges", async () => {+ const root = protobuf.parse(await readFile(new URL("../../server/src/__tests__/fixtures/otlp-observability.proto", import.meta.url), "utf8")).root;+ const trace = new EvalAttemptTrace(config, identity, provenance);+ trace.phase("setup"); trace.phase("execution"); trace.phase("feedback"); trace.phase("cleanup");+ let captured: RequestInit | undefined;+ const transport = vi.fn<typeof fetch>(async (_url, init) => { captured = init; return new Response(null, { status: 200 }); });+ expect(await trace.export(input, transport)).toMatchObject({ status: "accepted", verification: "unverified", nativeConfiguration: "not_started" });+ expect(captured).toMatchObject({ method: "POST", redirect: "error", headers: { authorization: `Bearer ${config.apiKey}`, "x-bt-parent": `project_id:${projectId}`, "content-type": "application/x-protobuf" } });+ const bytes = captured!.body as Buffer;+ expect(bytes.includes(config.apiKey)).toBe(false);+ const decoded = root.lookupType("paperclip.test.otlp.ExportTraceServiceRequest").decode(bytes).toJSON();+ const spans = decoded.resourceSpans[0].scopeSpans[0].spans;+ expect(spans.map((span: { name: string }) => span.name)).toEqual(["paperclip.eval.attempt", "paperclip.eval.setup", "paperclip.eval.execution", "paperclip.eval.feedback", "paperclip.eval.cleanup"]);+ expect(spans[0].parentSpanId).toBeUndefined();+ for (const span of spans.slice(1)) {+ expect(span.traceId).toBe(spans[0].traceId);+ expect(span.parentSpanId).toBe(spans[0].spanId);+ }+ expect(JSON.stringify(decoded)).toContain("paperclip.eval.attempt_id");+ const attributes = Object.fromEntries(spans[0].attributes.map((attribute: { key: string; value: { stringValue: string } }) => [attribute.key, attribute.value.stringValue]));+ expect(attributes).toMatchObject({+ "paperclip.eval.suite_id": provenance.suiteId, "paperclip.eval.suite_definition_hash": provenance.suiteDefinitionHash,+ "paperclip.eval.profile_id": provenance.profileId, "paperclip.eval.case_id": provenance.caseId,+ "paperclip.eval.source_revision": provenance.sourceRevision, "paperclip.eval.grader": "tests/runner-e2e/runner.spec.ts",+ });+ expect(JSON.stringify(decoded)).not.toMatch(/gen_ai\.usage|cost_usd|private-ingestion/);+ expect(transport).toHaveBeenCalledTimes(1);+});++it.each([+ [() => new Response("private-provider-error", { status: 403 }), "receiver_rejected"],+ [() => Response.json({ partialSuccess: { rejectedSpans: "1" } }), "partial_rejection"],+ [() => Response.json({ partialSuccess: { rejectedSpans: -1 } }), "export_failed"],+ [() => new Response("invalid", { status: 200, headers: { "content-type": "text/html" } }), "export_failed"],+ [() => new Response(Buffer.alloc(65_537)), "export_failed"],+])("retains export failure without raw receiver diagnostics", async (response, errorCode) => {+ const trace = new EvalAttemptTrace(config, identity);+ expect(await trace.export(input, async () => response())).toMatchObject({ status: "failed", verification: "unverified", errorCode });+ expect(JSON.stringify(trace.snapshot())).not.toMatch(/private-provider-error|private-ingestion/);+});++it("configures native export through a revision-fenced sensitive PUT before any credential or agent setup", async () => {+ const order: string[] = [];+ const trace = new EvalAttemptTrace(config, identity);+ const api = {+ async get() { return [{ id: "local", driver: "local" }]; },+ async putSensitive(path: string, data: unknown) {+ order.push("observability");+ expect(path).toBe(`/api/companies/${companyId}/observability`);+ expect(data).toMatchObject({ expectedRevision: null, enabled: true,+ credentials: { mode: "replace", headers: [{ name: "authorization", value: `Bearer ${config.apiKey}` }, { name: "x-bt-parent", value: exportBraintrustAttemptParent(trace.snapshot()) }] },+ destination: { provider: "otlp", endpointBaseUrl: "https://api.braintrust.dev/otel" } });+ return { settings: { enabled: true, credentialsConfigured: true, revision: projectId, destination: { provider: "otlp", endpointBaseUrl: "https://api.braintrust.dev/otel" } } };+ },+ async postSensitive() { order.push("secret"); return { id: "secret" }; },+ async post(path: string) { order.push(path === "/api/companies" ? "company" : "agent"); return { id: companyId, name: "fixture" }; },+ } as unknown as RunnerApi;+ const execution = runnerMatrix.find(cell => cell.profile.id === "runner-codex" && cell.environment.id === "local" && cell.task.flow === "single_turn")!;+ expect(execution).toBeDefined();+ const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { OPENAI_API_KEY: "private-provider" }, onCompanyCreated: id => trace.configureCompany(api, id) });+ await fixtures.teardown();+ expect(order.slice(0, 3)).toEqual(["company", "observability", "secret"]);+ expect(trace.snapshot()).toMatchObject({ nativeConfiguration: "configured", companyId, revision: projectId });+});++it.each([+ { provider: "braintrust", projectId },+ { provider: "otlp", endpointBaseUrl: "https://elsewhere.invalid/otel" },+])("refuses a saved native destination that cannot carry the attempt parent", async destination => {+ const trace = new EvalAttemptTrace(config, identity);+ const api = { putSensitive: async () => ({ settings: { enabled: true, credentialsConfigured: true, revision: projectId, destination } }) } as unknown as RunnerApi;+ await expect(trace.configureCompany(api, companyId)).rejects.toThrow("configuration failed");+ expect(trace.snapshot().nativeConfiguration).toBe("failed");+});++it("never sends sensitive PUT bodies through Playwright and refuses redirects or foreign destinations", async () => {+ vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "43123");+ const playwright = { put: vi.fn() };+ const api = new RunnerApi(playwright as never);+ api.setBrowserSession("session=private");+ const transport = vi.fn().mockResolvedValue(Response.json({ ok: true })); vi.stubGlobal("fetch", transport);+ await api.putSensitive("/api/companies/fixture/observability", { apiKey: config.apiKey });+ expect(playwright.put).not.toHaveBeenCalled();+ expect(transport).toHaveBeenCalledWith(new URL("http://127.0.0.1:43123/api/companies/fixture/observability"), expect.objectContaining({ method: "PUT", redirect: "error", signal: expect.any(AbortSignal), headers: expect.objectContaining({ Cookie: "session=private" }) }));+ await expect(api.putSensitive("https://other.invalid/api/collect", { apiKey: config.apiKey })).rejects.toThrow("fixture API");+ transport.mockRejectedValue(new Error(config.apiKey));+ await expect(api.putSensitive("/api/fixture", {})).rejects.toThrow("details withheld");+});++it.each([+ { status: "failed", cleanup: "passed", failureClass: "permanent_infrastructure" },+ { status: "failed", cleanup: "failed", failureClass: "cleanup_failure" },+] as const)("updates a passing worker trace with the supervisor's $failureClass outcome", async outcome => {+ const root = protobuf.parse(await readFile(new URL("../../server/src/__tests__/fixtures/otlp-observability.proto", import.meta.url), "utf8")).root;+ const trace = new EvalAttemptTrace(config, identity, provenance);+ const original = await trace.export(input, async () => new Response(null, { status: 200 }));+ const saved = structuredClone(original);+ let captured: RequestInit | undefined;+ const finalization = await finalizeEvalAttemptTrace(config, { receipt: original, identity, provenance,+ result: { startedAt: new Date(Date.now() - 10_000).toISOString(), ...outcome, runIds: [companyId] },+ }, async (_url, init) => { captured = init; return new Response(null, { status: 200 }); });+ expect(finalization).toMatchObject({ status: "accepted", outcome: "failed", verification: "unverified" });+ expect(original).toEqual(saved);+ const decoded = root.lookupType("paperclip.test.otlp.ExportTraceServiceRequest").decode(captured!.body as Buffer).toJSON();+ const spans = decoded.resourceSpans[0].scopeSpans[0].spans;+ expect(spans.map((span: { name: string }) => span.name)).toEqual(["paperclip.eval.attempt", "paperclip.eval.finalization"]);+ expect(Buffer.from(spans[0].traceId, "base64").toString("hex")).toBe(original.traceId);+ expect(Buffer.from(spans[0].spanId, "base64").toString("hex")).toBe(original.spanId);+ expect(spans[1].parentSpanId).toBe(spans[0].spanId);+ expect(spans.every((span: { status: { code: number } }) => span.status.code === 2)).toBe(true);+ const attributes = Object.fromEntries(spans[0].attributes.map((attribute: { key: string; value: { stringValue: string } }) => [attribute.key, attribute.value.stringValue]));+ expect(attributes).toMatchObject({ "paperclip.eval.outcome_stage": "supervisor", "paperclip.eval.outcome": "failed",+ "paperclip.eval.failure_class": outcome.failureClass, "paperclip.eval.source_revision": provenance.sourceRevision });+ expect(JSON.stringify(decoded)).not.toMatch(/private-ingestion|gen_ai\.usage|cost_usd/);+});++it("retains finalization export failures without changing the recorded product result", async () => {+ const trace = new EvalAttemptTrace(config, identity, provenance);+ const result = { startedAt: new Date().toISOString(), status: "passed" as const, cleanup: "passed" as const };+ const saved = structuredClone(result);+ const receipt = await finalizeEvalAttemptTrace(config, { receipt: trace.snapshot(), identity, provenance, result }, async () => new Response("private-error", { status: 503 }));+ expect(receipt).toMatchObject({ status: "failed", outcome: "passed", errorCode: "receiver_rejected", httpStatus: 503 });+ expect(result).toEqual(saved);+ expect(JSON.stringify(receipt)).not.toMatch(/private-error|private-ingestion/);+ const transport = vi.fn<typeof fetch>();+ expect(await finalizeEvalAttemptTrace(config, { receipt: { ...trace.snapshot(), projectId: companyId }, identity, provenance, result }, transport))+ .toMatchObject({ status: "failed", errorCode: "invalid_finalization_context" });+ expect(transport).not.toHaveBeenCalled();+ expect(await finalizeEvalAttemptTrace(config, { receipt: trace.snapshot(), identity, provenance,+ result: { ...result, failureClass: "private-target-output" as never },+ }, transport)).toMatchObject({ status: "failed", errorCode: "invalid_finalization_context" });+ expect(transport).not.toHaveBeenCalled();+});diff --git a/tests/runner-e2e/braintrust.ts b/tests/runner-e2e/braintrust.tsnew file mode 100644index 0000000000..adb5d94e4b--- /dev/null+++ b/tests/runner-e2e/braintrust.ts@@ -0,0 +1,218 @@+import { randomBytes } from "node:crypto";+import { createRequire } from "node:module";+import type { RunnerApi } from "./api.js";+import type { FailureClass, RunnerE2EResult } from "./types.js";++export const EVAL_OBSERVABILITY_SECRET_NAMES = ["BRAINTRUST_API_KEY"] as const;+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+export interface EvalBraintrustConfig { projectId: string; apiKey: string; endpointBaseUrl?: string }+function evalOtlpEndpoint(config: EvalBraintrustConfig) {+ if (config.endpointBaseUrl !== undefined && !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(config.endpointBaseUrl)) {+ throw new Error("Eval OTLP relay must use the parent-owned loopback endpoint");+ }+ if (config.endpointBaseUrl && Number(new URL(config.endpointBaseUrl).port) > 65535) throw new Error("Invalid eval OTLP relay port");+ return config.endpointBaseUrl ?? "https://api.braintrust.dev/otel";+}+export function evalBraintrustConfig(env: NodeJS.ProcessEnv): EvalBraintrustConfig | undefined {+ const projectId = env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID;+ if (!projectId) return undefined;+ if (!uuid.test(projectId) || !env.BRAINTRUST_API_KEY?.trim() || /[\r\n]/.test(env.BRAINTRUST_API_KEY)) {+ throw new Error("Eval Braintrust export requires an explicit project UUID and API key");+ }+ const config = { projectId, apiKey: env.BRAINTRUST_API_KEY.trim(),+ ...(env.PAPERCLIP_E2E_OTLP_RELAY_URL ? { endpointBaseUrl: env.PAPERCLIP_E2E_OTLP_RELAY_URL } : {}) };+ evalOtlpEndpoint(config);+ return config;+}++type Attributes = Record<string, string | number | boolean | string[]>;+type Phase = "setup" | "execution" | "feedback" | "cleanup" | "finalization";+interface PhaseSpan { name: Phase; spanId: string; start: number; end?: number; failed?: boolean }+export interface EvalTraceIdentity { campaignId: string; attemptId: string; executionId: string; provider: string; model: string; environment: string }+export interface EvalTraceProvenance {+ suiteId: string; suiteDefinitionHash: string; profileId: string; caseId: string; sourceRevision: string | null;+}+export interface EvalTraceReceipt {+ projectId: string; traceId: string; spanId: string;+ status: "pending" | "accepted" | "failed";+ verification: "unverified";+ nativeConfiguration: "not_started" | "configured" | "failed";+ companyId?: string; revision?: string; errorCode?: string; httpStatus?: number;+ finalization?: EvalFinalizationReceipt;+}+export interface EvalFinalizationReceipt {+ spanId: string; status: "accepted" | "failed"; outcome: "passed" | "failed"; verification: "unverified";+ errorCode?: string; httpStatus?: number;+}+const require = createRequire(new URL("../../server/package.json", import.meta.url));+const time = (ms: number): [number, number] => [Math.floor(ms / 1000), (ms % 1000) * 1_000_000];++export function exportBraintrustAttemptParent(receipt: Pick<EvalTraceReceipt, "projectId" | "spanId" | "traceId">): string {+ if (!uuid.test(receipt.projectId) || !/^[0-9a-f]{16}$/.test(receipt.spanId) || !/^[0-9a-f]{32}$/.test(receipt.traceId)+ || /^0+$/.test(receipt.spanId) || /^0+$/.test(receipt.traceId)) throw new Error("Invalid eval trace parent identity");+ return Buffer.concat([Buffer.from([4, 2, 0]), Buffer.from(JSON.stringify({+ object_id: receipt.projectId, row_id: receipt.spanId, span_id: receipt.spanId, root_span_id: receipt.traceId,+ }))]).toString("base64");+}++export class EvalAttemptTrace {+ private readonly startedAt = Date.now();+ private readonly phases: PhaseSpan[] = [];+ private active?: PhaseSpan;+ private readonly receipt: EvalTraceReceipt;+ private readonly endpointBaseUrl: string;+ constructor(private readonly config: EvalBraintrustConfig, private readonly identity: EvalTraceIdentity, private readonly provenance?: EvalTraceProvenance,+ context?: { receipt: EvalTraceReceipt; startedAtMs: number }) {+ this.endpointBaseUrl = evalOtlpEndpoint(config);+ if (Object.values(identity).some(value => typeof value !== "string" || !value || value.length > 512 || /[\r\n]/.test(value))) throw new Error("Invalid eval trace identity");+ if (provenance && (Object.values(provenance).some(value => value !== null && (typeof value !== "string" || !value || value.length > 512 || /[\r\n]/.test(value)))+ || !/^[0-9a-f]{64}$/.test(provenance.suiteDefinitionHash)+ || provenance.sourceRevision !== null && !/^[0-9a-f]{40}$/.test(provenance.sourceRevision))) throw new Error("Invalid eval trace provenance");+ this.receipt = { projectId: config.projectId, traceId: randomBytes(16).toString("hex"), spanId: randomBytes(8).toString("hex"),+ status: "pending", verification: "unverified", nativeConfiguration: "not_started" };+ if (context) {+ const { receipt, startedAtMs } = context;+ exportBraintrustAttemptParent(receipt);+ if (receipt.projectId !== config.projectId || !Number.isFinite(startedAtMs) || startedAtMs < 0 || startedAtMs > Date.now()+ || !["not_started", "configured", "failed"].includes(receipt.nativeConfiguration)+ || receipt.companyId !== undefined && !uuid.test(receipt.companyId)+ || receipt.revision !== undefined && !uuid.test(receipt.revision)) throw new Error("Invalid eval finalization context");+ this.startedAt = startedAtMs;+ Object.assign(this.receipt, { traceId: receipt.traceId, spanId: receipt.spanId, nativeConfiguration: receipt.nativeConfiguration,+ ...(receipt.companyId ? { companyId: receipt.companyId } : {}), ...(receipt.revision ? { revision: receipt.revision } : {}) });+ }+ }+ snapshot(): EvalTraceReceipt { return { ...this.receipt }; }+ phase(name: Phase, previousFailed = false) {+ this.endPhase(previousFailed);+ this.active = { name, spanId: randomBytes(8).toString("hex"), start: Date.now() };+ this.phases.push(this.active);+ return this.active.spanId;+ }+ endPhase(failed = false) {+ if (this.active) { this.active.end = Date.now(); this.active.failed = failed; this.active = undefined; }+ }+ async configureCompany(api: Pick<RunnerApi, "putSensitive">, companyId: string) {+ if (!uuid.test(companyId)) throw new Error("Invalid eval trace company identity");+ this.receipt.companyId = companyId;+ try {+ const saved = await api.putSensitive<{ settings: { revision: string; enabled: boolean; credentialsConfigured: boolean; destination: { provider: string; endpointBaseUrl?: string } } }>(+ `/api/companies/${companyId}/observability`, {+ expectedRevision: null, enabled: true,+ destination: { provider: "otlp", endpointBaseUrl: this.endpointBaseUrl },+ credentials: { mode: "replace", headers: [+ { name: "authorization", value: `Bearer ${this.config.apiKey}` },+ { name: "x-bt-parent", value: exportBraintrustAttemptParent(this.receipt) },+ ] },+ capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true },+ });+ if (!saved.settings?.enabled || !saved.settings.credentialsConfigured || !uuid.test(saved.settings.revision)+ || saved.settings.destination.provider !== "otlp" || saved.settings.destination.endpointBaseUrl !== this.endpointBaseUrl) throw new Error("Unexpected observability setting");+ this.receipt.revision = saved.settings.revision;+ this.receipt.nativeConfiguration = "configured";+ } catch {+ this.receipt.nativeConfiguration = "failed";+ throw new Error("Eval native observability configuration failed; response withheld");+ }+ }+ async export(input: { failed: boolean; runIds: string[]; cleanup: string; complaints: number; suggestions: number;+ outcomeStage?: "worker" | "supervisor"; failureClass?: FailureClass }, transport: typeof fetch = fetch): Promise<EvalTraceReceipt> {+ this.endPhase(input.failed);+ const endedAt = Date.now();+ const { resourceFromAttributes } = require("@opentelemetry/resources") as { resourceFromAttributes(attributes: Attributes): unknown };+ const { ProtobufTraceSerializer: serializer } = require("@opentelemetry/otlp-transformer") as {+ ProtobufTraceSerializer: { serializeRequest(spans: unknown[]): Uint8Array | undefined; deserializeResponse(bytes: Uint8Array): unknown };+ };+ const attributes: Attributes = {+ "paperclip.eval.campaign_id": this.identity.campaignId, "paperclip.eval.attempt_id": this.identity.attemptId,+ "paperclip.eval.execution_id": this.identity.executionId, "paperclip.eval.provider": this.identity.provider,+ "paperclip.eval.model": this.identity.model, "paperclip.eval.environment": this.identity.environment,+ "paperclip.eval.outcome_stage": input.outcomeStage ?? "worker", "paperclip.eval.outcome": input.failed ? "failed" : "passed",+ ...(input.failureClass ? { "paperclip.eval.failure_class": input.failureClass } : {}),+ "paperclip.eval.cleanup": input.cleanup, "paperclip.eval.run_ids": input.runIds,+ "paperclip.eval.feedback.complaints": input.complaints, "paperclip.eval.feedback.suggestions": input.suggestions,+ ...(this.provenance ? {+ "paperclip.eval.suite_id": this.provenance.suiteId, "paperclip.eval.suite_definition_hash": this.provenance.suiteDefinitionHash,+ "paperclip.eval.profile_id": this.provenance.profileId, "paperclip.eval.case_id": this.provenance.caseId,+ "paperclip.eval.source_revision": this.provenance.sourceRevision ?? "unknown",+ "paperclip.eval.grader": "tests/runner-e2e/runner.spec.ts",+ } : {}),+ ...(this.receipt.companyId ? { "paperclip.company.id": this.receipt.companyId } : {}),+ };+ const resource = resourceFromAttributes({ "service.name": "paperclip-evals" });+ const instrumentationScope = { name: "paperclip.eval-operations", version: "1" };+ const rootContext = { traceId: this.receipt.traceId, spanId: this.receipt.spanId, traceFlags: 1 };+ const span = (name: string, spanId: string, start: number, end: number, failed: boolean, parent = false) => ({+ name, kind: 0, spanContext: () => ({ ...rootContext, spanId }), ...(parent ? { parentSpanContext: rootContext } : {}),+ startTime: time(start), endTime: time(end), duration: time(end - start), status: { code: failed ? 2 : 1 }, attributes,+ links: [], events: [], ended: true, resource, instrumentationScope,+ droppedAttributesCount: 0, droppedEventsCount: 0, droppedLinksCount: 0,+ });+ try {+ const body = serializer.serializeRequest([+ span("paperclip.eval.attempt", this.receipt.spanId, this.startedAt, endedAt, input.failed),+ ...this.phases.map(phase => span(`paperclip.eval.${phase.name}`, phase.spanId, phase.start, phase.end!, phase.failed ?? false, true)),+ ]);+ if (!body) throw new Error("serialization");+ const response = await transport(`${this.endpointBaseUrl}/v1/traces`, {+ method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000),+ headers: { authorization: `Bearer ${this.config.apiKey}`, "x-bt-parent": `project_id:${this.config.projectId}`,+ "content-type": "application/x-protobuf", accept: "application/x-protobuf, application/json" },+ body: Buffer.from(body),+ });+ this.receipt.httpStatus = response.status;+ if (!response.ok) { this.receipt.errorCode = "receiver_rejected"; await response.body?.cancel(); throw new Error("rejected"); }+ const contentType = response.headers.get("content-type")?.split(";")[0].trim();+ if (contentType && !["application/json", "application/x-protobuf"].includes(contentType)) { await response.body?.cancel(); throw new Error("invalid_response"); }+ const reader = response.body?.getReader();+ const chunks: Uint8Array[] = []; let length = 0;+ try {+ while (reader) {+ const next = await reader.read(); if (next.done) break;+ length += next.value.length; if (length > 65_536) throw new Error("response_too_large");+ chunks.push(next.value);+ }+ } finally { await reader?.cancel().catch(() => {}); }+ if (length) {+ const bytes = Buffer.concat(chunks);+ const decoded = contentType === "application/json" ? JSON.parse(bytes.toString("utf8"))+ : contentType === "application/x-protobuf" ? serializer.deserializeResponse(bytes) : null;+ if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_response");+ const partial = (decoded as { partialSuccess?: { rejectedSpans?: unknown } }).partialSuccess;+ if (partial !== undefined && (!partial || typeof partial !== "object" || Array.isArray(partial))) throw new Error("invalid_response");+ const rejected = partial?.rejectedSpans ?? 0;+ if (!/^[0-9]+$/.test(String(rejected)) || !Number.isSafeInteger(Number(rejected))) throw new Error("invalid_response");+ if (Number(rejected) !== 0) { this.receipt.errorCode = "partial_rejection"; throw new Error("partial_rejection"); }+ }+ this.receipt.status = "accepted";+ } catch {+ this.receipt.status = "failed";+ this.receipt.errorCode ??= "export_failed";+ }+ return this.snapshot();+ }+}++export async function finalizeEvalAttemptTrace(config: EvalBraintrustConfig, input: {+ receipt: EvalTraceReceipt; identity: EvalTraceIdentity; provenance: EvalTraceProvenance;+ result: Pick<RunnerE2EResult, "startedAt" | "status" | "cleanup" | "runIds" | "feedback" | "failureClass">;+}, transport: typeof fetch = fetch): Promise<EvalFinalizationReceipt> {+ const failed = input.result.status !== "passed" || input.result.cleanup !== "passed";+ const receipt: EvalFinalizationReceipt = { spanId: randomBytes(8).toString("hex"), status: "failed", outcome: failed ? "failed" : "passed", verification: "unverified" };+ try {+ if (!["passed", "failed"].includes(input.result.status) || !["not_started", "passed", "failed"].includes(input.result.cleanup)+ || input.result.failureClass !== undefined && !["candidate_failure", "provider_variance", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"].includes(input.result.failureClass)+ || input.result.runIds !== undefined && (!Array.isArray(input.result.runIds) || input.result.runIds.some(id => !uuid.test(id)))+ || [input.result.feedback?.complaints ?? 0, input.result.feedback?.suggestions ?? 0].some(count => !Number.isSafeInteger(count) || count < 0)) throw new Error("Invalid eval finalization result");+ const trace = new EvalAttemptTrace(config, input.identity, input.provenance, { receipt: input.receipt, startedAtMs: Date.parse(input.result.startedAt) });+ receipt.spanId = trace.phase("finalization");+ const exported = await trace.export({ failed, outcomeStage: "supervisor", failureClass: input.result.failureClass,+ runIds: input.result.runIds ?? [], cleanup: input.result.cleanup,+ complaints: input.result.feedback?.complaints ?? 0, suggestions: input.result.feedback?.suggestions ?? 0,+ }, transport);+ receipt.status = exported.status === "accepted" ? "accepted" : "failed";+ if (exported.httpStatus !== undefined) receipt.httpStatus = exported.httpStatus;+ if (exported.errorCode) receipt.errorCode = exported.errorCode;+ } catch { receipt.errorCode = "invalid_finalization_context"; }+ return receipt;+}diff --git a/tests/runner-e2e/budgeted-setup.test.ts b/tests/runner-e2e/budgeted-setup.test.tsnew file mode 100644index 0000000000..f7ddaf2c5b--- /dev/null+++ b/tests/runner-e2e/budgeted-setup.test.ts@@ -0,0 +1,75 @@+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, it, vi } from "vitest";+import type { RunnerApi } from "./api.js";+import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js";+import { finishBudgetedEvalSetup, type EvalSetupObservation } from "./budgeted-setup.js";++const identity = { executionId: "fixture-execution", attemptId: "fixture-attempt" };+const ready = { id: "agent", companyId: "company", status: "idle", lifecycleState: "ready", lifecycleError: null };+async function fixture() {+ const directory = await mkdtemp(path.join(tmpdir(), "budgeted-setup-"));+ const file = path.join(directory, "phase.json");+ await createEvalAttemptPhase(file, identity);+ return { directory, file, env: { PAPERCLIP_E2E_GATEWAY_BASE_URL: "http://127.0.0.1:12345",+ PAPERCLIP_E2E_GATEWAY_PHASE_PATH: file, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: identity.attemptId } };+}++it("keeps asynchronous hello probes in setup until the same agent is ready", async () => {+ const f = await fixture();+ const get = vi.fn().mockResolvedValueOnce({ ...ready, status: "paused", lifecycleState: "verifying", adapterConfig: { private: true } }).mockResolvedValue(ready);+ let saved: EvalSetupObservation[] = [];+ try {+ await finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",+ deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi,+ observe: async observations => {+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");+ saved = structuredClone(observations);+ } }, f.env);+ expect(saved.map(row => row.lifecycleState)).toEqual(["verifying", "ready"]);+ expect(saved[0]).not.toHaveProperty("adapterConfig");+ expect(get).toHaveBeenCalledTimes(2);+ expect(get.mock.calls.every(([url, options]) => url === "/api/agents/agent" && options.timeout > 0 && options.timeout <= 5_000)).toBe(true);+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("agent");+ } finally { await rm(f.directory, { recursive: true, force: true }); }+});++it.each([+ [{ ...ready, lifecycleError: "harness_command_unavailable" }, "setup failed"],+ [{ ...ready, lifecycleHolds: ["approval"] }, "setup is held"],+ [{ ...ready, companyId: "other" }, "scope changed"],+ [{ ...ready, lifecycleState: "terminated" }, "setup stopped"],+] as const)("preserves setup attribution and the failed state when readiness is rejected: %s", async (state, reason) => {+ const f = await fixture(), observe = vi.fn(async () => {}), get = vi.fn().mockResolvedValue(state);+ try {+ await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",+ deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi, observe }, f.env)).rejects.toThrow(reason);+ expect(get).toHaveBeenCalledOnce();+ expect(observe).toHaveBeenCalledOnce();+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");+ } finally { await rm(f.directory, { recursive: true, force: true }); }+});++it("does not start a readiness request after the attempt deadline", async () => {+ const f = await fixture(), get = vi.fn();+ try {+ await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",+ deadlineAt: Date.now() - 1, api: { get } as unknown as RunnerApi, observe: async () => {} }, f.env)).rejects.toThrow("Timed out");+ expect(get).not.toHaveBeenCalled();+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");+ } finally { await rm(f.directory, { recursive: true, force: true }); }+});++it("does not advance when saving readiness crosses the deadline", async () => {+ const f = await fixture(), get = vi.fn().mockResolvedValue(ready);+ const deadlineAt = Date.now() + 5_000;+ const now = vi.spyOn(Date, "now");+ try {+ await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company",+ deadlineAt, api: { get } as unknown as RunnerApi,+ observe: async () => { now.mockReturnValue(deadlineAt); } }, f.env)).rejects.toThrow("Timed out saving");+ expect(get).toHaveBeenCalledOnce();+ expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");+ } finally { now.mockRestore(); await rm(f.directory, { recursive: true, force: true }); }+});diff --git a/tests/runner-e2e/budgeted-setup.ts b/tests/runner-e2e/budgeted-setup.tsnew file mode 100644index 0000000000..42dc216ea5--- /dev/null+++ b/tests/runner-e2e/budgeted-setup.ts@@ -0,0 +1,44 @@+import type { Agent } from "../../packages/shared/src/types/agent.js";+import { pollUntil, type RunnerApi } from "./api.js";+import { beginEvalAgentPhase } from "./attempt-phase.js";++type SetupState = Pick<Agent, "id" | "companyId" | "status" | "lifecycleState" | "lifecycleError"> & { lifecycleHolds?: string[] };+export type EvalSetupObservation = SetupState & { observedAt: string };++export async function finishBudgetedEvalSetup(input: {+ executionId: string;+ agentId: string;+ companyId: string;+ deadlineAt: number;+ api: Pick<RunnerApi, "get">;+ observe: (observations: EvalSetupObservation[]) => Promise<void>;+}, env: NodeJS.ProcessEnv = process.env) {+ if (env.PAPERCLIP_E2E_GATEWAY_BASE_URL) {+ const deadlineAt = Math.min(input.deadlineAt, Date.now() + 45_000);+ const observations: EvalSetupObservation[] = [];+ let previous: string | undefined;+ const scoped = (agent: SetupState) => agent.id === input.agentId && agent.companyId === input.companyId;+ await pollUntil({ label: "budgeted eval agent setup", deadlineAt, intervalMs: 500,+ load: async () => {+ const agent = await input.api.get<SetupState>(`/api/agents/${input.agentId}`, { timeout: Math.max(1, Math.min(5_000, deadlineAt - Date.now())) });+ const state: SetupState = { id: agent.id, companyId: agent.companyId, status: agent.status,+ lifecycleState: agent.lifecycleState, lifecycleError: agent.lifecycleError, lifecycleHolds: agent.lifecycleHolds };+ const current = JSON.stringify(state);+ if (current !== previous) {+ observations.push({ ...state, observedAt: new Date().toISOString() });+ await input.observe(observations);+ previous = current;+ }+ return state;+ },+ accept: agent => scoped(agent) && agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status)+ && !agent.lifecycleError && !agent.lifecycleHolds?.length,+ reject: agent => !scoped(agent) ? "Agent scope changed"+ : agent.lifecycleError ? `Agent setup failed: ${agent.lifecycleError}`+ : agent.lifecycleHolds?.length ? "Agent setup is held"+ : ["paused", "rejected", "terminated"].includes(agent.lifecycleState ?? "") ? `Agent setup stopped: ${agent.lifecycleState}` : undefined,+ });+ if (Date.now() >= deadlineAt) throw new Error("Timed out saving budgeted eval agent readiness");+ }+ await beginEvalAgentPhase(input.executionId, env);+}diff --git a/tests/runner-e2e/campaign-timing.test.ts b/tests/runner-e2e/campaign-timing.test.tsnew file mode 100644index 0000000000..cf7409be76--- /dev/null+++ b/tests/runner-e2e/campaign-timing.test.ts@@ -0,0 +1,42 @@+import { describe, expect, it } from "vitest";+import { buildCampaignAccounting } from "./attempt-accounting.js";+import { campaignAttemptTimeLabel, summarizeCampaignTiming } from "./campaign-timing.js";+import type { RunnerE2EResult } from "./types.js";++function result(executionId: string, startMs: number, durationMs: number, attempt = 1): RunnerE2EResult {+ return {+ schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "timing-fixture",+ accountingIdentity: { campaignId: "timing-campaign", attemptId: `${executionId}-${attempt}` },+ attempt, status: "passed", profileId: "runner-codex", environmentId: "local",+ caseId: executionId, provider: "codex", model: "fixture-model", runtimeMode: "native",+ startedAt: new Date(startMs).toISOString(), finishedAt: new Date(startMs + durationMs).toISOString(),+ durationMs, cleanup: "passed",+ };+}++describe("campaign timing evidence", () => {+ it("separates overlapping attempt work from the observed window and retains a failed retry", () => {+ const failed = { ...result("a", 0, 30_000), status: "failed" as const };+ const selected = [result("a", 30_000, 120_000, 2), result("b", 60_000, 60_000)];+ const accounting = buildCampaignAccounting({ results: selected, attempts: [failed, ...selected] });+ expect(summarizeCampaignTiming(selected, accounting)).toEqual({+ retainedAttempts: 3, missingAttempts: 0, subtotalMs: 210_000, totalMs: 210_000, observedWindowMs: 150_000,+ });+ });++ it("keeps a missing retry unknown while exposing the retained subtotal", () => {+ const timing = summarizeCampaignTiming([result("a", 60_000, 30_000, 2)]);+ expect(timing).toEqual({ retainedAttempts: 1, missingAttempts: 1, subtotalMs: 30_000, totalMs: null, observedWindowMs: 30_000 });+ expect(campaignAttemptTimeLabel(timing)).toBe("Unknown (30s retained subtotal)");+ const missing = summarizeCampaignTiming([{ ...result("a", 0, 0, 0), status: "failed" }]);+ expect(missing.totalMs).toBeNull();+ expect(missing.observedWindowMs).toBeNull();+ expect(campaignAttemptTimeLabel(missing)).toBe("Unknown");+ });++ it("does not derive an observed window from reversed retained timestamps", () => {+ const timing = summarizeCampaignTiming([{ ...result("a", 1000, 1000), finishedAt: new Date(0).toISOString() }]);+ expect(timing.totalMs).toBe(1000);+ expect(timing.observedWindowMs).toBeNull();+ });+});diff --git a/tests/runner-e2e/campaign-timing.ts b/tests/runner-e2e/campaign-timing.tsnew file mode 100644index 0000000000..0958dede05--- /dev/null+++ b/tests/runner-e2e/campaign-timing.ts@@ -0,0 +1,83 @@+import { buildCampaignAccounting } from "./attempt-accounting.js";+import { durationLabel, html } from "./report-format.js";+import { WORKER_TIMING_PHASES } from "./worker-timing.js";+import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js";++export const campaignTimingScope = "Attempt durations include setup and cleanup captured inside each attempt. The observed window spans the earliest retained start to the latest retained finish, including gaps and overlap. Carried-forward evidence can span older runs, so this window is not the current workflow's duration. Neither measure is CI queue/setup time or provider-billed runtime.";++export function summarizeCampaignTiming(+ results: readonly RunnerE2EResult[],+ retained?: RunnerE2ECampaignAccounting,+) {+ const accounting = retained ?? buildCampaignAccounting({ results });+ const attempts = accounting.attempts;+ const subtotal = attempts.reduce((total, attempt) => total + attempt.durationMs, 0);+ const subtotalMs = Number.isFinite(subtotal) ? subtotal : null;+ const starts = attempts.map(attempt => Date.parse(attempt.startedAt));+ const finishes = attempts.map(attempt => Date.parse(attempt.finishedAt));+ const validWindows = attempts.length > 0 && starts.every((start, index) =>+ Number.isFinite(start) && Number.isFinite(finishes[index]) && finishes[index]! >= start);+ const observedWindowMs = validWindows ? Math.max(...finishes) - Math.min(...starts) : null;+ const complete = attempts.length > 0 && accounting.missingAttempts === 0 && subtotalMs !== null;+ return {+ retainedAttempts: attempts.length,+ missingAttempts: accounting.missingAttempts,+ subtotalMs,+ totalMs: complete ? subtotalMs : null,+ observedWindowMs,+ };+}++export function campaignAttemptTimeLabel(timing: ReturnType<typeof summarizeCampaignTiming>) {+ if (timing.totalMs !== null) return durationLabel(timing.totalMs);+ return timing.retainedAttempts > 0 && timing.subtotalMs !== null+ ? `Unknown (${durationLabel(timing.subtotalMs)} retained subtotal)`+ : "Unknown";+}++export function campaignTimingText(timing: ReturnType<typeof summarizeCampaignTiming>) {+ return [+ `Summed attempt time: ${campaignAttemptTimeLabel(timing)}; ${timing.retainedAttempts} retained attempts; ${timing.missingAttempts} missing attempts.`,+ `Observed attempt window: ${durationLabel(timing.observedWindowMs)}.`,+ campaignTimingScope,+ ];+}++export function summarizeWorkerPhaseTiming(results: readonly RunnerE2EResult[], retained?: RunnerE2ECampaignAccounting) {+ const accounting = retained ?? buildCampaignAccounting({ results });+ const receipts = accounting.attempts.map(attempt => attempt.workerTiming);+ const completeAttempts = receipts.filter(receipt => receipt?.status === "complete").length;+ const partialAttempts = receipts.filter(receipt => receipt?.status === "partial").length;+ const complete = receipts.length > 0 && completeAttempts === receipts.length && accounting.missingAttempts === 0;+ return {+ retainedAttempts: receipts.length, completeAttempts, partialAttempts, missingAttempts: accounting.missingAttempts,+ phases: WORKER_TIMING_PHASES.map(phase => {+ const segments = receipts.flatMap(receipt => receipt && receipt.status !== "unavailable"+ ? receipt.segments.filter(segment => segment.phase === phase && segment.finishedAt !== null) : []);+ const subtotalMs = segments.reduce((total, segment) => total + Date.parse(segment.finishedAt!) - Date.parse(segment.startedAt), 0);+ return { phase, totalMs: complete ? subtotalMs : null, subtotalMs, observedSegments: segments.length };+ }),+ };+}++export const workerTimingScope = "Worker phases cover the test callback through finalization before result publication. Browser/server startup, parent cleanup, CI queue/build time and provider-billed runtime are outside these measurements. An interrupted phase has no inferred end time.";++function workerPhaseLabel(phase: ReturnType<typeof summarizeWorkerPhaseTiming>["phases"][number]) {+ return phase.totalMs !== null ? durationLabel(phase.totalMs)+ : phase.observedSegments ? `Unknown (${durationLabel(phase.subtotalMs)} observed)` : "Unknown";+}++export function workerPhaseTimingText(timing: ReturnType<typeof summarizeWorkerPhaseTiming>) {+ return [+ `Worker phase coverage: ${timing.completeAttempts}/${timing.retainedAttempts} complete receipts; ${timing.partialAttempts} partial; ${timing.missingAttempts} missing attempts.`,+ ...timing.phases.map(phase => `Worker ${phase.phase} time: ${workerPhaseLabel(phase)}.`),+ workerTimingScope,+ ];+}++export function renderWorkerPhaseTiming(timing: ReturnType<typeof summarizeWorkerPhaseTiming>) {+ return `<section class="billing-overview" aria-label="Worker phase timing">+ ${timing.phases.map(phase => `<div class="billing-metric"><strong>${html(workerPhaseLabel(phase))}</strong><span>Worker ${phase.phase} time</span></div>`).join("")}+ <p class="billing-note">${timing.completeAttempts}/${timing.retainedAttempts} complete worker receipts; ${timing.partialAttempts} partial; ${timing.missingAttempts} missing attempts. ${html(workerTimingScope)}</p>+ </section>`;+}diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.tsindex b06ccb8b1f..8fe1d5957f 100644--- a/tests/runner-e2e/catalog.test.ts+++ b/tests/runner-e2e/catalog.test.ts@@ -219,10 +219,10 @@ describe("runner E2E catalog", () => {expect(localIntegrityTasks).toHaveLength(2);expect(openRouterBreadthTasks).toHaveLength(3);expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([- 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 4, 116, 2,+ 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2,]);- expect(validateRunnerCatalog()).toHaveLength(761);- expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(761);+ expect(validateRunnerCatalog()).toHaveLength(763);+ expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(763);expect(runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),).toHaveLength(48);diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.tsindex 8f33bc2ad3..c6bc7b4d29 100644--- a/tests/runner-e2e/catalog.ts+++ b/tests/runner-e2e/catalog.ts@@ -1,3 +1,4 @@+import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js";import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js";import { finalizationRecoveryTask, finalizationRecoveryDigest } from "./finalization-recovery.js";import { necessaryQuestionTask, necessaryQuestionDigest } from "./necessary-question.js";@@ -1664,6 +1665,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [},...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []),+ {+ id: FEEDBACK_PERSISTENCE_SUITE, label: "Feedback Persistence", manualOnly: true,+ description: "Real complaint and suggestion submissions, durable bodies, and retained tool receipts.",+ groups: ["native"],+ profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)),+ environments: [localEnvironment], tasks: [feedbackPersistenceTask(runnerTasks.find(task => task.id === "message-marker")!)], expectedMatrixSize: 2,+ definitionMetadata: { version: 1, owner: "runner-e2e-maintainers", scheduling: "explicit-only",+ sourceDigest: feedbackPersistenceDigest(runnerTasks.find(task => task.id === "message-marker")!), expectedWorkRuns: 1, automaticRetryPolicy: "single_attempt",+ budgetCents: 1_000, forecast: "unmeasured; gateway manifest required for bounded qualification",+ grading: "retained-exact-bodies-and-run-attributed-completed-tools", feedback: "synthetic-disposable-items" },+ },{id: "core-compatibility",label: "Core Runner Compatibility",diff --git a/tests/runner-e2e/cleanup-verification.ts b/tests/runner-e2e/cleanup-verification.tsindex d21191e99f..2e935b226b 100644--- a/tests/runner-e2e/cleanup-verification.ts+++ b/tests/runner-e2e/cleanup-verification.ts@@ -22,7 +22,7 @@ export function mustPreserveRecoveryState(input: {resourceAdmissionStarted?: boolean;results: readonly { cleanup: string; synthetic?: boolean }[];}) {- return input.processCleanupFailed || input.results.some(result => result.cleanup === "failed")+ return input.processCleanupFailed || input.results.some(result => result.cleanup === "failed" || result.synthetic)|| (input.resourceAdmissionStarted === true && (input.results.length === 0 || input.results.some(result => result.synthetic)));}diff --git a/tests/runner-e2e/codex-ci-sandbox.test.ts b/tests/runner-e2e/codex-ci-sandbox.test.tsindex 0594234598..94b043ed20 100644--- a/tests/runner-e2e/codex-ci-sandbox.test.ts+++ b/tests/runner-e2e/codex-ci-sandbox.test.ts@@ -34,7 +34,7 @@ describe("Codex CI sandbox trust boundary", () => {/- name: Provision Codex sandbox on the disposable trusted runner\n\s+if: ([^\n]+)/,)?.[1];expect(condition).toMatch(- /^matrix\.environmentId == 'local' && \(matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/,+ /^matrix\.environmentId == 'local' && \(needs\.catalog\.outputs\.operations_cadence == 'weekly' \|\| matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/,);const provisionedProfiles = new Set([...condition!.matchAll(/matrix\.profileId == '([^']+)'/g)].map((match) => match[1]),@@ -77,7 +77,7 @@ describe("Codex CI sandbox trust boundary", () => {const setup = workflow.indexOf("- name: Provision Codex sandbox on the disposable trusted runner",);- const paid = workflow.indexOf("- name: Run paid cell");+ const paid = workflow.indexOf("- name: Run bounded campaign and retain feedback review");expect(setup).toBeGreaterThan(workflow.indexOf("- name: Reauthorize paid execution before provider access",diff --git a/tests/runner-e2e/cost-ledger.test.ts b/tests/runner-e2e/cost-ledger.test.tsnew file mode 100644index 0000000000..c4c0346775--- /dev/null+++ b/tests/runner-e2e/cost-ledger.test.ts@@ -0,0 +1,197 @@+import { afterEach, expect, it, vi } from "vitest";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalCostObservation, type EvalSpendLimits } from "./cost-ledger.js";++const roots: string[] = [];+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });+const limits = (overrides: Partial<EvalSpendLimits> = {}): EvalSpendLimits => ({ campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2, ...overrides });+const identity = { executionId: "codex.local.fixture", attemptId: "attempt-1", phase: "setup" as const };+const observation = (overrides: Partial<EvalCostObservation> = {}): EvalCostObservation => ({ observationId: "receipt", requestId: "request", source: "provider", provider: "openai", model: "fixture", providerRequestId: "provider-request", currency: "USD", route: "direct", classification: "reported", costUsd: "0.000000123", final: true, ...overrides });+async function fixture(overrides: Partial<EvalSpendLimits> = {}) {+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-cost-ledger-")); roots.push(root);+ const file = path.join(root, "costs.jsonl");+ return { file, ledger: await openEvalCostLedger(file, limits(overrides)) };+}++it("persists a reservation before dispatch and counts setup spending with nanodollar precision", async () => {+ const { file, ledger } = await fixture();+ try {+ await ledger.reserve("request", identity, "0.1");+ const beforeDispatch = (await readEvalCostLedger(file)).state.snapshot();+ expect(beforeDispatch).toMatchObject({ committedUsd: "0.100000000", heldUsd: "0.100000000", unknownRequests: 1 });+ await ledger.observe(observation());+ await ledger.finish("request", "succeeded");+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.000000123", committedUsd: "0.000000123", heldUsd: "0.000000000",+ requests: [{ phase: "setup", receiptComplete: true }] });+ } finally { await ledger.close(); }+ expect((await readEvalCostLedger(file)).state.snapshot().knownUsd).toBe("0.000000123");+});++it("serializes competing admissions so concurrent requests cannot oversubscribe a cap", async () => {+ const { ledger } = await fixture();+ try {+ const admitted = await Promise.allSettled([ledger.reserve("one", identity, "0.6"), ledger.reserve("two", identity, "0.6")]);+ expect(admitted.map(value => value.status)).toEqual(["fulfilled", "rejected"]);+ expect(ledger.snapshot().committedUsd).toBe("0.600000000");+ } finally { await ledger.close(); }+});++it("durably caps attempts including zero-request failures and rejects unadmitted requests", async () => {+ const cap = limits({ maxAttemptsPerExecution: 2 });+ const { file, ledger } = await fixture({ maxAttemptsPerExecution: 2 });+ await expect(ledger.reserve("request", identity, "0.1")).rejects.toThrow("no admitted attempt");+ await ledger.admitAttempt(identity.executionId, "attempt-1");+ await ledger.close();+ const resumed = await openEvalCostLedger(file, cap);+ try {+ await expect(resumed.admitAttempt(identity.executionId, "attempt-1")).rejects.toThrow("already admitted");+ const competing = await Promise.allSettled([+ resumed.admitAttempt(identity.executionId, "attempt-2"),+ resumed.admitAttempt(identity.executionId, "attempt-3"),+ ]);+ expect(competing.map(result => result.status)).toEqual(["fulfilled", "rejected"]);+ await resumed.reserve("request", { ...identity, attemptId: "attempt-2" }, "0.1");+ await resumed.admitAttempt("another.cell", "another-attempt");+ await expect(resumed.reserve("wrong-cell", { ...identity, attemptId: "another-attempt" }, "0.1")).rejects.toThrow("no admitted attempt");+ expect((await readEvalCostLedger(file)).state.snapshot().admittedAttempts).toHaveLength(3);+ } finally { await resumed.close(); }+ await expect(openEvalCostLedger(file, limits({ maxAttemptsPerExecution: 3 }))).rejects.toThrow("limits changed");+});++it("holds unknown/cancelled charges after restart and counts failed retries independently", async () => {+ const { file, ledger } = await fixture();+ await ledger.reserve("request", identity, "0.5");+ await ledger.finish("request", "cancelled");+ await ledger.close();+ const resumed = await openEvalCostLedger(file, limits());+ try {+ expect(resumed.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 1, committedUsd: "0.500000000" });+ await resumed.reserve("retry", { ...identity, attemptId: "attempt-2" }, "0.5");+ await resumed.observe(observation({ observationId: "retry-cost", requestId: "retry", providerRequestId: "provider-retry", costUsd: "0.2" }));+ await resumed.finish("retry", "failed");+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.500000000", committedUsd: "0.700000000" });+ await resumed.observe(observation({ costUsd: "0.3" }));+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.500000000", heldUsd: "0.000000000" });+ } finally { await resumed.close(); }+});++it("deduplicates provider/gateway/native observations and preserves invoice adjustments", async () => {+ const { ledger } = await fixture();+ try {+ await ledger.reserve("request", identity, "1");+ await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" }));+ await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" }));+ await ledger.observe(observation({ observationId: "native", source: "native_otlp", costUsd: "0.2" }));+ await ledger.observe(observation({ observationId: "invoice", source: "invoice", classification: "reconciled", costUsd: "0.15" }));+ await ledger.finish("request", "succeeded");+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.150000000", breakdown: { estimated: "0.000000000", reported: "0.000000000", reconciled: "0.150000000" } });+ await expect(ledger.observe(observation({ costUsd: "0.7" }))).rejects.toThrow("identity conflict");+ } finally { await ledger.close(); }+});++it("retains partial costs and contradictory receipts without freeing their reservations", async () => {+ const { ledger } = await fixture();+ try {+ await ledger.reserve("request", identity, "0.5");+ await ledger.observe(observation({ final: false, costUsd: "0.2" }));+ await ledger.finish("request", "failed");+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.300000000", unknownRequests: 1 });+ await ledger.observe(observation({ observationId: "final1", costUsd: "0.3" }));+ await ledger.observe(observation({ observationId: "final2", costUsd: "0.4" }));+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.400000000", heldUsd: "0.100000000", conflictingRequests: 1 });+ await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("receipts conflict");+ } finally { await ledger.close(); }+});++it("rejects ambiguous request joins and stops on measured overshoot", async () => {+ const { ledger } = await fixture();+ try {+ await ledger.reserve("request", identity, "0.1");+ await ledger.reserve("another", identity, "0.1");+ await ledger.observe(observation({ costUsd: "1.1" }));+ await expect(ledger.observe(observation({ observationId: "duplicate-request", requestId: "another" }))).rejects.toThrow("another reservation");+ expect(ledger.snapshot().overBudget).toBe(true);+ await ledger.finish("request", "succeeded");+ await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("spending cap");+ } finally { await ledger.close(); }+});++it("enforces deadlines, request count, concurrency, and explicit pricing/usage invariants", async () => {+ const expired = await fixture({ deadlineAt: "2000-01-01T00:00:00Z" });+ await expect(expired.ledger.reserve("request", identity, "0.1")).rejects.toThrow("deadline");+ await expired.ledger.close();+ const { ledger } = await fixture({ maxRequests: 2, maxConcurrent: 1 });+ try {+ await ledger.reserve("request", identity, "0.1");+ await expect(ledger.reserve("two", identity, "0.1")).rejects.toThrow("concurrency");+ await expect(ledger.observe(observation({ classification: "estimated" }))).rejects.toThrow("pricing provenance");+ await expect(ledger.observe(observation({ tokens: { output: 1, reasoning: 2 } }))).rejects.toThrow("token usage");+ await ledger.finish("request", "failed");+ await ledger.reserve("two", identity, "0.1");+ await ledger.finish("two", "failed");+ await expect(ledger.reserve("three", identity, "0.1")).rejects.toThrow("request cap");+ } finally { await ledger.close(); }+});++it("fails closed on concurrent writers, changed limits, incomplete writes, and tampering", async () => {+ const { file, ledger } = await fixture();+ await expect(openEvalCostLedger(file, limits())).rejects.toThrow("writer lock");+ await ledger.reserve("request", identity, "0.1");+ await ledger.close();+ await expect(openEvalCostLedger(file, limits({ maxUsd: "2" }))).rejects.toThrow("limits changed");+ const contents = await readFile(file, "utf8");+ await writeFile(file, contents.slice(0, -1));+ await expect(readEvalCostLedger(file)).rejects.toThrow("incomplete");+ await writeFile(file, contents.replace('"maxUsd":"1"', '"maxUsd":"2"'));+ await expect(readEvalCostLedger(file)).rejects.toThrow("integrity");+});++it("rejects accidental raw payload fields and accepts equivalent receipt key order", async () => {+ const { file, ledger } = await fixture();+ try {+ await ledger.reserve("request", identity, "0.1");+ await expect(ledger.observe({ ...observation(), rawResponse: "private response" } as EvalCostObservation)).rejects.toThrow("Unknown ledger field");+ await ledger.observe(observation());+ await ledger.observe(Object.fromEntries(Object.entries(observation()).reverse()) as unknown as EvalCostObservation);+ expect(await readFile(file, "utf8")).not.toContain("private response");+ } finally { await ledger.close(); }+});++it("does not let a terminal estimate erase a stronger partial provider charge", async () => {+ const { ledger } = await fixture();+ try {+ await ledger.reserve("request", identity, "0.8");+ await ledger.observe(observation({ observationId: "provider-partial", final: false, costUsd: "0.6" }));+ await ledger.observe(observation({ observationId: "gateway-final", source: "gateway", classification: "estimated", pricingVersion: "fixture", costUsd: "0.1" }));+ await ledger.finish("request", "succeeded");+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.600000000", heldUsd: "0.200000000", unknownRequests: 1, breakdown: { reported: "0.600000000", estimated: "0.000000000" } });+ await expect(ledger.reserve("next", identity, "0.3")).rejects.toThrow("spending cap");+ await ledger.observe(observation({ observationId: "provider-final", costUsd: "0.65" }));+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.650000000", heldUsd: "0.000000000", unknownRequests: 0 });+ } finally { await ledger.close(); }+});++it("replays admissions when the deadline crosses between validation and journal writing", async () => {+ const admittedAt = Date.parse("2098-12-31T23:59:59.999Z");+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(admittedAt);+ const { file, ledger } = await fixture();+ const validate = EvalSpendState.prototype.validate;+ const validation = vi.spyOn(EvalSpendState.prototype, "validate").mockImplementation(function (this: EvalSpendState, event, now) {+ validate.call(this, event, now);+ vi.setSystemTime(admittedAt + 2);+ });+ try {+ await ledger.reserve("request", identity, "0.1");+ validation.mockRestore();+ expect((await readEvalCostLedger(file)).state.snapshot().committedUsd).toBe("0.100000000");+ const records = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));+ expect(records[1].at).toBe(new Date(admittedAt).toISOString());+ } finally {+ validation.mockRestore();+ vi.useRealTimers();+ await ledger.close();+ }+});diff --git a/tests/runner-e2e/cost-ledger.ts b/tests/runner-e2e/cost-ledger.tsnew file mode 100644index 0000000000..c132a6257f--- /dev/null+++ b/tests/runner-e2e/cost-ledger.ts@@ -0,0 +1,293 @@+import { createHash } from "node:crypto";+import { constants } from "node:fs";+import { mkdir, open, readFile, rm, type FileHandle } from "node:fs/promises";+import path from "node:path";+import { usdToUnits } from "../../packages/shared/src/money.js";+import { validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";++export interface EvalSpendLimits {+ campaignId: string;+ maxUsd: string;+ deadlineAt: string;+ maxRequests: number;+ maxConcurrent: number;+ maxAttemptsPerExecution?: number;+ overhead?: EvalOperatingOverhead;+}+export interface EvalChargeIdentity {+ executionId: string;+ attemptId: string;+ phase: "setup" | "agent" | "judge" | "feedback_review" | "tool" | "infrastructure";+ runId?: string;+}+export interface EvalLegacyAttemptAudit {+ previousJournalHash: string;+ evidenceSha256: string;+ maxAttemptsPerExecution: number;+ priorAttempts: Array<{ executionId: string; attemptId: string }>;+}+export interface EvalCostObservation {+ observationId: string;+ requestId: string;+ source: "provider" | "gateway" | "native_otlp" | "heartbeat" | "infrastructure" | "invoice";+ provider: string;+ model?: string;+ providerRequestId?: string;+ currency: "USD";+ route: "direct" | "braintrust" | "other_gateway" | "subscription" | "infrastructure";+ classification: "reported" | "estimated" | "allocated" | "reconciled" | "unknown";+ costUsd?: string;+ pricingVersion?: string;+ tokens?: { input?: number; cachedRead?: number; cachedWrite?: number; output?: number; reasoning?: number };+ final: boolean;+}++type Event =+ | { type: "campaign"; limits: EvalSpendLimits }+ | ({ type: "attempt_policy" } & EvalLegacyAttemptAudit)+ | { type: "attempt"; executionId: string; attemptId: string }+ | { type: "reserve"; requestId: string; identity: EvalChargeIdentity; maxUsd: string }+ | { type: "observe"; observation: EvalCostObservation }+ | { type: "finish"; requestId: string; outcome: "succeeded" | "failed" | "cancelled" };+interface Line { seq: number; at: string; previous: string; event: Event; hash: string }+interface RequestState { identity: EvalChargeIdentity; reserved: bigint; observations: EvalCostObservation[]; finished: boolean }+const usd = (units: bigint) => `${units / 1_000_000_000n}.${String(units % 1_000_000_000n).padStart(9, "0")}`;+export const validEvalLedgerId = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value);+const validId = validEvalLedgerId;+const amount = (value: string) => {+ if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(value)) throw new Error("Invalid ledger USD amount");+ return usdToUnits(value);+};+const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value, (_key, item) =>+ item && typeof item === "object" && !Array.isArray(item)+ ? Object.fromEntries(Object.keys(item).sort().map(key => [key, item[key]])) : item)).digest("hex");+const fields = (value: object, allowed: string[]) => {+ if (Object.keys(value).some(key => !allowed.includes(key))) throw new Error("Unknown ledger field; refusing unreviewed data");+};+const ranks = { unknown: 0, allocated: 1, estimated: 2, reported: 3, reconciled: 4 };++export class EvalAdmissionError extends Error {}++export class EvalSpendState {+ readonly limits: EvalSpendLimits;+ readonly requests = new Map<string, RequestState>();+ readonly observations = new Map<string, EvalCostObservation>();+ readonly attempts = new Map<string, string>();+ private providerRequests = new Map<string, string>();+ constructor(limits: EvalSpendLimits) {+ this.limits = structuredClone(limits);+ fields(limits, ["campaignId", "maxUsd", "deadlineAt", "maxRequests", "maxConcurrent", "maxAttemptsPerExecution", "overhead"]);+ if (!validId(limits.campaignId) || amount(limits.maxUsd) <= 0n || !Number.isFinite(Date.parse(limits.deadlineAt))+ || !Number.isSafeInteger(limits.maxRequests) || limits.maxRequests < 1+ || !Number.isSafeInteger(limits.maxConcurrent) || limits.maxConcurrent < 1+ || limits.maxAttemptsPerExecution !== undefined && (!Number.isSafeInteger(limits.maxAttemptsPerExecution) || limits.maxAttemptsPerExecution < 1)) throw new Error("Invalid eval spend limits");+ if (this.limits.overhead !== undefined) {+ validateOperatingOverhead(this.limits.overhead, this.limits.maxUsd);+ if (this.limits.maxAttemptsPerExecution === undefined || this.limits.maxRequests <= this.limits.overhead.components.length) {+ throw new Error("Operating overhead requires bounded attempts and room for model requests");+ }+ for (const component of this.limits.overhead.components) this.requests.set(`overhead:${component.component}`, {+ identity: { executionId: "campaign-overhead", attemptId: component.component, phase: "infrastructure" },+ reserved: amount(component.maxUsd), observations: [], finished: true,+ });+ }+ }+ private authoritative(request: RequestState) {+ const priced = request.observations.filter(value => value.costUsd !== undefined);+ const bestRank = Math.max(-1, ...priced.map(value => ranks[value.classification]));+ const strongest = priced.filter(value => ranks[value.classification] === bestRank);+ const finalized = strongest.filter(value => value.final);+ const best = finalized.length ? finalized : strongest;+ if (!best.length) return { cost: null, classification: "unknown" as const, conflict: false, resolved: false };+ const amounts = new Set(best.map(value => amount(value.costUsd!).toString()));+ const cost = best.reduce((maximum, value) => amount(value.costUsd!) > maximum ? amount(value.costUsd!) : maximum, 0n);+ return { cost, classification: best[0]!.classification, conflict: finalized.length > 0 && amounts.size !== 1,+ resolved: finalized.length > 0 && amounts.size === 1 };+ }+ snapshot() {+ let known = 0n, held = 0n, committed = 0n, inflight = 0, unknown = 0, conflicts = 0;+ const breakdown: Record<"reported" | "estimated" | "allocated" | "reconciled", bigint> = { reported: 0n, estimated: 0n, allocated: 0n, reconciled: 0n };+ const requests = [...this.requests].map(([requestId, request]) => {+ const receipt = this.authoritative(request);+ if (!request.finished) inflight++;+ if (receipt.conflict) conflicts++;+ if (!receipt.resolved) unknown++;+ const cost = receipt.cost ?? 0n;+ known += cost;+ if (receipt.classification !== "unknown") breakdown[receipt.classification] += cost;+ const commitment = !request.finished || !receipt.resolved+ ? (request.reserved > cost ? request.reserved : cost) : cost;+ committed += commitment;+ held += commitment - cost;+ return { requestId, ...request.identity, reservedUsd: usd(request.reserved), costUsd: receipt.cost === null ? null : usd(cost),+ classification: receipt.classification, conflict: receipt.conflict, receiptComplete: receipt.resolved, finished: request.finished };+ });+ return { campaignId: this.limits.campaignId, limitUsd: usd(amount(this.limits.maxUsd)), knownUsd: usd(known), heldUsd: usd(held), committedUsd: usd(committed),+ overBudget: committed > amount(this.limits.maxUsd), unknownRequests: unknown, conflictingRequests: conflicts, inflight,+ breakdown: Object.fromEntries(Object.entries(breakdown).map(([key, value]) => [key, usd(value)])), requests,+ admittedAttempts: [...this.attempts].map(([attemptId, executionId]) => ({ executionId, attemptId })) };+ }+ validate(event: Event, now = Date.now()) {+ if ((event.type === "attempt" || event.type === "reserve") && this.limits.overhead+ && (now < Date.parse(this.limits.overhead.verifiedAt) || now >= Date.parse(this.limits.overhead.expiresAt))) {+ throw new EvalAdmissionError("Operating overhead allowance needs review before paid admission");+ }+ if (event.type === "campaign") throw new Error("Duplicate campaign event");+ if (event.type === "attempt_policy") {+ fields(event, ["type", "previousJournalHash", "evidenceSha256", "maxAttemptsPerExecution", "priorAttempts"]);+ if (this.limits.maxAttemptsPerExecution !== undefined || this.attempts.size) throw new Error("Eval attempt policy already established");+ if (!/^[0-9a-f]{64}$/.test(event.previousJournalHash) || !/^[0-9a-f]{64}$/.test(event.evidenceSha256)+ || !Number.isSafeInteger(event.maxAttemptsPerExecution) || event.maxAttemptsPerExecution < 1+ || !Array.isArray(event.priorAttempts)) throw new Error("Invalid legacy attempt audit");+ const audited = new Map<string, string>();+ for (const attempt of event.priorAttempts) {+ fields(attempt, ["executionId", "attemptId"]);+ if (!validId(attempt.executionId) || !validId(attempt.attemptId) || audited.has(attempt.attemptId)) throw new Error("Invalid legacy attempt identity");+ audited.set(attempt.attemptId, attempt.executionId);+ }+ for (const request of this.requests.values()) {+ if (!request.finished) throw new Error("Legacy attempt audit requires stopped requests");+ if (audited.get(request.identity.attemptId) !== request.identity.executionId) throw new Error("Legacy attempt audit omits a recorded request");+ }+ } else if (event.type === "attempt") {+ fields(event, ["type", "executionId", "attemptId"]);+ if (!validId(event.executionId) || !validId(event.attemptId) || this.limits.maxAttemptsPerExecution === undefined) throw new Error("Invalid attempt admission");+ if (this.attempts.has(event.attemptId)) throw new EvalAdmissionError("Eval attempt already admitted");+ if (now >= Date.parse(this.limits.deadlineAt)) throw new EvalAdmissionError("Eval campaign deadline reached");+ if ([...this.attempts.values()].filter(id => id === event.executionId).length >= this.limits.maxAttemptsPerExecution) throw new EvalAdmissionError("Eval campaign attempt cap reached");+ } else if (event.type === "reserve") {+ fields(event.identity, ["executionId", "attemptId", "phase", "runId"]);+ if (!validId(event.requestId) || !validId(event.identity.executionId) || !validId(event.identity.attemptId)+ || event.identity.runId !== undefined && !validId(event.identity.runId)+ || !["setup", "agent", "judge", "feedback_review", "tool", "infrastructure"].includes(event.identity.phase)) throw new Error("Invalid request identity");+ if (this.limits.maxAttemptsPerExecution !== undefined && this.attempts.get(event.identity.attemptId) !== event.identity.executionId) throw new EvalAdmissionError("Eval request has no admitted attempt");+ if (this.requests.has(event.requestId)) throw new Error("Request already reserved");+ if (amount(event.maxUsd) <= 0n) throw new Error("Request reservation must be positive");+ const current = this.snapshot();+ if (now >= Date.parse(this.limits.deadlineAt)) throw new EvalAdmissionError("Eval campaign deadline reached");+ if (this.requests.size >= this.limits.maxRequests) throw new EvalAdmissionError("Eval campaign request cap reached");+ if (current.inflight >= this.limits.maxConcurrent) throw new EvalAdmissionError("Eval campaign concurrency cap reached");+ if (current.conflictingRequests) throw new EvalAdmissionError("Eval cost receipts conflict; admission stopped");+ if (amount(current.committedUsd) + amount(event.maxUsd) > amount(this.limits.maxUsd)) throw new EvalAdmissionError("Eval campaign spending cap reached");+ } else if (event.type === "observe") {+ const o = event.observation;+ fields(o, ["observationId", "requestId", "source", "provider", "model", "providerRequestId", "currency", "route", "classification", "costUsd", "pricingVersion", "tokens", "final"]);+ if (o.tokens) fields(o.tokens, ["input", "cachedRead", "cachedWrite", "output", "reasoning"]);+ if (!this.requests.has(o.requestId) || !validId(o.observationId) || !validId(o.provider)+ || o.model !== undefined && !validId(o.model) || o.providerRequestId !== undefined && !validId(o.providerRequestId)+ || o.currency !== "USD" || !Object.hasOwn(ranks, o.classification) || typeof o.final !== "boolean"+ || !["provider", "gateway", "native_otlp", "heartbeat", "infrastructure", "invoice"].includes(o.source)+ || !["direct", "braintrust", "other_gateway", "subscription", "infrastructure"].includes(o.route)) throw new Error("Invalid cost observation");+ if (o.classification === "unknown" ? o.costUsd !== undefined : o.costUsd === undefined) throw new Error("Cost classification requires an explicit amount or unknown");+ if (o.costUsd !== undefined) amount(o.costUsd);+ if ((o.classification === "estimated" || o.classification === "allocated") && !validId(o.pricingVersion)) throw new Error("Estimate requires pricing provenance");+ if (o.tokens && (Object.values(o.tokens).some(value => !Number.isSafeInteger(value) || Number(value) < 0)+ || o.tokens.reasoning !== undefined && o.tokens.output !== undefined && o.tokens.reasoning > o.tokens.output)) throw new Error("Invalid normalized token usage");+ const previous = this.observations.get(o.observationId);+ if (previous && digest(previous) !== digest(o)) throw new Error("Observation identity conflict");+ if (o.providerRequestId) {+ const request = this.providerRequests.get(`${o.provider}:${o.providerRequestId}`);+ if (request && request !== o.requestId) throw new Error("Provider request belongs to another reservation; reconcile identities before import");+ }+ } else if (event.type === "finish") {+ if (!this.requests.has(event.requestId) || !["succeeded", "failed", "cancelled"].includes(event.outcome)) throw new Error("Invalid request completion");+ } else throw new Error("Unknown ledger event");+ }+ apply(event: Event) {+ if (event.type === "attempt_policy") {+ this.limits.maxAttemptsPerExecution = event.maxAttemptsPerExecution;+ for (const attempt of event.priorAttempts) this.attempts.set(attempt.attemptId, attempt.executionId);+ } else if (event.type === "attempt") this.attempts.set(event.attemptId, event.executionId);+ else if (event.type === "reserve") this.requests.set(event.requestId, { identity: event.identity, reserved: amount(event.maxUsd), observations: [], finished: false });+ else if (event.type === "observe") {+ const o = event.observation;+ if (this.observations.has(o.observationId)) return;+ this.observations.set(o.observationId, o);+ this.requests.get(o.requestId)!.observations.push(o);+ if (o.providerRequestId) this.providerRequests.set(`${o.provider}:${o.providerRequestId}`, o.requestId);+ } else if (event.type === "finish") this.requests.get(event.requestId)!.finished = true;+ }+}++export async function readEvalCostLedger(file: string) {+ const raw = await readFile(file, "utf8");+ if (!raw.endsWith("\n")) throw new Error("Eval ledger has an incomplete durable record");+ const lines = raw.trimEnd().split("\n").map(value => JSON.parse(value) as Line);+ let previous = "", state: EvalSpendState | undefined;+ for (const [index, line] of lines.entries()) {+ const { hash, ...body } = line;+ if (line.seq !== index || line.previous !== previous || digest(body) !== hash || !Number.isFinite(Date.parse(line.at))) throw new Error("Eval ledger integrity check failed");+ if (index === 0) {+ if (line.event.type !== "campaign") throw new Error("Eval ledger lacks campaign limits");+ state = new EvalSpendState(line.event.limits);+ } else {+ if (line.event.type === "attempt_policy" && line.event.previousJournalHash !== previous) throw new Error("Legacy attempt audit checkpoint changed");+ state!.validate(line.event, Date.parse(line.at));+ state!.apply(line.event);+ }+ previous = hash;+ }+ return { state: state!, sequence: lines.length, previous };+}++export async function openEvalCostLedger(file: string, limits: EvalSpendLimits) {+ await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });+ const lockPath = `${file}.lock`;+ let lock: FileHandle;+ try { lock = await open(lockPath, "wx", 0o600); }+ catch { throw new Error("Eval ledger writer lock unavailable; inspect prior writer before recovery"); }+ let handle: FileHandle | undefined;+ try {+ await lock.writeFile(`${process.pid}\n`);+ await lock.sync();+ let state = new EvalSpendState(limits), sequence = 0, previous = "";+ try {+ const loaded = await readEvalCostLedger(file);+ if (digest(loaded.state.limits) !== digest(limits)) throw new Error("Eval campaign limits changed during resume");+ ({ state, sequence, previous } = loaded);+ } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }+ handle = await open(file, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);+ const write = async (event: Event, at = Date.now()) => {+ const body = { seq: sequence, at: new Date(at).toISOString(), previous, event };+ const hash = digest(body);+ await handle!.appendFile(`${JSON.stringify({ ...body, hash })}\n`);+ await handle!.sync();+ previous = hash;+ sequence++;+ };+ if (!sequence) await write({ type: "campaign", limits });+ let queue = Promise.resolve(), closed = false, poisoned = false;+ const append = (event: Event) => {+ if (closed) return Promise.reject(new Error("Eval ledger writer unavailable"));+ const operation = queue.then(async () => {+ if (poisoned) throw new Error("Eval ledger writer unavailable");+ const admittedAt = Date.now();+ if (event.type === "attempt_policy" && event.previousJournalHash !== previous) throw new Error("Legacy attempt audit checkpoint changed");+ state.validate(event, admittedAt);+ try { await write(event, admittedAt); } catch (error) { poisoned = true; throw error; }+ state.apply(event);+ });+ queue = operation.catch(() => {});+ return operation;+ };+ return {+ adoptLegacyAttemptPolicy: (audit: EvalLegacyAttemptAudit) => append({ ...structuredClone(audit), type: "attempt_policy" }),+ admitAttempt: (executionId: string, attemptId: string) => append({ type: "attempt", executionId, attemptId }),+ reserve: (requestId: string, identity: EvalChargeIdentity, maxUsd: string) => append({ type: "reserve", requestId, identity: structuredClone(identity), maxUsd }),+ observe: (observation: EvalCostObservation) => append({ type: "observe", observation: structuredClone(observation) }),+ finish: (requestId: string, outcome: "succeeded" | "failed" | "cancelled") => append({ type: "finish", requestId, outcome }),+ snapshot: () => state.snapshot(),+ async close() {+ if (closed) return;+ closed = true;+ await queue;+ try { await handle!.close(); } finally { await lock.close(); await rm(lockPath); }+ },+ };+ } catch (error) {+ await handle?.close();+ await lock.close();+ await rm(lockPath);+ throw error;+ }+}diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.tsindex 96976b8b9b..2913bd9645 100644--- a/tests/runner-e2e/dashboard-regenerate.ts+++ b/tests/runner-e2e/dashboard-regenerate.ts@@ -2,6 +2,7 @@ import path from "node:path";import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";import { runnerMatrix } from "./catalog.js";import { renderRunnerE2EDashboard } from "./dashboard.js";+import { campaignFeedbackReviewText } from "./feedback-dispositions.js";import {buildRunnerCampaign,canonicalExecutionId,@@ -25,6 +26,10 @@ interface PublishedCampaign {generatedAt: string;expected: string[];results: PublishedResult[];+ accounting?: RunnerE2ECampaign["accounting"];+ operations?: RunnerE2ECampaign["operations"];+ feedbackReview?: RunnerE2ECampaign["feedbackReview"];+ braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"];}async function relativeFiles(root: string, current = root): Promise<string[]> {@@ -57,6 +62,7 @@ export async function regenerateRunnerDashboard(input: {historyFile?: string | null;outputDirectory?: string;evidenceHrefPrefix?: string;+ feedbackReviewFile?: string;}) {const bundle = path.resolve(input.bundle);const outputDirectory = path.resolve(input.outputDirectory ?? bundle);@@ -93,6 +99,12 @@ export async function regenerateRunnerDashboard(input: {generatedAt: normalized.generatedAt,expected,results,+ accounting: normalized.accounting,+ operations: normalized.operations,+ braintrustExperiment: normalized.braintrustExperiment,+ feedbackReview: input.feedbackReviewFile+ ? JSON.parse(await readFile(input.feedbackReviewFile, "utf8"))+ : normalized.feedbackReview,});// A rendering refresh must not inherit the renderer's checkout or CI event.const retainedSource = results.find((result) =>@@ -168,10 +180,21 @@ export async function regenerateRunnerDashboard(input: {evidenceErrors: normalized.results[index]?.evidenceErrors ?? [],})),};+ const summary = await readFile(path.join(bundle, "summary.md"), "utf8").then(text => text.replace(+ /^(Feedback across retained attempts:.*?); [0-9]+ distinct items awaiting review;/m, "$1;",+ )).catch((error: NodeJS.ErrnoException) => {+ if (error.code === "ENOENT") return undefined;+ throw error;+ });+ const reviewStatus = /^(?:[0-9]+ distinct feedback items await review\.|Feedback reviewed by (?:human|agent) at ).*$/m;+ const refreshedSummary = summary === undefined ? undefined : reviewStatus.test(summary)+ ? summary.replace(reviewStatus, () => campaignFeedbackReviewText(campaign))+ : `${summary.trimEnd()}\n\n## Campaign feedback review\n\n${campaignFeedbackReviewText(campaign)}\n`;await mkdir(outputDirectory, { recursive: true });await Promise.all([writeFile(path.join(outputDirectory, "index.html"), dashboard, "utf8"),writeFile(path.join(outputDirectory, "dashboard.html"), dashboard, "utf8"),+ ...(refreshedSummary === undefined ? [] : [writeFile(path.join(outputDirectory, "summary.md"), refreshedSummary, "utf8")]),writeFile(path.join(outputDirectory, "normalized-results.json"),`${JSON.stringify(upgraded, null, 2)}\n`,@@ -192,7 +215,7 @@ async function main() {);if (!bundleArgument) {throw new Error(- "Usage: pnpm test:e2e:runner:dashboard -- <published-bundle-directory> [--history <history.json>]",+ "Usage: pnpm test:e2e:runner:dashboard -- <published-bundle-directory> [--history <history.json>] [--feedback-review <review.json>]",);}const historyIndex = arguments_.indexOf("--history");@@ -200,7 +223,10 @@ async function main() {historyIndex >= 0 ? arguments_[historyIndex + 1] : undefined;if (historyIndex >= 0 && !historyFile)throw new Error("--history requires a path");- await regenerateRunnerDashboard({ bundle: bundleArgument, historyFile });+ const reviewIndex = arguments_.indexOf("--feedback-review");+ const feedbackReviewFile = reviewIndex >= 0 ? arguments_[reviewIndex + 1] : undefined;+ if (reviewIndex >= 0 && !feedbackReviewFile) throw new Error("--feedback-review requires a path");+ await regenerateRunnerDashboard({ bundle: bundleArgument, historyFile, feedbackReviewFile });}if (diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.tsindex c949f6a07b..b8cb4ac92b 100644--- a/tests/runner-e2e/dashboard.ts+++ b/tests/runner-e2e/dashboard.ts@@ -1,3 +1,4 @@+import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, summarizeCampaignTiming, summarizeWorkerPhaseTiming } from "./campaign-timing.js";import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js";import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js";import { reportBrowserScript } from "./report-browser.js";@@ -8,6 +9,10 @@ import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js";import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js";import { STARTUP_REUSE_LAYERS, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js";import { renderFirstTaskDetails } from "./first-task-report.js";+import { summarizeCampaignFeedback } from "./feedback.js";+import { renderCampaignFeedbackReview } from "./feedback-dispositions.js";+import { renderEvalOperationsReport } from "./operations-report.js";+import { renderBraintrustExperiment } from "./braintrust-experiment.js";import {discoverReportCatalog,type ReportExecution,@@ -20,6 +25,7 @@ import type {} from "./types.js";import {aggregateCampaignBilling,+ billingCoverageLabel,summarizeExecutionBilling,} from "./billing.js";@@ -269,6 +275,7 @@ function renderCase(? `<div class="billing-strip" aria-label="Billing for ${html(execution.id)}"><div><span>Tokens</span><strong>${html(tokenLabel(billing.llm.inputTokens, billing.llm))} in · ${html(tokenLabel(billing.llm.outputTokens, billing.llm))} out</strong><small>${html(tokenLabel(billing.llm.cachedInputTokens, billing.llm))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered</small></div><div><span>LLM spend</span><strong>${html(llmCostLabel(billing.reportedCostUsd, billing.llm))}</strong><small>${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced</small></div>+ ${(billing.llm.runsWithEstimatedCost ?? 0) > 0 ? `<div><span>LLM estimate</span><strong>${html(billingCoverageLabel(usdLabel(billing.llm.estimatedCostUsd ?? 0), billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount))}</strong><small>Estimated from captured usage</small></div>` : ""}<div><span>Execution</span><strong>${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}</strong><small>${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}</small></div>${billing.assistant ? `<div><span>External assistant</span><strong>${html(usdLabel(billing.assistant.estimatedCostUsd))} est.</strong><small>${html(billing.assistant.model)} · ${billing.assistant.requests} requests · ${html(tokenLabel(billing.assistant.inputTokens + billing.assistant.cachedInputTokens))} in / ${html(tokenLabel(billing.assistant.outputTokens))} out</small></div>` : ""}</div>`@@ -301,6 +308,9 @@ function renderCase(<summary><span>Test details</span><span class="case-detail-summary">${matcherResults.length ? `${passedMatchers}/${matcherResults.length} checks${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded"}${entry ? ` · ${html(durationLabel(entry.result.durationMs))}` : ""}</span></summary>${passed ? outcome : ""}${billingStrip}+ ${entry ? `<p class="detail">Worker phase timing: ${entry.result.workerTiming ? html(entry.result.workerTiming.status === "unavailable" ? `unavailable (${entry.result.workerTiming.reason})` : entry.result.workerTiming.status) : "not captured"}. ${entry.result.workerTiming?.status === "partial" ? "Only completed phases have observed durations." : ""}</p>` : ""}+ ${entry?.result.feedback ? `<p class="detail">Feedback: ${entry.result.feedback.complaints} complaints · ${entry.result.feedback.suggestions} suggestions · ${entry.result.feedback.review.length} distinct items; see campaign feedback review. Saved-feedback collection: ${html(entry.result.feedback.collection)}; invocation coverage: ${html(entry.result.feedback.invocationCollection)}. Tool receipts: ${entry.result.feedback.invocations.failed} failed · ${entry.result.feedback.invocations.unknown} unresolved. Full bodies: access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.</p>` : entry ? `<p class="detail">Feedback was not captured for this attempt.</p>` : ""}+ ${entry?.result.tracing ? `<p class="detail">Braintrust worker export: ${html(entry.result.tracing.status)}; native configuration: ${html(entry.result.tracing.nativeConfiguration)}. ${entry.result.tracing.finalization ? `Supervisor outcome: ${html(entry.result.tracing.finalization.outcome)}; finalization export: ${html(entry.result.tracing.finalization.status)}.` : "Supervisor finalization was not captured."} Trace ${html(entry.result.tracing.traceId)} in project ${html(entry.result.tracing.projectId)}. Indexing and native-span correlation require independent verification.</p>` : ""}${links}${renderFirstTaskDetails(entry?.result)}<code class="execution-id">${html(execution.id)}</code>@@ -324,7 +334,7 @@ function renderCase(${latencyRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Event</th><th>Timing verdict</th><th>Possible latency</th><th>Target</th><th>Resolution</th></tr></thead><tbody>${latencyRows}</tbody></table></div>` : ""}${startupReuseRows ? `<div class="matcher-wrap"><table class="matchers"><caption>Startup reuse evidence</caption><thead><tr><th>Run</th><th>Warmth</th><th>Environment/lease</th><th>Workspace</th><th>Runner</th><th>Provider process</th><th>Provider session</th><th>Provider launches</th></tr></thead><tbody>${startupReuseRows}</tbody></table></div>` : ""}${matcherRows ? `<div class="matcher-wrap"><table class="matchers"><thead><tr><th>Result</th><th>Matcher</th><th>Expectation</th><th>Detail</th></tr></thead><tbody>${matcherRows}</tbody></table></div>` : `<p class="detail">No matcher result was recorded.</p>`}- ${entry ? `<details class="usage"><summary>Usage and billing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}</pre></details>` : ""}+ ${entry ? `<details class="usage"><summary>Usage, billing and timing metadata</summary><pre>${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null, workerTiming: entry.result.workerTiming ?? null }, null, 2))}</pre></details>` : ""}</details></article>`;}@@ -342,7 +352,7 @@ function renderSuiteResults(input: {? `<div class="suite-summary" aria-label="${html(suite.label)} current campaign summary"><div><span>Pass rate</span><strong>${summary.selected > 0 ? ((summary.passed / summary.selected) * 100).toFixed(1) : "0.0"}%</strong><small>${summary.passed}/${summary.selected} passed${summary.incomplete ? ` · ${summary.incomplete} incomplete` : ""}</small></div><div><span>Tokens</span><strong>${html(tokenLabel(summary.billing.llm.totalTokens, summary.billing.llm))}</strong><small>${html(tokenLabel(summary.billing.llm.inputTokens, summary.billing.llm))} input · ${html(tokenLabel(summary.billing.llm.outputTokens, summary.billing.llm))} output</small></div>- <div><span>Known spend</span><strong>${html(summary.billing.llm.runsWithReportedCost > 0 || summary.billing.estimatedRuntimeCostUsd > 0 || summary.billing.judge ? `${usdLabel(summary.billing.observedAndEstimatedCostUsd)}${summary.billing.testsWithCompleteBilling < summary.billing.testCount ? " (partial)" : ""}` : "Unavailable")}</strong><small>reported LLM + runtime${summary.billing.judge ? " + judge" : ""} estimate</small></div>+ <div><span>Known spend</span><strong>${html(summary.billing.llm.runsWithReportedCost > 0 || (summary.billing.llm.runsWithEstimatedCost ?? 0) > 0 || summary.billing.estimatedRuntimeCostUsd > 0 || summary.billing.judge || summary.billing.assistant ? `${usdLabel(summary.billing.observedAndEstimatedCostUsd)}${summary.billing.testsWithCompleteBilling < summary.billing.testCount ? " (partial)" : ""}` : "Unavailable")}</strong><small>reported LLM + model/runtime estimates${summary.billing.judge ? " + judge" : ""} estimate</small></div><div><span>Agent time</span><strong>${html(durationLabel(summary.billing.agentRunDurationMs))}</strong><small>${html(durationLabel(summary.billing.leaseDurationMs))} lease</small></div><div><span>Execution</span><strong>${summary.executed}/${summary.selected}</strong><small>${summary.retries} retries · cleanup ${summary.cleanupPassed ? "passed" : "failed"}</small></div></div>`@@ -386,15 +396,13 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {!entry.valid && isIncompleteFirstTaskResult(entry.result, entry.errors),).length;const failed = input.expected.length - passed - incomplete;- const totalDuration = selectedEntries.reduce(- (total, entry) => total + entry.result.durationMs,- 0,- );+ const timing = summarizeCampaignTiming(selectedEntries.map(entry => entry.result), input.campaign?.accounting);const screenshotCount = selectedEntries.reduce((total, entry) => total + resolveScreenshots(entry).length,0,);- const campaignBilling = aggregateCampaignBilling(+ const campaignFeedback = summarizeCampaignFeedback(input.campaign?.accounting?.attempts ?? input.entries.map(entry => entry.result));+ const campaignBilling = input.campaign?.billing ?? aggregateCampaignBilling(selectedEntries.map((entry) => entry.result),);const suites = [@@ -834,7 +842,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {<div class="metric"><strong>${passed}/${input.expected.length}</strong><span>Passed</span></div><div class="metric"><strong>${failed}</strong><span>Failed</span></div>${incomplete ? `<div class="metric"><strong>${incomplete}</strong><span>Incomplete</span></div>` : ""}- <div class="metric"><strong>${html(durationLabel(totalDuration))}</strong><span>Test time</span></div>+ <div class="metric"><strong>${html(campaignAttemptTimeLabel(timing))}</strong><span>Summed attempt time</span></div></div><button class="gallery-launch" type="button" data-gallery-open ${screenshotCount === 0 ? "disabled" : ""}>${screenshotCount === 0 ? "Visual evidence · workflow artifact only" : `View gallery · ${screenshotCount}`}</button></div>@@ -842,18 +850,23 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", false)}<details class="campaign-billing-details"><summary>Campaign usage and spend</summary><section class="billing-overview" aria-label="Campaign billing summary">+ <div class="billing-metric"><strong>${campaignFeedback.complaints} / ${campaignFeedback.suggestions}</strong><span>Complaints / suggestions · retained attempts</span><small>${campaignFeedback.review.length} distinct items · ${campaignFeedback.incomplete} attempts with incomplete collection · ${campaignFeedback.failedSubmissions} failed submissions · ${campaignFeedback.unknownSubmissions} unresolved submissions</small></div>+ ${input.campaign?.accounting ? `<div class="billing-metric"><strong>${input.campaign.accounting.attempts.length}</strong><span>${input.campaign.accounting.scope === "retained_attempts" ? "Retained attempts" : "Selected results only"}</span><small>${input.campaign.accounting.missingAttempts} missing attempts · ${input.campaign.accounting.excludedArtifacts} out-of-scope artifacts</small></div>` : ""}<div class="billing-metric"><strong>${html(tokenLabel(campaignBilling.llm.inputTokens, campaignBilling.llm))}</strong><span>Input tokens</span></div><div class="billing-metric"><strong>${html(tokenLabel(campaignBilling.llm.outputTokens, campaignBilling.llm))}</strong><span>Output tokens</span></div><div class="billing-metric"><strong>${html(tokenLabel(campaignBilling.llm.cachedInputTokens, campaignBilling.llm))}</strong><span>Cached tokens</span></div><div class="billing-metric"><strong>${html(llmCostLabel(campaignBilling.reportedLlmCostUsd, campaignBilling.llm))}</strong><span>LLM reported subtotal</span></div>+ ${(campaignBilling.llm.runsWithEstimatedCost ?? 0) > 0 ? `<div class="billing-metric"><strong>${html(billingCoverageLabel(usdLabel(campaignBilling.estimatedLlmCostUsd ?? 0), campaignBilling.llm.runsWithEstimatedCost ?? 0, campaignBilling.llm.runCount))}</strong><span>LLM estimated subtotal</span></div>` : ""}<div class="billing-metric"><strong>${html(usdLabel(campaignBilling.estimatedRuntimeCostUsd))}</strong><span>Daytona list estimate</span></div>${campaignBilling.judge ? `<div class="billing-metric"><strong>${html(usdLabel(campaignBilling.judge.estimatedCostUsd))}</strong><span>Judge estimate · ${campaignBilling.judge.attempts} attempts · ${campaignBilling.judge.attemptsWithUnknownUsage} unknown usage</span><small>${html(tokenLabel(campaignBilling.judge.inputTokens))} in / ${html(tokenLabel(campaignBilling.judge.outputTokens))} out · $${campaignBilling.judge.reservedCostUsd.toFixed(6)} reserved</small></div>` : ""}${campaignBilling.assistant ? `<div class="billing-metric"><strong>${html(usdLabel(campaignBilling.assistant.estimatedCostUsd))}</strong><span>External assistant estimate · ${campaignBilling.assistant.requests} requests</span><small>${html(tokenLabel(campaignBilling.assistant.inputTokens + campaignBilling.assistant.cachedInputTokens))} in / ${html(tokenLabel(campaignBilling.assistant.outputTokens))} out</small></div>` : ""}+ <div class="billing-metric"><strong>${html(durationLabel(timing.observedWindowMs))}</strong><span>Observed attempt window</span><small>${timing.retainedAttempts} retained attempts · ${timing.missingAttempts} missing attempts</small></div>+ <p class="billing-note">${html(campaignTimingScope)}</p><div class="billing-metric"><strong>${html(durationLabel(campaignBilling.agentRunDurationMs))}</strong><span>Agent execution time</span></div><div class="billing-metric"><strong>${html(durationLabel(campaignBilling.leaseDurationMs))}</strong><span>Daytona lease time</span></div><div class="billing-metric"><strong>${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}</strong><span>Runs provider-priced</span></div>- <p class="billing-note">Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.</p>- </section></details>+ <p class="billing-note">Campaign costs include retained attempts, including failed retries. Result cards show the selected attempt. Provider-reported amounts and model estimates are separate; unpriced or unavailable runs are excluded, never counted as free. Coverage describes captured usage; unobserved setup, subscriptions, tools, CI and storage require additional accounting. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.</p>+ </section>${renderWorkerPhaseTiming(summarizeWorkerPhaseTiming(selectedEntries.map(entry => entry.result), input.campaign?.accounting))}${input.campaign?.operations ? renderEvalOperationsReport(input.campaign.operations) : ""}${input.campaign ? renderCampaignFeedbackReview(input.campaign) + renderBraintrustExperiment(input.campaign) : ""}</details></section><div class="results-heading" id="results"><h2 data-results-title>All results</h2><span data-results-description></span></div><div data-feature-coverage>${renderJourneyFeatureCoverage(catalog, entryById)}</div>diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.tsindex 4ca6a41d37..8d3daf45be 100644--- a/tests/runner-e2e/daytona-image-content.ts+++ b/tests/runner-e2e/daytona-image-content.ts@@ -58,8 +58,12 @@ export const DAYTONA_IMAGE_INPUT_PATHS = ["packages/paperclip-runner/test/fixtures/copilot-profile-v47-identity.json","packages/paperclip-runner/test/fixtures/copilot-profile-v48-identity.json","packages/paperclip-runner/test/fixtures/copilot-profile-v49-identity.json",+ "packages/paperclip-runner/test/fixtures/copilot-profile-v50-identity.json",+ "packages/paperclip-runner/test/fixtures/copilot-profile-v51-identity.json",+ "packages/paperclip-runner/test/fixtures/copilot-profile-v52-identity.json","packages/paperclip-runner/test/fixtures/copilot-profile-v54-identity.json","packages/paperclip-runner/test/fixtures/copilot-profile-v55-identity.json",+ "packages/paperclip-runner/test/fixtures/copilot-profile-v57-identity.json","packages/paperclip-runner/test/fixtures/copilot-profile-v58-identity.json","packages/paperclip-runner/scripts/materialize-cursor-distribution.mjs","packages/paperclip-runner/scripts/cursor-runtime-patch.mjs",diff --git a/tests/runner-e2e/eval-finalization-relay.test.ts b/tests/runner-e2e/eval-finalization-relay.test.tsnew file mode 100644index 0000000000..d2453eb78e--- /dev/null+++ b/tests/runner-e2e/eval-finalization-relay.test.ts@@ -0,0 +1,40 @@+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, it, vi } from "vitest";+import { EVAL_FINALIZATION_LIMITS, finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js";++it("meters finalization after the worker has closed without changing the original trace or failed outcome", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-finalization-ingestion-"));+ try {+ const projectId = "11111111-1111-4111-8111-111111111111";+ const input = { receipt: { projectId, traceId: "a".repeat(32), spanId: "b".repeat(16), status: "pending" as const,+ verification: "unverified" as const, nativeConfiguration: "configured" as const },+ identity: { campaignId: "campaign", attemptId: "attempt", executionId: "fixture.local", provider: "fixture", model: "fixture", environment: "local" },+ provenance: { suiteId: "fixture", suiteDefinitionHash: "c".repeat(64), profileId: "fixture", caseId: "case", sourceRevision: "d".repeat(40) },+ result: { startedAt: new Date(Date.now() - 60_000).toISOString(), status: "failed" as const, cleanup: "failed" as const } };+ const before = structuredClone(input);+ let sentBytes = 0;+ const upstream = vi.fn<typeof fetch>(async (url, init) => {+ expect(String(url)).toBe("https://api.braintrust.dev/otel/v1/traces");+ expect(init!.headers).toMatchObject({ authorization: "Bearer upstream-private" });+ const body = Buffer.from(init!.body as Uint8Array); sentBytes = body.length;+ expect(body.toString()).toContain("paperclip.eval.finalization");+ expect(body.includes(Buffer.from(input.receipt.traceId, "hex"))).toBe(true);+ return new Response(null, { status: 200, headers: { "content-type": "application/x-protobuf" } });+ });+ const file = path.join(root, "finalization.jsonl");+ const result = await finalizeEvalAttemptThroughRelay({ projectId, apiKey: "upstream-private" }, input, file, upstream);+ expect(result).toMatchObject({ status: "accepted", outcome: "failed" });+ expect(input).toEqual(before);+ const raw = await readFile(file, "utf8"), records = raw.trim().split("\n").map(line => JSON.parse(line));+ expect(records[0]).toMatchObject({ campaignId: "campaign", attemptId: "attempt", limits: EVAL_FINALIZATION_LIMITS });+ expect(records.at(-1)).toMatchObject({ type: "closed", admittedRequests: 1, admittedBytes: sentBytes, acceptedRequests: 1 });+ expect(raw).not.toContain("upstream-private");+ expect((await stat(file)).mode & 0o777).toBe(0o600);+ expect(await finalizeEvalAttemptThroughRelay({ projectId, apiKey: "upstream-private" }, input, file, upstream))+ .toMatchObject({ status: "failed", errorCode: "finalization_ingestion_unavailable" });+ expect(upstream).toHaveBeenCalledTimes(1);+ expect(await readFile(file, "utf8")).toBe(raw);+ } finally { await rm(root, { recursive: true, force: true }); }+});diff --git a/tests/runner-e2e/eval-finalization-relay.ts b/tests/runner-e2e/eval-finalization-relay.tsnew file mode 100644index 0000000000..db0c2119f8--- /dev/null+++ b/tests/runner-e2e/eval-finalization-relay.ts@@ -0,0 +1,24 @@+import { randomBytes } from "node:crypto";+import { finalizeEvalAttemptTrace, type EvalBraintrustConfig, type EvalFinalizationReceipt } from "./braintrust.js";+import { startEvalOtlpRelay } from "./eval-otlp-relay.js";++export const EVAL_FINALIZATION_LIMITS = { maxRequests: 1, maxRequestBytes: 64 * 1024,+ maxTotalBytes: 64 * 1024, maxConcurrent: 1, maxExpandedBytes: 1024 * 1024, maxRecords: 256 } as const;++export async function finalizeEvalAttemptThroughRelay(config: EvalBraintrustConfig,+ input: Parameters<typeof finalizeEvalAttemptTrace>[1], receiptPath: string, transport?: typeof fetch): Promise<EvalFinalizationReceipt> {+ let relay: Awaited<ReturnType<typeof startEvalOtlpRelay>> | undefined;+ let receipt: EvalFinalizationReceipt = { spanId: randomBytes(8).toString("hex"), status: "failed",+ outcome: input.result.status === "passed" && input.result.cleanup === "passed" ? "passed" : "failed", verification: "unverified" };+ try {+ relay = await startEvalOtlpRelay({ projectId: config.projectId, apiKey: config.apiKey,+ campaignId: input.identity.campaignId, attemptId: input.identity.attemptId, receiptPath,+ deadlineAt: Date.now() + 30_000, limits: EVAL_FINALIZATION_LIMITS, transport });+ receipt = await finalizeEvalAttemptTrace({ ...config, apiKey: relay.token, endpointBaseUrl: relay.baseUrl }, input);+ } catch { receipt.errorCode = "finalization_ingestion_unavailable"; }+ finally {+ try { await relay?.close(); }+ catch { receipt.status = "failed"; receipt.errorCode = "finalization_ingestion_persistence_failed"; }+ }+ return receipt;+}diff --git a/tests/runner-e2e/eval-gateway.test.ts b/tests/runner-e2e/eval-gateway.test.tsnew file mode 100644index 0000000000..4fa02239ca--- /dev/null+++ b/tests/runner-e2e/eval-gateway.test.ts@@ -0,0 +1,315 @@+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { connect } from "node:net";+import { afterEach, expect, it, vi } from "vitest";+import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js";+import { beginEvalAgentPhase, createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js";+import { evalRequestReservation, EvalUsageStream, forwardEvalRequest, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js";+const directories: string[] = [], closes: Array<() => Promise<void>> = [];+afterEach(async () => { for (const close of closes.splice(0).reverse()) await close(); for (const dir of directories.splice(0)) await rm(dir, { recursive: true, force: true }); });+const identity = { executionId: "hello", attemptId: "attempt", phase: "agent" as const };+const route: EvalGatewayRoute = { provider: "openai", mode: "braintrust", apiKey: "private-upstream-fixture", projectId: "11111111-1111-4111-8111-111111111111", endpointName: "OPENAI_API_KEY",+ maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 2000,+ price: { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.invalid/prices", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" } };+const usage = { object: "response", status: "completed", id: "provider-response", model: "fixture", usage: { input_tokens: 10, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 5, output_tokens_details: { reasoning_tokens: 3 } } };+const response = () => Response.json(usage, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY", "x-bt-cached": "MISS", "x-bt-request-id": "gateway-request", "x-bt-span-id": "gateway-span" } });+async function fixture(maxUsd = "1", maxConcurrent = 2) {+ const dir = await mkdtemp(path.join(tmpdir(), "eval-forward-test-")); directories.push(dir);+ const file = path.join(dir, "ledger.jsonl"), deadlineAt = Date.now() + 30_000;+ const ledger = await openEvalCostLedger(file, { campaignId: "campaign", maxUsd, deadlineAt: new Date(deadlineAt).toISOString(), maxRequests: 10, maxConcurrent });+ closes.push(() => ledger.close());+ const receipts: EvalGatewayReceipt[] = [];+ return { ledger, file, deadlineAt, identity, route, receipts, payload: { model: "fixture", input: "hello", max_output_tokens: 500 },+ persist: async (receipt: EvalGatewayReceipt) => { receipts.push(structuredClone(receipt)); }, onHeaders: vi.fn(), onChunk: vi.fn(async () => {}) };+}+it("fsyncs reservation before dispatch, pins gateway routing, and replaces reservation with one estimate", async () => {+ const f = await fixture();+ let captured: RequestInit | undefined;+ let reservedBeforeDispatch = false;+ const transport = vi.fn<typeof fetch>(async (_url, init) => { captured = init; reservedBeforeDispatch = (await readEvalCostLedger(f.file)).state.snapshot().inflight === 1; return response(); });+ const result = await forwardEvalRequest({ ...f, transport });+ expect(reservedBeforeDispatch).toBe(true);+ expect(transport.mock.calls[0]![0]).toBe("https://gateway.braintrust.dev/responses");+ expect(captured).toMatchObject({ redirect: "error", headers: { authorization: `Bearer ${route.apiKey}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", "x-bt-project-id": route.projectId, "x-bt-endpoint-name": route.endpointName, "x-bt-parent": `project_id:${route.projectId}` } });+ expect(JSON.parse(captured!.body as string)).toMatchObject({ max_output_tokens: 100, service_tier: "default" });+ expect(result).toMatchObject({ outcome: "succeeded", costUsd: "0.000068000", final: true, usageComplete: true, gatewaySpanId: "gateway-span" });+ expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000068000", heldUsd: "0.000000000", inflight: 0 });+ expect(await readFile(f.file, "utf8")).not.toContain(route.apiKey);+});+it("stops at the campaign cap before any upstream request", async () => {+ const f = await fixture("0.002"), transport = vi.fn<typeof fetch>();+ expect(evalRequestReservation(route)).toBe("0.003000000");+ await expect(forwardEvalRequest({ ...f, transport })).rejects.toThrow("spending cap");+ expect(transport).not.toHaveBeenCalled();+});+it.each(["direct", "braintrust"] as const)("retains request-policy cost provenance without changing the %s cache policy", async mode => {+ const f = await fixture();+ const pricedRoute = { ...route, mode, price: { ...route.price, model: "gpt-5.6-sol", inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4",+ outputUsdPerMillion: "20", cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10" } };+ const payload = { ...f.payload, model: pricedRoute.price.model, prompt_cache_retention: "24h", stream: true };+ const body = { ...usage, model: pricedRoute.price.model, usage: { ...usage.usage, input_tokens_details: { cache_write_tokens: 6, cached_tokens: 2 } } };+ const transport = vi.fn<typeof fetch>(async () => new Response(`data: ${JSON.stringify({ type: "response.completed", response: body })}\n\n`, {+ headers: { "content-type": "text/event-stream", "x-bt-used-endpoint": "OPENAI_API_KEY", "x-bt-cached": "MISS" },+ }));+ const receipt = await forwardEvalRequest({ ...f, route: pricedRoute, payload, transport });+ expect(JSON.parse(transport.mock.calls[0]![1]!.body as string)).toEqual({ ...payload, max_output_tokens: 100, service_tier: "default" });+ expect(receipt).toMatchObject({ outcome: "succeeded", final: true, costUsd: "0.000138800", usage: { cacheWrite30m: 6, cacheWrite12h: 0,+ cacheWriteAttribution: { source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl: "30m" } } });+ expect(f.receipts.at(-1)).toEqual(receipt);+ expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000138800", heldUsd: "0.000000000", inflight: 0 });+});+it("retains cancelled and provider-error requests as unknown, including repeated retries", async () => {+ const f = await fixture();+ const first = await forwardEvalRequest({ ...f, transport: async () => { throw new Error(route.apiKey); } });+ const second = await forwardEvalRequest({ ...f, transport: async () => Response.json({ error: "private-provider-message" }, { status: 429 }) });+ expect(first).toMatchObject({ outcome: "failed", errorCode: "request_failed", costUsd: null });+ expect(second).toMatchObject({ outcome: "failed", errorCode: "upstream_rejected", costUsd: null });+ expect(first.requestId).not.toBe(second.requestId);+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.006000000", knownUsd: "0.000000000", unknownRequests: 2 });+ expect(JSON.stringify(f.receipts)).not.toMatch(/private-upstream|private-provider-message/);+});+it.each([+ { model: "other" }, { model: "fixture", background: true }, { model: "fixture", service_tier: "priority" },+ { model: "fixture", tools: [{ type: "web_search" }] }, { model: "fixture", input: [{ type: "input_image", image_url: "https://image.invalid" }] },+ { model: "fixture", speed: "fast" }, { model: "fixture", inference_geo: "us" },+])("rejects unpriced request shapes before admission", async payload => {+ const f = await fixture(), transport = vi.fn<typeof fetch>();+ await expect(forwardEvalRequest({ ...f, payload, transport })).rejects.toThrow();+ expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0);+});+it("does not dispatch when the private pre-request receipt cannot be persisted", async () => {+ const f = await fixture(), transport = vi.fn<typeof fetch>();+ await expect(forwardEvalRequest({ ...f, persist: async () => { throw new Error("disk unavailable"); }, transport })).rejects.toThrow("disk unavailable");+ expect(transport).not.toHaveBeenCalled();+ expect(f.ledger.snapshot()).toMatchObject({ unknownRequests: 1, heldUsd: "0.003000000" });+});+it("parses actual fragmented SSE without summing cumulative counters", async () => {+ const f = await fixture();+ const wire = `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "hello 👋" })}\r\n\r\ndata: ${JSON.stringify({ type: "response.completed", response: usage })}\r\n\r\ndata: [DONE]\r\n\r\n`;+ const bytes = Buffer.from(wire);+ const body = new ReadableStream<Uint8Array>({ start(controller) { for (let i = 0; i < bytes.length; i += 3) controller.enqueue(bytes.subarray(i, i + 3)); controller.close(); } });+ const result = await forwardEvalRequest({ ...f, transport: async () => new Response(body, { headers: { "content-type": "text/event-stream", "x-bt-used-endpoint": "OPENAI_API_KEY" } }) });+ expect(result).toMatchObject({ costUsd: "0.000068000", final: true });+ expect(f.onChunk.mock.calls.length).toBeGreaterThan(1);+ const truncated = new EvalUsageStream(() => {});+ expect(() => truncated.push(Buffer.from('data: {"type":"response.completed"}'), true)).toThrow("incomplete_usage_stream");+});+it.each([{ "x-bt-cached": "HIT" }, { "x-bt-used-endpoint": "FOREIGN" }, { "x-bt-failover-to": "fixture/FOREIGN" }])("rejects cached/failover/foreign route evidence without releasing the reservation", async headers => {+ const f = await fixture();+ const result = await forwardEvalRequest({ ...f, transport: async () => Response.json(usage, { headers: Object.fromEntries(Object.entries({ "x-bt-used-endpoint": "OPENAI_API_KEY", ...headers }).filter((pair): pair is [string, string] => pair[1] !== undefined)) }) });+ expect(result).toMatchObject({ outcome: "failed", errorCode: "gateway_route_not_qualified", costUsd: null });+ expect(f.onChunk).not.toHaveBeenCalled();+ expect(f.ledger.snapshot().heldUsd).toBe("0.003000000");+});+it("uses the loopback gateway through real HTTP while withholding upstream credentials from the client", async () => {+ const f = await fixture();+ const transport = vi.fn<typeof fetch>(async () => response());+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());+ const denied = await fetch(`${gateway.baseUrl}/v1/responses`, { method: "POST", body: "{}" });+ expect(denied.status).toBe(401); expect(transport).not.toHaveBeenCalled();+ const accepted = await fetch(`${gateway.baseUrl}/v1/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}`, "content-type": "application/json" }, body: JSON.stringify(f.payload) });+ expect(accepted.status).toBe(200); expect(await accepted.json()).toEqual(usage);+ expect(gateway.token).not.toBe(route.apiKey);+ expect(accepted.headers.get("x-paperclip-eval-request-id")).toBeTruthy();+ expect([...accepted.headers.values()].join(" ")).not.toContain(route.apiKey);+});+it.each([+ { status: 400, type: "text/plain", body: "private fixture rejection" },+ { status: 503, type: "text/html", body: "<html>private fixture rejection</html>" },+ { status: 400, type: "application/json", body: "invalid JSON fixture rejection" },+])("finishes non-JSON upstream errors without truncating the client response: $status $type", async error => {+ const f = await fixture();+ const transport = vi.fn<typeof fetch>().mockResolvedValueOnce(new Response(error.body, { status: error.status, headers: { "content-type": error.type } }))+ .mockImplementation(async () => response());+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());+ const request = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });+ const rejected = await request();+ expect(rejected.status).toBe(error.status);+ expect(await rejected.text()).toBe(error.body);+ expect(f.receipts.at(-1)).toMatchObject({ outcome: "failed", errorCode: "upstream_rejected", costUsd: null, usageComplete: false });+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });+ expect(JSON.stringify(f.receipts)).not.toContain(error.body);+ expect(await (await request()).json()).toEqual(usage);+ expect(transport).toHaveBeenCalledTimes(2);+});+it("still fails successful responses with invalid usage JSON and holds their reservation", async () => {+ const f = await fixture();+ const result = await forwardEvalRequest({ ...f, transport: async () => new Response("invalid successful response", {+ headers: { "content-type": "application/json", "x-bt-used-endpoint": "OPENAI_API_KEY" },+ }) });+ expect(result).toMatchObject({ outcome: "failed", errorCode: "request_failed", costUsd: null });+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });+});++it("reserves the highest cache-write and long-context rates before dispatch", () => {+ expect(evalRequestReservation({ ...route, price: { ...route.price, cacheWrite12hUsdPerMillion: "5",+ longContext: { aboveInputTokens: 100, rates: { inputUsdPerMillion: "4", cachedReadUsdPerMillion: "2", cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10", outputUsdPerMillion: "15" } } } })).toBe("0.011500000");+ expect(() => evalRequestReservation({ ...route, price: { ...route.price, cacheWrite12hUsdPerMillion: undefined } })).toThrow("both provider cache-write");+});+it("propagates cancellation to an in-flight upstream request and retains its reservation", async () => {+ const f = await fixture(), controller = new AbortController();+ const transport = vi.fn<typeof fetch>(async (_url, init) => {+ controller.abort();+ init!.signal!.throwIfAborted();+ throw new Error("unreachable");+ });+ const receipt = await forwardEvalRequest({ ...f, signal: controller.signal, transport });+ expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled", final: false, costUsd: null });+ expect(transport).toHaveBeenCalledOnce();+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });+});+it("does not release a reservation for an otherwise final response missing cache-write usage", async () => {+ const f = await fixture(), body = structuredClone(usage);+ delete (body.usage.input_tokens_details as any).cache_write_tokens;+ const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json(body, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } }) });+ expect(receipt).toMatchObject({ outcome: "succeeded", usageComplete: false, final: true, costUsd: null });+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1 });+});+it.each([{ version: "prices with spaces" }, { model: "model with spaces" }])("rejects invalid ledger price identities before reserving or dispatching", async invalid => {+ const f = await fixture(), transport = vi.fn<typeof fetch>();+ const price = { ...route.price, ...invalid };+ await expect(forwardEvalRequest({ ...f, route: { ...route, price }, payload: { model: price.model }, transport })).rejects.toThrow("price identity");+ expect(transport).not.toHaveBeenCalled();+ expect(f.ledger.snapshot().requests).toHaveLength(0);+});+it("retains invalid reasoning counters privately and finishes an unknown charge", async () => {+ const f = await fixture(), body = structuredClone(usage);+ body.usage.output_tokens_details.reasoning_tokens = 6;+ const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json(body, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } }) });+ expect(receipt).toMatchObject({ outcome: "succeeded", errorCode: "invalid_usage", usageComplete: false, final: true, costUsd: null,+ usage: { tokens: { output: 5, reasoning: 6 } } });+ expect(f.receipts.at(-1)).toEqual(receipt);+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });+ const journal = await readFile(f.file, "utf8");+ expect(journal).not.toContain('"reasoning":6');+ expect(journal).toContain('"type":"finish"');+});+it("retains malformed provider identifiers without losing the final receipt or ledger finish", async () => {+ const f = await fixture();+ const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json({ ...usage, model: "unexpected model name" },+ { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY", "x-request-id": "request with spaces" } }) });+ expect(receipt).toMatchObject({ providerRequestId: "request with spaces", usage: { model: "unexpected model name" }, final: true, costUsd: null });+ expect(f.receipts.at(-1)).toEqual(receipt);+ expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 });+});+it("admits later HTTP requests after temporary concurrency exhaustion", async () => {+ const f = await fixture("1", 1);+ let admitted!: () => void, release!: () => void;+ const started = new Promise<void>(resolve => { admitted = resolve; });+ const blocked = new Promise<void>(resolve => { release = resolve; });+ const transport = vi.fn<typeof fetch>(async () => { admitted(); await blocked; return response(); });+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());+ const request = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });+ const first = request(); await started;+ try {+ const denied = await request();+ expect(denied.status).toBe(429); expect(transport).toHaveBeenCalledOnce();+ } finally { release(); }+ expect(await (await first).json()).toEqual(usage);+ const later = await request();+ expect(later.status).toBe(200); expect(await later.json()).toEqual(usage);+ expect(transport).toHaveBeenCalledTimes(2);+ expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 0 });+});+it("rejects malformed, interrupted and unqualified uploads without disabling later valid requests", async () => {+ const f = await fixture(), transport = vi.fn<typeof fetch>(async () => response());+ const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close());+ const headers = { authorization: `Bearer ${gateway.token}` };+ for (const body of ["{", JSON.stringify({ model: "other" }), JSON.stringify({ model: "fixture", tools: [{ type: "web_search" }] })]) {+ const rejected = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers, body });+ expect(rejected.status).toBe(400); await rejected.text();+ }+ await new Promise<void>((resolve, reject) => {+ const socket = connect(Number(new URL(gateway.baseUrl).port), "127.0.0.1");+ socket.on("error", reject); socket.on("close", () => resolve()); socket.resume();+ socket.on("connect", () => socket.end(`POST /responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ${gateway.token}\r\nContent-Length: 100\r\n\r\n{`));+ });+ expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0);+ const valid = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers, body: JSON.stringify(f.payload) });+ expect(valid.status).toBe(200); expect(await valid.json()).toEqual(usage);+ expect(transport).toHaveBeenCalledOnce(); expect(f.ledger.snapshot().requests).toHaveLength(1);+});+it("still disables the HTTP gateway after a receipt persistence failure", async () => {+ const f = await fixture(), transport = vi.fn<typeof fetch>(async () => response());+ const gateway = await startEvalGateway({ ...f, transport, persist: async () => { throw new Error("storage failure"); } }); closes.push(() => gateway.close());+ for (let i = 0; i < 2; i++) {+ const denied = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });+ expect(denied.status).toBe(503); await denied.text();+ }+ expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(1);+});+it("applies the request deadline while waiting for downstream backpressure", async () => {+ const f = await fixture();+ const receipt = await forwardEvalRequest({ ...f, route: { ...f.route, timeoutMs: 50 }, transport: async () => response(),+ onChunk: async (_chunk, signal) => {+ signal.throwIfAborted();+ await new Promise<void>((_resolve, reject) => signal.addEventListener("abort", () => reject(signal.reason), { once: true }));+ },+ });+ expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled" });+ expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 1, heldUsd: "0.003000000" });+});+it("forwards Claude's Messages beta query without admitting unrelated endpoints or queries", async () => {+ const f = await fixture();+ const anthropicRoute: EvalGatewayRoute = { ...route, provider: "anthropic", mode: "direct", price: { provider: "anthropic", model: "fixture",+ version: "fixture", sourceUrl: "https://provider.invalid/prices", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10",+ cacheWrite5mUsdPerMillion: "2.5", cacheWrite1hUsdPerMillion: "4" } };+ const body = { type: "message", model: "fixture", usage: { input_tokens: 2, output_tokens: 3, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } };+ const transport = vi.fn<typeof fetch>(async () => Response.json(body));+ const gateway = await startEvalGateway({ ...f, route: anthropicRoute, transport }); closes.push(() => gateway.close());+ const request = (pathname: string) => fetch(gateway.baseUrl + pathname, { method: "POST", headers: { "x-api-key": gateway.token, "anthropic-beta": "fixture-beta" }, body: JSON.stringify({ model: "fixture", messages: [] }) });+ expect((await request("/v1/messages?beta=true&extra=true")).status).toBe(404);+ const accepted = await request("/v1/messages?beta=true");+ expect(accepted.status).toBe(200); expect(await accepted.json()).toEqual(body);+ expect(transport).toHaveBeenCalledOnce();+ expect(transport.mock.calls[0]![0]).toBe("https://api.anthropic.com/v1/messages?beta=true");+ expect(transport.mock.calls[0]![1]!.headers).toMatchObject({ "anthropic-beta": "fixture-beta" });+ expect(f.ledger.snapshot()).toMatchObject({ unknownRequests: 0, inflight: 0 });+});++it("attributes setup at admission, preserves it during execution, and blocks missing phase evidence", async () => {+ const f = await fixture();+ const phasePath = path.join(path.dirname(f.file), "phase.json");+ await createEvalAttemptPhase(phasePath, identity);+ const env = { PAPERCLIP_E2E_GATEWAY_PHASE_PATH: phasePath, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: identity.attemptId };+ let started!: () => void, release!: () => void, calls = 0;+ const firstStarted = new Promise<void>(resolve => { started = resolve; });+ const continuation = new Promise<void>(resolve => { release = resolve; });+ const gateway = await startEvalGateway({ ...f, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity),+ transport: async () => {+ calls++;+ if (calls === 1) { started(); await continuation; }+ return Response.json({ ...usage, id: `provider-${calls}` }, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } });+ },+ });+ closes.push(() => gateway.close());+ const send = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });+ const first = send();+ await firstStarted;+ await beginEvalAgentPhase(identity.executionId, env);+ release();+ expect((await first).status).toBe(200);+ expect((await send()).status).toBe(200);+ expect(f.ledger.snapshot().requests.map(request => request.phase)).toEqual(["setup", "agent"]);+ await writeFile(phasePath, JSON.stringify({ version: 1, ...identity, attemptId: "foreign-attempt" }));+ expect((await send()).status).toBe(503);+ expect(calls).toBe(2);+ expect((await readEvalCostLedger(f.file)).state.snapshot().requests.map(request => request.phase)).toEqual(["setup", "agent"]);+});++it("refuses missing or unreviewed attribution checkpoints before request dispatch", async () => {+ const f = await fixture(), transport = vi.fn<typeof fetch>();+ const phasePath = path.join(path.dirname(f.file), "missing-phase.json");+ const gateway = await startEvalGateway({ ...f, transport, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity) });+ closes.push(() => gateway.close());+ const response = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) });+ expect(response.status).toBe(503);+ expect(transport).not.toHaveBeenCalled();+ expect(f.ledger.snapshot().requests).toEqual([]);+ await expect(beginEvalAgentPhase(identity.executionId, { PAPERCLIP_E2E_GATEWAY_BASE_URL: gateway.baseUrl })).rejects.toThrow("Missing eval phase");+ await createEvalAttemptPhase(phasePath, identity);+ await expect(createEvalAttemptPhase(phasePath, identity)).rejects.toThrow();+ await writeFile(phasePath, JSON.stringify({ version: 1, ...identity, providerResponse: "private" }));+ await expect(readEvalAttemptPhase(phasePath, identity)).rejects.toThrow("Invalid eval phase");+});diff --git a/tests/runner-e2e/eval-gateway.ts b/tests/runner-e2e/eval-gateway.tsnew file mode 100644index 0000000000..7252890866--- /dev/null+++ b/tests/runner-e2e/eval-gateway.ts@@ -0,0 +1,256 @@+import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto";+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";+import { once } from "node:events";+import { usdToUnits } from "../../packages/shared/src/money.js";+import { estimateProviderCost, ProviderUsageAccumulator, type EvalModelPrice, type ProviderUsageReceipt } from "./provider-usage.js";+import { EvalAdmissionError, validEvalLedgerId, type EvalChargeIdentity, type openEvalCostLedger } from "./cost-ledger.js";++type Ledger = Awaited<ReturnType<typeof openEvalCostLedger>>;+export interface EvalGatewayRoute {+ provider: "openai" | "anthropic";+ mode: "direct" | "braintrust";+ apiKey: string;+ projectId?: string;+ endpointName?: string;+ price: EvalModelPrice;+ maximumInputTokens: number;+ maximumOutputTokens: number;+ timeoutMs: number;+}+export interface EvalGatewayReceipt {+ requestId: string; status?: number; route: "direct" | "braintrust";+ gatewayRequestId?: string; gatewaySpanId?: string; providerRequestId?: string;+ endpoint?: string; cache?: string; failoverFrom?: string; failoverTo?: string; errorOrigin?: string;+ outcome: "succeeded" | "failed" | "cancelled"; errorCode?: string;+ usageComplete: boolean; final: boolean; costUsd: string | null;+ usage?: ProviderUsageReceipt;+}+const money = (n: bigint) => `${n / 1_000_000_000n}.${String(n % 1_000_000_000n).padStart(9, "0")}`;+const safeHeader = (value: string | null) => value && /^[A-Za-z0-9_.:/ -]{1,512}$/.test(value) ? value : undefined;+const row = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};+class EvalRequestError extends Error {}++export function evalRequestReservation(route: EvalGatewayRoute): string {+ if (![route.maximumInputTokens, route.maximumOutputTokens, route.timeoutMs].every(n => Number.isSafeInteger(n) && n > 0)+ || route.timeoutMs > 300_000 || route.provider !== route.price.provider || !validEvalLedgerId(route.price.version) || !validEvalLedgerId(route.price.model)) throw new Error("Invalid eval gateway limits or price identity");+ const parse = (rate: string) => {+ if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(rate)) throw new Error("Invalid eval gateway price");+ return usdToUnits(rate);+ };+ if (route.price.longContext && (!Number.isSafeInteger(route.price.longContext.aboveInputTokens) || route.price.longContext.aboveInputTokens < 1)) throw new Error("Invalid long-context threshold");+ const tiers = [route.price, ...(route.price.longContext ? [route.price.longContext.rates] : [])];+ let maximumInputRate = 0n, maximumOutputRate = 0n;+ for (const tier of tiers) {+ const writes = route.provider === "openai" ? [tier.cacheWrite30mUsdPerMillion, tier.cacheWrite12hUsdPerMillion] : [tier.cacheWrite5mUsdPerMillion, tier.cacheWrite1hUsdPerMillion];+ if (writes.some(rate => rate === undefined)) throw new Error("Reservation requires both provider cache-write prices");+ for (const rate of [tier.inputUsdPerMillion, tier.cachedReadUsdPerMillion, ...writes]) {+ const units = parse(rate!); if (units > maximumInputRate) maximumInputRate = units;+ }+ const output = parse(tier.outputUsdPerMillion); if (output > maximumOutputRate) maximumOutputRate = output;+ }+ const numerator = BigInt(route.maximumInputTokens) * maximumInputRate + BigInt(route.maximumOutputTokens) * maximumOutputRate;+ return money((numerator + 999_999n) / 1_000_000n);+}++function boundedPayload(value: unknown, route: EvalGatewayRoute) {+ const body = row(value);+ if (body.model !== route.price.model) throw new EvalRequestError("unqualified_model");+ if (body.background === true || body.mcp_servers !== undefined || body.service_tier !== undefined && !["auto", "default", "standard_only"].includes(String(body.service_tier))) throw new EvalRequestError("unqualified_billing_mode");+ if (body.speed !== undefined && body.speed !== "standard" || body.inference_geo !== undefined) throw new EvalRequestError("unqualified_billing_mode");+ if (body.tools !== undefined && (!Array.isArray(body.tools) || body.tools.some(tool => {+ const type = row(tool).type;+ return route.provider === "openai" ? !["function", "custom"].includes(String(type)) : type !== undefined && type !== "custom";+ }))) throw new EvalRequestError("unqualified_hosted_tool");+ const inspect = (item: unknown): void => {+ if (Array.isArray(item)) { item.forEach(inspect); return; }+ if (!item || typeof item !== "object") return;+ const r = row(item);+ if (["input_image", "input_audio", "input_video", "image", "audio", "document", "video", "input_file"].includes(String(r.type))) throw new EvalRequestError("unqualified_multimodal_input");+ Object.values(r).forEach(inspect);+ };+ inspect(body);+ const field = route.provider === "openai" ? "max_output_tokens" : "max_tokens";+ if (body[field] !== undefined && (!Number.isSafeInteger(body[field]) || Number(body[field]) < 1)) throw new EvalRequestError("invalid_output_limit");+ return { ...body, [field]: Math.min(Number(body[field] ?? route.maximumOutputTokens), route.maximumOutputTokens),+ service_tier: route.provider === "openai" ? "default" : "standard_only" };+}++export class EvalUsageStream {+ private readonly decoder = new TextDecoder();+ private pending = "";+ constructor(private readonly observe: (value: unknown) => void) {}+ push(bytes: Uint8Array, final = false) {+ this.pending += this.decoder.decode(bytes, { stream: !final });+ if (this.pending.length > 4 * 1024 * 1024) throw new Error("usage_frame_too_large");+ for (;;) {+ const match = /\r?\n\r?\n/.exec(this.pending);+ if (!match) break;+ const frame = this.pending.slice(0, match.index); this.pending = this.pending.slice(match.index + match[0].length);+ const data = frame.split(/\r?\n/).filter(line => line.startsWith("data:")).map(line => line.slice(5).trimStart()).join("\n");+ if (data && data !== "[DONE]") this.observe(JSON.parse(data));+ }+ if (final && this.pending.trim()) throw new Error("incomplete_usage_stream");+ }+}++export async function forwardEvalRequest(input: {+ ledger: Ledger; identity: EvalChargeIdentity; route: EvalGatewayRoute; payload: unknown;+ deadlineAt: number; signal?: AbortSignal; anthropicBeta?: string; anthropicBetaQuery?: boolean;+ onHeaders: (status: number, headers: Record<string, string>) => void;+ onChunk: (chunk: Uint8Array, signal: AbortSignal) => Promise<void>;+ persist: (receipt: EvalGatewayReceipt) => Promise<void>;+ transport?: typeof fetch;+}): Promise<EvalGatewayReceipt> {+ const { route, ledger } = input;+ let body: ReturnType<typeof boundedPayload>;+ try { body = boundedPayload(input.payload, route); }+ catch (error) { throw error instanceof EvalRequestError ? error : new EvalRequestError("invalid_request_payload"); }+ const reservation = evalRequestReservation(route);+ if (!route.apiKey || /[\r\n]/.test(route.apiKey)) throw new Error("missing_gateway_credential");+ if (route.mode === "braintrust" && (!/^[0-9a-f-]{36}$/i.test(route.projectId ?? "") || !/^[A-Za-z0-9_-]+$/.test(route.endpointName ?? ""))) throw new Error("invalid_project_provider");+ if (input.signal?.aborted) throw new EvalRequestError("request_cancelled");+ if (Date.now() >= input.deadlineAt) throw new EvalAdmissionError("request_deadline_reached");+ const requestId = randomUUID();+ const receipt: EvalGatewayReceipt = { requestId, route: route.mode, outcome: "failed", usageComplete: false, final: false, costUsd: null };+ const usage = new ProviderUsageAccumulator(route.provider, body);+ await ledger.reserve(requestId, input.identity, reservation);+ const abort = new AbortController();+ const signal = AbortSignal.any([abort.signal, AbortSignal.timeout(Math.max(1, Math.min(route.timeoutMs, input.deadlineAt - Date.now()))), ...(input.signal ? [input.signal] : [])]);+ let response: Response | undefined;+ try {+ await input.persist(receipt);+ const headers: Record<string, string> = { "content-type": "application/json", "accept-encoding": "identity" };+ if (route.mode === "braintrust") Object.assign(headers, {+ authorization: `Bearer ${route.apiKey}`, "x-bt-project-id": route.projectId!, "x-bt-endpoint-name": route.endpointName!,+ "x-bt-parent": `project_id:${route.projectId}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store",+ });+ else if (route.provider === "openai") headers.authorization = `Bearer ${route.apiKey}`;+ else headers["x-api-key"] = route.apiKey;+ if (route.provider === "anthropic") {+ headers["anthropic-version"] = "2023-06-01";+ if (input.anthropicBeta && /^[A-Za-z0-9, _-]{1,2048}$/.test(input.anthropicBeta)) headers["anthropic-beta"] = input.anthropicBeta;+ }+ const base = route.mode === "braintrust" ? "https://gateway.braintrust.dev" : route.provider === "openai" ? "https://api.openai.com/v1" : "https://api.anthropic.com/v1";+ const endpoint = route.provider === "openai" ? "/responses" : input.anthropicBetaQuery ? "/messages?beta=true" : "/messages";+ response = await (input.transport ?? fetch)(base + endpoint, { method: "POST", headers, body: JSON.stringify(body), redirect: "error", signal });+ receipt.status = response.status;+ receipt.gatewayRequestId = safeHeader(response.headers.get("x-bt-request-id"));+ receipt.gatewaySpanId = safeHeader(response.headers.get("x-bt-span-id"));+ receipt.providerRequestId = safeHeader(response.headers.get("x-request-id") ?? response.headers.get("request-id"));+ receipt.endpoint = safeHeader(response.headers.get("x-bt-used-endpoint"));+ receipt.cache = safeHeader(response.headers.get("x-bt-cached"));+ receipt.failoverFrom = safeHeader(response.headers.get("x-bt-failover-from"));+ receipt.failoverTo = safeHeader(response.headers.get("x-bt-failover-to"));+ receipt.errorOrigin = safeHeader(response.headers.get("x-bt-error-origin"));+ await input.persist(receipt);+ if (route.mode === "braintrust" && (receipt.cache === "HIT" || receipt.failoverFrom || receipt.failoverTo+ || response.ok && receipt.endpoint !== route.endpointName)) throw new Error("gateway_route_not_qualified");+ const contentType = response.headers.get("content-type") ?? "application/json";+ input.onHeaders(response.status, { "content-type": contentType, "cache-control": "no-store", "x-paperclip-eval-request-id": requestId });+ const reader = response.body?.getReader();+ const stream = contentType.startsWith("text/event-stream") ? new EvalUsageStream(value => usage.observe(value)) : undefined;+ const chunks: Uint8Array[] = []; let size = 0;+ try {+ while (reader) {+ const next = await reader.read(); if (next.done) break;+ size += next.value.length; if (size > 32 * 1024 * 1024) throw new Error("response_too_large");+ if (stream) stream.push(next.value); else chunks.push(next.value);+ await input.onChunk(next.value, signal);+ }+ if (stream) stream.push(new Uint8Array(), true);+ else if (size) {+ try { usage.observe(JSON.parse(Buffer.concat(chunks).toString("utf8"))); }+ catch (error) { if (response.ok) throw error; }+ }+ } finally { await reader?.cancel().catch(() => {}); }+ receipt.outcome = response.ok ? "succeeded" : "failed";+ if (!response.ok) receipt.errorCode = "upstream_rejected";+ } catch (error) {+ receipt.outcome = signal.aborted ? "cancelled" : "failed";+ const code = error instanceof Error ? error.message : "";+ receipt.errorCode = ["gateway_route_not_qualified", "response_too_large", "usage_frame_too_large", "incomplete_usage_stream"].includes(code) ? code : signal.aborted ? "request_cancelled" : "request_failed";+ } finally {+ abort.abort();+ await response?.body?.cancel().catch(() => {});+ const observed = usage.receipt();+ receipt.usage = observed;+ receipt.usageComplete = observed.complete;+ receipt.final = observed.final;+ receipt.costUsd = estimateProviderCost(observed, route.price);+ const tokens = { ...observed.tokens };+ if (tokens.reasoning !== undefined && tokens.output !== undefined && tokens.reasoning > tokens.output) {+ delete tokens.reasoning;+ receipt.errorCode ??= "invalid_usage";+ }+ await input.persist(receipt);+ await ledger.observe({ observationId: `${requestId}:usage`, requestId, source: route.mode === "braintrust" ? "gateway" : "provider",+ provider: route.provider,+ ...(validEvalLedgerId(observed.model) ? { model: observed.model } : {}),+ ...(validEvalLedgerId(receipt.providerRequestId) ? { providerRequestId: receipt.providerRequestId } : {}),+ currency: "USD", route: route.mode, classification: receipt.costUsd === null ? "unknown" : "estimated",+ ...(receipt.costUsd !== null ? { costUsd: receipt.costUsd, pricingVersion: route.price.version } : {}), tokens, final: observed.final });+ await ledger.finish(requestId, receipt.outcome);+ }+ return receipt;+}++async function requestJson(req: IncomingMessage) {+ const chunks: Buffer[] = []; let size = 0;+ for await (const chunk of req) {+ size += chunk.length; if (size > 4 * 1024 * 1024) throw new Error("request_too_large");+ chunks.push(chunk);+ }+ return JSON.parse(Buffer.concat(chunks).toString("utf8"));+}++export async function startEvalGateway(input: Omit<Parameters<typeof forwardEvalRequest>[0], "payload" | "onHeaders" | "onChunk" | "signal" | "anthropicBeta" | "anthropicBetaQuery"> & {+ resolveIdentity?: () => Promise<EvalChargeIdentity>;+}) {+ const token = randomBytes(32).toString("hex"), pending = new Set<Promise<void>>(), controllers = new Set<AbortController>();+ let closed = false, poisoned = false;+ const respond = (res: ServerResponse, status: number, code: string) => {+ if (res.destroyed) return;+ if (!res.headersSent) { res.writeHead(status, { "content-type": "application/json" }); res.end(JSON.stringify({ error: { type: "eval_gateway", message: code } })); }+ else res.destroy();+ };+ const server = createServer((req, res) => {+ const operation = (async () => {+ const auth = String(req.headers.authorization ?? "").replace(/^Bearer /, "") || String(req.headers["x-api-key"] ?? "");+ if (Buffer.byteLength(auth) !== Buffer.byteLength(token) || !timingSafeEqual(Buffer.from(auth), Buffer.from(token))) { respond(res, 401, "unauthorized"); return; }+ const endpoint = input.route.provider === "openai" ? "responses" : "messages";+ if (closed || poisoned) { respond(res, 503, "gateway_unavailable"); return; }+ const paths = [`/${endpoint}`, `/v1/${endpoint}`];+ if (input.route.provider === "anthropic") paths.push("/messages?beta=true", "/v1/messages?beta=true");+ if (req.method !== "POST" || !paths.includes(req.url ?? "")) { respond(res, 404, "unqualified_endpoint"); return; }+ const abort = new AbortController(); controllers.add(abort);+ const onClose = () => { if (!res.writableEnded) abort.abort(); }; res.on("close", onClose);+ try {+ let payload: unknown;+ try { payload = await requestJson(req); }+ catch { respond(res, 400, "invalid_request_body"); return; }+ const identity = input.resolveIdentity ? await input.resolveIdentity() : input.identity;+ if (identity.executionId !== input.identity.executionId || identity.attemptId !== input.identity.attemptId) throw new Error("Invalid eval request attribution");+ const receipt = await forwardEvalRequest({ ...input, identity, payload, signal: abort.signal,+ anthropicBeta: typeof req.headers["anthropic-beta"] === "string" ? req.headers["anthropic-beta"] : undefined,+ anthropicBetaQuery: input.route.provider === "anthropic" && req.url?.endsWith("?beta=true"),+ onHeaders: (status, headers) => { res.writeHead(status, headers); },+ onChunk: async (chunk, signal) => { if (!res.write(chunk)) await once(res, "drain", { signal }); },+ });+ if (receipt.outcome === "succeeded" || receipt.errorCode === "upstream_rejected") res.end();+ else respond(res, 502, receipt.errorCode ?? "request_failed");+ } catch (error) {+ if (error instanceof EvalRequestError) { respond(res, 400, "invalid_request_payload"); return; }+ if (error instanceof EvalAdmissionError) { respond(res, 429, "eval_admission_denied"); return; }+ poisoned = true; respond(res, 503, "eval_admission_or_persistence_failed");+ } finally { controllers.delete(abort); res.off("close", onClose); }+ })();+ pending.add(operation); void operation.then(() => pending.delete(operation), () => { poisoned = true; respond(res, 503, "gateway_unavailable"); pending.delete(operation); });+ });+ server.requestTimeout = 15_000; server.headersTimeout = 10_000;+ server.listen(0, "127.0.0.1"); await once(server, "listening");+ const address = server.address(); if (!address || typeof address === "string") throw new Error("Eval gateway did not bind loopback");+ return { baseUrl: `http://127.0.0.1:${address.port}`, token,+ async close() { closed = true; controllers.forEach(controller => controller.abort()); server.closeAllConnections();+ await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve())); await Promise.allSettled([...pending]); },+ };+}diff --git a/tests/runner-e2e/eval-otlp-payload.ts b/tests/runner-e2e/eval-otlp-payload.tsnew file mode 100644index 0000000000..345734f73e--- /dev/null+++ b/tests/runner-e2e/eval-otlp-payload.ts@@ -0,0 +1,127 @@+import { evalOtlpSchema as root, protobuf } from "./eval-otlp-schema.js";++export type EvalOtlpSignal = "traces" | "logs";+export const isEvalProjectId = (value: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value);++export function validateEvalOtlpParent(value: string, projectId: string): string {+ if (!isEvalProjectId(projectId)) throw new Error("invalid_otlp_project");+ if (value === `project_id:${projectId}`) return value;+ if (value.length > 2048 || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) throw new Error("invalid_otlp_parent");+ const bytes = Buffer.from(value, "base64");+ if (bytes.toString("base64") !== value || !bytes.subarray(0, 3).equals(Buffer.from([4, 2, 0]))) throw new Error("invalid_otlp_parent");+ const data = JSON.parse(bytes.subarray(3).toString("utf8"));+ if (!data || Array.isArray(data) || Object.keys(data).sort().join(",") !== "object_id,root_span_id,row_id,span_id"+ || data.object_id !== projectId || data.row_id !== data.span_id+ || !/^[0-9a-f]{16}$/.test(data.span_id) || /^0+$/.test(data.span_id)+ || !/^[0-9a-f]{32}$/.test(data.root_span_id) || /^0+$/.test(data.root_span_id)) throw new Error("invalid_otlp_parent");+ return value;+}++function validateWire(body: Buffer, type: import("../../server/node_modules/protobufjs/index.js").Type) {+ const reader = protobuf.Reader.create(body);+ let fields = 0;+ const inspect = (message: typeof type, end: number, depth: number): void => {+ if (depth > 64) throw new Error("otlp_structure_limit");+ const seen = new Set<number | string>();+ while (reader.pos < end) {+ if (++fields > 200_000) throw new Error("otlp_structure_limit");+ const tag = reader.uint32(), id = tag >>> 3, wire = tag & 7, field = message.fieldsById[id];+ if (!id || ![0, 1, 2, 5].includes(wire)) throw new Error("invalid_otlp_wire");+ if (field) {+ if (!field.repeated && seen.has(id) || field.partOf && seen.has(field.partOf.name)) throw new Error("ambiguous_otlp_wire");+ seen.add(id); if (field.partOf) seen.add(field.partOf.name);+ const nested = field.resolvedType instanceof protobuf.Type ? field.resolvedType : undefined;+ const expected = nested || ["string", "bytes"].includes(field.type) ? 2+ : ["double", "fixed64", "sfixed64"].includes(field.type) ? 1+ : ["float", "fixed32", "sfixed32"].includes(field.type) ? 5 : 0;+ if (wire !== expected) throw new Error("invalid_otlp_wire");+ if (nested) {+ const length = reader.uint32(), next = reader.pos + length;+ if (next > end) throw new Error("invalid_otlp_wire");+ inspect(nested, next, depth + 1);+ } else reader.skipType(wire);+ } else throw new Error("unqualified_otlp_field");+ if (reader.pos > end) throw new Error("invalid_otlp_wire");+ }+ };+ inspect(type, body.length, 0);+}++export function validateEvalOtlpPayload(body: Buffer, contentType: string, signal: EvalOtlpSignal, projectId: string) {+ const type = root.lookupType(signal === "traces" ? "TracesData" : "LogsData");+ if (contentType !== "application/json") validateWire(body, type);+ const decoded: unknown = contentType === "application/json" ? JSON.parse(body.toString("utf8")) : type.decode(body).toJSON();+ if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_otlp_payload");+ let visited = 0;+ const inspect = (value: unknown, depth: number): void => {+ if (++visited > 200_000 || depth > 64) throw new Error("otlp_structure_limit");+ if (Array.isArray(value)) { value.forEach(item => inspect(item, depth + 1)); return; }+ if (!value || typeof value !== "object") return;+ const row = value as Record<string, unknown>;+ if (row.key === "braintrust.parent") {+ const parent = row.value as { stringValue?: unknown; string_value?: unknown } | undefined;+ if (!parent || Object.keys(parent).length !== 1) throw new Error("invalid_otlp_parent");+ const text = parent.stringValue ?? parent.string_value;+ if (typeof text !== "string") throw new Error("invalid_otlp_parent");+ validateEvalOtlpParent(text, projectId);+ }+ if (Object.hasOwn(row, "braintrust.parent")) {+ if (typeof row["braintrust.parent"] !== "string") throw new Error("invalid_otlp_parent");+ validateEvalOtlpParent(row["braintrust.parent"], projectId);+ }+ Object.values(row).forEach(item => inspect(item, depth + 1));+ };+ inspect(decoded, 0);+ const canonical = Buffer.from(JSON.stringify(decoded));+ const sizing = expandedEvalOtlpSize(decoded as Record<string, unknown>, signal);+ return { body: contentType === "application/json" ? canonical : body, ...sizing };+}++function expandedEvalOtlpSize(decoded: Record<string, unknown>, signal: EvalOtlpSignal) {+ const bytes = (value: unknown) => Buffer.byteLength(JSON.stringify(value));+ const object = (value: unknown): Record<string, unknown> => {+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid_otlp_record");+ return value as Record<string, unknown>;+ };+ const take = (row: Record<string, unknown>, camel: string, snake = camel) => {+ if (camel !== snake && Object.hasOwn(row, camel) && Object.hasOwn(row, snake)) throw new Error("ambiguous_otlp_collection");+ const value = Object.hasOwn(row, camel) ? row[camel] : Object.hasOwn(row, snake) ? row[snake] : [];+ if (!Array.isArray(value)) throw new Error("invalid_otlp_collection");+ const rest = Object.fromEntries(Object.entries(row).filter(([key]) => key !== camel && key !== snake));+ return { rows: value, rest };+ };+ const traces = signal === "traces";+ const resources = take(decoded, traces ? "resourceSpans" : "resourceLogs", traces ? "resource_spans" : "resource_logs");+ let recordCount = 0, expandedBytes = bytes(decoded);+ for (const resource of resources.rows) {+ const scopes = take(object(resource), traces ? "scopeSpans" : "scopeLogs", traces ? "scope_spans" : "scope_logs");+ for (const scope of scopes.rows) {+ const records = traces ? take(object(scope), "spans") : take(object(scope), "logRecords", "log_records");+ const envelopeBytes = bytes({ source: resources.rest, resource: scopes.rest, scope: records.rest, record: {} }) - 2;+ for (const record of records.rows) {+ recordCount++;+ expandedBytes += envelopeBytes + bytes(object(record));+ }+ }+ }+ return { recordCount, expandedBytes };+}++export function evalOtlpRejected(body: Buffer, contentType: string, signal: EvalOtlpSignal): number {+ if (!body.length) return 0;+ const decoded = contentType === "application/json" ? JSON.parse(body.toString("utf8"))+ : root.lookupType("Response").decode(body).toJSON();+ if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_otlp_response");+ const partial = decoded.partialSuccess;+ if (partial !== undefined && (!partial || typeof partial !== "object" || Array.isArray(partial))) throw new Error("invalid_otlp_response");+ const rejected = partial?.[signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"] ?? partial?.rejected ?? 0;+ if (!/^[0-9]+$/.test(String(rejected)) || !Number.isSafeInteger(Number(rejected))) throw new Error("invalid_otlp_response");+ return Number(rejected);+}++export function evalOtlpResponse(rejected: number, contentType: string, signal: EvalOtlpSignal): Buffer {+ if (contentType === "application/json") return Buffer.from(JSON.stringify(rejected+ ? { partialSuccess: { [signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"]: String(rejected) } } : {}));+ const type = root.lookupType("Response");+ return Buffer.from(type.encode(type.fromObject(rejected ? { partialSuccess: { rejected: String(rejected) } } : {})).finish());+}diff --git a/tests/runner-e2e/eval-otlp-relay.test.ts b/tests/runner-e2e/eval-otlp-relay.test.tsnew file mode 100644index 0000000000..d241934242--- /dev/null+++ b/tests/runner-e2e/eval-otlp-relay.test.ts@@ -0,0 +1,258 @@+import { afterEach, expect, it, vi } from "vitest";+import { mkdtemp, open, readFile, rm, stat } from "node:fs/promises";+import { createRequire } from "node:module";+import { tmpdir } from "node:os";+import path from "node:path";+import { createServer } from "node:http";+import { once } from "node:events";+import { gzipSync } from "node:zlib";+import { setTimeout as delay } from "node:timers/promises";+import { EVAL_OTLP_LIMITS, startEvalOtlpRelay } from "./eval-otlp-relay.js";+import { EvalAttemptTrace, exportBraintrustAttemptParent } from "./braintrust.js";+import { evalOtlpRejected, evalOtlpResponse } from "./eval-otlp-payload.js";+import { evalOtlpSchema } from "./eval-otlp-schema.js";++const projectId = "11111111-1111-4111-8111-111111111111", apiKey = "upstream-private-fixture";+const protobuf = createRequire(new URL("../../server/package.json", import.meta.url))("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js");+const message = (field: number, ...parts: Uint8Array[]) => Buffer.from(protobuf.Writer.create().uint32(field * 8 + 2).bytes(Buffer.concat(parts)).finish());+const roots: string[] = [], closers: Array<() => Promise<void>> = [];+afterEach(async () => { await Promise.allSettled(closers.splice(0).map(close => close()));+ await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); vi.restoreAllMocks(); });+async function fixture(overrides: Partial<Parameters<typeof startEvalOtlpRelay>[0]> = {}) {+ const root = await mkdtemp(path.join(tmpdir(), "eval-otlp-")); roots.push(root);+ const policy = { projectId, apiKey, campaignId: "campaign", attemptId: "attempt", receiptPath: path.join(root, "receipts.jsonl"),+ deadlineAt: Date.now() + 60_000, ...overrides };+ const relay = await startEvalOtlpRelay(policy); closers.push(relay.close);+ const send = (body = "{}", headers: Record<string, string> = {}, endpoint = "/v1/traces") => fetch(relay.baseUrl + endpoint, {+ method: "POST", headers: { authorization: `Bearer ${relay.token}`, "content-type": "application/json", ...headers }, body,+ });+ const records = async () => (await readFile(policy.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line));+ return { relay, policy, send, records };+}++it("forwards real attempt protobuf over HTTP using only the controller credential and retains body-free receipts", async () => {+ const received: Array<{ authorization?: string; parent?: string; body: Buffer }> = [];+ const server = createServer(async (req, res) => {+ const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk));+ received.push({ authorization: req.headers.authorization, parent: req.headers["x-bt-parent"] as string, body: Buffer.concat(chunks) });+ res.writeHead(200, { "content-type": "application/x-protobuf" }); res.end();+ });+ server.listen(0, "127.0.0.1"); await once(server, "listening");+ closers.push(async () => { server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve())); });+ const address = server.address(); if (!address || typeof address === "string") throw new Error("fixture_bind_failed");+ const upstream = vi.fn<typeof fetch>((url, init) => {+ expect(String(url)).toBe("https://api.braintrust.dev/otel/v1/traces");+ expect(init?.redirect).toBe("error");+ return fetch(`http://127.0.0.1:${address.port}`, init);+ });+ const f = await fixture({ transport: upstream });+ const trace = new EvalAttemptTrace({ projectId, apiKey: f.relay.token, endpointBaseUrl: f.relay.baseUrl },+ { campaignId: "campaign", attemptId: "attempt", executionId: "fixture.local", provider: "fixture", model: "fixture", environment: "local" });+ trace.phase("setup");+ expect(await trace.export({ failed: false, runIds: [], cleanup: "passed", complaints: 0, suggestions: 0 })).toMatchObject({ status: "accepted" });+ expect(received).toHaveLength(1);+ expect(received[0]).toMatchObject({ authorization: `Bearer ${apiKey}`, parent: `project_id:${projectId}` });+ expect(received[0]!.body.toString()).toContain("paperclip.eval.attempt");+ expect(received[0]!.body.toString()).not.toContain(f.relay.token);+ await f.relay.close();+ const records = await f.records();+ expect(records.map(row => row.type)).toEqual(["policy", "admitted", "outcome", "closed"]);+ expect(records[2]).toMatchObject({ outcome: "accepted", dispatched: true, httpStatus: 200, rejectedRecords: 0 });+ expect(records[3]).toMatchObject({ admittedRequests: 1, admittedBytes: received[0]!.body.length,+ attemptedForwardBytes: received[0]!.body.length, acceptedRequests: 1, incompleteRequests: 0 });+ expect(JSON.stringify(records)).not.toMatch(/upstream-private-fixture|paperclip.eval.attempt/);+ expect((await stat(f.policy.receiptPath)).mode & 0o777).toBe(0o600);+ await expect(startEvalOtlpRelay(f.policy)).rejects.toMatchObject({ code: "EEXIST" });+});++it("refuses token, endpoint, header and payload routing bypasses before contacting Braintrust", async () => {+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));+ const f = await fixture({ transport: upstream });+ expect((await f.send("{}", { authorization: `Bearer ${apiKey}` })).status).toBe(401);+ expect((await f.send("{}", {}, "/../api/project")).status).toBe(404);+ expect((await f.send("{}", {}, "/v1/traces?target=other")).status).toBe(404);+ expect((await f.send("{}", { "x-bt-parent": "project_name:another" })).status).toBe(400);+ for (const attributes of [+ { resourceSpans: [{ resource: { attributes: [{ key: "braintrust.parent", value: { stringValue: "project_id:22222222-2222-4222-8222-222222222222" } }] } }] },+ { resourceSpans: [{ scopeSpans: [{ spans: [{ attributes: [{ key: "braintrust.parent", value: { stringValue: "experiment_id:private" } }] }] }] }] },+ ]) expect((await f.send(JSON.stringify(attributes))).status).toBe(400);+ expect(upstream).not.toHaveBeenCalled();+ const parent = exportBraintrustAttemptParent({ projectId, spanId: "1234567890abcdef", traceId: "abcdef0123456789abcdef0123456789" });+ expect((await f.send("{}", { "x-bt-parent": parent })).status).toBe(200);+ expect(upstream.mock.calls[0]?.[1]?.headers).toMatchObject({ "x-bt-parent": parent });+});++it("counts decompressed payload bytes and refuses a compressed body above its allowance", async () => {+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRequestBytes: 1024 } });+ const body = JSON.stringify({ resourceLogs: [{ resource: { attributes: [{ key: "test", value: { stringValue: "private-content-canary" } }] } }] });+ const request = (bytes: Buffer) => fetch(f.relay.baseUrl + "/v1/logs", { method: "POST", body: Buffer.from(bytes),+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/json", "content-encoding": "gzip" } });+ expect((await request(gzipSync(body))).status).toBe(200);+ expect((await request(gzipSync(JSON.stringify({ payload: "x".repeat(2048) })))).status).toBe(400);+ expect(upstream).toHaveBeenCalledTimes(1);+ expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array).toString()).toBe(body);+ const records = await f.records();+ expect(records[1]).toMatchObject({ signal: "logs", bodyBytes: Buffer.byteLength(body), wireBytes: gzipSync(body).length });+ expect(JSON.stringify(records)).not.toContain("private-content-canary");+});++it("rejects protobuf routing metadata and ambiguous singular fields before forwarding", async () => {+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));+ const f = await fixture({ transport: upstream });+ const attribute = message(1, Buffer.from("braintrust.parent"));+ const foreign = message(2, message(1, Buffer.from("project_id:22222222-2222-4222-8222-222222222222")));+ const local = message(2, message(1, Buffer.from(`project_id:${projectId}`)));+ const resource = message(1, message(1, attribute, foreign));+ const bodies = [+ message(1, resource),+ message(1, resource, message(1)),+ message(1, message(1, message(1, attribute, foreign, local))),+ message(1, message(2, message(2, message(9, attribute, foreign)))),+ message(1, message(2, message(1, message(3, attribute, foreign)))),+ ];+ for (const body of bodies) {+ const response = await fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body: Buffer.from(body),+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/x-protobuf" } });+ expect(response.status).toBe(400);+ }+ expect(upstream).not.toHaveBeenCalled();+});++it("canonicalizes duplicate JSON keys and refuses conflicting parent value aliases", async () => {+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));+ const f = await fixture({ transport: upstream });+ const foreign = "project_id:22222222-2222-4222-8222-222222222222";+ const body = `{"resourceSpans":[{"resource":{"attributes":[{"key":"braintrust.parent","value":{"stringValue":"${foreign}","stringValue":"project_id:${projectId}"}}]}}]}`;+ expect((await f.send(body)).status).toBe(200);+ expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array).toString()).not.toContain(foreign);+ const ambiguous = JSON.stringify({ resourceSpans: [{ resource: { attributes: [{ key: "braintrust.parent",+ value: { stringValue: `project_id:${projectId}`, string_value: foreign } }] } }] });+ expect((await f.send(ambiguous)).status).toBe(400);+ expect(upstream).toHaveBeenCalledTimes(1);+});++it("does not dispatch when durable admission fails", async () => {+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));+ const f = await fixture({ transport: upstream });+ const handle = await open(path.join(path.dirname(f.policy.receiptPath), "fixture"), "w");+ const sync = vi.spyOn(Object.getPrototypeOf(handle), "sync").mockRejectedValueOnce(new Error("fixture_disk_failure"));+ try {+ expect((await f.send()).status).toBe(503);+ expect((await f.send()).status).toBe(503);+ expect(upstream).not.toHaveBeenCalled();+ await expect(f.relay.close()).rejects.toThrow("fixture_disk_failure");+ } finally { sync.mockRestore(); await handle.close(); }+});++it("bounds simultaneous exports without retrying the refused request", async () => {+ let entered!: () => void, release!: () => void;+ const active = new Promise<void>(resolve => { entered = resolve; });+ const wait = new Promise<void>(resolve => { release = resolve; });+ const upstream = vi.fn<typeof fetch>(async () => { entered(); await wait; return new Response(null, { status: 200 }); });+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxConcurrent: 1 } });+ const first = f.send(); await active;+ try { expect((await f.send()).status).toBe(429); } finally { release(); }+ expect((await first).status).toBe(200);+ expect(upstream).toHaveBeenCalledTimes(1);+});++it.each(["count", "bytes"])("retains failed attempts against the %s allowance without retrying or resetting", async kind => {+ const upstream = vi.fn<typeof fetch>(async () => new Response("private-vendor-error", { status: 503 }));+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS,+ ...(kind === "count" ? { maxRequests: 1 } : { maxTotalBytes: 3 }) } });+ const first = await f.send(); expect(first.status).toBe(502); expect(await first.text()).not.toContain("private-vendor-error");+ expect((await f.send()).status).toBe(429);+ expect((await f.send()).status).toBe(429);+ expect(upstream).toHaveBeenCalledTimes(1);+ await f.relay.close();+ const records = await f.records();+ expect(records.filter(row => row.type === "quota_exhausted")).toHaveLength(1);+ expect(records.find(row => row.type === "outcome")).toMatchObject({ outcome: "rejected", httpStatus: 503 });+ expect(records.at(-1)).toMatchObject({ admittedRequests: 1, admittedBytes: 2, incompleteRequests: 1 });+});++it.each(["traces", "logs"] as const)("preserves partial %s rejection without returning upstream diagnostic text", async signal => {+ const upstream = vi.fn<typeof fetch>(async () => Response.json({ partialSuccess: {+ [signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"]: "2", errorMessage: "private-upstream-diagnostic" } }));+ const f = await fixture({ transport: upstream });+ const response = await f.send("{}", {}, `/v1/${signal}`);+ expect(response.status).toBe(200);+ const text = await response.text(); expect(text).not.toContain("private-upstream-diagnostic");+ expect(evalOtlpRejected(Buffer.from(text), "application/json", signal)).toBe(2);+ const protobuf = evalOtlpResponse(2, "application/x-protobuf", signal);+ expect(evalOtlpRejected(protobuf, "application/x-protobuf", signal)).toBe(2);+ expect((await f.records()).find(row => row.type === "outcome")).toMatchObject({ outcome: "partial", rejectedRecords: 2 });+});++it("aborts an in-flight export on close and retains its uncertain outcome", async () => {+ let started!: () => void, aborted = false;+ const active = new Promise<void>(resolve => { started = resolve; });+ const upstream = vi.fn<typeof fetch>(async (_url, init) => {+ started(); await new Promise<void>((_resolve, reject) => init!.signal!.addEventListener("abort", () => { aborted = true; reject(new Error("fixture_abort")); }, { once: true }));+ return new Response(null, { status: 200 });+ });+ const f = await fixture({ transport: upstream });+ const request = f.send().catch(() => undefined); await active;+ await f.relay.close(); await request;+ expect(aborted).toBe(true);+ expect((await f.records()).find(row => row.type === "outcome")).toMatchObject({ outcome: "uncertain", dispatched: true });+ await f.relay.close();+ expect(upstream).toHaveBeenCalledTimes(1);+});++it("retires the listening endpoint at its deadline without a supervisor close", async () => {+ const upstream = vi.fn<typeof fetch>(async () => new Response(null, { status: 200 }));+ const f = await fixture({ transport: upstream, deadlineAt: Date.now() + 120 });+ await delay(180);+ await expect(f.send()).rejects.toThrow();+ await f.relay.close();+ expect(upstream).not.toHaveBeenCalled();+ expect((await f.records()).at(-1)).toMatchObject({ type: "closed", admittedRequests: 0 });+});++it.each(["json", "protobuf"])("meters repeated resource/scope data and full record fields in %s before dispatch", async encoding => {+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxExpandedBytes: 40_000 } });+ const payload = { resourceSpans: [{ resource: { attributes: [{ key: "shared", value: { stringValue: "r".repeat(3000) } }] },+ scopeSpans: [{ scope: { name: "s".repeat(1000), version: "fixture" }, spans: Array.from({ length: 4 }, () => ({+ name: "n".repeat(1000), traceId: "AQEBAQEBAQEBAQEBAQEBAQ==", spanId: "AgICAgICAgI=", flags: 1,+ startTimeUnixNano: "1234567890123456789", endTimeUnixNano: "1234567890123456790",+ events: [{ name: "e".repeat(1000), timeUnixNano: "1234567890123456789" }],+ links: [{ traceId: "AwMDAwMDAwMDAwMDAwMDAw==", spanId: "BAQEBAQEBAQ=", flags: 1 }],+ })) }] }] };+ const type = evalOtlpSchema.lookupType("TracesData");+ const body = encoding === "json" ? Buffer.from(JSON.stringify(payload)) : Buffer.from(type.encode(type.fromObject(payload)).finish());+ const send = () => fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body: Buffer.from(body),+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": encoding === "json" ? "application/json" : "application/x-protobuf" } });+ expect(body.length).toBeLessThan(20_000);+ expect((await send()).status).toBe(200);+ expect((await send()).status).toBe(429);+ expect(upstream).toHaveBeenCalledTimes(1);+ expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array)).toEqual(body);+ await f.relay.close();+ const records = await f.records(), admitted = records.find(row => row.type === "admitted");+ expect(admitted.recordCount).toBe(4);+ expect(admitted.expandedBytes).toBeGreaterThan(35_000);+ expect(records.at(-1)).toMatchObject({ admittedRecords: 4, admittedExpandedBytes: admitted.expandedBytes });+});++it("accepts thousands of small native records while bounding their total count", async () => {+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));+ const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRecords: 3000 } });+ const body = JSON.stringify({ resource_logs: [{ scope_logs: [{ log_records: Array.from({ length: 2533 }, () => ({ body: { stringValue: "event" } })) }] }] });+ expect((await f.send(body, {}, "/v1/logs")).status).toBe(200);+ expect((await f.send(body, {}, "/v1/logs")).status).toBe(429);+ expect(upstream).toHaveBeenCalledTimes(1);+ expect((await f.records()).find(row => row.type === "admitted").recordCount).toBe(2533);+});++it("rejects unknown protobuf fields instead of excluding forwarded data from sizing", async () => {+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));+ const f = await fixture({ transport: upstream });+ const body = message(1, message(2, message(2, message(100, Buffer.from("unknown-content")))));+ const response = await fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body,+ headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/x-protobuf" } });+ expect(response.status).toBe(400);+ expect(upstream).not.toHaveBeenCalled();+});diff --git a/tests/runner-e2e/eval-otlp-relay.ts b/tests/runner-e2e/eval-otlp-relay.tsnew file mode 100644index 0000000000..36e61d9dde--- /dev/null+++ b/tests/runner-e2e/eval-otlp-relay.ts@@ -0,0 +1,139 @@+import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";+import { once } from "node:events";+import { mkdir, open } from "node:fs/promises";+import { createServer, type ServerResponse } from "node:http";+import path from "node:path";+import { gunzipSync } from "node:zlib";+import { evalOtlpRejected, evalOtlpResponse, isEvalProjectId, validateEvalOtlpParent, validateEvalOtlpPayload, type EvalOtlpSignal } from "./eval-otlp-payload.js";++export const EVAL_OTLP_LIMITS = { maxRequests: 2048, maxRequestBytes: 8 * 1024 * 1024,+ maxTotalBytes: 64 * 1024 * 1024, maxConcurrent: 4,+ maxExpandedBytes: 256 * 1024 * 1024, maxRecords: 16_384 } as const;++export async function startEvalOtlpRelay(input: {+ projectId: string; apiKey: string; campaignId: string; attemptId: string; receiptPath: string; deadlineAt: number;+ limits?: { [K in keyof typeof EVAL_OTLP_LIMITS]: number }; transport?: typeof fetch;+}) {+ const limits = input.limits ?? EVAL_OTLP_LIMITS;+ if (!isEvalProjectId(input.projectId) || !input.apiKey.trim() || /[\r\n]/.test(input.apiKey)+ || ![input.campaignId, input.attemptId].every(value => /^[A-Za-z0-9_.:-]{1,512}$/.test(value))+ || !path.isAbsolute(input.receiptPath) || !Number.isFinite(input.deadlineAt)+ || input.deadlineAt <= Date.now() || input.deadlineAt > Date.now() + 4 * 3_600_000+ || Object.keys(limits).sort().join(",") !== Object.keys(EVAL_OTLP_LIMITS).sort().join(",")+ || Object.entries(EVAL_OTLP_LIMITS).some(([key, maximum]) => !Number.isSafeInteger(limits[key as keyof typeof limits])+ || limits[key as keyof typeof limits] < 1 || limits[key as keyof typeof limits] > maximum)) throw new Error("Invalid eval OTLP relay policy");+ await mkdir(path.dirname(input.receiptPath), { recursive: true, mode: 0o700 });+ const file = await open(input.receiptPath, "ax", 0o600);+ let writes = Promise.resolve(), poisoned = false, closed = false, quotaRecorded = false;+ const record = (value: object) => {+ writes = writes.then(async () => { await file.writeFile(JSON.stringify(value) + "\n"); await file.sync(); })+ .catch(error => { poisoned = true; throw error; });+ return writes;+ };+ try { await record({ type: "policy", version: 2, sizing: "otlp-v1.9-expanded-source-json", projectId: input.projectId, campaignId: input.campaignId,+ attemptId: input.attemptId, limits, deadlineAt: new Date(input.deadlineAt).toISOString(), openedAt: new Date().toISOString() }); }+ catch (error) { await file.close(); throw error; }+ const token = randomBytes(32).toString("hex");+ const pending = new Set<Promise<void>>(), controllers = new Set<AbortController>();+ const usage = { admittedRequests: 0, admittedWireBytes: 0, admittedBytes: 0, admittedExpandedBytes: 0,+ admittedRecords: 0, attemptedForwardBytes: 0, acceptedRequests: 0, incompleteRequests: 0 };+ const deny = (response: ServerResponse, status: number, code: string) => {+ if (response.destroyed) return;+ response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" });+ response.end(JSON.stringify({ error: { type: "eval_otlp_relay", message: code } }));+ };+ const server = createServer((request, response) => {+ const operation = (async () => {+ const auth = typeof request.headers.authorization === "string" ? request.headers.authorization.replace(/^Bearer /, "") : "";+ if (Buffer.byteLength(auth) !== Buffer.byteLength(token) || !timingSafeEqual(Buffer.from(auth), Buffer.from(token))) return deny(response, 401, "unauthorized");+ if (closed || poisoned || Date.now() >= input.deadlineAt) return deny(response, 503, "relay_unavailable");+ const signal: EvalOtlpSignal | undefined = request.url === "/otel/v1/traces" ? "traces" : request.url === "/otel/v1/logs" ? "logs" : undefined;+ if (request.method !== "POST" || !signal) return deny(response, 404, "unqualified_endpoint");+ if (pending.size >= limits.maxConcurrent) return deny(response, 429, "concurrency_limit");+ const contentType = request.headers["content-type"]?.split(";")[0].trim();+ const encoding = request.headers["content-encoding"];+ if (!["application/json", "application/x-protobuf"].includes(contentType ?? "") || encoding && encoding !== "gzip" && encoding !== "identity") return deny(response, 415, "unqualified_encoding");+ let body: Buffer, parent: string, wireBytes = 0, expandedBytes = 0, recordCount = 0;+ try {+ parent = validateEvalOtlpParent(String(request.headers["x-bt-parent"] ?? `project_id:${input.projectId}`), input.projectId);+ const chunks: Buffer[] = [];+ for await (const chunk of request) {+ wireBytes += chunk.length;+ if (wireBytes > limits.maxRequestBytes) throw new Error("request_limit");+ chunks.push(Buffer.from(chunk));+ }+ body = Buffer.concat(chunks);+ if (encoding === "gzip") body = gunzipSync(body, { maxOutputLength: limits.maxRequestBytes });+ const validated = validateEvalOtlpPayload(body, contentType!, signal, input.projectId);+ ({ body, expandedBytes, recordCount } = validated);+ if (body.length > limits.maxRequestBytes) throw new Error("request_limit");+ } catch { return deny(response, 400, "invalid_or_oversized_payload"); }+ if (closed || poisoned || Date.now() >= input.deadlineAt) return deny(response, 503, "relay_unavailable");+ if (usage.admittedRequests >= limits.maxRequests || usage.admittedBytes + body.length > limits.maxTotalBytes+ || usage.admittedExpandedBytes + expandedBytes > limits.maxExpandedBytes+ || usage.admittedRecords + recordCount > limits.maxRecords) {+ if (!quotaRecorded) { quotaRecorded = true; await record({ type: "quota_exhausted", at: new Date().toISOString(), ...usage }); }+ return deny(response, 429, "ingestion_allowance_exhausted");+ }+ const id = randomUUID();+ usage.admittedRequests++; usage.admittedWireBytes += wireBytes; usage.admittedBytes += body.length;+ usage.admittedExpandedBytes += expandedBytes; usage.admittedRecords += recordCount;+ await record({ type: "admitted", id, signal, at: new Date().toISOString(), wireBytes, bodyBytes: body.length,+ expandedBytes, recordCount,+ payloadSha256: createHash("sha256").update(body).digest("hex") });+ const abort = new AbortController(); controllers.add(abort);+ const disconnected = () => { if (!response.writableEnded) abort.abort(); };+ response.on("close", disconnected);+ let status: number | undefined, outcome = "uncertain", rejected: number | undefined, dispatched = false;+ try {+ if (closed || poisoned || response.destroyed || Date.now() >= input.deadlineAt) throw new Error("relay_unavailable");+ dispatched = true; usage.attemptedForwardBytes += body.length;+ const upstream = await (input.transport ?? fetch)(`https://api.braintrust.dev/otel/v1/${signal}`, {+ method: "POST", redirect: "error", signal: AbortSignal.any([abort.signal,+ AbortSignal.timeout(Math.max(1, Math.min(10_000, input.deadlineAt - Date.now())))]),+ headers: { authorization: `Bearer ${input.apiKey}`, "x-bt-parent": parent, "content-type": contentType!,+ "accept-encoding": "identity", accept: "application/x-protobuf, application/json" }, body: Buffer.from(body),+ });+ status = upstream.status;+ if (!upstream.ok) { await upstream.body?.cancel(); outcome = "rejected"; throw new Error("receiver_rejected"); }+ const type = upstream.headers.get("content-type")?.split(";")[0].trim() ?? contentType!;+ if (!["application/json", "application/x-protobuf"].includes(type)) { await upstream.body?.cancel(); throw new Error("invalid_response"); }+ const reader = upstream.body?.getReader(), chunks: Uint8Array[] = []; let bytes = 0;+ try {+ while (reader) {+ const next = await reader.read(); if (next.done) break;+ bytes += next.value.length; if (bytes > 65_536) throw new Error("response_limit");+ chunks.push(next.value);+ }+ } finally { await reader?.cancel().catch(() => {}); }+ rejected = evalOtlpRejected(Buffer.concat(chunks), type, signal);+ outcome = rejected ? "partial" : "accepted";+ } catch { if (!dispatched) outcome = "not_dispatched"; }+ finally { controllers.delete(abort); response.off("close", disconnected); }+ if (outcome === "accepted") usage.acceptedRequests++; else usage.incompleteRequests++;+ await record({ type: "outcome", id, at: new Date().toISOString(), outcome, dispatched,+ ...(status === undefined ? {} : { httpStatus: status }), ...(rejected === undefined ? {} : { rejectedRecords: rejected }) });+ if (response.destroyed) return;+ if (outcome !== "accepted" && outcome !== "partial") return deny(response, 502, "export_unconfirmed");+ response.writeHead(200, { "content-type": contentType!, "cache-control": "no-store" });+ response.end(evalOtlpResponse(rejected!, contentType!, signal));+ })();+ pending.add(operation);+ void operation.then(() => pending.delete(operation), () => { poisoned = true; pending.delete(operation); deny(response, 503, "receipt_persistence_failed"); });+ });+ server.requestTimeout = 15_000; server.headersTimeout = 10_000; server.maxConnections = limits.maxConcurrent + 8;+ try { server.listen(0, "127.0.0.1"); await once(server, "listening"); }+ catch (error) { await file.close(); throw error; }+ const address = server.address();+ if (!address || typeof address === "string") { await file.close(); throw new Error("Eval OTLP relay did not bind loopback"); }+ let closing: Promise<void> | undefined;+ const close = () => closing ??= (async () => {+ closed = true; clearTimeout(timer); controllers.forEach(controller => controller.abort()); server.closeAllConnections();+ await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()));+ await Promise.allSettled([...pending]);+ try { await record({ type: "closed", at: new Date().toISOString(), ...usage, poisoned }); }+ finally { await file.close(); }+ })();+ const timer = setTimeout(() => { void close().catch(() => {}); }, Math.max(1, input.deadlineAt - Date.now())); timer.unref();+ return { baseUrl: `http://127.0.0.1:${address.port}/otel`, token, receiptPath: input.receiptPath, close };+}diff --git a/tests/runner-e2e/eval-otlp-schema.ts b/tests/runner-e2e/eval-otlp-schema.tsnew file mode 100644index 0000000000..a277cbd7f1--- /dev/null+++ b/tests/runner-e2e/eval-otlp-schema.ts@@ -0,0 +1,170 @@+import { createRequire } from "node:module";+const require = createRequire(new URL("../../server/package.json", import.meta.url));+export const protobuf = require("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js");+export const evalOtlpSchema = protobuf.parse(`syntax = "proto3";+message AnyValue {+oneof value {+string string_value = 1;+bool bool_value = 2;+int64 int_value = 3;+double double_value = 4;+ArrayValue array_value = 5;+KeyValueList kvlist_value = 6;+bytes bytes_value = 7;+}+}+message ArrayValue {+repeated AnyValue values = 1;+}+message KeyValueList {+repeated KeyValue values = 1;+}+message KeyValue {+string key = 1;+AnyValue value = 2;+}+message InstrumentationScope {+string name = 1;+string version = 2;+repeated KeyValue attributes = 3;+uint32 dropped_attributes_count = 4;+}+message EntityRef {+string schema_url = 1;+string type = 2;+repeated string id_keys = 3;+repeated string description_keys = 4;+}+message Resource {+repeated KeyValue attributes = 1;+uint32 dropped_attributes_count = 2;+repeated EntityRef entity_refs = 3;+}+message TracesData {+repeated ResourceSpans resource_spans = 1;+}+message ResourceSpans {+Resource resource = 1;+repeated ScopeSpans scope_spans = 2;+string schema_url = 3;+}+message ScopeSpans {+InstrumentationScope scope = 1;+repeated Span spans = 2;+string schema_url = 3;+}+message Span {+bytes trace_id = 1;+bytes span_id = 2;+string trace_state = 3;+bytes parent_span_id = 4;+fixed32 flags = 16;+string name = 5;+enum SpanKind {+SPAN_KIND_UNSPECIFIED = 0;+SPAN_KIND_INTERNAL = 1;+SPAN_KIND_SERVER = 2;+SPAN_KIND_CLIENT = 3;+SPAN_KIND_PRODUCER = 4;+SPAN_KIND_CONSUMER = 5;+}+SpanKind kind = 6;+fixed64 start_time_unix_nano = 7;+fixed64 end_time_unix_nano = 8;+repeated KeyValue attributes = 9;+uint32 dropped_attributes_count = 10;+message Event {+fixed64 time_unix_nano = 1;+string name = 2;+repeated KeyValue attributes = 3;+uint32 dropped_attributes_count = 4;+}+repeated Event events = 11;+uint32 dropped_events_count = 12;+message Link {+bytes trace_id = 1;+bytes span_id = 2;+string trace_state = 3;+repeated KeyValue attributes = 4;+uint32 dropped_attributes_count = 5;+fixed32 flags = 6;+}+repeated Link links = 13;+uint32 dropped_links_count = 14;+Status status = 15;+}+message Status {+string message = 2;+enum StatusCode {+STATUS_CODE_UNSET = 0;+STATUS_CODE_OK = 1;+STATUS_CODE_ERROR = 2;+};+StatusCode code = 3;+}+enum SpanFlags {+SPAN_FLAGS_DO_NOT_USE = 0;+SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF;+SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100;+SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200;+}+message LogsData {+repeated ResourceLogs resource_logs = 1;+}+message ResourceLogs {+Resource resource = 1;+repeated ScopeLogs scope_logs = 2;+string schema_url = 3;+}+message ScopeLogs {+InstrumentationScope scope = 1;+repeated LogRecord log_records = 2;+string schema_url = 3;+}+enum SeverityNumber {+SEVERITY_NUMBER_UNSPECIFIED = 0;+SEVERITY_NUMBER_TRACE = 1;+SEVERITY_NUMBER_TRACE2 = 2;+SEVERITY_NUMBER_TRACE3 = 3;+SEVERITY_NUMBER_TRACE4 = 4;+SEVERITY_NUMBER_DEBUG = 5;+SEVERITY_NUMBER_DEBUG2 = 6;+SEVERITY_NUMBER_DEBUG3 = 7;+SEVERITY_NUMBER_DEBUG4 = 8;+SEVERITY_NUMBER_INFO = 9;+SEVERITY_NUMBER_INFO2 = 10;+SEVERITY_NUMBER_INFO3 = 11;+SEVERITY_NUMBER_INFO4 = 12;+SEVERITY_NUMBER_WARN = 13;+SEVERITY_NUMBER_WARN2 = 14;+SEVERITY_NUMBER_WARN3 = 15;+SEVERITY_NUMBER_WARN4 = 16;+SEVERITY_NUMBER_ERROR = 17;+SEVERITY_NUMBER_ERROR2 = 18;+SEVERITY_NUMBER_ERROR3 = 19;+SEVERITY_NUMBER_ERROR4 = 20;+SEVERITY_NUMBER_FATAL = 21;+SEVERITY_NUMBER_FATAL2 = 22;+SEVERITY_NUMBER_FATAL3 = 23;+SEVERITY_NUMBER_FATAL4 = 24;+}+enum LogRecordFlags {+LOG_RECORD_FLAGS_DO_NOT_USE = 0;+LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 0x000000FF;+}+message LogRecord {+fixed64 time_unix_nano = 1;+fixed64 observed_time_unix_nano = 11;+SeverityNumber severity_number = 2;+string severity_text = 3;+AnyValue body = 5;+repeated KeyValue attributes = 6;+uint32 dropped_attributes_count = 7;+fixed32 flags = 8;+bytes trace_id = 9;+bytes span_id = 10;+string event_name = 12;+}+message PartialSuccess { int64 rejected = 1; }+message Response { PartialSuccess partial_success = 1; }+`).root.resolveAll();diff --git a/tests/runner-e2e/experiment-ingestion.test.ts b/tests/runner-e2e/experiment-ingestion.test.tsnew file mode 100644index 0000000000..984bcb91a7--- /dev/null+++ b/tests/runner-e2e/experiment-ingestion.test.ts@@ -0,0 +1,108 @@+import { afterEach, expect, it, vi } from "vitest";+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { EVAL_EXPERIMENT_LIMITS, openExperimentIngestion } from "./experiment-ingestion.js";++const projectId = "11111111-1111-4111-8111-111111111111", experimentId = "22222222-2222-4222-8222-222222222222";+const roots: string[] = [], meters: Array<Awaited<ReturnType<typeof openExperimentIngestion>>> = [];+afterEach(async () => { await Promise.allSettled(meters.splice(0).map(meter => meter.close()));+ await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });+async function fixture(transport: typeof fetch = async () => Response.json({ ok: true })) {+ const root = await mkdtemp(path.join(tmpdir(), "experiment-ingestion-")); roots.push(root);+ const policy = { projectId, campaignId: "campaign", deadlineAt: Date.now() + 60_000, receiptPath: path.join(root, "receipts.jsonl"), transport };+ const meter = await openExperimentIngestion(policy); meters.push(meter);+ return { ...meter, policy, records: async () => (await readFile(policy.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)),+ send: (payload: unknown, endpoint = "experiment") => meter.transport(`https://api.braintrust.dev/v1/${endpoint}`, {+ method: "POST", body: JSON.stringify(payload), headers: { authorization: "Bearer private-secret" },+ }) };+}++it("durably meters payloads and scores before dispatch while retaining body-free private evidence", async () => {+ const upstream = vi.fn<typeof fetch>(async (_url, init) => {+ expect((await f.records()).at(-1)).toMatchObject({ type: "admitted", bodyBytes: init?.body === undefined ? 0 : Buffer.byteLength(String(init.body)) });+ expect(init!.redirect).toBe("error");+ return Response.json({ ok: true });+ });+ const f = await fixture(upstream);+ const create = { project_id: projectId, name: "private-payload-canary" };+ const insert = { events: [{ scores: { product_pass: 0, cleanup_pass: 0 }, input: "private-input-canary" }] };+ await f.send(create); await f.send(insert, `experiment/${experimentId}/insert`);+ await f.transport(`https://api.braintrust.dev/v1/experiment/${experimentId}/summarize`, { method: "GET" });+ await f.close();+ const records = await f.records();+ expect(records.at(-1)).toMatchObject({ type: "closed", admittedRequests: 3, admittedScores: 2,+ admittedBytes: Buffer.byteLength(JSON.stringify(create)) + Buffer.byteLength(JSON.stringify(insert)), unresolvedRequests: 0 });+ const raw = await readFile(f.policy.receiptPath, "utf8");+ expect(raw).not.toMatch(/private-secret|private-payload-canary|private-input-canary/);+ expect((await stat(f.policy.receiptPath)).mode & 0o777).toBe(0o600);+ await expect(openExperimentIngestion(f.policy)).rejects.toMatchObject({ code: "EEXIST" });+ expect(await readFile(f.policy.receiptPath, "utf8")).toBe(raw);+});++it("keeps failed forwarding charged to the finite allowance and records an unresolved outcome", async () => {+ const upstream = vi.fn<typeof fetch>(async () => { throw new Error("private-upstream-error"); });+ const f = await fixture(upstream);+ for (let i = 0; i < EVAL_EXPERIMENT_LIMITS.maxRequests; i++) {+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed");+ }+ await expect(f.send({ project_id: projectId })).rejects.toThrow("allowance exhausted");+ expect(upstream).toHaveBeenCalledTimes(3);+ await f.close();+ expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 3, responses: 0, unresolvedRequests: 3 });+});++it("retains an unresolved allowance when response headers arrive but the body disconnects", async () => {+ const f = await fixture(async () => new Response(new ReadableStream({ pull(controller) {+ controller.error(new Error("private-disconnect-error"));+ } }), { status: 200 }));+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed");+ await f.close();+ const rows = await f.records();+ expect(rows[2]).toMatchObject({ type: "outcome", outcome: "unconfirmed" });+ expect(rows.at(-1)).toMatchObject({ admittedRequests: 1, responses: 0, unresolvedRequests: 1 });+ expect(JSON.stringify(rows)).not.toContain("private-disconnect-error");+});++it("keeps a pending response body unresolved until the whole response is read", async () => {+ let finish!: () => void;+ const f = await fixture(async () => new Response(new ReadableStream({ start(controller) {+ controller.enqueue(new TextEncoder().encode('{"ok":'));+ finish = () => { controller.enqueue(new TextEncoder().encode('true}')); controller.close(); };+ } })));+ const pending = f.send({ project_id: projectId });+ await vi.waitFor(() => expect(finish).toBeTypeOf("function"));+ expect((await f.records()).at(-1)).toMatchObject({ type: "admitted" });+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable");+ finish();+ expect(await (await pending).json()).toEqual({ ok: true });+ await f.close();+ expect((await f.records()).at(-1)).toMatchObject({ responses: 1, unresolvedRequests: 0 });+});++it("refuses excess bytes, scores, foreign project and unqualified endpoints before dispatch", async () => {+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));+ const f = await fixture(upstream);+ await expect(f.send({ project_id: experimentId })).rejects.toThrow("project mismatch");+ await expect(f.send({ project_id: projectId, data: "x".repeat(EVAL_EXPERIMENT_LIMITS.maxTotalBytes) })).rejects.toThrow("byte limit");+ await expect(f.send({ events: Array.from({ length: 9 }, () => ({ scores: { pass: 1 } })) }, `experiment/${experimentId}/insert`))+ .rejects.toThrow("allowance exhausted");+ await expect(f.send({}, "project")).rejects.toThrow("Unqualified");+ expect(upstream).not.toHaveBeenCalled();+ await f.close();+ expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 0, admittedBytes: 0, admittedScores: 0 });+});++it("refuses concurrent requests and expired windows without resetting reservations", async () => {+ let release!: () => void;+ const upstream = vi.fn<typeof fetch>(async () => { await new Promise<void>(resolve => { release = resolve; }); return Response.json({}); });+ const f = await fixture(upstream);+ const pending = f.send({ project_id: projectId });+ await vi.waitFor(() => expect(upstream).toHaveBeenCalledOnce());+ await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable");+ release(); await pending;+ const now = vi.spyOn(Date, "now").mockReturnValue(f.policy.deadlineAt);+ try { await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); }+ finally { now.mockRestore(); }+ expect(upstream).toHaveBeenCalledOnce();+});diff --git a/tests/runner-e2e/experiment-ingestion.ts b/tests/runner-e2e/experiment-ingestion.tsnew file mode 100644index 0000000000..7cb1c102bb--- /dev/null+++ b/tests/runner-e2e/experiment-ingestion.ts@@ -0,0 +1,91 @@+import { createHash, randomUUID } from "node:crypto";+import { mkdir, open } from "node:fs/promises";+import path from "node:path";+import { isEvalProjectId } from "./eval-otlp-payload.js";++export const EVAL_EXPERIMENT_LIMITS = { maxRequests: 3, maxTotalBytes: 4 * 1024 * 1024, maxScores: 8 } as const;++export async function openExperimentIngestion(input: {+ receiptPath: string; campaignId: string; projectId: string; deadlineAt: number; transport?: typeof fetch;+}) {+ if (!path.isAbsolute(input.receiptPath) || !/^[A-Za-z0-9_.:-]{1,512}$/.test(input.campaignId)+ || !isEvalProjectId(input.projectId) || !Number.isFinite(input.deadlineAt)+ || input.deadlineAt <= Date.now() || input.deadlineAt > Date.now() + 60_000) throw new Error("Invalid experiment ingestion policy");+ await mkdir(path.dirname(input.receiptPath), { recursive: true, mode: 0o700 });+ const file = await open(input.receiptPath, "ax", 0o600);+ let writes = Promise.resolve(), closed = false, poisoned = false, active = false;+ let admittedRequests = 0, admittedBytes = 0, admittedScores = 0, responses = 0;+ const record = (value: object) => {+ writes = writes.then(async () => { await file.writeFile(JSON.stringify(value) + "\n"); await file.sync(); })+ .catch(error => { poisoned = true; throw error; });+ return writes;+ };+ try { await record({ type: "policy", version: 1, kind: "experiment", projectId: input.projectId, campaignId: input.campaignId,+ limits: EVAL_EXPERIMENT_LIMITS, openedAt: new Date().toISOString(), deadlineAt: new Date(input.deadlineAt).toISOString() }); }+ catch (error) { await file.close(); throw error; }+ const transport: typeof fetch = async (target, init) => {+ if (closed || poisoned || active || Date.now() >= input.deadlineAt) throw new Error("Experiment ingestion unavailable");+ const url = String(target), method = init?.method;+ const create = url === "https://api.braintrust.dev/v1/experiment" && method === "POST";+ const insert = /^https:\/\/api\.braintrust\.dev\/v1\/experiment\/[0-9a-f-]{36}\/insert$/.test(url) && method === "POST";+ const summarize = /^https:\/\/api\.braintrust\.dev\/v1\/experiment\/[0-9a-f-]{36}\/summarize$/.test(url) && method === "GET";+ if (!create && !insert && !summarize || summarize && init?.body !== undefined+ || !summarize && typeof init?.body !== "string") throw new Error("Unqualified experiment ingestion request");+ const body = summarize ? "" : init!.body as string;+ const bytes = Buffer.byteLength(body);+ if (bytes > EVAL_EXPERIMENT_LIMITS.maxTotalBytes) throw new Error("Experiment ingestion byte limit");+ let scores = 0;+ if (body) {+ const payload = JSON.parse(body);+ if (create && payload.project_id !== input.projectId) throw new Error("Experiment ingestion project mismatch");+ if (insert) {+ if (!Array.isArray(payload.events)) throw new Error("Invalid experiment ingestion events");+ scores = payload.events.reduce((sum: number, event: { scores?: object }) => sum + Object.keys(event.scores ?? {}).length, 0);+ }+ }+ if (admittedRequests >= EVAL_EXPERIMENT_LIMITS.maxRequests || admittedBytes + bytes > EVAL_EXPERIMENT_LIMITS.maxTotalBytes+ || admittedScores + scores > EVAL_EXPERIMENT_LIMITS.maxScores) throw new Error("Experiment ingestion allowance exhausted");+ active = true;+ const id = randomUUID();+ admittedRequests++; admittedBytes += bytes; admittedScores += scores;+ try {+ await record({ type: "admitted", id, at: new Date().toISOString(), operation: create ? "create" : insert ? "insert" : "summarize",+ bodyBytes: bytes, scores, payloadSha256: createHash("sha256").update(body).digest("hex") });+ if (closed || poisoned || Date.now() >= input.deadlineAt) throw new Error("Experiment ingestion unavailable");+ let response: Response;+ try {+ const upstream = await (input.transport ?? fetch)(target, { ...init, redirect: "error", signal: AbortSignal.any([+ AbortSignal.timeout(Math.max(1, Math.min(15_000, input.deadlineAt - Date.now()))), ...(init?.signal ? [init.signal] : []),+ ]) });+ const reader = upstream.body?.getReader(), chunks: Uint8Array[] = [];+ let responseBytes = 0;+ try {+ while (reader) {+ const next = await reader.read();+ if (next.done) break;+ responseBytes += next.value.length;+ if (responseBytes > 1_048_576) throw new Error("Oversized experiment response");+ chunks.push(next.value);+ }+ response = new Response(upstream.body === null ? null : Buffer.concat(chunks), { status: upstream.status, headers: upstream.headers });+ } finally { await reader?.cancel().catch(() => {}); }+ } catch {+ await record({ type: "outcome", id, at: new Date().toISOString(), outcome: "unconfirmed" });+ throw new Error("Experiment ingestion response unconfirmed");+ }+ try { await record({ type: "outcome", id, at: new Date().toISOString(), outcome: "response", httpStatus: response.status }); }+ catch (error) { await response.body?.cancel().catch(() => {}); throw error; }+ responses++;+ return response;+ } finally { active = false; }+ };+ let closing: Promise<void> | undefined;+ return { transport, close: () => closing ??= (async () => {+ closed = true;+ try {+ if (active) throw new Error("Experiment ingestion is still in flight");+ await record({ type: "closed", at: new Date().toISOString(), admittedRequests, admittedBytes, admittedScores,+ responses, unresolvedRequests: admittedRequests - responses });+ } finally { await file.close(); }+ })() };+}diff --git a/tests/runner-e2e/feedback-dispositions.test.ts b/tests/runner-e2e/feedback-dispositions.test.tsnew file mode 100644index 0000000000..743ac95fb8--- /dev/null+++ b/tests/runner-e2e/feedback-dispositions.test.ts@@ -0,0 +1,129 @@+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, it } from "vitest";+import { runnerMatrix } from "./catalog.js";+import { emptyFeedbackSummary } from "./feedback.js";+import { buildRunnerCampaign } from "./history.js";+import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";+import { campaignFeedbackEvidence, campaignFeedbackReviewText, feedbackReviewTemplate, validateCampaignFeedbackReview, type CampaignFeedbackReview } from "./feedback-dispositions.js";+import type { RunnerE2EResult } from "./types.js";++function fixture() {+ const cell = runnerMatrix[0]!;+ const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id, suiteId: cell.suite.id,+ suiteDefinitionHash: cell.suiteDefinitionHash, attempt: 1, status: "failed", profileId: cell.profile.id,+ environmentId: cell.environment.id, caseId: cell.task.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,+ runIds: [], startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed",+ feedback: { ...emptyFeedbackSummary(), collection: "partial", invocationCollection: "complete", complaints: 2, suggestions: 1, bodiesCollected: 3,+ review: ["a", "b", "c"].map(value => ({ fingerprint: value.repeat(64), kind: value === "c" ? "suggestion" : "complaint", occurrences: 1, disposition: "needs_investigation" })) } };+ return buildRunnerCampaign({ campaignId: "feedback-campaign", generatedAt: result.finishedAt, expected: [cell.id], results: [result] });+}+function reviewed(campaign = fixture()): CampaignFeedbackReview {+ return { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "agent",+ decisions: [+ { fingerprint: "a".repeat(64), disposition: "actionable", issueNumber: 123 },+ { fingerprint: "b".repeat(64), disposition: "duplicate", duplicateOf: "a".repeat(64) },+ { fingerprint: "c".repeat(64), disposition: "unsupported" },+ ] };+}++it("keeps an unreviewed template invalid until a reviewer records the decision metadata", () => {+ const campaign = fixture();+ expect(() => validateCampaignFeedbackReview(campaign, feedbackReviewTemplate(campaign) as never)).toThrow("Invalid or stale");+ expect(campaignFeedbackReviewText(campaign)).toContain("3 distinct feedback items await review");+ validateCampaignFeedbackReview(campaign, reviewed(campaign));+});++it("preserves dispositions on regeneration without changing failed grades or incomplete collection", async () => {+ const campaign = fixture(); campaign.feedbackReview = reviewed(campaign);+ const root = await mkdtemp(path.join(tmpdir(), "feedback-review-"));+ try {+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify({ ...campaign, feedbackReview: undefined }));+ const retainedHeader = "# Existing campaign\n\nProvider-reported LLM cost: $0.123456\n\n";+ await writeFile(path.join(root, "summary.md"), `${retainedHeader}${campaignFeedbackReviewText({ ...campaign, feedbackReview: undefined })}\n`);+ const feedbackReviewFile = path.join(root, "review.json");+ await writeFile(feedbackReviewFile, JSON.stringify(campaign.feedbackReview));+ await regenerateRunnerDashboard({ bundle: root, historyFile: null, feedbackReviewFile });+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });+ const retained = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));+ expect(retained.feedbackReview).toEqual(campaign.feedbackReview);+ expect(retained.results[0].status).toBe("failed");+ expect(retained.accounting.attempts[0].feedback.collection).toBe("partial");+ const html = await readFile(path.join(root, "dashboard.html"), "utf8");+ expect(html).toContain("1 actionable; 1 duplicate; 0 already fixed; 1 unsupported; 0 needs investigation");+ expect(html).toContain("1 attempts have incomplete collection");+ expect(html).toContain("https://github.com/paperclipai/paperclip/issues/123");+ const summary = await readFile(path.join(root, "summary.md"), "utf8");+ expect(summary).toBe(`${retainedHeader}${campaignFeedbackReviewText(campaign)}\n`);+ expect(summary).not.toContain("await review");+ } finally { await rm(root, { recursive: true, force: true }); }+});++it("rejects stale reviews when attempts or collection evidence change, but permits a render refresh", () => {+ const campaign = fixture(), review = reviewed(campaign);+ campaign.generatedAt = "2026-10-21T00:00:00Z";+ validateCampaignFeedbackReview(campaign, review);+ const changed = structuredClone(campaign);+ changed.accounting!.attempts[0]!.feedback!.collection = "complete";+ expect(() => validateCampaignFeedbackReview(changed, review)).toThrow("Invalid or stale");+ changed.accounting = structuredClone(campaign.accounting);+ changed.accounting!.missingAttempts++;+ expect(() => validateCampaignFeedbackReview(changed, review)).toThrow("Invalid or stale");+ expect(campaignFeedbackEvidence({ ...campaign, expected: [...campaign.expected].reverse() })).toBe(review.evidenceHash);+});++it("requires exactly one closed decision per group and refuses raw notes and duplicate cycles", () => {+ const campaign = fixture(), review = reviewed(campaign);+ for (const changed of [+ { ...review, rationale: "private notes" },+ { ...review, decisions: review.decisions.slice(1) },+ { ...review, decisions: [...review.decisions, review.decisions[0]!] },+ { ...review, decisions: review.decisions.map(item => ({ ...item, rawBody: "private body" })) },+ { ...review, decisions: review.decisions.map(item => item.fingerprint === "a".repeat(64) ? { ...item, disposition: "duplicate", duplicateOf: "b".repeat(64) } : item) },+ ]) expect(() => validateCampaignFeedbackReview(campaign, changed as never)).toThrow("Invalid or stale");+});++it.each(["2026-10-20T01:00:00Z", "2026-10-20T01:00:00.936739+00:00", "2026-10-19T20:00:00-05:00"])("accepts explicit ISO review timestamps: %s", reviewedAt => {+ const campaign = fixture();+ validateCampaignFeedbackReview(campaign, { ...reviewed(campaign), reviewedAt });+});++it("rejects private timestamp notes before a review can alter the public report", async () => {+ const campaign = fixture();+ const reviewedAt = "Tue, 20 Oct 2026 01:00:00 GMT (Private rationale: customer account was suspended)";+ expect(Number.isFinite(Date.parse(reviewedAt))).toBe(true);+ const root = await mkdtemp(path.join(tmpdir(), "feedback-private-timestamp-"));+ try {+ const original = JSON.stringify(campaign);+ await writeFile(path.join(root, "normalized-results.json"), original);+ const feedbackReviewFile = path.join(root, "review.json");+ await writeFile(feedbackReviewFile, JSON.stringify({ ...reviewed(campaign), reviewedAt }));+ await expect(regenerateRunnerDashboard({ bundle: root, historyFile: null, feedbackReviewFile })).rejects.toThrow("Invalid or stale");+ expect(await readFile(path.join(root, "normalized-results.json"), "utf8")).toBe(original);+ for (const invalid of ["2026-10-20T01:00:00Z\n", "2026-10-20", "2026-10-20T24:00:00Z"]) {+ expect(() => validateCampaignFeedbackReview(campaign, { ...reviewed(campaign), reviewedAt: invalid })).toThrow("Invalid or stale");+ }+ } finally { await rm(root, { recursive: true, force: true }); }+});++it("records an explicit empty review without implying that uncaptured feedback was absent", () => {+ const campaign = fixture(); campaign.accounting!.attempts[0]!.feedback = emptyFeedbackSummary();+ campaign.feedbackReview = { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "human", decisions: [] };+ validateCampaignFeedbackReview(campaign, campaign.feedbackReview);+ expect(campaignFeedbackReviewText(campaign)).toContain("1 attempts have incomplete collection");+});++it("refreshes an older summary that kept the awaiting-review count inline", async () => {+ const campaign = fixture(); campaign.feedbackReview = reviewed(campaign);+ const root = await mkdtemp(path.join(tmpdir(), "feedback-legacy-summary-"));+ try {+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));+ await writeFile(path.join(root, "summary.md"), "# Existing campaign\n\nFeedback across retained attempts: 2 complaints; 1 suggestions; 3 distinct items awaiting review; 1 attempts with incomplete collection.\n");+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });+ const summary = await readFile(path.join(root, "summary.md"), "utf8");+ expect(summary).toContain("2 complaints; 1 suggestions; 1 attempts with incomplete collection.");+ expect(summary).toContain(campaignFeedbackReviewText(campaign));+ expect(summary).not.toContain("awaiting review");+ } finally { await rm(root, { recursive: true, force: true }); }+});diff --git a/tests/runner-e2e/feedback-dispositions.ts b/tests/runner-e2e/feedback-dispositions.tsnew file mode 100644index 0000000000..243a79a70d--- /dev/null+++ b/tests/runner-e2e/feedback-dispositions.ts@@ -0,0 +1,95 @@+import { createHash } from "node:crypto";+import { readFile, writeFile } from "node:fs/promises";+import { pathToFileURL } from "node:url";+import { summarizeCampaignFeedback } from "./feedback.js";+import { html, isPublicReportTimestamp } from "./report-format.js";+import type { RunnerE2ECampaign } from "./types.js";++const dispositions = ["actionable", "duplicate", "already_fixed", "unsupported", "needs_investigation"] as const;+export interface CampaignFeedbackReview {+ version: 1;+ campaignId: string;+ evidenceHash: string;+ reviewedAt: string;+ reviewedBy: "human" | "agent";+ decisions: Array<{+ fingerprint: string;+ disposition: typeof dispositions[number];+ duplicateOf?: string;+ issueNumber?: number;+ }>;+}++export function campaignFeedbackEvidence(campaign: RunnerE2ECampaign) {+ if (!campaign.accounting) throw new Error("Feedback review requires retained attempt accounting");+ const attempts = campaign.accounting.attempts.map(attempt => ({+ id: attempt.id, executionId: attempt.executionId, attempt: attempt.attempt,+ feedback: attempt.feedback ? {+ ...attempt.feedback,+ review: [...attempt.feedback.review].sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)),+ } : null,+ })).sort((a, b) => a.id.localeCompare(b.id));+ const evidence = { campaignId: campaign.campaignId, expected: [...campaign.expected].sort(),+ scope: campaign.accounting.scope, missingAttempts: campaign.accounting.missingAttempts, attempts };+ const canonical = (_key: string, value: unknown) => value && typeof value === "object" && !Array.isArray(value)+ ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) : value;+ return createHash("sha256").update(JSON.stringify(evidence, canonical)).digest("hex");+}++export function feedbackReviewTemplate(campaign: RunnerE2ECampaign) {+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);+ return { version: 1, campaignId: campaign.campaignId, evidenceHash: campaignFeedbackEvidence(campaign),+ reviewedAt: null, reviewedBy: null,+ decisions: feedback.review.map(({ fingerprint }) => ({ fingerprint, disposition: "needs_investigation" })) };+}++export function validateCampaignFeedbackReview(campaign: RunnerE2ECampaign, review: CampaignFeedbackReview): void {+ const fail = (): never => { throw new Error("Invalid or stale campaign feedback review"); };+ const fields = (value: unknown, allowed: string[]) => {+ if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(key => !allowed.includes(key))) fail();+ };+ fields(review, ["version", "campaignId", "evidenceHash", "reviewedAt", "reviewedBy", "decisions"]);+ if (review.version !== 1 || review.campaignId !== campaign.campaignId || review.evidenceHash !== campaignFeedbackEvidence(campaign)+ || !isPublicReportTimestamp(review.reviewedAt)+ || !["human", "agent"].includes(review.reviewedBy) || !Array.isArray(review.decisions)) fail();+ const groups = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results).review;+ const expected = new Set(groups.map(group => group.fingerprint)), seen = new Set<string>();+ for (const decision of review.decisions) {+ fields(decision, ["fingerprint", "disposition", "duplicateOf", "issueNumber"]);+ if (!expected.has(decision.fingerprint) || seen.has(decision.fingerprint) || !dispositions.includes(decision.disposition)) fail();+ seen.add(decision.fingerprint);+ if (decision.issueNumber !== undefined && (!Number.isSafeInteger(decision.issueNumber) || decision.issueNumber < 1)) fail();+ if (decision.disposition === "duplicate") {+ if (!decision.duplicateOf || decision.duplicateOf === decision.fingerprint || !expected.has(decision.duplicateOf)+ || review.decisions.find(item => item.fingerprint === decision.duplicateOf)?.disposition === "duplicate") fail();+ } else if (decision.duplicateOf !== undefined) fail();+ }+ if (seen.size !== expected.size) fail();+}++export function campaignFeedbackReviewText(campaign: RunnerE2ECampaign): string {+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);+ const review = campaign.feedbackReview;+ if (!review) return `${feedback.review.length} distinct feedback items await review. Collection gaps and failed submissions remain separate findings.`;+ validateCampaignFeedbackReview(campaign, review);+ const count = (disposition: typeof dispositions[number]) => review.decisions.filter(item => item.disposition === disposition).length;+ return `Feedback reviewed by ${review.reviewedBy} at ${review.reviewedAt}: ${dispositions.map(value => `${count(value)} ${value.replaceAll("_", " ")}`).join("; ")}. ${feedback.incomplete} attempts have incomplete collection; review does not close these gaps or change product grades.`;+}++export function renderCampaignFeedbackReview(campaign: RunnerE2ECampaign): string {+ const summary = campaignFeedbackReviewText(campaign);+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);+ const rows = feedback.review.map(group => {+ const decision = campaign.feedbackReview?.decisions.find(item => item.fingerprint === group.fingerprint);+ return `<tr><td>${html(group.kind)}</td><td><code>${html(group.fingerprint)}</code></td><td>${group.occurrences}</td><td>${html(decision?.disposition ?? "unreviewed")}</td><td>${decision?.duplicateOf ? html(decision.duplicateOf) : ""}${decision?.issueNumber ? `<a href="https://github.com/paperclipai/paperclip/issues/${decision.issueNumber}">#${decision.issueNumber}</a>` : ""}</td></tr>`;+ }).join("");+ return `<section aria-label="Campaign feedback review"><h2>Campaign feedback review</h2><p>${html(summary)}</p><p>Full bodies and review rationale stay in private evidence. Exact duplicate bodies are grouped once; related causes can be linked by a reviewed duplicate disposition.</p>${rows ? `<div class="matcher-wrap" tabindex="0" role="region" aria-label="Feedback review decisions"><table><thead><tr><th>Kind</th><th>Fingerprint</th><th>Occurrences</th><th>Disposition</th><th>Follow-up</th></tr></thead><tbody>${rows}</tbody></table></div>` : ""}</section>`;+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const [campaignFile, outputFile] = process.argv.slice(2);+ if (!campaignFile || !outputFile) throw new Error("Usage: feedback-dispositions.ts <normalized-results.json> <private-review.json>");+ const campaign = JSON.parse(await readFile(campaignFile, "utf8")) as RunnerE2ECampaign;+ await writeFile(outputFile, `${JSON.stringify(feedbackReviewTemplate(campaign), null, 2)}\n`, { mode: 0o600, flag: "wx" });+ console.log("Created an unreviewed template. Inspect private feedback bodies, fill dispositions and review metadata, then import with PAPERCLIP_E2E_FEEDBACK_REVIEW.");+}diff --git a/tests/runner-e2e/feedback-persistence.test.ts b/tests/runner-e2e/feedback-persistence.test.tsnew file mode 100644index 0000000000..ca178d36a7--- /dev/null+++ b/tests/runner-e2e/feedback-persistence.test.ts@@ -0,0 +1,68 @@+import { describe, expect, it } from "vitest";+import { expectedFixtureFeedback, feedbackPersistenceDigest, gradeFeedbackPersistence } from "./feedback-persistence.js";+import { runnerMatrix, runnerTasks, suiteDefinitionHash } from "./catalog.js";+import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";++function fixture() {+ const nonce = "fixture-nonce", bodies = expectedFixtureFeedback(nonce);+ return { nonce, companyId: "company", agentId: "agent", taskId: "task", runIds: ["run"],+ evidence: { version: 1, companyId: "company", summary: { collection: "complete", invocationCollection: "complete" },+ entries: (["complaint", "suggestion"] as const).map(kind => ({ id: kind, kind, body: bodies[kind],+ agent: { id: "agent" }, task: { id: "task" }, runId: "run" })),+ invocations: (["complaint", "suggestion"] as const).map(kind => ({ callId: `${kind}-call`, kind, runId: "run", outcome: "completed" })),+ },+ };+}++describe("feedback persistence oracle", () => {+ it("changes the suite hash when the inherited prompt or matchers change", () => {+ const base = runnerTasks.find(task => task.id === "message-marker")!;+ const suite = runnerMatrix.find(execution => execution.suite.id === "feedback-persistence")!.suite;+ const baseline = suiteDefinitionHash(suite);+ expect(suite.definitionMetadata?.sourceDigest).toBe(feedbackPersistenceDigest(base));+ for (const changed of [+ { ...base, buildPrompt: () => "A different completion contract" },+ { ...base, buildMatchers: () => [] },+ ]) {+ expect(suiteDefinitionHash({ ...suite, definitionMetadata: {+ ...suite.definitionMetadata, sourceDigest: feedbackPersistenceDigest(changed),+ } })).not.toBe(baseline);+ }+ });+ it("accepts independently retained bodies with same-run real tool receipts", () => {+ expect(gradeFeedbackPersistence(fixture()).every(check => check.passed)).toBe(true);+ });+ it.each([+ ["missing saved evidence", (f: ReturnType<typeof fixture>) => { f.evidence.entries = []; }],+ ["prose without tool receipts", (f: ReturnType<typeof fixture>) => { f.evidence.invocations = []; }],+ ["wrong saved body", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.body = "submitted successfully"; }],+ ["foreign company", (f: ReturnType<typeof fixture>) => { f.evidence.companyId = "other"; }],+ ["foreign task", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.task.id = "other"; }],+ ["foreign agent", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.agent.id = "other"; }],+ ["foreign run", (f: ReturnType<typeof fixture>) => { f.evidence.entries[0]!.runId = "other"; }],+ ["unrelated successful receipt", (f: ReturnType<typeof fixture>) => { f.evidence.invocations[0]!.runId = "other"; }],+ ["failed tool", (f: ReturnType<typeof fixture>) => { f.evidence.invocations[0]!.outcome = "failed"; }],+ ["partial collection", (f: ReturnType<typeof fixture>) => { f.evidence.summary.collection = "partial"; }],+ ["partial invocation scan", (f: ReturnType<typeof fixture>) => { f.evidence.summary.invocationCollection = "partial"; }],+ ["duplicate submission", (f: ReturnType<typeof fixture>) => { f.evidence.invocations.push(f.evidence.invocations[0]!); }],+ ["duplicate saved row", (f: ReturnType<typeof fixture>) => { f.evidence.entries.push(f.evidence.entries[0]!); }],+ ] as const)("rejects %s", (_name, mutate) => {+ const f = fixture(); mutate(f);+ expect(gradeFeedbackPersistence(f).some(check => !check.passed)).toBe(true);+ });+ it("fails closed on unavailable or malformed retained evidence", () => {+ for (const evidence of [undefined, null, {}, { entries: [null], invocations: [null] }]) {+ expect(gradeFeedbackPersistence({ ...fixture(), evidence }).some(check => !check.passed)).toBe(true);+ }+ });+ it("registers only two explicit local native cases with no automatic retry", () => {+ const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", "feedback-persistence"]));+ expect(selected.map(execution => execution.id)).toEqual([+ "feedback-persistence.runner-codex.local.complaint-and-suggestion",+ "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion",+ ]);+ expect(selected.every(execution => execution.task.automaticRetryPolicy === "single_attempt")).toBe(true);+ expect(runnerMatrix).toEqual(expect.arrayContaining(selected));+ expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "feedback-persistence")).toBe(false);+ });+});diff --git a/tests/runner-e2e/feedback-persistence.ts b/tests/runner-e2e/feedback-persistence.tsnew file mode 100644index 0000000000..fd4faf1ee7--- /dev/null+++ b/tests/runner-e2e/feedback-persistence.ts@@ -0,0 +1,78 @@+import { createHash } from "node:crypto";+import { readFileSync } from "node:fs";+import type { RunnerTaskFixture } from "./types.js";++export const FEEDBACK_PERSISTENCE_SUITE = "feedback-persistence";+const feedbackPersistenceSourceDigest = createHash("sha256")+ .update(readFileSync(new URL(import.meta.url)))+ .update(readFileSync(new URL("./catalog.ts", import.meta.url)))+ .update(readFileSync(new URL("./feedback.ts", import.meta.url)))+ .update(readFileSync(new URL("./runner.spec.ts", import.meta.url)))+ .digest("hex");++export function feedbackPersistenceDigest(base: RunnerTaskFixture) {+ return createHash("sha256").update(feedbackPersistenceSourceDigest)+ .update(JSON.stringify(base, (_key, value) => typeof value === "function" ? value.toString() : value))+ .digest("hex");+}++export function expectedFixtureFeedback(nonce: string) {+ return {+ complaint: `Synthetic eval complaint ${nonce}: the fixture export omitted its receipt. This is test data, not a real product incident.`,+ suggestion: `Synthetic eval suggestion ${nonce}: add a keyboard shortcut to the fixture preview. This is a separate test idea, not a real feature request.`,+ };+}++export function feedbackPersistenceTask(base: RunnerTaskFixture): RunnerTaskFixture {+ return {+ ...base, id: "complaint-and-suggestion", label: "Retain submitted feedback", automaticRetryPolicy: "single_attempt",+ attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 },+ buildPrompt(nonce) {+ const bodies = expectedFixtureFeedback(nonce);+ return [+ "This disposable acceptance task verifies two different feedback tools. Before completing the task, discover the feedback tools and submit exactly one complaint and one suggestion through their real tool-call channel.",+ `Call submit_complaint with body ${JSON.stringify(bodies.complaint)} and idempotencyKey ${JSON.stringify(`complaint-${nonce}`)}.`,+ `Call submit_suggestion with body ${JSON.stringify(bodies.suggestion)} and idempotencyKey ${JSON.stringify(`suggestion-${nonce}`)}.`,+ "Wait for both saved acknowledgments. Do not substitute prose, a task comment, direct API calls, or files for these two tool submissions. These are distinct synthetic items; do not submit additional feedback.",+ base.buildPrompt(nonce),+ ].join("\n");+ },+ };+}++const row = (value: unknown): Record<string, unknown> =>+ value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};++export function gradeFeedbackPersistence(input: {+ evidence: unknown; nonce: string; companyId: string | undefined; agentId: string | undefined;+ taskId: string | undefined; runIds: string[];+}) {+ const evidence = row(input.evidence), summary = row(evidence.summary);+ const entries = Array.isArray(evidence.entries) ? evidence.entries.map(row) : [];+ const invocations = Array.isArray(evidence.invocations) ? evidence.invocations.map(row) : [];+ const expected = expectedFixtureFeedback(input.nonce);+ const checks: Array<{ id: string; passed: boolean; detail: string }> = [];+ const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail });+ check("complete-retained-evidence", evidence.version === 1 && Boolean(input.companyId)+ && evidence.companyId === input.companyId && summary.collection === "complete" && summary.invocationCollection === "complete",+ "Both collection passes completed and the saved evidence belongs to this company.");+ check("one-of-each", entries.length === 2 && new Set(entries.map(entry => entry.id)).size === 2+ && entries.every(entry => typeof entry.id === "string" && entry.id.length > 0),+ "Exactly two distinct saved feedback records remain after teardown.");+ for (const kind of ["complaint", "suggestion"] as const) {+ const saved = entries.filter(entry => entry.kind === kind);+ const entry = saved[0] ?? {};+ const calls = invocations.filter(call => call.kind === kind);+ check(`${kind}-body-and-attribution`, saved.length === 1 && entry.body === expected[kind]+ && Boolean(input.agentId) && row(entry.agent).id === input.agentId && Boolean(input.taskId)+ && row(entry.task).id === input.taskId && typeof entry.runId === "string" && input.runIds.includes(entry.runId),+ "The independently read saved body has the exact nonce, agent, task, and observed run.");+ check(`${kind}-completed-tool`, calls.length === 1 && calls[0]!.outcome === "completed"+ && typeof calls[0]!.callId === "string" && Boolean(calls[0]!.callId)+ && typeof entry.runId === "string" && calls[0]!.runId === entry.runId,+ "A real completed tool receipt from the same run corroborates the saved feedback.");+ }+ check("only-expected-submissions", invocations.length === 2 && new Set(invocations.map(call => call.callId)).size === 2,+ "There are no duplicate or extra feedback submissions.");+ return checks;+}diff --git a/tests/runner-e2e/feedback-recovery.test.ts b/tests/runner-e2e/feedback-recovery.test.tsnew file mode 100644index 0000000000..21e23ea200--- /dev/null+++ b/tests/runner-e2e/feedback-recovery.test.ts@@ -0,0 +1,111 @@+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";+import path from "node:path";+import os from "node:os";+import { afterEach, expect, it, vi } from "vitest";+import { recoverInterruptedFeedback, readRecoveredFeedback } from "./feedback-recovery.js";+import { mustPreserveRecoveryState } from "./cleanup-verification.js";+import { isHistoricalBundlePathAllowed } from "./history-publish.js";+import type { RunnerApi } from "./api.js";++vi.mock("node:fs/promises", async (importOriginal) => {+ const actual = await importOriginal<typeof import("node:fs/promises")>();+ return { ...actual, writeFile: vi.fn(actual.writeFile) };+});++const roots: string[] = [];+afterEach(async () => { vi.mocked(writeFile).mockReset(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });+const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution" };+const companyId = "88c1939b-2d9f-4bb3-8ebc-8a7a85efeeac";+const api = (get: (url: string) => unknown): Pick<RunnerApi, "get"> => ({ async get<T>(url: string) { return get(url) as T; } });++async function fixture() {+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-feedback-recovery-")); roots.push(root);+ await mkdir(path.join(root, "snapshots"));+ await writeFile(path.join(root, "snapshots/fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId }));+ return root;+}++it("retains saved feedback after worker loss while active-run coverage stays partial", async () => {+ const root = await fixture();+ const entry = { id: "saved", kind: "complaint", runId: "run", body: "Synthetic lost-worker complaint. private-canary-value" };+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000,+ secrets: ["private-canary-value"], api: api(url => {+ if (url.includes("agent-commentary?")) return { items: [entry], total: 1 };+ if (url.endsWith("agent-commentary/saved")) return entry;+ if (url.endsWith("heartbeat-runs?limit=1000")) return [{ id: "run" }];+ if (url.endsWith("heartbeat-runs/run")) return { id: "run", runtimeMode: "native", status: "running" };+ return [];+ }),+ });+ const summary = await readRecoveredFeedback(root, identity);+ expect(summary).toMatchObject({ collection: "partial", invocationCollection: "partial", complaints: 1, bodiesCollected: 1 });+ expect(summary?.review).toHaveLength(1);+ const saved = await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8");+ expect(saved).toContain("Synthetic lost-worker complaint.");+ expect(saved).not.toContain("private-canary-value");+ expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/agent-feedback-recovered.json")).toBe(false);+ expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/feedback-recovery.json")).toBe(false);+ await writeFile(path.join(root, "snapshots/agent-feedback.json"), saved);+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [],+ api: api(() => { throw new Error("controller stopped after the worker checkpoint"); }),+ });+ expect(await readRecoveredFeedback(root, identity)).toMatchObject({ complaints: 1, bodiesCollected: 1, collection: "partial" });+ expect(await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8")).toContain("Synthetic lost-worker complaint.");+ expect(await readRecoveredFeedback(root, { ...identity, attemptId: "different" })).toBeUndefined();+ const receiptPath = path.join(root, "snapshots/feedback-recovery.json");+ const receipt = JSON.parse(await readFile(receiptPath, "utf8"));+ receipt.summary.collection = "complete";+ await writeFile(receiptPath, JSON.stringify(receipt));+ expect(await readRecoveredFeedback(root, identity)).toBeUndefined();+});++it("preserves completed attempt evidence without making another API request", async () => {+ const root = await fixture();+ await writeFile(path.join(root, "result.json"), "retained original");+ let calls = 0;+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [],+ api: api(() => { calls++; throw new Error("must not read"); }),+ });+ expect(calls).toBe(0);+ expect(await readFile(path.join(root, "result.json"), "utf8")).toBe("retained original");+ expect(await readRecoveredFeedback(root, identity)).toBeUndefined();+});++it("reports the earlier atomic checkpoint when a later write exhausts the recovery deadline", async () => {+ const root = await fixture();+ const actual = await vi.importActual<typeof import("node:fs/promises")>("node:fs/promises");+ let checkpointWrites = 0;+ vi.mocked(writeFile).mockImplementation(async (file, data, options) => {+ if (String(file).endsWith("agent-feedback-recovered.json.pending") && ++checkpointWrites === 3) {+ const signal = typeof options === "object" && options ? options.signal : undefined;+ if (!signal?.aborted) await new Promise(resolve => signal!.addEventListener("abort", resolve, { once: true }));+ signal!.throwIfAborted();+ }+ return actual.writeFile(file, data, options);+ });+ const entry = { id: "saved", kind: "suggestion", runId: "run", body: "Keep this earlier durable checkpoint." };+ await expect(recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 1000,+ secrets: [], api: api(url => {+ if (url.includes("agent-commentary?")) return { items: [entry], total: 1 };+ if (url.endsWith("agent-commentary/saved")) return entry;+ return [];+ }),+ })).rejects.toThrow();+ expect(checkpointWrites).toBe(3);+ expect(await readRecoveredFeedback(root, identity)).toMatchObject({ suggestions: 1, bodiesCollected: 1,+ collection: "partial", invocationCollection: "unavailable" });+ expect(await readRecoveredFeedback(root, { ...identity, attemptId: "different" })).toBeUndefined();+ await expect(readFile(path.join(root, "snapshots/feedback-recovery.json"))).rejects.toMatchObject({ code: "ENOENT" });+ const saved = JSON.parse(await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8"));+ expect(saved.entries[0].body).toBe(entry.body);+});++it("reports unavailable recovery without fabricating an empty successful collection", async () => {+ const root = await fixture();+ await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [],+ api: api(() => { throw new Error("controller unavailable"); }),+ });+ expect(await readRecoveredFeedback(root, identity)).toMatchObject({ collection: "unavailable", invocationCollection: "unavailable" });+ expect(mustPreserveRecoveryState({ processCleanupFailed: false, results: [{ cleanup: "passed", synthetic: true }] })).toBe(true);+ expect(mustPreserveRecoveryState({ processCleanupFailed: false, results: [{ cleanup: "passed", synthetic: false }] })).toBe(false);+});diff --git a/tests/runner-e2e/feedback-recovery.ts b/tests/runner-e2e/feedback-recovery.tsnew file mode 100644index 0000000000..0d8c571fd9--- /dev/null+++ b/tests/runner-e2e/feedback-recovery.ts@@ -0,0 +1,101 @@+import { access, readFile, rename, writeFile } from "node:fs/promises";+import path from "node:path";+import type { RunnerApi } from "./api.js";+import { collectAttemptFeedback, emptyFeedbackSummary, type FeedbackEvidence, type FeedbackSummary } from "./feedback.js";+import { assertSecretFree, sanitizeJson } from "./redaction.js";++export interface FeedbackRecoveryIdentity {+ campaignId: string;+ attemptId: string;+ executionId: string;+}++const exists = (file: string) => access(file).then(() => true, () => false);++export async function recoverInterruptedFeedback(input: {+ identity: FeedbackRecoveryIdentity;+ caseDirectory: string;+ api: Pick<RunnerApi, "get">;+ secrets: readonly string[];+ deadlineAt: number;+}) {+ const deadline = AbortSignal.timeout(Math.max(1, input.deadlineAt - Date.now()));+ if (await exists(path.join(input.caseDirectory, "result.json"))) return;+ const snapshots = path.join(input.caseDirectory, "snapshots");+ let fixture: { executionId?: string; companyId?: string };+ try { fixture = JSON.parse(await readFile(path.join(snapshots, "fixtures.json"), { encoding: "utf8", signal: deadline })); }+ catch { return; }+ if (fixture.executionId !== input.identity.executionId || !/^[a-f0-9-]{36}$/i.test(fixture.companyId ?? "")) return;+ const companyId = fixture.companyId!;+ let summary = emptyFeedbackSummary();+ let checkpoint: FeedbackEvidence | undefined;+ const recoveryReceipt = (summary: FeedbackSummary) => ({ version: 1, ...input.identity, companyId,+ reason: "worker_exited_without_result", summary });+ const save = async (name: string, value: unknown, signal: AbortSignal = deadline) => {+ const text = JSON.stringify(sanitizeJson(value, input.secrets), null, 2) + "\n";+ assertSecretFree(text, input.secrets, name);+ await writeFile(path.join(snapshots, `${name}.pending`), text, { mode: 0o600, signal });+ signal.throwIfAborted();+ await rename(path.join(snapshots, `${name}.pending`), path.join(snapshots, name));+ };+ const saveCheckpoint = (evidence: FeedbackEvidence, signal: AbortSignal = deadline) =>+ save("agent-feedback-recovered.json", { ...evidence, recovery: recoveryReceipt({ ...evidence.summary,+ collection: evidence.summary.bodiesCollected ? "partial" : "unavailable",+ invocationCollection: evidence.invocations.length ? "partial" : "unavailable" }) }, signal);+ try {+ const original = JSON.parse(await readFile(path.join(snapshots, "agent-feedback.json"), { encoding: "utf8", signal: deadline }));+ if (original.version === 1 && original.companyId === companyId && original.summary?.version === 1+ && Array.isArray(original.entries) && Array.isArray(original.invocations)+ && original.summary.bodiesCollected === original.entries.length) {+ await saveCheckpoint(original);+ checkpoint = original;+ summary = structuredClone(original.summary);+ }+ } catch {}+ try {+ const collected = await collectAttemptFeedback({ api: input.api, companyId, deadlineAt: input.deadlineAt,+ persist: async (evidence, signal) => {+ if (checkpoint && (!checkpoint.entries.every(entry => evidence.entries.some(next => next.id === entry.id))+ || !checkpoint.invocations.every(call => evidence.invocations.some(next => next.runId === call.runId && next.callId === call.callId)))) return;+ await saveCheckpoint(evidence, signal);+ checkpoint = structuredClone(evidence);+ summary = structuredClone(evidence.summary);+ },+ });+ summary.collection = summary.bodiesCollected ? "partial" : collected.collection;+ summary.invocationCollection = summary.invocations.completed + summary.invocations.failed + summary.invocations.unknown+ ? "partial" : collected.invocationCollection;+ } catch {+ summary.collection = summary.bodiesCollected ? "partial" : "unavailable";+ summary.invocationCollection = "unavailable";+ }+ await save("feedback-recovery.json", recoveryReceipt(summary));+}++export async function readRecoveredFeedback(caseDirectory: string, identity: FeedbackRecoveryIdentity): Promise<FeedbackSummary | undefined> {+ try {+ const snapshots = path.join(caseDirectory, "snapshots");+ const evidence = JSON.parse(await readFile(path.join(snapshots, "agent-feedback-recovered.json"), "utf8"));+ let receipt = evidence.recovery;+ try { receipt = JSON.parse(await readFile(path.join(snapshots, "feedback-recovery.json"), "utf8")); }+ catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") return; }+ if (receipt?.version !== 1 || evidence.version !== 1 || receipt.reason !== "worker_exited_without_result"+ || !receipt.companyId || receipt.companyId !== evidence.companyId+ || Object.entries(identity).some(([key, value]) => receipt[key] !== value)) return;+ const summary = evidence.summary as FeedbackSummary;+ const statuses = ["complete", "partial", "unavailable", "not_started"];+ if (summary?.version !== 1 || !Array.isArray(evidence.entries) || !Array.isArray(evidence.invocations)+ || summary.bodiesCollected !== evidence.entries.length || !Array.isArray(summary.review)+ || ![summary.complaints, summary.suggestions, summary.bodiesCollected, summary.runsInspected,+ summary.invocations?.completed, summary.invocations?.failed, summary.invocations?.unknown]+ .every(value => Number.isSafeInteger(value) && value >= 0)+ || !["observed", "unknown"].includes(summary.toolAvailability)+ || !["collection", "invocationCollection"].every(key => {+ const original = summary[key as "collection" | "invocationCollection"];+ const recovered = receipt.summary?.[key];+ return statuses.includes(original) && statuses.includes(recovered)+ && (recovered === original || recovered === "partial" || recovered === "unavailable");+ })) return;+ return { ...summary, collection: receipt.summary.collection, invocationCollection: receipt.summary.invocationCollection };+ } catch { return; }+}diff --git a/tests/runner-e2e/feedback-shutdown-process.test.ts b/tests/runner-e2e/feedback-shutdown-process.test.tsnew file mode 100644index 0000000000..ffd6119416--- /dev/null+++ b/tests/runner-e2e/feedback-shutdown-process.test.ts@@ -0,0 +1,113 @@+import { spawn } from "node:child_process";+import { createRequire } from "node:module";+import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { expect, it } from "vitest";+import { readRecoveredFeedback } from "./feedback-recovery.js";+import { runnerE2EWebServerCommand, runnerE2EPlaywrightInvocation, runnerE2EWebServerGracefulShutdown, runnerE2EPlaywrightShutdownSignal } from "./web-server-command.js";+import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";++it.skipIf(process.platform === "win32")("retains feedback before shutdown through the real Playwright launch chain", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "eval feedback 'shutdown-"));+ const repository = path.resolve(import.meta.dirname, "../..");+ const caseDirectory = path.join(root, "case");+ const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution" };+ const companyId = "88c1939b-2d9f-4bb3-8ebc-8a7a85efeeac";+ let child: ReturnType<typeof spawn> | undefined;+ let timer: ReturnType<typeof setTimeout> | undefined;+ let supervisorPid: number | undefined;+ let owner: ReturnType<typeof createProcessTreeOwner> | undefined;+ try {+ await mkdir(path.join(root, "tests/runner-e2e"), { recursive: true });+ await mkdir(path.join(caseDirectory, "snapshots"), { recursive: true });+ await symlink(path.join(repository, "cli"), path.join(root, "cli"));+ await writeFile(path.join(root, "package.json"), JSON.stringify({ type: "module" }));+ await writeFile(path.join(caseDirectory, "snapshots/fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId }));+ await writeFile(path.join(root, "tests/runner-e2e/server.ts"), `+ import { createServer } from 'node:http';+ import { writeFile } from 'node:fs/promises';+ import { recoverInterruptedFeedback } from ${JSON.stringify(pathToFileURL(path.join(repository, "tests/runner-e2e/feedback-recovery.ts")).href)};+ const entries = ['complaint', 'suggestion'].map(kind => ({ id: kind, kind, runId: 'run', body: 'Saved ' + kind + ' private-canary-value' }));+ const server = createServer((req, res) => {+ setTimeout(() => {+ res.setHeader('content-type', 'application/json');+ res.end(JSON.stringify(req.url.includes('agent-commentary?') ? { items: entries, total: 2 }+ : req.url.includes('agent-commentary/') ? entries.find(entry => req.url.endsWith('/' + entry.id))+ : req.url.endsWith('/heartbeat-runs/run') ? { id: 'run', runtimeMode: 'native', status: 'running' }+ : req.url.includes('heartbeat-runs?') ? [{ id: 'run' }] : []));+ }, 100);+ });+ let closing;+ process.on('SIGTERM', () => closing ??= (async () => {+ await recoverInterruptedFeedback({ identity: ${JSON.stringify(identity)}, caseDirectory: ${JSON.stringify(caseDirectory)},+ deadlineAt: Date.now() + 5000, secrets: ['private-canary-value'], api: {+ async get(endpoint, options) {+ return (await fetch('http://127.0.0.1:' + server.address().port + endpoint,+ { signal: AbortSignal.timeout(options?.timeout ?? 1000) })).json();+ }+ } });+ server.close();+ server.closeAllConnections();+ })());+ server.listen(0, '127.0.0.1', async () => {+ await writeFile(${JSON.stringify(path.join(root, "ready.json"))}, JSON.stringify({ pid: process.pid }));+ console.log('SUPERVISOR_READY');+ });+ `);+ const testModule = createRequire(import.meta.url).resolve("@playwright/test");+ const configPath = path.join(root, "playwright.config.cjs");+ await writeFile(configPath, `module.exports = {+ testDir: ${JSON.stringify(root)}, testMatch: 'interrupted.spec.cjs', workers: 1, reporter: 'line', timeout: 30000,+ webServer: { command: ${JSON.stringify(runnerE2EWebServerCommand(root))},+ wait: { stdout: /SUPERVISOR_READY/ }, timeout: 10000,+ gracefulShutdown: ${JSON.stringify(runnerE2EWebServerGracefulShutdown)} }+ };`);+ await writeFile(path.join(root, "interrupted.spec.cjs"), `+ const { test } = require(${JSON.stringify(testModule)});+ const { writeFile } = require('node:fs/promises');+ test('interrupted worker', async () => {+ await writeFile(${JSON.stringify(path.join(root, "worker-ready"))}, 'ready');+ await new Promise(() => {});+ });+ `);+ const invocation = runnerE2EPlaywrightInvocation(repository, ["test", "--config", configPath]);+ child = spawn(invocation.command, invocation.args, { cwd: repository, detached: true, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, CI: "1" } });+ owner = createProcessTreeOwner(child);+ let output = "";+ child.stdout!.on("data", chunk => { output += chunk; });+ child.stderr!.on("data", chunk => { output += chunk; });+ const closed = new Promise<void>((resolve, reject) => {+ child!.once("error", reject);+ child!.once("close", () => resolve());+ });+ const deadline = Date.now() + 10_000;+ let workerReady = false;+ while ((!supervisorPid || !workerReady) && Date.now() < deadline) {+ const ready = await readFile(path.join(root, "ready.json"), "utf8").catch(() => null);+ if (ready) supervisorPid = JSON.parse(ready).pid;+ workerReady = await readFile(path.join(root, "worker-ready"), "utf8").then(() => true, () => false);+ if (!supervisorPid || !workerReady) await new Promise(resolve => setTimeout(resolve, 25));+ }+ expect(supervisorPid, output).toBeTypeOf("number");+ expect(workerReady, output).toBe(true);+ await stopOwnedProcessTree(child, owner, 10_000, 2_000, runnerE2EPlaywrightShutdownSignal);+ await Promise.race([closed, new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error(`Shutdown did not finish: ${output}`)), 10_000);+ })]);+ expect(await readRecoveredFeedback(caseDirectory, identity), output).toMatchObject({+ collection: "partial", invocationCollection: "partial", complaints: 1, suggestions: 1, bodiesCollected: 2,+ });+ const evidence = await readFile(path.join(caseDirectory, "snapshots/agent-feedback-recovered.json"), "utf8");+ expect(evidence).toContain("Saved complaint");+ expect(evidence).not.toContain("private-canary-value");+ expect(() => process.kill(supervisorPid!, 0)).toThrow();+ } finally {+ owner?.stopObserving();+ if (timer) clearTimeout(timer);+ if (owner) await owner.signal("SIGKILL").catch(() => {});+ if (child?.pid && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");+ await rm(root, { recursive: true, force: true });+ }+}, 35_000);diff --git a/tests/runner-e2e/feedback.test.ts b/tests/runner-e2e/feedback.test.tsnew file mode 100644index 0000000000..6cb8ead6bb--- /dev/null+++ b/tests/runner-e2e/feedback.test.ts@@ -0,0 +1,319 @@+import { expect, it, vi } from "vitest";+import type { AgentCommentaryDetail } from "../../packages/shared/src/validators/agent-commentary.js";+import type { RunnerApi } from "./api.js";+import { collectAttemptFeedback, emptyFeedbackSummary, feedbackReview, observeFeedbackInvocations, summarizeCampaignFeedback, FEEDBACK_COLLECTION_LIMITS, type FeedbackEvidence } from "./feedback.js";+import { runCleanupWithObservers } from "./cleanup-verification.js";+import { isHistoricalBundlePathAllowed } from "./history-publish.js";+import { canonicalProviderEventsFromOpenCodePart } from "../../packages/paperclip-runner/src/provider-events.js";++function entry(id: string, body = "Please preserve the original tool error."): AgentCommentaryDetail {+ return { id, kind: "complaint", preview: body.slice(0, 280), body,+ createdAt: "2026-10-20T00:00:00.000Z", agent: { id: "agent", name: "Fixture", icon: null }, runId: "run", task: null };+}+const api = (read: (path: string) => unknown | Promise<unknown>) => ({ get: async <T>(path: string) => await read(path) as T }) as Pick<RunnerApi, "get">;+const event = (seq: number, eventType: string, payload: unknown, runId = "run") => ({ seq, eventType,+ payload: { prpEvent: { runId, sourceKind: "runner", eventType, payload } } });++it("collects full paginated company feedback, including feedback outside the selected task, and checkpoints it", async () => {+ const entries = Array.from({ length: 101 }, (_, i) => ({ ...entry(String(i), "a".repeat(300)), runId: `child-${i}` }));+ const checkpoints: FeedbackEvidence[] = [];+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { checkpoints.push(structuredClone(value)); },+ api: api(path => {+ if (path.includes("agent-commentary?")) {+ const offset = Number(new URL(path, "http://fixture").searchParams.get("offset"));+ return { items: entries.slice(offset, offset + 100).map(({ body: _body, ...rest }) => rest), total: 101 };+ }+ if (path.includes("agent-commentary/")) return entries[Number(path.split("/").at(-1))];+ return [];+ }),+ });+ expect(summary).toMatchObject({ collection: "complete", complaints: 101, bodiesCollected: 101, toolAvailability: "unknown" });+ expect(summary.review).toEqual([{ fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), occurrences: 101, kind: "complaint", disposition: "needs_investigation" }]);+ expect(checkpoints.at(-1)!.entries[100]!.body).toHaveLength(300);+ expect(checkpoints.some(value => value.entries.length === 10)).toBe(true);+ expect(JSON.stringify(summary)).not.toContain("aaa");+});++it("keeps saved-feedback absence separate from unavailable reads and unknown tool availability", async () => {+ const empty = await collectAttemptFeedback({ companyId: "company", persist: async () => {},+ api: api(path => path.includes("agent-commentary") ? { items: [], total: 0 } : []),+ });+ expect(empty).toMatchObject({ collection: "complete", complaints: 0, suggestions: 0, toolAvailability: "unknown" });+ const unavailable = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(() => { throw new Error("unavailable"); }) });+ expect(unavailable).toMatchObject({ collection: "unavailable", invocationCollection: "unavailable", complaints: 0 });+});++it("retains partial bodies and failed submissions without inventing persistence from a completed tool", async () => {+ let retained: FeedbackEvidence | undefined;+ const events = [+ event(1, "tool.execution.started", { name: "submit_suggestion", executionId: "failed" }),+ event(2, "tool.execution.completed", { name: "submit_suggestion", executionId: "failed", status: "failed" }),+ event(3, "tool.execution.started", { name: "submit_complaint", executionId: "done" }),+ event(4, "tool.execution.completed", { name: "submit_complaint", executionId: "done", status: "completed" }),+ event(5, "tool.execution.started", { name: "submit_suggestion", executionId: "interrupted" }),+ ];+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); },+ api: api(path => {+ if (path.includes("agent-commentary?")) return { items: [entry("1"), entry("2")], total: 2 };+ if (path.endsWith("agent-commentary/1")) return entry("1");+ if (path.endsWith("agent-commentary/2")) throw new Error("body unavailable");+ if (path.includes("/events?")) return events;+ if (path === "/api/heartbeat-runs/run") return { id: "run", runtimeMode: "native", status: "failed" };+ return [{ id: "run", status: "failed" }];+ }),+ });+ expect(summary).toMatchObject({ collection: "partial", complaints: 2, bodiesCollected: 1, invocationCollection: "complete",+ invocations: { completed: 1, failed: 1, unknown: 1 } });+ expect(retained!.diagnostics).toContain("feedback_detail_failed");+});++it("binds native invocations to their run, ignores prose, and recognizes compatibility receipts", () => {+ const events = [+ event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "call", name: "paperclip.submit_complaint" } }),+ event(2, "item.completed", { kind: "tool_result", item: { type: "tool_result", id: "call", status: "completed", isError: true } }),+ event(3, "tool.execution.started", { name: "submit_suggestion", executionId: "foreign" }, "foreign-run"),+ event(4, "item.completed", { item: { type: "agentMessage", text: "I used submit_complaint" } }),+ ];+ expect(observeFeedbackInvocations("run", events)).toEqual([{ runId: "run", callId: "call", kind: "complaint", outcome: "failed" }]);+});++it.each(["paperclip_", "paperclip__", "mcp__paperclip__"])("maps namespaced feedback receipts through native events: %s", prefix => {+ const events = [+ event(1, "tool.execution.started", { executionId: "failed", name: `${prefix}submit_complaint` }),+ event(2, "tool.execution.completed", { executionId: "failed", name: `${prefix}submit_complaint`, status: "failed" }),+ event(3, "item.started", { kind: "tool_call", item: { id: "done", name: `${prefix}submit_suggestion` } }),+ event(4, "item.completed", { kind: "tool_result", item: { id: "done", status: "completed" } }),+ event(5, "tool.execution.started", { executionId: "interrupted", name: `${prefix}submit_suggestion` }),+ event(6, "tool.execution.started", { executionId: "unrelated", name: "other_submit_complaint" }),+ ];+ expect(observeFeedbackInvocations("run", events)).toEqual([+ { runId: "run", callId: "failed", kind: "complaint", outcome: "failed" },+ { runId: "run", callId: "done", kind: "suggestion", outcome: "completed" },+ { runId: "run", callId: "interrupted", kind: "suggestion", outcome: "unknown" },+ ]);+});++it.each([{ runtimeMode: "legacy", status: "succeeded" }, { runtimeMode: "native", status: "running" }])("does not claim complete invocation coverage for legacy helpers or still-active work: %o", async run => {+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(path => {+ if (path.includes("agent-commentary")) return { items: [], total: 0 };+ if (path.includes("/events?")) return [];+ if (path === "/api/heartbeat-runs/run") return { id: "run", ...run };+ return [{ id: "run", status: run.status }];+ }) });+ expect(summary.invocationCollection).toBe("partial");+});++it("captures feedback after cancellation failure but before destructive teardown, and still tears down if persistence fails", async () => {+ const calls: string[] = [];+ const result = await runCleanupWithObservers({+ retireRuns: async () => { throw new Error("cancel failed"); }, assertions: [],+ teardown: async () => {+ try {+ await collectAttemptFeedback({ companyId: "company", api: api(() => []), persist: async () => { calls.push("capture"); throw new Error("disk full"); } });+ } finally { calls.push("teardown"); }+ },+ });+ expect(calls).toEqual(["capture", "teardown"]);+ expect(result.errors).toHaveLength(2);+});++it("bounds collection by a deadline and retains content-free diagnostics", async () => {+ let requests = 0, last: FeedbackEvidence | undefined;+ const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 0,+ persist: async value => { last = structuredClone(value); }, api: api(() => { requests++; throw new Error("secret response"); }),+ });+ expect(requests).toBe(0);+ expect(summary.collection).toBe("unavailable");+ expect(last).toBeUndefined();+});++it("preserves the read prefix when slow detail requests exhaust the read budget mid-page", async () => {+ let now = 0, detailReads = 0;+ const clock = vi.spyOn(Date, "now").mockImplementation(() => now);+ const checkpoints: FeedbackEvidence[] = [];+ const slowApi = { get: async (url: string, options: { timeout: number }) => {+ if (url.includes("agent-commentary?")) return { items: Array.from({ length: 100 }, (_, index) => entry(String(index))), total: 100 };+ if (!url.includes("agent-commentary/")) throw new Error("No read should start after the reserved cutoff");+ detailReads++;+ now += Math.min(400, options.timeout);+ if (options.timeout < 400) throw new Error("request timeout");+ return entry(url.split("/").at(-1)!);+ } } as Pick<RunnerApi, "get">;+ try {+ const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000, api: slowApi,+ persist: async evidence => { checkpoints.push(structuredClone(evidence)); now += 20; },+ });+ expect(summary).toMatchObject({ collection: "partial", invocationCollection: "unavailable" });+ expect(summary.bodiesCollected).toBeGreaterThan(10);+ expect(summary.bodiesCollected).toBeLessThan(100);+ expect(checkpoints[1].entries).toHaveLength(10);+ expect(checkpoints.at(-1)!.entries).toHaveLength(summary.bodiesCollected);+ expect(checkpoints.at(-1)!.entries.every(value => value.body === entry("fixture").body)).toBe(true);+ expect(checkpoints.at(-1)!.diagnostics).toContain("deadline_reached");+ expect(detailReads).toBe(summary.bodiesCollected + 1);+ expect(now).toBeLessThan(30_000);+ } finally { clock.mockRestore(); }+});++it("groups identical feedback for one review without conflating kinds", () => {+ expect(feedbackReview([entry("1", " slow request "), entry("2", "slow request"), { ...entry("3", "slow request"), kind: "suggestion" }]))+ .toMatchObject([{ kind: "complaint", occurrences: 2 }, { kind: "suggestion", occurrences: 1 }]);+});++it("groups feedback across retries without omitting incomplete collection and keeps bodies out of public history", () => {+ const feedback = { ...emptyFeedbackSummary(), complaints: 1, review: feedbackReview([entry("1")]) };+ const aggregate = summarizeCampaignFeedback([{ feedback }, { feedback }, {}]);+ expect(aggregate).toMatchObject({ complaints: 2, incomplete: 3, review: [{ occurrences: 2 }] });+ expect(feedback.review[0]!.occurrences).toBe(1);+ expect(JSON.stringify(aggregate)).not.toContain(entry("1").body);+ expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/agent-feedback.json")).toBe(false);+});++it("uses run details for native coverage when the real list projection omits runtimeMode", async () => {+ const reads: string[] = [];+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(path => {+ reads.push(path);+ if (path.includes("agent-commentary")) return { items: [], total: 0 };+ if (path.includes("/events?")) return [];+ if (path === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" };+ return [{ id: "run", status: "succeeded" }];+ }) });+ expect(reads).toContain("/api/heartbeat-runs/run");+ expect(summary).toMatchObject({ collection: "complete", invocationCollection: "complete", runsInspected: 1 });+});+it("retains partial evidence if the run-detail lookup fails", async () => {+ let retained: FeedbackEvidence | undefined;+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, api: api(path => {+ if (path.includes("agent-commentary")) return { items: [], total: 0 };+ if (path.includes("/events?")) return [event(1, "tool.execution.started", { name: "submit_complaint", executionId: "call" })];+ if (path === "/api/heartbeat-runs/run") throw new Error("unavailable");+ return [{ id: "run", status: "succeeded" }];+ }) });+ expect(summary).toMatchObject({ collection: "partial", invocationCollection: "partial", invocations: { unknown: 1 }, runsInspected: 1 });+ expect(retained!.diagnostics).toContain("run_detail_failed");+});++it("retains OpenCode feedback whose first observed update is running or terminal", () => {+ const updates = [+ { id: "running", status: "running", tool: "paperclip_submit_suggestion" },+ { id: "failed", status: "error", tool: "paperclip_submit_complaint" },+ { id: "completed", status: "completed", tool: "paperclip_submit_suggestion" },+ ].flatMap(update => canonicalProviderEventsFromOpenCodePart({+ id: update.id, messageID: "message", type: "tool", callID: update.id, tool: update.tool,+ state: { status: update.status, input: {}, output: "fixture result", error: "fixture failure", time: { start: 1, end: 2 } },+ }));+ expect(updates.map(update => update.eventType)).toEqual(["tool.execution.progressed", "tool.execution.completed", "tool.execution.completed"]);+ expect(observeFeedbackInvocations("run", updates.map((update, index) => event(index, update.eventType, update.payload)))).toEqual([+ { runId: "run", callId: "running", kind: "suggestion", outcome: "unknown" },+ { runId: "run", callId: "failed", kind: "complaint", outcome: "failed" },+ { runId: "run", callId: "completed", kind: "suggestion", outcome: "completed" },+ ]);+});++it("bounds retained bytes and checkpoint writes for maximum-size feedback bodies", async () => {+ const body = "x".repeat(524_288);+ let detailReads = 0, writtenBytes = 0, writes = 0, retained: FeedbackEvidence | undefined;+ const summary = await collectAttemptFeedback({ companyId: "company",+ persist: async evidence => { writes++; writtenBytes += Buffer.byteLength(JSON.stringify(evidence, null, 2)); retained = structuredClone(evidence); },+ api: api(url => {+ if (url.includes("agent-commentary?")) return { items: Array.from({ length: 100 }, (_, index) => entry(String(index), "preview")), total: 10_000 };+ if (url.includes("agent-commentary/")) { detailReads++; return entry(url.split("/").at(-1)!, body); }+ return [];+ }),+ });+ expect(detailReads).toBeLessThanOrEqual(16);+ expect(writes).toBeLessThanOrEqual(4);+ expect(writtenBytes).toBeLessThan(3 * FEEDBACK_COLLECTION_LIMITS.retainedBytes);+ expect(Buffer.byteLength(JSON.stringify(retained!.entries))).toBeLessThanOrEqual(FEEDBACK_COLLECTION_LIMITS.retainedBytes);+ expect(summary.collection).toBe("partial");+ expect(retained!.diagnostics).toContain("evidence_limit_reached");+ expect(retained!.entries.every(item => item.body === body)).toBe(true);+});++it("caps entry count and stops requesting further feedback pages", async () => {+ let pages = 0, retained: FeedbackEvidence | undefined;+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async evidence => { retained = structuredClone(evidence); },+ api: api(url => {+ if (url.includes("agent-commentary?")) {+ const offset = Number(new URL(url, "http://fixture").searchParams.get("offset")); pages++;+ return { items: Array.from({ length: 100 }, (_, index) => entry(String(offset + index))), total: 2000 };+ }+ if (url.includes("agent-commentary/")) return entry(url.split("/").at(-1)!);+ return [];+ }),+ });+ expect(pages).toBe(10);+ expect(summary.bodiesCollected).toBe(FEEDBACK_COLLECTION_LIMITS.entries);+ expect(summary.collection).toBe("partial");+ expect(retained!.diagnostics).toContain("evidence_limit_reached");+});++it("aborts stalled evidence persistence at the collection deadline", async () => {+ let signal: AbortSignal | undefined, reads = 0;+ await expect(collectAttemptFeedback({ companyId: "company", deadlineAt: Date.now() + 50,+ persist: async (_evidence, currentSignal) => { signal = currentSignal; await new Promise<void>(() => {}); },+ api: api(() => { reads++; return []; }),+ })).rejects.toThrow("Feedback persistence deadline reached");+ expect(signal?.aborted).toBe(true);+ expect(reads).toBe(0);+});++it("acknowledges only successfully persisted feedback when a later checkpoint fails", async () => {+ let retained: FeedbackEvidence | undefined, acknowledged = emptyFeedbackSummary();+ const entries = Array.from({ length: 11 }, (_, index) => entry(String(index)));+ await expect(collectAttemptFeedback({ companyId: "company",+ onCheckpoint: summary => { acknowledged = summary; },+ persist: async evidence => {+ if (evidence.entries.length > 10) throw new Error("disk full");+ retained = structuredClone(evidence);+ },+ api: api(url => url.includes("agent-commentary?") ? { items: entries, total: entries.length }+ : entries[Number(url.split("/").at(-1))]),+ })).rejects.toThrow("disk full");+ expect(retained!.entries).toHaveLength(10);+ expect(acknowledged).toEqual(retained!.summary);+ expect(acknowledged.bodiesCollected).toBe(10);+ expect(acknowledged.review).toMatchObject([{ occurrences: 10 }]);+});++it("returns saved review counts when the deadline expires before a later checkpoint", async () => {+ let now = 0, retained: FeedbackEvidence | undefined;+ const clock = vi.spyOn(Date, "now").mockImplementation(() => now);+ const entries = Array.from({ length: 11 }, (_, index) => entry(String(index)));+ try {+ const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000,+ persist: async evidence => { retained = structuredClone(evidence); },+ api: api(url => {+ if (url.includes("agent-commentary?")) return { items: entries, total: entries.length };+ const index = Number(url.split("/").at(-1));+ if (index === 10) now = 30_001;+ return entries[index];+ }),+ });+ expect(retained!.entries).toHaveLength(10);+ expect(summary).toMatchObject({ bodiesCollected: 10, collection: "partial", invocationCollection: "unavailable" });+ expect(summary.review).toEqual(retained!.summary.review);+ } finally { clock.mockRestore(); }+});++it("bounds raw event inspection while retaining the observed feedback receipt prefix", async () => {+ let pages = 0, retained: FeedbackEvidence | undefined;+ const summary = await collectAttemptFeedback({ companyId: "company", persist: async evidence => { retained = structuredClone(evidence); },+ api: api(url => {+ if (url.includes("agent-commentary?")) return { items: [], total: 0 };+ if (url.includes("/events?")) {+ pages++;+ return Array.from({ length: 100 }, (_, index) => event(index + 1, "tool.execution.started", {+ executionId: `call-${index}`, name: "submit_complaint", input: "x".repeat(200_000),+ }));+ }+ if (url === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" };+ return [{ id: "run", status: "succeeded" }];+ }),+ });+ expect(pages).toBe(1);+ expect(summary.invocationCollection).toBe("partial");+ expect(summary.invocations.unknown).toBeGreaterThan(0);+ expect(summary.invocations.unknown).toBeLessThan(100);+ expect(retained!.diagnostics).toContain("evidence_limit_reached");+});diff --git a/tests/runner-e2e/feedback.ts b/tests/runner-e2e/feedback.tsnew file mode 100644index 0000000000..7a24b21cb6--- /dev/null+++ b/tests/runner-e2e/feedback.ts@@ -0,0 +1,268 @@+import { createHash } from "node:crypto";+import type { AgentCommentaryDetail, AgentCommentaryPage } from "../../packages/shared/src/validators/agent-commentary.js";+import type { RunnerApi } from "./api.js";++type Row = Record<string, unknown>;+const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {};+export type FeedbackCollectionStatus = "complete" | "partial" | "unavailable" | "not_started";+export interface FeedbackSummary {+ version: 1;+ collection: FeedbackCollectionStatus;+ invocationCollection: FeedbackCollectionStatus;+ toolAvailability: "observed" | "unknown";+ complaints: number;+ suggestions: number;+ bodiesCollected: number;+ runsInspected: number;+ invocations: { completed: number; failed: number; unknown: number };+ review: Array<{ fingerprint: string; kind: "complaint" | "suggestion"; occurrences: number; disposition: "needs_investigation" }>;+}+interface FeedbackInvocation {+ runId: string;+ callId: string;+ kind: "complaint" | "suggestion";+ outcome: "completed" | "failed" | "unknown";+}+export interface FeedbackEvidence {+ version: 1;+ companyId: string;+ entries: AgentCommentaryDetail[];+ invocations: FeedbackInvocation[];+ diagnostics: Array<"feedback_read_failed" | "feedback_detail_failed" | "feedback_pagination_incomplete" | "run_read_failed" | "run_detail_failed" | "run_limit_reached" | "events_read_failed" | "deadline_reached" | "evidence_limit_reached">;+ summary: FeedbackSummary;+}++export const FEEDBACK_COLLECTION_LIMITS = {+ entries: 1000, retainedBytes: 8 * 1024 * 1024, eventBytes: 16 * 1024 * 1024, events: 10_000,+} as const;++export function emptyFeedbackSummary(): FeedbackSummary {+ return { version: 1, collection: "not_started", invocationCollection: "not_started", toolAvailability: "unknown",+ complaints: 0, suggestions: 0, bodiesCollected: 0, runsInspected: 0,+ invocations: { completed: 0, failed: 0, unknown: 0 }, review: [] };+}++function feedbackKind(name: unknown): FeedbackInvocation["kind"] | undefined {+ if (typeof name !== "string") return undefined;+ const canonicalName = name.replace(/^(?:mcp__paperclip__|paperclip__|paperclip_)/, "");+ if (/(?:^|[.\/])submit_complaint$/.test(canonicalName)) return "complaint";+ if (/(?:^|[.\/])submit_suggestion$/.test(canonicalName)) return "suggestion";+ return undefined;+}++export function observeFeedbackInvocations(runId: string, events: readonly Row[]): FeedbackInvocation[] {+ const calls = new Map<string, FeedbackInvocation>();+ for (const event of events) {+ const envelope = row(row(event.payload).prpEvent);+ if (envelope.runId !== runId || envelope.sourceKind !== "runner" || envelope.eventType !== event.eventType) continue;+ const payload = row(envelope.payload), item = row(payload.item);+ const execution = typeof event.eventType === "string" && event.eventType.startsWith("tool.execution.");+ const callId = execution ? payload.executionId : item.id;+ if (typeof callId !== "string" || !callId) continue;+ const kind = feedbackKind(execution ? payload.name : item.name);+ const started = event.eventType === "tool.execution.started" || event.eventType === "item.started" && (payload.kind === "tool_call" || item.type === "tool_call");+ const completed = event.eventType === "tool.execution.completed" || event.eventType === "item.completed" && (payload.kind === "tool_result" || item.type === "tool_result");+ if ((execution || started || completed) && kind) calls.set(callId, calls.get(callId) ?? { runId, callId, kind, outcome: "unknown" });+ const call = calls.get(callId);+ if (!completed || !call || kind && kind !== call.kind) continue;+ const status = execution ? payload.status : item.status;+ if (payload.isError === true || item.isError === true || ["failed", "error", "cancelled"].includes(String(status))) call.outcome = "failed";+ else if (status === "completed") call.outcome = "completed";+ }+ return [...calls.values()];+}++export function feedbackReview(entries: readonly AgentCommentaryDetail[]): FeedbackSummary["review"] {+ const groups = new Map<string, FeedbackSummary["review"][number]>();+ for (const entry of entries) {+ const fingerprint = createHash("sha256").update(`${entry.kind}\n${entry.body.trim().replace(/\s+/g, " ")}`).digest("hex");+ const group = groups.get(fingerprint);+ if (group) group.occurrences++;+ else groups.set(fingerprint, { fingerprint, kind: entry.kind, occurrences: 1, disposition: "needs_investigation" });+ }+ return [...groups.values()];+}++export function summarizeCampaignFeedback(attempts: readonly { feedback?: FeedbackSummary }[]) {+ const review = new Map<string, FeedbackSummary["review"][number]>();+ let complaints = 0, suggestions = 0, incomplete = 0, failedSubmissions = 0, unknownSubmissions = 0;+ for (const { feedback } of attempts) {+ if (!feedback || feedback.collection !== "complete" || feedback.invocationCollection !== "complete") incomplete++;+ if (!feedback) continue;+ complaints += feedback.complaints;+ suggestions += feedback.suggestions;+ failedSubmissions += feedback.invocations.failed;+ unknownSubmissions += feedback.invocations.unknown;+ for (const group of feedback.review) {+ const previous = review.get(group.fingerprint);+ if (previous) previous.occurrences += group.occurrences;+ else review.set(group.fingerprint, { ...group });+ }+ }+ return { complaints, suggestions, incomplete, failedSubmissions, unknownSubmissions, review: [...review.values()] };+}++export async function collectAttemptFeedback(input: {+ api: Pick<RunnerApi, "get">;+ companyId: string;+ persist: (evidence: FeedbackEvidence, signal: AbortSignal) => Promise<void>;+ onCheckpoint?: (summary: FeedbackSummary) => void;+ deadlineAt?: number;+}): Promise<FeedbackSummary> {+ const deadlineAt = input.deadlineAt ?? Date.now() + 30_000;+ const readDeadlineAt = deadlineAt - Math.min(5_000, Math.max(0, Math.floor((deadlineAt - Date.now()) / 5)));+ const summary = emptyFeedbackSummary();+ let checkpointSummary = emptyFeedbackSummary();+ const retainedSummary = () => ({ ...structuredClone(checkpointSummary),+ ...(summary.collection !== "complete" ? { collection: summary.collection } : {}),+ ...(summary.invocationCollection !== "complete" ? { invocationCollection: summary.invocationCollection } : {}),+ });+ const evidence: FeedbackEvidence = { version: 1, companyId: input.companyId, entries: [], invocations: [], diagnostics: [], summary };+ let retainedBytes = 0, checkpointBytes = 0, eventBytes = 0, eventCount = 0;+ const diagnose = (code: FeedbackEvidence["diagnostics"][number]) => { if (!evidence.diagnostics.includes(code)) evidence.diagnostics.push(code); };+ const get = <T>(url: string) => {+ const remaining = readDeadlineAt - Date.now();+ if (remaining <= 0) { diagnose("deadline_reached"); throw new Error("Feedback collection deadline reached"); }+ return input.api.get<T>(url, { timeout: Math.min(5_000, remaining) });+ };+ const persist = async (force = false) => {+ summary.bodiesCollected = evidence.entries.length;+ summary.invocations = { completed: 0, failed: 0, unknown: 0 };+ for (const invocation of evidence.invocations) summary.invocations[invocation.outcome]++;+ summary.toolAvailability = evidence.invocations.length ? "observed" : "unknown";+ if (Date.now() >= deadlineAt) { diagnose("deadline_reached"); return false; }+ if (!force && (!retainedBytes || checkpointBytes > 0 && retainedBytes < Math.max(1024 * 1024, checkpointBytes * 2))) return true;+ summary.review = feedbackReview(evidence.entries);+ const remaining = deadlineAt - Date.now();+ if (remaining <= 0) { diagnose("deadline_reached"); return false; }+ const abort = new AbortController();+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ await Promise.race([+ new Promise<never>((_resolve, reject) => { timer = setTimeout(() => {+ abort.abort(); reject(new Error("Feedback persistence deadline reached"));+ }, remaining); }),+ input.persist(evidence, abort.signal),+ ]);+ if (Date.now() >= deadlineAt) { abort.abort(); diagnose("deadline_reached"); return false; }+ checkpointBytes = retainedBytes;+ checkpointSummary = structuredClone(summary);+ input.onCheckpoint?.(structuredClone(checkpointSummary));+ } finally { clearTimeout(timer); }+ return true;+ };+ if (!await persist(true)) {+ summary.collection = "unavailable";+ summary.invocationCollection = "unavailable";+ return summary;+ }+ const seen = new Set<string>();+ let expectedTotal: number | undefined;+ let detailsFailed = false;+ feedbackPages: for (let offset = 0; offset < FEEDBACK_COLLECTION_LIMITS.entries; offset += 100) {+ let page: AgentCommentaryPage;+ try {+ page = await get<AgentCommentaryPage>(`/api/companies/${input.companyId}/agent-commentary?limit=100&offset=${offset}`);+ if (!Array.isArray(page.items) || !Number.isSafeInteger(page.total) || page.total < 0 || page.items.length > 100) throw new Error("Invalid feedback page");+ } catch {+ summary.collection = seen.size ? "partial" : "unavailable";+ diagnose("feedback_read_failed");+ break;+ }+ expectedTotal ??= page.total;+ summary.collection = "partial";+ for (const entry of page.items) {+ if (Date.now() >= readDeadlineAt) { diagnose("deadline_reached"); break feedbackPages; }+ if (seen.has(entry.id)) continue;+ seen.add(entry.id);+ if (entry.kind === "complaint") summary.complaints++;+ else if (entry.kind === "suggestion") summary.suggestions++;+ else { detailsFailed = true; continue; }+ try {+ const detail = await get<AgentCommentaryDetail>(`/api/companies/${input.companyId}/agent-commentary/${encodeURIComponent(entry.id)}`);+ if (detail.id !== entry.id || detail.kind !== entry.kind || detail.runId !== entry.runId || typeof detail.body !== "string") throw new Error("Invalid feedback detail");+ const bytes = Buffer.byteLength(JSON.stringify(detail)) + 1;+ if (retainedBytes + bytes + 4 > FEEDBACK_COLLECTION_LIMITS.retainedBytes) { diagnose("evidence_limit_reached"); break feedbackPages; }+ evidence.entries.push(detail);+ retainedBytes += bytes;+ } catch {+ detailsFailed = true;+ diagnose("feedback_detail_failed");+ }+ if (evidence.entries.length % 10 === 0 && !await persist()) break feedbackPages;+ }+ if (!await persist()) break;+ if (offset + page.items.length >= page.total || page.items.length < 100) {+ if (seen.size === page.total && expectedTotal === page.total && !detailsFailed) summary.collection = "complete";+ else diagnose("feedback_pagination_incomplete");+ break;+ }+ if (offset + 100 >= FEEDBACK_COLLECTION_LIMITS.entries) diagnose("evidence_limit_reached");+ }+ if (!await persist(true)) summary.collection = evidence.entries.length ? "partial" : "unavailable";+ let runs: Array<{ id: string; status?: string }>;+ try {+ runs = await get(`/api/companies/${input.companyId}/heartbeat-runs?limit=1000`);+ if (!Array.isArray(runs) || runs.length > 1000 || runs.some(run => typeof run.id !== "string")) throw new Error("Invalid runs");+ } catch {+ summary.invocationCollection = "unavailable";+ diagnose("run_read_failed");+ await persist(true);+ return retainedSummary();+ }+ summary.invocationCollection = runs.length < 1000 ? "complete" : "partial";+ if (runs.length >= 1000) diagnose("run_limit_reached");+ for (const listed of runs) {+ let run: { id: string; status?: string; runtimeMode?: string } = listed;+ try {+ const detail = await get<typeof run>(`/api/heartbeat-runs/${encodeURIComponent(listed.id)}`);+ if (detail.id !== listed.id) throw new Error("Invalid run detail");+ run = detail;+ } catch {+ summary.invocationCollection = "partial";+ summary.collection = summary.collection === "complete" ? "partial" : summary.collection;+ diagnose("run_detail_failed");+ }+ if (!["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(run.status ?? "")) {+ summary.collection = summary.collection === "complete" ? "partial" : summary.collection;+ summary.invocationCollection = "partial";+ }+ if (run.runtimeMode !== "native") summary.invocationCollection = "partial";+ const events: Row[] = [];+ let afterSeq = 0;+ try {+ for (let pageNumber = 0; pageNumber < 100; pageNumber++) {+ const page = await get<Row[]>(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=100`);+ if (!Array.isArray(page) || page.length > 100) throw new Error("Invalid events");+ for (const event of page) {+ if (!Number.isSafeInteger(event.seq) || Number(event.seq) <= afterSeq) throw new Error("Invalid event sequence");+ const bytes = Buffer.byteLength(JSON.stringify(event));+ if (eventCount >= FEEDBACK_COLLECTION_LIMITS.events || eventBytes + bytes > FEEDBACK_COLLECTION_LIMITS.eventBytes) {+ diagnose("evidence_limit_reached"); throw new Error("Feedback event limit reached");+ }+ eventCount++; eventBytes += bytes;+ afterSeq = Number(event.seq);+ events.push(event);+ }+ if (page.length < 100) break;+ if (pageNumber === 99) { diagnose("evidence_limit_reached"); throw new Error("Event limit reached"); }+ }+ summary.runsInspected++;+ } catch {+ summary.invocationCollection = "partial";+ diagnose("events_read_failed");+ }+ for (const invocation of observeFeedbackInvocations(run.id, events)) {+ const bytes = Buffer.byteLength(JSON.stringify(invocation)) + 1;+ if (retainedBytes + bytes + 4 > FEEDBACK_COLLECTION_LIMITS.retainedBytes) { diagnose("evidence_limit_reached"); break; }+ retainedBytes += bytes;+ evidence.invocations.push(invocation);+ }+ if (!await persist() || evidence.diagnostics.includes("evidence_limit_reached")) {+ summary.invocationCollection = "partial";+ break;+ }+ }+ if (!await persist(true)) summary.invocationCollection = "partial";+ return retainedSummary();+}diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.tsindex 11c16a0792..f131ee9131 100644--- a/tests/runner-e2e/first-task-flow.ts+++ b/tests/runner-e2e/first-task-flow.ts@@ -276,6 +276,7 @@ export async function setupFirstTaskFixtures(input: {daytonaImage?: string;deadlineAt?: number;observe: (fixtures: LiveFixtureValues) => void;+ onCompanyCreated?: (companyId: string) => Promise<void>;}): Promise<FirstTaskFixtureValues> {const { page, api, execution, nonce } = input;await api.patch("/api/instance/settings/experimental", {@@ -296,6 +297,7 @@ export async function setupFirstTaskFixtures(input: {const companies = await api.get<Row[]>("/api/companies");const company = companies.find((c) => c.name === `First task ${nonce}`);if (!company) throw new Error("Onboarding fixture company missing");+ await input.onCompanyCreated?.(company.id);const fixtures = await provisionFirstTaskFixtures({api,execution,diff --git a/tests/runner-e2e/github-operating-usage.test.ts b/tests/runner-e2e/github-operating-usage.test.tsnew file mode 100644index 0000000000..dac23f7ba3--- /dev/null+++ b/tests/runner-e2e/github-operating-usage.test.ts@@ -0,0 +1,64 @@+import { expect, it } from "vitest";+import { readGitHubOperatingUsage } from "./github-operating-usage.js";++const now = Date.parse("2026-10-21T12:00:00Z"), sha = "a".repeat(40);+const run = { id: 123, repository: { id: 456, full_name: "paperclipai/paperclip" },+ path: ".github/workflows/runner-full-stack-e2e.yml", run_attempt: 1, status: "completed", conclusion: "failure", head_sha: sha,+ created_at: "2026-10-21T10:00:00Z", run_started_at: "2026-10-21T10:01:00Z", updated_at: "2026-10-21T10:10:00Z" };+const job = (id = 1) => ({ id, run_id: 123, run_attempt: 1, status: "completed", conclusion: "success", runner_id: 1,+ runner_name: "GitHub Actions 1", runner_group_name: "GitHub Actions", labels: ["ubuntu-latest"],+ started_at: "2026-10-21T10:01:00Z", completed_at: "2026-10-21T10:02:00Z", name: "unretained arbitrary user text" });+const artifact = { id: 9, workflow_run: { id: 123, head_sha: sha }, size_in_bytes: 1024, expired: false,+ created_at: "2026-10-21T10:03:00Z", expires_at: "2026-11-20T10:03:00Z", name: "unretained arbitrary artifact text" };+function api(overrides: Record<string, any> = {}) {+ const reads: string[] = [];+ return { reads, read: async (endpoint: string) => {+ reads.push(endpoint);+ if (endpoint === "repos/paperclipai/paperclip") return { id: 456, full_name: "paperclipai/paperclip", private: false, ...overrides.repo };+ if (endpoint.endsWith("/runs/123")) return { ...run, ...overrides.run, ...(reads.filter(row => row.endsWith("/runs/123")).length === 2 ? overrides.finalRun : {}) };+ if (endpoint.includes("/jobs?")) {+ const jobs = overrides.jobs ?? [job()];+ const page = Number(new URL(`https://api.github.com/${endpoint}`).searchParams.get("page"));+ return { total_count: jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100), ...overrides.jobPage };+ }+ if (endpoint.includes("/artifacts?")) return { total_count: 1, artifacts: [artifact], ...overrides.artifactPage };+ throw new Error("Unexpected read");+ } };+}++it("collects failed-run usage without equating job time, current rates or inventory with billed cost", async () => {+ const source = api();+ const usage = await readGitHubOperatingUsage("123", source.read, now);+ expect(usage).toMatchObject({ runAttempt: 1, sourceSha: sha, queueSeconds: 60, summedJobSeconds: 60,+ allStartedJobsStandardHosted: true, artifacts: { retainedBytes: 1024, lifetimeCoverage: "incomplete" },+ billing: { ciComputeUsd: null, artifactStorageUsd: null, publicStandardComputeRateUsdPerMinute: "0" } });+ expect(JSON.stringify(usage)).not.toContain("unretained");+ expect(source.reads).toHaveLength(5);+});++it("paginates all jobs and preserves nonstandard compute as unpriced", async () => {+ const jobs = Array.from({ length: 101 }, (_, index) => job(index + 1));+ jobs[100]!.runner_group_name = "custom-fleet";+ const source = api({ jobs });+ expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 6060,+ allStartedJobsStandardHosted: false, billing: { publicStandardComputeRateUsdPerMinute: null } });+ expect(source.reads.some(endpoint => endpoint.includes("page=2"))).toBe(true);+ expect((await readGitHubOperatingUsage("123", api({ repo: { private: true } }).read, now)).billing.publicStandardComputeRateUsdPerMinute).toBeNull();+});++it("rejects changing, truncated, duplicated, misattributed, rerun and uncompleted evidence", async () => {+ for (const override of [{ run: { run_attempt: 2 } }, { run: { status: "in_progress" } }, { run: { path: "other.yml" } },+ { finalRun: { head_sha: "b".repeat(40) } }, { jobPage: { total_count: 2 } }, { jobs: [job(), job()] },+ { jobs: [{ ...job(), run_id: 999 }] }, { jobs: [{ ...job(), completed_at: null }] }, { jobPage: { total_count: 1001 } },+ { artifactPage: { artifacts: [{ ...artifact, workflow_run: { id: 999, head_sha: sha } }] } },+ { artifactPage: { artifacts: [{ ...artifact, size_in_bytes: -1 }] } }]) {+ await expect(readGitHubOperatingUsage("123", api(override).read, now)).rejects.toThrow();+ }+});++it("accounts skipped jobs without requiring a fictitious VM and keeps expired artifact identity", async () => {+ const source = api({ jobs: [{ ...job(), conclusion: "skipped", runner_id: 0, started_at: null, completed_at: null }],+ artifactPage: { artifacts: [{ ...artifact, expired: true }] } });+ expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 0,+ jobs: [{ runner: "not_started" }], artifacts: { retainedBytes: 0, inventory: [{ id: 9, expired: true }] } });+});diff --git a/tests/runner-e2e/github-operating-usage.ts b/tests/runner-e2e/github-operating-usage.tsnew file mode 100644index 0000000000..e55ead9a50--- /dev/null+++ b/tests/runner-e2e/github-operating-usage.ts@@ -0,0 +1,100 @@+import { execFile } from "node:child_process";+import { promisify } from "node:util";++const repository = "paperclipai/paperclip";+type Json = Record<string, any>;+type ReadGitHub = (endpoint: string) => Promise<Json>;+const integer = (value: unknown) => Number.isSafeInteger(value) && Number(value) >= 0;+const timestamp = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value));++async function inventory(read: ReadGitHub, endpoint: string, field: string) {+ let total: number | undefined;+ const rows: Json[] = [], ids = new Set<number>();+ for (let page = 1; page <= 10; page++) {+ const response = await read(`${endpoint}${endpoint.includes("?") ? "&" : "?"}per_page=100&page=${page}`);+ if (!integer(response.total_count) || response.total_count > 1_000 || total !== undefined && total !== response.total_count+ || !Array.isArray(response[field]) || response[field].length > 100) throw new Error("Incomplete or changing GitHub inventory");+ total = response.total_count;+ for (const row of response[field]) {+ if (!row || !integer(row.id) || row.id < 1 || ids.has(row.id)) throw new Error("Invalid or duplicate GitHub inventory row");+ ids.add(row.id); rows.push(row);+ }+ if (rows.length === total) return rows;+ if (rows.length > total! || response[field].length < 100) throw new Error("Incomplete GitHub inventory");+ }+ throw new Error("GitHub inventory exceeds collection bound");+}++export async function readGitHubOperatingUsage(runId: string, read: ReadGitHub, now = Date.now()) {+ if (!/^[1-9][0-9]{0,15}$/.test(runId) || !Number.isSafeInteger(Number(runId)) || !Number.isFinite(now)) throw new Error("Invalid workflow run identity");+ const prefix = `repos/${repository}`;+ const repo = await read(prefix);+ const run = await read(`${prefix}/actions/runs/${runId}`);+ const validRun = (value: Json) => value.id === Number(runId) && value.repository?.id === repo.id+ && value.repository?.full_name === repository && value.path === ".github/workflows/runner-full-stack-e2e.yml"+ && value.run_attempt === 1 && value.status === "completed" && /^[a-f0-9]{40}$/.test(value.head_sha ?? "")+ && timestamp(value.created_at) && timestamp(value.run_started_at) && timestamp(value.updated_at)+ && Date.parse(value.created_at) <= Date.parse(value.run_started_at)+ && Date.parse(value.run_started_at) <= Date.parse(value.updated_at) && Date.parse(value.updated_at) <= now;+ if (!integer(repo.id) || repo.id < 1 || repo.full_name !== repository || typeof repo.private !== "boolean" || !validRun(run)) {+ throw new Error("Operating usage requires an identified, completed first-attempt Product E2E workflow");+ }+ const rawJobs = await inventory(read, `${prefix}/actions/runs/${runId}/attempts/1/jobs`, "jobs");+ if (!rawJobs.length) throw new Error("Missing workflow jobs");+ const jobs = rawJobs.map(job => {+ if (job.run_id !== run.id || job.run_attempt !== 1 || job.status !== "completed" || !integer(job.runner_id)+ || !Array.isArray(job.labels) || job.labels.some((label: unknown) => typeof label !== "string")) throw new Error("Invalid workflow job identity");+ const skipped = job.conclusion === "skipped" && job.runner_id === 0;+ if (!skipped && (!timestamp(job.started_at) || !timestamp(job.completed_at)+ || Date.parse(job.started_at) < Date.parse(run.created_at) || Date.parse(job.completed_at) > now+ || Date.parse(job.completed_at) < Date.parse(job.started_at))) throw new Error("Incomplete workflow job timing");+ const standard = job.runner_group_name === "GitHub Actions" && /^GitHub Actions [0-9]+$/.test(job.runner_name ?? "")+ && job.labels.length === 1 && job.labels[0] === "ubuntu-latest";+ return { id: job.id as number, runner: skipped ? "not_started" : standard ? "standard_github_hosted" : "other_or_unknown",+ elapsedSeconds: skipped ? 0 : (Date.parse(job.completed_at) - Date.parse(job.started_at)) / 1_000 };+ });+ const rawArtifacts = await inventory(read, `${prefix}/actions/runs/${runId}/artifacts`, "artifacts");+ const artifacts = rawArtifacts.map(artifact => {+ if (artifact.workflow_run?.id !== run.id || artifact.workflow_run?.head_sha !== run.head_sha || !integer(artifact.size_in_bytes)+ || typeof artifact.expired !== "boolean" || !timestamp(artifact.created_at) || !timestamp(artifact.expires_at)+ || Date.parse(artifact.created_at) < Date.parse(run.created_at) || Date.parse(artifact.created_at) > now+ || Date.parse(artifact.expires_at) < Date.parse(artifact.created_at)) throw new Error("Invalid workflow artifact identity or retention");+ return { id: artifact.id as number, sizeBytes: artifact.size_in_bytes as number,+ createdAt: artifact.created_at as string, expiresAt: artifact.expires_at as string, expired: artifact.expired as boolean };+ });+ const finalRun = await read(`${prefix}/actions/runs/${runId}`);+ if (!validRun(finalRun) || ["id", "head_sha", "run_attempt", "status", "updated_at", "conclusion"].some(key => finalRun[key] !== run[key])) {+ throw new Error("Workflow changed during operating usage collection");+ }+ const retainedBytes = artifacts.filter(row => !row.expired).reduce((sum, row) => sum + row.sizeBytes, 0);+ if (!Number.isSafeInteger(retainedBytes)) throw new Error("Artifact inventory byte count overflow");+ return {+ version: 1, source: "github_actions", repository, repositoryId: repo.id as number,+ observedAt: new Date(now).toISOString(), repositoryCurrentlyPublic: !repo.private,+ runId: run.id as number, runAttempt: 1, sourceSha: run.head_sha as string,+ workflowUrl: `https://github.com/${repository}/actions/runs/${runId}`,+ startedAt: run.run_started_at as string, completedSnapshotAt: run.updated_at as string,+ queueSeconds: (Date.parse(run.run_started_at) - Date.parse(run.created_at)) / 1_000,+ summedJobSeconds: jobs.reduce((sum, row) => sum + row.elapsedSeconds, 0),+ allStartedJobsStandardHosted: jobs.every(row => row.runner !== "other_or_unknown"), jobs,+ artifacts: { inventory: artifacts, retainedBytes, lifetimeCoverage: "incomplete" },+ billing: { ciComputeUsd: null, artifactStorageUsd: null, classification: "unknown",+ publicStandardComputeRateUsdPerMinute: !repo.private && jobs.every(row => row.runner !== "other_or_unknown") ? "0" : null,+ rateSource: "https://docs.github.com/en/billing/concepts/product-billing/github-actions",+ limitations: ["Current repository visibility does not prove visibility throughout the billing window.",+ "Job elapsed time is not VM billed lifetime; queue and summed job time are separate.",+ "Artifacts already deleted are absent; returned compressed bytes and expiry are an inventory, not billed byte-hours.",+ "Caches, logs, external S3 history, model calls, observability and shared subscriptions require separate evidence."] },+ };+}++export async function collectGitHubOperatingUsage(runId: string) {+ const run = promisify(execFile);+ return readGitHubOperatingUsage(runId, async endpoint => {+ try {+ const result = await run("gh", ["api", "--hostname", "github.com", "--method", "GET", endpoint],+ { timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: "utf8" });+ return JSON.parse(result.stdout) as Json;+ } catch { throw new Error("GitHub operating usage read failed; no retry or paid dispatch was attempted"); }+ });+}diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.tsindex c52a430276..2ea3cd91a4 100644--- a/tests/runner-e2e/harness-env.ts+++ b/tests/runner-e2e/harness-env.ts@@ -11,6 +11,9 @@ const AMBIENT_PAPERCLIP_CREDENTIAL_KEYS = ["PAPERCLIP_API_KEY","PAPERCLIP_AGENT_API_KEY","PAPERCLIP_TASK_BRIDGE_TOKEN",+ "PAPERCLIP_E2E_GATEWAY_TOKEN",+ "PAPERCLIP_E2E_GATEWAY_PHASE_PATH",+ "PAPERCLIP_E2E_OTLP_RELAY_URL","PAPERCLIP_SETUP_TOKEN","PAPERCLIP_SECRETS_MASTER_KEY","PAPERCLIP_SECRETS_MASTER_KEY_FILE",@@ -28,7 +31,25 @@ const AMBIENT_EXTERNAL_STATE_KEYS = ["PAPERCLIP_STORAGE_S3_PREFIX","PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE",] as const;-const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/;+const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|BRAINTRUST|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/;++export function applyEvalGatewayEnvironment(env: NodeJS.ProcessEnv, credential: string, gateway: {+ token: string; baseUrl: string; phasePath: string; observability?: { token: string; baseUrl: string };+}) {+ for (const name of Object.keys(env)) {+ if (CREDENTIAL_NAMES.includes(name as typeof CREDENTIAL_NAMES[number])+ || PROVIDER_SECRET_KEY.test(name) && /KEY|TOKEN|SECRET|PASSWORD/.test(name)) delete env[name];+ }+ delete env.PAPERCLIP_E2E_OTLP_RELAY_URL;+ env[credential] = gateway.token;+ env.PAPERCLIP_E2E_GATEWAY_BASE_URL = gateway.baseUrl;+ env.PAPERCLIP_E2E_GATEWAY_TOKEN = gateway.token;+ env.PAPERCLIP_E2E_GATEWAY_PHASE_PATH = gateway.phasePath;+ if (gateway.observability) {+ env.BRAINTRUST_API_KEY = gateway.observability.token;+ env.PAPERCLIP_E2E_OTLP_RELAY_URL = gateway.observability.baseUrl;+ }+}export function runnerE2EServerControlPaths(temporaryRoot: string) {const controlDirectory = path.join(temporaryRoot, "control");@@ -110,6 +131,8 @@ export function buildRunnerE2EProcessEnvironment(const installed = usesInstalledCli(source);if (installed) assertInstalledCliSelection(source, executions);const result = { ...source };+ delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH;+ delete result.PAPERCLIP_E2E_OTLP_RELAY_URL;// Announcements are unrelated to the scenarios and obscure screenshot evidence.result.PAPERCLIP_ANNOUNCEMENTS_ENABLED = "false";delete result.OPENCODE_ALLOW_ALL_MODELS;diff --git a/tests/runner-e2e/history-budget.test.ts b/tests/runner-e2e/history-budget.test.tsnew file mode 100644index 0000000000..3b6b1aa190--- /dev/null+++ b/tests/runner-e2e/history-budget.test.ts@@ -0,0 +1,170 @@+import { mkdir, mkdtemp, readFile, rm, truncate, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { afterEach, describe, expect, it, vi } from "vitest";+import { BoundedHistoryPublisher, checkHistoryPayload, HISTORY_LIMITS, validateBoundedHistoryRetention } from "./history-budget.js";+import { createBundleManifest, uploadImmutableBundle } from "./history-publish.js";+import { buildRunnerCampaign, campaignHistoryRecord, emptyRunnerHistory, mergeRunnerHistory } from "./history.js";++const roots: string[] = [];+async function temporary() {+ const root = await mkdtemp(path.join(os.tmpdir(), "history-budget-"));+ roots.push(root);+ return root;+}+afterEach(async () => {+ vi.restoreAllMocks();+ await Promise.all(roots.splice(0).map(root => rm(root, { force: true, recursive: true })));+});++function lifecycle() {+ return { Rules: [+ { Status: "Enabled", Filter: { Prefix: "" }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 7 } },+ { Status: "Enabled", Filter: { Prefix: "runner-e2e/bounded-v1/" }, Expiration: { Days: 30 },+ NoncurrentVersionExpiration: { NoncurrentDays: 1 }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } },+ { Status: "Enabled", Filter: { Prefix: "runner-e2e/bounded-v1/" }, Expiration: { ExpiredObjectDeleteMarker: true } },+ ] };+}+function providerError(code: string) { return Object.assign(new Error("provider failure"), { stderr: `An error occurred (${code})` }); }++describe("bounded public history", () => {+ it("rejects excess bytes, count, invalid sizes and duplicate paths", () => {+ expect(checkHistoryPayload([{ path: "a", bytes: 8 }], 8, 1, "report")).toEqual({ files: 1, bytes: 8 });+ for (const files of [[{ path: "a", bytes: 9 }], [{ path: "a", bytes: -1 }],+ [{ path: "a", bytes: 0 }, { path: "b", bytes: 0 }], [{ path: "a", bytes: 0 }, { path: "a", bytes: 0 }]]) {+ expect(() => checkHistoryPayload(files, 8, 1, "report")).toThrow();+ }+ });++ it("rejects oversized real bundle files before hashing them", async () => {+ const root = await temporary();+ await writeFile(path.join(root, "index.html"), "");+ await truncate(path.join(root, "index.html"), HISTORY_LIMITS.campaignBytes + 1);+ await expect(createBundleManifest(root, "oversized", false, new Set(), true)).rejects.toThrow("publication boundary");+ });++ it("uploads an allowlisted uppercase PNG with the same raster content type", async () => {+ const root = await temporary(); const bundle = path.join(root, "bundle");+ const screenshot = "evidence/execution/attempt-1/shot.PNG";+ await mkdir(path.dirname(path.join(bundle, screenshot)), { recursive: true });+ await writeFile(path.join(bundle, screenshot), Buffer.from("89504e470d0a1a0a0000000d494844520000000100000001", "hex"));+ const manifest = await createBundleManifest(bundle, "uppercase", false, new Set([screenshot]), true);+ const runner = vi.fn(async () => ({ stdout: "{}" }));+ await uploadImmutableBundle("bucket", "campaigns/uppercase", bundle, manifest,+ new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner));+ expect(runner).toHaveBeenCalledWith(expect.arrayContaining(["--content-type", "image/png", "--key", `campaigns/uppercase/${screenshot}`]), expect.any(Number));+ });++ it("requires expiration for both current and noncurrent versions in the isolated prefix", () => {+ expect(() => validateBoundedHistoryRetention(lifecycle(), "runner-e2e/bounded-v1")).not.toThrow();+ expect(() => validateBoundedHistoryRetention(lifecycle(), "runner-e2e")).toThrow("isolated");+ for (const mutate of [+ (config: ReturnType<typeof lifecycle>) => { config.Rules[1]!.Status = "Disabled"; },+ (config: ReturnType<typeof lifecycle>) => { config.Rules[1]!.NoncurrentVersionExpiration = undefined; },+ (config: ReturnType<typeof lifecycle>) => { Object.assign(config.Rules[1]!.NoncurrentVersionExpiration!, { NewerNoncurrentVersions: 10 }); },+ (config: ReturnType<typeof lifecycle>) => { Object.assign(config.Rules[1]!.Filter, { Tag: { Key: "optional", Value: "true" } }); },+ (config: ReturnType<typeof lifecycle>) => { config.Rules.pop(); },+ ]) {+ const config = lifecycle(); mutate(config);+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("expiry");+ }+ });++ it("rejects overlapping transitions or earlier expiration without affecting unrelated prefixes", () => {+ const config = lifecycle();+ Object.assign(config.Rules[0]!, { Transitions: [{ Days: 1, StorageClass: "GLACIER" }] });+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("overlapping");+ config.Rules[0]!.Filter.Prefix = "storybook/";+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).not.toThrow();+ Object.assign(config.Rules[2]!.Expiration!, { Days: 1 });+ expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("overlapping");+ });++ it("rebuilds pointers after removing old, future and excess history records", () => {+ const now = Date.parse("2026-10-21T12:00:00Z");+ const record = (id: string, age: number) => campaignHistoryRecord(buildRunnerCampaign({ campaignId: id,+ generatedAt: new Date(now - age * 86_400_000).toISOString(), expected: [], results: [] }), "https://example.test/bounded-v1");+ const old = { ...record("expired-green", 29), complete: true, failed: 0 };+ const current = { ...emptyRunnerHistory(), campaigns: [old, record("future", -2), record("yesterday", 1), record("earlier", 2)] };+ const result = mergeRunnerHistory(current, record("today", 0), { now, maxAgeMs: 28 * 86_400_000, maxCampaigns: 2 });+ expect(result.campaigns.map(row => row.campaignId)).toEqual(["today", "yesterday"]);+ expect(result.latestCampaignId).toBe("today");+ expect(result.latestGreenCampaignId).not.toBe("expired-green");+ expect(current.campaigns).toHaveLength(4);+ });++ it("distinguishes absent lock/replication from denied inspection", async () => {+ const root = await temporary();+ const runner = vi.fn(async (args: string[]) => {+ if (args[1] === "get-bucket-versioning") return { stdout: '{"Status":"Enabled"}' };+ if (args[1] === "get-bucket-lifecycle-configuration") return { stdout: JSON.stringify(lifecycle()) };+ throw providerError(args[1] === "get-object-lock-configuration" ? "ObjectLockConfigurationNotFoundError" : "ReplicationConfigurationNotFoundError");+ });+ await new BoundedHistoryPublisher(path.join(root, "allowed.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1");+ runner.mockImplementation(async () => { throw providerError("AccessDenied"); });+ await expect(new BoundedHistoryPublisher(path.join(root, "denied.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1")).rejects.toThrow();+ });++ it.each(["get-object-lock-configuration", "get-bucket-replication"])("rejects configured %s", async blocked => {+ const root = await temporary();+ const runner = async (args: string[]) => {+ if (args[1] === "get-bucket-versioning") return { stdout: '{"Status":"Enabled"}' };+ if (args[1] === "get-bucket-lifecycle-configuration") return { stdout: JSON.stringify(lifecycle()) };+ if (args[1] === blocked) return { stdout: "{}" };+ throw providerError("ObjectLockConfigurationNotFoundError");+ };+ await expect(new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1")).rejects.toThrow("cannot prove");+ });++ it("bounds history reads and preserves permission errors", async () => {+ const root = await temporary(); const destination = path.join(root, "download.json");+ const runner = vi.fn(async (args: string[]) => {+ expect(args).toContain(`bytes=0-${HISTORY_LIMITS.jsonBytes}`);+ await writeFile(destination, ""); await truncate(destination, HISTORY_LIMITS.jsonBytes + 1);+ return { stdout: "{}" };+ });+ const publisher = new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner);+ await expect(publisher.downloadJson("bucket", "history.json", destination)).rejects.toThrow("read boundary");+ runner.mockImplementation(async () => { throw providerError("NoSuchKey"); });+ await expect(publisher.downloadJson("bucket", "missing", destination)).resolves.toBeNull();+ runner.mockImplementation(async () => { throw providerError("AccessDenied"); });+ await expect(publisher.downloadJson("bucket", "denied", destination)).rejects.toThrow();+ });++ it("retains an attempted upload before dispatch and preserves unknown failures", async () => {+ const root = await temporary(); const receipt = path.join(root, "receipt.json"); const source = path.join(root, "source.json");+ await writeFile(source, "12345");+ const runner = vi.fn(async (args: string[], timeout: number) => {+ expect(args).toContain("--if-none-match"); expect(args).toContain("STANDARD"); expect(timeout).toBeLessThanOrEqual(30_000);+ expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ attemptedSourceBytes: 5, requests: [{ status: "unknown" }] });+ throw new Error("lost response");+ });+ const publisher = new BoundedHistoryPublisher(receipt, runner);+ await expect(publisher.upload("bucket", "campaign/a", source, "application/json", "no-cache", true)).rejects.toThrow("lost response");+ expect(runner).toHaveBeenCalledTimes(1);+ expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ billedUsd: null, attemptedSourceBytes: 5,+ requests: [{ operation: "put-object", status: "failed_or_unknown", sourceBytes: 5 }] });+ });++ it("stops before another command when the request or elapsed-time boundary is reached", async () => {+ const root = await temporary(); const runner = vi.fn(async () => ({ stdout: "{}" }));+ const publisher = new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner);+ for (let i = 0; i < HISTORY_LIMITS.requests; i++) await publisher.run(["s3api", "head-object"]);+ await expect(publisher.run(["s3api", "head-object"])).rejects.toThrow("allowance");+ expect(runner).toHaveBeenCalledTimes(HISTORY_LIMITS.requests);+ const timed = new BoundedHistoryPublisher(path.join(root, "timed.json"), runner);+ vi.spyOn(Date, "now").mockReturnValue(Date.now() + HISTORY_LIMITS.deadlineMs + 1);+ await expect(timed.run(["s3api", "head-object"])).rejects.toThrow("allowance");+ expect(runner).toHaveBeenCalledTimes(HISTORY_LIMITS.requests);+ });++ it("retains the object identity and provider version separately from billed cost", async () => {+ const root = await temporary(); const receipt = path.join(root, "receipt.json"); const source = path.join(root, "source.json");+ await writeFile(source, "{}");+ const publisher = new BoundedHistoryPublisher(receipt, async () => ({ stdout: '{"VersionId":"version-1","ETag":"etag-1"}' }));+ await publisher.upload("bucket", "campaign/result.json", source, "application/json", "no-cache", true);+ expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ billedUsd: null, requests: [{+ operation: "put-object", status: "accepted", bucket: "bucket", key: "campaign/result.json", versionId: "version-1", etag: "etag-1", sourceBytes: 2,+ }] });+ });+});diff --git a/tests/runner-e2e/history-budget.ts b/tests/runner-e2e/history-budget.tsnew file mode 100644index 0000000000..b7ffdf89d1--- /dev/null+++ b/tests/runner-e2e/history-budget.ts@@ -0,0 +1,177 @@+import { execFile } from "node:child_process";+import { open, readFile, rename, stat } from "node:fs/promises";+import { promisify } from "node:util";++export const HISTORY_LIMITS = Object.freeze({+ campaignBytes: 64 * 1024 * 1024,+ campaignFiles: 256,+ mutableBytes: 16 * 1024 * 1024,+ pagesBytes: 64 * 1024 * 1024,+ jsonBytes: 4 * 1024 * 1024,+ indexDays: 28,+ indexCampaigns: 64,+ requests: 280,+ deadlineMs: 10 * 60_000,+});++export function checkHistoryPayload(+ files: readonly { path: string; bytes: number }[],+ maxBytes: number,+ maxFiles: number,+ label: string,+) {+ let bytes = 0;+ const paths = new Set<string>();+ for (const file of files) {+ if (paths.has(file.path) || !Number.isSafeInteger(file.bytes) || file.bytes < 0) {+ throw new Error(`${label} contains invalid or duplicate file metadata`);+ }+ paths.add(file.path);+ bytes += file.bytes;+ }+ if (!Number.isSafeInteger(bytes) || bytes > maxBytes || files.length > maxFiles) {+ throw new Error(`${label} exceeds the publication boundary (${files.length} files, ${bytes} bytes)`);+ }+ return { files: files.length, bytes };+}++interface LifecycleRule {+ Status?: string;+ Prefix?: string;+ Filter?: { Prefix?: string; And?: { Prefix?: string } };+ Expiration?: { Days?: number; Date?: string; ExpiredObjectDeleteMarker?: boolean };+ NoncurrentVersionExpiration?: { NoncurrentDays?: number; NewerNoncurrentVersions?: number };+ AbortIncompleteMultipartUpload?: { DaysAfterInitiation?: number };+ Transitions?: unknown[];+ NoncurrentVersionTransitions?: unknown[];+}++export function validateBoundedHistoryRetention(configuration: unknown, prefix: string) {+ const rules = (configuration as { Rules?: LifecycleRule[] })?.Rules;+ if (!Array.isArray(rules) || !prefix.endsWith("/bounded-v1")) {+ throw new Error("Bounded history requires an isolated bounded-v1 prefix and lifecycle rules");+ }+ const scoped = rules.filter(rule => rule.Status === "Enabled" &&+ rule.Filter?.Prefix === `${prefix}/` && Object.keys(rule.Filter).length === 1);+ const retention = scoped.some(rule => rule.Expiration?.Days === 30 &&+ rule.NoncurrentVersionExpiration?.NoncurrentDays === 1 &&+ rule.NoncurrentVersionExpiration?.NewerNoncurrentVersions === undefined &&+ rule.AbortIncompleteMultipartUpload?.DaysAfterInitiation === 1);+ const markers = scoped.some(rule => rule.Expiration?.ExpiredObjectDeleteMarker === true);+ if (!retention || !markers) {+ throw new Error("Bounded history needs 30-day current, 1-day noncurrent, delete-marker and multipart expiry");+ }+ for (const rule of rules.filter(rule => rule.Status === "Enabled")) {+ const filterPrefix = rule.Filter?.Prefix ?? rule.Filter?.And?.Prefix ?? rule.Prefix ?? "";+ if (!`${prefix}/`.startsWith(filterPrefix) && !filterPrefix.startsWith(`${prefix}/`)) continue;+ if (rule.Transitions?.length || rule.NoncurrentVersionTransitions?.length ||+ (rule.Expiration && !scoped.includes(rule))) {+ throw new Error("An overlapping lifecycle rule changes bounded history retention or storage class");+ }+ if (rule.Expiration?.Days !== undefined && rule.Expiration.Days !== 30 || rule.Expiration?.Date) {+ throw new Error("An overlapping lifecycle rule changes bounded history expiration");+ }+ }+}++type AwsRunner = (args: string[], timeout: number) => Promise<{ stdout: string }>;+const execFileAsync = promisify(execFile);+const defaultRunner: AwsRunner = async (args, timeout) => execFileAsync("aws", args, {+ timeout, maxBuffer: 1024 * 1024,+ env: { ...process.env, AWS_MAX_ATTEMPTS: "1", AWS_RETRY_MODE: "standard", AWS_PAGER: "" },+});++function errorDetail(error: unknown) {+ return String((error as { stderr?: string })?.stderr ?? (error instanceof Error ? error.message : error));+}++export class BoundedHistoryPublisher {+ private readonly startedAt = Date.now();+ private readonly observations: Array<{ operation: string; status: string; bucket?: string; key?: string; sourceBytes?: number; versionId?: string; etag?: string }> = [];++ constructor(private readonly receipt: string, private readonly runner: AwsRunner = defaultRunner) {}++ async run(args: string[], sourceBytes?: number) {+ const remaining = HISTORY_LIMITS.deadlineMs - (Date.now() - this.startedAt);+ if (remaining <= 0 || this.observations.length >= HISTORY_LIMITS.requests) {+ throw new Error("Bounded history publisher exhausted its time or request allowance");+ }+ const argument = (name: string) => args.includes(name) ? args[args.indexOf(name) + 1] : undefined;+ const observation: typeof this.observations[number] = { operation: args[1]!, status: "unknown",+ bucket: argument("--bucket"), key: argument("--key"), ...(sourceBytes === undefined ? {} : { sourceBytes }) };+ this.observations.push(observation);+ await this.save();+ try {+ const result = await this.runner([...args, "--cli-connect-timeout", "10", "--cli-read-timeout", "30"], Math.min(30_000, remaining));+ observation.status = "accepted";+ if (observation.operation === "put-object") {+ const response = JSON.parse(result.stdout) as { VersionId?: unknown; ETag?: unknown };+ if (typeof response.VersionId === "string") observation.versionId = response.VersionId.slice(0, 1024);+ if (typeof response.ETag === "string") observation.etag = response.ETag.slice(0, 1024);+ }+ return result;+ } catch (error) {+ observation.status = /\((?:NoSuchKey|404|NotFound|ObjectLockConfigurationNotFoundError|ReplicationConfigurationNotFoundError)\)/.test(errorDetail(error))+ ? "absent" : "failed_or_unknown";+ throw error;+ } finally {+ await this.save();+ }+ }++ async save() {+ const file = await open(`${this.receipt}.tmp`, "w", 0o600);+ try {+ await file.writeFile(`${JSON.stringify({+ schema: "paperclip.runner-e2e.history-publication/v1",+ startedAt: new Date(this.startedAt).toISOString(), observedAt: new Date().toISOString(),+ workflowRunId: process.env.GITHUB_RUN_ID ?? null, workflowAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,+ limits: HISTORY_LIMITS, billedUsd: null, requests: this.observations,+ attemptedSourceBytes: this.observations.reduce((sum, entry) => sum + (entry.sourceBytes ?? 0), 0),+ }, null, 2)}\n`);+ await file.sync();+ } finally {+ await file.close();+ }+ await rename(`${this.receipt}.tmp`, this.receipt);+ }++ async verifyRetention(bucket: string, prefix: string) {+ const config = async (operation: string) => JSON.parse((await this.run(["s3api", operation, "--bucket", bucket])).stdout);+ if ((await config("get-bucket-versioning")).Status !== "Enabled") {+ throw new Error("Bounded history requires versioning-enabled storage");+ }+ validateBoundedHistoryRetention(await config("get-bucket-lifecycle-configuration"), prefix);+ for (const [operation, absence] of [+ ["get-object-lock-configuration", "ObjectLockConfigurationNotFoundError"],+ ["get-bucket-replication", "ReplicationConfigurationNotFoundError"],+ ]) {+ try {+ await config(operation!);+ } catch (error) {+ if (errorDetail(error).includes(`(${absence})`)) continue;+ throw error;+ }+ throw new Error(`Bounded history cannot prove expiration with ${operation} configured`);+ }+ }++ async downloadJson<T>(bucket: string, key: string, destination: string): Promise<T | null> {+ try {+ await this.run(["s3api", "get-object", "--bucket", bucket, "--key", key,+ "--range", `bytes=0-${HISTORY_LIMITS.jsonBytes}`, destination]);+ } catch (error) {+ if (/\((?:NoSuchKey|404|NotFound)\)/.test(errorDetail(error))) return null;+ throw error;+ }+ if ((await stat(destination)).size > HISTORY_LIMITS.jsonBytes) throw new Error("History JSON exceeds the read boundary");+ return JSON.parse(await readFile(destination, "utf8")) as T;+ }++ async upload(bucket: string, key: string, file: string, contentType: string, cacheControl: string, immutable = false) {+ const bytes = (await stat(file)).size;+ return this.run(["s3api", "put-object", "--bucket", bucket, "--key", key, "--body", file,+ "--content-length", String(bytes), "--content-type", contentType, "--cache-control", cacheControl,+ "--storage-class", "STANDARD", ...(immutable ? ["--if-none-match", "*"] : [])], bytes);+ }+}diff --git a/tests/runner-e2e/history-index.ts b/tests/runner-e2e/history-index.tsindex 5c72cb9dde..2845cc834f 100644--- a/tests/runner-e2e/history-index.ts+++ b/tests/runner-e2e/history-index.ts@@ -25,7 +25,8 @@ function usd(value: number | null) {return `$${value.toFixed(value < 0.01 ? 6 : 2)}`;}-function duration(durationMs: number) {+function duration(durationMs: number | null) {+ if (durationMs === null) return "Unknown";const seconds = Math.round(durationMs / 1_000);if (seconds < 60) return `${seconds}s`;const minutes = Math.floor(seconds / 60);diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.tsindex 80159a5f0a..a39dee6ab7 100644--- a/tests/runner-e2e/history-publish.ts+++ b/tests/runner-e2e/history-publish.ts@@ -1,4 +1,6 @@-import { validateRetainedRunnerResult } from "./result-validation.js";+import { validateCampaignAccounting, validateRetainedRunnerResult } from "./result-validation.js";+import { validateEvalOperationsReport } from "./operations-report.js";+import { validateCampaignFeedbackReview } from "./feedback-dispositions.js";import { createHash } from "node:crypto";import { execFile } from "node:child_process";import path from "node:path";@@ -22,6 +24,7 @@ import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";import { renderRunnerHistoryIndex } from "./history-index.js";import { validateHistoryDestination } from "./history-destination.js";import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js";+import { BoundedHistoryPublisher, checkHistoryPayload, HISTORY_LIMITS } from "./history-budget.js";import {campaignHistoryRecord,emptyRunnerHistory,@@ -277,10 +280,12 @@ async function relativeFiles(current = root,allowPublicSummary = false,publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,+ boundary?: { entries: number },): Promise<string[]> {const entries = await readdir(current, { withFileTypes: true });const files: string[] = [];for (const entry of entries) {+ if (boundary && ++boundary.entries > 1024) throw new Error("Public bundle exceeds the directory-entry boundary");const absolute = path.join(current, entry.name);if (entry.isSymbolicLink()) {throw new Error(`Refusing to publish symbolic link ${entry.name}`);@@ -292,6 +297,7 @@ async function relativeFiles(absolute,allowPublicSummary,publicScreenshots,+ boundary,)),);} else if (entry.isFile()) {@@ -319,11 +325,17 @@ export async function createBundleManifest(campaignId: string,allowPublicSummary = false,publicScreenshots: ReadonlySet<string> = EMPTY_PUBLIC_SCREENSHOTS,+ bounded = false,): Promise<BundleManifest> {if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(campaignId)) {throw new Error("Campaign ID is unsafe for immutable object storage");}- const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots);+ const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots, bounded ? { entries: 0 } : undefined);+ if (bounded) {+ checkHistoryPayload(await Promise.all(retainedPaths.map(async file => ({+ path: file, bytes: (await stat(path.join(root, file))).size,+ }))), HISTORY_LIMITS.campaignBytes, HISTORY_LIMITS.campaignFiles, "Public campaign");+ }const retained = new Set(retainedPaths);const missingScreenshots = [...publicScreenshots].filter((relative) => !retained.has(relative));if (missingScreenshots.length > 0) {@@ -431,7 +443,9 @@ async function downloadJson<T>(bucket: string,key: string,destination: string,+ bounded?: BoundedHistoryPublisher,) {+ if (bounded) return bounded.downloadJson<T>(bucket, key, destination);if (!(await objectExists(bucket, key))) return null;await execFileAsync("aws", ["s3",@@ -472,11 +486,24 @@ async function uploadFile(]);}-async function uploadImmutableBundle(+export async function uploadImmutableBundle(bucket: string,key: string,directory: string,+ manifest?: BundleManifest,+ bounded?: BoundedHistoryPublisher,) {+ if (bounded && manifest) {+ for (const file of manifest.files) {+ const extension = path.extname(file.path).toLowerCase();+ const contentType = ({ ".png": "image/png", ".json": "application/json", ".html": "text/html; charset=utf-8",+ ".woff2": "font/woff2", ".xml": "application/xml", ".md": "text/plain; charset=utf-8" } as Record<string, string>)[extension];+ if (!contentType) throw new Error(`Missing public content type for ${file.path}`);+ await bounded.upload(bucket, `${key}/${file.path}`, path.join(directory, file.path), contentType,+ "public,max-age=2592000,immutable", true);+ }+ return;+ }const common = ["--recursive","--only-show-errors",@@ -524,10 +551,19 @@ async function main() {prefix: process.env.RUNNER_E2E_HISTORY_PREFIX ?? "runner-e2e",publicBaseUrl: process.env.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL ?? "",});+ const bounded = process.env.PAPERCLIP_E2E_BOUNDED_HISTORY === "true";+ if (bounded && process.env.GITHUB_ACTIONS === "true" && process.env.GITHUB_RUN_ATTEMPT !== "1") {+ throw new Error("Bounded history publication requires the original workflow attempt; retain failed publication evidence");+ }+ const publisher = bounded ? new BoundedHistoryPublisher(path.join(path.dirname(reportRoot), "history-publication-receipt.json")) : undefined;+ if (publisher) await publisher.verifyRetention(bucket, destination.prefix);const campaign = JSON.parse(await readFile(path.join(reportRoot, "normalized-results.json"), "utf8"),) as RunnerE2ECampaign;for (const result of campaign.results ?? []) validateRetainedRunnerResult(result);+ if (campaign.accounting) validateCampaignAccounting(campaign.accounting);+ if (campaign.operations) validateEvalOperationsReport(campaign.operations);+ if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview);assertPublicCampaign(campaign);if (campaign.schema !== "paperclip.runner-e2e.campaign/v2") {throw new Error("Historical publishing requires a v2 normalized campaign");@@ -542,6 +578,7 @@ async function main() {bucket,historyKey,path.join(temporary, "current-history.json"),+ publisher,)) ?? emptyRunnerHistory();const history = mergeRunnerHistory(current,@@ -549,7 +586,11 @@ async function main() {campaign,`${destination.publicBaseUrl}/${destination.prefix}`,),+ bounded ? { now: Date.now(), maxAgeMs: HISTORY_LIMITS.indexDays * 86_400_000, maxCampaigns: HISTORY_LIMITS.indexCampaigns } : undefined,);+ if (bounded && !history.campaigns.some(entry => entry.campaignId === campaign.campaignId)) {+ throw new Error("Bounded campaign is outside the retained history window");+ }const publicScreenshots = publicScreenshotPaths(campaign);// Campaign bundles are immutable and must not capture a mutable history@@ -581,6 +622,7 @@ async function main() {campaign.campaignId,true,publicScreenshots,+ bounded,);const campaignPrefix = `${destination.prefix}/campaigns/${campaign.campaignId}`;const manifestKey = `${campaignPrefix}/bundle-manifest.json`;@@ -588,6 +630,7 @@ async function main() {bucket,manifestKey,path.join(temporary, "existing-manifest.json"),+ publisher,);if (existingManifest &&@@ -597,17 +640,8 @@ async function main() {`Immutable campaign ${campaign.campaignId} already exists with a different digest`,);}- if (!existingManifest) {- await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot);- const manifestFile = path.join(temporary, "bundle-manifest.json");- await writeFile(manifestFile, json(manifest), "utf8");- await uploadJson(- bucket,- manifestKey,- manifestFile,- "public,max-age=31536000,immutable",- );- }+ const manifestFile = path.join(temporary, "bundle-manifest.json");+ await writeFile(manifestFile, json(manifest), "utf8");// Pages uses the same declared-screenshot boundary in a sibling stage. The// downloaded access-controlled report stays intact for this whole job.@@ -630,11 +664,12 @@ async function main() {writeFile(latestGreenFile, json(pointers.latestGreen), "utf8"),]);await regenerateRunnerDashboard({ bundle: pagesRoot, historyFile });- await createBundleManifest(+ const pagesManifest = await createBundleManifest(pagesRoot,campaign.campaignId,false,publicScreenshots,+ bounded,);const landingDirectory = path.join(temporary, "landing");await regenerateRunnerDashboard({@@ -648,6 +683,37 @@ async function main() {renderRunnerHistoryIndex(history),"utf8",);+ if (publisher) {+ const mutableFiles = [+ { key: historyKey, file: historyFile, contentType: "application/json" },+ { key: `${destination.prefix}/latest.json`, file: latestFile, contentType: "application/json" },+ { key: `${destination.prefix}/latest-green.json`, file: latestGreenFile, contentType: "application/json" },+ ...["index.html", "dashboard.html", "normalized-results.json"].map(file => ({ key: `${destination.prefix}/${file}`,+ file: path.join(landingDirectory, file), contentType: file.endsWith("json") ? "application/json" : "text/html; charset=utf-8" })),+ ...["favicon-32x32.png", "InterVariable.woff2"].map(file => ({ key: `${destination.prefix}/assets/${file}`,+ file: path.join(s3ReportRoot, "assets", file), contentType: file.endsWith("png") ? "image/png" : "font/woff2" })),+ ];+ const metadata = async (file: string) => ({ path: file, bytes: (await stat(file)).size });+ const campaignSize = checkHistoryPayload([...manifest.files, await metadata(manifestFile)],+ HISTORY_LIMITS.campaignBytes, HISTORY_LIMITS.campaignFiles, "Public campaign with manifest");+ const mutableMetadata = await Promise.all(mutableFiles.map(file => metadata(file.file)));+ const mutableSize = checkHistoryPayload(mutableMetadata, HISTORY_LIMITS.mutableBytes, 8, "Mutable history");+ const pointerMetadata = await Promise.all([historyFile, latestFile, latestGreenFile].map(metadata));+ checkHistoryPayload(pointerMetadata, HISTORY_LIMITS.jsonBytes, 3, "History JSON");+ const pagesSize = checkHistoryPayload([...pagesManifest.files, ...pointerMetadata],+ HISTORY_LIMITS.pagesBytes, HISTORY_LIMITS.campaignFiles + 3, "Pages bundle");+ if (!existingManifest) {+ await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot, manifest, publisher);+ await publisher.upload(bucket, manifestKey, manifestFile, "application/json", "public,max-age=2592000,immutable", true);+ }+ for (const file of mutableFiles) await publisher.upload(bucket, file.key, file.file, file.contentType, "no-cache");+ console.log(JSON.stringify({ campaign: campaignSize, mutable: mutableSize, pages: pagesSize, billedUsd: null }));+ return;+ }+ if (!existingManifest) {+ await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot);+ await uploadJson(bucket, manifestKey, manifestFile, "public,max-age=31536000,immutable");+ }await Promise.all([uploadJson(bucket, historyKey, historyFile, "no-cache"),uploadJson(diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.tsindex c60278769d..c717441e32 100644--- a/tests/runner-e2e/history.test.ts+++ b/tests/runner-e2e/history.test.ts@@ -69,6 +69,41 @@ function result(execution: MatrixExecution, status: "passed" | "failed") {}describe("runner E2E campaign history", () => {+ it("retains the supervisor trace outcome independently of the worker export through regeneration", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-finalized-trace-"));+ temporaryDirectories.push(root);+ const execution = runnerMatrix[0]!;+ const attempt: RunnerE2EResult = { ...result(execution, "failed"), failureClass: "permanent_infrastructure",+ tracing: { projectId: "11111111-1111-4111-8111-111111111111", traceId: "a".repeat(32), spanId: "b".repeat(16),+ status: "accepted", verification: "unverified", nativeConfiguration: "configured",+ finalization: { spanId: "c".repeat(16), status: "accepted", outcome: "failed", verification: "unverified" } },+ };+ const campaign = buildRunnerCampaign({ campaignId: "finalized-trace", generatedAt: attempt.finishedAt,+ expected: [execution.id], results: [attempt], attempts: [attempt] });+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });+ const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));+ expect(rebuilt.results[0].tracing).toEqual(attempt.tracing);+ expect(rebuilt.accounting.attempts[0].tracing).toEqual(attempt.tracing);+ expect(rebuilt.failed).toBe(1);+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("Supervisor outcome: failed; finalization export: accepted");+ });++ it("preserves all-attempt costs when regenerating a retained report", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-attempt-costs-"));+ temporaryDirectories.push(root);+ const execution = runnerMatrix[0]!;+ const first = result(execution, "failed");+ const last = { ...result(execution, "passed"), attempt: 2, usage: { inputTokens: 200, outputTokens: 50, costUsd: 0.02 } };+ const campaign = buildRunnerCampaign({ campaignId: "retries", generatedAt: last.finishedAt,+ expected: [execution.id], results: [last], attempts: [first, last] });+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));+ await regenerateRunnerDashboard({ bundle: root });+ const regenerated = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));+ expect(regenerated.accounting).toEqual(campaign.accounting);+ expect(regenerated.billing.reportedLlmCostUsd).toBe(0.03);+ expect(regenerated.results[0].billing.reportedCostUsd).toBe(0.02);+ });it("does not let empty legacy provenance hide a later measured source", async () => {const root = await mkdtemp(path.join(os.tmpdir(), "runner-legacy-source-"));temporaryDirectories.push(root);diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.tsindex 90476f4ee8..822bd81b7e 100644--- a/tests/runner-e2e/history.ts+++ b/tests/runner-e2e/history.ts@@ -1,7 +1,11 @@import { isIncompleteFirstTaskResult } from "./case-outcome.js";+import { buildCampaignAccounting } from "./attempt-accounting.js";+import { validateEvalOperationsReport } from "./operations-report.js";+import { validateCampaignFeedbackReview } from "./feedback-dispositions.js";+import { validateBraintrustExperimentReceipt } from "./braintrust-experiment.js";import { classifyLatency } from "./latency-timing.js";import { summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js";-import { validateRetainedRunnerResult } from "./result-validation.js";+import { validateCampaignAccounting, validateRetainedRunnerResult } from "./result-validation.js";import {discoverReportCatalog,parseReportExecutionId,@@ -9,12 +13,13 @@ import {} from "./report-catalog.js";import { runnerMatrix, runnerSuites } from "./catalog.js";import {- aggregateCampaignBilling,+ aggregateBillingSummaries,summarizeExecutionBilling,} from "./billing.js";import { resolveRunnerE2ESource } from "./source.js";import type {RunnerE2ECampaign,+ RunnerE2ECampaignAccounting,RunnerE2EHistoryCampaign,RunnerE2EHistoryIndex,RunnerE2EResult,@@ -71,6 +76,12 @@ export function buildRunnerCampaign(input: {generatedAt: string;expected: readonly string[];results: readonly RunnerE2EResult[];+ attempts?: readonly RunnerE2EResult[];+ accounting?: RunnerE2ECampaignAccounting;+ operations?: RunnerE2ECampaign["operations"];+ feedbackReview?: RunnerE2ECampaign["feedbackReview"];+ braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"];+ excludedArtifacts?: number;eventName?: string | null;catalog?: readonly ReportExecution[];}): RunnerE2ECampaign {@@ -79,6 +90,28 @@ export function buildRunnerCampaign(input: {...upgradeRunnerResult(result),billing: result.billing ?? summarizeExecutionBilling(result),}));+ const accounting = input.accounting ?? buildCampaignAccounting({+ results,+ attempts: input.attempts?.map(upgradeRunnerResult),+ excludedArtifacts: input.excludedArtifacts,+ });+ validateCampaignAccounting(accounting);+ if (input.operations) validateEvalOperationsReport(input.operations);+ const billingFor = (selectedResults: readonly RunnerE2EResult[]) => {+ const ids = new Set(selectedResults.map(result => result.executionId));+ const receipts = accounting.attempts.filter(attempt => ids.has(attempt.executionId));+ const billing = aggregateBillingSummaries([+ ...receipts,+ ...selectedResults.filter(result => !receipts.some(attempt => attempt.executionId === result.executionId))+ .map(result => ({ executionId: result.executionId, billing: summarizeExecutionBilling(result) })),+ ]);+ billing.testsWithCompleteBilling = selectedResults.filter(result => {+ const retained = receipts.filter(attempt => attempt.executionId === result.executionId);+ return retained.length > 0 && retained.every(attempt => attempt.billing.complete) &&+ new Set(retained.map(attempt => attempt.attempt)).size >= Math.max(result.attempt, ...retained.map(attempt => attempt.attempt));+ }).length;+ return billing;+ };const knownCatalog = input.catalog ?? runnerMatrix;const catalog = discoverReportCatalog({catalog: knownCatalog,@@ -150,7 +183,7 @@ export function buildRunnerCampaign(input: {(total, result) => total + result.durationMs,0,),- billing: aggregateCampaignBilling(suiteResults),+ billing: billingFor(suiteResults),};}).filter((suite): suite is NonNullable<typeof suite> => Boolean(suite));@@ -175,7 +208,7 @@ export function buildRunnerCampaign(input: {),).values(),];- return {+ const campaign: RunnerE2ECampaign = {schema: "paperclip.runner-e2e.campaign/v2",campaignId: input.campaignId,generatedAt: input.generatedAt,@@ -195,10 +228,17 @@ export function buildRunnerCampaign(input: {),cleanupPassed: results.every((result) => result.cleanup === "passed"),rankingSnapshots,- billing: aggregateCampaignBilling(results),+ billing: billingFor(results),+ accounting,+ ...(input.operations ? { operations: input.operations } : {}),+ ...(input.feedbackReview ? { feedbackReview: input.feedbackReview } : {}),+ ...(input.braintrustExperiment ? { braintrustExperiment: input.braintrustExperiment } : {}),suites,results,};+ if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview);+ if (campaign.braintrustExperiment) validateBraintrustExperimentReceipt(campaign, campaign.braintrustExperiment);+ return campaign;}export function campaignHistoryRecord(@@ -253,6 +293,7 @@ export function emptyRunnerHistory(): RunnerE2EHistoryIndex {export function mergeRunnerHistory(current: RunnerE2EHistoryIndex | null | undefined,campaign: RunnerE2EHistoryCampaign,+ retention?: { now: number; maxAgeMs: number; maxCampaigns: number },): RunnerE2EHistoryIndex {const base =current?.schema === "paperclip.runner-e2e.history/v1"@@ -263,10 +304,14 @@ export function mergeRunnerHistory(...base.campaigns.filter((candidate) => candidate.campaignId !== campaign.campaignId,),- ].sort(+ ].filter(candidate => !retention || (+ Number.isFinite(Date.parse(candidate.generatedAt)) &&+ Date.parse(candidate.generatedAt) > retention.now - retention.maxAgeMs &&+ Date.parse(candidate.generatedAt) <= retention.now + 60_000+ )).sort((left, right) =>Date.parse(right.generatedAt) - Date.parse(left.generatedAt),- );+ ).slice(0, retention?.maxCampaigns);const latestBySuite: Record<string, string> = {};const latestGreenBySuite: Record<string, string> = {};for (const candidate of campaigns) {@@ -279,7 +324,7 @@ export function mergeRunnerHistory(}return {schema: "paperclip.runner-e2e.history/v1",- updatedAt: new Date().toISOString(),+ updatedAt: new Date(retention?.now ?? Date.now()).toISOString(),latestCampaignId: campaigns[0]?.campaignId ?? null,latestGreenCampaignId:campaigns.find(diff --git a/tests/runner-e2e/ingestion-usage.test.ts b/tests/runner-e2e/ingestion-usage.test.tsnew file mode 100644index 0000000000..fdb327f454--- /dev/null+++ b/tests/runner-e2e/ingestion-usage.test.ts@@ -0,0 +1,108 @@+import { createHash } from "node:crypto";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { afterEach, expect, it } from "vitest";+import { openEvalCostLedger } from "./cost-ledger.js";+import { startEvalOtlpRelay } from "./eval-otlp-relay.js";+import { EVAL_FINALIZATION_LIMITS } from "./eval-finalization-relay.js";+import { openExperimentIngestion } from "./experiment-ingestion.js";+import { collectScheduledIngestion } from "./ingestion-usage.js";++const projectId = "11111111-1111-4111-8111-111111111111";+const roots: string[] = [];+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });+async function fixture() {+ const root = await mkdtemp(path.join(tmpdir(), "scheduled-ingestion-")); roots.push(root);+ const input = { ledgerPath: path.join(root, "ledger.jsonl"), campaignId: "campaign", projectId, executionIds: ["fixture.local"] };+ const ledger = await openEvalCostLedger(input.ledgerPath, { campaignId: input.campaignId, maxUsd: "1",+ deadlineAt: new Date(Date.now() + 60_000).toISOString(), maxRequests: 1, maxConcurrent: 1, maxAttemptsPerExecution: 1 });+ try { await ledger.admitAttempt(input.executionIds[0]!, "attempt"); } finally { await ledger.close(); }+ const name = createHash("sha256").update("attempt").digest("hex"), files: string[] = [];+ for (const kind of ["native", "finalization"] as const) {+ const receiptPath = path.join(`${input.ledgerPath}.otlp`, `${name}${kind === "finalization" ? "-finalization" : ""}.jsonl`);+ files.push(receiptPath);+ const relay = await startEvalOtlpRelay({ ...input, attemptId: "attempt", apiKey: "private-fixture", receiptPath,+ deadlineAt: Date.now() + 60_000, ...(kind === "finalization" ? { limits: EVAL_FINALIZATION_LIMITS } : {}),+ transport: async () => Response.json({}) });+ try {+ expect((await fetch(relay.baseUrl + "/v1/traces", { method: "POST", body: "{}",+ headers: { authorization: `Bearer ${relay.token}`, "content-type": "application/json" } })).status).toBe(200);+ } finally { await relay.close(); }+ }+ const experiment = await openExperimentIngestion({ ...input, receiptPath: `${input.ledgerPath}.experiment-ingestion.jsonl`,+ deadlineAt: Date.now() + 60_000, transport: async () => Response.json({}) });+ const body = JSON.stringify({ project_id: projectId });+ try { await experiment.transport("https://api.braintrust.dev/v1/experiment", { method: "POST", body }); }+ finally { await experiment.close(); }+ return { input, files, expectedBytes: 4 + Buffer.byteLength(body) };+}++it("reconciles real durable receipts for all three transport paths without inventing vendor billing", async () => {+ const f = await fixture(), usage = await collectScheduledIngestion(f.input);+ expect(usage).toMatchObject({ status: "complete", observedAdmittedPayloadBytes: f.expectedBytes,+ totalAdmittedPayloadBytes: f.expectedBytes, billedCostUsd: null, billingStatus: "unreconciled", missingExecutions: [] });+ expect(usage.receipts.map(row => row.kind)).toEqual(["native", "finalization", "experiment"]);+ expect(usage.receipts[0]).toMatchObject({ admittedExpandedSourceBytes: 2, admittedRecords: 0 });+ expect(usage.receipts.every(row => "receiptSha256" in row && /^[a-f0-9]{64}$/.test(row.receiptSha256!))).toBe(true);+ expect(usage.ledgerHash).toMatch(/^[a-f0-9]{64}$/);+ expect(JSON.stringify(usage)).not.toContain("private-fixture");+});++it.each(["missing", "unclosed", "unresolved", "foreign-project", "bad-total", "bad-expanded-total", "missing-record-count", "duplicate-outcome"])(+ "retains other measured traffic and an unknown total for a %s receipt", async reason => {+ const f = await fixture(), file = f.files[0]!;+ if (reason === "missing") await rm(file);+ else {+ const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));+ if (reason === "unclosed") rows.pop();+ if (reason === "unresolved") rows[2].outcome = "uncertain";+ if (reason === "foreign-project") rows[0].projectId = "22222222-2222-4222-8222-222222222222";+ if (reason === "bad-total") rows.at(-1).admittedBytes = 0;+ if (reason === "bad-expanded-total") rows.at(-1).admittedExpandedBytes = 0;+ if (reason === "missing-record-count") delete rows[1].recordCount;+ if (reason === "duplicate-outcome") rows.splice(3, 0, rows[2]);+ await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n");+ }+ const usage = await collectScheduledIngestion(f.input);+ expect(usage).toMatchObject({ status: "incomplete", totalAdmittedPayloadBytes: null, billedCostUsd: null });+ expect(usage.observedAdmittedPayloadBytes).toBeGreaterThan(0);+ expect(usage.receipts[0]!.status).not.toBe("complete");+ expect(usage.receipts.slice(1).every(row => row.status === "complete")).toBe(true);+ });++it("binds the expected receipt set to admitted attempts and retains unstarted selections", async () => {+ const f = await fixture();+ expect(await collectScheduledIngestion({ ...f.input, executionIds: [...f.input.executionIds, "missing.local"] }))+ .toMatchObject({ status: "incomplete", missingExecutions: ["missing.local"], totalAdmittedPayloadBytes: null });+ await expect(collectScheduledIngestion({ ...f.input, executionIds: [] })).rejects.toThrow("selection");+ await expect(collectScheduledIngestion({ ...f.input, executionIds: ["foreign.local"] })).rejects.toThrow("Foreign ingestion attempt");+ await expect(collectScheduledIngestion({ ...f.input, campaignId: "foreign" })).rejects.toThrow("campaign mismatch");+});++it("keeps version 1 transport receipts readable without inventing expanded data measurements", async () => {+ const f = await fixture(), file = f.files[0]!;+ const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));+ rows[0].version = 1; delete rows[0].sizing;+ delete rows[0].limits.maxExpandedBytes; delete rows[0].limits.maxRecords;+ for (const row of rows) {+ delete row.expandedBytes; delete row.recordCount;+ delete row.admittedExpandedBytes; delete row.admittedRecords;+ }+ await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n");+ const usage = await collectScheduledIngestion(f.input);+ expect(usage).toMatchObject({ status: "complete", billedCostUsd: null });+ expect(usage.receipts[0]).toMatchObject({ admittedPayloadBytes: 2, admittedExpandedSourceBytes: null, admittedRecords: null });+});++it.each(["outcome", "closed"])("retains the verified prefix and digest when the final %s line is truncated", async kind => {+ const f = await fixture(), file = f.files[0]!;+ const lines = (await readFile(file, "utf8")).trim().split("\n");+ const prefix = lines.slice(0, kind === "outcome" ? 2 : 3);+ const truncated = prefix.join("\n") + '\n{"type":"' + kind;+ await writeFile(file, truncated);+ const usage = await collectScheduledIngestion(f.input);+ expect(usage).toMatchObject({ status: "incomplete", observedAdmittedPayloadBytes: f.expectedBytes, totalAdmittedPayloadBytes: null });+ expect(usage.receipts[0]).toMatchObject({ status: "incomplete", admittedRequests: 1, admittedPayloadBytes: 2,+ unresolvedRequests: kind === "outcome" ? 1 : 0, receiptSha256: createHash("sha256").update(truncated).digest("hex") });+});diff --git a/tests/runner-e2e/ingestion-usage.ts b/tests/runner-e2e/ingestion-usage.tsnew file mode 100644index 0000000000..f5c925844c--- /dev/null+++ b/tests/runner-e2e/ingestion-usage.ts@@ -0,0 +1,98 @@+import { createHash } from "node:crypto";+import { readFile } from "node:fs/promises";+import path from "node:path";+import { readEvalCostLedger, validEvalLedgerId } from "./cost-ledger.js";+import { EVAL_OTLP_LIMITS } from "./eval-otlp-relay.js";+import { EVAL_FINALIZATION_LIMITS } from "./eval-finalization-relay.js";+import { EVAL_EXPERIMENT_LIMITS } from "./experiment-ingestion.js";++type Kind = "native" | "finalization" | "experiment";+interface ReceiptScope { kind: Kind; file: string; campaignId: string; projectId: string; attemptId?: string }+async function readReceipt(scope: ReceiptScope) {+ const bytes = await readFile(scope.file);+ if (bytes.length > 4 * 1024 * 1024) throw new Error("Oversized ingestion receipt");+ const rows: Array<Record<string, any>> = [];+ let malformed = false;+ for (const line of bytes.toString("utf8").trim().split("\n")) {+ try { rows.push(JSON.parse(line)); }+ catch { malformed = true; break; }+ }+ const policy = rows[0], closed = rows.at(-1);+ const maximum = scope.kind === "native" ? EVAL_OTLP_LIMITS : scope.kind === "finalization" ? EVAL_FINALIZATION_LIMITS : EVAL_EXPERIMENT_LIMITS;+ const expanded = scope.kind !== "experiment" && policy?.version === 2;+ if (policy?.type !== "policy" || !(policy.version === 1 || expanded && policy.sizing === "otlp-v1.9-expanded-source-json")+ || policy.campaignId !== scope.campaignId || policy.projectId !== scope.projectId+ || (scope.kind === "experiment" ? policy?.kind !== "experiment" : policy?.attemptId !== scope.attemptId)) {+ throw new Error("Invalid ingestion receipt identity");+ }+ if (!policy.limits || Object.entries(maximum).filter(([key]) => expanded || !["maxExpandedBytes", "maxRecords"].includes(key))+ .some(([key, max]) => !Number.isSafeInteger(policy.limits[key])+ || policy.limits[key] < 1 || policy.limits[key] > max)) throw new Error("Invalid ingestion receipt limits");+ const admissions = new Map<string, { bodyBytes: number; scores: number }>();+ const outcomes = new Map<string, string>();+ let payloadBytes = 0, scores = 0, expandedBytes = 0, recordCount = 0;+ const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0;+ for (let index = 1; index < rows.length; index++) {+ const row = rows[index];+ if (row.type === "admitted") {+ const scoreCount = scope.kind === "experiment" ? row.scores : 0;+ if (typeof row.id !== "string" || admissions.has(row.id) || !count(row.bodyBytes) || !count(scoreCount)+ || scope.kind !== "experiment" && row.bodyBytes > policy.limits.maxRequestBytes+ || expanded && (!count(row.expandedBytes) || !count(row.recordCount))+ || !/^[0-9a-f]{64}$/.test(row.payloadSha256)) throw new Error("Invalid ingestion admission");+ admissions.set(row.id, { bodyBytes: row.bodyBytes, scores: scoreCount });+ payloadBytes += row.bodyBytes; scores += scoreCount;+ if (expanded) { expandedBytes += row.expandedBytes; recordCount += row.recordCount; }+ } else if (row.type === "outcome") {+ const allowed = scope.kind === "experiment" ? ["response", "unconfirmed"] : ["accepted", "partial", "rejected", "uncertain", "not_dispatched"];+ if (!admissions.has(row.id) || outcomes.has(row.id) || !allowed.includes(row.outcome)+ || ["response", "accepted", "partial", "rejected"].includes(row.outcome)+ && (!Number.isInteger(row.httpStatus) || row.httpStatus < 100 || row.httpStatus > 599)) throw new Error("Invalid ingestion outcome");+ outcomes.set(row.id, row.outcome);+ } else if (row.type === "closed") {+ if (index !== rows.length - 1 || row.admittedRequests !== admissions.size || row.admittedBytes !== payloadBytes+ || expanded && (row.admittedExpandedBytes !== expandedBytes || row.admittedRecords !== recordCount)+ || scope.kind === "experiment" && row.admittedScores !== scores || row.poisoned) throw new Error("Invalid ingestion closure");+ } else if (row.type !== "quota_exhausted" || scope.kind === "experiment") throw new Error("Unexpected ingestion record");+ }+ if (admissions.size > policy.limits.maxRequests || payloadBytes > policy.limits.maxTotalBytes+ || expanded && (expandedBytes > policy.limits.maxExpandedBytes || recordCount > policy.limits.maxRecords)+ || scope.kind === "experiment" && scores > policy.limits.maxScores) throw new Error("Ingestion allowance exceeded");+ const unresolved = [...admissions.keys()].filter(id => !outcomes.has(id) || ["uncertain", "unconfirmed"].includes(outcomes.get(id)!)).length;+ return { kind: scope.kind, ...(scope.attemptId ? { attemptId: scope.attemptId } : {}),+ receiptSha256: createHash("sha256").update(bytes).digest("hex"),+ status: !malformed && closed?.type === "closed" && unresolved === 0 ? "complete" : "incomplete",+ admittedRequests: admissions.size, admittedPayloadBytes: payloadBytes, admittedScores: scope.kind === "experiment" ? scores : null,+ admittedExpandedSourceBytes: expanded ? expandedBytes : null, admittedRecords: expanded ? recordCount : null,+ unresolvedRequests: unresolved };+}++export async function collectScheduledIngestion(input: {+ ledgerPath: string; campaignId: string; projectId: string; executionIds: string[];+}) {+ if (!input.executionIds.length || new Set(input.executionIds).size !== input.executionIds.length+ || !input.executionIds.every(validEvalLedgerId)) throw new Error("Invalid ingestion execution selection");+ const ledger = await readEvalCostLedger(input.ledgerPath);+ if (ledger.state.limits.campaignId !== input.campaignId) throw new Error("Ingestion ledger campaign mismatch");+ const attempts = ledger.state.snapshot().admittedAttempts;+ if (attempts.some(row => !input.executionIds.includes(row.executionId))) throw new Error("Foreign ingestion attempt");+ const missingExecutions = input.executionIds.filter(id => !attempts.some(row => row.executionId === id));+ const scopes: ReceiptScope[] = attempts.flatMap(row => {+ const name = createHash("sha256").update(row.attemptId).digest("hex");+ return (["native", "finalization"] as const).map(kind => ({ kind, attemptId: row.attemptId,+ campaignId: input.campaignId, projectId: input.projectId,+ file: path.join(`${input.ledgerPath}.otlp`, `${name}${kind === "finalization" ? "-finalization" : ""}.jsonl`) }));+ });+ scopes.push({ kind: "experiment", file: `${input.ledgerPath}.experiment-ingestion.jsonl`, campaignId: input.campaignId, projectId: input.projectId });+ const receipts = await Promise.all(scopes.map(async scope => {+ try { return await readReceipt(scope); }+ catch { return { kind: scope.kind, ...(scope.attemptId ? { attemptId: scope.attemptId } : {}), status: "unavailable" }; }+ }));+ const complete = missingExecutions.length === 0 && receipts.every(row => row.status === "complete");+ return { version: 1, campaignId: input.campaignId, projectId: input.projectId, ledgerHash: ledger.previous,+ status: complete ? "complete" : "incomplete", missingExecutions, receipts,+ observedAdmittedPayloadBytes: receipts.reduce((sum, row) => sum + ("admittedPayloadBytes" in row ? row.admittedPayloadBytes : 0), 0),+ totalAdmittedPayloadBytes: complete ? receipts.reduce((sum, row) => sum + ("admittedPayloadBytes" in row ? row.admittedPayloadBytes : 0), 0) : null,+ billedCostUsd: null, billingStatus: "unreconciled",+ scope: "Bounded attempt relay, controller finalization and retained experiment requests; not vendor processed bytes or provider model charges" };+}diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.tsindex b1ab4577a8..83770cd98a 100644--- a/tests/runner-e2e/installed-cli.test.ts+++ b/tests/runner-e2e/installed-cli.test.ts@@ -108,10 +108,9 @@ it("uses the direct public Playwright JS bin without bin-shim Node injection", aconst f = await fixture(); const cli = join(f.root,"node_modules/@playwright/test/cli.js");await mkdir(join(f.root,"node_modules/@playwright/test"),{recursive:true});await writeFile(cli,"console.log(JSON.stringify({nodePath:process.env.NODE_PATH??null,nodeOptions:process.env.NODE_OPTIONS??null,args:process.argv.slice(2)}))");- const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"],true);+ const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"]);expect(invocation.command).toBe(process.execPath);expect(JSON.parse(execFileSync(invocation.command,invocation.args,{env:{},timeout:5000,encoding:"utf8"}))).toEqual({nodePath:null,nodeOptions:null,args:["--version"]});- expect(runnerE2EPlaywrightInvocation(f.root,["--version"],false)).toEqual({command:"pnpm",args:["exec","playwright","--version"]});for (const key of ["NODE_PATH","NODE_OPTIONS"]) expect(()=>assertInstalledCliSelection({...f.env,[key]:""},[cells[0]!])).toThrow("ambient Node injection");});diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.tsindex 092ce425f9..30c489ad92 100644--- a/tests/runner-e2e/journey-features.ts+++ b/tests/runner-e2e/journey-features.ts@@ -34,6 +34,9 @@ associate(["local-session-integrity", "extended-harnesses", "grok-qualification"associate(["question-resume"], {"question-answer-resume": ["questions-and-approvals/questions", "questions-and-approvals/continuation"],});+associate(["feedback-persistence"], {+ "complaint-and-suggestion": ["activity/agent-commentary"],+});associate(["continuation"], {"answer-updates-scope": ["questions-and-approvals/questions", "questions-and-approvals/continuation"],"clarification-not-approval": ["questions-and-approvals/questions", "questions-and-approvals/confirmations"],diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.tsindex 1d0971ab97..934daa1dc8 100644--- a/tests/runner-e2e/launch.ts+++ b/tests/runner-e2e/launch.ts@@ -1,9 +1,15 @@-import { runnerE2EPlaywrightInvocation } from "./web-server-command.js";+import { recoverWorkerTiming } from "./worker-timing.js";+import { openEvalOperations, type EvalOperations } from "./operations.js";+import { readRecoveredFeedback } from "./feedback-recovery.js";+import { readInterruptedAttemptTrace } from "./trace-recovery.js";+import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js";+import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js";+import { runnerE2EPlaywrightInvocation, runnerE2EPlaywrightShutdownSignal } from "./web-server-command.js";import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js";import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";import { createRunnerE2ETemporaryRoot } from "./server-config.js";-import { randomBytes } from "node:crypto";+import { createHash, randomBytes } from "node:crypto";import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";import { spawn } from "node:child_process";import { createWriteStream } from "node:fs";@@ -30,7 +36,9 @@ import { classifyFailure } from "./failure-classifier.js";import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js";import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js";import { buildRunnerCampaign } from "./history.js";+import { readEvalOperationsReport } from "./operations-report.js";import {+ applyEvalGatewayEnvironment,buildRunnerE2EProcessEnvironment,resolvePaperclipRemoteRunnerBinaryForHarness,resolvePaperclipRunnerBinaryForHarness,@@ -305,7 +313,7 @@ async function runProcess(const cleanupFinished = new Promise<number>(resolve => { cleanupSettled = () => resolve(1); });const stopChildTree = (_diagnostic?: ProcessTreeDiagnostic) => {if (boundedCleanup) return;- boundedCleanup = stopOwnedProcessTree(child, processOwner)+ boundedCleanup = stopOwnedProcessTree(child, processOwner, 45_000, 5_000, runnerE2EPlaywrightShutdownSignal).then(() => null, error => error instanceof Error ? error.message : String(error)).finally(cleanupSettled);activeProcessCleanup.set(child.pid!, boundedCleanup);@@ -467,8 +475,12 @@ async function runAttempt(input: {attempt: number;campaignId: string;options: ReturnType<typeof parseRunnerSelectors>;+ operations?: EvalOperations;}): Promise<RunnerE2EResult[]> {const { executions, attempt, campaignId, options } = input;+ const accountingCampaignId = input.operations?.manifest.limits.campaignId ?? process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? campaignId;+ const accountingAttemptId = randomBytes(16).toString("hex");+ const traceConfig = evalBraintrustConfig({ ...process.env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined });const execution = executions[0];if (!execution) throw new Error("A runner E2E cell must contain a task case");if (@@ -494,6 +506,7 @@ async function runAttempt(input: {let attemptSecrets: string[] = [];let processCleanupFailed = false;let startupReceiptPath: string | undefined;+ let gateway: Awaited<ReturnType<EvalOperations["start"]>> | undefined;const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = [];try {const paperclipHome = path.join(temporaryRoot, "paperclip-home");@@ -523,8 +536,11 @@ async function runAttempt(input: {const decisionSigningSecret = secret(48);const toolActionSigningSecret = secret(48);const betterAuthSecret = secret(48);+ gateway = await input.operations?.start(execution, accountingAttemptId, attempt);const credentialSources = {- ...Object.fromEntries(CREDENTIAL_NAMES.map((name) => [name, process.env[name]])),+ PAPERCLIP_E2E_GATEWAY_TOKEN: gateway?.token,+ PAPERCLIP_E2E_OTLP_RELAY_TOKEN: gateway?.observability?.token,+ ...Object.fromEntries([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => [name, process.env[name]])),PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret,PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret,PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,@@ -545,6 +561,8 @@ async function runAttempt(input: {executions.map((candidate) => candidate.id),),PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt),+ PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: accountingCampaignId,+ PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: accountingAttemptId,PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY: options.installedStartupOnly ? "1" : undefined,PAPERCLIP_RUNNER_E2E_PUBLIC_MCP: executions.some(candidate => candidate.task.flow === "public_mcp") ? "1" : "0",PAPERCLIP_RUNNER_E2E_PORT: String(port),@@ -575,6 +593,9 @@ async function runAttempt(input: {PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,BETTER_AUTH_SECRET: betterAuthSecret,};+ if (gateway) {+ applyEvalGatewayEnvironment(childEnv, execution.profile.credential, gateway);+ }// The database URLs are stripped here and again at the Playwright web-server// boundary so a developer's shell can never redirect this paid test.delete childEnv.DATABASE_URL;@@ -600,7 +621,7 @@ async function runAttempt(input: {) +5 * 60_000;const processResult = await runProcess(- runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs, Boolean(installedCli)),+ runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs),childEnv,watchdog,path.join(privateDir, "playwright.log"),@@ -653,6 +674,16 @@ async function runAttempt(input: {processFailureClass,);const result = await readResult(resultPath, fallback);+ if (!result.workerTiming) result.workerTiming = await recoverWorkerTiming(path.join(path.dirname(resultPath), "snapshots", "worker-timing.json"),+ { executionId: candidate.id, attempt, startedAt: result.startedAt, finishedAt: result.finishedAt });+ if (result === fallback) {+ result.feedback = await readRecoveredFeedback(path.dirname(resultPath), {+ campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,+ });+ if (traceConfig) result.tracing = await readInterruptedAttemptTrace(path.dirname(resultPath), {+ projectId: traceConfig.projectId, executionId: candidate.id,+ });+ }// Keep cleanup authority before evidence copying/publication can fail.rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback, status: result.status });return enforceResultProcessIntegrity(result, processResult);@@ -712,7 +743,10 @@ async function runAttempt(input: {}}for (const [index, candidate] of executions.entries()) {- let result = results[index];+ let result = {+ ...results[index],+ accountingIdentity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId },+ };if (isolationError &&result.failureClass !== "cleanup_failure" &&@@ -836,6 +870,8 @@ async function runAttempt(input: {}return [...publishedResults];} finally {+ let gatewayCleanupError: unknown;+ try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; }if (!processCleanupFailed) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline);let cleanupError: unknown;const resourceAdmissionStarted = await access(path.join(temporaryRoot, "artifacts-private", "resource-admission-started"))@@ -877,6 +913,7 @@ async function runAttempt(input: {`Refusing to remove unexpected temporary path ${temporaryRoot}`,);}+ if (gatewayCleanupError) cleanupError = new AggregateError([cleanupError, gatewayCleanupError].filter(Boolean), "Eval gateway cleanup failed");if (startupReceiptPath) {const receipt = JSON.parse(await readFile(startupReceiptPath, "utf8"));receipt.status = cleanupError ? "installed_launch_chain_cleanup_failed" : "installed_launch_chain_passed";@@ -909,6 +946,26 @@ async function runAttempt(input: {console.error(message);if (options.installedStartupOnly) throw new Error(message);}+ for (const result of publishedResults) {+ const resultPath = publishedResultPaths.get(result.executionId);+ const candidate = executions.find(candidate => candidate.id === result.executionId);+ if (!traceConfig || !result.tracing || !resultPath || !candidate) continue;+ const finalizationInput = {+ receipt: result.tracing,+ identity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,+ provider: candidate.profile.provider, model: candidate.profile.model, environment: candidate.environment.id },+ provenance: { suiteId: candidate.suite.id, suiteDefinitionHash: candidate.suiteDefinitionHash, profileId: candidate.profile.id,+ caseId: candidate.task.id, sourceRevision: resolveRunnerE2ESource().sha },+ result,+ };+ result.tracing.finalization = input.operations+ ? await finalizeEvalAttemptThroughRelay(traceConfig, finalizationInput,+ path.join(`${input.operations.manifest.ledgerPath}.otlp`, `${createHash("sha256").update(accountingAttemptId).digest("hex")}-finalization.jsonl`))+ : await finalizeEvalAttemptTrace(traceConfig, finalizationInput);+ const safeResult = `${JSON.stringify(sanitizeJson(result, attemptSecrets), null, 2)}\n`;+ assertSecretFree(safeResult, attemptSecrets, resultPath);+ await writeFile(resultPath, safeResult, "utf8");+ }}}@@ -932,6 +989,8 @@ async function runExecutionWithRetry(input: {execution: MatrixExecution;campaignId: string;options: ReturnType<typeof parseRunnerSelectors>;+ onAttempt: (result: RunnerE2EResult) => void;+ operations?: EvalOperations;}): Promise<RunnerE2EResult> {const { execution, campaignId, options } = input;return executeWithAutomaticRetry({@@ -942,8 +1001,9 @@ async function runExecutionWithRetry(input: {`Retrying ${execution.id} in a fresh isolated harness after ${result.failureClass!.replaceAll("_", " ")}`,),runAttempt: async attempt => {- const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options });+ const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options, operations: input.operations });if (!result) throw new Error(`No result produced for ${execution.id} attempt ${attempt}`);+ input.onAttempt(result);return result;},});@@ -968,7 +1028,9 @@ async function runWithConcurrency<T, R>(}},);- await Promise.all(workers);+ const settled = await Promise.allSettled(workers);+ const failures = settled.filter((result): result is PromiseRejectedResult => result.status === "rejected");+ if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Eval campaign workers failed");return results;}@@ -1011,6 +1073,7 @@ async function main() {return;}+ if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST && executions.some(execution => execution.task.flow === "provider_connection")) throw new Error("Provider connection campaign has no budgeted routing qualification");if (executions.some(execution => execution.task.flow === "provider_connection")) {const { runConnectionCampaign } = await import("./connection-launch.js");await runConnectionCampaign({ executions, catalog: runnerMatrix, configFile: options.connectionConfig, repositoryRoot });@@ -1041,7 +1104,7 @@ async function main() {await loadLocalEnvironment(process.env);assertRemoteNativeEvidencePrerequisites(executions, process.env);- const missingCredentials = [+ const missingCredentials = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST ? [] : [...new Set(executions.flatMap((execution) => execution.requiredCredentials),),@@ -1084,21 +1147,35 @@ async function main() {console.log(`Running ${executions.length} isolated execution(s) with max parallelism ${requestedParallelism}`,);- const finalResults = await runWithConcurrency(- executions,- requestedParallelism,- (execution) => runExecutionWithRetry({ execution, campaignId, options }),- );+ const attempts: RunnerE2EResult[] = [];+ const operations = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST+ ? await openEvalOperations(process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, executions, options.maxAutomaticRetries, process.env) : undefined;+ let finalResults: RunnerE2EResult[] = [];+ let campaignFailed = false;+ try {+ if (operations) await writeFile(path.join(summaryDir, "operations-preflight.json"), `${JSON.stringify(operations.manifest, null, 2)}\n`, { mode: 0o600 });+ finalResults = await runWithConcurrency(executions, operations ? Math.min(requestedParallelism, operations.manifest.limits.maxConcurrent) : requestedParallelism,+ execution => runExecutionWithRetry({ execution, campaignId, options, operations, onAttempt: result => attempts.push(result) }));+ } catch {+ campaignFailed = true;+ finalResults = [...new Map(attempts.map(result => [result.executionId, result])).values()];+ } finally {+ try { await operations?.close(); } catch { campaignFailed = true; }+ }const generatedAt = new Date().toISOString();+ const operationsReport = operations ? await readEvalOperationsReport(operations.manifest.ledgerPath) : undefined;+ if (operationsReport?.status === "unavailable") campaignFailed = true;const campaign = buildRunnerCampaign({campaignId,generatedAt,expected: executions.map((execution) => execution.id),results: finalResults,+ attempts,+ ...(operationsReport ? { operations: operationsReport } : {}),});const campaignSecrets = normalizedSecrets(- CREDENTIAL_NAMES.map((name) => process.env[name]),+ [...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => process.env[name]),);const safeCampaign = sanitizeJson(campaign,@@ -1138,7 +1215,8 @@ async function main() {console.log(`Campaign ${campaignId}: ${safeCampaign.passed}/${safeCampaign.selected} passed`,);- if (safeCampaign.failed > 0) process.exitCode = 1;+ if (campaignFailed) console.error("Campaign finalization incomplete; inspect retained attempt artifacts and the operations ledger.");+ if (campaignFailed || safeCampaign.failed > 0) process.exitCode = 1;}await main().catch((error) => {diff --git a/tests/runner-e2e/ledger-attempt-policy.test.ts b/tests/runner-e2e/ledger-attempt-policy.test.tsnew file mode 100644index 0000000000..138398236c--- /dev/null+++ b/tests/runner-e2e/ledger-attempt-policy.test.ts@@ -0,0 +1,120 @@+import { afterEach, expect, it } from "vitest";+import { createHash } from "node:crypto";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js";+import { adoptLegacyEvalAttemptPolicy, type LegacyAttemptAuditFile } from "./ledger-attempt-policy.js";++const roots: string[] = [];+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });+const limits: EvalSpendLimits = { campaignId: "legacy-pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 3, maxConcurrent: 1 };+const identity = { executionId: "claude.local.fixture", attemptId: "old-request-attempt", phase: "agent" as const };+const hash = (value: string) => createHash("sha256").update(value).digest("hex");++async function fixture(overrides: Partial<EvalSpendLimits> = {}) {+ const root = await mkdtemp(path.join(os.tmpdir(), "legacy-eval-attempts-")); roots.push(root);+ const file = path.join(root, "ledger.jsonl");+ const originalLimits = { ...limits, ...overrides };+ const ledger = await openEvalCostLedger(file, originalLimits);+ await ledger.reserve("cancelled-request", identity, "0.6");+ await ledger.observe({ observationId: "partial", requestId: "cancelled-request", source: "provider", provider: "anthropic",+ currency: "USD", route: "direct", classification: "reported", costUsd: "0.2", final: false });+ await ledger.finish("cancelled-request", "cancelled");+ await ledger.close();+ const evidencePath = path.join(root, "reviewed-attempts.md");+ const evidence = "Reviewed retained campaigns, including the earlier zero-request startup failure.";+ await writeFile(evidencePath, evidence);+ const loaded = await readEvalCostLedger(file);+ const audit: LegacyAttemptAuditFile = { version: 1, ledgerPath: file, evidencePath, evidenceSha256: hash(evidence),+ previousJournalHash: loaded.previous, maxAttemptsPerExecution: 3,+ priorAttempts: [identity, { executionId: identity.executionId, attemptId: "old-zero-request-attempt" }]+ .map(({ executionId, attemptId }) => ({ executionId, attemptId })) };+ return { file, audit, originalLimits, before: loaded.state.snapshot(), prefix: await readFile(file, "utf8") };+}++it("appends an audited policy without rewriting charges, held reservations, or original limits", async () => {+ const { file, audit, before, prefix, originalLimits } = await fixture();+ const migrated = await adoptLegacyEvalAttemptPolicy(audit);+ expect((await readFile(file, "utf8")).startsWith(prefix)).toBe(true);+ expect(migrated.limits).toEqual({ ...originalLimits, maxAttemptsPerExecution: 3 });+ expect({ ...migrated.accounting, admittedAttempts: [] }).toEqual(before);+ expect(migrated.accounting.admittedAttempts).toEqual(audit.priorAttempts);+ const resumed = await openEvalCostLedger(file, migrated.limits);+ try {+ await resumed.admitAttempt(identity.executionId, "new-attempt");+ await expect(resumed.admitAttempt(identity.executionId, "extra-attempt")).rejects.toThrow("attempt cap");+ await expect(resumed.reserve("new-request", { ...identity, attemptId: "new-attempt" }, "0.5")).rejects.toThrow("spending cap");+ await resumed.reserve("new-request", { ...identity, attemptId: "new-attempt" }, "0.4");+ expect(resumed.snapshot().committedUsd).toBe("1.000000000");+ } finally { await resumed.close(); }+ const replayed = await readEvalCostLedger(file);+ expect(replayed.state.snapshot().admittedAttempts).toHaveLength(3);+ expect(replayed.state.snapshot().knownUsd).toBe("0.200000000");+ await expect(openEvalCostLedger(file, originalLimits)).rejects.toThrow("limits changed");+ await expect(openEvalCostLedger(file, { ...migrated.limits, maxUsd: "2" })).rejects.toThrow("limits changed");+});++it("requires all recorded request identities and rejects duplicate or foreign attempts", async () => {+ const { file, audit, prefix } = await fixture();+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [] })).rejects.toThrow("omits a recorded request");+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [audit.priorAttempts[0]!, audit.priorAttempts[0]!] })).rejects.toThrow("Invalid legacy attempt identity");+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [{ ...audit.priorAttempts[0]!, executionId: "other-cell" }] })).rejects.toThrow("omits a recorded request");+ expect(await readFile(file, "utf8")).toBe(prefix);+});++it("rejects changed evidence and changed journal checkpoints without appending", async () => {+ const { file, audit, prefix, originalLimits } = await fixture();+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, evidenceSha256: "a".repeat(64) })).rejects.toThrow("evidence changed");+ expect(await readFile(file, "utf8")).toBe(prefix);+ const writer = await openEvalCostLedger(file, originalLimits);+ try {+ await writer.observe({ observationId: "late", requestId: "cancelled-request", source: "invoice", provider: "anthropic",+ currency: "USD", route: "direct", classification: "reconciled", costUsd: "0.1", final: true });+ await expect(writer.adoptLegacyAttemptPolicy({ previousJournalHash: audit.previousJournalHash, evidenceSha256: audit.evidenceSha256,+ maxAttemptsPerExecution: 3, priorAttempts: audit.priorAttempts })).rejects.toThrow("checkpoint changed");+ } finally { await writer.close(); }+ await expect(adoptLegacyEvalAttemptPolicy(audit)).rejects.toThrow("checkpoint changed");+ expect((await readEvalCostLedger(file)).state.limits.maxAttemptsPerExecution).toBeUndefined();+});++it("refuses concurrent writers and unfinished requests", async () => {+ const { file, audit, originalLimits } = await fixture();+ const writer = await openEvalCostLedger(file, originalLimits);+ try {+ await expect(adoptLegacyEvalAttemptPolicy(audit)).rejects.toThrow("writer lock");+ await writer.reserve("inflight", identity, "0.1");+ } finally { await writer.close(); }+ const changed = await readEvalCostLedger(file);+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, previousJournalHash: changed.previous })).rejects.toThrow("stopped requests");+ expect((await readEvalCostLedger(file)).state.snapshot().inflight).toBe(1);+});++it("does not change established caps or admit extra attempts after an over-cap history", async () => {+ const { file, audit } = await fixture();+ const migrated = await adoptLegacyEvalAttemptPolicy({ ...audit, maxAttemptsPerExecution: 1 });+ const writer = await openEvalCostLedger(file, migrated.limits);+ try { await expect(writer.admitAttempt(identity.executionId, "extra")).rejects.toThrow("attempt cap"); }+ finally { await writer.close(); }+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, previousJournalHash: migrated.journalHash, maxAttemptsPerExecution: 10 })).rejects.toThrow("already established");+});++it("preserves an expired campaign deadline during a historical audit", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "expired-eval-attempts-")); roots.push(root);+ const file = path.join(root, "ledger.jsonl"), evidencePath = path.join(root, "audit.md");+ const writer = await openEvalCostLedger(file, { ...limits, deadlineAt: "2000-01-01T00:00:00Z" });+ await writer.close(); await writeFile(evidencePath, "No launched attempts.");+ const original = await readEvalCostLedger(file);+ const migrated = await adoptLegacyEvalAttemptPolicy({ version: 1, ledgerPath: file, evidencePath, evidenceSha256: hash("No launched attempts."),+ previousJournalHash: original.previous, maxAttemptsPerExecution: 1, priorAttempts: [] });+ const resumed = await openEvalCostLedger(file, migrated.limits);+ try { await expect(resumed.admitAttempt("cell", "attempt")).rejects.toThrow("deadline"); }+ finally { await resumed.close(); }+});++it("rejects extra policy fields instead of accepting limit changes or private notes", async () => {+ const { file, audit, prefix } = await fixture();+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, maxUsd: "100" } as LegacyAttemptAuditFile)).rejects.toThrow("Invalid legacy attempt audit file");+ await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [{ ...audit.priorAttempts[0]!, note: "private" }] } as unknown as LegacyAttemptAuditFile)).rejects.toThrow("Unknown ledger field");+ expect(await readFile(file, "utf8")).toBe(prefix);+});diff --git a/tests/runner-e2e/ledger-attempt-policy.ts b/tests/runner-e2e/ledger-attempt-policy.tsnew file mode 100644index 0000000000..3bcbc28676--- /dev/null+++ b/tests/runner-e2e/ledger-attempt-policy.ts@@ -0,0 +1,40 @@+import { createHash } from "node:crypto";+import { readFile } from "node:fs/promises";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { openEvalCostLedger, readEvalCostLedger, type EvalLegacyAttemptAudit } from "./cost-ledger.js";++export interface LegacyAttemptAuditFile extends EvalLegacyAttemptAudit {+ version: 1;+ ledgerPath: string;+ evidencePath: string;+}++export async function adoptLegacyEvalAttemptPolicy(audit: LegacyAttemptAuditFile) {+ const allowed = ["version", "ledgerPath", "evidencePath", "previousJournalHash", "evidenceSha256", "maxAttemptsPerExecution", "priorAttempts"];+ if (!audit || typeof audit !== "object" || Array.isArray(audit) || Object.keys(audit).some(key => !allowed.includes(key))+ || audit.version !== 1 || typeof audit.ledgerPath !== "string" || !path.isAbsolute(audit.ledgerPath)+ || typeof audit.evidencePath !== "string" || !path.isAbsolute(audit.evidencePath)) throw new Error("Invalid legacy attempt audit file");+ const evidence = await readFile(audit.evidencePath);+ if (createHash("sha256").update(evidence).digest("hex") !== audit.evidenceSha256) throw new Error("Legacy attempt audit evidence changed");+ const original = await readEvalCostLedger(audit.ledgerPath);+ if (original.previous !== audit.previousJournalHash) throw new Error("Legacy attempt audit checkpoint changed");+ const ledger = await openEvalCostLedger(audit.ledgerPath, original.state.limits);+ try {+ await ledger.adoptLegacyAttemptPolicy({ previousJournalHash: audit.previousJournalHash, evidenceSha256: audit.evidenceSha256,+ maxAttemptsPerExecution: audit.maxAttemptsPerExecution, priorAttempts: audit.priorAttempts });+ } finally { await ledger.close(); }+ const updated = await readEvalCostLedger(audit.ledgerPath);+ return { journalHash: updated.previous, records: updated.sequence, limits: updated.state.limits, accounting: updated.state.snapshot() };+}++if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {+ try {+ if (process.argv.length !== 3) throw new Error("Usage: ledger-attempt-policy.ts /absolute/private-audit.json");+ const result = await adoptLegacyEvalAttemptPolicy(JSON.parse(await readFile(process.argv[2]!, "utf8")));+ process.stdout.write(`Legacy attempt policy saved at journal record ${result.records}. Existing costs and limits are preserved.\n`);+ } catch {+ process.stderr.write("Legacy attempt audit failed. Inspect the private audit and original journal before retrying.\n");+ process.exitCode = 1;+ }+}diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.tsindex 5cc7ff5172..cc861c6232 100644--- a/tests/runner-e2e/live-fixtures.ts+++ b/tests/runner-e2e/live-fixtures.ts@@ -1,3 +1,4 @@+import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js";import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js";import path from "node:path";@@ -44,11 +45,7 @@ interface AgentRecord {}interface ManagedAccountFixture {connectionId: string;- binding: {- provider: "openai" | "anthropic" | "github" | "openrouter";- method: "api_key";- mode: "responsible_user";- };+ binding: AiConnectionBinding;}interface ProjectRecord {@@ -143,6 +140,7 @@ export async function setupLiveFixtures(input: {workspacePath: string;credentials: Partial<Record<CredentialName, string>>;daytonaImage?: string;+ onCompanyCreated?: (companyId: string) => Promise<void>;}): Promise<LiveFixtureValues> {const { api, execution } = input;const receiptSmoke = execution.suite.id === "case-handoffs" && execution.task.id === caseReceiptTask.id;@@ -166,15 +164,17 @@ export async function setupLiveFixtures(input: {registry.register<CompanyRecord>({id: "company",async setup() {- return api.post<CompanyRecord>("/api/companies", {+ const company = await api.post<CompanyRecord>("/api/companies", {name: `Runner E2E ${execution.id} ${input.executionNonce}`,description: "Ephemeral paid full-stack runner acceptance fixture",budgetMonthlyCents: receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS: ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id)|| (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS: execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS- : ["stock-harness", "task-workspaces", "case-handoffs"].includes(execution.suite.id) ? 1_000 : 0,+ : ["stock-harness", "task-workspaces", "case-handoffs", "feedback-persistence"].includes(execution.suite.id) ? 1_000 : 0,});+ await input.onCompanyCreated?.(company.id);+ return company;},async teardown() {// The isolated instance/database is removed by the launcher. Do not call@@ -275,7 +275,12 @@ export async function setupLiveFixtures(input: {const managedHiring = isManagedHiringCase(execution.suite.id, execution.task.id);const savedCopilot = execution.profile.qualificationCandidate === "copilot";- if (managedHiring || savedCopilot) {+ const gatewayUrl = process.env.PAPERCLIP_E2E_GATEWAY_BASE_URL;+ const gatewayToken = process.env.PAPERCLIP_E2E_GATEWAY_TOKEN;+ const budgetedGateway = Boolean(gatewayUrl || gatewayToken);+ if (budgetedGateway && (!gatewayUrl || !gatewayToken || !/^http:\/\/127\.0\.0\.1:[0-9]+$/.test(gatewayUrl)+ || execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id))) throw new Error("Unqualified fixture gateway route");+ if (managedHiring || savedCopilot || budgetedGateway) {registry.register<ManagedAccountFixture>({id: "ai-connection",dependencies: ["company", ...(savedCopilot ? ["environment"] : [])],@@ -288,22 +293,26 @@ export async function setupLiveFixtures(input: {if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`);const apiKey = input.credentials[key];if (!apiKey) throw new Error(`Missing credential ${key}`);- const account = await api.postSensitive<{ connectionId: string }>(+ const account = await api.postSensitive<{ connectionId: string; grantId: string }>(`/api/companies/${company.id}/ai-connections`,{provider,method: "api_key",name: `Runner E2E account ${input.executionNonce}`,- ownership: "personal",+ ownership: budgetedGateway ? "shared" : "personal",+ ...(budgetedGateway ? { routing: { kind: "gateway", protocol: provider === "anthropic" ? "messages" : "responses",+ baseUrl: provider === "anthropic" ? gatewayUrl : `${gatewayUrl}/v1`, auth: "bearer", models: [{ id: execution.profile.model }] } } : {}),...(savedCopilot ? { environmentId: value<EnvironmentRecord>(resolved, "environment").id } : {}),- apiKey,+ apiKey: budgetedGateway ? gatewayToken : apiKey,agentIds: [],- allAgents: false,+ allAgents: budgetedGateway,},);return {connectionId: account.connectionId,- binding: { provider, method: "api_key", mode: "responsible_user" },+ binding: budgetedGateway+ ? { provider, method: "api_key", mode: "shared", connectionId: account.connectionId, grantId: account.grantId }+ : { provider, method: "api_key", mode: "responsible_user" },};},});@@ -316,7 +325,7 @@ export async function setupLiveFixtures(input: {"secrets","environment",...((grokSubscription || codexSubscription) ? ["subscription-login"] : []),- ...(managedHiring || savedCopilot ? ["ai-connection"] : []),+ ...(managedHiring || savedCopilot || budgetedGateway ? ["ai-connection"] : []),],async setup(resolved) {const company = value<CompanyRecord>(resolved, "company");@@ -356,7 +365,7 @@ export async function setupLiveFixtures(input: {setupProbe.defaultRunner = selected.defaultRunner;}}- if (["stock-harness", "native-connection-guidance"].includes(execution.suite.id)+ if (["stock-harness", "native-connection-guidance", "feedback-persistence"].includes(execution.suite.id)|| (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) {agent.budgetMonthlyCents = 1_000;}@@ -369,7 +378,7 @@ export async function setupLiveFixtures(input: {heartbeat: { ...((runtimeConfig.heartbeat ?? {}) as Record<string, unknown>),enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 1, maxDailyCostCents: CASE_RECEIPT_BUDGET_CENTS } };}- if (managedHiring || savedCopilot) {+ if (managedHiring || savedCopilot || budgetedGateway) {const account = value<ManagedAccountFixture>(resolved, "ai-connection");const config = agent.adapterConfig as Record<string, unknown>;delete config.env;diff --git a/tests/runner-e2e/operating-overhead.test.ts b/tests/runner-e2e/operating-overhead.test.tsnew file mode 100644index 0000000000..c837450da0--- /dev/null+++ b/tests/runner-e2e/operating-overhead.test.ts@@ -0,0 +1,119 @@+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, it } from "vitest";+import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js";+import { EVAL_OVERHEAD_COMPONENTS, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";+import { readEvalOperationsReport, renderEvalOperationsReport } from "./operations-report.js";++const now = Date.parse("2026-10-21T00:00:00Z");+const allowance = (): EvalOperatingOverhead => ({ version: 1, reference: "fixture-only", runner: "aws-fleet",+ verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",+ components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) });+const currentAllowance = (time = Date.now()): EvalOperatingOverhead => ({ ...allowance(),+ verifiedAt: new Date(time - 86_400_000).toISOString(), expiresAt: new Date(time + 30 * 86_400_000).toISOString() });+const runner = "ubuntu-latest";+const modelIdentity = { executionId: "claude.local.fixture", attemptId: "attempt-1", phase: "agent" as const };++it("keeps operating overhead inside the same cap through model completion and a journal restart", async () => {+ const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-"));+ const file = path.join(directory, "ledger.jsonl");+ const limits: EvalSpendLimits = { campaignId: "fixture", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z",+ maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() };+ try {+ const ledger = await openEvalCostLedger(file, limits);+ try {+ expect((await readEvalCostLedger(file)).state.snapshot()).toMatchObject({ knownUsd: "0.000000000", heldUsd: "4.000000000", inflight: 0, unknownRequests: 4 });+ await ledger.admitAttempt(modelIdentity.executionId, modelIdentity.attemptId);+ await expect(ledger.reserve("too-large", modelIdentity, "17")).rejects.toThrow("spending cap");+ await ledger.reserve("model", modelIdentity, "16");+ await ledger.observe({ observationId: "model-receipt", requestId: "model", source: "provider", provider: "anthropic",+ currency: "USD", route: "direct", classification: "reported", costUsd: "1", final: true });+ await ledger.finish("model", "succeeded");+ expect(ledger.snapshot()).toMatchObject({ knownUsd: "1.000000000", heldUsd: "4.000000000", committedUsd: "5.000000000" });+ } finally { await ledger.close(); }+ const changed = structuredClone(limits); changed.overhead!.components[0]!.maxUsd = "0";+ await expect(openEvalCostLedger(file, changed)).rejects.toThrow("limits changed");+ const resumed = await openEvalCostLedger(file, limits);+ try {+ await resumed.finish("overhead:ci_compute", "cancelled");+ expect(resumed.snapshot().heldUsd).toBe("4.000000000");+ await resumed.observe({ observationId: "ci-partial", requestId: "overhead:ci_compute", source: "infrastructure", provider: "aws",+ currency: "USD", route: "infrastructure", classification: "estimated", pricingVersion: "fixture", costUsd: "0.2", final: false });+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "1.200000000", heldUsd: "3.800000000", committedUsd: "5.000000000", unknownRequests: 4 });+ await resumed.observe({ observationId: "ci-invoice", requestId: "overhead:ci_compute", source: "invoice", provider: "aws",+ currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "0.5", final: true });+ expect(resumed.snapshot()).toMatchObject({ knownUsd: "1.500000000", heldUsd: "3.000000000", committedUsd: "4.500000000", unknownRequests: 3 });+ } finally { await resumed.close(); }+ const report = await readEvalOperationsReport(file);+ expect(report).toMatchObject({ requests: 5, unresolvedRequests: 3, knownUsd: "1.500000000", heldUsd: "3.000000000" });+ expect(renderEvalOperationsReport(report)).toContain("campaign-overhead");+ expect(renderEvalOperationsReport(report)).toContain("Held reservations are not charges");+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it("stops further model admission when a later infrastructure receipt exceeds the shared cap", async () => {+ const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-overrun-"));+ try {+ const ledger = await openEvalCostLedger(path.join(directory, "ledger.jsonl"), { campaignId: "overrun", maxUsd: "20",+ deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() });+ try {+ await ledger.admitAttempt(modelIdentity.executionId, modelIdentity.attemptId);+ await ledger.observe({ observationId: "late-invoice", requestId: "overhead:ci_compute", source: "invoice", provider: "aws",+ currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "21", final: true });+ expect(ledger.snapshot()).toMatchObject({ overBudget: true, committedUsd: "24.000000000" });+ await expect(ledger.reserve("next", modelIdentity, "0.1")).rejects.toThrow("spending cap");+ } finally { await ledger.close(); }+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it("requires each cost category once and rejects unknown policy fields or missing provenance", () => {+ expect(validateOperatingOverhead(allowance(), "20")).toBe(4_000_000_000n);+ expect(() => validateOperatingOverhead({ ...allowance(), components: allowance().components.slice(1) }, "20")).toThrow("Incomplete");+ const duplicate = allowance(); duplicate.components[0] = duplicate.components[1]!;+ expect(() => validateOperatingOverhead(duplicate, "20")).toThrow("component");+ expect(() => validateOperatingOverhead({ ...allowance(), apiKey: "never-allowed" } as never, "20")).toThrow("fields");+ const missing = allowance(); missing.components[0]!.reference = "";+ expect(() => validateOperatingOverhead(missing, "20")).toThrow("component");+ expect(() => validateOperatingOverhead(null as never, "20")).toThrow("fields");+ expect(() => validateOperatingOverhead({ ...allowance(), expiresAt: "2027-10-20T00:00:00Z" }, "20")).toThrow("Incomplete");+ expect(() => validateOperatingOverhead(allowance(), "4")).toThrow("no model allowance");+});++it("requires fresh reviewed allowances tied to the actual CI runner before scheduled work", () => {+ const hosted = { ...allowance(), runner: "github-hosted" as const };+ const raw = JSON.stringify(hosted);+ expect(scheduledOperatingOverhead(raw, runner, "20", now)).toEqual(hosted);+ expect(() => scheduledOperatingOverhead(undefined, runner, "20", now)).toThrow("Missing reviewed");+ expect(() => scheduledOperatingOverhead("not json", runner, "20", now)).toThrow("JSON");+ expect(() => scheduledOperatingOverhead(JSON.stringify(allowance()), runner, "20", now)).toThrow("runner");+ expect(() => scheduledOperatingOverhead(raw, "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", "20", now)).toThrow("runner");+ expect(() => scheduledOperatingOverhead(raw, "unrecognized", "20", now)).toThrow("runner");+ expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-10-19T23:59:59Z"))).toThrow("needs review");+ expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-11-18T20:00:00Z"))).toThrow("needs review");+});++it("does not turn a zero allowance into an observed zero charge", async () => {+ const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-zero-"));+ const overhead = allowance(); overhead.components[0]!.maxUsd = "0";+ try {+ const ledger = await openEvalCostLedger(path.join(directory, "ledger.jsonl"), { campaignId: "zero", maxUsd: "20",+ deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead });+ try {+ expect(ledger.snapshot().requests.find(row => row.requestId === "overhead:ci_compute"))+ .toMatchObject({ costUsd: null, classification: "unknown", receiptComplete: false, reservedUsd: "0.000000000" });+ } finally { await ledger.close(); }+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it("stops new admission after allowance expiry while preserving late invoice reconciliation", () => {+ const state = new EvalSpendState({ campaignId: "expired", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z",+ maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: allowance() });+ const admitted = { type: "attempt" as const, executionId: modelIdentity.executionId, attemptId: modelIdentity.attemptId };+ state.validate(admitted, now); state.apply(admitted);+ const expiredAt = Date.parse(allowance().expiresAt);+ expect(() => state.validate({ ...admitted, attemptId: "next" }, expiredAt)).toThrow("needs review");+ expect(() => state.validate({ type: "reserve", requestId: "model", identity: modelIdentity, maxUsd: "1" }, expiredAt)).toThrow("needs review");+ expect(() => state.validate({ type: "observe", observation: { observationId: "late-invoice", requestId: "overhead:ci_compute",+ source: "invoice", provider: "aws", currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "0.5", final: true } }, expiredAt)).not.toThrow();+});diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.tsnew file mode 100644index 0000000000..08396df0ef--- /dev/null+++ b/tests/runner-e2e/operating-overhead.ts@@ -0,0 +1,53 @@+import { usdToUnits } from "../../packages/shared/src/money.js";++export const EVAL_OVERHEAD_COMPONENTS = ["ci_compute", "artifact_storage", "observability", "shared_subscription"] as const;+export interface EvalOperatingOverhead {+ version: 1;+ reference: string;+ runner: "github-hosted" | "aws-fleet";+ verifiedAt: string;+ expiresAt: string;+ components: Array<{ component: typeof EVAL_OVERHEAD_COMPONENTS[number]; maxUsd: string; reference: string }>;+}++const object = (value: unknown, allowed: string[]) => {+ if (!value || typeof value !== "object" || Array.isArray(value)+ || Object.keys(value).some(key => !allowed.includes(key))) throw new Error("Invalid operating overhead fields");+};+const reference = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value);++export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd: string): bigint {+ object(value, ["version", "reference", "runner", "verifiedAt", "expiresAt", "components"]);+ if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet"].includes(value.runner)+ || typeof value.verifiedAt !== "string" || typeof value.expiresAt !== "string"+ || !Number.isFinite(Date.parse(value.verifiedAt)) || !Number.isFinite(Date.parse(value.expiresAt))+ || Date.parse(value.expiresAt) <= Date.parse(value.verifiedAt)+ || Date.parse(value.expiresAt) - Date.parse(value.verifiedAt) > 31 * 86_400_000+ || !Array.isArray(value.components) || value.components.length !== EVAL_OVERHEAD_COMPONENTS.length) {+ throw new Error("Incomplete operating overhead allowance");+ }+ const seen = new Set<string>();+ let reserved = 0n;+ for (const component of value.components) {+ object(component, ["component", "maxUsd", "reference"]);+ if (!EVAL_OVERHEAD_COMPONENTS.includes(component.component) || seen.has(component.component)+ || !reference(component.reference) || typeof component.maxUsd !== "string"+ || !/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(component.maxUsd)) throw new Error("Invalid operating overhead component");+ seen.add(component.component);+ reserved += usdToUnits(component.maxUsd);+ }+ if (reserved >= usdToUnits(totalUsd)) throw new Error("Operating overhead leaves no model allowance");+ return reserved;+}++export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number): EvalOperatingOverhead {+ if (!raw || raw.length > 16_384) throw new Error("Missing reviewed operating overhead allowance");+ let value: EvalOperatingOverhead;+ try { value = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }+ validateOperatingOverhead(value, totalUsd);+ if (runner !== "ubuntu-latest" || value.runner !== "github-hosted") throw new Error("Bounded campaigns require the standard GitHub-hosted runner allowance");+ if (!Number.isFinite(now) || now < Date.parse(value.verifiedAt) || now + 4 * 3_600_000 >= Date.parse(value.expiresAt)) {+ throw new Error("Operating overhead allowance needs review before paid admission");+ }+ return value;+}diff --git a/tests/runner-e2e/operating-receipts.test.ts b/tests/runner-e2e/operating-receipts.test.tsnew file mode 100644index 0000000000..4ce448949f--- /dev/null+++ b/tests/runner-e2e/operating-receipts.test.ts@@ -0,0 +1,216 @@+import { createHash } from "node:crypto";+import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";+import * as fs from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, it, vi } from "vitest";+import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js";+import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js";+import { importOperatingReceipt, operatingReceiptObservation, type EvalOperatingReceipt } from "./operating-receipts.js";++vi.mock("node:fs/promises", async (importOriginal) => {+ const actual = await importOriginal<typeof fs>();+ return { ...actual, open: vi.fn(actual.open) };+});++const evidence = Buffer.from("fixture-only infrastructure receipt");+const overhead: EvalOperatingOverhead = { version: 1, runner: "github-hosted", reference: "fixture-allowance",+ verifiedAt: "2026-10-01T00:00:00Z", expiresAt: "2026-11-01T00:00:00Z",+ components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) };+const receipt = (fields: Partial<EvalOperatingReceipt> = {}): EvalOperatingReceipt => ({ version: 2, costBasis: "component_cumulative", campaignId: "fixture",+ allowanceReference: overhead.reference, component: "artifact_storage", evidenceSha256: createHash("sha256").update(evidence).digest("hex"),+ windowStartedAt: "2026-10-01T00:00:00Z", windowEndedAt: "2026-10-02T00:00:00Z", coverage: "partial",+ source: "infrastructure", provider: "fixture", classification: "estimated", costUsd: "0.2", pricingVersion: "fixture-pricing", ...fields });+const now = Date.parse("2026-10-03T00:00:00Z");++it("binds operating observations to the campaign, allowance, evidence and closed observation window", () => {+ expect(operatingReceiptObservation(receipt(), "fixture", overhead, evidence, now)).toMatchObject({ requestId: "overhead:artifact_storage", final: false });+ for (const changed of [receipt({ campaignId: "other" }), receipt({ allowanceReference: "other" }), receipt({ evidenceSha256: "0".repeat(64) }),+ receipt({ windowEndedAt: "2026-10-04T00:00:00Z" }), receipt({ windowStartedAt: "2026-10-03T00:00:00Z" }),+ receipt({ component: "model" as never }), receipt({ version: 1 as never }), receipt({ costBasis: undefined }),+ receipt({ costBasis: "window_charge" as never }), { ...receipt(), apiKey: "disallowed" }]) {+ expect(() => operatingReceiptObservation(changed, "fixture", overhead, evidence, now)).toThrow();+ }+ expect(() => operatingReceiptObservation(receipt(), "fixture", overhead, Buffer.from("different"), now)).toThrow();+});++it("does not upgrade missing or partial evidence to an invoice or completed receipt", () => {+ for (const changed of [receipt({ classification: "reconciled" }), receipt({ classification: "unknown" }),+ receipt({ classification: "unknown", costUsd: undefined, coverage: "complete" }), receipt({ pricingVersion: undefined }), receipt({ costUsd: "-1" })]) {+ expect(() => operatingReceiptObservation(changed, "fixture", overhead, evidence, now)).toThrow();+ }+ expect(operatingReceiptObservation(receipt({ classification: "unknown", costUsd: undefined }), "fixture", overhead, evidence, now))+ .toMatchObject({ classification: "unknown", final: false });+ expect(operatingReceiptObservation(receipt({ component: "shared_subscription", classification: "allocated", coverage: "complete" }), "fixture", overhead, evidence, now))+ .toMatchObject({ classification: "allocated", route: "subscription", final: true });+});++async function fixture(maxUsd = "20") {+ const directory = await mkdtemp(path.join(tmpdir(), "operating-receipts-"));+ const ledgerPath = path.join(directory, "ledger.jsonl");+ const ledger = await openEvalCostLedger(ledgerPath, { campaignId: "fixture", maxUsd, deadlineAt: "2099-01-01T00:00:00Z",+ maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead });+ await ledger.close();+ return { directory, ledgerPath };+}++it("retains private source evidence, imports idempotently and keeps partial holds until invoice reconciliation", async () => {+ const { directory, ledgerPath } = await fixture();+ try {+ expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000", unknownRequests: 4 });+ const before = await readFile(ledgerPath, "utf8");+ await importOperatingReceipt(ledgerPath, Object.fromEntries(Object.entries(receipt()).reverse()) as unknown as EvalOperatingReceipt, evidence);+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ const retained = path.join(`${ledgerPath}.overhead`, `${receipt().evidenceSha256}.evidence`);+ expect(await readFile(retained)).toEqual(evidence);+ expect((await stat(retained)).mode & 0o777).toBe(0o600);+ expect(await readdir(`${ledgerPath}.overhead`)).toHaveLength(2);+ expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.3", coverage: "complete" }), evidence))+ .toMatchObject({ knownUsd: "0.300000000", heldUsd: "3.000000000", unknownRequests: 3 });+ expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.4", coverage: "complete" }), evidence))+ .toMatchObject({ knownUsd: "0.400000000", heldUsd: "3.600000000", conflictingRequests: 1 });+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it("fails before ledger mutation when retained evidence is corrupt or another writer owns the journal", async () => {+ const { directory, ledgerPath } = await fixture();+ try {+ await importOperatingReceipt(ledgerPath, receipt(), evidence);+ const before = await readFile(ledgerPath, "utf8");+ await writeFile(path.join(`${ledgerPath}.overhead`, `${receipt().evidenceSha256}.evidence`), "corrupted");+ await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "0.5" }), evidence)).rejects.toThrow("conflicts");+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits);+ try { await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("writer lock"); }+ finally { await writer.close(); }+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it("rejects separate partial windows and accounts for their cumulative total before admitting more work", async () => {+ const { directory, ledgerPath } = await fixture("4.5");+ try {+ await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8" }), evidence);+ const before = await readFile(ledgerPath, "utf8");+ await expect(importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8",+ windowStartedAt: "2026-10-02T00:00:00Z", windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).rejects.toThrow("cumulative component window");+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "1.6",+ windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).toMatchObject({ knownUsd: "1.600000000", heldUsd: "3.000000000", committedUsd: "4.600000000", overBudget: true });+ const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits);+ try {+ await writer.admitAttempt("fixture-cell", "next");+ await expect(writer.reserve("next", { executionId: "fixture-cell", attemptId: "next", phase: "agent" }, "0.01"))+ .rejects.toThrow("spending cap reached");+ } finally { await writer.close(); }+ await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "0.9" }), evidence)).rejects.toThrow("cumulative component window");+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it.each(["partial", "complete"] as const)("rejects a larger cumulative estimate hidden by an earlier %s invoice", async coverage => {+ const { directory, ledgerPath } = await fixture("4.5");+ try {+ await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8", coverage }), evidence);+ const before = await readFile(ledgerPath, "utf8"), retained = await readdir(`${ledgerPath}.overhead`);+ await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "1.6", coverage,+ windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).rejects.toThrow("would understate the cumulative total");+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ expect(await readdir(`${ledgerPath}.overhead`)).toEqual(retained);+ const updated = await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "1.6", coverage,+ windowEndedAt: "2026-10-03T00:00:00Z" }), evidence);+ expect(updated).toMatchObject({ knownUsd: "1.600000000", committedUsd: "4.600000000", overBudget: true });+ const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits);+ try {+ await writer.admitAttempt("fixture-cell", "next");+ await expect(writer.reserve("next", { executionId: "fixture-cell", attemptId: "next", phase: "agent" }, "0.01"))+ .rejects.toThrow(coverage === "complete" ? "receipts conflict" : "spending cap reached");+ } finally { await writer.close(); }+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it.each(["evidence", "receipt.json"])("retries must persist the existing %s file before recording its charge", async suffix => {+ const { directory, ledgerPath } = await fixture();+ const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual<typeof fs>("node:fs/promises")).open;+ const value = receipt(), canonical = Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)));+ const observation = operatingReceiptObservation(value, "fixture", overhead, evidence, now);+ await mkdir(`${ledgerPath}.overhead`, { mode: 0o700 });+ await writeFile(path.join(`${ledgerPath}.overhead`, suffix === "evidence" ? `${value.evidenceSha256}.evidence`+ : `${observation.observationId.split(":")[1]}.receipt.json`), suffix === "evidence" ? evidence : `${JSON.stringify(canonical, null, 2)}\n`, { mode: 0o600 });+ let syncs = 0;+ const opened = vi.mocked(fs.open).mockImplementation(async (...args) => {+ const handle = await originalOpen(...args);+ if (String(args[0]).endsWith(`.${suffix}`)) {+ const originalSync = handle.sync.bind(handle);+ vi.spyOn(handle, "sync").mockImplementation(async () => {+ if (++syncs <= 2) throw new Error("fixture file sync failure");+ await originalSync();+ });+ }+ return handle;+ });+ try {+ for (let attempt = 0; attempt < 2; attempt++) {+ await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("file sync failure");+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ }+ expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000" });+ expect(syncs).toBe(3);+ } finally {+ opened.mockImplementation(originalOpen);+ await rm(directory, { recursive: true, force: true });+ }+});++it.each(["evidence", "receipt.json"])("recovers an incomplete %s write without publishing corrupt evidence or requiring manual deletion", async suffix => {+ const { directory, ledgerPath } = await fixture();+ const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual<typeof fs>("node:fs/promises")).open;+ let injected = false;+ const opened = vi.mocked(fs.open).mockImplementation(async (...args) => {+ const handle = await originalOpen(...args);+ if (!injected && String(args[0]).includes(`.${suffix}.`) && String(args[0]).endsWith(".tmp")) {+ const originalWrite = handle.writeFile.bind(handle);+ vi.spyOn(handle, "writeFile").mockImplementation(async () => {+ injected = true;+ await originalWrite("partial");+ throw new Error("fixture partial write");+ });+ }+ return handle;+ });+ try {+ await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("partial write");+ expect(injected).toBe(true);+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ expect((await readdir(`${ledgerPath}.overhead`)).some(name => name.endsWith(`.${suffix}`) || name.endsWith(".tmp"))).toBe(false);+ expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000" });+ expect(await readdir(`${ledgerPath}.overhead`)).toHaveLength(2);+ } finally {+ opened.mockImplementation(originalOpen);+ await rm(directory, { recursive: true, force: true });+ }+});++it("retains the hold when the first evidence directory cannot be persisted in its parent", async () => {+ const { directory, ledgerPath } = await fixture();+ const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual<typeof fs>("node:fs/promises")).open;+ let parentSyncs = 0;+ const opened = vi.mocked(fs.open).mockImplementation(async (...args) => {+ const handle = await originalOpen(...args);+ if (args[0] === directory && args[1] === "r") {+ vi.spyOn(handle, "sync").mockImplementation(async () => {+ parentSyncs++;+ throw new Error("fixture parent directory sync failure");+ });+ }+ return handle;+ });+ try {+ await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("parent directory sync failure");+ expect(parentSyncs).toBe(1);+ expect(await readFile(ledgerPath, "utf8")).toBe(before);+ expect((await readEvalCostLedger(ledgerPath)).state.snapshot()).toMatchObject({ knownUsd: "0.000000000", heldUsd: "4.000000000" });+ } finally {+ opened.mockImplementation(originalOpen);+ await rm(directory, { recursive: true, force: true });+ }+});diff --git a/tests/runner-e2e/operating-receipts.ts b/tests/runner-e2e/operating-receipts.tsnew file mode 100644index 0000000000..8806292b0c--- /dev/null+++ b/tests/runner-e2e/operating-receipts.ts@@ -0,0 +1,136 @@+import { createHash, randomUUID } from "node:crypto";+import { link, mkdir, open, readFile, rm } from "node:fs/promises";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { usdToUnits } from "../../packages/shared/src/money.js";+import { openEvalCostLedger, readEvalCostLedger, validEvalLedgerId, type EvalCostObservation } from "./cost-ledger.js";+import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js";+import { collectGitHubOperatingUsage } from "./github-operating-usage.js";++export interface EvalOperatingReceipt {+ version: 2;+ costBasis: "component_cumulative";+ campaignId: string;+ allowanceReference: string;+ component: typeof EVAL_OVERHEAD_COMPONENTS[number];+ evidenceSha256: string;+ windowStartedAt: string;+ windowEndedAt: string;+ coverage: "complete" | "partial";+ source: "infrastructure" | "invoice";+ provider: string;+ classification: EvalCostObservation["classification"];+ costUsd?: string;+ pricingVersion?: string;+}+const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex");+const canonical = (receipt: EvalOperatingReceipt) => Object.fromEntries(Object.entries(receipt).sort(([left], [right]) => left.localeCompare(right)));++export function operatingReceiptObservation(receipt: EvalOperatingReceipt, campaignId: string, overhead: EvalOperatingOverhead,+ evidence: Buffer, now = Date.now()): EvalCostObservation {+ if (!receipt || typeof receipt !== "object" || Array.isArray(receipt) || Object.keys(receipt).some(key => ![+ "version", "costBasis", "campaignId", "allowanceReference", "component", "evidenceSha256", "windowStartedAt", "windowEndedAt",+ "coverage", "source", "provider", "classification", "costUsd", "pricingVersion",+ ].includes(key))) throw new Error("Invalid operating receipt fields");+ if (receipt.version !== 2 || receipt.costBasis !== "component_cumulative"+ || receipt.campaignId !== campaignId || receipt.allowanceReference !== overhead.reference+ || !EVAL_OVERHEAD_COMPONENTS.includes(receipt.component) || !validEvalLedgerId(receipt.provider)+ || !["infrastructure", "invoice"].includes(receipt.source) || !["complete", "partial"].includes(receipt.coverage)+ || receipt.evidenceSha256 !== hash(evidence) || !/^[a-f0-9]{64}$/.test(receipt.evidenceSha256)+ || !Number.isFinite(now) || typeof receipt.windowStartedAt !== "string" || typeof receipt.windowEndedAt !== "string"+ || !Number.isFinite(Date.parse(receipt.windowStartedAt)) || !Number.isFinite(Date.parse(receipt.windowEndedAt))+ || Date.parse(receipt.windowStartedAt) > Date.parse(receipt.windowEndedAt) || Date.parse(receipt.windowEndedAt) > now) {+ throw new Error("Operating receipt scope, evidence or window mismatch");+ }+ if (!["reported", "estimated", "allocated", "reconciled", "unknown"].includes(receipt.classification)+ || receipt.classification === "reconciled" && receipt.source !== "invoice"+ || receipt.classification === "unknown" && (receipt.costUsd !== undefined || receipt.coverage === "complete")+ || receipt.classification !== "unknown" && (typeof receipt.costUsd !== "string" || !/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(receipt.costUsd))+ || ["estimated", "allocated"].includes(receipt.classification) && !validEvalLedgerId(receipt.pricingVersion)+ || receipt.pricingVersion !== undefined && !validEvalLedgerId(receipt.pricingVersion)) throw new Error("Invalid operating cost classification or provenance");+ return { observationId: `operating-receipt:${hash(JSON.stringify(canonical(receipt)))}`, requestId: `overhead:${receipt.component}`,+ source: receipt.source, provider: receipt.provider, currency: "USD",+ route: receipt.component === "shared_subscription" ? "subscription" : "infrastructure",+ classification: receipt.classification, ...(receipt.costUsd === undefined ? {} : { costUsd: receipt.costUsd }),+ ...(receipt.pricingVersion === undefined ? {} : { pricingVersion: receipt.pricingVersion }), final: receipt.coverage === "complete" };+}++async function retain(file: string, bytes: Buffer) {+ const temporary = `${file}.${randomUUID()}.tmp`;+ const handle = await open(temporary, "wx", 0o600);+ try {+ try { await handle.writeFile(bytes); await handle.sync(); } finally { await handle.close(); }+ // Hard-link publication is atomic and cannot replace previously retained evidence.+ try { await link(temporary, file); }+ catch (error) {+ if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;+ const existing = await open(file, "r+");+ try {+ if (!(await existing.readFile()).equals(bytes)) throw new Error("Retained operating evidence conflicts");+ await existing.sync();+ } finally { await existing.close(); }+ }+ } finally { await rm(temporary, { force: true }); }+}++export async function importOperatingReceipt(ledgerPath: string, receipt: EvalOperatingReceipt, evidence: Buffer) {+ if (evidence.length < 1 || evidence.length > 8 * 1024 * 1024) throw new Error("Operating evidence must be between one byte and 8 MiB");+ const loaded = await readEvalCostLedger(ledgerPath);+ if (!loaded.state.limits.overhead) throw new Error("Campaign has no declared operating overhead");+ const observation = operatingReceiptObservation(receipt, loaded.state.limits.campaignId, loaded.state.limits.overhead, evidence);+ const ledger = await openEvalCostLedger(ledgerPath, loaded.state.limits);+ try {+ const directory = `${ledgerPath}.overhead`;+ const current = (await readEvalCostLedger(ledgerPath)).state;+ const alreadyImported = current.observations.has(observation.observationId);+ if (!alreadyImported) {+ for (const previous of current.observations.values()) {+ if (previous.requestId !== observation.requestId) continue;+ if (!/^operating-receipt:[a-f0-9]{64}$/.test(previous.observationId)) {+ throw new Error("Operating receipt lacks cumulative window evidence");+ }+ const prior = JSON.parse(await readFile(path.join(directory, `${previous.observationId.split(":")[1]}.receipt.json`), "utf8")) as EvalOperatingReceipt;+ if (previous.observationId !== `operating-receipt:${hash(JSON.stringify(canonical(prior)))}`+ || prior.version !== 2 || prior.costBasis !== "component_cumulative") {+ throw new Error("Retained operating receipt lacks verified cumulative window evidence");+ }+ if (Date.parse(receipt.windowStartedAt) !== Date.parse(prior.windowStartedAt)+ || Date.parse(receipt.windowEndedAt) < Date.parse(prior.windowEndedAt)+ || prior.coverage === "complete" && receipt.coverage !== "complete") {+ throw new Error("Operating receipts must cover one cumulative component window; combine separate charges before importing");+ }+ }+ // Preview the ledger's own precedence rules before retaining or appending a receipt.+ current.validate({ type: "observe", observation });+ current.apply({ type: "observe", observation });+ const counted = current.snapshot().requests.find(request => request.requestId === observation.requestId)?.costUsd;+ if (observation.costUsd !== undefined && (counted == null || usdToUnits(counted) < usdToUnits(observation.costUsd))) {+ throw new Error("Operating receipt would understate the cumulative total; reconcile the entire component window before importing");+ }+ }+ await mkdir(directory, { recursive: true, mode: 0o700 });+ await retain(path.join(directory, `${receipt.evidenceSha256}.evidence`), evidence);+ await retain(path.join(directory, `${observation.observationId.split(":")[1]}.receipt.json`), Buffer.from(`${JSON.stringify(canonical(receipt), null, 2)}\n`));+ for (const durableDirectory of [directory, path.dirname(directory)]) {+ const directoryHandle = await open(durableDirectory, "r");+ try { await directoryHandle.sync(); } finally { await directoryHandle.close(); }+ }+ if (!alreadyImported) await ledger.observe(observation);+ return ledger.snapshot();+ } finally { await ledger.close(); }+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const [command, first, second, third, ...rest] = process.argv.slice(2);+ if (command === "github" && first && second && !third) {+ const usage = await collectGitHubOperatingUsage(first);+ await retain(path.resolve(second), Buffer.from(`${JSON.stringify(usage, null, 2)}\n`));+ console.log(JSON.stringify({ source: usage.workflowUrl, summedJobSeconds: usage.summedJobSeconds,+ retainedArtifactBytes: usage.artifacts.retainedBytes, billing: "unknown", output: path.resolve(second) }));+ } else if (command === "import" && first && second && third && !rest.length) {+ const receipt = JSON.parse(await readFile(second, "utf8")) as EvalOperatingReceipt;+ const result = await importOperatingReceipt(path.resolve(first), receipt, await readFile(third));+ console.log(JSON.stringify({ campaignId: result.campaignId, knownUsd: result.knownUsd, heldUsd: result.heldUsd,+ committedUsd: result.committedUsd, unknownRequests: result.unknownRequests, conflictingRequests: result.conflictingRequests }));+ } else throw new Error("Usage: operating-receipts.ts github <run-id> <private-output> | import <ledger> <receipt> <evidence>");+}diff --git a/tests/runner-e2e/operations-report.test.ts b/tests/runner-e2e/operations-report.test.tsnew file mode 100644index 0000000000..a310ddd35e--- /dev/null+++ b/tests/runner-e2e/operations-report.test.ts@@ -0,0 +1,93 @@+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, it } from "vitest";+import { openEvalCostLedger } from "./cost-ledger.js";+import { readEvalOperationsReport, renderEvalOperationsReport, validateEvalOperationsReport } from "./operations-report.js";+import { buildRunnerCampaign } from "./history.js";+import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";++it("reports shared setup/retry spending once, preserves unknown holds and omits private provider receipts", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-"));+ const file = path.join(root, "ledger.jsonl");+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 });+ try {+ await ledger.reserve("setup", { executionId: "cell", attemptId: "first", phase: "setup" }, "0.2");+ await ledger.observe({ observationId: "gateway", requestId: "setup", provider: "fixture", providerRequestId: "private-provider-id", source: "gateway",+ currency: "USD", route: "direct", classification: "estimated", pricingVersion: "fixture", costUsd: "0.05", final: true });+ await ledger.observe({ observationId: "native", requestId: "setup", provider: "fixture", providerRequestId: "private-provider-id", source: "native_otlp",+ currency: "USD", route: "direct", classification: "reported", costUsd: "0.05", final: true });+ await ledger.finish("setup", "failed");+ await ledger.reserve("retry", { executionId: "cell", attemptId: "retry", phase: "agent" }, "0.2");+ await ledger.finish("retry", "cancelled");+ const report = await readEvalOperationsReport(file);+ validateEvalOperationsReport(report);+ if (report.status === "unavailable") throw new Error("Expected readable fixture ledger");+ expect(report).toMatchObject({ knownUsd: "0.050000000", heldUsd: "0.200000000", committedUsd: "0.250000000", requests: 2,+ unresolvedRequests: 1, inflight: 0, breakdown: { reported: "0.050000000", estimated: "0.000000000" },+ attempts: [{ attemptId: "first", requests: 1, knownUsd: "0.050000000" }, { attemptId: "retry", unresolvedRequests: 1, heldUsd: "0.200000000" }] });+ expect(JSON.stringify(report)).not.toContain("private-provider-id");+ const campaign = buildRunnerCampaign({ campaignId: "retry-campaign", generatedAt: new Date().toISOString(), expected: [], results: [], operations: report });+ expect(campaign.operations).toEqual(report);+ expect(campaign.billing.reportedLlmCostUsd).toBe(0);+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });+ const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));+ expect(rebuilt.operations).toEqual(report);+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("$0.200000000 additional held reservations");+ expect(renderEvalOperationsReport(report)).toContain("Do not add its model subtotal to heartbeat totals");+ expect(renderEvalOperationsReport(report)).toContain("Unrecorded infrastructure, storage, subscriptions");+ expect(() => validateEvalOperationsReport({ ...report, secret: "forbidden" } as never)).toThrow("Invalid retained");+ expect(() => validateEvalOperationsReport({ ...report, knownUsd: "0.060000000" })).toThrow("Invalid retained");+ expect(() => validateEvalOperationsReport({ ...report, attempts: [] })).toThrow("Invalid retained");+ expect(() => validateEvalOperationsReport({ ...report, deadlineAt: "Tue, 20 Oct 2026 01:00:00 GMT (private note)" })).toThrow("Invalid retained");+ } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); }+});++it("publishes the parsed deadline without private text accepted by the journal's date parser", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl");+ const deadlineAt = "Thu, 01 Jan 2099 00:00:00 GMT (private budget rationale)";+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt, maxRequests: 10, maxConcurrent: 2 });+ try {+ const report = await readEvalOperationsReport(file);+ expect(report).toMatchObject({ deadlineAt: "2099-01-01T00:00:00.000Z" });+ expect(JSON.stringify(report)).not.toContain("private budget rationale");+ expect(await readFile(file, "utf8")).toContain(deadlineAt);+ } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); }+});++it("preserves reports with explicit unknown accounting after a partial journal write", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl");+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 });+ await ledger.reserve("request", { executionId: "cell", attemptId: "attempt", phase: "setup" }, "0.2");+ await ledger.close();+ try {+ await writeFile(file, '{"partial-record":', { flag: "a" });+ const damaged = await readFile(file, "utf8");+ const report = await readEvalOperationsReport(file);+ expect(report).toEqual({ version: 1, scope: "shared_gateway_ledger", status: "unavailable" });+ validateEvalOperationsReport(report);+ expect(renderEvalOperationsReport(report)).toContain("Request amounts, counts, and held reservations are unknown");+ expect(renderEvalOperationsReport(report)).not.toContain("$0");+ expect(() => validateEvalOperationsReport({ ...report, knownUsd: "0.000000000" } as never)).toThrow("Invalid retained");+ const campaign = buildRunnerCampaign({ campaignId: "damaged-ledger", generatedAt: new Date().toISOString(), expected: [], results: [], operations: report });+ await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign));+ await regenerateRunnerDashboard({ bundle: root, historyFile: null });+ const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8"));+ expect(rebuilt.operations).toEqual(report);+ expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("Request ledger accounting unavailable");+ expect(await readFile(file, "utf8")).toBe(damaged);+ } finally { await rm(root, { recursive: true, force: true }); }+});++it("keeps an in-flight final receipt reserved until the forwarding request finishes", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl");+ const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 });+ try {+ await ledger.reserve("request", { executionId: "cell", attemptId: "attempt", phase: "agent" }, "0.2");+ await ledger.observe({ observationId: "usage", requestId: "request", provider: "fixture", source: "provider", currency: "USD", route: "direct", classification: "reported", costUsd: "0.05", final: true });+ const report = await readEvalOperationsReport(file);+ validateEvalOperationsReport(report);+ expect(report).toMatchObject({ inflight: 1, knownUsd: "0.050000000", heldUsd: "0.150000000", committedUsd: "0.200000000", unresolvedRequests: 0 });+ } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); }+});diff --git a/tests/runner-e2e/operations-report.ts b/tests/runner-e2e/operations-report.tsnew file mode 100644index 0000000000..97d8a3df77--- /dev/null+++ b/tests/runner-e2e/operations-report.ts@@ -0,0 +1,114 @@+import { readEvalCostLedger, validEvalLedgerId } from "./cost-ledger.js";+import { usdToUnits } from "../../packages/shared/src/money.js";+import { html, isPublicReportTimestamp } from "./report-format.js";++const money = (n: bigint) => `${n / 1_000_000_000n}.${String(n % 1_000_000_000n).padStart(9, "0")}`;+interface EvalOperationsAvailableReport {+ version: 1;+ scope: "shared_gateway_ledger";+ status?: never;+ ledgerCampaignId: string;+ journalHash: string;+ records: number;+ deadlineAt: string;+ limitUsd: string;+ knownUsd: string;+ heldUsd: string;+ committedUsd: string;+ requests: number;+ unresolvedRequests: number;+ conflictingRequests: number;+ inflight: number;+ breakdown: { reported: string; estimated: string; allocated: string; reconciled: string };+ attempts: Array<{ executionId: string; attemptId: string; requests: number; unresolvedRequests: number; knownUsd: string; heldUsd: string }>;+}+export type EvalOperationsReport = EvalOperationsAvailableReport | {+ version: 1; scope: "shared_gateway_ledger"; status: "unavailable";+};++export function validateEvalOperationsReport(report: EvalOperationsReport): void {+ const fail = (): never => { throw new Error("Invalid retained eval operations report"); };+ const fields = (value: unknown, allowed: string[]) => {+ if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(key => !allowed.includes(key))) fail();+ };+ if (report?.status === "unavailable") {+ fields(report, ["version", "scope", "status"]);+ if (report.version !== 1 || report.scope !== "shared_gateway_ledger") fail();+ return;+ }+ const dollars = (value: unknown): bigint => typeof value === "string" && /^(?:0|[1-9][0-9]*)\.[0-9]{9}$/.test(value) ? usdToUnits(value) : fail();+ const count = (value: unknown) => { if (!Number.isSafeInteger(value) || Number(value) < 0) fail(); };+ fields(report, ["version", "scope", "ledgerCampaignId", "journalHash", "records", "deadlineAt", "limitUsd", "knownUsd", "heldUsd", "committedUsd", "requests", "unresolvedRequests", "conflictingRequests", "inflight", "breakdown", "attempts"]);+ if (report.version !== 1 || report.scope !== "shared_gateway_ledger" || !validEvalLedgerId(report.ledgerCampaignId)+ || !/^[a-f0-9]{64}$/.test(report.journalHash) || !isPublicReportTimestamp(report.deadlineAt) || !Array.isArray(report.attempts)) fail();+ [report.records, report.requests, report.unresolvedRequests, report.conflictingRequests, report.inflight].forEach(count);+ if (report.records < 1 || report.unresolvedRequests > report.requests || report.conflictingRequests > report.unresolvedRequests || report.inflight > report.requests) fail();+ dollars(report.limitUsd);+ const known = dollars(report.knownUsd), held = dollars(report.heldUsd);+ if (known + held !== dollars(report.committedUsd)) fail();+ fields(report.breakdown, ["reported", "estimated", "allocated", "reconciled"]);+ if (["reported", "estimated", "allocated", "reconciled"].reduce((sum, key) => sum + dollars(report.breakdown[key as keyof typeof report.breakdown]), 0n) !== known) fail();+ let requests = 0, unresolved = 0, rowKnown = 0n, rowHeld = 0n;+ const seen = new Set<string>();+ for (const attempt of report.attempts) {+ fields(attempt, ["executionId", "attemptId", "requests", "unresolvedRequests", "knownUsd", "heldUsd"]);+ if (!validEvalLedgerId(attempt.executionId) || !validEvalLedgerId(attempt.attemptId)) fail();+ const key = JSON.stringify([attempt.executionId, attempt.attemptId]); if (seen.has(key)) fail(); seen.add(key);+ count(attempt.requests); count(attempt.unresolvedRequests);+ if (attempt.requests < 1 || attempt.unresolvedRequests > attempt.requests) fail();+ requests += attempt.requests; unresolved += attempt.unresolvedRequests;+ rowKnown += dollars(attempt.knownUsd); rowHeld += dollars(attempt.heldUsd);+ }+ if (requests !== report.requests || unresolved !== report.unresolvedRequests || rowKnown !== known || rowHeld !== held) fail();+}++export async function readEvalOperationsReport(file: string): Promise<EvalOperationsReport> {+ try { return await readAvailableEvalOperationsReport(file); }+ catch { return { version: 1, scope: "shared_gateway_ledger", status: "unavailable" }; }+}++async function readAvailableEvalOperationsReport(file: string): Promise<EvalOperationsAvailableReport> {+ const loaded = await readEvalCostLedger(file), snapshot = loaded.state.snapshot();+ const groups = new Map<string, EvalOperationsAvailableReport["attempts"][number]>();+ for (const request of snapshot.requests) {+ const key = JSON.stringify([request.executionId, request.attemptId]);+ const group = groups.get(key) ?? { executionId: request.executionId, attemptId: request.attemptId,+ requests: 0, unresolvedRequests: 0, knownUsd: money(0n), heldUsd: money(0n) };+ const cost = usdToUnits(request.costUsd ?? "0"), reserved = usdToUnits(request.reservedUsd);+ const held = (!request.finished || !request.receiptComplete) && reserved > cost ? reserved - cost : 0n;+ group.requests++; if (!request.receiptComplete) group.unresolvedRequests++;+ group.knownUsd = money(usdToUnits(group.knownUsd) + cost);+ group.heldUsd = money(usdToUnits(group.heldUsd) + held);+ groups.set(key, group);+ }+ const report: EvalOperationsAvailableReport = { version: 1, scope: "shared_gateway_ledger", ledgerCampaignId: snapshot.campaignId,+ journalHash: loaded.previous, records: loaded.sequence, deadlineAt: new Date(loaded.state.limits.deadlineAt).toISOString(),+ limitUsd: snapshot.limitUsd, knownUsd: snapshot.knownUsd, heldUsd: snapshot.heldUsd, committedUsd: snapshot.committedUsd,+ requests: snapshot.requests.length, unresolvedRequests: snapshot.unknownRequests, conflictingRequests: snapshot.conflictingRequests,+ inflight: snapshot.inflight, breakdown: snapshot.breakdown as EvalOperationsAvailableReport["breakdown"], attempts: [...groups.values()] };+ validateEvalOperationsReport(report);+ return report;+}++export function operationsReportText(report: EvalOperationsReport): string[] {+ if (report.status === "unavailable") return [+ "Request ledger accounting unavailable: the journal could not be read or validated. Request amounts, counts, and held reservations are unknown.",+ "Retained test results are preserved. Recover and validate the original journal before resuming paid work; do not reset the allowance or treat missing amounts as zero.",+ ];+ return [+ `Shared request ledger ${report.ledgerCampaignId}: $${report.knownUsd} known subtotal; $${report.heldUsd} additional held reservations; $${report.committedUsd} committed against $${report.limitUsd}.`,+ `Charges: $${report.breakdown.reconciled} reconciled; $${report.breakdown.reported} reported; $${report.breakdown.estimated} estimated; $${report.breakdown.allocated} allocated.`,+ `${report.requests} recorded requests or overhead reservations; ${report.unresolvedRequests} unresolved; ${report.conflictingRequests} conflicting; ${report.inflight} in flight. Deadline ${report.deadlineAt}.`,+ "This shared ledger includes earlier attempts and setup probes routed through the gateway, plus any declared campaign-overhead reservations. Do not add its model subtotal to heartbeat totals. Held reservations are not charges. Unrecorded infrastructure, storage, subscriptions and requests outside the gateway still need accounting; zero recorded requests does not prove zero cost.",+ `Journal checkpoint: ${report.records} records; SHA-256 ${report.journalHash}.`,+ ];+}++export function renderEvalOperationsReport(report: EvalOperationsReport): string {+ validateEvalOperationsReport(report);+ if (report.status === "unavailable") return `<section aria-label="Request ledger accounting"><h2>Request ledger accounting unavailable</h2>${operationsReportText(report).map(line => `<p>${html(line)}</p>`).join("")}</section>`;+ return `<section aria-label="Request ledger accounting"><h2>Request ledger accounting</h2>${operationsReportText(report).map(line => `<p>${html(line)}</p>`).join("")}+ <table><thead><tr><th>Execution</th><th>Attempt or component</th><th>Requests / reservations</th><th>Known subtotal</th><th>Additional held</th><th>Unresolved</th></tr></thead><tbody>+ ${report.attempts.map(attempt => `<tr><td>${html(attempt.executionId)}</td><td>${html(attempt.attemptId)}</td><td>${attempt.requests}</td><td>$${html(attempt.knownUsd)}</td><td>$${html(attempt.heldUsd)}</td><td>${attempt.unresolvedRequests}</td></tr>`).join("")}+ </tbody></table></section>`;+}diff --git a/tests/runner-e2e/operations.test.ts b/tests/runner-e2e/operations.test.tsnew file mode 100644index 0000000000..7193837a01--- /dev/null+++ b/tests/runner-e2e/operations.test.ts@@ -0,0 +1,173 @@+import { afterEach, expect, it, vi } from "vitest";+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { createHash } from "node:crypto";+import path from "node:path";+import { tmpdir } from "node:os";+import { runnerMatrix } from "./catalog.js";+import { validateEvalOperationsManifest, openEvalOperations, type EvalOperationsManifest } from "./operations.js";+import { setupLiveFixtures } from "./live-fixtures.js";+import { createAiConnectionSchema, aiConnectionBindingSchema } from "../../packages/shared/src/ai-connections.js";+import type { RunnerApi } from "./api.js";+import { beginEvalAgentPhase, readEvalAttemptPhase } from "./attempt-phase.js";+import { applyEvalGatewayEnvironment, buildPaperclipServerEnvironment } from "./harness-env.js";+const cell = runnerMatrix.find(e => e.suite.id === "core-compatibility" && e.profile.id === "runner-codex" && e.environment.id === "local")!;+function manifest(): EvalOperationsManifest {+ return { version: 1, limits: { campaignId: "pilot", maxUsd: "100", deadlineAt: new Date(Date.now() + 60_000).toISOString(), maxRequests: 20, maxConcurrent: 1 },+ ledgerPath: "/tmp/fixture-not-opened.jsonl", executionIds: [cell.id], maxAttemptsPerExecution: 2, route: "direct",+ models: [{ provider: "openai", maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 1000,+ price: { provider: "openai", model: cell.profile.model, version: "fixture-only", sourceUrl: "https://provider.invalid/fixture", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" } }] };+}+afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); });+it("admits only the exact bounded local selection and explicitly priced models", () => {+ expect(validateEvalOperationsManifest(manifest(), [cell], 1)).toMatchObject({ route: "direct" });+ expect(() => validateEvalOperationsManifest({ ...manifest(), executionIds: [] }, [cell], 1)).toThrow("selection");+ expect(() => validateEvalOperationsManifest(manifest(), [cell], 2)).toThrow("limits");+ const remote = runnerMatrix.find(e => e.suite.id === cell.suite.id && e.profile.id === cell.profile.id && e.environment.id === "daytona")!;+ expect(() => validateEvalOperationsManifest({ ...manifest(), executionIds: [remote.id] }, [remote], 1)).toThrow("Unqualified");+ const changed = manifest(); changed.models[0]!.price.model = "other";+ expect(() => validateEvalOperationsManifest(changed, [cell], 1)).toThrow("Missing exact model");+});+it("rejects credentials in the manifest and unknown pricing/billing policy fields", () => {+ expect(() => validateEvalOperationsManifest({ ...manifest(), apiKey: "private" } as never, [cell], 0)).toThrow("fields");+ const changed = manifest(); Object.assign(changed.models[0]!, { apiKey: "private" });+ expect(() => validateEvalOperationsManifest(changed, [cell], 0)).toThrow("fields");+ expect(() => validateEvalOperationsManifest({ ...manifest(), route: "braintrust" }, [cell], 0)).toThrow("project UUID");+});++it("rejects an unaffordable selected request before opening a ledger or checking credentials", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-unaffordable-preflight-"));+ try {+ const value = manifest(); value.limits.maxUsd = "0.002999999"; value.ledgerPath = path.join(root, "ledger.jsonl");+ const file = path.join(root, "manifest.json"); await writeFile(file, JSON.stringify(value));+ await expect(openEvalOperations(file, [cell], 0, {})).rejects.toThrow("Insufficient initial model allowance");+ await expect(access(value.ledgerPath)).rejects.toMatchObject({ code: "ENOENT" });+ await expect(access(`${value.ledgerPath}.requests`)).rejects.toMatchObject({ code: "ENOENT" });+ value.limits.maxUsd = "0.003";+ expect(validateEvalOperationsManifest(value, [cell], 0)).toBe(value);+ } finally { await rm(root, { recursive: true, force: true }); }+});++it("subtracts declared operating holds and checks only models used by the selection", () => {+ const value = manifest(); value.limits.maxUsd = "0.007"; value.limits.maxAttemptsPerExecution = 2;+ value.limits.overhead = { version: 1, reference: "fixture", runner: "github-hosted",+ verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",+ components: (["ci_compute", "artifact_storage", "observability", "shared_subscription"] as const).map(component => ({+ component, maxUsd: "0.001", reference: "fixture" })) };+ value.models.push({ ...structuredClone(value.models[0]!), maximumInputTokens: 1_000_000,+ price: { ...value.models[0]!.price, model: "unused-model" } });+ expect(validateEvalOperationsManifest(value, [cell], 0)).toBe(value);+ value.limits.maxUsd = "0.006999999";+ expect(() => validateEvalOperationsManifest(value, [cell], 0)).toThrow("Insufficient initial model allowance");+});+it("retains a campaign summary and refuses changing its budget on restart", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-ops-test-"));+ try {+ const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl") }, file = path.join(root, "manifest.json");+ await writeFile(file, JSON.stringify(value));+ const operations = await openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" });+ const endpoint = await operations.start(cell, "attempt", 1);+ expect(endpoint.baseUrl).toMatch(/^http:\/\/127\.0\.0\.1:/);+ expect(await readEvalAttemptPhase(endpoint.phasePath, { executionId: cell.id, attemptId: "attempt" })).toMatchObject({ phase: "setup" });+ await beginEvalAgentPhase(cell.id, { PAPERCLIP_E2E_GATEWAY_PHASE_PATH: endpoint.phasePath, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: "attempt" });+ expect(await readEvalAttemptPhase(endpoint.phasePath, { executionId: cell.id, attemptId: "attempt" })).toMatchObject({ phase: "agent" });+ await endpoint.close(); await operations.close();+ expect(JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8"))).toMatchObject({ limitUsd: "100.000000000", knownUsd: "0.000000000", requests: [] });+ const resumed = await openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" });+ const retry = await resumed.start(cell, "retry-after-restart", 1);+ await retry.close();+ await expect(resumed.start(cell, "third-attempt", 1)).rejects.toThrow("attempt cap");+ await resumed.close();+ const summary = JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8"));+ expect(summary.admittedAttempts).toEqual([{ executionId: cell.id, attemptId: "attempt" }, { executionId: cell.id, attemptId: "retry-after-restart" }]);+ await writeFile(file, JSON.stringify({ ...value, maxAttemptsPerExecution: 3 }));+ await expect(openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" })).rejects.toThrow("limits changed");+ await writeFile(file, JSON.stringify({ ...value, limits: { ...value.limits, maxUsd: "101" } }));+ await expect(openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" })).rejects.toThrow("limits changed");+ } finally { await rm(root, { recursive: true, force: true }); }+});++it("keeps both upstream keys on the controller and owns the trace relay lifetime", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-ops-relay-"));+ const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl") }, file = path.join(root, "manifest.json");+ const env = { OPENAI_API_KEY: "provider-private", BRAINTRUST_API_KEY: "braintrust-private", BRAINTRUST_OTHER_KEY: "other-private",+ GITHUB_SHA: "a".repeat(40), PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "11111111-1111-4111-8111-111111111111",+ PAPERCLIP_E2E_OTLP_RELAY_URL: "https://unowned.invalid/otel" };+ let operations: Awaited<ReturnType<typeof openEvalOperations>> | undefined;+ try {+ await writeFile(file, JSON.stringify(value));+ operations = await openEvalOperations(file, [cell], 0, env);+ const endpoint = await operations.start(cell, "with-observability", 1);+ const worker: NodeJS.ProcessEnv = { ...env };+ applyEvalGatewayEnvironment(worker, cell.profile.credential, endpoint);+ expect(worker).toMatchObject({ OPENAI_API_KEY: endpoint.token, BRAINTRUST_API_KEY: endpoint.observability!.token,+ PAPERCLIP_E2E_OTLP_RELAY_URL: endpoint.observability!.baseUrl, GITHUB_SHA: env.GITHUB_SHA });+ expect(JSON.stringify(worker)).not.toMatch(/provider-private|braintrust-private|other-private/);+ expect(buildPaperclipServerEnvironment(worker).BRAINTRUST_API_KEY).toBeUndefined();+ expect((await fetch(endpoint.observability!.baseUrl + "/v1/traces", { method: "POST" })).status).toBe(401);+ await operations.close(); operations = undefined;+ await expect(fetch(endpoint.baseUrl)).rejects.toThrow();+ await expect(fetch(endpoint.observability!.baseUrl)).rejects.toThrow();+ const receipts = (await readFile(endpoint.observability!.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line));+ expect(receipts.map(row => row.type)).toEqual(["policy", "closed"]);+ expect(receipts[0]).toMatchObject({ campaignId: "pilot", attemptId: "with-observability", projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID });+ const withoutTracing: NodeJS.ProcessEnv = { ...env };+ applyEvalGatewayEnvironment(withoutTracing, cell.profile.credential, { ...endpoint, observability: undefined });+ expect(withoutTracing.BRAINTRUST_API_KEY).toBeUndefined();+ expect(withoutTracing.PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined();+ } finally { await operations?.close(); await rm(root, { recursive: true, force: true }); }+});++it("rejects mismatched trace projects before opening the journal and retains failed relay admission", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "eval-ops-relay-failure-"));+ const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl"), route: "braintrust" as const,+ projectId: "11111111-1111-4111-8111-111111111111", maxAttemptsPerExecution: 1 }, file = path.join(root, "manifest.json");+ const env = { BRAINTRUST_API_KEY: "braintrust-private", PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "22222222-2222-4222-8222-222222222222" };+ let operations: Awaited<ReturnType<typeof openEvalOperations>> | undefined;+ try {+ await writeFile(file, JSON.stringify(value));+ await expect(openEvalOperations(file, [cell], 0, env)).rejects.toThrow("projects differ");+ await expect(access(value.ledgerPath)).rejects.toMatchObject({ code: "ENOENT" });+ await mkdir(`${value.ledgerPath}.otlp`);+ const receipt = path.join(`${value.ledgerPath}.otlp`, `${createHash("sha256").update("collision").digest("hex")}.jsonl`);+ await writeFile(receipt, "retained-receipt");+ operations = await openEvalOperations(file, [cell], 0, { ...env, PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: value.projectId });+ await expect(operations.start(cell, "collision", 1)).rejects.toMatchObject({ code: "EEXIST" });+ await expect(operations.start(cell, "replacement", 1)).rejects.toThrow("attempt cap");+ await operations.close(); operations = undefined;+ expect(await readFile(receipt, "utf8")).toBe("retained-receipt");+ expect(JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8")).admittedAttempts)+ .toEqual([{ executionId: cell.id, attemptId: "collision" }]);+ } finally { await operations?.close(); await rm(root, { recursive: true, force: true }); }+});+it.each([+ cell.id,+ "feedback-persistence.runner-codex.local.complaint-and-suggestion",+ "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion",+])("routes %s through a shared connection without ambient provider bindings", async executionId => {+ const selected = runnerMatrix.find(execution => execution.id === executionId)!;+ const anthropic = selected.profile.credential === "ANTHROPIC_API_KEY";+ const admitted = manifest(); admitted.executionIds = [selected.id];+ admitted.models[0]!.provider = anthropic ? "anthropic" : "openai";+ admitted.models[0]!.price.provider = admitted.models[0]!.provider;+ admitted.models[0]!.price.model = selected.profile.model;+ if (anthropic) Object.assign(admitted.models[0]!.price, { cacheWrite5mUsdPerMillion: "2", cacheWrite1hUsdPerMillion: "2" });+ expect(validateEvalOperationsManifest(admitted, [selected], 0)).toBe(admitted);+ vi.stubEnv("PAPERCLIP_E2E_GATEWAY_BASE_URL", "http://127.0.0.1:43123");+ vi.stubEnv("PAPERCLIP_E2E_GATEWAY_TOKEN", "ephemeral-fixture-token");+ const id = "11111111-1111-4111-8111-111111111111", grantId = "22222222-2222-4222-8222-222222222222";+ let connection: unknown, agent: Record<string, any> | undefined, company: Record<string, unknown> | undefined;+ const api = {+ async get() { return [{ id, driver: "local" }]; },+ async postSensitive(url: string, body: unknown) { if (url.endsWith("/ai-connections")) { connection = body; return { connectionId: id, grantId }; } return { id }; },+ async post(url: string, body: Record<string, unknown>) { if (url.endsWith("/agents")) agent = body; if (url === "/api/companies") company = body; return { id, name: "fixture" }; },+ } as unknown as RunnerApi;+ const fixtures = await setupLiveFixtures({ api, execution: selected, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { [selected.profile.credential]: "ephemeral-fixture-token" } });+ await fixtures.teardown();+ expect(createAiConnectionSchema.parse(connection)).toMatchObject({ ownership: "shared", apiKey: "ephemeral-fixture-token", routing: { kind: "gateway", protocol: anthropic ? "messages" : "responses", baseUrl: `http://127.0.0.1:43123${anthropic ? "" : "/v1"}`, auth: "bearer" } });+ expect(aiConnectionBindingSchema.parse(agent!.runtimeConfig.aiConnection)).toEqual({ provider: anthropic ? "anthropic" : "openai", method: "api_key", mode: "shared", connectionId: id, grantId });+ expect(agent!.adapterConfig.env).toBeUndefined();+ if (selected.suite.id === "feedback-persistence") {+ expect(company!.budgetMonthlyCents).toBe(1_000);+ expect(agent!.budgetMonthlyCents).toBe(1_000);+ }+});diff --git a/tests/runner-e2e/operations.ts b/tests/runner-e2e/operations.tsnew file mode 100644index 0000000000..2ab81eb267--- /dev/null+++ b/tests/runner-e2e/operations.ts@@ -0,0 +1,123 @@+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";+import path from "node:path";+import { createHash } from "node:crypto";+import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js";+import { EvalSpendState, openEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js";+import { evalRequestReservation, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js";+import type { MatrixExecution } from "./types.js";+import { evalBraintrustConfig } from "./braintrust.js";+import { startEvalOtlpRelay } from "./eval-otlp-relay.js";+import { usdToUnits } from "../../packages/shared/src/money.js";++export interface EvalOperationsManifest {+ version: 1;+ limits: EvalSpendLimits;+ ledgerPath: string;+ executionIds: string[];+ maxAttemptsPerExecution: number;+ route: "direct" | "braintrust";+ projectId?: string;+ models: Array<Omit<EvalGatewayRoute, "apiKey" | "projectId" | "endpointName" | "mode">>;+}+const fields = (value: object, names: string[]) => {+ if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(name => !names.includes(name))) throw new Error("Invalid eval operations manifest fields");+};+export function validateEvalOperationsManifest(value: EvalOperationsManifest, executions: readonly MatrixExecution[], maxRetries: number) {+ fields(value, ["version", "limits", "ledgerPath", "executionIds", "maxAttemptsPerExecution", "route", "projectId", "models"]);+ const initial = new EvalSpendState(value.limits).snapshot();+ const initialModelAllowance = usdToUnits(value.limits.maxUsd) - usdToUnits(initial.committedUsd);+ if (value.version !== 1 || !["direct", "braintrust"].includes(value.route) || !path.isAbsolute(value.ledgerPath)+ || !Array.isArray(value.executionIds) || new Set(value.executionIds).size !== value.executionIds.length+ || JSON.stringify([...value.executionIds].sort()) !== JSON.stringify(executions.map(e => e.id).sort())+ || !Number.isSafeInteger(value.maxAttemptsPerExecution) || value.maxAttemptsPerExecution < 1+ || value.limits.maxAttemptsPerExecution !== undefined && value.limits.maxAttemptsPerExecution !== value.maxAttemptsPerExecution+ || maxRetries + 1 > value.maxAttemptsPerExecution || !Array.isArray(value.models) || value.models.length === 0) throw new Error("Eval operations selection or limits mismatch");+ if (value.route === "braintrust" && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value.projectId ?? "")) throw new Error("Braintrust gateway requires an explicit project UUID");+ const models = new Map<string, bigint>();+ for (const model of value.models) {+ fields(model, ["provider", "price", "maximumInputTokens", "maximumOutputTokens", "timeoutMs"]);+ const rateFields = ["inputUsdPerMillion", "cachedReadUsdPerMillion", "outputUsdPerMillion", "cacheWrite5mUsdPerMillion", "cacheWrite1hUsdPerMillion", "cacheWrite30mUsdPerMillion", "cacheWrite12hUsdPerMillion"];+ fields(model.price, ["provider", "model", "version", "sourceUrl", "longContext", ...rateFields]);+ if (model.price.longContext) { fields(model.price.longContext, ["aboveInputTokens", "rates"]); fields(model.price.longContext.rates, rateFields); }+ const url = new URL(model.price.sourceUrl);+ if (!["openai", "anthropic"].includes(model.provider) || url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("Invalid model price provenance");+ const key = `${model.provider}:${model.price.model}`;+ if (models.has(key)) throw new Error("Duplicate eval model price");+ models.set(key, usdToUnits(evalRequestReservation({ ...model, mode: value.route, apiKey: "validation" })));+ }+ for (const execution of executions) {+ if (execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id)+ || !["runner-codex", "runner-acpx-claude", "legacy-codex", "legacy-claude"].includes(execution.profile.id)+ || execution.requiredCredentials.length !== 1) throw new Error(`Unqualified budgeted eval route: ${execution.id}`);+ const provider = execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai";+ if (!models.has(`${provider}:${execution.profile.model}`)) throw new Error(`Missing exact model pricing: ${execution.id}`);+ if (models.get(`${provider}:${execution.profile.model}`)! > initialModelAllowance) {+ throw new Error(`Insufficient initial model allowance for selected eval: ${execution.id}`);+ }+ }+ return value;+}++export async function openEvalOperations(file: string, executions: readonly MatrixExecution[], maxRetries: number, env: NodeJS.ProcessEnv) {+ const manifest = validateEvalOperationsManifest(JSON.parse(await readFile(file, "utf8")), executions, maxRetries);+ if (Date.now() >= Date.parse(manifest.limits.deadlineAt)) throw new Error("Eval campaign deadline reached");+ if (manifest.limits.overhead && (Date.now() < Date.parse(manifest.limits.overhead.verifiedAt)+ || Date.parse(manifest.limits.deadlineAt) >= Date.parse(manifest.limits.overhead.expiresAt))) {+ throw new Error("Operating overhead allowance does not cover the campaign window");+ }+ const required = manifest.route === "braintrust" ? ["BRAINTRUST_API_KEY"] : executions.map(execution => execution.profile.credential);+ if (required.some(name => !env[name]?.trim())) throw new Error("Missing eval gateway upstream credential");+ const observability = evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined });+ if (observability && manifest.route === "braintrust" && observability.projectId !== manifest.projectId) throw new Error("Eval gateway and OTLP projects differ");+ const receiptDirectory = `${manifest.ledgerPath}.requests`;+ await mkdir(receiptDirectory, { recursive: true, mode: 0o700 });+ const phaseDirectory = `${manifest.ledgerPath}.attempts`;+ await mkdir(phaseDirectory, { recursive: true, mode: 0o700 });+ const ledger = await openEvalCostLedger(manifest.ledgerPath, { ...manifest.limits, maxAttemptsPerExecution: manifest.maxAttemptsPerExecution });+ const active = new Set<{ close(): Promise<void> }>();+ const atomicJson = async (file: string, value: unknown) => {+ await writeFile(`${file}.tmp`, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });+ await rename(`${file}.tmp`, file);+ };+ return {+ manifest,+ async start(execution: MatrixExecution, attemptId: string, attempt: number) {+ if (!Number.isSafeInteger(attempt) || attempt < 1 || attempt > manifest.maxAttemptsPerExecution || Date.now() >= Date.parse(manifest.limits.deadlineAt)) throw new Error("Eval operations attempt/deadline limit reached");+ if (!manifest.executionIds.includes(execution.id)) throw new Error("Eval operation is outside the admitted selection");+ const provider = execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai";+ const model = manifest.models.find(m => m.provider === provider && m.price.model === execution.profile.model)!;+ const route: EvalGatewayRoute = { ...model, mode: manifest.route, projectId: manifest.projectId,+ endpointName: provider === "openai" ? "OPENAI_API_KEY" : "ANTHROPIC_API_KEY",+ apiKey: env[manifest.route === "braintrust" ? "BRAINTRUST_API_KEY" : execution.profile.credential] ?? "" };+ if (!route.apiKey) throw new Error("Missing eval gateway upstream credential");+ await ledger.admitAttempt(execution.id, attemptId);+ const identity = { executionId: execution.id, attemptId, phase: "setup" as const };+ const phasePath = path.join(phaseDirectory, `${createHash("sha256").update(attemptId).digest("hex")}.json`);+ await createEvalAttemptPhase(phasePath, { executionId: execution.id, attemptId });+ const gateway = await startEvalGateway({ ledger, route,+ identity, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity), deadlineAt: Date.parse(manifest.limits.deadlineAt),+ persist: (receipt: EvalGatewayReceipt) => atomicJson(path.join(receiptDirectory, `${receipt.requestId}.json`), receipt),+ });+ let relay: Awaited<ReturnType<typeof startEvalOtlpRelay>> | undefined;+ try {+ if (observability) relay = await startEvalOtlpRelay({ projectId: observability.projectId, apiKey: observability.apiKey,+ campaignId: manifest.limits.campaignId, attemptId, deadlineAt: Math.min(Date.parse(manifest.limits.deadlineAt), Date.now() + 4 * 3_600_000),+ receiptPath: path.join(`${manifest.ledgerPath}.otlp`, `${createHash("sha256").update(attemptId).digest("hex")}.jsonl`) });+ } catch (error) { await gateway.close(); throw error; }+ let closing: Promise<void> | undefined;+ const endpoint = { ...gateway, phasePath, observability: relay, close: () => closing ??= (async () => {+ const results = await Promise.allSettled([gateway.close(), relay?.close()]);+ active.delete(endpoint);+ if (results.some(result => result.status === "rejected")) throw new Error("Eval gateway or OTLP relay cleanup failed");+ })() };+ active.add(endpoint);+ return endpoint;+ },+ async close() {+ const results = await Promise.allSettled([...active].map(gateway => gateway.close())); active.clear();+ try { await atomicJson(`${manifest.ledgerPath}.summary.json`, ledger.snapshot()); } finally { await ledger.close(); }+ if (results.some(result => result.status === "rejected")) throw new Error("Eval gateway cleanup failed");+ },+ };+}+export type EvalOperations = Awaited<ReturnType<typeof openEvalOperations>>;diff --git a/tests/runner-e2e/paid-runner-routing.test.mjs b/tests/runner-e2e/paid-runner-routing.test.mjsnew file mode 100644index 0000000000..669ade6481--- /dev/null+++ b/tests/runner-e2e/paid-runner-routing.test.mjs@@ -0,0 +1,51 @@+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { spawnSync } from "node:child_process";+import { expect, it } from "vitest";++const workflow = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8");+const step = workflow.split(" - name: Select paid test runner\n")[1]?.split("\n target_lock:")[0];+if (!step?.includes(" run: |\n")) throw new Error("Missing actual workflow runner selection");+const script = step.split(" run: |\n")[1].split("\n").map(line => line.replace(/^ /, "")).join("\n");++async function select(overrides) {+ const root = await mkdtemp(path.join(tmpdir(), "eval-runner-routing-"));+ try {+ const output = path.join(root, "outputs");+ const result = spawnSync("bash", ["-c", script], { encoding: "utf8", timeout: 5000, env: {+ PATH: process.env.PATH, GITHUB_OUTPUT: output, AWS_PAID_RUNNER_ENABLED: "true",+ EVENT_NAME: "workflow_dispatch", EVAL_CADENCE: "manual", REPOSITORY_PRIVATE: "false", ...overrides,+ } });+ const saved = await readFile(output, "utf8").catch(error => { if (error.code === "ENOENT") return ""; throw error; });+ return { ...result, outputs: Object.fromEntries(saved.trim().split("\n").filter(Boolean).map(line => {+ const separator = line.indexOf("="); return [line.slice(0, separator), line.slice(separator + 1)];+ })) };+ } finally { await rm(root, { recursive: true, force: true }); }+}++it.each([+ { EVENT_NAME: "schedule" },+ { EVAL_CADENCE: "daily" },+ { EVAL_CADENCE: "weekly" },+])("routes bounded $EVENT_NAME/$EVAL_CADENCE campaigns to standard hosted compute despite the AWS switch", async trigger => {+ const result = await select(trigger);+ expect(result.status).toBe(0);+ expect(result.outputs).toMatchObject({ runner: "ubuntu-latest", playwright_channel: "" });+});++it.each(["true", "", "unknown"])("refuses bounded campaigns when repository privacy is %s", async REPOSITORY_PRIVATE => {+ const result = await select({ EVAL_CADENCE: "daily", REPOSITORY_PRIVATE });+ expect(result.status).not.toBe(0);+ expect(result.stderr).toContain("require a public repository");+ expect(result.outputs).toEqual({});+});++it("preserves the explicit manual AWS and GitHub options", async () => {+ const aws = await select({ REPOSITORY_PRIVATE: "true" });+ expect(aws.status).toBe(0);+ expect(aws.outputs).toMatchObject({ runner: "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", playwright_channel: "chrome" });+ const hosted = await select({ AWS_PAID_RUNNER_ENABLED: "false" });+ expect(hosted.status).toBe(0);+ expect(hosted.outputs).toMatchObject({ runner: "ubuntu-latest", playwright_channel: "" });+});diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.tsindex 599d8c6fc2..bae4834456 100644--- a/tests/runner-e2e/pi-controls-evidence.test.ts+++ b/tests/runner-e2e/pi-controls-evidence.test.ts@@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => {expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true);expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]);expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false);- expect(validateRunnerCatalog()).toHaveLength(761);+ expect(validateRunnerCatalog()).toHaveLength(763);for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow();});diff --git a/tests/runner-e2e/private-history-archive.py b/tests/runner-e2e/private-history-archive.pynew file mode 100644index 0000000000..02c3271234--- /dev/null+++ b/tests/runner-e2e/private-history-archive.py@@ -0,0 +1,148 @@+import json+import os+from pathlib import Path, PurePosixPath+import shutil+import stat+import struct+import sys+import tempfile+import zipfile+++MAX_BYTES = 256 * 1024 * 1024+MAX_ARCHIVE_BYTES = MAX_BYTES + 8 * 1024 * 1024+MAX_FILES = 4097+++def safe_path(name):+ parts = name.split("/")+ if (not name or len(name) > 512 or "\\" in name or ":" in name+ or any(ord(c) < 32 or ord(c) == 127 for c in name)+ or any(p in ("", ".", "..") for p in parts)):+ raise ValueError("Unsafe private archive path")+ return PurePosixPath(name)+++def pack(source, destination):+ source, destination = Path(source), Path(destination)+ if source.is_symlink() or not source.is_dir():+ raise ValueError("Private archive source must be a directory")+ root = source.resolve()+ if destination.resolve().is_relative_to(root):+ raise ValueError("Private archive must be outside its source")+ files = []+ names_seen = set()+ total = 0+ visited = 0+ for directory, directories, names in os.walk(source, followlinks=False):+ visited += len(directories) + len(names)+ if visited > MAX_FILES:+ raise ValueError("Private archive exceeds entry allowance")+ for name in directories:+ entry = Path(directory) / name+ safe_path(entry.relative_to(source).as_posix())+ if entry.is_symlink():+ raise ValueError("Private archive cannot contain symlinks")+ for name in names:+ entry = Path(directory) / name+ relative = entry.relative_to(source).as_posix()+ safe_path(relative)+ if relative.casefold() in names_seen:+ raise ValueError("Private archive has colliding paths")+ names_seen.add(relative.casefold())+ metadata = entry.lstat()+ if not stat.S_ISREG(metadata.st_mode) or not entry.resolve().is_relative_to(root):+ raise ValueError("Private archive requires regular files")+ total += metadata.st_size+ if total > MAX_BYTES:+ raise ValueError("Private archive exceeds byte allowance")+ files.append((entry, relative, metadata))+ if not files:+ raise ValueError("Private archive is empty")+ output = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)+ try:+ with os.fdopen(output, "wb") as stream, zipfile.ZipFile(stream, "w", zipfile.ZIP_STORED, allowZip64=False) as archive:+ for entry, relative, before in sorted(files):+ with os.fdopen(os.open(entry, os.O_RDONLY | os.O_NOFOLLOW), "rb") as data:+ observed = os.fstat(data.fileno())+ if (observed.st_ino, observed.st_size, observed.st_mtime_ns) != (before.st_ino, before.st_size, before.st_mtime_ns):+ raise ValueError("Private archive source changed")+ info = zipfile.ZipInfo(relative)+ info.external_attr = (stat.S_IFREG | 0o600) << 16+ copied = 0+ with archive.open(info, "w") as target:+ while chunk := data.read(64 * 1024):+ copied += len(chunk)+ if copied > before.st_size:+ raise ValueError("Private archive source grew")+ target.write(chunk)+ after = os.fstat(data.fileno())+ if copied != before.st_size or (after.st_size, after.st_mtime_ns, after.st_ctime_ns) != (before.st_size, before.st_mtime_ns, before.st_ctime_ns):+ raise ValueError("Private archive source changed")+ if destination.stat().st_size > MAX_ARCHIVE_BYTES:+ raise ValueError("Private archive exceeds transfer allowance")+ except BaseException:+ destination.unlink(missing_ok=True)+ raise+ return {"files": len(files), "sourceBytes": total, "archiveBytes": destination.stat().st_size}+++def unpack(source, destination):+ source, destination = Path(source), Path(destination)+ if source.is_symlink() or not source.is_file() or source.stat().st_size > MAX_ARCHIVE_BYTES:+ raise ValueError("Invalid private archive file")+ if destination.exists() or destination.is_symlink():+ raise ValueError("Private archive destination already exists")+ with source.open("rb") as data:+ if source.stat().st_size < 22:+ raise ValueError("Private archive is truncated")+ data.seek(-22, os.SEEK_END)+ signature, disk, start_disk, disk_entries, entries, central_bytes, central_offset, comment_bytes = struct.unpack("<4s4H2LH", data.read(22))+ if (signature != b"PK\x05\x06" or disk or start_disk or comment_bytes+ or entries != disk_entries or not 0 < entries <= MAX_FILES+ or central_bytes > 8 * 1024 * 1024+ or central_offset + central_bytes != source.stat().st_size - 22):+ raise ValueError("Private archive directory exceeds its boundary")+ destination.parent.mkdir(parents=True, exist_ok=True)+ temporary = Path(tempfile.mkdtemp(prefix=".private-eval-", dir=destination.parent))+ try:+ with zipfile.ZipFile(source) as archive:+ entries = archive.infolist()+ if not entries or len(entries) != disk_entries or len(entries) > MAX_FILES:+ raise ValueError("Private archive exceeds entry allowance")+ paths = set()+ total = 0+ for entry in entries:+ relative = safe_path(entry.filename)+ mode = entry.external_attr >> 16+ if (entry.compress_type != zipfile.ZIP_STORED or entry.flag_bits & 1+ or stat.S_IFMT(mode) not in (0, stat.S_IFREG)+ or str(relative).casefold() in paths):+ raise ValueError("Private archive has unsupported or duplicate entries")+ paths.add(str(relative).casefold())+ total += entry.file_size+ if total > MAX_BYTES:+ raise ValueError("Private archive exceeds byte allowance")+ for entry in entries:+ target = temporary.joinpath(*safe_path(entry.filename).parts)+ target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)+ copied = 0+ with archive.open(entry) as data, os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as output:+ while chunk := data.read(64 * 1024):+ copied += len(chunk)+ if copied > entry.file_size:+ raise ValueError("Private archive entry exceeded its declared size")+ output.write(chunk)+ if copied != entry.file_size:+ raise ValueError("Private archive entry was truncated")+ os.rename(temporary, destination)+ except BaseException:+ shutil.rmtree(temporary)+ raise+ return {"files": len(entries), "sourceBytes": total}+++if __name__ == "__main__":+ if len(sys.argv) != 4 or sys.argv[1] not in ("pack", "unpack"):+ raise SystemExit("Usage: private-history-archive.py pack|unpack SOURCE DESTINATION")+ print(json.dumps((pack if sys.argv[1] == "pack" else unpack)(sys.argv[2], sys.argv[3])))diff --git a/tests/runner-e2e/private-history.test.ts b/tests/runner-e2e/private-history.test.tsnew file mode 100644index 0000000000..9af459a01b--- /dev/null+++ b/tests/runner-e2e/private-history.test.ts@@ -0,0 +1,162 @@+import { execFile } from "node:child_process";+import { copyFile, lstat, mkdir, mkdtemp, readFile, rm, symlink, truncate, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { promisify } from "node:util";+import { afterEach, expect, it, vi } from "vitest";+import { BoundedHistoryPublisher } from "./history-budget.js";+import { archivePrivateEvidence, PRIVATE_HISTORY_PREFIX, privateArtifactKey, retainPrivateTransfer, transferPrivateArtifact, verifyPrivateHistory } from "./private-history.js";++const execute = promisify(execFile);+const roots: string[] = [];+async function temporary() {+ const root = await mkdtemp(path.join(os.tmpdir(), "private-history-test-")); roots.push(root); return root;+}+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });++function configurations() {+ return {+ "get-bucket-versioning": { Status: "Enabled" },+ "get-bucket-lifecycle-configuration": { Rules: [+ { Status: "Enabled", Filter: { Prefix: `${PRIVATE_HISTORY_PREFIX}/` }, Expiration: { Days: 30 },+ NoncurrentVersionExpiration: { NoncurrentDays: 1 }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } },+ { Status: "Enabled", Filter: { Prefix: `${PRIVATE_HISTORY_PREFIX}/` }, Expiration: { ExpiredObjectDeleteMarker: true } },+ ] },+ "get-public-access-block": { PublicAccessBlockConfiguration: { BlockPublicAcls: true, IgnorePublicAcls: true, BlockPublicPolicy: true, RestrictPublicBuckets: true } },+ "get-bucket-ownership-controls": { OwnershipControls: { Rules: [{ ObjectOwnership: "BucketOwnerEnforced" }] } },+ "get-bucket-policy": { Policy: JSON.stringify({ Statement: [{ Effect: "Deny" }] }) },+ };+}+function configurationRunner(config: Record<string, unknown>) {+ return async (args: string[]) => {+ if (args[1] === "get-object-lock-configuration" || args[1] === "get-bucket-replication") {+ throw Object.assign(new Error("Not configured"), { stderr: `(${args[1] === "get-object-lock-configuration" ? "ObjectLockConfigurationNotFoundError" : "ReplicationConfigurationNotFoundError"})` });+ }+ if (!(args[1]! in config)) throw new Error(`Unexpected AWS operation ${args[1]}`);+ return { stdout: JSON.stringify(config[args[1]!]) };+ };+}++it("round trips binary evidence and a ledger without exposing unrelated files", async () => {+ const root = await temporary(), source = path.join(root, "source"), archive = path.join(root, "evidence.zip"), destination = path.join(root, "restored");+ await mkdir(path.join(source, "scheduled"), { recursive: true });+ await writeFile(path.join(source, "scheduled/ledger.jsonl"), '{"cost":"unknown"}\n');+ await writeFile(path.join(source, "binary.png"), Buffer.from([0, 255, 17, 0]));+ await writeFile(path.join(root, "outside"), "private outside material");+ expect(await archivePrivateEvidence("pack", source, archive)).toMatchObject({ files: 2 });+ expect(await archivePrivateEvidence("unpack", archive, destination)).toMatchObject({ files: 2 });+ expect(await readFile(path.join(destination, "binary.png"))).toEqual(Buffer.from([0, 255, 17, 0]));+ expect(await readFile(path.join(destination, "scheduled/ledger.jsonl"), "utf8")).toBe('{"cost":"unknown"}\n');+ expect((await lstat(path.join(destination, "binary.png"))).mode & 0o777).toBe(0o600);+ await expect(lstat(path.join(destination, "outside"))).rejects.toThrow();+ await expect(archivePrivateEvidence("unpack", archive, destination)).rejects.toThrow("already exists");+});++it("rejects symlink sources and oversized source files before producing an archive", async () => {+ const root = await temporary(), source = path.join(root, "source"), archive = path.join(root, "evidence.zip");+ await mkdir(source); await writeFile(path.join(root, "outside"), "outside");+ await symlink(path.join(root, "outside"), path.join(source, "link"));+ await expect(archivePrivateEvidence("pack", source, archive)).rejects.toThrow("regular files");+ await rm(path.join(source, "link")); await writeFile(path.join(source, "huge"), "");+ await truncate(path.join(source, "huge"), 256 * 1024 * 1024 + 1);+ await expect(archivePrivateEvidence("pack", source, archive)).rejects.toThrow("byte allowance");+ await expect(lstat(archive)).rejects.toThrow();+});++it.each(["traversal", "absolute", "symlink", "compressed", "duplicate", "case_collision", "comment", "many_entries", "truncated"])("rejects %s archives without retaining a partial extraction", async kind => {+ const root = await temporary(), archive = path.join(root, "bad.zip"), destination = path.join(root, "restored");+ await execute("python3", ["-c", `+import sys,zipfile,stat+p,kind=sys.argv[1:]+with zipfile.ZipFile(p,'w') as z:+ name={'traversal':'../outside','absolute':'/tmp/outside'}.get(kind,'safe.txt')+ info=zipfile.ZipInfo(name)+ if kind=='symlink': info.external_attr=(stat.S_IFLNK|0o777)<<16+ if kind=='compressed': info.compress_type=zipfile.ZIP_DEFLATED+ z.writestr(info,'data')+ if kind=='duplicate': z.writestr(name,'other')+ if kind=='case_collision': z.writestr('SAFE.txt','other')+ if kind=='comment': z.comment=b'extra'+ if kind=='many_entries':+ for n in range(4097): z.writestr(str(n),'')+if kind=='truncated':+ with open(p,'r+b') as f: f.truncate(12)+`, archive, kind]);+ await expect(archivePrivateEvidence("unpack", archive, destination)).rejects.toThrow();+ await expect(lstat(destination)).rejects.toThrow();+ await expect(lstat(path.join(root, "outside"))).rejects.toThrow();+});++it("fails closed on public access, ACLs, a reader grant, or unavailable inspection", async () => {+ const root = await temporary();+ await verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, "allowed.json"), configurationRunner(configurations())), "private-bucket");+ for (const operation of ["get-public-access-block", "get-bucket-ownership-controls", "get-bucket-policy"]) {+ const config: Record<string, unknown> = configurations();+ config[operation] = operation === "get-bucket-policy" ? { Policy: '{"Statement":[{"Effect":"Allow"}]}' } : {};+ const runner = vi.fn(configurationRunner(config));+ await expect(verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, `${operation}.json`), runner), "private-bucket")).rejects.toThrow();+ expect(runner.mock.calls.some(([args]) => args[1] === "put-object")).toBe(false);+ }+ await expect(verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, "denied.json"), async () => { throw new Error("AccessDenied"); }), "private-bucket")).rejects.toThrow("AccessDenied");+});++it("uses immutable private keys and restores the uploaded archive through the actual transfer path", async () => {+ const root = await temporary(), source = path.join(root, "source"), stored = path.join(root, "stored.zip"), destination = path.join(root, "restored");+ await mkdir(source); await writeFile(path.join(source, "ledger.jsonl"), "retained attempt\n");+ const config = configurationRunner(configurations());+ const runner = vi.fn(async (args: string[]) => {+ if (args[1] === "put-object") {+ expect(args).toEqual(expect.arrayContaining(["--if-none-match", "*", "--cache-control", "private, no-store"]));+ expect(args[args.indexOf("--key") + 1]).toBe(privateArtifactKey("evidence", "12345"));+ await copyFile(args[args.indexOf("--body") + 1]!, stored);+ return { stdout: '{"VersionId":"retained-version","ETag":"tag"}' };+ }+ if (args[1] === "get-object") {+ expect(args).toContain(`bytes=0-${264 * 1024 * 1024}`);+ await copyFile(stored, args[args.indexOf("--cli-connect-timeout") - 1]!);+ return { stdout: '{"VersionId":"retained-version"}' };+ }+ return config(args);+ });+ const upload = new BoundedHistoryPublisher(path.join(root, "upload.json"), runner);+ expect(await transferPrivateArtifact({ operation: "upload", kind: "evidence", runId: "12345", directory: source, bucket: "private-bucket", publisher: upload }))+ .toMatchObject({ private: true, versionId: "retained-version", files: 1, sha256: expect.stringMatching(/^[0-9a-f]{64}$/) });+ await transferPrivateArtifact({ operation: "download", kind: "evidence", runId: "12345", directory: destination, bucket: "private-bucket", publisher: new BoundedHistoryPublisher(path.join(root, "download.json"), runner) });+ expect(await readFile(path.join(destination, "ledger.jsonl"), "utf8")).toBe("retained attempt\n");+ expect(runner).toHaveBeenCalledTimes(16);+});++it("rejects unsafe run identities and other artifact kinds", () => {+ for (const id of ["../123", "0", "1/2", "123-2"]) expect(() => privateArtifactKey("report", id)).toThrow();+ expect(() => privateArtifactKey("public", "123")).toThrow();+});++it.each([false, true])("retains an unknown transfer without retrying it when receipt upload fails=%s", async receiptFails => {+ const root = await temporary(), source = path.join(root, "source"), receipt = path.join(root, "transfer.json");+ await mkdir(source); await writeFile(path.join(source, "ledger.jsonl"), "original evidence\n");+ const config = configurationRunner(configurations());+ const runner = vi.fn(async (args: string[]) => {+ if (args[1] !== "put-object") return config(args);+ await copyFile(args[args.indexOf("--body") + 1]!, path.join(root, "accepted.zip"));+ throw new Error("Response lost after server accepted archive");+ });+ const retain = vi.fn(async (args: string[]) => {+ expect(args[args.indexOf("--key") + 1]).toBe(`${privateArtifactKey("evidence", "12345")}.upload-receipt.json`);+ const snapshot = JSON.parse(await readFile(args[args.indexOf("--body") + 1]!, "utf8"));+ expect(snapshot.transfer.status).toBe("failed_or_unknown");+ expect(snapshot.requests.at(-1)).toMatchObject({ operation: "put-object", status: "failed_or_unknown" });+ expect(snapshot.billedUsd).toBeNull();+ expect(snapshot.receiptUpload.status).toBe("pending_at_snapshot");+ if (receiptFails) throw new Error("Receipt response also lost");+ return { stdout: '{"VersionId":"receipt-version"}' };+ });+ await expect(retainPrivateTransfer({ operation: "upload", kind: "evidence", runId: "12345", directory: source,+ bucket: "private-bucket", publisher: new BoundedHistoryPublisher(receipt, runner) }, receipt,+ () => new BoundedHistoryPublisher(`${receipt}.retention.json`, retain)))+ .rejects.toThrow(receiptFails ? "receipt retention failed" : "Response lost");+ expect(runner.mock.calls.filter(([args]) => args[1] === "put-object")).toHaveLength(1);+ expect(retain).toHaveBeenCalledTimes(1);+ expect((await lstat(`${receipt}.transfer.json`)).mode & 0o777).toBe(0o600);+ expect(JSON.parse(await readFile(`${receipt}.retention.json`, "utf8")).requests[0].status)+ .toBe(receiptFails ? "failed_or_unknown" : "accepted");+});diff --git a/tests/runner-e2e/private-history.ts b/tests/runner-e2e/private-history.tsnew file mode 100644index 0000000000..85224bf997--- /dev/null+++ b/tests/runner-e2e/private-history.ts@@ -0,0 +1,142 @@+import { execFile } from "node:child_process";+import { createReadStream } from "node:fs";+import { createHash } from "node:crypto";+import { lstat, mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { promisify } from "node:util";+import { pathToFileURL } from "node:url";+import { BoundedHistoryPublisher } from "./history-budget.js";++const execFileAsync = promisify(execFile);+export const PRIVATE_HISTORY_PREFIX = "runner-e2e-private/bounded-v1";+export const PRIVATE_ARCHIVE_BYTES = 264 * 1024 * 1024;++export function privateArtifactKey(kind: string, runId: string) {+ if (!["evidence", "report"].includes(kind) || !/^[1-9][0-9]{0,19}$/.test(runId)) {+ throw new Error("Private history needs an evidence/report kind and numeric workflow run ID");+ }+ return `${PRIVATE_HISTORY_PREFIX}/campaigns/gha-${runId}-1/${kind}.zip`;+}++export async function archivePrivateEvidence(operation: "pack" | "unpack", source: string, destination: string) {+ const result = await execFileAsync("python3", [path.join(import.meta.dirname, "private-history-archive.py"), operation, source, destination], {+ timeout: 60_000, maxBuffer: 64 * 1024,+ });+ return JSON.parse(result.stdout) as { files: number; sourceBytes: number; archiveBytes?: number };+}++export async function verifyPrivateHistory(publisher: BoundedHistoryPublisher, bucket: string) {+ if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) throw new Error("Invalid private history bucket");+ await publisher.verifyRetention(bucket, PRIVATE_HISTORY_PREFIX);+ const configuration = async (operation: string) => JSON.parse((await publisher.run(["s3api", operation, "--bucket", bucket])).stdout);+ const block = (await configuration("get-public-access-block")).PublicAccessBlockConfiguration;+ if (!["BlockPublicAcls", "IgnorePublicAcls", "BlockPublicPolicy", "RestrictPublicBuckets"].every(key => block?.[key] === true)) {+ throw new Error("Private history requires all bucket public-access blocks");+ }+ const ownership = (await configuration("get-bucket-ownership-controls")).OwnershipControls?.Rules;+ if (!Array.isArray(ownership) || ownership.length !== 1 || ownership[0]?.ObjectOwnership !== "BucketOwnerEnforced") {+ throw new Error("Private history requires disabled object ACLs");+ }+ const policy = JSON.parse((await configuration("get-bucket-policy")).Policy);+ if (!Array.isArray(policy.Statement) || policy.Statement.some((statement: { Effect?: string }) => statement.Effect !== "Deny")) {+ throw new Error("Private history bucket policy must not grant access to external readers or a CDN");+ }+}++type TransferOptions = {+ operation: "upload" | "download";+ kind: "evidence" | "report";+ runId: string;+ directory: string;+ bucket: string;+ publisher: BoundedHistoryPublisher;+};++export async function transferPrivateArtifact(options: TransferOptions) {+ privateArtifactKey(options.kind, options.runId);+ await verifyPrivateHistory(options.publisher, options.bucket);+ return transferVerifiedArtifact(options);+}++async function transferVerifiedArtifact(options: TransferOptions) {+ const key = privateArtifactKey(options.kind, options.runId);+ const temporary = await mkdtemp(path.join(os.tmpdir(), "paperclip-private-history-"));+ const archive = path.join(temporary, "archive.zip");+ try {+ if (options.operation === "upload") {+ const metadata = await archivePrivateEvidence("pack", options.directory, archive);+ const hash = createHash("sha256");+ for await (const chunk of createReadStream(archive)) hash.update(chunk);+ const result = await options.publisher.upload(options.bucket, key, archive, "application/zip", "private, no-store", true);+ const response = JSON.parse(result.stdout) as { VersionId?: string };+ if (!response.VersionId) throw new Error("Private archive upload did not return a version identity");+ return { private: true, ...metadata, uri: `s3://${options.bucket}/${key}`, versionId: response.VersionId, sha256: hash.digest("hex") };+ }+ await options.publisher.run(["s3api", "get-object", "--bucket", options.bucket, "--key", key,+ "--range", `bytes=0-${PRIVATE_ARCHIVE_BYTES}`, archive]);+ if ((await lstat(archive)).size > PRIVATE_ARCHIVE_BYTES) throw new Error("Private archive exceeds transfer allowance");+ const metadata = await archivePrivateEvidence("unpack", archive, options.directory);+ return { ...metadata, uri: `s3://${options.bucket}/${key}` };+ } finally {+ await rm(temporary, { recursive: true, force: true });+ }+}++export async function retainPrivateTransfer(options: TransferOptions, receipt: string, receiptPublisher?: () => BoundedHistoryPublisher) {+ const key = privateArtifactKey(options.kind, options.runId);+ await verifyPrivateHistory(options.publisher, options.bucket);+ let result: Awaited<ReturnType<typeof transferVerifiedArtifact>> | undefined;+ let failure: unknown;+ try {+ result = await transferVerifiedArtifact(options);+ } catch (error) {+ failure = error;+ }+ if (receiptPublisher) {+ await options.publisher.save();+ const snapshot = JSON.stringify({ ...JSON.parse(await readFile(receipt, "utf8")),+ transfer: result ? { status: "complete", ...result } : { status: "failed_or_unknown" },+ receiptUpload: { status: "pending_at_snapshot", maxBytes: 1024 * 1024 } });+ if (Buffer.byteLength(snapshot) > 1024 * 1024) throw new Error("Private transfer receipt exceeds allowance");+ await writeFile(`${receipt}.transfer.json`, snapshot, { mode: 0o600, flag: "wx" });+ try {+ await receiptPublisher().upload(options.bucket, `${key}.${options.operation}-receipt.json`, `${receipt}.transfer.json`, "application/json", "private, no-store", true);+ } catch (retentionError) {+ throw new AggregateError(failure ? [failure, retentionError] : [retentionError], "Private transfer receipt retention failed; local receipts remain authoritative");+ }+ }+ if (failure) throw failure;+ return result!;+}++async function main() {+ const [operation, kind, runId, directory] = process.argv.slice(2);+ if (!["verify", "upload", "download"].includes(operation ?? "") || !kind || !runId || !directory) {+ throw new Error("Usage: private-history.ts verify|upload|download evidence|report RUN_ID DIRECTORY");+ }+ privateArtifactKey(kind, runId);+ if (process.env.GITHUB_ACTIONS === "true" && (process.env.GITHUB_RUN_ATTEMPT !== "1" || process.env.GITHUB_RUN_ID !== runId)) {+ throw new Error("Private bounded history accepts only the initial attempt of this workflow run");+ }+ const bucket = process.env.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET ?? "";+ const receipt = path.resolve(`private-history-${operation}-${kind}-${runId}.json`);+ await (await open(receipt, "wx", 0o600)).close();+ const publisher = new BoundedHistoryPublisher(receipt);+ if (operation === "verify") {+ await verifyPrivateHistory(publisher, bucket);+ console.log("Private retention and access checks passed");+ return;+ }+ const receiptPublisher = operation === "upload" || process.env.GITHUB_ACTIONS === "true"+ ? () => new BoundedHistoryPublisher(`${receipt}.retention.json`) : undefined;+ const result = await retainPrivateTransfer({ operation: operation as "upload" | "download", kind: kind as "evidence" | "report", runId, directory, bucket, publisher }, receipt, receiptPublisher);+ console.log(JSON.stringify(process.env.GITHUB_ACTIONS === "true" ? { version: 1, privateTransferCompleted: true } : result));+}++if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {+ main().catch(error => {+ console.error(process.env.GITHUB_ACTIONS === "true" ? "Private history transfer failed; inspect private retained evidence." : error instanceof Error ? error.message : String(error));+ process.exitCode = 1;+ });+}diff --git a/tests/runner-e2e/process-tree-owner.ts b/tests/runner-e2e/process-tree-owner.tsindex 86b44902bc..3dcae17791 100644--- a/tests/runner-e2e/process-tree-owner.ts+++ b/tests/runner-e2e/process-tree-owner.ts@@ -177,13 +177,13 @@ export function createProcessTreeOwner(root: ChildProcess, options: {}let graceComplete = false;let directGraceDelivered = false;- async function signalGracefully() {+ async function signalGracefully(signal: "SIGTERM" | "SIGINT" = "SIGTERM") {if (graceComplete) return;await observe();// Choose and deliver from one validated table. ESRCH grants no ownership// coverage: the next grace poll can select that branch's surviving owner.const selected = new Set([...gracefulRoots()].filter(pid => !covered.has(pid)));- const delivered = signalSnapshot("SIGTERM", selected);+ const delivered = signalSnapshot(signal, selected);for (const pid of delivered) {if (pid === root.pid) directGraceDelivered = true;covered.add(pid);@@ -220,9 +220,10 @@ export async function stopKnownDirectChild(child: ChildProcess, phase: { gracefu}export async function incompleteTreeFallback(error: unknown, child: ChildProcess,- phase: { gracefulSent: boolean; forcedDeadline?: number }, deadline: number, forcedMs: number) {+ phase: { gracefulSent: boolean; forcedDeadline?: number }, deadline: number, forcedMs: number,+ signal: "SIGTERM" | "SIGINT" = "SIGTERM") {let fallbackError: unknown;- try { await stopKnownDirectChild(child, phase, deadline, forcedMs); }+ try { await stopKnownDirectChild(child, phase, deadline, forcedMs, signal); }catch (failure) { fallbackError = failure; }throw new Error(`Owned tree cleanup incomplete: ${String(error)}${fallbackError ? `; ${String(fallbackError)}` : ""}`);}@@ -234,13 +235,14 @@ export async function stopOwnedProcessTree(owner: ReturnType<typeof createProcessTreeOwner>,gracefulMs = 45_000,forcedMs = 5_000,+ gracefulSignal: "SIGTERM" | "SIGINT" = "SIGTERM",): Promise<void> {const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));const deadline = Date.now() + gracefulMs;const phase: { gracefulSent: boolean; forcedDeadline?: number } = { gracefulSent: false };try {if (process.platform === "win32") {- await stopKnownDirectChild(child, phase, deadline, forcedMs);+ await stopKnownDirectChild(child, phase, deadline, forcedMs, gracefulSignal);return;}const stopped = async () => {@@ -248,7 +250,7 @@ export async function stopOwnedProcessTree(return (child.exitCode !== null || child.signalCode !== null) && owner.liveGroups().length === 0;};do {- await owner.signalGracefully();+ await owner.signalGracefully(gracefulSignal);phase.gracefulSent ||= owner.directGraceDelivered();if (await stopped()) return;await wait(50);@@ -262,6 +264,6 @@ export async function stopOwnedProcessTree(throw new Error("Observed launcher descendants remained after bounded cleanup");} catch (error) {phase.gracefulSent ||= owner.directGraceDelivered();- await incompleteTreeFallback(error, child, phase, deadline, forcedMs);+ await incompleteTreeFallback(error, child, phase, deadline, forcedMs, gracefulSignal);}}diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.tsindex 82762c533f..5e90e3c2e2 100644--- a/tests/runner-e2e/product-journeys/contracts.test.ts+++ b/tests/runner-e2e/product-journeys/contracts.test.ts@@ -143,6 +143,8 @@ describe("deployment journey contracts", () => {onboarding.runIds=["run-1"]; onboarding.usage={runs:[{runId:"run-1",usage:{costUsd:0.5,inputTokens:10,outputTokens:10}}]};onboarding.productJourney!.operations={campaignId:"campaign",attemptId:"attempt",modelCostUsd:0.7,setupCostUsd:0.2,requestCount:2,feedback:{collection:"complete",complaints:0,suggestions:1},traceIds:[]};expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeCloseTo(0.7);+ onboarding.usage={runs:[{runId:"run-1",usage:{costUsdExact:"0.5",costStatus:"estimated",inputTokens:10,outputTokens:10}}]};+ expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeCloseTo(0.7);expect(()=>validateJourneyCollection({journeyId:"cloud-onboarding",runIds:["run"],usage:{runs:[{runId:"run",usage:{apiKey:"not-retainable"} as any}]}})).toThrow("invalid_collection_receipt");onboarding.productJourney!.operations.modelCostUsd=null;expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeNull();diff --git a/tests/runner-e2e/provider-usage.test.ts b/tests/runner-e2e/provider-usage.test.tsnew file mode 100644index 0000000000..b651c9e8a8--- /dev/null+++ b/tests/runner-e2e/provider-usage.test.ts@@ -0,0 +1,113 @@+import { expect, it } from "vitest";+import { estimateProviderCost, normalizeProviderUsage, ProviderUsageAccumulator, type EvalModelPrice } from "./provider-usage.js";+const price: EvalModelPrice = { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.test/prices", inputUsdPerMillion: "2.5", cachedReadUsdPerMillion: "0.25", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" };++it("excludes cached reads from OpenAI input and treats reasoning as a subset of output", () => {+ const usage = normalizeProviderUsage("openai", { id: "response", model: "fixture", usage: { input_tokens: 100, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 40 }, output_tokens: 20, output_tokens_details: { reasoning_tokens: 15 } } });+ expect(usage).toMatchObject({ complete: true, tokens: { input: 60, cachedRead: 40, cachedWrite: 0, output: 20, reasoning: 15 } });+ expect(estimateProviderCost(usage, price)).toBe("0.000360000");+});+it("never turns missing usage or incompatible model/pricing into a free charge", () => {+ const absent = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, output_tokens: 0 } });+ expect(absent.complete).toBe(false);+ expect(estimateProviderCost(absent, price)).toBeNull();+ const zero = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 0 } });+ expect(estimateProviderCost(zero, price)).toBe("0.000000000");+ expect(estimateProviderCost(zero, { ...price, model: "other" })).toBeNull();+});+it("does not add Anthropic cumulative output counters and prices cache writes by TTL", () => {+ const stream = new ProviderUsageAccumulator("anthropic");+ stream.observe({ type: "message_start", message: { id: "message", model: "fixture", usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 20, cache_creation_input_tokens: 30, cache_creation: { ephemeral_5m_input_tokens: 20, ephemeral_1h_input_tokens: 10 } } } });+ stream.observe({ type: "message_delta", usage: { output_tokens: 5 } });+ stream.observe({ type: "message_delta", usage: { output_tokens: 8 } });+ expect(stream.receipt().final).toBe(false);+ stream.observe({ type: "message_stop" });+ expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 10, cachedRead: 20, cachedWrite: 30, output: 8 } });+ expect(estimateProviderCost(stream.receipt(), { ...price, provider: "anthropic", cacheWrite5mUsdPerMillion: "3.125", cacheWrite1hUsdPerMillion: "5" })).toBe("0.000222500");+});+it("retains missing cache TTL and malformed token subsets as unknown", () => {+ const write = normalizeProviderUsage("anthropic", { model: "fixture", usage: { input_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 4, output_tokens: 1 } });+ expect(estimateProviderCost(write, { ...price, provider: "anthropic" })).toBeNull();+ expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 1 } }).complete).toBe(false);+ expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1, output_tokens_details: { reasoning_tokens: 2 } } }).complete).toBe(false);+});+it("uses one terminal Responses usage receipt regardless of streamed text or repeat notifications", () => {+ const stream = new ProviderUsageAccumulator("openai");+ stream.observe({ type: "response.output_text.delta", delta: "not a usage receipt" });+ expect(stream.receipt().complete).toBe(false);+ const terminal = { type: "response.completed", response: { id: "response", model: "fixture", usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1 } } };+ stream.observe(terminal); stream.observe(terminal);+ expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 1, output: 1 } });+ expect(estimateProviderCost(stream.receipt(), price)).toBe("0.000012500");+});++const modernPrice: EvalModelPrice = { ...price, inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4", outputUsdPerMillion: "20",+ cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10",+ longContext: { aboveInputTokens: 272_000, rates: { inputUsdPerMillion: "8", cachedReadUsdPerMillion: "0.8", outputUsdPerMillion: "30", cacheWrite30mUsdPerMillion: "10", cacheWrite12hUsdPerMillion: "20" } } };+const modernUsage = (total = 12_000, read = 4000, shortWrite = 3000, longWrite = 4000) => ({ model: "fixture", usage: {+ input_tokens: total, input_tokens_details: { cached_tokens: read, cache_write_tokens: shortWrite + longWrite,+ cache_write_tokens_details: { ttl_30m_tokens: shortWrite, ttl_12h_tokens: longWrite } }, output_tokens: 100,+} });+it("separates modern OpenAI ordinary input and mixed-TTL writes without adding the base rate twice", () => {+ const receipt = normalizeProviderUsage("openai", modernUsage());+ expect(receipt).toMatchObject({ complete: true, tokens: { input: 1000, cachedRead: 4000, cachedWrite: 7000 }, cacheWrite30m: 3000, cacheWrite12h: 4000 });+ expect(estimateProviderCost(receipt, modernPrice)).toBe("0.062600000");+});+it("applies long-context pricing to the entire request using total input including cache", () => {+ expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_000, 270_000, 0, 1000)), modernPrice)).toBe("0.124000000");+ expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_001, 270_000, 0, 1000)), modernPrice)).toBe("0.247008000");+});+it("keeps missing modern write counters or missing/inconsistent TTL splits unknown", () => {+ const missing = modernUsage(); delete (missing.usage.input_tokens_details as any).cache_write_tokens;+ expect(normalizeProviderUsage("openai", missing).complete).toBe(false);+ expect(estimateProviderCost(normalizeProviderUsage("openai", missing), modernPrice)).toBeNull();+ const noTtl = modernUsage(); delete (noTtl.usage.input_tokens_details as any).cache_write_tokens_details;+ expect(estimateProviderCost(normalizeProviderUsage("openai", noTtl), modernPrice)).toBeNull();+ const mismatch = modernUsage(); mismatch.usage.input_tokens_details.cache_write_tokens_details.ttl_12h_tokens++;+ expect(normalizeProviderUsage("openai", mismatch).complete).toBe(false);+});++const policyPrice = { ...modernPrice, model: "gpt-5.6-sol" };+function requestPolicyReceipt(request: Record<string, unknown> = {}, details: Record<string, unknown> = {}) {+ const stream = new ProviderUsageAccumulator("openai", { model: policyPrice.model, input: "hello", ...request });+ stream.observe({ type: "response.completed", response: { model: policyPrice.model, usage: {+ input_tokens: 12_000, input_tokens_details: { cached_tokens: 4000, cache_write_tokens: 7000, ...details }, output_tokens: 100,+ } } });+ return stream.receipt();+}+it.each([+ [{}, "30m", "0.042600000"],+ [{ prompt_cache_retention: "24h" }, "30m", "0.042600000"],+ [{ prompt_cache_options: { mode: "implicit", ttl: "12h" } }, "12h", "0.077600000"],+] as const)("prices omitted write TTLs from a documented visible request policy: %j", (request, ttl, cost) => {+ const receipt = requestPolicyReceipt(request);+ expect(receipt).toMatchObject({ final: true, complete: true, cacheWriteAttribution: {+ source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl,+ } });+ expect(estimateProviderCost(receipt, policyPrice)).toBe(cost);+});+it.each([+ { model: "unqualified-model" }, { previous_response_id: "stored-response" }, { conversation: "stored-conversation" },+ { prompt: { id: "stored-prompt" } }, { context_management: [] },+ { input: [{ type: "item_reference", id: "stored-item" }] },+ { input: [{ type: "compaction", encrypted_content: "opaque" }] },+ { input: [{ role: "user", content: [{ type: "input_text", text: "hello", prompt_cache_breakpoint: { mode: "explicit", ttl: "12h" } }] }] },+ { prompt_cache_options: { mode: "explicit" } }, { prompt_cache_options: { ttl: "unknown" } },+ { prompt_cache_options: { ttl: null } }, { prompt_cache_options: null },+ { prompt_cache_options: { future_option: true } },+])("keeps hidden, explicit, or unqualified cache policies unknown: %j", request => {+ const receipt = requestPolicyReceipt(request);+ expect(receipt.cacheWriteAttribution).toBeUndefined();+ expect(estimateProviderCost(receipt, policyPrice)).toBeNull();+});+it("prefers provider TTL splits and never repairs malformed or partial provider counters", () => {+ const reported = requestPolicyReceipt({}, { cache_write_tokens_details: { ttl_30m_tokens: 3000, ttl_12h_tokens: 4000 } });+ expect(reported.cacheWriteAttribution).toBeUndefined();+ expect(estimateProviderCost(reported, policyPrice)).toBe("0.062600000");+ for (const details of [{ cache_write_tokens: undefined }, { cache_write_tokens_details: null },+ { cache_write_tokens_details: { ttl_30m_tokens: 7000 } }, { cache_write_tokens_details: { ttl_30m_tokens: 7000, ttl_12h_tokens: 1 } }]) {+ const receipt = requestPolicyReceipt({}, details);+ expect(receipt.cacheWriteAttribution).toBeUndefined();+ expect(estimateProviderCost(receipt, policyPrice)).toBeNull();+ }+});diff --git a/tests/runner-e2e/provider-usage.ts b/tests/runner-e2e/provider-usage.tsnew file mode 100644index 0000000000..bb3453a761--- /dev/null+++ b/tests/runner-e2e/provider-usage.ts@@ -0,0 +1,154 @@+import { usdToUnits } from "../../packages/shared/src/money.js";+import type { EvalCostObservation } from "./cost-ledger.js";++type Tokens = NonNullable<EvalCostObservation["tokens"]>;+type Row = Record<string, unknown>;+const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {};+const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;+export interface ProviderUsageReceipt {+ provider: "openai" | "anthropic";+ tokens: Tokens;+ complete: boolean;+ providerResponseId?: string;+ model?: string;+ cacheWrite5m?: number;+ cacheWrite1h?: number;+ cacheWrite30m?: number;+ cacheWrite12h?: number;+ cacheWriteAttribution?: { source: "request_policy"; version: "openai-prompt-cache-2026-10-20"; ttl: "30m" | "12h" };+}++function openAiRequestCachePolicy(payload: unknown): { model: string; ttl: "30m" | "12h" } | undefined {+ const body = row(payload);+ if (body.model !== "gpt-5.6-sol" || body.previous_response_id !== undefined || body.conversation !== undefined || body.prompt !== undefined+ || body.context_management !== undefined || !(typeof body.input === "string" || Array.isArray(body.input))) return;+ const options = row(body.prompt_cache_options);+ if (body.prompt_cache_options !== undefined && body.prompt_cache_options !== options) return;+ if (Object.keys(options).some(key => !["mode", "ttl"].includes(key)) || options.mode !== undefined && options.mode !== "implicit") return;+ const ttl = options.ttl === undefined ? "30m" : options.ttl;+ if (ttl !== "30m" && ttl !== "12h") return;+ const hasUnqualifiedInput = (value: unknown): boolean => {+ if (Array.isArray(value)) return value.some(hasUnqualifiedInput);+ if (!value || typeof value !== "object") return false;+ const item = row(value);+ return "prompt_cache_breakpoint" in item || item.type === "compaction" || item.type === "item_reference"+ || Object.values(item).some(hasUnqualifiedInput);+ };+ if (hasUnqualifiedInput(body.input)) return;+ return { model: body.model, ttl };+}++export function normalizeProviderUsage(provider: "openai" | "anthropic", payload: unknown): ProviderUsageReceipt {+ const body = row(payload), usage = row(body.usage);+ const output = count(usage.output_tokens);+ let tokens: Tokens;+ let cacheWrite5m: number | undefined, cacheWrite1h: number | undefined;+ let cacheWrite30m: number | undefined, cacheWrite12h: number | undefined;+ if (provider === "openai") {+ const totalInput = count(usage.input_tokens);+ const details = row(usage.input_tokens_details);+ const cachedRead = count(details.cached_tokens);+ let cachedWrite = count(details.cache_write_tokens);+ const writes = row(details.cache_write_tokens_details);+ cacheWrite30m = count(writes.ttl_30m_tokens);+ cacheWrite12h = count(writes.ttl_12h_tokens);+ if (cachedWrite !== undefined && cacheWrite30m !== undefined && cacheWrite12h !== undefined && cacheWrite30m + cacheWrite12h !== cachedWrite) cachedWrite = undefined;+ const reasoning = count(row(usage.output_tokens_details).reasoning_tokens);+ tokens = { input: totalInput !== undefined && cachedRead !== undefined && cachedWrite !== undefined && totalInput >= cachedRead + cachedWrite ? totalInput - cachedRead - cachedWrite : undefined,+ cachedRead, cachedWrite, output, reasoning };+ } else {+ const cache = row(usage.cache_creation);+ cacheWrite5m = count(cache.ephemeral_5m_input_tokens);+ cacheWrite1h = count(cache.ephemeral_1h_input_tokens);+ tokens = { input: count(usage.input_tokens), cachedRead: count(usage.cache_read_input_tokens), cachedWrite: count(usage.cache_creation_input_tokens), output };+ if (tokens.cachedWrite !== undefined && cacheWrite5m !== undefined && cacheWrite1h !== undefined && cacheWrite5m + cacheWrite1h !== tokens.cachedWrite) tokens.cachedWrite = undefined;+ }+ const complete = tokens.input !== undefined && tokens.cachedRead !== undefined && tokens.cachedWrite !== undefined && output !== undefined+ && !(tokens.reasoning !== undefined && tokens.reasoning > output);+ return { provider, tokens: Object.fromEntries(Object.entries(tokens).filter(([, value]) => value !== undefined)), complete,+ ...(typeof body.id === "string" ? { providerResponseId: body.id } : {}),+ ...(typeof body.model === "string" ? { model: body.model } : {}),+ ...(cacheWrite5m !== undefined ? { cacheWrite5m } : {}), ...(cacheWrite1h !== undefined ? { cacheWrite1h } : {}),+ ...(cacheWrite30m !== undefined ? { cacheWrite30m } : {}), ...(cacheWrite12h !== undefined ? { cacheWrite12h } : {}) };+}++export class ProviderUsageAccumulator {+ private body: Row = {};+ private usage: Row = {};+ private final = false;+ private readonly cachePolicy: ReturnType<typeof openAiRequestCachePolicy>;+ constructor(readonly provider: "openai" | "anthropic", request?: unknown) {+ this.cachePolicy = provider === "openai" ? openAiRequestCachePolicy(request) : undefined;+ }+ observe(event: unknown) {+ const value = row(event);+ if (this.provider === "openai") {+ const response = row(value.response);+ if (["response.completed", "response.incomplete", "response.failed"].includes(String(value.type))) {+ this.body = response;+ this.usage = row(response.usage);+ this.final = true;+ } else if (value.object === "response") {+ this.body = value;+ this.usage = row(value.usage);+ this.final = ["completed", "incomplete", "failed"].includes(String(value.status));+ }+ } else {+ if (value.type === "message_start") {+ this.body = row(value.message);+ this.usage = { ...this.usage, ...row(this.body.usage) };+ } else if (value.type === "message_delta") this.usage = { ...this.usage, ...row(value.usage) };+ else if (value.type === "message_stop") this.final = true;+ else if (value.type === "message") { this.body = value; this.usage = row(value.usage); this.final = true; }+ }+ }+ receipt() {+ const receipt = normalizeProviderUsage(this.provider, { ...this.body, usage: this.usage });+ const details = row(this.usage.input_tokens_details);+ if (this.cachePolicy && receipt.model === this.cachePolicy.model && receipt.complete && receipt.tokens.cachedWrite! > 0+ && !("cache_write_tokens_details" in details)) {+ receipt.cacheWrite30m = this.cachePolicy.ttl === "30m" ? receipt.tokens.cachedWrite : 0;+ receipt.cacheWrite12h = this.cachePolicy.ttl === "12h" ? receipt.tokens.cachedWrite : 0;+ receipt.cacheWriteAttribution = { source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl: this.cachePolicy.ttl };+ }+ return { ...receipt, final: this.final };+ }+}++export interface EvalTokenRates {+ inputUsdPerMillion: string;+ cachedReadUsdPerMillion: string;+ outputUsdPerMillion: string;+ cacheWrite5mUsdPerMillion?: string;+ cacheWrite1hUsdPerMillion?: string;+ cacheWrite30mUsdPerMillion?: string;+ cacheWrite12hUsdPerMillion?: string;+}+export interface EvalModelPrice extends EvalTokenRates {+ provider: "openai" | "anthropic";+ model: string;+ version: string;+ sourceUrl: string;+ longContext?: { aboveInputTokens: number; rates: EvalTokenRates };+}++export function estimateProviderCost(receipt: ProviderUsageReceipt, price: EvalModelPrice): string | null {+ if (!receipt.complete || receipt.model !== price.model || receipt.provider !== price.provider) return null;+ const { tokens } = receipt;+ const totalInput = tokens.input! + tokens.cachedRead! + tokens.cachedWrite!;+ if (!Number.isSafeInteger(totalInput)) return null;+ const rates = price.longContext && totalInput > price.longContext.aboveInputTokens ? price.longContext.rates : price;+ const priced = (count: number, rate: string) => {+ if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(rate)) throw new Error("Invalid nonnegative model price");+ return BigInt(count) * usdToUnits(rate);+ };+ let numerator = priced(tokens.input!, rates.inputUsdPerMillion) + priced(tokens.cachedRead!, rates.cachedReadUsdPerMillion) + priced(tokens.output!, rates.outputUsdPerMillion);+ if (tokens.cachedWrite! > 0) {+ const counts = receipt.provider === "openai" ? [receipt.cacheWrite30m, receipt.cacheWrite12h] : [receipt.cacheWrite5m, receipt.cacheWrite1h];+ const writeRates = receipt.provider === "openai" ? [rates.cacheWrite30mUsdPerMillion, rates.cacheWrite12hUsdPerMillion] : [rates.cacheWrite5mUsdPerMillion, rates.cacheWrite1hUsdPerMillion];+ if (counts[0] === undefined || counts[1] === undefined || counts[0] + counts[1] !== tokens.cachedWrite || writeRates[0] === undefined || writeRates[1] === undefined) return null;+ numerator += priced(counts[0], writeRates[0]) + priced(counts[1], writeRates[1]);+ }+ const units = (numerator + 999_999n) / 1_000_000n;+ return `${units / 1_000_000_000n}.${String(units % 1_000_000_000n).padStart(9, "0")}`;+}diff --git a/tests/runner-e2e/report-format.ts b/tests/runner-e2e/report-format.tsindex 4b50371ed5..7237d45697 100644--- a/tests/runner-e2e/report-format.ts+++ b/tests/runner-e2e/report-format.ts@@ -1,6 +1,12 @@import type { RunnerE2EBillingSummary } from "./types.js";import { billingCoverageLabel } from "./billing.js";+export function isPublicReportTimestamp(value: unknown): value is string {+ return typeof value === "string" && value === value.trim()+ && /^\d{4}-\d{2}-\d{2}T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d{1,9})?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/.test(value)+ && Number.isFinite(Date.parse(value));+}+export function html(value: unknown) {return String(value ?? "").replaceAll("&", "&")@@ -10,7 +16,8 @@ export function html(value: unknown) {.replaceAll("'", "'");}-export function durationLabel(durationMs: number) {+export function durationLabel(durationMs: number | null) {+ if (durationMs === null) return "Unknown";if (durationMs < 1_000) return `${durationMs}ms`;const seconds = Math.round(durationMs / 1_000);if (seconds < 60) return `${seconds}s`;@@ -23,7 +30,7 @@ export function tokenLabel(value: number, llm?: RunnerE2EBillingSummary["llm"])}export function llmCostLabel(value: number, llm: RunnerE2EBillingSummary["llm"]) {- return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount);+ return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount, (llm.runsWithPartialReportedCost ?? 0) > 0);}export function usdLabel(value: number | null) {diff --git a/tests/runner-e2e/report-playwright.config.ts b/tests/runner-e2e/report-playwright.config.tsnew file mode 100644index 0000000000..390ee13208--- /dev/null+++ b/tests/runner-e2e/report-playwright.config.ts@@ -0,0 +1,7 @@+import { defineConfig } from "@playwright/test";++export default defineConfig({+ testDir: "../e2e", testMatch: "runner-e2e-dashboard.spec.ts", workers: 1, retries: 0,+ timeout: 30_000, outputDir: "../../test-results/eval-report-navigation",+ use: { browserName: "chromium", headless: true }, reporter: "list",+});diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.tsindex 2677ec7d3b..f3c516fbd8 100644--- a/tests/runner-e2e/report.test.ts+++ b/tests/runner-e2e/report.test.ts@@ -1,3 +1,4 @@+import { createWorkerTiming } from "./worker-timing.js";import { execFile } from "node:child_process";import { promisify } from "node:util";import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";@@ -20,6 +21,50 @@ afterEach(async () => {});describe("runner E2E report aggregation", () => {+ it("publishes retained passing results but exits unsuccessfully when the request journal is damaged", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-damaged-ledger-report-"));+ cleanupDirectories.push(root);+ const executionId = "core-compatibility.runner-codex.local.message-marker";+ const directory = path.join(root, "attempt-1");+ await mkdir(directory);+ const result: RunnerE2EResult = {+ schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "core-compatibility",+ attempt: 1, status: "passed", profileId: "runner-codex", environmentId: "local",+ caseId: "message-marker", provider: "codex", model: "fixture-model", runtimeMode: "native",+ startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed",+ };+ const phaseStart = Date.parse(result.startedAt);+ const timing = await createWorkerTiming(path.join(directory, "worker-timing.json"), result, phaseStart);+ await timing.enter("execution", phaseStart + 200);+ await timing.enter("cleanup", phaseStart + 800);+ await timing.enter("finalization", phaseStart + 900);+ result.workerTiming = await timing.finish(phaseStart + 1000);+ const original = JSON.stringify(result);+ await writeFile(path.join(directory, "result.json"), original);+ await writeFile(path.join(directory, "final-state.png"), "fake-png");+ await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] }));+ const journal = path.join(root, "journal.jsonl"), output = path.join(root, "merged");+ await writeFile(journal, '{"damaged-journal":');+ await expect(execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {+ cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,+ PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), PAPERCLIP_E2E_OPERATIONS_LEDGER: journal },+ })).rejects.toMatchObject({ code: 1 });+ const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8"));+ expect(normalized).toMatchObject({ passed: 1, failed: 0, operations: { status: "unavailable" } });+ expect(normalized.results[0]).toMatchObject({ executionId, status: "passed" });+ expect(normalized.accounting.attempts[0].workerTiming).toEqual(result.workerTiming);+ expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("Worker setup time: 200ms.");+ expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("<strong>200ms</strong><span>Worker setup time</span>");+ expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("Worker phase timing: complete.");+ expect(normalized.billing.agentRunDurationMs).toBeNull();+ expect(await readFile(path.join(output, "index.html"), "utf8"))+ .toContain("<strong>Unknown</strong><span>Agent execution time</span>");+ for (const file of ["index.html", "summary.md"]) {+ expect(await readFile(path.join(output, file), "utf8")).toContain("Request ledger accounting unavailable");+ }+ expect(await readFile(path.join(directory, "result.json"), "utf8")).toBe(original);+ });+it("corrects overconfident timing verdicts without changing retained bounds or original files", async () => {const root = await mkdtemp(path.join(os.tmpdir(), "runner-latency-report-"));cleanupDirectories.push(root);@@ -331,13 +376,14 @@ describe("runner E2E report aggregation", () => {},});expect(normalized.billing).toMatchObject({- reportedLlmCostUsd: 0.0125,+ reportedLlmCostUsd: 0.025,llm: {- inputTokens: 1_250,- outputTokens: 75,- runsWithReportedCost: 1,+ inputTokens: 2_500,+ outputTokens: 150,+ runsWithReportedCost: 2,},});+ expect(normalized.accounting.attempts).toHaveLength(2);expect(normalized.results[0]).toMatchObject({attempt: 2,evidenceValid: true,@@ -353,6 +399,10 @@ describe("runner E2E report aggregation", () => {"utf8",);expect(dashboard).toContain("Runner Full-Stack E2E");+ expect(dashboard).toMatch(/\$0\.0250*<\/strong><span>LLM reported subtotal/);+ expect(dashboard).toContain("<strong>2s</strong><span>Summed attempt time</span>");+ expect(dashboard).toContain("Carried-forward evidence can span older runs");+ expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("Summed attempt time: 2s; 2 retained attempts; 0 missing attempts.");expect(dashboard).toContain(executionId);expect(dashboard).toContain("case-passed");expect(dashboard).not.toContain(@@ -668,3 +718,45 @@ describe("runner E2E report aggregation", () => {expect(junit).not.toContain("<?xml-stylesheet");});});++it("retains selector-approved successes and their spending when only failed workflow jobs rerun", async () => {+ const { selectRerunArtifacts } = await import("./select-rerun-artifacts.js");+ const root = await mkdtemp(path.join(os.tmpdir(), "runner-partial-rerun-"));+ cleanupDirectories.push(root);+ const artifactRoot = path.join(root, "downloaded"), selectedRoot = path.join(root, "selected");+ const executions = ["core-compatibility.runner-codex.local.message-marker", "core-compatibility.runner-claude.local.message-marker"];+ const source = { sha: "a".repeat(40), ref: "master", workflowRunUrl: "https://github.com/paperclipai/paperclip/actions/runs/12345" };+ for (const [index, executionId] of executions.entries()) {+ const workflowAttempt = index + 1;+ const artifactName = `runner-e2e-12345-${workflowAttempt}-${executionId}`;+ const directory = path.join(artifactRoot, artifactName, `gha-12345-${workflowAttempt}-${executionId}`, "attempt-1");+ await mkdir(directory, { recursive: true });+ const result: RunnerE2EResult = {+ schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "core-compatibility", source,+ accountingIdentity: { campaignId: `gha-12345-${workflowAttempt}`, attemptId: `attempt-${index}` },+ attempt: 1, status: "passed", profileId: index ? "runner-claude" : "runner-codex", environmentId: "local",+ caseId: "message-marker", provider: index ? "claude" : "codex", model: "fixture", runtimeMode: "native",+ startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed",+ usage: { inputTokens: 100, outputTokens: 10, costUsd: 0.01 * workflowAttempt },+ };+ await writeFile(path.join(directory, "result.json"), JSON.stringify(result));+ await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] }));+ await writeFile(path.join(directory, "final-state.png"), "fixture-png");+ }+ await selectRerunArtifacts({ artifactRoot, selectedRoot, expectedExecutionIds: executions, workflowRunId: "12345", workflowRunAttempt: 2,+ sourceSha: source.sha, sourceRef: source.ref, workflowRunUrl: source.workflowRunUrl,+ jobs: { attempts: [{ run_attempt: 1, run_started_at: "2026-10-20T00:00:00Z" }, { run_attempt: 2, run_started_at: "2026-10-20T01:00:00Z" }],+ jobs: [{ name: executions[0]!, run_attempt: 1, started_at: "2026-10-20T00:01:00Z" },+ { name: executions[1]!, run_attempt: 1, started_at: "2026-10-20T00:01:00Z" },+ { name: executions[1]!, run_attempt: 2, started_at: "2026-10-20T01:01:00Z" }] },+ });+ const output = path.join(root, "report");+ await execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {+ cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_E2E_CAMPAIGN_ID: "gha-12345-2", PAPERCLIP_RUNNER_E2E_REPORT_ROOT: selectedRoot,+ PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify(executions) },+ });+ const campaign = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8"));+ expect(campaign).toMatchObject({ passed: 2, failed: 0, billing: { reportedLlmCostUsd: 0.03 }, accounting: { excludedArtifacts: 0 } });+ expect(campaign.accounting.attempts).toHaveLength(2);+ expect(campaign.results[0].accountingIdentity.campaignId).toBe("gha-12345-1");+});diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.tsindex 81103993b2..a8a9664c3a 100644--- a/tests/runner-e2e/report.ts+++ b/tests/runner-e2e/report.ts@@ -1,3 +1,4 @@+import { campaignTimingText, summarizeCampaignTiming, summarizeWorkerPhaseTiming, workerPhaseTimingText } from "./campaign-timing.js";import { stageDashboardBrandAssets } from "./report-assets.js";import { discoverReportCatalog } from "./report-catalog.js";import path from "node:path";@@ -12,6 +13,10 @@ import {} from "node:fs/promises";import { runnerMatrix } from "./catalog.js";import { billingCoverageLabel, summarizeExecutionBilling } from "./billing.js";+import { isSelectedCarryForward, retainedCampaignAttempts } from "./attempt-accounting.js";+import { summarizeCampaignFeedback } from "./feedback.js";+import { campaignFeedbackReviewText } from "./feedback-dispositions.js";+import { operationsReportText, readEvalOperationsReport } from "./operations-report.js";import { renderRunnerE2EDashboard } from "./dashboard.js";import {buildRunnerCampaign,@@ -195,6 +200,12 @@ async function main() {const resultFiles = (await walk(root)).filter((file) => path.basename(file) === "result.json",);+ const requestedCampaignId = process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??+ (process.env.GITHUB_RUN_ID ? `gha-${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT ?? "1"}` : undefined);+ const selection = await readFile(path.join(root, "selected-artifacts.json"), "utf8")+ .then(value => JSON.parse(value) as unknown)+ .catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; });+ let excludedArtifacts = 0;const candidates = new Map<string, AggregatedResult[]>();for (const resultFile of resultFiles) {const parsed = JSON.parse(@@ -206,6 +217,13 @@ async function main() {)continue;const result = upgradeRunnerResult(parsed);+ if (requestedCampaignId && result.accountingIdentity &&+ result.accountingIdentity.campaignId !== requestedCampaignId &&+ !isSelectedCarryForward({ requestedCampaignId, result, selection,+ relativeResultPath: path.relative(root, resultFile).split(path.sep).join("/") })) {+ excludedArtifacts += 1;+ continue;+ }const directory = path.dirname(resultFile);const evidence = await readFile(path.join(directory, "evidence-manifest.json"),@@ -314,17 +332,24 @@ async function main() {billing: summarizeExecutionBilling(entry.result),}));const generatedAt = new Date().toISOString();+ const retained = retainedCampaignAttempts(+ selected.map(entry => entry.result),+ [...candidates.values()].flatMap(entries => entries.map(entry => entry.result)),+ );const campaign = buildRunnerCampaign({campaignId:- process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??- (process.env.GITHUB_RUN_ID- ? `gha-${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT ?? "1"}`- : `report-${generatedAt.replace(/[:.]/g, "-")}`),+ requestedCampaignId ?? `report-${generatedAt.replace(/[:.]/g, "-")}`,generatedAt,expected,results: resolvedResults,+ attempts: retained.attempts,+ excludedArtifacts: excludedArtifacts + retained.excludedArtifacts,+ ...(process.env.PAPERCLIP_E2E_OPERATIONS_LEDGER ? { operations: await readEvalOperationsReport(process.env.PAPERCLIP_E2E_OPERATIONS_LEDGER) } : {}),+ ...(process.env.PAPERCLIP_E2E_FEEDBACK_REVIEW ? { feedbackReview: JSON.parse(await readFile(process.env.PAPERCLIP_E2E_FEEDBACK_REVIEW, "utf8")) } : {}),+ ...(process.env.PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT ? { braintrustExperiment: JSON.parse(await readFile(process.env.PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT, "utf8")) } : {}),});const billing = campaign.billing;+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);const normalized = {...campaign,results: selected.map((entry, index) => ({@@ -386,7 +411,15 @@ async function main() {: []),`Tokens: ${billingCoverageLabel(`${billing.llm.inputTokens} input / ${billing.llm.outputTokens} output / ${billing.llm.cachedInputTokens} cached`, billing.llm.runsWithTokenUsage, billing.llm.runCount)}`,"",- `Provider-reported LLM cost: ${billingCoverageLabel(`$${billing.reportedLlmCostUsd.toFixed(6)}`, billing.llm.runsWithReportedCost, billing.llm.runCount)}`,+ `Provider-reported LLM cost: ${billingCoverageLabel(`$${billing.reportedLlmCostUsd.toFixed(6)}`, billing.llm.runsWithReportedCost, billing.llm.runCount, (billing.llm.runsWithPartialReportedCost ?? 0) > 0)}`,+ `Estimated LLM cost: ${billingCoverageLabel(`$${(billing.estimatedLlmCostUsd ?? 0).toFixed(6)}`, billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount)}`,+ `Accounting: ${campaign.accounting!.attempts.length} retained attempts; ${campaign.accounting!.missingAttempts} missing attempts; ${campaign.accounting!.excludedArtifacts} out-of-scope artifacts excluded. Costs cover captured usage, not unobserved setup or external charges.`,+ ...campaignTimingText(summarizeCampaignTiming(campaign.results, campaign.accounting)),+ ...workerPhaseTimingText(summarizeWorkerPhaseTiming(campaign.results, campaign.accounting)),+ ...(campaign.operations ? operationsReportText(campaign.operations) : []),+ "",+ `Feedback across retained attempts: ${feedback.complaints} complaints; ${feedback.suggestions} suggestions; ${feedback.incomplete} attempts with incomplete collection. Tool submissions: ${feedback.failedSubmissions} failed, ${feedback.unknownSubmissions} unresolved. Full bodies are in access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.`,+ campaignFeedbackReviewText(campaign),"",`Estimated Daytona list-price runtime cost: $${billing.estimatedRuntimeCostUsd.toFixed(6)}`,...(billing.judge ? [`Estimated judge cost: ${billing.judge.estimatedCostUsd === null ? "unknown" : `$${billing.judge.estimatedCostUsd.toFixed(6)}`}; ${billing.judge.attempts} attempts; ${billing.judge.attemptsWithUnknownUsage} with unknown usage; $${billing.judge.reservedCostUsd.toFixed(6)} reserved`] : []),@@ -401,7 +434,7 @@ async function main() {const cell = publicCampaignUrl? `[${resolved.executionId}](${publicCampaignUrl}#execution-${encodeURIComponent(resolved.executionId)})`: resolved.executionId;- return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${billingCoverageLabel(`${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens}`, cellBilling.llm.runsWithTokenUsage, cellBilling.llm.runCount)} | ${billingCoverageLabel(`$${cellBilling.reportedCostUsd.toFixed(6)}`, cellBilling.llm.runsWithReportedCost, cellBilling.llm.runCount)} | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;+ return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${billingCoverageLabel(`${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens}`, cellBilling.llm.runsWithTokenUsage, cellBilling.llm.runCount)} | ${billingCoverageLabel(`$${cellBilling.reportedCostUsd.toFixed(6)}`, cellBilling.llm.runsWithReportedCost, cellBilling.llm.runCount, (cellBilling.llm.runsWithPartialReportedCost ?? 0) > 0)} | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`;}),"",];@@ -419,7 +452,7 @@ async function main() {const junit = `<?xml version="1.0" encoding="UTF-8"?><testsuite name="Runner Full-Stack E2E" tests="${selected.length}" failures="${failures}">${cases}</testsuite>\n`;await writeFile(path.join(output, "junit.xml"), junit);console.log(summaryLines.join("\n"));- if (failures > 0 || (dependable && (!dependable.complete || dependable.unclassifiedResults.length > 0))) process.exitCode = 1;+ if (failures > 0 || campaign.operations?.status === "unavailable" || (dependable && (!dependable.complete || dependable.unclassifiedResults.length > 0))) process.exitCode = 1;}await main().catch((error) => {diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.tsindex eb06745b10..460855184f 100644--- a/tests/runner-e2e/result-validation.ts+++ b/tests/runner-e2e/result-validation.ts@@ -1,6 +1,7 @@+import { validateWorkerTiming } from "./worker-timing.js";import { connectionCheckpoints, connectionEvidencePasses, connectionRunSignals } from "./connection-evidence.js";import { STARTUP_REUSE_LAYERS, STARTUP_REUSE_REASONS } from "../../packages/adapter-utils/src/startup-reuse.js";-import type { RunnerE2EResult } from "./types.js";+import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js";import { DEPENDABLE_CELLS } from "./dependable-qualification.js";type Rule = (value: unknown, at: string) => void;@@ -117,7 +118,7 @@ const relativeFile: Rule = (value, at) => {};const runtime = shape({provider: oneOf("local", "daytona"),- agentRunDurationMs: number,+ agentRunDurationMs: nullable(number),leaseDurationMs: nullable(number),leaseCount: integer,cpuCores: optional(number),@@ -138,12 +139,14 @@ const llm = shape({runCount: integer,runsWithTokenUsage: integer,runsWithReportedCost: integer,+ runsWithEstimatedCost: optional(integer),inputTokens: number,outputTokens: number,cachedInputTokens: number,totalTokens: number,reportedCostUsd: number,- costStatus: oneOf("reported", "partial", "unpriced", "unavailable"),+ estimatedCostUsd: optional(number),+ costStatus: oneOf("reported", "estimated", "partial", "unpriced", "unavailable"),});const billing = shape({judge: optional(shape({ inputTokens: nullable(number), outputTokens: nullable(number), estimatedCostUsd: nullable(number), reservedCostUsd: number })),@@ -155,6 +158,43 @@ const billing = shape({observedAndEstimatedCostUsd: nullable(number),complete: boolean,});+const tracing = shape({+ projectId: string, traceId: string, spanId: string, status: oneOf("pending", "accepted", "failed"),+ verification: oneOf("unverified"), nativeConfiguration: oneOf("not_started", "configured", "failed"),+ companyId: optional(string), revision: optional(string), errorCode: optional(string), httpStatus: optional(number),+ finalization: optional(shape({ spanId: string, status: oneOf("accepted", "failed"), outcome: oneOf("passed", "failed"),+ verification: oneOf("unverified"), errorCode: optional(string), httpStatus: optional(number) })),+});+const feedback = shape({+ version: (value, at) => { if (value !== 1) invalid(at); },+ collection: oneOf("complete", "partial", "unavailable", "not_started"),+ invocationCollection: oneOf("complete", "partial", "unavailable", "not_started"),+ toolAvailability: oneOf("observed", "unknown"),+ complaints: integer, suggestions: integer, bodiesCollected: integer, runsInspected: integer,+ invocations: shape({ completed: integer, failed: integer, unknown: integer }),+ review: array(shape({ fingerprint: (value, at) => { if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) invalid(at); },+ kind: oneOf("complaint", "suggestion"), occurrences: integer, disposition: oneOf("needs_investigation") })),+});++export function validateCampaignAccounting(value: RunnerE2ECampaignAccounting) {+ shape({+ version: (version, at) => { if (version !== 1) invalid(at); },+ scope: oneOf("retained_attempts", "selected_results"),+ missingAttempts: integer,+ excludedArtifacts: integer,+ attempts: array(shape({+ id: string, executionId: string, attempt: integer,+ startedAt: date, finishedAt: date, durationMs: number,+ status: oneOf("passed", "failed"), cleanup: oneOf("not_started", "passed", "failed"), billing, feedback: optional(feedback), tracing: optional(tracing),+ })),+ })(value, "accounting");+ if (new Set(value.attempts.map(attempt => attempt.id)).size !== value.attempts.length) invalid("accounting.attempts duplicate id");+ for (const attempt of value.attempts) {+ if (attempt.workerTiming !== undefined) {+ try { validateWorkerTiming(attempt.workerTiming, attempt); } catch { invalid("accounting.attempts.workerTiming"); }+ }+ }+}const matcher: Rule = (value, at) => {object(value, at);const item = value as Record<string, unknown>;@@ -182,6 +222,7 @@ const matcher: Rule = (value, at) => {rule(value, at);};const fields = {+ accountingIdentity: optional(shape({ campaignId: string, attemptId: string })),publicMcp: optional(assistant),schema: oneOf("paperclip.runner-e2e.result/v1",@@ -268,11 +309,13 @@ const fields = {startedAt: date,finishedAt: date,durationMs: number,+ workerTiming: optional(object),// Provider usage is intentionally opaque. Billing reads only finite numbers;// the dashboard shows opaque diagnostics through escaped JSON.usage: optional(nullable(object)),runtimeUsage: optional(runtime),billing: optional(billing),+ feedback: optional(feedback), tracing: optional(tracing),matcherResults: optional(array(shape({ matcher, passed: boolean, detail: string })),),@@ -344,6 +387,9 @@ export function validateRetainedRunnerResult(): asserts value is RunnerE2EResult {result(value, "result");const checked = value as RunnerE2EResult;+ if (checked.workerTiming !== undefined) {+ try { validateWorkerTiming(checked.workerTiming, checked); } catch { invalid("result.workerTiming"); }+ }if (checked.suiteId === "product-journeys" && checked.status === "passed") {const evidence = checked.productJourney;if (!evidence) return invalid("result.productJourney");diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjsindex d2a1f01848..ee7e6dfd1f 100644--- a/tests/runner-e2e/run-unit.mjs+++ b/tests/runner-e2e/run-unit.mjs@@ -20,7 +20,7 @@ delete env.NODE_OPTIONS;for (const [name, command] of [["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]],- ["Node tests", ["--test", "tests/runner-e2e/native-completion-checks.test.mjs",+ ["Node tests", ["--test", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs","tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"]],]) {const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" });diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.tsindex b4b0824653..420d8c97cf 100644--- a/tests/runner-e2e/runner.spec.ts+++ b/tests/runner-e2e/runner.spec.ts@@ -1,3 +1,6 @@+import { createWorkerTiming } from "./worker-timing.js";+import { FEEDBACK_PERSISTENCE_SUITE, gradeFeedbackPersistence } from "./feedback-persistence.js";+import { EvalAttemptTrace, evalBraintrustConfig, EVAL_OBSERVABILITY_SECRET_NAMES } from "./braintrust.js";import { captureFollowupStopFailureApiState } from "./followup-stop-evidence.js";import { runFollowupStopResumeFlow } from "./followup-stop-resume-flow.js";import { runFinalizationRecoveryFlow } from "./finalization-recovery-flow.js";@@ -58,6 +61,8 @@ import path from "node:path";import { expect, test, type Page } from "@playwright/test";import { RunnerApi, pollUntil } from "./api.js";import { buildRuntimeUsage, summarizeExecutionBilling } from "./billing.js";+import { collectAttemptFeedback, emptyFeedbackSummary } from "./feedback.js";+import { finishBudgetedEvalSetup } from "./budgeted-setup.js";import { establishPublicMcpSession, runPublicMcpFlow } from "./public-mcp-flow.js";import { assistantUsage } from "./public-mcp-model.js";import { runnerExecutionById } from "./catalog.js";@@ -359,7 +364,9 @@ async function writeSanitizedJson(name: string,value: unknown,secrets: readonly string[],+ signal?: AbortSignal,) {+ signal?.throwIfAborted();const raw = `${JSON.stringify(value)}\n`;// Public API payloads may contain harmless provider-shaped fixtures or// opaque generated tokens. Reject exact campaign credentials before writing,@@ -368,8 +375,9 @@ async function writeSanitizedJson(const safe = `${JSON.stringify(sanitizeJson(value, secrets), null, 2)}\n`;const leak = findSecretLeakInJsonValues(JSON.parse(safe), secrets);if (leak) throw new Error(`Secret leak in ${name}: ${leak}`);+ signal?.throwIfAborted();await mkdir(directory, { recursive: true });- await writeFile(path.join(directory, name), safe, "utf8");+ await writeFile(path.join(directory, name), safe, { encoding: "utf8", signal });}async function submitTaskRevision(page: Page, body: string): Promise<number> {@@ -593,6 +601,7 @@ for (const execution of executions) {}const startedAtMs = Date.now();const startedAt = new Date(startedAtMs).toISOString();+ const workerTiming = await createWorkerTiming(path.join(snapshotsDir, "worker-timing.json"), { executionId: execution.id, attempt }, startedAtMs);const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;const marker = execution.task.buildVisibleMarker(nonce);const title = execution.task.buildTitle(nonce);@@ -609,7 +618,20 @@ for (const execution of executions) {verifyNativeInstructionPreflight(process.env[NATIVE_INSTRUCTION_PREFLIGHT_ENV]);}const credentials = credentialValues();- const secrets = normalizedSecrets(Object.values(credentials));+ const secrets = normalizedSecrets([...Object.values(credentials), ...EVAL_OBSERVABILITY_SECRET_NAMES.map(name => process.env[name]), process.env.PAPERCLIP_E2E_GATEWAY_TOKEN]);+ const traceConfig = evalBraintrustConfig(process.env);+ const trace = traceConfig ? new EvalAttemptTrace(traceConfig, {+ campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? "",+ attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ?? "",+ executionId: execution.id, provider: execution.profile.provider, model: execution.profile.model, environment: execution.environment.id,+ }, {+ suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash,+ profileId: execution.profile.id, caseId: execution.task.id, sourceRevision: resolveRunnerE2ESource().sha,+ }) : undefined;+ const saveTrace = async () => { if (trace) await writeSanitizedJson(snapshotsDir, "braintrust-trace.json", trace.snapshot(), secrets); };+ const onCompanyCreated = async (id: string) => {+ if (trace) { try { await trace.configureCompany(api, id); } finally { await saveTrace(); } }+ };const api = new RunnerApi(request);const companyRunFlow = execution.suite.id === "task-titles" || ["plan_task_guidance", "blocker_guidance", "continuation_accounting", "continuation", "context_integrity", "repository_skills", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "pi_controls", "copilot_protection", "cursor_native", "native_active_stop", "native_provider_loss", "public_mcp", "case_handoff"].includes(execution.task.flow);const publicMcpUsage = execution.task.flow === "public_mcp" ? assistantUsage(execution.profile.provider === "claude" ? "anthropic" : "openai", execution.profile.model) : undefined;@@ -626,6 +648,7 @@ for (const execution of executions) {let issue: IssueRecord | undefined;let workflowCase: { id: string; identifier?: string } | undefined;let selectedRuns: RunRecord[] = [];+ let feedback = emptyFeedbackSummary();let runtimeLeases: EnvironmentLeaseRecord[] = [];let matcherResults: MatcherResult[] = [];let downloadedResponseProof: Awaited<ReturnType<typeof readResponseProof>> | undefined;@@ -891,6 +914,8 @@ for (const execution of executions) {});try {+ trace?.phase("setup");+ await saveTrace();if (execution.task.flow === "public_mcp") publicMcpUserId = await establishPublicMcpSession(api, page, secrets);if (execution.suite.id === "stock-harness") {const receipt = verifyStockHarnessPreflight(process.env[STOCK_PREFLIGHT_ENV]);@@ -920,7 +945,7 @@ for (const execution of executions) {await setupGitStreamingWorkspace(workspacePath);}fixtures = execution.task.flow === "first_task"- ? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, deadlineAt: startedAtMs + deadlineMs, observe: value => { fixtures = value; } })+ ? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, deadlineAt: startedAtMs + deadlineMs, observe: value => { fixtures = value; }, onCompanyCreated }): await setupLiveFixtures({api,execution,@@ -928,7 +953,13 @@ for (const execution of executions) {workspacePath,credentials,daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE,+ onCompanyCreated,});+ await finishBudgetedEvalSetup({ executionId: execution.id, agentId: fixtures.agent.id,+ companyId: fixtures.company.id, deadlineAt: startedAtMs + deadlineMs, api,+ observe: observations => writeSanitizedJson(snapshotsDir, "budgeted-setup.json", { observations }, secrets) });+ await workerTiming.enter("execution");+ trace?.phase("execution");if (["native-completion", NATIVE_INSTRUCTION_SUITE].includes(execution.suite.id)) {const receipt = await captureNativeDefault({ api, agentId: fixtures.agent.id, companyId: fixtures.company.id });@@ -3078,6 +3109,8 @@ for (const execution of executions) {}}} finally {+ await workerTiming.enter("cleanup");+ trace?.phase("cleanup", Boolean(primaryError));await responseTiming?.stop();await reviewProvider?.close();try {@@ -3136,7 +3169,29 @@ for (const execution of executions) {}},assertions: beforeEnvironmentTeardownAssertions,- teardown: () => fixtures!.teardown(),+ teardown: async () => {+ await workerTiming.enter("feedback");+ trace?.phase("feedback");+ try {+ feedback = await collectAttemptFeedback({ api, companyId: fixtures!.company.id,+ onCheckpoint: summary => { feedback = summary; },+ persist: async (evidence, signal) => {+ await writeSanitizedJson(snapshotsDir, "agent-feedback.pending.json", evidence, secrets, signal);+ signal.throwIfAborted();+ await rename(path.join(snapshotsDir, "agent-feedback.pending.json"), path.join(snapshotsDir, "agent-feedback.json"));+ signal.throwIfAborted();+ },+ });+ } catch {+ feedback.collection = feedback.bodiesCollected ? "partial" : "unavailable";+ feedback.invocationCollection = "unavailable";+ networkDiagnostics.push({ feedbackCaptureError: "feedback_evidence_write_failed" });+ } finally {+ await workerTiming.enter("cleanup");+ trace?.phase("cleanup", !["complete", "not_started"].includes(feedback.collection));+ await fixtures!.teardown();+ }+ },});matcherResults.push(...verification.checks.map(check => ({matcher: { kind: "json_path" as const, path: `remoteCleanup.${check.id}`, expected: true }, passed: check.passed, detail: check.detail,@@ -3202,6 +3257,20 @@ for (const execution of executions) {}}+ await workerTiming.enter("finalization");+ if (execution.suite.id === FEEDBACK_PERSISTENCE_SUITE) {+ let evidence: unknown;+ try { evidence = JSON.parse(await readFile(path.join(snapshotsDir, "agent-feedback.json"), "utf8")); } catch { evidence = undefined; }+ const checks = gradeFeedbackPersistence({ evidence, nonce, companyId: fixtures?.company.id,+ agentId: fixtures?.agent.id, taskId: issue?.id, runIds: selectedRuns.map(run => run.id) });+ matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const,+ path: `feedbackPersistence.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })));+ if (checks.some(check => !check.passed)) primaryError ??= new Error("Feedback persistence matcher failed");+ }++ const traceReceipt = await trace?.export({ failed: Boolean(primaryError), runIds: selectedRuns.map(run => run.id),+ cleanup, complaints: feedback.complaints, suggestions: feedback.suggestions });+ await saveTrace();const finishedAtMs = Date.now();const runtimeUsage = fixtureRuntimeCostCoverage(buildRuntimeUsage({environmentId: execution.environment.id,@@ -3211,6 +3280,12 @@ for (const execution of executions) {}), fixtures ?? {});const resultWithoutBilling: RunnerE2EResult = {schema: "paperclip.runner-e2e.result/v2",+ ...(process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID && process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ? {+ accountingIdentity: {+ campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID,+ attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID,+ },+ } : {}),executionId: execution.id,suiteId: execution.suite.id,suiteDefinitionHash: execution.suiteDefinitionHash,@@ -3248,6 +3323,7 @@ for (const execution of executions) {startedAt,finishedAt: new Date(finishedAtMs).toISOString(),durationMs: finishedAtMs - startedAtMs,+ workerTiming: await workerTiming.finish(finishedAtMs),usage:selectedRuns.length === 1? (selectedRuns[0]?.usageJson ?? null)@@ -3258,6 +3334,8 @@ for (const execution of executions) {})),},runtimeUsage,+ feedback,+ ...(traceReceipt ? { tracing: traceReceipt } : {}),matcherResults,screenshots,cleanup,diff --git a/tests/runner-e2e/scheduled-finalization.test.ts b/tests/runner-e2e/scheduled-finalization.test.tsnew file mode 100644index 0000000000..8b90c9fd37--- /dev/null+++ b/tests/runner-e2e/scheduled-finalization.test.ts@@ -0,0 +1,54 @@+import { expect, it } from "vitest";+import { buildRunnerCampaign } from "./history.js";+import { runnerMatrix } from "./catalog.js";+import { emptyFeedbackSummary, feedbackReview, type FeedbackEvidence } from "./feedback.js";+import { expectedFixtureFeedback } from "./feedback-persistence.js";+import { reviewScheduledFeedback, scheduledCampaignFindings } from "./scheduled-finalization.js";+import type { RunnerE2EResult } from "./types.js";++function fixture() {+ const cell = runnerMatrix.find(row => row.id === "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion")!;+ const nonce = "abcdef123-1", bodies = expectedFixtureFeedback(nonce);+ const entries = ["complaint", "suggestion"].map(kind => ({ kind, body: bodies[kind as keyof typeof bodies] })) as FeedbackEvidence["entries"];+ const feedback = { version: 1, companyId: "company", entries, invocations: [], diagnostics: [], summary: emptyFeedbackSummary() } as FeedbackEvidence;+ const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id,+ suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash,+ accountingIdentity: { campaignId: "scheduled-fixture", attemptId: "one" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null },+ attempt: 1, status: "passed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id,+ environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode,+ startedAt: "2026-10-21T00:00:00.000Z", finishedAt: "2026-10-21T00:00:01.000Z", durationMs: 1000,+ feedback: { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete", complaints: 1, suggestions: 1,+ bodiesCollected: 2, review: feedbackReview(entries) },+ matcherResults: [{ matcher: { kind: "message_exact", expected: `PAPERCLIP_E2E_OK_${nonce}` }, passed: true, detail: "matched" },+ ...["complete-retained-evidence", "one-of-each", "complaint-body-and-attribution", "complaint-completed-tool",+ "suggestion-body-and-attribution", "suggestion-completed-tool", "only-expected-submissions"].map(id => ({+ matcher: { kind: "json_path" as const, path: `feedbackPersistence.${id}`, expected: true }, passed: true, detail: "matched" }))] };+ const campaign = buildRunnerCampaign({ campaignId: "scheduled-fixture", generatedAt: result.finishedAt,+ expected: [cell.id], results: [result], attempts: [result] });+ return { result, feedback, campaign };+}++it("reviews independently retained exact synthetic bodies without a model call", () => {+ const { result, feedback, campaign } = fixture();+ const review = reviewScheduledFeedback(campaign, [{ result, feedback }], result.finishedAt);+ expect(review.decisions.map(row => row.disposition)).toEqual(["unsupported", "unsupported"]);+ expect(JSON.stringify(review)).not.toContain("Synthetic eval");+});++it.each(["body-changed", "wrong-suite", "missing-evidence", "failed-grade", "partial-collection"])("leaves %s feedback for investigation", reason => {+ const { result, feedback, campaign } = fixture();+ if (reason === "body-changed") feedback.entries[0]!.body = "A real product problem";+ if (reason === "wrong-suite") result.suiteId = "core-compatibility";+ if (reason === "failed-grade") result.matcherResults![2]!.passed = false;+ if (reason === "partial-collection") result.feedback!.collection = "partial";+ const review = reviewScheduledFeedback(campaign, [{ result, feedback: reason === "missing-evidence" ? null : feedback }], result.finishedAt);+ expect(review.decisions.every(row => row.disposition === "needs_investigation")).toBe(true);+});++it("keeps incomplete accounting and trace export visible independently of a passing product grade", () => {+ const { result, feedback, campaign } = fixture();+ campaign.feedbackReview = reviewScheduledFeedback(campaign, [{ result, feedback }], result.finishedAt);+ expect(scheduledCampaignFindings(campaign)).toEqual(["accounting_unavailable", "trace_export_incomplete"]);+ campaign.feedbackReview.decisions[0]!.disposition = "needs_investigation";+ expect(scheduledCampaignFindings(campaign)).toContain("feedback_needs_review");+});diff --git a/tests/runner-e2e/scheduled-finalization.ts b/tests/runner-e2e/scheduled-finalization.tsnew file mode 100644index 0000000000..b9653b1772--- /dev/null+++ b/tests/runner-e2e/scheduled-finalization.ts@@ -0,0 +1,117 @@+import { readFile, writeFile } from "node:fs/promises";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { evalBraintrustConfig } from "./braintrust.js";+import { openExperimentIngestion } from "./experiment-ingestion.js";+import { collectScheduledIngestion } from "./ingestion-usage.js";+import { importBraintrustExperiment, type BraintrustExperimentReceipt } from "./braintrust-experiment.js";+import { verifyBraintrustIndexing } from "./braintrust-indexing.js";+import { feedbackReviewTemplate, validateCampaignFeedbackReview, type CampaignFeedbackReview } from "./feedback-dispositions.js";+import { expectedFixtureFeedback } from "./feedback-persistence.js";+import { feedbackReview, summarizeCampaignFeedback, type FeedbackEvidence } from "./feedback.js";+import type { EvalOperationsManifest } from "./operations.js";+import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js";++export function reviewScheduledFeedback(campaign: RunnerE2ECampaign,+ evidence: Array<{ result: RunnerE2EResult; feedback: FeedbackEvidence | null }>, now: string): CampaignFeedbackReview {+ const synthetic = new Set<string>();+ for (const { result, feedback } of evidence) {+ if (result.suiteId !== "feedback-persistence" || !feedback || result.feedback?.collection !== "complete"+ || result.feedback.invocationCollection !== "complete") continue;+ const marker = result.matcherResults?.find(row => row.matcher.kind === "message_exact")?.matcher;+ const nonce = marker?.kind === "message_exact" ? /^PAPERCLIP_E2E_OK_([a-z0-9]+-[1-9][0-9]*)$/.exec(marker.expected)?.[1] : undefined;+ if (!nonce) continue;+ const checks = ["complete-retained-evidence", "one-of-each", "complaint-body-and-attribution", "complaint-completed-tool",+ "suggestion-body-and-attribution", "suggestion-completed-tool", "only-expected-submissions"];+ if (!checks.every(id => result.matcherResults?.some(row => row.matcher.kind === "json_path"+ && row.matcher.path === `feedbackPersistence.${id}` && row.passed))) continue;+ const bodies = expectedFixtureFeedback(nonce);+ if (feedback.entries.length !== 2 || !["complaint", "suggestion"].every(kind => feedback.entries.filter(entry =>+ entry.kind === kind && entry.body === bodies[kind as keyof typeof bodies]).length === 1)) continue;+ for (const group of feedbackReview(feedback.entries)) synthetic.add(group.fingerprint);+ }+ const template = feedbackReviewTemplate(campaign);+ const review: CampaignFeedbackReview = { ...template, version: 1, reviewedAt: now, reviewedBy: "agent",+ decisions: template.decisions.map(row => ({ fingerprint: row.fingerprint,+ disposition: synthetic.has(row.fingerprint) ? "unsupported" : "needs_investigation" })) };+ validateCampaignFeedbackReview(campaign, review);+ return review;+}++export function scheduledCampaignFindings(campaign: RunnerE2ECampaign): string[] {+ const findings: string[] = [];+ if (campaign.failed || campaign.results.length !== campaign.expected.length || campaign.accounting?.missingAttempts) findings.push("product_or_missing_attempt");+ if (campaign.accounting?.attempts.some(row => row.cleanup !== "passed")) findings.push("cleanup_failure");+ const operations = campaign.operations;+ if (!operations || operations.status === "unavailable") findings.push("accounting_unavailable");+ else if (operations.unresolvedRequests || operations.conflictingRequests || operations.inflight) findings.push("accounting_incomplete");+ const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results);+ if (feedback.incomplete || feedback.failedSubmissions || feedback.unknownSubmissions) findings.push("feedback_collection_incomplete");+ if (!campaign.feedbackReview || campaign.feedbackReview.decisions.some(row => ["actionable", "needs_investigation"].includes(row.disposition))) findings.push("feedback_needs_review");+ if (campaign.results.some(row => row.tracing?.status !== "accepted" || row.tracing.nativeConfiguration !== "configured"+ || row.tracing.finalization?.status !== "accepted")) findings.push("trace_export_incomplete");+ return findings;+}++export async function finalizeScheduledCampaign(env: NodeJS.ProcessEnv) {+ if (!env.PAPERCLIP_E2E_OPERATIONS_MANIFEST || !env.GITHUB_WORKSPACE) throw new Error("Missing scheduled campaign manifest");+ const manifest = JSON.parse(await readFile(env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, "utf8")) as EvalOperationsManifest;+ const directory = path.dirname(env.PAPERCLIP_E2E_OPERATIONS_MANIFEST);+ const root = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results", manifest.limits.campaignId);+ const campaign = JSON.parse(await readFile(path.join(root, "campaign.json"), "utf8")) as RunnerE2ECampaign;+ const originals: RunnerE2EResult[] = [], evidence: Array<{ result: RunnerE2EResult; feedback: FeedbackEvidence | null }> = [];+ for (const selected of campaign.results) {+ if (selected.attempt < 1) continue;+ if (!manifest.executionIds.includes(selected.executionId) || selected.attempt !== 1) throw new Error("Unexpected scheduled attempt");+ const resultPath = path.join(root, ...selected.executionId.split("."), "attempt-1");+ const result = JSON.parse(await readFile(path.join(resultPath, "result.json"), "utf8")) as RunnerE2EResult;+ originals.push(result);+ let feedback: FeedbackEvidence | null = null;+ try { feedback = JSON.parse(await readFile(path.join(resultPath, "snapshots/agent-feedback.json"), "utf8")); } catch {}+ evidence.push({ result, feedback });+ }+ campaign.feedbackReview = reviewScheduledFeedback(campaign, evidence, new Date().toISOString());+ await writeFile(path.join(directory, "feedback-review.json"), `${JSON.stringify(campaign.feedbackReview, null, 2)}\n`, { mode: 0o600 });+ const findings = scheduledCampaignFindings(campaign);+ const config = evalBraintrustConfig(env);+ let experiment: BraintrustExperimentReceipt | undefined;+ let ingestion: Awaited<ReturnType<typeof openExperimentIngestion>> | undefined;+ try {+ if (!config) throw new Error("Missing Braintrust export configuration");+ if (config.projectId !== manifest.projectId || campaign.campaignId !== manifest.limits.campaignId) throw new Error("Scheduled ingestion identity mismatch");+ ingestion = await openExperimentIngestion({ projectId: config.projectId, campaignId: manifest.limits.campaignId,+ receiptPath: `${manifest.ledgerPath}.experiment-ingestion.jsonl`, deadlineAt: Date.now() + 60_000 });+ await importBraintrustExperiment(config, campaign, originals, async receipt => {+ await writeFile(path.join(directory, "experiment-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });+ experiment = receipt;+ }, ingestion.transport);+ } catch { findings.push("experiment_import_incomplete"); }+ finally {+ try { await ingestion?.close(); }+ catch { findings.push("experiment_ingestion_accounting_incomplete"); }+ }+ try {+ if (!config) throw new Error("Missing Braintrust ingestion configuration");+ const usage = await collectScheduledIngestion({ ledgerPath: manifest.ledgerPath, campaignId: manifest.limits.campaignId,+ projectId: config.projectId, executionIds: manifest.executionIds });+ await writeFile(path.join(directory, "ingestion-usage.json"), `${JSON.stringify(usage, null, 2)}\n`, { mode: 0o600 });+ if (usage.status !== "complete") findings.push("ingestion_accounting_incomplete");+ } catch { findings.push("ingestion_accounting_unavailable"); }+ let indexing = "unavailable";+ try {+ if (!config) throw new Error("Missing Braintrust query configuration");+ const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment);+ indexing = receipt.status;+ await writeFile(path.join(directory, "indexing-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });+ } catch {}+ if (indexing !== "verified") findings.push("trace_or_experiment_indexing_incomplete");+ await writeFile(path.join(directory, "findings.json"), `${JSON.stringify({ version: 1, findings,+ nativeIndexVerification: indexing, invoiceReconciliation: "pending",+ feedbackReviewMethod: "Only exact synthetic fixture bodies with all corroborating graders are automatically dismissed; all other feedback requires review" }, null, 2)}\n`, { mode: 0o600 });+ for (const finding of findings) console.error(`::error title=Eval operations::${finding}`);+ return findings;+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ if ((await finalizeScheduledCampaign(process.env)).length) process.exitCode = 1;+}diff --git a/tests/runner-e2e/scheduled-forecast.test.ts b/tests/runner-e2e/scheduled-forecast.test.tsnew file mode 100644index 0000000000..daf5b007a1--- /dev/null+++ b/tests/runner-e2e/scheduled-forecast.test.ts@@ -0,0 +1,91 @@+import { expect, it } from "vitest";+import { runnerMatrix } from "./catalog.js";+import { scheduledForecast } from "./scheduled-forecast.js";+import { scheduledSelection } from "./scheduled-operations.js";+import measurements from "./scheduled-measurements.json" with { type: "json" };+import prices from "./scheduled-prices.json" with { type: "json" };+import type { EvalOperationsManifest } from "./operations.js";++const models = prices.models as EvalOperationsManifest["models"];+const selected = (cadence: "daily" | "weekly") => runnerMatrix.filter(cell => scheduledSelection(cadence).includes(cell.id));++it.each([+ ["daily", "0.427624800", 72757], ["weekly", "0.791539400", 127069],+] as const)("retains exact historical %s measurements without inventing hosted or tail estimates", (cadence, modelUsd, summedAttemptMs) => {+ const forecast = scheduledForecast(selected(cadence), models);+ expect(forecast.historicalSuccessfulSelection).toEqual({ modelUsd, summedAttemptMs });+ expect(forecast.measurementSnapshotSha256).toMatch(/^[0-9a-f]{64}$/);+ expect(forecast).toMatchObject({ hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null, costStatus: "estimated" });+ for (const cell of forecast.cells) {+ expect(cell.historicalSuccessful.samples).toBe(1);+ expect(cell.historicalSuccessful.modelUsd.p95).toBeNull();+ expect(cell.historicalSuccessful.attemptMs.p95).toBeNull();+ expect(cell.samples[0]!.resultSha256).toMatch(/^[0-9a-f]{64}$/);+ }+});++it("does not treat the older feedback definition as a current measurement", () => {+ const forecast = scheduledForecast(selected("daily"), models);+ const feedback = forecast.cells.find(cell => cell.executionId.startsWith("feedback-"))!;+ expect(feedback.samples[0]!.mismatches).toContain("definition");+ expect(feedback.compatibleSuccessful.samples).toBe(0);+ expect(forecast.compatibleSuccessfulSelection).toEqual({ modelUsd: null, summedAttemptMs: null });+});++it.each(["definitionHash", "model", "environmentId", "runtimeMode", "route", "priceVersion"] as const)("excludes incompatible %s samples without dropping their evidence", field => {+ const snapshot = structuredClone(measurements);+ const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!;+ const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!;+ sample[field] = field === "definitionHash" ? "a".repeat(64) : "different";+ const forecast = scheduledForecast([cell], models, snapshot);+ expect(forecast.cells[0]!.compatibleSuccessful.samples).toBe(0);+ expect(forecast.cells[0]!.samples).toHaveLength(1);+ expect(forecast.compatibleSuccessfulSelection.modelUsd).toBeNull();+});++it("keeps missing cells and empty selections unknown", () => {+ const snapshot = structuredClone(measurements);+ snapshot.samples = [];+ expect(scheduledForecast(selected("daily"), models, snapshot).historicalSuccessfulSelection)+ .toEqual({ modelUsd: null, summedAttemptMs: null });+ expect(scheduledForecast([], models).compatibleSuccessfulSelection)+ .toEqual({ modelUsd: null, summedAttemptMs: null });+});++it("computes distributions within one cohort and does not sum P95s", () => {+ const snapshot = structuredClone(measurements);+ const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!;+ const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!;+ snapshot.samples = Array.from({ length: 20 }, (_, index) => ({ ...sample, attemptId: `attempt-${index}`,+ knownModelUsd: String(index + 1), attemptDurationMs: (index + 1) * 1000 }));+ const forecast = scheduledForecast([cell], models, snapshot);+ expect(forecast.cells[0]!.compatibleSuccessful).toEqual({ samples: 20,+ modelUsd: { mean: "10.500000000", median: "10.500000000", p95: "19.000000000" },+ attemptMs: { mean: 10500, median: 10500, p95: 19000 } });+ expect(forecast.selectionP95).toBeNull();+ snapshot.samples.push({ ...sample, attemptId: "other-cohort", definitionHash: "a".repeat(64) });+ const mixed = scheduledForecast([cell], models, snapshot);+ expect(mixed.cells[0]!.historicalCohorts).toHaveLength(2);+ expect(mixed.historicalSuccessfulSelection.modelUsd).toBeNull();+ expect(mixed.cells[0]!.compatibleSuccessful.samples).toBe(20);+});++it("rejects duplicate artifacts and invalid measurement provenance", () => {+ const snapshot = structuredClone(measurements);+ snapshot.samples.push(snapshot.samples[0]!);+ expect(() => scheduledForecast(selected("daily"), models, snapshot)).toThrow("Duplicate forecast attempt");+ snapshot.samples.pop();+ snapshot.samples[0]!.resultSha256 = "missing";+ expect(() => scheduledForecast(selected("daily"), models, snapshot)).toThrow("provenance");+});++it("preserves nanodollar costs and genuine measured zero", () => {+ const snapshot = structuredClone(measurements);+ const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!;+ const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!;+ snapshot.samples = [{ ...sample, knownModelUsd: "0.000000001", attemptDurationMs: 0 }];+ expect(scheduledForecast([cell], models, snapshot).compatibleSuccessfulSelection)+ .toEqual({ modelUsd: "0.000000001", summedAttemptMs: 0 });+ snapshot.samples[0]!.knownModelUsd = "0";+ expect(scheduledForecast([cell], models, snapshot).compatibleSuccessfulSelection.modelUsd).toBe("0.000000000");+});diff --git a/tests/runner-e2e/scheduled-forecast.ts b/tests/runner-e2e/scheduled-forecast.tsnew file mode 100644index 0000000000..7cc1acf23f--- /dev/null+++ b/tests/runner-e2e/scheduled-forecast.ts@@ -0,0 +1,92 @@+import { createHash } from "node:crypto";+import { centsToUsd, unitsToCents, usdToUnits } from "../../packages/shared/src/money.js";+import measurements from "./scheduled-measurements.json" with { type: "json" };+import type { MatrixExecution } from "./types.js";+import type { EvalOperationsManifest } from "./operations.js";++type Sample = (typeof measurements.samples)[number];+const cohortKey = (sample: Sample) => JSON.stringify([sample.definitionHash, sample.model, sample.environmentId,+ sample.runtimeMode, sample.route, sample.priceVersion]);+const usd = (units: bigint) => centsToUsd(unitsToCents(units));+const average = (values: bigint[]) => values.length+ ? (values.reduce((sum, value) => sum + value, 0n) + BigInt(Math.floor(values.length / 2))) / BigInt(values.length) : null;++function statistics(values: bigint[]) {+ const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);+ const middle = Math.floor(sorted.length / 2);+ return {+ count: sorted.length,+ mean: average(sorted),+ median: sorted.length ? average(sorted.length % 2 ? [sorted[middle]!] : [sorted[middle - 1]!, sorted[middle]!]) : null,+ p95: sorted.length >= 20 ? sorted[Math.ceil(sorted.length * 0.95) - 1]! : null,+ };+}++function summarize(samples: Sample[]) {+ const cost = statistics(samples.map(sample => usdToUnits(sample.knownModelUsd)));+ const time = statistics(samples.map(sample => BigInt(sample.attemptDurationMs)));+ return {+ samples: cost.count,+ modelUsd: { mean: cost.mean === null ? null : usd(cost.mean), median: cost.median === null ? null : usd(cost.median),+ p95: cost.p95 === null ? null : usd(cost.p95) },+ attemptMs: { mean: time.mean === null ? null : Number(time.mean), median: time.median === null ? null : Number(time.median),+ p95: time.p95 === null ? null : Number(time.p95) },+ };+}++function total(rows: ReturnType<typeof summarize>[]) {+ return {+ modelUsd: rows.length && rows.every(row => row.modelUsd.mean !== null)+ ? usd(rows.reduce((sum, row) => sum + usdToUnits(row.modelUsd.mean!), 0n)) : null,+ summedAttemptMs: rows.length && rows.every(row => row.attemptMs.mean !== null)+ ? rows.reduce((sum, row) => sum + row.attemptMs.mean!, 0) : null,+ };+}++export function scheduledForecast(+ executions: MatrixExecution[], models: EvalOperationsManifest["models"], snapshot = measurements,+) {+ const seen = new Set<string>();+ for (const sample of snapshot.samples) {+ if (seen.has(sample.attemptId)) throw new Error("Duplicate forecast attempt");+ seen.add(sample.attemptId);+ if (!Number.isSafeInteger(sample.attemptDurationMs) || sample.attemptDurationMs < 0 || usdToUnits(sample.knownModelUsd) < 0n+ || !/^[0-9a-f]{40}$/.test(sample.sourceRevision)+ || [sample.definitionHash, sample.resultSha256, sample.ledgerSha256, sample.ledgerFinalHash].some(hash => !/^[0-9a-f]{64}$/.test(hash))) {+ throw new Error("Invalid forecast measurement or provenance");+ }+ }+ if (new Set(executions.map(execution => execution.id)).size !== executions.length) throw new Error("Duplicate forecast execution");+ const cells = executions.map(execution => {+ const retained = snapshot.samples.filter(sample => sample.executionId === execution.id);+ const cohorts = [...new Set(retained.map(cohortKey))].map(key => ({+ key, ...summarize(retained.filter(sample => cohortKey(sample) === key)),+ }));+ const samples = retained.map(sample => {+ const mismatches = [+ sample.definitionHash === execution.suiteDefinitionHash ? null : "definition",+ sample.model === execution.profile.model ? null : "model",+ sample.environmentId === execution.environment.id ? null : "environment",+ sample.runtimeMode === execution.profile.expectedRuntimeMode ? null : "runtime",+ sample.route === "braintrust" ? null : "route",+ models.some(model => model.price.model === sample.model && model.price.version === sample.priceVersion) ? null : "pricing",+ ].filter(value => value !== null);+ return { ...sample, mismatches };+ });+ return { executionId: execution.id, currentDefinitionHash: execution.suiteDefinitionHash,+ historicalSuccessful: summarize(cohorts.length === 1 ? retained : []), historicalCohorts: cohorts,+ compatibleSuccessful: summarize(samples.filter(sample => sample.mismatches.length === 0)), samples };+ });+ return {+ measurementSnapshotSha256: createHash("sha256").update(JSON.stringify(snapshot)).digest("hex"),+ measuredHost: snapshot.host, costStatus: snapshot.costStatus, sampleScope: snapshot.sampleScope,+ historicalSuccessfulSelection: total(cells.map(cell => cell.historicalSuccessful)),+ compatibleSuccessfulSelection: total(cells.map(cell => cell.compatibleSuccessful)),+ hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null,+ cells, limitations: [...snapshot.limitations,+ "Compatibility compares definition, model, environment, runtime, route and price version; source revisions remain explicit",+ "Per-cell P95 uses nearest rank only with at least 20 samples; it is descriptive, not a confidence bound",+ "Summed attempt time is serial attempt work, not campaign wall time, summed agent time or billed runner time",+ "Historical successful samples do not estimate failure frequency; every failure remains in campaign accounting"],+ };+}diff --git a/tests/runner-e2e/scheduled-measurements.json b/tests/runner-e2e/scheduled-measurements.jsonnew file mode 100644index 0000000000..afd9b43aaf--- /dev/null+++ b/tests/runner-e2e/scheduled-measurements.json@@ -0,0 +1,84 @@+{+ "version": 1,+ "recordedAt": "2026-10-21T18:13:01.055Z",+ "costStatus": "estimated",+ "sampleScope": "successful local native qualification attempts only",+ "host": "linux-arm64-local-container",+ "samples": [+ {+ "executionId": "feedback-persistence.runner-codex.local.complaint-and-suggestion",+ "attemptId": "96850eb44e6a9f9856fcf24a40fb4903",+ "sourceRevision": "276b8ff966c177b8cb34af12479915d9c8f2c9a2",+ "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92",+ "knownModelUsd": "0.189173400",+ "attemptDurationMs": 26770,+ "model": "gpt-5.6-sol",+ "environmentId": "local",+ "runtimeMode": "native",+ "route": "braintrust",+ "priceVersion": "openai-standard-2026-10-20",+ "measuredAt": "2026-10-21T16:32:44.767Z",+ "resultSha256": "e1748f875784065f2d40ebd7cc83cb9c3f84b1ce8dd05629482a97e818608188",+ "ledgerSha256": "b5f42a64e8a72c5ae23c4dbf33fda1c561a7e2833d499e86b63627166144f1ef",+ "ledgerFinalHash": "f06b687f8063926b77c0378946f1d6a9bfb718e1d8b9ca7d6e2aa344187be2dc"+ },+ {+ "executionId": "core-compatibility.runner-acpx-claude.local.message-marker",+ "attemptId": "8da10d462119f3cf6bb3a4669c43b6b5",+ "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c",+ "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa",+ "knownModelUsd": "0.123679200",+ "attemptDurationMs": 34495,+ "model": "claude-sonnet-5",+ "environmentId": "local",+ "runtimeMode": "native",+ "route": "braintrust",+ "priceVersion": "anthropic-standard-2026-10-20",+ "measuredAt": "2026-10-21T16:54:26.516Z",+ "resultSha256": "89aac65b9a703d812a5a3c83c51732031e9b6fcadc6bac3c14a86c419f169d7f",+ "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756",+ "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e"+ },+ {+ "executionId": "core-compatibility.runner-codex.local.message-marker",+ "attemptId": "03720091468148fe7a0d2b1fa68d3966",+ "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c",+ "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa",+ "knownModelUsd": "0.174741200",+ "attemptDurationMs": 27542,+ "model": "gpt-5.6-sol",+ "environmentId": "local",+ "runtimeMode": "native",+ "route": "braintrust",+ "priceVersion": "openai-standard-2026-10-20",+ "measuredAt": "2026-10-21T16:53:20.764Z",+ "resultSha256": "188a0655a7aebe37ec21c582e6b4afef1633210fa8f93e1049b9db0078657c4d",+ "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756",+ "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e"+ },+ {+ "executionId": "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion",+ "attemptId": "3849bc83e59ccddfa964120053d3428c",+ "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c",+ "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92",+ "knownModelUsd": "0.303945600",+ "attemptDurationMs": 38262,+ "model": "claude-sonnet-5",+ "environmentId": "local",+ "runtimeMode": "native",+ "route": "braintrust",+ "priceVersion": "anthropic-standard-2026-10-20",+ "measuredAt": "2026-10-21T16:52:21.240Z",+ "resultSha256": "c086648b3dfc67a6d6ee04bb66a1914b3db69f9d40457198e2ce53c3a752ac24",+ "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756",+ "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e"+ }+ ],+ "limitations": [+ "One successful sample per cell; no measured slow-tail distribution or predicted failure rate",+ "Gateway estimates include setup requests; historical phase labels may be inaccurate",+ "Attempt time includes setup and teardown; it excludes queue, dependency installation and compilation",+ "No hosted timing qualification or complete infrastructure, storage and subscription allocation",+ "No change to the conservative per-request reservations or approved budgets"+ ]+}diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.tsnew file mode 100644index 0000000000..72ebe6d4d1--- /dev/null+++ b/tests/runner-e2e/scheduled-operations.test.ts@@ -0,0 +1,77 @@+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, it } from "vitest";+import { prepareScheduledCampaign, scheduledCadence, scheduledManifest, scheduledMatrix, scheduledSelection, validateScheduledBudget } from "./scheduled-operations.js";+import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";+import { runnerMatrix } from "./catalog.js";+import { evalRequestReservation } from "./eval-gateway.js";+import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js";++const overhead: EvalOperatingOverhead = { version: 1, reference: "fixture-only", runner: "github-hosted",+ verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",+ components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) };++const input = { cadence: "daily" as const, campaignId: "scheduled-fixture", ledgerPath: "/tmp/scheduled-fixture/ledger.jsonl",+ projectId: "11111111-1111-4111-8111-111111111111", now: Date.parse("2026-10-21T00:00:00Z"), runAttempt: "1", overhead };++it.each(["daily", "weekly"] as const)("runs the exact %s selection through one shared budget and catalog job", cadence => {+ const manifest = scheduledManifest({ ...input, cadence });+ const selected = selectRunnerExecutions(parseRunnerSelectors(manifest.executionIds.flatMap(id => ["--id", id])), runnerMatrix);+ expect(selected.map(row => row.id).sort()).toEqual([...scheduledSelection(cadence)].sort());+ expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxConcurrent: 1, maxAttemptsPerExecution: 1 });+ expect(Date.parse(manifest.limits.deadlineAt) - input.now).toBe(30 * 60_000);+ expect(scheduledMatrix(cadence).include).toHaveLength(1);+ expect(manifest.executionIds).toHaveLength(cadence === "daily" ? 2 : 4);+ expect(manifest.models.every(model => Number(evalRequestReservation({ ...model, mode: "braintrust", apiKey: "fixture" })) <= Number(manifest.limits.maxUsd))).toBe(true);+});++it("does not reduce Codex's verified context ceiling to fit the daily allowance", () => {+ const codex = scheduledManifest({ ...input, cadence: "weekly" }).models.find(row => row.provider === "openai")!;+ expect(codex.maximumInputTokens).toBe(1_050_000);+ expect(Number(evalRequestReservation({ ...codex, mode: "braintrust", apiKey: "fixture" }))).toBeGreaterThan(20);+ expect(scheduledManifest(input).models.map(model => model.provider)).toEqual(["anthropic"]);+});++it.each(["2", "0", "", "01"])("refuses workflow attempt %s before creating a fresh allowance", runAttempt => {+ expect(() => scheduledManifest({ ...input, runAttempt })).toThrow("fresh allowance");+});++it.each(["2026-10-19T23:59:59Z", "2026-11-19T00:00:00Z"])("rejects pricing outside its reviewed window at %s", time => {+ expect(() => scheduledManifest({ ...input, now: Date.parse(time) })).toThrow("pricing needs review");+});++it("rejects unknown triggers and maps the reviewed schedule expressions", () => {+ expect(scheduledCadence("schedule", "47 8 * * *")).toBe("daily");+ expect(scheduledCadence("schedule", "47 9 * * 0")).toBe("weekly");+ expect(scheduledCadence("workflow_dispatch", "", "manual")).toBeUndefined();+ expect(() => scheduledCadence("schedule", "* * * * *")).toThrow();+ expect(() => scheduledCadence("pull_request", "", "daily")).toThrow();+});++it("fails closed before paid work if disabled, missing project identity, or preflight is repeated", async () => {+ const directory = await mkdtemp(path.join(tmpdir(), "eval-schedule-"));+ const env = { EVAL_EVENT: "schedule", EVAL_SCHEDULE: "47 8 * * *", EVAL_OPERATIONS_ENABLED: "true",+ GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory,+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(overhead), EVAL_TEST_RUNNER: "ubuntu-latest",+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId };+ try {+ await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATIONS_ENABLED: "false" }, input.now)).rejects.toThrow("not enabled");+ await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATING_OVERHEAD_JSON: "" }, input.now)).rejects.toThrow("Missing reviewed");+ await expect(prepareScheduledCampaign({ ...env, PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "" }, input.now)).rejects.toThrow("UUID");+ const prepared = await prepareScheduledCampaign(env, input.now);+ expect(prepared.manifest.limits.campaignId).toBe(prepared.campaignId);+ const forecast = JSON.parse(await readFile(path.join(path.dirname(prepared.manifestPath), "forecast.json"), "utf8"));+ expect(forecast.measurements.historicalSuccessfulSelection).toEqual({ modelUsd: "0.427624800", summedAttemptMs: 72757 });+ expect(forecast.measurements.hostedAttemptMs).toBeNull();+ expect(forecast.maxUsd).toBe("20");+ await expect(prepareScheduledCampaign(env, input.now)).rejects.toThrow("EEXIST");+ } finally { await rm(directory, { recursive: true, force: true }); }+});++it("rejects an overhead allowance that cannot leave room for a full selected request before emitting the matrix", () => {+ const expensive = structuredClone(overhead);+ expensive.components[0]!.maxUsd = "13";+ expect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance");+ expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") });+});diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.tsnew file mode 100644index 0000000000..84bb950e08--- /dev/null+++ b/tests/runner-e2e/scheduled-operations.ts@@ -0,0 +1,121 @@+import { mkdir, readFile, writeFile } from "node:fs/promises";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { runnerMatrix } from "./catalog.js";+import { validateEvalOperationsManifest, type EvalOperationsManifest } from "./operations.js";+import { evalRequestReservation } from "./eval-gateway.js";+import prices from "./scheduled-prices.json" with { type: "json" };+import { scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";+import { usdToUnits } from "../../packages/shared/src/money.js";+import { scheduledForecast } from "./scheduled-forecast.js";++export type EvalCadence = "daily" | "weekly";+export const EVAL_SCHEDULES = { daily: "47 8 * * *", weekly: "47 9 * * 0" } as const;+const budget = (cadence: EvalCadence) => cadence === "daily" ? "20" : "100";+const marker = (profile: string) => `core-compatibility.${profile}.local.message-marker`;+const feedback = (profile: string) => `feedback-persistence.${profile}.local.complaint-and-suggestion`;++export function scheduledSelection(cadence: EvalCadence) {+ if (!["daily", "weekly"].includes(cadence)) throw new Error("Unknown eval cadence");+ return [marker("runner-acpx-claude"), feedback("runner-acpx-claude"),+ ...(cadence === "weekly" ? [marker("runner-codex"), feedback("runner-codex")] : [])];+}++export function scheduledCadence(event: string, schedule: string, requested = "manual"): EvalCadence | undefined {+ if (event === "schedule") {+ const found = Object.entries(EVAL_SCHEDULES).find(([, expression]) => expression === schedule);+ if (!found) throw new Error("Unrecognized paid eval schedule");+ return found[0] as EvalCadence;+ }+ if (event !== "workflow_dispatch" || !["manual", "daily", "weekly"].includes(requested)) throw new Error("Unrecognized eval trigger");+ return requested === "manual" ? undefined : requested as EvalCadence;+}++export function validateScheduledBudget(input: { cadence: EvalCadence; now: number; overhead: EvalOperatingOverhead }) {+ if (!Number.isFinite(input.now) || input.now < Date.parse(prices.verifiedAt) || input.now >= Date.parse(prices.expiresAt)) {+ throw new Error("Scheduled eval pricing needs review before paid admission");+ }+ const executionIds = scheduledSelection(input.cadence);+ const executions = executionIds.map(id => {+ const execution = runnerMatrix.find(row => row.id === id);+ if (!execution) throw new Error(`Scheduled execution is absent: ${id}`);+ return execution;+ });+ const models = prices.models.filter(model => executions.some(execution => execution.profile.model === model.price.model)) as EvalOperationsManifest["models"];+ if (executions.some(execution => !models.some(model => model.price.model === execution.profile.model+ && model.provider === (execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai")))) {+ throw new Error("Missing exact scheduled model pricing");+ }+ const reserved = validateOperatingOverhead(input.overhead, budget(input.cadence));+ if (input.now < Date.parse(input.overhead.verifiedAt) || input.now + 30 * 60_000 >= Date.parse(input.overhead.expiresAt)) {+ throw new Error("Operating overhead allowance does not cover the campaign window");+ }+ if (models.some(model => reserved + usdToUnits(evalRequestReservation({ ...model, apiKey: "forecast", mode: "braintrust" })) > usdToUnits(budget(input.cadence)))) {+ throw new Error("Operating overhead leaves insufficient allowance for the selected model");+ }+ return { executionIds, executions, models };+}++export function scheduledManifest(input: {+ cadence: EvalCadence; campaignId: string; ledgerPath: string; projectId: string; now: number; runAttempt: string;+ overhead: EvalOperatingOverhead;+}): EvalOperationsManifest {+ if (input.runAttempt !== "1") throw new Error("Paid campaign reruns require the retained journal; a fresh allowance is refused");+ const { executionIds, executions, models } = validateScheduledBudget(input);+ const manifest: EvalOperationsManifest = {+ version: 1, limits: { campaignId: input.campaignId, maxUsd: budget(input.cadence), overhead: input.overhead,+ deadlineAt: new Date(input.now + 30 * 60_000).toISOString(), maxRequests: input.cadence === "daily" ? 60 : 120,+ maxConcurrent: 1, maxAttemptsPerExecution: 1 },+ ledgerPath: input.ledgerPath, executionIds, maxAttemptsPerExecution: 1, route: "braintrust", projectId: input.projectId,+ models,+ };+ validateEvalOperationsManifest(manifest, executions, 0);+ return manifest;+}++export function scheduledMatrix(cadence: EvalCadence) {+ return { include: [{ executionId: "scheduled-campaign", suiteId: "core-compatibility", profileId: "runner-acpx-claude",+ credentialName: "", environmentId: "local", caseId: "scheduled-campaign", timeoutMinutes: 60, needsDaytona: false }],+ needsDaytona: false, executionIds: scheduledSelection(cadence) };+}++export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Date.now()) {+ const cadence = scheduledCadence(env.EVAL_EVENT ?? "", env.EVAL_SCHEDULE ?? "", env.EVAL_CADENCE);+ if (!cadence || env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled");+ if (!/^[1-9][0-9]*$/.test(env.GITHUB_RUN_ID ?? "") || !env.GITHUB_WORKSPACE) throw new Error("Missing workflow campaign identity");+ const campaignId = `gha-${env.GITHUB_RUN_ID}-1-scheduled-campaign`;+ const directory = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results/scheduled");+ const manifest = scheduledManifest({ cadence, campaignId, ledgerPath: path.join(directory, "ledger.jsonl"),+ projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "", now, runAttempt: env.GITHUB_RUN_ATTEMPT ?? "",+ overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now) });+ await mkdir(directory, { recursive: true, mode: 0o700 });+ const manifestPath = path.join(directory, "manifest.json");+ await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" });+ const forecast = { cadence, executionIds: manifest.executionIds, maxUsd: manifest.limits.maxUsd,+ overhead: manifest.limits.overhead, budgetScope: "model requests plus declared operating overhead reservations",+ measurements: scheduledForecast(runnerMatrix.filter(execution => manifest.executionIds.includes(execution.id)), manifest.models),+ basis: "Retained local pilot measurements; successful samples and matching configurations are reported separately",+ uncertainty: "Incomplete provider receipts and no invoice reconciliation; estimates are not a guarantee",+ maximumRequestReservations: manifest.models.map(model => ({ model: model.price.model,+ usd: evalRequestReservation({ ...model, apiKey: "forecast", mode: "braintrust" }) })),+ maximumMonthlyTotalAllowanceUsd: "1120", pricingExpiresAt: prices.expiresAt };+ await writeFile(path.join(directory, "forecast.json"), `${JSON.stringify(forecast, null, 2)}\n`, { mode: 0o600, flag: "wx" });+ return { manifest, manifestPath, campaignId, cadence };+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const command = process.argv[2];+ if (command === "matrix") {+ const cadence = scheduledCadence(process.env.EVAL_EVENT ?? "", process.env.EVAL_SCHEDULE ?? "", process.env.EVAL_CADENCE);+ if (!cadence || process.env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled");+ const now = Date.now();+ const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now);+ validateScheduledBudget({ cadence, overhead, now });+ console.log(JSON.stringify({ ...scheduledMatrix(cadence), overhead }));+ } else if (command === "prepare") {+ const prepared = await prepareScheduledCampaign(process.env);+ if (!process.env.GITHUB_ENV) throw new Error("Missing workflow environment file");+ const prior = await readFile(process.env.GITHUB_ENV, "utf8");+ await writeFile(process.env.GITHUB_ENV, `${prior}PAPERCLIP_E2E_OPERATIONS_MANIFEST=${prepared.manifestPath}\nPAPERCLIP_E2E_CAMPAIGN_ID=${prepared.campaignId}\nPAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID=${prepared.campaignId}\n`);+ } else throw new Error("Usage: scheduled-operations.ts <matrix|prepare>");+}diff --git a/tests/runner-e2e/scheduled-prices.json b/tests/runner-e2e/scheduled-prices.jsonnew file mode 100644index 0000000000..c1d437be77--- /dev/null+++ b/tests/runner-e2e/scheduled-prices.json@@ -0,0 +1,51 @@+{+ "version": 1,+ "verifiedAt": "2026-10-20T00:00:00Z",+ "expiresAt": "2026-11-19T00:00:00Z",+ "models": [+ {+ "provider": "openai",+ "maximumInputTokens": 1050000,+ "maximumOutputTokens": 16384,+ "timeoutMs": 180000,+ "price": {+ "provider": "openai",+ "model": "gpt-5.6-sol",+ "version": "openai-standard-2026-10-20",+ "sourceUrl": "https://developers.openai.com/api/docs/pricing",+ "inputUsdPerMillion": "4",+ "cachedReadUsdPerMillion": "0.4",+ "outputUsdPerMillion": "20",+ "cacheWrite30mUsdPerMillion": "5",+ "cacheWrite12hUsdPerMillion": "10",+ "longContext": {+ "aboveInputTokens": 272000,+ "rates": {+ "inputUsdPerMillion": "8",+ "cachedReadUsdPerMillion": "0.8",+ "outputUsdPerMillion": "30",+ "cacheWrite30mUsdPerMillion": "10",+ "cacheWrite12hUsdPerMillion": "20"+ }+ }+ }+ },+ {+ "provider": "anthropic",+ "maximumInputTokens": 1000000,+ "maximumOutputTokens": 16384,+ "timeoutMs": 180000,+ "price": {+ "provider": "anthropic",+ "model": "claude-sonnet-5",+ "version": "anthropic-standard-2026-10-20",+ "sourceUrl": "https://platform.claude.com/docs/en/models/sonnet-5/overview",+ "inputUsdPerMillion": "2",+ "cachedReadUsdPerMillion": "0.2",+ "outputUsdPerMillion": "10",+ "cacheWrite5mUsdPerMillion": "2.5",+ "cacheWrite1hUsdPerMillion": "4"+ }+ }+ ]+}diff --git a/tests/runner-e2e/select-rerun-artifacts.ts b/tests/runner-e2e/select-rerun-artifacts.tsindex 3745881532..9dab414180 100644--- a/tests/runner-e2e/select-rerun-artifacts.ts+++ b/tests/runner-e2e/select-rerun-artifacts.ts@@ -1,4 +1,4 @@-import { cp, mkdir, readFile, readdir } from "node:fs/promises";+import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";import path from "node:path";import { pathToFileURL } from "node:url";import type { RunnerE2EResult } from "./types.js";@@ -307,6 +307,11 @@ export async function selectRerunArtifacts(input: SelectRerunArtifactsInput) {});selections.push({ executionId, workflowAttempt, artifactName });}+ await mkdir(input.selectedRoot, { recursive: true });+ await writeFile(path.join(input.selectedRoot, "selected-artifacts.json"), JSON.stringify({+ version: 1, workflowRunId: runId, workflowRunAttempt: currentAttempt,+ sourceSha: input.sourceSha, sourceRef: input.sourceRef, workflowRunUrl: input.workflowRunUrl, selections,+ }));return selections;}diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.tsindex 05a82316c1..8956ac3f5a 100644--- a/tests/runner-e2e/server.ts+++ b/tests/runner-e2e/server.ts@@ -1,5 +1,9 @@import { parseFinalizationCrashBinding, readFinalizationCrashBoundary, type FinalizationCrashBinding } from "./finalization-crash-boundary.js";import { parseRestartRunnerIdentity, type RestartRunnerIdentity } from "./process-tree-owner.js";+import { recoverInterruptedFeedback } from "./feedback-recovery.js";+import { normalizedSecrets } from "./redaction.js";+import { CREDENTIAL_NAMES } from "./types.js";+import { EVAL_OBSERVABILITY_SECRET_NAMES } from "./braintrust.js";import { usesInstalledCli, verifyInstalledCli } from "./installed-cli.js";import { runnerMatrix } from "./catalog.js";import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server-stop.js";@@ -117,6 +121,43 @@ let child: ChildProcess | null = null;let unexpectedChildFailure: Error | null = null;let shutdownSignal: NodeJS.Signals | null = null;let activeRestartRequestId: string | null = null;+let feedbackRecovery: Promise<void> | undefined;++function recoverFeedbackBeforeShutdown() {+ return feedbackRecovery ??= (async () => {+ const privateDir = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR;+ const campaignId = process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID;+ const attemptId = process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID;+ if (!privateDir || !campaignId || !attemptId || process.env.PAPERCLIP_RUNNER_E2E_PUBLIC_MCP === "1") return;+ const deadlineAt = Date.now() + 5_000;+ const secrets = normalizedSecrets([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES,+ "PAPERCLIP_E2E_GATEWAY_TOKEN", "PAPERCLIP_AGENT_JWT_SECRET", "PAPERCLIP_DECISION_SIGNING_SECRET",+ "PAPERCLIP_TOOL_ACTION_SIGNING_SECRET", "BETTER_AUTH_SECRET"].map(name => process.env[name]));+ for (const executionId of executionIds) {+ if (Date.now() >= deadlineAt) break;+ const execution = runnerMatrix.find(row => row.id === executionId);+ if (!execution) continue;+ try {+ await recoverInterruptedFeedback({ identity: { campaignId, attemptId, executionId },+ caseDirectory: path.join(privateDir, "cases", execution.task.id), secrets, deadlineAt,+ api: { async get<T>(endpoint: string, options?: { timeout: number }): Promise<T> {+ const base = `http://127.0.0.1:${port}`;+ const url = new URL(endpoint, base);+ if (url.origin !== base || !url.pathname.startsWith("/api/")) throw new Error("Invalid feedback recovery endpoint");+ const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(Math.max(1, Math.min(options?.timeout ?? 1_000, deadlineAt - Date.now()))) });+ if (!response.ok) { await response.body?.cancel(); throw new Error("Feedback recovery read failed"); }+ return response.json() as Promise<T>;+ } },+ });+ } catch { appendLog("Feedback recovery checkpoint unavailable; retain private recovery state.\n"); }+ }+ })();+}++async function stopWithFeedback(candidate: ChildProcess, signal: NodeJS.Signals) {+ await recoverFeedbackBeforeShutdown();+ await stopServer(candidate, signal);+}function appendLog(message: string) {process.stderr.write(message);@@ -186,7 +227,7 @@ async function startServer() {// that race create an unowned replacement server.if (shutdownSignal) {// The main/error path awaits this same promise and reports any failure.- void stopServer(candidate, shutdownSignal).catch(() => {});+ void stopWithFeedback(candidate, shutdownSignal).catch(() => {});}await stopServer.watch(candidate);return candidate;@@ -344,9 +385,7 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {if (shutdownSignal) return;shutdownSignal = signal;if (!child) return;- // Begin immediately, including during a health/restart wait. The final- // cleanup joins this promise instead of sending a second graceful signal.- void stopServer(child, signal).catch(() => {});+ void stopWithFeedback(child, signal).catch(() => {});});}@@ -374,6 +413,7 @@ async function supervise() {await delay(200);}+ await recoverFeedbackBeforeShutdown();await stopServer.stopAll(shutdownSignal ?? "SIGTERM");}@@ -394,6 +434,7 @@ try {}}try {+ await recoverFeedbackBeforeShutdown();await stopServer.stopAll();} catch (stopError) {appendLog(`Failed to stop Paperclip after supervisor failure: ${stopError instanceof Error ? stopError.message : String(stopError)}\n`);diff --git a/tests/runner-e2e/trace-recovery.test.ts b/tests/runner-e2e/trace-recovery.test.tsnew file mode 100644index 0000000000..da24f6f824--- /dev/null+++ b/tests/runner-e2e/trace-recovery.test.ts@@ -0,0 +1,46 @@+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { expect, it, vi } from "vitest";+import { EvalAttemptTrace, finalizeEvalAttemptTrace } from "./braintrust.js";+import { readInterruptedAttemptTrace } from "./trace-recovery.js";++it("finalizes a lost worker's original trace while rejecting unrelated or malformed snapshots", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "eval-trace-recovery-"));+ const projectId = "11111111-1111-4111-8111-111111111111";+ const companyId = "22222222-2222-4222-8222-222222222222";+ const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution", provider: "acpx", model: "claude", environment: "local" };+ const config = { projectId, apiKey: "synthetic-canary" };+ const trace = new EvalAttemptTrace(config, identity).snapshot();+ const saved = { ...trace, companyId, nativeConfiguration: "configured", privateCanary: "must-not-copy", finalization: { status: "accepted" } };+ const fixture = { companyId, executionId: identity.executionId };+ const snapshot = path.join(root, "snapshots/braintrust-trace.json");+ try {+ await mkdir(path.join(root, "snapshots"));+ await writeFile(path.join(root, "snapshots/fixtures.json"), JSON.stringify(fixture));+ await writeFile(snapshot, JSON.stringify(saved));+ const recovered = await readInterruptedAttemptTrace(root, { projectId, executionId: identity.executionId });+ expect(recovered).toMatchObject({ traceId: trace.traceId, spanId: trace.spanId, status: "pending", verification: "unverified" });+ expect(recovered).not.toHaveProperty("privateCanary");+ expect(recovered).not.toHaveProperty("finalization");+ const transport = vi.fn<typeof fetch>(async (_url, init) => {+ const body = init!.body as Buffer;+ expect(body.includes(Buffer.from(trace.traceId, "hex"))).toBe(true);+ expect(body.includes(Buffer.from(trace.spanId, "hex"))).toBe(true);+ expect(body.toString()).not.toContain("must-not-copy");+ return new Response(null, { status: 200 });+ });+ expect(await finalizeEvalAttemptTrace(config, { receipt: recovered!, identity,+ provenance: { suiteId: "suite", suiteDefinitionHash: "a".repeat(64), profileId: "profile", caseId: "case", sourceRevision: "b".repeat(40) },+ result: { startedAt: new Date(Date.now() - 1000).toISOString(), status: "failed", cleanup: "failed", failureClass: "cleanup_failure" },+ }, transport)).toMatchObject({ status: "accepted", outcome: "failed", verification: "unverified" });+ expect(transport).toHaveBeenCalledOnce();+ expect(JSON.parse(await readFile(snapshot, "utf8"))).toEqual(saved);+ expect(await readInterruptedAttemptTrace(root, { projectId: companyId, executionId: identity.executionId })).toBeUndefined();+ expect(await readInterruptedAttemptTrace(root, { projectId, executionId: "other" })).toBeUndefined();+ for (const delta of [{ companyId: projectId }, { spanId: "0".repeat(16) }, { traceId: "invalid" }, { revision: "invalid" }, { nativeConfiguration: "unknown" }]) {+ await writeFile(snapshot, JSON.stringify({ ...saved, ...delta }));+ expect(await readInterruptedAttemptTrace(root, { projectId, executionId: identity.executionId })).toBeUndefined();+ }+ } finally { await rm(root, { recursive: true, force: true }); }+});diff --git a/tests/runner-e2e/trace-recovery.ts b/tests/runner-e2e/trace-recovery.tsnew file mode 100644index 0000000000..2f9feef9ee--- /dev/null+++ b/tests/runner-e2e/trace-recovery.ts@@ -0,0 +1,23 @@+import { readFile } from "node:fs/promises";+import path from "node:path";+import { exportBraintrustAttemptParent, type EvalTraceReceipt } from "./braintrust.js";++export async function readInterruptedAttemptTrace(caseDirectory: string, expected: {+ projectId: string; executionId: string;+}): Promise<EvalTraceReceipt | undefined> {+ try {+ const snapshots = path.join(caseDirectory, "snapshots");+ const [fixture, trace] = await Promise.all(["fixtures.json", "braintrust-trace.json"].map(async name =>+ JSON.parse(await readFile(path.join(snapshots, name), "utf8"))));+ const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+ if (fixture.executionId !== expected.executionId || !uuid.test(fixture.companyId)+ || trace.projectId !== expected.projectId || trace.companyId !== fixture.companyId+ || trace.revision !== undefined && !uuid.test(trace.revision)+ || !["pending", "accepted", "failed"].includes(trace.status)+ || !["not_started", "configured", "failed"].includes(trace.nativeConfiguration)) return;+ exportBraintrustAttemptParent(trace);+ return { projectId: trace.projectId, traceId: trace.traceId, spanId: trace.spanId,+ companyId: trace.companyId, ...(trace.revision ? { revision: trace.revision } : {}),+ status: trace.status, nativeConfiguration: trace.nativeConfiguration, verification: "unverified" };+ } catch { return; }+}diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.tsindex df280261e5..f54348ef35 100644--- a/tests/runner-e2e/types.ts+++ b/tests/runner-e2e/types.ts@@ -244,7 +244,7 @@ export type RunnerE2ECostStatus =export interface RunnerE2ERuntimeUsage {provider: RunnerEnvironmentId;/** Sum of the selected Paperclip heartbeat-run spans. */- agentRunDurationMs: number;+ agentRunDurationMs: number | null;/** Sum of provider lease windows when the environment exposes leases. */leaseDurationMs: number | null;leaseCount: number;@@ -267,15 +267,17 @@ export interface RunnerE2EBillingSummary {runCount: number;runsWithTokenUsage: number;runsWithReportedCost: number;+ runsWithEstimatedCost?: number;+ runsWithPartialReportedCost?: number;inputTokens: number;outputTokens: number;cachedInputTokens: number;totalTokens: number;reportedCostUsd: number;- costStatus: Exclude<RunnerE2ECostStatus, "estimated" | "not_metered">;+ estimatedCostUsd?: number;+ costStatus: Exclude<RunnerE2ECostStatus, "not_metered">;};runtime: RunnerE2ERuntimeUsage;- /** Provider-reported model spend only; never includes unknown/unpriced runs. */reportedCostUsd: number;/** Public-list-price estimate for metered execution infrastructure. */estimatedRuntimeCostUsd: number;@@ -288,6 +290,7 @@ export interface RunnerE2EBillingSummary {export interface RunnerE2EResult {schema: "paperclip.runner-e2e.result/v1" | "paperclip.runner-e2e.result/v2";+ accountingIdentity?: { campaignId: string; attemptId: string };executionId: string;suiteId?: string;suiteDefinitionHash?: string;@@ -338,9 +341,12 @@ export interface RunnerE2EResult {startedAt: string;finishedAt: string;durationMs: number;+ workerTiming?: import("./worker-timing.js").WorkerTiming;usage?: Record<string, unknown> | null;runtimeUsage?: RunnerE2ERuntimeUsage;billing?: RunnerE2EBillingSummary;+ feedback?: import("./feedback.js").FeedbackSummary;+ tracing?: import("./braintrust.js").EvalTraceReceipt;matcherResults?: Array<{matcher: Matcher;passed: boolean;@@ -393,15 +399,39 @@ export interface RunnerE2EAggregateBillingSummary {assistant?: { requests: number; inputTokens: number; outputTokens: number; cachedInputTokens: number; estimatedCostUsd: number };judge?: RunnerE2EJudgeBillingSummary;testCount: number;- agentRunDurationMs: number;- leaseDurationMs: number;+ agentRunDurationMs: number | null;+ leaseDurationMs: number | null;llm: RunnerE2EBillingSummary["llm"];reportedLlmCostUsd: number;+ estimatedLlmCostUsd?: number;estimatedRuntimeCostUsd: number;observedAndEstimatedCostUsd: number | null;testsWithCompleteBilling: number;}+export interface RunnerE2EAttemptReceipt {+ id: string;+ executionId: string;+ attempt: number;+ startedAt: string;+ finishedAt: string;+ durationMs: number;+ workerTiming?: import("./worker-timing.js").WorkerTiming;+ status: RunnerE2EResult["status"];+ cleanup: RunnerE2EResult["cleanup"];+ billing: RunnerE2EBillingSummary;+ feedback?: import("./feedback.js").FeedbackSummary;+ tracing?: import("./braintrust.js").EvalTraceReceipt;+}++export interface RunnerE2ECampaignAccounting {+ version: 1;+ scope: "retained_attempts" | "selected_results";+ attempts: RunnerE2EAttemptReceipt[];+ missingAttempts: number;+ excludedArtifacts: number;+}+export interface RunnerE2ECampaign {schema: "paperclip.runner-e2e.campaign/v2";campaignId: string;@@ -427,6 +457,10 @@ export interface RunnerE2ECampaign {sourceUrl: string;}>;billing: RunnerE2EAggregateBillingSummary;+ accounting?: RunnerE2ECampaignAccounting;+ operations?: import("./operations-report.js").EvalOperationsReport;+ feedbackReview?: import("./feedback-dispositions.js").CampaignFeedbackReview;+ braintrustExperiment?: import("./braintrust-experiment.js").BraintrustExperimentReceipt;suites: RunnerE2ESuiteSummary[];results: RunnerE2EResult[];}diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.tsindex 7d0dcdbc0f..448f2df923 100644--- a/tests/runner-e2e/vitest.config.ts+++ b/tests/runner-e2e/vitest.config.ts@@ -6,7 +6,7 @@ export default defineConfig({name: "runner-e2e-support",include: ["**/*.test.ts", "**/*.test.mjs"],// These use node:test and run separately in test:e2e:runner:unit.- exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs"],+ exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs"],testTimeout: 30_000,},});diff --git a/tests/runner-e2e/web-server-command.ts b/tests/runner-e2e/web-server-command.tsindex 7735119d24..df8425695a 100644--- a/tests/runner-e2e/web-server-command.ts+++ b/tests/runner-e2e/web-server-command.ts@@ -1,5 +1,7 @@import path from "node:path";+export const runnerE2EPlaywrightShutdownSignal = "SIGINT" as const;+// Let the restart-aware supervisor reap Paperclip and finish its log streams.// Its stop path allows 30 seconds for SIGTERM and 5 seconds for SIGKILL.export const runnerE2EWebServerGracefulShutdown = {@@ -22,11 +24,6 @@ export function runnerE2ETypeScriptProcessArgs(repositoryRoot: string, entry: streturn ["--import", path.join(repositoryRoot, "cli/node_modules/tsx/dist/loader.mjs"), entry, ...args];}-// pnpm's generated playwright bin shim adds NODE_PATH, even when the caller-// rejected ambient injection. The installed lane invokes the public JS bin-// with the current Node directly, keeping the WebServer environment closed.-export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[], installed: boolean) {- return installed- ? { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] }- : { command: "pnpm", args: ["exec", "playwright", ...args] };+export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[]) {+ return { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] };}diff --git a/tests/runner-e2e/worker-timing.test.ts b/tests/runner-e2e/worker-timing.test.tsnew file mode 100644index 0000000000..2a188ea582--- /dev/null+++ b/tests/runner-e2e/worker-timing.test.ts@@ -0,0 +1,127 @@+import { spawn } from "node:child_process";+import { once } from "node:events";+import { mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";+import { createRequire } from "node:module";+import os from "node:os";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { afterEach, describe, expect, it } from "vitest";+import { buildCampaignAccounting } from "./attempt-accounting.js";+import { summarizeWorkerPhaseTiming, workerPhaseTimingText } from "./campaign-timing.js";+import { validateRetainedRunnerResult } from "./result-validation.js";+import { createWorkerTiming, recoverWorkerTiming, validateWorkerTiming } from "./worker-timing.js";+import type { RunnerE2EResult } from "./types.js";++const temporary: string[] = [];+afterEach(async () => { await Promise.all(temporary.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });+async function fixture() {+ const dir = await mkdtemp(path.join(os.tmpdir(), "worker-timing-")); temporary.push(dir);+ const result: RunnerE2EResult = {+ schema: "paperclip.runner-e2e.result/v2", executionId: "timing-case", suiteId: "timing-fixture", attempt: 1,+ accountingIdentity: { campaignId: "timing-campaign", attemptId: "timing-attempt" },+ status: "passed", profileId: "runner-codex", environmentId: "local", caseId: "marker", provider: "codex",+ model: "fixture-model", runtimeMode: "native", startedAt: new Date(1000).toISOString(),+ finishedAt: new Date(1500).toISOString(), durationMs: 500, cleanup: "passed",+ };+ return { file: path.join(dir, "snapshots", "worker-timing.json"), dir, result };+}++async function complete(file: string, result: RunnerE2EResult) {+ const recorder = await createWorkerTiming(file, result, 1000);+ await recorder.enter("execution", 1100);+ await recorder.enter("cleanup", 1300);+ await recorder.enter("feedback", 1350);+ await recorder.enter("cleanup", 1430);+ await recorder.enter("finalization", 1470);+ return recorder.finish(1500);+}++describe("worker phase timing", () => {+ it("retains and validates all phase boundaries, including both cleanup segments", async () => {+ const { file, result } = await fixture(); result.workerTiming = await complete(file, result);+ validateRetainedRunnerResult(result);+ expect(await recoverWorkerTiming(file, result)).toEqual(result.workerTiming);+ const accounting = buildCampaignAccounting({ results: [result], attempts: [result] });+ expect(accounting.attempts[0]?.workerTiming).toEqual(result.workerTiming);+ const summary = summarizeWorkerPhaseTiming([result], accounting);+ expect(summary).toMatchObject({ completeAttempts: 1, partialAttempts: 0, missingAttempts: 0 });+ expect(summary.phases.map(phase => [phase.phase, phase.totalMs])).toEqual([+ ["setup", 100], ["execution", 200], ["feedback", 80], ["cleanup", 90], ["finalization", 30],+ ]);+ expect(workerPhaseTimingText(summary)).toContain("Worker cleanup time: 90ms.");+ });++ it("recovers only completed segments after a real worker process is killed", async () => {+ const { file, result } = await fixture();+ const loader = createRequire(new URL("../../cli/package.json", import.meta.url)).resolve("tsx");+ const module = pathToFileURL(path.join(import.meta.dirname, "worker-timing.ts")).href;+ const program = `import { createWorkerTiming } from ${JSON.stringify(module)};+ const recorder = await createWorkerTiming(${JSON.stringify(file)}, { executionId: "timing-case", attempt: 1 }, 1000);+ await recorder.enter("execution", 1200); process.stdout.write("checkpoint-ready"); setInterval(() => {}, 1000);`;+ const child = spawn(process.execPath, ["--import", loader, "--input-type=module", "-e", program], { stdio: ["ignore", "pipe", "pipe"] });+ const closed = once(child, "close");+ try {+ await new Promise<void>((resolve, reject) => {+ const timer = setTimeout(() => reject(new Error("Worker checkpoint was not written")), 10_000);+ child.once("error", error => { clearTimeout(timer); reject(error); });+ child.stdout!.once("data", chunk => {+ clearTimeout(timer); String(chunk).includes("checkpoint-ready") ? resolve() : reject(new Error("Unexpected worker output"));+ });+ child.once("exit", () => { clearTimeout(timer); reject(new Error("Worker exited before interruption")); });+ });+ child.kill("SIGKILL"); await closed;+ result.status = "failed"; result.workerTiming = await recoverWorkerTiming(file, result);+ expect(result.workerTiming).toMatchObject({ status: "partial", finishedAt: null,+ segments: [{ phase: "setup", finishedAt: new Date(1200).toISOString() }, { phase: "execution", finishedAt: null }] });+ validateRetainedRunnerResult(result);+ const summary = summarizeWorkerPhaseTiming([result]);+ expect(summary).toMatchObject({ completeAttempts: 0, partialAttempts: 1 });+ expect(summary.phases.every(phase => phase.totalMs === null)).toBe(true);+ expect(workerPhaseTimingText(summary)).toContain("Worker setup time: Unknown (200ms observed).");+ expect(workerPhaseTimingText(summary)).toContain("Worker execution time: Unknown.");+ } finally { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); await closed; }+ });++ it("rejects wrong identities, overlapping segments and out-of-window timestamps before reporting", async () => {+ const { file, result } = await fixture(); const receipt = await complete(file, result);+ if (receipt.status === "unavailable") throw new Error("Missing fixture timing");+ const other = structuredClone(receipt); other.executionId = "another-case";+ const overlap = structuredClone(receipt); overlap.segments[1]!.startedAt = new Date(1050).toISOString();+ const future = structuredClone(receipt); future.finishedAt = new Date(1600).toISOString(); future.segments.at(-1)!.finishedAt = future.finishedAt;+ for (const invalid of [other, overlap, future, { ...receipt, extra: "unapproved field" }]) {+ expect(() => validateWorkerTiming(invalid, result)).toThrow("Invalid worker timing");+ expect(() => validateRetainedRunnerResult({ ...result, workerTiming: invalid })).toThrow("result.workerTiming");+ }+ });++ it("bounds recovery reads and refuses symlinks or mismatched attempt files", async () => {+ const { file, dir, result } = await fixture(); await complete(file, result);+ expect(await recoverWorkerTiming(file, { ...result, attempt: 2 })).toMatchObject({ status: "unavailable", reason: "invalid_evidence" });+ const link = path.join(dir, "linked.json"); await symlink(file, link);+ expect(await recoverWorkerTiming(link, result)).toMatchObject({ status: "unavailable", reason: "invalid_evidence" });+ await writeFile(file, " ".repeat(8193));+ expect(await recoverWorkerTiming(file, result)).toMatchObject({ status: "unavailable", reason: "invalid_evidence" });+ expect(await recoverWorkerTiming(path.join(dir, "missing.json"), result)).toBeUndefined();+ });++ it("keeps timing failures from interrupting cleanup or overwriting earlier evidence", async () => {+ const { file, result } = await fixture(); await complete(file, result); const original = await readFile(file, "utf8");+ const recorder = await createWorkerTiming(file, result, 1000);+ await recorder.enter("cleanup", 1100);+ expect(await recorder.finish(1500)).toMatchObject({ status: "unavailable", reason: "write_failed" });+ expect(await readFile(file, "utf8")).toBe(original);+ const clockFile = file + ".clock"; const clock = await createWorkerTiming(clockFile, result, 1000);+ await clock.enter("execution", 900);+ expect(await clock.finish(1500)).toMatchObject({ status: "unavailable", reason: "clock_regression" });+ });++ it("preserves a timing observation through duplicate legacy copies and rejects contradictions", async () => {+ const { file, result } = await fixture(); const measured = { ...result, workerTiming: await complete(file, result) };+ for (const attempts of [[measured, result], [result, measured]]) {+ expect(buildCampaignAccounting({ results: [measured], attempts }).attempts[0]?.workerTiming).toEqual(measured.workerTiming);+ }+ expect(() => buildCampaignAccounting({ results: [measured], attempts: [measured, { ...measured,+ workerTiming: { version: 1, scope: "worker", status: "unavailable", reason: "write_failed" },+ }] })).toThrow("Conflicting timing receipts");+ });+});diff --git a/tests/runner-e2e/worker-timing.ts b/tests/runner-e2e/worker-timing.tsnew file mode 100644index 0000000000..8bbe9abaec--- /dev/null+++ b/tests/runner-e2e/worker-timing.ts@@ -0,0 +1,148 @@+import { randomUUID } from "node:crypto";+import { constants } from "node:fs";+import { mkdir, open, rename, rm, writeFile } from "node:fs/promises";+import path from "node:path";++export const WORKER_TIMING_PHASES = ["setup", "execution", "feedback", "cleanup", "finalization"] as const;+export type WorkerTimingPhase = typeof WORKER_TIMING_PHASES[number];+type TimingContext = { executionId: string; attempt: number; startedAt: string; finishedAt: string };+export type WorkerTiming = {+ version: 1;+ scope: "worker";+ status: "complete" | "partial";+ executionId: string;+ attempt: number;+ startedAt: string;+ finishedAt: string | null;+ segments: Array<{ phase: WorkerTimingPhase; startedAt: string; finishedAt: string | null }>;+} | {+ version: 1;+ scope: "worker";+ status: "unavailable";+ reason: "write_failed" | "invalid_evidence" | "clock_regression";+};++function invalid(): never { throw new Error("Invalid worker timing receipt"); }+function shape(value: unknown, keys: string[]): Record<string, unknown> {+ if (!value || typeof value !== "object" || Array.isArray(value)) invalid();+ const record = value as Record<string, unknown>;+ if (Object.keys(record).length !== keys.length || keys.some(key => !Object.hasOwn(record, key))) invalid();+ return record;+}+function timestamp(value: unknown) {+ if (typeof value !== "string") invalid();+ const ms = Date.parse(value);+ if (!Number.isFinite(ms) || new Date(ms).toISOString() !== value) invalid();+ return ms;+}++export function validateWorkerTiming(value: unknown, context: TimingContext): asserts value is WorkerTiming {+ if (!value || typeof value !== "object" || Array.isArray(value)) invalid();+ const status = (value as Record<string, unknown>).status;+ const record = shape(value, status === "unavailable"+ ? ["version", "scope", "status", "reason"]+ : ["version", "scope", "status", "executionId", "attempt", "startedAt", "finishedAt", "segments"]);+ if (record.version !== 1 || record.scope !== "worker") invalid();+ if (status === "unavailable") {+ if (typeof record.reason !== "string" || !["write_failed", "invalid_evidence", "clock_regression"].includes(record.reason)) invalid();+ return;+ }+ if (status !== "complete" && status !== "partial" || record.executionId !== context.executionId || record.attempt !== context.attempt+ || typeof record.executionId !== "string" || record.executionId.length < 1 || record.executionId.length > 512+ || !Number.isSafeInteger(record.attempt) || Number(record.attempt) < 1) invalid();+ const start = timestamp(record.startedAt), lower = Date.parse(context.startedAt), upper = Date.parse(context.finishedAt);+ if (!Number.isFinite(lower) || !Number.isFinite(upper) || start < lower || start > upper) invalid();+ if (!Array.isArray(record.segments) || record.segments.length < 1 || record.segments.length > 16) invalid();+ let cursor = start;+ record.segments.forEach((raw, index) => {+ const segment = shape(raw, ["phase", "startedAt", "finishedAt"]);+ if (!WORKER_TIMING_PHASES.includes(segment.phase as WorkerTimingPhase) || index === 0 && segment.phase !== "setup") invalid();+ if (timestamp(segment.startedAt) !== cursor || cursor > upper) invalid();+ if (segment.finishedAt === null) {+ if (status !== "partial" || index !== (record.segments as unknown[]).length - 1) invalid();+ } else {+ const end = timestamp(segment.finishedAt);+ if (end < cursor || end > upper || status === "partial" && index === (record.segments as unknown[]).length - 1) invalid();+ cursor = end;+ }+ });+ if (status === "partial" ? record.finishedAt !== null : timestamp(record.finishedAt) !== cursor) invalid();+}++function unavailable(reason: Extract<WorkerTiming, { status: "unavailable" }>["reason"]): WorkerTiming {+ return { version: 1, scope: "worker", status: "unavailable", reason };+}++export async function createWorkerTiming(+ file: string,+ identity: Pick<TimingContext, "executionId" | "attempt">,+ startedAtMs: number,+) {+ let receipt: WorkerTiming = {+ version: 1, scope: "worker", status: "partial", executionId: identity.executionId, attempt: identity.attempt,+ startedAt: new Date(startedAtMs).toISOString(), finishedAt: null,+ segments: [{ phase: "setup", startedAt: new Date(startedAtMs).toISOString(), finishedAt: null }],+ };+ async function persist(initial = false) {+ if (receipt.status === "unavailable") return;+ const temporary = `${file}.${randomUUID()}.pending`;+ try {+ const data = JSON.stringify(receipt);+ if (initial) {+ await mkdir(path.dirname(file), { recursive: true });+ await writeFile(file, data, { mode: 0o600, flag: "wx" });+ }+ else {+ await writeFile(temporary, data, { mode: 0o600, flag: "wx" });+ await rename(temporary, file);+ }+ } catch { receipt = unavailable("write_failed"); }+ finally { await rm(temporary, { force: true }).catch(() => undefined); }+ }+ function closeCurrent(at: number) {+ if (receipt.status === "unavailable" || receipt.status === "complete") return false;+ const active = receipt.segments.at(-1)!;+ if (!Number.isSafeInteger(at) || !Number.isFinite(new Date(at).getTime()) || at < Date.parse(active.startedAt)) {+ receipt = unavailable("clock_regression");+ return false;+ }+ active.finishedAt = new Date(at).toISOString();+ return true;+ }+ await persist(true);+ return {+ async enter(phase: WorkerTimingPhase, at = Date.now()) {+ if (!closeCurrent(at) || receipt.status === "unavailable") return;+ if (receipt.segments.length >= 16) { receipt = unavailable("invalid_evidence"); return; }+ receipt.segments.push({ phase, startedAt: new Date(at).toISOString(), finishedAt: null });+ await persist();+ },+ async finish(at = Date.now()): Promise<WorkerTiming> {+ if (closeCurrent(at) && receipt.status !== "unavailable") {+ receipt.status = "complete";+ receipt.finishedAt = new Date(at).toISOString();+ await persist();+ }+ return structuredClone(receipt);+ },+ };+}++export async function recoverWorkerTiming(file: string, context: TimingContext): Promise<WorkerTiming | undefined> {+ try {+ if (typeof constants.O_NOFOLLOW !== "number" || typeof constants.O_NONBLOCK !== "number") return unavailable("invalid_evidence");+ const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);+ try {+ const stat = await handle.stat();+ if (!stat.isFile() || stat.size > 8192) return unavailable("invalid_evidence");+ const buffer = Buffer.alloc(8193);+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);+ if (bytesRead > 8192) return unavailable("invalid_evidence");+ const receipt: unknown = JSON.parse(buffer.subarray(0, bytesRead).toString("utf8"));+ validateWorkerTiming(receipt, context);+ return receipt;+ } finally { await handle.close(); }+ } catch (error) {+ return (error as NodeJS.ErrnoException).code === "ENOENT" ? undefined : unavailable("invalid_evidence");+ }+}diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.tsindex b89cc7c3b1..e6c8b1c16e 100644--- a/tests/runner-e2e/workflow-security.test.ts+++ b/tests/runner-e2e/workflow-security.test.ts@@ -93,7 +93,7 @@ describe("public repository paid workflow security", () => {const workflows = [{name: "runner-full-stack-e2e.yml",- expectedCachedSetupNodeSteps: 4,+ expectedCachedSetupNodeSteps: 3,},{name: "pr-trusted.yml",@@ -311,7 +311,7 @@ describe("public repository paid workflow security", () => {const everydayOraclePreparation = paidJob.indexOf("- name: Prepare pinned Python artifact oracle image",);- const paidExecution = paidJob.indexOf("- name: Run paid cell");+ const paidExecution = paidJob.indexOf("- name: Run bounded campaign and retain feedback review");expect(paidInstall).toBeGreaterThan(0);expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall);expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation);@@ -588,7 +588,7 @@ describe("public repository paid workflow security", () => {}});- it("runs paid scheduled campaigns only on Sundays", async () => {+ it("uses bounded daily and weekly Product schedules while preserving Runner schedules", async () => {const workflows = await Promise.all(["runner-full-stack-e2e.yml",@@ -598,12 +598,20 @@ describe("public repository paid workflow security", () => {readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),),);- for (const workflow of workflows) {- const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(- (match) => match[1]!,- );- expect(crons).toHaveLength(1);- expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);+ for (const [index, workflow] of workflows.entries()) {+ const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(match => match[1]!);+ if (index === 0) {+ expect(crons).toEqual(["47 8 * * *", "47 9 * * 0"]);+ expect(workflow).toContain("vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'");+ expect(workflow).toContain('test "$TARGET_SHA" = "$GITHUB_SHA"');+ expect(workflow).toContain('echo "max_parallel=1"');+ expect(workflow).toContain("--max-automatic-retries 0");+ const paid = workflow.split("- name: Run bounded campaign and retain feedback review")[1]!.split("- name: Run paid cell")[0]!;+ expect([...paid.matchAll(/secrets\.([A-Z_]+)/g)].map(match => match[1])).toEqual(["BRAINTRUST_API_KEY"]);+ } else {+ expect(crons).toHaveLength(1);+ expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);+ }expect(workflow).toContain("workflow_dispatch:");}});@@ -720,18 +728,18 @@ describe("public repository paid workflow security", () => {);});- it("resolves each trusted reporting lockfile before frozen install and AWS credentials", async () => {+ it("keeps reporting jobs on the committed lockfile without registry re-resolution", async () => {const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8",);for (const jobName of ["report", "publish_history"]) {const job = workflow.split(`\n ${jobName}:`)[1]!.split(/\n [a-z_]+:/u)[0]!;- const resolve = job.indexOf("pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile");- const install = job.indexOf("pnpm install --frozen-lockfile");+ const install = job.indexOf("pnpm install --frozen-lockfile --ignore-scripts");expect(job).toContain("ref: ${{ github.sha }}");expect(job).not.toContain("resolved-target-lockfile");- expect(resolve, jobName).toBeGreaterThan(-1);- expect(install, jobName).toBeGreaterThan(resolve);+ expect(job, jobName).not.toContain("--no-frozen-lockfile");+ expect(job, jobName).not.toContain("--lockfile-only");+ expect(install, jobName).toBeGreaterThan(job.indexOf("ref: ${{ github.sha }}"));const credentials = job.indexOf("aws-actions/configure-aws-credentials@");if (credentials >= 0) expect(install).toBeLessThan(credentials);}@@ -767,11 +775,12 @@ describe("public repository paid workflow security", () => {"PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e",);expect(report).toContain(- "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}",- );- expect(report).toContain(- "PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}",+ "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}",);+ const historyPrefix = "${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}";+ expect(report).toContain(`PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`);+ expect(publisher).toContain(`RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`);+ expect(publisher).toContain("needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'");expect(report.indexOf("Select latest workflow attempt per cell"),).toBeLessThan(report.indexOf("Collect blob reports"));@@ -789,6 +798,64 @@ describe("public repository paid workflow security", () => {);});+ it("keeps recurring reports private while preserving a content-free liveness receipt", async () => {+ const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");+ const paid = workflow.split("\n test:")[1]!.split("\n report:")[0]!;+ const manual = paid.split("\n test_recurring:")[0]!;+ const recurring = paid.split("\n test_recurring:")[1]!;+ expect(manual).not.toContain("id-token: write");+ expect(manual).toContain("if: needs.catalog.outputs.operations_cadence == ''");+ expect(recurring).toContain("id-token: write");+ expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha");+ expect(recurring).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)");+ expect(recurring).toContain("steps: *paid_test_steps");+ expect(manual).toContain("steps: &paid_test_steps");+ const report = workflow.split("\n report:")[1]!.split("\n publish_history:")[0]!;+ const step = (source: string, name: string) => source.split(` - name: ${name}\n`)[1]!.split(/\n - /u)[0]!;+ for (const [job, name] of [[paid, "Upload access-controlled packaged cell evidence"], [report, "Upload access-controlled merged report"]]) {+ expect(step(job!, name!)).toContain("if: always() && needs.catalog.outputs.operations_cadence == ''");+ }+ expect(step(report, "Download cell evidence")).toContain("if: needs.catalog.outputs.operations_cadence == ''");+ expect(step(paid, "Run bounded campaign and retain feedback review")).not.toContain("AWS_");+ for (const job of [paid, report]) {+ const credentials = step(job, "Obtain scoped private history credentials");+ expect(credentials).toContain("output-env-credentials: false");+ expect(credentials).toContain("disable-retry: true");+ expect(credentials).toContain("RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN");+ }+ expect(paid.indexOf("Verify private evidence retention before paid execution")).toBeLessThan(paid.indexOf("Run bounded campaign and retain feedback review"));+ expect(report).toContain("report > runner-e2e-merged-report/report.log 2>&1");+ const marker = step(report, "Retain report availability for the schedule monitor");+ expect(marker).toContain("steps.private_report.outcome == 'success'");+ expect(marker).toContain("path: private-report-status.json");+ expect(report).toContain('echo \'{"version":1,"privateReportRetained":true}\' > private-report-status.json');+ expect(marker).not.toMatch(/BUCKET|ROLE_ARN|private-report-reference/);+ });++ it("isolates private archive trust from workflows that execute target-branch code", async () => {+ const workflows = path.join(repositoryRoot, ".github/workflows");+ const source = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8");+ const trust = await readFile(path.join(repositoryRoot, ".github/infra/runner-e2e-private-history.yml"), "utf8");+ const environments = ["runner-e2e-private-paid", "runner-e2e-private-history"];+ const subjects = [...trust.matchAll(/!Sub 'repo:\$\{GitHubRepository\}:environment:([^']+)'/gu)].map(match => match[1]);+ expect(subjects).toEqual(environments);+ const recurring = source.split("\n test_recurring:")[1]!.split("\n report:")[0]!;+ const report = source.split("\n report:")[1]!.split("\n publish_history:")[0]!;+ for (const [index, job] of [recurring, report].entries()) {+ expect(job).toContain(`name: ${environments[index]}`);+ expect(job).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)");+ }+ expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha");+ for (const filename of await readdir(workflows)) {+ if (!/\.ya?ml$/u.test(filename) || filename === "runner-full-stack-e2e.yml") continue;+ const other = await readFile(path.join(workflows, filename), "utf8");+ for (const environment of environments) expect(other, filename).not.toContain(environment);+ }+ const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8");+ expect(protocol).toContain("name: runner-e2e-paid");+ expect(protocol).toContain("needs.authorize.outputs.target_sha");+ });+it("uses environment-scoped OIDC for a no-delete history publisher", async () => {const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),@@ -827,4 +894,26 @@ describe("public repository paid workflow security", () => {workflow.indexOf("pnpm test:e2e:runner:history:publish"),).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@"));});++ it("does not restore dependency caches or run install scripts in OIDC-authorized jobs", async () => {+ const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");+ const jobs = workflow.split(/\n(?= [a-z_]+:)/u);+ const paidSteps = workflow.split("steps: &paid_test_steps")[1]!.split("\n test_recurring:")[0]!;+ const oidcJobs = jobs.filter(job => job.includes("id-token: write"));+ expect(oidcJobs.length).toBeGreaterThanOrEqual(3);+ for (const job of oidcJobs) {+ const steps = job + (job.includes("steps: *paid_test_steps") ? paidSteps : "");+ expect(steps, job.split("\n")[0]).not.toMatch(/cache:\s*(?:pnpm|npm|yarn)\b/u);+ expect(steps).not.toMatch(/actions\/cache(?:\/restore)?@/u);+ for (const install of steps.matchAll(/\bpnpm install[^\n]*/gu)) {+ expect(install[0]).toContain("--ignore-scripts");+ }+ }+ for (const name of ["report", "publish_history"]) {+ const job = jobs.find(value => value.startsWith(` ${name}:`))!;+ const installs = [...job.matchAll(/\bpnpm install[^\n]*/gu)];+ expect(installs).toHaveLength(1);+ for (const install of installs) expect(install[0]).toContain('--store-dir "${{ runner.temp }}/eval-');+ }+ });});diff --git a/tests/runner-e2e/workflow-target.test.ts b/tests/runner-e2e/workflow-target.test.tsnew file mode 100644index 0000000000..88aa5b6cb5--- /dev/null+++ b/tests/runner-e2e/workflow-target.test.ts@@ -0,0 +1,49 @@+import { spawnSync } from "node:child_process";+import { mkdtemp, readFile, rm } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { describe, expect, it } from "vitest";++const queuedSha = "a".repeat(40);+const advancedSha = "b".repeat(40);++async function resolveTarget(event: string, cadence: string, branch = "master") {+ const workflow = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8");+ const step = workflow.split(" - name: Resolve requested repository branch to an immutable commit\n")[1]!.split("\n - name:")[0]!;+ const script = step.split(" run: |\n")[1]!.replace(/^ {10}/gm, "");+ const directory = await mkdtemp(path.join(os.tmpdir(), "eval-workflow-target-"));+ try {+ const output = path.join(directory, "output");+ const result = spawnSync("bash", ["-c", `gh() { printf '%s\\n' "$MOCK_BRANCH_SHA"; }\n${script}`], {+ encoding: "utf8", env: { ...process.env, EVENT_NAME: event, EVAL_CADENCE: cadence,+ TARGET_BRANCH: branch, DEFAULT_BRANCH: "master", GITHUB_SHA: queuedSha,+ MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip" },+ });+ return { status: result.status, stderr: result.stderr,+ output: await readFile(output, "utf8").catch(() => "") };+ } finally { await rm(directory, { recursive: true, force: true }); }+}++describe("paid workflow target authorization", () => {+ it.each([["schedule", "manual"], ["workflow_dispatch", "daily"], ["workflow_dispatch", "weekly"]])(+ "pins %s/%s to the queued revision when the branch advances", async (event, cadence) => {+ const result = await resolveTarget(event, cadence);+ expect(result.status, result.stderr).toBe(0);+ expect(result.output).toContain(`sha=${queuedSha}\n`);+ expect(result.output).not.toContain(advancedSha);+ });++ it("rejects custom bounded targets before emitting a commit", async () => {+ const result = await resolveTarget("workflow_dispatch", "daily", "development");+ expect(result.status).toBe(1);+ expect(result.output).toBe("");+ expect(result.stderr).toContain("default-branch commit");+ });++ it("still resolves explicit manual targets at authorization time", async () => {+ const result = await resolveTarget("workflow_dispatch", "manual", "development");+ expect(result.status, result.stderr).toBe(0);+ expect(result.output).toContain(`sha=${advancedSha}\n`);+ expect(result.output).toContain("ref=refs/heads/development\n");+ });+});diff --git a/ui/src/pages/Cases.test.tsx b/ui/src/pages/Cases.test.tsxindex c952f558d1..613314a8cd 100644--- a/ui/src/pages/Cases.test.tsx+++ b/ui/src/pages/Cases.test.tsx@@ -278,6 +278,7 @@ describe("Read-only cases list", () => {})));expect(container.querySelector('[data-testid="location"]')?.textContent).toBe("/PAP/cases?sort=title&dir=asc");await escape();+ await waitFor(() => expect(document.activeElement).toBe(labelledButton("Sort")));await click(labelledButton("Group"));const type = [...document.body.querySelectorAll('[aria-label="Group cases"] button')].find(element => element.textContent === "Type")!;