Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

benchmark prime agent performance on pull request updates (#2160)

ktf9c7e06b58b9
diff --git a/.github/workflows/benchmark-completed.yml b/.github/workflows/benchmark-completed.yml
new file mode 100644
index 0000000000..cc00300d56
--- /dev/null
+++ b/.github/workflows/benchmark-completed.yml
@@ -0,0 +1,81 @@
+name: Finalize Prime Agent benchmarks
+
+on:
+ workflow_run:
+ workflows: [Prime Agent benchmarks]
+ types: [completed]
+
+permissions: {}
+
+jobs:
+ cleanup:
+ if: contains(fromJSON('["pull_request_target", "workflow_dispatch"]'), github.event.workflow_run.event)
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ ref: main
+ persist-credentials: false
+ - name: Install benchmark tooling
+ run: |
+ python3 -m venv .benchmark-bootstrap
+ .benchmark-bootstrap/bin/pip install uv==0.12.9
+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks
+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"
+ - name: Delete remaining run sandboxes
+ working-directory: scripts/benchmarks
+ run: uv run --locked cli.py cleanup
+ env:
+ PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}
+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}
+
+ publish:
+ if: contains(fromJSON('["pull_request_target", "workflow_dispatch"]'), github.event.workflow_run.event)
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ concurrency:
+ group: prime-agent-benchmark-comment-${{ github.event.workflow_run.display_title }}
+ cancel-in-progress: false
+ permissions:
+ contents: read
+ actions: read
+ pull-requests: write
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ ref: main
+ persist-credentials: false
+ - name: Download request
+ id: request
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0
+ with:
+ name: benchmark-request-${{ github.event.workflow_run.run_attempt }}
+ path: scripts/benchmarks/request
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ github.token }}
+ - name: Download results
+ if: steps.request.outcome == 'success'
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0
+ with:
+ name: benchmark-results-${{ github.event.workflow_run.run_attempt }}
+ path: scripts/benchmarks/results
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ github.token }}
+ - name: Install benchmark tooling
+ if: steps.request.outcome == 'success'
+ run: |
+ python3 -m venv .benchmark-bootstrap
+ .benchmark-bootstrap/bin/pip install uv==0.12.9
+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks
+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"
+ - name: Publish the current result
+ if: steps.request.outcome == 'success'
+ working-directory: scripts/benchmarks
+ run: uv run --locked cli.py finalize
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
new file mode 100644
index 0000000000..3bc5e5a15a
--- /dev/null
+++ b/.github/workflows/benchmarks.yml
@@ -0,0 +1,121 @@
+name: Prime Agent benchmarks
+run-name: 'Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}'
+
+on:
+ pull_request_target:
+ branches: [main]
+ types: [opened, synchronize, reopened, ready_for_review]
+ workflow_dispatch:
+ inputs:
+ pr:
+ description: Open PR number to benchmark against current main
+ required: true
+ type: number
+
+concurrency:
+ group: prime-agent-benchmark-${{ github.event.pull_request.number || inputs.pr }}
+ cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+ request:
+ if: github.event_name == 'pull_request_target' || github.ref == 'refs/heads/main'
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ concurrency:
+ group: 'prime-agent-benchmark-comment-Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}'
+ cancel-in-progress: false
+ permissions:
+ contents: read
+ actions: read
+ pull-requests: write
+ outputs:
+ allowed: ${{ steps.resolve.outputs.needed == 'true' && steps.vouch.outputs.vouched == 'true' }}
+ harness: ${{ steps.resolve.outputs.harness }}
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+ - name: Install benchmark tooling
+ run: |
+ python3 -m venv .benchmark-bootstrap
+ .benchmark-bootstrap/bin/pip install uv==0.12.9
+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks
+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"
+ - name: Resolve exact revisions
+ id: resolve
+ working-directory: scripts/benchmarks
+ run: uv run --locked cli.py resolve --results request
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ - name: Check contributor trust
+ id: vouch
+ if: steps.resolve.outputs.needed == 'true'
+ uses: mitchellh/vouch/action/check-user@d66fa29a64600490892131ad87597c30c91fcac4 # v1
+ with:
+ user: ${{ steps.resolve.outputs.author }}
+ allow-fail: true
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ - name: Update benchmark comment
+ if: steps.resolve.outputs.needed == 'true'
+ working-directory: scripts/benchmarks
+ run: |
+ if [ "$VOUCHED" = true ]; then
+ uv run --locked cli.py publish --results request
+ else
+ uv run --locked cli.py publish --results request --pending-trust
+ fi
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ VOUCHED: ${{ steps.vouch.outputs.vouched }}
+ - name: Save benchmark request
+ if: steps.resolve.outputs.needed == 'true'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0
+ with:
+ name: benchmark-request-${{ github.run_attempt }}
+ path: scripts/benchmarks/request/report.json
+ retention-days: 14
+
+ benchmark:
+ needs: request
+ if: needs.request.outputs.allowed == 'true'
+ runs-on: ubuntu-24.04
+ timeout-minutes: 25
+ permissions:
+ contents: read
+ actions: read
+ pull-requests: read
+ steps:
+ # The controller is trusted main code. PR code executes only in Prime sandboxes.
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ ref: ${{ needs.request.outputs.harness }}
+ persist-credentials: false
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0
+ with:
+ name: benchmark-request-${{ github.run_attempt }}
+ path: scripts/benchmarks/request
+ - name: Install benchmark tooling
+ run: |
+ python3 -m venv .benchmark-bootstrap
+ .benchmark-bootstrap/bin/pip install uv==0.12.9
+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks
+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"
+ - name: Benchmark main and PR in Prime sandboxes
+ working-directory: scripts/benchmarks
+ run: uv run --locked cli.py run
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}
+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}
+ - name: Save benchmark results
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0
+ with:
+ name: benchmark-results-${{ github.run_attempt }}
+ path: scripts/benchmarks/results/
+ if-no-files-found: warn
+ retention-days: 14
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index af521a547c..2fca23f717 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -143,6 +143,14 @@ jobs:
working-directory: ${{ matrix.package }}
run: ${{ matrix.command }}
+ - name: Test benchmark harness
+ if: matrix.name == 'runtime python'
+ working-directory: scripts/benchmarks
+ run: |
+ uv run --locked ruff check .
+ uv run --locked ruff format --check .
+ uv run --locked python -m unittest discover -s tests -v
+
build-check-test:
name: build-check-test
if: always() && needs.trust.outputs.allowed == 'true'
diff --git a/scripts/benchmarks/.gitignore b/scripts/benchmarks/.gitignore
new file mode 100644
index 0000000000..b5c701661a
--- /dev/null
+++ b/scripts/benchmarks/.gitignore
@@ -0,0 +1,3 @@
+.venv/
+results/
+.ruff_cache/
diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md
new file mode 100644
index 0000000000..b1deb3863a
--- /dev/null
+++ b/scripts/benchmarks/README.md
@@ -0,0 +1,153 @@
+# PR performance benchmarks
+
+Each push to an open, vouched PR starts an informational Prime Agent benchmark. Multiple commits in
+one push produce one run for the final head. Draft PRs are included. A two-second debounce and
+per-PR cancellation avoid finishing obsolete runs. `workflow_dispatch` reruns an open PR by number.
+Identical automatic requests reuse the completed comment when both SHAs, harness, and configuration
+match. Manual dispatch and GitHub reruns force fresh measurements. Unvouched authors
+receive a pending-trust comment; a maintainer can rerun after vouching.
+
+The controller resolves current `main` and the PR head to full SHAs, builds both in separate Prime
+sandboxes, and alternates their measurements. Both use the same trusted harness revision, image
+digest and resource allocation. No performance gate blocks merging.
+
+## Enable in GitHub
+
+Add the repository secret `PRIME_SANDBOX_API_KEY`: a dedicated key with Sandbox permissions for
+provisioning and cleanup. Optional `PRIME_BENCHMARK_TEAM_ID` selects the sandbox billing workspace.
+No inference key, model configuration, or login is required.
+
+The workflows must first land on `main`: both `pull_request_target` and the completion listener run
+trusted default-branch code. Trigger `Prime Agent benchmarks` manually with an open PR number after
+configuring the sandbox secret. The first rollout should include a main-versus-main calibration.
+
+No credentials are injected into the sandboxes. The sandbox control key and GitHub token stay on the
+trusted controller; a separate publisher owns GitHub comment write permission. No PR checkout,
+build script, installer, executable, or archive is executed/extracted on a privileged GitHub runner.
+The existing Vouch gate controls who can trigger compute usage.
+
+## Measurements
+
+The default configuration uses two Linux x64 containers, each with 4 vCPU, 8 GB RAM, and 20 GB disk.
+`config.json` pins the image and sampling policy. Harness dependencies are locked with a seven-day
+release cutoff. Provisioning, harness setup, and source compilation have separate recorded durations
+outside the timed installation interval. Interactive runs use the same small committed Git fixture.
+
+The base image supplies Node, npm, Git, curl, and Python. Untimed sandbox setup adds Python venv
+support, Cairo/Pango/JPEG/GIF/SVG development libraries, ripgrep, and fd. These prerequisites are
+shared within each sandbox; the normal installer still prepares each user's stock Python environment
+and other missing tools. Their setup time and disk usage are outside the installation metrics.
+
+- **Cold startup:** process launch until the editor visibly echoes a typed marker, after stopping all
+ processes owned by the benchmark user. OS filesystem caches are not flushed.
+- **Warm startup:** the same input-ready measurement while retaining the daemon and stopping its
+ previous active sessions. Each TUI launch opens a fresh conversation.
+- **Installation:** the normal installer and Python/tool bootstrap in three new user homes, each
+ with empty npm and uv caches. Unpublished candidate release tarballs are served over loopback;
+ npm/Python dependencies use the real network. This does not measure public release-CDN latency.
+- **Compressed artifacts:** total bytes of the four tarballs produced by the release packer, using
+ an identical synthetic version and download origin for both sides. External registry dependencies
+ are not included in these tarballs.
+- **Installed footprint:** apparent bytes added after first use in the first fresh home, including stock Python,
+ runtime, and tool assets; excluding download caches, session history, and logs. Shared system
+ dependencies supplied by the base image and the fixture repository are excluded.
+- **Idle memory:** summed RSS across the benchmark user's entire process tree after input readiness
+ and a one-second settle. Raw results include each process and PSS when Linux permits reading it.
+ The controller, PTY harness, artifact server, and build user are excluded. RSS can double-count
+ shared pages.
+
+Startup and memory use 10 trials per revision. Installation uses three; sizes are measured once.
+Stock tools, skills, daemon, persistence, and Python bootstrap remain enabled. Homes contain no
+credentials, extensions, MCP servers, or personal skills. The onboarding splash is marked as already
+shown before timing, so startup measures the editor rather than waiting for a person to sign in.
+The editor runs without a selected model; no prompts are submitted. These measurements cover the
+local startup path, not authenticated provider discovery or inference.
+
+### Python runtime
+
+A separate probe drives the **installed** `python -m rlm.repl` over its JSONL protocol. It measures the
+CPython kernel and stock `rlm.bash` implementation, including IPC, serialization, and output capture.
+It does not include the TypeScript kernel manager, TUI rendering, or model execution. The harness
+fails visibly if either revision does not support the required runtime protocol; it never substitutes
+a plain Python process or omits a failed metric.
+
+Each of the 10 trials starts with all benchmark-user processes stopped and a new kernel:
+
+- **Kernel startup:** process spawn through the protocol-ready handshake. Includes the small
+ `runuser` launch overhead; excludes Python installation and host-side agent bootstrap.
+- **Python cell round trip:** mean of 50 sequential `pass` cells through the complete request/done
+ protocol. Five Python cells and one empty bash command warm the runtime before execution timings.
+- **Empty bash command:** mean of five `await bash(':')` calls with successful, empty results.
+- **Bash git status:** mean of five `git status --porcelain` calls in the fixed, clean Git fixture.
+- **Bash 32 KiB output:** mean of five shell commands producing exactly 32,768 bytes. The runtime
+ captures and forwards the entire output, and the probe verifies its byte count.
+- **35 cells / 9 shell calls:** total time for a fixed mix of 26 Python no-ops and nine git-status
+ calls, alternating one shell call with three Python cells.
+- **Interrupt to done:** after an executing cell signals readiness, send an interrupt and wait for
+ its KeyboardInterrupt/done acknowledgement. A subsequent cell must still execute successfully.
+- **State snapshot:** serialize a 10,000-row, eight-column integer pandas DataFrame and a
+ 10,000-integer list to disk. Includes first-use serialization imports.
+- **State restore:** restore that state into a fresh kernel, including first-use pandas imports but
+ excluding kernel startup. Verify the recovered dimensions, sum, and list values after timing.
+- **Kernel idle RSS:** memory immediately after the ready handshake, before workload imports.
+- **Kernel RSS after pandas workload:** memory after the mixed calls and construction of the same
+ DataFrame/list used for snapshotting. Both memory measurements include only the isolated user's
+ Python process; raw per-process measurements are retained.
+
+The runtime rows show medians across the 10 independent trials. Batch iteration counts and fixture
+sizes are fixed in `kernel.py`; failures and incomplete output remain failed samples. These are
+subsystem measurements alongside the normal installed CLI benchmarks, not end-to-end tool latency.
+
+The comment shows medians, signed absolute/percentage deltas, successful/attempted counts, and spread.
+`↓` means improvement, bold `↑` means regression, `≈` means no clear change, and `—` means unavailable
+or incomplete. Arrows require a change larger than the metric's absolute floor, relative floor,
+and observed spread. The initial relative floor is 20% for timings and memory, 0.5% for artifact
+size, and 1% for disk footprint. A same-revision calibration on separate sandboxes showed roughly
+7–18% variation across several timings; the conservative floor avoids labeling that as a code regression.
+Smaller signed differences remain visible. Revisit these floors after collecting more control runs. This is a practical noise filter, not a statistical significance test. Inspect
+raw trials before acting on small changes; sandbox scheduling and filesystem caches still introduce noise.
+
+## Lifecycle and costs
+
+The main workflow posts a single marked comment and updates it in place. The completion workflow
+publishes the final result or a cancellation/failure notice. Publishers serialize by PR and check the
+current head, latest run ID, attempt, and existing comment generation before writing. A rerun of an
+older commit cannot overwrite a newer result, including when the newer run has the same head SHA.
+
+Results and logs are retained as GitHub artifacts for 14 days. They include exact source and harness
+SHAs, environment details, installed dependency inventory, transcripts, raw observations, failures,
+and estimated sandbox costs. The publisher validates schema, identity, finite numbers, and report
+size and escapes sandbox-provided text.
+
+The controller stops scheduling work at 20 minutes or its $1 estimated budget target. Every sandbox
+has a 30-minute TTL. Teardown runs in `finally`; the independent completion workflow deletes any
+remaining sandboxes with the exact repository/run/attempt labels, including after cancellation.
+Cleanup enumerates all pages before deleting so pagination cannot skip a sandbox.
+
+At the configured list rates, two sandboxes cost about $0.01/minute together. A 10-minute run costs
+about $0.10 in sandbox compute. There are no inference charges. Full sandbox lifetimes are recorded,
+including provisioning, setup, building, measurements, and cleanup. The $1 target is an estimate,
+not a hard billing cap; scheduling and deletion latency can increase the final bill.
+
+## Local development
+
+The existing runtime Python CI job runs the harness checks. To run them locally from this directory:
+
+```sh
+uv sync --locked
+uv run --locked ruff check .
+uv run --locked ruff format --check .
+uv run --locked python -m unittest discover -s tests -v
+```
+
+Tests use fake GitHub/SDK responses and terminal streams; they never invoke inference or provision
+sandboxes. A live run is separate and requires only the `PRIME_SANDBOX_API_KEY` environment variable:
+
+```sh
+uv run --locked cli.py local --base FULL_MAIN_SHA --head FULL_HEAD_SHA --results results/local
+```
+
+Use identical SHAs to calibrate main against itself. `--config /path/to/config.json` can reduce trials
+for a smoke run; the report records the effective configuration. Local mode never posts GitHub
+comments. `results/`, `.venv/`, and `.ruff_cache/` are ignored. Run the repository's `npm run check`
+after code changes as well.
diff --git a/scripts/benchmarks/cli.py b/scripts/benchmarks/cli.py
new file mode 100644
index 0000000000..1c63836ccd
--- /dev/null
+++ b/scripts/benchmarks/cli.py
@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+from datetime import UTC, datetime
+from pathlib import Path
+
+from prime_sandboxes import APIClient, SandboxClient
+
+from controller import Controller, cleanup
+from github import WORKFLOW, GitHub
+from report import render
+from schema import Config, Report, Side, load_report, write_json
+
+
+def workflow_source(event: dict) -> dict:
+ run = event["workflow_run"]
+ if (
+ run["event"] not in ("pull_request_target", "workflow_dispatch")
+ or run["path"].split("@")[0] != f".github/workflows/{WORKFLOW}"
+ or run["repository"]["full_name"] != os.environ["GITHUB_REPOSITORY"]
+ or (run["event"] == "workflow_dispatch" and run["head_branch"] != "main")
+ ):
+ raise ValueError("Unexpected source workflow")
+ return run
+
+
+def validate_completion(report: Report, request: Report, run: dict) -> None:
+ identity = {
+ "repository",
+ "head_repository",
+ "pr",
+ "run_id",
+ "attempt",
+ "harness_sha",
+ "base_sha",
+ "head_sha",
+ "started_at",
+ "config",
+ }
+ if report.model_dump(include=identity) != request.model_dump(include=identity):
+ raise ValueError("Result identity does not match the trusted benchmark request")
+ if (request.run_id, request.attempt) != (run["id"], run["run_attempt"]) or run[
+ "display_title"
+ ] != f"Prime Agent benchmarks · PR #{request.pr}":
+ raise ValueError("Benchmark request does not match the completed workflow")
+
+
+def completed_report(result_path: Path, request_path: Path, run: dict) -> Report:
+ request = load_report(request_path)
+ validate_completion(request, request, run)
+ try:
+ report = load_report(result_path) if result_path.exists() else request
+ validate_completion(report, request, run)
+ except (ValueError, OSError):
+ report = request
+ report.status = "failed"
+ report.errors.append("Benchmark report failed validation; see the workflow artifacts")
+ if report.status == "running":
+ report.status = "canceled" if run["conclusion"] == "cancelled" else "failed"
+ report.errors.append("Benchmark did not produce a final report; see the workflow logs")
+ if report.status != "pending-trust" and report.finished_at is None:
+ report.finished_at = datetime.now(UTC)
+ return report
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("command", choices=("resolve", "run", "publish", "finalize", "cleanup", "local"))
+ parser.add_argument("--results", type=Path, default=Path("results"))
+ parser.add_argument("--request", type=Path, default=Path("request/report.json"))
+ parser.add_argument("--base")
+ parser.add_argument("--head")
+ parser.add_argument("--config", type=Path)
+ parser.add_argument("--pending-trust", action="store_true")
+ args = parser.parse_args()
+ args.results.mkdir(parents=True, exist_ok=True)
+ repository = os.environ.get("GITHUB_REPOSITORY", "PrimeIntellect-ai/prime-agent")
+ config = Config.load(args.config) if args.config else Config.load()
+ if args.command == "local":
+ if not args.base or not args.head:
+ parser.error("local requires exact --base and --head commit SHAs")
+ report = Report(
+ repository=repository,
+ head_repository=repository,
+ pr=0,
+ run_id=int(datetime.now(UTC).timestamp()),
+ attempt=1,
+ harness_sha=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
+ base_sha=args.base,
+ head_sha=args.head,
+ started_at=datetime.now(UTC),
+ config=config,
+ main=Side(sha=args.base),
+ pr_head=Side(sha=args.head),
+ )
+ Controller(report, args.results, live_github=False).run()
+ return
+ event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
+ if args.command == "cleanup":
+ run = workflow_source(event)
+ client = SandboxClient(APIClient(api_key=os.environ["PRIME_SANDBOX_API_KEY"]))
+ deleted = cleanup(client, repository, run["id"], run["run_attempt"])
+ print(f"Cleaned up {len(deleted)} remaining benchmark sandboxes")
+ return
+ github = GitHub(repository)
+ if args.command == "resolve":
+ pr = int(event.get("pull_request", {}).get("number") or event["inputs"]["pr"])
+ if pr <= 0:
+ raise ValueError("Invalid PR number")
+ report, author = github.resolve(
+ pr,
+ os.environ["GITHUB_SHA"],
+ int(os.environ["GITHUB_RUN_ID"]),
+ int(os.environ["GITHUB_RUN_ATTEMPT"]),
+ config,
+ )
+ write_json(args.results / "report.json", report)
+ manual = os.environ.get("GITHUB_EVENT_NAME") == "workflow_dispatch" or report.attempt > 1
+ needed = manual or not github.duplicate(report)
+ with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output:
+ output.write(
+ f"pr={pr}\nauthor={author}\nharness={report.harness_sha}\nneeded={str(needed).lower()}\n"
+ )
+ return
+ report_path = args.results / "report.json"
+ report = (
+ completed_report(report_path, args.request, workflow_source(event))
+ if args.command == "finalize"
+ else load_report(report_path if report_path.exists() else args.request)
+ )
+ if report.repository != repository:
+ raise ValueError("Result repository does not match the workflow")
+ if args.command == "finalize":
+ github.publish(report)
+ else:
+ if (report.run_id, report.attempt) != (
+ int(os.environ["GITHUB_RUN_ID"]),
+ int(os.environ["GITHUB_RUN_ATTEMPT"]),
+ ):
+ raise ValueError("Result identity does not match the active workflow")
+ if args.command == "run":
+ if not os.environ.get("PRIME_SANDBOX_API_KEY"):
+ report.status = "failed"
+ report.errors.append("Repository secret PRIME_SANDBOX_API_KEY is not configured")
+ if report.status == "running":
+ controller = Controller(report, args.results)
+ controller.run()
+ report = controller.report
+ else:
+ write_json(report_path, report)
+ else:
+ if args.pending_trust:
+ report.status = "pending-trust"
+ write_json(report_path, report)
+ github.publish(report)
+ summary = os.environ.get("GITHUB_STEP_SUMMARY")
+ if summary:
+ Path(summary).write_text(render(report))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/benchmarks/config.json b/scripts/benchmarks/config.json
new file mode 100644
index 0000000000..f0a6962115
--- /dev/null
+++ b/scripts/benchmarks/config.json
@@ -0,0 +1,16 @@
+{
+ "image": "node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2",
+ "region": "us",
+ "cpu_cores": 4,
+ "memory_gb": 8,
+ "disk_gb": 20,
+ "trials": 10,
+ "install_trials": 3,
+ "debounce_seconds": 2,
+ "timeout_seconds": 1200,
+ "ttl_minutes": 30,
+ "budget_usd": 1,
+ "cpu_usd_per_hour": 0.05,
+ "memory_usd_per_gb_hour": 0.01,
+ "disk_usd_per_gb_hour": 0.001
+}
diff --git a/scripts/benchmarks/controller.py b/scripts/benchmarks/controller.py
new file mode 100644
index 0000000000..258af25850
--- /dev/null
+++ b/scripts/benchmarks/controller.py
@@ -0,0 +1,370 @@
+from __future__ import annotations
+
+import os
+import shlex
+import signal
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+from prime_sandboxes import APIClient, CreateSandboxRequest, SandboxClient
+from prime_sandboxes.models import BackgroundJob, Sandbox
+
+from github import GitHub
+from report import render
+from schema import (
+ ROOT,
+ UV_VERSION,
+ Report,
+ Request,
+ Result,
+ SandboxUsage,
+ Side,
+ write_json,
+)
+
+REMOTE = "/opt/prime-benchmark"
+OWNER_LABEL = "prime-agent-benchmarks-v1"
+FILES = ("schema.py", "terminal.py", "kernel.py", "worker.py", "pyproject.toml", "uv.lock")
+
+
+def elapsed_seconds(start: datetime) -> float:
+ return max(0, (datetime.now(UTC) - start.replace(tzinfo=start.tzinfo or UTC)).total_seconds())
+
+
+class Canceled(Exception):
+ pass
+
+
+def labels(repository: str, run_id: int, attempt: int) -> list[str]:
+ return [OWNER_LABEL, f"repository:{repository}", f"run:{run_id}", f"attempt:{attempt}"]
+
+
+def cleanup(client: SandboxClient, repository: str, run_id: int, attempt: int) -> list[str]:
+ expected = labels(repository, run_id, attempt)
+ owned = []
+ page = 1
+ while True:
+ response = client.list(labels=expected, page=page, per_page=100, exclude_terminated=True)
+ owned.extend(sandbox.id for sandbox in response.sandboxes if set(expected) <= set(sandbox.labels))
+ if not response.has_next:
+ break
+ page += 1
+ failed = []
+ for sandbox_id in owned:
+ try:
+ client.delete(sandbox_id)
+ except Exception:
+ failed.append(sandbox_id)
+ if failed:
+ raise RuntimeError(f"Cleanup failed for {len(failed)} sandbox(es); TTL remains active")
+ return owned
+
+
+class Controller:
+ def __init__(self, report: Report, results: Path, *, live_github: bool = True):
+ self.report = report
+ self.results = results
+ self.results.mkdir(parents=True, exist_ok=True)
+ self.harness = self.results / "harness"
+ self.harness.mkdir(exist_ok=True)
+ for name in FILES:
+ (self.harness / name).write_bytes((ROOT / name).read_bytes())
+ self.client = SandboxClient(APIClient(api_key=os.environ["PRIME_SANDBOX_API_KEY"]))
+ self.github = GitHub(report.repository) if live_github else None
+ self.sandboxes: dict[str, Sandbox] = {}
+ self.requests: dict[str, Request] = {}
+ self.started = time.monotonic()
+ self.deadline = self.started + report.config.timeout_seconds
+ self.last_fresh_check = 0.0
+
+ def checkpoint(self) -> None:
+ now = time.monotonic()
+ if now > self.deadline:
+ raise TimeoutError("Benchmark exceeded its runtime limit")
+ if self.github and now - self.last_fresh_check > 60:
+ self.last_fresh_check = now
+ if not self.github.fresh(self.report):
+ raise Canceled("A newer PR head or benchmark run superseded this run")
+
+ def save(self) -> None:
+ self.report.errors = self.report.errors[:30]
+ self.report = Report.model_validate(self.report.model_dump())
+ write_json(self.results / "report.json", self.report)
+ (self.results / "comment.md").write_text(render(self.report))
+
+ def command(self, sandbox: Sandbox, args: list[str], timeout: int = 60) -> None:
+ self.checkpoint()
+ result = self.client.execute_command(sandbox.id, shlex.join(args), timeout=timeout)
+ if result.exit_code:
+ raise RuntimeError(f"Sandbox setup command failed with exit code {result.exit_code}")
+
+ def start(self, role: str) -> Sandbox:
+ started = time.monotonic()
+ side = self.report.main if role == "main" else self.report.pr_head
+ config = self.report.config
+ sha = self.report.base_sha if role == "main" else self.report.head_sha
+ request = Request(
+ repository=self.report.repository,
+ source_repository=self.report.repository if role == "main" else self.report.head_repository,
+ sha=sha,
+ harness_sha=self.report.harness_sha,
+ pr=self.report.pr,
+ run_id=self.report.run_id,
+ attempt=self.report.attempt,
+ role=role,
+ config=config,
+ )
+ self.requests[role] = request
+ self.checkpoint()
+ sandbox = self.client.create(
+ CreateSandboxRequest(
+ name=f"agent-bench-{self.report.run_id}-{self.report.attempt}-{role}",
+ docker_image=config.image,
+ cpu_cores=config.cpu_cores,
+ memory_gb=config.memory_gb,
+ disk_size_gb=config.disk_gb,
+ vm=False,
+ region=config.region,
+ timeout_minutes=config.ttl_minutes,
+ labels=labels(self.report.repository, self.report.run_id, self.report.attempt)
+ + [f"role:{role}"],
+ idempotency_key=f"agent-bench-{self.report.repository}-{self.report.run_id}-{self.report.attempt}-{role}",
+ )
+ )
+ self.sandboxes[role] = sandbox
+ print(f"Created {role} sandbox {sandbox.id}", flush=True)
+ while self.client.get(sandbox.id).status != "RUNNING":
+ self.checkpoint()
+ state = self.client.get(sandbox.id).status
+ if state in ("ERROR", "FAILED", "TERMINATED", "STOPPED", "TIMEOUT"):
+ raise RuntimeError(f"{role} sandbox failed during provisioning")
+ time.sleep(2)
+ side.timings["provision"] = time.monotonic() - started
+ started = time.monotonic()
+ self.command(sandbox, ["mkdir", "-p", REMOTE])
+ for name in FILES:
+ self.client.upload_file(sandbox.id, f"{REMOTE}/{name}", str(self.harness / name))
+ request_path = self.results / f"request-{role}.json"
+ write_json(request_path, request)
+ self.client.upload_file(sandbox.id, f"{REMOTE}/request.json", str(request_path))
+ setup = (
+ "set -eu\n"
+ "apt-get update -qq\n"
+ "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-venv "
+ "libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep\n"
+ "ln -sf /usr/bin/fdfind /usr/local/bin/fd\n"
+ "useradd --create-home --uid 1500 --shell /bin/bash builder\n"
+ "python3 -m venv /opt/benchmark-bootstrap\n"
+ f"/opt/benchmark-bootstrap/bin/pip -q install uv=={UV_VERSION}\n"
+ f"/opt/benchmark-bootstrap/bin/uv sync --locked --no-dev --project {REMOTE}\n"
+ )
+ command = f"{shlex.join(['sh', '-c', setup])} > {REMOTE}/setup.log 2>&1"
+ job = self.client.start_background_job(sandbox.id, command)
+ self.wait(sandbox, job, "setup")
+ side.timings["setup"] = time.monotonic() - started
+ return sandbox
+
+ def wait(self, sandbox: Sandbox, job: BackgroundJob, label: str) -> None:
+ while True:
+ self.checkpoint()
+ status = self.client.get_background_job(sandbox.id, job, timeout=30)
+ if status.completed:
+ if status.exit_code:
+ raise RuntimeError(f"{label} failed with exit code {status.exit_code}")
+ return
+ time.sleep(2)
+
+ def phase(self, role: str, phase: str, trial: int = 0) -> None:
+ started = time.monotonic()
+ sandbox = self.sandboxes[role]
+ args = [f"{REMOTE}/.venv/bin/python", f"{REMOTE}/worker.py", phase, "--trial", str(trial)]
+ command = f"{shlex.join(args)} > {REMOTE}/{phase}.log 2>&1"
+ job = self.client.start_background_job(sandbox.id, command)
+ try:
+ self.wait(sandbox, job, f"{role} {phase} {trial}")
+ except (Canceled, TimeoutError):
+ raise
+ except Exception:
+ try:
+ self.collect_result(role)
+ except Exception:
+ pass
+ raise
+ else:
+ self.collect_result(role)
+ finally:
+ if phase == "prepare":
+ side = self.report.main if role == "main" else self.report.pr_head
+ side.timings["build"] = time.monotonic() - started
+ self.save()
+
+ def collect_result(self, role: str) -> None:
+ path = self.results / f"result-{role}.json"
+ self.client.download_file(self.sandboxes[role].id, f"{REMOTE}/results/result.json", str(path))
+ if path.stat().st_size > 2_000_000:
+ raise ValueError("Sandbox result exceeds size limit")
+ result = Result.model_validate_json(path.read_text())
+ if result.request != self.requests[role] or result.side.sha != self.requests[role].sha:
+ raise ValueError("Sandbox result does not match this run's request")
+ previous = self.report.main if role == "main" else self.report.pr_head
+ result.side.timings = previous.timings.copy()
+ if role == "main":
+ self.report.main = result.side
+ else:
+ self.report.pr_head = result.side
+ self.save()
+
+ def logs(self, role: str) -> None:
+ sandbox = self.sandboxes[role]
+ # Never unpack PR-produced archives on the GitHub runner.
+ args = [
+ "python3",
+ "-c",
+ "import pathlib,tarfile; "
+ f"root=pathlib.Path({REMOTE!r}); "
+ "paths=[p for p in root.glob('*.log') if p.is_file() and p.stat().st_size < 8000000]; "
+ "paths += [p for p in (root/'results').glob('*') "
+ "if p.is_file() and p.stat().st_size < 8000000]; "
+ "archive=tarfile.open(root/'logs.tar.gz','w:gz'); "
+ "[archive.add(p,arcname=str(p.relative_to(root)),recursive=False) for p in paths[:1000]]; "
+ "archive.close()",
+ ]
+ result = self.client.execute_command(sandbox.id, shlex.join(args), timeout=30)
+ if result.exit_code == 0:
+ self.client.download_file(
+ sandbox.id, f"{REMOTE}/logs.tar.gz", str(self.results / f"{role}-logs.tar.gz")
+ )
+
+ def run(self) -> None:
+ old_handler = signal.signal(signal.SIGTERM, cancel)
+ try:
+ self.save()
+ while time.monotonic() - self.started < self.report.config.debounce_seconds:
+ self.checkpoint()
+ time.sleep(1)
+ ready = []
+ for role in ("main", "pr"):
+ try:
+ self.start(role)
+ self.phase(role, "prepare")
+ server = shlex.join(
+ [
+ "runuser",
+ "-u",
+ "builder",
+ "--",
+ "python3",
+ "-m",
+ "http.server",
+ "18741",
+ "--bind",
+ "127.0.0.1",
+ "--directory",
+ f"{REMOTE}/www",
+ ]
+ )
+ self.client.start_background_job(self.sandboxes[role].id, server)
+ ready.append(role)
+ except (Canceled, TimeoutError):
+ raise
+ except Exception as error:
+ self.report.errors.append(f"{role} setup: {type(error).__name__}")
+ for phase, count in (
+ ("install", self.report.config.install_trials),
+ ("measure", self.report.config.trials),
+ ("runtime", self.report.config.trials),
+ ):
+ for trial in range(count):
+ for role in ready if trial % 2 == 0 else list(reversed(ready)):
+ print(f"{role}: {phase} trial {trial + 1}/{count}", flush=True)
+ try:
+ self.phase(role, phase, trial)
+ except (Canceled, TimeoutError):
+ raise
+ except Exception as error:
+ if len(self.report.errors) < 25:
+ self.report.errors.append(f"{role} {phase} {trial}: {type(error).__name__}")
+ compute = sum(
+ elapsed_seconds(sandbox.created_at) * self.report.config.hourly_cost() / 3600
+ for sandbox in self.sandboxes.values()
+ )
+ if compute >= self.report.config.budget_usd:
+ raise TimeoutError("Reached the estimated run budget")
+ complete = all(
+ side_complete(side, self.report.config.trials, self.report.config.install_trials)
+ for side in (self.report.main, self.report.pr_head)
+ )
+ self.report.status = "completed" if complete and not self.report.errors else "partial"
+ if not ready:
+ self.report.status = "failed"
+ except Canceled:
+ self.report.status = "canceled"
+ except Exception as error:
+ self.report.status = "failed"
+ message = str(error)
+ for key in ("PRIME_SANDBOX_API_KEY", "GITHUB_TOKEN"):
+ if os.environ.get(key):
+ message = message.replace(os.environ[key], "[REDACTED]")
+ self.report.errors.append(message[:500])
+ finally:
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ for role, sandbox in self.sandboxes.items():
+ deleted = False
+ try:
+ self.logs(role)
+ except Exception:
+ self.report.errors.append(f"{role} logs could not be collected")
+ try:
+ self.client.delete(sandbox.id)
+ deleted = True
+ except Exception:
+ self.report.errors.append(
+ f"{role} sandbox cleanup deferred to the cleanup workflow or TTL"
+ )
+ seconds = elapsed_seconds(sandbox.created_at)
+ if not deleted:
+ seconds = max(seconds, self.report.config.ttl_minutes * 60)
+ self.report.sandboxes.append(
+ SandboxUsage(
+ id=sandbox.id,
+ role=role,
+ seconds=seconds,
+ deleted=deleted,
+ estimated_usd=seconds * self.report.config.hourly_cost() / 3600,
+ )
+ )
+ self.report.finished_at = datetime.now(UTC)
+ self.save()
+ signal.signal(signal.SIGTERM, old_handler)
+
+
+def cancel(_signum: int, _frame: object) -> None:
+ raise Canceled("Workflow was canceled")
+
+
+def side_complete(side: Side, trials: int, installs: int) -> bool:
+ expected = {
+ "cold": trials,
+ "warm": trials,
+ "kernel_start": trials,
+ "kernel_exec": trials,
+ "bash": trials,
+ "git_status": trials,
+ "output": trials,
+ "mixed": trials,
+ "interrupt": trials,
+ "snapshot": trials,
+ "restore": trials,
+ "kernel_rss": trials,
+ "loaded_rss": trials,
+ "rss": trials,
+ "install": installs,
+ "disk": 1,
+ "bundle": 1,
+ }
+ return not side.error and all(
+ len(side.metrics.get(metric, [])) == count
+ and all(sample.value is not None for sample in side.metrics[metric])
+ for metric, count in expected.items()
+ )
diff --git a/scripts/benchmarks/github.py b/scripts/benchmarks/github.py
new file mode 100644
index 0000000000..3e9ff6baa0
--- /dev/null
+++ b/scripts/benchmarks/github.py
@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import os
+import re
+from datetime import UTC, datetime
+
+import httpx
+
+from report import MARKER, fingerprint, render
+from schema import Config, Report, Side
+
+WORKFLOW = "benchmarks.yml"
+TITLE = "Prime Agent benchmarks · PR #"
+GENERATION = re.compile(r"<!-- run:(\d+):(\d+) head:([a-f0-9]{40}) -->")
+
+
+class GitHub:
+ def __init__(self, repository: str):
+ self.repository = repository
+ self.client = httpx.Client(
+ base_url=f"https://api.github.com/repos/{repository}/",
+ headers={
+ "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}",
+ "Accept": "application/vnd.github+json",
+ "X-GitHub-Api-Version": "2022-11-28",
+ },
+ timeout=30,
+ )
+
+ def request(self, method: str, path: str, body: dict | None = None) -> dict | list:
+ response = self.client.request(method, path, json=body)
+ response.raise_for_status()
+ return response.json()
+
+ def pages(self, path: str, key: str | None = None):
+ separator = "&" if "?" in path else "?"
+ page = 1
+ while True:
+ data = self.request("GET", f"{path}{separator}per_page=100&page={page}")
+ items = data[key] if key else data
+ yield items
+ if len(items) < 100:
+ return
+ page += 1
+
+ def resolve(
+ self, pr: int, harness_sha: str, run_id: int, attempt: int, config: Config
+ ) -> tuple[Report, str]:
+ pull = self.request("GET", f"pulls/{pr}")
+ if pull["state"] != "open" or pull["base"]["ref"] != "main":
+ raise ValueError("Benchmark requires an open PR targeting main")
+ if pull["base"]["repo"]["full_name"] != self.repository:
+ raise ValueError("PR belongs to another repository")
+ base = self.request("GET", "git/ref/heads/main")["object"]["sha"]
+ report = Report(
+ repository=self.repository,
+ head_repository=pull["head"]["repo"]["full_name"],
+ pr=pr,
+ run_id=run_id,
+ attempt=attempt,
+ harness_sha=harness_sha,
+ base_sha=base,
+ head_sha=pull["head"]["sha"],
+ started_at=datetime.now(UTC),
+ config=config,
+ main=Side(sha=base),
+ pr_head=Side(sha=pull["head"]["sha"]),
+ )
+ return report, pull["user"]["login"]
+
+ def fresh(self, report: Report) -> bool:
+ pull = self.request("GET", f"pulls/{report.pr}")
+ if pull["state"] != "open" or pull["head"]["sha"] != report.head_sha:
+ return False
+ run = self.request("GET", f"actions/runs/{report.run_id}")
+ if run["run_attempt"] != report.attempt:
+ return False
+ for runs in self.pages(f"actions/workflows/{WORKFLOW}/runs", "workflow_runs"):
+ if any(
+ candidate["display_title"] == f"{TITLE}{report.pr}"
+ and (candidate["run_number"], candidate["run_attempt"]) > (run["run_number"], report.attempt)
+ for candidate in runs
+ ):
+ return False
+ if not runs or min(candidate["run_number"] for candidate in runs) < run["run_number"]:
+ break
+ return True
+
+ def comment(self, pr: int) -> dict | None:
+ comments = [
+ comment
+ for page in self.pages(f"issues/{pr}/comments")
+ for comment in page
+ if comment["user"]["login"] == "github-actions[bot]"
+ and comment.get("body", "").startswith(MARKER)
+ ]
+ return comments[-1] if comments else None
+
+ def duplicate(self, report: Report) -> bool:
+ comment = self.comment(report.pr)
+ return bool(
+ comment and f"<!-- comparison:{fingerprint(report)} status:completed -->" in comment["body"]
+ )
+
+ def publish(self, report: Report) -> bool:
+ if not self.fresh(report):
+ return False
+ comment = self.comment(report.pr)
+ if comment:
+ generation = GENERATION.search(comment["body"])
+ if generation and tuple(map(int, generation.groups()[:2])) > (report.run_id, report.attempt):
+ return False
+ body = render(report)
+ if comment and comment["body"] == body:
+ return False
+ if not self.fresh(report):
+ return False
+ if comment:
+ self.request("PATCH", f"issues/comments/{comment['id']}", {"body": body})
+ else:
+ self.request("POST", f"issues/{report.pr}/comments", {"body": body})
+ return True
diff --git a/scripts/benchmarks/kernel.py b/scripts/benchmarks/kernel.py
new file mode 100644
index 0000000000..559c3935ac
--- /dev/null
+++ b/scripts/benchmarks/kernel.py
@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import json
+import os
+import selectors
+import signal
+import subprocess
+import time
+from collections import deque
+from contextlib import suppress
+from pathlib import Path
+
+CELL_REPEATS = 50
+SHELL_REPEATS = 5
+MIXED_CELLS = 35
+OUTPUT_BYTES = 32 * 1024
+FRAME_ROWS = 10_000
+FRAME_COLUMNS = 8
+
+BASH_EMPTY = "_r = await bash(':')\nassert _r.exit_code == 0 and _r.output == ''"
+GIT_STATUS = "_r = await bash('git status --porcelain')\nassert _r.exit_code == 0 and _r.output == ''"
+BASH_OUTPUT = (
+ f"_r = await bash(\"printf '%{OUTPUT_BYTES}s' ''\")\n"
+ f"assert _r.exit_code == 0 and len(_r.output) == {OUTPUT_BYTES}\nprint(_r.output, end='')"
+)
+
+
+class Kernel:
+ def __init__(self, command: list[str], cwd: Path, env: dict[str, str], transcript: Path):
+ self.transcript = transcript
+ self.stderr = transcript.with_suffix(".stderr").open("wb")
+ self.selector = selectors.DefaultSelector()
+ self.raw: list[bytes] = []
+ self.pending = b""
+ self.lines: deque[bytes] = deque()
+ self.bytes = 0
+ self.sequence = 0
+ self.started = time.perf_counter()
+ try:
+ self.process = subprocess.Popen(
+ command,
+ cwd=cwd,
+ env=env,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=self.stderr,
+ bufsize=0,
+ start_new_session=True,
+ )
+ except Exception:
+ self.stderr.close()
+ self.selector.close()
+ raise
+ assert self.process.stdout is not None
+ self.selector.register(self.process.stdout, selectors.EVENT_READ)
+
+ def event(self, deadline: float) -> dict:
+ while not self.lines:
+ remaining = deadline - time.perf_counter()
+ if remaining <= 0 or not self.selector.select(remaining):
+ raise TimeoutError("Timed out waiting for the kernel protocol")
+ assert self.process.stdout is not None
+ chunk = os.read(self.process.stdout.fileno(), 65536)
+ if not chunk:
+ raise RuntimeError("Kernel exited before completing the request")
+ self.bytes += len(chunk)
+ if self.bytes > 8_000_000:
+ raise RuntimeError("Kernel transcript exceeds 8 MB")
+ self.raw.append(chunk)
+ chunks = (self.pending + chunk).split(b"\n")
+ self.pending = chunks.pop()
+ self.lines.extend(chunks)
+ event = json.loads(self.lines.popleft())
+ if not isinstance(event, dict) or not isinstance(event.get("event"), str):
+ raise ValueError("Invalid kernel protocol event")
+ return event
+
+ def ready(self) -> tuple[float, str]:
+ event = self.event(time.perf_counter() + 30)
+ if event.get("event") != "ready" or event.get("protocol") != 3:
+ raise ValueError("Expected kernel protocol 3 ready event")
+ return time.perf_counter() - self.started, str(event.get("python", "unknown"))
+
+ def send(self, request: dict) -> None:
+ assert self.process.stdin is not None
+ data = memoryview((json.dumps(request) + "\n").encode())
+ while data:
+ written = self.process.stdin.write(data)
+ if not written:
+ raise RuntimeError("Kernel protocol write failed")
+ data = data[written:]
+
+ def done(self, request_id: str, *, interrupted: bool = False) -> list[dict]:
+ deadline = time.perf_counter() + 30
+ events = []
+ while True:
+ event = self.event(deadline)
+ events.append(event)
+ if event.get("event") == "done" and event.get("id") == request_id:
+ if interrupted:
+ if event.get("status") != "error" or not any(
+ e.get("ename") == "KeyboardInterrupt" and e.get("id") == request_id for e in events
+ ):
+ raise RuntimeError("Kernel did not acknowledge the interrupt")
+ elif event.get("status") != "ok":
+ raise RuntimeError(f"Kernel request failed: {str(events)[-400:]}")
+ return events
+
+ def request(self, kind: str, **fields: object) -> list[dict]:
+ self.sequence += 1
+ request_id = str(self.sequence)
+ self.send({"type": kind, "id": request_id, **fields})
+ return self.done(request_id)
+
+ def execute(self, code: str) -> list[dict]:
+ return self.request("execute", code=code)
+
+ def batch(self, code: str, repeats: int, *, output_bytes: int | None = None) -> float:
+ started = time.perf_counter()
+ for _ in range(repeats):
+ events = self.execute(code)
+ if output_bytes is not None:
+ output = "".join(e.get("text", "") for e in events if e["event"] == "stdout")
+ if len(output.encode()) != output_bytes:
+ raise RuntimeError("Kernel output was truncated or missing")
+ return (time.perf_counter() - started) / repeats
+
+ def mixed(self) -> float:
+ started = time.perf_counter()
+ for index in range(MIXED_CELLS):
+ self.execute(GIT_STATUS if index % 4 == 0 else "pass")
+ return time.perf_counter() - started
+
+ def interrupt(self) -> float:
+ self.sequence += 1
+ request_id = str(self.sequence)
+ self.send(
+ {
+ "type": "execute",
+ "id": request_id,
+ "code": "import asyncio\nprint('interrupt-ready', flush=True)\nawait asyncio.sleep(60)",
+ }
+ )
+ deadline = time.perf_counter() + 30
+ while True:
+ event = self.event(deadline)
+ if (
+ event.get("id") == request_id
+ and event["event"] == "stdout"
+ and "interrupt-ready" in event.get("text", "")
+ ):
+ break
+ if event["event"] in ("error", "done"):
+ raise RuntimeError("Interrupt fixture did not reach the running state")
+ started = time.perf_counter()
+ self.send({"type": "interrupt", "id": request_id})
+ self.done(request_id, interrupted=True)
+ elapsed = time.perf_counter() - started
+ self.execute("assert 1 + 1 == 2")
+ return elapsed
+
+ def close(self) -> None:
+ try:
+ if self.process.poll() is None:
+ self.send({"type": "shutdown"})
+ self.process.wait(timeout=5)
+ except (OSError, subprocess.TimeoutExpired):
+ with suppress(ProcessLookupError):
+ os.killpg(self.process.pid, signal.SIGKILL)
+ self.process.wait(timeout=5)
+ finally:
+ for stream in (self.process.stdin, self.process.stdout):
+ if stream is not None:
+ stream.close()
+ self.selector.close()
+ self.stderr.close()
+ self.transcript.with_suffix(".jsonl").write_bytes(b"".join(self.raw))
diff --git a/scripts/benchmarks/pyproject.toml b/scripts/benchmarks/pyproject.toml
new file mode 100644
index 0000000000..1453224541
--- /dev/null
+++ b/scripts/benchmarks/pyproject.toml
@@ -0,0 +1,25 @@
+[project]
+name = "prime-agent-benchmarks"
+version = "0.1.0"
+requires-python = ">=3.11"
+dependencies = [
+ "httpx==0.28.1",
+ "pexpect==4.9.0",
+ "prime-sandboxes==0.2.40",
+ "pydantic==2.13.5",
+ "pyte==0.8.2",
+]
+
+[dependency-groups]
+dev = ["ruff==0.16.5"]
+
+[tool.uv]
+package = false
+exclude-newer = "2026-09-02T00:00:00Z"
+
+[tool.ruff]
+line-length = 110
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
diff --git a/scripts/benchmarks/report.py b/scripts/benchmarks/report.py
new file mode 100644
index 0000000000..5e34c15890
--- /dev/null
+++ b/scripts/benchmarks/report.py
@@ -0,0 +1,308 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import statistics
+from dataclasses import dataclass
+
+from schema import Metric, Observation, Report
+
+MARKER = "<!-- prime-agent-benchmark:v1 -->"
+PERFORMANCE_NOISE_FLOOR = 0.2
+
+
+def fingerprint(report: Report) -> str:
+ fields = {
+ "repository",
+ "head_repository",
+ "base_sha",
+ "head_sha",
+ "harness_sha",
+ "config",
+ }
+ data = json.dumps(report.model_dump(include=fields), sort_keys=True, separators=(",", ":"))
+ return hashlib.sha256(data.encode()).hexdigest()
+
+
+@dataclass(frozen=True)
+class Definition:
+ key: Metric
+ title: str
+ scale: float
+ unit: str
+ absolute: float
+ relative: float
+ better: str
+ worse: str
+
+
+METRICS = (
+ Definition("cold", "Cold startup", 1000, "ms", 0.1, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),
+ Definition("warm", "Warm startup", 1000, "ms", 0.1, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),
+ Definition("install", "Installation", 1, "s", 1, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),
+ Definition("bundle", "Compressed release artifacts", 1e-6, "MB", 65536, 0.005, "smaller", "larger"),
+ Definition("disk", "Installed footprint", 1e-6, "MB", 1048576, 0.01, "smaller", "larger"),
+ Definition(
+ "rss",
+ "Idle memory, summed RSS",
+ 1e-6,
+ "MB",
+ 10485760,
+ PERFORMANCE_NOISE_FLOOR,
+ "less memory",
+ "more memory",
+ ),
+)
+RUNTIME_METRICS = (
+ Definition(
+ "kernel_start",
+ "Python kernel startup",
+ 1000,
+ "ms",
+ 0.005,
+ PERFORMANCE_NOISE_FLOOR,
+ "faster",
+ "slower",
+ ),
+ Definition(
+ "kernel_exec",
+ "Python cell round trip",
+ 1000,
+ "ms",
+ 0.0001,
+ PERFORMANCE_NOISE_FLOOR,
+ "faster",
+ "slower",
+ ),
+ Definition("bash", "Empty bash command", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),
+ Definition(
+ "git_status", "Bash git status", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"
+ ),
+ Definition(
+ "output", "Bash 32 KiB output", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"
+ ),
+ Definition(
+ "mixed", "35 cells / 9 shell calls", 1000, "ms", 0.005, PERFORMANCE_NOISE_FLOOR, "faster", "slower"
+ ),
+ Definition(
+ "interrupt",
+ "Python interrupt to done",
+ 1000,
+ "ms",
+ 0.0005,
+ PERFORMANCE_NOISE_FLOOR,
+ "faster",
+ "slower",
+ ),
+ Definition(
+ "snapshot", "Python state snapshot", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"
+ ),
+ Definition(
+ "restore", "Python state restore", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"
+ ),
+ Definition(
+ "kernel_rss",
+ "Python idle RSS",
+ 1e-6,
+ "MB",
+ 1048576,
+ PERFORMANCE_NOISE_FLOOR,
+ "less memory",
+ "more memory",
+ ),
+ Definition(
+ "loaded_rss",
+ "Python RSS after pandas workload",
+ 1e-6,
+ "MB",
+ 1048576,
+ PERFORMANCE_NOISE_FLOOR,
+ "less memory",
+ "more memory",
+ ),
+)
+
+
+def escape(text: str) -> str:
+ escaped = (
+ text.replace("&", "&amp;")
+ .replace("<", "&lt;")
+ .replace(">", "&gt;")
+ .replace("|", "&#124;")
+ .replace("`", "&#96;")
+ .replace("\r", " ")
+ .replace("\n", " ")
+ )
+ for character in ("\\", "[", "]", "(", ")", "*", "_", "!"):
+ escaped = escaped.replace(character, f"\\{character}")
+ return escaped
+
+
+def values(samples: list[Observation]) -> list[float]:
+ return [s.value for s in samples if s.value is not None]
+
+
+def spread(numbers: list[float]) -> float:
+ if len(numbers) < 4:
+ return max(numbers) - min(numbers) if numbers else 0
+ q1, _, q3 = statistics.quantiles(numbers, n=4, method="inclusive")
+ return q3 - q1
+
+
+def dispersion(samples: list[Observation], definition: Definition) -> str:
+ numbers = values(samples)
+ if len(numbers) < 2:
+ return "—"
+ kind = "IQR" if len(numbers) >= 4 else "range"
+ return f"{kind} {number(spread(numbers), definition)} {definition.unit}"
+
+
+def number(value: float, definition: Definition, signed: bool = False) -> str:
+ scaled = value * definition.scale
+ precision = (
+ 3
+ if definition.unit == "ms" and definition.absolute < 0.001
+ else (1 if definition.unit == "ms" else 2)
+ )
+ if scaled and abs(scaled) < 10 ** (-precision):
+ return f"{scaled:+.2g}" if signed else f"{scaled:.2g}"
+ return f"{scaled:+,.{precision}f}" if signed else f"{scaled:,.{precision}f}"
+
+
+def comparison(
+ definition: Definition, baseline: list[Observation], candidate: list[Observation], expected: int
+) -> tuple[str, str, str, str, str]:
+ left, right = values(baseline), values(candidate)
+ main = statistics.median(left) if left else None
+ head = statistics.median(right) if right else None
+ main_text = "—" if main is None else f"{number(main, definition)} {definition.unit}"
+ head_text = "—" if head is None else f"{number(head, definition)} {definition.unit}"
+ if main is None or head is None:
+ return main_text, head_text, "—", "N/A", "unavailable"
+ delta = head - main
+ percentage = f"{delta / main * 100:+.2f}%" if main else "N/A"
+ if len(left) != expected or len(right) != expected:
+ return (
+ main_text,
+ head_text,
+ f"— {number(delta, definition, True)} {definition.unit}",
+ percentage,
+ "incomplete",
+ )
+ threshold = max(definition.absolute, main * definition.relative, spread(left), spread(right))
+ signal, outcome = "≈", "no clear change"
+ if abs(delta) > threshold:
+ signal, outcome = ("↑", definition.worse) if delta > 0 else ("↓", definition.better)
+ change = f"{signal} {number(delta, definition, True)} {definition.unit}"
+ if signal == "↑":
+ change, outcome = f"**{change}**", f"**{outcome}**"
+ return main_text, head_text, change, percentage, outcome
+
+
+def render(report: Report) -> str:
+ run_url = f"https://github.com/{report.repository}/actions/runs/{report.run_id}"
+ lines = [
+ MARKER,
+ f"<!-- run:{report.run_id}:{report.attempt} head:{report.head_sha} -->",
+ f"<!-- comparison:{fingerprint(report)} status:{report.status} -->",
+ f"### Prime Agent performance — {report.status}",
+ "",
+ f"PR `{report.head_sha[:8]}` compared with main `{report.base_sha[:8]}`.",
+ *(
+ ["Waiting for contributor vouch before credentials or sandboxes are allocated.", ""]
+ if report.status == "pending-trust"
+ else []
+ ),
+ "↓ improved · ↑ regressed · ≈ no clear change · — unavailable",
+ "",
+ "| Metric | Main | This PR | Change | Change % | Result |",
+ "| --- | ---: | ---: | ---: | ---: | --- |",
+ ]
+ for definition in (*METRICS, *RUNTIME_METRICS):
+ if definition == RUNTIME_METRICS[0]:
+ lines.extend(
+ [
+ "",
+ "**Python runtime**",
+ "",
+ "| Metric | Main | This PR | Change | Change % | Result |",
+ "| --- | ---: | ---: | ---: | ---: | --- |",
+ ]
+ )
+ expected = report.config.install_trials if definition.key == "install" else report.config.trials
+ if definition.key in ("bundle", "disk"):
+ expected = 1
+ cells = comparison(
+ definition,
+ report.main.metrics.get(definition.key, []),
+ report.pr_head.metrics.get(definition.key, []),
+ expected,
+ )
+ lines.append(f"| {definition.title} | {' | '.join(cells)} |")
+ compute = sum(s.estimated_usd for s in report.sandboxes)
+ cost_text = (
+ f"**Sandbox cost: ~${compute:.4f}** — no inference calls."
+ if report.sandboxes
+ else "Cost pending or unavailable; sandbox usage has not been collected."
+ )
+ result_link = (
+ f"[Run, logs, and downloadable raw results]({run_url})"
+ if report.pr
+ else "Local run; raw results are stored beside this report."
+ )
+ lines.extend(
+ [
+ "",
+ cost_text,
+ result_link,
+ "",
+ "<details><summary>Methodology and samples</summary>",
+ "",
+ f"Main resolved at {report.started_at.isoformat()}. Harness `{report.harness_sha[:8]}`.",
+ f"Linux x64, {report.config.cpu_cores:g} vCPU, {report.config.memory_gb:g} GB RAM, "
+ f"{report.config.disk_gb:g} GB disk; region {escape(report.config.region)}.",
+ f"Image: `{escape(report.config.image)}`.",
+ "Stock tools, skills, daemon, and Python bootstrap enabled; fresh homes and a fixed Git fixture.",
+ "Onboarding is dismissed; the editor starts without a selected model or submitted prompt.",
+ "Medians shown. Arrows require a 20% timing/memory change plus absolute floors and IQR.",
+ "These practical noise floors are not a statistical significance test.",
+ "Cold means stopped Prime processes; OS filesystem caches are not flushed.",
+ "No model requests or credentials. Installation excludes build/setup time.",
+ "Installer tarballs use loopback; npm/Python downloads use the network with fresh caches.",
+ "Artifact size counts release tarballs; footprint after first use includes registry packages.",
+ "MB is decimal. Summed RSS can double-count shared pages; PSS is recorded when available.",
+ "Provisioning, setup, and build durations are recorded separately in the raw results.",
+ "Kernel probes use the installed JSONL runtime, outside the TUI/TypeScript host.",
+ "Per trial: 50 Python cells, 5 calls per shell case, and one 35-cell mix (9 git status calls).",
+ "Cell/shell values are batch means; other runtime timings are single operations.",
+ "State fixture: a 10,000-row × 8-column integer DataFrame and a 10,000-integer list.",
+ "Restore runs in a fresh kernel, including pandas imports; kernel startup is excluded.",
+ "Kernel RSS covers the isolated Python process; loaded RSS follows the pandas workload.",
+ "Costs estimate full sandbox lifetimes at configured rates, including setup and build.",
+ f"Budget target: ${report.config.budget_usd:g}; not a billing cap. Checks are informational.",
+ "",
+ "| Metric | Main successful/attempted | PR successful/attempted | Main spread | PR spread |",
+ "| --- | ---: | ---: | ---: | ---: |",
+ ]
+ )
+ for definition in (*METRICS, *RUNTIME_METRICS):
+ left = report.main.metrics.get(definition.key, [])
+ right = report.pr_head.metrics.get(definition.key, [])
+ lines.append(
+ f"| {definition.title} | {len(values(left))}/{len(left)} | {len(values(right))}/{len(right)} | "
+ f"{dispersion(left, definition)} | {dispersion(right, definition)} |"
+ )
+ errors = list(report.errors)
+ for name, side in (("main", report.main), ("PR", report.pr_head)):
+ if side.error:
+ errors.append(f"{name}: {side.error}")
+ errors.extend(
+ f"{name} {metric} trial {sample.trial}: {sample.error}"
+ for metric, samples in side.metrics.items()
+ for sample in samples
+ if sample.error
+ )
+ if errors:
+ lines.extend(["", "Failures:", ""] + [f"- {escape(error)}" for error in errors[:30]])
+ lines.extend(["", "</details>", ""])
+ return "\n".join(lines)
diff --git a/scripts/benchmarks/schema.py b/scripts/benchmarks/schema.py
new file mode 100644
index 0000000000..3446ea63c8
--- /dev/null
+++ b/scripts/benchmarks/schema.py
@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator
+
+SHA = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{40}$")]
+Repository = Annotated[str, StringConstraints(pattern=r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")]
+Metric = Literal[
+ "cold",
+ "warm",
+ "install",
+ "bundle",
+ "disk",
+ "rss",
+ "pss",
+ "kernel_start",
+ "kernel_exec",
+ "bash",
+ "git_status",
+ "output",
+ "mixed",
+ "interrupt",
+ "snapshot",
+ "restore",
+ "kernel_rss",
+ "loaded_rss",
+]
+NonNegative = Annotated[float, Field(ge=0, le=1e15, allow_inf_nan=False)]
+ROOT = Path(__file__).resolve().parent
+UV_VERSION = "0.12.9"
+
+
+class StrictModel(BaseModel):
+ model_config = ConfigDict(extra="forbid", allow_inf_nan=False, strict=True)
+
+
+class Config(StrictModel):
+ image: Annotated[str, StringConstraints(pattern=r"^.+@sha256:[0-9a-f]{64}$")]
+ region: str
+ cpu_cores: Annotated[float, Field(gt=0, le=16)]
+ memory_gb: Annotated[float, Field(gt=0, le=64)]
+ disk_gb: Annotated[float, Field(gt=0, le=1000)]
+ trials: Annotated[int, Field(ge=3, le=50)]
+ install_trials: Annotated[int, Field(ge=1, le=10)]
+ debounce_seconds: Annotated[int, Field(ge=0, le=300)]
+ timeout_seconds: Annotated[int, Field(ge=60, le=3600)]
+ ttl_minutes: Annotated[int, Field(ge=1, le=120)]
+ budget_usd: Annotated[float, Field(gt=0, le=20)]
+ cpu_usd_per_hour: NonNegative
+ memory_usd_per_gb_hour: NonNegative
+ disk_usd_per_gb_hour: NonNegative
+
+ @model_validator(mode="after")
+ def validate_limits(self) -> Config:
+ if self.ttl_minutes * 60 <= self.timeout_seconds:
+ raise ValueError("Sandbox TTL must exceed the run timeout")
+ return self
+
+ @classmethod
+ def load(cls, path: Path = ROOT / "config.json") -> Config:
+ return cls.model_validate_json(path.read_text())
+
+ def hourly_cost(self) -> float:
+ return (
+ self.cpu_cores * self.cpu_usd_per_hour
+ + self.memory_gb * self.memory_usd_per_gb_hour
+ + self.disk_gb * self.disk_usd_per_gb_hour
+ )
+
+
+class Observation(StrictModel):
+ trial: Annotated[int, Field(ge=0, le=100)]
+ value: NonNegative | None = None
+ error: Annotated[str, Field(max_length=500)] | None = None
+
+ @model_validator(mode="after")
+ def validate_outcome(self) -> Observation:
+ if (self.value is None) == (self.error is None):
+ raise ValueError("An observation must contain either a value or an error")
+ return self
+
+
+class ProcessMemory(StrictModel):
+ pid: Annotated[int, Field(gt=0)]
+ name: Annotated[str, Field(max_length=80)]
+ rss: NonNegative
+ pss: NonNegative | None = None
+
+
+class Side(StrictModel):
+ sha: SHA
+ metrics: dict[Metric, list[Observation]] = Field(default_factory=dict)
+ processes: list[ProcessMemory] = Field(default_factory=list, max_length=128)
+ runtime: dict[str, str] = Field(default_factory=dict)
+ timings: dict[Literal["provision", "setup", "build"], NonNegative] = Field(default_factory=dict)
+ artifacts: dict[str, NonNegative] = Field(default_factory=dict)
+ error: Annotated[str, Field(max_length=500)] | None = None
+
+
+class Request(StrictModel):
+ repository: Repository
+ source_repository: Repository
+ sha: SHA
+ harness_sha: SHA
+ pr: Annotated[int, Field(ge=0)]
+ run_id: Annotated[int, Field(ge=1)]
+ attempt: Annotated[int, Field(ge=1)]
+ role: Literal["main", "pr"]
+ config: Config
+
+
+class Result(StrictModel):
+ request: Request
+ side: Side
+
+
+class SandboxUsage(StrictModel):
+ id: Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9_-]+$")]
+ role: Literal["main", "pr"]
+ seconds: NonNegative
+ estimated_usd: NonNegative
+ deleted: bool
+
+
+class Report(StrictModel):
+ schema_version: Literal[1] = 1
+ repository: Repository
+ head_repository: Repository
+ pr: Annotated[int, Field(ge=0)]
+ run_id: Annotated[int, Field(ge=1)]
+ attempt: Annotated[int, Field(ge=1)] = 1
+ harness_sha: SHA
+ base_sha: SHA
+ head_sha: SHA
+ started_at: datetime
+ finished_at: datetime | None = None
+ config: Config
+ status: Literal["pending-trust", "running", "completed", "partial", "failed", "canceled"] = "running"
+ main: Side
+ pr_head: Side
+ sandboxes: list[SandboxUsage] = Field(default_factory=list, max_length=2)
+ errors: list[Annotated[str, Field(max_length=500)]] = Field(default_factory=list, max_length=30)
+
+ @model_validator(mode="after")
+ def validate_revisions(self) -> Report:
+ if self.main.sha != self.base_sha or self.pr_head.sha != self.head_sha:
+ raise ValueError("Result revisions do not match the request")
+ for side in (self.main, self.pr_head):
+ for metric, samples in side.metrics.items():
+ expected = self.config.install_trials if metric == "install" else self.config.trials
+ if metric in ("bundle", "disk"):
+ expected = 1
+ if (
+ len(samples) > expected
+ or len({s.trial for s in samples}) != len(samples)
+ or any(sample.trial >= expected for sample in samples)
+ ):
+ raise ValueError("Duplicate or excessive observations")
+ return self
+
+
+def write_json(path: Path, value: BaseModel) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ temporary = path.with_suffix(".tmp")
+ temporary.write_text(value.model_dump_json(indent=2) + "\n")
+ temporary.replace(path)
+
+
+def load_report(path: Path) -> Report:
+ if path.stat().st_size > 2_000_000:
+ raise ValueError("Report exceeds the size limit")
+ return Report.model_validate_json(path.read_text())
diff --git a/scripts/benchmarks/terminal.py b/scripts/benchmarks/terminal.py
new file mode 100644
index 0000000000..0c92ba7244
--- /dev/null
+++ b/scripts/benchmarks/terminal.py
@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+import pexpect
+import pyte
+
+QUERIES = {
+ "\x1b[6n": "\x1b[1;1R",
+ "\x1b[?6n": "\x1b[?1;1R",
+ "\x1b[c": "\x1b[?1;2c",
+ "\x1b[0c": "\x1b[?1;2c",
+ "\x1b[>c": "\x1b[>0;276;0c",
+ "\x1b[>0c": "\x1b[>0;276;0c",
+ "\x1b[?u": "\x1b[?0u",
+ "\x1b[?2026$p": "\x1b[?2026;2$y",
+ "\x1b[14t": "\x1b[4;640;960t",
+ "\x1b[16t": "\x1b[6;16;8t",
+ "\x1b[18t": "\x1b[8;40;120t",
+}
+for terminator in ("\x07", "\x1b\\"):
+ QUERIES[f"\x1b]11;?{terminator}"] = "\x1b]11;rgb:0000/0000/0000\x1b\\"
+ QUERIES[f"\x1b]10;?{terminator}"] = "\x1b]10;rgb:eeee/eeee/eeee\x1b\\"
+
+
+class Display:
+ def __init__(self, reply: Callable[[str], object]):
+ self.screen = pyte.Screen(120, 40)
+ self.stream = pyte.Stream(self.screen)
+ self.reply = reply
+ self.pending = ""
+
+ def feed(self, chunk: str) -> None:
+ self.pending += chunk
+ while True:
+ matches = [(self.pending.find(query), query) for query in QUERIES if query in self.pending]
+ if not matches:
+ break
+ position, query = min(matches)
+ self.stream.feed(self.pending[:position])
+ self.reply(QUERIES[query])
+ self.pending = self.pending[position + len(query) :]
+ keep = max(
+ (
+ length
+ for query in QUERIES
+ for length in range(1, len(query))
+ if self.pending.endswith(query[:length])
+ ),
+ default=0,
+ )
+ if keep:
+ self.stream.feed(self.pending[:-keep])
+ self.pending = self.pending[-keep:]
+ else:
+ self.stream.feed(self.pending)
+ self.pending = ""
+
+ def text(self) -> str:
+ return "\n".join(row.rstrip() for row in self.screen.display)
+
+
+class Terminal:
+ def __init__(self, command: list[str], cwd: Path, env: dict[str, str], transcript: Path):
+ self.started = time.perf_counter()
+ self.child = pexpect.spawn(
+ command[0],
+ command[1:],
+ cwd=str(cwd),
+ env=env,
+ encoding="utf-8",
+ codec_errors="replace",
+ dimensions=(40, 120),
+ echo=False,
+ )
+ self.child.delaybeforesend = 0
+ self.display = Display(self.child.send)
+ self.transcript = transcript
+ self.raw: list[str] = []
+ self.bytes = 0
+
+ def pump(self) -> bool:
+ try:
+ chunk = self.child.read_nonblocking(65536, timeout=0.005)
+ except pexpect.TIMEOUT:
+ return False
+ except pexpect.EOF as error:
+ raise RuntimeError("Prime Agent exited before the measurement completed") from error
+ self.bytes += len(chunk)
+ if self.bytes > 8_000_000:
+ raise RuntimeError("Terminal transcript exceeds 8 MB")
+ self.raw.append(chunk)
+ self.display.feed(chunk)
+ return True
+
+ def until(self, predicate: Callable[[Display], bool], seconds: float) -> float:
+ deadline = time.perf_counter() + seconds
+ while time.perf_counter() < deadline:
+ if self.pump() and predicate(self.display):
+ return time.perf_counter()
+ raise TimeoutError("Timed out waiting for the expected terminal state")
+
+ def settle(self, seconds: float) -> None:
+ deadline = time.perf_counter() + seconds
+ while time.perf_counter() < deadline:
+ self.pump()
+
+ def ready(self) -> float:
+ self.until(lambda display: "agents/resume" in display.text(), 30)
+ self.child.send("benchready")
+ echoed = self.until(lambda display: "benchready" in display.text(), 5)
+ self.child.send("\x7f" * len("benchready"))
+ self.until(lambda display: "benchready" not in display.text(), 5)
+ return echoed - self.started
+
+ def close(self) -> None:
+ raw, snapshot = "".join(self.raw), self.display.text()
+ self.transcript.with_suffix(".raw").write_text(raw)
+ self.transcript.with_suffix(".txt").write_text(snapshot)
+ try:
+ if self.child.isalive():
+ self.child.sendcontrol("c")
+ time.sleep(0.15)
+ if self.child.isalive():
+ self.child.sendcontrol("c")
+ except (OSError, pexpect.EOF):
+ pass
+ finally:
+ self.child.close(force=True)
diff --git a/scripts/benchmarks/tests/calibration.json b/scripts/benchmarks/tests/calibration.json
new file mode 100644
index 0000000000..badaa4d414
--- /dev/null
+++ b/scripts/benchmarks/tests/calibration.json
@@ -0,0 +1,398 @@
+{
+ "baseline_sha": "427ea4c72cc606ac061a14287c0c15c471eff002",
+ "recorded_at": "2026-09-09T21:30:42.632129Z",
+ "metrics": {
+ "bundle": {
+ "main": [
+ 11096130.0
+ ],
+ "pr": [
+ 11096130.0
+ ]
+ },
+ "install": {
+ "main": [
+ 27.877250564000008,
+ 28.70503522299998,
+ 29.183944166000003
+ ],
+ "pr": [
+ 30.62683971300001,
+ 32.43433822399999,
+ 36.622502448999995
+ ]
+ },
+ "cold": {
+ "main": [
+ 2.9279603130000282,
+ 2.9515973970000005,
+ 2.864970442000015,
+ 2.9641698160000374,
+ 2.843047642999977,
+ 2.769899554999995,
+ 2.87473634600002,
+ 2.977558033999969,
+ 2.922310485999958,
+ 3.0185121560000425
+ ],
+ "pr": [
+ 3.200634946999969,
+ 3.0652513259999523,
+ 3.3256505719999723,
+ 3.0905467590000057,
+ 3.5100812420000125,
+ 3.551760985000044,
+ 3.3178279680000173,
+ 3.235152325999991,
+ 3.2145795829999884,
+ 3.216496445999951
+ ]
+ },
+ "rss": {
+ "main": [
+ 1099157504.0,
+ 1038544896.0,
+ 961441792.0,
+ 1009164288.0,
+ 1042464768.0,
+ 987299840.0,
+ 968232960.0,
+ 909803520.0,
+ 971235328.0,
+ 1116012544.0
+ ],
+ "pr": [
+ 1045270528.0,
+ 1166790656.0,
+ 1030512640.0,
+ 989421568.0,
+ 1112367104.0,
+ 1099530240.0,
+ 1153560576.0,
+ 1118666752.0,
+ 1084510208.0,
+ 1133363200.0
+ ]
+ },
+ "warm": {
+ "main": [
+ 1.7345433609999645,
+ 1.7651198379999755,
+ 1.7687801519999766,
+ 1.762977601999978,
+ 1.6944376189999844,
+ 1.642891163999991,
+ 1.6524441209999736,
+ 1.7300561220000077,
+ 1.6743516009999553,
+ 1.7221875839999257
+ ],
+ "pr": [
+ 1.7706765419999897,
+ 1.8812752079999768,
+ 1.8848265259999835,
+ 1.773448736999967,
+ 1.854215795000016,
+ 1.9367081579999876,
+ 1.8299759000000222,
+ 1.9682038950000447,
+ 1.8523304409999923,
+ 1.8055133089999913
+ ]
+ },
+ "disk": {
+ "main": [
+ 597316531.0
+ ],
+ "pr": [
+ 597315364.0
+ ]
+ },
+ "kernel_start": {
+ "main": [
+ 0.12121383599992441,
+ 0.12723471499998595,
+ 0.1409247950000463,
+ 0.12776771299991196,
+ 0.12564238899994962,
+ 0.13082783800007292,
+ 0.12394591999998283,
+ 0.1299987349999583,
+ 0.11980933699999241,
+ 0.1293355980000115
+ ],
+ "pr": [
+ 0.14848755700000993,
+ 0.1377638199999751,
+ 0.12577888999999232,
+ 0.14028942499999175,
+ 0.12999939600001653,
+ 0.18843824300000733,
+ 0.16141352499994355,
+ 0.1650210259999767,
+ 0.15177539300009357,
+ 0.12403774500000964
+ ]
+ },
+ "kernel_rss": {
+ "main": [
+ 35270656.0,
+ 31350784.0,
+ 37015552.0,
+ 35385344.0,
+ 37322752.0,
+ 35954688.0,
+ 37765120.0,
+ 36163584.0,
+ 38535168.0,
+ 35065856.0
+ ],
+ "pr": [
+ 35020800.0,
+ 36134912.0,
+ 30023680.0,
+ 36614144.0,
+ 34148352.0,
+ 33996800.0,
+ 33341440.0,
+ 38506496.0,
+ 32722944.0,
+ 36675584.0
+ ]
+ },
+ "kernel_exec": {
+ "main": [
+ 0.0005549562799978957,
+ 0.000465069780000249,
+ 0.0005641982199995255,
+ 0.000583243199998833,
+ 0.0005096060599998963,
+ 0.0005911986399996749,
+ 0.0005406058400012625,
+ 0.0004923386999985269,
+ 0.0007612591199995223,
+ 0.0005891538199989555
+ ],
+ "pr": [
+ 0.0007702345799998512,
+ 0.0006107251399998858,
+ 0.0005365984600007323,
+ 0.000595193220000283,
+ 0.000712963140000511,
+ 0.0006294563200003723,
+ 0.000602465059998849,
+ 0.000630622300000141,
+ 0.0006821516199988764,
+ 0.0006074899800000822
+ ]
+ },
+ "bash": {
+ "main": [
+ 0.012265951599988511,
+ 0.011434986400013258,
+ 0.012523944600002324,
+ 0.012188101599986113,
+ 0.012133405999998103,
+ 0.013145084400002815,
+ 0.011684339800012822,
+ 0.010691210599998158,
+ 0.014503767800010791,
+ 0.013724902000012662
+ ],
+ "pr": [
+ 0.015500308199989377,
+ 0.012726768200002425,
+ 0.013086532799991345,
+ 0.012979604999998173,
+ 0.012928957999997692,
+ 0.013177246399993691,
+ 0.014024491799978023,
+ 0.012863739800013718,
+ 0.014560996599993815,
+ 0.013492744999985007
+ ]
+ },
+ "git_status": {
+ "main": [
+ 0.017970771399996012,
+ 0.01924374820000594,
+ 0.018154279200007294,
+ 0.017899456400004964,
+ 0.017645751600002767,
+ 0.018604173399990032,
+ 0.01732922300000155,
+ 0.017314914799999314,
+ 0.01831345020000299,
+ 0.019714569600000686
+ ],
+ "pr": [
+ 0.02181655580000097,
+ 0.018099323399997046,
+ 0.021713121400000545,
+ 0.01987777339999184,
+ 0.019425245200000062,
+ 0.01948336739999377,
+ 0.0200670867999861,
+ 0.02184636980000505,
+ 0.018714642200006893,
+ 0.020085525600006805
+ ]
+ },
+ "output": {
+ "main": [
+ 0.01278957699998955,
+ 0.012213238399999682,
+ 0.011799004199997399,
+ 0.014527103199998236,
+ 0.013296984999988127,
+ 0.013471006999998281,
+ 0.012347434999992402,
+ 0.011561371400011922,
+ 0.012395569399996021,
+ 0.012976578400002835
+ ],
+ "pr": [
+ 0.019721852200007106,
+ 0.01687608419999833,
+ 0.015285829599997669,
+ 0.012512077600001703,
+ 0.014360300200007715,
+ 0.016800876600007085,
+ 0.013386880000007295,
+ 0.016856159600001773,
+ 0.01287674860000152,
+ 0.014394774199990934
+ ]
+ },
+ "mixed": {
+ "main": [
+ 0.18327940200003923,
+ 0.1656647150000481,
+ 0.18445866100000785,
+ 0.1879419299999654,
+ 0.18152489999999943,
+ 0.18734949900010633,
+ 0.18109903399999894,
+ 0.16189230999998472,
+ 0.17762886299999536,
+ 0.19014139399996566
+ ],
+ "pr": [
+ 0.24294485600000826,
+ 0.19827466099997082,
+ 0.24033736699999508,
+ 0.2019193049999899,
+ 0.20282605300002388,
+ 0.2027119749999997,
+ 0.20539985499999602,
+ 0.20633829200005493,
+ 0.1809149719999823,
+ 0.21170844000005218
+ ]
+ },
+ "interrupt": {
+ "main": [
+ 0.0017507960000102685,
+ 0.0015083110000659872,
+ 0.0017416609999827415,
+ 0.0016143720000627582,
+ 0.0018679830000110087,
+ 0.0016709809999611025,
+ 0.0024465850000296996,
+ 0.0013896130000148332,
+ 0.001642629000002671,
+ 0.0018307729999378353
+ ],
+ "pr": [
+ 0.001693358000011358,
+ 0.0017097159999934775,
+ 0.001485645000002478,
+ 0.001690604000032181,
+ 0.0017368419999570506,
+ 0.0015665369999737777,
+ 0.0014975740000409132,
+ 0.0015429750000066633,
+ 0.0015613769999163196,
+ 0.001896525000006477
+ ]
+ },
+ "loaded_rss": {
+ "main": [
+ 107814912.0,
+ 93478912.0,
+ 98910208.0,
+ 97415168.0,
+ 99360768.0,
+ 98050048.0,
+ 99872768.0,
+ 98254848.0,
+ 100601856.0,
+ 97099776.0
+ ],
+ "pr": [
+ 107479040.0,
+ 98242560.0,
+ 90345472.0,
+ 96727040.0,
+ 96256000.0,
+ 96059392.0,
+ 95399936.0,
+ 100548608.0,
+ 94867456.0,
+ 98779136.0
+ ]
+ },
+ "snapshot": {
+ "main": [
+ 0.078061469999966,
+ 0.025671378000083678,
+ 0.029300445999979274,
+ 0.027639665000037894,
+ 0.025642438999966544,
+ 0.026381777999972655,
+ 0.025477492000050006,
+ 0.027599351999924693,
+ 0.025847754999972494,
+ 0.028655434000029345
+ ],
+ "pr": [
+ 0.09319571300000007,
+ 0.03817890999999918,
+ 0.02773127599999725,
+ 0.03216677000000345,
+ 0.035073244000045634,
+ 0.029259709999962524,
+ 0.02817944000003081,
+ 0.028553541999940535,
+ 0.026783507000004647,
+ 0.02539736100004575
+ ]
+ },
+ "restore": {
+ "main": [
+ 0.36483748199998445,
+ 0.3538136629999826,
+ 0.3790124850000893,
+ 0.3602016209999874,
+ 0.3554445230000738,
+ 0.3486341860000266,
+ 0.3896290149999686,
+ 0.3843983080000726,
+ 0.40788538100002825,
+ 0.37759137900002315
+ ],
+ "pr": [
+ 0.48703021900001886,
+ 0.44926845599997023,
+ 0.39550326900001664,
+ 0.42711700600000313,
+ 0.4238764670000137,
+ 0.4049739440000053,
+ 0.42803659699995933,
+ 0.38498759599997356,
+ 0.39857957699996405,
+ 0.3963605919999509
+ ]
+ }
+ }
+}
diff --git a/scripts/benchmarks/tests/test_benchmarks.py b/scripts/benchmarks/tests/test_benchmarks.py
new file mode 100644
index 0000000000..554db5d3a1
--- /dev/null
+++ b/scripts/benchmarks/tests/test_benchmarks.py
@@ -0,0 +1,511 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+from pydantic import ValidationError
+
+from cli import completed_report, main, validate_completion, workflow_source
+from controller import Canceled, Controller, cleanup, labels, side_complete
+from github import TITLE, GitHub
+from report import MARKER, METRICS, RUNTIME_METRICS, comparison, render
+from schema import (
+ Config,
+ Observation,
+ ProcessMemory,
+ Report,
+ Request,
+ Side,
+ load_report,
+ write_json,
+)
+from terminal import QUERIES, Display, Terminal
+from worker import environment, install, measure
+
+SHA = "a" * 40
+HEAD = "b" * 40
+
+
+def fixture() -> Report:
+ return Report(
+ repository="PrimeIntellect-ai/prime-agent",
+ head_repository="PrimeIntellect-ai/prime-agent",
+ pr=42,
+ run_id=100,
+ attempt=1,
+ harness_sha=SHA,
+ base_sha=SHA,
+ head_sha=HEAD,
+ started_at=datetime(2026, 9, 9, tzinfo=UTC),
+ config=Config.load(),
+ main=Side(sha=SHA),
+ pr_head=Side(sha=HEAD),
+ )
+
+
+def observations(*numbers: float) -> list[Observation]:
+ return [Observation(trial=index, value=value) for index, value in enumerate(numbers)]
+
+
+class TerminalTests(unittest.TestCase):
+ def test_real_pty_detects_injected_startup_delays_without_submitting(self):
+ script = """
+import os, sys, time, tty
+tty.setraw(0)
+time.sleep(float(sys.argv[1]))
+os.write(1, b'agents/resume\\r\\n> ')
+while True:
+ byte = os.read(0, 1)
+ if byte == b'\\x7f':
+ os.write(1, b'\\b \\b')
+ elif byte == b'\\r':
+ raise RuntimeError('benchmark submitted a prompt')
+ elif byte == b'\\x03':
+ break
+ else:
+ os.write(1, byte)
+"""
+ measurements = []
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ path = root / "fixture.py"
+ path.write_text(script)
+ for delay in (0.05, 0.6):
+ terminal = Terminal(
+ [sys.executable, str(path), str(delay)],
+ root,
+ os.environ.copy(),
+ root / f"transcript-{delay}",
+ )
+ try:
+ measurements.append(terminal.ready())
+ finally:
+ terminal.close()
+ self.assertGreater(measurements[1] - measurements[0], 0.25)
+
+ def test_queries_split_at_every_boundary(self):
+ for query, reply in QUERIES.items():
+ for boundary in range(len(query) + 1):
+ with self.subTest(query=query, boundary=boundary):
+ replies = []
+ display = Display(replies.append)
+ display.feed("prefix" + query[:boundary])
+ display.feed(query[boundary:] + "suffix")
+ self.assertEqual(replies, [reply])
+ self.assertTrue(display.text().startswith("prefixsuffix"))
+
+
+class ReportTests(unittest.TestCase):
+ def test_same_revision_sandbox_calibration_does_not_report_regressions(self):
+ calibration = json.loads(Path(__file__).with_name("calibration.json").read_text())
+ for definition in (*METRICS, *RUNTIME_METRICS):
+ with self.subTest(metric=definition.key):
+ samples = calibration["metrics"][definition.key]
+ result = comparison(
+ definition,
+ observations(*samples["main"]),
+ observations(*samples["pr"]),
+ len(samples["main"]),
+ )
+ self.assertEqual(result[-1], "no clear change")
+
+ def test_slower_and_faster_have_consistent_signs_and_arrows(self):
+ baseline = observations(*([2.0] * 10))
+ slower = comparison(METRICS[0], baseline, observations(*([2.5] * 10)), 10)
+ faster = comparison(METRICS[0], baseline, observations(*([1.5] * 10)), 10)
+ self.assertEqual(slower[2:], ("**↑ +500.0 ms**", "+25.00%", "**slower**"))
+ self.assertEqual(faster[2:], ("↓ -500.0 ms", "-25.00%", "faster"))
+
+ def test_noise_and_partial_results_are_not_regressions(self):
+ noisy = observations(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+ shifted = observations(1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 9.2, 10.2)
+ self.assertEqual(comparison(METRICS[0], noisy, shifted, 10)[4], "no clear change")
+ shifted[-1] = Observation(trial=9, error="timed out")
+ self.assertEqual(comparison(METRICS[0], noisy, shifted, 10)[4], "incomplete")
+ self.assertEqual(comparison(METRICS[0], [], shifted, 10)[4], "unavailable")
+
+ def test_zero_baseline_and_small_bundle_changes(self):
+ cells = comparison(METRICS[3], observations(0), observations(65537), 1)
+ self.assertEqual(cells[3], "N/A")
+ self.assertIn("larger", cells[4])
+ unchanged = comparison(METRICS[3], observations(10_000_000), observations(10_001_000), 1)
+ self.assertEqual(unchanged[4], "no clear change")
+
+ def test_comment_escapes_untrusted_text_and_reports_failures(self):
+ report = fixture()
+ report.pr_head.error = "<img src=x> | ![click](https://example.test)\n<!-- comment -->"
+ text = render(report)
+ self.assertTrue(text.startswith(MARKER))
+ self.assertNotIn("<img", text)
+ self.assertNotIn("![click]", text)
+ self.assertIn("&#124;", text)
+ self.assertIn("0/0", text)
+ self.assertNotIn("TTFT", text)
+ self.assertNotIn("Pinference", text)
+ for definition in RUNTIME_METRICS:
+ self.assertIn(definition.title, text)
+
+ def test_schema_rejects_nonfinite_negative_duplicate_and_wrong_revision(self):
+ for value in (float("nan"), float("inf"), -1, True, "1"):
+ with self.assertRaises(ValidationError):
+ Observation(trial=0, value=value)
+ report = fixture().model_dump()
+ report["main"]["sha"] = HEAD
+ with self.assertRaises(ValidationError):
+ Report.model_validate(report)
+ report = fixture().model_dump()
+ report["main"]["metrics"] = {"cold": [{"trial": 0, "value": 1}] * 2}
+ with self.assertRaises(ValidationError):
+ Report.model_validate(report)
+
+ def test_report_round_trip_and_size_limit(self):
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory) / "report.json"
+ report = fixture()
+ write_json(path, report)
+ self.assertEqual(load_report(path), report)
+ path.write_bytes(b" " * 2_000_001)
+ with self.assertRaises(ValueError):
+ load_report(path)
+
+
+class FakeGitHub(GitHub):
+ def __init__(self):
+ self.head = HEAD
+ self.state = "open"
+ self.attempt = 1
+ self.runs = [{"id": 100, "run_number": 10, "run_attempt": 1, "display_title": f"{TITLE}42"}]
+ self.comments = []
+ self.writes = []
+
+ def request(self, method, path, body=None):
+ if method != "GET":
+ self.writes.append((method, path, body))
+ return {}
+ if path == "pulls/42":
+ return {"state": self.state, "head": {"sha": self.head}}
+ if path == "actions/runs/100":
+ return {"run_attempt": self.attempt, "run_number": 10}
+ raise AssertionError(path)
+
+ def pages(self, path, key=None):
+ yield self.runs if key else self.comments
+
+
+class PublishingTests(unittest.TestCase):
+ def test_superseded_commit_and_same_commit_rerun_do_not_publish(self):
+ for case in ("head", "new_run", "attempt", "closed"):
+ with self.subTest(case=case):
+ github = FakeGitHub()
+ if case == "head":
+ github.head = SHA
+ elif case == "new_run":
+ github.runs.append(
+ {"id": 101, "run_number": 11, "run_attempt": 1, "display_title": f"{TITLE}42"}
+ )
+ elif case == "attempt":
+ github.attempt = 2
+ else:
+ github.state = "closed"
+ self.assertFalse(github.publish(fixture()))
+ self.assertEqual(github.writes, [])
+
+ def test_updates_only_its_own_sticky_comment(self):
+ github = FakeGitHub()
+ github.comments = [
+ {"id": 1, "user": {"login": "someone"}, "body": MARKER},
+ {"id": 2, "user": {"login": "github-actions[bot]"}, "body": MARKER},
+ ]
+ self.assertTrue(github.publish(fixture()))
+ self.assertEqual(github.writes[0][:2], ("PATCH", "issues/comments/2"))
+
+ def test_newer_comment_is_not_overwritten(self):
+ github = FakeGitHub()
+ github.comments = [
+ {
+ "id": 2,
+ "user": {"login": "github-actions[bot]"},
+ "body": f"{MARKER}\n<!-- run:101:1 head:{HEAD} -->",
+ }
+ ]
+ self.assertFalse(github.publish(fixture()))
+ self.assertEqual(github.writes, [])
+
+
+class LifecycleTests(unittest.TestCase):
+ def test_sandbox_creation_does_not_inject_credentials(self):
+ with (
+ tempfile.TemporaryDirectory() as directory,
+ patch.dict(os.environ, {"PRIME_SANDBOX_API_KEY": "fake", "PINFERENCE_API_KEY": "must-not-use"}),
+ ):
+ controller = Controller(fixture(), Path(directory), live_github=False)
+ controller.client = Mock()
+ controller.client.create.return_value = SimpleNamespace(id="one")
+ controller.client.get.return_value = SimpleNamespace(status="RUNNING")
+ controller.client.execute_command.return_value = SimpleNamespace(exit_code=0)
+ controller.wait = Mock()
+ controller.start("main")
+ request = controller.client.create.call_args.args[0]
+ self.assertIsNone(request.secrets)
+ self.assertIsNone(request.environment_vars)
+
+ def test_invalid_results_get_a_failure_notice_and_missing_results_preserve_pending_trust(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ request, result = root / "request.json", root / "result.json"
+ run = {"id": 100, "run_attempt": 1, "display_title": f"{TITLE}42", "conclusion": "cancelled"}
+ write_json(request, fixture())
+ self.assertEqual(completed_report(result, request, run).status, "canceled")
+ result.write_text('{"forged": true}')
+ report = completed_report(result, request, run)
+ self.assertEqual(report.status, "failed")
+ self.assertIn("validation", report.errors[0])
+ result.unlink()
+ pending = fixture()
+ pending.status = "pending-trust"
+ write_json(request, pending)
+ self.assertEqual(completed_report(result, request, run).status, "pending-trust")
+ self.assertIn("Waiting for contributor vouch", render(pending))
+
+ def test_automatic_duplicates_skip_but_manual_runs_and_new_comparisons_do_not(self):
+ for case in ("duplicate", "manual", "attempt", "main", "harness", "config", "failed"):
+ with self.subTest(case=case), tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ event, output = root / "event.json", root / "output.txt"
+ event.write_text('{"pull_request": {"number": 42}}')
+ old = fixture()
+ old.status = "completed" if case != "failed" else "partial"
+ github = FakeGitHub()
+ github.comments = [{"user": {"login": "github-actions[bot]"}, "body": render(old)}]
+ report = fixture()
+ if case == "main":
+ report.base_sha = report.main.sha = "c" * 40
+ elif case == "harness":
+ report.harness_sha = "c" * 40
+ elif case == "config":
+ report.config.trials = 11
+ elif case == "attempt":
+ report.attempt = 2
+ github.resolve = Mock(return_value=(report, "kevin"))
+ with (
+ patch("cli.GitHub", return_value=github),
+ patch.object(sys, "argv", ["cli.py", "resolve", "--results", directory]),
+ patch.dict(
+ os.environ,
+ {
+ "GITHUB_EVENT_PATH": str(event),
+ "GITHUB_OUTPUT": str(output),
+ "GITHUB_EVENT_NAME": "workflow_dispatch"
+ if case == "manual"
+ else "pull_request_target",
+ "GITHUB_SHA": SHA,
+ "GITHUB_RUN_ID": "100",
+ "GITHUB_RUN_ATTEMPT": str(report.attempt),
+ },
+ ),
+ ):
+ main()
+ self.assertIn(f"needed={'false' if case == 'duplicate' else 'true'}", output.read_text())
+ self.assertEqual(github.writes, [])
+
+ def test_cleanup_paginates_before_deleting_and_checks_all_labels(self):
+ client = Mock()
+ expected = labels("owner/repo", 100, 2)
+ first = SimpleNamespace(id="one", labels=expected)
+ wrong = SimpleNamespace(id="other-attempt", labels=labels("owner/repo", 100, 1))
+ second = SimpleNamespace(id="two", labels=expected)
+ client.list.side_effect = [
+ SimpleNamespace(sandboxes=[first, wrong], has_next=True),
+ SimpleNamespace(sandboxes=[second], has_next=False),
+ ]
+ self.assertEqual(cleanup(client, "owner/repo", 100, 2), ["one", "two"])
+ self.assertEqual([call[0] for call in client.mock_calls], ["list", "list", "delete", "delete"])
+
+ def test_cleanup_attempts_remaining_deletions_after_failure(self):
+ client = Mock()
+ expected = labels("owner/repo", 1, 1)
+ client.list.return_value = SimpleNamespace(
+ sandboxes=[
+ SimpleNamespace(id="one", labels=expected),
+ SimpleNamespace(id="two", labels=expected),
+ ],
+ has_next=False,
+ )
+ client.delete.side_effect = [RuntimeError("unavailable"), None]
+ with self.assertRaises(RuntimeError):
+ cleanup(client, "owner/repo", 1, 1)
+ self.assertEqual(client.delete.call_count, 2)
+
+ def test_cancellation_still_deletes_a_created_sandbox(self):
+ with (
+ tempfile.TemporaryDirectory() as directory,
+ patch.dict(os.environ, {"PRIME_SANDBOX_API_KEY": "fake"}),
+ ):
+ report = fixture()
+ report.config.debounce_seconds = 0
+ controller = Controller(report, Path(directory), live_github=False)
+ controller.client = Mock()
+ controller.logs = Mock()
+ sandbox = SimpleNamespace(id="one", created_at=datetime.now(UTC).replace(tzinfo=None))
+
+ def start(_role):
+ controller.sandboxes["main"] = sandbox
+ raise Canceled()
+
+ controller.start = start
+ controller.run()
+ controller.client.delete.assert_called_once_with("one")
+ self.assertEqual(load_report(Path(directory) / "report.json").status, "canceled")
+
+ def test_workflow_source_rejects_untrusted_triggers(self):
+ event = {
+ "workflow_run": {
+ "event": "pull_request",
+ "path": ".github/workflows/benchmarks.yml",
+ "head_repository": {"full_name": "owner/repo"},
+ }
+ }
+ with patch.dict(os.environ, {"GITHUB_REPOSITORY": "owner/repo"}), self.assertRaises(ValueError):
+ workflow_source(event)
+
+ def test_fork_metadata_does_not_confuse_pr_head_with_trusted_harness(self):
+ run = {
+ "event": "pull_request_target",
+ "path": ".github/workflows/benchmarks.yml",
+ "repository": {"full_name": "PrimeIntellect-ai/prime-agent"},
+ "head_repository": {"full_name": "contributor/prime-agent"},
+ "head_sha": HEAD,
+ "id": 100,
+ "run_attempt": 1,
+ "display_title": f"{TITLE}42",
+ }
+ with patch.dict(os.environ, {"GITHUB_REPOSITORY": "PrimeIntellect-ai/prime-agent"}):
+ self.assertEqual(workflow_source({"workflow_run": run}), run)
+ validate_completion(fixture(), fixture(), run)
+ forged = fixture()
+ forged.harness_sha = HEAD
+ with self.assertRaises(ValueError):
+ validate_completion(forged, fixture(), run)
+
+ def test_dispatch_from_a_feature_branch_cannot_publish_or_clean_up(self):
+ run = {
+ "event": "workflow_dispatch",
+ "path": ".github/workflows/benchmarks.yml",
+ "repository": {"full_name": "owner/repo"},
+ "head_branch": "untrusted",
+ }
+ with patch.dict(os.environ, {"GITHUB_REPOSITORY": "owner/repo"}), self.assertRaises(ValueError):
+ workflow_source({"workflow_run": run})
+
+
+class MeasurementTests(unittest.TestCase):
+ def test_disk_footprint_is_measured_after_interactive_first_use(self):
+ order = []
+ terminal = Mock()
+ terminal.ready.return_value = 0.5
+ terminal.close.side_effect = lambda *_args: order.append("closed")
+ side = Side(
+ sha=SHA, metrics={"install": observations(1)}, runtime={"home_before_install_bytes": "100"}
+ )
+ report = fixture()
+ request = Request(
+ repository=report.repository,
+ source_repository=report.repository,
+ sha=SHA,
+ harness_sha=SHA,
+ pr=42,
+ run_id=100,
+ attempt=1,
+ role="main",
+ config=report.config,
+ )
+
+ def disk(_home):
+ self.assertEqual(order, ["closed", "closed"])
+ return 700
+
+ with (
+ patch("worker.Terminal", return_value=terminal) as launch,
+ patch("worker.memory", return_value=[ProcessMemory(pid=1, name="agent", rss=100)]),
+ patch("worker.pwd.getpwnam", return_value=SimpleNamespace(pw_uid=1)),
+ patch("worker.stop_processes"),
+ patch("worker.stop_agents"),
+ patch("worker.write_json"),
+ patch("worker.disk_bytes", side_effect=disk),
+ ):
+ measure(request, side, 0)
+ self.assertEqual(side.metrics["disk"][0].value, 600)
+ self.assertIsNotNone(side.metrics["cold"][0].value)
+ terminal.child.send.assert_not_called()
+ for call in launch.call_args_list:
+ self.assertEqual(call.args[0], ["/usr/sbin/runuser", "-u", "benchmark1", "--", "prime-agent"])
+ self.assertNotIn("PRIME_API_KEY", call.args[2])
+
+ def test_dependency_inventory_warning_preserves_successful_installation(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ home = root / "benchmark1"
+ (home / ".prime/agent/kernel-venv/bin").mkdir(parents=True)
+ (home / ".prime/agent/kernel-venv/bin/python").touch()
+ side = Side(sha=SHA)
+
+ def run_as(_user, command, *_args, **_kwargs):
+ if command[:2] == ["npm", "ls"]:
+ raise subprocess.CalledProcessError(1, command)
+ return "0.0.0-benchmark"
+
+ with (
+ patch("worker.HOMES", root),
+ patch("worker.RESULTS", root),
+ patch("worker.subprocess.run"),
+ patch("worker.run_as", side_effect=run_as),
+ patch("worker.disk_bytes", side_effect=[100, 300]),
+ patch("worker.os.chown"),
+ patch("worker.pwd.getpwnam", return_value=SimpleNamespace(pw_uid=1, pw_gid=1)),
+ patch("worker.stop_processes"),
+ ):
+ install(SimpleNamespace(config=Config.load()), side, 0)
+ self.assertIsNotNone(side.metrics["install"][0].value)
+ self.assertIsNone(side.metrics["install"][0].error)
+ self.assertIn("warnings", side.runtime["dependency_inventory"])
+ self.assertTrue((home / "workspace/example.py").exists())
+ self.assertNotIn("disk", side.metrics)
+
+ def test_child_environment_never_inherits_credentials_or_personal_settings(self):
+ with patch.dict(
+ os.environ,
+ {
+ "PRIME_API_KEY": "fake",
+ "PINFERENCE_API_KEY": "fake",
+ "PRIME_SANDBOX_API_KEY": "fake",
+ "GITHUB_TOKEN": "fake",
+ "PRIME_TEAM_ID": "private-team",
+ "PI_CODING_AGENT_DIR": "/personal",
+ },
+ ):
+ env = environment("benchmark1")
+ self.assertFalse(any("KEY" in name or "TOKEN" in name or "TEAM" in name for name in env))
+ self.assertNotIn("PI_CODING_AGENT_DIR", env)
+
+ def test_completion_requires_every_metric(self):
+ side = Side(sha=SHA)
+ for definition in (*METRICS, *RUNTIME_METRICS):
+ count = 1 if definition.key in ("bundle", "disk") else (3 if definition.key == "install" else 10)
+ side.metrics[definition.key] = observations(*([1] * count))
+ self.assertTrue(side_complete(side, 10, 3))
+ for definition in (*METRICS, *RUNTIME_METRICS):
+ incomplete = side.model_copy(deep=True)
+ incomplete.metrics[definition.key][-1] = Observation(trial=0, error="failed")
+ self.assertFalse(side_complete(incomplete, 10, 3))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/benchmarks/tests/test_kernel.py b/scripts/benchmarks/tests/test_kernel.py
new file mode 100644
index 0000000000..fd78506591
--- /dev/null
+++ b/scripts/benchmarks/tests/test_kernel.py
@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+from kernel import OUTPUT_BYTES, Kernel
+
+FIXTURE = r"""
+import json, os, sys, time
+
+def emit(event):
+ data = (json.dumps(event) + "\n").encode()
+ for index in range(0, len(data), 17):
+ os.write(1, data[index:index + 17])
+
+time.sleep(float(sys.argv[1]))
+emit({"event": "ready", "protocol": int(sys.argv[3]), "python": "fixture"})
+for line in sys.stdin:
+ request = json.loads(line)
+ rid, kind = request.get("id"), request["type"]
+ if kind == "shutdown":
+ break
+ if kind == "interrupt":
+ time.sleep(0.03)
+ emit({"event": "error", "id": rid, "ename": "KeyboardInterrupt"})
+ emit({"event": "done", "id": rid, "status": "error"})
+ continue
+ code = request.get("code", "")
+ if "interrupt-ready" in code:
+ emit({"event": "stdout", "id": rid, "text": "interrupt-ready"})
+ continue
+ if code == "exit":
+ break
+ if code == "invalid":
+ os.write(1, b'not json\n')
+ continue
+ time.sleep(float(sys.argv[2]))
+ if code.startswith("output:"):
+ emit({"event": "stdout", "id": rid, "text": "x" * int(code.split(":")[1])})
+ emit({"event": "done", "id": rid, "status": "error" if code == "fail" else "ok"})
+"""
+
+
+class KernelTests(unittest.TestCase):
+ def setUp(self):
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+ self.script = self.root / "fixture.py"
+ self.script.write_text(FIXTURE)
+ self.count = 0
+
+ def launch(self, startup=0, execution=0, protocol=3):
+ self.count += 1
+ kernel = Kernel(
+ [sys.executable, str(self.script), str(startup), str(execution), str(protocol)],
+ self.root,
+ os.environ.copy(),
+ self.root / f"kernel-{self.count}",
+ )
+ self.addCleanup(kernel.close)
+ return kernel
+
+ def test_real_protocol_distinguishes_startup_from_execution_regressions(self):
+ timings = []
+ for startup, execution in ((0, 0), (0.3, 0.02)):
+ kernel = self.launch(startup, execution)
+ ready, version = kernel.ready()
+ self.assertEqual(version, "fixture")
+ timings.append((ready, kernel.batch("pass", 5)))
+ self.assertGreater(timings[1][0] - timings[0][0], 0.2)
+ self.assertGreater(timings[1][1] - timings[0][1], 0.015)
+
+ def test_complete_output_is_required_even_across_split_frames(self):
+ kernel = self.launch()
+ kernel.ready()
+ self.assertGreater(kernel.batch(f"output:{OUTPUT_BYTES}", 2, output_bytes=OUTPUT_BYTES), 0)
+ with self.assertRaisesRegex(RuntimeError, "truncated or missing"):
+ kernel.batch("output:31", 1, output_bytes=OUTPUT_BYTES)
+
+ def test_failed_execution_and_invalid_protocol_are_not_timings(self):
+ for code, error in (("fail", RuntimeError), ("invalid", ValueError), ("exit", RuntimeError)):
+ with self.subTest(code=code):
+ kernel = self.launch()
+ kernel.ready()
+ with self.assertRaises(error):
+ kernel.batch(code, 1)
+
+ def test_interrupt_waits_for_running_cell_and_acknowledgement(self):
+ kernel = self.launch()
+ kernel.ready()
+ self.assertGreaterEqual(kernel.interrupt(), 0.025)
+ self.assertGreater(kernel.batch("pass", 1), 0)
+
+ def test_ready_version_and_protocol_deadline_are_enforced(self):
+ wrong = self.launch(protocol=2)
+ with self.assertRaisesRegex(ValueError, "protocol 3"):
+ wrong.ready()
+ kernel = self.launch()
+ kernel.ready()
+ with self.assertRaises(TimeoutError):
+ kernel.event(time.perf_counter() + 0.02)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/benchmarks/uv.lock b/scripts/benchmarks/uv.lock
new file mode 100644
index 0000000000..efb86effe0
--- /dev/null
+++ b/scripts/benchmarks/uv.lock
@@ -0,0 +1,494 @@
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[options]
+exclude-newer = "2026-09-02T00:00:00Z"
+
+[[package]]
+name = "aiofiles"
+version = "25.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" },
+]
+
+[[package]]
+name = "annotated-types"
+version = "0.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" },
+]
+
+[[package]]
+name = "anyio"
+version = "4.14.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "idna" },
+ { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
+]
+
+[[package]]
+name = "certifi"
+version = "2026.7.22"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },
+]
+
+[[package]]
+name = "connectrpc"
+version = "0.11.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf-py" },
+ { name = "pyqwest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/b5/63e14ab9d4d4cc58818562db4120a35fa7454e3035633da41bdc6b712abd/connectrpc-0.11.1.tar.gz", hash = "sha256:18277f7838847b4271ca38d40c7d2387b5a2ea6a29f240689c19e1ec84aaff66", size = 46222, upload-time = "2026-07-15T06:33:31.601Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5c/ac/aa647675812cd075f2cb9acb00d62f4f2cbcbc6736049a62342b7371ce5b/connectrpc-0.11.1-py3-none-any.whl", hash = "sha256:8a52e2e92a485fa9681c1101a79a5ebeb31807e3ea3d5aabd41f484a6398bc7b", size = 64991, upload-time = "2026-07-15T06:33:29.396Z" },
+]
+
+[[package]]
+name = "h11"
+version = "0.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
+]
+
+[[package]]
+name = "httpcore"
+version = "1.0.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "certifi" },
+ { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },
+]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+ { name = "certifi" },
+ { name = "httpcore" },
+ { name = "idna" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
+]
+
+[[package]]
+name = "idna"
+version = "3.19"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" },
+]
+
+[[package]]
+name = "opentelemetry-api"
+version = "1.44.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" },
+]
+
+[[package]]
+name = "pexpect"
+version = "4.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "ptyprocess" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" },
+]
+
+[[package]]
+name = "prime-agent-benchmarks"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+ { name = "httpx" },
+ { name = "pexpect" },
+ { name = "prime-sandboxes" },
+ { name = "pydantic" },
+ { name = "pyte" },
+]
+
+[package.dev-dependencies]
+dev = [
+ { name = "ruff" },
+]
+
+[package.metadata]
+requires-dist = [
+ { name = "httpx", specifier = "==0.28.1" },
+ { name = "pexpect", specifier = "==4.9.0" },
+ { name = "prime-sandboxes", specifier = "==0.2.40" },
+ { name = "pydantic", specifier = "==2.13.5" },
+ { name = "pyte", specifier = "==0.8.2" },
+]
+
+[package.metadata.requires-dev]
+dev = [{ name = "ruff", specifier = "==0.16.5" }]
+
+[[package]]
+name = "prime-sandboxes"
+version = "0.2.40"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "aiofiles" },
+ { name = "certifi" },
+ { name = "connectrpc" },
+ { name = "httpx" },
+ { name = "protobuf" },
+ { name = "pydantic" },
+ { name = "pyqwest" },
+ { name = "tenacity" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/64/8c/19e812273ecf633da7b8767b6c6b14b55a4e9753f7c9791349a6ce9e0353/prime_sandboxes-0.2.40.tar.gz", hash = "sha256:4096f8afc4887590028e2b4eb1f424c7db40022aee6784beb2a071e6023e9791", size = 107334, upload-time = "2026-08-27T18:12:08.044Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c6/ba/6a5ce07386b457f0eae76d025b8b0af79bd82b8ce9fc2b9f0772e17a214b/prime_sandboxes-0.2.40-py3-none-any.whl", hash = "sha256:62d3afc9ce8a5b39dc292208ab8ba23779320e91cf2f7bd21564ed20ac6faa75", size = 60144, upload-time = "2026-08-27T18:12:06.818Z" },
+]
+
+[[package]]
+name = "protobuf"
+version = "7.36.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/86/73/f66c748df06e7fe24e658eddd600d19c4b40bad836c97ce2d0ad9851fb6b/protobuf-7.36.1.tar.gz", hash = "sha256:d0f6470f0ce2b84e3feaea2d4b816378b37ba4d4aa08a274305373de93e2d524", size = 512499, upload-time = "2026-08-31T22:40:04.667Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f7/6c/3a54a58f2948b0f485df9ecdd06590f15d0a7abf46a89d50c3de709ff4ff/protobuf-7.36.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:3cf2ee25d006cee57294a1196ea43b37feb78e0dcd1e8af5c1aeddb777655aca", size = 456046, upload-time = "2026-08-31T22:39:56.865Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/08/9f9548793c771095245c0eeaf0c84b76b16a5ef26158043d456f551344a0/protobuf-7.36.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:43d3d37b1eb24c113b9b7d02008cac44e423f00b611b7781ae998d7623972969", size = 344226, upload-time = "2026-08-31T22:39:58.263Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/51/1bdbd612fa3c51e42ea6f45d05e84dc748ddcd2663e1aa1e89a00a33facd/protobuf-7.36.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:39c518c05586c016d7874ff6079ee115bcec1ea5fbb1d177fbf7867ef4c67e44", size = 357229, upload-time = "2026-08-31T22:39:59.198Z" },
+ { url = "https://files.pythonhosted.org/packages/22/df/c799fe7a05ef16ba853a59db01f3a2c5f7d0676469589ccc4874f76a2a88/protobuf-7.36.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:97198b77e369a0abd8e262b8f6c7266c55ddb796a3a12c76d7b8881188ed83aa", size = 343228, upload-time = "2026-08-31T22:40:00.179Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/33/d4724ec5d86d496fe4108e220618aa0837ad3423a7af7ccdd9684ccc77c8/protobuf-7.36.1-cp310-abi3-win32.whl", hash = "sha256:0b53ce95272aad50ad25d7ff03373743209822e8ba42ea7fad27d2bee1547d00", size = 443002, upload-time = "2026-08-31T22:40:01.32Z" },
+ { url = "https://files.pythonhosted.org/packages/db/37/155788a0d8daded960375af604202805308169f9b859419ea0aa370946e2/protobuf-7.36.1-cp310-abi3-win_amd64.whl", hash = "sha256:51139351435d9b43d88a55eaa49fb6f737fbb478fb0cbf2cf694d1a04a9d3363", size = 456518, upload-time = "2026-08-31T22:40:02.494Z" },
+ { url = "https://files.pythonhosted.org/packages/39/ca/c47f91d3cab175b01fd8c4f0d80fdf8613be876cc616e66ad281a59c5ddf/protobuf-7.36.1-py3-none-any.whl", hash = "sha256:7d951e46b3f963d6c264c367c437921de9d5aedd9c3f9612b9077736b4e3ad5c", size = 179813, upload-time = "2026-08-31T22:40:03.54Z" },
+]
+
+[[package]]
+name = "protobuf-py"
+version = "0.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf-py-ext", marker = "(platform_machine == 'arm64' and platform_python_implementation == 'CPython' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'AMD64' and platform_python_implementation == 'CPython' and sys_platform == 'win32') or (platform_machine == 'ARM64' and platform_python_implementation == 'CPython' and sys_platform == 'win32')" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/72/ed/02fd902d9c51b7ff53dfc9a745eb11490722edfd30073af889e171f07b8e/protobuf_py-0.1.1.tar.gz", hash = "sha256:6bd08ac4d8f1661965bbe2685429d79043704cdd1ee720a7a89617331742240b", size = 133525, upload-time = "2026-06-24T19:02:15.271Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fd/00/1b3775aca1c70e3007e06ef5996f6bb9b3a32341eb0cce3ffb6effad8dec/protobuf_py-0.1.1-py3-none-any.whl", hash = "sha256:efc4f50f275ed6dae10a1f30bb81ad1a75368557b3ff22a532b7a472050368f1", size = 181656, upload-time = "2026-06-24T19:01:29.556Z" },
+]
+
+[[package]]
+name = "protobuf-py-ext"
+version = "0.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2f/05/6dc9ccff1e8159eb9a144e6d3c4acfd2211cd4fcd20c34fe9155d17d6a7f/protobuf_py_ext-0.1.1.tar.gz", hash = "sha256:e85bfdfdb3ed50634db8ccc7429dd9286520109489c735463971a418707b4fef", size = 31912, upload-time = "2026-06-24T19:02:16.321Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/19/70/2b5d62a60a2e0d88ecde1ae98db3132bcd2672fb39c8d581b82b34ac0bd8/protobuf_py_ext-0.1.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:310039e03cb15181781a0b78017419f6d4ee302e988c3c70b87f1facdf05532d", size = 306008, upload-time = "2026-06-24T19:01:31.399Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/d6/73c06bb4cac2e04c0adc154965fe8b6520224bd737fd7e73bba405056321/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89ec8348d1ba045f79b2fedd14e40cca36f2a41b52f2c4fdf55a60c58add2353", size = 314769, upload-time = "2026-06-24T19:01:32.89Z" },
+ { url = "https://files.pythonhosted.org/packages/18/c5/e4e6bc6096b66d1c82639a1b501147f16ee65d32567304b987d002e2c666/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:182798e4861aba72d05855bd06febe4926aa7265e6f444a1b8af5252beee4f7e", size = 328122, upload-time = "2026-06-24T19:01:34.394Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/4b/1d792a40d0f0a0f914f1dfa8bb5e9573ca0ecd5fe5cecf80d77193212abd/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9eb25c3a329c0551cc86b209a5e5d8ecb8d834b9924a3aa019377853a703b6d3", size = 492338, upload-time = "2026-06-24T19:01:36.103Z" },
+ { url = "https://files.pythonhosted.org/packages/52/51/5ad329223c905e5c530cae38ae24cde3584d8ab7e09457f2a01afce80e60/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aaecbde82bef10c7c40578cbb61b7a19896bf7fa450972050a3bb302acb7d5d6", size = 541578, upload-time = "2026-06-24T19:01:37.481Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/c7/01bd8a8bfe2df4b07aafac12ccfb7b7ebe2303f65296a9e02fcafa28ff3e/protobuf_py_ext-0.1.1-cp310-abi3-win_amd64.whl", hash = "sha256:6b0c615c48e95acc53cf33e9310eeaff8b30d2d7555bf93e7bca8fb4f40e9a5c", size = 251319, upload-time = "2026-06-24T19:01:38.946Z" },
+ { url = "https://files.pythonhosted.org/packages/24/dc/914065538b5db54b6a920b5af38c1ef142252ea1eea711820435fa259fac/protobuf_py_ext-0.1.1-cp310-abi3-win_arm64.whl", hash = "sha256:72956cd0af5dee24b41c6f5ba5e42622d17e6d555002b5efc1634e27a1446de2", size = 241635, upload-time = "2026-06-24T19:01:40.301Z" },
+ { url = "https://files.pythonhosted.org/packages/13/a5/0b5d73cab815fd50615cb87a02e04e8332f9e27380c890aba1459cf7e919/protobuf_py_ext-0.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6f0cd58620f415a3534d195358338f4999a774e12510f91c592b38d13d37388", size = 304903, upload-time = "2026-06-24T19:01:41.796Z" },
+ { url = "https://files.pythonhosted.org/packages/37/a9/14c120b9ac36a0e11bb05e482fa6ae322de583a8e1068e4859035c289947/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21572764f625d829604fc4d83635f533f35775f11c6da142345b3f3c8d64bd09", size = 316148, upload-time = "2026-06-24T19:01:43.247Z" },
+ { url = "https://files.pythonhosted.org/packages/35/54/7c00a1a9783c3ba74f6b2f5d2f049f09037bbd489c465c09a34f32fb611b/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f14f00c2678bfbe7ad46f057b9f9938c1677bcf39e405e163e272c6f9814b8", size = 326458, upload-time = "2026-06-24T19:01:44.655Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/0e/81fa50c0d6c4d664e5be6d0a3c4f2c7edfef87ab12d0bac764abca1f2955/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1610865622e2e27568277ca63d8d2d23dcc55eaa767398865c21539ddf4ce24d", size = 493596, upload-time = "2026-06-24T19:01:46.344Z" },
+ { url = "https://files.pythonhosted.org/packages/56/19/3183cf6e4de62846c20549654a1d573dae51ca06aaf7fed06accdfab74f6/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8995efba9476e1ea18ef9873306871dba697308994bc2766558fec3387acc3de", size = 539656, upload-time = "2026-06-24T19:01:48.21Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/6c/09841f3dbe7c3d4b3f2779f8f2832ac23fb96ac8ab71674e91af732cf9ff/protobuf_py_ext-0.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba61d49dace8f874361583030a7c48139b42eb37c9ffbb1e7e8a227a51576f44", size = 305017, upload-time = "2026-06-24T19:01:49.84Z" },
+ { url = "https://files.pythonhosted.org/packages/52/43/e450b5e202f6715274acc9acd9f9769908cbdeab861a53c798fcbd467d35/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a9c2f026096ca4c595c89a297067ef371e241d3ff8e1f6d7c779aa8419cdca7", size = 316215, upload-time = "2026-06-24T19:01:51.249Z" },
+ { url = "https://files.pythonhosted.org/packages/97/5c/23e79b35f1b5755b9bdc0c0c9b8cd8abababaef1a1639608d8a96bb61a9d/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf78707a040b9294e5e1ec4a1875f0046acfe52e92150cea27de4e9fc9db39bb", size = 326419, upload-time = "2026-06-24T19:01:52.93Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/de/5493de0ec12920a3b1dd72608ce0c1e8cdb7e0eb9e87accaecc5216df29e/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae4373845cbb85bdade3ef5368cc2f4b5f80bf173383afc1ab063f95644e5599", size = 493734, upload-time = "2026-06-24T19:01:54.301Z" },
+ { url = "https://files.pythonhosted.org/packages/98/89/31da55b414e6332aad11d858e9a86bd36fbb324f52fa3fc6d8f1c57840a9/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2a4cc478eef7a2acc1daaebcde479a3ac2396d47e6bdc7e776ca4c4147ba8b4c", size = 539703, upload-time = "2026-06-24T19:01:55.707Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/71/39f231838ef06476d46ac40dd814894277a732a3688c0a0c994850b3b62f/protobuf_py_ext-0.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:359dccdc1c3eafed2a913c570bb082b8df848a1d27d548ce8385f246a7d68be2", size = 302145, upload-time = "2026-06-24T19:01:57.116Z" },
+ { url = "https://files.pythonhosted.org/packages/87/f1/01c81ff5f420600366a0e6a613ce2af20834534d2b3d7523f4f3b4ddb54f/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91c38b4a10a306366443273ee03ca554537a0965bf05b7ada8e7dbdee04cc93e", size = 314541, upload-time = "2026-06-24T19:01:58.745Z" },
+ { url = "https://files.pythonhosted.org/packages/86/d8/1cbf5a0298ceddc0cdbb28bf1ecaf8bd3cff54ed4ced6a1392d5226172fc/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79eee3bcbb289d6ea114eb8fe3a1469c5b59bf53e207238469f567d9c53ba56f", size = 325092, upload-time = "2026-06-24T19:02:00.382Z" },
+ { url = "https://files.pythonhosted.org/packages/af/23/8b1694616044cbfec2d18d4c6fffb03e05089c8bdb5970c6bafd60cc7fd5/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:10992141a8282a71ac3e3530d1a489efb27618d84000b9a47918cf70e5816d9b", size = 491684, upload-time = "2026-06-24T19:02:01.862Z" },
+ { url = "https://files.pythonhosted.org/packages/49/d8/99997a7a6cf6989c944d9183c5deb6a045c27460add0316c7b34763891b3/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4411ffe0e06a774b83c5c71c546ce097640a25f596c45f95f53d3e3148e3f22d", size = 538048, upload-time = "2026-06-24T19:02:03.362Z" },
+ { url = "https://files.pythonhosted.org/packages/15/3c/9e99ecbb68e1d5b46016d821eb1cbba9a91e6b50e71e75faf0c1e6189f0d/protobuf_py_ext-0.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55a17d80ea419501ff221a6627523f38a431bb33e6aa3de81ae3a7f271c49c75", size = 296337, upload-time = "2026-06-24T19:02:04.787Z" },
+ { url = "https://files.pythonhosted.org/packages/af/f2/305338a28225fb54b28d6c3f5948109b9088b329a2b6f77ca610c2bdcd34/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbb518b5638403ceaa08ac4fc7dac626f45ed9b856b3517de3906cf3de4d632", size = 308961, upload-time = "2026-06-24T19:02:06.185Z" },
+ { url = "https://files.pythonhosted.org/packages/60/f9/416103c93677ff2ea407704ea64fa6de9e700dc030c48360a182d91ce373/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4adc65ab6a5e4885c67fc808bcacb83d755d05b566d312a0e10c2a873f2ad4", size = 321895, upload-time = "2026-06-24T19:02:07.714Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/83/eb3e81bb2f83834b7deff4cee2d56eeb1adcbc847492a56b7f910ab257db/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e97f45c49676efacfb8e95bfcb1a002bc337e618a6780be1e55df2ba5ebd2f1e", size = 486091, upload-time = "2026-06-24T19:02:09.243Z" },
+ { url = "https://files.pythonhosted.org/packages/99/96/bd88fca38556b3105e4dd41d6a176e31dcc583fe979e830aeedd2f8c20ee/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:53a6b3590f6aa7f97b8ed60f62fef9b096babfeae82f7283fd3a4c405827d4f8", size = 534582, upload-time = "2026-06-24T19:02:11.227Z" },
+]
+
+[[package]]
+name = "ptyprocess"
+version = "0.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" },
+]
+
+[[package]]
+name = "pydantic"
+version = "2.13.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "annotated-types" },
+ { name = "pydantic-core" },
+ { name = "typing-extensions" },
+ { name = "typing-inspection" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" },
+]
+
+[[package]]
+name = "pydantic-core"
+version = "2.46.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a2/b6/81d2d19ea0be2c03664381b59f65fa72fc7969decedae00bc2c4ad835708/pydantic_core-2.46.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f", size = 2074737, upload-time = "2026-08-28T09:57:57.711Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/18/b70da8300e292df4099684ea11b1958043580d2f50d2dc8bf7e542bdd84a/pydantic_core-2.46.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f", size = 1921751, upload-time = "2026-08-28T09:57:59.265Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/1a/0d590341b6ffa4b4aca83508e6b8db4761aaeacfc15a25ca3815876d4797/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061", size = 1948231, upload-time = "2026-08-28T09:58:00.678Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/1d/02eb35761c51f2f7b1b042d6ab4cda6600f0c8c88a2243b3f734376201e5/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be", size = 2020708, upload-time = "2026-08-28T09:58:02.267Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/ea/f86073830e35d508cc8ddf9c3d9e6e6840fcb88d34bf726b0b4710186f27/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a", size = 2194914, upload-time = "2026-08-28T09:58:03.934Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/d7/fc36240d7791ce90939e51608568c33bfdae26202016f9770c229a487d86/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b", size = 2235622, upload-time = "2026-08-28T09:58:05.516Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/bc/3fa2d76b83162820a17da7f645b28d1cba99fc8e1e5fc6517067ec450fa1/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c", size = 2062091, upload-time = "2026-08-28T09:58:07.135Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/9a/095d557bb492c90cd8a70a6dd048bf793d433d03d86c81c11e912e4cd049/pydantic_core-2.46.5-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee", size = 2089904, upload-time = "2026-08-28T09:58:08.814Z" },
+ { url = "https://files.pythonhosted.org/packages/24/98/7b76b1ad10a19a617a52aaa1d80e159115af939b095e86f8e756fd52e0df/pydantic_core-2.46.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e", size = 2132244, upload-time = "2026-08-28T09:58:10.435Z" },
+ { url = "https://files.pythonhosted.org/packages/20/32/7d6ca365fadba186a0c8f85de1a701663bce81efd309d9479be58687622f/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2", size = 2143901, upload-time = "2026-08-28T09:58:12.033Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/09/eb9a6aa57f22fd1541a9c0aa2a1f3aeef3ec65347d33e10a6da2f43e0ee9/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689", size = 2299425, upload-time = "2026-08-28T09:58:13.614Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/f9/548a5bb9d4ba8cd26e26daf48052236f6b38bb61e7b7241fbc3c995719eb/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec", size = 2318566, upload-time = "2026-08-28T09:58:15.199Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/20/06454d18834c02c406c9133f1a3b485305fd9ee984f9636c2f730bef6a9d/pydantic_core-2.46.5-cp311-cp311-win32.whl", hash = "sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129", size = 1954258, upload-time = "2026-08-28T09:58:16.813Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/c2/718b9deb4b72453b5d8c7447a3b14cb77bef36917ef5f514e0948a4096a0/pydantic_core-2.46.5-cp311-cp311-win_amd64.whl", hash = "sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c", size = 2041030, upload-time = "2026-08-28T09:58:18.288Z" },
+ { url = "https://files.pythonhosted.org/packages/67/ea/c1d1a5b72d6e1ff7f377a4d9199f6591f095beb5b409a8a5d89f7238d939/pydantic_core-2.46.5-cp311-cp311-win_arm64.whl", hash = "sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8", size = 2009234, upload-time = "2026-08-28T09:58:19.929Z" },
+ { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" },
+ { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" },
+ { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" },
+ { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" },
+ { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" },
+ { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" },
+ { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" },
+ { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" },
+ { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" },
+ { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" },
+ { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" },
+ { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" },
+ { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" },
+ { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" },
+ { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" },
+ { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" },
+ { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" },
+ { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" },
+ { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" },
+ { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" },
+ { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" },
+ { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" },
+ { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" },
+ { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" },
+ { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" },
+ { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" },
+ { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" },
+ { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" },
+ { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" },
+ { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" },
+ { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" },
+ { url = "https://files.pythonhosted.org/packages/af/1e/ecca01fce348f7e8afa9572441ff6f7d1cc70d21e4859f33944d10877e1e/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2", size = 2075342, upload-time = "2026-08-28T10:00:51.353Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/4c/af80c7a8032dfc897040ad5cb772bebde529a381186499e6e29987f23f8c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c", size = 1907219, upload-time = "2026-08-28T10:00:53.438Z" },
+ { url = "https://files.pythonhosted.org/packages/be/3e/54d89e2b092e778716bf6153634ef479e955f48c261090be23aa1e0fb0b5/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47", size = 1953393, upload-time = "2026-08-28T10:00:55.58Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/89/828ee90cda28ce17bdefaa3a6eaf74fe430e113295a10e6126beca559d6c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a", size = 2099024, upload-time = "2026-08-28T10:00:57.794Z" },
+ { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" },
+ { url = "https://files.pythonhosted.org/packages/20/21/22102e9950b3049526d20e811b95396508377d87651edd2b80d2b3d28659/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f", size = 2071333, upload-time = "2026-08-28T10:01:09.636Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/18/87aefa427d191e6d3ab1447f1efc1cdcac86af1069239b133e8a0fd7f7c9/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0", size = 1912713, upload-time = "2026-08-28T10:01:12.285Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/93/fd89e9ad49b1805ca94d24ce1088b7d305f05c35ffafcedb9819d03588a0/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4", size = 2090926, upload-time = "2026-08-28T10:01:15.19Z" },
+ { url = "https://files.pythonhosted.org/packages/6f/45/8e59dab6acf8d35f02f0a958980074f31038968bdb2c983fcae9d1efee03/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25", size = 2131303, upload-time = "2026-08-28T10:01:17.937Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/a5/e1d4dc5180dd887a9522efc1f8716b8692b7606b1d3273d7862eaf66be44/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6", size = 2145128, upload-time = "2026-08-28T10:01:20.694Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/d7/ad493864a7fb21c0c4df98f965e2db430cb25a9d7369b5778d5016c09fd9/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e", size = 2294560, upload-time = "2026-08-28T10:01:23.495Z" },
+ { url = "https://files.pythonhosted.org/packages/02/8e/b41c84c913f29973a268e6c2b5bbf13c95adb9956c126d10da11ba3b2bef/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda", size = 2317531, upload-time = "2026-08-28T10:01:26.334Z" },
+ { url = "https://files.pythonhosted.org/packages/db/1d/068464f23075f66a8f1b806935e9cd9363ee446636ea70d2c22ee8659dbf/pydantic_core-2.46.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266", size = 2140686, upload-time = "2026-08-28T10:01:28.947Z" },
+]
+
+[[package]]
+name = "pyqwest"
+version = "0.10.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-api" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/07/ee/0ff9facfa9e7a4f6df2a770d4eaf1ad0f74165da7e8c28e888461f07604c/pyqwest-0.10.0.tar.gz", hash = "sha256:6c1a693be17d57d2c2eca4085e32c2809c53090c16719a907c90ebcf1f40dc01", size = 482248, upload-time = "2026-08-21T06:09:20.656Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/43/ee/b1a28f57c689606cfd065d8a553841150f7daaa91d20e58dcc2c5ea191f8/pyqwest-0.10.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:aa492d5777dd145a60795ed95d9d4707a3cd1091fdcdfc93a82ac7fdc43ebacd", size = 5261059, upload-time = "2026-08-21T06:08:04.999Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/13/9c5046cfd6ef705bde0b620ba8a794335bcabc0839342a2a647f2427b27e/pyqwest-0.10.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:59f3f16628e518c674102e7b5fcff2101bba6abb4f6737ec5fade9b9278e6a53", size = 5134207, upload-time = "2026-08-21T06:08:06.955Z" },
+ { url = "https://files.pythonhosted.org/packages/93/7d/50021dd88d82d6966ab1c27593ceaee9d1ed62fbe597c40e8dc187cfa5fd/pyqwest-0.10.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6e7db305a8318b1f3218053e87501f8f245ca8bd63e948e0282d04bf0883470", size = 5640730, upload-time = "2026-08-21T06:08:09.135Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/3f/5bf6c32e9e701837a8c47ce6e3ad38978cfec8eb7bc6596181e5f9e1eaeb/pyqwest-0.10.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5c757cfac5f53c8671dcb4850d5fc4c4339ea3e90636331c9318f8e3ddabc06", size = 5561462, upload-time = "2026-08-21T06:08:10.836Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/61/ca9ba5721461b7ce5cfaac373ab3a1723ddcc434af7430f8bd628da6e623/pyqwest-0.10.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:234b3f71e3f314d997c203d8cf829b7117edd041153f9c277d0060ab90134148", size = 5801847, upload-time = "2026-08-21T06:08:12.502Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/44/95593919b996a417093f598d887822b9b899e8d025588c9bfaf8c60dd812/pyqwest-0.10.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5637256a0dac0ef57e0eaa02b032014965e4a4c995e1deca1b1b97e6d1765f78", size = 5978692, upload-time = "2026-08-21T06:08:14.383Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/b9/b2821ce5188457168ebb25d5ff65b1ca1bf27bc6b4a33df4bcc2357e625c/pyqwest-0.10.0-cp310-abi3-win_amd64.whl", hash = "sha256:7ea761937acf3a00d1a7e70e982949d18946e5471d1419266ab3a78bbfa19759", size = 4876627, upload-time = "2026-08-21T06:08:16.084Z" },
+ { url = "https://files.pythonhosted.org/packages/86/b4/16ccef1c203fa258ce46a86aefc1a79c13b5f0b8d49627347d90eef25efd/pyqwest-0.10.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a21f1f15252a8303623b4f17b9c6de595ace11b3ade07f2adb6d07121e8191aa", size = 5274815, upload-time = "2026-08-21T06:08:17.777Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/61/6a87f84f571441ea43279587d4bfcad4543505918ae2b83a1ebdcfa98be5/pyqwest-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eb472c6e5d6833ebfec79db310e426eb17b01ac64c0e2c251bd9192c0d2ee0c5", size = 5123656, upload-time = "2026-08-21T06:08:19.501Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/f2/ab69e581cf9b798b0e169f7b27fd3f8b6f9f1631bd4d3b6e22e5abaf8d8b/pyqwest-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83578e24cccd5e0dc04d60a0af7bfb43325b5f22d03ff74ff79ed0ecf553b50d", size = 5641253, upload-time = "2026-08-21T06:08:21.627Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/dd/f1a62eebf8321ace506bd94551a01431f6a45b882225455eae3ea6e8c6d1/pyqwest-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bb511c434f79c641efb5573e5795e56dc972252f4b96e52a9636d4ece5231a4", size = 5567341, upload-time = "2026-08-21T06:08:23.346Z" },
+ { url = "https://files.pythonhosted.org/packages/4c/48/51d767691973e046887f5e6d96e32142fe296823b163ccba732233a6ef72/pyqwest-0.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1aaccd8a9db9430b2aedb5bad8ead80742cbc056b85c229516c70dc80539f906", size = 5803874, upload-time = "2026-08-21T06:08:25.096Z" },
+ { url = "https://files.pythonhosted.org/packages/58/0a/d2834ccc6e59ad110718895cc65ff2a68aa6e010f0ba8fbe42a57ea33c21/pyqwest-0.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:73d9eb438ab4a957a1ce0619d3af8c1c1126bfb9181033b123d792fcf4224531", size = 5981518, upload-time = "2026-08-21T06:08:27.594Z" },
+ { url = "https://files.pythonhosted.org/packages/75/f3/274b4c268e9a55fbdb1b3637ac50b5bf42cd3a85d1cfbdc15c602a7b0d9c/pyqwest-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:317a74d633abe3bc5bccabf479e069c515dab9e6a755274b0ccb1d8a5bbfede3", size = 4870638, upload-time = "2026-08-21T06:08:29.421Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/3d/8b1092f25159bf61a9470ebd35438c669b91ef553a7ee205bdec8006107b/pyqwest-0.10.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3978e794b9cfd8eaa500fb5d7aee63bc6172c605efa0abc1f62d85485bc049e1", size = 5273599, upload-time = "2026-08-21T06:08:31.107Z" },
+ { url = "https://files.pythonhosted.org/packages/75/10/54a9786123942b124c2afb9562b74e158afec7be40ef0caa0d37f615d379/pyqwest-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:715991fd4f04862cd7a9d7452daabcdbd74dff4dff55eb20c22d60382dc2a4ed", size = 5122920, upload-time = "2026-08-21T06:08:33.025Z" },
+ { url = "https://files.pythonhosted.org/packages/a2/5b/6a6bd76f91e068b9a619f62aef9fe5ef201f859ddbb6b0a11ad3875ecdda/pyqwest-0.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c04798bed79c1dfa0e5b0e30fb137124311083490d44d6dfbe068d3dd254349e", size = 5639577, upload-time = "2026-08-21T06:08:34.896Z" },
+ { url = "https://files.pythonhosted.org/packages/68/28/12277a24a8dd74b0a7f124c624d9ed58eccb41e2087ff1087a14d348c778/pyqwest-0.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35b472877e73dd63fed089c2bc8fa198407f005c8c19e0a93f025ebefde01a81", size = 5565835, upload-time = "2026-08-21T06:08:36.567Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/b6/b7da4a3f1fe43600154ae75e91ba7969024d886d60077dd3a1ba8e66d170/pyqwest-0.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:564ec360b7848b35e009038ffbca00466305a9708ab21829477f64aa8cad4c64", size = 5803078, upload-time = "2026-08-21T06:08:38.362Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/6f/0c9ba210f49f232289afaa8f06369c5f135ac786e1ca0cb22243b7f1fe2c/pyqwest-0.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5c80e88a5967c1cadb3237c450f91a84a3683f8838c8dca96f09fee3612e762", size = 5980431, upload-time = "2026-08-21T06:08:39.967Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/f5/306eeed41a3cd3100247e6e442f4345277b70f1d24efe1641181b14839cd/pyqwest-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc3d80b402fb59dbe015e25993ac8147456fb231a4c949f92a89f31315ad50f9", size = 4870198, upload-time = "2026-08-21T06:08:41.687Z" },
+ { url = "https://files.pythonhosted.org/packages/64/18/0086a408e7cbf39dab18fa5b7e42c969a98382da5a4e6debe40f05acc6a1/pyqwest-0.10.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:23a28beb55fa6d975949bffae4adfb69378f3229bb5cbd71231e95bf66f5b26c", size = 5274542, upload-time = "2026-08-21T06:08:43.323Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/7b/e1b9aaaf7596e4faaa53cefc2efaca4e3cde721e308e6385e366361cfdcc/pyqwest-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e4415ae40b8eedb1713dab14d7f9fecc3f79d26f3206c561087b88b99d5ce24b", size = 5127922, upload-time = "2026-08-21T06:08:45.116Z" },
+ { url = "https://files.pythonhosted.org/packages/75/cd/e6d68bb1de5dd26100fcfc878cbd67c402a928774edf1e8ae304c5a84f5b/pyqwest-0.10.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14b875d2273212d7fa8e4b755d8d736ffd226b1c707a9c0017dfdc8393a96eca", size = 5645856, upload-time = "2026-08-21T06:08:47.055Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/48/8c9f9f0467c41f6a563146d57a52f8f6d60c0cb09d0fd3ef88ad5a1c442f/pyqwest-0.10.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5071491e416ea54e3b95bf9ffbed0bd065b093cb96e10a75c3d8f2cbe3c9823", size = 5569831, upload-time = "2026-08-21T06:08:48.78Z" },
+ { url = "https://files.pythonhosted.org/packages/16/ac/c85ab70c6c72078d49a82da76b820e46aaf95a3f6fe271dac955ac195d21/pyqwest-0.10.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b68b5e68d513a4c63a072f8f40e38015160cf90bfbf7e8ef7c3935ca87e9e022", size = 5806735, upload-time = "2026-08-21T06:08:50.652Z" },
+ { url = "https://files.pythonhosted.org/packages/66/db/ad7375b22fb2d0807431dcc9bc2aaf840e374c298cd15071024d5f6dd6d1/pyqwest-0.10.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c48910d27820b9c46fcd001b0fe514a3cf47d4784f59512dcdb8c91c395f82e4", size = 5984786, upload-time = "2026-08-21T06:08:52.377Z" },
+ { url = "https://files.pythonhosted.org/packages/17/88/c449a772afe129683fd7acc657cbc7c69bec085dc75b6e8710a50fbb44e7/pyqwest-0.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:d03ba2cd17948b623a6210981d342eb122546d8a8e910ec77511aff4b1acdd00", size = 4872288, upload-time = "2026-08-21T06:08:54.101Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/f8/439ffc0ee12cd7d9b57ac07ccea78ad3ba66b0d6817d429dd661d73308c4/pyqwest-0.10.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:07a0eb595f4096232c2d22549b6e4612c1ecada7934e46462c2c37ce14a89cfb", size = 5256823, upload-time = "2026-08-21T06:08:55.681Z" },
+ { url = "https://files.pythonhosted.org/packages/df/2b/72ecd27d104d2b4284710194cce796d607674966c3ea66436768e812a66a/pyqwest-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:26401baf7dafc71c8d12d2e8389519d141e6f7c14094d0dd4cf9ec1d3b5555bd", size = 5112624, upload-time = "2026-08-21T06:08:57.366Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/24/0fb89c3f7d5a0410fcb7560588b4c741ef24b19195c307d4263f04e75c2b/pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e3c436e041d8873ce5bb0fdcf9f9e86f5604e8f0ef9e03149efebd8cb474f6", size = 5631844, upload-time = "2026-08-21T06:08:59.165Z" },
+ { url = "https://files.pythonhosted.org/packages/4c/4f/921d14754a186f0143ad62b50108dd808328e347e98e9dafab3897eeb405/pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09364115761579eabfc79d1e954cdb3ded508dac1903fac7285d4c6f058c683f", size = 5555869, upload-time = "2026-08-21T06:09:00.783Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/70/504780417319a626fe9549a7e6f9020a3d448eddf8a09617238a3426e90c/pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559674a98a8b1217e1830ecd41c9905bf2b60983c6b8017063dfac199f00727c", size = 5793738, upload-time = "2026-08-21T06:09:03.324Z" },
+ { url = "https://files.pythonhosted.org/packages/45/f7/8d0a5b8a3289f4300dc9005ebde75d316f2371a2671617f942036337731a/pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f399a696392fff3db3eef0a18ef65b8a3b8396d129193487d966b8eb11006376", size = 5972889, upload-time = "2026-08-21T06:09:05.2Z" },
+ { url = "https://files.pythonhosted.org/packages/89/c4/f4c781e475c451cb5f4762a2f814a1750ef375b8db4db09bb2dcea03c4e5/pyqwest-0.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0f9163d6dd991bf1bf27308ba38ba021af660b15fffa47ebca98e41cf6f00309", size = 4858036, upload-time = "2026-08-21T06:09:06.926Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/6f/29f605665f33aab894db8daf11b3b64bdca015fb11135815c53a150191a6/pyqwest-0.10.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cfcc7ba0229baa17831582befb046ace167b368140dae022d0b89b8d586ba12c", size = 5264903, upload-time = "2026-08-21T06:09:09.183Z" },
+ { url = "https://files.pythonhosted.org/packages/c8/c8/4ce4b40f21397a6482da910fae18b32c28dcdaddf3498aef2fe38597a9e0/pyqwest-0.10.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eff9ccf427604d34c635954def07b6113d4754f968073eef2df40bdb80b05bf5", size = 5142291, upload-time = "2026-08-21T06:09:10.943Z" },
+ { url = "https://files.pythonhosted.org/packages/41/34/7205db9cddc5a2286da459ca013af629e20f09498751c60093b952379add/pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78662158093f9d5c742368f4dd9956aa595f44c6ac0860777c98853ddd5e1610", size = 5648118, upload-time = "2026-08-21T06:09:12.616Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/94/75c6cd5d01cf68c99c0cfd66b165dce3fd9d9fce97cd4973117cab68355d/pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0879a3f0b37876372aa328b9fee956165db174c1ab72464146fe515f49399ddb", size = 5565620, upload-time = "2026-08-21T06:09:14.223Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/a6/20a910d2f096908cf25ec47fa84b35b1bbb1af63dc9713f53f38ed97802e/pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:850b8de6ade09a60bdb2f969871a177c2c304b594b2034ba3f5962c7bea75551", size = 5809262, upload-time = "2026-08-21T06:09:15.897Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/f7/f92fe4004d93f08c5118e750feefdbc72a73438b7fd7083300728e999911/pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:399802647ea646c6ac9b5460e541b7c209b7a13563c667b2690ded2060185f2e", size = 5985550, upload-time = "2026-08-21T06:09:17.579Z" },
+ { url = "https://files.pythonhosted.org/packages/47/3e/2c896e54dbe3f1ba6e3bd10e9d412f422a2e33827ff591d59f67b13e0fa5/pyqwest-0.10.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c26f3de1feb5d066d7a66802a47407a93ba043696064ad80beda4a0a4bf10056", size = 4873529, upload-time = "2026-08-21T06:09:19.161Z" },
+]
+
+[[package]]
+name = "pyte"
+version = "0.8.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "wcwidth" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ab/ab/b599762933eba04de7dc5b31ae083112a6c9a9db15b01d3109ad797559d9/pyte-0.8.2.tar.gz", hash = "sha256:5af970e843fa96a97149d64e170c984721f20e52227a2f57f0a54207f08f083f", size = 92301, upload-time = "2023-11-12T09:33:43.217Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/59/d0/bb522283b90853afbf506cd5b71c650cf708829914efd0003d615cf426cd/pyte-0.8.2-py3-none-any.whl", hash = "sha256:85db42a35798a5aafa96ac4d8da78b090b2c933248819157fc0e6f78876a0135", size = 31627, upload-time = "2023-11-12T09:33:41.096Z" },
+]
+
+[[package]]
+name = "ruff"
+version = "0.16.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f3/85/c8e12473c93018f92d19dd988a294202e1c27426c47ec4de53ffb847b8d8/ruff-0.16.5.tar.gz", hash = "sha256:1b88500f9ffbcab3dedb0082c9f9492e91ec3d618aac1236a3e0189938f7040b", size = 4912003, upload-time = "2026-08-27T16:34:18.258Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c6/b6/77c90a970fe2dae17a723acbd011043ea97c98d7deacccefdc4ba74ec512/ruff-0.16.5-py3-none-linux_armv6l.whl", hash = "sha256:12e5f673e774c35fbb62f288809c7653b73445f8ecec6b6063fd6ea3521aa14b", size = 10011941, upload-time = "2026-08-27T16:33:41.287Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/46/6cf67cf6411885a1d6f7f6d801682f155536a85176d10b605e2ceffed8bd/ruff-0.16.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:eda58a5802de40e7ed5b32b64e0b32539338cc6fcd2c78f61e3ad6a0d79f51c3", size = 10204049, upload-time = "2026-08-27T16:33:44.056Z" },
+ { url = "https://files.pythonhosted.org/packages/46/fd/c8720ca7a090abf0c2fef4abe8a5ef6e5127ed15196d8886ff75a2b370e2/ruff-0.16.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5ae9a7b9a8875131f40f8fe967cc86abf899779efd663cb7ce3d572d01da7eb", size = 9809037, upload-time = "2026-08-27T16:33:46.257Z" },
+ { url = "https://files.pythonhosted.org/packages/43/45/a684caacdedaca180f52bacccc40bf0789d2c5a7c75f25324853e9eaedb5/ruff-0.16.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b719b0a1f4d59710d283ab2965f621684a108a9e41da622e3b23f0326cd0025", size = 9964129, upload-time = "2026-08-27T16:33:48.352Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/f2/5d2bcdaca6b5b93d1b4dfc166cd2aebf7680143a1b38a28759df13a94d31/ruff-0.16.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2298f2780ed1be0c5cb1361e32ab7b1467f3cce7dabe101d2210a314f2fe42e9", size = 9821518, upload-time = "2026-08-27T16:33:50.57Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/ff/011cce29accf9257d5974145b733fc653a37985ed6825413a3987cefbfe0/ruff-0.16.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:258f29035a2dd021e7861e631b227a5b3f14e50c1184c9a6a122c5f4576154d7", size = 10534835, upload-time = "2026-08-27T16:33:52.522Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/5a/f0cf109bada9bba0e96c90c21c9f9251803f57225c32d293327a03c710d6/ruff-0.16.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9a4f0432966834019c74d1b7e5c51224305d7713f3d7faf3e7451f1a3be3cde", size = 11252550, upload-time = "2026-08-27T16:33:54.521Z" },
+ { url = "https://files.pythonhosted.org/packages/63/4d/1d481aaea2046c6a7ed7c291f9004c669cce3c087b6b376ed5b08271e3fe/ruff-0.16.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5eb3a8c3d0ade9cea42b591fd530368e8798380e30e0a308b85a5cf718f09ea", size = 10777949, upload-time = "2026-08-27T16:33:56.88Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/34/ee245ca55f64443233034b3d02b03236b19242004281247c079390b7facd/ruff-0.16.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0f69e191a13a3c9816f63163c88790cb12cd157bbbb384e9c44745702ab105", size = 10311656, upload-time = "2026-08-27T16:33:59.12Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/4d/c33a333e341c0a2b96c715b52d89a606f5a34cd4ac493cd9b8d0187186b8/ruff-0.16.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0eeab41fbea2c42f98dfb9822cdccda9d24ba38d49f6dc945b5c236d48f0ef29", size = 10532125, upload-time = "2026-08-27T16:34:01.166Z" },
+ { url = "https://files.pythonhosted.org/packages/30/e1/a64cef78b40192497bb98a27a8aa8f2c98ee9ee15bc97f7712d94ef32937/ruff-0.16.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f0768e9df4300713fff30733c87575f68b6f1d8de41184e505b7fdd9c0c95eaf", size = 10097648, upload-time = "2026-08-27T16:34:03.16Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/4e/4cdc9ed3c3e109d2f71e62572a37457298d7bc7501ec3138babb7ed32bbd/ruff-0.16.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:95cc70cdc7aa80c338de356279d2adbeb2de0f520b9ecd8aba75b94e95e02f91", size = 9829344, upload-time = "2026-08-27T16:34:05.134Z" },
+ { url = "https://files.pythonhosted.org/packages/39/4a/31ed35ce31729955fc583ee0d176d6e784c1290cb0b0a75cb2134c1ab72a/ruff-0.16.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d185c8398ded1bfd91c0c2cb258346307571eccc473a8490af8c3977399c384a", size = 10277117, upload-time = "2026-08-27T16:34:07.425Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/a0/60356d86687b4b666d593df213f4dc3041750d024cb7bf2cfa81cfd65c2e/ruff-0.16.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fb8e3a3c4c6a784150a7ced53b015f4b253fc2bf97a610886419ead64b4756ef", size = 10711653, upload-time = "2026-08-27T16:34:09.712Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/20/656d67f5b25ca9bda4e02b1de25867b2954e1d19e03648060f167ad0f4cc/ruff-0.16.5-py3-none-win32.whl", hash = "sha256:288b0a5f080492fe5635db849f9e2e84aa3cce7b7f0e955997d416c507c76a26", size = 10034250, upload-time = "2026-08-27T16:34:11.8Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/42/ee8e68a207b9127fcde6c3d7e197def432f346cb1af159e1fa14ca0d1cdc/ruff-0.16.5-py3-none-win_amd64.whl", hash = "sha256:ddc6385fb2137f616357ca03d6c74f4be987f80fed4008566b754f6032b8546f", size = 10516714, upload-time = "2026-08-27T16:34:13.963Z" },
+ { url = "https://files.pythonhosted.org/packages/73/e3/7df5a396e445b9ba49ce9a9437439a4d80042c61c0ade199abf8d16de1ac/ruff-0.16.5-py3-none-win_arm64.whl", hash = "sha256:a64abe90968719b851bb7cedffaa8753fbdbdadab483089682db623f3edc587e", size = 10391564, upload-time = "2026-08-27T16:34:16.064Z" },
+]
+
+[[package]]
+name = "tenacity"
+version = "9.1.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" },
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.16.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },
+]
+
+[[package]]
+name = "typing-inspection"
+version = "0.4.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },
+]
+
+[[package]]
+name = "wcwidth"
+version = "0.8.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/36/57/ed58088fafdf4c55a0ad6bde846502567645424d7ebf325230b9237f4085/wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb", size = 1458450, upload-time = "2026-08-28T18:10:06.875Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" },
+]
diff --git a/scripts/benchmarks/worker.py b/scripts/benchmarks/worker.py
new file mode 100644
index 0000000000..9c6f954370
--- /dev/null
+++ b/scripts/benchmarks/worker.py
@@ -0,0 +1,501 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pwd
+import shutil
+import signal
+import subprocess
+import time
+from pathlib import Path
+
+from kernel import (
+ BASH_EMPTY,
+ BASH_OUTPUT,
+ CELL_REPEATS,
+ FRAME_COLUMNS,
+ FRAME_ROWS,
+ GIT_STATUS,
+ OUTPUT_BYTES,
+ SHELL_REPEATS,
+ Kernel,
+)
+from schema import (
+ ROOT,
+ Metric,
+ Observation,
+ ProcessMemory,
+ Request,
+ Result,
+ Side,
+ write_json,
+)
+from terminal import Terminal
+
+HOMES = Path("/home")
+SOURCE = HOMES / "builder/source"
+RESULTS = ROOT / "results"
+VERSION = "0.0.0-benchmark"
+ORIGIN = "http://127.0.0.1:18741"
+
+
+def clean_error(error: Exception) -> str:
+ return f"{type(error).__name__}: {error}"[:500]
+
+
+def environment(user: str) -> dict[str, str]:
+ home = str(HOMES / user)
+ env = {
+ "HOME": home,
+ "USER": user,
+ "LOGNAME": user,
+ "SHELL": "/bin/bash",
+ "PATH": f"{home}/.local/bin:/usr/local/bin:/usr/bin:/bin",
+ "TERM": "xterm-256color",
+ "COLORTERM": "truecolor",
+ "LANG": "C.UTF-8",
+ "NPM_CONFIG_PREFIX": f"{home}/.local",
+ "NPM_CONFIG_CACHE": f"{home}/.npm",
+ "UV_CACHE_DIR": f"{home}/.cache/uv",
+ "UV_LINK_MODE": "copy",
+ }
+ return env
+
+
+def run_as(
+ user: str,
+ args: list[str],
+ cwd: Path,
+ *,
+ timeout: int = 60,
+ log: Path | None = None,
+ extra_env: dict[str, str] | None = None,
+ merge_output: bool = False,
+) -> str:
+ env = environment(user) | (extra_env or {})
+ command = ["/usr/sbin/runuser", "-u", user, "--", *args]
+ if log:
+ with log.open("a") as stream:
+ subprocess.run(
+ command,
+ cwd=cwd,
+ env=env,
+ stdin=subprocess.DEVNULL,
+ stdout=stream,
+ stderr=subprocess.STDOUT,
+ check=True,
+ timeout=timeout,
+ )
+ return ""
+ return subprocess.check_output(
+ command,
+ cwd=cwd,
+ env=env,
+ stderr=subprocess.STDOUT if merge_output else subprocess.PIPE,
+ text=True,
+ timeout=timeout,
+ )
+
+
+def stop_processes(user: str) -> None:
+ uid = pwd.getpwnam(user).pw_uid
+ for sig, delay in ((signal.SIGTERM, 3), (signal.SIGKILL, 2)):
+ for process in memory(uid):
+ try:
+ os.kill(process.pid, sig)
+ except ProcessLookupError:
+ pass
+ deadline = time.monotonic() + delay
+ while time.monotonic() < deadline:
+ if not memory(uid):
+ return
+ time.sleep(0.05)
+ if memory(uid):
+ raise RuntimeError(f"Processes remain for {user}")
+
+
+def memory(uid: int) -> list[ProcessMemory]:
+ processes = []
+ for path in Path("/proc").glob("[0-9]*/status"):
+ try:
+ fields = dict(line.split(":", 1) for line in path.read_text().splitlines() if ":" in line)
+ if int(fields["Uid"].split()[0]) != uid or fields["State"].strip().startswith("Z"):
+ continue
+ pss = None
+ try:
+ for line in (path.parent / "smaps_rollup").read_text().splitlines():
+ if line.startswith("Pss:"):
+ pss = int(line.split()[1]) * 1024
+ except (PermissionError, FileNotFoundError, ProcessLookupError):
+ pass
+ processes.append(
+ ProcessMemory(
+ pid=int(path.parent.name),
+ name=fields["Name"].strip(),
+ rss=int(fields.get("VmRSS", "0").split()[0]) * 1024,
+ pss=pss,
+ )
+ )
+ except (PermissionError, FileNotFoundError, ProcessLookupError):
+ continue
+ return processes
+
+
+def prepare(request: Request, side: Side) -> None:
+ log = RESULTS / "build.log"
+ source_url = f"https://github.com/{request.source_repository}.git"
+ SOURCE.mkdir(parents=True, exist_ok=True)
+ builder = pwd.getpwnam("builder")
+ os.chown(SOURCE, builder.pw_uid, builder.pw_gid)
+ for args in (
+ ["git", "init", "--quiet"],
+ ["git", "fetch", "--depth=1", source_url, request.sha],
+ ["git", "checkout", "--detach", "FETCH_HEAD"],
+ ["npm", "ci", "--no-audit", "--no-fund"],
+ ):
+ run_as("builder", args, SOURCE, timeout=600, log=log)
+ actual = run_as("builder", ["git", "rev-parse", "HEAD"], SOURCE).strip()
+ if actual != request.sha:
+ raise RuntimeError("Checkout did not resolve to the requested commit")
+ for package in ("tui", "ai", "agent", "coding-agent"):
+ run_as(
+ "builder",
+ [str(SOURCE / "node_modules/.bin/tsgo"), "-p", "tsconfig.build.json"],
+ SOURCE / "packages" / package,
+ timeout=180,
+ log=log,
+ )
+ agent = SOURCE / "packages/coding-agent"
+ run_as("builder", ["chmod", "+x", "dist/cli.js"], agent)
+ for script in ("copy-assets", "bundle"):
+ run_as("builder", ["npm", "run", script], agent, timeout=180, log=log)
+ run_as(
+ "builder",
+ [
+ "node",
+ "scripts/pack-prime-agent-release.mjs",
+ "--base-url",
+ ORIGIN,
+ "--version",
+ VERSION,
+ "--out-dir",
+ "packages/coding-agent/release/benchmark",
+ ],
+ SOURCE,
+ timeout=180,
+ log=log,
+ )
+ artifacts = agent / "release/benchmark/artifacts"
+ side.artifacts = {path.name: path.stat().st_size for path in sorted(artifacts.glob("*.tgz"))}
+ if len(side.artifacts) != 4:
+ raise RuntimeError("Expected the four release package tarballs")
+ record(side, "bundle", 0, sum(side.artifacts.values()))
+ release = ROOT / "www/releases" / f"v{VERSION}"
+ release.parent.mkdir(parents=True, exist_ok=True)
+ release.symlink_to(artifacts)
+ for name in ("node", "npm"):
+ side.runtime[name] = run_as("builder", [name, "--version"], SOURCE).strip()
+ side.runtime["kernel"] = os.uname().release
+ side.runtime["machine"] = os.uname().machine
+ side.runtime["artifact_format"] = "npm-tarballs"
+ cpu = Path("/proc/cpuinfo").read_text()
+ side.runtime["cpu"] = next(
+ (line.partition(":")[2].strip() for line in cpu.splitlines() if line.startswith("model name")),
+ "unavailable",
+ )
+ if side.runtime["machine"] != "x86_64":
+ raise RuntimeError("Benchmark requires Linux x64")
+
+
+def disk_bytes(home: Path) -> int:
+ output = subprocess.check_output(
+ [
+ "du",
+ "--apparent-size",
+ "--summarize",
+ "--block-size=1",
+ "--exclude=.cache",
+ "--exclude=.npm",
+ "--exclude=*.log",
+ "--exclude=sessions",
+ "--exclude=session-artifacts",
+ "--exclude=workspace",
+ str(home),
+ ],
+ text=True,
+ )
+ return int(output.split()[0])
+
+
+def install(request: Request, side: Side, trial: int) -> None:
+ user = f"benchmark{trial + 1}"
+ subprocess.run(
+ ["useradd", "--create-home", "--uid", str(2001 + trial), "--shell", "/bin/bash", user], check=True
+ )
+ home = HOMES / user
+ before = disk_bytes(home)
+ started = time.perf_counter()
+ try:
+ run_as(
+ user,
+ ["sh", str(SOURCE / "install.sh"), VERSION],
+ home,
+ timeout=240,
+ log=RESULTS / f"install-{trial}.log",
+ extra_env={
+ "PRIME_AGENT_DOWNLOAD_BASE_URL": ORIGIN,
+ "PRIME_AGENT_INSTALLER_PLAIN": "1",
+ "PRIME_AGENT_BOOTSTRAP_KERNEL_ON_INSTALL": "1",
+ "PRIME_AGENT_BOOTSTRAP_TOOLS_ON_INSTALL": "1",
+ "PRIME_AGENT_INSTALL_UV": "1",
+ },
+ )
+ elapsed = time.perf_counter() - started
+ version = run_as(user, ["prime-agent", "--version"], home, merge_output=True).strip()
+ if VERSION not in version:
+ raise RuntimeError(f"Installed version does not match the packed release: {version[:100]}")
+ if not (home / ".prime/agent/kernel-venv/bin/python").exists():
+ raise RuntimeError("The installer's Python bootstrap did not complete")
+ record(side, "install", trial, elapsed)
+ if trial == 0:
+ side.runtime["home_before_install_bytes"] = str(before)
+ info = pwd.getpwnam(user)
+ settings = home / ".prime/agent/settings.json"
+ settings.write_text(json.dumps({"onboardingShown": True}) + "\n")
+ os.chown(settings, info.pw_uid, info.pw_gid)
+ workspace = home / "workspace"
+ workspace.mkdir()
+ os.chown(workspace, info.pw_uid, info.pw_gid)
+ (workspace / "README.md").write_text(
+ "# Benchmark fixture\n\nA small repository for CLI benchmarks.\n"
+ )
+ (workspace / "example.py").write_text("def add(left, right):\n return left + right\n")
+ for path in workspace.iterdir():
+ os.chown(path, info.pw_uid, info.pw_gid)
+ for command in (
+ ["git", "init", "--quiet", "--initial-branch=main"],
+ ["git", "add", "README.md", "example.py"],
+ [
+ "git",
+ "-c",
+ "user.name=Benchmark",
+ "-c",
+ "commit",
+ "--quiet",
+ "-m",
+ "chore: initialize benchmark fixture",
+ ],
+ ):
+ run_as(
+ user,
+ command,
+ workspace,
+ extra_env={
+ "GIT_AUTHOR_DATE": "2000-01-01T00:00:00Z",
+ "GIT_COMMITTER_DATE": "2000-01-01T00:00:00Z",
+ },
+ )
+ side.runtime["installed"] = version
+ try:
+ run_as(
+ user,
+ ["npm", "ls", "--global", "--all", "--json"],
+ home,
+ log=RESULTS / "installed-dependencies.json",
+ )
+ except subprocess.CalledProcessError:
+ side.runtime["dependency_inventory"] = "npm ls reported dependency warnings; see inventory"
+ except Exception as error:
+ record(side, "install", trial, error=clean_error(error))
+ finally:
+ stop_processes(user)
+
+
+def record(
+ side: Side, metric: Metric, trial: int, value: float | None = None, error: str | None = None
+) -> None:
+ samples = side.metrics.setdefault(metric, [])
+ samples[:] = [sample for sample in samples if sample.trial != trial]
+ samples.append(Observation(trial=trial, value=value, error=error))
+
+
+def stop_agents(home: Path) -> None:
+ listing = json.loads(run_as("benchmark1", ["prime-agent", "list", "--json"], home))
+ for session in listing["sessions"]:
+ if session.get("activeSessionId"):
+ run_as(
+ "benchmark1",
+ ["prime-agent", "stop", session["activeSessionId"], "--json"],
+ home,
+ )
+
+
+def measure(request: Request, side: Side, trial: int) -> None:
+ if not any(s.value is not None and s.trial == 0 for s in side.metrics.get("install", [])):
+ raise RuntimeError("The first installation must succeed before interactive measurements")
+ home = HOMES / "benchmark1"
+ stop_processes("benchmark1")
+ for mode in ("cold", "warm"):
+ terminal = Terminal(
+ [
+ "/usr/sbin/runuser",
+ "-u",
+ "benchmark1",
+ "--",
+ "prime-agent",
+ ],
+ home / "workspace",
+ environment("benchmark1"),
+ RESULTS / f"{mode}-{trial}",
+ )
+ try:
+ record(side, mode, trial, terminal.ready())
+ terminal.settle(1)
+ if mode == "cold":
+ processes = memory(pwd.getpwnam("benchmark1").pw_uid)
+ if not processes:
+ raise RuntimeError("No owned processes found for memory measurement")
+ record(side, "rss", trial, sum(process.rss for process in processes))
+ if all(process.pss is not None for process in processes):
+ record(side, "pss", trial, sum(process.pss or 0 for process in processes))
+ side.processes = processes
+ write_json(RESULTS / f"memory-{trial}.json", Result(request=request, side=side))
+ except Exception as error:
+ record(side, mode, trial, error=clean_error(error))
+ finally:
+ terminal.close()
+ if mode == "cold":
+ stop_agents(home)
+ stop_processes("benchmark1")
+ if trial == 0 and any(sample.value is not None for sample in side.metrics.get("cold", [])):
+ record(side, "disk", 0, disk_bytes(home) - int(side.runtime["home_before_install_bytes"]))
+
+
+def runtime(side: Side, trial: int) -> None:
+ home = HOMES / "benchmark1"
+ user = pwd.getpwnam("benchmark1")
+ stop_processes("benchmark1")
+ state = home / f"runtime-benchmark-{trial}"
+ state.mkdir()
+ os.chown(state, user.pw_uid, user.pw_gid)
+ command = [
+ "/usr/sbin/runuser",
+ "-u",
+ "benchmark1",
+ "--",
+ str(home / ".prime/agent/kernel-venv/bin/python"),
+ "-m",
+ "rlm.repl",
+ ]
+ kernel = None
+ metric: Metric = "kernel_start"
+
+ def rss() -> int:
+ processes = memory(user.pw_uid)
+ if not processes:
+ raise RuntimeError("No kernel processes found for memory measurement")
+ (RESULTS / f"{metric}-{trial}.json").write_text(
+ json.dumps([process.model_dump() for process in processes], indent=2) + "\n"
+ )
+ return sum(process.rss for process in processes)
+
+ try:
+ kernel = Kernel(command, home / "workspace", environment("benchmark1"), RESULTS / f"kernel-{trial}")
+ elapsed, python = kernel.ready()
+ record(side, metric, trial, elapsed)
+ side.runtime["python"] = python
+ metric = "kernel_rss"
+ record(side, metric, trial, rss())
+ kernel.execute("from rlm import bash")
+ kernel.batch("pass", 5)
+ kernel.batch(BASH_EMPTY, 1)
+ for metric, code, repeats, output in (
+ ("kernel_exec", "pass", CELL_REPEATS, None),
+ ("bash", BASH_EMPTY, SHELL_REPEATS, None),
+ ("git_status", GIT_STATUS, SHELL_REPEATS, None),
+ ("output", BASH_OUTPUT, SHELL_REPEATS, OUTPUT_BYTES),
+ ):
+ record(side, metric, trial, kernel.batch(code, repeats, output_bytes=output))
+ metric = "mixed"
+ record(side, metric, trial, kernel.mixed())
+ metric = "interrupt"
+ record(side, metric, trial, kernel.interrupt())
+ metric = "loaded_rss"
+ kernel.execute(
+ f"import pandas as pd\nframe = pd.DataFrame({{str(i): range({FRAME_ROWS}) "
+ f"for i in range({FRAME_COLUMNS})}})\npayload = list(range({FRAME_ROWS}))\n"
+ f"assert frame.shape == ({FRAME_ROWS}, {FRAME_COLUMNS})"
+ )
+ record(side, metric, trial, rss())
+ metric = "snapshot"
+ started = time.perf_counter()
+ events = kernel.request(
+ "snapshot", path=str(state / "state.pkl"), manifest_path=str(state / "manifest.json")
+ )
+ elapsed = time.perf_counter() - started
+ if not {"frame", "payload"} <= set(events[-1].get("saved", [])):
+ raise RuntimeError("Snapshot did not save the complete state fixture")
+ record(side, metric, trial, elapsed)
+ kernel.close()
+ kernel = None
+ metric = "restore"
+ kernel = Kernel(command, home / "workspace", environment("benchmark1"), RESULTS / f"restore-{trial}")
+ kernel.ready()
+ started = time.perf_counter()
+ events = kernel.request("restore", path=str(state / "state.pkl"))
+ elapsed = time.perf_counter() - started
+ if events[-1].get("failed") or not {"frame", "payload"} <= set(events[-1].get("restored", [])):
+ raise RuntimeError("Restore did not recover the complete state fixture")
+ kernel.execute(
+ f"assert frame.shape == ({FRAME_ROWS}, {FRAME_COLUMNS})\n"
+ f"assert int(frame.sum().sum()) == {FRAME_COLUMNS * FRAME_ROWS * (FRAME_ROWS - 1) // 2}\n"
+ f"assert payload == list(range({FRAME_ROWS}))"
+ )
+ record(side, metric, trial, elapsed)
+ except Exception as error:
+ record(side, metric, trial, error=clean_error(error))
+ finally:
+ try:
+ if kernel:
+ kernel.close()
+ finally:
+ stop_processes("benchmark1")
+ shutil.rmtree(state)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("phase", choices=("prepare", "install", "measure", "runtime"))
+ parser.add_argument("--trial", type=int, default=0)
+ args = parser.parse_args()
+ RESULTS.mkdir(exist_ok=True)
+ request = Request.model_validate_json((ROOT / "request.json").read_text())
+ path = RESULTS / "result.json"
+ result = (
+ Result.model_validate_json(path.read_text())
+ if path.exists()
+ else Result(
+ request=request,
+ side=Side(sha=request.sha),
+ )
+ )
+ try:
+ if args.phase == "prepare":
+ prepare(request, result.side)
+ elif args.phase == "install":
+ install(request, result.side, args.trial)
+ elif args.phase == "measure":
+ measure(request, result.side, args.trial)
+ else:
+ runtime(result.side, args.trial)
+ except Exception as error:
+ result.side.error = clean_error(error)
+ raise
+ finally:
+ write_json(path, result)
+
+
+if __name__ == "__main__":
+ main()