A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
benchmark prime agent performance on pull request updates (#2160)
ktf9c7e06b58b9
diff --git a/.github/workflows/benchmark-completed.yml b/.github/workflows/benchmark-completed.ymlnew file mode 100644index 0000000000..cc00300d56--- /dev/null+++ b/.github/workflows/benchmark-completed.yml@@ -0,0 +1,81 @@+name: Finalize Prime Agent benchmarks++on:+ workflow_run:+ workflows: [Prime Agent benchmarks]+ types: [completed]++permissions: {}++jobs:+ cleanup:+ if: contains(fromJSON('["pull_request_target", "workflow_dispatch"]'), github.event.workflow_run.event)+ runs-on: ubuntu-24.04+ timeout-minutes: 5+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0+ with:+ ref: main+ persist-credentials: false+ - name: Install benchmark tooling+ run: |+ python3 -m venv .benchmark-bootstrap+ .benchmark-bootstrap/bin/pip install uv==0.12.9+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"+ - name: Delete remaining run sandboxes+ working-directory: scripts/benchmarks+ run: uv run --locked cli.py cleanup+ env:+ PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}++ publish:+ if: contains(fromJSON('["pull_request_target", "workflow_dispatch"]'), github.event.workflow_run.event)+ runs-on: ubuntu-24.04+ timeout-minutes: 5+ concurrency:+ group: prime-agent-benchmark-comment-${{ github.event.workflow_run.display_title }}+ cancel-in-progress: false+ permissions:+ contents: read+ actions: read+ pull-requests: write+ steps:+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0+ with:+ ref: main+ persist-credentials: false+ - name: Download request+ id: request+ continue-on-error: true+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0+ with:+ name: benchmark-request-${{ github.event.workflow_run.run_attempt }}+ path: scripts/benchmarks/request+ run-id: ${{ github.event.workflow_run.id }}+ github-token: ${{ github.token }}+ - name: Download results+ if: steps.request.outcome == 'success'+ continue-on-error: true+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0+ with:+ name: benchmark-results-${{ github.event.workflow_run.run_attempt }}+ path: scripts/benchmarks/results+ run-id: ${{ github.event.workflow_run.id }}+ github-token: ${{ github.token }}+ - name: Install benchmark tooling+ if: steps.request.outcome == 'success'+ run: |+ python3 -m venv .benchmark-bootstrap+ .benchmark-bootstrap/bin/pip install uv==0.12.9+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"+ - name: Publish the current result+ if: steps.request.outcome == 'success'+ working-directory: scripts/benchmarks+ run: uv run --locked cli.py finalize+ env:+ GITHUB_TOKEN: ${{ github.token }}diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.ymlnew file mode 100644index 0000000000..3bc5e5a15a--- /dev/null+++ b/.github/workflows/benchmarks.yml@@ -0,0 +1,121 @@+name: Prime Agent benchmarks+run-name: 'Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}'++on:+ pull_request_target:+ branches: [main]+ types: [opened, synchronize, reopened, ready_for_review]+ workflow_dispatch:+ inputs:+ pr:+ description: Open PR number to benchmark against current main+ required: true+ type: number++concurrency:+ group: prime-agent-benchmark-${{ github.event.pull_request.number || inputs.pr }}+ cancel-in-progress: true++permissions: {}++jobs:+ request:+ if: github.event_name == 'pull_request_target' || github.ref == 'refs/heads/main'+ runs-on: ubuntu-24.04+ timeout-minutes: 5+ concurrency:+ group: 'prime-agent-benchmark-comment-Prime Agent benchmarks · PR #${{ github.event.pull_request.number || inputs.pr }}'+ cancel-in-progress: false+ permissions:+ contents: read+ actions: read+ pull-requests: write+ outputs:+ allowed: ${{ steps.resolve.outputs.needed == 'true' && steps.vouch.outputs.vouched == 'true' }}+ harness: ${{ steps.resolve.outputs.harness }}+ steps:+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0+ with:+ ref: ${{ github.sha }}+ persist-credentials: false+ - name: Install benchmark tooling+ run: |+ python3 -m venv .benchmark-bootstrap+ .benchmark-bootstrap/bin/pip install uv==0.12.9+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"+ - name: Resolve exact revisions+ id: resolve+ working-directory: scripts/benchmarks+ run: uv run --locked cli.py resolve --results request+ env:+ GITHUB_TOKEN: ${{ github.token }}+ - name: Check contributor trust+ id: vouch+ if: steps.resolve.outputs.needed == 'true'+ uses: mitchellh/vouch/action/check-user@d66fa29a64600490892131ad87597c30c91fcac4 # v1+ with:+ user: ${{ steps.resolve.outputs.author }}+ allow-fail: true+ env:+ GITHUB_TOKEN: ${{ github.token }}+ - name: Update benchmark comment+ if: steps.resolve.outputs.needed == 'true'+ working-directory: scripts/benchmarks+ run: |+ if [ "$VOUCHED" = true ]; then+ uv run --locked cli.py publish --results request+ else+ uv run --locked cli.py publish --results request --pending-trust+ fi+ env:+ GITHUB_TOKEN: ${{ github.token }}+ VOUCHED: ${{ steps.vouch.outputs.vouched }}+ - name: Save benchmark request+ if: steps.resolve.outputs.needed == 'true'+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0+ with:+ name: benchmark-request-${{ github.run_attempt }}+ path: scripts/benchmarks/request/report.json+ retention-days: 14++ benchmark:+ needs: request+ if: needs.request.outputs.allowed == 'true'+ runs-on: ubuntu-24.04+ timeout-minutes: 25+ permissions:+ contents: read+ actions: read+ pull-requests: read+ steps:+ # The controller is trusted main code. PR code executes only in Prime sandboxes.+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0+ with:+ ref: ${{ needs.request.outputs.harness }}+ persist-credentials: false+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.0+ with:+ name: benchmark-request-${{ github.run_attempt }}+ path: scripts/benchmarks/request+ - name: Install benchmark tooling+ run: |+ python3 -m venv .benchmark-bootstrap+ .benchmark-bootstrap/bin/pip install uv==0.12.9+ .benchmark-bootstrap/bin/uv sync --locked --project scripts/benchmarks+ echo "$PWD/.benchmark-bootstrap/bin" >> "$GITHUB_PATH"+ - name: Benchmark main and PR in Prime sandboxes+ working-directory: scripts/benchmarks+ run: uv run --locked cli.py run+ env:+ GITHUB_TOKEN: ${{ github.token }}+ PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}+ - name: Save benchmark results+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0+ with:+ name: benchmark-results-${{ github.run_attempt }}+ path: scripts/benchmarks/results/+ if-no-files-found: warn+ retention-days: 14diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex af521a547c..2fca23f717 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -143,6 +143,14 @@ jobs:working-directory: ${{ matrix.package }}run: ${{ matrix.command }}+ - name: Test benchmark harness+ if: matrix.name == 'runtime python'+ working-directory: scripts/benchmarks+ run: |+ uv run --locked ruff check .+ uv run --locked ruff format --check .+ uv run --locked python -m unittest discover -s tests -v+build-check-test:name: build-check-testif: always() && needs.trust.outputs.allowed == 'true'diff --git a/scripts/benchmarks/.gitignore b/scripts/benchmarks/.gitignorenew file mode 100644index 0000000000..b5c701661a--- /dev/null+++ b/scripts/benchmarks/.gitignore@@ -0,0 +1,3 @@+.venv/+results/+.ruff_cache/diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.mdnew file mode 100644index 0000000000..b1deb3863a--- /dev/null+++ b/scripts/benchmarks/README.md@@ -0,0 +1,153 @@+# PR performance benchmarks++Each push to an open, vouched PR starts an informational Prime Agent benchmark. Multiple commits in+one push produce one run for the final head. Draft PRs are included. A two-second debounce and+per-PR cancellation avoid finishing obsolete runs. `workflow_dispatch` reruns an open PR by number.+Identical automatic requests reuse the completed comment when both SHAs, harness, and configuration+match. Manual dispatch and GitHub reruns force fresh measurements. Unvouched authors+receive a pending-trust comment; a maintainer can rerun after vouching.++The controller resolves current `main` and the PR head to full SHAs, builds both in separate Prime+sandboxes, and alternates their measurements. Both use the same trusted harness revision, image+digest and resource allocation. No performance gate blocks merging.++## Enable in GitHub++Add the repository secret `PRIME_SANDBOX_API_KEY`: a dedicated key with Sandbox permissions for+provisioning and cleanup. Optional `PRIME_BENCHMARK_TEAM_ID` selects the sandbox billing workspace.+No inference key, model configuration, or login is required.++The workflows must first land on `main`: both `pull_request_target` and the completion listener run+trusted default-branch code. Trigger `Prime Agent benchmarks` manually with an open PR number after+configuring the sandbox secret. The first rollout should include a main-versus-main calibration.++No credentials are injected into the sandboxes. The sandbox control key and GitHub token stay on the+trusted controller; a separate publisher owns GitHub comment write permission. No PR checkout,+build script, installer, executable, or archive is executed/extracted on a privileged GitHub runner.+The existing Vouch gate controls who can trigger compute usage.++## Measurements++The default configuration uses two Linux x64 containers, each with 4 vCPU, 8 GB RAM, and 20 GB disk.+`config.json` pins the image and sampling policy. Harness dependencies are locked with a seven-day+release cutoff. Provisioning, harness setup, and source compilation have separate recorded durations+outside the timed installation interval. Interactive runs use the same small committed Git fixture.++The base image supplies Node, npm, Git, curl, and Python. Untimed sandbox setup adds Python venv+support, Cairo/Pango/JPEG/GIF/SVG development libraries, ripgrep, and fd. These prerequisites are+shared within each sandbox; the normal installer still prepares each user's stock Python environment+and other missing tools. Their setup time and disk usage are outside the installation metrics.++- **Cold startup:** process launch until the editor visibly echoes a typed marker, after stopping all+ processes owned by the benchmark user. OS filesystem caches are not flushed.+- **Warm startup:** the same input-ready measurement while retaining the daemon and stopping its+ previous active sessions. Each TUI launch opens a fresh conversation.+- **Installation:** the normal installer and Python/tool bootstrap in three new user homes, each+ with empty npm and uv caches. Unpublished candidate release tarballs are served over loopback;+ npm/Python dependencies use the real network. This does not measure public release-CDN latency.+- **Compressed artifacts:** total bytes of the four tarballs produced by the release packer, using+ an identical synthetic version and download origin for both sides. External registry dependencies+ are not included in these tarballs.+- **Installed footprint:** apparent bytes added after first use in the first fresh home, including stock Python,+ runtime, and tool assets; excluding download caches, session history, and logs. Shared system+ dependencies supplied by the base image and the fixture repository are excluded.+- **Idle memory:** summed RSS across the benchmark user's entire process tree after input readiness+ and a one-second settle. Raw results include each process and PSS when Linux permits reading it.+ The controller, PTY harness, artifact server, and build user are excluded. RSS can double-count+ shared pages.++Startup and memory use 10 trials per revision. Installation uses three; sizes are measured once.+Stock tools, skills, daemon, persistence, and Python bootstrap remain enabled. Homes contain no+credentials, extensions, MCP servers, or personal skills. The onboarding splash is marked as already+shown before timing, so startup measures the editor rather than waiting for a person to sign in.+The editor runs without a selected model; no prompts are submitted. These measurements cover the+local startup path, not authenticated provider discovery or inference.++### Python runtime++A separate probe drives the **installed** `python -m rlm.repl` over its JSONL protocol. It measures the+CPython kernel and stock `rlm.bash` implementation, including IPC, serialization, and output capture.+It does not include the TypeScript kernel manager, TUI rendering, or model execution. The harness+fails visibly if either revision does not support the required runtime protocol; it never substitutes+a plain Python process or omits a failed metric.++Each of the 10 trials starts with all benchmark-user processes stopped and a new kernel:++- **Kernel startup:** process spawn through the protocol-ready handshake. Includes the small+ `runuser` launch overhead; excludes Python installation and host-side agent bootstrap.+- **Python cell round trip:** mean of 50 sequential `pass` cells through the complete request/done+ protocol. Five Python cells and one empty bash command warm the runtime before execution timings.+- **Empty bash command:** mean of five `await bash(':')` calls with successful, empty results.+- **Bash git status:** mean of five `git status --porcelain` calls in the fixed, clean Git fixture.+- **Bash 32 KiB output:** mean of five shell commands producing exactly 32,768 bytes. The runtime+ captures and forwards the entire output, and the probe verifies its byte count.+- **35 cells / 9 shell calls:** total time for a fixed mix of 26 Python no-ops and nine git-status+ calls, alternating one shell call with three Python cells.+- **Interrupt to done:** after an executing cell signals readiness, send an interrupt and wait for+ its KeyboardInterrupt/done acknowledgement. A subsequent cell must still execute successfully.+- **State snapshot:** serialize a 10,000-row, eight-column integer pandas DataFrame and a+ 10,000-integer list to disk. Includes first-use serialization imports.+- **State restore:** restore that state into a fresh kernel, including first-use pandas imports but+ excluding kernel startup. Verify the recovered dimensions, sum, and list values after timing.+- **Kernel idle RSS:** memory immediately after the ready handshake, before workload imports.+- **Kernel RSS after pandas workload:** memory after the mixed calls and construction of the same+ DataFrame/list used for snapshotting. Both memory measurements include only the isolated user's+ Python process; raw per-process measurements are retained.++The runtime rows show medians across the 10 independent trials. Batch iteration counts and fixture+sizes are fixed in `kernel.py`; failures and incomplete output remain failed samples. These are+subsystem measurements alongside the normal installed CLI benchmarks, not end-to-end tool latency.++The comment shows medians, signed absolute/percentage deltas, successful/attempted counts, and spread.+`↓` means improvement, bold `↑` means regression, `≈` means no clear change, and `—` means unavailable+or incomplete. Arrows require a change larger than the metric's absolute floor, relative floor,+and observed spread. The initial relative floor is 20% for timings and memory, 0.5% for artifact+size, and 1% for disk footprint. A same-revision calibration on separate sandboxes showed roughly+7–18% variation across several timings; the conservative floor avoids labeling that as a code regression.+Smaller signed differences remain visible. Revisit these floors after collecting more control runs. This is a practical noise filter, not a statistical significance test. Inspect+raw trials before acting on small changes; sandbox scheduling and filesystem caches still introduce noise.++## Lifecycle and costs++The main workflow posts a single marked comment and updates it in place. The completion workflow+publishes the final result or a cancellation/failure notice. Publishers serialize by PR and check the+current head, latest run ID, attempt, and existing comment generation before writing. A rerun of an+older commit cannot overwrite a newer result, including when the newer run has the same head SHA.++Results and logs are retained as GitHub artifacts for 14 days. They include exact source and harness+SHAs, environment details, installed dependency inventory, transcripts, raw observations, failures,+and estimated sandbox costs. The publisher validates schema, identity, finite numbers, and report+size and escapes sandbox-provided text.++The controller stops scheduling work at 20 minutes or its $1 estimated budget target. Every sandbox+has a 30-minute TTL. Teardown runs in `finally`; the independent completion workflow deletes any+remaining sandboxes with the exact repository/run/attempt labels, including after cancellation.+Cleanup enumerates all pages before deleting so pagination cannot skip a sandbox.++At the configured list rates, two sandboxes cost about $0.01/minute together. A 10-minute run costs+about $0.10 in sandbox compute. There are no inference charges. Full sandbox lifetimes are recorded,+including provisioning, setup, building, measurements, and cleanup. The $1 target is an estimate,+not a hard billing cap; scheduling and deletion latency can increase the final bill.++## Local development++The existing runtime Python CI job runs the harness checks. To run them locally from this directory:++```sh+uv sync --locked+uv run --locked ruff check .+uv run --locked ruff format --check .+uv run --locked python -m unittest discover -s tests -v+```++Tests use fake GitHub/SDK responses and terminal streams; they never invoke inference or provision+sandboxes. A live run is separate and requires only the `PRIME_SANDBOX_API_KEY` environment variable:++```sh+uv run --locked cli.py local --base FULL_MAIN_SHA --head FULL_HEAD_SHA --results results/local+```++Use identical SHAs to calibrate main against itself. `--config /path/to/config.json` can reduce trials+for a smoke run; the report records the effective configuration. Local mode never posts GitHub+comments. `results/`, `.venv/`, and `.ruff_cache/` are ignored. Run the repository's `npm run check`+after code changes as well.diff --git a/scripts/benchmarks/cli.py b/scripts/benchmarks/cli.pynew file mode 100644index 0000000000..1c63836ccd--- /dev/null+++ b/scripts/benchmarks/cli.py@@ -0,0 +1,165 @@+from __future__ import annotations++import argparse+import json+import os+import subprocess+from datetime import UTC, datetime+from pathlib import Path++from prime_sandboxes import APIClient, SandboxClient++from controller import Controller, cleanup+from github import WORKFLOW, GitHub+from report import render+from schema import Config, Report, Side, load_report, write_json+++def workflow_source(event: dict) -> dict:+ run = event["workflow_run"]+ if (+ run["event"] not in ("pull_request_target", "workflow_dispatch")+ or run["path"].split("@")[0] != f".github/workflows/{WORKFLOW}"+ or run["repository"]["full_name"] != os.environ["GITHUB_REPOSITORY"]+ or (run["event"] == "workflow_dispatch" and run["head_branch"] != "main")+ ):+ raise ValueError("Unexpected source workflow")+ return run+++def validate_completion(report: Report, request: Report, run: dict) -> None:+ identity = {+ "repository",+ "head_repository",+ "pr",+ "run_id",+ "attempt",+ "harness_sha",+ "base_sha",+ "head_sha",+ "started_at",+ "config",+ }+ if report.model_dump(include=identity) != request.model_dump(include=identity):+ raise ValueError("Result identity does not match the trusted benchmark request")+ if (request.run_id, request.attempt) != (run["id"], run["run_attempt"]) or run[+ "display_title"+ ] != f"Prime Agent benchmarks · PR #{request.pr}":+ raise ValueError("Benchmark request does not match the completed workflow")+++def completed_report(result_path: Path, request_path: Path, run: dict) -> Report:+ request = load_report(request_path)+ validate_completion(request, request, run)+ try:+ report = load_report(result_path) if result_path.exists() else request+ validate_completion(report, request, run)+ except (ValueError, OSError):+ report = request+ report.status = "failed"+ report.errors.append("Benchmark report failed validation; see the workflow artifacts")+ if report.status == "running":+ report.status = "canceled" if run["conclusion"] == "cancelled" else "failed"+ report.errors.append("Benchmark did not produce a final report; see the workflow logs")+ if report.status != "pending-trust" and report.finished_at is None:+ report.finished_at = datetime.now(UTC)+ return report+++def main() -> None:+ parser = argparse.ArgumentParser()+ parser.add_argument("command", choices=("resolve", "run", "publish", "finalize", "cleanup", "local"))+ parser.add_argument("--results", type=Path, default=Path("results"))+ parser.add_argument("--request", type=Path, default=Path("request/report.json"))+ parser.add_argument("--base")+ parser.add_argument("--head")+ parser.add_argument("--config", type=Path)+ parser.add_argument("--pending-trust", action="store_true")+ args = parser.parse_args()+ args.results.mkdir(parents=True, exist_ok=True)+ repository = os.environ.get("GITHUB_REPOSITORY", "PrimeIntellect-ai/prime-agent")+ config = Config.load(args.config) if args.config else Config.load()+ if args.command == "local":+ if not args.base or not args.head:+ parser.error("local requires exact --base and --head commit SHAs")+ report = Report(+ repository=repository,+ head_repository=repository,+ pr=0,+ run_id=int(datetime.now(UTC).timestamp()),+ attempt=1,+ harness_sha=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),+ base_sha=args.base,+ head_sha=args.head,+ started_at=datetime.now(UTC),+ config=config,+ main=Side(sha=args.base),+ pr_head=Side(sha=args.head),+ )+ Controller(report, args.results, live_github=False).run()+ return+ event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())+ if args.command == "cleanup":+ run = workflow_source(event)+ client = SandboxClient(APIClient(api_key=os.environ["PRIME_SANDBOX_API_KEY"]))+ deleted = cleanup(client, repository, run["id"], run["run_attempt"])+ print(f"Cleaned up {len(deleted)} remaining benchmark sandboxes")+ return+ github = GitHub(repository)+ if args.command == "resolve":+ pr = int(event.get("pull_request", {}).get("number") or event["inputs"]["pr"])+ if pr <= 0:+ raise ValueError("Invalid PR number")+ report, author = github.resolve(+ pr,+ os.environ["GITHUB_SHA"],+ int(os.environ["GITHUB_RUN_ID"]),+ int(os.environ["GITHUB_RUN_ATTEMPT"]),+ config,+ )+ write_json(args.results / "report.json", report)+ manual = os.environ.get("GITHUB_EVENT_NAME") == "workflow_dispatch" or report.attempt > 1+ needed = manual or not github.duplicate(report)+ with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output:+ output.write(+ f"pr={pr}\nauthor={author}\nharness={report.harness_sha}\nneeded={str(needed).lower()}\n"+ )+ return+ report_path = args.results / "report.json"+ report = (+ completed_report(report_path, args.request, workflow_source(event))+ if args.command == "finalize"+ else load_report(report_path if report_path.exists() else args.request)+ )+ if report.repository != repository:+ raise ValueError("Result repository does not match the workflow")+ if args.command == "finalize":+ github.publish(report)+ else:+ if (report.run_id, report.attempt) != (+ int(os.environ["GITHUB_RUN_ID"]),+ int(os.environ["GITHUB_RUN_ATTEMPT"]),+ ):+ raise ValueError("Result identity does not match the active workflow")+ if args.command == "run":+ if not os.environ.get("PRIME_SANDBOX_API_KEY"):+ report.status = "failed"+ report.errors.append("Repository secret PRIME_SANDBOX_API_KEY is not configured")+ if report.status == "running":+ controller = Controller(report, args.results)+ controller.run()+ report = controller.report+ else:+ write_json(report_path, report)+ else:+ if args.pending_trust:+ report.status = "pending-trust"+ write_json(report_path, report)+ github.publish(report)+ summary = os.environ.get("GITHUB_STEP_SUMMARY")+ if summary:+ Path(summary).write_text(render(report))+++if __name__ == "__main__":+ main()diff --git a/scripts/benchmarks/config.json b/scripts/benchmarks/config.jsonnew file mode 100644index 0000000000..f0a6962115--- /dev/null+++ b/scripts/benchmarks/config.json@@ -0,0 +1,16 @@+{+ "image": "node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2",+ "region": "us",+ "cpu_cores": 4,+ "memory_gb": 8,+ "disk_gb": 20,+ "trials": 10,+ "install_trials": 3,+ "debounce_seconds": 2,+ "timeout_seconds": 1200,+ "ttl_minutes": 30,+ "budget_usd": 1,+ "cpu_usd_per_hour": 0.05,+ "memory_usd_per_gb_hour": 0.01,+ "disk_usd_per_gb_hour": 0.001+}diff --git a/scripts/benchmarks/controller.py b/scripts/benchmarks/controller.pynew file mode 100644index 0000000000..258af25850--- /dev/null+++ b/scripts/benchmarks/controller.py@@ -0,0 +1,370 @@+from __future__ import annotations++import os+import shlex+import signal+import time+from datetime import UTC, datetime+from pathlib import Path++from prime_sandboxes import APIClient, CreateSandboxRequest, SandboxClient+from prime_sandboxes.models import BackgroundJob, Sandbox++from github import GitHub+from report import render+from schema import (+ ROOT,+ UV_VERSION,+ Report,+ Request,+ Result,+ SandboxUsage,+ Side,+ write_json,+)++REMOTE = "/opt/prime-benchmark"+OWNER_LABEL = "prime-agent-benchmarks-v1"+FILES = ("schema.py", "terminal.py", "kernel.py", "worker.py", "pyproject.toml", "uv.lock")+++def elapsed_seconds(start: datetime) -> float:+ return max(0, (datetime.now(UTC) - start.replace(tzinfo=start.tzinfo or UTC)).total_seconds())+++class Canceled(Exception):+ pass+++def labels(repository: str, run_id: int, attempt: int) -> list[str]:+ return [OWNER_LABEL, f"repository:{repository}", f"run:{run_id}", f"attempt:{attempt}"]+++def cleanup(client: SandboxClient, repository: str, run_id: int, attempt: int) -> list[str]:+ expected = labels(repository, run_id, attempt)+ owned = []+ page = 1+ while True:+ response = client.list(labels=expected, page=page, per_page=100, exclude_terminated=True)+ owned.extend(sandbox.id for sandbox in response.sandboxes if set(expected) <= set(sandbox.labels))+ if not response.has_next:+ break+ page += 1+ failed = []+ for sandbox_id in owned:+ try:+ client.delete(sandbox_id)+ except Exception:+ failed.append(sandbox_id)+ if failed:+ raise RuntimeError(f"Cleanup failed for {len(failed)} sandbox(es); TTL remains active")+ return owned+++class Controller:+ def __init__(self, report: Report, results: Path, *, live_github: bool = True):+ self.report = report+ self.results = results+ self.results.mkdir(parents=True, exist_ok=True)+ self.harness = self.results / "harness"+ self.harness.mkdir(exist_ok=True)+ for name in FILES:+ (self.harness / name).write_bytes((ROOT / name).read_bytes())+ self.client = SandboxClient(APIClient(api_key=os.environ["PRIME_SANDBOX_API_KEY"]))+ self.github = GitHub(report.repository) if live_github else None+ self.sandboxes: dict[str, Sandbox] = {}+ self.requests: dict[str, Request] = {}+ self.started = time.monotonic()+ self.deadline = self.started + report.config.timeout_seconds+ self.last_fresh_check = 0.0++ def checkpoint(self) -> None:+ now = time.monotonic()+ if now > self.deadline:+ raise TimeoutError("Benchmark exceeded its runtime limit")+ if self.github and now - self.last_fresh_check > 60:+ self.last_fresh_check = now+ if not self.github.fresh(self.report):+ raise Canceled("A newer PR head or benchmark run superseded this run")++ def save(self) -> None:+ self.report.errors = self.report.errors[:30]+ self.report = Report.model_validate(self.report.model_dump())+ write_json(self.results / "report.json", self.report)+ (self.results / "comment.md").write_text(render(self.report))++ def command(self, sandbox: Sandbox, args: list[str], timeout: int = 60) -> None:+ self.checkpoint()+ result = self.client.execute_command(sandbox.id, shlex.join(args), timeout=timeout)+ if result.exit_code:+ raise RuntimeError(f"Sandbox setup command failed with exit code {result.exit_code}")++ def start(self, role: str) -> Sandbox:+ started = time.monotonic()+ side = self.report.main if role == "main" else self.report.pr_head+ config = self.report.config+ sha = self.report.base_sha if role == "main" else self.report.head_sha+ request = Request(+ repository=self.report.repository,+ source_repository=self.report.repository if role == "main" else self.report.head_repository,+ sha=sha,+ harness_sha=self.report.harness_sha,+ pr=self.report.pr,+ run_id=self.report.run_id,+ attempt=self.report.attempt,+ role=role,+ config=config,+ )+ self.requests[role] = request+ self.checkpoint()+ sandbox = self.client.create(+ CreateSandboxRequest(+ name=f"agent-bench-{self.report.run_id}-{self.report.attempt}-{role}",+ docker_image=config.image,+ cpu_cores=config.cpu_cores,+ memory_gb=config.memory_gb,+ disk_size_gb=config.disk_gb,+ vm=False,+ region=config.region,+ timeout_minutes=config.ttl_minutes,+ labels=labels(self.report.repository, self.report.run_id, self.report.attempt)+ + [f"role:{role}"],+ idempotency_key=f"agent-bench-{self.report.repository}-{self.report.run_id}-{self.report.attempt}-{role}",+ )+ )+ self.sandboxes[role] = sandbox+ print(f"Created {role} sandbox {sandbox.id}", flush=True)+ while self.client.get(sandbox.id).status != "RUNNING":+ self.checkpoint()+ state = self.client.get(sandbox.id).status+ if state in ("ERROR", "FAILED", "TERMINATED", "STOPPED", "TIMEOUT"):+ raise RuntimeError(f"{role} sandbox failed during provisioning")+ time.sleep(2)+ side.timings["provision"] = time.monotonic() - started+ started = time.monotonic()+ self.command(sandbox, ["mkdir", "-p", REMOTE])+ for name in FILES:+ self.client.upload_file(sandbox.id, f"{REMOTE}/{name}", str(self.harness / name))+ request_path = self.results / f"request-{role}.json"+ write_json(request_path, request)+ self.client.upload_file(sandbox.id, f"{REMOTE}/request.json", str(request_path))+ setup = (+ "set -eu\n"+ "apt-get update -qq\n"+ "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-venv "+ "libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep\n"+ "ln -sf /usr/bin/fdfind /usr/local/bin/fd\n"+ "useradd --create-home --uid 1500 --shell /bin/bash builder\n"+ "python3 -m venv /opt/benchmark-bootstrap\n"+ f"/opt/benchmark-bootstrap/bin/pip -q install uv=={UV_VERSION}\n"+ f"/opt/benchmark-bootstrap/bin/uv sync --locked --no-dev --project {REMOTE}\n"+ )+ command = f"{shlex.join(['sh', '-c', setup])} > {REMOTE}/setup.log 2>&1"+ job = self.client.start_background_job(sandbox.id, command)+ self.wait(sandbox, job, "setup")+ side.timings["setup"] = time.monotonic() - started+ return sandbox++ def wait(self, sandbox: Sandbox, job: BackgroundJob, label: str) -> None:+ while True:+ self.checkpoint()+ status = self.client.get_background_job(sandbox.id, job, timeout=30)+ if status.completed:+ if status.exit_code:+ raise RuntimeError(f"{label} failed with exit code {status.exit_code}")+ return+ time.sleep(2)++ def phase(self, role: str, phase: str, trial: int = 0) -> None:+ started = time.monotonic()+ sandbox = self.sandboxes[role]+ args = [f"{REMOTE}/.venv/bin/python", f"{REMOTE}/worker.py", phase, "--trial", str(trial)]+ command = f"{shlex.join(args)} > {REMOTE}/{phase}.log 2>&1"+ job = self.client.start_background_job(sandbox.id, command)+ try:+ self.wait(sandbox, job, f"{role} {phase} {trial}")+ except (Canceled, TimeoutError):+ raise+ except Exception:+ try:+ self.collect_result(role)+ except Exception:+ pass+ raise+ else:+ self.collect_result(role)+ finally:+ if phase == "prepare":+ side = self.report.main if role == "main" else self.report.pr_head+ side.timings["build"] = time.monotonic() - started+ self.save()++ def collect_result(self, role: str) -> None:+ path = self.results / f"result-{role}.json"+ self.client.download_file(self.sandboxes[role].id, f"{REMOTE}/results/result.json", str(path))+ if path.stat().st_size > 2_000_000:+ raise ValueError("Sandbox result exceeds size limit")+ result = Result.model_validate_json(path.read_text())+ if result.request != self.requests[role] or result.side.sha != self.requests[role].sha:+ raise ValueError("Sandbox result does not match this run's request")+ previous = self.report.main if role == "main" else self.report.pr_head+ result.side.timings = previous.timings.copy()+ if role == "main":+ self.report.main = result.side+ else:+ self.report.pr_head = result.side+ self.save()++ def logs(self, role: str) -> None:+ sandbox = self.sandboxes[role]+ # Never unpack PR-produced archives on the GitHub runner.+ args = [+ "python3",+ "-c",+ "import pathlib,tarfile; "+ f"root=pathlib.Path({REMOTE!r}); "+ "paths=[p for p in root.glob('*.log') if p.is_file() and p.stat().st_size < 8000000]; "+ "paths += [p for p in (root/'results').glob('*') "+ "if p.is_file() and p.stat().st_size < 8000000]; "+ "archive=tarfile.open(root/'logs.tar.gz','w:gz'); "+ "[archive.add(p,arcname=str(p.relative_to(root)),recursive=False) for p in paths[:1000]]; "+ "archive.close()",+ ]+ result = self.client.execute_command(sandbox.id, shlex.join(args), timeout=30)+ if result.exit_code == 0:+ self.client.download_file(+ sandbox.id, f"{REMOTE}/logs.tar.gz", str(self.results / f"{role}-logs.tar.gz")+ )++ def run(self) -> None:+ old_handler = signal.signal(signal.SIGTERM, cancel)+ try:+ self.save()+ while time.monotonic() - self.started < self.report.config.debounce_seconds:+ self.checkpoint()+ time.sleep(1)+ ready = []+ for role in ("main", "pr"):+ try:+ self.start(role)+ self.phase(role, "prepare")+ server = shlex.join(+ [+ "runuser",+ "-u",+ "builder",+ "--",+ "python3",+ "-m",+ "http.server",+ "18741",+ "--bind",+ "127.0.0.1",+ "--directory",+ f"{REMOTE}/www",+ ]+ )+ self.client.start_background_job(self.sandboxes[role].id, server)+ ready.append(role)+ except (Canceled, TimeoutError):+ raise+ except Exception as error:+ self.report.errors.append(f"{role} setup: {type(error).__name__}")+ for phase, count in (+ ("install", self.report.config.install_trials),+ ("measure", self.report.config.trials),+ ("runtime", self.report.config.trials),+ ):+ for trial in range(count):+ for role in ready if trial % 2 == 0 else list(reversed(ready)):+ print(f"{role}: {phase} trial {trial + 1}/{count}", flush=True)+ try:+ self.phase(role, phase, trial)+ except (Canceled, TimeoutError):+ raise+ except Exception as error:+ if len(self.report.errors) < 25:+ self.report.errors.append(f"{role} {phase} {trial}: {type(error).__name__}")+ compute = sum(+ elapsed_seconds(sandbox.created_at) * self.report.config.hourly_cost() / 3600+ for sandbox in self.sandboxes.values()+ )+ if compute >= self.report.config.budget_usd:+ raise TimeoutError("Reached the estimated run budget")+ complete = all(+ side_complete(side, self.report.config.trials, self.report.config.install_trials)+ for side in (self.report.main, self.report.pr_head)+ )+ self.report.status = "completed" if complete and not self.report.errors else "partial"+ if not ready:+ self.report.status = "failed"+ except Canceled:+ self.report.status = "canceled"+ except Exception as error:+ self.report.status = "failed"+ message = str(error)+ for key in ("PRIME_SANDBOX_API_KEY", "GITHUB_TOKEN"):+ if os.environ.get(key):+ message = message.replace(os.environ[key], "[REDACTED]")+ self.report.errors.append(message[:500])+ finally:+ signal.signal(signal.SIGTERM, signal.SIG_IGN)+ for role, sandbox in self.sandboxes.items():+ deleted = False+ try:+ self.logs(role)+ except Exception:+ self.report.errors.append(f"{role} logs could not be collected")+ try:+ self.client.delete(sandbox.id)+ deleted = True+ except Exception:+ self.report.errors.append(+ f"{role} sandbox cleanup deferred to the cleanup workflow or TTL"+ )+ seconds = elapsed_seconds(sandbox.created_at)+ if not deleted:+ seconds = max(seconds, self.report.config.ttl_minutes * 60)+ self.report.sandboxes.append(+ SandboxUsage(+ id=sandbox.id,+ role=role,+ seconds=seconds,+ deleted=deleted,+ estimated_usd=seconds * self.report.config.hourly_cost() / 3600,+ )+ )+ self.report.finished_at = datetime.now(UTC)+ self.save()+ signal.signal(signal.SIGTERM, old_handler)+++def cancel(_signum: int, _frame: object) -> None:+ raise Canceled("Workflow was canceled")+++def side_complete(side: Side, trials: int, installs: int) -> bool:+ expected = {+ "cold": trials,+ "warm": trials,+ "kernel_start": trials,+ "kernel_exec": trials,+ "bash": trials,+ "git_status": trials,+ "output": trials,+ "mixed": trials,+ "interrupt": trials,+ "snapshot": trials,+ "restore": trials,+ "kernel_rss": trials,+ "loaded_rss": trials,+ "rss": trials,+ "install": installs,+ "disk": 1,+ "bundle": 1,+ }+ return not side.error and all(+ len(side.metrics.get(metric, [])) == count+ and all(sample.value is not None for sample in side.metrics[metric])+ for metric, count in expected.items()+ )diff --git a/scripts/benchmarks/github.py b/scripts/benchmarks/github.pynew file mode 100644index 0000000000..3e9ff6baa0--- /dev/null+++ b/scripts/benchmarks/github.py@@ -0,0 +1,122 @@+from __future__ import annotations++import os+import re+from datetime import UTC, datetime++import httpx++from report import MARKER, fingerprint, render+from schema import Config, Report, Side++WORKFLOW = "benchmarks.yml"+TITLE = "Prime Agent benchmarks · PR #"+GENERATION = re.compile(r"<!-- run:(\d+):(\d+) head:([a-f0-9]{40}) -->")+++class GitHub:+ def __init__(self, repository: str):+ self.repository = repository+ self.client = httpx.Client(+ base_url=f"https://api.github.com/repos/{repository}/",+ headers={+ "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}",+ "Accept": "application/vnd.github+json",+ "X-GitHub-Api-Version": "2022-11-28",+ },+ timeout=30,+ )++ def request(self, method: str, path: str, body: dict | None = None) -> dict | list:+ response = self.client.request(method, path, json=body)+ response.raise_for_status()+ return response.json()++ def pages(self, path: str, key: str | None = None):+ separator = "&" if "?" in path else "?"+ page = 1+ while True:+ data = self.request("GET", f"{path}{separator}per_page=100&page={page}")+ items = data[key] if key else data+ yield items+ if len(items) < 100:+ return+ page += 1++ def resolve(+ self, pr: int, harness_sha: str, run_id: int, attempt: int, config: Config+ ) -> tuple[Report, str]:+ pull = self.request("GET", f"pulls/{pr}")+ if pull["state"] != "open" or pull["base"]["ref"] != "main":+ raise ValueError("Benchmark requires an open PR targeting main")+ if pull["base"]["repo"]["full_name"] != self.repository:+ raise ValueError("PR belongs to another repository")+ base = self.request("GET", "git/ref/heads/main")["object"]["sha"]+ report = Report(+ repository=self.repository,+ head_repository=pull["head"]["repo"]["full_name"],+ pr=pr,+ run_id=run_id,+ attempt=attempt,+ harness_sha=harness_sha,+ base_sha=base,+ head_sha=pull["head"]["sha"],+ started_at=datetime.now(UTC),+ config=config,+ main=Side(sha=base),+ pr_head=Side(sha=pull["head"]["sha"]),+ )+ return report, pull["user"]["login"]++ def fresh(self, report: Report) -> bool:+ pull = self.request("GET", f"pulls/{report.pr}")+ if pull["state"] != "open" or pull["head"]["sha"] != report.head_sha:+ return False+ run = self.request("GET", f"actions/runs/{report.run_id}")+ if run["run_attempt"] != report.attempt:+ return False+ for runs in self.pages(f"actions/workflows/{WORKFLOW}/runs", "workflow_runs"):+ if any(+ candidate["display_title"] == f"{TITLE}{report.pr}"+ and (candidate["run_number"], candidate["run_attempt"]) > (run["run_number"], report.attempt)+ for candidate in runs+ ):+ return False+ if not runs or min(candidate["run_number"] for candidate in runs) < run["run_number"]:+ break+ return True++ def comment(self, pr: int) -> dict | None:+ comments = [+ comment+ for page in self.pages(f"issues/{pr}/comments")+ for comment in page+ if comment["user"]["login"] == "github-actions[bot]"+ and comment.get("body", "").startswith(MARKER)+ ]+ return comments[-1] if comments else None++ def duplicate(self, report: Report) -> bool:+ comment = self.comment(report.pr)+ return bool(+ comment and f"<!-- comparison:{fingerprint(report)} status:completed -->" in comment["body"]+ )++ def publish(self, report: Report) -> bool:+ if not self.fresh(report):+ return False+ comment = self.comment(report.pr)+ if comment:+ generation = GENERATION.search(comment["body"])+ if generation and tuple(map(int, generation.groups()[:2])) > (report.run_id, report.attempt):+ return False+ body = render(report)+ if comment and comment["body"] == body:+ return False+ if not self.fresh(report):+ return False+ if comment:+ self.request("PATCH", f"issues/comments/{comment['id']}", {"body": body})+ else:+ self.request("POST", f"issues/{report.pr}/comments", {"body": body})+ return Truediff --git a/scripts/benchmarks/kernel.py b/scripts/benchmarks/kernel.pynew file mode 100644index 0000000000..559c3935ac--- /dev/null+++ b/scripts/benchmarks/kernel.py@@ -0,0 +1,177 @@+from __future__ import annotations++import json+import os+import selectors+import signal+import subprocess+import time+from collections import deque+from contextlib import suppress+from pathlib import Path++CELL_REPEATS = 50+SHELL_REPEATS = 5+MIXED_CELLS = 35+OUTPUT_BYTES = 32 * 1024+FRAME_ROWS = 10_000+FRAME_COLUMNS = 8++BASH_EMPTY = "_r = await bash(':')\nassert _r.exit_code == 0 and _r.output == ''"+GIT_STATUS = "_r = await bash('git status --porcelain')\nassert _r.exit_code == 0 and _r.output == ''"+BASH_OUTPUT = (+ f"_r = await bash(\"printf '%{OUTPUT_BYTES}s' ''\")\n"+ f"assert _r.exit_code == 0 and len(_r.output) == {OUTPUT_BYTES}\nprint(_r.output, end='')"+)+++class Kernel:+ def __init__(self, command: list[str], cwd: Path, env: dict[str, str], transcript: Path):+ self.transcript = transcript+ self.stderr = transcript.with_suffix(".stderr").open("wb")+ self.selector = selectors.DefaultSelector()+ self.raw: list[bytes] = []+ self.pending = b""+ self.lines: deque[bytes] = deque()+ self.bytes = 0+ self.sequence = 0+ self.started = time.perf_counter()+ try:+ self.process = subprocess.Popen(+ command,+ cwd=cwd,+ env=env,+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=self.stderr,+ bufsize=0,+ start_new_session=True,+ )+ except Exception:+ self.stderr.close()+ self.selector.close()+ raise+ assert self.process.stdout is not None+ self.selector.register(self.process.stdout, selectors.EVENT_READ)++ def event(self, deadline: float) -> dict:+ while not self.lines:+ remaining = deadline - time.perf_counter()+ if remaining <= 0 or not self.selector.select(remaining):+ raise TimeoutError("Timed out waiting for the kernel protocol")+ assert self.process.stdout is not None+ chunk = os.read(self.process.stdout.fileno(), 65536)+ if not chunk:+ raise RuntimeError("Kernel exited before completing the request")+ self.bytes += len(chunk)+ if self.bytes > 8_000_000:+ raise RuntimeError("Kernel transcript exceeds 8 MB")+ self.raw.append(chunk)+ chunks = (self.pending + chunk).split(b"\n")+ self.pending = chunks.pop()+ self.lines.extend(chunks)+ event = json.loads(self.lines.popleft())+ if not isinstance(event, dict) or not isinstance(event.get("event"), str):+ raise ValueError("Invalid kernel protocol event")+ return event++ def ready(self) -> tuple[float, str]:+ event = self.event(time.perf_counter() + 30)+ if event.get("event") != "ready" or event.get("protocol") != 3:+ raise ValueError("Expected kernel protocol 3 ready event")+ return time.perf_counter() - self.started, str(event.get("python", "unknown"))++ def send(self, request: dict) -> None:+ assert self.process.stdin is not None+ data = memoryview((json.dumps(request) + "\n").encode())+ while data:+ written = self.process.stdin.write(data)+ if not written:+ raise RuntimeError("Kernel protocol write failed")+ data = data[written:]++ def done(self, request_id: str, *, interrupted: bool = False) -> list[dict]:+ deadline = time.perf_counter() + 30+ events = []+ while True:+ event = self.event(deadline)+ events.append(event)+ if event.get("event") == "done" and event.get("id") == request_id:+ if interrupted:+ if event.get("status") != "error" or not any(+ e.get("ename") == "KeyboardInterrupt" and e.get("id") == request_id for e in events+ ):+ raise RuntimeError("Kernel did not acknowledge the interrupt")+ elif event.get("status") != "ok":+ raise RuntimeError(f"Kernel request failed: {str(events)[-400:]}")+ return events++ def request(self, kind: str, **fields: object) -> list[dict]:+ self.sequence += 1+ request_id = str(self.sequence)+ self.send({"type": kind, "id": request_id, **fields})+ return self.done(request_id)++ def execute(self, code: str) -> list[dict]:+ return self.request("execute", code=code)++ def batch(self, code: str, repeats: int, *, output_bytes: int | None = None) -> float:+ started = time.perf_counter()+ for _ in range(repeats):+ events = self.execute(code)+ if output_bytes is not None:+ output = "".join(e.get("text", "") for e in events if e["event"] == "stdout")+ if len(output.encode()) != output_bytes:+ raise RuntimeError("Kernel output was truncated or missing")+ return (time.perf_counter() - started) / repeats++ def mixed(self) -> float:+ started = time.perf_counter()+ for index in range(MIXED_CELLS):+ self.execute(GIT_STATUS if index % 4 == 0 else "pass")+ return time.perf_counter() - started++ def interrupt(self) -> float:+ self.sequence += 1+ request_id = str(self.sequence)+ self.send(+ {+ "type": "execute",+ "id": request_id,+ "code": "import asyncio\nprint('interrupt-ready', flush=True)\nawait asyncio.sleep(60)",+ }+ )+ deadline = time.perf_counter() + 30+ while True:+ event = self.event(deadline)+ if (+ event.get("id") == request_id+ and event["event"] == "stdout"+ and "interrupt-ready" in event.get("text", "")+ ):+ break+ if event["event"] in ("error", "done"):+ raise RuntimeError("Interrupt fixture did not reach the running state")+ started = time.perf_counter()+ self.send({"type": "interrupt", "id": request_id})+ self.done(request_id, interrupted=True)+ elapsed = time.perf_counter() - started+ self.execute("assert 1 + 1 == 2")+ return elapsed++ def close(self) -> None:+ try:+ if self.process.poll() is None:+ self.send({"type": "shutdown"})+ self.process.wait(timeout=5)+ except (OSError, subprocess.TimeoutExpired):+ with suppress(ProcessLookupError):+ os.killpg(self.process.pid, signal.SIGKILL)+ self.process.wait(timeout=5)+ finally:+ for stream in (self.process.stdin, self.process.stdout):+ if stream is not None:+ stream.close()+ self.selector.close()+ self.stderr.close()+ self.transcript.with_suffix(".jsonl").write_bytes(b"".join(self.raw))diff --git a/scripts/benchmarks/pyproject.toml b/scripts/benchmarks/pyproject.tomlnew file mode 100644index 0000000000..1453224541--- /dev/null+++ b/scripts/benchmarks/pyproject.toml@@ -0,0 +1,25 @@+[project]+name = "prime-agent-benchmarks"+version = "0.1.0"+requires-python = ">=3.11"+dependencies = [+ "httpx==0.28.1",+ "pexpect==4.9.0",+ "prime-sandboxes==0.2.40",+ "pydantic==2.13.5",+ "pyte==0.8.2",+]++[dependency-groups]+dev = ["ruff==0.16.5"]++[tool.uv]+package = false+exclude-newer = "2026-09-02T00:00:00Z"++[tool.ruff]+line-length = 110+target-version = "py311"++[tool.ruff.lint]+select = ["E", "F", "I", "UP", "B"]diff --git a/scripts/benchmarks/report.py b/scripts/benchmarks/report.pynew file mode 100644index 0000000000..5e34c15890--- /dev/null+++ b/scripts/benchmarks/report.py@@ -0,0 +1,308 @@+from __future__ import annotations++import hashlib+import json+import statistics+from dataclasses import dataclass++from schema import Metric, Observation, Report++MARKER = "<!-- prime-agent-benchmark:v1 -->"+PERFORMANCE_NOISE_FLOOR = 0.2+++def fingerprint(report: Report) -> str:+ fields = {+ "repository",+ "head_repository",+ "base_sha",+ "head_sha",+ "harness_sha",+ "config",+ }+ data = json.dumps(report.model_dump(include=fields), sort_keys=True, separators=(",", ":"))+ return hashlib.sha256(data.encode()).hexdigest()+++@dataclass(frozen=True)+class Definition:+ key: Metric+ title: str+ scale: float+ unit: str+ absolute: float+ relative: float+ better: str+ worse: str+++METRICS = (+ Definition("cold", "Cold startup", 1000, "ms", 0.1, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),+ Definition("warm", "Warm startup", 1000, "ms", 0.1, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),+ Definition("install", "Installation", 1, "s", 1, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),+ Definition("bundle", "Compressed release artifacts", 1e-6, "MB", 65536, 0.005, "smaller", "larger"),+ Definition("disk", "Installed footprint", 1e-6, "MB", 1048576, 0.01, "smaller", "larger"),+ Definition(+ "rss",+ "Idle memory, summed RSS",+ 1e-6,+ "MB",+ 10485760,+ PERFORMANCE_NOISE_FLOOR,+ "less memory",+ "more memory",+ ),+)+RUNTIME_METRICS = (+ Definition(+ "kernel_start",+ "Python kernel startup",+ 1000,+ "ms",+ 0.005,+ PERFORMANCE_NOISE_FLOOR,+ "faster",+ "slower",+ ),+ Definition(+ "kernel_exec",+ "Python cell round trip",+ 1000,+ "ms",+ 0.0001,+ PERFORMANCE_NOISE_FLOOR,+ "faster",+ "slower",+ ),+ Definition("bash", "Empty bash command", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"),+ Definition(+ "git_status", "Bash git status", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"+ ),+ Definition(+ "output", "Bash 32 KiB output", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"+ ),+ Definition(+ "mixed", "35 cells / 9 shell calls", 1000, "ms", 0.005, PERFORMANCE_NOISE_FLOOR, "faster", "slower"+ ),+ Definition(+ "interrupt",+ "Python interrupt to done",+ 1000,+ "ms",+ 0.0005,+ PERFORMANCE_NOISE_FLOOR,+ "faster",+ "slower",+ ),+ Definition(+ "snapshot", "Python state snapshot", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"+ ),+ Definition(+ "restore", "Python state restore", 1000, "ms", 0.001, PERFORMANCE_NOISE_FLOOR, "faster", "slower"+ ),+ Definition(+ "kernel_rss",+ "Python idle RSS",+ 1e-6,+ "MB",+ 1048576,+ PERFORMANCE_NOISE_FLOOR,+ "less memory",+ "more memory",+ ),+ Definition(+ "loaded_rss",+ "Python RSS after pandas workload",+ 1e-6,+ "MB",+ 1048576,+ PERFORMANCE_NOISE_FLOOR,+ "less memory",+ "more memory",+ ),+)+++def escape(text: str) -> str:+ escaped = (+ text.replace("&", "&")+ .replace("<", "<")+ .replace(">", ">")+ .replace("|", "|")+ .replace("`", "`")+ .replace("\r", " ")+ .replace("\n", " ")+ )+ for character in ("\\", "[", "]", "(", ")", "*", "_", "!"):+ escaped = escaped.replace(character, f"\\{character}")+ return escaped+++def values(samples: list[Observation]) -> list[float]:+ return [s.value for s in samples if s.value is not None]+++def spread(numbers: list[float]) -> float:+ if len(numbers) < 4:+ return max(numbers) - min(numbers) if numbers else 0+ q1, _, q3 = statistics.quantiles(numbers, n=4, method="inclusive")+ return q3 - q1+++def dispersion(samples: list[Observation], definition: Definition) -> str:+ numbers = values(samples)+ if len(numbers) < 2:+ return "—"+ kind = "IQR" if len(numbers) >= 4 else "range"+ return f"{kind} {number(spread(numbers), definition)} {definition.unit}"+++def number(value: float, definition: Definition, signed: bool = False) -> str:+ scaled = value * definition.scale+ precision = (+ 3+ if definition.unit == "ms" and definition.absolute < 0.001+ else (1 if definition.unit == "ms" else 2)+ )+ if scaled and abs(scaled) < 10 ** (-precision):+ return f"{scaled:+.2g}" if signed else f"{scaled:.2g}"+ return f"{scaled:+,.{precision}f}" if signed else f"{scaled:,.{precision}f}"+++def comparison(+ definition: Definition, baseline: list[Observation], candidate: list[Observation], expected: int+) -> tuple[str, str, str, str, str]:+ left, right = values(baseline), values(candidate)+ main = statistics.median(left) if left else None+ head = statistics.median(right) if right else None+ main_text = "—" if main is None else f"{number(main, definition)} {definition.unit}"+ head_text = "—" if head is None else f"{number(head, definition)} {definition.unit}"+ if main is None or head is None:+ return main_text, head_text, "—", "N/A", "unavailable"+ delta = head - main+ percentage = f"{delta / main * 100:+.2f}%" if main else "N/A"+ if len(left) != expected or len(right) != expected:+ return (+ main_text,+ head_text,+ f"— {number(delta, definition, True)} {definition.unit}",+ percentage,+ "incomplete",+ )+ threshold = max(definition.absolute, main * definition.relative, spread(left), spread(right))+ signal, outcome = "≈", "no clear change"+ if abs(delta) > threshold:+ signal, outcome = ("↑", definition.worse) if delta > 0 else ("↓", definition.better)+ change = f"{signal} {number(delta, definition, True)} {definition.unit}"+ if signal == "↑":+ change, outcome = f"**{change}**", f"**{outcome}**"+ return main_text, head_text, change, percentage, outcome+++def render(report: Report) -> str:+ run_url = f"https://github.com/{report.repository}/actions/runs/{report.run_id}"+ lines = [+ MARKER,+ f"<!-- run:{report.run_id}:{report.attempt} head:{report.head_sha} -->",+ f"<!-- comparison:{fingerprint(report)} status:{report.status} -->",+ f"### Prime Agent performance — {report.status}",+ "",+ f"PR `{report.head_sha[:8]}` compared with main `{report.base_sha[:8]}`.",+ *(+ ["Waiting for contributor vouch before credentials or sandboxes are allocated.", ""]+ if report.status == "pending-trust"+ else []+ ),+ "↓ improved · ↑ regressed · ≈ no clear change · — unavailable",+ "",+ "| Metric | Main | This PR | Change | Change % | Result |",+ "| --- | ---: | ---: | ---: | ---: | --- |",+ ]+ for definition in (*METRICS, *RUNTIME_METRICS):+ if definition == RUNTIME_METRICS[0]:+ lines.extend(+ [+ "",+ "**Python runtime**",+ "",+ "| Metric | Main | This PR | Change | Change % | Result |",+ "| --- | ---: | ---: | ---: | ---: | --- |",+ ]+ )+ expected = report.config.install_trials if definition.key == "install" else report.config.trials+ if definition.key in ("bundle", "disk"):+ expected = 1+ cells = comparison(+ definition,+ report.main.metrics.get(definition.key, []),+ report.pr_head.metrics.get(definition.key, []),+ expected,+ )+ lines.append(f"| {definition.title} | {' | '.join(cells)} |")+ compute = sum(s.estimated_usd for s in report.sandboxes)+ cost_text = (+ f"**Sandbox cost: ~${compute:.4f}** — no inference calls."+ if report.sandboxes+ else "Cost pending or unavailable; sandbox usage has not been collected."+ )+ result_link = (+ f"[Run, logs, and downloadable raw results]({run_url})"+ if report.pr+ else "Local run; raw results are stored beside this report."+ )+ lines.extend(+ [+ "",+ cost_text,+ result_link,+ "",+ "<details><summary>Methodology and samples</summary>",+ "",+ f"Main resolved at {report.started_at.isoformat()}. Harness `{report.harness_sha[:8]}`.",+ f"Linux x64, {report.config.cpu_cores:g} vCPU, {report.config.memory_gb:g} GB RAM, "+ f"{report.config.disk_gb:g} GB disk; region {escape(report.config.region)}.",+ f"Image: `{escape(report.config.image)}`.",+ "Stock tools, skills, daemon, and Python bootstrap enabled; fresh homes and a fixed Git fixture.",+ "Onboarding is dismissed; the editor starts without a selected model or submitted prompt.",+ "Medians shown. Arrows require a 20% timing/memory change plus absolute floors and IQR.",+ "These practical noise floors are not a statistical significance test.",+ "Cold means stopped Prime processes; OS filesystem caches are not flushed.",+ "No model requests or credentials. Installation excludes build/setup time.",+ "Installer tarballs use loopback; npm/Python downloads use the network with fresh caches.",+ "Artifact size counts release tarballs; footprint after first use includes registry packages.",+ "MB is decimal. Summed RSS can double-count shared pages; PSS is recorded when available.",+ "Provisioning, setup, and build durations are recorded separately in the raw results.",+ "Kernel probes use the installed JSONL runtime, outside the TUI/TypeScript host.",+ "Per trial: 50 Python cells, 5 calls per shell case, and one 35-cell mix (9 git status calls).",+ "Cell/shell values are batch means; other runtime timings are single operations.",+ "State fixture: a 10,000-row × 8-column integer DataFrame and a 10,000-integer list.",+ "Restore runs in a fresh kernel, including pandas imports; kernel startup is excluded.",+ "Kernel RSS covers the isolated Python process; loaded RSS follows the pandas workload.",+ "Costs estimate full sandbox lifetimes at configured rates, including setup and build.",+ f"Budget target: ${report.config.budget_usd:g}; not a billing cap. Checks are informational.",+ "",+ "| Metric | Main successful/attempted | PR successful/attempted | Main spread | PR spread |",+ "| --- | ---: | ---: | ---: | ---: |",+ ]+ )+ for definition in (*METRICS, *RUNTIME_METRICS):+ left = report.main.metrics.get(definition.key, [])+ right = report.pr_head.metrics.get(definition.key, [])+ lines.append(+ f"| {definition.title} | {len(values(left))}/{len(left)} | {len(values(right))}/{len(right)} | "+ f"{dispersion(left, definition)} | {dispersion(right, definition)} |"+ )+ errors = list(report.errors)+ for name, side in (("main", report.main), ("PR", report.pr_head)):+ if side.error:+ errors.append(f"{name}: {side.error}")+ errors.extend(+ f"{name} {metric} trial {sample.trial}: {sample.error}"+ for metric, samples in side.metrics.items()+ for sample in samples+ if sample.error+ )+ if errors:+ lines.extend(["", "Failures:", ""] + [f"- {escape(error)}" for error in errors[:30]])+ lines.extend(["", "</details>", ""])+ return "\n".join(lines)diff --git a/scripts/benchmarks/schema.py b/scripts/benchmarks/schema.pynew file mode 100644index 0000000000..3446ea63c8--- /dev/null+++ b/scripts/benchmarks/schema.py@@ -0,0 +1,175 @@+from __future__ import annotations++from datetime import datetime+from pathlib import Path+from typing import Annotated, Literal++from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator++SHA = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{40}$")]+Repository = Annotated[str, StringConstraints(pattern=r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")]+Metric = Literal[+ "cold",+ "warm",+ "install",+ "bundle",+ "disk",+ "rss",+ "pss",+ "kernel_start",+ "kernel_exec",+ "bash",+ "git_status",+ "output",+ "mixed",+ "interrupt",+ "snapshot",+ "restore",+ "kernel_rss",+ "loaded_rss",+]+NonNegative = Annotated[float, Field(ge=0, le=1e15, allow_inf_nan=False)]+ROOT = Path(__file__).resolve().parent+UV_VERSION = "0.12.9"+++class StrictModel(BaseModel):+ model_config = ConfigDict(extra="forbid", allow_inf_nan=False, strict=True)+++class Config(StrictModel):+ image: Annotated[str, StringConstraints(pattern=r"^.+@sha256:[0-9a-f]{64}$")]+ region: str+ cpu_cores: Annotated[float, Field(gt=0, le=16)]+ memory_gb: Annotated[float, Field(gt=0, le=64)]+ disk_gb: Annotated[float, Field(gt=0, le=1000)]+ trials: Annotated[int, Field(ge=3, le=50)]+ install_trials: Annotated[int, Field(ge=1, le=10)]+ debounce_seconds: Annotated[int, Field(ge=0, le=300)]+ timeout_seconds: Annotated[int, Field(ge=60, le=3600)]+ ttl_minutes: Annotated[int, Field(ge=1, le=120)]+ budget_usd: Annotated[float, Field(gt=0, le=20)]+ cpu_usd_per_hour: NonNegative+ memory_usd_per_gb_hour: NonNegative+ disk_usd_per_gb_hour: NonNegative++ @model_validator(mode="after")+ def validate_limits(self) -> Config:+ if self.ttl_minutes * 60 <= self.timeout_seconds:+ raise ValueError("Sandbox TTL must exceed the run timeout")+ return self++ @classmethod+ def load(cls, path: Path = ROOT / "config.json") -> Config:+ return cls.model_validate_json(path.read_text())++ def hourly_cost(self) -> float:+ return (+ self.cpu_cores * self.cpu_usd_per_hour+ + self.memory_gb * self.memory_usd_per_gb_hour+ + self.disk_gb * self.disk_usd_per_gb_hour+ )+++class Observation(StrictModel):+ trial: Annotated[int, Field(ge=0, le=100)]+ value: NonNegative | None = None+ error: Annotated[str, Field(max_length=500)] | None = None++ @model_validator(mode="after")+ def validate_outcome(self) -> Observation:+ if (self.value is None) == (self.error is None):+ raise ValueError("An observation must contain either a value or an error")+ return self+++class ProcessMemory(StrictModel):+ pid: Annotated[int, Field(gt=0)]+ name: Annotated[str, Field(max_length=80)]+ rss: NonNegative+ pss: NonNegative | None = None+++class Side(StrictModel):+ sha: SHA+ metrics: dict[Metric, list[Observation]] = Field(default_factory=dict)+ processes: list[ProcessMemory] = Field(default_factory=list, max_length=128)+ runtime: dict[str, str] = Field(default_factory=dict)+ timings: dict[Literal["provision", "setup", "build"], NonNegative] = Field(default_factory=dict)+ artifacts: dict[str, NonNegative] = Field(default_factory=dict)+ error: Annotated[str, Field(max_length=500)] | None = None+++class Request(StrictModel):+ repository: Repository+ source_repository: Repository+ sha: SHA+ harness_sha: SHA+ pr: Annotated[int, Field(ge=0)]+ run_id: Annotated[int, Field(ge=1)]+ attempt: Annotated[int, Field(ge=1)]+ role: Literal["main", "pr"]+ config: Config+++class Result(StrictModel):+ request: Request+ side: Side+++class SandboxUsage(StrictModel):+ id: Annotated[str, StringConstraints(pattern=r"^[a-zA-Z0-9_-]+$")]+ role: Literal["main", "pr"]+ seconds: NonNegative+ estimated_usd: NonNegative+ deleted: bool+++class Report(StrictModel):+ schema_version: Literal[1] = 1+ repository: Repository+ head_repository: Repository+ pr: Annotated[int, Field(ge=0)]+ run_id: Annotated[int, Field(ge=1)]+ attempt: Annotated[int, Field(ge=1)] = 1+ harness_sha: SHA+ base_sha: SHA+ head_sha: SHA+ started_at: datetime+ finished_at: datetime | None = None+ config: Config+ status: Literal["pending-trust", "running", "completed", "partial", "failed", "canceled"] = "running"+ main: Side+ pr_head: Side+ sandboxes: list[SandboxUsage] = Field(default_factory=list, max_length=2)+ errors: list[Annotated[str, Field(max_length=500)]] = Field(default_factory=list, max_length=30)++ @model_validator(mode="after")+ def validate_revisions(self) -> Report:+ if self.main.sha != self.base_sha or self.pr_head.sha != self.head_sha:+ raise ValueError("Result revisions do not match the request")+ for side in (self.main, self.pr_head):+ for metric, samples in side.metrics.items():+ expected = self.config.install_trials if metric == "install" else self.config.trials+ if metric in ("bundle", "disk"):+ expected = 1+ if (+ len(samples) > expected+ or len({s.trial for s in samples}) != len(samples)+ or any(sample.trial >= expected for sample in samples)+ ):+ raise ValueError("Duplicate or excessive observations")+ return self+++def write_json(path: Path, value: BaseModel) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ temporary = path.with_suffix(".tmp")+ temporary.write_text(value.model_dump_json(indent=2) + "\n")+ temporary.replace(path)+++def load_report(path: Path) -> Report:+ if path.stat().st_size > 2_000_000:+ raise ValueError("Report exceeds the size limit")+ return Report.model_validate_json(path.read_text())diff --git a/scripts/benchmarks/terminal.py b/scripts/benchmarks/terminal.pynew file mode 100644index 0000000000..0c92ba7244--- /dev/null+++ b/scripts/benchmarks/terminal.py@@ -0,0 +1,131 @@+from __future__ import annotations++import time+from collections.abc import Callable+from pathlib import Path++import pexpect+import pyte++QUERIES = {+ "\x1b[6n": "\x1b[1;1R",+ "\x1b[?6n": "\x1b[?1;1R",+ "\x1b[c": "\x1b[?1;2c",+ "\x1b[0c": "\x1b[?1;2c",+ "\x1b[>c": "\x1b[>0;276;0c",+ "\x1b[>0c": "\x1b[>0;276;0c",+ "\x1b[?u": "\x1b[?0u",+ "\x1b[?2026$p": "\x1b[?2026;2$y",+ "\x1b[14t": "\x1b[4;640;960t",+ "\x1b[16t": "\x1b[6;16;8t",+ "\x1b[18t": "\x1b[8;40;120t",+}+for terminator in ("\x07", "\x1b\\"):+ QUERIES[f"\x1b]11;?{terminator}"] = "\x1b]11;rgb:0000/0000/0000\x1b\\"+ QUERIES[f"\x1b]10;?{terminator}"] = "\x1b]10;rgb:eeee/eeee/eeee\x1b\\"+++class Display:+ def __init__(self, reply: Callable[[str], object]):+ self.screen = pyte.Screen(120, 40)+ self.stream = pyte.Stream(self.screen)+ self.reply = reply+ self.pending = ""++ def feed(self, chunk: str) -> None:+ self.pending += chunk+ while True:+ matches = [(self.pending.find(query), query) for query in QUERIES if query in self.pending]+ if not matches:+ break+ position, query = min(matches)+ self.stream.feed(self.pending[:position])+ self.reply(QUERIES[query])+ self.pending = self.pending[position + len(query) :]+ keep = max(+ (+ length+ for query in QUERIES+ for length in range(1, len(query))+ if self.pending.endswith(query[:length])+ ),+ default=0,+ )+ if keep:+ self.stream.feed(self.pending[:-keep])+ self.pending = self.pending[-keep:]+ else:+ self.stream.feed(self.pending)+ self.pending = ""++ def text(self) -> str:+ return "\n".join(row.rstrip() for row in self.screen.display)+++class Terminal:+ def __init__(self, command: list[str], cwd: Path, env: dict[str, str], transcript: Path):+ self.started = time.perf_counter()+ self.child = pexpect.spawn(+ command[0],+ command[1:],+ cwd=str(cwd),+ env=env,+ encoding="utf-8",+ codec_errors="replace",+ dimensions=(40, 120),+ echo=False,+ )+ self.child.delaybeforesend = 0+ self.display = Display(self.child.send)+ self.transcript = transcript+ self.raw: list[str] = []+ self.bytes = 0++ def pump(self) -> bool:+ try:+ chunk = self.child.read_nonblocking(65536, timeout=0.005)+ except pexpect.TIMEOUT:+ return False+ except pexpect.EOF as error:+ raise RuntimeError("Prime Agent exited before the measurement completed") from error+ self.bytes += len(chunk)+ if self.bytes > 8_000_000:+ raise RuntimeError("Terminal transcript exceeds 8 MB")+ self.raw.append(chunk)+ self.display.feed(chunk)+ return True++ def until(self, predicate: Callable[[Display], bool], seconds: float) -> float:+ deadline = time.perf_counter() + seconds+ while time.perf_counter() < deadline:+ if self.pump() and predicate(self.display):+ return time.perf_counter()+ raise TimeoutError("Timed out waiting for the expected terminal state")++ def settle(self, seconds: float) -> None:+ deadline = time.perf_counter() + seconds+ while time.perf_counter() < deadline:+ self.pump()++ def ready(self) -> float:+ self.until(lambda display: "agents/resume" in display.text(), 30)+ self.child.send("benchready")+ echoed = self.until(lambda display: "benchready" in display.text(), 5)+ self.child.send("\x7f" * len("benchready"))+ self.until(lambda display: "benchready" not in display.text(), 5)+ return echoed - self.started++ def close(self) -> None:+ raw, snapshot = "".join(self.raw), self.display.text()+ self.transcript.with_suffix(".raw").write_text(raw)+ self.transcript.with_suffix(".txt").write_text(snapshot)+ try:+ if self.child.isalive():+ self.child.sendcontrol("c")+ time.sleep(0.15)+ if self.child.isalive():+ self.child.sendcontrol("c")+ except (OSError, pexpect.EOF):+ pass+ finally:+ self.child.close(force=True)diff --git a/scripts/benchmarks/tests/calibration.json b/scripts/benchmarks/tests/calibration.jsonnew file mode 100644index 0000000000..badaa4d414--- /dev/null+++ b/scripts/benchmarks/tests/calibration.json@@ -0,0 +1,398 @@+{+ "baseline_sha": "427ea4c72cc606ac061a14287c0c15c471eff002",+ "recorded_at": "2026-09-09T21:30:42.632129Z",+ "metrics": {+ "bundle": {+ "main": [+ 11096130.0+ ],+ "pr": [+ 11096130.0+ ]+ },+ "install": {+ "main": [+ 27.877250564000008,+ 28.70503522299998,+ 29.183944166000003+ ],+ "pr": [+ 30.62683971300001,+ 32.43433822399999,+ 36.622502448999995+ ]+ },+ "cold": {+ "main": [+ 2.9279603130000282,+ 2.9515973970000005,+ 2.864970442000015,+ 2.9641698160000374,+ 2.843047642999977,+ 2.769899554999995,+ 2.87473634600002,+ 2.977558033999969,+ 2.922310485999958,+ 3.0185121560000425+ ],+ "pr": [+ 3.200634946999969,+ 3.0652513259999523,+ 3.3256505719999723,+ 3.0905467590000057,+ 3.5100812420000125,+ 3.551760985000044,+ 3.3178279680000173,+ 3.235152325999991,+ 3.2145795829999884,+ 3.216496445999951+ ]+ },+ "rss": {+ "main": [+ 1099157504.0,+ 1038544896.0,+ 961441792.0,+ 1009164288.0,+ 1042464768.0,+ 987299840.0,+ 968232960.0,+ 909803520.0,+ 971235328.0,+ 1116012544.0+ ],+ "pr": [+ 1045270528.0,+ 1166790656.0,+ 1030512640.0,+ 989421568.0,+ 1112367104.0,+ 1099530240.0,+ 1153560576.0,+ 1118666752.0,+ 1084510208.0,+ 1133363200.0+ ]+ },+ "warm": {+ "main": [+ 1.7345433609999645,+ 1.7651198379999755,+ 1.7687801519999766,+ 1.762977601999978,+ 1.6944376189999844,+ 1.642891163999991,+ 1.6524441209999736,+ 1.7300561220000077,+ 1.6743516009999553,+ 1.7221875839999257+ ],+ "pr": [+ 1.7706765419999897,+ 1.8812752079999768,+ 1.8848265259999835,+ 1.773448736999967,+ 1.854215795000016,+ 1.9367081579999876,+ 1.8299759000000222,+ 1.9682038950000447,+ 1.8523304409999923,+ 1.8055133089999913+ ]+ },+ "disk": {+ "main": [+ 597316531.0+ ],+ "pr": [+ 597315364.0+ ]+ },+ "kernel_start": {+ "main": [+ 0.12121383599992441,+ 0.12723471499998595,+ 0.1409247950000463,+ 0.12776771299991196,+ 0.12564238899994962,+ 0.13082783800007292,+ 0.12394591999998283,+ 0.1299987349999583,+ 0.11980933699999241,+ 0.1293355980000115+ ],+ "pr": [+ 0.14848755700000993,+ 0.1377638199999751,+ 0.12577888999999232,+ 0.14028942499999175,+ 0.12999939600001653,+ 0.18843824300000733,+ 0.16141352499994355,+ 0.1650210259999767,+ 0.15177539300009357,+ 0.12403774500000964+ ]+ },+ "kernel_rss": {+ "main": [+ 35270656.0,+ 31350784.0,+ 37015552.0,+ 35385344.0,+ 37322752.0,+ 35954688.0,+ 37765120.0,+ 36163584.0,+ 38535168.0,+ 35065856.0+ ],+ "pr": [+ 35020800.0,+ 36134912.0,+ 30023680.0,+ 36614144.0,+ 34148352.0,+ 33996800.0,+ 33341440.0,+ 38506496.0,+ 32722944.0,+ 36675584.0+ ]+ },+ "kernel_exec": {+ "main": [+ 0.0005549562799978957,+ 0.000465069780000249,+ 0.0005641982199995255,+ 0.000583243199998833,+ 0.0005096060599998963,+ 0.0005911986399996749,+ 0.0005406058400012625,+ 0.0004923386999985269,+ 0.0007612591199995223,+ 0.0005891538199989555+ ],+ "pr": [+ 0.0007702345799998512,+ 0.0006107251399998858,+ 0.0005365984600007323,+ 0.000595193220000283,+ 0.000712963140000511,+ 0.0006294563200003723,+ 0.000602465059998849,+ 0.000630622300000141,+ 0.0006821516199988764,+ 0.0006074899800000822+ ]+ },+ "bash": {+ "main": [+ 0.012265951599988511,+ 0.011434986400013258,+ 0.012523944600002324,+ 0.012188101599986113,+ 0.012133405999998103,+ 0.013145084400002815,+ 0.011684339800012822,+ 0.010691210599998158,+ 0.014503767800010791,+ 0.013724902000012662+ ],+ "pr": [+ 0.015500308199989377,+ 0.012726768200002425,+ 0.013086532799991345,+ 0.012979604999998173,+ 0.012928957999997692,+ 0.013177246399993691,+ 0.014024491799978023,+ 0.012863739800013718,+ 0.014560996599993815,+ 0.013492744999985007+ ]+ },+ "git_status": {+ "main": [+ 0.017970771399996012,+ 0.01924374820000594,+ 0.018154279200007294,+ 0.017899456400004964,+ 0.017645751600002767,+ 0.018604173399990032,+ 0.01732922300000155,+ 0.017314914799999314,+ 0.01831345020000299,+ 0.019714569600000686+ ],+ "pr": [+ 0.02181655580000097,+ 0.018099323399997046,+ 0.021713121400000545,+ 0.01987777339999184,+ 0.019425245200000062,+ 0.01948336739999377,+ 0.0200670867999861,+ 0.02184636980000505,+ 0.018714642200006893,+ 0.020085525600006805+ ]+ },+ "output": {+ "main": [+ 0.01278957699998955,+ 0.012213238399999682,+ 0.011799004199997399,+ 0.014527103199998236,+ 0.013296984999988127,+ 0.013471006999998281,+ 0.012347434999992402,+ 0.011561371400011922,+ 0.012395569399996021,+ 0.012976578400002835+ ],+ "pr": [+ 0.019721852200007106,+ 0.01687608419999833,+ 0.015285829599997669,+ 0.012512077600001703,+ 0.014360300200007715,+ 0.016800876600007085,+ 0.013386880000007295,+ 0.016856159600001773,+ 0.01287674860000152,+ 0.014394774199990934+ ]+ },+ "mixed": {+ "main": [+ 0.18327940200003923,+ 0.1656647150000481,+ 0.18445866100000785,+ 0.1879419299999654,+ 0.18152489999999943,+ 0.18734949900010633,+ 0.18109903399999894,+ 0.16189230999998472,+ 0.17762886299999536,+ 0.19014139399996566+ ],+ "pr": [+ 0.24294485600000826,+ 0.19827466099997082,+ 0.24033736699999508,+ 0.2019193049999899,+ 0.20282605300002388,+ 0.2027119749999997,+ 0.20539985499999602,+ 0.20633829200005493,+ 0.1809149719999823,+ 0.21170844000005218+ ]+ },+ "interrupt": {+ "main": [+ 0.0017507960000102685,+ 0.0015083110000659872,+ 0.0017416609999827415,+ 0.0016143720000627582,+ 0.0018679830000110087,+ 0.0016709809999611025,+ 0.0024465850000296996,+ 0.0013896130000148332,+ 0.001642629000002671,+ 0.0018307729999378353+ ],+ "pr": [+ 0.001693358000011358,+ 0.0017097159999934775,+ 0.001485645000002478,+ 0.001690604000032181,+ 0.0017368419999570506,+ 0.0015665369999737777,+ 0.0014975740000409132,+ 0.0015429750000066633,+ 0.0015613769999163196,+ 0.001896525000006477+ ]+ },+ "loaded_rss": {+ "main": [+ 107814912.0,+ 93478912.0,+ 98910208.0,+ 97415168.0,+ 99360768.0,+ 98050048.0,+ 99872768.0,+ 98254848.0,+ 100601856.0,+ 97099776.0+ ],+ "pr": [+ 107479040.0,+ 98242560.0,+ 90345472.0,+ 96727040.0,+ 96256000.0,+ 96059392.0,+ 95399936.0,+ 100548608.0,+ 94867456.0,+ 98779136.0+ ]+ },+ "snapshot": {+ "main": [+ 0.078061469999966,+ 0.025671378000083678,+ 0.029300445999979274,+ 0.027639665000037894,+ 0.025642438999966544,+ 0.026381777999972655,+ 0.025477492000050006,+ 0.027599351999924693,+ 0.025847754999972494,+ 0.028655434000029345+ ],+ "pr": [+ 0.09319571300000007,+ 0.03817890999999918,+ 0.02773127599999725,+ 0.03216677000000345,+ 0.035073244000045634,+ 0.029259709999962524,+ 0.02817944000003081,+ 0.028553541999940535,+ 0.026783507000004647,+ 0.02539736100004575+ ]+ },+ "restore": {+ "main": [+ 0.36483748199998445,+ 0.3538136629999826,+ 0.3790124850000893,+ 0.3602016209999874,+ 0.3554445230000738,+ 0.3486341860000266,+ 0.3896290149999686,+ 0.3843983080000726,+ 0.40788538100002825,+ 0.37759137900002315+ ],+ "pr": [+ 0.48703021900001886,+ 0.44926845599997023,+ 0.39550326900001664,+ 0.42711700600000313,+ 0.4238764670000137,+ 0.4049739440000053,+ 0.42803659699995933,+ 0.38498759599997356,+ 0.39857957699996405,+ 0.3963605919999509+ ]+ }+ }+}diff --git a/scripts/benchmarks/tests/test_benchmarks.py b/scripts/benchmarks/tests/test_benchmarks.pynew file mode 100644index 0000000000..554db5d3a1--- /dev/null+++ b/scripts/benchmarks/tests/test_benchmarks.py@@ -0,0 +1,511 @@+from __future__ import annotations++import json+import os+import subprocess+import sys+import tempfile+import unittest+from datetime import UTC, datetime+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import Mock, patch++from pydantic import ValidationError++from cli import completed_report, main, validate_completion, workflow_source+from controller import Canceled, Controller, cleanup, labels, side_complete+from github import TITLE, GitHub+from report import MARKER, METRICS, RUNTIME_METRICS, comparison, render+from schema import (+ Config,+ Observation,+ ProcessMemory,+ Report,+ Request,+ Side,+ load_report,+ write_json,+)+from terminal import QUERIES, Display, Terminal+from worker import environment, install, measure++SHA = "a" * 40+HEAD = "b" * 40+++def fixture() -> Report:+ return Report(+ repository="PrimeIntellect-ai/prime-agent",+ head_repository="PrimeIntellect-ai/prime-agent",+ pr=42,+ run_id=100,+ attempt=1,+ harness_sha=SHA,+ base_sha=SHA,+ head_sha=HEAD,+ started_at=datetime(2026, 9, 9, tzinfo=UTC),+ config=Config.load(),+ main=Side(sha=SHA),+ pr_head=Side(sha=HEAD),+ )+++def observations(*numbers: float) -> list[Observation]:+ return [Observation(trial=index, value=value) for index, value in enumerate(numbers)]+++class TerminalTests(unittest.TestCase):+ def test_real_pty_detects_injected_startup_delays_without_submitting(self):+ script = """+import os, sys, time, tty+tty.setraw(0)+time.sleep(float(sys.argv[1]))+os.write(1, b'agents/resume\\r\\n> ')+while True:+ byte = os.read(0, 1)+ if byte == b'\\x7f':+ os.write(1, b'\\b \\b')+ elif byte == b'\\r':+ raise RuntimeError('benchmark submitted a prompt')+ elif byte == b'\\x03':+ break+ else:+ os.write(1, byte)+"""+ measurements = []+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ path = root / "fixture.py"+ path.write_text(script)+ for delay in (0.05, 0.6):+ terminal = Terminal(+ [sys.executable, str(path), str(delay)],+ root,+ os.environ.copy(),+ root / f"transcript-{delay}",+ )+ try:+ measurements.append(terminal.ready())+ finally:+ terminal.close()+ self.assertGreater(measurements[1] - measurements[0], 0.25)++ def test_queries_split_at_every_boundary(self):+ for query, reply in QUERIES.items():+ for boundary in range(len(query) + 1):+ with self.subTest(query=query, boundary=boundary):+ replies = []+ display = Display(replies.append)+ display.feed("prefix" + query[:boundary])+ display.feed(query[boundary:] + "suffix")+ self.assertEqual(replies, [reply])+ self.assertTrue(display.text().startswith("prefixsuffix"))+++class ReportTests(unittest.TestCase):+ def test_same_revision_sandbox_calibration_does_not_report_regressions(self):+ calibration = json.loads(Path(__file__).with_name("calibration.json").read_text())+ for definition in (*METRICS, *RUNTIME_METRICS):+ with self.subTest(metric=definition.key):+ samples = calibration["metrics"][definition.key]+ result = comparison(+ definition,+ observations(*samples["main"]),+ observations(*samples["pr"]),+ len(samples["main"]),+ )+ self.assertEqual(result[-1], "no clear change")++ def test_slower_and_faster_have_consistent_signs_and_arrows(self):+ baseline = observations(*([2.0] * 10))+ slower = comparison(METRICS[0], baseline, observations(*([2.5] * 10)), 10)+ faster = comparison(METRICS[0], baseline, observations(*([1.5] * 10)), 10)+ self.assertEqual(slower[2:], ("**↑ +500.0 ms**", "+25.00%", "**slower**"))+ self.assertEqual(faster[2:], ("↓ -500.0 ms", "-25.00%", "faster"))++ def test_noise_and_partial_results_are_not_regressions(self):+ noisy = observations(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)+ shifted = observations(1.2, 2.2, 3.2, 4.2, 5.2, 6.2, 7.2, 8.2, 9.2, 10.2)+ self.assertEqual(comparison(METRICS[0], noisy, shifted, 10)[4], "no clear change")+ shifted[-1] = Observation(trial=9, error="timed out")+ self.assertEqual(comparison(METRICS[0], noisy, shifted, 10)[4], "incomplete")+ self.assertEqual(comparison(METRICS[0], [], shifted, 10)[4], "unavailable")++ def test_zero_baseline_and_small_bundle_changes(self):+ cells = comparison(METRICS[3], observations(0), observations(65537), 1)+ self.assertEqual(cells[3], "N/A")+ self.assertIn("larger", cells[4])+ unchanged = comparison(METRICS[3], observations(10_000_000), observations(10_001_000), 1)+ self.assertEqual(unchanged[4], "no clear change")++ def test_comment_escapes_untrusted_text_and_reports_failures(self):+ report = fixture()+ report.pr_head.error = "<img src=x> | \n<!-- comment -->"+ text = render(report)+ self.assertTrue(text.startswith(MARKER))+ self.assertNotIn("<img", text)+ self.assertNotIn("![click]", text)+ self.assertIn("|", text)+ self.assertIn("0/0", text)+ self.assertNotIn("TTFT", text)+ self.assertNotIn("Pinference", text)+ for definition in RUNTIME_METRICS:+ self.assertIn(definition.title, text)++ def test_schema_rejects_nonfinite_negative_duplicate_and_wrong_revision(self):+ for value in (float("nan"), float("inf"), -1, True, "1"):+ with self.assertRaises(ValidationError):+ Observation(trial=0, value=value)+ report = fixture().model_dump()+ report["main"]["sha"] = HEAD+ with self.assertRaises(ValidationError):+ Report.model_validate(report)+ report = fixture().model_dump()+ report["main"]["metrics"] = {"cold": [{"trial": 0, "value": 1}] * 2}+ with self.assertRaises(ValidationError):+ Report.model_validate(report)++ def test_report_round_trip_and_size_limit(self):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "report.json"+ report = fixture()+ write_json(path, report)+ self.assertEqual(load_report(path), report)+ path.write_bytes(b" " * 2_000_001)+ with self.assertRaises(ValueError):+ load_report(path)+++class FakeGitHub(GitHub):+ def __init__(self):+ self.head = HEAD+ self.state = "open"+ self.attempt = 1+ self.runs = [{"id": 100, "run_number": 10, "run_attempt": 1, "display_title": f"{TITLE}42"}]+ self.comments = []+ self.writes = []++ def request(self, method, path, body=None):+ if method != "GET":+ self.writes.append((method, path, body))+ return {}+ if path == "pulls/42":+ return {"state": self.state, "head": {"sha": self.head}}+ if path == "actions/runs/100":+ return {"run_attempt": self.attempt, "run_number": 10}+ raise AssertionError(path)++ def pages(self, path, key=None):+ yield self.runs if key else self.comments+++class PublishingTests(unittest.TestCase):+ def test_superseded_commit_and_same_commit_rerun_do_not_publish(self):+ for case in ("head", "new_run", "attempt", "closed"):+ with self.subTest(case=case):+ github = FakeGitHub()+ if case == "head":+ github.head = SHA+ elif case == "new_run":+ github.runs.append(+ {"id": 101, "run_number": 11, "run_attempt": 1, "display_title": f"{TITLE}42"}+ )+ elif case == "attempt":+ github.attempt = 2+ else:+ github.state = "closed"+ self.assertFalse(github.publish(fixture()))+ self.assertEqual(github.writes, [])++ def test_updates_only_its_own_sticky_comment(self):+ github = FakeGitHub()+ github.comments = [+ {"id": 1, "user": {"login": "someone"}, "body": MARKER},+ {"id": 2, "user": {"login": "github-actions[bot]"}, "body": MARKER},+ ]+ self.assertTrue(github.publish(fixture()))+ self.assertEqual(github.writes[0][:2], ("PATCH", "issues/comments/2"))++ def test_newer_comment_is_not_overwritten(self):+ github = FakeGitHub()+ github.comments = [+ {+ "id": 2,+ "user": {"login": "github-actions[bot]"},+ "body": f"{MARKER}\n<!-- run:101:1 head:{HEAD} -->",+ }+ ]+ self.assertFalse(github.publish(fixture()))+ self.assertEqual(github.writes, [])+++class LifecycleTests(unittest.TestCase):+ def test_sandbox_creation_does_not_inject_credentials(self):+ with (+ tempfile.TemporaryDirectory() as directory,+ patch.dict(os.environ, {"PRIME_SANDBOX_API_KEY": "fake", "PINFERENCE_API_KEY": "must-not-use"}),+ ):+ controller = Controller(fixture(), Path(directory), live_github=False)+ controller.client = Mock()+ controller.client.create.return_value = SimpleNamespace(id="one")+ controller.client.get.return_value = SimpleNamespace(status="RUNNING")+ controller.client.execute_command.return_value = SimpleNamespace(exit_code=0)+ controller.wait = Mock()+ controller.start("main")+ request = controller.client.create.call_args.args[0]+ self.assertIsNone(request.secrets)+ self.assertIsNone(request.environment_vars)++ def test_invalid_results_get_a_failure_notice_and_missing_results_preserve_pending_trust(self):+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ request, result = root / "request.json", root / "result.json"+ run = {"id": 100, "run_attempt": 1, "display_title": f"{TITLE}42", "conclusion": "cancelled"}+ write_json(request, fixture())+ self.assertEqual(completed_report(result, request, run).status, "canceled")+ result.write_text('{"forged": true}')+ report = completed_report(result, request, run)+ self.assertEqual(report.status, "failed")+ self.assertIn("validation", report.errors[0])+ result.unlink()+ pending = fixture()+ pending.status = "pending-trust"+ write_json(request, pending)+ self.assertEqual(completed_report(result, request, run).status, "pending-trust")+ self.assertIn("Waiting for contributor vouch", render(pending))++ def test_automatic_duplicates_skip_but_manual_runs_and_new_comparisons_do_not(self):+ for case in ("duplicate", "manual", "attempt", "main", "harness", "config", "failed"):+ with self.subTest(case=case), tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ event, output = root / "event.json", root / "output.txt"+ event.write_text('{"pull_request": {"number": 42}}')+ old = fixture()+ old.status = "completed" if case != "failed" else "partial"+ github = FakeGitHub()+ github.comments = [{"user": {"login": "github-actions[bot]"}, "body": render(old)}]+ report = fixture()+ if case == "main":+ report.base_sha = report.main.sha = "c" * 40+ elif case == "harness":+ report.harness_sha = "c" * 40+ elif case == "config":+ report.config.trials = 11+ elif case == "attempt":+ report.attempt = 2+ github.resolve = Mock(return_value=(report, "kevin"))+ with (+ patch("cli.GitHub", return_value=github),+ patch.object(sys, "argv", ["cli.py", "resolve", "--results", directory]),+ patch.dict(+ os.environ,+ {+ "GITHUB_EVENT_PATH": str(event),+ "GITHUB_OUTPUT": str(output),+ "GITHUB_EVENT_NAME": "workflow_dispatch"+ if case == "manual"+ else "pull_request_target",+ "GITHUB_SHA": SHA,+ "GITHUB_RUN_ID": "100",+ "GITHUB_RUN_ATTEMPT": str(report.attempt),+ },+ ),+ ):+ main()+ self.assertIn(f"needed={'false' if case == 'duplicate' else 'true'}", output.read_text())+ self.assertEqual(github.writes, [])++ def test_cleanup_paginates_before_deleting_and_checks_all_labels(self):+ client = Mock()+ expected = labels("owner/repo", 100, 2)+ first = SimpleNamespace(id="one", labels=expected)+ wrong = SimpleNamespace(id="other-attempt", labels=labels("owner/repo", 100, 1))+ second = SimpleNamespace(id="two", labels=expected)+ client.list.side_effect = [+ SimpleNamespace(sandboxes=[first, wrong], has_next=True),+ SimpleNamespace(sandboxes=[second], has_next=False),+ ]+ self.assertEqual(cleanup(client, "owner/repo", 100, 2), ["one", "two"])+ self.assertEqual([call[0] for call in client.mock_calls], ["list", "list", "delete", "delete"])++ def test_cleanup_attempts_remaining_deletions_after_failure(self):+ client = Mock()+ expected = labels("owner/repo", 1, 1)+ client.list.return_value = SimpleNamespace(+ sandboxes=[+ SimpleNamespace(id="one", labels=expected),+ SimpleNamespace(id="two", labels=expected),+ ],+ has_next=False,+ )+ client.delete.side_effect = [RuntimeError("unavailable"), None]+ with self.assertRaises(RuntimeError):+ cleanup(client, "owner/repo", 1, 1)+ self.assertEqual(client.delete.call_count, 2)++ def test_cancellation_still_deletes_a_created_sandbox(self):+ with (+ tempfile.TemporaryDirectory() as directory,+ patch.dict(os.environ, {"PRIME_SANDBOX_API_KEY": "fake"}),+ ):+ report = fixture()+ report.config.debounce_seconds = 0+ controller = Controller(report, Path(directory), live_github=False)+ controller.client = Mock()+ controller.logs = Mock()+ sandbox = SimpleNamespace(id="one", created_at=datetime.now(UTC).replace(tzinfo=None))++ def start(_role):+ controller.sandboxes["main"] = sandbox+ raise Canceled()++ controller.start = start+ controller.run()+ controller.client.delete.assert_called_once_with("one")+ self.assertEqual(load_report(Path(directory) / "report.json").status, "canceled")++ def test_workflow_source_rejects_untrusted_triggers(self):+ event = {+ "workflow_run": {+ "event": "pull_request",+ "path": ".github/workflows/benchmarks.yml",+ "head_repository": {"full_name": "owner/repo"},+ }+ }+ with patch.dict(os.environ, {"GITHUB_REPOSITORY": "owner/repo"}), self.assertRaises(ValueError):+ workflow_source(event)++ def test_fork_metadata_does_not_confuse_pr_head_with_trusted_harness(self):+ run = {+ "event": "pull_request_target",+ "path": ".github/workflows/benchmarks.yml",+ "repository": {"full_name": "PrimeIntellect-ai/prime-agent"},+ "head_repository": {"full_name": "contributor/prime-agent"},+ "head_sha": HEAD,+ "id": 100,+ "run_attempt": 1,+ "display_title": f"{TITLE}42",+ }+ with patch.dict(os.environ, {"GITHUB_REPOSITORY": "PrimeIntellect-ai/prime-agent"}):+ self.assertEqual(workflow_source({"workflow_run": run}), run)+ validate_completion(fixture(), fixture(), run)+ forged = fixture()+ forged.harness_sha = HEAD+ with self.assertRaises(ValueError):+ validate_completion(forged, fixture(), run)++ def test_dispatch_from_a_feature_branch_cannot_publish_or_clean_up(self):+ run = {+ "event": "workflow_dispatch",+ "path": ".github/workflows/benchmarks.yml",+ "repository": {"full_name": "owner/repo"},+ "head_branch": "untrusted",+ }+ with patch.dict(os.environ, {"GITHUB_REPOSITORY": "owner/repo"}), self.assertRaises(ValueError):+ workflow_source({"workflow_run": run})+++class MeasurementTests(unittest.TestCase):+ def test_disk_footprint_is_measured_after_interactive_first_use(self):+ order = []+ terminal = Mock()+ terminal.ready.return_value = 0.5+ terminal.close.side_effect = lambda *_args: order.append("closed")+ side = Side(+ sha=SHA, metrics={"install": observations(1)}, runtime={"home_before_install_bytes": "100"}+ )+ report = fixture()+ request = Request(+ repository=report.repository,+ source_repository=report.repository,+ sha=SHA,+ harness_sha=SHA,+ pr=42,+ run_id=100,+ attempt=1,+ role="main",+ config=report.config,+ )++ def disk(_home):+ self.assertEqual(order, ["closed", "closed"])+ return 700++ with (+ patch("worker.Terminal", return_value=terminal) as launch,+ patch("worker.memory", return_value=[ProcessMemory(pid=1, name="agent", rss=100)]),+ patch("worker.pwd.getpwnam", return_value=SimpleNamespace(pw_uid=1)),+ patch("worker.stop_processes"),+ patch("worker.stop_agents"),+ patch("worker.write_json"),+ patch("worker.disk_bytes", side_effect=disk),+ ):+ measure(request, side, 0)+ self.assertEqual(side.metrics["disk"][0].value, 600)+ self.assertIsNotNone(side.metrics["cold"][0].value)+ terminal.child.send.assert_not_called()+ for call in launch.call_args_list:+ self.assertEqual(call.args[0], ["/usr/sbin/runuser", "-u", "benchmark1", "--", "prime-agent"])+ self.assertNotIn("PRIME_API_KEY", call.args[2])++ def test_dependency_inventory_warning_preserves_successful_installation(self):+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ home = root / "benchmark1"+ (home / ".prime/agent/kernel-venv/bin").mkdir(parents=True)+ (home / ".prime/agent/kernel-venv/bin/python").touch()+ side = Side(sha=SHA)++ def run_as(_user, command, *_args, **_kwargs):+ if command[:2] == ["npm", "ls"]:+ raise subprocess.CalledProcessError(1, command)+ return "0.0.0-benchmark"++ with (+ patch("worker.HOMES", root),+ patch("worker.RESULTS", root),+ patch("worker.subprocess.run"),+ patch("worker.run_as", side_effect=run_as),+ patch("worker.disk_bytes", side_effect=[100, 300]),+ patch("worker.os.chown"),+ patch("worker.pwd.getpwnam", return_value=SimpleNamespace(pw_uid=1, pw_gid=1)),+ patch("worker.stop_processes"),+ ):+ install(SimpleNamespace(config=Config.load()), side, 0)+ self.assertIsNotNone(side.metrics["install"][0].value)+ self.assertIsNone(side.metrics["install"][0].error)+ self.assertIn("warnings", side.runtime["dependency_inventory"])+ self.assertTrue((home / "workspace/example.py").exists())+ self.assertNotIn("disk", side.metrics)++ def test_child_environment_never_inherits_credentials_or_personal_settings(self):+ with patch.dict(+ os.environ,+ {+ "PRIME_API_KEY": "fake",+ "PINFERENCE_API_KEY": "fake",+ "PRIME_SANDBOX_API_KEY": "fake",+ "GITHUB_TOKEN": "fake",+ "PRIME_TEAM_ID": "private-team",+ "PI_CODING_AGENT_DIR": "/personal",+ },+ ):+ env = environment("benchmark1")+ self.assertFalse(any("KEY" in name or "TOKEN" in name or "TEAM" in name for name in env))+ self.assertNotIn("PI_CODING_AGENT_DIR", env)++ def test_completion_requires_every_metric(self):+ side = Side(sha=SHA)+ for definition in (*METRICS, *RUNTIME_METRICS):+ count = 1 if definition.key in ("bundle", "disk") else (3 if definition.key == "install" else 10)+ side.metrics[definition.key] = observations(*([1] * count))+ self.assertTrue(side_complete(side, 10, 3))+ for definition in (*METRICS, *RUNTIME_METRICS):+ incomplete = side.model_copy(deep=True)+ incomplete.metrics[definition.key][-1] = Observation(trial=0, error="failed")+ self.assertFalse(side_complete(incomplete, 10, 3))+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/benchmarks/tests/test_kernel.py b/scripts/benchmarks/tests/test_kernel.pynew file mode 100644index 0000000000..fd78506591--- /dev/null+++ b/scripts/benchmarks/tests/test_kernel.py@@ -0,0 +1,110 @@+from __future__ import annotations++import os+import sys+import tempfile+import time+import unittest+from pathlib import Path++from kernel import OUTPUT_BYTES, Kernel++FIXTURE = r"""+import json, os, sys, time++def emit(event):+ data = (json.dumps(event) + "\n").encode()+ for index in range(0, len(data), 17):+ os.write(1, data[index:index + 17])++time.sleep(float(sys.argv[1]))+emit({"event": "ready", "protocol": int(sys.argv[3]), "python": "fixture"})+for line in sys.stdin:+ request = json.loads(line)+ rid, kind = request.get("id"), request["type"]+ if kind == "shutdown":+ break+ if kind == "interrupt":+ time.sleep(0.03)+ emit({"event": "error", "id": rid, "ename": "KeyboardInterrupt"})+ emit({"event": "done", "id": rid, "status": "error"})+ continue+ code = request.get("code", "")+ if "interrupt-ready" in code:+ emit({"event": "stdout", "id": rid, "text": "interrupt-ready"})+ continue+ if code == "exit":+ break+ if code == "invalid":+ os.write(1, b'not json\n')+ continue+ time.sleep(float(sys.argv[2]))+ if code.startswith("output:"):+ emit({"event": "stdout", "id": rid, "text": "x" * int(code.split(":")[1])})+ emit({"event": "done", "id": rid, "status": "error" if code == "fail" else "ok"})+"""+++class KernelTests(unittest.TestCase):+ def setUp(self):+ temporary = tempfile.TemporaryDirectory()+ self.addCleanup(temporary.cleanup)+ self.root = Path(temporary.name)+ self.script = self.root / "fixture.py"+ self.script.write_text(FIXTURE)+ self.count = 0++ def launch(self, startup=0, execution=0, protocol=3):+ self.count += 1+ kernel = Kernel(+ [sys.executable, str(self.script), str(startup), str(execution), str(protocol)],+ self.root,+ os.environ.copy(),+ self.root / f"kernel-{self.count}",+ )+ self.addCleanup(kernel.close)+ return kernel++ def test_real_protocol_distinguishes_startup_from_execution_regressions(self):+ timings = []+ for startup, execution in ((0, 0), (0.3, 0.02)):+ kernel = self.launch(startup, execution)+ ready, version = kernel.ready()+ self.assertEqual(version, "fixture")+ timings.append((ready, kernel.batch("pass", 5)))+ self.assertGreater(timings[1][0] - timings[0][0], 0.2)+ self.assertGreater(timings[1][1] - timings[0][1], 0.015)++ def test_complete_output_is_required_even_across_split_frames(self):+ kernel = self.launch()+ kernel.ready()+ self.assertGreater(kernel.batch(f"output:{OUTPUT_BYTES}", 2, output_bytes=OUTPUT_BYTES), 0)+ with self.assertRaisesRegex(RuntimeError, "truncated or missing"):+ kernel.batch("output:31", 1, output_bytes=OUTPUT_BYTES)++ def test_failed_execution_and_invalid_protocol_are_not_timings(self):+ for code, error in (("fail", RuntimeError), ("invalid", ValueError), ("exit", RuntimeError)):+ with self.subTest(code=code):+ kernel = self.launch()+ kernel.ready()+ with self.assertRaises(error):+ kernel.batch(code, 1)++ def test_interrupt_waits_for_running_cell_and_acknowledgement(self):+ kernel = self.launch()+ kernel.ready()+ self.assertGreaterEqual(kernel.interrupt(), 0.025)+ self.assertGreater(kernel.batch("pass", 1), 0)++ def test_ready_version_and_protocol_deadline_are_enforced(self):+ wrong = self.launch(protocol=2)+ with self.assertRaisesRegex(ValueError, "protocol 3"):+ wrong.ready()+ kernel = self.launch()+ kernel.ready()+ with self.assertRaises(TimeoutError):+ kernel.event(time.perf_counter() + 0.02)+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/benchmarks/uv.lock b/scripts/benchmarks/uv.locknew file mode 100644index 0000000000..efb86effe0--- /dev/null+++ b/scripts/benchmarks/uv.lock@@ -0,0 +1,494 @@+version = 1+revision = 3+requires-python = ">=3.11"++[options]+exclude-newer = "2026-09-02T00:00:00Z"++[[package]]+name = "aiofiles"+version = "25.1.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" },+]++[[package]]+name = "annotated-types"+version = "0.8.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" },+]++[[package]]+name = "anyio"+version = "4.14.2"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "idna" },+ { name = "typing-extensions", marker = "python_full_version < '3.13'" },+]+sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },+]++[[package]]+name = "certifi"+version = "2026.7.22"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },+]++[[package]]+name = "connectrpc"+version = "0.11.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "protobuf-py" },+ { name = "pyqwest" },+]+sdist = { url = "https://files.pythonhosted.org/packages/8b/b5/63e14ab9d4d4cc58818562db4120a35fa7454e3035633da41bdc6b712abd/connectrpc-0.11.1.tar.gz", hash = "sha256:18277f7838847b4271ca38d40c7d2387b5a2ea6a29f240689c19e1ec84aaff66", size = 46222, upload-time = "2026-07-15T06:33:31.601Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/5c/ac/aa647675812cd075f2cb9acb00d62f4f2cbcbc6736049a62342b7371ce5b/connectrpc-0.11.1-py3-none-any.whl", hash = "sha256:8a52e2e92a485fa9681c1101a79a5ebeb31807e3ea3d5aabd41f484a6398bc7b", size = 64991, upload-time = "2026-07-15T06:33:29.396Z" },+]++[[package]]+name = "h11"+version = "0.16.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },+]++[[package]]+name = "httpcore"+version = "1.0.9"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "certifi" },+ { name = "h11" },+]+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" },+]++[[package]]+name = "httpx"+version = "0.28.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "anyio" },+ { name = "certifi" },+ { name = "httpcore" },+ { name = "idna" },+]+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },+]++[[package]]+name = "idna"+version = "3.19"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" },+]++[[package]]+name = "opentelemetry-api"+version = "1.44.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" },+]++[[package]]+name = "pexpect"+version = "4.9.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "ptyprocess" },+]+sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" },+]++[[package]]+name = "prime-agent-benchmarks"+version = "0.1.0"+source = { virtual = "." }+dependencies = [+ { name = "httpx" },+ { name = "pexpect" },+ { name = "prime-sandboxes" },+ { name = "pydantic" },+ { name = "pyte" },+]++[package.dev-dependencies]+dev = [+ { name = "ruff" },+]++[package.metadata]+requires-dist = [+ { name = "httpx", specifier = "==0.28.1" },+ { name = "pexpect", specifier = "==4.9.0" },+ { name = "prime-sandboxes", specifier = "==0.2.40" },+ { name = "pydantic", specifier = "==2.13.5" },+ { name = "pyte", specifier = "==0.8.2" },+]++[package.metadata.requires-dev]+dev = [{ name = "ruff", specifier = "==0.16.5" }]++[[package]]+name = "prime-sandboxes"+version = "0.2.40"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "aiofiles" },+ { name = "certifi" },+ { name = "connectrpc" },+ { name = "httpx" },+ { name = "protobuf" },+ { name = "pydantic" },+ { name = "pyqwest" },+ { name = "tenacity" },+]+sdist = { url = "https://files.pythonhosted.org/packages/64/8c/19e812273ecf633da7b8767b6c6b14b55a4e9753f7c9791349a6ce9e0353/prime_sandboxes-0.2.40.tar.gz", hash = "sha256:4096f8afc4887590028e2b4eb1f424c7db40022aee6784beb2a071e6023e9791", size = 107334, upload-time = "2026-08-27T18:12:08.044Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c6/ba/6a5ce07386b457f0eae76d025b8b0af79bd82b8ce9fc2b9f0772e17a214b/prime_sandboxes-0.2.40-py3-none-any.whl", hash = "sha256:62d3afc9ce8a5b39dc292208ab8ba23779320e91cf2f7bd21564ed20ac6faa75", size = 60144, upload-time = "2026-08-27T18:12:06.818Z" },+]++[[package]]+name = "protobuf"+version = "7.36.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/86/73/f66c748df06e7fe24e658eddd600d19c4b40bad836c97ce2d0ad9851fb6b/protobuf-7.36.1.tar.gz", hash = "sha256:d0f6470f0ce2b84e3feaea2d4b816378b37ba4d4aa08a274305373de93e2d524", size = 512499, upload-time = "2026-08-31T22:40:04.667Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/f7/6c/3a54a58f2948b0f485df9ecdd06590f15d0a7abf46a89d50c3de709ff4ff/protobuf-7.36.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:3cf2ee25d006cee57294a1196ea43b37feb78e0dcd1e8af5c1aeddb777655aca", size = 456046, upload-time = "2026-08-31T22:39:56.865Z" },+ { url = "https://files.pythonhosted.org/packages/6e/08/9f9548793c771095245c0eeaf0c84b76b16a5ef26158043d456f551344a0/protobuf-7.36.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:43d3d37b1eb24c113b9b7d02008cac44e423f00b611b7781ae998d7623972969", size = 344226, upload-time = "2026-08-31T22:39:58.263Z" },+ { url = "https://files.pythonhosted.org/packages/fe/51/1bdbd612fa3c51e42ea6f45d05e84dc748ddcd2663e1aa1e89a00a33facd/protobuf-7.36.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:39c518c05586c016d7874ff6079ee115bcec1ea5fbb1d177fbf7867ef4c67e44", size = 357229, upload-time = "2026-08-31T22:39:59.198Z" },+ { url = "https://files.pythonhosted.org/packages/22/df/c799fe7a05ef16ba853a59db01f3a2c5f7d0676469589ccc4874f76a2a88/protobuf-7.36.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:97198b77e369a0abd8e262b8f6c7266c55ddb796a3a12c76d7b8881188ed83aa", size = 343228, upload-time = "2026-08-31T22:40:00.179Z" },+ { url = "https://files.pythonhosted.org/packages/3d/33/d4724ec5d86d496fe4108e220618aa0837ad3423a7af7ccdd9684ccc77c8/protobuf-7.36.1-cp310-abi3-win32.whl", hash = "sha256:0b53ce95272aad50ad25d7ff03373743209822e8ba42ea7fad27d2bee1547d00", size = 443002, upload-time = "2026-08-31T22:40:01.32Z" },+ { url = "https://files.pythonhosted.org/packages/db/37/155788a0d8daded960375af604202805308169f9b859419ea0aa370946e2/protobuf-7.36.1-cp310-abi3-win_amd64.whl", hash = "sha256:51139351435d9b43d88a55eaa49fb6f737fbb478fb0cbf2cf694d1a04a9d3363", size = 456518, upload-time = "2026-08-31T22:40:02.494Z" },+ { url = "https://files.pythonhosted.org/packages/39/ca/c47f91d3cab175b01fd8c4f0d80fdf8613be876cc616e66ad281a59c5ddf/protobuf-7.36.1-py3-none-any.whl", hash = "sha256:7d951e46b3f963d6c264c367c437921de9d5aedd9c3f9612b9077736b4e3ad5c", size = 179813, upload-time = "2026-08-31T22:40:03.54Z" },+]++[[package]]+name = "protobuf-py"+version = "0.1.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "protobuf-py-ext", marker = "(platform_machine == 'arm64' and platform_python_implementation == 'CPython' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'AMD64' and platform_python_implementation == 'CPython' and sys_platform == 'win32') or (platform_machine == 'ARM64' and platform_python_implementation == 'CPython' and sys_platform == 'win32')" },+]+sdist = { url = "https://files.pythonhosted.org/packages/72/ed/02fd902d9c51b7ff53dfc9a745eb11490722edfd30073af889e171f07b8e/protobuf_py-0.1.1.tar.gz", hash = "sha256:6bd08ac4d8f1661965bbe2685429d79043704cdd1ee720a7a89617331742240b", size = 133525, upload-time = "2026-06-24T19:02:15.271Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/fd/00/1b3775aca1c70e3007e06ef5996f6bb9b3a32341eb0cce3ffb6effad8dec/protobuf_py-0.1.1-py3-none-any.whl", hash = "sha256:efc4f50f275ed6dae10a1f30bb81ad1a75368557b3ff22a532b7a472050368f1", size = 181656, upload-time = "2026-06-24T19:01:29.556Z" },+]++[[package]]+name = "protobuf-py-ext"+version = "0.1.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/2f/05/6dc9ccff1e8159eb9a144e6d3c4acfd2211cd4fcd20c34fe9155d17d6a7f/protobuf_py_ext-0.1.1.tar.gz", hash = "sha256:e85bfdfdb3ed50634db8ccc7429dd9286520109489c735463971a418707b4fef", size = 31912, upload-time = "2026-06-24T19:02:16.321Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/19/70/2b5d62a60a2e0d88ecde1ae98db3132bcd2672fb39c8d581b82b34ac0bd8/protobuf_py_ext-0.1.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:310039e03cb15181781a0b78017419f6d4ee302e988c3c70b87f1facdf05532d", size = 306008, upload-time = "2026-06-24T19:01:31.399Z" },+ { url = "https://files.pythonhosted.org/packages/2d/d6/73c06bb4cac2e04c0adc154965fe8b6520224bd737fd7e73bba405056321/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89ec8348d1ba045f79b2fedd14e40cca36f2a41b52f2c4fdf55a60c58add2353", size = 314769, upload-time = "2026-06-24T19:01:32.89Z" },+ { url = "https://files.pythonhosted.org/packages/18/c5/e4e6bc6096b66d1c82639a1b501147f16ee65d32567304b987d002e2c666/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:182798e4861aba72d05855bd06febe4926aa7265e6f444a1b8af5252beee4f7e", size = 328122, upload-time = "2026-06-24T19:01:34.394Z" },+ { url = "https://files.pythonhosted.org/packages/b0/4b/1d792a40d0f0a0f914f1dfa8bb5e9573ca0ecd5fe5cecf80d77193212abd/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9eb25c3a329c0551cc86b209a5e5d8ecb8d834b9924a3aa019377853a703b6d3", size = 492338, upload-time = "2026-06-24T19:01:36.103Z" },+ { url = "https://files.pythonhosted.org/packages/52/51/5ad329223c905e5c530cae38ae24cde3584d8ab7e09457f2a01afce80e60/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aaecbde82bef10c7c40578cbb61b7a19896bf7fa450972050a3bb302acb7d5d6", size = 541578, upload-time = "2026-06-24T19:01:37.481Z" },+ { url = "https://files.pythonhosted.org/packages/ed/c7/01bd8a8bfe2df4b07aafac12ccfb7b7ebe2303f65296a9e02fcafa28ff3e/protobuf_py_ext-0.1.1-cp310-abi3-win_amd64.whl", hash = "sha256:6b0c615c48e95acc53cf33e9310eeaff8b30d2d7555bf93e7bca8fb4f40e9a5c", size = 251319, upload-time = "2026-06-24T19:01:38.946Z" },+ { url = "https://files.pythonhosted.org/packages/24/dc/914065538b5db54b6a920b5af38c1ef142252ea1eea711820435fa259fac/protobuf_py_ext-0.1.1-cp310-abi3-win_arm64.whl", hash = "sha256:72956cd0af5dee24b41c6f5ba5e42622d17e6d555002b5efc1634e27a1446de2", size = 241635, upload-time = "2026-06-24T19:01:40.301Z" },+ { url = "https://files.pythonhosted.org/packages/13/a5/0b5d73cab815fd50615cb87a02e04e8332f9e27380c890aba1459cf7e919/protobuf_py_ext-0.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6f0cd58620f415a3534d195358338f4999a774e12510f91c592b38d13d37388", size = 304903, upload-time = "2026-06-24T19:01:41.796Z" },+ { url = "https://files.pythonhosted.org/packages/37/a9/14c120b9ac36a0e11bb05e482fa6ae322de583a8e1068e4859035c289947/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21572764f625d829604fc4d83635f533f35775f11c6da142345b3f3c8d64bd09", size = 316148, upload-time = "2026-06-24T19:01:43.247Z" },+ { url = "https://files.pythonhosted.org/packages/35/54/7c00a1a9783c3ba74f6b2f5d2f049f09037bbd489c465c09a34f32fb611b/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f14f00c2678bfbe7ad46f057b9f9938c1677bcf39e405e163e272c6f9814b8", size = 326458, upload-time = "2026-06-24T19:01:44.655Z" },+ { url = "https://files.pythonhosted.org/packages/0d/0e/81fa50c0d6c4d664e5be6d0a3c4f2c7edfef87ab12d0bac764abca1f2955/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1610865622e2e27568277ca63d8d2d23dcc55eaa767398865c21539ddf4ce24d", size = 493596, upload-time = "2026-06-24T19:01:46.344Z" },+ { url = "https://files.pythonhosted.org/packages/56/19/3183cf6e4de62846c20549654a1d573dae51ca06aaf7fed06accdfab74f6/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8995efba9476e1ea18ef9873306871dba697308994bc2766558fec3387acc3de", size = 539656, upload-time = "2026-06-24T19:01:48.21Z" },+ { url = "https://files.pythonhosted.org/packages/a4/6c/09841f3dbe7c3d4b3f2779f8f2832ac23fb96ac8ab71674e91af732cf9ff/protobuf_py_ext-0.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba61d49dace8f874361583030a7c48139b42eb37c9ffbb1e7e8a227a51576f44", size = 305017, upload-time = "2026-06-24T19:01:49.84Z" },+ { url = "https://files.pythonhosted.org/packages/52/43/e450b5e202f6715274acc9acd9f9769908cbdeab861a53c798fcbd467d35/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a9c2f026096ca4c595c89a297067ef371e241d3ff8e1f6d7c779aa8419cdca7", size = 316215, upload-time = "2026-06-24T19:01:51.249Z" },+ { url = "https://files.pythonhosted.org/packages/97/5c/23e79b35f1b5755b9bdc0c0c9b8cd8abababaef1a1639608d8a96bb61a9d/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf78707a040b9294e5e1ec4a1875f0046acfe52e92150cea27de4e9fc9db39bb", size = 326419, upload-time = "2026-06-24T19:01:52.93Z" },+ { url = "https://files.pythonhosted.org/packages/e8/de/5493de0ec12920a3b1dd72608ce0c1e8cdb7e0eb9e87accaecc5216df29e/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae4373845cbb85bdade3ef5368cc2f4b5f80bf173383afc1ab063f95644e5599", size = 493734, upload-time = "2026-06-24T19:01:54.301Z" },+ { url = "https://files.pythonhosted.org/packages/98/89/31da55b414e6332aad11d858e9a86bd36fbb324f52fa3fc6d8f1c57840a9/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2a4cc478eef7a2acc1daaebcde479a3ac2396d47e6bdc7e776ca4c4147ba8b4c", size = 539703, upload-time = "2026-06-24T19:01:55.707Z" },+ { url = "https://files.pythonhosted.org/packages/1c/71/39f231838ef06476d46ac40dd814894277a732a3688c0a0c994850b3b62f/protobuf_py_ext-0.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:359dccdc1c3eafed2a913c570bb082b8df848a1d27d548ce8385f246a7d68be2", size = 302145, upload-time = "2026-06-24T19:01:57.116Z" },+ { url = "https://files.pythonhosted.org/packages/87/f1/01c81ff5f420600366a0e6a613ce2af20834534d2b3d7523f4f3b4ddb54f/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91c38b4a10a306366443273ee03ca554537a0965bf05b7ada8e7dbdee04cc93e", size = 314541, upload-time = "2026-06-24T19:01:58.745Z" },+ { url = "https://files.pythonhosted.org/packages/86/d8/1cbf5a0298ceddc0cdbb28bf1ecaf8bd3cff54ed4ced6a1392d5226172fc/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79eee3bcbb289d6ea114eb8fe3a1469c5b59bf53e207238469f567d9c53ba56f", size = 325092, upload-time = "2026-06-24T19:02:00.382Z" },+ { url = "https://files.pythonhosted.org/packages/af/23/8b1694616044cbfec2d18d4c6fffb03e05089c8bdb5970c6bafd60cc7fd5/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:10992141a8282a71ac3e3530d1a489efb27618d84000b9a47918cf70e5816d9b", size = 491684, upload-time = "2026-06-24T19:02:01.862Z" },+ { url = "https://files.pythonhosted.org/packages/49/d8/99997a7a6cf6989c944d9183c5deb6a045c27460add0316c7b34763891b3/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4411ffe0e06a774b83c5c71c546ce097640a25f596c45f95f53d3e3148e3f22d", size = 538048, upload-time = "2026-06-24T19:02:03.362Z" },+ { url = "https://files.pythonhosted.org/packages/15/3c/9e99ecbb68e1d5b46016d821eb1cbba9a91e6b50e71e75faf0c1e6189f0d/protobuf_py_ext-0.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55a17d80ea419501ff221a6627523f38a431bb33e6aa3de81ae3a7f271c49c75", size = 296337, upload-time = "2026-06-24T19:02:04.787Z" },+ { url = "https://files.pythonhosted.org/packages/af/f2/305338a28225fb54b28d6c3f5948109b9088b329a2b6f77ca610c2bdcd34/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbb518b5638403ceaa08ac4fc7dac626f45ed9b856b3517de3906cf3de4d632", size = 308961, upload-time = "2026-06-24T19:02:06.185Z" },+ { url = "https://files.pythonhosted.org/packages/60/f9/416103c93677ff2ea407704ea64fa6de9e700dc030c48360a182d91ce373/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4adc65ab6a5e4885c67fc808bcacb83d755d05b566d312a0e10c2a873f2ad4", size = 321895, upload-time = "2026-06-24T19:02:07.714Z" },+ { url = "https://files.pythonhosted.org/packages/5a/83/eb3e81bb2f83834b7deff4cee2d56eeb1adcbc847492a56b7f910ab257db/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e97f45c49676efacfb8e95bfcb1a002bc337e618a6780be1e55df2ba5ebd2f1e", size = 486091, upload-time = "2026-06-24T19:02:09.243Z" },+ { url = "https://files.pythonhosted.org/packages/99/96/bd88fca38556b3105e4dd41d6a176e31dcc583fe979e830aeedd2f8c20ee/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:53a6b3590f6aa7f97b8ed60f62fef9b096babfeae82f7283fd3a4c405827d4f8", size = 534582, upload-time = "2026-06-24T19:02:11.227Z" },+]++[[package]]+name = "ptyprocess"+version = "0.7.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" },+]++[[package]]+name = "pydantic"+version = "2.13.5"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "annotated-types" },+ { name = "pydantic-core" },+ { name = "typing-extensions" },+ { name = "typing-inspection" },+]+sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" },+]++[[package]]+name = "pydantic-core"+version = "2.46.5"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/a2/b6/81d2d19ea0be2c03664381b59f65fa72fc7969decedae00bc2c4ad835708/pydantic_core-2.46.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f", size = 2074737, upload-time = "2026-08-28T09:57:57.711Z" },+ { url = "https://files.pythonhosted.org/packages/0c/18/b70da8300e292df4099684ea11b1958043580d2f50d2dc8bf7e542bdd84a/pydantic_core-2.46.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f", size = 1921751, upload-time = "2026-08-28T09:57:59.265Z" },+ { url = "https://files.pythonhosted.org/packages/e7/1a/0d590341b6ffa4b4aca83508e6b8db4761aaeacfc15a25ca3815876d4797/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061", size = 1948231, upload-time = "2026-08-28T09:58:00.678Z" },+ { url = "https://files.pythonhosted.org/packages/7d/1d/02eb35761c51f2f7b1b042d6ab4cda6600f0c8c88a2243b3f734376201e5/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be", size = 2020708, upload-time = "2026-08-28T09:58:02.267Z" },+ { url = "https://files.pythonhosted.org/packages/4a/ea/f86073830e35d508cc8ddf9c3d9e6e6840fcb88d34bf726b0b4710186f27/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a", size = 2194914, upload-time = "2026-08-28T09:58:03.934Z" },+ { url = "https://files.pythonhosted.org/packages/bb/d7/fc36240d7791ce90939e51608568c33bfdae26202016f9770c229a487d86/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b", size = 2235622, upload-time = "2026-08-28T09:58:05.516Z" },+ { url = "https://files.pythonhosted.org/packages/cf/bc/3fa2d76b83162820a17da7f645b28d1cba99fc8e1e5fc6517067ec450fa1/pydantic_core-2.46.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c", size = 2062091, upload-time = "2026-08-28T09:58:07.135Z" },+ { url = "https://files.pythonhosted.org/packages/ab/9a/095d557bb492c90cd8a70a6dd048bf793d433d03d86c81c11e912e4cd049/pydantic_core-2.46.5-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee", size = 2089904, upload-time = "2026-08-28T09:58:08.814Z" },+ { url = "https://files.pythonhosted.org/packages/24/98/7b76b1ad10a19a617a52aaa1d80e159115af939b095e86f8e756fd52e0df/pydantic_core-2.46.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e", size = 2132244, upload-time = "2026-08-28T09:58:10.435Z" },+ { url = "https://files.pythonhosted.org/packages/20/32/7d6ca365fadba186a0c8f85de1a701663bce81efd309d9479be58687622f/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2", size = 2143901, upload-time = "2026-08-28T09:58:12.033Z" },+ { url = "https://files.pythonhosted.org/packages/f8/09/eb9a6aa57f22fd1541a9c0aa2a1f3aeef3ec65347d33e10a6da2f43e0ee9/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689", size = 2299425, upload-time = "2026-08-28T09:58:13.614Z" },+ { url = "https://files.pythonhosted.org/packages/8a/f9/548a5bb9d4ba8cd26e26daf48052236f6b38bb61e7b7241fbc3c995719eb/pydantic_core-2.46.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec", size = 2318566, upload-time = "2026-08-28T09:58:15.199Z" },+ { url = "https://files.pythonhosted.org/packages/4a/20/06454d18834c02c406c9133f1a3b485305fd9ee984f9636c2f730bef6a9d/pydantic_core-2.46.5-cp311-cp311-win32.whl", hash = "sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129", size = 1954258, upload-time = "2026-08-28T09:58:16.813Z" },+ { url = "https://files.pythonhosted.org/packages/9e/c2/718b9deb4b72453b5d8c7447a3b14cb77bef36917ef5f514e0948a4096a0/pydantic_core-2.46.5-cp311-cp311-win_amd64.whl", hash = "sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c", size = 2041030, upload-time = "2026-08-28T09:58:18.288Z" },+ { url = "https://files.pythonhosted.org/packages/67/ea/c1d1a5b72d6e1ff7f377a4d9199f6591f095beb5b409a8a5d89f7238d939/pydantic_core-2.46.5-cp311-cp311-win_arm64.whl", hash = "sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8", size = 2009234, upload-time = "2026-08-28T09:58:19.929Z" },+ { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" },+ { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" },+ { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" },+ { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" },+ { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" },+ { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" },+ { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" },+ { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" },+ { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" },+ { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" },+ { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" },+ { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" },+ { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" },+ { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" },+ { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" },+ { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" },+ { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" },+ { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" },+ { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" },+ { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" },+ { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" },+ { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" },+ { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" },+ { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" },+ { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" },+ { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" },+ { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" },+ { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" },+ { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" },+ { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" },+ { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" },+ { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" },+ { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" },+ { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" },+ { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" },+ { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" },+ { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" },+ { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" },+ { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" },+ { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" },+ { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" },+ { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" },+ { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" },+ { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" },+ { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" },+ { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" },+ { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" },+ { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" },+ { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" },+ { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" },+ { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" },+ { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" },+ { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" },+ { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" },+ { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" },+ { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" },+ { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" },+ { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" },+ { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" },+ { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" },+ { url = "https://files.pythonhosted.org/packages/af/1e/ecca01fce348f7e8afa9572441ff6f7d1cc70d21e4859f33944d10877e1e/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2", size = 2075342, upload-time = "2026-08-28T10:00:51.353Z" },+ { url = "https://files.pythonhosted.org/packages/1f/4c/af80c7a8032dfc897040ad5cb772bebde529a381186499e6e29987f23f8c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c", size = 1907219, upload-time = "2026-08-28T10:00:53.438Z" },+ { url = "https://files.pythonhosted.org/packages/be/3e/54d89e2b092e778716bf6153634ef479e955f48c261090be23aa1e0fb0b5/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47", size = 1953393, upload-time = "2026-08-28T10:00:55.58Z" },+ { url = "https://files.pythonhosted.org/packages/ea/89/828ee90cda28ce17bdefaa3a6eaf74fe430e113295a10e6126beca559d6c/pydantic_core-2.46.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a", size = 2099024, upload-time = "2026-08-28T10:00:57.794Z" },+ { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" },+ { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" },+ { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" },+ { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" },+ { url = "https://files.pythonhosted.org/packages/20/21/22102e9950b3049526d20e811b95396508377d87651edd2b80d2b3d28659/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f", size = 2071333, upload-time = "2026-08-28T10:01:09.636Z" },+ { url = "https://files.pythonhosted.org/packages/d8/18/87aefa427d191e6d3ab1447f1efc1cdcac86af1069239b133e8a0fd7f7c9/pydantic_core-2.46.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0", size = 1912713, upload-time = "2026-08-28T10:01:12.285Z" },+ { url = "https://files.pythonhosted.org/packages/1f/93/fd89e9ad49b1805ca94d24ce1088b7d305f05c35ffafcedb9819d03588a0/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4", size = 2090926, upload-time = "2026-08-28T10:01:15.19Z" },+ { url = "https://files.pythonhosted.org/packages/6f/45/8e59dab6acf8d35f02f0a958980074f31038968bdb2c983fcae9d1efee03/pydantic_core-2.46.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25", size = 2131303, upload-time = "2026-08-28T10:01:17.937Z" },+ { url = "https://files.pythonhosted.org/packages/d5/a5/e1d4dc5180dd887a9522efc1f8716b8692b7606b1d3273d7862eaf66be44/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6", size = 2145128, upload-time = "2026-08-28T10:01:20.694Z" },+ { url = "https://files.pythonhosted.org/packages/c2/d7/ad493864a7fb21c0c4df98f965e2db430cb25a9d7369b5778d5016c09fd9/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e", size = 2294560, upload-time = "2026-08-28T10:01:23.495Z" },+ { url = "https://files.pythonhosted.org/packages/02/8e/b41c84c913f29973a268e6c2b5bbf13c95adb9956c126d10da11ba3b2bef/pydantic_core-2.46.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda", size = 2317531, upload-time = "2026-08-28T10:01:26.334Z" },+ { url = "https://files.pythonhosted.org/packages/db/1d/068464f23075f66a8f1b806935e9cd9363ee446636ea70d2c22ee8659dbf/pydantic_core-2.46.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266", size = 2140686, upload-time = "2026-08-28T10:01:28.947Z" },+]++[[package]]+name = "pyqwest"+version = "0.10.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "opentelemetry-api" },+]+sdist = { url = "https://files.pythonhosted.org/packages/07/ee/0ff9facfa9e7a4f6df2a770d4eaf1ad0f74165da7e8c28e888461f07604c/pyqwest-0.10.0.tar.gz", hash = "sha256:6c1a693be17d57d2c2eca4085e32c2809c53090c16719a907c90ebcf1f40dc01", size = 482248, upload-time = "2026-08-21T06:09:20.656Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/43/ee/b1a28f57c689606cfd065d8a553841150f7daaa91d20e58dcc2c5ea191f8/pyqwest-0.10.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:aa492d5777dd145a60795ed95d9d4707a3cd1091fdcdfc93a82ac7fdc43ebacd", size = 5261059, upload-time = "2026-08-21T06:08:04.999Z" },+ { url = "https://files.pythonhosted.org/packages/dc/13/9c5046cfd6ef705bde0b620ba8a794335bcabc0839342a2a647f2427b27e/pyqwest-0.10.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:59f3f16628e518c674102e7b5fcff2101bba6abb4f6737ec5fade9b9278e6a53", size = 5134207, upload-time = "2026-08-21T06:08:06.955Z" },+ { url = "https://files.pythonhosted.org/packages/93/7d/50021dd88d82d6966ab1c27593ceaee9d1ed62fbe597c40e8dc187cfa5fd/pyqwest-0.10.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6e7db305a8318b1f3218053e87501f8f245ca8bd63e948e0282d04bf0883470", size = 5640730, upload-time = "2026-08-21T06:08:09.135Z" },+ { url = "https://files.pythonhosted.org/packages/ff/3f/5bf6c32e9e701837a8c47ce6e3ad38978cfec8eb7bc6596181e5f9e1eaeb/pyqwest-0.10.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5c757cfac5f53c8671dcb4850d5fc4c4339ea3e90636331c9318f8e3ddabc06", size = 5561462, upload-time = "2026-08-21T06:08:10.836Z" },+ { url = "https://files.pythonhosted.org/packages/5f/61/ca9ba5721461b7ce5cfaac373ab3a1723ddcc434af7430f8bd628da6e623/pyqwest-0.10.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:234b3f71e3f314d997c203d8cf829b7117edd041153f9c277d0060ab90134148", size = 5801847, upload-time = "2026-08-21T06:08:12.502Z" },+ { url = "https://files.pythonhosted.org/packages/d2/44/95593919b996a417093f598d887822b9b899e8d025588c9bfaf8c60dd812/pyqwest-0.10.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5637256a0dac0ef57e0eaa02b032014965e4a4c995e1deca1b1b97e6d1765f78", size = 5978692, upload-time = "2026-08-21T06:08:14.383Z" },+ { url = "https://files.pythonhosted.org/packages/a5/b9/b2821ce5188457168ebb25d5ff65b1ca1bf27bc6b4a33df4bcc2357e625c/pyqwest-0.10.0-cp310-abi3-win_amd64.whl", hash = "sha256:7ea761937acf3a00d1a7e70e982949d18946e5471d1419266ab3a78bbfa19759", size = 4876627, upload-time = "2026-08-21T06:08:16.084Z" },+ { url = "https://files.pythonhosted.org/packages/86/b4/16ccef1c203fa258ce46a86aefc1a79c13b5f0b8d49627347d90eef25efd/pyqwest-0.10.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a21f1f15252a8303623b4f17b9c6de595ace11b3ade07f2adb6d07121e8191aa", size = 5274815, upload-time = "2026-08-21T06:08:17.777Z" },+ { url = "https://files.pythonhosted.org/packages/5f/61/6a87f84f571441ea43279587d4bfcad4543505918ae2b83a1ebdcfa98be5/pyqwest-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eb472c6e5d6833ebfec79db310e426eb17b01ac64c0e2c251bd9192c0d2ee0c5", size = 5123656, upload-time = "2026-08-21T06:08:19.501Z" },+ { url = "https://files.pythonhosted.org/packages/8b/f2/ab69e581cf9b798b0e169f7b27fd3f8b6f9f1631bd4d3b6e22e5abaf8d8b/pyqwest-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83578e24cccd5e0dc04d60a0af7bfb43325b5f22d03ff74ff79ed0ecf553b50d", size = 5641253, upload-time = "2026-08-21T06:08:21.627Z" },+ { url = "https://files.pythonhosted.org/packages/9f/dd/f1a62eebf8321ace506bd94551a01431f6a45b882225455eae3ea6e8c6d1/pyqwest-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bb511c434f79c641efb5573e5795e56dc972252f4b96e52a9636d4ece5231a4", size = 5567341, upload-time = "2026-08-21T06:08:23.346Z" },+ { url = "https://files.pythonhosted.org/packages/4c/48/51d767691973e046887f5e6d96e32142fe296823b163ccba732233a6ef72/pyqwest-0.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1aaccd8a9db9430b2aedb5bad8ead80742cbc056b85c229516c70dc80539f906", size = 5803874, upload-time = "2026-08-21T06:08:25.096Z" },+ { url = "https://files.pythonhosted.org/packages/58/0a/d2834ccc6e59ad110718895cc65ff2a68aa6e010f0ba8fbe42a57ea33c21/pyqwest-0.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:73d9eb438ab4a957a1ce0619d3af8c1c1126bfb9181033b123d792fcf4224531", size = 5981518, upload-time = "2026-08-21T06:08:27.594Z" },+ { url = "https://files.pythonhosted.org/packages/75/f3/274b4c268e9a55fbdb1b3637ac50b5bf42cd3a85d1cfbdc15c602a7b0d9c/pyqwest-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:317a74d633abe3bc5bccabf479e069c515dab9e6a755274b0ccb1d8a5bbfede3", size = 4870638, upload-time = "2026-08-21T06:08:29.421Z" },+ { url = "https://files.pythonhosted.org/packages/8d/3d/8b1092f25159bf61a9470ebd35438c669b91ef553a7ee205bdec8006107b/pyqwest-0.10.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3978e794b9cfd8eaa500fb5d7aee63bc6172c605efa0abc1f62d85485bc049e1", size = 5273599, upload-time = "2026-08-21T06:08:31.107Z" },+ { url = "https://files.pythonhosted.org/packages/75/10/54a9786123942b124c2afb9562b74e158afec7be40ef0caa0d37f615d379/pyqwest-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:715991fd4f04862cd7a9d7452daabcdbd74dff4dff55eb20c22d60382dc2a4ed", size = 5122920, upload-time = "2026-08-21T06:08:33.025Z" },+ { url = "https://files.pythonhosted.org/packages/a2/5b/6a6bd76f91e068b9a619f62aef9fe5ef201f859ddbb6b0a11ad3875ecdda/pyqwest-0.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c04798bed79c1dfa0e5b0e30fb137124311083490d44d6dfbe068d3dd254349e", size = 5639577, upload-time = "2026-08-21T06:08:34.896Z" },+ { url = "https://files.pythonhosted.org/packages/68/28/12277a24a8dd74b0a7f124c624d9ed58eccb41e2087ff1087a14d348c778/pyqwest-0.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35b472877e73dd63fed089c2bc8fa198407f005c8c19e0a93f025ebefde01a81", size = 5565835, upload-time = "2026-08-21T06:08:36.567Z" },+ { url = "https://files.pythonhosted.org/packages/9e/b6/b7da4a3f1fe43600154ae75e91ba7969024d886d60077dd3a1ba8e66d170/pyqwest-0.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:564ec360b7848b35e009038ffbca00466305a9708ab21829477f64aa8cad4c64", size = 5803078, upload-time = "2026-08-21T06:08:38.362Z" },+ { url = "https://files.pythonhosted.org/packages/0c/6f/0c9ba210f49f232289afaa8f06369c5f135ac786e1ca0cb22243b7f1fe2c/pyqwest-0.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5c80e88a5967c1cadb3237c450f91a84a3683f8838c8dca96f09fee3612e762", size = 5980431, upload-time = "2026-08-21T06:08:39.967Z" },+ { url = "https://files.pythonhosted.org/packages/9b/f5/306eeed41a3cd3100247e6e442f4345277b70f1d24efe1641181b14839cd/pyqwest-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc3d80b402fb59dbe015e25993ac8147456fb231a4c949f92a89f31315ad50f9", size = 4870198, upload-time = "2026-08-21T06:08:41.687Z" },+ { url = "https://files.pythonhosted.org/packages/64/18/0086a408e7cbf39dab18fa5b7e42c969a98382da5a4e6debe40f05acc6a1/pyqwest-0.10.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:23a28beb55fa6d975949bffae4adfb69378f3229bb5cbd71231e95bf66f5b26c", size = 5274542, upload-time = "2026-08-21T06:08:43.323Z" },+ { url = "https://files.pythonhosted.org/packages/0c/7b/e1b9aaaf7596e4faaa53cefc2efaca4e3cde721e308e6385e366361cfdcc/pyqwest-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e4415ae40b8eedb1713dab14d7f9fecc3f79d26f3206c561087b88b99d5ce24b", size = 5127922, upload-time = "2026-08-21T06:08:45.116Z" },+ { url = "https://files.pythonhosted.org/packages/75/cd/e6d68bb1de5dd26100fcfc878cbd67c402a928774edf1e8ae304c5a84f5b/pyqwest-0.10.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14b875d2273212d7fa8e4b755d8d736ffd226b1c707a9c0017dfdc8393a96eca", size = 5645856, upload-time = "2026-08-21T06:08:47.055Z" },+ { url = "https://files.pythonhosted.org/packages/f7/48/8c9f9f0467c41f6a563146d57a52f8f6d60c0cb09d0fd3ef88ad5a1c442f/pyqwest-0.10.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5071491e416ea54e3b95bf9ffbed0bd065b093cb96e10a75c3d8f2cbe3c9823", size = 5569831, upload-time = "2026-08-21T06:08:48.78Z" },+ { url = "https://files.pythonhosted.org/packages/16/ac/c85ab70c6c72078d49a82da76b820e46aaf95a3f6fe271dac955ac195d21/pyqwest-0.10.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b68b5e68d513a4c63a072f8f40e38015160cf90bfbf7e8ef7c3935ca87e9e022", size = 5806735, upload-time = "2026-08-21T06:08:50.652Z" },+ { url = "https://files.pythonhosted.org/packages/66/db/ad7375b22fb2d0807431dcc9bc2aaf840e374c298cd15071024d5f6dd6d1/pyqwest-0.10.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c48910d27820b9c46fcd001b0fe514a3cf47d4784f59512dcdb8c91c395f82e4", size = 5984786, upload-time = "2026-08-21T06:08:52.377Z" },+ { url = "https://files.pythonhosted.org/packages/17/88/c449a772afe129683fd7acc657cbc7c69bec085dc75b6e8710a50fbb44e7/pyqwest-0.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:d03ba2cd17948b623a6210981d342eb122546d8a8e910ec77511aff4b1acdd00", size = 4872288, upload-time = "2026-08-21T06:08:54.101Z" },+ { url = "https://files.pythonhosted.org/packages/3d/f8/439ffc0ee12cd7d9b57ac07ccea78ad3ba66b0d6817d429dd661d73308c4/pyqwest-0.10.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:07a0eb595f4096232c2d22549b6e4612c1ecada7934e46462c2c37ce14a89cfb", size = 5256823, upload-time = "2026-08-21T06:08:55.681Z" },+ { url = "https://files.pythonhosted.org/packages/df/2b/72ecd27d104d2b4284710194cce796d607674966c3ea66436768e812a66a/pyqwest-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:26401baf7dafc71c8d12d2e8389519d141e6f7c14094d0dd4cf9ec1d3b5555bd", size = 5112624, upload-time = "2026-08-21T06:08:57.366Z" },+ { url = "https://files.pythonhosted.org/packages/a4/24/0fb89c3f7d5a0410fcb7560588b4c741ef24b19195c307d4263f04e75c2b/pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e3c436e041d8873ce5bb0fdcf9f9e86f5604e8f0ef9e03149efebd8cb474f6", size = 5631844, upload-time = "2026-08-21T06:08:59.165Z" },+ { url = "https://files.pythonhosted.org/packages/4c/4f/921d14754a186f0143ad62b50108dd808328e347e98e9dafab3897eeb405/pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09364115761579eabfc79d1e954cdb3ded508dac1903fac7285d4c6f058c683f", size = 5555869, upload-time = "2026-08-21T06:09:00.783Z" },+ { url = "https://files.pythonhosted.org/packages/b8/70/504780417319a626fe9549a7e6f9020a3d448eddf8a09617238a3426e90c/pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559674a98a8b1217e1830ecd41c9905bf2b60983c6b8017063dfac199f00727c", size = 5793738, upload-time = "2026-08-21T06:09:03.324Z" },+ { url = "https://files.pythonhosted.org/packages/45/f7/8d0a5b8a3289f4300dc9005ebde75d316f2371a2671617f942036337731a/pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f399a696392fff3db3eef0a18ef65b8a3b8396d129193487d966b8eb11006376", size = 5972889, upload-time = "2026-08-21T06:09:05.2Z" },+ { url = "https://files.pythonhosted.org/packages/89/c4/f4c781e475c451cb5f4762a2f814a1750ef375b8db4db09bb2dcea03c4e5/pyqwest-0.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0f9163d6dd991bf1bf27308ba38ba021af660b15fffa47ebca98e41cf6f00309", size = 4858036, upload-time = "2026-08-21T06:09:06.926Z" },+ { url = "https://files.pythonhosted.org/packages/7a/6f/29f605665f33aab894db8daf11b3b64bdca015fb11135815c53a150191a6/pyqwest-0.10.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cfcc7ba0229baa17831582befb046ace167b368140dae022d0b89b8d586ba12c", size = 5264903, upload-time = "2026-08-21T06:09:09.183Z" },+ { url = "https://files.pythonhosted.org/packages/c8/c8/4ce4b40f21397a6482da910fae18b32c28dcdaddf3498aef2fe38597a9e0/pyqwest-0.10.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eff9ccf427604d34c635954def07b6113d4754f968073eef2df40bdb80b05bf5", size = 5142291, upload-time = "2026-08-21T06:09:10.943Z" },+ { url = "https://files.pythonhosted.org/packages/41/34/7205db9cddc5a2286da459ca013af629e20f09498751c60093b952379add/pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78662158093f9d5c742368f4dd9956aa595f44c6ac0860777c98853ddd5e1610", size = 5648118, upload-time = "2026-08-21T06:09:12.616Z" },+ { url = "https://files.pythonhosted.org/packages/1e/94/75c6cd5d01cf68c99c0cfd66b165dce3fd9d9fce97cd4973117cab68355d/pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0879a3f0b37876372aa328b9fee956165db174c1ab72464146fe515f49399ddb", size = 5565620, upload-time = "2026-08-21T06:09:14.223Z" },+ { url = "https://files.pythonhosted.org/packages/7a/a6/20a910d2f096908cf25ec47fa84b35b1bbb1af63dc9713f53f38ed97802e/pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:850b8de6ade09a60bdb2f969871a177c2c304b594b2034ba3f5962c7bea75551", size = 5809262, upload-time = "2026-08-21T06:09:15.897Z" },+ { url = "https://files.pythonhosted.org/packages/e4/f7/f92fe4004d93f08c5118e750feefdbc72a73438b7fd7083300728e999911/pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:399802647ea646c6ac9b5460e541b7c209b7a13563c667b2690ded2060185f2e", size = 5985550, upload-time = "2026-08-21T06:09:17.579Z" },+ { url = "https://files.pythonhosted.org/packages/47/3e/2c896e54dbe3f1ba6e3bd10e9d412f422a2e33827ff591d59f67b13e0fa5/pyqwest-0.10.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c26f3de1feb5d066d7a66802a47407a93ba043696064ad80beda4a0a4bf10056", size = 4873529, upload-time = "2026-08-21T06:09:19.161Z" },+]++[[package]]+name = "pyte"+version = "0.8.2"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "wcwidth" },+]+sdist = { url = "https://files.pythonhosted.org/packages/ab/ab/b599762933eba04de7dc5b31ae083112a6c9a9db15b01d3109ad797559d9/pyte-0.8.2.tar.gz", hash = "sha256:5af970e843fa96a97149d64e170c984721f20e52227a2f57f0a54207f08f083f", size = 92301, upload-time = "2023-11-12T09:33:43.217Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/59/d0/bb522283b90853afbf506cd5b71c650cf708829914efd0003d615cf426cd/pyte-0.8.2-py3-none-any.whl", hash = "sha256:85db42a35798a5aafa96ac4d8da78b090b2c933248819157fc0e6f78876a0135", size = 31627, upload-time = "2023-11-12T09:33:41.096Z" },+]++[[package]]+name = "ruff"+version = "0.16.5"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/f3/85/c8e12473c93018f92d19dd988a294202e1c27426c47ec4de53ffb847b8d8/ruff-0.16.5.tar.gz", hash = "sha256:1b88500f9ffbcab3dedb0082c9f9492e91ec3d618aac1236a3e0189938f7040b", size = 4912003, upload-time = "2026-08-27T16:34:18.258Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c6/b6/77c90a970fe2dae17a723acbd011043ea97c98d7deacccefdc4ba74ec512/ruff-0.16.5-py3-none-linux_armv6l.whl", hash = "sha256:12e5f673e774c35fbb62f288809c7653b73445f8ecec6b6063fd6ea3521aa14b", size = 10011941, upload-time = "2026-08-27T16:33:41.287Z" },+ { url = "https://files.pythonhosted.org/packages/4b/46/6cf67cf6411885a1d6f7f6d801682f155536a85176d10b605e2ceffed8bd/ruff-0.16.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:eda58a5802de40e7ed5b32b64e0b32539338cc6fcd2c78f61e3ad6a0d79f51c3", size = 10204049, upload-time = "2026-08-27T16:33:44.056Z" },+ { url = "https://files.pythonhosted.org/packages/46/fd/c8720ca7a090abf0c2fef4abe8a5ef6e5127ed15196d8886ff75a2b370e2/ruff-0.16.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5ae9a7b9a8875131f40f8fe967cc86abf899779efd663cb7ce3d572d01da7eb", size = 9809037, upload-time = "2026-08-27T16:33:46.257Z" },+ { url = "https://files.pythonhosted.org/packages/43/45/a684caacdedaca180f52bacccc40bf0789d2c5a7c75f25324853e9eaedb5/ruff-0.16.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b719b0a1f4d59710d283ab2965f621684a108a9e41da622e3b23f0326cd0025", size = 9964129, upload-time = "2026-08-27T16:33:48.352Z" },+ { url = "https://files.pythonhosted.org/packages/9e/f2/5d2bcdaca6b5b93d1b4dfc166cd2aebf7680143a1b38a28759df13a94d31/ruff-0.16.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2298f2780ed1be0c5cb1361e32ab7b1467f3cce7dabe101d2210a314f2fe42e9", size = 9821518, upload-time = "2026-08-27T16:33:50.57Z" },+ { url = "https://files.pythonhosted.org/packages/aa/ff/011cce29accf9257d5974145b733fc653a37985ed6825413a3987cefbfe0/ruff-0.16.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:258f29035a2dd021e7861e631b227a5b3f14e50c1184c9a6a122c5f4576154d7", size = 10534835, upload-time = "2026-08-27T16:33:52.522Z" },+ { url = "https://files.pythonhosted.org/packages/d7/5a/f0cf109bada9bba0e96c90c21c9f9251803f57225c32d293327a03c710d6/ruff-0.16.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9a4f0432966834019c74d1b7e5c51224305d7713f3d7faf3e7451f1a3be3cde", size = 11252550, upload-time = "2026-08-27T16:33:54.521Z" },+ { url = "https://files.pythonhosted.org/packages/63/4d/1d481aaea2046c6a7ed7c291f9004c669cce3c087b6b376ed5b08271e3fe/ruff-0.16.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5eb3a8c3d0ade9cea42b591fd530368e8798380e30e0a308b85a5cf718f09ea", size = 10777949, upload-time = "2026-08-27T16:33:56.88Z" },+ { url = "https://files.pythonhosted.org/packages/ee/34/ee245ca55f64443233034b3d02b03236b19242004281247c079390b7facd/ruff-0.16.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0f69e191a13a3c9816f63163c88790cb12cd157bbbb384e9c44745702ab105", size = 10311656, upload-time = "2026-08-27T16:33:59.12Z" },+ { url = "https://files.pythonhosted.org/packages/a7/4d/c33a333e341c0a2b96c715b52d89a606f5a34cd4ac493cd9b8d0187186b8/ruff-0.16.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0eeab41fbea2c42f98dfb9822cdccda9d24ba38d49f6dc945b5c236d48f0ef29", size = 10532125, upload-time = "2026-08-27T16:34:01.166Z" },+ { url = "https://files.pythonhosted.org/packages/30/e1/a64cef78b40192497bb98a27a8aa8f2c98ee9ee15bc97f7712d94ef32937/ruff-0.16.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f0768e9df4300713fff30733c87575f68b6f1d8de41184e505b7fdd9c0c95eaf", size = 10097648, upload-time = "2026-08-27T16:34:03.16Z" },+ { url = "https://files.pythonhosted.org/packages/cc/4e/4cdc9ed3c3e109d2f71e62572a37457298d7bc7501ec3138babb7ed32bbd/ruff-0.16.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:95cc70cdc7aa80c338de356279d2adbeb2de0f520b9ecd8aba75b94e95e02f91", size = 9829344, upload-time = "2026-08-27T16:34:05.134Z" },+ { url = "https://files.pythonhosted.org/packages/39/4a/31ed35ce31729955fc583ee0d176d6e784c1290cb0b0a75cb2134c1ab72a/ruff-0.16.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d185c8398ded1bfd91c0c2cb258346307571eccc473a8490af8c3977399c384a", size = 10277117, upload-time = "2026-08-27T16:34:07.425Z" },+ { url = "https://files.pythonhosted.org/packages/a8/a0/60356d86687b4b666d593df213f4dc3041750d024cb7bf2cfa81cfd65c2e/ruff-0.16.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fb8e3a3c4c6a784150a7ced53b015f4b253fc2bf97a610886419ead64b4756ef", size = 10711653, upload-time = "2026-08-27T16:34:09.712Z" },+ { url = "https://files.pythonhosted.org/packages/ed/20/656d67f5b25ca9bda4e02b1de25867b2954e1d19e03648060f167ad0f4cc/ruff-0.16.5-py3-none-win32.whl", hash = "sha256:288b0a5f080492fe5635db849f9e2e84aa3cce7b7f0e955997d416c507c76a26", size = 10034250, upload-time = "2026-08-27T16:34:11.8Z" },+ { url = "https://files.pythonhosted.org/packages/5b/42/ee8e68a207b9127fcde6c3d7e197def432f346cb1af159e1fa14ca0d1cdc/ruff-0.16.5-py3-none-win_amd64.whl", hash = "sha256:ddc6385fb2137f616357ca03d6c74f4be987f80fed4008566b754f6032b8546f", size = 10516714, upload-time = "2026-08-27T16:34:13.963Z" },+ { url = "https://files.pythonhosted.org/packages/73/e3/7df5a396e445b9ba49ce9a9437439a4d80042c61c0ade199abf8d16de1ac/ruff-0.16.5-py3-none-win_arm64.whl", hash = "sha256:a64abe90968719b851bb7cedffaa8753fbdbdadab483089682db623f3edc587e", size = 10391564, upload-time = "2026-08-27T16:34:16.064Z" },+]++[[package]]+name = "tenacity"+version = "9.1.4"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" },+]++[[package]]+name = "typing-extensions"+version = "4.16.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },+]++[[package]]+name = "typing-inspection"+version = "0.4.4"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" },+]++[[package]]+name = "wcwidth"+version = "0.8.3"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/36/57/ed58088fafdf4c55a0ad6bde846502567645424d7ebf325230b9237f4085/wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb", size = 1458450, upload-time = "2026-08-28T18:10:06.875Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" },+]diff --git a/scripts/benchmarks/worker.py b/scripts/benchmarks/worker.pynew file mode 100644index 0000000000..9c6f954370--- /dev/null+++ b/scripts/benchmarks/worker.py@@ -0,0 +1,501 @@+from __future__ import annotations++import argparse+import json+import os+import pwd+import shutil+import signal+import subprocess+import time+from pathlib import Path++from kernel import (+ BASH_EMPTY,+ BASH_OUTPUT,+ CELL_REPEATS,+ FRAME_COLUMNS,+ FRAME_ROWS,+ GIT_STATUS,+ OUTPUT_BYTES,+ SHELL_REPEATS,+ Kernel,+)+from schema import (+ ROOT,+ Metric,+ Observation,+ ProcessMemory,+ Request,+ Result,+ Side,+ write_json,+)+from terminal import Terminal++HOMES = Path("/home")+SOURCE = HOMES / "builder/source"+RESULTS = ROOT / "results"+VERSION = "0.0.0-benchmark"+ORIGIN = "http://127.0.0.1:18741"+++def clean_error(error: Exception) -> str:+ return f"{type(error).__name__}: {error}"[:500]+++def environment(user: str) -> dict[str, str]:+ home = str(HOMES / user)+ env = {+ "HOME": home,+ "USER": user,+ "LOGNAME": user,+ "SHELL": "/bin/bash",+ "PATH": f"{home}/.local/bin:/usr/local/bin:/usr/bin:/bin",+ "TERM": "xterm-256color",+ "COLORTERM": "truecolor",+ "LANG": "C.UTF-8",+ "NPM_CONFIG_PREFIX": f"{home}/.local",+ "NPM_CONFIG_CACHE": f"{home}/.npm",+ "UV_CACHE_DIR": f"{home}/.cache/uv",+ "UV_LINK_MODE": "copy",+ }+ return env+++def run_as(+ user: str,+ args: list[str],+ cwd: Path,+ *,+ timeout: int = 60,+ log: Path | None = None,+ extra_env: dict[str, str] | None = None,+ merge_output: bool = False,+) -> str:+ env = environment(user) | (extra_env or {})+ command = ["/usr/sbin/runuser", "-u", user, "--", *args]+ if log:+ with log.open("a") as stream:+ subprocess.run(+ command,+ cwd=cwd,+ env=env,+ stdin=subprocess.DEVNULL,+ stdout=stream,+ stderr=subprocess.STDOUT,+ check=True,+ timeout=timeout,+ )+ return ""+ return subprocess.check_output(+ command,+ cwd=cwd,+ env=env,+ stderr=subprocess.STDOUT if merge_output else subprocess.PIPE,+ text=True,+ timeout=timeout,+ )+++def stop_processes(user: str) -> None:+ uid = pwd.getpwnam(user).pw_uid+ for sig, delay in ((signal.SIGTERM, 3), (signal.SIGKILL, 2)):+ for process in memory(uid):+ try:+ os.kill(process.pid, sig)+ except ProcessLookupError:+ pass+ deadline = time.monotonic() + delay+ while time.monotonic() < deadline:+ if not memory(uid):+ return+ time.sleep(0.05)+ if memory(uid):+ raise RuntimeError(f"Processes remain for {user}")+++def memory(uid: int) -> list[ProcessMemory]:+ processes = []+ for path in Path("/proc").glob("[0-9]*/status"):+ try:+ fields = dict(line.split(":", 1) for line in path.read_text().splitlines() if ":" in line)+ if int(fields["Uid"].split()[0]) != uid or fields["State"].strip().startswith("Z"):+ continue+ pss = None+ try:+ for line in (path.parent / "smaps_rollup").read_text().splitlines():+ if line.startswith("Pss:"):+ pss = int(line.split()[1]) * 1024+ except (PermissionError, FileNotFoundError, ProcessLookupError):+ pass+ processes.append(+ ProcessMemory(+ pid=int(path.parent.name),+ name=fields["Name"].strip(),+ rss=int(fields.get("VmRSS", "0").split()[0]) * 1024,+ pss=pss,+ )+ )+ except (PermissionError, FileNotFoundError, ProcessLookupError):+ continue+ return processes+++def prepare(request: Request, side: Side) -> None:+ log = RESULTS / "build.log"+ source_url = f"https://github.com/{request.source_repository}.git"+ SOURCE.mkdir(parents=True, exist_ok=True)+ builder = pwd.getpwnam("builder")+ os.chown(SOURCE, builder.pw_uid, builder.pw_gid)+ for args in (+ ["git", "init", "--quiet"],+ ["git", "fetch", "--depth=1", source_url, request.sha],+ ["git", "checkout", "--detach", "FETCH_HEAD"],+ ["npm", "ci", "--no-audit", "--no-fund"],+ ):+ run_as("builder", args, SOURCE, timeout=600, log=log)+ actual = run_as("builder", ["git", "rev-parse", "HEAD"], SOURCE).strip()+ if actual != request.sha:+ raise RuntimeError("Checkout did not resolve to the requested commit")+ for package in ("tui", "ai", "agent", "coding-agent"):+ run_as(+ "builder",+ [str(SOURCE / "node_modules/.bin/tsgo"), "-p", "tsconfig.build.json"],+ SOURCE / "packages" / package,+ timeout=180,+ log=log,+ )+ agent = SOURCE / "packages/coding-agent"+ run_as("builder", ["chmod", "+x", "dist/cli.js"], agent)+ for script in ("copy-assets", "bundle"):+ run_as("builder", ["npm", "run", script], agent, timeout=180, log=log)+ run_as(+ "builder",+ [+ "node",+ "scripts/pack-prime-agent-release.mjs",+ "--base-url",+ ORIGIN,+ "--version",+ VERSION,+ "--out-dir",+ "packages/coding-agent/release/benchmark",+ ],+ SOURCE,+ timeout=180,+ log=log,+ )+ artifacts = agent / "release/benchmark/artifacts"+ side.artifacts = {path.name: path.stat().st_size for path in sorted(artifacts.glob("*.tgz"))}+ if len(side.artifacts) != 4:+ raise RuntimeError("Expected the four release package tarballs")+ record(side, "bundle", 0, sum(side.artifacts.values()))+ release = ROOT / "www/releases" / f"v{VERSION}"+ release.parent.mkdir(parents=True, exist_ok=True)+ release.symlink_to(artifacts)+ for name in ("node", "npm"):+ side.runtime[name] = run_as("builder", [name, "--version"], SOURCE).strip()+ side.runtime["kernel"] = os.uname().release+ side.runtime["machine"] = os.uname().machine+ side.runtime["artifact_format"] = "npm-tarballs"+ cpu = Path("/proc/cpuinfo").read_text()+ side.runtime["cpu"] = next(+ (line.partition(":")[2].strip() for line in cpu.splitlines() if line.startswith("model name")),+ "unavailable",+ )+ if side.runtime["machine"] != "x86_64":+ raise RuntimeError("Benchmark requires Linux x64")+++def disk_bytes(home: Path) -> int:+ output = subprocess.check_output(+ [+ "du",+ "--apparent-size",+ "--summarize",+ "--block-size=1",+ "--exclude=.cache",+ "--exclude=.npm",+ "--exclude=*.log",+ "--exclude=sessions",+ "--exclude=session-artifacts",+ "--exclude=workspace",+ str(home),+ ],+ text=True,+ )+ return int(output.split()[0])+++def install(request: Request, side: Side, trial: int) -> None:+ user = f"benchmark{trial + 1}"+ subprocess.run(+ ["useradd", "--create-home", "--uid", str(2001 + trial), "--shell", "/bin/bash", user], check=True+ )+ home = HOMES / user+ before = disk_bytes(home)+ started = time.perf_counter()+ try:+ run_as(+ user,+ ["sh", str(SOURCE / "install.sh"), VERSION],+ home,+ timeout=240,+ log=RESULTS / f"install-{trial}.log",+ extra_env={+ "PRIME_AGENT_DOWNLOAD_BASE_URL": ORIGIN,+ "PRIME_AGENT_INSTALLER_PLAIN": "1",+ "PRIME_AGENT_BOOTSTRAP_KERNEL_ON_INSTALL": "1",+ "PRIME_AGENT_BOOTSTRAP_TOOLS_ON_INSTALL": "1",+ "PRIME_AGENT_INSTALL_UV": "1",+ },+ )+ elapsed = time.perf_counter() - started+ version = run_as(user, ["prime-agent", "--version"], home, merge_output=True).strip()+ if VERSION not in version:+ raise RuntimeError(f"Installed version does not match the packed release: {version[:100]}")+ if not (home / ".prime/agent/kernel-venv/bin/python").exists():+ raise RuntimeError("The installer's Python bootstrap did not complete")+ record(side, "install", trial, elapsed)+ if trial == 0:+ side.runtime["home_before_install_bytes"] = str(before)+ info = pwd.getpwnam(user)+ settings = home / ".prime/agent/settings.json"+ settings.write_text(json.dumps({"onboardingShown": True}) + "\n")+ os.chown(settings, info.pw_uid, info.pw_gid)+ workspace = home / "workspace"+ workspace.mkdir()+ os.chown(workspace, info.pw_uid, info.pw_gid)+ (workspace / "README.md").write_text(+ "# Benchmark fixture\n\nA small repository for CLI benchmarks.\n"+ )+ (workspace / "example.py").write_text("def add(left, right):\n return left + right\n")+ for path in workspace.iterdir():+ os.chown(path, info.pw_uid, info.pw_gid)+ for command in (+ ["git", "init", "--quiet", "--initial-branch=main"],+ ["git", "add", "README.md", "example.py"],+ [+ "git",+ "-c",+ "user.name=Benchmark",+ "-c",+ "[email protected]",+ "commit",+ "--quiet",+ "-m",+ "chore: initialize benchmark fixture",+ ],+ ):+ run_as(+ user,+ command,+ workspace,+ extra_env={+ "GIT_AUTHOR_DATE": "2000-01-01T00:00:00Z",+ "GIT_COMMITTER_DATE": "2000-01-01T00:00:00Z",+ },+ )+ side.runtime["installed"] = version+ try:+ run_as(+ user,+ ["npm", "ls", "--global", "--all", "--json"],+ home,+ log=RESULTS / "installed-dependencies.json",+ )+ except subprocess.CalledProcessError:+ side.runtime["dependency_inventory"] = "npm ls reported dependency warnings; see inventory"+ except Exception as error:+ record(side, "install", trial, error=clean_error(error))+ finally:+ stop_processes(user)+++def record(+ side: Side, metric: Metric, trial: int, value: float | None = None, error: str | None = None+) -> None:+ samples = side.metrics.setdefault(metric, [])+ samples[:] = [sample for sample in samples if sample.trial != trial]+ samples.append(Observation(trial=trial, value=value, error=error))+++def stop_agents(home: Path) -> None:+ listing = json.loads(run_as("benchmark1", ["prime-agent", "list", "--json"], home))+ for session in listing["sessions"]:+ if session.get("activeSessionId"):+ run_as(+ "benchmark1",+ ["prime-agent", "stop", session["activeSessionId"], "--json"],+ home,+ )+++def measure(request: Request, side: Side, trial: int) -> None:+ if not any(s.value is not None and s.trial == 0 for s in side.metrics.get("install", [])):+ raise RuntimeError("The first installation must succeed before interactive measurements")+ home = HOMES / "benchmark1"+ stop_processes("benchmark1")+ for mode in ("cold", "warm"):+ terminal = Terminal(+ [+ "/usr/sbin/runuser",+ "-u",+ "benchmark1",+ "--",+ "prime-agent",+ ],+ home / "workspace",+ environment("benchmark1"),+ RESULTS / f"{mode}-{trial}",+ )+ try:+ record(side, mode, trial, terminal.ready())+ terminal.settle(1)+ if mode == "cold":+ processes = memory(pwd.getpwnam("benchmark1").pw_uid)+ if not processes:+ raise RuntimeError("No owned processes found for memory measurement")+ record(side, "rss", trial, sum(process.rss for process in processes))+ if all(process.pss is not None for process in processes):+ record(side, "pss", trial, sum(process.pss or 0 for process in processes))+ side.processes = processes+ write_json(RESULTS / f"memory-{trial}.json", Result(request=request, side=side))+ except Exception as error:+ record(side, mode, trial, error=clean_error(error))+ finally:+ terminal.close()+ if mode == "cold":+ stop_agents(home)+ stop_processes("benchmark1")+ if trial == 0 and any(sample.value is not None for sample in side.metrics.get("cold", [])):+ record(side, "disk", 0, disk_bytes(home) - int(side.runtime["home_before_install_bytes"]))+++def runtime(side: Side, trial: int) -> None:+ home = HOMES / "benchmark1"+ user = pwd.getpwnam("benchmark1")+ stop_processes("benchmark1")+ state = home / f"runtime-benchmark-{trial}"+ state.mkdir()+ os.chown(state, user.pw_uid, user.pw_gid)+ command = [+ "/usr/sbin/runuser",+ "-u",+ "benchmark1",+ "--",+ str(home / ".prime/agent/kernel-venv/bin/python"),+ "-m",+ "rlm.repl",+ ]+ kernel = None+ metric: Metric = "kernel_start"++ def rss() -> int:+ processes = memory(user.pw_uid)+ if not processes:+ raise RuntimeError("No kernel processes found for memory measurement")+ (RESULTS / f"{metric}-{trial}.json").write_text(+ json.dumps([process.model_dump() for process in processes], indent=2) + "\n"+ )+ return sum(process.rss for process in processes)++ try:+ kernel = Kernel(command, home / "workspace", environment("benchmark1"), RESULTS / f"kernel-{trial}")+ elapsed, python = kernel.ready()+ record(side, metric, trial, elapsed)+ side.runtime["python"] = python+ metric = "kernel_rss"+ record(side, metric, trial, rss())+ kernel.execute("from rlm import bash")+ kernel.batch("pass", 5)+ kernel.batch(BASH_EMPTY, 1)+ for metric, code, repeats, output in (+ ("kernel_exec", "pass", CELL_REPEATS, None),+ ("bash", BASH_EMPTY, SHELL_REPEATS, None),+ ("git_status", GIT_STATUS, SHELL_REPEATS, None),+ ("output", BASH_OUTPUT, SHELL_REPEATS, OUTPUT_BYTES),+ ):+ record(side, metric, trial, kernel.batch(code, repeats, output_bytes=output))+ metric = "mixed"+ record(side, metric, trial, kernel.mixed())+ metric = "interrupt"+ record(side, metric, trial, kernel.interrupt())+ metric = "loaded_rss"+ kernel.execute(+ f"import pandas as pd\nframe = pd.DataFrame({{str(i): range({FRAME_ROWS}) "+ f"for i in range({FRAME_COLUMNS})}})\npayload = list(range({FRAME_ROWS}))\n"+ f"assert frame.shape == ({FRAME_ROWS}, {FRAME_COLUMNS})"+ )+ record(side, metric, trial, rss())+ metric = "snapshot"+ started = time.perf_counter()+ events = kernel.request(+ "snapshot", path=str(state / "state.pkl"), manifest_path=str(state / "manifest.json")+ )+ elapsed = time.perf_counter() - started+ if not {"frame", "payload"} <= set(events[-1].get("saved", [])):+ raise RuntimeError("Snapshot did not save the complete state fixture")+ record(side, metric, trial, elapsed)+ kernel.close()+ kernel = None+ metric = "restore"+ kernel = Kernel(command, home / "workspace", environment("benchmark1"), RESULTS / f"restore-{trial}")+ kernel.ready()+ started = time.perf_counter()+ events = kernel.request("restore", path=str(state / "state.pkl"))+ elapsed = time.perf_counter() - started+ if events[-1].get("failed") or not {"frame", "payload"} <= set(events[-1].get("restored", [])):+ raise RuntimeError("Restore did not recover the complete state fixture")+ kernel.execute(+ f"assert frame.shape == ({FRAME_ROWS}, {FRAME_COLUMNS})\n"+ f"assert int(frame.sum().sum()) == {FRAME_COLUMNS * FRAME_ROWS * (FRAME_ROWS - 1) // 2}\n"+ f"assert payload == list(range({FRAME_ROWS}))"+ )+ record(side, metric, trial, elapsed)+ except Exception as error:+ record(side, metric, trial, error=clean_error(error))+ finally:+ try:+ if kernel:+ kernel.close()+ finally:+ stop_processes("benchmark1")+ shutil.rmtree(state)+++def main() -> None:+ parser = argparse.ArgumentParser()+ parser.add_argument("phase", choices=("prepare", "install", "measure", "runtime"))+ parser.add_argument("--trial", type=int, default=0)+ args = parser.parse_args()+ RESULTS.mkdir(exist_ok=True)+ request = Request.model_validate_json((ROOT / "request.json").read_text())+ path = RESULTS / "result.json"+ result = (+ Result.model_validate_json(path.read_text())+ if path.exists()+ else Result(+ request=request,+ side=Side(sha=request.sha),+ )+ )+ try:+ if args.phase == "prepare":+ prepare(request, result.side)+ elif args.phase == "install":+ install(request, result.side, args.trial)+ elif args.phase == "measure":+ measure(request, result.side, args.trial)+ else:+ runtime(result.side, args.trial)+ except Exception as error:+ result.side.error = clean_error(error)+ raise+ finally:+ write_json(path, result)+++if __name__ == "__main__":+ main()