High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
refactor: cleaning up a bit and merging various branches
Luigi Agosti6652ef6bb61d
diff --git a/dev b/devindex c19f46e9..74958968 100755--- a/dev+++ b/dev@@ -34,6 +34,10 @@ COMMANDS=(" api-extended:π³:API extended (multi-instance)"" cli-extended:π³:CLI extended tests"" infra-extended:π³:Infra extended (multi-instance)"+ "benchmark:π:Benchmark lanes"+ " baseline:π:Run benchmark baseline lane"+ " agent:π:Run PinchTab benchmark via Anthropic Messages API"+ " agent-browser:π:Run agent-browser benchmark via Anthropic Messages API""doctor:π©Ί:Setup dev environment""check go:π§ͺ:Go only""check dashboard:π§ͺ:Dashboard only"@@ -159,6 +163,17 @@ run_command() {echo ""exec bash scripts/docker-smoke.sh "$image";;+ "benchmark")+ echo " ${ACCENT}${BOLD}π Benchmark${NC}"+ echo ""+ exec bash scripts/dev-benchmark.sh help+ ;;+ "benchmark "*)+ echo " ${ACCENT}${BOLD}π Benchmark${NC}"+ echo ""+ local benchmark_args="${target#benchmark }"+ exec bash scripts/dev-benchmark.sh ${benchmark_args}+ ;;"e2e"*)local suite="release"local filter=""@@ -306,6 +321,13 @@ if [ $# -gt 0 ]; thenrun_command "binary"fi;;+ benchmark)+ if [ $# -gt 1 ]; then+ exec bash scripts/dev-benchmark.sh "${@:2}"+ else+ exec bash scripts/dev-benchmark.sh help+ fi+ ;;*) run_command "$1" ;;esacfidiff --git a/docs/deep-dive/agent-browser-benchmark.md b/docs/deep-dive/agent-browser-benchmark.mdindex 08afcc85..9103f9f2 100644--- a/docs/deep-dive/agent-browser-benchmark.md+++ b/docs/deep-dive/agent-browser-benchmark.md@@ -117,59 +117,172 @@ diagnostics (instance management, tab IDs, auth verification) don't apply.### Per-Step Metrics+The benchmark now uses an execution/verification split rather than a single+pass/fail write:+```bash-./scripts/record-step.sh --type <lane> <group> <step> <pass|fail> "notes"+./scripts/record-step.sh --type <lane> <group> <step> answer "<observed result>" "notes"+./scripts/verify-step.sh --type <lane> <group> <step> <pass|fail|skip> "verification notes"```-- **Group/Step**: Task identifier (e.g., 3.1 = Group 3, Step 1)-- **Status**: pass, fail, or skip-- **Notes**: Human-readable result description-- **Tool Calls**: Browser commands executed (agent-browser only, auto-counted)+Per-step fields:++- **Group/Step**: task identifier (for example `3.1`)+- **Status**:+ - execution: `answer`, `fail`, or `skip`+ - verification: `pass`, `fail`, or `skip`+- **Answer**: raw observed result recorded by the executing lane+- **Notes**: short human-readable execution or verification note+- **Tool Calls**: browser commands executed (auto-counted for `agent-browser`)++Baseline now records the same shape as the agent lanes, but verifies each step+immediately after recording the answer.++### Run-Level Usage++Step-level token fields still exist in the JSON schema for compatibility, but+the benchmark's precise token accounting lives at **run level** under+`run_usage`.++This is populated only when the lane runs through a provider API runner, not by+manual shell execution.++Current runner:-**Note**: Per-step token tracking is not currently implemented. The harness-supports `--tokens <in> <out>` arguments, but agents running inside Claude-Code don't have access to their own API usage during execution.+- `tests/benchmark/scripts/run-api-benchmark.ts`++Supported providers:++- Anthropic Messages API+- OpenAI Responses API++Run-level fields:++| Field | Meaning |+|-------|---------|+| `input_tokens` | Fresh uncached input tokens |+| `cache_creation_input_tokens` | Prompt-cache write tokens |+| `cache_read_input_tokens` | Prompt-cache hit/read tokens |+| `total_input_tokens` | Sum of fresh + cache create + cache read |+| `output_tokens` | Output tokens |+| `total_tokens` | Sum of total input + output |+| `request_count` | API requests made by the runner |### Aggregate Metrics| Metric | Description ||--------|-------------|-| Steps Passed | Tasks completed successfully |-| Steps Failed | Tasks that did not meet verification criteria |-| Steps Skipped | Tasks not attempted |-| Total Tokens | Sum of input + output tokens (from API) |-| Tool Uses | Total LLM tool invocations |-| Duration | Wall-clock execution time |+| Steps Answered | Steps with an observed result |+| Execution Failed | Steps the runner could not complete |+| Execution Skipped | Steps intentionally skipped |+| Verified Passed | Answered steps that matched the oracle |+| Verified Failed | Answered steps that did not match the oracle |+| Pending Verification | Answered steps not yet verified |+| Tool Calls | Browser-tool commands executed |+| Run Usage | Exact provider usage for the whole run |-### How Total Tokens Are Measured+### How Token Usage Is Measured with API Keys-The `total_tokens` metric is **exact**, not estimated. It comes directly-from the Anthropic API:+When running with provider API keys, the benchmark records usage from the API+response envelope itself rather than asking the model to self-report usage.+Provider selection:++- `OPENAI_API_KEY` -> OpenAI Responses API+- `ANTHROPIC_API_KEY` -> Anthropic Messages API+- if both are set, the runner requires explicit `--provider`++Examples:++```bash+./dev benchmark baseline++OPENAI_API_KEY=... ./dev benchmark agent --provider openai+OPENAI_API_KEY=... ./dev benchmark agent-browser --provider openai++ANTHROPIC_API_KEY=... ./dev benchmark agent --provider anthropic+ANTHROPIC_API_KEY=... ./dev benchmark agent-browser --provider anthropic```-βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-β Agent Execution Flow β-βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€-β β-β Agent Turn 1 βββΆ Anthropic API βββΆ usage.input: 5,234 β-β usage.output: 892 β-β β-β Agent Turn 2 βββΆ Anthropic API βββΆ usage.input: 6,102 β-β usage.output: 1,204 β-β β-β ... (281 tool uses) ... β-β β-β Claude Code sums all usage βββΆ total_tokens: 104,695 β-β β-βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-```-Each API response includes exact token counts. Claude Code accumulates-these across all turns and reports the total when the agent completes.+#### Anthropic++The runner reads:++- `input_tokens`+- `output_tokens`+- `cache_creation_input_tokens`+- `cache_read_input_tokens`++These come from Anthropic's `usage` object on each response and are summed+across the run.++#### OpenAI++The runner reads:++- `usage.input_tokens`+- `usage.output_tokens`+- `usage.input_tokens_details.cached_tokens`++The benchmark maps OpenAI cached input tokens into the shared+`cache_read_input_tokens` field.++### What the Token Total Actually Represents-This makes `total_tokens` a reliable metric for comparing browser tool-efficiency, even without per-step breakdown. The total represents the-**real cost** of completing all 85 tasks with each tool surface.+The run-level total is the cost of the **entire agent loop**, not a pure+"browser tool only" number. It includes:++- system prompt+- benchmark instructions+- tool request/response turns+- execution + verification reasoning+- retries and recovery turns+- any runner-added context summaries++It does **not** include:++- purely local shell execution cost+- Docker CPU time+- a separate local token cache outside the provider API++This means the benchmark compares the **real operational cost** of running the+lane with an agent, not an abstract idealized tool cost.++### Harness Overhead vs Tool Overhead++When using the same API-key runner for both `agent` and `agent-browser`, the+harness-added overhead is intentionally similar:++- same provider API wrapper+- same system prompt style+- same execution/verification workflow+- same conversation-compaction logic+- same report-based progress summaries++What can still differ is the tool-under-test overhead:++- tool output shape and verbosity+- number of tool calls needed+- recovery behavior after errors+- command composition efficiency++That remaining difference is the thing the benchmark wants to measure.++### Cost Control and Context Compaction++Naive API runners can become expensive because they resend the entire running+conversation, including long tool outputs, on every request. The current+TypeScript API runner reduces this in three ways:++1. tool output is truncated before being fed back to the model+2. old history is compacted into a short progress summary derived from the+ benchmark report+3. Anthropic defaults to a cheaper model (`claude-haiku-4-5-20251001`) unless a+ different model is specified++This means a benchmark run with API keys is now measuring a more realistic+agent loop instead of an unnecessarily bloated replay of every prior tool+result.## Results: April 15, 2026 Run@@ -278,6 +391,40 @@ session management is simpler but provides less visibility.PinchTab's Go implementation with direct Chrome DevTools Protocol accessis inherently faster than Node.js with Playwright's abstraction layer.+### 4.1 PinchTab-Specific Execution Gotchas++Two PinchTab behaviors are important when interpreting agent traces:++#### Navigation 409s after successful clicks++A navigation-triggering click can return:++```text+Error 409: unexpected page navigation: http://fixtures/wiki.html -> http://fixtures/wiki-go.html+```++This does **not** necessarily mean the click failed. It usually means:++1. the click succeeded+2. the page navigated immediately+3. the server aborted the in-flight click response because the page changed++Benchmark runners should treat this as "likely success" and verify the result+with a fresh snapshot or text read.++#### Download endpoint output++The PinchTab download endpoint returns JSON containing base64-encoded file+content, not a ready-made local file path. A runner that assumes a file already+exists on disk will fail incorrectly.++The correct handling is:++1. call the download endpoint+2. parse the JSON response+3. decode the `data` field from base64+4. verify the decoded content+### 5. Docker and Wrapper OverheadBoth lanes run through Docker, which adds overhead that wouldn't exist indiff --git a/scripts/dev-benchmark.sh b/scripts/dev-benchmark.shnew file mode 100755index 00000000..7c58fd04--- /dev/null+++ b/scripts/dev-benchmark.sh@@ -0,0 +1,70 @@+#!/usr/bin/env bash+set -euo pipefail++ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"+BENCH_DIR="${ROOT_DIR}/tests/benchmark"+RESULTS_DIR="${BENCH_DIR}/results"++resolve_current_report() {+ local ptr="$1"+ if [[ -f "${ptr}" ]]; then+ tr -d '[:space:]' < "${ptr}"+ return 0+ fi+ return 1+}++usage() {+ cat <<'EOF'+Usage:+ ./dev benchmark baseline+ OPENAI_API_KEY=... ./dev benchmark agent [extra runner args...]+ OPENAI_API_KEY=... ./dev benchmark agent-browser [extra runner args...]+ ANTHROPIC_API_KEY=... ./dev benchmark agent [extra runner args...]+ ANTHROPIC_API_KEY=... ./dev benchmark agent-browser [extra runner args...]++Examples:+ ./dev benchmark baseline+ OPENAI_API_KEY=... ./dev benchmark agent --model gpt-5+ OPENAI_API_KEY=... ./dev benchmark agent-browser --model gpt-5+ ANTHROPIC_API_KEY=... ./dev benchmark agent --model claude-haiku-4-5-20251001+ ANTHROPIC_API_KEY=... ./dev benchmark agent-browser --model claude-haiku-4-5-20251001+ ANTHROPIC_API_KEY=... ./dev benchmark agent --provider anthropic --turn-delay-ms 3000+ ANTHROPIC_API_KEY=... ./dev benchmark agent --provider anthropic --profile common10+ ANTHROPIC_API_KEY=... ./dev benchmark agent-browser --provider anthropic --groups 0,1,2,3+EOF+}++if [[ $# -lt 1 ]]; then+ usage+ exit 1+fi++mode="$1"+shift++cd "${BENCH_DIR}"++case "${mode}" in+ baseline)+ ./scripts/run-optimization.sh+ ./scripts/baseline.sh+ BASELINE_REPORT="$(resolve_current_report "${RESULTS_DIR}/current_baseline_report.txt")"+ ./scripts/finalize-report.sh "${BASELINE_REPORT}"+ ;;+ agent|agent-browser)+ if [[ -z "${OPENAI_API_KEY:-}" && -z "${ANTHROPIC_API_KEY:-}" ]]; then+ echo "ERROR: OPENAI_API_KEY or ANTHROPIC_API_KEY is required for benchmark ${mode}" >&2+ exit 1+ fi+ exec node --experimental-strip-types ./scripts/run-api-benchmark.ts --lane "${mode}" --finalize "$@"+ ;;+ -h|--help|help)+ usage+ ;;+ *)+ echo "ERROR: unknown benchmark mode: ${mode}" >&2+ usage+ exit 1+ ;;+esacdiff --git a/tests/benchmark/BASELINE_TASKS.md b/tests/benchmark/BASELINE_TASKS.mdindex fc22ceba..16efe850 100644--- a/tests/benchmark/BASELINE_TASKS.md+++ b/tests/benchmark/BASELINE_TASKS.md@@ -21,11 +21,11 @@ docker compose up -d --build## Recording```bash-# Baseline (no tokens, optional response bytes):-./scripts/record-step.sh --type baseline <group> <step> <pass|fail> "notes"+# Baseline execution answer:+./scripts/record-step.sh --type baseline <group> <step> answer "<observed result>" "notes"-# Minimal (just pass/fail):-./scripts/record-step.sh <group> <step> <pass|fail> "notes"+# Immediate verification:+./scripts/verify-step.sh --type baseline <group> <step> <pass|fail|skip> "verification notes"```**On failure, include in notes:**diff --git a/tests/benchmark/CRON_TASK.md b/tests/benchmark/CRON_TASK.mdindex 00eaf9c6..75e31857 100644--- a/tests/benchmark/CRON_TASK.md+++ b/tests/benchmark/CRON_TASK.md@@ -1,63 +1,85 @@# PinchTab Optimization Cron Task-**Goal: Close the gap between baseline (explicit curl) and agent (natural language).**+**Goal: improve the agent lane against a known-good benchmark environment.**-Every run must bring the agent closer to 100%. If already at 100%, expand tests.+Baseline is not part of the recurring agent task. It is a direct shell+verification that should be run after infrastructure, fixture, or benchmark+changes to confirm the environment still behaves as expected.----## Task for Agent+## Baseline Gate++Run baseline manually when any of these change:++- PinchTab server or CLI behavior+- benchmark fixtures+- `tests/benchmark/scripts/baseline.sh`+- benchmark Docker setup++Command:++```bash+cd ~/dev/pinchtab/tests/benchmark+./scripts/run-optimization.sh+./scripts/baseline.sh+./scripts/finalize-report.sh "$(cat ./results/current_baseline_report.txt)"+```++Only continue with optimization work if baseline is green.++---++## Recurring Optimization Task### Step 1 β Setup```bashcd ~/dev/pinchtab/tests/benchmarkgit checkout feat/benchmark && git pull --rebase origin feat/benchmark-./run-optimization.sh+./scripts/run-optimization.sh# Note the TIMESTAMP from output```-### Step 2 β Run Baseline Benchmark-Execute BASELINE_TASKS.md exactly as written.-- Token: `benchmark-token`-- Record each step: `./record-step.sh <group> <step> <pass|fail> <in> <out> "notes"`-- For failures: include HTTP status, expected string, actual response in notes+### Step 2 β Run Agent Benchmark+Execute `AGENT_TASKS.md` using `skills/pinchtab/SKILL.md` as the guide.-### Step 3 β Run Agent Benchmark-Execute AGENT_TASKS.md using SKILL.md as your only guide.-- **Do NOT look at BASELINE_TASKS.md**-- Figure out commands from skill documentation-- Record: `./record-agent-step.sh <group> <step> <pass|fail> <in> <out> "commands used" "result"`-- Log every curl/command executed+- Use the benchmark wrapper `./scripts/pt`+- Record results with `record-step.sh --type agent`+- Verify answers with `verify-step.sh`+- Log every command executed+- Do not use baseline as an execution guide-### Step 4 β Gap Analysis-Compare results and classify each agent failure:+### Step 3 β Gap Analysis+Compare agent results against the latest green baseline report and classify failures:| Failure Type | Cause | Fix ||---|---|---|-| **Wrong endpoint** | Agent used `/text` when should use `/snapshot` | Improve SKILL.md |-| **Wrong selector** | Agent guessed selector incorrectly | Improve SKILL.md or fixture |-| **Missing step** | Agent skipped a required action | Clarify AGENT_TASKS.md |-| **Wrong URL** | Agent used wrong fixture path | Clarify AGENT_TASKS.md |+| **Wrong endpoint** | Agent used `/text` when should use `/snapshot` | Improve `SKILL.md` |+| **Wrong selector** | Agent guessed selector incorrectly | Improve `SKILL.md` or fixture |+| **Missing step** | Agent skipped a required action | Clarify `AGENT_TASKS.md` |+| **Wrong URL** | Agent used wrong fixture path | Clarify `AGENT_TASKS.md` || **API bug** | Endpoint behaves unexpectedly | Fix PinchTab code |-| **Test ambiguity** | Verification string hard to find | Fix fixture or test |+| **Test ambiguity** | Verification string hard to find | Fix fixture or `scripts/baseline.sh` |-### Step 5 β Make Exactly 1 Change+### Step 4 β Make Exactly 1 ChangePriority order:1. **API Bug** β Fix PinchTab Go code, commit as `fix: <description>`-2. **Skill Gap** β Improve SKILL.md with clearer guidance or example, commit as `docs(skill): <description>`-3. **Test Ambiguity** β Fix fixture HTML or BASELINE_TASKS.md verification, commit as `test: <description>`-4. **Agent Task Clarity** β Improve AGENT_TASKS.md phrasing, commit as `test(agent): <description>`-5. **No Gaps Found** β Add 2-3 new test cases covering uncovered scenarios, commit as `test: add cases for <scenario>`+2. **Skill Gap** β Improve `SKILL.md`, commit as `docs(skill): <description>`+3. **Test Ambiguity** β Fix fixture HTML or `scripts/baseline.sh`, commit as `test: <description>`+4. **Agent Task Clarity** β Improve `AGENT_TASKS.md`, commit as `test(agent): <description>`+5. **No Gaps Found** β Add new benchmark coverage, with matching baseline + agent cases-**If agent is already at 100%**: Add harder cases (nested interactions, state persistence, multi-page flows).+**If agent is already at 100%**: add harder cases.-### Step 6 β Verify the Fix Makes Sense-Before committing, ask: "Will this change help the agent pick the right tool next time?"-- If you fixed a skill doc: check that the added text directly addresses the failure-- If you fixed a test: check the verification string is reachable via snapshot+### Step 5 β Verify the Fix Makes Sense+Before committing, ask:-### Step 7 β Commit & Push+- will this change help the agent pick the right tool next time?+- if this is a skill/docs fix, does it directly address the observed failure?+- if this is a benchmark fix, is the expected signal reachable via the real tool flow?++### Step 6 β Commit & Push```bashcd ~/dev/pinchtabgit add -A@@ -65,14 +87,14 @@ git commit -m "<type>: <clear description of what changed and why>"git push origin feat/benchmark```-### Step 8 β Log the Run+### Step 7 β Log the RunAppend to `results/optimization_log.md`:```markdown## Run #N β YYYY-MM-DD HH:MM**Results:**-- Baseline: X/Y (Z%)+- Baseline reference: X/Y (Z%)- Agent: X/Y (Z%)- Gap: N steps@@ -93,11 +115,12 @@ Append to `results/optimization_log.md`:[What to watch in the next run]```-### Step 9 β Report to User+### Step 8 β Report to UserSend a concise update:-```++```textRun #N complete-Baseline: X% | Agent: Y% | Gap: N steps+Baseline reference: X% | Agent: Y% | Gap: N stepsFailure: [brief description]Fix: [what changed] ([commit])```@@ -107,12 +130,13 @@ Fix: [what changed] ([commit])## Success Criteria- **Short term**: Agent pass rate β₯ 95%-- **Long term**: Agent matches baseline on all existing tests-- **Ongoing**: When gap closes, increase test complexity+- **Long term**: Agent matches the latest green baseline on all existing tests+- **Ongoing**: When the gap closes, increase test complexity## What NOT to do-- β Don't make multiple changes per run β one focused fix only-- β Don't skip the analysis step β root cause first, then fix-- β Don't change tests just to make them easier β the goal is better skill/API-- β Don't commit if the change doesn't directly address an observed failure+- Do not run baseline as an agent task+- Do not make multiple changes per run+- Do not skip root-cause analysis+- Do not weaken tests just to improve the score+- Do not commit a change that does not directly address an observed failurediff --git a/tests/benchmark/OPTIMIZATION_LOOP.md b/tests/benchmark/OPTIMIZATION_LOOP.mdindex def048f7..d8a1175e 100644--- a/tests/benchmark/OPTIMIZATION_LOOP.md+++ b/tests/benchmark/OPTIMIZATION_LOOP.md@@ -74,24 +74,32 @@ curl -sf -H "Authorization: Bearer benchmark-token" http://localhost:9867/health| 5 Implement | **Main agent** | Make the change, leave uncommitted || 6 Log | **Main agent** | Append to optimization_log.md |-## Baseline is a Prerequisite, Not Part of the Loop+## Baseline is a Shell Verification, Not an Agent Run-The baseline suite (`BASELINE_TASKS.md`) validates that the benchmark infrastructure itself works β fixtures serve correctly, PinchTab APIs behave as expected, and the test conditions are reachable. It's deterministic (no LLM involved), so once it's at 100% it stays at 100% unless something changes underneath it.+The baseline suite (`scripts/baseline.sh`) validates that the benchmark infrastructure itself works β fixtures serve correctly, PinchTab APIs behave as expected, and the test conditions are reachable. It's deterministic (no LLM involved), so once it's at 100% it stays at 100% unless something changes underneath it.-**The loop does NOT re-run the baseline each iteration.** Instead, run the baseline manually when any of these change:+Run baseline directly as a shell command:++```bash+./scripts/run-optimization.sh+./scripts/baseline.sh+```++**The optimization loop does NOT treat baseline as an agent task and does NOT re-run it every iteration.** Instead, run the baseline manually when any of these change:- PinchTab server code (anything in `internal/` that affects the HTTP API)- Fixtures (anything in `tests/benchmark/fixtures/`)-- `BASELINE_TASKS.md` itself (adding/modifying tests)+- `scripts/baseline.sh` itself (adding/modifying tests)- `docker-compose.yml` or the Docker image**Workflow for adding a new test case:**-1. Write the new test in `BASELINE_TASKS.md` (curl commands + pass condition)+1. Write the executable baseline case in `scripts/baseline.sh`.+ If you keep `BASELINE_TASKS.md` around as reference material, update it second.2. Write the matching task in `AGENT_TASKS.md` (natural language)3. Write the summary row in `TEST_CASES.md`4. Add any new fixture files needed-5. **Run the baseline** β it must reach 100% before the case is considered valid+5. **Run the baseline shell verification** β it must reach 100% before the case is considered valid6. If baseline fails: fix the test/fixture/PinchTab until it passes7. Commit the new case β now the agent loop can use it@@ -146,12 +154,13 @@ Each run appends to `results/optimization_log.md`:## Files+- `scripts/baseline.sh` β Executable baseline lane source of truth- `scripts/run-optimization.sh` β Main loop script (initializes reports)- `scripts/record-step.sh` β Appends a step result to the current report- `results/optimization_log.md` β Run history (includes per-run metrics table)- `results/best_score.txt` β High-water mark for regression detection- Changes left uncommitted for manual review and commit-Agent runs use the PinchTab CLI directly via the standard env vars-(`PINCHTAB_TOKEN`, `PINCHTAB_SERVER`, `PINCHTAB_TAB`) β no helper file to-source. See `AGENT_TASKS.md` "Recommended setup" for the pattern.+Agent runs use the benchmark wrapper `./scripts/pt`, which forwards to the+PinchTab CLI inside Docker while preserving `PINCHTAB_TAB` targeting. See+`AGENT_TASKS.md` "Recommended setup" for the exact pattern.diff --git a/tests/benchmark/README.md b/tests/benchmark/README.mdindex b653f212..b54fadf8 100644--- a/tests/benchmark/README.md+++ b/tests/benchmark/README.md@@ -8,13 +8,33 @@ browser-control surfaces against the same fixture suite.```bashcd tests/benchmark-# PinchTab lane+# initialize PinchTab benchmark reports + Docker./scripts/run-optimization.sh+# deterministic baseline verification after infrastructure/product changes+./scripts/baseline.sh+./scripts/finalize-report.sh "$(cat ./results/current_baseline_report.txt)"+# agent-browser lane./scripts/run-agent-browser-benchmark.sh# Then read AGENT_BROWSER_INSTRUCTIONS.md and run tasks from AGENT_BROWSER_TASKS.md with ./scripts/ab./scripts/finalize-report.sh++# optional provider API runner+OPENAI_API_KEY=... node --experimental-strip-types ./scripts/run-api-benchmark.ts --lane agent --finalize+OPENAI_API_KEY=... node --experimental-strip-types ./scripts/run-api-benchmark.ts --lane agent-browser --finalize+ANTHROPIC_API_KEY=... node --experimental-strip-types ./scripts/run-api-benchmark.ts --lane agent --model claude-haiku-4-5-20251001 --finalize+ANTHROPIC_API_KEY=... node --experimental-strip-types ./scripts/run-api-benchmark.ts --lane agent-browser --model claude-haiku-4-5-20251001 --finalize++# repo entrypoints+./dev benchmark baseline+OPENAI_API_KEY=... ./dev benchmark agent+OPENAI_API_KEY=... ./dev benchmark agent-browser+ANTHROPIC_API_KEY=... ./dev benchmark agent+ANTHROPIC_API_KEY=... ./dev benchmark agent-browser+ANTHROPIC_API_KEY=... ./dev benchmark agent --provider anthropic --turn-delay-ms 3000+ANTHROPIC_API_KEY=... ./dev benchmark agent --provider anthropic --profile common10+ANTHROPIC_API_KEY=... ./dev benchmark agent-browser --provider anthropic --groups 0,1,2,3```## MANDATORY: Docker Environment@@ -35,12 +55,15 @@ If Docker build fails or is skipped, the benchmark is **INVALID**.|------|---------|| `../../skills/pinchtab/SKILL.md` | PinchTab skill (same as shipped product) || `AGENT_BROWSER_INSTRUCTIONS.md` | Benchmark-local instructions for the `agent-browser` lane |-| `BASELINE_TASKS.md` | Standalone task list (same as skill) |+| `BASELINE_TASKS.md` | Legacy descriptive baseline spec; not executed by the harness || `AGENT_BROWSER_TASKS.md` | Equivalent task lane for `agent-browser` |+| `scripts/baseline.sh` | Executable baseline lane source of truth || `scripts/run-optimization.sh` | Initialize PinchTab benchmark reports || `scripts/run-agent-browser-benchmark.sh` | Start fixtures + `agent-browser` and initialize a fresh report || `scripts/ab` | Docker-backed `agent-browser` wrapper with tool-call logging || `scripts/record-step.sh` | Record step results and tool-call counts |+| `scripts/record-run-usage.sh` | Stamp exact run-level usage into a report |+| `scripts/run-api-benchmark.ts` | Optional OpenAI/Anthropic API runner with exact run usage capture || `scripts/finalize-report.sh` | Generate final summary report || `config/pinchtab.json` | PinchTab configuration || `agent-browser/Dockerfile` | `agent-browser` benchmark image |@@ -52,20 +75,54 @@ If Docker build fails or is skipped, the benchmark is **INVALID**.The benchmark is designed to run in a fresh agent context:1. Initialize the relevant benchmark lane-2. Execute the natural-language tasks with the target browser surface-3. Record each step's raw answer/result-4. Verify observed steps in a separate pass-5. Let the harness count browser/tool calls where possible+2. Run baseline as a direct shell verification when you need to confirm the benchmark environment still works after a change+3. Execute the natural-language tasks only for the agent and agent-browser lanes+4. Record each step's raw answer/result+5. Verify observed steps in a separate pass+6. Let the harness count browser/tool calls where possibleThis measures the **real cost** of using a browser tool with an agent, including:- Context loading overhead- Browser/tool-call count- Total benchmark cost-For agent lanes, the recommended flow is two-phase:+Baseline is not an agent lane. It is a deterministic shell script verification:++- executable source of truth: `scripts/baseline.sh`+- typical use: run it after PinchTab, fixtures, or benchmark-harness changes+- purpose: confirm the benchmark environment still behaves as expected before spending agent budget++For the agent lanes, the recommended flow is two-phase:++1. execution records each step as `answer`+2. verification stamps each answered step with `verify-step.sh`++Baseline runs do both phases inline inside `scripts/baseline.sh`. Agent lanes keep+execution and verification separate by default.++## Token Usage++The benchmark distinguishes between:++- **step data**: execution answers, tool calls, pass/fail/skip+- **run usage**: exact model usage reported by the provider for the full run-1. execution agent records each step as `answer`-2. verifier agent reads the report and stamps each step with `verify-step.sh`+Step-level `--tokens` on `record-step.sh` remains only as backward-compatible+schema baggage. The preferred path for precise token accounting is run-level+usage recorded after the run completes.++When the optional provider runner is used, the report stores exact usage from+the backing API:++- `input_tokens` (uncached input after the cache breakpoint)+- `cache_creation_input_tokens`+- `cache_read_input_tokens`+- `output_tokens`+- `total_input_tokens`+- `total_tokens`++Those values live under `run_usage` in the report JSON and are summarized by+`finalize-report.sh`.## Environment@@ -79,16 +136,16 @@ The benchmark runs PinchTab in Docker with:## Step Recording-Every step must record the answer/result:+Every step should record the observed answer/result:```bash-./scripts/record-step.sh <group> <step> <pass|fail|skip|answer> "notes"+./scripts/record-step.sh --type agent <group> <step> <answer|fail|skip> "answer" "notes"```Example:```bash-./scripts/record-step.sh 1 1 pass "Navigation completed in 1.2s"-./scripts/record-step.sh 2 3 fail "Element not found"+./scripts/record-step.sh --type agent 1 1 answer "Navigation completed in 1.2s" "observed output"+./scripts/record-step.sh --type agent 2 3 fail "Element not found"```Deferred-verification example:@@ -101,6 +158,13 @@ Deferred-verification example:"Answer satisfies the benchmark oracle"```+Baseline automation uses the same report shape, but verifies inline:++```bash+./scripts/record-step.sh --type baseline 1 1 answer "ok" "Health endpoint returned ok"+./scripts/verify-step.sh --type baseline 1 1 pass "Health endpoint matched oracle"+```+## ReportsReports are generated in `results/`:@@ -131,11 +195,70 @@ Reports are generated in `results/`:For automated benchmarks, you can:-1. Parse `BASELINE_TASKS.md` for curl commands-2. Execute each command-3. Parse responses for pass/fail+1. Run `scripts/baseline.sh` for deterministic shell verification+2. Treat `scripts/baseline.sh` as the executable baseline contract+3. Use the API runner only for `agent` and `agent-browser`4. Call `scripts/record-step.sh` with results-5. Run `scripts/finalize-report.sh`+5. Call `scripts/verify-step.sh` for answered steps+6. Run `scripts/finalize-report.sh`++### Optional Provider Runner++The API runner executes the benchmark lane through the OpenAI Responses API or+Anthropic Messages API with a persistent local shell tool. It is useful when+you want exact provider-reported usage at the end of the run.++Examples:++```bash+cd tests/benchmark++# PinchTab lane+OPENAI_API_KEY=... node --experimental-strip-types ./scripts/run-api-benchmark.ts \+ --lane agent \+ --model gpt-5 \+ --finalize++# PinchTab lane via Anthropic+ANTHROPIC_API_KEY=... node --experimental-strip-types ./scripts/run-api-benchmark.ts \+ --lane agent \+ --model claude-haiku-4-5-20251001 \+ --finalize++# agent-browser lane via Anthropic+ANTHROPIC_API_KEY=... node --experimental-strip-types ./scripts/run-api-benchmark.ts \+ --lane agent-browser \+ --model claude-haiku-4-5-20251001 \+ --finalize+```++By default the runner enables provider prompt caching when supported, so the+final report can include cached and uncached input token counts.++If you hit provider rate limits, add a delay between API turns:++```bash+ANTHROPIC_API_KEY=... ./dev benchmark agent --provider anthropic --turn-delay-ms 3000+```++The runner also honors provider `retry-after` headers automatically on `429`+responses and prints the resolved provider/model/report path before execution.+It also writes a tool-command trace to `results/agent_commands.ndjson` and+stops early if too many turns pass without recording any benchmark progress.++You can also run only a subset of benchmark groups:++```bash+# Representative ~10% subset of groups+ANTHROPIC_API_KEY=... ./dev benchmark agent --provider anthropic --profile common10++# Exact group selection+ANTHROPIC_API_KEY=... ./dev benchmark agent-browser --provider anthropic --groups 0,1,2,3+```++Current subset preset:++- `common10` -> groups `0,1,2,3` (setup, reading/extraction, search, form)## Reproducibilitydiff --git a/tests/benchmark/continuous-loop-plan.md b/tests/benchmark/continuous-loop-plan.mdindex a142ed83..feaf1b85 100644--- a/tests/benchmark/continuous-loop-plan.md+++ b/tests/benchmark/continuous-loop-plan.md@@ -47,8 +47,8 @@ regressions that poison later loops.- For server changes: update action in `internal/bridge/...`, update`internal/handlers/...` if HTTP contract changes, add tests.- For fixture cases: add the HTML/JS fixture in- `tests/benchmark/fixtures/<name>.html`, add the baseline step in- `BASELINE_TASKS.md`, add the matching agent task in `AGENT_TASKS.md`,+ `tests/benchmark/fixtures/<name>.html`, add the executable baseline step in+ `scripts/baseline.sh`, add the matching agent task in `AGENT_TASKS.md`,update `TEST_CASES.md` summary, update the verification-strings table.- Run `go test ./internal/...` β must pass.- Rebuild Docker: `cd tests/benchmark && docker compose up -d --build`.diff --git a/tests/benchmark/scripts/baseline_all.sh b/tests/benchmark/scripts/baseline.shsimilarity index 95%rename from tests/benchmark/scripts/baseline_all.shrename to tests/benchmark/scripts/baseline.shindex c3bffbda..046fb255 100755--- a/tests/benchmark/scripts/baseline_all.sh+++ b/tests/benchmark/scripts/baseline.sh@@ -2,6 +2,13 @@# Runs every baseline task against the running PinchTab container and records# each step into the most recent baseline_*.json report (see run-optimization.sh).#+# IMPORTANT:+# - This script is the executable source of truth for the baseline lane.+# - `tests/benchmark/BASELINE_TASKS.md` is descriptive/spec documentation only;+# `./dev benchmark baseline` does not parse that markdown file.+# - If baseline behavior changes, update this script first and then update+# any markdown/docs that describe the baseline contract.+## Prerequisites:# - `bash scripts/run-optimization.sh` has been run first (initializes the# empty baseline_*.json this script appends into).@@ -17,11 +24,33 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."AUTH="Authorization: Bearer benchmark-token"BASE="http://localhost:9867"-REC() { ./scripts/record-step.sh --type baseline "$@" > /dev/null; }+REC() {+ local group="$1"+ local step="$2"+ local verdict="$3"+ shift 3++ local answer="${1:-}"+ local notes="${2:-$answer}"++ case "${verdict}" in+ pass|fail)+ ./scripts/record-step.sh --type baseline "${group}" "${step}" answer "${answer}" "${notes}" > /dev/null+ ./scripts/verify-step.sh --type baseline "${group}" "${step}" "${verdict}" "${notes}" > /dev/null+ ;;+ skip)+ ./scripts/record-step.sh --type baseline "${group}" "${step}" skip "${notes}" > /dev/null+ ;;+ *)+ echo "ERROR: unsupported baseline verdict: ${verdict}" >&2+ return 1+ ;;+ esac+}# VERIFY_MARKER <group> <step> <text> <marker> ["extra notes"]-# Grep for the marker in the text; record pass with the marker as evidence-# (so the verifier can audit baseline results), or fail with what was expected.+# Grep for the marker in the text; record the observed marker as the baseline+# answer and immediately stamp verification.VERIFY_MARKER() {local g="$1" s="$2" text="$3" marker="$4" extra="${5:-}"if printf '%s' "$text" | grep -q "$marker"; thendiff --git a/tests/benchmark/scripts/finalize-report.sh b/tests/benchmark/scripts/finalize-report.shindex e8cf56fc..d45c4ab3 100755--- a/tests/benchmark/scripts/finalize-report.sh+++ b/tests/benchmark/scripts/finalize-report.sh@@ -66,13 +66,19 @@ jq -r 'def pct($a; $b):if $b == 0 then "0.0%" else (((1000 * $a) / $b | round) / 10 | tostring) + "%" end;. as $root |- ($root.totals.steps_passed + $root.totals.steps_failed + $root.totals.steps_skipped) as $baseline_total |- (($root.totals.steps_answered // 0) + $root.totals.steps_failed + $root.totals.steps_skipped) as $agent_total |+ ($root.run_usage // {}) as $usage |+ ($root.totals.steps_answered // 0) as $answered |+ ($root.totals.steps_failed // 0) as $execution_failed |+ ($root.totals.steps_skipped // 0) as $execution_skipped |+ ($root.totals.steps_passed // 0) as $legacy_passed |+ ($root.benchmark.type == "baseline" and $answered == 0) as $legacy_baseline |+ ($legacy_passed + $execution_failed + $execution_skipped) as $legacy_total |+ ($answered + $execution_failed + $execution_skipped) as $execution_total |($root.totals.steps_verified_passed // 0) as $verified_passed |($root.totals.steps_verified_failed // 0) as $verified_failed |($root.totals.steps_verified_skipped // 0) as $verified_skipped |($root.totals.steps_pending_verification // 0) as $pending |- (if $root.benchmark.type == "baseline" then+ (if $legacy_baseline then["# Benchmark Summary","",@@ -80,10 +86,10 @@ jq -r '"|--------|-------|","| Type | \($root.benchmark.type) |","| Model | \($root.benchmark.model) |",- "| Steps Passed | \($root.totals.steps_passed) |",- "| Steps Failed | \($root.totals.steps_failed) |",- "| Steps Skipped | \($root.totals.steps_skipped) |",- "| Pass Rate | \(pct($root.totals.steps_passed; $baseline_total)) |",+ "| Steps Passed | \($legacy_passed) |",+ "| Steps Failed | \($execution_failed) |",+ "| Steps Skipped | \($execution_skipped) |",+ "| Pass Rate | \(pct($legacy_passed; $legacy_total)) |","| Tool Calls | \($root.totals.tool_calls // 0) |"]else@@ -94,19 +100,41 @@ jq -r '"|--------|-------|","| Type | \($root.benchmark.type) |","| Model | \($root.benchmark.model) |",- "| Steps Answered | \($root.totals.steps_answered // 0) |",- "| Steps Failed | \($root.totals.steps_failed) |",- "| Steps Skipped | \($root.totals.steps_skipped) |",- "| Answer Rate | \(pct($root.totals.steps_answered // 0; $agent_total)) |",+ "| Steps Answered | \($answered) |",+ "| Execution Failed | \($execution_failed) |",+ "| Execution Skipped | \($execution_skipped) |",+ "| Answer Rate | \(pct($answered; $execution_total)) |","| Verified Passed | \($verified_passed) |","| Verified Failed | \($verified_failed) |","| Verified Skipped | \($verified_skipped) |","| Pending Verification | \($pending) |",- "| Verification Pass Rate | \(pct($verified_passed; $agent_total)) |",+ "| Verification Pass Rate | \(pct($verified_passed; $execution_total)) |","| Tool Calls | \($root.totals.tool_calls // 0) |"]end)[],"",+ "## Run Usage",+ "",+ (+ if (($usage.provider // "") == "" and ($usage.total_tokens // 0) == 0 and ($usage.request_count // 0) == 0) then+ ["- none recorded"]+ else+ [+ "| Metric | Value |",+ "|--------|-------|",+ "| Source | \($usage.source // "unknown") |",+ "| Provider | \($usage.provider // "unknown") |",+ "| API Requests | \($usage.request_count // 0) |",+ "| Input Tokens (uncached) | \($usage.input_tokens // 0) |",+ "| Cache Creation Input Tokens | \($usage.cache_creation_input_tokens // 0) |",+ "| Cache Read Input Tokens | \($usage.cache_read_input_tokens // 0) |",+ "| Total Input Tokens | \($usage.total_input_tokens // 0) |",+ "| Output Tokens | \($usage.output_tokens // 0) |",+ "| Total Tokens | \($usage.total_tokens // 0) |"+ ]+ end+ )[],+ "","## Pending Verification","",(@@ -130,3 +158,5 @@ jq -r '' "${REPORT_FILE}" > "${SUMMARY_FILE}"echo "Wrote ${SUMMARY_FILE}"+echo ""+cat "${SUMMARY_FILE}"diff --git a/tests/benchmark/scripts/record-run-usage.sh b/tests/benchmark/scripts/record-run-usage.shnew file mode 100755index 00000000..21b8fab0--- /dev/null+++ b/tests/benchmark/scripts/record-run-usage.sh@@ -0,0 +1,152 @@+#!/usr/bin/env bash+#+# Records exact run-level usage for a benchmark report.+#+# Usage:+# ./record-run-usage.sh --type agent|agent-browser \+# --provider anthropic \+# --input-tokens 123 \+# --output-tokens 45 \+# --cache-creation-input-tokens 67 \+# --cache-read-input-tokens 890 \+# [--request-count 12] \+# [--report-file path]++set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+RESULTS_DIR="${SCRIPT_DIR}/../results"+CURRENT_AGENT_PTR="${RESULTS_DIR}/current_agent_report.txt"+CURRENT_AGENT_BROWSER_PTR="${RESULTS_DIR}/current_agent_browser_report.txt"++REPORT_TYPE=""+REPORT_FILE=""+PROVIDER=""+SOURCE="external"+REQUEST_COUNT=0+INPUT_TOKENS=0+OUTPUT_TOKENS=0+CACHE_CREATION_INPUT_TOKENS=0+CACHE_READ_INPUT_TOKENS=0++resolve_current_report() {+ local ptr="$1"+ if [[ -f "${ptr}" ]]; then+ tr -d '[:space:]' < "${ptr}"+ return 0+ fi+ return 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --type)+ REPORT_TYPE="$2"+ shift 2+ ;;+ --report-file)+ REPORT_FILE="$2"+ shift 2+ ;;+ --provider)+ PROVIDER="$2"+ shift 2+ ;;+ --source)+ SOURCE="$2"+ shift 2+ ;;+ --request-count)+ REQUEST_COUNT="$2"+ shift 2+ ;;+ --input-tokens)+ INPUT_TOKENS="$2"+ shift 2+ ;;+ --output-tokens)+ OUTPUT_TOKENS="$2"+ shift 2+ ;;+ --cache-creation-input-tokens)+ CACHE_CREATION_INPUT_TOKENS="$2"+ shift 2+ ;;+ --cache-read-input-tokens)+ CACHE_READ_INPUT_TOKENS="$2"+ shift 2+ ;;+ *)+ echo "Unknown option: $1"+ exit 1+ ;;+ esac+done++if [[ -z "${PROVIDER}" ]]; then+ echo "ERROR: --provider is required"+ exit 1+fi++if [[ -z "${REPORT_FILE}" ]]; then+ case "${REPORT_TYPE}" in+ agent)+ REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_PTR}" || true)"+ [[ -n "${REPORT_FILE}" ]] || REPORT_FILE=$(ls -t "${RESULTS_DIR}"/agent_benchmark_*.json 2>/dev/null | head -1)+ ;;+ agent-browser|agent_browser)+ REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_BROWSER_PTR}" || true)"+ [[ -n "${REPORT_FILE}" ]] || REPORT_FILE=$(ls -t "${RESULTS_DIR}"/agent_browser_benchmark_*.json 2>/dev/null | head -1)+ ;;+ *)+ echo "ERROR: --type must be 'agent' or 'agent-browser' unless --report-file is provided"+ exit 1+ ;;+ esac+fi++if [[ -z "${REPORT_FILE}" || ! -f "${REPORT_FILE}" ]]; then+ echo "ERROR: no benchmark report found"+ exit 1+fi++TOTAL_INPUT_TOKENS=$((INPUT_TOKENS + CACHE_CREATION_INPUT_TOKENS + CACHE_READ_INPUT_TOKENS))+TOTAL_TOKENS=$((TOTAL_INPUT_TOKENS + OUTPUT_TOKENS))++TMP_FILE=$(mktemp)+jq \+ --arg provider "${PROVIDER}" \+ --arg source "${SOURCE}" \+ --argjson request_count "${REQUEST_COUNT}" \+ --argjson input_tokens "${INPUT_TOKENS}" \+ --argjson output_tokens "${OUTPUT_TOKENS}" \+ --argjson cache_creation_input_tokens "${CACHE_CREATION_INPUT_TOKENS}" \+ --argjson cache_read_input_tokens "${CACHE_READ_INPUT_TOKENS}" \+ --argjson total_input_tokens "${TOTAL_INPUT_TOKENS}" \+ --argjson total_tokens "${TOTAL_TOKENS}" \+ '+ .benchmark.runner = $source |+ .run_usage = {+ source: $source,+ provider: $provider,+ request_count: $request_count,+ input_tokens: $input_tokens,+ output_tokens: $output_tokens,+ cache_creation_input_tokens: $cache_creation_input_tokens,+ cache_read_input_tokens: $cache_read_input_tokens,+ total_input_tokens: $total_input_tokens,+ total_tokens: $total_tokens+ }+ ' "${REPORT_FILE}" > "${TMP_FILE}"++mv "${TMP_FILE}" "${REPORT_FILE}"++echo "Recorded run usage in ${REPORT_FILE}"+echo " Provider: ${PROVIDER}"+echo " Requests: ${REQUEST_COUNT}"+echo " Input (uncached): ${INPUT_TOKENS}"+echo " Cache creation input: ${CACHE_CREATION_INPUT_TOKENS}"+echo " Cache read input: ${CACHE_READ_INPUT_TOKENS}"+echo " Total input: ${TOTAL_INPUT_TOKENS}"+echo " Output: ${OUTPUT_TOKENS}"+echo " Total: ${TOTAL_TOKENS}"diff --git a/tests/benchmark/scripts/record-step.sh b/tests/benchmark/scripts/record-step.shindex 75df4742..f83f2d5e 100755--- a/tests/benchmark/scripts/record-step.sh+++ b/tests/benchmark/scripts/record-step.sh@@ -7,17 +7,17 @@## Options:# --type baseline|agent|agent-browser Report type (default: auto-detect most recent)-# --tokens <in> <out> Deprecated and ignored (kept for compatibility)+# --tokens <in> <out> Deprecated compatibility field; benchmark reporting now prefers exact run-level usage# --bytes <n> HTTP response size in bytes (baseline runs)# --tool-calls <n> Override auto-counted tool calls# --observed <text> Deprecated alias for answer text# --answer <text> Optional explicit answer text## Examples:-# ./record-step.sh 1 1 pass "Navigation completed"+# ./record-step.sh 1 1 answer "Navigation completed" "Observed output"# ./record-step.sh --type agent 2 3 fail "Element not found"# ./record-step.sh --type agent 2 4 answer "Robert Griesemer, 2009" "read infobox"-# ./record-step.sh --type baseline 1 2 pass --bytes 4520 "Snapshot returned"+# ./record-step.sh --type baseline 1 2 answer --bytes 4520 "Snapshot returned" "Observed output"set -euo pipefail@@ -158,9 +158,9 @@ BENCHMARK_TYPE=$(jq -r '.benchmark.type' "${REPORT_FILE}")case "${BENCHMARK_TYPE}" inbaseline)case "${STATUS}" in- pass|fail|skip) ;;+ pass|fail|skip|answer) ;;*)- echo "ERROR: baseline steps must use pass, fail, or skip"+ echo "ERROR: baseline steps must use answer, fail, skip, or legacy pass"exit 1;;esacdiff --git a/tests/benchmark/scripts/run-agent-browser-benchmark.sh b/tests/benchmark/scripts/run-agent-browser-benchmark.shindex 3de99c43..2e0d8335 100755--- a/tests/benchmark/scripts/run-agent-browser-benchmark.sh+++ b/tests/benchmark/scripts/run-agent-browser-benchmark.sh@@ -51,7 +51,23 @@ cat > "${REPORT_FILE}" << EOF"tool_calls": 0,"steps_passed": 0,"steps_failed": 0,- "steps_skipped": 0+ "steps_skipped": 0,+ "steps_answered": 0,+ "steps_verified_passed": 0,+ "steps_verified_failed": 0,+ "steps_verified_skipped": 0,+ "steps_pending_verification": 0+ },+ "run_usage": {+ "source": "none",+ "provider": "",+ "request_count": 0,+ "input_tokens": 0,+ "output_tokens": 0,+ "cache_creation_input_tokens": 0,+ "cache_read_input_tokens": 0,+ "total_input_tokens": 0,+ "total_tokens": 0},"steps": []}@@ -67,5 +83,5 @@ echo " 1. Read ./AGENT_BROWSER_INSTRUCTIONS.md"echo " 2. Load the live CLI skill with ./scripts/ab skills get agent-browser --full"echo " 3. Use ./scripts/ab ... to drive agent-browser inside Docker"echo " 4. Record each completed benchmark step with:"-echo " ./scripts/record-step.sh --type agent-browser <group> <step> <pass|fail|skip|answer> \"what you saw\" \"notes\""+echo " ./scripts/record-step.sh --type agent-browser <group> <step> <answer|fail|skip> \"what you saw\" \"notes\""echo " 5. Summarize the report with ./scripts/finalize-report.sh"diff --git a/tests/benchmark/scripts/run-api-benchmark.ts b/tests/benchmark/scripts/run-api-benchmark.tsnew file mode 100644index 00000000..ded4285b--- /dev/null+++ b/tests/benchmark/scripts/run-api-benchmark.ts@@ -0,0 +1,1185 @@+#!/usr/bin/env -S node --experimental-strip-types++import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";+import { spawn, spawnSync } from "node:child_process";+import { dirname, resolve } from "node:path";+import { fileURLToPath } from "node:url";++type Lane = "agent" | "agent-browser";+type Provider = "anthropic" | "openai";++type ToolCall = {+ id: string;+ command: string;+ timeoutSeconds: number;+};++type ToolExecutionResult = {+ id: string;+ isError: boolean;+ content: string;+};++type Args = {+ lane: Lane;+ provider?: Provider;+ model?: string;+ groups?: number[];+ profile?: string;+ maxTokens: number;+ temperature: number;+ maxTurns: number;+ maxIdleTurns: number;+ timeoutSeconds: number;+ turnDelayMs: number;+ reportFile?: string;+ skipInit: boolean;+ noPromptCaching: boolean;+ finalize: boolean;+};++const SCRIPT_PATH = fileURLToPath(import.meta.url);+const SCRIPT_DIR = dirname(SCRIPT_PATH);+const BENCH_DIR = resolve(SCRIPT_DIR, "..");+const REPO_ROOT = resolve(BENCH_DIR, "..", "..");+const RESULTS_DIR = resolve(BENCH_DIR, "results");+const CURRENT_AGENT_PTR = resolve(RESULTS_DIR, "current_agent_report.txt");+const CURRENT_AGENT_BROWSER_PTR = resolve(RESULTS_DIR, "current_agent_browser_report.txt");+const AGENT_COMMANDS_LOG = resolve(RESULTS_DIR, "agent_commands.ndjson");+const MAX_TOOL_OUTPUT_CHARS = 2400;+const COMPACT_AFTER_ANTHROPIC_MESSAGES = 10;+const KEEP_RECENT_ANTHROPIC_MESSAGES = 6;+const COMPACT_AFTER_OPENAI_ITEMS = 16;+const KEEP_RECENT_OPENAI_ITEMS = 10;+const MAX_SUMMARY_STEPS = 12;+const MAX_SUMMARY_TEXT_CHARS = 220;+const AGENT_TASKS_FILE = resolve(BENCH_DIR, "AGENT_TASKS.md");+const AGENT_BROWSER_TASKS_FILE = resolve(BENCH_DIR, "AGENT_BROWSER_TASKS.md");++function usage(exitCode = 1): never {+ const stream = exitCode === 0 ? console.log : console.error;+ stream(`Usage:+ node --experimental-strip-types ./tests/benchmark/scripts/run-api-benchmark.ts --lane agent [options]+ node --experimental-strip-types ./tests/benchmark/scripts/run-api-benchmark.ts --lane agent-browser [options]++Options:+ --provider anthropic|openai+ --model MODEL+ --groups 0,1,2,3+ --profile common10+ --max-tokens N+ --temperature N+ --max-turns N+ --max-idle-turns N+ --timeout-seconds N+ --turn-delay-ms N+ --report-file PATH+ --skip-init+ --no-prompt-caching+ --finalize+`);+ process.exit(exitCode);+}++function parseArgs(argv: string[]): Args {+ let lane: Lane | undefined;+ let provider: Provider | undefined;+ let model: string | undefined;+ let groups: number[] | undefined;+ let profile: string | undefined;+ let maxTokens = 4096;+ let temperature = 0;+ let maxTurns = 300;+ let maxIdleTurns = 25;+ let timeoutSeconds = 120;+ let turnDelayMs = 1500;+ let reportFile: string | undefined;+ let skipInit = false;+ let noPromptCaching = false;+ let finalize = false;++ for (let i = 0; i < argv.length; i += 1) {+ const arg = argv[i];+ switch (arg) {+ case "--lane":+ lane = argv[++i] as Lane;+ break;+ case "--provider":+ provider = argv[++i] as Provider;+ break;+ case "--model":+ model = argv[++i];+ break;+ case "--groups":+ groups = argv[++i]+ .split(",")+ .map((value) => Number(value.trim()))+ .filter((value) => Number.isInteger(value));+ break;+ case "--profile":+ profile = argv[++i];+ break;+ case "--max-tokens":+ maxTokens = Number(argv[++i]);+ break;+ case "--temperature":+ temperature = Number(argv[++i]);+ break;+ case "--max-turns":+ maxTurns = Number(argv[++i]);+ break;+ case "--max-idle-turns":+ maxIdleTurns = Number(argv[++i]);+ break;+ case "--timeout-seconds":+ timeoutSeconds = Number(argv[++i]);+ break;+ case "--turn-delay-ms":+ turnDelayMs = Number(argv[++i]);+ break;+ case "--report-file":+ reportFile = argv[++i];+ break;+ case "--skip-init":+ skipInit = true;+ break;+ case "--no-prompt-caching":+ noPromptCaching = true;+ break;+ case "--finalize":+ finalize = true;+ break;+ case "-h":+ case "--help":+ case "help":+ usage(0);+ break;+ default:+ throw new Error(`Unknown argument: ${arg}`);+ }+ }++ if (lane !== "agent" && lane !== "agent-browser") {+ throw new Error("--lane must be 'agent' or 'agent-browser'");+ }++ return {+ lane,+ provider,+ model,+ groups,+ profile,+ maxTokens,+ temperature,+ maxTurns,+ maxIdleTurns,+ timeoutSeconds,+ turnDelayMs,+ reportFile,+ skipInit,+ noPromptCaching,+ finalize,+ };+}++function resolveGroups(args: Args): number[] | undefined {+ if (args.groups && args.groups.length > 0) {+ return [...new Set(args.groups)].sort((a, b) => a - b);+ }+ if (!args.profile) {+ return undefined;+ }+ switch (args.profile) {+ case "common10":+ return [0, 1, 2, 3];+ default:+ throw new Error(`Unknown profile: ${args.profile}`);+ }+}++function formatGroups(groups?: number[]): string {+ if (!groups || groups.length === 0) {+ return "all";+ }+ return groups.join(",");+}++function extractGroupSections(markdownPath: string, groups: number[]): string {+ const lines = readText(markdownPath).split("\n");+ const wanted = new Set(groups);+ const collected: string[] = [];+ let capture = false;++ for (const line of lines) {+ const match = line.match(/^## Group (\d+):/);+ if (match) {+ const group = Number(match[1]);+ capture = wanted.has(group);+ }+ if (capture) {+ collected.push(line);+ }+ }++ return collected.join("\n").trim();+}++function extractSingleGroupSection(markdownPath: string, group: number): string {+ const lines = readText(markdownPath).split("\n");+ const collected: string[] = [];+ let capture = false;++ for (const line of lines) {+ const match = line.match(/^## Group (\d+):/);+ if (match) {+ const currentGroup = Number(match[1]);+ if (capture && currentGroup !== group) {+ break;+ }+ capture = currentGroup === group;+ }+ if (capture) {+ collected.push(line);+ }+ }++ return collected.join("\n").trim();+}++function readText(path: string): string {+ return readFileSync(path, "utf8").trim();+}++function readJsonFile(path: string): any {+ return JSON.parse(readFileSync(path, "utf8"));+}++function resolveReportPath(lane: Lane): string {+ const ptr = lane === "agent" ? CURRENT_AGENT_PTR : CURRENT_AGENT_BROWSER_PTR;+ return readText(ptr);+}++function runChecked(cmd: string[], env?: NodeJS.ProcessEnv): void {+ const result = spawnSync(cmd[0], cmd.slice(1), {+ cwd: REPO_ROOT,+ env: env ?? process.env,+ stdio: "inherit",+ });+ if (result.status !== 0) {+ throw new Error(`command failed: ${cmd.join(" ")}`);+ }+}++function resolveProvider(args: Args): Provider {+ if (args.provider) {+ return args.provider;+ }+ const hasOpenAI = (process.env.OPENAI_API_KEY ?? "").trim().length > 0;+ const hasAnthropic = (process.env.ANTHROPIC_API_KEY ?? "").trim().length > 0;+ const configured = [hasOpenAI, hasAnthropic].filter(Boolean).length;+ if (configured > 1) {+ throw new Error("Multiple providers appear configured. Pass --provider openai|anthropic explicitly.");+ }+ if (hasOpenAI) {+ return "openai";+ }+ if (hasAnthropic) {+ return "anthropic";+ }+ throw new Error("Set OPENAI_API_KEY or ANTHROPIC_API_KEY, or pass --provider");+}++function resolveModel(provider: Provider, explicit?: string): string {+ if (explicit) {+ return explicit;+ }+ if (provider === "openai") {+ return process.env.OPENAI_MODEL?.trim() || "gpt-5";+ }+ return process.env.ANTHROPIC_MODEL?.trim() || "claude-haiku-4-5-20251001";+}++async function listAnthropicModels(apiKey: string): Promise<string[]> {+ const response = await fetch("https://api.anthropic.com/v1/models", {+ method: "GET",+ headers: {+ "x-api-key": apiKey,+ "anthropic-version": "2023-06-01",+ },+ });++ const payload = await response.json();+ if (!response.ok) {+ throw new Error(`Anthropic models API error ${response.status}: ${JSON.stringify(payload)}`);+ }++ return Array.isArray(payload.data)+ ? payload.data+ .map((item: any) => String(item.id ?? "").trim())+ .filter((id: string) => id.length > 0)+ : [];+}++async function resolveAnthropicModel(explicit?: string): Promise<string> {+ if (explicit) {+ return explicit;+ }++ const envModel = process.env.ANTHROPIC_MODEL?.trim();+ if (envModel) {+ return envModel;+ }++ const apiKey = (process.env.ANTHROPIC_API_KEY ?? "").trim();+ if (!apiKey) {+ return "claude-haiku-4-5-20251001";+ }++ try {+ const available = await listAnthropicModels(apiKey);+ const preferred = [+ "claude-haiku-4-5-20251001",+ "claude-3-5-haiku-20241022",+ "claude-3-5-haiku-latest",+ "claude-3-haiku-20240307",+ "claude-sonnet-4-6",+ "claude-sonnet-4-20250514",+ "claude-sonnet-4-5-20250929",+ "claude-3-7-sonnet-20250219",+ "claude-3-7-sonnet-latest",+ "claude-3-5-sonnet-20241022",+ "claude-3-5-sonnet-latest",+ "claude-3-5-sonnet-20240620",+ "claude-opus-4-7",+ "claude-opus-4-6",+ "claude-opus-4-5-20251101",+ "claude-opus-4-1-20250805",+ ];++ for (const candidate of preferred) {+ if (available.includes(candidate)) {+ return candidate;+ }+ }++ if (available.length > 0) {+ return available[0];+ }+ } catch {+ // Fall back to a documented default if model discovery fails.+ }++ return "claude-haiku-4-5-20251001";+}++function runnerSource(provider: Provider): string {+ if (provider === "openai") {+ return "openai-responses";+ }+ return "anthropic-messages";+}++function initializeLane(args: Args, provider: Provider, model: string): string {+ const env = { ...process.env, BENCHMARK_MODEL: model, BENCHMARK_RUNNER: runnerSource(provider) };+ if (args.lane === "agent") {+ runChecked([resolve(BENCH_DIR, "scripts", "run-optimization.sh")], env);+ } else {+ runChecked([resolve(BENCH_DIR, "scripts", "run-agent-browser-benchmark.sh")], env);+ }+ return resolveReportPath(args.lane);+}++class PersistentShell {+ private proc = spawn("/bin/bash", [], {+ cwd: BENCH_DIR,+ stdio: ["pipe", "pipe", "pipe"],+ });++ constructor() {+ this.proc.stderr?.pipe(process.stderr);+ }++ async run(command: string, timeoutSeconds: number): Promise<{ output: string; exitCode: number }> {+ const marker = `__BENCH_DONE__${Math.random().toString(16).slice(2)}`;+ const wrapped = `${command}\nprintf '\\n${marker}:%s\\n' "$?"\n`;+ const stdout = this.proc.stdout;+ const stdin = this.proc.stdin;+ if (!stdout || !stdin) {+ throw new Error("persistent shell unavailable");+ }++ stdin.write(wrapped);+ const deadline = Date.now() + timeoutSeconds * 1000;+ let buffer = "";++ while (Date.now() < deadline) {+ const chunk = await new Promise<string>((resolveChunk, rejectChunk) => {+ const onData = (data: Buffer) => {+ cleanup();+ resolveChunk(data.toString("utf8"));+ };+ const onExit = () => {+ cleanup();+ rejectChunk(new Error("persistent shell exited unexpectedly"));+ };+ const cleanup = () => {+ clearTimeout(timer);+ stdout.off("data", onData);+ this.proc.off("exit", onExit);+ };+ const timer = setTimeout(() => {+ cleanup();+ resolveChunk("");+ }, 100);+ stdout.once("data", onData);+ this.proc.once("exit", onExit);+ });++ if (chunk) {+ buffer += chunk;+ const match = buffer.match(new RegExp(`\\n${marker}:(\\d+)\\r?\\n`));+ if (match) {+ const output = buffer.slice(0, match.index).replace(/^\n/, "");+ return { output, exitCode: Number(match[1]) };+ }+ }+ }++ this.close(true);+ throw new Error(`shell command timed out after ${timeoutSeconds}s: ${command}`);+ }++ close(force = false): void {+ if (this.proc.exitCode !== null) {+ return;+ }+ if (force) {+ this.proc.kill("SIGKILL");+ return;+ }+ this.proc.stdin?.end("exit\n");+ }+}++abstract class ApiRunner {+ requestCount = 0;+ inputTokens = 0;+ outputTokens = 0;+ cacheCreationInputTokens = 0;+ cacheReadInputTokens = 0;+ provider: Provider;+ source: string;+ apiKey: string;+ model: string;+ maxTokens: number;+ temperature: number;+ promptCaching: boolean;++ constructor(provider: Provider, source: string, apiKey: string, model: string, maxTokens: number, temperature: number, promptCaching: boolean) {+ this.provider = provider;+ this.source = source;+ this.apiKey = apiKey;+ this.model = model;+ this.maxTokens = maxTokens;+ this.temperature = temperature;+ this.promptCaching = promptCaching;+ }++ abstract toolDefinitions(): unknown[];+ abstract initialConversation(userPrompt: string): unknown[];+ abstract send(systemPrompt: string, conversation: unknown[]): Promise<any>;+ abstract extractToolCalls(response: any, defaultTimeout: number): ToolCall[];+ abstract appendToolResults(conversation: unknown[], response: any, results: ToolExecutionResult[]): void;+ abstract extractFinalText(response: any): string;+}++class AnthropicRunner extends ApiRunner {+ constructor(apiKey: string, model: string, maxTokens: number, temperature: number, promptCaching: boolean) {+ super("anthropic", "anthropic-messages", apiKey, model, maxTokens, temperature, promptCaching);+ }++ toolDefinitions(): unknown[] {+ return [+ {+ name: "run_command",+ description:+ "Run a shell command in a persistent bash session rooted at tests/benchmark. Environment changes such as cd/export persist across calls.",+ input_schema: {+ type: "object",+ properties: {+ command: { type: "string" },+ timeout_seconds: { type: "integer", minimum: 1, maximum: 600 },+ },+ required: ["command"],+ },+ },+ ];+ }++ initialConversation(userPrompt: string): unknown[] {+ return [{ role: "user", content: userPrompt }];+ }++ async send(systemPrompt: string, conversation: unknown[]): Promise<any> {+ const body: Record<string, unknown> = {+ model: this.model,+ max_tokens: this.maxTokens,+ temperature: this.temperature,+ system: systemPrompt,+ tools: this.toolDefinitions(),+ messages: conversation,+ };+ if (this.promptCaching) {+ body.cache_control = { type: "ephemeral" };+ }++ const response = await fetchWithRetry("https://api.anthropic.com/v1/messages", {+ method: "POST",+ headers: {+ "content-type": "application/json",+ "x-api-key": this.apiKey,+ "anthropic-version": "2023-06-01",+ },+ body: JSON.stringify(body),+ });++ const payload = await response.json();+ if (!response.ok) {+ throw new Error(`Anthropic API error ${response.status}: ${JSON.stringify(payload)}`);+ }++ const usage = payload.usage ?? {};+ this.requestCount += 1;+ this.inputTokens += Number(usage.input_tokens ?? 0);+ this.outputTokens += Number(usage.output_tokens ?? 0);+ this.cacheCreationInputTokens += Number(usage.cache_creation_input_tokens ?? 0);+ this.cacheReadInputTokens += Number(usage.cache_read_input_tokens ?? 0);+ return payload;+ }++ extractToolCalls(response: any, defaultTimeout: number): ToolCall[] {+ return (response.content ?? [])+ .filter((item: any) => item.type === "tool_use")+ .map((item: any) => ({+ id: String(item.id),+ command: String(item.input?.command ?? "").trim(),+ timeoutSeconds: Number(item.input?.timeout_seconds ?? defaultTimeout),+ }))+ .filter((call: ToolCall) => call.command.length > 0);+ }++ appendToolResults(conversation: unknown[], response: any, results: ToolExecutionResult[]): void {+ conversation.push({ role: "assistant", content: response.content });+ if (results.length > 0) {+ conversation.push({+ role: "user",+ content: results.map((result) => ({+ type: "tool_result",+ tool_use_id: result.id,+ is_error: result.isError,+ content: result.content,+ })),+ });+ }+ }++ extractFinalText(response: any): string {+ return (response.content ?? [])+ .filter((item: any) => item.type === "text" && item.text)+ .map((item: any) => String(item.text))+ .join("\n")+ .trim();+ }+}++class OpenAIRunner extends ApiRunner {+ private baseUrl: string;++ constructor(provider: "openai", source: string, baseUrl: string, apiKey: string, model: string, maxTokens: number, temperature: number, promptCaching: boolean) {+ super(provider, source, apiKey, model, maxTokens, temperature, promptCaching);+ this.baseUrl = baseUrl;+ }++ toolDefinitions(): unknown[] {+ return [+ {+ type: "function",+ name: "run_command",+ description:+ "Run a shell command in a persistent bash session rooted at tests/benchmark. Environment changes such as cd/export persist across calls.",+ parameters: {+ type: "object",+ properties: {+ command: { type: "string" },+ timeout_seconds: { type: "integer", minimum: 1, maximum: 600 },+ },+ required: ["command"],+ },+ },+ ];+ }++ initialConversation(userPrompt: string): unknown[] {+ return [{ role: "user", content: [{ type: "input_text", text: userPrompt }] }];+ }++ async send(systemPrompt: string, conversation: unknown[]): Promise<any> {+ const body: Record<string, unknown> = {+ model: this.model,+ instructions: systemPrompt,+ tools: this.toolDefinitions(),+ input: conversation,+ max_output_tokens: this.maxTokens,+ };+ if (this.promptCaching) {+ body.prompt_cache_retention = "24h";+ }++ const response = await fetchWithRetry(this.baseUrl, {+ method: "POST",+ headers: {+ "content-type": "application/json",+ authorization: `Bearer ${this.apiKey}`,+ },+ body: JSON.stringify(body),+ });++ const payload = await response.json();+ if (!response.ok) {+ throw new Error(`OpenAI API error ${response.status}: ${JSON.stringify(payload)}`);+ }++ const usage = payload.usage ?? {};+ const totalInput = Number(usage.input_tokens ?? 0);+ const cached = Number(usage.input_tokens_details?.cached_tokens ?? 0);+ this.requestCount += 1;+ this.inputTokens += Math.max(0, totalInput - cached);+ this.outputTokens += Number(usage.output_tokens ?? 0);+ this.cacheReadInputTokens += cached;+ return payload;+ }++ extractToolCalls(response: any, defaultTimeout: number): ToolCall[] {+ return (response.output ?? [])+ .filter((item: any) => item.type === "function_call")+ .map((item: any) => {+ const parsed = item.arguments ? JSON.parse(item.arguments) : {};+ return {+ id: String(item.call_id ?? item.id),+ command: String(parsed.command ?? "").trim(),+ timeoutSeconds: Number(parsed.timeout_seconds ?? defaultTimeout),+ };+ })+ .filter((call: ToolCall) => call.command.length > 0);+ }++ appendToolResults(conversation: unknown[], response: any, results: ToolExecutionResult[]): void {+ conversation.push(...(response.output ?? []));+ for (const result of results) {+ conversation.push({+ type: "function_call_output",+ call_id: result.id,+ output: result.content,+ });+ }+ }++ extractFinalText(response: any): string {+ if (typeof response.output_text === "string" && response.output_text.trim()) {+ return response.output_text.trim();+ }+ return (response.output ?? [])+ .flatMap((item: any) => (item.type === "message" ? item.content ?? [] : []))+ .filter((part: any) => (part.type === "output_text" || part.type === "text") && part.text)+ .map((part: any) => String(part.text))+ .join("\n")+ .trim();+ }+}++function trimToolOutput(text: string): string {+ if (text.length <= MAX_TOOL_OUTPUT_CHARS) {+ return text;+ }+ return `${text.slice(0, MAX_TOOL_OUTPUT_CHARS)}\n\n[output truncated: ${text.length - MAX_TOOL_OUTPUT_CHARS} more chars]`;+}++function parseRetryAfterMs(value: string | null): number {+ if (!value) {+ return 0;+ }+ const seconds = Number(value);+ if (Number.isFinite(seconds) && seconds > 0) {+ return Math.ceil(seconds * 1000);+ }+ const dateMs = Date.parse(value);+ if (!Number.isNaN(dateMs)) {+ return Math.max(0, dateMs - Date.now());+ }+ return 0;+}++function sleep(ms: number): Promise<void> {+ return new Promise((resolvePromise) => setTimeout(resolvePromise, ms));+}++async function fetchWithRetry(url: string, init: RequestInit, maxRetries = 3): Promise<Response> {+ let attempt = 0;+ while (true) {+ const response = await fetch(url, init);+ if (response.status !== 429 || attempt >= maxRetries) {+ return response;+ }++ const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"));+ const backoffMs = retryAfterMs > 0 ? retryAfterMs : Math.min(15000, 2000 * (attempt + 1));+ await sleep(backoffMs);+ attempt += 1;+ }+}++async function executeToolCalls(shell: PersistentShell, calls: ToolCall[]): Promise<ToolExecutionResult[]> {+ const results: ToolExecutionResult[] = [];+ for (const call of calls) {+ try {+ const { output, exitCode } = await shell.run(call.command, call.timeoutSeconds);+ const result = {+ id: call.id,+ isError: exitCode !== 0,+ content: `$ ${call.command}\n[exit_code=${exitCode}]\n${trimToolOutput(output)}`,+ };+ results.push(result);+ appendCommandLog(call.command, exitCode, trimToolOutput(output));+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ const result = {+ id: call.id,+ isError: true,+ content: `$ ${call.command}\n[runner_error]\n${message}`,+ };+ results.push(result);+ appendCommandLog(call.command, -1, message);+ }+ }+ return results;+}++function appendCommandLog(command: string, exitCode: number, output: string): void {+ const entry = {+ timestamp: new Date().toISOString(),+ command,+ exit_code: exitCode,+ output,+ };+ appendFileSync(AGENT_COMMANDS_LOG, `${JSON.stringify(entry)}\n`, "utf8");+}++function recordUsage(reportFile: string, runner: ApiRunner): void {+ runChecked([+ resolve(BENCH_DIR, "scripts", "record-run-usage.sh"),+ "--report-file",+ reportFile,+ "--provider",+ runner.provider,+ "--source",+ runner.source,+ "--request-count",+ String(runner.requestCount),+ "--input-tokens",+ String(runner.inputTokens),+ "--output-tokens",+ String(runner.outputTokens),+ "--cache-creation-input-tokens",+ String(runner.cacheCreationInputTokens),+ "--cache-read-input-tokens",+ String(runner.cacheReadInputTokens),+ ]);+}++function shorten(text: string, maxChars = MAX_SUMMARY_TEXT_CHARS): string {+ const normalized = text.replace(/\s+/g, " ").trim();+ if (normalized.length <= maxChars) {+ return normalized;+ }+ return `${normalized.slice(0, maxChars - 3)}...`;+}++function buildProgressSummary(reportFile: string): string {+ if (!existsSync(reportFile)) {+ return "No benchmark report file exists yet.";+ }++ const report = JSON.parse(readText(reportFile));+ const totals = report.totals ?? {};+ const steps = Array.isArray(report.steps) ? report.steps : [];+ const recentSteps = steps.slice(-MAX_SUMMARY_STEPS).map((step: any) => {+ const verification = step.verification?.status ? ` / verify=${step.verification.status}` : "";+ const answer = step.answer || step.notes || "";+ return `- ${step.id}: status=${step.status}${verification}; ${shorten(String(answer))}`;+ });++ return [+ "Benchmark progress summary from the external report.",+ `- answered=${totals.steps_answered ?? 0}`,+ `- execution_failed=${totals.steps_failed ?? 0}`,+ `- execution_skipped=${totals.steps_skipped ?? 0}`,+ `- verified_passed=${totals.steps_verified_passed ?? 0}`,+ `- verified_failed=${totals.steps_verified_failed ?? 0}`,+ `- verified_skipped=${totals.steps_verified_skipped ?? 0}`,+ `- pending_verification=${totals.steps_pending_verification ?? 0}`,+ recentSteps.length > 0 ? "Recent recorded steps:" : "No steps recorded yet.",+ ...recentSteps,+ "Use the report file and benchmark wrappers as the source of truth.",+ "Do not re-read large documentation files unless you are blocked on a specific syntax detail.",+ ].join("\n");+}++function readProgress(reportFile: string): { answered: number; failed: number; skipped: number; verifiedPassed: number } {+ if (!existsSync(reportFile)) {+ return { answered: 0, failed: 0, skipped: 0, verifiedPassed: 0 };+ }+ const report = readJsonFile(reportFile);+ const totals = report.totals ?? {};+ return {+ answered: Number(totals.steps_answered ?? 0),+ failed: Number(totals.steps_failed ?? 0),+ skipped: Number(totals.steps_skipped ?? 0),+ verifiedPassed: Number(totals.steps_verified_passed ?? 0),+ };+}++function compactAnthropicConversation(conversation: unknown[], reportFile: string): void {+ if (conversation.length <= COMPACT_AFTER_ANTHROPIC_MESSAGES) {+ return;+ }+ const head = conversation[0];+ const recent = conversation.slice(-KEEP_RECENT_ANTHROPIC_MESSAGES);+ conversation.splice(+ 0,+ conversation.length,+ head,+ {+ role: "user",+ content: buildProgressSummary(reportFile),+ },+ ...recent,+ );+}++function compactOpenAIConversation(conversation: unknown[], reportFile: string): void {+ if (conversation.length <= COMPACT_AFTER_OPENAI_ITEMS) {+ return;+ }+ const head = conversation[0];+ const recent = conversation.slice(-KEEP_RECENT_OPENAI_ITEMS);+ conversation.splice(+ 0,+ conversation.length,+ head,+ {+ role: "user",+ content: [{ type: "input_text", text: buildProgressSummary(reportFile) }],+ },+ ...recent,+ );+}++function compactConversation(provider: Provider, conversation: unknown[], reportFile: string): void {+ if (provider === "anthropic") {+ compactAnthropicConversation(conversation, reportFile);+ return;+ }+ compactOpenAIConversation(conversation, reportFile);+}++function laneSubsetInstructions(groups?: number[]): string {+ if (!groups || groups.length === 0) {+ return "Execute the full benchmark task set.";+ }++ return [+ `Execute only these benchmark groups: ${groups.join(", ")}.`,+ "Do not attempt groups outside this subset.",+ "Treat all other groups as out of scope for this run rather than as failures.",+ "For the selected groups, execute every step in the group unless blocked.",+ ].join("\n");+}++function laneTaskSourceInstructions(lane: Lane, groups?: number[]): string {+ const fullTaskFile = lane === "agent" ? AGENT_TASKS_FILE : AGENT_BROWSER_TASKS_FILE;+ if (!groups || groups.length === 0) {+ return [+ `Read the full task file at ${fullTaskFile}.`,+ "Execute the full benchmark task set.",+ ].join("\n");+ }++ if (lane === "agent") {+ const subset = extractGroupSections(AGENT_TASKS_FILE, groups);+ return [+ `Use only this selected task subset from ${AGENT_TASKS_FILE}:`,+ "",+ subset,+ ].join("\n");+ }++ const sections: string[] = [];+ for (const group of groups) {+ if (group === 0) {+ sections.push([+ "## Group 0: Setup & Diagnosis (agent-browser lane)",+ "",+ "### 0.1 Open fixtures home",+ "Run `./scripts/ab open http://fixtures/`.",+ "",+ "**Verify**: Open succeeds and the home page loads.",+ "",+ "### 0.2 Snapshot interactive refs",+ "Run `./scripts/ab snapshot -i -c` on the home page.",+ "",+ "**Verify**: Interactive refs are returned without error.",+ "",+ "### 0.3 Session persists across commands",+ "Use the same wrapper session across multiple commands and confirm browser state persists.",+ "",+ "**Verify**: A follow-up action can use the existing page/session state.",+ ].join("\n"));+ continue;+ }++ const section = extractSingleGroupSection(AGENT_TASKS_FILE, group);+ if (section) {+ sections.push(section);+ }+ }++ return [+ `Use only this selected task subset for agent-browser: Group 0 from ${AGENT_BROWSER_TASKS_FILE}, Groups 1+ from ${AGENT_TASKS_FILE}.`,+ "",+ sections.join("\n\n"),+ ].join("\n");+}++type LanePromptConfig = {+ label: string;+ skillInstruction: string;+ wrapper: string;+ recordType: string;+ workflowSummary: string[];+ adapterNotes: string[];+ bootstrapCommands: string[];+};++function lanePromptConfig(lane: Lane): LanePromptConfig {+ if (lane === "agent") {+ return {+ label: "PinchTab agent",+ skillInstruction: `Read ${resolve(REPO_ROOT, "skills", "pinchtab", "SKILL.md")} exactly once before acting.`,+ wrapper: "./scripts/pt",+ recordType: "agent",+ workflowSummary: [+ "use only ./scripts/pt for browser actions",+ "keep one shared tab via PINCHTAB_TAB",+ "use snap -i -c for actionable refs",+ "use text or text --full for reading content",+ "refresh refs after any navigation or DOM change",+ ],+ adapterNotes: [+ 'a navigation-expected click can return "Error 409: unexpected page navigation ..."; treat that as likely success and verify with a fresh snapshot/text read',+ "do not assume every command returns JSON; only parse JSON when the command actually returned JSON",+ 'the download endpoint returns JSON with base64 content in "data", not a local file path',+ 'this environment uses BSD/macOS userland tools; avoid GNU-only flags such as "head -n -1"',+ ],+ bootstrapCommands: [+ "./scripts/pt health",+ "./scripts/pt tab",+ "export PINCHTAB_TAB=$(./scripts/pt nav http://fixtures/)",+ "./scripts/pt snap -i -c",+ ],+ };+ }++ return {+ label: "agent-browser",+ skillInstruction: "Load the official agent-browser skill exactly once with `./scripts/ab skills get agent-browser --full` before acting.",+ wrapper: "./scripts/ab",+ recordType: "agent-browser",+ workflowSummary: [+ "use only ./scripts/ab for browser actions",+ "keep the shared benchmark session across commands",+ "use snapshot -i -c for actionable refs",+ "use fresh refs after any navigation or DOM change",+ "reuse the current browser state instead of reopening pages unless needed",+ ],+ adapterNotes: [+ "after navigation-triggering clicks, verify the resulting page with a fresh snapshot/text read instead of trusting the click response alone",+ "do not assume every command returns JSON; parse only when the command actually returned JSON",+ "for downloads, inspect returned content instead of assuming a file already exists locally",+ 'this environment uses BSD/macOS userland tools; avoid GNU-only flags such as "head -n -1"',+ ],+ bootstrapCommands: [+ "./scripts/ab skills get agent-browser --full",+ "./scripts/ab open http://fixtures/",+ "./scripts/ab snapshot -i -c",+ ],+ };+}++function laneUserPrompt(lane: Lane, reportFile: string, groups?: number[]): string {+ const config = lanePromptConfig(lane);+ const firstGroup = groups && groups.length > 0 ? groups[0] : 0;+ const bootstrap = config.bootstrapCommands.map((command, index) => `${index + 1}. ${command}`).join("\n");++ return `+Work in this repo: ${REPO_ROOT}++Benchmark lane: ${config.label} execution.++Requirements:+- Follow a linear execution flow: skill once, selected groups, execute, record, verify, continue.+- Do not read README, browse directories, or inspect unrelated files unless a command path is missing.+- ${config.skillInstruction}+- Tool wrapper:+ - ${config.wrapper}+- Workflow summary:+ - ${config.workflowSummary.join("\n - ")}+- Adapter notes:+ - ${config.adapterNotes.join("\n - ")}+- Task scope:+ ${laneTaskSourceInstructions(lane, groups).replace(/\n/g, "\n ")}+- For each completed step:+ 1. record the observed answer with:+ - ./scripts/record-step.sh --report-file ${reportFile} --type ${config.recordType} <group> <step> answer "<what you saw>" "notes"+ 2. immediately verify it with:+ - ./scripts/verify-step.sh --report-file ${reportFile} --type ${config.recordType} <group> <step> <pass|fail|skip> "verification notes"+- If a step cannot be completed, record fail or skip in the same report.+- Do not leave answered steps pending verification.+- Keep commands concise. Prefer rg/sed/cat only when you must inspect a specific file.+- After the skill step, begin actual benchmark execution immediately.+- Start with this bootstrap command sequence before attempting the selected steps:+${bootstrap}+- After the bootstrap, immediately execute Group ${firstGroup} step 1.+- Subset selection:+ - ${laneSubsetInstructions(groups).replace(/\n/g, "\n - ")}+- Finish when all selected steps are executed or when you are blocked.++Your final answer should briefly summarize completion status and the main blockers, if any.+`.trim();+}++function systemPrompt(): string {+ return `+You are a precise benchmark execution agent. Use tools to inspect the repo and run the benchmark lane exactly as instructed.++Rules:+- Never fabricate command output or task results.+- Use the shell tool for all file reads and command execution.+- Do not use destructive commands such as rm -rf, git reset, or checkout.+- After recording an answer, verify it immediately against the task oracle.+- Prefer factual command output over long reasoning.+`.trim();+}++function createRunner(provider: Provider, model: string, args: Args): ApiRunner {+ if (provider === "openai") {+ const apiKey = (process.env.OPENAI_API_KEY ?? "").trim();+ if (!apiKey) {+ throw new Error("OPENAI_API_KEY is required for provider=openai");+ }+ return new OpenAIRunner(+ "openai",+ "openai-responses",+ "https://api.openai.com/v1/responses",+ apiKey,+ model,+ args.maxTokens,+ args.temperature,+ !args.noPromptCaching,+ );+ }++ const apiKey = (process.env.ANTHROPIC_API_KEY ?? "").trim();+ if (!apiKey) {+ throw new Error("ANTHROPIC_API_KEY is required for provider=anthropic");+ }+ return new AnthropicRunner(apiKey, model, args.maxTokens, args.temperature, !args.noPromptCaching);+}++async function main(): Promise<number> {+ try {+ const args = parseArgs(process.argv.slice(2));+ const provider = resolveProvider(args);+ const model = provider === "anthropic" ? await resolveAnthropicModel(args.model) : resolveModel(provider, args.model);+ const groups = resolveGroups(args);+ const reportFile = args.reportFile || (args.skipInit ? resolveReportPath(args.lane) : initializeLane(args, provider, model));+ writeFileSync(AGENT_COMMANDS_LOG, "", "utf8");++ console.log(`[benchmark-runner] provider=${provider} model=${model} lane=${args.lane} groups=${formatGroups(groups)} report=${reportFile}`);++ const runner = createRunner(provider, model, args);+ const shell = new PersistentShell();+ const conversation = runner.initialConversation(laneUserPrompt(args.lane, reportFile, groups));++ let finalText = "";+ let exitCode = 0;+ let idleTurns = 0;+ let lastAnswered = readProgress(reportFile).answered;++ try {+ for (let turn = 1; turn <= args.maxTurns; turn += 1) {+ if (turn > 1 && args.turnDelayMs > 0) {+ await sleep(args.turnDelayMs);+ }+ const response = await runner.send(systemPrompt(), conversation);+ const toolCalls = runner.extractToolCalls(response, args.timeoutSeconds);+ if (toolCalls.length > 0) {+ const results = await executeToolCalls(shell, toolCalls);+ runner.appendToolResults(conversation, response, results);+ compactConversation(runner.provider, conversation, reportFile);+ const progress = readProgress(reportFile);+ if (progress.answered > lastAnswered) {+ idleTurns = 0;+ lastAnswered = progress.answered;+ } else {+ idleTurns += 1;+ }+ if (idleTurns >= args.maxIdleTurns) {+ finalText = `Stopped after ${idleTurns} consecutive turns without recording a benchmark step. Check ${AGENT_COMMANDS_LOG} for the command trace.`;+ exitCode = 3;+ break;+ }+ continue;+ }++ finalText = runner.extractFinalText(response);+ break;+ }++ if (!finalText) {+ finalText = `Stopped after reaching max turns (${args.maxTurns}).`;+ exitCode = 2;+ }+ } finally {+ shell.close();+ if (existsSync(reportFile)) {+ recordUsage(reportFile, runner);+ }+ if (args.finalize && existsSync(reportFile)) {+ runChecked([resolve(BENCH_DIR, "scripts", "finalize-report.sh"), reportFile]);+ }+ }++ if (finalText) {+ console.log(finalText);+ }+ console.log(+ `\n[run-usage] provider=${runner.provider} requests=${runner.requestCount} input=${runner.inputTokens} cache_create=${runner.cacheCreationInputTokens} cache_read=${runner.cacheReadInputTokens} output=${runner.outputTokens} total=${runner.inputTokens + runner.cacheCreationInputTokens + runner.cacheReadInputTokens + runner.outputTokens}`,+ );+ return exitCode;+ } catch (error) {+ console.error(error instanceof Error ? error.message : String(error));+ return 1;+ }+}++process.exitCode = await main();diff --git a/tests/benchmark/scripts/run-optimization.sh b/tests/benchmark/scripts/run-optimization.shindex 41d5ef99..43016e2e 100755--- a/tests/benchmark/scripts/run-optimization.sh+++ b/tests/benchmark/scripts/run-optimization.sh@@ -7,6 +7,7 @@set -euo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+BENCH_DIR="${SCRIPT_DIR}/.."RESULTS_DIR="${SCRIPT_DIR}/../results"mkdir -p "${RESULTS_DIR}"LOG_FILE="${RESULTS_DIR}/optimization_log.md"@@ -19,11 +20,8 @@ RUN_NUMBER=$((RUN_NUMBER + 1))echo "=== PinchTab Optimization Run #${RUN_NUMBER} ==="echo "Timestamp: ${TIMESTAMP}"-# Run on current branch-cd ~/dev/pinchtab-# Ensure Docker is running-cd "${SCRIPT_DIR}"+cd "${BENCH_DIR}"if ! docker compose ps 2>/dev/null | grep -q "running"; thenecho "Starting Docker..."docker compose up -d --build@@ -48,16 +46,34 @@ cat > "${BASELINE_REPORT}" << EOF"type": "baseline","run_number": ${RUN_NUMBER},"timestamp": "${TIMESTAMP}",- "model": "claude-haiku-4-5"+ "model": "${BENCHMARK_MODEL:-baseline}",+ "runner": "${BENCHMARK_RUNNER:-manual}"},"totals": {"input_tokens": 0,"output_tokens": 0,"total_tokens": 0,"estimated_cost_usd": 0,+ "tool_calls": 0,"steps_passed": 0,"steps_failed": 0,- "steps_skipped": 0+ "steps_skipped": 0,+ "steps_answered": 0,+ "steps_verified_passed": 0,+ "steps_verified_failed": 0,+ "steps_verified_skipped": 0,+ "steps_pending_verification": 0+ },+ "run_usage": {+ "source": "none",+ "provider": "",+ "request_count": 0,+ "input_tokens": 0,+ "output_tokens": 0,+ "cache_creation_input_tokens": 0,+ "cache_read_input_tokens": 0,+ "total_input_tokens": 0,+ "total_tokens": 0},"steps": []}@@ -69,16 +85,34 @@ cat > "${AGENT_REPORT}" << EOF"type": "agent","run_number": ${RUN_NUMBER},"timestamp": "${TIMESTAMP}",- "model": "claude-haiku-4-5"+ "model": "${BENCHMARK_MODEL:-unknown}",+ "runner": "${BENCHMARK_RUNNER:-manual}"},"totals": {"input_tokens": 0,"output_tokens": 0,"total_tokens": 0,"estimated_cost_usd": 0,+ "tool_calls": 0,"steps_passed": 0,"steps_failed": 0,- "steps_skipped": 0+ "steps_skipped": 0,+ "steps_answered": 0,+ "steps_verified_passed": 0,+ "steps_verified_failed": 0,+ "steps_verified_skipped": 0,+ "steps_pending_verification": 0+ },+ "run_usage": {+ "source": "none",+ "provider": "",+ "request_count": 0,+ "input_tokens": 0,+ "output_tokens": 0,+ "cache_creation_input_tokens": 0,+ "cache_read_input_tokens": 0,+ "total_input_tokens": 0,+ "total_tokens": 0},"steps": []}@@ -87,8 +121,8 @@ EOFprintf '%s\n' "${BASELINE_REPORT}" > "${CURRENT_BASELINE_PTR}"printf '%s\n' "${AGENT_REPORT}" > "${CURRENT_AGENT_PTR}"-# Clear previous agent commands log-rm -f "${RESULTS_DIR}/agent_commands.log"+# Clear previous agent command trace+rm -f "${RESULTS_DIR}/agent_commands.ndjson"echo "Reports initialized:"echo " Baseline: ${BASELINE_REPORT}"diff --git a/tests/benchmark/scripts/verify-step.sh b/tests/benchmark/scripts/verify-step.shindex 1e1b9dd3..d6439d40 100755--- a/tests/benchmark/scripts/verify-step.sh+++ b/tests/benchmark/scripts/verify-step.sh@@ -4,12 +4,13 @@# step.## Usage:-# ./verify-step.sh [--type agent|agent-browser] [--report-file <path>] <group> <step> <pass|fail|skip> "notes"+# ./verify-step.sh [--type baseline|agent|agent-browser] [--report-file <path>] <group> <step> <pass|fail|skip> "notes"set -euo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"RESULTS_DIR="${SCRIPT_DIR}/../results"+CURRENT_BASELINE_PTR="${RESULTS_DIR}/current_baseline_report.txt"CURRENT_AGENT_PTR="${RESULTS_DIR}/current_agent_report.txt"CURRENT_AGENT_BROWSER_PTR="${RESULTS_DIR}/current_agent_browser_report.txt"@@ -34,7 +35,7 @@ while [[ $# -gt 0 && "$1" == --* ]]; dodoneif [[ $# -lt 3 ]]; then- echo "Usage: $0 [--type agent|agent-browser] [--report-file <path>] <group> <step> <pass|fail|skip> [notes]"+ echo "Usage: $0 [--type baseline|agent|agent-browser] [--report-file <path>] <group> <step> <pass|fail|skip> [notes]"exit 1fi@@ -65,6 +66,10 @@ resolve_current_report() {if [[ -z "${REPORT_FILE}" ]]; thenif [[ -n "${REPORT_TYPE}" ]]; thencase "${REPORT_TYPE}" in+ baseline)+ REPORT_FILE="$(resolve_current_report "${CURRENT_BASELINE_PTR}" || true)"+ [[ -n "${REPORT_FILE}" ]] || REPORT_FILE=$(ls -t "${RESULTS_DIR}"/baseline_*.json 2>/dev/null | head -1)+ ;;agent)REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_PTR}" || true)"[[ -n "${REPORT_FILE}" ]] || REPORT_FILE=$(ls -t "${RESULTS_DIR}"/agent_benchmark_*.json 2>/dev/null | head -1)@@ -74,19 +79,20 @@ if [[ -z "${REPORT_FILE}" ]]; then[[ -n "${REPORT_FILE}" ]] || REPORT_FILE=$(ls -t "${RESULTS_DIR}"/agent_browser_benchmark_*.json 2>/dev/null | head -1);;*)- echo "ERROR: --type must be 'agent' or 'agent-browser'"+ echo "ERROR: --type must be 'baseline', 'agent', or 'agent-browser'"exit 1;;esacelseREPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_BROWSER_PTR}" || true)"[[ -n "${REPORT_FILE}" ]] || REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_PTR}" || true)"- [[ -n "${REPORT_FILE}" ]] || REPORT_FILE=$(ls -t "${RESULTS_DIR}"/agent_browser_benchmark_*.json "${RESULTS_DIR}"/agent_benchmark_*.json 2>/dev/null | head -1)+ [[ -n "${REPORT_FILE}" ]] || REPORT_FILE="$(resolve_current_report "${CURRENT_BASELINE_PTR}" || true)"+ [[ -n "${REPORT_FILE}" ]] || REPORT_FILE=$(ls -t "${RESULTS_DIR}"/agent_browser_benchmark_*.json "${RESULTS_DIR}"/agent_benchmark_*.json "${RESULTS_DIR}"/baseline_*.json 2>/dev/null | head -1)fifiif [[ -z "${REPORT_FILE:-}" || ! -f "${REPORT_FILE}" ]]; then- echo "ERROR: no agent benchmark report found"+ echo "ERROR: no benchmark report found"exit 1fi@@ -119,4 +125,9 @@ jq --arg id "${STEP_ID}" \' "${REPORT_FILE}" > "${TMP_FILE}"mv "${TMP_FILE}" "${REPORT_FILE}"++if [[ "${STATUS}" == "fail" ]]; then+ echo "[${TIMESTAMP}] Step ${STEP_ID} VERIFICATION FAILED: ${NOTES}" >> "${RESULTS_DIR}/errors.log"+fi+echo "Verified: Step ${STEP_ID} = ${STATUS}"