Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat: add opt summarize/inject-usage commands and port finalize to go

Mariof7f591e2aff3
diff --git a/.gitignore b/.gitignore
index 84abeb70..b1bafa45 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,4 +53,5 @@ pinchtab.exe
plugins/*/.venv/
tests/e2e/results/*
/runner
+tests/tools/scripts/runner
skills/agent-browser
diff --git a/internal/bridge/tab_manager.go b/internal/bridge/tab_manager.go
index 5300ff62..028339ae 100644
--- a/internal/bridge/tab_manager.go
+++ b/internal/bridge/tab_manager.go
@@ -134,7 +134,7 @@ func (tm *TabManager) CreateTab(url string) (string, context.Context, context.Ca
// Use target.CreateTarget CDP protocol call to create a new tab.
// This works for both local and remote (CDP_URL) allocators.
var targetID target.ID
- createCtx, createCancel := context.WithTimeout(tm.browserCtx, 10*time.Second)
+ createCtx, createCancel := context.WithTimeout(tm.browserCtx, 30*time.Second)
if err := chromedp.Run(createCtx,
chromedp.ActionFunc(func(ctx context.Context) error {
var err error
diff --git a/scripts/check.sh b/scripts/check.sh
index 30ca8288..7c4cc3a2 100755
--- a/scripts/check.sh
+++ b/scripts/check.sh
@@ -33,9 +33,10 @@ echo -e " ${INFO}Running pre-push checks (matches GitHub Actions CI)...${NC}"
section "Format"
-mapfile -t go_files < <(git ls-files -- '*.go' | while read -r file; do
- [ -f "$file" ] && printf '%s\n' "$file"
-done)
+go_files=()
+while IFS= read -r file; do
+ [ -f "$file" ] && go_files+=("$file")
+done < <(git ls-files -- '*.go')
unformatted=""
if [ ${#go_files[@]} -gt 0 ]; then
unformatted=$(gofmt -l "${go_files[@]}")
diff --git a/scripts/dev-opt.sh b/scripts/dev-opt.sh
index c516f6b9..33b3b0f4 100755
--- a/scripts/dev-opt.sh
+++ b/scripts/dev-opt.sh
@@ -39,7 +39,12 @@ case "${mode}" in
./scripts/run-optimization.sh
./scripts/baseline.sh
BASELINE_REPORT="$(resolve_current_report "${RESULTS_DIR}/current_baseline_report.txt")"
- ./scripts/finalize-report.sh "${BASELINE_REPORT}"
+ if [[ -n "${BASELINE_REPORT}" && -f "${BASELINE_REPORT}" ]]; then
+ echo ""
+ echo "Baseline complete:"
+ jq -r '" steps: \(.totals.steps_answered // .totals.steps_passed // 0)/87 verified: \(.totals.steps_verified_passed // 0)/87 elapsed: \(.totals.elapsed_ms // 0)ms"' "${BASELINE_REPORT}"
+ echo " report: ${BASELINE_REPORT}"
+ fi
;;
-h|--help|help)
usage
diff --git a/skills/pinchtab-opt/SKILL.md b/skills/pinchtab-opt/SKILL.md
new file mode 100644
index 00000000..f0bec351
--- /dev/null
+++ b/skills/pinchtab-opt/SKILL.md
@@ -0,0 +1,125 @@
+---
+name: pinchtab-opt
+description: "Run the PinchTab optimization loop. Spawns blind subagents that execute 87 browser automation steps across 39 groups using only the PinchTab skill, then reports pass/fail results and operation counts vs baseline. Use when asked to 'run optimization', 'run the opt loop', 'benchmark the agent', or 'test pinchtab agent'."
+---
+
+# PinchTab Optimization Loop
+
+Run blind subagents against 87 browser automation steps (39 groups) to measure how well an AI agent can drive PinchTab without hand-held selectors.
+
+## Path Resolution
+
+All paths below are relative to the **project root** (git root). Resolve it first:
+
+```bash
+PROJECT_ROOT=$(git rev-parse --show-toplevel)
+TOOLS_DIR="$PROJECT_ROOT/tests/tools"
+```
+
+The subagents must run with `$TOOLS_DIR` as their working directory because `./scripts/pt` and `./scripts/runner` live there.
+
+## Prerequisites
+
+Docker services must be running. Verify before spawning agents:
+
+```bash
+$TOOLS_DIR/scripts/pt health
+```
+
+If unhealthy, start the services:
+
+```bash
+docker compose -f "$TOOLS_DIR/docker-compose.yml" up -d --build
+```
+
+Wait a few seconds and re-check health.
+
+## Execution
+
+### 0. Create per-agent report files
+
+Before spawning agents, create isolated report files so concurrent writes don't corrupt a shared file:
+
+```bash
+RESULTS_DIR="$TOOLS_DIR/../benchmark/results"
+TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)
+mkdir -p "$RESULTS_DIR"
+
+for agent in A B C; do
+ cat > "$RESULTS_DIR/agent${agent}_${TIMESTAMP}.json" <<SEED
+{
+ "benchmark": {"type": "pinchtab", "timestamp": "${TIMESTAMP}", "agent": "${agent}"},
+ "totals": {"steps_answered": 0},
+ "steps": []
+}
+SEED
+done
+```
+
+Save the three file paths — you'll pass one to each subagent.
+
+### 1. Spawn 3 parallel subagents
+
+Use the **Agent** tool with `run_in_background: true`. Split the 39 groups into three batches:
+
+- **Batch A**: groups 0-12 (39 steps)
+- **Batch B**: groups 13-25 (26 steps)
+- **Batch C**: groups 26-38 (22 steps)
+
+Each subagent gets the **same prompt template** — only the group range and `{REPORT_FILE}` change. Replace `{START}`, `{END}`, `{START_PAD}`, `{END_PAD}`, `{PROJECT_ROOT}`, and `{REPORT_FILE}` with actual values:
+
+```
+You are running PinchTab optimization tasks. Your job is to execute groups {START} through {END}.
+
+CRITICAL: Your working directory MUST be {PROJECT_ROOT}/tests/tools for all commands. Prefix every shell command with `cd {PROJECT_ROOT}/tests/tools && `.
+
+Your report file is: {REPORT_FILE}
+Use `--report-file {REPORT_FILE}` on every `./scripts/runner step-end` call.
+
+Start by reading these files to understand your tools and tasks:
+1. Read `{PROJECT_ROOT}/tests/optimization/subagent-context.md` — environment, wrapper, and recording format.
+2. Read `{PROJECT_ROOT}/skills/pinchtab/SKILL.md` — full PinchTab command reference.
+3. Read each group file from `{PROJECT_ROOT}/tests/optimization/group-{START_PAD}.md` through `{PROJECT_ROOT}/tests/optimization/group-{END_PAD}.md`.
+
+DO NOT read `{PROJECT_ROOT}/tests/tools/scripts/baseline.sh` or any file under `{PROJECT_ROOT}/tests/benchmark/`.
+
+After reading the above files, execute each step in each group sequentially:
+- Always cd to {PROJECT_ROOT}/tests/tools before running commands.
+- Use `./scripts/pt` as the wrapper for all PinchTab commands.
+- After each step, record the result with `./scripts/runner step-end --report-file {REPORT_FILE} <group> <step> answer "<observation>" pass "notes"` (or fail if it didn't work).
+- Use your judgment to figure out the right PinchTab commands from the skill doc. The group files describe WHAT to do, not HOW.
+
+Work through every step in groups {START}-{END}. Do not skip any.
+```
+
+### 2. Monitor progress
+
+While agents run, periodically count step-end recordings in each agent's output file:
+
+```bash
+grep -c "step-end" <output_file>
+```
+
+Expected totals: Batch A ~39, Batch B ~26, Batch C ~22 = 87 total.
+
+### 3. Collect and summarize
+
+Once all 3 agents complete, use `./scripts/runner` subcommands to merge reports (`merge-reports`), inject token usage from subagent transcripts (`inject-usage`), and print the final comparison table (`opt summarize`). Present the summarize output to the user as-is.
+
+## Reference Numbers
+
+- **Baseline**: 87/87 steps, 246 browser ops, 2.8 ops/step
+- **Expected agent range**: 350-500 browser ops, 4-6 ops/step (agent must explore pages before acting)
+- **Group count**: 39 groups, 87 total steps
+
+## File Locations (relative to project root)
+
+| Path | Purpose |
+|------|---------|
+| `tests/optimization/subagent-context.md` | Subagent instructions (env, wrapper, recording) |
+| `tests/optimization/index.md` | Group listing |
+| `tests/optimization/group-00.md` .. `group-38.md` | Task descriptions |
+| `skills/pinchtab/SKILL.md` | PinchTab command reference (read by subagent) |
+| `tests/tools/scripts/pt` | PinchTab wrapper (CWD must be `tests/tools`) |
+| `tests/tools/scripts/runner` | Step recorder (CWD must be `tests/tools`) |
+| `tests/tools/scripts/baseline.sh` | Baseline (subagent must NOT read this) |
diff --git a/skills/pinchtab/SKILL.md b/skills/pinchtab/SKILL.md
index 764cec83..c54b10ad 100644
--- a/skills/pinchtab/SKILL.md
+++ b/skills/pinchtab/SKILL.md
@@ -100,6 +100,8 @@ pinchtab instance navigate <instance-id> <url>
Tab state is automatic: `nav` persists the tab ID to a state file, and subsequent commands read it. Just run `pinchtab nav URL` then `pinchtab snap -i -c` — the tab is remembered. For explicit control, use `--tab <id>`.
+**Parallel agents**: when multiple agents share the same browser instance, the automatic tab state file becomes a race condition — one agent's `nav` overwrites another's tab ID. Each agent must manage its own tab ID explicitly: open with `nav <url> --new-tab`, capture the tab ID from the output, and pass `--tab <id>` on every subsequent command (`snap`, `click`, `fill`, `text`, `eval`, `press`, `scroll`, `frame`, etc.).
+
### Observation
```bash
diff --git a/tests/optimization/subagent-context.md b/tests/optimization/subagent-context.md
index 7ce4e986..064b0918 100644
--- a/tests/optimization/subagent-context.md
+++ b/tests/optimization/subagent-context.md
@@ -25,25 +25,49 @@ Always use `./scripts/pt ...` — never call `pinchtab` directly.
The wrapper executes pinchtab inside the Docker service and forwards `PINCHTAB_TOKEN` and `PINCHTAB_SERVER` automatically.
-Tab state is automatic — `nav` persists the tab ID to a state file, and subsequent commands read it. No need to manage tab IDs manually.
+### Tab isolation (critical for parallel agents)
+
+Multiple subagents run in parallel against the same browser instance. The default tab state file is shared, so if you rely on it another agent's `nav` will overwrite your tab ID and your subsequent commands will hit the wrong tab or fail with "tab not found".
+
+**You must manage tab IDs explicitly:**
+
+1. On your first `nav`, use `--new-tab` to get a dedicated tab. Capture the tab ID from the output.
+2. Pass `--tab <your-tab-id>` on **every** subsequent command (`snap`, `click`, `fill`, `text`, `eval`, `press`, `scroll`, `frame`, etc.).
+3. Never rely on the automatic state file — always be explicit.
+
+Example:
+```bash
+# First navigation — open a new tab and capture the ID
+./scripts/pt nav "http://fixtures/wiki.html" --new-tab --snap
+# Output includes tab ID, e.g. A1B2C3D4...
+
+# All subsequent commands use --tab explicitly
+./scripts/pt click e3 --tab A1B2C3D4... --snap-diff
+./scripts/pt text --tab A1B2C3D4...
+```
+
+If you lose your tab (404 error), re-navigate with `--new-tab` and update your tab ID.
## Recording
+Each agent writes to its own report file to avoid concurrent-write corruption. Your report file path will be provided as `REPORT_FILE` when you are launched. Pass it on every `step-end` call with `--report-file`.
+
Record every step result immediately after completion:
```bash
-./scripts/runner step-end <group> <step> answer "<what you observed>" <pass|fail|skip> "verification notes"
+./scripts/runner step-end --report-file "$REPORT_FILE" <group> <step> answer "<what you observed>" <pass|fail|skip> "verification notes"
```
For failures:
```bash
-./scripts/runner step-end <group> <step> fail "<what went wrong>" skip ""
+./scripts/runner step-end --report-file "$REPORT_FILE" <group> <step> fail "<what went wrong>" skip ""
```
- `<group>` is the group number (e.g., `0`, `15`, `38`)
- `<step>` is the step number within the group (e.g., `1`, `2`, `3`)
- Keep answers factual — do not self-grade in the answer payload.
+- **Quote actual output, don't paraphrase.** The answer field must include the literal text or marker from the tool output. For example, if the server returns `status: ok`, write `status: ok` in the answer — not "Server responded with ok". Verification patterns match against exact substrings, so paraphrasing causes false failures.
## Execution approach
diff --git a/tests/tools/runner/internal/bench/finalize.go b/tests/tools/runner/internal/bench/finalize.go
new file mode 100644
index 00000000..52bdf827
--- /dev/null
+++ b/tests/tools/runner/internal/bench/finalize.go
@@ -0,0 +1,186 @@
+package bench
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+)
+
+func runFinalize(reportFile string, stdout, stderr io.Writer) {
+ data, err := os.ReadFile(reportFile)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "[finalize] cannot read %s: %v\n", reportFile, err)
+ return
+ }
+ var report map[string]any
+ if err := json.Unmarshal(data, &report); err != nil {
+ _, _ = fmt.Fprintf(stderr, "[finalize] cannot parse %s: %v\n", reportFile, err)
+ return
+ }
+
+ totals, _ := report["totals"].(map[string]any)
+ getF := func(key string) float64 {
+ if v, ok := totals[key].(float64); ok {
+ return v
+ }
+ return 0
+ }
+
+ usage, _ := report["run_usage"].(map[string]any)
+ getU := func(key string) float64 {
+ if usage == nil {
+ return 0
+ }
+ if v, ok := usage[key].(float64); ok {
+ return v
+ }
+ return 0
+ }
+
+ bench, _ := report["benchmark"].(map[string]any)
+ benchType, _ := bench["type"].(string)
+ model, _ := bench["model"].(string)
+
+ answered := getF("steps_answered")
+ execFailed := getF("steps_failed")
+ execSkipped := getF("steps_skipped")
+ legacyPassed := getF("steps_passed")
+ verifiedPassed := getF("steps_verified_passed")
+ verifiedFailed := getF("steps_verified_failed")
+ verifiedSkipped := getF("steps_verified_skipped")
+ pending := getF("steps_pending_verification")
+ toolCalls := getF("tool_calls")
+
+ isLegacyBaseline := benchType == "baseline" && answered == 0
+ legacyTotal := legacyPassed + execFailed + execSkipped
+ execTotal := answered + execFailed + execSkipped
+
+ pct := func(a, b float64) string {
+ if b == 0 {
+ return "0.0%"
+ }
+ return fmt.Sprintf("%.1f%%", (a/b)*100)
+ }
+
+ var sb strings.Builder
+ sb.WriteString("# Benchmark Summary\n\n")
+ sb.WriteString("| Metric | Value |\n")
+ sb.WriteString("|--------|-------|\n")
+ fmt.Fprintf(&sb, "| Type | %s |\n", benchType)
+ fmt.Fprintf(&sb, "| Model | %s |\n", model)
+
+ if isLegacyBaseline {
+ fmt.Fprintf(&sb, "| Steps Passed | %.0f |\n", legacyPassed)
+ fmt.Fprintf(&sb, "| Steps Failed | %.0f |\n", execFailed)
+ fmt.Fprintf(&sb, "| Steps Skipped | %.0f |\n", execSkipped)
+ fmt.Fprintf(&sb, "| Pass Rate | %s |\n", pct(legacyPassed, legacyTotal))
+ fmt.Fprintf(&sb, "| Tool Calls | %.0f |\n", toolCalls)
+ } else {
+ fmt.Fprintf(&sb, "| Steps Answered | %.0f |\n", answered)
+ fmt.Fprintf(&sb, "| Execution Failed | %.0f |\n", execFailed)
+ fmt.Fprintf(&sb, "| Execution Skipped | %.0f |\n", execSkipped)
+ fmt.Fprintf(&sb, "| Answer Rate | %s |\n", pct(answered, execTotal))
+ fmt.Fprintf(&sb, "| Verified Passed | %.0f |\n", verifiedPassed)
+ fmt.Fprintf(&sb, "| Verified Failed | %.0f |\n", verifiedFailed)
+ fmt.Fprintf(&sb, "| Verified Skipped | %.0f |\n", verifiedSkipped)
+ fmt.Fprintf(&sb, "| Pending Verification | %.0f |\n", pending)
+ fmt.Fprintf(&sb, "| Verification Pass Rate | %s |\n", pct(verifiedPassed, execTotal))
+ fmt.Fprintf(&sb, "| Tool Calls | %.0f |\n", toolCalls)
+ }
+
+ sb.WriteString("\n## Run Usage\n\n")
+ if usage == nil || (getU("total_tokens") == 0 && getU("request_count") == 0) {
+ sb.WriteString("- none recorded\n")
+ } else {
+ sb.WriteString("| Metric | Value |\n")
+ sb.WriteString("|--------|-------|\n")
+ source, _ := usage["source"].(string)
+ provider, _ := usage["provider"].(string)
+ if source == "" {
+ source = "unknown"
+ }
+ if provider == "" {
+ provider = "unknown"
+ }
+ fmt.Fprintf(&sb, "| Source | %s |\n", source)
+ fmt.Fprintf(&sb, "| Provider | %s |\n", provider)
+ fmt.Fprintf(&sb, "| API Requests | %.0f |\n", getU("request_count"))
+ fmt.Fprintf(&sb, "| Input Tokens (uncached) | %.0f |\n", getU("input_tokens"))
+ fmt.Fprintf(&sb, "| Cache Creation Input Tokens | %.0f |\n", getU("cache_creation_input_tokens"))
+ fmt.Fprintf(&sb, "| Cache Read Input Tokens | %.0f |\n", getU("cache_read_input_tokens"))
+ fmt.Fprintf(&sb, "| Total Input Tokens | %.0f |\n", getU("total_input_tokens"))
+ fmt.Fprintf(&sb, "| Output Tokens | %.0f |\n", getU("output_tokens"))
+ fmt.Fprintf(&sb, "| Total Tokens | %.0f |\n", getU("total_tokens"))
+ }
+
+ steps, _ := report["steps"].([]any)
+
+ sb.WriteString("\n## Pending Verification\n\n")
+ var hasPending bool
+ for _, s := range steps {
+ step, _ := s.(map[string]any)
+ status, _ := step["status"].(string)
+ ver, _ := step["verification"].(map[string]any)
+ verStatus, _ := ver["status"].(string)
+ if verStatus == "" {
+ verStatus = "pending"
+ }
+ if status == "answer" && verStatus == "pending" {
+ id, _ := step["id"].(string)
+ answer, _ := step["answer"].(string)
+ if answer == "" {
+ answer, _ = step["notes"].(string)
+ }
+ fmt.Fprintf(&sb, "- %s: %s\n", id, answer)
+ hasPending = true
+ }
+ }
+ if !hasPending {
+ sb.WriteString("- none\n")
+ }
+
+ sb.WriteString("\n## Failed Steps\n\n")
+ var hasFailed bool
+ for _, s := range steps {
+ step, _ := s.(map[string]any)
+ status, _ := step["status"].(string)
+ if status == "fail" {
+ id, _ := step["id"].(string)
+ notes, _ := step["notes"].(string)
+ fmt.Fprintf(&sb, "- %s: %s\n", id, notes)
+ hasFailed = true
+ }
+ }
+ if !hasFailed {
+ sb.WriteString("- none\n")
+ }
+
+ sb.WriteString("\n## Verification Failures\n\n")
+ var hasVerFail bool
+ for _, s := range steps {
+ step, _ := s.(map[string]any)
+ status, _ := step["status"].(string)
+ ver, _ := step["verification"].(map[string]any)
+ verStatus, _ := ver["status"].(string)
+ if status == "answer" && verStatus == "fail" {
+ id, _ := step["id"].(string)
+ notes, _ := ver["notes"].(string)
+ fmt.Fprintf(&sb, "- %s: %s\n", id, notes)
+ hasVerFail = true
+ }
+ }
+ if !hasVerFail {
+ sb.WriteString("- none\n")
+ }
+
+ summaryFile := strings.TrimSuffix(reportFile, ".json") + "_summary.md"
+ if err := os.WriteFile(summaryFile, []byte(sb.String()), 0644); err != nil {
+ _, _ = fmt.Fprintf(stderr, "[finalize] cannot write %s: %v\n", summaryFile, err)
+ return
+ }
+
+ _, _ = fmt.Fprintf(stdout, "Wrote %s\n\n", summaryFile)
+ _, _ = fmt.Fprint(stdout, sb.String())
+}
diff --git a/tests/tools/runner/internal/bench/loop.go b/tests/tools/runner/internal/bench/loop.go
index f8797641..aeea7e32 100644
--- a/tests/tools/runner/internal/bench/loop.go
+++ b/tests/tools/runner/internal/bench/loop.go
@@ -151,10 +151,7 @@ func RunLoop(cfg LoopConfig, runner Runner, shell *PersistentShell) LoopResult {
if _, err := os.Stat(cfg.ReportFile); err == nil {
recordUsage(cfg.ToolsDir, cfg.ReportFile, runner)
if cfg.Finalize {
- // Stream finalize-report.sh's output through our stdout/stderr so
- // the generated markdown summary actually reaches the terminal.
- // (The old finalizeReport() used cmd.Run() which discards output.)
- runFinalizeReport(cfg.ToolsDir, cfg.ReportFile, cfg.Stdout, cfg.Stderr)
+ runFinalize(cfg.ReportFile, cfg.Stdout, cfg.Stderr)
}
}
@@ -168,9 +165,8 @@ func RunLoop(cfg LoopConfig, runner Runner, shell *PersistentShell) LoopResult {
}
out.Summary(runner.Usage(), runner.Provider())
- // Harness-generated end-of-run summary: runs regardless of whether
- // finalize-report.sh succeeded, reads the JSON report directly so it
- // always reflects recorded state.
+ // Harness-generated end-of-run summary: reads the JSON report directly
+ // so it always reflects recorded state.
if _, err := os.Stat(cfg.ReportFile); err == nil {
PrintEndBanner(cfg.Stdout, cfg.ReportFile)
}
diff --git a/tests/tools/runner/internal/bench/summary.go b/tests/tools/runner/internal/bench/summary.go
index b6b93070..c3793c2a 100644
--- a/tests/tools/runner/internal/bench/summary.go
+++ b/tests/tools/runner/internal/bench/summary.go
@@ -6,7 +6,6 @@ import (
"fmt"
"io"
"os"
- "os/exec"
"path/filepath"
"regexp"
"sort"
@@ -142,9 +141,7 @@ type reportSummary struct {
} `json:"steps"`
}
-// PrintEndBanner prints a compact pass/fail table at end-of-run — independent
-// of finalize-report.sh, so the user always sees it even if finalize is
-// skipped or its markdown summary is piped elsewhere.
+// PrintEndBanner prints a compact pass/fail table at end-of-run.
func PrintEndBanner(w io.Writer, reportFile string) {
data, err := os.ReadFile(reportFile)
if err != nil {
@@ -207,15 +204,3 @@ func trimOneLine(s string, max int) string {
}
return s
}
-
-// runFinalizeReport invokes finalize-report.sh and streams its stdout/stderr
-// through the provided writers. Previously loop.go::finalizeReport used
-// cmd.Run() which silently discarded the summary output — this is the fix.
-func runFinalizeReport(toolsDir, reportFile string, stdout, stderr io.Writer) {
- script := filepath.Join(toolsDir, "scripts", "finalize-report.sh")
- cmd := exec.Command(script, reportFile) // #nosec G204 -- script path is constructed from known toolsDir
- cmd.Dir = toolsDir
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- _ = cmd.Run()
-}
diff --git a/tests/tools/runner/internal/opt/injectusage.go b/tests/tools/runner/internal/opt/injectusage.go
new file mode 100644
index 00000000..564accfd
--- /dev/null
+++ b/tests/tools/runner/internal/opt/injectusage.go
@@ -0,0 +1,155 @@
+package opt
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {
+ var reportFile string
+ var transcriptFiles []string
+
+ i := 0
+ for i < len(argv) && strings.HasPrefix(argv[i], "-") {
+ switch argv[i] {
+ case "--report", "-r":
+ if i+1 >= len(argv) {
+ _, _ = fmt.Fprintf(stderr, "inject-usage: %s requires a value\n", argv[i])
+ return 1
+ }
+ reportFile = argv[i+1]
+ i += 2
+ default:
+ _, _ = fmt.Fprintf(stderr, "inject-usage: unknown option: %s\n", argv[i])
+ return 1
+ }
+ }
+
+ transcriptFiles = argv[i:]
+ if len(transcriptFiles) == 0 {
+ _, _ = fmt.Fprintln(stderr, "usage: inject-usage [-r report.json] <transcript1.jsonl> [transcript2.jsonl ...]")
+ return 1
+ }
+
+ // Expand globs
+ var files []string
+ for _, pattern := range transcriptFiles {
+ matches, err := filepath.Glob(pattern)
+ if err != nil || len(matches) == 0 {
+ if _, statErr := os.Stat(pattern); statErr == nil {
+ files = append(files, pattern)
+ } else {
+ _, _ = fmt.Fprintf(stderr, "inject-usage: no files matched %q\n", pattern)
+ return 1
+ }
+ } else {
+ files = append(files, matches...)
+ }
+ }
+
+ var totals struct {
+ InputTokens int64 `json:"input_tokens"`
+ CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`
+ CacheReadInputTokens int64 `json:"cache_read_input_tokens"`
+ OutputTokens int64 `json:"output_tokens"`
+ RequestCount int64 `json:"request_count"`
+ }
+
+ for _, f := range files {
+ fh, err := os.Open(f)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to open %s: %v\n", f, err)
+ return 1
+ }
+
+ var fileRequests int64
+ scanner := bufio.NewScanner(fh)
+ scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)
+ for scanner.Scan() {
+ var entry struct {
+ Message struct {
+ Role string `json:"role"`
+ Usage struct {
+ InputTokens int64 `json:"input_tokens"`
+ CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`
+ CacheReadInputTokens int64 `json:"cache_read_input_tokens"`
+ OutputTokens int64 `json:"output_tokens"`
+ } `json:"usage"`
+ } `json:"message"`
+ }
+ if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
+ continue
+ }
+ u := entry.Message.Usage
+ if u.InputTokens == 0 && u.CacheCreationInputTokens == 0 && u.CacheReadInputTokens == 0 && u.OutputTokens == 0 {
+ continue
+ }
+ totals.InputTokens += u.InputTokens
+ totals.CacheCreationInputTokens += u.CacheCreationInputTokens
+ totals.CacheReadInputTokens += u.CacheReadInputTokens
+ totals.OutputTokens += u.OutputTokens
+ fileRequests++
+ }
+ _ = fh.Close()
+
+ totals.RequestCount += fileRequests
+ _, _ = fmt.Fprintf(stdout, "Parsed %d API responses from %s\n", fileRequests, filepath.Base(f))
+ }
+
+ totalInput := totals.InputTokens + totals.CacheCreationInputTokens + totals.CacheReadInputTokens
+ totalTokens := totalInput + totals.OutputTokens
+
+ usage := map[string]any{
+ "source": "subagent-transcripts",
+ "provider": "anthropic",
+ "request_count": totals.RequestCount,
+ "input_tokens": totals.InputTokens,
+ "cache_creation_input_tokens": totals.CacheCreationInputTokens,
+ "cache_read_input_tokens": totals.CacheReadInputTokens,
+ "total_input_tokens": totalInput,
+ "output_tokens": totals.OutputTokens,
+ "total_tokens": totalTokens,
+ }
+
+ _, _ = fmt.Fprintf(stdout, "\nToken usage:\n")
+ _, _ = fmt.Fprintf(stdout, " API requests: %d\n", totals.RequestCount)
+ _, _ = fmt.Fprintf(stdout, " Input (uncached): %d\n", totals.InputTokens)
+ _, _ = fmt.Fprintf(stdout, " Cache write: %d\n", totals.CacheCreationInputTokens)
+ _, _ = fmt.Fprintf(stdout, " Cache read: %d\n", totals.CacheReadInputTokens)
+ _, _ = fmt.Fprintf(stdout, " Total input: %d\n", totalInput)
+ _, _ = fmt.Fprintf(stdout, " Output: %d\n", totals.OutputTokens)
+ _, _ = fmt.Fprintf(stdout, " Total tokens: %d\n", totalTokens)
+
+ if reportFile != "" {
+ data, err := os.ReadFile(reportFile)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to read %s: %v\n", reportFile, err)
+ return 1
+ }
+ var report map[string]any
+ if err := json.Unmarshal(data, &report); err != nil {
+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to parse %s: %v\n", reportFile, err)
+ return 1
+ }
+
+ report["run_usage"] = usage
+
+ output, err := json.MarshalIndent(report, "", " ")
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to marshal: %v\n", err)
+ return 1
+ }
+ if err := os.WriteFile(reportFile, output, 0644); err != nil {
+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to write %s: %v\n", reportFile, err)
+ return 1
+ }
+ _, _ = fmt.Fprintf(stdout, "\nInjected run_usage into %s\n", reportFile)
+ }
+
+ return 0
+}
diff --git a/tests/tools/runner/internal/opt/mergereports.go b/tests/tools/runner/internal/opt/mergereports.go
new file mode 100644
index 00000000..0c0e5db1
--- /dev/null
+++ b/tests/tools/runner/internal/opt/mergereports.go
@@ -0,0 +1,147 @@
+package opt
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+)
+
+type MergeReportsArgs struct {
+ OutputFile string
+ InputFiles []string
+}
+
+func ParseMergeReportsArgs(argv []string) (MergeReportsArgs, error) {
+ var args MergeReportsArgs
+
+ i := 0
+ for i < len(argv) && strings.HasPrefix(argv[i], "-") {
+ switch argv[i] {
+ case "--output", "-o":
+ if i+1 >= len(argv) {
+ return args, fmt.Errorf("%s requires a value", argv[i])
+ }
+ args.OutputFile = argv[i+1]
+ i += 2
+ default:
+ return args, fmt.Errorf("unknown option: %s", argv[i])
+ }
+ }
+
+ args.InputFiles = argv[i:]
+ if len(args.InputFiles) == 0 {
+ return args, fmt.Errorf("usage: merge-reports [-o output.json] <report1.json> <report2.json>")
+ }
+
+ return args, nil
+}
+
+func RunMergeReports(argv []string, stdout, stderr io.Writer) int {
+ args, err := ParseMergeReportsArgs(argv)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "merge-reports: %v\n", err)
+ return 1
+ }
+
+ // Also support glob patterns (e.g. agent*_20260429.json)
+ var files []string
+ for _, pattern := range args.InputFiles {
+ matches, err := filepath.Glob(pattern)
+ if err != nil || len(matches) == 0 {
+ if _, statErr := os.Stat(pattern); statErr == nil {
+ files = append(files, pattern)
+ } else {
+ _, _ = fmt.Fprintf(stderr, "merge-reports: no files matched %q\n", pattern)
+ return 1
+ }
+ } else {
+ files = append(files, matches...)
+ }
+ }
+
+ var allSteps []map[string]any
+ seen := make(map[string]bool)
+
+ for _, f := range files {
+ data, err := os.ReadFile(f)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to read %s: %v\n", f, err)
+ return 1
+ }
+ var report map[string]any
+ if err := json.Unmarshal(data, &report); err != nil {
+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to parse %s: %v\n", f, err)
+ return 1
+ }
+
+ steps, _ := report["steps"].([]any)
+ for _, s := range steps {
+ step, ok := s.(map[string]any)
+ if !ok {
+ continue
+ }
+ id, _ := step["id"].(string)
+ if id == "" {
+ g, _ := step["group"].(float64)
+ s, _ := step["step"].(float64)
+ id = fmt.Sprintf("%d.%d", int(g), int(s))
+ }
+ if !seen[id] {
+ seen[id] = true
+ allSteps = append(allSteps, step)
+ }
+ }
+ _, _ = fmt.Fprintf(stdout, "Loaded %d steps from %s\n", len(steps), filepath.Base(f))
+ }
+
+ // Sort by group.step
+ sort.Slice(allSteps, func(i, j int) bool {
+ gi, _ := allSteps[i]["group"].(float64)
+ si, _ := allSteps[i]["step"].(float64)
+ gj, _ := allSteps[j]["group"].(float64)
+ sj, _ := allSteps[j]["step"].(float64)
+ if gi != gj {
+ return gi < gj
+ }
+ return si < sj
+ })
+
+ // Build merged report
+ merged := map[string]any{
+ "benchmark": map[string]any{
+ "type": "pinchtab",
+ "timestamp": time.Now().UTC().Format("20060102_150405"),
+ "merged": true,
+ "sources": len(files),
+ },
+ "totals": map[string]any{
+ "steps_answered": len(allSteps),
+ },
+ "steps": allSteps,
+ }
+
+ output, err := json.MarshalIndent(merged, "", " ")
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to marshal: %v\n", err)
+ return 1
+ }
+
+ if args.OutputFile != "" {
+ if err := os.WriteFile(args.OutputFile, output, 0644); err != nil {
+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to write %s: %v\n", args.OutputFile, err)
+ return 1
+ }
+ _, _ = fmt.Fprintf(stdout, "Merged %d unique steps into %s\n", len(allSteps), args.OutputFile)
+ } else {
+ _, _ = fmt.Fprintf(stdout, "Merged %d unique steps (use -o to write to file)\n", len(allSteps))
+ _, _ = stdout.Write(output)
+ _, _ = fmt.Fprintln(stdout)
+ }
+
+ return 0
+}
diff --git a/tests/tools/runner/internal/opt/opt.go b/tests/tools/runner/internal/opt/opt.go
new file mode 100644
index 00000000..c78aadc9
--- /dev/null
+++ b/tests/tools/runner/internal/opt/opt.go
@@ -0,0 +1,28 @@
+package opt
+
+import (
+ "fmt"
+ "io"
+)
+
+func Run(argv []string, stdout, stderr io.Writer) int {
+ if len(argv) == 0 {
+ _, _ = fmt.Fprintln(stderr, "usage: runner opt <merge-reports|verify-answers> [args...]")
+ return 1
+ }
+
+ switch argv[0] {
+ case "merge-reports":
+ return RunMergeReports(argv[1:], stdout, stderr)
+ case "verify-answers":
+ return RunVerifyAnswers(argv[1:], stdout, stderr)
+ case "inject-usage":
+ return RunInjectUsage(argv[1:], stdout, stderr)
+ case "summarize":
+ return RunSummarize(argv[1:], stdout, stderr)
+ default:
+ _, _ = fmt.Fprintf(stderr, "runner opt: unknown subcommand %q\n", argv[0])
+ _, _ = fmt.Fprintln(stderr, "Use one of: merge-reports, verify-answers, inject-usage, summarize")
+ return 1
+ }
+}
diff --git a/tests/tools/runner/internal/opt/summarize.go b/tests/tools/runner/internal/opt/summarize.go
new file mode 100644
index 00000000..dc2d69a3
--- /dev/null
+++ b/tests/tools/runner/internal/opt/summarize.go
@@ -0,0 +1,295 @@
+package opt
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+)
+
+func RunSummarize(argv []string, stdout, stderr io.Writer) int {
+ var reportFile string
+ var transcriptFiles []string
+
+ i := 0
+ for i < len(argv) && strings.HasPrefix(argv[i], "-") {
+ switch argv[i] {
+ case "--report", "-r":
+ if i+1 >= len(argv) {
+ _, _ = fmt.Fprintf(stderr, "summarize: %s requires a value\n", argv[i])
+ return 1
+ }
+ reportFile = argv[i+1]
+ i += 2
+ default:
+ _, _ = fmt.Fprintf(stderr, "summarize: unknown option: %s\n", argv[i])
+ return 1
+ }
+ }
+ transcriptFiles = argv[i:]
+
+ if reportFile == "" {
+ _, _ = fmt.Fprintln(stderr, "usage: summarize -r <merged-report.json> [transcript1.jsonl ...]")
+ return 1
+ }
+
+ data, err := os.ReadFile(reportFile)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "summarize: %v\n", err)
+ return 1
+ }
+ var report map[string]any
+ if err := json.Unmarshal(data, &report); err != nil {
+ _, _ = fmt.Fprintf(stderr, "summarize: %v\n", err)
+ return 1
+ }
+
+ steps, _ := report["steps"].([]any)
+ var answered, failed, skipped int
+ seen := make(map[string]string)
+ for _, s := range steps {
+ step, _ := s.(map[string]any)
+ status, _ := step["status"].(string)
+ switch status {
+ case "answer":
+ answered++
+ case "fail":
+ failed++
+ case "skip":
+ skipped++
+ }
+ id, _ := step["id"].(string)
+ if id == "" {
+ g, _ := step["group"].(float64)
+ st, _ := step["step"].(float64)
+ id = fmt.Sprintf("%d.%d", int(g), int(st))
+ }
+ answer, _ := step["answer"].(string)
+ seen[id] = answer
+ }
+ totalSteps := answered + failed + skipped
+
+ var verifyPass, verifyFail int
+ var verifyFailures []string
+ for id, answer := range seen {
+ pattern, has := expectPatterns[id]
+ if !has {
+ continue
+ }
+ re, err := regexp.Compile("(?i)" + pattern)
+ if err != nil || !re.MatchString(answer) {
+ verifyFail++
+ truncated := answer
+ if len(truncated) > 80 {
+ truncated = truncated[:77] + "..."
+ }
+ verifyFailures = append(verifyFailures, fmt.Sprintf("%s: [%s]", id, truncated))
+ } else {
+ verifyPass++
+ }
+ }
+
+ var missing []string
+ for g := 0; g <= 38; g++ {
+ count := groupSizes[g]
+ for s := 1; s <= count; s++ {
+ id := fmt.Sprintf("%d.%d", g, s)
+ if _, ok := seen[id]; !ok {
+ missing = append(missing, id)
+ }
+ }
+ }
+
+ // Parse browser ops from transcripts
+ ptRe := regexp.MustCompile(`\./scripts/pt\s+(\w+)`)
+ var totalOps int
+ cmdTypes := make(map[string]int)
+ var hasTranscripts bool
+
+ for _, f := range expandGlobs(transcriptFiles, stderr) {
+ fh, err := os.Open(f)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "summarize: warning: cannot open %s: %v\n", f, err)
+ continue
+ }
+ scanner := bufio.NewScanner(fh)
+ scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)
+ for scanner.Scan() {
+ var entry struct {
+ Message struct {
+ Content json.RawMessage `json:"content"`
+ } `json:"message"`
+ }
+ if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
+ continue
+ }
+ var blocks []struct {
+ Type string `json:"type"`
+ Name string `json:"name"`
+ Input struct {
+ Command string `json:"command"`
+ } `json:"input"`
+ }
+ if err := json.Unmarshal(entry.Message.Content, &blocks); err != nil {
+ continue
+ }
+ for _, b := range blocks {
+ if b.Type == "tool_use" && b.Name == "Bash" {
+ for _, m := range ptRe.FindAllStringSubmatch(b.Input.Command, -1) {
+ totalOps++
+ cmdTypes[m[1]]++
+ }
+ }
+ }
+ }
+ _ = fh.Close()
+ hasTranscripts = true
+ }
+
+ usage, _ := report["run_usage"].(map[string]any)
+ getInt := func(key string) int64 {
+ if v, ok := usage[key].(float64); ok {
+ return int64(v)
+ }
+ return 0
+ }
+
+ // Build rows for a single combined table
+ type row struct {
+ metric, baseline, agent string
+ }
+ rows := []row{
+ {"Steps completed", "87/87", fmt.Sprintf("%d/87", totalSteps)},
+ {"Verified pass", "87/87", fmt.Sprintf("%d/87", verifyPass)},
+ }
+ if hasTranscripts {
+ rows = append(rows,
+ row{"Browser ops", "246", fmtInt(int64(totalOps))},
+ row{"Ops/step", "2.8", fmt.Sprintf("%.1f", float64(totalOps)/87.0)},
+ row{"Ratio", "1.0x", fmt.Sprintf("%.2fx", float64(totalOps)/246.0)},
+ )
+ }
+ rows = append(rows, row{"Errors", "0", fmt.Sprintf("%d", failed)})
+
+ if usage != nil {
+ rows = append(rows,
+ row{"", "", ""},
+ row{"API requests", "", fmtInt(getInt("request_count"))},
+ row{"Input (uncached)", "", fmtInt(getInt("input_tokens"))},
+ row{"Cache write", "", fmtInt(getInt("cache_creation_input_tokens"))},
+ row{"Cache read", "", fmtInt(getInt("cache_read_input_tokens"))},
+ row{"Total input", "", fmtInt(getInt("total_input_tokens"))},
+ row{"Output", "", fmtInt(getInt("output_tokens"))},
+ row{"Total tokens", "", fmtInt(getInt("total_tokens"))},
+ )
+ }
+
+ // Compute column widths
+ colW := [3]int{len("Metric"), len("Baseline"), len("Agent")}
+ for _, r := range rows {
+ if r.metric == "" {
+ continue
+ }
+ if len(r.metric) > colW[0] {
+ colW[0] = len(r.metric)
+ }
+ if len(r.baseline) > colW[1] {
+ colW[1] = len(r.baseline)
+ }
+ if len(r.agent) > colW[2] {
+ colW[2] = len(r.agent)
+ }
+ }
+
+ hLine := fmt.Sprintf("+-%-s-+-%-s-+-%-s-+", strings.Repeat("-", colW[0]), strings.Repeat("-", colW[1]), strings.Repeat("-", colW[2]))
+ fmtRow := func(a, b, c string) string {
+ return fmt.Sprintf("| %-*s | %-*s | %-*s |", colW[0], a, colW[1], b, colW[2], c)
+ }
+
+ _, _ = fmt.Fprintln(stdout)
+ _, _ = fmt.Fprintln(stdout, hLine)
+ _, _ = fmt.Fprintln(stdout, fmtRow("Metric", "Baseline", "Agent"))
+ _, _ = fmt.Fprintln(stdout, hLine)
+ for _, r := range rows {
+ if r.metric == "" {
+ _, _ = fmt.Fprintln(stdout, hLine)
+ continue
+ }
+ _, _ = fmt.Fprintln(stdout, fmtRow(r.metric, r.baseline, r.agent))
+ }
+ _, _ = fmt.Fprintln(stdout, hLine)
+
+ // Ops breakdown (compact, one line)
+ if hasTranscripts {
+ type kv struct {
+ k string
+ v int
+ }
+ var sortedOps []kv
+ for k, v := range cmdTypes {
+ sortedOps = append(sortedOps, kv{k, v})
+ }
+ sort.Slice(sortedOps, func(i, j int) bool { return sortedOps[i].v > sortedOps[j].v })
+ _, _ = fmt.Fprintln(stdout)
+ _, _ = fmt.Fprint(stdout, "Ops: ")
+ for i, s := range sortedOps {
+ if i > 0 {
+ _, _ = fmt.Fprint(stdout, ", ")
+ }
+ _, _ = fmt.Fprintf(stdout, "%s %d", s.k, s.v)
+ }
+ _, _ = fmt.Fprintln(stdout)
+ }
+
+ if len(missing) > 0 {
+ _, _ = fmt.Fprintf(stdout, "\nMissing: %s\n", strings.Join(missing, ", "))
+ }
+
+ if len(verifyFailures) > 0 {
+ _, _ = fmt.Fprintln(stdout, "\nVerification failures:")
+ for _, f := range verifyFailures {
+ _, _ = fmt.Fprintf(stdout, " %s\n", f)
+ }
+ }
+
+ _, _ = fmt.Fprintln(stdout)
+ return 0
+}
+
+func fmtInt(n int64) string {
+ if n == 0 {
+ return "0"
+ }
+ s := fmt.Sprintf("%d", n)
+ if n < 0 {
+ return s
+ }
+ var b strings.Builder
+ for i, c := range s {
+ if i > 0 && (len(s)-i)%3 == 0 {
+ b.WriteByte(',')
+ }
+ b.WriteRune(c)
+ }
+ return b.String()
+}
+
+func expandGlobs(patterns []string, stderr io.Writer) []string {
+ var out []string
+ for _, p := range patterns {
+ matches, err := filepath.Glob(p)
+ if err != nil || len(matches) == 0 {
+ if _, statErr := os.Stat(p); statErr == nil {
+ out = append(out, p)
+ }
+ } else {
+ out = append(out, matches...)
+ }
+ }
+ return out
+}
diff --git a/tests/tools/runner/internal/opt/verifyanswers.go b/tests/tools/runner/internal/opt/verifyanswers.go
new file mode 100644
index 00000000..5195ef38
--- /dev/null
+++ b/tests/tools/runner/internal/opt/verifyanswers.go
@@ -0,0 +1,330 @@
+package opt
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+)
+
+// expectPatterns maps step ID → regex pattern for answer verification.
+// Ported from verify-answers.sh — patterns are case-insensitive.
+var expectPatterns = map[string]string{
+ // Group 0: Setup & diagnosis
+ "0.1": `status.*ok|status:.*ok`,
+ "0.2": `401|missing_token|unauth`,
+ "0.3": `200|authed.*200`,
+ "0.4": `running`,
+ "0.5": `tab.*list|tabs.*returned|active tab|listed.*tab|\d+ tab.*found|tabs:`,
+ "0.6": `cleaned|closed|no stale|reused.*tab|no cleanup|single active|is empty|already empty|clean.*state|cannot close last|one tab`,
+ "0.7": `VERIFY_HOME_LOADED_12345|navigated to fixtures|fixtures.*home|PinchTab Benchmark|fixtures/`,
+ "0.8": `[0-9A-F]{32}|tab.*id.*captured`,
+ // Group 1: Reading & Extracting
+ "1.1": `COUNT_LANGUAGES_12|Programming Languages 12|12 articles`,
+ "1.2": `wiki-go|Go \(programming language\)|409 success|clicked`,
+ "1.3": `Robert Griesemer.*2009|2009.*Griesemer`,
+ "1.4": `FEATURE_COUNT_6|6 key features|6 features`,
+ "1.5": `Artificial Intelligence.*Climate Action.*Mars|Artificial Intelligence, Climate Action`,
+ "1.6": `24,582.*1,284,930|24,582|Revenue \$1,284,930`,
+ // Group 2: Search
+ "2.1": `wiki-go|VERIFY_WIKI_GO_LANG_88888`,
+ "2.2": `No results|no results|xyznonexistent`,
+ "2.3": `Artificial Intelligence|ARTIFICIAL_INTELLIGENCE`,
+ // Group 3: Form
+ "3.1": `submitted|VERIFY_FORM_SUBMITTED_SUCCESS|FORM_SUBMITTED|SUBMISSION_DATA`,
+ "3.2": `[Rr]eset.*button|#reset-btn|[Rr]eset.*present|reset-btn`,
+ // Group 4: SPA
+ "4.1": `TASK_STATS_TOTAL_3|Total.*3.*Active.*2.*Done.*1|3.*2.*1`,
+ "4.2": `TASK_ADDED|AUTOMATE|DEPLOYMENT|high`,
+ "4.3": `deleted.*task|TASK_STATS_TOTAL_3|All Tasks.*3|3.*tasks|4 to 3|tests deleted`,
+ // Group 5: Login
+ "5.1": `INVALID_CREDENTIALS_ERROR|Invalid`,
+ "5.2": `VERIFY_LOGIN_SUCCESS_DASHBOARD|SESSION_TOKEN_ACTIVE_TRUE|login success|Dashboard`,
+ // Group 6: E-commerce
+ "6.1": `149.99.*299.99|Wireless.*Smart Watch|Portable Charger`,
+ "6.2": `449.98|CART_ITEM_WIRELESS`,
+ "6.3": `VERIFY_CHECKOUT_SUCCESS_ORDER|checkout`,
+ // Group 7: Content + Interaction
+ "7.1": `COMMENT_POSTED_RATING_5`,
+ "7.2": `Developer Tools.*15|15.*Developer|wiki-go|VERIFY_WIKI_GO`,
+ // Group 8: Error handling
+ "8.1": `404|not found`,
+ "8.2": `element.*not found|no element found|no element|clear error`,
+ // Group 9: Export
+ "9.1": `[Ss]creenshot|\.png.*[0-9]{4,} bytes`,
+ "9.2": `[Pp][Dd][Ff]|\.pdf.*[0-9]{4,} bytes`,
+ // Group 10: Modals
+ "10.1": `Dashboard Settings`,
+ "10.2": `THEME_DARK_APPLIED`,
+ // Group 11: Persistence
+ "11.1": `TASK_PERSISTENT_TEST_FOUND`,
+ "11.2": `SESSION_RENEWED|VERIFY_LOGIN_SUCCESS_DASHBOARD`,
+ // Group 12: Multi-page nav
+ "12.1": `home|VERIFY_HOME_LOADED_12345|PinchTab Benchmark - Home`,
+ "12.2": `COUNT_LANGUAGES_12|12.*Programming|Artificial Intelligence|Total Articles|12\+8\+15|COMPARISON_DATA_FOUND`,
+ // Group 13: Form validation
+ "13.1": `blocked|display:none|valueMissing|not submitted`,
+ "13.2": `VERIFY_FORM_SUBMITTED_SUCCESS|OPTIONAL_FIELD_SKIPPED_SUCCESS|submitted`,
+ // Group 14: Dynamic content
+ "14.1": `ADDITIONAL_PRODUCTS_LOADED`,
+ "14.2": `CART_UPDATED_WITH_LAZY_PRODUCT|USB-C`,
+ // Group 15: Data aggregation
+ "15.1": `1,284,930.*384,930|PROFIT_MARGIN_CALCULATED|Revenue.*Profit`,
+ "15.2": `FEATURE_COUNT_6.*7.*5|Go=6.*Python=7.*Rust=5|6, 7, 5|COMPARISON_TABLE_BUILT`,
+ "15.3": `VERIFY_ARTICLE_PAGE_41414`,
+ "15.4": `flex`,
+ // Group 16: Hover
+ "16.1": `HOVER_REVEALED_USER_1`,
+ "16.2": `HOVER_REVEALED_USER_2`,
+ // Group 17: Scroll
+ "17.1": `SCROLL_MIDDLE_MARKER`,
+ "17.2": `SCROLL_REACHED_FOOTER`,
+ // Group 18: Download
+ "18.1": `DOWNLOAD_FILE_CONTENT_VERIFIED|143 bytes`,
+ // Group 19: iFrame
+ "19.1": `IFRAME_INNER_CONTENT_LOADED`,
+ "19.2": `IFRAME_INPUT_RECEIVED_HELLO_WORLD`,
+ // Group 20: Dialogs
+ "20.1": `DIALOG_ALERT_DISMISSED`,
+ "20.2": `DIALOG_CONFIRM_CANCELLED`,
+ // Group 21: Async
+ "21.1": `ASYNC_PAYLOAD_READY_42`,
+ "21.2": `ASYNC_USER_NAME_ADA`,
+ // Group 22: Drag
+ "22.1": `DROP_ZONE_A_OK`,
+ "22.2": `DROP_SEQUENCE=DROP_ZONE_A_OK,DROP_ZONE_B_OK,DROP_ZONE_C_OK`,
+ // Group 23: Loading
+ "23.1": `VERIFY_LOADING_COMPLETE_88888`,
+ // Group 24: Keyboard
+ "24.1": `KEYBOARD_ESCAPE_PRESSED`,
+ "24.2": `KEYBOARD_KEY_A_PRESSED|KEYBOARD_ENTER_PRESSED|ESC.*A.*ENTER`,
+ // Group 25: Tabs
+ "25.1": `TAB_SETTINGS_CONTENT`,
+ "25.2": `TAB_BILLING_CONTENT`,
+ // Group 26: Accordion
+ "26.1": `ACCORDION_SECTION_A_OPEN`,
+ "26.2": `ACCORDION_SECTION_B_OPEN.*aria-expanded=false|Section A aria-expanded=false`,
+ // Group 27: Editor
+ "27.1": `EDITOR_CHARS=15|Hello rich text`,
+ "27.2": `EDITOR_COMMITTED=Hello rich text`,
+ // Group 28: Range
+ "28.1": `RANGE_VALUE_90.*BUCKET_HIGH|RANGE_VALUE_90`,
+ "28.2": `RANGE_VALUE_10.*BUCKET_LOW|RANGE_VALUE_10`,
+ // Group 29: Pagination
+ "29.1": `PAGE_2_FIRST_ITEM|PAGE_2_OF_3`,
+ "29.2": `PAGE_3_FIRST_ITEM|disabled=true`,
+ // Group 30: Dropdown
+ "30.1": `DROPDOWN_SELECTED=BETA`,
+ "30.2": `DROPDOWN_SELECTED=GAMMA`,
+ // Group 31-34: Iframe variants
+ "31.1": `DEEP_CLICKED=YES_LEVEL_3`,
+ "32.1": `IFRAME_INPUT_RECEIVED_LATE_WORLD`,
+ "33.1": `INLINE_RECEIVED_SRCDOC`,
+ "34.1": `SANDBOX_CLICKED=YES`,
+ // Group 35: Text-heavy
+ "35.1": `ARTICLE_PUBLISHED_2026_04_15.*ARTICLE_WORD_COUNT_MARKER_323|ARTICLE_WORD_COUNT_MARKER_323`,
+ "35.2": `FOOTER_COPYRIGHT_MARKER`,
+ // Group 36: SERP
+ "36.1": `RESULT_3_TITLE.*RESULT_3_SNIPPET_MARKER|RESULT_3`,
+ "36.2": `RESULT_1.*RESULT_6|SERP_RESULT_COUNT_6`,
+ // Group 37: Q&A
+ "37.1": `a-2`,
+ "37.2": `ANSWER_2_BODY_MARKER.*ACCEPTED_ANSWER_ID_A2|ACCEPTED_ANSWER_ID_A2`,
+ // Group 38: Pricing
+ "38.1": `PLAN_PRO_PRICE_29`,
+ "38.2": `PLAN_FREE_PRICE_0.*PLAN_PRO_PRICE_29.*PLAN_ENTERPRISE_PRICE_CUSTOM|PLAN_PRO_PRICE_29`,
+}
+
+var groupSizes = map[int]int{
+ 0: 8, 1: 6, 2: 3, 3: 2, 4: 3, 5: 2, 6: 3, 7: 2, 8: 2, 9: 2, 10: 2, 11: 2, 12: 2,
+ 13: 2, 14: 2, 15: 4, 16: 2, 17: 2, 18: 1, 19: 2, 20: 2, 21: 2, 22: 2, 23: 1, 24: 2, 25: 2,
+ 26: 2, 27: 2, 28: 2, 29: 2, 30: 2, 31: 1, 32: 1, 33: 1, 34: 1, 35: 2, 36: 2, 37: 2, 38: 2,
+}
+
+type VerifyAnswersArgs struct {
+ ReportFiles []string
+}
+
+func ParseVerifyAnswersArgs(argv []string) (VerifyAnswersArgs, error) {
+ var args VerifyAnswersArgs
+ args.ReportFiles = argv
+
+ if len(args.ReportFiles) == 0 {
+ return args, fmt.Errorf("usage: verify-answers <report1.json> [report2.json ...]")
+ }
+
+ return args, nil
+}
+
+type verifyResult struct {
+ ID string
+ Status string // "pass", "fail", "skip"
+ Answer string
+ Pattern string
+}
+
+func RunVerifyAnswers(argv []string, stdout, stderr io.Writer) int {
+ args, err := ParseVerifyAnswersArgs(argv)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "verify-answers: %v\n", err)
+ return 1
+ }
+
+ // Expand globs
+ var files []string
+ for _, pattern := range args.ReportFiles {
+ matches, err := filepath.Glob(pattern)
+ if err != nil || len(matches) == 0 {
+ if _, statErr := os.Stat(pattern); statErr == nil {
+ files = append(files, pattern)
+ } else {
+ _, _ = fmt.Fprintf(stderr, "verify-answers: no files matched %q\n", pattern)
+ return 1
+ }
+ } else {
+ files = append(files, matches...)
+ }
+ }
+
+ // Load all steps, deduplicate by ID (keep first seen)
+ type stepData struct {
+ ID string `json:"id"`
+ Answer string `json:"answer"`
+ }
+ seen := make(map[string]stepData)
+
+ for _, f := range files {
+ data, err := os.ReadFile(f)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "verify-answers: failed to read %s: %v\n", f, err)
+ return 1
+ }
+ var report struct {
+ Steps []stepData `json:"steps"`
+ }
+ if err := json.Unmarshal(data, &report); err != nil {
+ _, _ = fmt.Fprintf(stderr, "verify-answers: failed to parse %s: %v\n", f, err)
+ return 1
+ }
+ for _, s := range report.Steps {
+ if _, exists := seen[s.ID]; !exists {
+ seen[s.ID] = s
+ }
+ }
+ }
+
+ // Verify each step against expected pattern
+ var results []verifyResult
+ for id, s := range seen {
+ pattern, hasPattern := expectPatterns[id]
+ if !hasPattern {
+ results = append(results, verifyResult{
+ ID: id,
+ Status: "skip",
+ Answer: s.Answer,
+ })
+ continue
+ }
+
+ re, err := regexp.Compile("(?i)" + pattern)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "verify-answers: bad pattern for %s: %v\n", id, err)
+ results = append(results, verifyResult{
+ ID: id,
+ Status: "fail",
+ Answer: s.Answer,
+ Pattern: pattern,
+ })
+ continue
+ }
+
+ if re.MatchString(s.Answer) {
+ results = append(results, verifyResult{
+ ID: id,
+ Status: "pass",
+ Answer: s.Answer,
+ Pattern: pattern,
+ })
+ } else {
+ results = append(results, verifyResult{
+ ID: id,
+ Status: "fail",
+ Answer: s.Answer,
+ Pattern: pattern,
+ })
+ }
+ }
+
+ // Sort results by group.step
+ sort.Slice(results, func(i, j int) bool {
+ gi, si := parseStepID(results[i].ID)
+ gj, sj := parseStepID(results[j].ID)
+ if gi != gj {
+ return gi < gj
+ }
+ return si < sj
+ })
+
+ // Count totals
+ var pass, fail, skip int
+ var failures []verifyResult
+ for _, r := range results {
+ switch r.Status {
+ case "pass":
+ pass++
+ case "fail":
+ fail++
+ failures = append(failures, r)
+ case "skip":
+ skip++
+ }
+ }
+
+ total := pass + fail + skip
+ _, _ = fmt.Fprintf(stdout, "Verified: %d/%d steps\n", total, 87)
+ _, _ = fmt.Fprintf(stdout, " pass: %d\n", pass)
+ _, _ = fmt.Fprintf(stdout, " fail: %d\n", fail)
+ _, _ = fmt.Fprintf(stdout, " skip: %d (no pattern)\n", skip)
+
+ // Find missing steps
+ var missing []string
+ for g := 0; g <= 38; g++ {
+ count := groupSizes[g]
+ for s := 1; s <= count; s++ {
+ id := fmt.Sprintf("%d.%d", g, s)
+ if _, ok := seen[id]; !ok {
+ missing = append(missing, id)
+ }
+ }
+ }
+
+ if len(missing) > 0 {
+ _, _ = fmt.Fprintf(stdout, " missing: %s\n", strings.Join(missing, ", "))
+ }
+
+ if len(failures) > 0 {
+ _, _ = fmt.Fprintf(stdout, "\nFailed steps:\n")
+ for _, f := range failures {
+ answer := f.Answer
+ if len(answer) > 100 {
+ answer = answer[:97] + "..."
+ }
+ _, _ = fmt.Fprintf(stdout, " %s: answer=[%s] expected=[%s]\n", f.ID, answer, f.Pattern)
+ }
+ }
+
+ if fail > 0 {
+ return 1
+ }
+ return 0
+}
+
+func parseStepID(id string) (int, int) {
+ var g, s int
+ _, _ = fmt.Sscanf(id, "%d.%d", &g, &s)
+ return g, s
+}
diff --git a/tests/tools/runner/main.go b/tests/tools/runner/main.go
index d3de424f..61704016 100644
--- a/tests/tools/runner/main.go
+++ b/tests/tools/runner/main.go
@@ -7,6 +7,7 @@ import (
"github.com/pinchtab/pinchtab/tests/tools/runner/internal/bench"
"github.com/pinchtab/pinchtab/tests/tools/runner/internal/e2e"
+ "github.com/pinchtab/pinchtab/tests/tools/runner/internal/opt"
)
func main() {
@@ -25,6 +26,10 @@ func run(argv []string, stdout, stderr io.Writer) int {
return bench.Run(argv[1:], stdout, stderr)
case "verify-step", "record-step", "step-end":
return bench.Run(argv, stdout, stderr)
+ case "opt":
+ return opt.Run(argv[1:], stdout, stderr)
+ case "merge-reports", "verify-answers", "inject-usage":
+ return opt.Run(argv, stdout, stderr)
default:
if argv[0] == "--lane" || argv[0] == "--help" || argv[0] == "-h" {
return bench.Run(argv, stdout, stderr)
@@ -38,6 +43,8 @@ func run(argv []string, stdout, stderr io.Writer) int {
_, _ = fmt.Fprintln(stderr, " runner bench --lane pinchtab")
_, _ = fmt.Fprintln(stderr, " runner --lane pinchtab")
_, _ = fmt.Fprintln(stderr, " runner step-end ...")
+ _, _ = fmt.Fprintln(stderr, " runner opt merge-reports ...")
+ _, _ = fmt.Fprintln(stderr, " runner opt verify-answers ...")
return 1
}
}
diff --git a/tests/tools/scripts/finalize-report.sh b/tests/tools/scripts/finalize-report.sh
deleted file mode 100755
index 331dd6d6..00000000
--- a/tests/tools/scripts/finalize-report.sh
+++ /dev/null
@@ -1,162 +0,0 @@
-#!/usr/bin/env bash
-#
-# Renders a short markdown summary for the latest benchmark JSON report.
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-BENCHMARK_DIR="${SCRIPT_DIR}/../../benchmark"
-RESULTS_DIR="${BENCHMARK_DIR}/results"
-CURRENT_BASELINE_PTR="${RESULTS_DIR}/current_baseline_report.txt"
-CURRENT_AGENT_PTR="${RESULTS_DIR}/current_pinchtab_report.txt"
-CURRENT_AGENT_BROWSER_PTR="${RESULTS_DIR}/current_agent_browser_report.txt"
-
-resolve_current_report() {
- local ptr="$1"
- if [[ -f "${ptr}" ]]; then
- tr -d '[:space:]' < "${ptr}"
- return 0
- fi
- return 1
-}
-
-REPORT_FILE="${1:-}"
-if [[ -z "${REPORT_FILE}" ]]; then
- REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_BROWSER_PTR}" || true)"
- [[ -n "${REPORT_FILE}" ]] || REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_PTR}" || true)"
- [[ -n "${REPORT_FILE}" ]] || REPORT_FILE="$(resolve_current_report "${CURRENT_BASELINE_PTR}" || true)"
- if [[ -z "${REPORT_FILE}" ]]; then
- shopt -s nullglob
- candidates=(
- "${RESULTS_DIR}"/agent_browser_benchmark_*.json
- "${RESULTS_DIR}"/pinchtab_benchmark_*.json
- "${RESULTS_DIR}"/baseline_*.json
- )
- shopt -u nullglob
-
- if [[ ${#candidates[@]} -gt 0 ]]; then
- REPORT_FILE=$(ls -t "${candidates[@]}" | head -1)
- fi
- fi
-fi
-
-if [[ -z "${REPORT_FILE}" || ! -f "${REPORT_FILE}" ]]; then
- echo "ERROR: no benchmark report found"
- exit 1
-fi
-
-# Auto-verify pending agent answers before rendering the summary.
-# This ensures the "Verification Pass Rate" in the summary reflects actual
-# grading, not just the answer count. For baseline reports (which use
-# pass/fail directly) this is a no-op.
-PENDING=$(jq -r '.totals.steps_pending_verification // 0' "${REPORT_FILE}")
-if [[ "${PENDING}" -gt 0 ]]; then
- VERIFY_SCRIPT="${SCRIPT_DIR}/verify-answers.sh"
- if [[ -x "${VERIFY_SCRIPT}" ]]; then
- echo "Auto-verifying ${PENDING} pending answers..."
- "${VERIFY_SCRIPT}" "${REPORT_FILE}"
- else
- echo "WARNING: ${PENDING} answers pending verification but verify-answers.sh not found"
- fi
-fi
-
-SUMMARY_FILE="${REPORT_FILE%.json}_summary.md"
-
-jq -r '
- def pct($a; $b):
- if $b == 0 then "0.0%" else (((1000 * $a) / $b | round) / 10 | tostring) + "%" end;
- . as $root |
- ($root.run_usage // {}) as $usage |
- ($root.totals.steps_answered // 0) as $answered |
- ($root.totals.steps_failed // 0) as $execution_failed |
- ($root.totals.steps_skipped // 0) as $execution_skipped |
- ($root.totals.steps_passed // 0) as $legacy_passed |
- ($root.benchmark.type == "baseline" and $answered == 0) as $legacy_baseline |
- ($legacy_passed + $execution_failed + $execution_skipped) as $legacy_total |
- ($answered + $execution_failed + $execution_skipped) as $execution_total |
- ($root.totals.steps_verified_passed // 0) as $verified_passed |
- ($root.totals.steps_verified_failed // 0) as $verified_failed |
- ($root.totals.steps_verified_skipped // 0) as $verified_skipped |
- ($root.totals.steps_pending_verification // 0) as $pending |
- (if $legacy_baseline then
- [
- "# Benchmark Summary",
- "",
- "| Metric | Value |",
- "|--------|-------|",
- "| Type | \($root.benchmark.type) |",
- "| Model | \($root.benchmark.model) |",
- "| Steps Passed | \($legacy_passed) |",
- "| Steps Failed | \($execution_failed) |",
- "| Steps Skipped | \($execution_skipped) |",
- "| Pass Rate | \(pct($legacy_passed; $legacy_total)) |",
- "| Tool Calls | \($root.totals.tool_calls // 0) |"
- ]
- else
- [
- "# Benchmark Summary",
- "",
- "| Metric | Value |",
- "|--------|-------|",
- "| Type | \($root.benchmark.type) |",
- "| Model | \($root.benchmark.model) |",
- "| Steps Answered | \($answered) |",
- "| Execution Failed | \($execution_failed) |",
- "| Execution Skipped | \($execution_skipped) |",
- "| Answer Rate | \(pct($answered; $execution_total)) |",
- "| Verified Passed | \($verified_passed) |",
- "| Verified Failed | \($verified_failed) |",
- "| Verified Skipped | \($verified_skipped) |",
- "| Pending Verification | \($pending) |",
- "| Verification Pass Rate | \(pct($verified_passed; $execution_total)) |",
- "| Tool Calls | \($root.totals.tool_calls // 0) |"
- ]
- end)[],
- "",
- "## Run Usage",
- "",
- (
- if (($usage.provider // "") == "" and ($usage.total_tokens // 0) == 0 and ($usage.request_count // 0) == 0) then
- ["- none recorded"]
- else
- [
- "| Metric | Value |",
- "|--------|-------|",
- "| Source | \($usage.source // "unknown") |",
- "| Provider | \($usage.provider // "unknown") |",
- "| API Requests | \($usage.request_count // 0) |",
- "| Input Tokens (uncached) | \($usage.input_tokens // 0) |",
- "| Cache Creation Input Tokens | \($usage.cache_creation_input_tokens // 0) |",
- "| Cache Read Input Tokens | \($usage.cache_read_input_tokens // 0) |",
- "| Total Input Tokens | \($usage.total_input_tokens // 0) |",
- "| Output Tokens | \($usage.output_tokens // 0) |",
- "| Total Tokens | \($usage.total_tokens // 0) |"
- ]
- end
- )[],
- "",
- "## Pending Verification",
- "",
- (
- [ $root.steps[] | select(.status == "answer" and ((.verification.status // "pending") == "pending")) | "- \(.id): \(.answer // .notes // "")" ] |
- if length == 0 then ["- none"] else . end
- )[],
- "",
- "## Failed Steps",
- "",
- (
- [ $root.steps[] | select(.status == "fail") | "- \(.id): \(.notes)" ] |
- if length == 0 then ["- none"] else . end
- )[],
- "",
- "## Verification Failures",
- "",
- (
- [ $root.steps[] | select(.status == "answer" and ((.verification.status // "") == "fail")) | "- \(.id): \(.verification.notes // "")" ] |
- if length == 0 then ["- none"] else . end
- )[]
-' "${REPORT_FILE}" > "${SUMMARY_FILE}"
-
-echo "Wrote ${SUMMARY_FILE}"
-echo ""
-cat "${SUMMARY_FILE}"
diff --git a/tests/tools/scripts/run-agent-browser-benchmark.sh b/tests/tools/scripts/run-agent-browser-benchmark.sh
index 66a19d94..9aa5fbab 100755
--- a/tests/tools/scripts/run-agent-browser-benchmark.sh
+++ b/tests/tools/scripts/run-agent-browser-benchmark.sh
@@ -96,4 +96,4 @@ echo " 2. The agent-browser skill is refreshed by runner (DownloadAgentBrowserS
echo " 3. Use ./scripts/ab ... to drive agent-browser inside Docker"
echo " 4. Record and verify each completed benchmark step in a single call with:"
echo " ./scripts/ab step-end <group> <step> answer \"what you saw\" <pass|fail|skip> \"verification notes\""
-echo " 5. Summarize the report with ./scripts/finalize-report.sh"
+echo " 5. Report is finalized automatically at end of run"
diff --git a/tests/tools/scripts/runner b/tests/tools/scripts/runner
deleted file mode 100755
index 6c006576..00000000
--- a/tests/tools/scripts/runner
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-#
-# Thin wrapper for the Go runner.
-# Usage: ./scripts/runner <subcommand> [args...]
-#
-# Subcommands:
-# step-end Record and verify a step in one call
-# record-step Record a step result
-# verify-step Verify a previously recorded step
-# --lane ... Run the benchmark loop (existing behavior)
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-TOOLS_DIR="${SCRIPT_DIR}/.."
-
-exec go run "${TOOLS_DIR}/runner" "$@"
diff --git a/tests/tools/scripts/verify-answers.sh b/tests/tools/scripts/verify-answers.sh
deleted file mode 100755
index b8ed6e90..00000000
--- a/tests/tools/scripts/verify-answers.sh
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/env bash
-#
-# One-off verifier for Loop #31 answers. Reads the agent report, matches each
-# step's `.answer` against an expected marker/substring set, and invokes
-# verify-step.sh with pass/fail. The marker map below is hand-derived from
-# the baseline benchmark oracle implemented in scripts/baseline.sh.
-#
-# Usage:
-# ./verify_answers_for_loop31.sh results/pinchtab_benchmark_20260415_233421.json
-
-set -euo pipefail
-
-REPORT="${1:-results/pinchtab_benchmark_20260415_233421.json}"
-
-declare -A EXPECT
-# Setup & diagnosis — tightened after manual audit (see Loop #33 notes)
-EXPECT[0.1]='status.*ok|status:.*ok'
-EXPECT[0.2]='401|missing_token|unauth'
-EXPECT[0.3]='200|authed.*200'
-EXPECT[0.4]='running'
-EXPECT[0.5]='tab.*list|tabs.*returned|active tab|listed.*tab'
-EXPECT[0.6]='cleaned|closed|no stale|reused.*tab|no cleanup|single active|already empty'
-EXPECT[0.7]='VERIFY_HOME_LOADED_12345|navigated to fixtures|fixtures.*home'
-EXPECT[0.8]='[0-9A-F]{32}|tab.*id.*captured'
-# Reading & Extracting
-EXPECT[1.1]='COUNT_LANGUAGES_12|Programming Languages 12|12 articles'
-EXPECT[1.2]='wiki-go|Go \(programming language\)|409 success|clicked'
-EXPECT[1.3]='Robert Griesemer.*2009|2009.*Griesemer'
-EXPECT[1.4]='FEATURE_COUNT_6|6 key features|6 features'
-EXPECT[1.5]='Artificial Intelligence.*Climate Action.*Mars|Artificial Intelligence, Climate Action'
-EXPECT[1.6]='24,582.*1,284,930|24,582|Revenue \$1,284,930'
-# Search
-EXPECT[2.1]='wiki-go|VERIFY_WIKI_GO_LANG_88888'
-EXPECT[2.2]='No results|no results|xyznonexistent'
-EXPECT[2.3]='Artificial Intelligence|ARTIFICIAL_INTELLIGENCE'
-# Form
-EXPECT[3.1]='submitted|VERIFY_FORM_SUBMITTED_SUCCESS|FORM_SUBMITTED|SUBMISSION_DATA'
-EXPECT[3.2]='[Rr]eset.*button|#reset-btn|[Rr]eset.*present|reset-btn'
-# SPA
-EXPECT[4.1]='TASK_STATS_TOTAL_3|Total.*3.*Active.*2.*Done.*1|3.*2.*1'
-EXPECT[4.2]='TASK_ADDED|AUTOMATE|DEPLOYMENT|high'
-EXPECT[4.3]='deleted.*task|TASK_STATS_TOTAL_3|All Tasks.*3|3.*tasks'
-# Login
-EXPECT[5.1]='INVALID_CREDENTIALS_ERROR|Invalid'
-EXPECT[5.2]='VERIFY_LOGIN_SUCCESS_DASHBOARD|SESSION_TOKEN_ACTIVE_TRUE|login success|Dashboard'
-# E-commerce
-EXPECT[6.1]='149.99.*299.99|Wireless.*Smart Watch|Portable Charger'
-EXPECT[6.2]='449.98|CART_ITEM_WIRELESS'
-EXPECT[6.3]='VERIFY_CHECKOUT_SUCCESS_ORDER|checkout'
-# Content + Interaction
-EXPECT[7.1]='COMMENT_POSTED_RATING_5'
-EXPECT[7.2]='Developer Tools.*15|15.*Developer|wiki-go|VERIFY_WIKI_GO'
-# Error handling
-EXPECT[8.1]='404|not found'
-EXPECT[8.2]='element.*not found|no element found|no element|clear error'
-# Export
-EXPECT[9.1]='[Ss]creenshot|\.png.*[0-9]{4,} bytes'
-EXPECT[9.2]='[Pp][Dd][Ff]|\.pdf.*[0-9]{4,} bytes'
-# Modals
-EXPECT[10.1]='Dashboard Settings'
-EXPECT[10.2]='THEME_DARK_APPLIED'
-# Persistence
-EXPECT[11.1]='TASK_PERSISTENT_TEST_FOUND'
-EXPECT[11.2]='SESSION_RENEWED|VERIFY_LOGIN_SUCCESS_DASHBOARD'
-# Multi-page nav
-EXPECT[12.1]='home|VERIFY_HOME_LOADED_12345|PinchTab Benchmark - Home'
-EXPECT[12.2]='COUNT_LANGUAGES_12|12.*Programming|Artificial Intelligence|Total Articles|12\+8\+15|COMPARISON_DATA_FOUND'
-# Form validation
-EXPECT[13.1]='blocked|display:none|valueMissing|not submitted'
-EXPECT[13.2]='VERIFY_FORM_SUBMITTED_SUCCESS|OPTIONAL_FIELD_SKIPPED_SUCCESS|submitted'
-# Dynamic content
-EXPECT[14.1]='ADDITIONAL_PRODUCTS_LOADED'
-EXPECT[14.2]='CART_UPDATED_WITH_LAZY_PRODUCT|USB-C'
-# Data aggregation
-EXPECT[15.1]='1,284,930.*384,930|PROFIT_MARGIN_CALCULATED|Revenue.*Profit'
-EXPECT[15.2]='FEATURE_COUNT_6.*7.*5|Go=6.*Python=7.*Rust=5|6, 7, 5|COMPARISON_TABLE_BUILT'
-# Hover
-EXPECT[16.1]='HOVER_REVEALED_USER_1'
-EXPECT[16.2]='HOVER_REVEALED_USER_2'
-# Scroll
-EXPECT[17.1]='SCROLL_MIDDLE_MARKER'
-EXPECT[17.2]='SCROLL_REACHED_FOOTER'
-# Download
-EXPECT[18.1]='DOWNLOAD_FILE_CONTENT_VERIFIED|143 bytes'
-# iFrame
-EXPECT[19.1]='IFRAME_INNER_CONTENT_LOADED'
-EXPECT[19.2]='IFRAME_INPUT_RECEIVED_HELLO_WORLD'
-# Dialogs
-EXPECT[20.1]='DIALOG_ALERT_DISMISSED'
-EXPECT[20.2]='DIALOG_CONFIRM_CANCELLED'
-# Async / awaitPromise
-EXPECT[21.1]='ASYNC_PAYLOAD_READY_42'
-EXPECT[21.2]='ASYNC_USER_NAME_ADA'
-# Drag
-EXPECT[22.1]='DROP_ZONE_A_OK'
-EXPECT[22.2]='DROP_SEQUENCE=DROP_ZONE_A_OK,DROP_ZONE_B_OK,DROP_ZONE_C_OK'
-# Loading
-EXPECT[23.1]='VERIFY_LOADING_COMPLETE_88888'
-# Keyboard
-EXPECT[24.1]='KEYBOARD_ESCAPE_PRESSED'
-EXPECT[24.2]='KEYBOARD_KEY_A_PRESSED|KEYBOARD_ENTER_PRESSED|ESC.*A.*ENTER'
-# Tabs
-EXPECT[25.1]='TAB_SETTINGS_CONTENT'
-EXPECT[25.2]='TAB_BILLING_CONTENT'
-# Accordion
-EXPECT[26.1]='ACCORDION_SECTION_A_OPEN'
-EXPECT[26.2]='ACCORDION_SECTION_B_OPEN.*aria-expanded=false|Section A aria-expanded=false'
-# Editor
-EXPECT[27.1]='EDITOR_CHARS=15|Hello rich text'
-EXPECT[27.2]='EDITOR_COMMITTED=Hello rich text'
-# Range
-EXPECT[28.1]='RANGE_VALUE_90.*BUCKET_HIGH|RANGE_VALUE_90'
-EXPECT[28.2]='RANGE_VALUE_10.*BUCKET_LOW|RANGE_VALUE_10'
-# Pagination
-EXPECT[29.1]='PAGE_2_FIRST_ITEM|PAGE_2_OF_3'
-EXPECT[29.2]='PAGE_3_FIRST_ITEM|disabled=true'
-# Dropdown
-EXPECT[30.1]='DROPDOWN_SELECTED=BETA'
-EXPECT[30.2]='DROPDOWN_SELECTED=GAMMA'
-# Iframe variants
-EXPECT[31.1]='DEEP_CLICKED=YES_LEVEL_3'
-EXPECT[32.1]='IFRAME_INPUT_RECEIVED_LATE_WORLD'
-EXPECT[33.1]='INLINE_RECEIVED_SRCDOC'
-EXPECT[34.1]='SANDBOX_CLICKED=YES'
-# Text-heavy
-EXPECT[35.1]='ARTICLE_PUBLISHED_2026_04_15.*ARTICLE_WORD_COUNT_MARKER_323|ARTICLE_WORD_COUNT_MARKER_323'
-EXPECT[35.2]='FOOTER_COPYRIGHT_MARKER'
-EXPECT[36.1]='RESULT_3_TITLE.*RESULT_3_SNIPPET_MARKER|RESULT_3'
-EXPECT[36.2]='RESULT_1.*RESULT_6|SERP_RESULT_COUNT_6'
-EXPECT[37.1]='a-2'
-EXPECT[37.2]='ANSWER_2_BODY_MARKER.*ACCEPTED_ANSWER_ID_A2|ACCEPTED_ANSWER_ID_A2'
-EXPECT[38.1]='PLAN_PRO_PRICE_29'
-EXPECT[38.2]='PLAN_FREE_PRICE_0.*PLAN_PRO_PRICE_29.*PLAN_ENTERPRISE_PRICE_CUSTOM|PLAN_PRO_PRICE_29'
-
-pass=0
-fail=0
-failed_ids=()
-
-while IFS='|' read -r id answer; do
- pattern="${EXPECT[$id]:-}"
- if [[ -z "$pattern" ]]; then
- echo "skip $id — no expected pattern defined"
- "$(dirname "${BASH_SOURCE[0]}")/verify-step.sh" --report-file "$REPORT" \
- "${id%.*}" "${id#*.}" skip "no expected pattern" > /dev/null
- continue
- fi
- if printf '%s' "$answer" | grep -qE "$pattern"; then
- "$(dirname "${BASH_SOURCE[0]}")/verify-step.sh" --report-file "$REPORT" \
- "${id%.*}" "${id#*.}" pass "auto: matched /$pattern/" > /dev/null
- pass=$((pass + 1))
- else
- "$(dirname "${BASH_SOURCE[0]}")/verify-step.sh" --report-file "$REPORT" \
- "${id%.*}" "${id#*.}" fail "auto: answer did not match /$pattern/" > /dev/null
- fail=$((fail + 1))
- failed_ids+=("$id")
- fi
-done < <(jq -r '.steps[] | "\(.id)|\(.answer)"' "$REPORT")
-
-echo
-echo "Verified: $pass pass, $fail fail"
-if (( fail > 0 )); then
- echo "Failed ids:"
- printf ' %s\n' "${failed_ids[@]}"
-fi