High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: add opt summarize/inject-usage commands and port finalize to go
Mariof7f591e2aff3
diff --git a/.gitignore b/.gitignoreindex 84abeb70..b1bafa45 100644--- a/.gitignore+++ b/.gitignore@@ -53,4 +53,5 @@ pinchtab.exeplugins/*/.venv/tests/e2e/results/*/runner+tests/tools/scripts/runnerskills/agent-browserdiff --git a/internal/bridge/tab_manager.go b/internal/bridge/tab_manager.goindex 5300ff62..028339ae 100644--- a/internal/bridge/tab_manager.go+++ b/internal/bridge/tab_manager.go@@ -134,7 +134,7 @@ func (tm *TabManager) CreateTab(url string) (string, context.Context, context.Ca// Use target.CreateTarget CDP protocol call to create a new tab.// This works for both local and remote (CDP_URL) allocators.var targetID target.ID- createCtx, createCancel := context.WithTimeout(tm.browserCtx, 10*time.Second)+ createCtx, createCancel := context.WithTimeout(tm.browserCtx, 30*time.Second)if err := chromedp.Run(createCtx,chromedp.ActionFunc(func(ctx context.Context) error {var err errordiff --git a/scripts/check.sh b/scripts/check.shindex 30ca8288..7c4cc3a2 100755--- a/scripts/check.sh+++ b/scripts/check.sh@@ -33,9 +33,10 @@ echo -e " ${INFO}Running pre-push checks (matches GitHub Actions CI)...${NC}"section "Format"-mapfile -t go_files < <(git ls-files -- '*.go' | while read -r file; do- [ -f "$file" ] && printf '%s\n' "$file"-done)+go_files=()+while IFS= read -r file; do+ [ -f "$file" ] && go_files+=("$file")+done < <(git ls-files -- '*.go')unformatted=""if [ ${#go_files[@]} -gt 0 ]; thenunformatted=$(gofmt -l "${go_files[@]}")diff --git a/scripts/dev-opt.sh b/scripts/dev-opt.shindex c516f6b9..33b3b0f4 100755--- a/scripts/dev-opt.sh+++ b/scripts/dev-opt.sh@@ -39,7 +39,12 @@ case "${mode}" in./scripts/run-optimization.sh./scripts/baseline.shBASELINE_REPORT="$(resolve_current_report "${RESULTS_DIR}/current_baseline_report.txt")"- ./scripts/finalize-report.sh "${BASELINE_REPORT}"+ if [[ -n "${BASELINE_REPORT}" && -f "${BASELINE_REPORT}" ]]; then+ echo ""+ echo "Baseline complete:"+ jq -r '" steps: \(.totals.steps_answered // .totals.steps_passed // 0)/87 verified: \(.totals.steps_verified_passed // 0)/87 elapsed: \(.totals.elapsed_ms // 0)ms"' "${BASELINE_REPORT}"+ echo " report: ${BASELINE_REPORT}"+ fi;;-h|--help|help)usagediff --git a/skills/pinchtab-opt/SKILL.md b/skills/pinchtab-opt/SKILL.mdnew file mode 100644index 00000000..f0bec351--- /dev/null+++ b/skills/pinchtab-opt/SKILL.md@@ -0,0 +1,125 @@+---+name: pinchtab-opt+description: "Run the PinchTab optimization loop. Spawns blind subagents that execute 87 browser automation steps across 39 groups using only the PinchTab skill, then reports pass/fail results and operation counts vs baseline. Use when asked to 'run optimization', 'run the opt loop', 'benchmark the agent', or 'test pinchtab agent'."+---++# PinchTab Optimization Loop++Run blind subagents against 87 browser automation steps (39 groups) to measure how well an AI agent can drive PinchTab without hand-held selectors.++## Path Resolution++All paths below are relative to the **project root** (git root). Resolve it first:++```bash+PROJECT_ROOT=$(git rev-parse --show-toplevel)+TOOLS_DIR="$PROJECT_ROOT/tests/tools"+```++The subagents must run with `$TOOLS_DIR` as their working directory because `./scripts/pt` and `./scripts/runner` live there.++## Prerequisites++Docker services must be running. Verify before spawning agents:++```bash+$TOOLS_DIR/scripts/pt health+```++If unhealthy, start the services:++```bash+docker compose -f "$TOOLS_DIR/docker-compose.yml" up -d --build+```++Wait a few seconds and re-check health.++## Execution++### 0. Create per-agent report files++Before spawning agents, create isolated report files so concurrent writes don't corrupt a shared file:++```bash+RESULTS_DIR="$TOOLS_DIR/../benchmark/results"+TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)+mkdir -p "$RESULTS_DIR"++for agent in A B C; do+ cat > "$RESULTS_DIR/agent${agent}_${TIMESTAMP}.json" <<SEED+{+ "benchmark": {"type": "pinchtab", "timestamp": "${TIMESTAMP}", "agent": "${agent}"},+ "totals": {"steps_answered": 0},+ "steps": []+}+SEED+done+```++Save the three file paths — you'll pass one to each subagent.++### 1. Spawn 3 parallel subagents++Use the **Agent** tool with `run_in_background: true`. Split the 39 groups into three batches:++- **Batch A**: groups 0-12 (39 steps)+- **Batch B**: groups 13-25 (26 steps)+- **Batch C**: groups 26-38 (22 steps)++Each subagent gets the **same prompt template** — only the group range and `{REPORT_FILE}` change. Replace `{START}`, `{END}`, `{START_PAD}`, `{END_PAD}`, `{PROJECT_ROOT}`, and `{REPORT_FILE}` with actual values:++```+You are running PinchTab optimization tasks. Your job is to execute groups {START} through {END}.++CRITICAL: Your working directory MUST be {PROJECT_ROOT}/tests/tools for all commands. Prefix every shell command with `cd {PROJECT_ROOT}/tests/tools && `.++Your report file is: {REPORT_FILE}+Use `--report-file {REPORT_FILE}` on every `./scripts/runner step-end` call.++Start by reading these files to understand your tools and tasks:+1. Read `{PROJECT_ROOT}/tests/optimization/subagent-context.md` — environment, wrapper, and recording format.+2. Read `{PROJECT_ROOT}/skills/pinchtab/SKILL.md` — full PinchTab command reference.+3. Read each group file from `{PROJECT_ROOT}/tests/optimization/group-{START_PAD}.md` through `{PROJECT_ROOT}/tests/optimization/group-{END_PAD}.md`.++DO NOT read `{PROJECT_ROOT}/tests/tools/scripts/baseline.sh` or any file under `{PROJECT_ROOT}/tests/benchmark/`.++After reading the above files, execute each step in each group sequentially:+- Always cd to {PROJECT_ROOT}/tests/tools before running commands.+- Use `./scripts/pt` as the wrapper for all PinchTab commands.+- After each step, record the result with `./scripts/runner step-end --report-file {REPORT_FILE} <group> <step> answer "<observation>" pass "notes"` (or fail if it didn't work).+- Use your judgment to figure out the right PinchTab commands from the skill doc. The group files describe WHAT to do, not HOW.++Work through every step in groups {START}-{END}. Do not skip any.+```++### 2. Monitor progress++While agents run, periodically count step-end recordings in each agent's output file:++```bash+grep -c "step-end" <output_file>+```++Expected totals: Batch A ~39, Batch B ~26, Batch C ~22 = 87 total.++### 3. Collect and summarize++Once all 3 agents complete, use `./scripts/runner` subcommands to merge reports (`merge-reports`), inject token usage from subagent transcripts (`inject-usage`), and print the final comparison table (`opt summarize`). Present the summarize output to the user as-is.++## Reference Numbers++- **Baseline**: 87/87 steps, 246 browser ops, 2.8 ops/step+- **Expected agent range**: 350-500 browser ops, 4-6 ops/step (agent must explore pages before acting)+- **Group count**: 39 groups, 87 total steps++## File Locations (relative to project root)++| Path | Purpose |+|------|---------|+| `tests/optimization/subagent-context.md` | Subagent instructions (env, wrapper, recording) |+| `tests/optimization/index.md` | Group listing |+| `tests/optimization/group-00.md` .. `group-38.md` | Task descriptions |+| `skills/pinchtab/SKILL.md` | PinchTab command reference (read by subagent) |+| `tests/tools/scripts/pt` | PinchTab wrapper (CWD must be `tests/tools`) |+| `tests/tools/scripts/runner` | Step recorder (CWD must be `tests/tools`) |+| `tests/tools/scripts/baseline.sh` | Baseline (subagent must NOT read this) |diff --git a/skills/pinchtab/SKILL.md b/skills/pinchtab/SKILL.mdindex 764cec83..c54b10ad 100644--- a/skills/pinchtab/SKILL.md+++ b/skills/pinchtab/SKILL.md@@ -100,6 +100,8 @@ pinchtab instance navigate <instance-id> <url>Tab state is automatic: `nav` persists the tab ID to a state file, and subsequent commands read it. Just run `pinchtab nav URL` then `pinchtab snap -i -c` — the tab is remembered. For explicit control, use `--tab <id>`.+**Parallel agents**: when multiple agents share the same browser instance, the automatic tab state file becomes a race condition — one agent's `nav` overwrites another's tab ID. Each agent must manage its own tab ID explicitly: open with `nav <url> --new-tab`, capture the tab ID from the output, and pass `--tab <id>` on every subsequent command (`snap`, `click`, `fill`, `text`, `eval`, `press`, `scroll`, `frame`, etc.).+### Observation```bashdiff --git a/tests/optimization/subagent-context.md b/tests/optimization/subagent-context.mdindex 7ce4e986..064b0918 100644--- a/tests/optimization/subagent-context.md+++ b/tests/optimization/subagent-context.md@@ -25,25 +25,49 @@ Always use `./scripts/pt ...` — never call `pinchtab` directly.The wrapper executes pinchtab inside the Docker service and forwards `PINCHTAB_TOKEN` and `PINCHTAB_SERVER` automatically.-Tab state is automatic — `nav` persists the tab ID to a state file, and subsequent commands read it. No need to manage tab IDs manually.+### Tab isolation (critical for parallel agents)++Multiple subagents run in parallel against the same browser instance. The default tab state file is shared, so if you rely on it another agent's `nav` will overwrite your tab ID and your subsequent commands will hit the wrong tab or fail with "tab not found".++**You must manage tab IDs explicitly:**++1. On your first `nav`, use `--new-tab` to get a dedicated tab. Capture the tab ID from the output.+2. Pass `--tab <your-tab-id>` on **every** subsequent command (`snap`, `click`, `fill`, `text`, `eval`, `press`, `scroll`, `frame`, etc.).+3. Never rely on the automatic state file — always be explicit.++Example:+```bash+# First navigation — open a new tab and capture the ID+./scripts/pt nav "http://fixtures/wiki.html" --new-tab --snap+# Output includes tab ID, e.g. A1B2C3D4...++# All subsequent commands use --tab explicitly+./scripts/pt click e3 --tab A1B2C3D4... --snap-diff+./scripts/pt text --tab A1B2C3D4...+```++If you lose your tab (404 error), re-navigate with `--new-tab` and update your tab ID.## Recording+Each agent writes to its own report file to avoid concurrent-write corruption. Your report file path will be provided as `REPORT_FILE` when you are launched. Pass it on every `step-end` call with `--report-file`.+Record every step result immediately after completion:```bash-./scripts/runner step-end <group> <step> answer "<what you observed>" <pass|fail|skip> "verification notes"+./scripts/runner step-end --report-file "$REPORT_FILE" <group> <step> answer "<what you observed>" <pass|fail|skip> "verification notes"```For failures:```bash-./scripts/runner step-end <group> <step> fail "<what went wrong>" skip ""+./scripts/runner step-end --report-file "$REPORT_FILE" <group> <step> fail "<what went wrong>" skip ""```- `<group>` is the group number (e.g., `0`, `15`, `38`)- `<step>` is the step number within the group (e.g., `1`, `2`, `3`)- Keep answers factual — do not self-grade in the answer payload.+- **Quote actual output, don't paraphrase.** The answer field must include the literal text or marker from the tool output. For example, if the server returns `status: ok`, write `status: ok` in the answer — not "Server responded with ok". Verification patterns match against exact substrings, so paraphrasing causes false failures.## Execution approachdiff --git a/tests/tools/runner/internal/bench/finalize.go b/tests/tools/runner/internal/bench/finalize.gonew file mode 100644index 00000000..52bdf827--- /dev/null+++ b/tests/tools/runner/internal/bench/finalize.go@@ -0,0 +1,186 @@+package bench++import (+ "encoding/json"+ "fmt"+ "io"+ "os"+ "strings"+)++func runFinalize(reportFile string, stdout, stderr io.Writer) {+ data, err := os.ReadFile(reportFile)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "[finalize] cannot read %s: %v\n", reportFile, err)+ return+ }+ var report map[string]any+ if err := json.Unmarshal(data, &report); err != nil {+ _, _ = fmt.Fprintf(stderr, "[finalize] cannot parse %s: %v\n", reportFile, err)+ return+ }++ totals, _ := report["totals"].(map[string]any)+ getF := func(key string) float64 {+ if v, ok := totals[key].(float64); ok {+ return v+ }+ return 0+ }++ usage, _ := report["run_usage"].(map[string]any)+ getU := func(key string) float64 {+ if usage == nil {+ return 0+ }+ if v, ok := usage[key].(float64); ok {+ return v+ }+ return 0+ }++ bench, _ := report["benchmark"].(map[string]any)+ benchType, _ := bench["type"].(string)+ model, _ := bench["model"].(string)++ answered := getF("steps_answered")+ execFailed := getF("steps_failed")+ execSkipped := getF("steps_skipped")+ legacyPassed := getF("steps_passed")+ verifiedPassed := getF("steps_verified_passed")+ verifiedFailed := getF("steps_verified_failed")+ verifiedSkipped := getF("steps_verified_skipped")+ pending := getF("steps_pending_verification")+ toolCalls := getF("tool_calls")++ isLegacyBaseline := benchType == "baseline" && answered == 0+ legacyTotal := legacyPassed + execFailed + execSkipped+ execTotal := answered + execFailed + execSkipped++ pct := func(a, b float64) string {+ if b == 0 {+ return "0.0%"+ }+ return fmt.Sprintf("%.1f%%", (a/b)*100)+ }++ var sb strings.Builder+ sb.WriteString("# Benchmark Summary\n\n")+ sb.WriteString("| Metric | Value |\n")+ sb.WriteString("|--------|-------|\n")+ fmt.Fprintf(&sb, "| Type | %s |\n", benchType)+ fmt.Fprintf(&sb, "| Model | %s |\n", model)++ if isLegacyBaseline {+ fmt.Fprintf(&sb, "| Steps Passed | %.0f |\n", legacyPassed)+ fmt.Fprintf(&sb, "| Steps Failed | %.0f |\n", execFailed)+ fmt.Fprintf(&sb, "| Steps Skipped | %.0f |\n", execSkipped)+ fmt.Fprintf(&sb, "| Pass Rate | %s |\n", pct(legacyPassed, legacyTotal))+ fmt.Fprintf(&sb, "| Tool Calls | %.0f |\n", toolCalls)+ } else {+ fmt.Fprintf(&sb, "| Steps Answered | %.0f |\n", answered)+ fmt.Fprintf(&sb, "| Execution Failed | %.0f |\n", execFailed)+ fmt.Fprintf(&sb, "| Execution Skipped | %.0f |\n", execSkipped)+ fmt.Fprintf(&sb, "| Answer Rate | %s |\n", pct(answered, execTotal))+ fmt.Fprintf(&sb, "| Verified Passed | %.0f |\n", verifiedPassed)+ fmt.Fprintf(&sb, "| Verified Failed | %.0f |\n", verifiedFailed)+ fmt.Fprintf(&sb, "| Verified Skipped | %.0f |\n", verifiedSkipped)+ fmt.Fprintf(&sb, "| Pending Verification | %.0f |\n", pending)+ fmt.Fprintf(&sb, "| Verification Pass Rate | %s |\n", pct(verifiedPassed, execTotal))+ fmt.Fprintf(&sb, "| Tool Calls | %.0f |\n", toolCalls)+ }++ sb.WriteString("\n## Run Usage\n\n")+ if usage == nil || (getU("total_tokens") == 0 && getU("request_count") == 0) {+ sb.WriteString("- none recorded\n")+ } else {+ sb.WriteString("| Metric | Value |\n")+ sb.WriteString("|--------|-------|\n")+ source, _ := usage["source"].(string)+ provider, _ := usage["provider"].(string)+ if source == "" {+ source = "unknown"+ }+ if provider == "" {+ provider = "unknown"+ }+ fmt.Fprintf(&sb, "| Source | %s |\n", source)+ fmt.Fprintf(&sb, "| Provider | %s |\n", provider)+ fmt.Fprintf(&sb, "| API Requests | %.0f |\n", getU("request_count"))+ fmt.Fprintf(&sb, "| Input Tokens (uncached) | %.0f |\n", getU("input_tokens"))+ fmt.Fprintf(&sb, "| Cache Creation Input Tokens | %.0f |\n", getU("cache_creation_input_tokens"))+ fmt.Fprintf(&sb, "| Cache Read Input Tokens | %.0f |\n", getU("cache_read_input_tokens"))+ fmt.Fprintf(&sb, "| Total Input Tokens | %.0f |\n", getU("total_input_tokens"))+ fmt.Fprintf(&sb, "| Output Tokens | %.0f |\n", getU("output_tokens"))+ fmt.Fprintf(&sb, "| Total Tokens | %.0f |\n", getU("total_tokens"))+ }++ steps, _ := report["steps"].([]any)++ sb.WriteString("\n## Pending Verification\n\n")+ var hasPending bool+ for _, s := range steps {+ step, _ := s.(map[string]any)+ status, _ := step["status"].(string)+ ver, _ := step["verification"].(map[string]any)+ verStatus, _ := ver["status"].(string)+ if verStatus == "" {+ verStatus = "pending"+ }+ if status == "answer" && verStatus == "pending" {+ id, _ := step["id"].(string)+ answer, _ := step["answer"].(string)+ if answer == "" {+ answer, _ = step["notes"].(string)+ }+ fmt.Fprintf(&sb, "- %s: %s\n", id, answer)+ hasPending = true+ }+ }+ if !hasPending {+ sb.WriteString("- none\n")+ }++ sb.WriteString("\n## Failed Steps\n\n")+ var hasFailed bool+ for _, s := range steps {+ step, _ := s.(map[string]any)+ status, _ := step["status"].(string)+ if status == "fail" {+ id, _ := step["id"].(string)+ notes, _ := step["notes"].(string)+ fmt.Fprintf(&sb, "- %s: %s\n", id, notes)+ hasFailed = true+ }+ }+ if !hasFailed {+ sb.WriteString("- none\n")+ }++ sb.WriteString("\n## Verification Failures\n\n")+ var hasVerFail bool+ for _, s := range steps {+ step, _ := s.(map[string]any)+ status, _ := step["status"].(string)+ ver, _ := step["verification"].(map[string]any)+ verStatus, _ := ver["status"].(string)+ if status == "answer" && verStatus == "fail" {+ id, _ := step["id"].(string)+ notes, _ := ver["notes"].(string)+ fmt.Fprintf(&sb, "- %s: %s\n", id, notes)+ hasVerFail = true+ }+ }+ if !hasVerFail {+ sb.WriteString("- none\n")+ }++ summaryFile := strings.TrimSuffix(reportFile, ".json") + "_summary.md"+ if err := os.WriteFile(summaryFile, []byte(sb.String()), 0644); err != nil {+ _, _ = fmt.Fprintf(stderr, "[finalize] cannot write %s: %v\n", summaryFile, err)+ return+ }++ _, _ = fmt.Fprintf(stdout, "Wrote %s\n\n", summaryFile)+ _, _ = fmt.Fprint(stdout, sb.String())+}diff --git a/tests/tools/runner/internal/bench/loop.go b/tests/tools/runner/internal/bench/loop.goindex f8797641..aeea7e32 100644--- a/tests/tools/runner/internal/bench/loop.go+++ b/tests/tools/runner/internal/bench/loop.go@@ -151,10 +151,7 @@ func RunLoop(cfg LoopConfig, runner Runner, shell *PersistentShell) LoopResult {if _, err := os.Stat(cfg.ReportFile); err == nil {recordUsage(cfg.ToolsDir, cfg.ReportFile, runner)if cfg.Finalize {- // Stream finalize-report.sh's output through our stdout/stderr so- // the generated markdown summary actually reaches the terminal.- // (The old finalizeReport() used cmd.Run() which discards output.)- runFinalizeReport(cfg.ToolsDir, cfg.ReportFile, cfg.Stdout, cfg.Stderr)+ runFinalize(cfg.ReportFile, cfg.Stdout, cfg.Stderr)}}@@ -168,9 +165,8 @@ func RunLoop(cfg LoopConfig, runner Runner, shell *PersistentShell) LoopResult {}out.Summary(runner.Usage(), runner.Provider())- // Harness-generated end-of-run summary: runs regardless of whether- // finalize-report.sh succeeded, reads the JSON report directly so it- // always reflects recorded state.+ // Harness-generated end-of-run summary: reads the JSON report directly+ // so it always reflects recorded state.if _, err := os.Stat(cfg.ReportFile); err == nil {PrintEndBanner(cfg.Stdout, cfg.ReportFile)}diff --git a/tests/tools/runner/internal/bench/summary.go b/tests/tools/runner/internal/bench/summary.goindex b6b93070..c3793c2a 100644--- a/tests/tools/runner/internal/bench/summary.go+++ b/tests/tools/runner/internal/bench/summary.go@@ -6,7 +6,6 @@ import ("fmt""io""os"- "os/exec""path/filepath""regexp""sort"@@ -142,9 +141,7 @@ type reportSummary struct {} `json:"steps"`}-// PrintEndBanner prints a compact pass/fail table at end-of-run — independent-// of finalize-report.sh, so the user always sees it even if finalize is-// skipped or its markdown summary is piped elsewhere.+// PrintEndBanner prints a compact pass/fail table at end-of-run.func PrintEndBanner(w io.Writer, reportFile string) {data, err := os.ReadFile(reportFile)if err != nil {@@ -207,15 +204,3 @@ func trimOneLine(s string, max int) string {}return s}--// runFinalizeReport invokes finalize-report.sh and streams its stdout/stderr-// through the provided writers. Previously loop.go::finalizeReport used-// cmd.Run() which silently discarded the summary output — this is the fix.-func runFinalizeReport(toolsDir, reportFile string, stdout, stderr io.Writer) {- script := filepath.Join(toolsDir, "scripts", "finalize-report.sh")- cmd := exec.Command(script, reportFile) // #nosec G204 -- script path is constructed from known toolsDir- cmd.Dir = toolsDir- cmd.Stdout = stdout- cmd.Stderr = stderr- _ = cmd.Run()-}diff --git a/tests/tools/runner/internal/opt/injectusage.go b/tests/tools/runner/internal/opt/injectusage.gonew file mode 100644index 00000000..564accfd--- /dev/null+++ b/tests/tools/runner/internal/opt/injectusage.go@@ -0,0 +1,155 @@+package opt++import (+ "bufio"+ "encoding/json"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "strings"+)++func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {+ var reportFile string+ var transcriptFiles []string++ i := 0+ for i < len(argv) && strings.HasPrefix(argv[i], "-") {+ switch argv[i] {+ case "--report", "-r":+ if i+1 >= len(argv) {+ _, _ = fmt.Fprintf(stderr, "inject-usage: %s requires a value\n", argv[i])+ return 1+ }+ reportFile = argv[i+1]+ i += 2+ default:+ _, _ = fmt.Fprintf(stderr, "inject-usage: unknown option: %s\n", argv[i])+ return 1+ }+ }++ transcriptFiles = argv[i:]+ if len(transcriptFiles) == 0 {+ _, _ = fmt.Fprintln(stderr, "usage: inject-usage [-r report.json] <transcript1.jsonl> [transcript2.jsonl ...]")+ return 1+ }++ // Expand globs+ var files []string+ for _, pattern := range transcriptFiles {+ matches, err := filepath.Glob(pattern)+ if err != nil || len(matches) == 0 {+ if _, statErr := os.Stat(pattern); statErr == nil {+ files = append(files, pattern)+ } else {+ _, _ = fmt.Fprintf(stderr, "inject-usage: no files matched %q\n", pattern)+ return 1+ }+ } else {+ files = append(files, matches...)+ }+ }++ var totals struct {+ InputTokens int64 `json:"input_tokens"`+ CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`+ CacheReadInputTokens int64 `json:"cache_read_input_tokens"`+ OutputTokens int64 `json:"output_tokens"`+ RequestCount int64 `json:"request_count"`+ }++ for _, f := range files {+ fh, err := os.Open(f)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to open %s: %v\n", f, err)+ return 1+ }++ var fileRequests int64+ scanner := bufio.NewScanner(fh)+ scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)+ for scanner.Scan() {+ var entry struct {+ Message struct {+ Role string `json:"role"`+ Usage struct {+ InputTokens int64 `json:"input_tokens"`+ CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`+ CacheReadInputTokens int64 `json:"cache_read_input_tokens"`+ OutputTokens int64 `json:"output_tokens"`+ } `json:"usage"`+ } `json:"message"`+ }+ if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {+ continue+ }+ u := entry.Message.Usage+ if u.InputTokens == 0 && u.CacheCreationInputTokens == 0 && u.CacheReadInputTokens == 0 && u.OutputTokens == 0 {+ continue+ }+ totals.InputTokens += u.InputTokens+ totals.CacheCreationInputTokens += u.CacheCreationInputTokens+ totals.CacheReadInputTokens += u.CacheReadInputTokens+ totals.OutputTokens += u.OutputTokens+ fileRequests+++ }+ _ = fh.Close()++ totals.RequestCount += fileRequests+ _, _ = fmt.Fprintf(stdout, "Parsed %d API responses from %s\n", fileRequests, filepath.Base(f))+ }++ totalInput := totals.InputTokens + totals.CacheCreationInputTokens + totals.CacheReadInputTokens+ totalTokens := totalInput + totals.OutputTokens++ usage := map[string]any{+ "source": "subagent-transcripts",+ "provider": "anthropic",+ "request_count": totals.RequestCount,+ "input_tokens": totals.InputTokens,+ "cache_creation_input_tokens": totals.CacheCreationInputTokens,+ "cache_read_input_tokens": totals.CacheReadInputTokens,+ "total_input_tokens": totalInput,+ "output_tokens": totals.OutputTokens,+ "total_tokens": totalTokens,+ }++ _, _ = fmt.Fprintf(stdout, "\nToken usage:\n")+ _, _ = fmt.Fprintf(stdout, " API requests: %d\n", totals.RequestCount)+ _, _ = fmt.Fprintf(stdout, " Input (uncached): %d\n", totals.InputTokens)+ _, _ = fmt.Fprintf(stdout, " Cache write: %d\n", totals.CacheCreationInputTokens)+ _, _ = fmt.Fprintf(stdout, " Cache read: %d\n", totals.CacheReadInputTokens)+ _, _ = fmt.Fprintf(stdout, " Total input: %d\n", totalInput)+ _, _ = fmt.Fprintf(stdout, " Output: %d\n", totals.OutputTokens)+ _, _ = fmt.Fprintf(stdout, " Total tokens: %d\n", totalTokens)++ if reportFile != "" {+ data, err := os.ReadFile(reportFile)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to read %s: %v\n", reportFile, err)+ return 1+ }+ var report map[string]any+ if err := json.Unmarshal(data, &report); err != nil {+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to parse %s: %v\n", reportFile, err)+ return 1+ }++ report["run_usage"] = usage++ output, err := json.MarshalIndent(report, "", " ")+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to marshal: %v\n", err)+ return 1+ }+ if err := os.WriteFile(reportFile, output, 0644); err != nil {+ _, _ = fmt.Fprintf(stderr, "inject-usage: failed to write %s: %v\n", reportFile, err)+ return 1+ }+ _, _ = fmt.Fprintf(stdout, "\nInjected run_usage into %s\n", reportFile)+ }++ return 0+}diff --git a/tests/tools/runner/internal/opt/mergereports.go b/tests/tools/runner/internal/opt/mergereports.gonew file mode 100644index 00000000..0c0e5db1--- /dev/null+++ b/tests/tools/runner/internal/opt/mergereports.go@@ -0,0 +1,147 @@+package opt++import (+ "encoding/json"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "sort"+ "strings"+ "time"+)++type MergeReportsArgs struct {+ OutputFile string+ InputFiles []string+}++func ParseMergeReportsArgs(argv []string) (MergeReportsArgs, error) {+ var args MergeReportsArgs++ i := 0+ for i < len(argv) && strings.HasPrefix(argv[i], "-") {+ switch argv[i] {+ case "--output", "-o":+ if i+1 >= len(argv) {+ return args, fmt.Errorf("%s requires a value", argv[i])+ }+ args.OutputFile = argv[i+1]+ i += 2+ default:+ return args, fmt.Errorf("unknown option: %s", argv[i])+ }+ }++ args.InputFiles = argv[i:]+ if len(args.InputFiles) == 0 {+ return args, fmt.Errorf("usage: merge-reports [-o output.json] <report1.json> <report2.json>")+ }++ return args, nil+}++func RunMergeReports(argv []string, stdout, stderr io.Writer) int {+ args, err := ParseMergeReportsArgs(argv)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "merge-reports: %v\n", err)+ return 1+ }++ // Also support glob patterns (e.g. agent*_20260429.json)+ var files []string+ for _, pattern := range args.InputFiles {+ matches, err := filepath.Glob(pattern)+ if err != nil || len(matches) == 0 {+ if _, statErr := os.Stat(pattern); statErr == nil {+ files = append(files, pattern)+ } else {+ _, _ = fmt.Fprintf(stderr, "merge-reports: no files matched %q\n", pattern)+ return 1+ }+ } else {+ files = append(files, matches...)+ }+ }++ var allSteps []map[string]any+ seen := make(map[string]bool)++ for _, f := range files {+ data, err := os.ReadFile(f)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to read %s: %v\n", f, err)+ return 1+ }+ var report map[string]any+ if err := json.Unmarshal(data, &report); err != nil {+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to parse %s: %v\n", f, err)+ return 1+ }++ steps, _ := report["steps"].([]any)+ for _, s := range steps {+ step, ok := s.(map[string]any)+ if !ok {+ continue+ }+ id, _ := step["id"].(string)+ if id == "" {+ g, _ := step["group"].(float64)+ s, _ := step["step"].(float64)+ id = fmt.Sprintf("%d.%d", int(g), int(s))+ }+ if !seen[id] {+ seen[id] = true+ allSteps = append(allSteps, step)+ }+ }+ _, _ = fmt.Fprintf(stdout, "Loaded %d steps from %s\n", len(steps), filepath.Base(f))+ }++ // Sort by group.step+ sort.Slice(allSteps, func(i, j int) bool {+ gi, _ := allSteps[i]["group"].(float64)+ si, _ := allSteps[i]["step"].(float64)+ gj, _ := allSteps[j]["group"].(float64)+ sj, _ := allSteps[j]["step"].(float64)+ if gi != gj {+ return gi < gj+ }+ return si < sj+ })++ // Build merged report+ merged := map[string]any{+ "benchmark": map[string]any{+ "type": "pinchtab",+ "timestamp": time.Now().UTC().Format("20060102_150405"),+ "merged": true,+ "sources": len(files),+ },+ "totals": map[string]any{+ "steps_answered": len(allSteps),+ },+ "steps": allSteps,+ }++ output, err := json.MarshalIndent(merged, "", " ")+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to marshal: %v\n", err)+ return 1+ }++ if args.OutputFile != "" {+ if err := os.WriteFile(args.OutputFile, output, 0644); err != nil {+ _, _ = fmt.Fprintf(stderr, "merge-reports: failed to write %s: %v\n", args.OutputFile, err)+ return 1+ }+ _, _ = fmt.Fprintf(stdout, "Merged %d unique steps into %s\n", len(allSteps), args.OutputFile)+ } else {+ _, _ = fmt.Fprintf(stdout, "Merged %d unique steps (use -o to write to file)\n", len(allSteps))+ _, _ = stdout.Write(output)+ _, _ = fmt.Fprintln(stdout)+ }++ return 0+}diff --git a/tests/tools/runner/internal/opt/opt.go b/tests/tools/runner/internal/opt/opt.gonew file mode 100644index 00000000..c78aadc9--- /dev/null+++ b/tests/tools/runner/internal/opt/opt.go@@ -0,0 +1,28 @@+package opt++import (+ "fmt"+ "io"+)++func Run(argv []string, stdout, stderr io.Writer) int {+ if len(argv) == 0 {+ _, _ = fmt.Fprintln(stderr, "usage: runner opt <merge-reports|verify-answers> [args...]")+ return 1+ }++ switch argv[0] {+ case "merge-reports":+ return RunMergeReports(argv[1:], stdout, stderr)+ case "verify-answers":+ return RunVerifyAnswers(argv[1:], stdout, stderr)+ case "inject-usage":+ return RunInjectUsage(argv[1:], stdout, stderr)+ case "summarize":+ return RunSummarize(argv[1:], stdout, stderr)+ default:+ _, _ = fmt.Fprintf(stderr, "runner opt: unknown subcommand %q\n", argv[0])+ _, _ = fmt.Fprintln(stderr, "Use one of: merge-reports, verify-answers, inject-usage, summarize")+ return 1+ }+}diff --git a/tests/tools/runner/internal/opt/summarize.go b/tests/tools/runner/internal/opt/summarize.gonew file mode 100644index 00000000..dc2d69a3--- /dev/null+++ b/tests/tools/runner/internal/opt/summarize.go@@ -0,0 +1,295 @@+package opt++import (+ "bufio"+ "encoding/json"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "regexp"+ "sort"+ "strings"+)++func RunSummarize(argv []string, stdout, stderr io.Writer) int {+ var reportFile string+ var transcriptFiles []string++ i := 0+ for i < len(argv) && strings.HasPrefix(argv[i], "-") {+ switch argv[i] {+ case "--report", "-r":+ if i+1 >= len(argv) {+ _, _ = fmt.Fprintf(stderr, "summarize: %s requires a value\n", argv[i])+ return 1+ }+ reportFile = argv[i+1]+ i += 2+ default:+ _, _ = fmt.Fprintf(stderr, "summarize: unknown option: %s\n", argv[i])+ return 1+ }+ }+ transcriptFiles = argv[i:]++ if reportFile == "" {+ _, _ = fmt.Fprintln(stderr, "usage: summarize -r <merged-report.json> [transcript1.jsonl ...]")+ return 1+ }++ data, err := os.ReadFile(reportFile)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "summarize: %v\n", err)+ return 1+ }+ var report map[string]any+ if err := json.Unmarshal(data, &report); err != nil {+ _, _ = fmt.Fprintf(stderr, "summarize: %v\n", err)+ return 1+ }++ steps, _ := report["steps"].([]any)+ var answered, failed, skipped int+ seen := make(map[string]string)+ for _, s := range steps {+ step, _ := s.(map[string]any)+ status, _ := step["status"].(string)+ switch status {+ case "answer":+ answered+++ case "fail":+ failed+++ case "skip":+ skipped+++ }+ id, _ := step["id"].(string)+ if id == "" {+ g, _ := step["group"].(float64)+ st, _ := step["step"].(float64)+ id = fmt.Sprintf("%d.%d", int(g), int(st))+ }+ answer, _ := step["answer"].(string)+ seen[id] = answer+ }+ totalSteps := answered + failed + skipped++ var verifyPass, verifyFail int+ var verifyFailures []string+ for id, answer := range seen {+ pattern, has := expectPatterns[id]+ if !has {+ continue+ }+ re, err := regexp.Compile("(?i)" + pattern)+ if err != nil || !re.MatchString(answer) {+ verifyFail+++ truncated := answer+ if len(truncated) > 80 {+ truncated = truncated[:77] + "..."+ }+ verifyFailures = append(verifyFailures, fmt.Sprintf("%s: [%s]", id, truncated))+ } else {+ verifyPass+++ }+ }++ var missing []string+ for g := 0; g <= 38; g++ {+ count := groupSizes[g]+ for s := 1; s <= count; s++ {+ id := fmt.Sprintf("%d.%d", g, s)+ if _, ok := seen[id]; !ok {+ missing = append(missing, id)+ }+ }+ }++ // Parse browser ops from transcripts+ ptRe := regexp.MustCompile(`\./scripts/pt\s+(\w+)`)+ var totalOps int+ cmdTypes := make(map[string]int)+ var hasTranscripts bool++ for _, f := range expandGlobs(transcriptFiles, stderr) {+ fh, err := os.Open(f)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "summarize: warning: cannot open %s: %v\n", f, err)+ continue+ }+ scanner := bufio.NewScanner(fh)+ scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)+ for scanner.Scan() {+ var entry struct {+ Message struct {+ Content json.RawMessage `json:"content"`+ } `json:"message"`+ }+ if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {+ continue+ }+ var blocks []struct {+ Type string `json:"type"`+ Name string `json:"name"`+ Input struct {+ Command string `json:"command"`+ } `json:"input"`+ }+ if err := json.Unmarshal(entry.Message.Content, &blocks); err != nil {+ continue+ }+ for _, b := range blocks {+ if b.Type == "tool_use" && b.Name == "Bash" {+ for _, m := range ptRe.FindAllStringSubmatch(b.Input.Command, -1) {+ totalOps+++ cmdTypes[m[1]]+++ }+ }+ }+ }+ _ = fh.Close()+ hasTranscripts = true+ }++ usage, _ := report["run_usage"].(map[string]any)+ getInt := func(key string) int64 {+ if v, ok := usage[key].(float64); ok {+ return int64(v)+ }+ return 0+ }++ // Build rows for a single combined table+ type row struct {+ metric, baseline, agent string+ }+ rows := []row{+ {"Steps completed", "87/87", fmt.Sprintf("%d/87", totalSteps)},+ {"Verified pass", "87/87", fmt.Sprintf("%d/87", verifyPass)},+ }+ if hasTranscripts {+ rows = append(rows,+ row{"Browser ops", "246", fmtInt(int64(totalOps))},+ row{"Ops/step", "2.8", fmt.Sprintf("%.1f", float64(totalOps)/87.0)},+ row{"Ratio", "1.0x", fmt.Sprintf("%.2fx", float64(totalOps)/246.0)},+ )+ }+ rows = append(rows, row{"Errors", "0", fmt.Sprintf("%d", failed)})++ if usage != nil {+ rows = append(rows,+ row{"", "", ""},+ row{"API requests", "", fmtInt(getInt("request_count"))},+ row{"Input (uncached)", "", fmtInt(getInt("input_tokens"))},+ row{"Cache write", "", fmtInt(getInt("cache_creation_input_tokens"))},+ row{"Cache read", "", fmtInt(getInt("cache_read_input_tokens"))},+ row{"Total input", "", fmtInt(getInt("total_input_tokens"))},+ row{"Output", "", fmtInt(getInt("output_tokens"))},+ row{"Total tokens", "", fmtInt(getInt("total_tokens"))},+ )+ }++ // Compute column widths+ colW := [3]int{len("Metric"), len("Baseline"), len("Agent")}+ for _, r := range rows {+ if r.metric == "" {+ continue+ }+ if len(r.metric) > colW[0] {+ colW[0] = len(r.metric)+ }+ if len(r.baseline) > colW[1] {+ colW[1] = len(r.baseline)+ }+ if len(r.agent) > colW[2] {+ colW[2] = len(r.agent)+ }+ }++ hLine := fmt.Sprintf("+-%-s-+-%-s-+-%-s-+", strings.Repeat("-", colW[0]), strings.Repeat("-", colW[1]), strings.Repeat("-", colW[2]))+ fmtRow := func(a, b, c string) string {+ return fmt.Sprintf("| %-*s | %-*s | %-*s |", colW[0], a, colW[1], b, colW[2], c)+ }++ _, _ = fmt.Fprintln(stdout)+ _, _ = fmt.Fprintln(stdout, hLine)+ _, _ = fmt.Fprintln(stdout, fmtRow("Metric", "Baseline", "Agent"))+ _, _ = fmt.Fprintln(stdout, hLine)+ for _, r := range rows {+ if r.metric == "" {+ _, _ = fmt.Fprintln(stdout, hLine)+ continue+ }+ _, _ = fmt.Fprintln(stdout, fmtRow(r.metric, r.baseline, r.agent))+ }+ _, _ = fmt.Fprintln(stdout, hLine)++ // Ops breakdown (compact, one line)+ if hasTranscripts {+ type kv struct {+ k string+ v int+ }+ var sortedOps []kv+ for k, v := range cmdTypes {+ sortedOps = append(sortedOps, kv{k, v})+ }+ sort.Slice(sortedOps, func(i, j int) bool { return sortedOps[i].v > sortedOps[j].v })+ _, _ = fmt.Fprintln(stdout)+ _, _ = fmt.Fprint(stdout, "Ops: ")+ for i, s := range sortedOps {+ if i > 0 {+ _, _ = fmt.Fprint(stdout, ", ")+ }+ _, _ = fmt.Fprintf(stdout, "%s %d", s.k, s.v)+ }+ _, _ = fmt.Fprintln(stdout)+ }++ if len(missing) > 0 {+ _, _ = fmt.Fprintf(stdout, "\nMissing: %s\n", strings.Join(missing, ", "))+ }++ if len(verifyFailures) > 0 {+ _, _ = fmt.Fprintln(stdout, "\nVerification failures:")+ for _, f := range verifyFailures {+ _, _ = fmt.Fprintf(stdout, " %s\n", f)+ }+ }++ _, _ = fmt.Fprintln(stdout)+ return 0+}++func fmtInt(n int64) string {+ if n == 0 {+ return "0"+ }+ s := fmt.Sprintf("%d", n)+ if n < 0 {+ return s+ }+ var b strings.Builder+ for i, c := range s {+ if i > 0 && (len(s)-i)%3 == 0 {+ b.WriteByte(',')+ }+ b.WriteRune(c)+ }+ return b.String()+}++func expandGlobs(patterns []string, stderr io.Writer) []string {+ var out []string+ for _, p := range patterns {+ matches, err := filepath.Glob(p)+ if err != nil || len(matches) == 0 {+ if _, statErr := os.Stat(p); statErr == nil {+ out = append(out, p)+ }+ } else {+ out = append(out, matches...)+ }+ }+ return out+}diff --git a/tests/tools/runner/internal/opt/verifyanswers.go b/tests/tools/runner/internal/opt/verifyanswers.gonew file mode 100644index 00000000..5195ef38--- /dev/null+++ b/tests/tools/runner/internal/opt/verifyanswers.go@@ -0,0 +1,330 @@+package opt++import (+ "encoding/json"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "regexp"+ "sort"+ "strings"+)++// expectPatterns maps step ID → regex pattern for answer verification.+// Ported from verify-answers.sh — patterns are case-insensitive.+var expectPatterns = map[string]string{+ // Group 0: Setup & diagnosis+ "0.1": `status.*ok|status:.*ok`,+ "0.2": `401|missing_token|unauth`,+ "0.3": `200|authed.*200`,+ "0.4": `running`,+ "0.5": `tab.*list|tabs.*returned|active tab|listed.*tab|\d+ tab.*found|tabs:`,+ "0.6": `cleaned|closed|no stale|reused.*tab|no cleanup|single active|is empty|already empty|clean.*state|cannot close last|one tab`,+ "0.7": `VERIFY_HOME_LOADED_12345|navigated to fixtures|fixtures.*home|PinchTab Benchmark|fixtures/`,+ "0.8": `[0-9A-F]{32}|tab.*id.*captured`,+ // Group 1: Reading & Extracting+ "1.1": `COUNT_LANGUAGES_12|Programming Languages 12|12 articles`,+ "1.2": `wiki-go|Go \(programming language\)|409 success|clicked`,+ "1.3": `Robert Griesemer.*2009|2009.*Griesemer`,+ "1.4": `FEATURE_COUNT_6|6 key features|6 features`,+ "1.5": `Artificial Intelligence.*Climate Action.*Mars|Artificial Intelligence, Climate Action`,+ "1.6": `24,582.*1,284,930|24,582|Revenue \$1,284,930`,+ // Group 2: Search+ "2.1": `wiki-go|VERIFY_WIKI_GO_LANG_88888`,+ "2.2": `No results|no results|xyznonexistent`,+ "2.3": `Artificial Intelligence|ARTIFICIAL_INTELLIGENCE`,+ // Group 3: Form+ "3.1": `submitted|VERIFY_FORM_SUBMITTED_SUCCESS|FORM_SUBMITTED|SUBMISSION_DATA`,+ "3.2": `[Rr]eset.*button|#reset-btn|[Rr]eset.*present|reset-btn`,+ // Group 4: SPA+ "4.1": `TASK_STATS_TOTAL_3|Total.*3.*Active.*2.*Done.*1|3.*2.*1`,+ "4.2": `TASK_ADDED|AUTOMATE|DEPLOYMENT|high`,+ "4.3": `deleted.*task|TASK_STATS_TOTAL_3|All Tasks.*3|3.*tasks|4 to 3|tests deleted`,+ // Group 5: Login+ "5.1": `INVALID_CREDENTIALS_ERROR|Invalid`,+ "5.2": `VERIFY_LOGIN_SUCCESS_DASHBOARD|SESSION_TOKEN_ACTIVE_TRUE|login success|Dashboard`,+ // Group 6: E-commerce+ "6.1": `149.99.*299.99|Wireless.*Smart Watch|Portable Charger`,+ "6.2": `449.98|CART_ITEM_WIRELESS`,+ "6.3": `VERIFY_CHECKOUT_SUCCESS_ORDER|checkout`,+ // Group 7: Content + Interaction+ "7.1": `COMMENT_POSTED_RATING_5`,+ "7.2": `Developer Tools.*15|15.*Developer|wiki-go|VERIFY_WIKI_GO`,+ // Group 8: Error handling+ "8.1": `404|not found`,+ "8.2": `element.*not found|no element found|no element|clear error`,+ // Group 9: Export+ "9.1": `[Ss]creenshot|\.png.*[0-9]{4,} bytes`,+ "9.2": `[Pp][Dd][Ff]|\.pdf.*[0-9]{4,} bytes`,+ // Group 10: Modals+ "10.1": `Dashboard Settings`,+ "10.2": `THEME_DARK_APPLIED`,+ // Group 11: Persistence+ "11.1": `TASK_PERSISTENT_TEST_FOUND`,+ "11.2": `SESSION_RENEWED|VERIFY_LOGIN_SUCCESS_DASHBOARD`,+ // Group 12: Multi-page nav+ "12.1": `home|VERIFY_HOME_LOADED_12345|PinchTab Benchmark - Home`,+ "12.2": `COUNT_LANGUAGES_12|12.*Programming|Artificial Intelligence|Total Articles|12\+8\+15|COMPARISON_DATA_FOUND`,+ // Group 13: Form validation+ "13.1": `blocked|display:none|valueMissing|not submitted`,+ "13.2": `VERIFY_FORM_SUBMITTED_SUCCESS|OPTIONAL_FIELD_SKIPPED_SUCCESS|submitted`,+ // Group 14: Dynamic content+ "14.1": `ADDITIONAL_PRODUCTS_LOADED`,+ "14.2": `CART_UPDATED_WITH_LAZY_PRODUCT|USB-C`,+ // Group 15: Data aggregation+ "15.1": `1,284,930.*384,930|PROFIT_MARGIN_CALCULATED|Revenue.*Profit`,+ "15.2": `FEATURE_COUNT_6.*7.*5|Go=6.*Python=7.*Rust=5|6, 7, 5|COMPARISON_TABLE_BUILT`,+ "15.3": `VERIFY_ARTICLE_PAGE_41414`,+ "15.4": `flex`,+ // Group 16: Hover+ "16.1": `HOVER_REVEALED_USER_1`,+ "16.2": `HOVER_REVEALED_USER_2`,+ // Group 17: Scroll+ "17.1": `SCROLL_MIDDLE_MARKER`,+ "17.2": `SCROLL_REACHED_FOOTER`,+ // Group 18: Download+ "18.1": `DOWNLOAD_FILE_CONTENT_VERIFIED|143 bytes`,+ // Group 19: iFrame+ "19.1": `IFRAME_INNER_CONTENT_LOADED`,+ "19.2": `IFRAME_INPUT_RECEIVED_HELLO_WORLD`,+ // Group 20: Dialogs+ "20.1": `DIALOG_ALERT_DISMISSED`,+ "20.2": `DIALOG_CONFIRM_CANCELLED`,+ // Group 21: Async+ "21.1": `ASYNC_PAYLOAD_READY_42`,+ "21.2": `ASYNC_USER_NAME_ADA`,+ // Group 22: Drag+ "22.1": `DROP_ZONE_A_OK`,+ "22.2": `DROP_SEQUENCE=DROP_ZONE_A_OK,DROP_ZONE_B_OK,DROP_ZONE_C_OK`,+ // Group 23: Loading+ "23.1": `VERIFY_LOADING_COMPLETE_88888`,+ // Group 24: Keyboard+ "24.1": `KEYBOARD_ESCAPE_PRESSED`,+ "24.2": `KEYBOARD_KEY_A_PRESSED|KEYBOARD_ENTER_PRESSED|ESC.*A.*ENTER`,+ // Group 25: Tabs+ "25.1": `TAB_SETTINGS_CONTENT`,+ "25.2": `TAB_BILLING_CONTENT`,+ // Group 26: Accordion+ "26.1": `ACCORDION_SECTION_A_OPEN`,+ "26.2": `ACCORDION_SECTION_B_OPEN.*aria-expanded=false|Section A aria-expanded=false`,+ // Group 27: Editor+ "27.1": `EDITOR_CHARS=15|Hello rich text`,+ "27.2": `EDITOR_COMMITTED=Hello rich text`,+ // Group 28: Range+ "28.1": `RANGE_VALUE_90.*BUCKET_HIGH|RANGE_VALUE_90`,+ "28.2": `RANGE_VALUE_10.*BUCKET_LOW|RANGE_VALUE_10`,+ // Group 29: Pagination+ "29.1": `PAGE_2_FIRST_ITEM|PAGE_2_OF_3`,+ "29.2": `PAGE_3_FIRST_ITEM|disabled=true`,+ // Group 30: Dropdown+ "30.1": `DROPDOWN_SELECTED=BETA`,+ "30.2": `DROPDOWN_SELECTED=GAMMA`,+ // Group 31-34: Iframe variants+ "31.1": `DEEP_CLICKED=YES_LEVEL_3`,+ "32.1": `IFRAME_INPUT_RECEIVED_LATE_WORLD`,+ "33.1": `INLINE_RECEIVED_SRCDOC`,+ "34.1": `SANDBOX_CLICKED=YES`,+ // Group 35: Text-heavy+ "35.1": `ARTICLE_PUBLISHED_2026_04_15.*ARTICLE_WORD_COUNT_MARKER_323|ARTICLE_WORD_COUNT_MARKER_323`,+ "35.2": `FOOTER_COPYRIGHT_MARKER`,+ // Group 36: SERP+ "36.1": `RESULT_3_TITLE.*RESULT_3_SNIPPET_MARKER|RESULT_3`,+ "36.2": `RESULT_1.*RESULT_6|SERP_RESULT_COUNT_6`,+ // Group 37: Q&A+ "37.1": `a-2`,+ "37.2": `ANSWER_2_BODY_MARKER.*ACCEPTED_ANSWER_ID_A2|ACCEPTED_ANSWER_ID_A2`,+ // Group 38: Pricing+ "38.1": `PLAN_PRO_PRICE_29`,+ "38.2": `PLAN_FREE_PRICE_0.*PLAN_PRO_PRICE_29.*PLAN_ENTERPRISE_PRICE_CUSTOM|PLAN_PRO_PRICE_29`,+}++var groupSizes = map[int]int{+ 0: 8, 1: 6, 2: 3, 3: 2, 4: 3, 5: 2, 6: 3, 7: 2, 8: 2, 9: 2, 10: 2, 11: 2, 12: 2,+ 13: 2, 14: 2, 15: 4, 16: 2, 17: 2, 18: 1, 19: 2, 20: 2, 21: 2, 22: 2, 23: 1, 24: 2, 25: 2,+ 26: 2, 27: 2, 28: 2, 29: 2, 30: 2, 31: 1, 32: 1, 33: 1, 34: 1, 35: 2, 36: 2, 37: 2, 38: 2,+}++type VerifyAnswersArgs struct {+ ReportFiles []string+}++func ParseVerifyAnswersArgs(argv []string) (VerifyAnswersArgs, error) {+ var args VerifyAnswersArgs+ args.ReportFiles = argv++ if len(args.ReportFiles) == 0 {+ return args, fmt.Errorf("usage: verify-answers <report1.json> [report2.json ...]")+ }++ return args, nil+}++type verifyResult struct {+ ID string+ Status string // "pass", "fail", "skip"+ Answer string+ Pattern string+}++func RunVerifyAnswers(argv []string, stdout, stderr io.Writer) int {+ args, err := ParseVerifyAnswersArgs(argv)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "verify-answers: %v\n", err)+ return 1+ }++ // Expand globs+ var files []string+ for _, pattern := range args.ReportFiles {+ matches, err := filepath.Glob(pattern)+ if err != nil || len(matches) == 0 {+ if _, statErr := os.Stat(pattern); statErr == nil {+ files = append(files, pattern)+ } else {+ _, _ = fmt.Fprintf(stderr, "verify-answers: no files matched %q\n", pattern)+ return 1+ }+ } else {+ files = append(files, matches...)+ }+ }++ // Load all steps, deduplicate by ID (keep first seen)+ type stepData struct {+ ID string `json:"id"`+ Answer string `json:"answer"`+ }+ seen := make(map[string]stepData)++ for _, f := range files {+ data, err := os.ReadFile(f)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "verify-answers: failed to read %s: %v\n", f, err)+ return 1+ }+ var report struct {+ Steps []stepData `json:"steps"`+ }+ if err := json.Unmarshal(data, &report); err != nil {+ _, _ = fmt.Fprintf(stderr, "verify-answers: failed to parse %s: %v\n", f, err)+ return 1+ }+ for _, s := range report.Steps {+ if _, exists := seen[s.ID]; !exists {+ seen[s.ID] = s+ }+ }+ }++ // Verify each step against expected pattern+ var results []verifyResult+ for id, s := range seen {+ pattern, hasPattern := expectPatterns[id]+ if !hasPattern {+ results = append(results, verifyResult{+ ID: id,+ Status: "skip",+ Answer: s.Answer,+ })+ continue+ }++ re, err := regexp.Compile("(?i)" + pattern)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "verify-answers: bad pattern for %s: %v\n", id, err)+ results = append(results, verifyResult{+ ID: id,+ Status: "fail",+ Answer: s.Answer,+ Pattern: pattern,+ })+ continue+ }++ if re.MatchString(s.Answer) {+ results = append(results, verifyResult{+ ID: id,+ Status: "pass",+ Answer: s.Answer,+ Pattern: pattern,+ })+ } else {+ results = append(results, verifyResult{+ ID: id,+ Status: "fail",+ Answer: s.Answer,+ Pattern: pattern,+ })+ }+ }++ // Sort results by group.step+ sort.Slice(results, func(i, j int) bool {+ gi, si := parseStepID(results[i].ID)+ gj, sj := parseStepID(results[j].ID)+ if gi != gj {+ return gi < gj+ }+ return si < sj+ })++ // Count totals+ var pass, fail, skip int+ var failures []verifyResult+ for _, r := range results {+ switch r.Status {+ case "pass":+ pass+++ case "fail":+ fail+++ failures = append(failures, r)+ case "skip":+ skip+++ }+ }++ total := pass + fail + skip+ _, _ = fmt.Fprintf(stdout, "Verified: %d/%d steps\n", total, 87)+ _, _ = fmt.Fprintf(stdout, " pass: %d\n", pass)+ _, _ = fmt.Fprintf(stdout, " fail: %d\n", fail)+ _, _ = fmt.Fprintf(stdout, " skip: %d (no pattern)\n", skip)++ // Find missing steps+ var missing []string+ for g := 0; g <= 38; g++ {+ count := groupSizes[g]+ for s := 1; s <= count; s++ {+ id := fmt.Sprintf("%d.%d", g, s)+ if _, ok := seen[id]; !ok {+ missing = append(missing, id)+ }+ }+ }++ if len(missing) > 0 {+ _, _ = fmt.Fprintf(stdout, " missing: %s\n", strings.Join(missing, ", "))+ }++ if len(failures) > 0 {+ _, _ = fmt.Fprintf(stdout, "\nFailed steps:\n")+ for _, f := range failures {+ answer := f.Answer+ if len(answer) > 100 {+ answer = answer[:97] + "..."+ }+ _, _ = fmt.Fprintf(stdout, " %s: answer=[%s] expected=[%s]\n", f.ID, answer, f.Pattern)+ }+ }++ if fail > 0 {+ return 1+ }+ return 0+}++func parseStepID(id string) (int, int) {+ var g, s int+ _, _ = fmt.Sscanf(id, "%d.%d", &g, &s)+ return g, s+}diff --git a/tests/tools/runner/main.go b/tests/tools/runner/main.goindex d3de424f..61704016 100644--- a/tests/tools/runner/main.go+++ b/tests/tools/runner/main.go@@ -7,6 +7,7 @@ import ("github.com/pinchtab/pinchtab/tests/tools/runner/internal/bench""github.com/pinchtab/pinchtab/tests/tools/runner/internal/e2e"+ "github.com/pinchtab/pinchtab/tests/tools/runner/internal/opt")func main() {@@ -25,6 +26,10 @@ func run(argv []string, stdout, stderr io.Writer) int {return bench.Run(argv[1:], stdout, stderr)case "verify-step", "record-step", "step-end":return bench.Run(argv, stdout, stderr)+ case "opt":+ return opt.Run(argv[1:], stdout, stderr)+ case "merge-reports", "verify-answers", "inject-usage":+ return opt.Run(argv, stdout, stderr)default:if argv[0] == "--lane" || argv[0] == "--help" || argv[0] == "-h" {return bench.Run(argv, stdout, stderr)@@ -38,6 +43,8 @@ func run(argv []string, stdout, stderr io.Writer) int {_, _ = fmt.Fprintln(stderr, " runner bench --lane pinchtab")_, _ = fmt.Fprintln(stderr, " runner --lane pinchtab")_, _ = fmt.Fprintln(stderr, " runner step-end ...")+ _, _ = fmt.Fprintln(stderr, " runner opt merge-reports ...")+ _, _ = fmt.Fprintln(stderr, " runner opt verify-answers ...")return 1}}diff --git a/tests/tools/scripts/finalize-report.sh b/tests/tools/scripts/finalize-report.shdeleted file mode 100755index 331dd6d6..00000000--- a/tests/tools/scripts/finalize-report.sh+++ /dev/null@@ -1,162 +0,0 @@-#!/usr/bin/env bash-#-# Renders a short markdown summary for the latest benchmark JSON report.--set -euo pipefail--SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-BENCHMARK_DIR="${SCRIPT_DIR}/../../benchmark"-RESULTS_DIR="${BENCHMARK_DIR}/results"-CURRENT_BASELINE_PTR="${RESULTS_DIR}/current_baseline_report.txt"-CURRENT_AGENT_PTR="${RESULTS_DIR}/current_pinchtab_report.txt"-CURRENT_AGENT_BROWSER_PTR="${RESULTS_DIR}/current_agent_browser_report.txt"--resolve_current_report() {- local ptr="$1"- if [[ -f "${ptr}" ]]; then- tr -d '[:space:]' < "${ptr}"- return 0- fi- return 1-}--REPORT_FILE="${1:-}"-if [[ -z "${REPORT_FILE}" ]]; then- REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_BROWSER_PTR}" || true)"- [[ -n "${REPORT_FILE}" ]] || REPORT_FILE="$(resolve_current_report "${CURRENT_AGENT_PTR}" || true)"- [[ -n "${REPORT_FILE}" ]] || REPORT_FILE="$(resolve_current_report "${CURRENT_BASELINE_PTR}" || true)"- if [[ -z "${REPORT_FILE}" ]]; then- shopt -s nullglob- candidates=(- "${RESULTS_DIR}"/agent_browser_benchmark_*.json- "${RESULTS_DIR}"/pinchtab_benchmark_*.json- "${RESULTS_DIR}"/baseline_*.json- )- shopt -u nullglob-- if [[ ${#candidates[@]} -gt 0 ]]; then- REPORT_FILE=$(ls -t "${candidates[@]}" | head -1)- fi- fi-fi--if [[ -z "${REPORT_FILE}" || ! -f "${REPORT_FILE}" ]]; then- echo "ERROR: no benchmark report found"- exit 1-fi--# Auto-verify pending agent answers before rendering the summary.-# This ensures the "Verification Pass Rate" in the summary reflects actual-# grading, not just the answer count. For baseline reports (which use-# pass/fail directly) this is a no-op.-PENDING=$(jq -r '.totals.steps_pending_verification // 0' "${REPORT_FILE}")-if [[ "${PENDING}" -gt 0 ]]; then- VERIFY_SCRIPT="${SCRIPT_DIR}/verify-answers.sh"- if [[ -x "${VERIFY_SCRIPT}" ]]; then- echo "Auto-verifying ${PENDING} pending answers..."- "${VERIFY_SCRIPT}" "${REPORT_FILE}"- else- echo "WARNING: ${PENDING} answers pending verification but verify-answers.sh not found"- fi-fi--SUMMARY_FILE="${REPORT_FILE%.json}_summary.md"--jq -r '- def pct($a; $b):- if $b == 0 then "0.0%" else (((1000 * $a) / $b | round) / 10 | tostring) + "%" end;- . as $root |- ($root.run_usage // {}) as $usage |- ($root.totals.steps_answered // 0) as $answered |- ($root.totals.steps_failed // 0) as $execution_failed |- ($root.totals.steps_skipped // 0) as $execution_skipped |- ($root.totals.steps_passed // 0) as $legacy_passed |- ($root.benchmark.type == "baseline" and $answered == 0) as $legacy_baseline |- ($legacy_passed + $execution_failed + $execution_skipped) as $legacy_total |- ($answered + $execution_failed + $execution_skipped) as $execution_total |- ($root.totals.steps_verified_passed // 0) as $verified_passed |- ($root.totals.steps_verified_failed // 0) as $verified_failed |- ($root.totals.steps_verified_skipped // 0) as $verified_skipped |- ($root.totals.steps_pending_verification // 0) as $pending |- (if $legacy_baseline then- [- "# Benchmark Summary",- "",- "| Metric | Value |",- "|--------|-------|",- "| Type | \($root.benchmark.type) |",- "| Model | \($root.benchmark.model) |",- "| Steps Passed | \($legacy_passed) |",- "| Steps Failed | \($execution_failed) |",- "| Steps Skipped | \($execution_skipped) |",- "| Pass Rate | \(pct($legacy_passed; $legacy_total)) |",- "| Tool Calls | \($root.totals.tool_calls // 0) |"- ]- else- [- "# Benchmark Summary",- "",- "| Metric | Value |",- "|--------|-------|",- "| Type | \($root.benchmark.type) |",- "| Model | \($root.benchmark.model) |",- "| Steps Answered | \($answered) |",- "| Execution Failed | \($execution_failed) |",- "| Execution Skipped | \($execution_skipped) |",- "| Answer Rate | \(pct($answered; $execution_total)) |",- "| Verified Passed | \($verified_passed) |",- "| Verified Failed | \($verified_failed) |",- "| Verified Skipped | \($verified_skipped) |",- "| Pending Verification | \($pending) |",- "| Verification Pass Rate | \(pct($verified_passed; $execution_total)) |",- "| Tool Calls | \($root.totals.tool_calls // 0) |"- ]- end)[],- "",- "## Run Usage",- "",- (- if (($usage.provider // "") == "" and ($usage.total_tokens // 0) == 0 and ($usage.request_count // 0) == 0) then- ["- none recorded"]- else- [- "| Metric | Value |",- "|--------|-------|",- "| Source | \($usage.source // "unknown") |",- "| Provider | \($usage.provider // "unknown") |",- "| API Requests | \($usage.request_count // 0) |",- "| Input Tokens (uncached) | \($usage.input_tokens // 0) |",- "| Cache Creation Input Tokens | \($usage.cache_creation_input_tokens // 0) |",- "| Cache Read Input Tokens | \($usage.cache_read_input_tokens // 0) |",- "| Total Input Tokens | \($usage.total_input_tokens // 0) |",- "| Output Tokens | \($usage.output_tokens // 0) |",- "| Total Tokens | \($usage.total_tokens // 0) |"- ]- end- )[],- "",- "## Pending Verification",- "",- (- [ $root.steps[] | select(.status == "answer" and ((.verification.status // "pending") == "pending")) | "- \(.id): \(.answer // .notes // "")" ] |- if length == 0 then ["- none"] else . end- )[],- "",- "## Failed Steps",- "",- (- [ $root.steps[] | select(.status == "fail") | "- \(.id): \(.notes)" ] |- if length == 0 then ["- none"] else . end- )[],- "",- "## Verification Failures",- "",- (- [ $root.steps[] | select(.status == "answer" and ((.verification.status // "") == "fail")) | "- \(.id): \(.verification.notes // "")" ] |- if length == 0 then ["- none"] else . end- )[]-' "${REPORT_FILE}" > "${SUMMARY_FILE}"--echo "Wrote ${SUMMARY_FILE}"-echo ""-cat "${SUMMARY_FILE}"diff --git a/tests/tools/scripts/run-agent-browser-benchmark.sh b/tests/tools/scripts/run-agent-browser-benchmark.shindex 66a19d94..9aa5fbab 100755--- a/tests/tools/scripts/run-agent-browser-benchmark.sh+++ b/tests/tools/scripts/run-agent-browser-benchmark.sh@@ -96,4 +96,4 @@ echo " 2. The agent-browser skill is refreshed by runner (DownloadAgentBrowserSecho " 3. Use ./scripts/ab ... to drive agent-browser inside Docker"echo " 4. Record and verify each completed benchmark step in a single call with:"echo " ./scripts/ab step-end <group> <step> answer \"what you saw\" <pass|fail|skip> \"verification notes\""-echo " 5. Summarize the report with ./scripts/finalize-report.sh"+echo " 5. Report is finalized automatically at end of run"diff --git a/tests/tools/scripts/runner b/tests/tools/scripts/runnerdeleted file mode 100755index 6c006576..00000000--- a/tests/tools/scripts/runner+++ /dev/null@@ -1,17 +0,0 @@-#!/usr/bin/env bash-#-# Thin wrapper for the Go runner.-# Usage: ./scripts/runner <subcommand> [args...]-#-# Subcommands:-# step-end Record and verify a step in one call-# record-step Record a step result-# verify-step Verify a previously recorded step-# --lane ... Run the benchmark loop (existing behavior)--set -euo pipefail--SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-TOOLS_DIR="${SCRIPT_DIR}/.."--exec go run "${TOOLS_DIR}/runner" "$@"diff --git a/tests/tools/scripts/verify-answers.sh b/tests/tools/scripts/verify-answers.shdeleted file mode 100755index b8ed6e90..00000000--- a/tests/tools/scripts/verify-answers.sh+++ /dev/null@@ -1,164 +0,0 @@-#!/usr/bin/env bash-#-# One-off verifier for Loop #31 answers. Reads the agent report, matches each-# step's `.answer` against an expected marker/substring set, and invokes-# verify-step.sh with pass/fail. The marker map below is hand-derived from-# the baseline benchmark oracle implemented in scripts/baseline.sh.-#-# Usage:-# ./verify_answers_for_loop31.sh results/pinchtab_benchmark_20260415_233421.json--set -euo pipefail--REPORT="${1:-results/pinchtab_benchmark_20260415_233421.json}"--declare -A EXPECT-# Setup & diagnosis — tightened after manual audit (see Loop #33 notes)-EXPECT[0.1]='status.*ok|status:.*ok'-EXPECT[0.2]='401|missing_token|unauth'-EXPECT[0.3]='200|authed.*200'-EXPECT[0.4]='running'-EXPECT[0.5]='tab.*list|tabs.*returned|active tab|listed.*tab'-EXPECT[0.6]='cleaned|closed|no stale|reused.*tab|no cleanup|single active|already empty'-EXPECT[0.7]='VERIFY_HOME_LOADED_12345|navigated to fixtures|fixtures.*home'-EXPECT[0.8]='[0-9A-F]{32}|tab.*id.*captured'-# Reading & Extracting-EXPECT[1.1]='COUNT_LANGUAGES_12|Programming Languages 12|12 articles'-EXPECT[1.2]='wiki-go|Go \(programming language\)|409 success|clicked'-EXPECT[1.3]='Robert Griesemer.*2009|2009.*Griesemer'-EXPECT[1.4]='FEATURE_COUNT_6|6 key features|6 features'-EXPECT[1.5]='Artificial Intelligence.*Climate Action.*Mars|Artificial Intelligence, Climate Action'-EXPECT[1.6]='24,582.*1,284,930|24,582|Revenue \$1,284,930'-# Search-EXPECT[2.1]='wiki-go|VERIFY_WIKI_GO_LANG_88888'-EXPECT[2.2]='No results|no results|xyznonexistent'-EXPECT[2.3]='Artificial Intelligence|ARTIFICIAL_INTELLIGENCE'-# Form-EXPECT[3.1]='submitted|VERIFY_FORM_SUBMITTED_SUCCESS|FORM_SUBMITTED|SUBMISSION_DATA'-EXPECT[3.2]='[Rr]eset.*button|#reset-btn|[Rr]eset.*present|reset-btn'-# SPA-EXPECT[4.1]='TASK_STATS_TOTAL_3|Total.*3.*Active.*2.*Done.*1|3.*2.*1'-EXPECT[4.2]='TASK_ADDED|AUTOMATE|DEPLOYMENT|high'-EXPECT[4.3]='deleted.*task|TASK_STATS_TOTAL_3|All Tasks.*3|3.*tasks'-# Login-EXPECT[5.1]='INVALID_CREDENTIALS_ERROR|Invalid'-EXPECT[5.2]='VERIFY_LOGIN_SUCCESS_DASHBOARD|SESSION_TOKEN_ACTIVE_TRUE|login success|Dashboard'-# E-commerce-EXPECT[6.1]='149.99.*299.99|Wireless.*Smart Watch|Portable Charger'-EXPECT[6.2]='449.98|CART_ITEM_WIRELESS'-EXPECT[6.3]='VERIFY_CHECKOUT_SUCCESS_ORDER|checkout'-# Content + Interaction-EXPECT[7.1]='COMMENT_POSTED_RATING_5'-EXPECT[7.2]='Developer Tools.*15|15.*Developer|wiki-go|VERIFY_WIKI_GO'-# Error handling-EXPECT[8.1]='404|not found'-EXPECT[8.2]='element.*not found|no element found|no element|clear error'-# Export-EXPECT[9.1]='[Ss]creenshot|\.png.*[0-9]{4,} bytes'-EXPECT[9.2]='[Pp][Dd][Ff]|\.pdf.*[0-9]{4,} bytes'-# Modals-EXPECT[10.1]='Dashboard Settings'-EXPECT[10.2]='THEME_DARK_APPLIED'-# Persistence-EXPECT[11.1]='TASK_PERSISTENT_TEST_FOUND'-EXPECT[11.2]='SESSION_RENEWED|VERIFY_LOGIN_SUCCESS_DASHBOARD'-# Multi-page nav-EXPECT[12.1]='home|VERIFY_HOME_LOADED_12345|PinchTab Benchmark - Home'-EXPECT[12.2]='COUNT_LANGUAGES_12|12.*Programming|Artificial Intelligence|Total Articles|12\+8\+15|COMPARISON_DATA_FOUND'-# Form validation-EXPECT[13.1]='blocked|display:none|valueMissing|not submitted'-EXPECT[13.2]='VERIFY_FORM_SUBMITTED_SUCCESS|OPTIONAL_FIELD_SKIPPED_SUCCESS|submitted'-# Dynamic content-EXPECT[14.1]='ADDITIONAL_PRODUCTS_LOADED'-EXPECT[14.2]='CART_UPDATED_WITH_LAZY_PRODUCT|USB-C'-# Data aggregation-EXPECT[15.1]='1,284,930.*384,930|PROFIT_MARGIN_CALCULATED|Revenue.*Profit'-EXPECT[15.2]='FEATURE_COUNT_6.*7.*5|Go=6.*Python=7.*Rust=5|6, 7, 5|COMPARISON_TABLE_BUILT'-# Hover-EXPECT[16.1]='HOVER_REVEALED_USER_1'-EXPECT[16.2]='HOVER_REVEALED_USER_2'-# Scroll-EXPECT[17.1]='SCROLL_MIDDLE_MARKER'-EXPECT[17.2]='SCROLL_REACHED_FOOTER'-# Download-EXPECT[18.1]='DOWNLOAD_FILE_CONTENT_VERIFIED|143 bytes'-# iFrame-EXPECT[19.1]='IFRAME_INNER_CONTENT_LOADED'-EXPECT[19.2]='IFRAME_INPUT_RECEIVED_HELLO_WORLD'-# Dialogs-EXPECT[20.1]='DIALOG_ALERT_DISMISSED'-EXPECT[20.2]='DIALOG_CONFIRM_CANCELLED'-# Async / awaitPromise-EXPECT[21.1]='ASYNC_PAYLOAD_READY_42'-EXPECT[21.2]='ASYNC_USER_NAME_ADA'-# Drag-EXPECT[22.1]='DROP_ZONE_A_OK'-EXPECT[22.2]='DROP_SEQUENCE=DROP_ZONE_A_OK,DROP_ZONE_B_OK,DROP_ZONE_C_OK'-# Loading-EXPECT[23.1]='VERIFY_LOADING_COMPLETE_88888'-# Keyboard-EXPECT[24.1]='KEYBOARD_ESCAPE_PRESSED'-EXPECT[24.2]='KEYBOARD_KEY_A_PRESSED|KEYBOARD_ENTER_PRESSED|ESC.*A.*ENTER'-# Tabs-EXPECT[25.1]='TAB_SETTINGS_CONTENT'-EXPECT[25.2]='TAB_BILLING_CONTENT'-# Accordion-EXPECT[26.1]='ACCORDION_SECTION_A_OPEN'-EXPECT[26.2]='ACCORDION_SECTION_B_OPEN.*aria-expanded=false|Section A aria-expanded=false'-# Editor-EXPECT[27.1]='EDITOR_CHARS=15|Hello rich text'-EXPECT[27.2]='EDITOR_COMMITTED=Hello rich text'-# Range-EXPECT[28.1]='RANGE_VALUE_90.*BUCKET_HIGH|RANGE_VALUE_90'-EXPECT[28.2]='RANGE_VALUE_10.*BUCKET_LOW|RANGE_VALUE_10'-# Pagination-EXPECT[29.1]='PAGE_2_FIRST_ITEM|PAGE_2_OF_3'-EXPECT[29.2]='PAGE_3_FIRST_ITEM|disabled=true'-# Dropdown-EXPECT[30.1]='DROPDOWN_SELECTED=BETA'-EXPECT[30.2]='DROPDOWN_SELECTED=GAMMA'-# Iframe variants-EXPECT[31.1]='DEEP_CLICKED=YES_LEVEL_3'-EXPECT[32.1]='IFRAME_INPUT_RECEIVED_LATE_WORLD'-EXPECT[33.1]='INLINE_RECEIVED_SRCDOC'-EXPECT[34.1]='SANDBOX_CLICKED=YES'-# Text-heavy-EXPECT[35.1]='ARTICLE_PUBLISHED_2026_04_15.*ARTICLE_WORD_COUNT_MARKER_323|ARTICLE_WORD_COUNT_MARKER_323'-EXPECT[35.2]='FOOTER_COPYRIGHT_MARKER'-EXPECT[36.1]='RESULT_3_TITLE.*RESULT_3_SNIPPET_MARKER|RESULT_3'-EXPECT[36.2]='RESULT_1.*RESULT_6|SERP_RESULT_COUNT_6'-EXPECT[37.1]='a-2'-EXPECT[37.2]='ANSWER_2_BODY_MARKER.*ACCEPTED_ANSWER_ID_A2|ACCEPTED_ANSWER_ID_A2'-EXPECT[38.1]='PLAN_PRO_PRICE_29'-EXPECT[38.2]='PLAN_FREE_PRICE_0.*PLAN_PRO_PRICE_29.*PLAN_ENTERPRISE_PRICE_CUSTOM|PLAN_PRO_PRICE_29'--pass=0-fail=0-failed_ids=()--while IFS='|' read -r id answer; do- pattern="${EXPECT[$id]:-}"- if [[ -z "$pattern" ]]; then- echo "skip $id — no expected pattern defined"- "$(dirname "${BASH_SOURCE[0]}")/verify-step.sh" --report-file "$REPORT" \- "${id%.*}" "${id#*.}" skip "no expected pattern" > /dev/null- continue- fi- if printf '%s' "$answer" | grep -qE "$pattern"; then- "$(dirname "${BASH_SOURCE[0]}")/verify-step.sh" --report-file "$REPORT" \- "${id%.*}" "${id#*.}" pass "auto: matched /$pattern/" > /dev/null- pass=$((pass + 1))- else- "$(dirname "${BASH_SOURCE[0]}")/verify-step.sh" --report-file "$REPORT" \- "${id%.*}" "${id#*.}" fail "auto: answer did not match /$pattern/" > /dev/null- fail=$((fail + 1))- failed_ids+=("$id")- fi-done < <(jq -r '.steps[] | "\(.id)|\(.answer)"' "$REPORT")--echo-echo "Verified: $pass pass, $fail fail"-if (( fail > 0 )); then- echo "Failed ids:"- printf ' %s\n' "${failed_ids[@]}"-fi