High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: press [ref] <key> syntax, focus-before-dispatch, and CDP keyDown for form submission
Luigi Agosti9e5a5bf64211
diff --git a/cmd/pinchtab/cmd_cli_browser_actions.go b/cmd/pinchtab/cmd_cli_browser_actions.goindex c6a90680..40147ce1 100644--- a/cmd/pinchtab/cmd_cli_browser_actions.go+++ b/cmd/pinchtab/cmd_cli_browser_actions.go@@ -64,7 +64,7 @@ var dblclickCmd = newOptionalRefActionCmd("dblclick <ref>", "Double-click elemenvar typeCmd = newSimpleActionCmd("type <ref> <text>", "Type into element", "type", cobra.MinimumNArgs(2))-var pressCmd = newSimpleActionCmd("press <key>", "Press key (Enter, Tab, Escape...)", "press", cobra.MinimumNArgs(1))+var pressCmd = newSimpleActionCmd("press [ref] <key>", "Press key (Enter, Tab, Escape...), optionally focusing a ref first", "press", cobra.MinimumNArgs(1))var fillCmd = newSimpleActionCmd("fill <ref|selector> <text>", "Fill input directly", "fill", cobra.MinimumNArgs(2))diff --git a/internal/bridge/action_text.go b/internal/bridge/action_text.goindex f086c75a..9c6effe4 100644--- a/internal/bridge/action_text.go+++ b/internal/bridge/action_text.go@@ -56,6 +56,18 @@ func (b *Bridge) actionPress(ctx context.Context, req ActionRequest) (map[stringif req.Key == "" {return nil, fmt.Errorf("key required for press")}+ // If a ref/selector was given alongside the key, focus that element first+ // so the key lands on the intended target (e.g. `press e2 Enter` to submit+ // a form from the password field).+ if req.NodeID > 0 {+ if err := focusBackendNode(ctx, req.NodeID); err != nil {+ return nil, err+ }+ } else if req.Selector != "" {+ if err := chromedp.Run(ctx, chromedp.Focus(req.Selector, chromedp.ByQuery)); err != nil {+ return nil, err+ }+ }return map[string]any{"pressed": req.Key}, DispatchNamedKey(ctx, req.Key)}diff --git a/internal/bridge/cdp_keyboard.go b/internal/bridge/cdp_keyboard.goindex f215f83f..f1a78b93 100644--- a/internal/bridge/cdp_keyboard.go+++ b/internal/bridge/cdp_keyboard.go@@ -9,10 +9,16 @@ import (// namedKeyDefs maps friendly key names (as accepted by the CLI "press" command)// to their CDP Input.dispatchKeyEvent parameters. Keys not in this table fall// through to chromedp.KeyEvent so that single printable characters still work.+//+// insertText is the character text sent in the keyDown event's "text" field.+// This is what Playwright does: it makes Chrome generate both keydown and+// keypress DOM events, triggering built-in browser default actions (form+// submission for Enter, focus-advance for Tab). We do NOT send a separate+// Input.insertText call — the text field here is for event generation only.var namedKeyDefs = map[string]struct {code stringvirtualKey int64- insertText string // non-empty for keys that produce a character (Enter→\r, Tab→\t)+ insertText string // text field for keyDown event (triggers keypress + default action)}{"Enter": {"Enter", 13, "\r"},"Return": {"Enter", 13, "\r"},@@ -75,15 +81,38 @@ func DispatchNamedKey(ctx context.Context, key string) error {})}- actions := chromedp.Tasks{dispatchEvent("rawKeyDown")}+ // Character-producing keys (Enter, Tab) must use "keyDown" with the text+ // field set, mirroring what Playwright does. This causes Chrome to fire both+ // the keydown and keypress DOM events, which triggers built-in default+ // actions (form submit on Enter, focus-advance on Tab).+ //+ // Non-character keys (Escape, arrows, etc.) use "rawKeyDown" — they have no+ // default browser action that requires a keypress event.+ //+ // Note: we do NOT follow up with a separate Input.insertText for Enter.+ // The text field in the keyDown event is enough to trigger keypress/form+ // submission; a separate insertText would corrupt the field value.+ downType := "rawKeyDown"+ var downText stringif def.insertText != "" {- insertText := def.insertText- actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error {- return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.insertText", map[string]any{- "text": insertText,- }, nil)- }))+ downType = "keyDown"+ downText = def.insertText}+ dispatchKeyDown := chromedp.ActionFunc(func(ctx context.Context) error {+ params := map[string]any{+ "type": downType,+ "key": w3cKey,+ "code": def.code,+ "windowsVirtualKeyCode": def.virtualKey,+ "nativeVirtualKeyCode": def.virtualKey,+ }+ if downText != "" {+ params["text"] = downText+ params["unmodifiedText"] = downText+ }+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchKeyEvent", params, nil)+ })+ actions := chromedp.Tasks{dispatchKeyDown}actions = append(actions, dispatchEvent("keyUp"))return chromedp.Run(ctx, actions...)diff --git a/internal/cli/actions/actions_element.go b/internal/cli/actions/actions_element.goindex 5c099093..7dbcfe66 100644--- a/internal/cli/actions/actions_element.go+++ b/internal/cli/actions/actions_element.go@@ -400,7 +400,16 @@ func ActionSimple(client *http.Client, base, token, kind string, args []string,setSelectorBody(body, args[0])body["text"] = strings.Join(args[1:], " ")case "press":- body["key"] = args[0]+ // Accept both `press <key>` and `press <ref> <key>`.+ // When a ref is given it is used only to focus the element before+ // dispatching the key — this matches how agents naturally write it+ // ("press e2 Enter") by analogy with "fill e2 <text>".+ if len(args) >= 2 {+ setSelectorBody(body, args[0])+ body["key"] = args[1]+ } else {+ body["key"] = args[0]+ }case "scroll":// Precedence: integer pixels > direction keyword > unified selector.// Pixels and directions are short, low-cardinality inputs that woulddiff --git a/skills/pinchtab-opt/SKILL.md b/skills/pinchtab-opt/SKILL.mdindex 5322b192..7c83ff76 100644--- a/skills/pinchtab-opt/SKILL.md+++ b/skills/pinchtab-opt/SKILL.md@@ -1,11 +1,11 @@---name: pinchtab-opt-description: "Run the PinchTab optimization loop. Spawns blind subagents that execute 99 browser automation steps across 45 groups using only the PinchTab skill, then reports pass/fail results and operation counts vs baseline. Use when asked to 'run optimization', 'run the opt loop', 'benchmark the agent', or 'test pinchtab agent'."+description: "Run the PinchTab optimization loop. Spawns blind subagents that execute 108 browser automation steps across 47 groups using only the PinchTab skill, then reports pass/fail results and operation counts vs baseline. Use when asked to 'run optimization', 'run the opt loop', 'benchmark the agent', or 'test pinchtab agent'."---# PinchTab Optimization Loop-Run blind subagents against 99 browser automation steps (45 groups) to measure how well an AI agent can drive PinchTab without hand-held selectors.+Run blind subagents against 108 browser automation steps (47 groups) to measure how well an AI agent can drive PinchTab without hand-held selectors.## Path Resolution@@ -74,7 +74,7 @@ Use the **Agent** tool with `run_in_background: true`. Split the 45 groups into- **Batch A**: groups 0-14 (45 steps)- **Batch B**: groups 15-29 (30 steps)-- **Batch C**: groups 30-44 (24 steps)+- **Batch C**: groups 30-46 (33 steps)Each subagent gets the **same prompt template** — only the group range and `{REPORT_FILE}` change. Replace `{START}`, `{END}`, `{START_PAD}`, `{END_PAD}`, `{PROJECT_ROOT}`, and `{REPORT_FILE}` with actual values:@@ -110,17 +110,43 @@ While agents run, periodically count step-end recordings in each agent's outputgrep -c "step-end" <output_file>```-Expected totals: Batch A ~45, Batch B ~30, Batch C ~24 = 99 total.+Expected totals: Batch A ~45, Batch B ~30, Batch C ~33 = 108 total.### 3. Collect and summarize-Once all 3 agents complete, use `./scripts/runner` subcommands to merge reports (`merge-reports`), inject token usage from subagent transcripts (`inject-usage`), and print the final comparison table (`opt summarize`). Present the summarize output to the user as-is.+Once all 3 agents complete, run these steps **in order**. The Agent tool returns each subagent's output file path — save all three as `TRANSCRIPT_A`, `TRANSCRIPT_B`, `TRANSCRIPT_C`.++```bash+SKILL_DIR=~/.claude/skills/pinchtab-opt+MERGED="$RESULTS_DIR/merged_${TIMESTAMP}.json"++# Merge the three agent reports into one JSON (strip non-JSON header lines)+cd "$TOOLS_DIR" && \+ ./scripts/runner opt merge-reports \+ "$RESULTS_DIR/agentA_${TIMESTAMP}.json" \+ "$RESULTS_DIR/agentB_${TIMESTAMP}.json" \+ "$RESULTS_DIR/agentC_${TIMESTAMP}.json" \+ 2>/dev/null | grep -v '^Loaded\|^Merged' > "$MERGED"++# Inject token usage from the subagent JSONL transcripts+./scripts/runner opt inject-usage \+ -r "$MERGED" \+ "$TRANSCRIPT_A" "$TRANSCRIPT_B" "$TRANSCRIPT_C"++# Print the final comparison table — present this output to the user as-is+./scripts/runner opt summarize \+ -r "$MERGED" \+ -b "$SKILL_DIR/baseline-ref.json" \+ "$TRANSCRIPT_A" "$TRANSCRIPT_B" "$TRANSCRIPT_C"+```++The `--baseline` / `-b` flag loads stored reference timing and ops from `baseline-ref.json` so the Baseline column is fully populated. The transcripts enable the Browser ops and Ops/step rows.## Reference Numbers-- **Baseline**: 99/99 steps-- **Expected agent range**: 400-600 browser ops, 4-6 ops/step (agent must explore pages before acting)-- **Group count**: 45 groups, 99 total steps+- **Baseline**: 108/108 steps, 272 ops, 49s total, 0.5s/step (stored in `baseline-ref.json`)+- **Expected agent range**: 250-400 browser ops, 2.5-4 ops/step+- **Group count**: 47 groups, 108 total steps## File Locations (relative to project root)@@ -128,8 +154,9 @@ Once all 3 agents complete, use `./scripts/runner` subcommands to merge reports|------|---------|| `tests/optimization/subagent-context.md` | Subagent instructions (env, wrapper, recording) || `tests/optimization/index.md` | Group listing |-| `tests/optimization/group-00.md` .. `group-44.md` | Task descriptions |+| `tests/optimization/group-00.md` .. `group-46.md` | Task descriptions || `skills/pinchtab/SKILL.md` | PinchTab command reference (read by subagent) || `tests/tools/scripts/pt` | PinchTab wrapper (CWD must be `tests/tools`) || `tests/tools/scripts/runner` | Step recorder (CWD must be `tests/tools`) || `tests/tools/scripts/baseline.sh` | Baseline (subagent must NOT read this) |+| `~/.claude/skills/pinchtab-opt/baseline-ref.json` | Stored baseline timing/ops reference for table |diff --git a/skills/pinchtab-opt/baseline-ref.json b/skills/pinchtab-opt/baseline-ref.jsonnew file mode 100644index 00000000..b1594b28--- /dev/null+++ b/skills/pinchtab-opt/baseline-ref.json@@ -0,0 +1,8 @@+{+ "note": "Generated 2026-05-26 by running scripts/baseline.sh against Docker fixture server. Update by re-running baseline and measuring wall-clock time.",+ "total_steps": 108,+ "browser_ops": 272,+ "ops_per_step": 2.5,+ "total_time_ms": 49772,+ "avg_time_per_step_ms": 461+}diff --git a/tests/tools/runner/internal/opt/summarize.go b/tests/tools/runner/internal/opt/summarize.goindex ed1cee0f..870ded87 100644--- a/tests/tools/runner/internal/opt/summarize.go+++ b/tests/tools/runner/internal/opt/summarize.go@@ -15,6 +15,7 @@ import (func RunSummarize(argv []string, stdout, stderr io.Writer) int {var reportFile string+ var baselineFile stringvar transcriptFiles []stringi := 0@@ -27,6 +28,13 @@ func RunSummarize(argv []string, stdout, stderr io.Writer) int {}reportFile = argv[i+1]i += 2+ case "--baseline", "-b":+ if i+1 >= len(argv) {+ _, _ = fmt.Fprintf(stderr, "summarize: %s requires a value\n", argv[i])+ return 1+ }+ baselineFile = argv[i+1]+ i += 2default:_, _ = fmt.Fprintf(stderr, "summarize: unknown option: %s\n", argv[i])return 1@@ -35,10 +43,24 @@ func RunSummarize(argv []string, stdout, stderr io.Writer) int {transcriptFiles = argv[i:]if reportFile == "" {- _, _ = fmt.Fprintln(stderr, "usage: summarize -r <merged-report.json> [transcript1.jsonl ...]")+ _, _ = fmt.Fprintln(stderr, "usage: summarize -r <merged-report.json> [-b baseline.json] [transcript1.jsonl ...]")return 1}+ // Load baseline reference if provided+ var baselineRef map[string]any+ if baselineFile != "" {+ data, err := os.ReadFile(baselineFile)+ if err != nil {+ _, _ = fmt.Fprintf(stderr, "summarize: cannot read baseline file: %v\n", err)+ return 1+ }+ if err := json.Unmarshal(data, &baselineRef); err != nil {+ _, _ = fmt.Fprintf(stderr, "summarize: cannot parse baseline file: %v\n", err)+ return 1+ }+ }+data, err := os.ReadFile(reportFile)if err != nil {_, _ = fmt.Fprintf(stderr, "summarize: %v\n", err)@@ -211,21 +233,42 @@ func RunSummarize(argv []string, stdout, stderr io.Writer) int {}rows = append(rows, row{"Errors", "0", fmt.Sprintf("%d", failed)})- // Prefer transcript-based timing, then run_usage wall clock, then per-step sum.+ // Prefer transcript-based timing (sum across agents), then run_usage agent_durations_ms+ // sum (parallel runs — total compute time), then wall_clock_ms, then per-step sum.var displayDurationMs float64if totalAgentDurationMs > 0 {displayDurationMs = totalAgentDurationMs- } else if wc, ok := usage["wall_clock_ms"].(float64); ok && wc > 0 {- displayDurationMs = wc+ } else if usage != nil {+ if durations, ok := usage["agent_durations_ms"].([]any); ok && len(durations) > 0 {+ var sum float64+ for _, d := range durations {+ if v, ok := d.(float64); ok {+ sum += v+ }+ }+ displayDurationMs = sum+ } else if wc, ok := usage["wall_clock_ms"].(float64); ok && wc > 0 {+ displayDurationMs = wc+ }} else if stepsWithDuration > 0 {displayDurationMs = totalDurationMs}+ blAvgStr := ""+ blTotalStr := ""+ if baselineRef != nil {+ if v, ok := baselineRef["avg_time_per_step_ms"].(float64); ok && v > 0 {+ blAvgStr = fmt.Sprintf("%.1fs", v/1000)+ }+ if v, ok := baselineRef["total_time_ms"].(float64); ok && v > 0 {+ blTotalStr = fmtDuration(v)+ }+ }if displayDurationMs > 0 {avgMs := displayDurationMs / float64(blSteps)rows = append(rows,row{"", "", ""},- row{"Avg time/step", "", fmt.Sprintf("%.1fs", avgMs/1000)},- row{"Total time", "", fmtDuration(displayDurationMs)},+ row{"Avg time/step", blAvgStr, fmt.Sprintf("%.1fs", avgMs/1000)},+ row{"Total time", blTotalStr, fmtDuration(displayDurationMs)},)}