Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: press [ref] <key> syntax, focus-before-dispatch, and CDP keyDown for form submission

Luigi Agosti9e5a5bf64211
diff --git a/cmd/pinchtab/cmd_cli_browser_actions.go b/cmd/pinchtab/cmd_cli_browser_actions.go
index c6a90680..40147ce1 100644
--- a/cmd/pinchtab/cmd_cli_browser_actions.go
+++ b/cmd/pinchtab/cmd_cli_browser_actions.go
@@ -64,7 +64,7 @@ var dblclickCmd = newOptionalRefActionCmd("dblclick <ref>", "Double-click elemen
var typeCmd = newSimpleActionCmd("type <ref> <text>", "Type into element", "type", cobra.MinimumNArgs(2))
-var pressCmd = newSimpleActionCmd("press <key>", "Press key (Enter, Tab, Escape...)", "press", cobra.MinimumNArgs(1))
+var pressCmd = newSimpleActionCmd("press [ref] <key>", "Press key (Enter, Tab, Escape...), optionally focusing a ref first", "press", cobra.MinimumNArgs(1))
var fillCmd = newSimpleActionCmd("fill <ref|selector> <text>", "Fill input directly", "fill", cobra.MinimumNArgs(2))
diff --git a/internal/bridge/action_text.go b/internal/bridge/action_text.go
index f086c75a..9c6effe4 100644
--- a/internal/bridge/action_text.go
+++ b/internal/bridge/action_text.go
@@ -56,6 +56,18 @@ func (b *Bridge) actionPress(ctx context.Context, req ActionRequest) (map[string
if req.Key == "" {
return nil, fmt.Errorf("key required for press")
}
+ // If a ref/selector was given alongside the key, focus that element first
+ // so the key lands on the intended target (e.g. `press e2 Enter` to submit
+ // a form from the password field).
+ if req.NodeID > 0 {
+ if err := focusBackendNode(ctx, req.NodeID); err != nil {
+ return nil, err
+ }
+ } else if req.Selector != "" {
+ if err := chromedp.Run(ctx, chromedp.Focus(req.Selector, chromedp.ByQuery)); err != nil {
+ return nil, err
+ }
+ }
return map[string]any{"pressed": req.Key}, DispatchNamedKey(ctx, req.Key)
}
diff --git a/internal/bridge/cdp_keyboard.go b/internal/bridge/cdp_keyboard.go
index f215f83f..f1a78b93 100644
--- a/internal/bridge/cdp_keyboard.go
+++ b/internal/bridge/cdp_keyboard.go
@@ -9,10 +9,16 @@ import (
// namedKeyDefs maps friendly key names (as accepted by the CLI "press" command)
// to their CDP Input.dispatchKeyEvent parameters. Keys not in this table fall
// through to chromedp.KeyEvent so that single printable characters still work.
+//
+// insertText is the character text sent in the keyDown event's "text" field.
+// This is what Playwright does: it makes Chrome generate both keydown and
+// keypress DOM events, triggering built-in browser default actions (form
+// submission for Enter, focus-advance for Tab). We do NOT send a separate
+// Input.insertText call — the text field here is for event generation only.
var namedKeyDefs = map[string]struct {
code string
virtualKey int64
- insertText string // non-empty for keys that produce a character (Enter→\r, Tab→\t)
+ insertText string // text field for keyDown event (triggers keypress + default action)
}{
"Enter": {"Enter", 13, "\r"},
"Return": {"Enter", 13, "\r"},
@@ -75,15 +81,38 @@ func DispatchNamedKey(ctx context.Context, key string) error {
})
}
- actions := chromedp.Tasks{dispatchEvent("rawKeyDown")}
+ // Character-producing keys (Enter, Tab) must use "keyDown" with the text
+ // field set, mirroring what Playwright does. This causes Chrome to fire both
+ // the keydown and keypress DOM events, which triggers built-in default
+ // actions (form submit on Enter, focus-advance on Tab).
+ //
+ // Non-character keys (Escape, arrows, etc.) use "rawKeyDown" — they have no
+ // default browser action that requires a keypress event.
+ //
+ // Note: we do NOT follow up with a separate Input.insertText for Enter.
+ // The text field in the keyDown event is enough to trigger keypress/form
+ // submission; a separate insertText would corrupt the field value.
+ downType := "rawKeyDown"
+ var downText string
if def.insertText != "" {
- insertText := def.insertText
- actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error {
- return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.insertText", map[string]any{
- "text": insertText,
- }, nil)
- }))
+ downType = "keyDown"
+ downText = def.insertText
}
+ dispatchKeyDown := chromedp.ActionFunc(func(ctx context.Context) error {
+ params := map[string]any{
+ "type": downType,
+ "key": w3cKey,
+ "code": def.code,
+ "windowsVirtualKeyCode": def.virtualKey,
+ "nativeVirtualKeyCode": def.virtualKey,
+ }
+ if downText != "" {
+ params["text"] = downText
+ params["unmodifiedText"] = downText
+ }
+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchKeyEvent", params, nil)
+ })
+ actions := chromedp.Tasks{dispatchKeyDown}
actions = append(actions, dispatchEvent("keyUp"))
return chromedp.Run(ctx, actions...)
diff --git a/internal/cli/actions/actions_element.go b/internal/cli/actions/actions_element.go
index 5c099093..7dbcfe66 100644
--- a/internal/cli/actions/actions_element.go
+++ b/internal/cli/actions/actions_element.go
@@ -400,7 +400,16 @@ func ActionSimple(client *http.Client, base, token, kind string, args []string,
setSelectorBody(body, args[0])
body["text"] = strings.Join(args[1:], " ")
case "press":
- body["key"] = args[0]
+ // Accept both `press <key>` and `press <ref> <key>`.
+ // When a ref is given it is used only to focus the element before
+ // dispatching the key — this matches how agents naturally write it
+ // ("press e2 Enter") by analogy with "fill e2 <text>".
+ if len(args) >= 2 {
+ setSelectorBody(body, args[0])
+ body["key"] = args[1]
+ } else {
+ body["key"] = args[0]
+ }
case "scroll":
// Precedence: integer pixels > direction keyword > unified selector.
// Pixels and directions are short, low-cardinality inputs that would
diff --git a/skills/pinchtab-opt/SKILL.md b/skills/pinchtab-opt/SKILL.md
index 5322b192..7c83ff76 100644
--- a/skills/pinchtab-opt/SKILL.md
+++ b/skills/pinchtab-opt/SKILL.md
@@ -1,11 +1,11 @@
---
name: pinchtab-opt
-description: "Run the PinchTab optimization loop. Spawns blind subagents that execute 99 browser automation steps across 45 groups using only the PinchTab skill, then reports pass/fail results and operation counts vs baseline. Use when asked to 'run optimization', 'run the opt loop', 'benchmark the agent', or 'test pinchtab agent'."
+description: "Run the PinchTab optimization loop. Spawns blind subagents that execute 108 browser automation steps across 47 groups using only the PinchTab skill, then reports pass/fail results and operation counts vs baseline. Use when asked to 'run optimization', 'run the opt loop', 'benchmark the agent', or 'test pinchtab agent'."
---
# PinchTab Optimization Loop
-Run blind subagents against 99 browser automation steps (45 groups) to measure how well an AI agent can drive PinchTab without hand-held selectors.
+Run blind subagents against 108 browser automation steps (47 groups) to measure how well an AI agent can drive PinchTab without hand-held selectors.
## Path Resolution
@@ -74,7 +74,7 @@ Use the **Agent** tool with `run_in_background: true`. Split the 45 groups into
- **Batch A**: groups 0-14 (45 steps)
- **Batch B**: groups 15-29 (30 steps)
-- **Batch C**: groups 30-44 (24 steps)
+- **Batch C**: groups 30-46 (33 steps)
Each subagent gets the **same prompt template** — only the group range and `{REPORT_FILE}` change. Replace `{START}`, `{END}`, `{START_PAD}`, `{END_PAD}`, `{PROJECT_ROOT}`, and `{REPORT_FILE}` with actual values:
@@ -110,17 +110,43 @@ While agents run, periodically count step-end recordings in each agent's output
grep -c "step-end" <output_file>
```
-Expected totals: Batch A ~45, Batch B ~30, Batch C ~24 = 99 total.
+Expected totals: Batch A ~45, Batch B ~30, Batch C ~33 = 108 total.
### 3. Collect and summarize
-Once all 3 agents complete, use `./scripts/runner` subcommands to merge reports (`merge-reports`), inject token usage from subagent transcripts (`inject-usage`), and print the final comparison table (`opt summarize`). Present the summarize output to the user as-is.
+Once all 3 agents complete, run these steps **in order**. The Agent tool returns each subagent's output file path — save all three as `TRANSCRIPT_A`, `TRANSCRIPT_B`, `TRANSCRIPT_C`.
+
+```bash
+SKILL_DIR=~/.claude/skills/pinchtab-opt
+MERGED="$RESULTS_DIR/merged_${TIMESTAMP}.json"
+
+# Merge the three agent reports into one JSON (strip non-JSON header lines)
+cd "$TOOLS_DIR" && \
+ ./scripts/runner opt merge-reports \
+ "$RESULTS_DIR/agentA_${TIMESTAMP}.json" \
+ "$RESULTS_DIR/agentB_${TIMESTAMP}.json" \
+ "$RESULTS_DIR/agentC_${TIMESTAMP}.json" \
+ 2>/dev/null | grep -v '^Loaded\|^Merged' > "$MERGED"
+
+# Inject token usage from the subagent JSONL transcripts
+./scripts/runner opt inject-usage \
+ -r "$MERGED" \
+ "$TRANSCRIPT_A" "$TRANSCRIPT_B" "$TRANSCRIPT_C"
+
+# Print the final comparison table — present this output to the user as-is
+./scripts/runner opt summarize \
+ -r "$MERGED" \
+ -b "$SKILL_DIR/baseline-ref.json" \
+ "$TRANSCRIPT_A" "$TRANSCRIPT_B" "$TRANSCRIPT_C"
+```
+
+The `--baseline` / `-b` flag loads stored reference timing and ops from `baseline-ref.json` so the Baseline column is fully populated. The transcripts enable the Browser ops and Ops/step rows.
## Reference Numbers
-- **Baseline**: 99/99 steps
-- **Expected agent range**: 400-600 browser ops, 4-6 ops/step (agent must explore pages before acting)
-- **Group count**: 45 groups, 99 total steps
+- **Baseline**: 108/108 steps, 272 ops, 49s total, 0.5s/step (stored in `baseline-ref.json`)
+- **Expected agent range**: 250-400 browser ops, 2.5-4 ops/step
+- **Group count**: 47 groups, 108 total steps
## File Locations (relative to project root)
@@ -128,8 +154,9 @@ Once all 3 agents complete, use `./scripts/runner` subcommands to merge reports
|------|---------|
| `tests/optimization/subagent-context.md` | Subagent instructions (env, wrapper, recording) |
| `tests/optimization/index.md` | Group listing |
-| `tests/optimization/group-00.md` .. `group-44.md` | Task descriptions |
+| `tests/optimization/group-00.md` .. `group-46.md` | Task descriptions |
| `skills/pinchtab/SKILL.md` | PinchTab command reference (read by subagent) |
| `tests/tools/scripts/pt` | PinchTab wrapper (CWD must be `tests/tools`) |
| `tests/tools/scripts/runner` | Step recorder (CWD must be `tests/tools`) |
| `tests/tools/scripts/baseline.sh` | Baseline (subagent must NOT read this) |
+| `~/.claude/skills/pinchtab-opt/baseline-ref.json` | Stored baseline timing/ops reference for table |
diff --git a/skills/pinchtab-opt/baseline-ref.json b/skills/pinchtab-opt/baseline-ref.json
new file mode 100644
index 00000000..b1594b28
--- /dev/null
+++ b/skills/pinchtab-opt/baseline-ref.json
@@ -0,0 +1,8 @@
+{
+ "note": "Generated 2026-05-26 by running scripts/baseline.sh against Docker fixture server. Update by re-running baseline and measuring wall-clock time.",
+ "total_steps": 108,
+ "browser_ops": 272,
+ "ops_per_step": 2.5,
+ "total_time_ms": 49772,
+ "avg_time_per_step_ms": 461
+}
diff --git a/tests/tools/runner/internal/opt/summarize.go b/tests/tools/runner/internal/opt/summarize.go
index ed1cee0f..870ded87 100644
--- a/tests/tools/runner/internal/opt/summarize.go
+++ b/tests/tools/runner/internal/opt/summarize.go
@@ -15,6 +15,7 @@ import (
func RunSummarize(argv []string, stdout, stderr io.Writer) int {
var reportFile string
+ var baselineFile string
var transcriptFiles []string
i := 0
@@ -27,6 +28,13 @@ func RunSummarize(argv []string, stdout, stderr io.Writer) int {
}
reportFile = argv[i+1]
i += 2
+ case "--baseline", "-b":
+ if i+1 >= len(argv) {
+ _, _ = fmt.Fprintf(stderr, "summarize: %s requires a value\n", argv[i])
+ return 1
+ }
+ baselineFile = argv[i+1]
+ i += 2
default:
_, _ = fmt.Fprintf(stderr, "summarize: unknown option: %s\n", argv[i])
return 1
@@ -35,10 +43,24 @@ func RunSummarize(argv []string, stdout, stderr io.Writer) int {
transcriptFiles = argv[i:]
if reportFile == "" {
- _, _ = fmt.Fprintln(stderr, "usage: summarize -r <merged-report.json> [transcript1.jsonl ...]")
+ _, _ = fmt.Fprintln(stderr, "usage: summarize -r <merged-report.json> [-b baseline.json] [transcript1.jsonl ...]")
return 1
}
+ // Load baseline reference if provided
+ var baselineRef map[string]any
+ if baselineFile != "" {
+ data, err := os.ReadFile(baselineFile)
+ if err != nil {
+ _, _ = fmt.Fprintf(stderr, "summarize: cannot read baseline file: %v\n", err)
+ return 1
+ }
+ if err := json.Unmarshal(data, &baselineRef); err != nil {
+ _, _ = fmt.Fprintf(stderr, "summarize: cannot parse baseline file: %v\n", err)
+ return 1
+ }
+ }
+
data, err := os.ReadFile(reportFile)
if err != nil {
_, _ = fmt.Fprintf(stderr, "summarize: %v\n", err)
@@ -211,21 +233,42 @@ func RunSummarize(argv []string, stdout, stderr io.Writer) int {
}
rows = append(rows, row{"Errors", "0", fmt.Sprintf("%d", failed)})
- // Prefer transcript-based timing, then run_usage wall clock, then per-step sum.
+ // Prefer transcript-based timing (sum across agents), then run_usage agent_durations_ms
+ // sum (parallel runs — total compute time), then wall_clock_ms, then per-step sum.
var displayDurationMs float64
if totalAgentDurationMs > 0 {
displayDurationMs = totalAgentDurationMs
- } else if wc, ok := usage["wall_clock_ms"].(float64); ok && wc > 0 {
- displayDurationMs = wc
+ } else if usage != nil {
+ if durations, ok := usage["agent_durations_ms"].([]any); ok && len(durations) > 0 {
+ var sum float64
+ for _, d := range durations {
+ if v, ok := d.(float64); ok {
+ sum += v
+ }
+ }
+ displayDurationMs = sum
+ } else if wc, ok := usage["wall_clock_ms"].(float64); ok && wc > 0 {
+ displayDurationMs = wc
+ }
} else if stepsWithDuration > 0 {
displayDurationMs = totalDurationMs
}
+ blAvgStr := ""
+ blTotalStr := ""
+ if baselineRef != nil {
+ if v, ok := baselineRef["avg_time_per_step_ms"].(float64); ok && v > 0 {
+ blAvgStr = fmt.Sprintf("%.1fs", v/1000)
+ }
+ if v, ok := baselineRef["total_time_ms"].(float64); ok && v > 0 {
+ blTotalStr = fmtDuration(v)
+ }
+ }
if displayDurationMs > 0 {
avgMs := displayDurationMs / float64(blSteps)
rows = append(rows,
row{"", "", ""},
- row{"Avg time/step", "", fmt.Sprintf("%.1fs", avgMs/1000)},
- row{"Total time", "", fmtDuration(displayDurationMs)},
+ row{"Avg time/step", blAvgStr, fmt.Sprintf("%.1fs", avgMs/1000)},
+ row{"Total time", blTotalStr, fmtDuration(displayDurationMs)},
)
}