High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: add keyboard commands (type, inserttext, keydown, keyup) (#293) (#304)
Alice Losasso46debf4e044b
diff --git a/cmd/pinchtab/cmd_cli.go b/cmd/pinchtab/cmd_cli.goindex d7420943..c24b214f 100644--- a/cmd/pinchtab/cmd_cli.go+++ b/cmd/pinchtab/cmd_cli.go@@ -428,6 +428,59 @@ var uncheckCmd = &cobra.Command{},}+var keyboardCmd = &cobra.Command{+ Use: "keyboard",+ Short: "Keyboard commands (type, inserttext)",+}++var keyboardTypeCmd = &cobra.Command{+ Use: "type <text>",+ Short: "Type text at current focus via keystroke events",+ Args: cobra.MinimumNArgs(1),+ Run: func(cmd *cobra.Command, args []string) {+ cfg := config.Load()+ runCLIWith(cfg, func(client *http.Client, base, token string) {+ browseractions.ActionSimple(client, base, token, "keyboard-type", args, cmd)+ })+ },+}++var keyboardInsertTextCmd = &cobra.Command{+ Use: "inserttext <text>",+ Short: "Insert text at current focus (paste-like, no key events)",+ Args: cobra.MinimumNArgs(1),+ Run: func(cmd *cobra.Command, args []string) {+ cfg := config.Load()+ runCLIWith(cfg, func(client *http.Client, base, token string) {+ browseractions.ActionSimple(client, base, token, "keyboard-inserttext", args, cmd)+ })+ },+}++var keydownCmd = &cobra.Command{+ Use: "keydown <key>",+ Short: "Hold a key down",+ Args: cobra.ExactArgs(1),+ Run: func(cmd *cobra.Command, args []string) {+ cfg := config.Load()+ runCLIWith(cfg, func(client *http.Client, base, token string) {+ browseractions.ActionSimple(client, base, token, "keydown", args, cmd)+ })+ },+}++var keyupCmd = &cobra.Command{+ Use: "keyup <key>",+ Short: "Release a key",+ Args: cobra.ExactArgs(1),+ Run: func(cmd *cobra.Command, args []string) {+ cfg := config.Load()+ runCLIWith(cfg, func(client *http.Client, base, token string) {+ browseractions.ActionSimple(client, base, token, "keyup", args, cmd)+ })+ },+}+func init() {quickCmd.GroupID = "browser"navCmd.GroupID = "browser"@@ -459,6 +512,9 @@ func init() {uncheckCmd.GroupID = "browser"networkCmd.GroupID = "browser"dialogCmd.GroupID = "browser"+ keyboardCmd.GroupID = "browser"+ keydownCmd.GroupID = "browser"+ keyupCmd.GroupID = "browser"tabsCmd.AddCommand(&cobra.Command{Use: "new [url]",@@ -563,6 +619,10 @@ func init() {evalCmd.Flags().String("tab", "", "Tab ID")checkCmd.Flags().String("tab", "", "Tab ID")uncheckCmd.Flags().String("tab", "", "Tab ID")+ keyboardTypeCmd.Flags().String("tab", "", "Tab ID")+ keyboardInsertTextCmd.Flags().String("tab", "", "Tab ID")+ keydownCmd.Flags().String("tab", "", "Tab ID")+ keyupCmd.Flags().String("tab", "", "Tab ID")rootCmd.AddCommand(quickCmd)rootCmd.AddCommand(navCmd)@@ -594,6 +654,12 @@ func init() {rootCmd.AddCommand(checkCmd)rootCmd.AddCommand(uncheckCmd)rootCmd.AddCommand(networkCmd)+ rootCmd.AddCommand(keyboardCmd)+ rootCmd.AddCommand(keydownCmd)+ rootCmd.AddCommand(keyupCmd)++ keyboardCmd.AddCommand(keyboardTypeCmd)+ keyboardCmd.AddCommand(keyboardInsertTextCmd)networkCmd.Flags().String("tab", "", "Tab ID")networkCmd.Flags().String("filter", "", "URL pattern filter")diff --git a/internal/bridge/actions.go b/internal/bridge/actions.goindex c55598b6..39f86add 100644--- a/internal/bridge/actions.go+++ b/internal/bridge/actions.go@@ -13,20 +13,24 @@ import ()const (- ActionClick = "click"- ActionDoubleClick = "dblclick"- ActionType = "type"- ActionFill = "fill"- ActionPress = "press"- ActionFocus = "focus"- ActionHover = "hover"- ActionSelect = "select"- ActionScroll = "scroll"- ActionDrag = "drag"- ActionHumanClick = "humanClick"- ActionHumanType = "humanType"- ActionCheck = "check"- ActionUncheck = "uncheck"+ ActionClick = "click"+ ActionDoubleClick = "dblclick"+ ActionType = "type"+ ActionFill = "fill"+ ActionPress = "press"+ ActionFocus = "focus"+ ActionHover = "hover"+ ActionSelect = "select"+ ActionScroll = "scroll"+ ActionDrag = "drag"+ ActionHumanClick = "humanClick"+ ActionHumanType = "humanType"+ ActionCheck = "check"+ ActionUncheck = "uncheck"+ ActionKeyboardType = "keyboard-type"+ ActionKeyboardInsert = "keyboard-inserttext"+ ActionKeyDown = "keydown"+ ActionKeyUp = "keyup")func (b *Bridge) InitActionRegistry() {@@ -247,6 +251,91 @@ func (b *Bridge) InitActionRegistry() {ActionUncheck: func(ctx context.Context, req ActionRequest) (map[string]any, error) {return checkUncheck(ctx, req, false)},+ ActionKeyboardType: func(ctx context.Context, req ActionRequest) (map[string]any, error) {+ if req.Text == "" {+ return nil, fmt.Errorf("text required for keyboard-type")+ }+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ for _, ch := range req.Text {+ s := string(ch)+ params := map[string]any{+ "type": "keyDown",+ "text": s,+ "key": s,+ "unmodifiedText": s,+ "windowsVirtualKeyCode": int(ch),+ "nativeVirtualKeyCode": int(ch),+ }+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchKeyEvent", params, nil); err != nil {+ return err+ }+ paramsUp := map[string]any{+ "type": "keyUp",+ "key": s,+ "windowsVirtualKeyCode": int(ch),+ "nativeVirtualKeyCode": int(ch),+ }+ if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchKeyEvent", paramsUp, nil); err != nil {+ return err+ }+ }+ return nil+ }))+ if err != nil {+ return nil, err+ }+ return map[string]any{"typed": req.Text}, nil+ },+ ActionKeyboardInsert: func(ctx context.Context, req ActionRequest) (map[string]any, error) {+ if req.Text == "" {+ return nil, fmt.Errorf("text required for keyboard-inserttext")+ }+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.insertText", map[string]any{+ "text": req.Text,+ }, nil)+ }))+ if err != nil {+ return nil, err+ }+ return map[string]any{"inserted": req.Text}, nil+ },+ ActionKeyDown: func(ctx context.Context, req ActionRequest) (map[string]any, error) {+ if req.Key == "" {+ return nil, fmt.Errorf("key required for keydown")+ }+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ params := map[string]any{"type": "keyDown", "key": req.Key}+ if def, ok := namedKeyDefs[req.Key]; ok {+ params["code"] = def.code+ params["windowsVirtualKeyCode"] = def.virtualKey+ params["nativeVirtualKeyCode"] = def.virtualKey+ }+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchKeyEvent", params, nil)+ }))+ if err != nil {+ return nil, err+ }+ return map[string]any{"keydown": req.Key}, nil+ },+ ActionKeyUp: func(ctx context.Context, req ActionRequest) (map[string]any, error) {+ if req.Key == "" {+ return nil, fmt.Errorf("key required for keyup")+ }+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ params := map[string]any{"type": "keyUp", "key": req.Key}+ if def, ok := namedKeyDefs[req.Key]; ok {+ params["code"] = def.code+ params["windowsVirtualKeyCode"] = def.virtualKey+ params["nativeVirtualKeyCode"] = def.virtualKey+ }+ return chromedp.FromContext(ctx).Target.Execute(ctx, "Input.dispatchKeyEvent", params, nil)+ }))+ if err != nil {+ return nil, err+ }+ return map[string]any{"keyup": req.Key}, nil+ },}}diff --git a/internal/bridge/actions_test.go b/internal/bridge/actions_test.goindex 2c76d0b9..456d0e39 100644--- a/internal/bridge/actions_test.go+++ b/internal/bridge/actions_test.go@@ -140,3 +140,117 @@ func TestXpathString(t *testing.T) {})}}++// ── Keyboard action tests ──────────────────────────────────────────────++func TestKeyboardTypeAction_Registered(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ if _, ok := b.Actions[ActionKeyboardType]; !ok {+ t.Fatal("ActionKeyboardType not registered in action registry")+ }+}++func TestKeyboardInsertAction_Registered(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ if _, ok := b.Actions[ActionKeyboardInsert]; !ok {+ t.Fatal("ActionKeyboardInsert not registered in action registry")+ }+}++func TestKeyDownAction_Registered(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ if _, ok := b.Actions[ActionKeyDown]; !ok {+ t.Fatal("ActionKeyDown not registered in action registry")+ }+}++func TestKeyUpAction_Registered(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ if _, ok := b.Actions[ActionKeyUp]; !ok {+ t.Fatal("ActionKeyUp not registered in action registry")+ }+}++func TestKeyboardTypeAction_RequiresText(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ _, err := b.Actions[ActionKeyboardType](context.Background(), ActionRequest{})+ if err == nil {+ t.Fatal("expected error when text is empty")+ }+ if !strings.Contains(err.Error(), "text required") {+ t.Fatalf("expected 'text required' error, got: %v", err)+ }+}++func TestKeyboardInsertAction_RequiresText(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ _, err := b.Actions[ActionKeyboardInsert](context.Background(), ActionRequest{})+ if err == nil {+ t.Fatal("expected error when text is empty")+ }+ if !strings.Contains(err.Error(), "text required") {+ t.Fatalf("expected 'text required' error, got: %v", err)+ }+}++func TestKeyDownAction_RequiresKey(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ _, err := b.Actions[ActionKeyDown](context.Background(), ActionRequest{})+ if err == nil {+ t.Fatal("expected error when key is empty")+ }+ if !strings.Contains(err.Error(), "key required") {+ t.Fatalf("expected 'key required' error, got: %v", err)+ }+}++func TestKeyUpAction_RequiresKey(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ _, err := b.Actions[ActionKeyUp](context.Background(), ActionRequest{})+ if err == nil {+ t.Fatal("expected error when key is empty")+ }+ if !strings.Contains(err.Error(), "key required") {+ t.Fatalf("expected 'key required' error, got: %v", err)+ }+}++func TestKeyboardTypeAction_WithCancelledContext(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ _, err := b.Actions[ActionKeyboardType](ctx, ActionRequest{Text: "hello"})+ if err == nil {+ t.Fatal("expected error from cancelled context")+ }+}++func TestKeyboardInsertAction_WithCancelledContext(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ _, err := b.Actions[ActionKeyboardInsert](ctx, ActionRequest{Text: "hello"})+ if err == nil {+ t.Fatal("expected error from cancelled context")+ }+}++func TestKeyDownAction_WithCancelledContext(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ _, err := b.Actions[ActionKeyDown](ctx, ActionRequest{Key: "Control"})+ if err == nil {+ t.Fatal("expected error from cancelled context")+ }+}++func TestKeyUpAction_WithCancelledContext(t *testing.T) {+ b := New(context.TODO(), nil, &config.RuntimeConfig{})+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ _, err := b.Actions[ActionKeyUp](ctx, ActionRequest{Key: "Control"})+ if err == nil {+ t.Fatal("expected error from cancelled context")+ }+}diff --git a/internal/cli/actions/actions_element.go b/internal/cli/actions/actions_element.goindex c7c4bf21..6a31ba4a 100644--- a/internal/cli/actions/actions_element.go+++ b/internal/cli/actions/actions_element.go@@ -100,6 +100,14 @@ func ActionSimple(client *http.Client, base, token, kind string, args []string,case "select":setSelectorBody(body, args[0])body["value"] = args[1]+ case "keyboard-type":+ body["text"] = strings.Join(args, " ")+ case "keyboard-inserttext":+ body["text"] = strings.Join(args, " ")+ case "keydown":+ body["key"] = args[0]+ case "keyup":+ body["key"] = args[0]}tabID, _ := cmd.Flags().GetString("tab")diff --git a/internal/cli/actions/actions_element_test.go b/internal/cli/actions/actions_element_test.goindex 8156a7e5..2146c523 100644--- a/internal/cli/actions/actions_element_test.go+++ b/internal/cli/actions/actions_element_test.go@@ -360,3 +360,117 @@ func TestSelect(t *testing.T) {t.Errorf("expected value=option2, got %v", body["value"])}}++// ── Keyboard command tests ─────────────────────────────────────────────++func TestKeyboardType(t *testing.T) {+ m := newMockServer()+ defer m.close()+ client := m.server.Client()++ cmd := newSimpleCmd()+ ActionSimple(client, m.base(), "", "keyboard-type", []string{"hello", "world"}, cmd)+ if m.lastPath != "/action" {+ t.Errorf("expected /action, got %s", m.lastPath)+ }+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["kind"] != "keyboard-type" {+ t.Errorf("expected kind=keyboard-type, got %v", body["kind"])+ }+ if body["text"] != "hello world" {+ t.Errorf("expected text='hello world', got %v", body["text"])+ }+ // Should not have selector or ref+ if _, has := body["selector"]; has {+ t.Error("keyboard-type should not have selector")+ }+ if _, has := body["ref"]; has {+ t.Error("keyboard-type should not have ref")+ }+}++func TestKeyboardInsertText(t *testing.T) {+ m := newMockServer()+ defer m.close()+ client := m.server.Client()++ cmd := newSimpleCmd()+ ActionSimple(client, m.base(), "", "keyboard-inserttext", []string{"pasted", "text"}, cmd)+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["kind"] != "keyboard-inserttext" {+ t.Errorf("expected kind=keyboard-inserttext, got %v", body["kind"])+ }+ if body["text"] != "pasted text" {+ t.Errorf("expected text='pasted text', got %v", body["text"])+ }+}++func TestKeyDown(t *testing.T) {+ m := newMockServer()+ defer m.close()+ client := m.server.Client()++ cmd := newSimpleCmd()+ ActionSimple(client, m.base(), "", "keydown", []string{"Control"}, cmd)+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["kind"] != "keydown" {+ t.Errorf("expected kind=keydown, got %v", body["kind"])+ }+ if body["key"] != "Control" {+ t.Errorf("expected key=Control, got %v", body["key"])+ }+}++func TestKeyUp(t *testing.T) {+ m := newMockServer()+ defer m.close()+ client := m.server.Client()++ cmd := newSimpleCmd()+ ActionSimple(client, m.base(), "", "keyup", []string{"Shift"}, cmd)+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["kind"] != "keyup" {+ t.Errorf("expected kind=keyup, got %v", body["kind"])+ }+ if body["key"] != "Shift" {+ t.Errorf("expected key=Shift, got %v", body["key"])+ }+}++func TestKeyDownWithTab(t *testing.T) {+ m := newMockServer()+ defer m.close()+ client := m.server.Client()++ cmd := newSimpleCmd()+ _ = cmd.Flags().Set("tab", "abc123")+ ActionSimple(client, m.base(), "", "keydown", []string{"Alt"}, cmd)+ if m.lastPath != "/tabs/abc123/action" {+ t.Errorf("expected /tabs/abc123/action, got %s", m.lastPath)+ }+ var body map[string]any+ _ = json.Unmarshal([]byte(m.lastBody), &body)+ if body["kind"] != "keydown" {+ t.Errorf("expected kind=keydown, got %v", body["kind"])+ }+ if body["key"] != "Alt" {+ t.Errorf("expected key=Alt, got %v", body["key"])+ }+}++func TestKeyboardTypeWithTab(t *testing.T) {+ m := newMockServer()+ defer m.close()+ client := m.server.Client()++ cmd := newSimpleCmd()+ _ = cmd.Flags().Set("tab", "tab42")+ ActionSimple(client, m.base(), "", "keyboard-type", []string{"test"}, cmd)+ if m.lastPath != "/tabs/tab42/action" {+ t.Errorf("expected /tabs/tab42/action, got %s", m.lastPath)+ }+}diff --git a/internal/mcp/handlers.go b/internal/mcp/handlers.goindex e84b6495..373cbe9f 100644--- a/internal/mcp/handlers.go+++ b/internal/mcp/handlers.go@@ -34,6 +34,12 @@ func handlerMap(c *Client) map[string]func(context.Context, mcp.CallToolRequest)"pinchtab_scroll": handleAction(c, "scroll"),"pinchtab_fill": handleAction(c, "fill"),+ // Keyboard (no selector)+ "pinchtab_keyboard_type": handleKeyboardText(c, "keyboard-type"),+ "pinchtab_keyboard_inserttext": handleKeyboardText(c, "keyboard-inserttext"),+ "pinchtab_keydown": handleKeyboardKey(c, "keydown"),+ "pinchtab_keyup": handleKeyboardKey(c, "keyup"),+// Content"pinchtab_eval": handleEval(c),"pinchtab_pdf": handlePDF(c),@@ -271,6 +277,44 @@ func handleAction(c *Client, kind string) func(context.Context, mcp.CallToolRequ}}+// ── Keyboard handlers (no selector) ────────────────────────────────────++func handleKeyboardText(c *Client, kind string) func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {+ return func(ctx context.Context, r mcp.CallToolRequest) (*mcp.CallToolResult, error) {+ text, err := r.RequireString("text")+ if err != nil {+ return mcp.NewToolResultError(err.Error()), nil+ }+ payload := map[string]any{"kind": kind, "text": text}+ if tabID := optString(r, "tabId"); tabID != "" {+ payload["tabId"] = tabID+ }+ body, code, err := c.Post(ctx, "/action", payload)+ if err != nil {+ return mcp.NewToolResultError(err.Error()), nil+ }+ return resultFromBytes(body, code)+ }+}++func handleKeyboardKey(c *Client, kind string) func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {+ return func(ctx context.Context, r mcp.CallToolRequest) (*mcp.CallToolResult, error) {+ key, err := r.RequireString("key")+ if err != nil {+ return mcp.NewToolResultError(err.Error()), nil+ }+ payload := map[string]any{"kind": kind, "key": key}+ if tabID := optString(r, "tabId"); tabID != "" {+ payload["tabId"] = tabID+ }+ body, code, err := c.Post(ctx, "/action", payload)+ if err != nil {+ return mcp.NewToolResultError(err.Error()), nil+ }+ return resultFromBytes(body, code)+ }+}+// ── Content handlers ───────────────────────────────────────────────────func handleEval(c *Client) func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) {diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.goindex 9dc1db1e..137ae4b6 100644--- a/internal/mcp/server_test.go+++ b/internal/mcp/server_test.go@@ -18,8 +18,8 @@ func TestNewServerRegistersAllTools(t *testing.T) {// The server should have registered all tools.// We verify by checking that NewServer doesn't panic — the panic// in NewServer fires if any tool lacks a handler.- if len(tools) != 26 {- t.Errorf("expected 26 tools, got %d", len(tools))+ if len(tools) != 30 {+ t.Errorf("expected 30 tools, got %d", len(tools))}}diff --git a/internal/mcp/tools.go b/internal/mcp/tools.goindex ce9bbf00..564ab282 100644--- a/internal/mcp/tools.go+++ b/internal/mcp/tools.go@@ -84,6 +84,28 @@ func allTools() []mcp.Tool {mcp.WithString("tabId", mcp.Description("Target tab ID")),),+ // ── Keyboard (no selector — targets focused element) ────────+ mcp.NewTool("pinchtab_keyboard_type",+ mcp.WithDescription("Type text at the currently focused element via keystroke events (keyDown + keyUp per character)"),+ mcp.WithString("text", mcp.Required(), mcp.Description("Text to type")),+ mcp.WithString("tabId", mcp.Description("Target tab ID")),+ ),+ mcp.NewTool("pinchtab_keyboard_inserttext",+ mcp.WithDescription("Insert text at the currently focused element without key events (paste-like)"),+ mcp.WithString("text", mcp.Required(), mcp.Description("Text to insert")),+ mcp.WithString("tabId", mcp.Description("Target tab ID")),+ ),+ mcp.NewTool("pinchtab_keydown",+ mcp.WithDescription("Hold a key down (e.g. modifier keys like Control, Shift, Alt)"),+ mcp.WithString("key", mcp.Required(), mcp.Description("Key to hold down (e.g. 'Control', 'Shift', 'Alt', 'a')")),+ mcp.WithString("tabId", mcp.Description("Target tab ID")),+ ),+ mcp.NewTool("pinchtab_keyup",+ mcp.WithDescription("Release a held key"),+ mcp.WithString("key", mcp.Required(), mcp.Description("Key to release (e.g. 'Control', 'Shift', 'Alt', 'a')")),+ mcp.WithString("tabId", mcp.Description("Target tab ID")),+ ),+// ── Content ─────────────────────────────────────────────────mcp.NewTool("pinchtab_eval",mcp.WithDescription("Execute JavaScript in the browser and return the result"),