Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.
TypeScript agentandroidandroid-automationappium-alternativeclaude-codecodexcursoremulatoriosios-automationmcpmcp-servermobilemobile-automationmobile-testingreal-devicesimulatorskills
feat: click on element by ref from list_elements_on_screen
gmegidish7feb092152d6
diff --git a/src/mobile-device.ts b/src/mobile-device.tsindex 03b166f..781236c 100644--- a/src/mobile-device.ts+++ b/src/mobile-device.ts@@ -259,6 +259,10 @@ export class MobileDevice implements Robot {this.runCommand(["io", "tap", `${Math.round(x)},${Math.round(y)}`]);}+ public async tapByRef(ref: string): Promise<void> {+ this.runCommand(["io", "tap", ref]);+ }+public async doubleTap(x: number, y: number): Promise<void> {// TODO: should move into mobilecli itself as "io doubletap"await this.tap(x, y);diff --git a/src/robot.ts b/src/robot.tsindex bee9b08..cbc0618 100644--- a/src/robot.ts+++ b/src/robot.ts@@ -160,6 +160,11 @@ export interface Robot {*/tap(x: number, y: number): Promise<void>;+ /**+ * Tap on the center of an element by its ref from the latest getElementsOnScreen (e.g. "@e5"). mobilecli only.+ */+ tapByRef?(ref: string): Promise<void>;+/*** Tap on a specific coordinate on the screen.*/diff --git a/src/server.ts b/src/server.tsindex 2a24ed1..26d27a4 100644--- a/src/server.ts+++ b/src/server.ts@@ -571,15 +571,29 @@ export const createMcpServer = (): McpServer => {tool("mobile_click_on_screen_at_coordinates","Click Screen",- "Click on the screen at given x,y coordinates. If clicking on an element, use the list_elements_on_screen tool to find the coordinates.",+ "Click on the screen, either at x,y coordinates or on an element by its ref (e.g. \"@e5\") from the latest list_elements_on_screen result. Prefer ref when the element is listed.",{device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),- x: z.coerce.number().min(0).describe("The x coordinate to click on the screen, in pixels"),- y: z.coerce.number().min(0).describe("The y coordinate to click on the screen, in pixels"),+ x: z.coerce.number().min(0).optional().describe("The x coordinate to click on the screen, in pixels. Required unless ref is given"),+ y: z.coerce.number().min(0).optional().describe("The y coordinate to click on the screen, in pixels. Required unless ref is given"),+ ref: z.string().optional().describe("Element ref from list_elements_on_screen, e.g. \"@e5\". Takes precedence over x,y"),},{ readOnlyHint: false, destructiveHint: false, openWorldHint: true },- async ({ device, x, y }) => {+ async ({ device, x, y, ref }) => {const robot = getRobotFromDevice(device);+ if (ref !== undefined) {+ if (!robot.tapByRef) {+ throw new ActionableError("Clicking by ref is not supported in legacy robot mode");+ }++ await robot.tapByRef(ref);+ return `Clicked on element ${ref}`;+ }++ if (x === undefined || y === undefined) {+ throw new ActionableError("Either ref or both x and y must be provided");+ }+await robot.tap(x, y);return `Clicked on screen at coordinates: ${x}, ${y}`;}@@ -624,7 +638,7 @@ export const createMcpServer = (): McpServer => {tool("mobile_list_elements_on_screen","List Screen Elements",- "List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.",+ "List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.",{device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")},diff --git a/test/mobile-device.test.ts b/test/mobile-device.test.tsindex 3525dff..619c2f1 100644--- a/test/mobile-device.test.ts+++ b/test/mobile-device.test.ts@@ -43,6 +43,13 @@ test.describe("MobileDevice", () => {expect(calls[0]).toEqual(["io", "swipe", "101,300,101,-100", "--device", "test-device"]);});+ test("tapByRef should pass the element ref to mobilecli io tap", async () => {+ const { device, calls } = createMockMobileDevice("");+ await device.tapByRef("@e5");++ expect(calls[0]).toEqual(["io", "tap", "@e5", "--device", "test-device"]);+ });+test("tap should keep integer coordinates unchanged", async () => {const { device, calls } = createMockMobileDevice("");await device.tap(450, 785);