Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

feat: click on element by ref from list_elements_on_screen

gmegidish7feb092152d6
diff --git a/src/mobile-device.ts b/src/mobile-device.ts
index 03b166f..781236c 100644
--- a/src/mobile-device.ts
+++ b/src/mobile-device.ts
@@ -259,6 +259,10 @@ export class MobileDevice implements Robot {
this.runCommand(["io", "tap", `${Math.round(x)},${Math.round(y)}`]);
}
+ public async tapByRef(ref: string): Promise<void> {
+ this.runCommand(["io", "tap", ref]);
+ }
+
public async doubleTap(x: number, y: number): Promise<void> {
// TODO: should move into mobilecli itself as "io doubletap"
await this.tap(x, y);
diff --git a/src/robot.ts b/src/robot.ts
index bee9b08..cbc0618 100644
--- a/src/robot.ts
+++ b/src/robot.ts
@@ -160,6 +160,11 @@ export interface Robot {
*/
tap(x: number, y: number): Promise<void>;
+ /**
+ * Tap on the center of an element by its ref from the latest getElementsOnScreen (e.g. "@e5"). mobilecli only.
+ */
+ tapByRef?(ref: string): Promise<void>;
+
/**
* Tap on a specific coordinate on the screen.
*/
diff --git a/src/server.ts b/src/server.ts
index 2a24ed1..26d27a4 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -571,15 +571,29 @@ export const createMcpServer = (): McpServer => {
tool(
"mobile_click_on_screen_at_coordinates",
"Click Screen",
- "Click on the screen at given x,y coordinates. If clicking on an element, use the list_elements_on_screen tool to find the coordinates.",
+ "Click on the screen, either at x,y coordinates or on an element by its ref (e.g. \"@e5\") from the latest list_elements_on_screen result. Prefer ref when the element is listed.",
{
device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
- x: z.coerce.number().min(0).describe("The x coordinate to click on the screen, in pixels"),
- y: z.coerce.number().min(0).describe("The y coordinate to click on the screen, in pixels"),
+ x: z.coerce.number().min(0).optional().describe("The x coordinate to click on the screen, in pixels. Required unless ref is given"),
+ y: z.coerce.number().min(0).optional().describe("The y coordinate to click on the screen, in pixels. Required unless ref is given"),
+ ref: z.string().optional().describe("Element ref from list_elements_on_screen, e.g. \"@e5\". Takes precedence over x,y"),
},
{ readOnlyHint: false, destructiveHint: false, openWorldHint: true },
- async ({ device, x, y }) => {
+ async ({ device, x, y, ref }) => {
const robot = getRobotFromDevice(device);
+ if (ref !== undefined) {
+ if (!robot.tapByRef) {
+ throw new ActionableError("Clicking by ref is not supported in legacy robot mode");
+ }
+
+ await robot.tapByRef(ref);
+ return `Clicked on element ${ref}`;
+ }
+
+ if (x === undefined || y === undefined) {
+ throw new ActionableError("Either ref or both x and y must be provided");
+ }
+
await robot.tap(x, y);
return `Clicked on screen at coordinates: ${x}, ${y}`;
}
@@ -624,7 +638,7 @@ export const createMcpServer = (): McpServer => {
tool(
"mobile_list_elements_on_screen",
"List Screen Elements",
- "List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.",
+ "List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.",
{
device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
},
diff --git a/test/mobile-device.test.ts b/test/mobile-device.test.ts
index 3525dff..619c2f1 100644
--- a/test/mobile-device.test.ts
+++ b/test/mobile-device.test.ts
@@ -43,6 +43,13 @@ test.describe("MobileDevice", () => {
expect(calls[0]).toEqual(["io", "swipe", "101,300,101,-100", "--device", "test-device"]);
});
+ test("tapByRef should pass the element ref to mobilecli io tap", async () => {
+ const { device, calls } = createMockMobileDevice("");
+ await device.tapByRef("@e5");
+
+ expect(calls[0]).toEqual(["io", "tap", "@e5", "--device", "test-device"]);
+ });
+
test("tap should keep integer coordinates unchanged", async () => {
const { device, calls } = createMockMobileDevice("");
await device.tap(450, 785);