Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

fix: improvements to detection of elements through "hint", "content-desc" and others; also fixing coordinates (#27)

Gil Megidishfc2a95889bed
diff --git a/.mocharc.yml b/.mocharc.yml
index df054db..57ecb90 100644
--- a/.mocharc.yml
+++ b/.mocharc.yml
@@ -1 +1 @@
-timeout: 20s
+timeout: 30s
diff --git a/src/android.ts b/src/android.ts
index 2898bf4..4da601f 100644
--- a/src/android.ts
+++ b/src/android.ts
@@ -3,19 +3,14 @@ import { execFileSync } from "child_process";
import * as xml from "fast-xml-parser";
-import { ActionableError, Button, Dimensions, InstalledApp, Robot, ScreenSize, SwipeDirection } from "./robot";
-
-interface UiAutomatorBounds {
- left: number;
- top: number;
- right: number;
- bottom: number;
-}
+import { ActionableError, Button, InstalledApp, Robot, ScreenElement, ScreenElementRect, ScreenSize, SwipeDirection } from "./robot";
interface UiAutomatorXmlNode {
node: UiAutomatorXmlNode[];
+ class?: string;
text?: string;
bounds?: string;
+ hint?: string;
"content-desc"?: string;
}
@@ -119,44 +114,48 @@ export class AndroidRobot implements Robot {
return this.adb("shell", "screencap", "-p");
}
- private collectElements(node: UiAutomatorXmlNode, screenSize: Dimensions): any[] {
- const elements: any[] = [];
+ private collectElements(node: UiAutomatorXmlNode): ScreenElement[] {
+ const elements: Array<ScreenElement> = [];
- const getCoordinates = (element: UiAutomatorXmlNode): UiAutomatorBounds => {
+ const getScreenElementRect = (element: UiAutomatorXmlNode): ScreenElementRect => {
const bounds = String(element.bounds);
const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
- return { left, top, right, bottom };
+ return {
+ x: left,
+ y: top,
+ width: right - left,
+ height: bottom - top,
+ };
};
if (node.node) {
if (Array.isArray(node.node)) {
for (const childNode of node.node) {
- elements.push(...this.collectElements(childNode, screenSize));
+ elements.push(...this.collectElements(childNode));
}
} else {
- elements.push(...this.collectElements(node.node, screenSize));
+ elements.push(...this.collectElements(node.node));
}
}
- if (node.text) {
- elements.push({
- "text": node.text,
- "coordinates": getCoordinates(node),
- });
- }
+ if (node.text || node["content-desc"] || node.hint) {
+ const element: ScreenElement = {
+ type: node.class || "text",
+ name: node.text,
+ label: node["content-desc"] || node.hint || "",
+ rect: getScreenElementRect(node),
+ };
- if (node["content-desc"]) {
- elements.push({
- "text": node["content-desc"],
- "coordinates": getCoordinates(node),
- });
+ if (element.rect.width > 0 && element.rect.height > 0) {
+ elements.push(element);
+ }
}
return elements;
}
- public async getElementsOnScreen(): Promise<any[]> {
+ public async getElementsOnScreen(): Promise<ScreenElement[]> {
const dump = this.adb("exec-out", "uiautomator", "dump", "/dev/tty");
const parser = new xml.XMLParser({
@@ -167,8 +166,7 @@ export class AndroidRobot implements Robot {
const parsedXml = parser.parse(dump) as UiAutomatorXml;
const hierarchy = parsedXml.hierarchy;
- const screenSize = await this.getScreenSize();
- const elements = this.collectElements(hierarchy.node, screenSize);
+ const elements = this.collectElements(hierarchy.node);
return elements;
}
diff --git a/src/ios.ts b/src/ios.ts
index 5df4105..cb01055 100644
--- a/src/ios.ts
+++ b/src/ios.ts
@@ -5,8 +5,8 @@ import { readFileSync, unlinkSync } from "fs";
import { execFileSync } from "child_process";
import { Socket } from "net";
-import { ScreenElement, WebDriverAgent } from "./webdriver-agent";
-import { ActionableError, Button, InstalledApp, Robot, ScreenSize, SwipeDirection } from "./robot";
+import { WebDriverAgent } from "./webdriver-agent";
+import { ActionableError, Button, InstalledApp, Robot, ScreenSize, SwipeDirection, ScreenElement } from "./robot";
const WDA_PORT = 8100;
const IOS_TUNNEL_PORT = 60105;
diff --git a/src/iphone-simulator.ts b/src/iphone-simulator.ts
index f051524..8e8d640 100644
--- a/src/iphone-simulator.ts
+++ b/src/iphone-simulator.ts
@@ -1,7 +1,7 @@
import { execFileSync } from "child_process";
import { WebDriverAgent } from "./webdriver-agent";
-import { Button, InstalledApp, Robot, ScreenSize, SwipeDirection } from "./robot";
+import { ActionableError, Button, InstalledApp, Robot, ScreenElement, ScreenSize, SwipeDirection } from "./robot";
export interface Simulator {
name: string;
@@ -33,14 +33,21 @@ interface AppInfo {
}
const TIMEOUT = 30000;
+const WDA_PORT = 8100;
const MAX_BUFFER_SIZE = 1024 * 1024 * 4;
export class Simctl implements Robot {
- private readonly wda: WebDriverAgent;
+ constructor(private readonly simulatorUuid: string) {}
- constructor(private readonly simulatorUuid: string) {
- this.wda = new WebDriverAgent("localhost", 8100);
+ private async wda(): Promise<WebDriverAgent> {
+ const wda = new WebDriverAgent("localhost", WDA_PORT);
+
+ if (!(await wda.isRunning())) {
+ throw new ActionableError("WebDriverAgent is not running on device (tunnel okay, port forwarding okay), please see https://github.com/mobile-next/mobile-mcp/wiki/");
+ }
+
+ return wda;
}
private simctl(...args: string[]): Buffer {
@@ -59,7 +66,8 @@ export class Simctl implements Robot {
}
public async openUrl(url: string) {
- await this.wda.openUrl(url);
+ const wda = await this.wda();
+ await wda.openUrl(url);
// alternative: this.simctl("openurl", this.simulatorUuid, url);
}
@@ -122,27 +130,33 @@ export class Simctl implements Robot {
}
public async getScreenSize(): Promise<ScreenSize> {
- return this.wda.getScreenSize();
+ const wda = await this.wda();
+ return wda.getScreenSize();
}
public async sendKeys(keys: string) {
- return this.wda.sendKeys(keys);
+ const wda = await this.wda();
+ return wda.sendKeys(keys);
}
public async swipe(direction: SwipeDirection) {
- return this.wda.swipe(direction);
+ const wda = await this.wda();
+ return wda.swipe(direction);
}
public async tap(x: number, y: number) {
- return this.wda.tap(x, y);
+ const wda = await this.wda();
+ return wda.tap(x, y);
}
public async pressButton(button: Button) {
- await this.wda.pressButton(button);
+ const wda = await this.wda();
+ return wda.pressButton(button);
}
- public async getElementsOnScreen(): Promise<any[]> {
- return await this.wda.getElementsOnScreen();
+ public async getElementsOnScreen(): Promise<ScreenElement[]> {
+ const wda = await this.wda();
+ return wda.getElementsOnScreen();
}
}
diff --git a/src/robot.ts b/src/robot.ts
index c07589c..9e868a0 100644
--- a/src/robot.ts
+++ b/src/robot.ts
@@ -1,9 +1,4 @@
-export interface ElementCoordinates {
- x: number,
- y: number;
-}
-
export interface Dimensions {
width: number;
height: number;
@@ -22,6 +17,21 @@ export type SwipeDirection = "up" | "down" | "left" | "right";
export type Button = "HOME" | "BACK" | "VOLUME_UP" | "VOLUME_DOWN" | "ENTER";
+export interface ScreenElementRect {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+}
+
+export interface ScreenElement {
+ type: string;
+ label?: string;
+ name?: string;
+ value?: string;
+ rect: ScreenElementRect;
+}
+
export class ActionableError extends Error {
constructor(message: string) {
super(message);
@@ -87,5 +97,5 @@ export interface Robot {
* Get all elements on the screen. Works only on native apps (not webviews). Will
* return a filtered list of elements that make sense to interact with.
*/
- getElementsOnScreen(): Promise<any[]>;
+ getElementsOnScreen(): Promise<ScreenElement[]>;
}
diff --git a/src/server.ts b/src/server.ts
index a6da7fe..c827e20 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -172,8 +172,8 @@ export const createMcpServer = (): McpServer => {
const elements = await robot!.getElementsOnScreen();
const result = elements.map(element => {
- const x = Number((element.rect.x0 + element.rect.x1) / 2).toFixed(3);
- const y = Number((element.rect.y0 + element.rect.y1) / 2).toFixed(3);
+ const x = Number((element.rect.x + element.rect.width / 2)).toFixed(3);
+ const y = Number((element.rect.y + element.rect.height / 2)).toFixed(3);
return {
text: element.label,
diff --git a/src/webdriver-agent.ts b/src/webdriver-agent.ts
index 5900789..fabd014 100644
--- a/src/webdriver-agent.ts
+++ b/src/webdriver-agent.ts
@@ -1,4 +1,4 @@
-import { ActionableError, SwipeDirection, ScreenSize } from "./robot";
+import { ActionableError, SwipeDirection, ScreenSize, ScreenElement } from "./robot";
export interface SourceTreeElementRect {
x: number;
@@ -14,7 +14,7 @@ export interface SourceTreeElement {
value?: string;
rawIdentifier?: string;
rect: SourceTreeElementRect;
-
+ isVisible?: string; // "0" or "1"
children?: Array<SourceTreeElement>;
}
@@ -22,19 +22,6 @@ export interface SourceTree {
value: SourceTreeElement;
}
-export interface ScreenElement {
- type: string;
- label?: string;
- name?: string;
- value?: string;
- rect: {
- x0: number;
- y0: number;
- x1: number;
- y1: number;
- };
-}
-
export class WebDriverAgent {
constructor(private readonly host: string, private readonly port: number) {
@@ -85,8 +72,8 @@ export class WebDriverAgent {
const response = await fetch(url);
const json = await response.json();
return {
- width: json.value.screenSize.width * json.value.scale,
- height: json.value.screenSize.height * json.value.scale,
+ width: json.value.screenSize.width,
+ height: json.value.screenSize.height,
scale: json.value.scale || 1,
};
});
@@ -175,7 +162,7 @@ export class WebDriverAgent {
const acceptedTypes = ["TextField", "Button", "Switch", "Icon", "SearchField"];
if (acceptedTypes.includes(source.type)) {
- if (this.isVisible(source.rect)) {
+ if (source.isVisible === "1" && this.isVisible(source.rect)) {
if (source.label !== null || source.name !== null) {
output.push({
type: source.type,
@@ -183,10 +170,10 @@ export class WebDriverAgent {
name: source.name,
value: source.value,
rect: {
- x0: source.rect.x,
- y0: source.rect.y,
- x1: source.rect.x + source.rect.width,
- y1: source.rect.y + source.rect.height,
+ x: source.rect.x,
+ y: source.rect.y,
+ width: source.rect.width,
+ height: source.rect.height,
},
});
}
diff --git a/test/android.ts b/test/android.ts
index d890c98..d4b53ac 100644
--- a/test/android.ts
+++ b/test/android.ts
@@ -43,45 +43,55 @@ describe("android", () => {
it("should be able to open a url", async function() {
hasOneAndroidDevice || this.skip();
- await android.adb("shell", "input", "keyevent", "KEYCODE_HOME");
+ await android.adb("shell", "input", "keyevent", "HOME");
await android.openUrl("https://www.example.com");
});
it("should be able to list elements on screen", async function() {
hasOneAndroidDevice || this.skip();
- await android.adb("shell", "input", "keyevent", "KEYCODE_HOME");
+ await android.adb("shell", "input", "keyevent", "HOME");
await android.openUrl("https://www.example.com");
const elements = await android.getElementsOnScreen();
-
- const foundTitle = elements.find(element => element.text.includes("This domain is for use in illustrative examples in documents"));
- assert.ok(foundTitle);
+ const foundTitle = elements.find(element => element.name?.includes("This domain is for use in illustrative examples in documents"));
+ assert.ok(foundTitle, "Title element not found");
// make sure navbar is present
- const foundNavbar = elements.find(element => element.text === "example.com");
- assert.ok(foundNavbar);
+ const foundNavbar = elements.find(element => element.label === "Search or type URL" && element.name?.includes("example.com"));
+ assert.ok(foundNavbar, "Navbar element not found");
// this is an icon, but has accessibility text
- const foundSecureIcon = elements.find(element => element.text === "Connection is secure");
- assert.ok(foundSecureIcon);
+ const foundSecureIcon = elements.find(element => element.name === "" && element.label === "New tab");
+ assert.ok(foundSecureIcon, "Secure icon not found");
});
- it("should be able to send keys", async function() {
+ it("should be able to send keys and tap", async function() {
hasOneAndroidDevice || this.skip();
- });
+ await android.terminateApp("com.android.chrome");
+ await android.launchApp("com.android.chrome");
- it("should be able to press a button", async function() {
- hasOneAndroidDevice || this.skip();
- });
+ const elements = await android.getElementsOnScreen();
+ const searchElement = elements.find(e => e.label === "Search or type URL");
+ assert.ok(searchElement !== undefined);
+ await android.tap(searchElement.rect.x + searchElement.rect.width / 2, searchElement.rect.y + searchElement.rect.height / 2);
- it("should be able to tap an element", async function() {
- hasOneAndroidDevice || this.skip();
- });
+ await android.sendKeys("never gonna give you up lyrics");
+ await android.pressButton("ENTER");
+ await new Promise(resolve => setTimeout(resolve, 3000));
- it("should be able to swipe", async function() {
- hasOneAndroidDevice || this.skip();
+ const elements2 = await android.getElementsOnScreen();
+ const index = elements2.findIndex(e => e.name?.startsWith("We're no strangers to love"));
+ assert.ok(index !== -1);
});
it("should be able to launch and terminate an app", async function() {
hasOneAndroidDevice || this.skip();
+ await android.terminateApp("com.android.chrome");
+ await android.launchApp("com.android.chrome");
+ await new Promise(resolve => setTimeout(resolve, 3000));
+ const elements = await android.getElementsOnScreen();
+ await android.terminateApp("com.android.chrome");
+
+ const searchElement = elements.find(e => e.label === "Search or type URL");
+ assert.ok(searchElement !== undefined);
});
});
diff --git a/test/iphone-simulator.ts b/test/iphone-simulator.ts
index 15d10ee..e931775 100644
--- a/test/iphone-simulator.ts
+++ b/test/iphone-simulator.ts
@@ -39,15 +39,28 @@ describe("iphone-simulator", () => {
// make sure "General" is not visible now
const elements2 = await simctl.getElementsOnScreen();
assert.ok(elements2.findIndex(e => e.name === "com.apple.settings.general") === -1);
+
+ // swipe up
+ await simctl.swipe("up");
+
+ // make sure "General" is visible again
+ const elements3 = await simctl.getElementsOnScreen();
+ assert.ok(elements3.findIndex(e => e.name === "com.apple.settings.general") !== -1);
});
it("should be able to send keys and press enter", async function() {
hasOneSimulator || this.skip();
await restartRemindersApp();
+ // find new reminder element
const elements = await simctl.getElementsOnScreen();
- const addressElement = elements.filter(e => e.label === "New Reminder");
- await simctl.tap(addressElement[0].rect.x0, addressElement[0].rect.y0);
+ const newElement = elements.find(e => e.label === "New Reminder");
+ assert.ok(newElement !== undefined, "should have found new reminder element");
+
+ // click on new reminder
+ await simctl.tap(newElement.rect.x, newElement.rect.y);
+
+ // wait for keyboard to appear
await new Promise(resolve => setTimeout(resolve, 1000));
// send keys with press button "Enter"
@@ -82,6 +95,8 @@ describe("iphone-simulator", () => {
const image = sharp(screenshot);
const metadata = await image.metadata();
const screenSize = await simctl.getScreenSize();
+ assert.equal(metadata.format, "png");
+ assert.equal(metadata.isProgressive, false);
assert.equal(metadata.width, screenSize.width * screenSize.scale);
assert.equal(metadata.height, screenSize.height * screenSize.scale);
});
@@ -90,13 +105,14 @@ describe("iphone-simulator", () => {
hasOneSimulator || this.skip();
// simply checking thato openurl with https:// launches safari
await simctl.openUrl("https://www.example.com");
+ await new Promise(resolve => setTimeout(resolve, 1000));
const elements = await simctl.getElementsOnScreen();
assert.ok(elements.length > 0);
const addressBar = elements.find(element => element.type === "TextField" && element.name === "TabBarItemTitle" && element.label === "Address");
assert.ok(addressBar !== undefined, "should have address bar");
- }).timeout(10000);
+ });
it("should be able to list apps", async function() {
hasOneSimulator || this.skip();