Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.
TypeScript agentandroidandroid-automationappium-alternativeclaude-codecodexcursoremulatoriosios-automationmcpmcp-servermobilemobile-automationmobile-testingreal-devicesimulatorskills
fix: improvements to detection of elements through "hint", "content-desc" and others; also fixing coordinates (#27)
Gil Megidishfc2a95889bed
diff --git a/.mocharc.yml b/.mocharc.ymlindex df054db..57ecb90 100644--- a/.mocharc.yml+++ b/.mocharc.yml@@ -1 +1 @@-timeout: 20s+timeout: 30sdiff --git a/src/android.ts b/src/android.tsindex 2898bf4..4da601f 100644--- a/src/android.ts+++ b/src/android.ts@@ -3,19 +3,14 @@ import { execFileSync } from "child_process";import * as xml from "fast-xml-parser";-import { ActionableError, Button, Dimensions, InstalledApp, Robot, ScreenSize, SwipeDirection } from "./robot";--interface UiAutomatorBounds {- left: number;- top: number;- right: number;- bottom: number;-}+import { ActionableError, Button, InstalledApp, Robot, ScreenElement, ScreenElementRect, ScreenSize, SwipeDirection } from "./robot";interface UiAutomatorXmlNode {node: UiAutomatorXmlNode[];+ class?: string;text?: string;bounds?: string;+ hint?: string;"content-desc"?: string;}@@ -119,44 +114,48 @@ export class AndroidRobot implements Robot {return this.adb("shell", "screencap", "-p");}- private collectElements(node: UiAutomatorXmlNode, screenSize: Dimensions): any[] {- const elements: any[] = [];+ private collectElements(node: UiAutomatorXmlNode): ScreenElement[] {+ const elements: Array<ScreenElement> = [];- const getCoordinates = (element: UiAutomatorXmlNode): UiAutomatorBounds => {+ const getScreenElementRect = (element: UiAutomatorXmlNode): ScreenElementRect => {const bounds = String(element.bounds);const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];- return { left, top, right, bottom };+ return {+ x: left,+ y: top,+ width: right - left,+ height: bottom - top,+ };};if (node.node) {if (Array.isArray(node.node)) {for (const childNode of node.node) {- elements.push(...this.collectElements(childNode, screenSize));+ elements.push(...this.collectElements(childNode));}} else {- elements.push(...this.collectElements(node.node, screenSize));+ elements.push(...this.collectElements(node.node));}}- if (node.text) {- elements.push({- "text": node.text,- "coordinates": getCoordinates(node),- });- }+ if (node.text || node["content-desc"] || node.hint) {+ const element: ScreenElement = {+ type: node.class || "text",+ name: node.text,+ label: node["content-desc"] || node.hint || "",+ rect: getScreenElementRect(node),+ };- if (node["content-desc"]) {- elements.push({- "text": node["content-desc"],- "coordinates": getCoordinates(node),- });+ if (element.rect.width > 0 && element.rect.height > 0) {+ elements.push(element);+ }}return elements;}- public async getElementsOnScreen(): Promise<any[]> {+ public async getElementsOnScreen(): Promise<ScreenElement[]> {const dump = this.adb("exec-out", "uiautomator", "dump", "/dev/tty");const parser = new xml.XMLParser({@@ -167,8 +166,7 @@ export class AndroidRobot implements Robot {const parsedXml = parser.parse(dump) as UiAutomatorXml;const hierarchy = parsedXml.hierarchy;- const screenSize = await this.getScreenSize();- const elements = this.collectElements(hierarchy.node, screenSize);+ const elements = this.collectElements(hierarchy.node);return elements;}diff --git a/src/ios.ts b/src/ios.tsindex 5df4105..cb01055 100644--- a/src/ios.ts+++ b/src/ios.ts@@ -5,8 +5,8 @@ import { readFileSync, unlinkSync } from "fs";import { execFileSync } from "child_process";import { Socket } from "net";-import { ScreenElement, WebDriverAgent } from "./webdriver-agent";-import { ActionableError, Button, InstalledApp, Robot, ScreenSize, SwipeDirection } from "./robot";+import { WebDriverAgent } from "./webdriver-agent";+import { ActionableError, Button, InstalledApp, Robot, ScreenSize, SwipeDirection, ScreenElement } from "./robot";const WDA_PORT = 8100;const IOS_TUNNEL_PORT = 60105;diff --git a/src/iphone-simulator.ts b/src/iphone-simulator.tsindex f051524..8e8d640 100644--- a/src/iphone-simulator.ts+++ b/src/iphone-simulator.ts@@ -1,7 +1,7 @@import { execFileSync } from "child_process";import { WebDriverAgent } from "./webdriver-agent";-import { Button, InstalledApp, Robot, ScreenSize, SwipeDirection } from "./robot";+import { ActionableError, Button, InstalledApp, Robot, ScreenElement, ScreenSize, SwipeDirection } from "./robot";export interface Simulator {name: string;@@ -33,14 +33,21 @@ interface AppInfo {}const TIMEOUT = 30000;+const WDA_PORT = 8100;const MAX_BUFFER_SIZE = 1024 * 1024 * 4;export class Simctl implements Robot {- private readonly wda: WebDriverAgent;+ constructor(private readonly simulatorUuid: string) {}- constructor(private readonly simulatorUuid: string) {- this.wda = new WebDriverAgent("localhost", 8100);+ private async wda(): Promise<WebDriverAgent> {+ const wda = new WebDriverAgent("localhost", WDA_PORT);++ if (!(await wda.isRunning())) {+ throw new ActionableError("WebDriverAgent is not running on device (tunnel okay, port forwarding okay), please see https://github.com/mobile-next/mobile-mcp/wiki/");+ }++ return wda;}private simctl(...args: string[]): Buffer {@@ -59,7 +66,8 @@ export class Simctl implements Robot {}public async openUrl(url: string) {- await this.wda.openUrl(url);+ const wda = await this.wda();+ await wda.openUrl(url);// alternative: this.simctl("openurl", this.simulatorUuid, url);}@@ -122,27 +130,33 @@ export class Simctl implements Robot {}public async getScreenSize(): Promise<ScreenSize> {- return this.wda.getScreenSize();+ const wda = await this.wda();+ return wda.getScreenSize();}public async sendKeys(keys: string) {- return this.wda.sendKeys(keys);+ const wda = await this.wda();+ return wda.sendKeys(keys);}public async swipe(direction: SwipeDirection) {- return this.wda.swipe(direction);+ const wda = await this.wda();+ return wda.swipe(direction);}public async tap(x: number, y: number) {- return this.wda.tap(x, y);+ const wda = await this.wda();+ return wda.tap(x, y);}public async pressButton(button: Button) {- await this.wda.pressButton(button);+ const wda = await this.wda();+ return wda.pressButton(button);}- public async getElementsOnScreen(): Promise<any[]> {- return await this.wda.getElementsOnScreen();+ public async getElementsOnScreen(): Promise<ScreenElement[]> {+ const wda = await this.wda();+ return wda.getElementsOnScreen();}}diff --git a/src/robot.ts b/src/robot.tsindex c07589c..9e868a0 100644--- a/src/robot.ts+++ b/src/robot.ts@@ -1,9 +1,4 @@-export interface ElementCoordinates {- x: number,- y: number;-}-export interface Dimensions {width: number;height: number;@@ -22,6 +17,21 @@ export type SwipeDirection = "up" | "down" | "left" | "right";export type Button = "HOME" | "BACK" | "VOLUME_UP" | "VOLUME_DOWN" | "ENTER";+export interface ScreenElementRect {+ x: number;+ y: number;+ width: number;+ height: number;+}++export interface ScreenElement {+ type: string;+ label?: string;+ name?: string;+ value?: string;+ rect: ScreenElementRect;+}+export class ActionableError extends Error {constructor(message: string) {super(message);@@ -87,5 +97,5 @@ export interface Robot {* Get all elements on the screen. Works only on native apps (not webviews). Will* return a filtered list of elements that make sense to interact with.*/- getElementsOnScreen(): Promise<any[]>;+ getElementsOnScreen(): Promise<ScreenElement[]>;}diff --git a/src/server.ts b/src/server.tsindex a6da7fe..c827e20 100644--- a/src/server.ts+++ b/src/server.ts@@ -172,8 +172,8 @@ export const createMcpServer = (): McpServer => {const elements = await robot!.getElementsOnScreen();const result = elements.map(element => {- const x = Number((element.rect.x0 + element.rect.x1) / 2).toFixed(3);- const y = Number((element.rect.y0 + element.rect.y1) / 2).toFixed(3);+ const x = Number((element.rect.x + element.rect.width / 2)).toFixed(3);+ const y = Number((element.rect.y + element.rect.height / 2)).toFixed(3);return {text: element.label,diff --git a/src/webdriver-agent.ts b/src/webdriver-agent.tsindex 5900789..fabd014 100644--- a/src/webdriver-agent.ts+++ b/src/webdriver-agent.ts@@ -1,4 +1,4 @@-import { ActionableError, SwipeDirection, ScreenSize } from "./robot";+import { ActionableError, SwipeDirection, ScreenSize, ScreenElement } from "./robot";export interface SourceTreeElementRect {x: number;@@ -14,7 +14,7 @@ export interface SourceTreeElement {value?: string;rawIdentifier?: string;rect: SourceTreeElementRect;-+ isVisible?: string; // "0" or "1"children?: Array<SourceTreeElement>;}@@ -22,19 +22,6 @@ export interface SourceTree {value: SourceTreeElement;}-export interface ScreenElement {- type: string;- label?: string;- name?: string;- value?: string;- rect: {- x0: number;- y0: number;- x1: number;- y1: number;- };-}-export class WebDriverAgent {constructor(private readonly host: string, private readonly port: number) {@@ -85,8 +72,8 @@ export class WebDriverAgent {const response = await fetch(url);const json = await response.json();return {- width: json.value.screenSize.width * json.value.scale,- height: json.value.screenSize.height * json.value.scale,+ width: json.value.screenSize.width,+ height: json.value.screenSize.height,scale: json.value.scale || 1,};});@@ -175,7 +162,7 @@ export class WebDriverAgent {const acceptedTypes = ["TextField", "Button", "Switch", "Icon", "SearchField"];if (acceptedTypes.includes(source.type)) {- if (this.isVisible(source.rect)) {+ if (source.isVisible === "1" && this.isVisible(source.rect)) {if (source.label !== null || source.name !== null) {output.push({type: source.type,@@ -183,10 +170,10 @@ export class WebDriverAgent {name: source.name,value: source.value,rect: {- x0: source.rect.x,- y0: source.rect.y,- x1: source.rect.x + source.rect.width,- y1: source.rect.y + source.rect.height,+ x: source.rect.x,+ y: source.rect.y,+ width: source.rect.width,+ height: source.rect.height,},});}diff --git a/test/android.ts b/test/android.tsindex d890c98..d4b53ac 100644--- a/test/android.ts+++ b/test/android.ts@@ -43,45 +43,55 @@ describe("android", () => {it("should be able to open a url", async function() {hasOneAndroidDevice || this.skip();- await android.adb("shell", "input", "keyevent", "KEYCODE_HOME");+ await android.adb("shell", "input", "keyevent", "HOME");await android.openUrl("https://www.example.com");});it("should be able to list elements on screen", async function() {hasOneAndroidDevice || this.skip();- await android.adb("shell", "input", "keyevent", "KEYCODE_HOME");+ await android.adb("shell", "input", "keyevent", "HOME");await android.openUrl("https://www.example.com");const elements = await android.getElementsOnScreen();-- const foundTitle = elements.find(element => element.text.includes("This domain is for use in illustrative examples in documents"));- assert.ok(foundTitle);+ const foundTitle = elements.find(element => element.name?.includes("This domain is for use in illustrative examples in documents"));+ assert.ok(foundTitle, "Title element not found");// make sure navbar is present- const foundNavbar = elements.find(element => element.text === "example.com");- assert.ok(foundNavbar);+ const foundNavbar = elements.find(element => element.label === "Search or type URL" && element.name?.includes("example.com"));+ assert.ok(foundNavbar, "Navbar element not found");// this is an icon, but has accessibility text- const foundSecureIcon = elements.find(element => element.text === "Connection is secure");- assert.ok(foundSecureIcon);+ const foundSecureIcon = elements.find(element => element.name === "" && element.label === "New tab");+ assert.ok(foundSecureIcon, "Secure icon not found");});- it("should be able to send keys", async function() {+ it("should be able to send keys and tap", async function() {hasOneAndroidDevice || this.skip();- });+ await android.terminateApp("com.android.chrome");+ await android.launchApp("com.android.chrome");- it("should be able to press a button", async function() {- hasOneAndroidDevice || this.skip();- });+ const elements = await android.getElementsOnScreen();+ const searchElement = elements.find(e => e.label === "Search or type URL");+ assert.ok(searchElement !== undefined);+ await android.tap(searchElement.rect.x + searchElement.rect.width / 2, searchElement.rect.y + searchElement.rect.height / 2);- it("should be able to tap an element", async function() {- hasOneAndroidDevice || this.skip();- });+ await android.sendKeys("never gonna give you up lyrics");+ await android.pressButton("ENTER");+ await new Promise(resolve => setTimeout(resolve, 3000));- it("should be able to swipe", async function() {- hasOneAndroidDevice || this.skip();+ const elements2 = await android.getElementsOnScreen();+ const index = elements2.findIndex(e => e.name?.startsWith("We're no strangers to love"));+ assert.ok(index !== -1);});it("should be able to launch and terminate an app", async function() {hasOneAndroidDevice || this.skip();+ await android.terminateApp("com.android.chrome");+ await android.launchApp("com.android.chrome");+ await new Promise(resolve => setTimeout(resolve, 3000));+ const elements = await android.getElementsOnScreen();+ await android.terminateApp("com.android.chrome");++ const searchElement = elements.find(e => e.label === "Search or type URL");+ assert.ok(searchElement !== undefined);});});diff --git a/test/iphone-simulator.ts b/test/iphone-simulator.tsindex 15d10ee..e931775 100644--- a/test/iphone-simulator.ts+++ b/test/iphone-simulator.ts@@ -39,15 +39,28 @@ describe("iphone-simulator", () => {// make sure "General" is not visible nowconst elements2 = await simctl.getElementsOnScreen();assert.ok(elements2.findIndex(e => e.name === "com.apple.settings.general") === -1);++ // swipe up+ await simctl.swipe("up");++ // make sure "General" is visible again+ const elements3 = await simctl.getElementsOnScreen();+ assert.ok(elements3.findIndex(e => e.name === "com.apple.settings.general") !== -1);});it("should be able to send keys and press enter", async function() {hasOneSimulator || this.skip();await restartRemindersApp();+ // find new reminder elementconst elements = await simctl.getElementsOnScreen();- const addressElement = elements.filter(e => e.label === "New Reminder");- await simctl.tap(addressElement[0].rect.x0, addressElement[0].rect.y0);+ const newElement = elements.find(e => e.label === "New Reminder");+ assert.ok(newElement !== undefined, "should have found new reminder element");++ // click on new reminder+ await simctl.tap(newElement.rect.x, newElement.rect.y);++ // wait for keyboard to appearawait new Promise(resolve => setTimeout(resolve, 1000));// send keys with press button "Enter"@@ -82,6 +95,8 @@ describe("iphone-simulator", () => {const image = sharp(screenshot);const metadata = await image.metadata();const screenSize = await simctl.getScreenSize();+ assert.equal(metadata.format, "png");+ assert.equal(metadata.isProgressive, false);assert.equal(metadata.width, screenSize.width * screenSize.scale);assert.equal(metadata.height, screenSize.height * screenSize.scale);});@@ -90,13 +105,14 @@ describe("iphone-simulator", () => {hasOneSimulator || this.skip();// simply checking thato openurl with https:// launches safariawait simctl.openUrl("https://www.example.com");+ await new Promise(resolve => setTimeout(resolve, 1000));const elements = await simctl.getElementsOnScreen();assert.ok(elements.length > 0);const addressBar = elements.find(element => element.type === "TextField" && element.name === "TabBarItemTitle" && element.label === "Address");assert.ok(addressBar !== undefined, "should have address bar");- }).timeout(10000);+ });it("should be able to list apps", async function() {hasOneSimulator || this.skip();