Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

fix: keep ref, selected, checked and enabled on screen elements (#421)

Gil Megidishde14577ba56f
diff --git a/src/mobile-device.ts b/src/mobile-device.ts
index 9fe74cb..3600b7b 100644
--- a/src/mobile-device.ts
+++ b/src/mobile-device.ts
@@ -30,6 +30,7 @@ interface DeviceInfoResponse {
}
interface UIElementResponse {
+ ref?: string;
type: string;
label?: string;
text?: string;
@@ -43,6 +44,9 @@ interface UIElementResponse {
height: number;
};
focused?: boolean;
+ selected?: boolean;
+ checked?: boolean;
+ enabled?: boolean;
children?: UIElementResponse[];
}
@@ -77,7 +81,11 @@ const flattenUIElement = (element: UIElementResponse): ScreenElement[] => {
value: element.value,
identifier: element.identifier,
rect: element.rect,
+ ref: element.ref,
focused: element.focused,
+ selected: element.selected,
+ checked: element.checked,
+ enabled: element.enabled,
};
if (!element.children) {
diff --git a/src/robot.ts b/src/robot.ts
index 563c8b0..fc32312 100644
--- a/src/robot.ts
+++ b/src/robot.ts
@@ -32,8 +32,14 @@ export interface ScreenElement {
identifier?: string;
rect: ScreenElementRect;
+ // short reference from the latest ui dump (e.g. "e5"), mobilecli only
+ ref?: string;
+
// currently only on android tv
focused?: boolean;
+ selected?: boolean;
+ checked?: boolean;
+ enabled?: boolean;
}
export class ActionableError extends Error {
diff --git a/src/server.ts b/src/server.ts
index 434bf34..600b3aa 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -624,6 +624,7 @@ export const createMcpServer = (): McpServer => {
const result = elements.map(element => {
const out: any = {
+ ref: element.ref,
type: element.type,
text: element.text,
label: element.label,
@@ -638,10 +639,23 @@ export const createMcpServer = (): McpServer => {
},
};
+ // only emit non-default states, otherwise don't confuse llm
if (element.focused) {
out.focused = true;
}
+ if (element.selected) {
+ out.selected = true;
+ }
+
+ if (element.checked) {
+ out.checked = true;
+ }
+
+ if (element.enabled === false) {
+ out.enabled = false;
+ }
+
return out;
});
diff --git a/test/mobile-device.test.ts b/test/mobile-device.test.ts
index 6ad616c..b77f908 100644
--- a/test/mobile-device.test.ts
+++ b/test/mobile-device.test.ts
@@ -50,6 +50,23 @@ test.describe("MobileDevice", () => {
});
});
+ test.describe("element state", () => {
+
+ test("getElementsOnScreen should keep ref, selected, checked and enabled from dump ui", async () => {
+ const dump = { status: "ok", data: { elements: [{
+ ref: "e1", type: "Switch", label: "Wi-Fi", rect: { x: 0, y: 0, width: 100, height: 40 },
+ selected: true, checked: true, enabled: false,
+ children: [{ ref: "e2", type: "Text", text: "child", rect: { x: 0, y: 0, width: 10, height: 10 } }],
+ }] } };
+ const { device } = createMockMobileDevice(JSON.stringify(dump));
+ const elements = await device.getElementsOnScreen();
+
+ expect(elements.map(e => e.ref)).toEqual(["e1", "e2"]);
+ expect(elements[0]).toMatchObject({ selected: true, checked: true, enabled: false });
+ expect(elements[1].selected).toBeUndefined();
+ });
+ });
+
test.describe("foreground app", () => {
test("getForegroundApp should call mobilecli apps foreground and fall back to package name when app name is missing", async () => {