Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

feat: expose raw UI hierarchy through MCP (#548)

Paul63bcc27003d7
diff --git a/src/mobile-device.ts b/src/mobile-device.ts
index 3ffe593..a58f543 100644
--- a/src/mobile-device.ts
+++ b/src/mobile-device.ts
@@ -56,6 +56,7 @@ interface DumpUIResponse {
// device screen is off or locked, so neither level can be relied on.
data?: {
elements?: UIElementResponse[];
+ rawData?: unknown;
};
}
@@ -314,6 +315,17 @@ export class MobileDevice implements Robot {
return elements.flatMap(element => flattenUIElement(element));
}
+ public async getRawUIHierarchy(): Promise<unknown> {
+ const response = this.runJsonCommand<DumpUIResponse>(["dump", "ui", "--format", "raw", "--full"]);
+ const rawData = response.data?.rawData;
+
+ if (rawData === undefined || rawData === null) {
+ throw new ActionableError("No raw UI hierarchy was returned for this device. Its screen may be off or locked, or the installed mobilecli version may not support raw UI dumps");
+ }
+
+ return rawData;
+ }
+
public async setOrientation(orientation: Orientation): Promise<void> {
this.runCommand(["device", "orientation", "set", orientation]);
}
diff --git a/src/robot.ts b/src/robot.ts
index fc67d3f..2320a66 100644
--- a/src/robot.ts
+++ b/src/robot.ts
@@ -212,6 +212,11 @@ export interface Robot {
*/
getElementsOnScreen(): Promise<ScreenElement[]>;
+ /**
+ * Get the unfiltered, platform-specific UI hierarchy. mobilecli only.
+ */
+ getRawUIHierarchy?(): Promise<unknown>;
+
/**
* Change the screen orientation of the device.
* @param orientation The desired orientation ("portrait" or "landscape")
diff --git a/src/server.ts b/src/server.ts
index 4f875d1..6e047c0 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -803,11 +803,18 @@ export const createMcpServer = (): McpServer => {
"List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates, mobile_double_tap_on_screen or mobile_long_press_on_screen_at_coordinates. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.",
{
device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
- format: z.enum(["text", "json"]).optional().describe("Output format. \"text\" (default) is one compact line per element, \"json\" is a json array"),
+ format: z.enum(["text", "json", "raw"]).optional().describe("Output format. \"text\" (default) is one compact line per element, \"json\" is a json array, \"raw\" is the unfiltered platform-specific hierarchy as JSON (mobilecli only). Raw nodes may be offscreen and do not have element refs"),
},
{ readOnlyHint: true, openWorldHint: true },
async ({ device, format = "text" }) => {
const robot = getRobotFromDevice(device);
+ if (format === "raw") {
+ if (!robot.getRawUIHierarchy) {
+ throw new ActionableError("Raw UI hierarchy is not supported in legacy robot mode");
+ }
+ return JSON.stringify(await robot.getRawUIHierarchy());
+ }
+
const elements = await robot.getElementsOnScreen();
return formatElements(elements, format);
}