Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

feat: text format for mobile_list_elements_on_screen (default)

gmegidishcf85dfb62987
diff --git a/src/format-elements.ts b/src/format-elements.ts
new file mode 100644
index 0000000..51eb9d0
--- /dev/null
+++ b/src/format-elements.ts
@@ -0,0 +1,105 @@
+import { ScreenElement } from "./robot";
+
+export type ElementsFormat = "text" | "json";
+
+const shortType = (type: string): string => type.substring(type.lastIndexOf(".") + 1);
+
+const quote = (value: string): string => JSON.stringify(value);
+
+const formatElementAsText = (element: ScreenElement): string => {
+ const parts: string[] = [];
+ if (element.ref) {
+ parts.push(element.ref);
+ }
+
+ parts.push(shortType(element.type));
+
+ if (element.text) {
+ parts.push(`text=${quote(element.text)}`);
+ }
+
+ if (element.label) {
+ parts.push(`label=${quote(element.label)}`);
+ }
+
+ if (element.name) {
+ parts.push(`name=${quote(element.name)}`);
+ }
+
+ if (element.value) {
+ parts.push(`value=${quote(element.value)}`);
+ }
+
+ if (element.identifier) {
+ parts.push(`id=${element.identifier}`);
+ }
+
+ parts.push(`at=${element.rect.x},${element.rect.y}`);
+ parts.push(`size=${element.rect.width}x${element.rect.height}`);
+
+ // only emit non-default states, otherwise don't confuse llm
+ if (element.focused) {
+ parts.push("focused");
+ }
+
+ if (element.selected) {
+ parts.push("selected");
+ }
+
+ if (element.checked) {
+ parts.push("checked");
+ }
+
+ if (element.enabled === false) {
+ parts.push("disabled");
+ }
+
+ return parts.join(" ");
+};
+
+const formatElementAsJson = (element: ScreenElement): any => {
+ const out: any = {
+ ref: element.ref,
+ type: element.type,
+ text: element.text,
+ label: element.label,
+ name: element.name,
+ value: element.value,
+ identifier: element.identifier,
+ coordinates: {
+ x: element.rect.x,
+ y: element.rect.y,
+ width: element.rect.width,
+ height: element.rect.height,
+ },
+ };
+
+ // only emit non-default states, otherwise don't confuse llm
+ if (element.focused) {
+ out.focused = true;
+ }
+
+ if (element.selected) {
+ out.selected = true;
+ }
+
+ if (element.checked) {
+ out.checked = true;
+ }
+
+ if (element.enabled === false) {
+ out.enabled = false;
+ }
+
+ return out;
+};
+
+const TEXT_HEADER = "One element per line: @ref Type text= label= name= value= id= at=x,y size=WxH [focused] [selected] [checked] [disabled]";
+
+export const formatElements = (elements: ScreenElement[], format: ElementsFormat): string => {
+ if (format === "json") {
+ return `Found these elements on screen: ${JSON.stringify(elements.map(formatElementAsJson))}`;
+ }
+
+ return [TEXT_HEADER, ...elements.map(formatElementAsText)].join("\n");
+};
diff --git a/src/robot.ts b/src/robot.ts
index cbc0618..f801de7 100644
--- a/src/robot.ts
+++ b/src/robot.ts
@@ -32,7 +32,7 @@ export interface ScreenElement {
identifier?: string;
rect: ScreenElementRect;
- // short reference from the latest ui dump (e.g. "e5"), mobilecli only
+ // short reference from the latest ui dump (e.g. "@e5"), mobilecli only
ref?: string;
// currently only on android tv
diff --git a/src/server.ts b/src/server.ts
index fb27e67..3abf9e6 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -14,6 +14,7 @@ import { PNG } from "./png";
import { Mobilecli } from "./mobilecli";
import { MobileDevice } from "./mobile-device";
import { validateOutputPath, validateFileExtension } from "./utils";
+import { formatElements } from "./format-elements";
const ALLOWED_LOG_EXTENSIONS = [".log", ".txt", ".jsonl"];
const DEFAULT_DEVICE_LOG_ENTRIES = 100;
@@ -660,51 +661,14 @@ export const createMcpServer = (): McpServer => {
"List Screen Elements",
"List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.",
{
- device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
+ device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
+ format: z.enum(["text", "json"]).optional().describe("Output format. \"text\" (default) is one compact line per element, \"json\" is a json array"),
},
{ readOnlyHint: true, openWorldHint: true },
- async ({ device }) => {
+ async ({ device, format = "text" }) => {
const robot = getRobotFromDevice(device);
const elements = await robot.getElementsOnScreen();
-
- const result = elements.map(element => {
- const out: any = {
- ref: element.ref,
- type: element.type,
- text: element.text,
- label: element.label,
- name: element.name,
- value: element.value,
- identifier: element.identifier,
- coordinates: {
- x: element.rect.x,
- y: element.rect.y,
- width: element.rect.width,
- height: element.rect.height,
- },
- };
-
- // only emit non-default states, otherwise don't confuse llm
- if (element.focused) {
- out.focused = true;
- }
-
- if (element.selected) {
- out.selected = true;
- }
-
- if (element.checked) {
- out.checked = true;
- }
-
- if (element.enabled === false) {
- out.enabled = false;
- }
-
- return out;
- });
-
- return `Found these elements on screen: ${JSON.stringify(result)}`;
+ return formatElements(elements, format);
}
);
diff --git a/test/format-elements.test.ts b/test/format-elements.test.ts
new file mode 100644
index 0000000..d95796f
--- /dev/null
+++ b/test/format-elements.test.ts
@@ -0,0 +1,36 @@
+import { expect, test } from "@playwright/test";
+
+import { formatElements } from "../src/format-elements";
+import { ScreenElement } from "../src/robot";
+
+const incrementButton: ScreenElement = {
+ ref: "@e42",
+ type: "android.widget.Button",
+ text: "+",
+ identifier: "com.mobilenext.playground:id/stepper_increment",
+ rect: { x: 1070, y: 1340, width: 120, height: 120 },
+};
+
+const focusedTextField: ScreenElement = {
+ ref: "@e32",
+ type: "android.widget.EditText",
+ text: "Hello World",
+ label: "text_field",
+ rect: { x: 48, y: 447, width: 1184, height: 149 },
+ focused: true,
+ enabled: false,
+};
+
+test("text format emits one self-describing line per element with coordinates", () => {
+ const lines = formatElements([incrementButton, focusedTextField], "text").split("\n");
+ expect(lines[0]).toContain("@ref Type");
+ expect(lines[1]).toBe("@e42 Button text=\"+\" id=com.mobilenext.playground:id/stepper_increment at=1070,1340 size=120x120");
+ expect(lines[2]).toBe("@e32 EditText text=\"Hello World\" label=\"text_field\" at=48,447 size=1184x149 focused disabled");
+});
+
+test("json format keeps the legacy shape", () => {
+ const text = formatElements([incrementButton], "json");
+ const parsed = JSON.parse(text.replace("Found these elements on screen: ", ""));
+ expect(parsed[0].coordinates).toEqual({ x: 1070, y: 1340, width: 120, height: 120 });
+ expect(parsed[0].enabled).toBeUndefined();
+});