Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.
TypeScript agentandroidandroid-automationappium-alternativeclaude-codecodexcursoremulatoriosios-automationmcpmcp-servermobilemobile-automationmobile-testingreal-devicesimulatorskills
feat: text format for mobile_list_elements_on_screen (default)
gmegidishcf85dfb62987
diff --git a/src/format-elements.ts b/src/format-elements.tsnew file mode 100644index 0000000..51eb9d0--- /dev/null+++ b/src/format-elements.ts@@ -0,0 +1,105 @@+import { ScreenElement } from "./robot";++export type ElementsFormat = "text" | "json";++const shortType = (type: string): string => type.substring(type.lastIndexOf(".") + 1);++const quote = (value: string): string => JSON.stringify(value);++const formatElementAsText = (element: ScreenElement): string => {+ const parts: string[] = [];+ if (element.ref) {+ parts.push(element.ref);+ }++ parts.push(shortType(element.type));++ if (element.text) {+ parts.push(`text=${quote(element.text)}`);+ }++ if (element.label) {+ parts.push(`label=${quote(element.label)}`);+ }++ if (element.name) {+ parts.push(`name=${quote(element.name)}`);+ }++ if (element.value) {+ parts.push(`value=${quote(element.value)}`);+ }++ if (element.identifier) {+ parts.push(`id=${element.identifier}`);+ }++ parts.push(`at=${element.rect.x},${element.rect.y}`);+ parts.push(`size=${element.rect.width}x${element.rect.height}`);++ // only emit non-default states, otherwise don't confuse llm+ if (element.focused) {+ parts.push("focused");+ }++ if (element.selected) {+ parts.push("selected");+ }++ if (element.checked) {+ parts.push("checked");+ }++ if (element.enabled === false) {+ parts.push("disabled");+ }++ return parts.join(" ");+};++const formatElementAsJson = (element: ScreenElement): any => {+ const out: any = {+ ref: element.ref,+ type: element.type,+ text: element.text,+ label: element.label,+ name: element.name,+ value: element.value,+ identifier: element.identifier,+ coordinates: {+ x: element.rect.x,+ y: element.rect.y,+ width: element.rect.width,+ height: element.rect.height,+ },+ };++ // only emit non-default states, otherwise don't confuse llm+ if (element.focused) {+ out.focused = true;+ }++ if (element.selected) {+ out.selected = true;+ }++ if (element.checked) {+ out.checked = true;+ }++ if (element.enabled === false) {+ out.enabled = false;+ }++ return out;+};++const TEXT_HEADER = "One element per line: @ref Type text= label= name= value= id= at=x,y size=WxH [focused] [selected] [checked] [disabled]";++export const formatElements = (elements: ScreenElement[], format: ElementsFormat): string => {+ if (format === "json") {+ return `Found these elements on screen: ${JSON.stringify(elements.map(formatElementAsJson))}`;+ }++ return [TEXT_HEADER, ...elements.map(formatElementAsText)].join("\n");+};diff --git a/src/robot.ts b/src/robot.tsindex cbc0618..f801de7 100644--- a/src/robot.ts+++ b/src/robot.ts@@ -32,7 +32,7 @@ export interface ScreenElement {identifier?: string;rect: ScreenElementRect;- // short reference from the latest ui dump (e.g. "e5"), mobilecli only+ // short reference from the latest ui dump (e.g. "@e5"), mobilecli onlyref?: string;// currently only on android tvdiff --git a/src/server.ts b/src/server.tsindex fb27e67..3abf9e6 100644--- a/src/server.ts+++ b/src/server.ts@@ -14,6 +14,7 @@ import { PNG } from "./png";import { Mobilecli } from "./mobilecli";import { MobileDevice } from "./mobile-device";import { validateOutputPath, validateFileExtension } from "./utils";+import { formatElements } from "./format-elements";const ALLOWED_LOG_EXTENSIONS = [".log", ".txt", ".jsonl"];const DEFAULT_DEVICE_LOG_ENTRIES = 100;@@ -660,51 +661,14 @@ export const createMcpServer = (): McpServer => {"List Screen Elements","List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.",{- device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")+ device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),+ format: z.enum(["text", "json"]).optional().describe("Output format. \"text\" (default) is one compact line per element, \"json\" is a json array"),},{ readOnlyHint: true, openWorldHint: true },- async ({ device }) => {+ async ({ device, format = "text" }) => {const robot = getRobotFromDevice(device);const elements = await robot.getElementsOnScreen();-- const result = elements.map(element => {- const out: any = {- ref: element.ref,- type: element.type,- text: element.text,- label: element.label,- name: element.name,- value: element.value,- identifier: element.identifier,- coordinates: {- x: element.rect.x,- y: element.rect.y,- width: element.rect.width,- height: element.rect.height,- },- };-- // only emit non-default states, otherwise don't confuse llm- if (element.focused) {- out.focused = true;- }-- if (element.selected) {- out.selected = true;- }-- if (element.checked) {- out.checked = true;- }-- if (element.enabled === false) {- out.enabled = false;- }-- return out;- });-- return `Found these elements on screen: ${JSON.stringify(result)}`;+ return formatElements(elements, format);});diff --git a/test/format-elements.test.ts b/test/format-elements.test.tsnew file mode 100644index 0000000..d95796f--- /dev/null+++ b/test/format-elements.test.ts@@ -0,0 +1,36 @@+import { expect, test } from "@playwright/test";++import { formatElements } from "../src/format-elements";+import { ScreenElement } from "../src/robot";++const incrementButton: ScreenElement = {+ ref: "@e42",+ type: "android.widget.Button",+ text: "+",+ identifier: "com.mobilenext.playground:id/stepper_increment",+ rect: { x: 1070, y: 1340, width: 120, height: 120 },+};++const focusedTextField: ScreenElement = {+ ref: "@e32",+ type: "android.widget.EditText",+ text: "Hello World",+ label: "text_field",+ rect: { x: 48, y: 447, width: 1184, height: 149 },+ focused: true,+ enabled: false,+};++test("text format emits one self-describing line per element with coordinates", () => {+ const lines = formatElements([incrementButton, focusedTextField], "text").split("\n");+ expect(lines[0]).toContain("@ref Type");+ expect(lines[1]).toBe("@e42 Button text=\"+\" id=com.mobilenext.playground:id/stepper_increment at=1070,1340 size=120x120");+ expect(lines[2]).toBe("@e32 EditText text=\"Hello World\" label=\"text_field\" at=48,447 size=1184x149 focused disabled");+});++test("json format keeps the legacy shape", () => {+ const text = formatElements([incrementButton], "json");+ const parsed = JSON.parse(text.replace("Found these elements on screen: ", ""));+ expect(parsed[0].coordinates).toEqual({ x: 1070, y: 1340, width: 120, height: 120 });+ expect(parsed[0].enabled).toBeUndefined();+});