Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.
TypeScript agentandroidandroid-automationappium-alternativeclaude-codecodexcursoremulatoriosios-automationmcpmcp-servermobilemobile-automationmobile-testingreal-devicesimulatorskills
Add AndroidTV specific tools 'navigate to item with label' and 'press dpad' (#38)
Suraj Kumar Sau399443d51928
diff --git a/src/android.ts b/src/android.tsindex 0b64525..8e12cbe 100644--- a/src/android.ts+++ b/src/android.ts@@ -11,6 +11,7 @@ interface UiAutomatorXmlNode {text?: string;bounds?: string;hint?: string;+ focused?: string;"content-desc"?: string;}@@ -40,9 +41,24 @@ const BUTTON_MAP: Record<Button, string> = {const TIMEOUT = 30000;const MAX_BUFFER_SIZE = 1024 * 1024 * 4;+type AndroidDeviceType = "tv" | "standard";++type DpadButton = "DPAD_UP" | "DPAD_DOWN" | "DPAD_LEFT" | "DPAD_RIGHT" | "DPAD_CENTER";+export class AndroidRobot implements Robot {+ public deviceType: AndroidDeviceType = "standard"; // Default to standard+public constructor(private deviceId: string) {+ // --- Device Type Detection ---+ try {+ const features = this.adb("shell", "pm", "list", "features").toString();+ if (features.includes("android.software.leanback") || features.includes("android.hardware.type.television")) {+ this.deviceType = "tv";+ }+ } catch (error: any) {+ // Defaulting to 'standard' is already set+ }}public adb(...args: string[]): Buffer {@@ -117,18 +133,6 @@ export class AndroidRobot implements Robot {private collectElements(node: UiAutomatorXmlNode): ScreenElement[] {const elements: Array<ScreenElement> = [];- const getScreenElementRect = (element: UiAutomatorXmlNode): ScreenElementRect => {- const bounds = String(element.bounds);-- const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];- return {- x: left,- y: top,- width: right - left,- height: bottom - top,- };- };-if (node.node) {if (Array.isArray(node.node)) {for (const childNode of node.node) {@@ -144,7 +148,7 @@ export class AndroidRobot implements Robot {type: node.class || "text",name: node.text,label: node["content-desc"] || node.hint || "",- rect: getScreenElementRect(node),+ rect: this.getScreenElementRect(node),};if (element.rect.width > 0 && element.rect.height > 0) {@@ -156,14 +160,7 @@ export class AndroidRobot implements Robot {}public async getElementsOnScreen(): Promise<ScreenElement[]> {- const dump = this.adb("exec-out", "uiautomator", "dump", "/dev/tty");-- const parser = new xml.XMLParser({- ignoreAttributes: false,- attributeNamePrefix: ""- });-- const parsedXml = parser.parse(dump) as UiAutomatorXml;+ const parsedXml = this.getParsedXml();const hierarchy = parsedXml.hierarchy;const elements = this.collectElements(hierarchy.node);@@ -237,6 +234,162 @@ export class AndroidRobot implements Robot {return rotation === "0" ? "portrait" : "landscape";}++ private getParsedXml(): UiAutomatorXml {+ const dump = this.adb("exec-out", "uiautomator", "dump", "/dev/tty");++ const parser = new xml.XMLParser({+ ignoreAttributes: false,+ attributeNamePrefix: ""+ });++ return parser.parse(dump) as UiAutomatorXml;+ }++ private getScreenElementRect(node: UiAutomatorXmlNode): ScreenElementRect {+ const bounds = String(node.bounds);++ const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];+ return {+ x: left,+ y: top,+ width: right - left,+ height: bottom - top,+ };+ }++ // --- TV Specific Methods ---++ public navigateToItemWithLabel(label: string) {+ this.requireAndroidTv();+ let currentDirection = this.getNextDpadDirectionToItemWithLabel(label);++ while (currentDirection) {+ this.pressDpadInternal(currentDirection);+ currentDirection = this.getNextDpadDirectionToItemWithLabel(label);+ }+ }++ public pressDpad(dpadButton: DpadButton) {+ this.requireAndroidTv();+ this.pressDpadInternal(dpadButton);+ }++ private getNextDpadDirectionToItemWithLabel(label: string): DpadButton | null {+ const parsedXml = this.getParsedXml();+ const targetElement = this.findElemenWithLabel(parsedXml.hierarchy.node, label);+ const focusedElement = this.findFocusedElement(parsedXml.hierarchy.node);++ if (!focusedElement || !targetElement) {+ return null;+ }++ const focusedRect = this.getScreenElementRect(focusedElement);+ const targetRect = this.getScreenElementRect(targetElement);++ return this.getDpadDirection(focusedRect, targetRect.x, targetRect.y);+ }++ /**+ * Find the element with the specified label in the UI hierarchy.+ *+ * @param node - The root node of the UI hierarchy.+ * @param label - The label to search for.+ * @returns The element node or null if not found.+ */+ private findElemenWithLabel(node: UiAutomatorXmlNode, label: string): UiAutomatorXmlNode | null {+ if (node["text"] === label || node["content-desc"] === label || node.hint === label) {+ return node;+ }++ if (node.node) {+ if (Array.isArray(node.node)) {+ for (const childNode of node.node) {+ const focusedChild = this.findElemenWithLabel(childNode, label);+ if (focusedChild) {+ return focusedChild;+ }+ }+ } else {+ const focusedChild = this.findElemenWithLabel(node.node, label);+ if (focusedChild) {+ return focusedChild;+ }+ }+ }++ return null;+ }++ /**+ * Find the focused element in the UI hierarchy.+ *+ * @param node - The root node of the UI hierarchy.+ * @returns The focused element node or null if not found.+ */+ private findFocusedElement(node: UiAutomatorXmlNode): UiAutomatorXmlNode | null {+ if (node["focused"] === "true") {+ return node;+ }++ if (node.node) {+ if (Array.isArray(node.node)) {+ for (const childNode of node.node) {+ const focusedChild = this.findFocusedElement(childNode);+ if (focusedChild) {+ return focusedChild;+ }+ }+ } else {+ const focusedChild = this.findFocusedElement(node.node);+ if (focusedChild) {+ return focusedChild;+ }+ }+ }++ return null;+ }++ /**+ * Get the dpad direction based on the target coordinates.+ *+ * @param focusedRect - The focused element.+ * @param targetX - The target x coordinate.+ * @param targetY - The target y coordinate.+ *+ * @returns The dpad direction or null if no dpad direction is needed.+ */+ private getDpadDirection(focusedRect: ScreenElementRect, targetX: number, targetY: number): DpadButton | null {+ // If target matches the focused element's coordinate, it means that we are already on the target.+ // No need to press any dpad button further.+ if (focusedRect.x === targetX && focusedRect.y === targetY) {+ return null;+ }++ if (focusedRect.x < targetX) {+ return "DPAD_RIGHT";+ } else if (focusedRect.x > targetX) {+ return "DPAD_LEFT";+ } else if (focusedRect.y < targetY) {+ return "DPAD_DOWN";+ } else if (focusedRect.y > targetY) {+ return "DPAD_UP";+ }++ // No further valid cases to be covered+ return null;+ }++ private async pressDpadInternal(dpadButton: DpadButton): Promise<void> {+ this.adb("shell", "input", "keyevent", dpadButton);+ }++ private requireAndroidTv() {+ if (this.deviceType !== "tv") {+ throw new ActionableError("This method is only supported on Android TV devices. Let the user about it and stop executing further commands.");+ }+ }}export const getConnectedDevices = (): string[] => {diff --git a/src/server.ts b/src/server.tsindex a1f9f3f..9451c33 100644--- a/src/server.ts+++ b/src/server.ts@@ -62,6 +62,13 @@ export const createMcpServer = (): McpServer => {}};+ const requireTvRobot = () => {+ requireRobot();+ if (!(robot instanceof AndroidRobot && robot.deviceType === "tv")) {+ throw new ActionableError("This tool is only supported on Android TV devices. Let user know about this and stop executing further commands.");+ }+ };+tool("mobile_list_available_devices","List all available devices. This includes both physical devices and simulators. If there is more than one device returned, you need to let the user select one of them.",@@ -96,7 +103,9 @@ export const createMcpServer = (): McpServer => {break;}- return `Selected device: ${device} (${deviceType})`;+ const isAndroidTv = (robot instanceof AndroidRobot && robot.deviceType === "tv");++ return `Selected device: ${device} (${deviceType}).${isAndroidTv ? " This is an AndroidTV. Use tv specific tools for navigation and selecting" : ""}`;});@@ -309,5 +318,33 @@ export const createMcpServer = (): McpServer => {});+ tool(+ "tv_dpad_navigate_to_item_with_label",+ "Navigate to an item on screen with a specific label using D-pad. This is specifically for TV devices which depend on D-pad based traversal.",+ {+ label: z.string().describe("The label of the item to navigate to"),+ },+ async ({ label }) => {+ requireTvRobot();+ (robot as AndroidRobot).navigateToItemWithLabel(label);++ return `Navigated with D-pad to item with label: ${label}`;+ }+ );++ tool(+ "tv_dpad_press_button",+ "Press a button on the D-pad. This is specifically for TV Devices which depend on D-pad.",+ {+ button: z.string().describe("The D-pad button to press. Supported buttons: DPAD_CENTER (center), DPAD_UP(up), DPAD_DOWN(down), DPAD_LEFT(left), DPAD_RIGHT(right)"),+ },+ async ({ button }) => {+ requireTvRobot();+ (robot as AndroidRobot).pressDpad(button);++ return `Pressed D-pad button: ${button}`;+ }+ );+return server;};