Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

Add AndroidTV specific tools 'navigate to item with label' and 'press dpad' (#38)

Suraj Kumar Sau399443d51928
diff --git a/src/android.ts b/src/android.ts
index 0b64525..8e12cbe 100644
--- a/src/android.ts
+++ b/src/android.ts
@@ -11,6 +11,7 @@ interface UiAutomatorXmlNode {
text?: string;
bounds?: string;
hint?: string;
+ focused?: string;
"content-desc"?: string;
}
@@ -40,9 +41,24 @@ const BUTTON_MAP: Record<Button, string> = {
const TIMEOUT = 30000;
const MAX_BUFFER_SIZE = 1024 * 1024 * 4;
+type AndroidDeviceType = "tv" | "standard";
+
+type DpadButton = "DPAD_UP" | "DPAD_DOWN" | "DPAD_LEFT" | "DPAD_RIGHT" | "DPAD_CENTER";
+
export class AndroidRobot implements Robot {
+ public deviceType: AndroidDeviceType = "standard"; // Default to standard
+
public constructor(private deviceId: string) {
+ // --- Device Type Detection ---
+ try {
+ const features = this.adb("shell", "pm", "list", "features").toString();
+ if (features.includes("android.software.leanback") || features.includes("android.hardware.type.television")) {
+ this.deviceType = "tv";
+ }
+ } catch (error: any) {
+ // Defaulting to 'standard' is already set
+ }
}
public adb(...args: string[]): Buffer {
@@ -117,18 +133,6 @@ export class AndroidRobot implements Robot {
private collectElements(node: UiAutomatorXmlNode): ScreenElement[] {
const elements: Array<ScreenElement> = [];
- const getScreenElementRect = (element: UiAutomatorXmlNode): ScreenElementRect => {
- const bounds = String(element.bounds);
-
- const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
- return {
- x: left,
- y: top,
- width: right - left,
- height: bottom - top,
- };
- };
-
if (node.node) {
if (Array.isArray(node.node)) {
for (const childNode of node.node) {
@@ -144,7 +148,7 @@ export class AndroidRobot implements Robot {
type: node.class || "text",
name: node.text,
label: node["content-desc"] || node.hint || "",
- rect: getScreenElementRect(node),
+ rect: this.getScreenElementRect(node),
};
if (element.rect.width > 0 && element.rect.height > 0) {
@@ -156,14 +160,7 @@ export class AndroidRobot implements Robot {
}
public async getElementsOnScreen(): Promise<ScreenElement[]> {
- const dump = this.adb("exec-out", "uiautomator", "dump", "/dev/tty");
-
- const parser = new xml.XMLParser({
- ignoreAttributes: false,
- attributeNamePrefix: ""
- });
-
- const parsedXml = parser.parse(dump) as UiAutomatorXml;
+ const parsedXml = this.getParsedXml();
const hierarchy = parsedXml.hierarchy;
const elements = this.collectElements(hierarchy.node);
@@ -237,6 +234,162 @@ export class AndroidRobot implements Robot {
return rotation === "0" ? "portrait" : "landscape";
}
+
+ private getParsedXml(): UiAutomatorXml {
+ const dump = this.adb("exec-out", "uiautomator", "dump", "/dev/tty");
+
+ const parser = new xml.XMLParser({
+ ignoreAttributes: false,
+ attributeNamePrefix: ""
+ });
+
+ return parser.parse(dump) as UiAutomatorXml;
+ }
+
+ private getScreenElementRect(node: UiAutomatorXmlNode): ScreenElementRect {
+ const bounds = String(node.bounds);
+
+ const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
+ return {
+ x: left,
+ y: top,
+ width: right - left,
+ height: bottom - top,
+ };
+ }
+
+ // --- TV Specific Methods ---
+
+ public navigateToItemWithLabel(label: string) {
+ this.requireAndroidTv();
+ let currentDirection = this.getNextDpadDirectionToItemWithLabel(label);
+
+ while (currentDirection) {
+ this.pressDpadInternal(currentDirection);
+ currentDirection = this.getNextDpadDirectionToItemWithLabel(label);
+ }
+ }
+
+ public pressDpad(dpadButton: DpadButton) {
+ this.requireAndroidTv();
+ this.pressDpadInternal(dpadButton);
+ }
+
+ private getNextDpadDirectionToItemWithLabel(label: string): DpadButton | null {
+ const parsedXml = this.getParsedXml();
+ const targetElement = this.findElemenWithLabel(parsedXml.hierarchy.node, label);
+ const focusedElement = this.findFocusedElement(parsedXml.hierarchy.node);
+
+ if (!focusedElement || !targetElement) {
+ return null;
+ }
+
+ const focusedRect = this.getScreenElementRect(focusedElement);
+ const targetRect = this.getScreenElementRect(targetElement);
+
+ return this.getDpadDirection(focusedRect, targetRect.x, targetRect.y);
+ }
+
+ /**
+ * Find the element with the specified label in the UI hierarchy.
+ *
+ * @param node - The root node of the UI hierarchy.
+ * @param label - The label to search for.
+ * @returns The element node or null if not found.
+ */
+ private findElemenWithLabel(node: UiAutomatorXmlNode, label: string): UiAutomatorXmlNode | null {
+ if (node["text"] === label || node["content-desc"] === label || node.hint === label) {
+ return node;
+ }
+
+ if (node.node) {
+ if (Array.isArray(node.node)) {
+ for (const childNode of node.node) {
+ const focusedChild = this.findElemenWithLabel(childNode, label);
+ if (focusedChild) {
+ return focusedChild;
+ }
+ }
+ } else {
+ const focusedChild = this.findElemenWithLabel(node.node, label);
+ if (focusedChild) {
+ return focusedChild;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Find the focused element in the UI hierarchy.
+ *
+ * @param node - The root node of the UI hierarchy.
+ * @returns The focused element node or null if not found.
+ */
+ private findFocusedElement(node: UiAutomatorXmlNode): UiAutomatorXmlNode | null {
+ if (node["focused"] === "true") {
+ return node;
+ }
+
+ if (node.node) {
+ if (Array.isArray(node.node)) {
+ for (const childNode of node.node) {
+ const focusedChild = this.findFocusedElement(childNode);
+ if (focusedChild) {
+ return focusedChild;
+ }
+ }
+ } else {
+ const focusedChild = this.findFocusedElement(node.node);
+ if (focusedChild) {
+ return focusedChild;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Get the dpad direction based on the target coordinates.
+ *
+ * @param focusedRect - The focused element.
+ * @param targetX - The target x coordinate.
+ * @param targetY - The target y coordinate.
+ *
+ * @returns The dpad direction or null if no dpad direction is needed.
+ */
+ private getDpadDirection(focusedRect: ScreenElementRect, targetX: number, targetY: number): DpadButton | null {
+ // If target matches the focused element's coordinate, it means that we are already on the target.
+ // No need to press any dpad button further.
+ if (focusedRect.x === targetX && focusedRect.y === targetY) {
+ return null;
+ }
+
+ if (focusedRect.x < targetX) {
+ return "DPAD_RIGHT";
+ } else if (focusedRect.x > targetX) {
+ return "DPAD_LEFT";
+ } else if (focusedRect.y < targetY) {
+ return "DPAD_DOWN";
+ } else if (focusedRect.y > targetY) {
+ return "DPAD_UP";
+ }
+
+ // No further valid cases to be covered
+ return null;
+ }
+
+ private async pressDpadInternal(dpadButton: DpadButton): Promise<void> {
+ this.adb("shell", "input", "keyevent", dpadButton);
+ }
+
+ private requireAndroidTv() {
+ if (this.deviceType !== "tv") {
+ throw new ActionableError("This method is only supported on Android TV devices. Let the user about it and stop executing further commands.");
+ }
+ }
}
export const getConnectedDevices = (): string[] => {
diff --git a/src/server.ts b/src/server.ts
index a1f9f3f..9451c33 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -62,6 +62,13 @@ export const createMcpServer = (): McpServer => {
}
};
+ const requireTvRobot = () => {
+ requireRobot();
+ if (!(robot instanceof AndroidRobot && robot.deviceType === "tv")) {
+ throw new ActionableError("This tool is only supported on Android TV devices. Let user know about this and stop executing further commands.");
+ }
+ };
+
tool(
"mobile_list_available_devices",
"List all available devices. This includes both physical devices and simulators. If there is more than one device returned, you need to let the user select one of them.",
@@ -96,7 +103,9 @@ export const createMcpServer = (): McpServer => {
break;
}
- return `Selected device: ${device} (${deviceType})`;
+ const isAndroidTv = (robot instanceof AndroidRobot && robot.deviceType === "tv");
+
+ return `Selected device: ${device} (${deviceType}).${isAndroidTv ? " This is an AndroidTV. Use tv specific tools for navigation and selecting" : ""}`;
}
);
@@ -309,5 +318,33 @@ export const createMcpServer = (): McpServer => {
}
);
+ tool(
+ "tv_dpad_navigate_to_item_with_label",
+ "Navigate to an item on screen with a specific label using D-pad. This is specifically for TV devices which depend on D-pad based traversal.",
+ {
+ label: z.string().describe("The label of the item to navigate to"),
+ },
+ async ({ label }) => {
+ requireTvRobot();
+ (robot as AndroidRobot).navigateToItemWithLabel(label);
+
+ return `Navigated with D-pad to item with label: ${label}`;
+ }
+ );
+
+ tool(
+ "tv_dpad_press_button",
+ "Press a button on the D-pad. This is specifically for TV Devices which depend on D-pad.",
+ {
+ button: z.string().describe("The D-pad button to press. Supported buttons: DPAD_CENTER (center), DPAD_UP(up), DPAD_DOWN(down), DPAD_LEFT(left), DPAD_RIGHT(right)"),
+ },
+ async ({ button }) => {
+ requireTvRobot();
+ (robot as AndroidRobot).pressDpad(button);
+
+ return `Pressed D-pad button: ${button}`;
+ }
+ );
+
return server;
};