Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

feat: swipe without appium

gmegidish1b9d3445391c
diff --git a/src/android.ts b/src/android.ts
index c9202e7..d486243 100644
--- a/src/android.ts
+++ b/src/android.ts
@@ -1,16 +1,27 @@
-import { execSync } from "child_process";
+import { exec, execSync } from "child_process";
+import * as xml from "fast-xml-parser";
-export const getConnectedDevices = () => {
- const devices = execSync(`adb devices`)
- .toString()
- .split("\n")
- .filter(line => !line.startsWith("List of devices attached"))
- .filter(line => line.trim() != "");
+interface Bounds {
+ left: number;
+ top: number;
+ right: number;
+ bottom: number;
+}
+
+interface ElementCoordinates {
+ x: number,
+ y: number;
+};
- return devices.length > 0;
+export const getConnectedDevices = (): string[] => {
+ return execSync(`adb devices`)
+ .toString()
+ .split("\n")
+ .filter(line => !line.startsWith("List of devices attached"))
+ .filter(line => line.trim() != "");
};
-export const resolveLaunchableActivities = async (packageName: string): Promise<string[]> => {
+export const resolveLaunchableActivities = (packageName: string): string[] => {
return execSync(`adb shell cmd package resolve-activity ${packageName}`)
.toString()
.split("\n")
@@ -18,3 +29,93 @@ export const resolveLaunchableActivities = async (packageName: string): Promise<
.filter(line => line.startsWith("name="))
.map(line => line.substring("name=".length));
}
+
+export const getScreenSize = (): [number, number] => {
+ const screenSize = execSync('adb shell wm size')
+ .toString()
+ .split(" ")
+ .pop();
+
+ if (!screenSize) {
+ throw new Error("Failed to get screen size");
+ }
+
+ const [width, height] = screenSize.split("x").map(Number);
+ return [width, height];
+}
+
+export const getElementCoordinates = (text: string): ElementCoordinates => {
+ const dump = execSync(`adb exec-out uiautomator dump /dev/tty`);
+
+ const parser = new xml.XMLParser({
+ ignoreAttributes: false,
+ attributeNamePrefix: ""
+ });
+
+ const parsedXml = parser.parse(dump);
+ const hierarchy = parsedXml.hierarchy;
+
+ // Function to recursively search for text elements in the UI hierarchy
+ const findTextElement = (node: any): any => {
+ // Base case: if node is null or undefined
+ if (!node) {
+ return null;
+ }
+
+ // Check if current node has the text we're looking for
+ if (node.text && node.text.includes(text)) {
+ return node;
+ }
+
+ // If node has a "node" property
+ if (node.node) {
+ // If node.node is an array, search in each element
+ if (Array.isArray(node.node)) {
+ for (const childNode of node.node) {
+ const result = findTextElement(childNode);
+ if (result) {
+ return result;
+ }
+ }
+ }
+ // If node.node is an object, recurse on it
+ else {
+ const result = findTextElement(node.node);
+ if (result) {
+ return result;
+ }
+ }
+ }
+
+ return null;
+ };
+
+ const textElement = findTextElement(hierarchy);
+
+ if (!textElement) {
+ console.log(`Element with text "${text}" not found`);
+ throw new Error(`Element with text "${text}" was not found on screen`);
+ }
+
+ const getCoordinates = (element: any): Bounds => {
+ const bounds = String(element.bounds);
+ const [_, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
+ return { left, top, right, bottom };
+ }
+
+ const getCenter = (coordinates: Bounds): ElementCoordinates => {
+ return {
+ x: Math.floor((coordinates.left + coordinates.right) / 2),
+ y: Math.floor((coordinates.top + coordinates.bottom) / 2),
+ };
+ }
+
+ return getCenter(getCoordinates(textElement));
+
+ /*
+ const jsonOutput = JSON.stringify(textElement, null, 2);
+ console.dir(jsonOutput.toString());
+ console.dir(getCoordinates(textElement));
+ console.dir(getCenter(getCoordinates(textElement)));
+ */
+}
diff --git a/src/server.ts b/src/server.ts
index a455d4f..18f2d9a 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -6,7 +6,7 @@ import { execSync } from 'child_process';
import { remote } from 'webdriverio';
import { trace } from './logger';
import { z, ZodRawShape, ZodTypeAny } from "zod";
-import { resolveLaunchableActivities } from './android';
+import { getElementCoordinates, getScreenSize, resolveLaunchableActivities } from './android';
enum APPIUM_SERVER_STATE {
STOPPED,
@@ -140,12 +140,13 @@ export const createMcpServer = (): McpServer => {
);
tool(
- "click-element",
- "Click on an element on device",
+ "find-element-on-screen",
+ "Find coordinates of an element on device by text",
{
- text: z.string().describe("Text of the element to click"),
+ text: z.string().describe("Text of the element to find"),
},
async ({ text }) => {
+ /*
const element = await driver.$(`//*[contains(@text, "${text}")]`);
if (!element) {
@@ -154,7 +155,9 @@ export const createMcpServer = (): McpServer => {
}
await element.click();
- return `Clicked on element with text "${text}"`;
+ */
+ const coordinates = getElementCoordinates(text);
+ return `Found element with text "${text}" at coordinates: ${coordinates.x},${coordinates.y}`;
}
);
@@ -185,25 +188,12 @@ export const createMcpServer = (): McpServer => {
"Swipe down on the screen",
{},
async ({}) => {
- const screenSize = await driver.getWindowSize();
- const centerX = screenSize.width / 2;
- const y0 = screenSize.height * 0.90;
- const y1 = screenSize.height * 0.10;
-
- await driver.performActions([{
- type: 'pointer',
- id: 'finger1',
- parameters: { pointerType: 'touch' },
- actions: [
- { type: 'pointerMove', duration: 0, x: centerX, y: y1 }, // start point
- { type: 'pointerDown', button: 0 },
- { type: 'pause', duration: 100 },
- { type: 'pointerMove', duration: 500, x: centerX, y: y0 }, // swipe up
- { type: 'pointerUp', button: 0 }
- ]
- }]);
-
- await driver.releaseActions();
+ const screenSize = getScreenSize();
+ const centerX = screenSize[0] / 2;
+ const y0 = screenSize[1] * 0.90;
+ const y1 = screenSize[1] * 0.10;
+
+ execSync(`adb shell input swipe ${centerX} ${y1} ${centerX} ${y0} 250`);
return `Swiped down on screen`;
}
);
@@ -246,7 +236,7 @@ export const createMcpServer = (): McpServer => {
async ({}) => {
try {
const screenshot = execSync(`adb exec-out screencap -p`);
- writeFileSync('/tmp/screenshot.png', screenshot);
+ // debug: writeFileSync('/tmp/screenshot.png', screenshot);
const screenshot64 = screenshot.toString('base64');
trace(`Screenshot taken: ${screenshot.length} bytes`);