Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

feat: improved swipe support with x,y,direction,duration parameters

Benjamin Myersafe162fa60a6
diff --git a/src/android.ts b/src/android.ts
index 6dc3292..4ffc8d9 100644
--- a/src/android.ts
+++ b/src/android.ts
@@ -120,7 +120,6 @@ export class AndroidRobot implements Robot {
public async swipe(direction: SwipeDirection): Promise<void> {
const screenSize = await this.getScreenSize();
const centerX = screenSize.width >> 1;
- // const centerY = screenSize[1] >> 1;
let x0: number, y0: number, x1: number, y1: number;
@@ -135,6 +134,55 @@ export class AndroidRobot implements Robot {
y0 = Math.floor(screenSize.height * 0.20);
y1 = Math.floor(screenSize.height * 0.80);
break;
+ case "left":
+ x0 = Math.floor(screenSize.width * 0.80);
+ x1 = Math.floor(screenSize.width * 0.20);
+ y0 = y1 = Math.floor(screenSize.height * 0.50);
+ break;
+ case "right":
+ x0 = Math.floor(screenSize.width * 0.20);
+ x1 = Math.floor(screenSize.width * 0.80);
+ y0 = y1 = Math.floor(screenSize.height * 0.50);
+ break;
+ default:
+ throw new ActionableError(`Swipe direction "${direction}" is not supported`);
+ }
+
+ this.adb("shell", "input", "swipe", `${x0}`, `${y0}`, `${x1}`, `${y1}`, "1000");
+ }
+
+ public async swipeFromCoordinate(x: number, y: number, direction: SwipeDirection, distance?: number): Promise<void> {
+ const screenSize = await this.getScreenSize();
+
+ let x0: number, y0: number, x1: number, y1: number;
+
+ // Use provided distance or default to 30% of screen dimension
+ const defaultDistanceY = Math.floor(screenSize.height * 0.3);
+ const defaultDistanceX = Math.floor(screenSize.width * 0.3);
+ const swipeDistanceY = distance || defaultDistanceY;
+ const swipeDistanceX = distance || defaultDistanceX;
+
+ switch (direction) {
+ case "up":
+ x0 = x1 = x;
+ y0 = y;
+ y1 = Math.max(0, y - swipeDistanceY);
+ break;
+ case "down":
+ x0 = x1 = x;
+ y0 = y;
+ y1 = Math.min(screenSize.height, y + swipeDistanceY);
+ break;
+ case "left":
+ x0 = x;
+ x1 = Math.max(0, x - swipeDistanceX);
+ y0 = y1 = y;
+ break;
+ case "right":
+ x0 = x;
+ x1 = Math.min(screenSize.width, x + swipeDistanceX);
+ y0 = y1 = y;
+ break;
default:
throw new ActionableError(`Swipe direction "${direction}" is not supported`);
}
diff --git a/src/ios.ts b/src/ios.ts
index 04f0a7d..58d1b4e 100644
--- a/src/ios.ts
+++ b/src/ios.ts
@@ -121,6 +121,11 @@ export class IosRobot implements Robot {
await wda.swipe(direction);
}
+ public async swipeFromCoordinate(x: number, y: number, direction: SwipeDirection, distance?: number): Promise<void> {
+ const wda = await this.wda();
+ await wda.swipeFromCoordinate(x, y, direction, distance);
+ }
+
public async listApps(): Promise<InstalledApp[]> {
await this.assertTunnelRunning();
diff --git a/src/iphone-simulator.ts b/src/iphone-simulator.ts
index 5d53199..912e570 100644
--- a/src/iphone-simulator.ts
+++ b/src/iphone-simulator.ts
@@ -98,11 +98,16 @@ export class Simctl implements Robot {
return wda.sendKeys(keys);
}
- public async swipe(direction: SwipeDirection) {
+ public async swipe(direction: SwipeDirection): Promise<void> {
const wda = await this.wda();
return wda.swipe(direction);
}
+ public async swipeFromCoordinate(x: number, y: number, direction: SwipeDirection, distance?: number): Promise<void> {
+ const wda = await this.wda();
+ return wda.swipeFromCoordinate(x, y, direction, distance);
+ }
+
public async tap(x: number, y: number) {
const wda = await this.wda();
return wda.tap(x, y);
diff --git a/src/robot.ts b/src/robot.ts
index 3ef5198..87b7097 100644
--- a/src/robot.ts
+++ b/src/robot.ts
@@ -55,6 +55,11 @@ export interface Robot {
*/
swipe(direction: SwipeDirection): Promise<void>;
+ /**
+ * Swipe from a specific coordinate in a direction.
+ */
+ swipeFromCoordinate(x: number, y: number, direction: SwipeDirection, distance?: number): Promise<void>;
+
/**
* Get a screenshot of the screen. Returns a Buffer that contains
* a PNG image of the screen. Will be same dimensions as getScreenSize().
diff --git a/src/server.ts b/src/server.ts
index 58940f4..2661970 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -278,12 +278,24 @@ export const createMcpServer = (): McpServer => {
"swipe_on_screen",
"Swipe on the screen",
{
- direction: z.enum(["up", "down"]).describe("The direction to swipe"),
+ direction: z.enum(["up", "down", "left", "right"]).describe("The direction to swipe"),
+ x: z.number().optional().describe("The x coordinate to start the swipe from, in pixels. If not provided, uses center of screen"),
+ y: z.number().optional().describe("The y coordinate to start the swipe from, in pixels. If not provided, uses center of screen"),
+ distance: z.number().optional().describe("The distance to swipe in pixels. Defaults to 400 pixels for iOS or 30% of screen dimension for Android"),
},
- async ({ direction }) => {
+ async ({ direction, x, y, distance }) => {
requireRobot();
- await robot!.swipe(direction);
- return `Swiped ${direction} on screen`;
+
+ if (x !== undefined && y !== undefined) {
+ // Use coordinate-based swipe
+ await robot!.swipeFromCoordinate(x, y, direction, distance);
+ const distanceText = distance ? ` ${distance} pixels` : "";
+ return `Swiped ${direction}${distanceText} from coordinates: ${x}, ${y}`;
+ } else {
+ // Use center-based swipe
+ await robot!.swipe(direction);
+ return `Swiped ${direction} on screen`;
+ }
}
);
diff --git a/src/webdriver-agent.ts b/src/webdriver-agent.ts
index 262d590..c7f2322 100644
--- a/src/webdriver-agent.ts
+++ b/src/webdriver-agent.ts
@@ -48,7 +48,16 @@ export class WebDriverAgent {
body: JSON.stringify({ capabilities: { alwaysMatch: { platformName: "iOS" } } }),
});
+ if (!response.ok) {
+ const errorText = await response.text();
+ throw new ActionableError(`Failed to create WebDriver session: ${response.status} ${errorText}`);
+ }
+
const json = await response.json();
+ if (!json.value || !json.value.sessionId) {
+ throw new ActionableError(`Invalid session response: ${JSON.stringify(json)}`);
+ }
+
return json.value.sessionId;
}
@@ -66,8 +75,8 @@ export class WebDriverAgent {
return result;
}
- public async getScreenSize(): Promise<ScreenSize> {
- return this.withinSession(async sessionUrl => {
+ public async getScreenSize(sessionUrl?: string): Promise<ScreenSize> {
+ if (sessionUrl) {
const url = `${sessionUrl}/wda/screen`;
const response = await fetch(url);
const json = await response.json();
@@ -76,7 +85,18 @@ export class WebDriverAgent {
height: json.value.screenSize.height,
scale: json.value.scale || 1,
};
- });
+ } else {
+ return this.withinSession(async sessionUrlInner => {
+ const url = `${sessionUrlInner}/wda/screen`;
+ const response = await fetch(url);
+ const json = await response.json();
+ return {
+ width: json.value.screenSize.width,
+ height: json.value.screenSize.height,
+ scale: json.value.scale || 1,
+ };
+ });
+ }
}
public async sendKeys(keys: string) {
@@ -211,22 +231,104 @@ export class WebDriverAgent {
});
}
- public async swipe(direction: SwipeDirection) {
+ public async swipe(direction: SwipeDirection): Promise<void> {
await this.withinSession(async sessionUrl => {
+ const screenSize = await this.getScreenSize(sessionUrl);
+ let x0: number, y0: number, x1: number, y1: number;
+ // Use 60% of the width/height for swipe distance
+ const verticalDistance = Math.floor(screenSize.height * 0.6);
+ const horizontalDistance = Math.floor(screenSize.width * 0.6);
+ const centerX = Math.floor(screenSize.width / 2);
+ const centerY = Math.floor(screenSize.height / 2);
+
+ switch (direction) {
+ case "up":
+ x0 = x1 = centerX;
+ y0 = centerY + Math.floor(verticalDistance / 2);
+ y1 = centerY - Math.floor(verticalDistance / 2);
+ break;
+ case "down":
+ x0 = x1 = centerX;
+ y0 = centerY - Math.floor(verticalDistance / 2);
+ y1 = centerY + Math.floor(verticalDistance / 2);
+ break;
+ case "left":
+ y0 = y1 = centerY;
+ x0 = centerX + Math.floor(horizontalDistance / 2);
+ x1 = centerX - Math.floor(horizontalDistance / 2);
+ break;
+ case "right":
+ y0 = y1 = centerY;
+ x0 = centerX - Math.floor(horizontalDistance / 2);
+ x1 = centerX + Math.floor(horizontalDistance / 2);
+ break;
+ default:
+ throw new ActionableError(`Swipe direction "${direction}" is not supported`);
+ }
- const x0 = 200;
- let y0 = 600;
- const x1 = 200;
- let y1 = 200;
+ const url = `${sessionUrl}/actions`;
+ const response = await fetch(url, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ actions: [
+ {
+ type: "pointer",
+ id: "finger1",
+ parameters: { pointerType: "touch" },
+ actions: [
+ { type: "pointerMove", duration: 0, x: x0, y: y0 },
+ { type: "pointerDown", button: 0 },
+ { type: "pointerMove", duration: 1000, x: x1, y: y1 },
+ { type: "pointerUp", button: 0 }
+ ]
+ }
+ ]
+ }),
+ });
- if (direction === "up") {
- const tmp = y0;
- y0 = y1;
- y1 = tmp;
+ if (!response.ok) {
+ const errorText = await response.text();
+ throw new ActionableError(`WebDriver actions request failed: ${response.status} ${errorText}`);
+ }
+
+ // Clear actions to ensure they complete
+ await fetch(`${sessionUrl}/actions`, {
+ method: "DELETE",
+ });
+ });
+ }
+
+ public async swipeFromCoordinate(x: number, y: number, direction: SwipeDirection, distance: number = 400): Promise<void> {
+ await this.withinSession(async sessionUrl => {
+ // Use simple coordinates like the working swipe method
+ const x0 = x;
+ const y0 = y;
+ let x1 = x;
+ let y1 = y;
+
+ // Calculate target position based on direction and distance
+ switch (direction) {
+ case "up":
+ y1 = y - distance; // Move up by specified distance
+ break;
+ case "down":
+ y1 = y + distance; // Move down by specified distance
+ break;
+ case "left":
+ x1 = x - distance; // Move left by specified distance
+ break;
+ case "right":
+ x1 = x + distance; // Move right by specified distance
+ break;
+ default:
+ throw new ActionableError(`Swipe direction "${direction}" is not supported`);
}
const url = `${sessionUrl}/actions`;
- await fetch(url, {
+ const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -240,14 +342,23 @@ export class WebDriverAgent {
actions: [
{ type: "pointerMove", duration: 0, x: x0, y: y0 },
{ type: "pointerDown", button: 0 },
- { type: "pointerMove", duration: 0, x: x1, y: y1 },
- { type: "pause", duration: 1000 },
+ { type: "pointerMove", duration: 1000, x: x1, y: y1 },
{ type: "pointerUp", button: 0 }
]
}
]
}),
});
+
+ if (!response.ok) {
+ const errorText = await response.text();
+ throw new ActionableError(`WebDriver actions request failed: ${response.status} ${errorText}`);
+ }
+
+ // Clear actions to ensure they complete
+ await fetch(`${sessionUrl}/actions`, {
+ method: "DELETE",
+ });
});
}