Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.
TypeScript agentandroidandroid-automationappium-alternativeclaude-codecodexcursoremulatoriosios-automationmcpmcp-servermobilemobile-automationmobile-testingreal-devicesimulatorskills
fix: return coordinate mapping with every screenshot
gmegidishf7b573cc1290
diff --git a/src/coordinate-mapping.ts b/src/coordinate-mapping.tsnew file mode 100644index 0000000..77af1af--- /dev/null+++ b/src/coordinate-mapping.ts@@ -0,0 +1,30 @@+import { Dimensions } from "./robot";++const RATIO_DECIMALS = 3;++const formatRatio = (ratio: number): string => Number(ratio.toFixed(RATIO_DECIMALS)).toString();++const isPositive = (size: Dimensions): boolean => size.width > 0 && size.height > 0;++/**+ * Plain-words instruction for a model that reads pixel positions off a+ * screenshot and taps in screen coordinates. The screenshot is usually+ * downscaled, and on iOS the screen is measured in points while a full-size+ * screenshot is in pixels, so the two spaces rarely agree.+ *+ * Returns null when either size is unknown, in which case no instruction is+ * better than a wrong one.+ */+export const describeCoordinateMapping = (screenshot: Dimensions, screen: Dimensions): string | null => {+ if (!isPositive(screenshot) || !isPositive(screen)) {+ return null;+ }++ const x = formatRatio(screen.width / screenshot.width);+ const y = formatRatio(screen.height / screenshot.height);+ if (x === "1" && y === "1") {+ return `Screenshot is ${screenshot.width}x${screenshot.height} and its coordinates match the screen.`;+ }++ return `Screenshot is ${screenshot.width}x${screenshot.height}. Screen coordinates are ${screen.width}x${screen.height}. To tap something you see in this screenshot, multiply its x by ${x} and y by ${y}.`;+};diff --git a/src/jpeg.ts b/src/jpeg.tsnew file mode 100644index 0000000..9d6f869--- /dev/null+++ b/src/jpeg.ts@@ -0,0 +1,41 @@+import { ActionableError, Dimensions } from "./robot";++const JPEG_SOI = 0xffd8;+const MARKER_PREFIX = 0xff;++// SOFn markers carry the frame header with the image dimensions. 0xc4 (DHT),+// 0xc8 (JPG) and 0xcc (DAC) sit in the same range but are not frame headers.+const isStartOfFrameMarker = (marker: number): boolean =>+ marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc;++export const getJpegDimensions = (buffer: Buffer): Dimensions => {+ if (buffer.length < 4 || buffer.readUInt16BE(0) !== JPEG_SOI) {+ throw new ActionableError("Invalid JPEG");+ }++ let offset = 2;+ while (offset + 4 <= buffer.length) {+ if (buffer[offset] !== MARKER_PREFIX) {+ throw new ActionableError("Invalid JPEG");+ }++ const marker = buffer[offset + 1];+ const segmentLength = buffer.readUInt16BE(offset + 2);++ if (isStartOfFrameMarker(marker)) {+ // segment: length(2) precision(1) height(2) width(2) ...+ if (offset + 9 > buffer.length) {+ throw new ActionableError("Invalid JPEG");+ }++ return {+ height: buffer.readUInt16BE(offset + 5),+ width: buffer.readUInt16BE(offset + 7),+ };+ }++ offset += 2 + segmentLength;+ }++ throw new ActionableError("Invalid JPEG");+};diff --git a/src/server.ts b/src/server.tsindex f2f63e3..01346ba 100644--- a/src/server.ts+++ b/src/server.ts@@ -8,14 +8,18 @@ import { ChildProcess } from "node:child_process";import { error, trace } from "./logger";import { AndroidRobot, AndroidDeviceManager } from "./android";-import { ActionableError, Robot, ScreenshotOptions } from "./robot";+import { ActionableError, Dimensions, Robot, ScreenshotOptions } from "./robot";import { IosManager, IosRobot } from "./ios";import { PNG } from "./png";+import { getJpegDimensions } from "./jpeg";+import { describeCoordinateMapping } from "./coordinate-mapping";import { Mobilecli } from "./mobilecli";import { MobileDevice } from "./mobile-device";import { validateOutputPath, validateFileExtension } from "./utils";import { formatElements } from "./format-elements";+type ScreenshotContent = { type: "text", text: string } | { type: "image", data: string, mimeType: string };+const ALLOWED_LOG_EXTENSIONS = [".log", ".txt", ".jsonl"];const DEFAULT_DEVICE_LOG_ENTRIES = 100;const MAX_DEVICE_LOG_ENTRIES = 10000;@@ -806,7 +810,7 @@ export const createMcpServer = (): McpServer => {"mobile_take_screenshot",{title: "Take Screenshot",- description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. Do not cache this result.",+ description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.",inputSchema: {device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),maxSize: z.number().int().positive().optional().describe(`Maximum width/height in pixels, keeping aspect ratio. Defaults to ${DEFAULT_SCREENSHOT_MAX_SIZE}.`),@@ -836,29 +840,41 @@ export const createMcpServer = (): McpServer => {const isJpeg = screenshot.length > 2 && screenshot[0] === 0xff && screenshot[1] === 0xd8;let mimeType = "image/jpeg";- if (!isJpeg) {+ let screenshotSize: Dimensions;+ if (isJpeg) {+ screenshotSize = getJpegDimensions(screenshot);+ } else {mimeType = "image/png";// validate we received a png, will throw exception otherwiseconst image = new PNG(screenshot);- const pngSize = image.getDimensions();- if (pngSize.width <= 0 || pngSize.height <= 0) {- throw new ActionableError("Screenshot is invalid. Please try again.");- }+ screenshotSize = image.getDimensions();+ }+ if (screenshotSize.width <= 0 || screenshotSize.height <= 0) {+ throw new ActionableError("Screenshot is invalid. Please try again.");}+ // the screenshot is downscaled (and on ios the screen is in points while pixels are not),+ // so tell the model how to map what it sees onto tap coordinates. see #29.+ const screenSize = await robot.getScreenSize();+ const mapping = describeCoordinateMapping(screenshotSize, screenSize);+const screenshot64 = screenshot.toString("base64");- trace(`Screenshot taken: ${screenshot.length} bytes`);+ trace(`Screenshot taken: ${screenshot.length} bytes, ${screenshotSize.width}x${screenshotSize.height}`);posthog("tool_invoked", {"ToolName": "mobile_take_screenshot","ScreenshotFilesize": screenshot64.length,"ScreenshotMimeType": mimeType,}).then();- return {- content: [{ type: "image", data: screenshot64, mimeType }]- };+ const content: ScreenshotContent[] = [];+ if (mapping !== null) {+ content.push({ type: "text", text: mapping });+ }++ content.push({ type: "image", data: screenshot64, mimeType });+ return { content };} catch (err: any) {error(`Error taking screenshot: ${err.message} ${err.stack}`);return {diff --git a/test/coordinate-mapping.ts b/test/coordinate-mapping.tsnew file mode 100644index 0000000..2365b5c--- /dev/null+++ b/test/coordinate-mapping.ts@@ -0,0 +1,27 @@+import { test, expect } from "@playwright/test";+import { describeCoordinateMapping } from "../src/coordinate-mapping";++test.describe("coordinate mapping", () => {+ test("tells the model to multiply when the screenshot is smaller than the screen", () => {+ const text = describeCoordinateMapping({ width: 590, height: 1278 }, { width: 1179, height: 2556 });+ expect(text).toBe("Screenshot is 590x1278. Screen coordinates are 1179x2556. To tap something you see in this screenshot, multiply its x by 1.998 and y by 2.");+ });++ test("says coordinates match when the screenshot is the same size as the screen", () => {+ const text = describeCoordinateMapping({ width: 1080, height: 2400 }, { width: 1080, height: 2400 });+ expect(text).toBe("Screenshot is 1080x2400 and its coordinates match the screen.");+ });++ test("handles a full-size pixel screenshot of a screen measured in points", () => {+ const text = describeCoordinateMapping({ width: 1179, height: 2556 }, { width: 393, height: 852 });+ expect(text).toBe("Screenshot is 1179x2556. Screen coordinates are 393x852. To tap something you see in this screenshot, multiply its x by 0.333 and y by 0.333.");+ });++ test("returns nothing when the screen size is unknown", () => {+ expect(describeCoordinateMapping({ width: 590, height: 1278 }, { width: 0, height: 0 })).toBeNull();+ });++ test("returns nothing when the screenshot size is unknown", () => {+ expect(describeCoordinateMapping({ width: 0, height: 0 }, { width: 1080, height: 2400 })).toBeNull();+ });+});diff --git a/test/fixtures/baseline.jpg b/test/fixtures/baseline.jpgnew file mode 100644index 0000000..31e3290Binary files /dev/null and b/test/fixtures/baseline.jpg differdiff --git a/test/fixtures/progressive.jpg b/test/fixtures/progressive.jpgnew file mode 100644index 0000000..3fd9783Binary files /dev/null and b/test/fixtures/progressive.jpg differdiff --git a/test/jpeg.ts b/test/jpeg.tsnew file mode 100644index 0000000..efe31dc--- /dev/null+++ b/test/jpeg.ts@@ -0,0 +1,29 @@+import { readFileSync } from "node:fs";+import { join } from "node:path";++import { test, expect } from "@playwright/test";+import { getJpegDimensions } from "../src/jpeg";++const loadFixture = (name: string): Buffer => readFileSync(join(__dirname, "fixtures", name));++test.describe("jpeg", () => {+ test("reads width and height of a baseline jpeg", () => {+ const dimensions = getJpegDimensions(loadFixture("baseline.jpg"));+ expect(dimensions).toEqual({ width: 96, height: 64 });+ });++ test("reads width and height of a progressive jpeg", () => {+ const dimensions = getJpegDimensions(loadFixture("progressive.jpg"));+ expect(dimensions).toEqual({ width: 30, height: 50 });+ });++ test("rejects a buffer that is not a jpeg", () => {+ const notAJpeg = Buffer.from("IAMADUCKIAMADUCKIAMADUCKIAMADUCKIAMADUCK");+ expect(() => getJpegDimensions(notAJpeg)).toThrow("Invalid JPEG");+ });++ test("rejects a jpeg that is cut off before its frame header", () => {+ const truncated = loadFixture("baseline.jpg").subarray(0, 20);+ expect(() => getJpegDimensions(truncated)).toThrow("Invalid JPEG");+ });+});