Skip to content

Mobile MCP: let AI agents drive your iOS and Android apps. MCP server for simulators, emulators and real devices. Works with Claude Code, Codex, Cursor and any MCP client.

fix: harden jpeg parser and match check after review

gmegidish2c74fe1c36d0
diff --git a/src/coordinate-mapping.ts b/src/coordinate-mapping.ts
index 77af1af..7812d5b 100644
--- a/src/coordinate-mapping.ts
+++ b/src/coordinate-mapping.ts
@@ -20,11 +20,11 @@ export const describeCoordinateMapping = (screenshot: Dimensions, screen: Dimens
return null;
}
- const x = formatRatio(screen.width / screenshot.width);
- const y = formatRatio(screen.height / screenshot.height);
- if (x === "1" && y === "1") {
+ if (screenshot.width === screen.width && screenshot.height === screen.height) {
return `Screenshot is ${screenshot.width}x${screenshot.height} and its coordinates match the screen.`;
}
+ const x = formatRatio(screen.width / screenshot.width);
+ const y = formatRatio(screen.height / screenshot.height);
return `Screenshot is ${screenshot.width}x${screenshot.height}. Screen coordinates are ${screen.width}x${screen.height}. To tap something you see in this screenshot, multiply its x by ${x} and y by ${y}.`;
};
diff --git a/src/jpeg.ts b/src/jpeg.ts
index 9d6f869..28c264b 100644
--- a/src/jpeg.ts
+++ b/src/jpeg.ts
@@ -3,6 +3,9 @@ import { ActionableError, Dimensions } from "./robot";
const JPEG_SOI = 0xffd8;
const MARKER_PREFIX = 0xff;
+// length(2) precision(1) height(2) width(2) components(1)
+const SOF_MIN_SEGMENT_LENGTH = 8;
+
// SOFn markers carry the frame header with the image dimensions. 0xc4 (DHT),
// 0xc8 (JPG) and 0xcc (DAC) sit in the same range but are not frame headers.
const isStartOfFrameMarker = (marker: number): boolean =>
@@ -19,12 +22,17 @@ export const getJpegDimensions = (buffer: Buffer): Dimensions => {
throw new ActionableError("Invalid JPEG");
}
+ // a marker may be preceded by any number of 0xff fill bytes
+ while (buffer[offset + 1] === MARKER_PREFIX && offset + 4 < buffer.length) {
+ offset++;
+ }
+
const marker = buffer[offset + 1];
const segmentLength = buffer.readUInt16BE(offset + 2);
if (isStartOfFrameMarker(marker)) {
// segment: length(2) precision(1) height(2) width(2) ...
- if (offset + 9 > buffer.length) {
+ if (segmentLength < SOF_MIN_SEGMENT_LENGTH || offset + 9 > buffer.length) {
throw new ActionableError("Invalid JPEG");
}
diff --git a/src/server.ts b/src/server.ts
index 01346ba..0710fde 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -810,7 +810,7 @@ export const createMcpServer = (): McpServer => {
"mobile_take_screenshot",
{
title: "Take Screenshot",
- description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.",
+ description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so when the device reports its screen size the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.",
inputSchema: {
device: z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
maxSize: z.number().int().positive().optional().describe(`Maximum width/height in pixels, keeping aspect ratio. Defaults to ${DEFAULT_SCREENSHOT_MAX_SIZE}.`),
diff --git a/test/coordinate-mapping.ts b/test/coordinate-mapping.ts
index 2365b5c..80e6b3b 100644
--- a/test/coordinate-mapping.ts
+++ b/test/coordinate-mapping.ts
@@ -12,6 +12,11 @@ test.describe("coordinate mapping", () => {
expect(text).toBe("Screenshot is 1080x2400 and its coordinates match the screen.");
});
+ test("does not claim a match when the sizes differ by a single pixel", () => {
+ const text = describeCoordinateMapping({ width: 10001, height: 2400 }, { width: 10000, height: 2400 });
+ expect(text).toBe("Screenshot is 10001x2400. Screen coordinates are 10000x2400. To tap something you see in this screenshot, multiply its x by 1 and y by 1.");
+ });
+
test("handles a full-size pixel screenshot of a screen measured in points", () => {
const text = describeCoordinateMapping({ width: 1179, height: 2556 }, { width: 393, height: 852 });
expect(text).toBe("Screenshot is 1179x2556. Screen coordinates are 393x852. To tap something you see in this screenshot, multiply its x by 0.333 and y by 0.333.");
diff --git a/test/jpeg.ts b/test/jpeg.ts
index efe31dc..d05e4ac 100644
--- a/test/jpeg.ts
+++ b/test/jpeg.ts
@@ -6,6 +6,28 @@ import { getJpegDimensions } from "../src/jpeg";
const loadFixture = (name: string): Buffer => readFileSync(join(__dirname, "fixtures", name));
+const SOF0 = Buffer.from([0xff, 0xc0]);
+
+const findFrameHeader = (jpeg: Buffer): number => {
+ const offset = jpeg.indexOf(SOF0);
+ if (offset < 0) {
+ throw new Error("fixture has no baseline frame header");
+ }
+
+ return offset;
+};
+
+const insertFillByteBeforeFrameHeader = (jpeg: Buffer): Buffer => {
+ const offset = findFrameHeader(jpeg);
+ return Buffer.concat([jpeg.subarray(0, offset), Buffer.from([0xff]), jpeg.subarray(offset)]);
+};
+
+const shrinkFrameHeaderLength = (jpeg: Buffer): Buffer => {
+ const copy = Buffer.from(jpeg);
+ copy.writeUInt16BE(2, findFrameHeader(copy) + 2);
+ return copy;
+};
+
test.describe("jpeg", () => {
test("reads width and height of a baseline jpeg", () => {
const dimensions = getJpegDimensions(loadFixture("baseline.jpg"));
@@ -22,6 +44,16 @@ test.describe("jpeg", () => {
expect(() => getJpegDimensions(notAJpeg)).toThrow("Invalid JPEG");
});
+ test("skips fill bytes that pad a marker", () => {
+ const padded = insertFillByteBeforeFrameHeader(loadFixture("baseline.jpg"));
+ expect(getJpegDimensions(padded)).toEqual({ width: 96, height: 64 });
+ });
+
+ test("rejects a frame header that is too short to hold dimensions", () => {
+ const undersized = shrinkFrameHeaderLength(loadFixture("baseline.jpg"));
+ expect(() => getJpegDimensions(undersized)).toThrow("Invalid JPEG");
+ });
+
test("rejects a jpeg that is cut off before its frame header", () => {
const truncated = loadFixture("baseline.jpg").subarray(0, 20);
expect(() => getJpegDimensions(truncated)).toThrow("Invalid JPEG");