Skip to content

Let an AI model use a real iPhone with a $7 ESP32: AirPlay eyes, USB HID hands, any OpenAI-compatible vision model. No Mac, no developer account, no jailbreak.

Add tests and CI for the server and firmware build

Henry1fecb9b63ea2
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..e8d6863
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,38 @@
+name: CI
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+
+jobs:
+ server:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python: ["3.10", "3.12"]
+ defaults:
+ run:
+ working-directory: server
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python }}
+ - run: pip install -e ".[dev]"
+ - run: ruff check .
+ - run: pytest -q
+
+ firmware:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: arduino/setup-arduino-cli@v2
+ - run: |
+ arduino-cli core update-index --additional-urls https://espressif.github.io/arduino-esp32/package_esp32_index.json
+ arduino-cli core install esp32:esp32 --additional-urls https://espressif.github.io/arduino-esp32/package_esp32_index.json
+ - name: Build
+ working-directory: hardware
+ run: |
+ cp hid_bridge/secrets.example.h hid_bridge/secrets.h
+ arduino-cli compile -b "esp32:esp32:esp32s3:USBMode=default,CDCOnBoot=cdc,FlashSize=8M" hid_bridge
diff --git a/server/tests/test_agent.py b/server/tests/test_agent.py
new file mode 100644
index 0000000..1984361
--- /dev/null
+++ b/server/tests/test_agent.py
@@ -0,0 +1,52 @@
+import math
+
+import pytest
+
+from hidagent.agent import circle, parse_actions, parse_command, parse_plan
+
+
+def test_parse_single_action():
+ acts = parse_actions("THINK: The Search pill is visible, so I tap it.\nDO: tap 500 814")
+ assert acts == [{"thought": "The Search pill is visible, so I tap it.", "thought_zh": "",
+ "action": "tap", "x": 500, "y": 814}]
+
+
+def test_parse_several_actions_keeps_thought_on_first():
+ acts = parse_actions("THINK: Type and search.\nDO: type Whiteboard\nDO: key enter")
+ assert [a["action"] for a in acts] == ["type", "key"]
+ assert acts[0]["text"] == "Whiteboard" and acts[1]["combo"] == "enter"
+ assert "thought" not in acts[1]
+
+
+def test_parse_caps_at_six_actions():
+ reply = "THINK: draw\n" + "\n".join(f"DO: tap {i} {i}" for i in range(10))
+ assert len(parse_actions(reply)) == 6
+
+
+def test_parse_path_and_circle():
+ assert parse_command("path 100 200 300 400 500 600")["points"] == [[100, 200], [300, 400], [500, 600]]
+ assert parse_command("circle 500 450 200") == {"action": "circle", "x": 500, "y": 450, "r": 200}
+
+
[email protected]("line", ["fly 1 2", "tap 500", "path 1 2 3"])
+def test_parse_rejects_bad_commands(line):
+ with pytest.raises(ValueError):
+ parse_command(line)
+
+
+def test_parse_requires_a_do_line():
+ with pytest.raises(ValueError):
+ parse_actions("THINK: I am not sure what to do.")
+
+
+def test_parse_plan_with_and_without_languages():
+ assert parse_plan("EN:\n1. Tap Search\nZH:\n1. 点搜索") == ("1. Tap Search", "1. 点搜索")
+ assert parse_plan("1. Tap Search") == ("1. Tap Search", "1. Tap Search")
+
+
+def test_circle_is_closed_and_round_on_a_tall_screen():
+ aspect = 0.46 # width / height of a portrait iPhone frame
+ pts = circle(0.5, 0.5, 0.2, aspect)
+ assert pts[0] == pytest.approx(pts[-1])
+ for x, y in pts: # equal radius in pixels: y offsets shrink by the aspect ratio
+ assert math.hypot(x - 0.5, (y - 0.5) / aspect) == pytest.approx(0.2)
diff --git a/server/tests/test_device.py b/server/tests/test_device.py
new file mode 100644
index 0000000..24d16b5
--- /dev/null
+++ b/server/tests/test_device.py
@@ -0,0 +1,41 @@
+import asyncio
+
+import pytest
+
+from hidagent.device import Device, DeviceError
+
+
+def test_timeout_grows_with_move_distance_and_text_length():
+ assert Device._timeout("click") == 5
+ assert Device._timeout("move 1000 -20") > Device._timeout("move 10 10") >= 5
+ assert Device._timeout("type " + "x" * 40) > Device._timeout("type hi")
+
+
+def test_send_without_board_fails_fast():
+ with pytest.raises(DeviceError, match="not connected"):
+ asyncio.run(Device().send("ping"))
+
+
+def test_round_trip_with_a_fake_board():
+ async def scenario():
+ device = Device()
+ server = await device.serve("127.0.0.1", 0)
+ port = server.sockets[0].getsockname()[1]
+ reader, writer = await asyncio.open_connection("127.0.0.1", port)
+
+ async def board(): # answers every command like the firmware does
+ while line := await reader.readline():
+ writer.write(b"ok\n" if line.strip() != b"bad" else b"err unknown command\n")
+ await writer.drain()
+
+ task = asyncio.create_task(board())
+ assert await device.wait_connected(2)
+ await device.send("ping")
+ with pytest.raises(DeviceError, match="unknown command"):
+ await device.send("bad")
+ assert device.links == 1
+ task.cancel()
+ writer.close()
+ server.close()
+
+ asyncio.run(scenario())
diff --git a/server/tests/test_vision.py b/server/tests/test_vision.py
new file mode 100644
index 0000000..994488a
--- /dev/null
+++ b/server/tests/test_vision.py
@@ -0,0 +1,49 @@
+import io
+
+import pytest
+from PIL import Image, ImageDraw
+
+from hidagent.vision import changed, pointer
+
+W, H = 498, 1080 # a mirrored iPhone frame
+
+
+def screen(pointer_at=None, button=False) -> bytes:
+ """A synthetic home screen: gradient background, optional button and pointer dot."""
+ img = Image.new("RGB", (W, H))
+ draw = ImageDraw.Draw(img)
+ for y in range(H):
+ draw.line([(0, y), (W, y)], fill=(40, 60 + y // 10, 120))
+ if button:
+ draw.rectangle([150, 400, 350, 600], fill=(250, 250, 250))
+ if pointer_at:
+ x, y = pointer_at
+ draw.ellipse([x - 12, y - 12, x + 12, y + 12], fill=(200, 200, 200))
+ buf = io.BytesIO()
+ img.save(buf, "JPEG", quality=75)
+ return buf.getvalue()
+
+
+def test_pointer_move_is_not_a_screen_change():
+ assert not changed(screen((260, 856)), screen((272, 856)))
+
+
+def test_new_content_is_a_screen_change():
+ assert changed(screen((260, 856)), screen((260, 856), button=True))
+
+
+def test_missing_frame_counts_as_changed():
+ assert changed(None, screen())
+
+
+def test_pointer_is_found_from_a_nudge():
+ shift = 12 / W
+ found = pointer(screen((262, 856)), screen((274, 856)), near=(0.5, 0.8), shift=shift)
+ assert found is not None
+ assert found[0] * W == pytest.approx(262, abs=3)
+ assert found[1] * H == pytest.approx(856, abs=3)
+
+
+def test_pointer_not_found_when_nothing_moved():
+ frame = screen((262, 856))
+ assert pointer(frame, frame, near=(0.5, 0.8), shift=12 / W) is None