Let an AI model use a real iPhone with a $7 ESP32: AirPlay eyes, USB HID hands, any OpenAI-compatible vision model. No Mac, no developer account, no jailbreak.
Add tests and CI for the server and firmware build
Henry1fecb9b63ea2
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlnew file mode 100644index 0000000..e8d6863--- /dev/null+++ b/.github/workflows/ci.yml@@ -0,0 +1,38 @@+name: CI++on:+ push:+ branches: [main]+ pull_request:++jobs:+ server:+ runs-on: ubuntu-latest+ strategy:+ matrix:+ python: ["3.10", "3.12"]+ defaults:+ run:+ working-directory: server+ steps:+ - uses: actions/checkout@v4+ - uses: actions/setup-python@v5+ with:+ python-version: ${{ matrix.python }}+ - run: pip install -e ".[dev]"+ - run: ruff check .+ - run: pytest -q++ firmware:+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v4+ - uses: arduino/setup-arduino-cli@v2+ - run: |+ arduino-cli core update-index --additional-urls https://espressif.github.io/arduino-esp32/package_esp32_index.json+ arduino-cli core install esp32:esp32 --additional-urls https://espressif.github.io/arduino-esp32/package_esp32_index.json+ - name: Build+ working-directory: hardware+ run: |+ cp hid_bridge/secrets.example.h hid_bridge/secrets.h+ arduino-cli compile -b "esp32:esp32:esp32s3:USBMode=default,CDCOnBoot=cdc,FlashSize=8M" hid_bridgediff --git a/server/tests/test_agent.py b/server/tests/test_agent.pynew file mode 100644index 0000000..1984361--- /dev/null+++ b/server/tests/test_agent.py@@ -0,0 +1,52 @@+import math++import pytest++from hidagent.agent import circle, parse_actions, parse_command, parse_plan+++def test_parse_single_action():+ acts = parse_actions("THINK: The Search pill is visible, so I tap it.\nDO: tap 500 814")+ assert acts == [{"thought": "The Search pill is visible, so I tap it.", "thought_zh": "",+ "action": "tap", "x": 500, "y": 814}]+++def test_parse_several_actions_keeps_thought_on_first():+ acts = parse_actions("THINK: Type and search.\nDO: type Whiteboard\nDO: key enter")+ assert [a["action"] for a in acts] == ["type", "key"]+ assert acts[0]["text"] == "Whiteboard" and acts[1]["combo"] == "enter"+ assert "thought" not in acts[1]+++def test_parse_caps_at_six_actions():+ reply = "THINK: draw\n" + "\n".join(f"DO: tap {i} {i}" for i in range(10))+ assert len(parse_actions(reply)) == 6+++def test_parse_path_and_circle():+ assert parse_command("path 100 200 300 400 500 600")["points"] == [[100, 200], [300, 400], [500, 600]]+ assert parse_command("circle 500 450 200") == {"action": "circle", "x": 500, "y": 450, "r": 200}++[email protected]("line", ["fly 1 2", "tap 500", "path 1 2 3"])+def test_parse_rejects_bad_commands(line):+ with pytest.raises(ValueError):+ parse_command(line)+++def test_parse_requires_a_do_line():+ with pytest.raises(ValueError):+ parse_actions("THINK: I am not sure what to do.")+++def test_parse_plan_with_and_without_languages():+ assert parse_plan("EN:\n1. Tap Search\nZH:\n1. 点搜索") == ("1. Tap Search", "1. 点搜索")+ assert parse_plan("1. Tap Search") == ("1. Tap Search", "1. Tap Search")+++def test_circle_is_closed_and_round_on_a_tall_screen():+ aspect = 0.46 # width / height of a portrait iPhone frame+ pts = circle(0.5, 0.5, 0.2, aspect)+ assert pts[0] == pytest.approx(pts[-1])+ for x, y in pts: # equal radius in pixels: y offsets shrink by the aspect ratio+ assert math.hypot(x - 0.5, (y - 0.5) / aspect) == pytest.approx(0.2)diff --git a/server/tests/test_device.py b/server/tests/test_device.pynew file mode 100644index 0000000..24d16b5--- /dev/null+++ b/server/tests/test_device.py@@ -0,0 +1,41 @@+import asyncio++import pytest++from hidagent.device import Device, DeviceError+++def test_timeout_grows_with_move_distance_and_text_length():+ assert Device._timeout("click") == 5+ assert Device._timeout("move 1000 -20") > Device._timeout("move 10 10") >= 5+ assert Device._timeout("type " + "x" * 40) > Device._timeout("type hi")+++def test_send_without_board_fails_fast():+ with pytest.raises(DeviceError, match="not connected"):+ asyncio.run(Device().send("ping"))+++def test_round_trip_with_a_fake_board():+ async def scenario():+ device = Device()+ server = await device.serve("127.0.0.1", 0)+ port = server.sockets[0].getsockname()[1]+ reader, writer = await asyncio.open_connection("127.0.0.1", port)++ async def board(): # answers every command like the firmware does+ while line := await reader.readline():+ writer.write(b"ok\n" if line.strip() != b"bad" else b"err unknown command\n")+ await writer.drain()++ task = asyncio.create_task(board())+ assert await device.wait_connected(2)+ await device.send("ping")+ with pytest.raises(DeviceError, match="unknown command"):+ await device.send("bad")+ assert device.links == 1+ task.cancel()+ writer.close()+ server.close()++ asyncio.run(scenario())diff --git a/server/tests/test_vision.py b/server/tests/test_vision.pynew file mode 100644index 0000000..994488a--- /dev/null+++ b/server/tests/test_vision.py@@ -0,0 +1,49 @@+import io++import pytest+from PIL import Image, ImageDraw++from hidagent.vision import changed, pointer++W, H = 498, 1080 # a mirrored iPhone frame+++def screen(pointer_at=None, button=False) -> bytes:+ """A synthetic home screen: gradient background, optional button and pointer dot."""+ img = Image.new("RGB", (W, H))+ draw = ImageDraw.Draw(img)+ for y in range(H):+ draw.line([(0, y), (W, y)], fill=(40, 60 + y // 10, 120))+ if button:+ draw.rectangle([150, 400, 350, 600], fill=(250, 250, 250))+ if pointer_at:+ x, y = pointer_at+ draw.ellipse([x - 12, y - 12, x + 12, y + 12], fill=(200, 200, 200))+ buf = io.BytesIO()+ img.save(buf, "JPEG", quality=75)+ return buf.getvalue()+++def test_pointer_move_is_not_a_screen_change():+ assert not changed(screen((260, 856)), screen((272, 856)))+++def test_new_content_is_a_screen_change():+ assert changed(screen((260, 856)), screen((260, 856), button=True))+++def test_missing_frame_counts_as_changed():+ assert changed(None, screen())+++def test_pointer_is_found_from_a_nudge():+ shift = 12 / W+ found = pointer(screen((262, 856)), screen((274, 856)), near=(0.5, 0.8), shift=shift)+ assert found is not None+ assert found[0] * W == pytest.approx(262, abs=3)+ assert found[1] * H == pytest.approx(856, abs=3)+++def test_pointer_not_found_when_nothing_moved():+ frame = screen((262, 856))+ assert pointer(frame, frame, near=(0.5, 0.8), shift=12 / W) is None