Let an AI model use a real iPhone with a $7 ESP32: AirPlay eyes, USB HID hands, any OpenAI-compatible vision model. No Mac, no developer account, no jailbreak.
Add ESP32-S3 USB HID bridge firmware
Henryca2fc3a6ef95
diff --git a/.gitignore b/.gitignorenew file mode 100644index 0000000..5912aea--- /dev/null+++ b/.gitignore@@ -0,0 +1,19 @@+# Python+__pycache__/+*.egg-info/+.venv/+.pytest_cache/+.ruff_cache/++# Local config and secrets+.env+hardware/hid_bridge/secrets.h++# Build output+hardware/build/+hardware/hid_bridge/build/++# Editors and OS+.DS_Store+.idea/+.vscode/diff --git a/hardware/flash.sh b/hardware/flash.shnew file mode 100755index 0000000..0ce2681--- /dev/null+++ b/hardware/flash.sh@@ -0,0 +1,18 @@+#!/usr/bin/env bash+# Build and flash the HID bridge (needs arduino-cli and the esp32 core). Usage: hardware/flash.sh [port] [--debug]+# A running bridge is a TinyUSB device, so it is first kicked into the ROM+# bootloader with a 1200-baud touch; a blank board is flashed directly.+set -euo pipefail+cd "$(dirname "$0")"+FQBN="esp32:esp32:esp32s3:USBMode=default,CDCOnBoot=cdc,FlashSize=8M"+port() { for p in /dev/cu.usbmodem* /dev/ttyACM*; do [[ -e $p ]] && { echo "$p"; return; }; done; }+PORT=${1:-$(port)}+FLAGS=(--build-property "compiler.cpp.extra_flags="); [[ "${2:-}" == --debug ]] && FLAGS=(--build-property "compiler.cpp.extra_flags=-DDEBUG_SERIAL=1")++arduino-cli compile -b "$FQBN" "${FLAGS[@]}" --output-dir build hid_bridge+OLD=$PORT+stty -f "$PORT" 1200 2>/dev/null || stty -F "$PORT" 1200 2>/dev/null || true+for _ in $(seq 20); do sleep 0.5; PORT=$(port); [[ -n "$PORT" && "$PORT" != "$OLD" ]] && break; done+sleep 1 # let the ROM bootloader's serial come up+PORT=$(port)+arduino-cli upload -b "$FQBN" -p "$PORT" --input-dir build hid_bridgediff --git a/hardware/hid_bridge/hid_bridge.ino b/hardware/hid_bridge/hid_bridge.inonew file mode 100644index 0000000..5a89e6a--- /dev/null+++ b/hardware/hid_bridge/hid_bridge.ino@@ -0,0 +1,121 @@+// HID bridge: an ESP32-S3 that shows up as a USB mouse + keyboard and takes+// line commands over TCP (Wi-Fi). It holds no logic; the server decides.+//+// Protocol (one command per line, the board answers "ok" or "err <why>"):+// ping | home | move <dx> <dy> | press | release | click+// scroll <v> <h> | type <ascii text> | key <combo, e.g. cmd+h, enter>++#include <WiFi.h>+#include "USB.h"+#include "USBHIDMouse.h"+#include "USBHIDKeyboard.h"+#include "secrets.h" // WIFI_SSID, WIFI_PASS, SERVER_HOST, SERVER_PORT++const int STEP = 10; // max pointer delta per report while moving+const int STEP_MS = 8; // report interval; constant pace keeps iOS acceleration predictable+const int HOME_REPORTS = 50;+#ifndef DEBUG_SERIAL+#define DEBUG_SERIAL 0 // 1: echo every command over USB serial (needs CDCOnBoot=cdc)+#endif++USBHIDMouse mouse;+USBHIDKeyboard keyboard;+WiFiClient server;++void movePaced(long dx, long dy) {+ while (dx || dy) {+ int sx = constrain(dx, -STEP, STEP), sy = constrain(dy, -STEP, STEP);+ mouse.move(sx, sy);+ dx -= sx;+ dy -= sy;+ delay(STEP_MS);+ }+}++uint8_t keyCode(const String &k) {+ if (k == "enter") return KEY_RETURN;+ if (k == "esc") return KEY_ESC;+ if (k == "tab") return KEY_TAB;+ if (k == "backspace") return KEY_BACKSPACE;+ if (k == "space") return ' ';+ if (k == "up") return KEY_UP_ARROW;+ if (k == "down") return KEY_DOWN_ARROW;+ if (k == "left") return KEY_LEFT_ARROW;+ if (k == "right") return KEY_RIGHT_ARROW;+ if (k == "cmd") return KEY_LEFT_GUI;+ if (k == "ctrl") return KEY_LEFT_CTRL;+ if (k == "alt") return KEY_LEFT_ALT;+ if (k == "shift") return KEY_LEFT_SHIFT;+ return k.length() == 1 ? k[0] : 0;+}++bool pressCombo(String combo) {+ combo += '+';+ for (int i; (i = combo.indexOf('+')) >= 0; combo = combo.substring(i + 1)) {+ uint8_t c = keyCode(combo.substring(0, i));+ if (!c) { keyboard.releaseAll(); return false; }+ keyboard.press(c);+ }+ delay(30);+ keyboard.releaseAll();+ return true;+}++String run(const String &line) {+ int sp = line.indexOf(' ');+ String cmd = sp < 0 ? line : line.substring(0, sp);+ String arg = sp < 0 ? "" : line.substring(sp + 1);+ long a = 0, b = 0;+ sscanf(arg.c_str(), "%ld %ld", &a, &b);++ if (cmd == "ping") return "ok";+ if (cmd == "home") { // slam the pointer into the top-left corner+ for (int i = 0; i < HOME_REPORTS; i++) { mouse.move(-127, -127); delay(2); }+ return "ok";+ }+ if (cmd == "move") { movePaced(a, b); return "ok"; }+ if (cmd == "press") { mouse.press(MOUSE_LEFT); return "ok"; }+ if (cmd == "release") { mouse.release(MOUSE_LEFT); return "ok"; }+ if (cmd == "click") { mouse.press(MOUSE_LEFT); delay(60); mouse.release(MOUSE_LEFT); return "ok"; }+ if (cmd == "scroll") { mouse.move(0, 0, a, b); return "ok"; }+ if (cmd == "type") { keyboard.print(arg); return "ok"; }+ if (cmd == "key") return pressCombo(arg) ? "ok" : "err unknown key";+ return "err unknown command";+}++void setup() {+#if DEBUG_SERIAL+ Serial.begin(115200);+#endif+ mouse.begin();+ keyboard.begin();+ USB.productName("HID Bridge");+ USB.begin();+ WiFi.setHostname("hid-bridge");+ WiFi.setSleep(false); // modem sleep adds ~100 ms per command+ WiFi.begin(WIFI_SSID, WIFI_PASS);+}++void loop() {+ if (WiFi.status() != WL_CONNECTED) { mouse.release(MOUSE_LEFT); delay(500); return; }+ if (!server.connected()) {+ mouse.release(MOUSE_LEFT); // a gesture cut off mid-way must not leave the button held (long press)+ keyboard.releaseAll();+ if (!server.connect(SERVER_HOST, SERVER_PORT)) { delay(2000); return; }+ server.setNoDelay(true);+#if DEBUG_SERIAL+ Serial.printf("connected to %s:%d as %s\n", SERVER_HOST, SERVER_PORT, WiFi.localIP().toString().c_str());+#endif+ }+ if (server.available()) {+ String line = server.readStringUntil('\n');+ line.trim();+ if (!line.length()) return;+ unsigned long started = millis();+ String reply = run(line);+ server.print(reply + "\n");+#if DEBUG_SERIAL+ Serial.printf("%s -> %s (%lu ms)\n", line.c_str(), reply.c_str(), millis() - started);+#endif+ }+}diff --git a/hardware/hid_bridge/secrets.example.h b/hardware/hid_bridge/secrets.example.hnew file mode 100644index 0000000..588402b--- /dev/null+++ b/hardware/hid_bridge/secrets.example.h@@ -0,0 +1,5 @@+// Copy to secrets.h (git-ignored) and fill in.+#define WIFI_SSID "your-wifi"+#define WIFI_PASS "your-password"+#define SERVER_HOST "192.168.1.10" // machine running `python -m hidagent`+#define SERVER_PORT 7300