Skip to content

Let an AI model use a real iPhone with a $7 ESP32: AirPlay eyes, USB HID hands, any OpenAI-compatible vision model. No Mac, no developer account, no jailbreak.

Add ESP32-S3 USB HID bridge firmware

Henryca2fc3a6ef95
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..5912aea
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,19 @@
+# Python
+__pycache__/
+*.egg-info/
+.venv/
+.pytest_cache/
+.ruff_cache/
+
+# Local config and secrets
+.env
+hardware/hid_bridge/secrets.h
+
+# Build output
+hardware/build/
+hardware/hid_bridge/build/
+
+# Editors and OS
+.DS_Store
+.idea/
+.vscode/
diff --git a/hardware/flash.sh b/hardware/flash.sh
new file mode 100755
index 0000000..0ce2681
--- /dev/null
+++ b/hardware/flash.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Build and flash the HID bridge (needs arduino-cli and the esp32 core). Usage: hardware/flash.sh [port] [--debug]
+# A running bridge is a TinyUSB device, so it is first kicked into the ROM
+# bootloader with a 1200-baud touch; a blank board is flashed directly.
+set -euo pipefail
+cd "$(dirname "$0")"
+FQBN="esp32:esp32:esp32s3:USBMode=default,CDCOnBoot=cdc,FlashSize=8M"
+port() { for p in /dev/cu.usbmodem* /dev/ttyACM*; do [[ -e $p ]] && { echo "$p"; return; }; done; }
+PORT=${1:-$(port)}
+FLAGS=(--build-property "compiler.cpp.extra_flags="); [[ "${2:-}" == --debug ]] && FLAGS=(--build-property "compiler.cpp.extra_flags=-DDEBUG_SERIAL=1")
+
+arduino-cli compile -b "$FQBN" "${FLAGS[@]}" --output-dir build hid_bridge
+OLD=$PORT
+stty -f "$PORT" 1200 2>/dev/null || stty -F "$PORT" 1200 2>/dev/null || true
+for _ in $(seq 20); do sleep 0.5; PORT=$(port); [[ -n "$PORT" && "$PORT" != "$OLD" ]] && break; done
+sleep 1 # let the ROM bootloader's serial come up
+PORT=$(port)
+arduino-cli upload -b "$FQBN" -p "$PORT" --input-dir build hid_bridge
diff --git a/hardware/hid_bridge/hid_bridge.ino b/hardware/hid_bridge/hid_bridge.ino
new file mode 100644
index 0000000..5a89e6a
--- /dev/null
+++ b/hardware/hid_bridge/hid_bridge.ino
@@ -0,0 +1,121 @@
+// HID bridge: an ESP32-S3 that shows up as a USB mouse + keyboard and takes
+// line commands over TCP (Wi-Fi). It holds no logic; the server decides.
+//
+// Protocol (one command per line, the board answers "ok" or "err <why>"):
+// ping | home | move <dx> <dy> | press | release | click
+// scroll <v> <h> | type <ascii text> | key <combo, e.g. cmd+h, enter>
+
+#include <WiFi.h>
+#include "USB.h"
+#include "USBHIDMouse.h"
+#include "USBHIDKeyboard.h"
+#include "secrets.h" // WIFI_SSID, WIFI_PASS, SERVER_HOST, SERVER_PORT
+
+const int STEP = 10; // max pointer delta per report while moving
+const int STEP_MS = 8; // report interval; constant pace keeps iOS acceleration predictable
+const int HOME_REPORTS = 50;
+#ifndef DEBUG_SERIAL
+#define DEBUG_SERIAL 0 // 1: echo every command over USB serial (needs CDCOnBoot=cdc)
+#endif
+
+USBHIDMouse mouse;
+USBHIDKeyboard keyboard;
+WiFiClient server;
+
+void movePaced(long dx, long dy) {
+ while (dx || dy) {
+ int sx = constrain(dx, -STEP, STEP), sy = constrain(dy, -STEP, STEP);
+ mouse.move(sx, sy);
+ dx -= sx;
+ dy -= sy;
+ delay(STEP_MS);
+ }
+}
+
+uint8_t keyCode(const String &k) {
+ if (k == "enter") return KEY_RETURN;
+ if (k == "esc") return KEY_ESC;
+ if (k == "tab") return KEY_TAB;
+ if (k == "backspace") return KEY_BACKSPACE;
+ if (k == "space") return ' ';
+ if (k == "up") return KEY_UP_ARROW;
+ if (k == "down") return KEY_DOWN_ARROW;
+ if (k == "left") return KEY_LEFT_ARROW;
+ if (k == "right") return KEY_RIGHT_ARROW;
+ if (k == "cmd") return KEY_LEFT_GUI;
+ if (k == "ctrl") return KEY_LEFT_CTRL;
+ if (k == "alt") return KEY_LEFT_ALT;
+ if (k == "shift") return KEY_LEFT_SHIFT;
+ return k.length() == 1 ? k[0] : 0;
+}
+
+bool pressCombo(String combo) {
+ combo += '+';
+ for (int i; (i = combo.indexOf('+')) >= 0; combo = combo.substring(i + 1)) {
+ uint8_t c = keyCode(combo.substring(0, i));
+ if (!c) { keyboard.releaseAll(); return false; }
+ keyboard.press(c);
+ }
+ delay(30);
+ keyboard.releaseAll();
+ return true;
+}
+
+String run(const String &line) {
+ int sp = line.indexOf(' ');
+ String cmd = sp < 0 ? line : line.substring(0, sp);
+ String arg = sp < 0 ? "" : line.substring(sp + 1);
+ long a = 0, b = 0;
+ sscanf(arg.c_str(), "%ld %ld", &a, &b);
+
+ if (cmd == "ping") return "ok";
+ if (cmd == "home") { // slam the pointer into the top-left corner
+ for (int i = 0; i < HOME_REPORTS; i++) { mouse.move(-127, -127); delay(2); }
+ return "ok";
+ }
+ if (cmd == "move") { movePaced(a, b); return "ok"; }
+ if (cmd == "press") { mouse.press(MOUSE_LEFT); return "ok"; }
+ if (cmd == "release") { mouse.release(MOUSE_LEFT); return "ok"; }
+ if (cmd == "click") { mouse.press(MOUSE_LEFT); delay(60); mouse.release(MOUSE_LEFT); return "ok"; }
+ if (cmd == "scroll") { mouse.move(0, 0, a, b); return "ok"; }
+ if (cmd == "type") { keyboard.print(arg); return "ok"; }
+ if (cmd == "key") return pressCombo(arg) ? "ok" : "err unknown key";
+ return "err unknown command";
+}
+
+void setup() {
+#if DEBUG_SERIAL
+ Serial.begin(115200);
+#endif
+ mouse.begin();
+ keyboard.begin();
+ USB.productName("HID Bridge");
+ USB.begin();
+ WiFi.setHostname("hid-bridge");
+ WiFi.setSleep(false); // modem sleep adds ~100 ms per command
+ WiFi.begin(WIFI_SSID, WIFI_PASS);
+}
+
+void loop() {
+ if (WiFi.status() != WL_CONNECTED) { mouse.release(MOUSE_LEFT); delay(500); return; }
+ if (!server.connected()) {
+ mouse.release(MOUSE_LEFT); // a gesture cut off mid-way must not leave the button held (long press)
+ keyboard.releaseAll();
+ if (!server.connect(SERVER_HOST, SERVER_PORT)) { delay(2000); return; }
+ server.setNoDelay(true);
+#if DEBUG_SERIAL
+ Serial.printf("connected to %s:%d as %s\n", SERVER_HOST, SERVER_PORT, WiFi.localIP().toString().c_str());
+#endif
+ }
+ if (server.available()) {
+ String line = server.readStringUntil('\n');
+ line.trim();
+ if (!line.length()) return;
+ unsigned long started = millis();
+ String reply = run(line);
+ server.print(reply + "\n");
+#if DEBUG_SERIAL
+ Serial.printf("%s -> %s (%lu ms)\n", line.c_str(), reply.c_str(), millis() - started);
+#endif
+ }
+}
diff --git a/hardware/hid_bridge/secrets.example.h b/hardware/hid_bridge/secrets.example.h
new file mode 100644
index 0000000..588402b
--- /dev/null
+++ b/hardware/hid_bridge/secrets.example.h
@@ -0,0 +1,5 @@
+// Copy to secrets.h (git-ignored) and fill in.
+#define WIFI_SSID "your-wifi"
+#define WIFI_PASS "your-password"
+#define SERVER_HOST "192.168.1.10" // machine running `python -m hidagent`
+#define SERVER_PORT 7300