The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
mobile: paste clipboard images on iOS
jaylfce9e7b733dbdb
diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swiftindex 3c09f949d..1076f4b55 100644--- a/mobile/ios/Runner/AppDelegate.swift+++ b/mobile/ios/Runner/AppDelegate.swift@@ -17,6 +17,40 @@ import UserNotificationsfunc didInitializeImplicitFlutterEngine(_ engineBridge: FlutterImplicitEngineBridge) {GeneratedPluginRegistrant.register(with: engineBridge.pluginRegistry)DeviceNameChannel.register(with: engineBridge.pluginRegistry)+ ClipboardImageChannel.register(with: engineBridge.pluginRegistry)+ }+}++/// `harness/clipboard_image` — the image on the clipboard, as PNG bytes (Dart:+/// `lib/clipboard/native_clipboard.dart`). Flutter's own clipboard reads `text/plain` only, so a+/// screenshot or a "Copy" from Photos was invisible to Paste. The same channel the desktop runners+/// answer; only `readImagePng` is implemented here, since nothing on the phone writes an image.+enum ClipboardImageChannel {+ static func register(with registry: FlutterPluginRegistry) {+ guard let messenger = registry.registrar(forPlugin: "HarnessClipboardImage")?.messenger() else { return }+ let channel = FlutterMethodChannel(name: "harness/clipboard_image", binaryMessenger: messenger)+ channel.setMethodCallHandler { call, result in+ guard call.method == "readImagePng" else { result(FlutterMethodNotImplemented); return }+ // `hasImages` answers without the system's paste prompt, so an empty clipboard never asks.+ let pasteboard = UIPasteboard.general+ guard pasteboard.hasImages else { result(nil); return }+ // An existing PNG is passed through untouched; anything else (a JPEG or HEIC from Photos) is+ // re-encoded, which for a full-size photo is slow enough to keep off the main thread.+ if let png = pasteboard.data(forPasteboardType: "public.png") {+ result(FlutterStandardTypedData(bytes: png))+ return+ }+ // From here an image IS on the clipboard, so a failure answers EMPTY bytes rather than nil:+ // Dart then says the image is unreadable instead of that there is nothing to paste.+ let unreadable = FlutterStandardTypedData(bytes: Data())+ guard let image = pasteboard.image else { result(unreadable); return }+ DispatchQueue.global(qos: .userInitiated).async {+ let png = image.pngData()+ DispatchQueue.main.async {+ result(png.map { FlutterStandardTypedData(bytes: $0) } ?? unreadable)+ }+ }+ }}}diff --git a/mobile/lib/clipboard/native_clipboard.dart b/mobile/lib/clipboard/native_clipboard.dartindex 0422c3508..eb61cb1be 100644--- a/mobile/lib/clipboard/native_clipboard.dart+++ b/mobile/lib/clipboard/native_clipboard.dart@@ -6,8 +6,9 @@ import 'package:flutter/services.dart';/// `widgets/terminal_panel.dart`'s `_paste()`. Reading a real IMAGE off the clipboard (a/// screenshot, "Copy Image" from a browser, ...) needs a native round trip instead: NSPasteboard on/// macOS (`macos/Runner/MainFlutterWindow.swift`), the GTK clipboard on Linux-/// (`linux/runner/my_application.cc`). Both answer over the same channel name and method, so this-/// wrapper is the one place call sites need to know about.+/// (`linux/runner/my_application.cc`), UIPasteboard on iOS (`ios/Runner/AppDelegate.swift`, read+/// only). All answer over the same channel name and method, so this wrapper is the one place call+/// sites need to know about.class NativeClipboard {NativeClipboard._();@@ -17,12 +18,16 @@ class NativeClipboard {/// Reads the system clipboard for an image, returned as PNG bytes.///+ /// EMPTY bytes (iOS) mean an image IS on the clipboard but could not be re-encoded, so a caller+ /// can say "unreadable" rather than "nothing there". Every other call site already treats empty+ /// as "no image".+ ////// Returns `null` on any platform without a native handler for this channel (Windows — the- /// runner is unexercised, see CLAUDE.md — and any platform that isn't macOS/Linux) or when the- /// clipboard genuinely holds no image, so call sites can use one check to fall through to+ /// runner is unexercised, see CLAUDE.md — and any platform that isn't macOS/Linux/iOS) or when+ /// the clipboard genuinely holds no image, so call sites can use one check to fall through to/// today's text-paste behavior either way.static Future<Uint8List?> readImagePng() async {- if (!Platform.isMacOS && !Platform.isLinux) return null;+ if (!Platform.isMacOS && !Platform.isLinux && !Platform.isIOS) return null;try {final bytes = await _channel.invokeMethod<Uint8List>('readImagePng');return bytes;diff --git a/mobile/lib/phone/terminal_paste.dart b/mobile/lib/phone/terminal_paste.dartindex 8c39c2616..d2d135275 100644--- a/mobile/lib/phone/terminal_paste.dart+++ b/mobile/lib/phone/terminal_paste.dart@@ -1,21 +1,24 @@import 'package:flutter/services.dart';+import 'package:harness_mobile/clipboard/native_clipboard.dart';import 'package:harness_mobile/state/app_state.dart' show MachineState;+import 'package:harness_mobile/terminal/image_transcode.dart';import 'package:harness_mobile/terminal/terminal_session.dart';/// Said when the terminal the paste was meant for is no longer the one to send it to.const pasteChangedMessage ='The terminal changed before the paste, so nothing was sent.';-/// Pastes the phone's clipboard text into [session], as one paste rather than as typing.+/// Pastes the phone's clipboard into [session]: its text as one paste rather than as typing, or,+/// when there is no text, its image, the same upload the key bar's image button makes.////// The Paste row of the terminal's actions sheet (`phone/terminal_page.dart`). iOS has no other way/// in: long-press on the terminal is selection there, and the soft keyboard has no paste key.////// Text is asked about first with [Clipboard.hasStrings], which iOS answers without its "Allow-/// Paste" prompt, so an empty clipboard never asks. Unlike the desktop's ⌘V there is no Ctrl+V-/// fallthrough: the agent runs on another machine, whose engine would read THAT machine's-/// clipboard rather than this phone's.+/// Paste" prompt, so the person is asked once, for the one thing actually read. Unlike the+/// desktop's ⌘V there is no Ctrl+V fallthrough: the agent runs on another machine, whose engine+/// would read THAT machine's clipboard rather than this phone's.////// [machine] is the session's machine, read for what its CLI can take at the moment of sending./// Every outcome worth telling the person about goes to [report]; a refused prompt says nothing.@@ -36,30 +39,66 @@ Future<void> pasteClipboard({bool stillOwnsPaste() =>stillShowing() && session.acceptsInput && session.streamId == streamId;- final String? text;try {- if (!await Clipboard.hasStrings()) {- report('There is no text on the clipboard.');+ if (await Clipboard.hasStrings()) {+ final text = (await Clipboard.getData(Clipboard.kTextPlain))?.text;+ // Empty here is a refused prompt as often as an empty string: nothing to say about either.+ if (text == null || text.isEmpty) return;+ if (!stillOwnsPaste()) {+ report(pasteChangedMessage);+ return;+ }+ // The same choice the desktop's paste makes: one atomic paste frame where the machine's CLI+ // knows it, otherwise xterm's own paste (bracketed when the program asked for it).+ if (machine != null && machine.terminalPasteRawAvailable) {+ if (!await session.pasteText(text)) {+ report('The paste could not be sent.');+ }+ } else {+ session.terminal.paste(text);+ }return;}- text = (await Clipboard.getData(Clipboard.kTextPlain))?.text;} on PlatformException {report('Could not read the clipboard.');return;}- // Empty here is a refused prompt as often as an empty string: nothing to say about either.- if (text == null || text.isEmpty) return;- if (!stillOwnsPaste()) {- report(pasteChangedMessage);++ // A plain shell, not an agent: the CLI delivers an image by replaying Ctrl+V, which a shell reads+ // as quoted-insert. The desktop skips image paste there for the same reason.+ if (session.engineId == 'terminal') {+ report('There is no text on the clipboard.');return;}- // The same choice the desktop's paste makes: one atomic paste frame where the machine's CLI- // knows it, otherwise xterm's own paste (bracketed when the program asked for it).- if (machine != null && machine.terminalPasteRawAvailable) {- if (!await session.pasteText(text)) {- report('The paste could not be sent.');- }- } else {- session.terminal.paste(text);+ final imageBytes = await NativeClipboard.readImagePng();+ if (imageBytes == null) {+ report('There is nothing on the clipboard to paste.');+ return;+ }+ // Empty is the native side saying an image IS there but could not be read, see+ // [NativeClipboard.readImagePng].+ if (imageBytes.isEmpty) {+ report("The clipboard's image isn't one this phone can read.");+ return;+ }+ if (machine == null || !machine.terminalImagePasteAvailable) {+ report("This machine's harness is too old to receive images.");+ return;+ }+ // Through the same transcode as a picked photo: a clipboard image is as often a full-size camera+ // shot as a screenshot, and it has to be scaled under the upload's ceiling either way.+ switch (await transcodeToPng(imageBytes)) {+ case ImageTranscodeUnreadable():+ report("The clipboard's image isn't one this phone can read.");+ case ImageTranscodeTooLarge():+ report('That image is too large to send, even scaled down.');+ case ImageTranscodeOk(:final pngBytes):+ if (!stillOwnsPaste()) {+ report(pasteChangedMessage);+ return;+ }+ if (!await session.pasteImage(pngBytes)) {+ report('The image could not be sent.');+ }}}diff --git a/mobile/lib/widgets/terminal_panel.dart b/mobile/lib/widgets/terminal_panel.dartindex 987799209..f6359f122 100644--- a/mobile/lib/widgets/terminal_panel.dart+++ b/mobile/lib/widgets/terminal_panel.dart@@ -1062,7 +1062,7 @@ class _TerminalPanelState extends State<TerminalPanel>/// text at all, so a naive body finds `null` and returns, silently: the single/// most common thing anyone pastes into a coding agent did nothing, with no/// error and nothing in a log. [NativeClipboard] closes that gap with a native- /// platform-channel read for an actual image (macOS/Linux only; see its doc).+ /// platform-channel read for an actual image (macOS, Linux, iOS; see its doc).////// The engines running in these panes read the system clipboard THEMSELVES —/// Claude Code attaches an image on Ctrl+V — so on a LOCAL pane that is already@@ -1074,10 +1074,10 @@ class _TerminalPanelState extends State<TerminalPanel>/// clipboard than this one — see `MachineState.isLocalMachine`./// An image handed over by the software keyboard's own clipboard.///- /// This is the phone's ONLY way to get an image into a pane: Flutter's- /// [Clipboard] reads `text/plain` and nothing else, and the native reader- /// [_paste] falls back on is macOS/Linux. Gboard hands the bytes over directly,- /// so there is no clipboard to read at all.+ /// Flutter's [Clipboard] reads `text/plain` and nothing else, and Gboard hands+ /// the bytes over directly, so there is no clipboard to read at all. (The+ /// actions sheet's Paste reads the system clipboard's image instead, through+ /// [NativeClipboard]: see `phone/terminal_paste.dart`.)////// ⚠️ A pane on a phone is ALWAYS remote — the agent runs on another machine,/// whose engine reads a different OS clipboard — so unlike the desktop there isdiff --git a/mobile/test/terminal_paste_test.dart b/mobile/test/terminal_paste_test.dartindex bca06902e..de56f82c4 100644--- a/mobile/test/terminal_paste_test.dart+++ b/mobile/test/terminal_paste_test.dart@@ -1,3 +1,10 @@+// NativeClipboard asks its channel only where a runner answers it, which a Windows host is not.+@TestOn('!windows')+library;++import 'dart:async';+import 'dart:convert';+import 'package:flutter/services.dart';import 'package:flutter_test/flutter_test.dart';import 'package:harness_mobile/core/models.dart';@@ -19,6 +26,24 @@ void main() {late List<String> reports;late TerminalSession session;late MachineState machine;+ late List<String> sentTypes;++ const imageChannel = MethodChannel('harness/clipboard_image');++ /// A 1x1 PNG, for an image the transcode can actually read.+ final onePixelPng = base64Decode(+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==',+ );++ /// The image on the clipboard as the native side answers it: null for none, empty for one it+ /// could not read. [onRead] runs during the read.+ void clipboardImage(Uint8List? bytes, {void Function()? onRead}) {+ messenger.setMockMethodCallHandler(imageChannel, (call) async {+ if (call.method != 'readImagePng') return null;+ onRead?.call();+ return bytes;+ });+ }/// The clipboard as Flutter's [Clipboard] sees it. [onRead] runs while the text is being read,/// the window in which iOS can be showing its paste prompt.@@ -45,7 +70,12 @@ void main() {setUp(() {frames = [];reports = [];+ sentTypes = [];session = controllingSession(+ send: (type, _) async {+ sentTypes.add(type);+ return true;+ },sendBinary: (frame) async {frames.add(frame);return true;@@ -62,6 +92,7 @@ void main() {tearDown(() {messenger.setMockMethodCallHandler(SystemChannels.platform, null);+ messenger.setMockMethodCallHandler(imageChannel, null);});test('text goes out as one paste frame where the CLI knows it', () async {@@ -90,13 +121,92 @@ void main() {test('an empty clipboard is said, not sent', () async {machine.terminalPasteRawAvailable = true;clipboardHolds(null);+ clipboardImage(null);await paste();expect(frames, isEmpty);+ expect(reports, ['There is nothing on the clipboard to paste.']);+ });++ test('a plain shell is never offered the image', () async {+ session =+ TerminalSession(+ machineId: 'm',+ agentId: 'a',+ agentName: 'Shell',+ engineId: 'terminal',+ send: (_, _) async => true,+ sendBinary: (_) async => true,+ )+ ..status = TerminalSessionStatus.controlling+ ..streamId = 's';+ clipboardHolds(null);+ var asked = false;+ clipboardImage(onePixelPng, onRead: () => asked = true);++ await paste();++ expect(asked, isFalse);expect(reports, ['There is no text on the clipboard.']);});+ test('an image the phone could not read is said to be unreadable', () async {+ machine.terminalImagePasteAvailable = true;+ clipboardHolds(null);+ clipboardImage(Uint8List(0));++ await paste();++ expect(sentTypes, isEmpty);+ expect(reports, ["The clipboard's image isn't one this phone can read."]);+ });++ test('bytes that are not a picture are unreadable too', () async {+ machine.terminalImagePasteAvailable = true;+ clipboardHolds(null);+ clipboardImage(Uint8List.fromList([1, 2, 3, 4]));++ await paste();++ expect(sentTypes, isEmpty);+ expect(reports, ["The clipboard's image isn't one this phone can read."]);+ });++ test('an older CLI is told it cannot take the image', () async {+ clipboardHolds(null);+ clipboardImage(onePixelPng);++ await paste();++ expect(sentTypes, isEmpty);+ expect(reports, ["This machine's harness is too old to receive images."]);+ });++ test('an image goes out as an upload', () async {+ machine.terminalImagePasteAvailable = true;+ clipboardHolds(null);+ clipboardImage(onePixelPng);++ // Nothing answers the upload's begin here, so only that it was asked for is checked.+ unawaited(paste());+ await pumpEventQueue();++ expect(sentTypes, contains('terminal_chunked_upload_begin'));+ expect(reports, isEmpty);+ });++ test('a stream that changed during the image read gets no upload', () async {+ machine.terminalImagePasteAvailable = true;+ clipboardHolds(null);+ clipboardImage(onePixelPng, onRead: () => session.streamId = 's2');++ await paste();++ expect(sentTypes, isEmpty);+ expect(reports, [pasteChangedMessage]);+ });+test('a stream that changed during the read is not pasted into', () async {machine.terminalPasteRawAvailable = true;// Reconnected while the prompt was up: same session, a different stream.