Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

mobile: paste clipboard images on iOS

jaylfce9e7b733dbdb
diff --git a/mobile/ios/Runner/AppDelegate.swift b/mobile/ios/Runner/AppDelegate.swift
index 3c09f949d..1076f4b55 100644
--- a/mobile/ios/Runner/AppDelegate.swift
+++ b/mobile/ios/Runner/AppDelegate.swift
@@ -17,6 +17,40 @@ import UserNotifications
func didInitializeImplicitFlutterEngine(_ engineBridge: FlutterImplicitEngineBridge) {
GeneratedPluginRegistrant.register(with: engineBridge.pluginRegistry)
DeviceNameChannel.register(with: engineBridge.pluginRegistry)
+ ClipboardImageChannel.register(with: engineBridge.pluginRegistry)
+ }
+}
+
+/// `harness/clipboard_image` — the image on the clipboard, as PNG bytes (Dart:
+/// `lib/clipboard/native_clipboard.dart`). Flutter's own clipboard reads `text/plain` only, so a
+/// screenshot or a "Copy" from Photos was invisible to Paste. The same channel the desktop runners
+/// answer; only `readImagePng` is implemented here, since nothing on the phone writes an image.
+enum ClipboardImageChannel {
+ static func register(with registry: FlutterPluginRegistry) {
+ guard let messenger = registry.registrar(forPlugin: "HarnessClipboardImage")?.messenger() else { return }
+ let channel = FlutterMethodChannel(name: "harness/clipboard_image", binaryMessenger: messenger)
+ channel.setMethodCallHandler { call, result in
+ guard call.method == "readImagePng" else { result(FlutterMethodNotImplemented); return }
+ // `hasImages` answers without the system's paste prompt, so an empty clipboard never asks.
+ let pasteboard = UIPasteboard.general
+ guard pasteboard.hasImages else { result(nil); return }
+ // An existing PNG is passed through untouched; anything else (a JPEG or HEIC from Photos) is
+ // re-encoded, which for a full-size photo is slow enough to keep off the main thread.
+ if let png = pasteboard.data(forPasteboardType: "public.png") {
+ result(FlutterStandardTypedData(bytes: png))
+ return
+ }
+ // From here an image IS on the clipboard, so a failure answers EMPTY bytes rather than nil:
+ // Dart then says the image is unreadable instead of that there is nothing to paste.
+ let unreadable = FlutterStandardTypedData(bytes: Data())
+ guard let image = pasteboard.image else { result(unreadable); return }
+ DispatchQueue.global(qos: .userInitiated).async {
+ let png = image.pngData()
+ DispatchQueue.main.async {
+ result(png.map { FlutterStandardTypedData(bytes: $0) } ?? unreadable)
+ }
+ }
+ }
}
}
diff --git a/mobile/lib/clipboard/native_clipboard.dart b/mobile/lib/clipboard/native_clipboard.dart
index 0422c3508..eb61cb1be 100644
--- a/mobile/lib/clipboard/native_clipboard.dart
+++ b/mobile/lib/clipboard/native_clipboard.dart
@@ -6,8 +6,9 @@ import 'package:flutter/services.dart';
/// `widgets/terminal_panel.dart`'s `_paste()`. Reading a real IMAGE off the clipboard (a
/// screenshot, "Copy Image" from a browser, ...) needs a native round trip instead: NSPasteboard on
/// macOS (`macos/Runner/MainFlutterWindow.swift`), the GTK clipboard on Linux
-/// (`linux/runner/my_application.cc`). Both answer over the same channel name and method, so this
-/// wrapper is the one place call sites need to know about.
+/// (`linux/runner/my_application.cc`), UIPasteboard on iOS (`ios/Runner/AppDelegate.swift`, read
+/// only). All answer over the same channel name and method, so this wrapper is the one place call
+/// sites need to know about.
class NativeClipboard {
NativeClipboard._();
@@ -17,12 +18,16 @@ class NativeClipboard {
/// Reads the system clipboard for an image, returned as PNG bytes.
///
+ /// EMPTY bytes (iOS) mean an image IS on the clipboard but could not be re-encoded, so a caller
+ /// can say "unreadable" rather than "nothing there". Every other call site already treats empty
+ /// as "no image".
+ ///
/// Returns `null` on any platform without a native handler for this channel (Windows — the
- /// runner is unexercised, see CLAUDE.md — and any platform that isn't macOS/Linux) or when the
- /// clipboard genuinely holds no image, so call sites can use one check to fall through to
+ /// runner is unexercised, see CLAUDE.md — and any platform that isn't macOS/Linux/iOS) or when
+ /// the clipboard genuinely holds no image, so call sites can use one check to fall through to
/// today's text-paste behavior either way.
static Future<Uint8List?> readImagePng() async {
- if (!Platform.isMacOS && !Platform.isLinux) return null;
+ if (!Platform.isMacOS && !Platform.isLinux && !Platform.isIOS) return null;
try {
final bytes = await _channel.invokeMethod<Uint8List>('readImagePng');
return bytes;
diff --git a/mobile/lib/phone/terminal_paste.dart b/mobile/lib/phone/terminal_paste.dart
index 8c39c2616..d2d135275 100644
--- a/mobile/lib/phone/terminal_paste.dart
+++ b/mobile/lib/phone/terminal_paste.dart
@@ -1,21 +1,24 @@
import 'package:flutter/services.dart';
+import 'package:harness_mobile/clipboard/native_clipboard.dart';
import 'package:harness_mobile/state/app_state.dart' show MachineState;
+import 'package:harness_mobile/terminal/image_transcode.dart';
import 'package:harness_mobile/terminal/terminal_session.dart';
/// Said when the terminal the paste was meant for is no longer the one to send it to.
const pasteChangedMessage =
'The terminal changed before the paste, so nothing was sent.';
-/// Pastes the phone's clipboard text into [session], as one paste rather than as typing.
+/// Pastes the phone's clipboard into [session]: its text as one paste rather than as typing, or,
+/// when there is no text, its image, the same upload the key bar's image button makes.
///
/// The Paste row of the terminal's actions sheet (`phone/terminal_page.dart`). iOS has no other way
/// in: long-press on the terminal is selection there, and the soft keyboard has no paste key.
///
/// Text is asked about first with [Clipboard.hasStrings], which iOS answers without its "Allow
-/// Paste" prompt, so an empty clipboard never asks. Unlike the desktop's ⌘V there is no Ctrl+V
-/// fallthrough: the agent runs on another machine, whose engine would read THAT machine's
-/// clipboard rather than this phone's.
+/// Paste" prompt, so the person is asked once, for the one thing actually read. Unlike the
+/// desktop's ⌘V there is no Ctrl+V fallthrough: the agent runs on another machine, whose engine
+/// would read THAT machine's clipboard rather than this phone's.
///
/// [machine] is the session's machine, read for what its CLI can take at the moment of sending.
/// Every outcome worth telling the person about goes to [report]; a refused prompt says nothing.
@@ -36,30 +39,66 @@ Future<void> pasteClipboard({
bool stillOwnsPaste() =>
stillShowing() && session.acceptsInput && session.streamId == streamId;
- final String? text;
try {
- if (!await Clipboard.hasStrings()) {
- report('There is no text on the clipboard.');
+ if (await Clipboard.hasStrings()) {
+ final text = (await Clipboard.getData(Clipboard.kTextPlain))?.text;
+ // Empty here is a refused prompt as often as an empty string: nothing to say about either.
+ if (text == null || text.isEmpty) return;
+ if (!stillOwnsPaste()) {
+ report(pasteChangedMessage);
+ return;
+ }
+ // The same choice the desktop's paste makes: one atomic paste frame where the machine's CLI
+ // knows it, otherwise xterm's own paste (bracketed when the program asked for it).
+ if (machine != null && machine.terminalPasteRawAvailable) {
+ if (!await session.pasteText(text)) {
+ report('The paste could not be sent.');
+ }
+ } else {
+ session.terminal.paste(text);
+ }
return;
}
- text = (await Clipboard.getData(Clipboard.kTextPlain))?.text;
} on PlatformException {
report('Could not read the clipboard.');
return;
}
- // Empty here is a refused prompt as often as an empty string: nothing to say about either.
- if (text == null || text.isEmpty) return;
- if (!stillOwnsPaste()) {
- report(pasteChangedMessage);
+
+ // A plain shell, not an agent: the CLI delivers an image by replaying Ctrl+V, which a shell reads
+ // as quoted-insert. The desktop skips image paste there for the same reason.
+ if (session.engineId == 'terminal') {
+ report('There is no text on the clipboard.');
return;
}
- // The same choice the desktop's paste makes: one atomic paste frame where the machine's CLI
- // knows it, otherwise xterm's own paste (bracketed when the program asked for it).
- if (machine != null && machine.terminalPasteRawAvailable) {
- if (!await session.pasteText(text)) {
- report('The paste could not be sent.');
- }
- } else {
- session.terminal.paste(text);
+ final imageBytes = await NativeClipboard.readImagePng();
+ if (imageBytes == null) {
+ report('There is nothing on the clipboard to paste.');
+ return;
+ }
+ // Empty is the native side saying an image IS there but could not be read, see
+ // [NativeClipboard.readImagePng].
+ if (imageBytes.isEmpty) {
+ report("The clipboard's image isn't one this phone can read.");
+ return;
+ }
+ if (machine == null || !machine.terminalImagePasteAvailable) {
+ report("This machine's harness is too old to receive images.");
+ return;
+ }
+ // Through the same transcode as a picked photo: a clipboard image is as often a full-size camera
+ // shot as a screenshot, and it has to be scaled under the upload's ceiling either way.
+ switch (await transcodeToPng(imageBytes)) {
+ case ImageTranscodeUnreadable():
+ report("The clipboard's image isn't one this phone can read.");
+ case ImageTranscodeTooLarge():
+ report('That image is too large to send, even scaled down.');
+ case ImageTranscodeOk(:final pngBytes):
+ if (!stillOwnsPaste()) {
+ report(pasteChangedMessage);
+ return;
+ }
+ if (!await session.pasteImage(pngBytes)) {
+ report('The image could not be sent.');
+ }
}
}
diff --git a/mobile/lib/widgets/terminal_panel.dart b/mobile/lib/widgets/terminal_panel.dart
index 987799209..f6359f122 100644
--- a/mobile/lib/widgets/terminal_panel.dart
+++ b/mobile/lib/widgets/terminal_panel.dart
@@ -1062,7 +1062,7 @@ class _TerminalPanelState extends State<TerminalPanel>
/// text at all, so a naive body finds `null` and returns, silently: the single
/// most common thing anyone pastes into a coding agent did nothing, with no
/// error and nothing in a log. [NativeClipboard] closes that gap with a native
- /// platform-channel read for an actual image (macOS/Linux only; see its doc).
+ /// platform-channel read for an actual image (macOS, Linux, iOS; see its doc).
///
/// The engines running in these panes read the system clipboard THEMSELVES —
/// Claude Code attaches an image on Ctrl+V — so on a LOCAL pane that is already
@@ -1074,10 +1074,10 @@ class _TerminalPanelState extends State<TerminalPanel>
/// clipboard than this one — see `MachineState.isLocalMachine`.
/// An image handed over by the software keyboard's own clipboard.
///
- /// This is the phone's ONLY way to get an image into a pane: Flutter's
- /// [Clipboard] reads `text/plain` and nothing else, and the native reader
- /// [_paste] falls back on is macOS/Linux. Gboard hands the bytes over directly,
- /// so there is no clipboard to read at all.
+ /// Flutter's [Clipboard] reads `text/plain` and nothing else, and Gboard hands
+ /// the bytes over directly, so there is no clipboard to read at all. (The
+ /// actions sheet's Paste reads the system clipboard's image instead, through
+ /// [NativeClipboard]: see `phone/terminal_paste.dart`.)
///
/// ⚠️ A pane on a phone is ALWAYS remote — the agent runs on another machine,
/// whose engine reads a different OS clipboard — so unlike the desktop there is
diff --git a/mobile/test/terminal_paste_test.dart b/mobile/test/terminal_paste_test.dart
index bca06902e..de56f82c4 100644
--- a/mobile/test/terminal_paste_test.dart
+++ b/mobile/test/terminal_paste_test.dart
@@ -1,3 +1,10 @@
+// NativeClipboard asks its channel only where a runner answers it, which a Windows host is not.
+@TestOn('!windows')
+library;
+
+import 'dart:async';
+import 'dart:convert';
+
import 'package:flutter/services.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:harness_mobile/core/models.dart';
@@ -19,6 +26,24 @@ void main() {
late List<String> reports;
late TerminalSession session;
late MachineState machine;
+ late List<String> sentTypes;
+
+ const imageChannel = MethodChannel('harness/clipboard_image');
+
+ /// A 1x1 PNG, for an image the transcode can actually read.
+ final onePixelPng = base64Decode(
+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==',
+ );
+
+ /// The image on the clipboard as the native side answers it: null for none, empty for one it
+ /// could not read. [onRead] runs during the read.
+ void clipboardImage(Uint8List? bytes, {void Function()? onRead}) {
+ messenger.setMockMethodCallHandler(imageChannel, (call) async {
+ if (call.method != 'readImagePng') return null;
+ onRead?.call();
+ return bytes;
+ });
+ }
/// The clipboard as Flutter's [Clipboard] sees it. [onRead] runs while the text is being read,
/// the window in which iOS can be showing its paste prompt.
@@ -45,7 +70,12 @@ void main() {
setUp(() {
frames = [];
reports = [];
+ sentTypes = [];
session = controllingSession(
+ send: (type, _) async {
+ sentTypes.add(type);
+ return true;
+ },
sendBinary: (frame) async {
frames.add(frame);
return true;
@@ -62,6 +92,7 @@ void main() {
tearDown(() {
messenger.setMockMethodCallHandler(SystemChannels.platform, null);
+ messenger.setMockMethodCallHandler(imageChannel, null);
});
test('text goes out as one paste frame where the CLI knows it', () async {
@@ -90,13 +121,92 @@ void main() {
test('an empty clipboard is said, not sent', () async {
machine.terminalPasteRawAvailable = true;
clipboardHolds(null);
+ clipboardImage(null);
await paste();
expect(frames, isEmpty);
+ expect(reports, ['There is nothing on the clipboard to paste.']);
+ });
+
+ test('a plain shell is never offered the image', () async {
+ session =
+ TerminalSession(
+ machineId: 'm',
+ agentId: 'a',
+ agentName: 'Shell',
+ engineId: 'terminal',
+ send: (_, _) async => true,
+ sendBinary: (_) async => true,
+ )
+ ..status = TerminalSessionStatus.controlling
+ ..streamId = 's';
+ clipboardHolds(null);
+ var asked = false;
+ clipboardImage(onePixelPng, onRead: () => asked = true);
+
+ await paste();
+
+ expect(asked, isFalse);
expect(reports, ['There is no text on the clipboard.']);
});
+ test('an image the phone could not read is said to be unreadable', () async {
+ machine.terminalImagePasteAvailable = true;
+ clipboardHolds(null);
+ clipboardImage(Uint8List(0));
+
+ await paste();
+
+ expect(sentTypes, isEmpty);
+ expect(reports, ["The clipboard's image isn't one this phone can read."]);
+ });
+
+ test('bytes that are not a picture are unreadable too', () async {
+ machine.terminalImagePasteAvailable = true;
+ clipboardHolds(null);
+ clipboardImage(Uint8List.fromList([1, 2, 3, 4]));
+
+ await paste();
+
+ expect(sentTypes, isEmpty);
+ expect(reports, ["The clipboard's image isn't one this phone can read."]);
+ });
+
+ test('an older CLI is told it cannot take the image', () async {
+ clipboardHolds(null);
+ clipboardImage(onePixelPng);
+
+ await paste();
+
+ expect(sentTypes, isEmpty);
+ expect(reports, ["This machine's harness is too old to receive images."]);
+ });
+
+ test('an image goes out as an upload', () async {
+ machine.terminalImagePasteAvailable = true;
+ clipboardHolds(null);
+ clipboardImage(onePixelPng);
+
+ // Nothing answers the upload's begin here, so only that it was asked for is checked.
+ unawaited(paste());
+ await pumpEventQueue();
+
+ expect(sentTypes, contains('terminal_chunked_upload_begin'));
+ expect(reports, isEmpty);
+ });
+
+ test('a stream that changed during the image read gets no upload', () async {
+ machine.terminalImagePasteAvailable = true;
+ clipboardHolds(null);
+ clipboardImage(onePixelPng, onRead: () => session.streamId = 's2');
+
+ await paste();
+
+ expect(sentTypes, isEmpty);
+ expect(reports, [pasteChangedMessage]);
+ });
+
test('a stream that changed during the read is not pasted into', () async {
machine.terminalPasteRawAvailable = true;
// Reconnected while the prompt was up: same session, a different stream.