Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat(mobile): let the phone's keyboard paste into a terminal, images included

bubu-dudud6fe5f85ed30
diff --git a/mobile/lib/widgets/terminal_panel.dart b/mobile/lib/widgets/terminal_panel.dart
index 1e123c3ac..dfab55f7a 100644
--- a/mobile/lib/widgets/terminal_panel.dart
+++ b/mobile/lib/widgets/terminal_panel.dart
@@ -934,6 +934,26 @@ class _TerminalPanelState extends State<TerminalPanel>
/// upload, daemon writes the far side's OS clipboard, daemon replays Ctrl+V) is
/// reserved for a genuinely REMOTE pane, whose engine reads a DIFFERENT
/// clipboard than this one — see `MachineState.isLocalMachine`.
+ /// An image handed over by the software keyboard's own clipboard.
+ ///
+ /// This is the phone's ONLY way to get an image into a pane: Flutter's
+ /// [Clipboard] reads `text/plain` and nothing else, and the native reader
+ /// [_paste] falls back on is macOS/Linux. Gboard hands the bytes over directly,
+ /// so there is no clipboard to read at all.
+ ///
+ /// ⚠️ A pane on a phone is ALWAYS remote — the agent runs on another machine,
+ /// whose engine reads a different OS clipboard — so unlike the desktop there is
+ /// no local shortcut here: every paste is the chunked upload.
+ void _onContentInserted(KeyboardInsertedContent content) {
+ final bytes = content.data;
+ if (bytes == null || bytes.isEmpty) return;
+ if (widget.readOnly || !widget.session.acceptsInput) return;
+ if (bytes.length > terminalLocalImagePasteMaxPayloadBytes) return;
+ final machine = widget.notifier.stateOf(widget.session.machineId);
+ if (machine == null || !machine.terminalImagePasteAvailable) return;
+ unawaited(widget.session.pasteImage(bytes));
+ }
+
Future<void> _paste() async {
if (widget.readOnly || !widget.session.acceptsInput) return;
final text = (await Clipboard.getData(Clipboard.kTextPlain))?.text;
@@ -1281,6 +1301,16 @@ class _TerminalPanelState extends State<TerminalPanel>
// holds from the very first frame — no scaled first paint
// and no startup resize.
textScaler: TextScaler.noScaling,
+ // ⚠️ PNG alone, and not because other types are rare.
+ // The daemon writes what it receives to a file it names
+ // `<uuid>.png` outright (`cli/src/lib/pasteDropFiles.ts`),
+ // so a JPEG would arrive on the far machine under a name
+ // that lies about it. Widening this means teaching the
+ // CLI the real type first — and an older CLI, which
+ // `terminalImagePasteAvailable` already gates on, would
+ // still not know it.
+ allowedMimeTypes: const ['image/png'],
+ onContentInserted: _onContentInserted,
onKeyEvent: _onTerminalKey,
onTapDown: _onLinkTapDown,
onTapUp: _onLinkTapUp,
diff --git a/mobile/test/terminal_ime_input_test.dart b/mobile/test/terminal_ime_input_test.dart
index 563e0fcdd..f61a06742 100644
--- a/mobile/test/terminal_ime_input_test.dart
+++ b/mobile/test/terminal_ime_input_test.dart
@@ -1,4 +1,5 @@
import 'package:flutter/material.dart';
+import 'package:flutter/services.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:xterm/xterm.dart';
@@ -119,4 +120,85 @@ void main() {
},
variant: TargetPlatformVariant.only(TargetPlatform.iOS),
);
+
+ testWidgets(
+ 'the keyboard keeps its clipboard, at the price of learning what is typed',
+ (tester) async {
+ final config = await attachedConfig(tester);
+
+ // `false` here is IME_FLAG_NO_PERSONALIZED_LEARNING, which Gboard reads
+ // as incognito — and incognito takes the toolbar, and with it the
+ // CLIPBOARD. A phone has no ⌘V and no readable clipboard beyond
+ // `text/plain`, so declining the learning also declined every paste.
+ expect(config['enableIMEPersonalizedLearning'], isTrue);
+ },
+ variant: TargetPlatformVariant.all(),
+ );
+
+ testWidgets(
+ 'an undeclared content type is refused before Dart sees it',
+ (tester) async {
+ final config = await attachedConfig(tester);
+
+ // Empty is Flutter's default, and it is what makes Gboard answer a
+ // clipboard image with "the current app does not allow pasting images
+ // here" — in the keyboard, with nothing reaching `insertContent` at all.
+ expect(config['contentCommitMimeTypes'], isEmpty);
+ },
+ variant: TargetPlatformVariant.all(),
+ );
+
+ testWidgets(
+ 'a declared type is offered and its bytes reach the host',
+ (tester) async {
+ KeyboardInsertedContent? received;
+ await tester.pumpWidget(
+ MaterialApp(
+ home: Scaffold(
+ body: SizedBox(
+ width: 400,
+ height: 320,
+ child: TerminalView(
+ newTerminal(),
+ autofocus: true,
+ allowedMimeTypes: const ['image/png'],
+ onContentInserted: (content) => received = content,
+ ),
+ ),
+ ),
+ ),
+ );
+ await tester.pump();
+
+ expect(
+ tester.testTextInput.setClientArgs!['contentCommitMimeTypes'],
+ ['image/png'],
+ );
+
+ // What the engine sends once the keyboard hands an image over. Delivered
+ // on the platform channel rather than through `testTextInput`, which has
+ // no helper for content.
+ final bytes = <int>[137, 80, 78, 71];
+ await tester.binding.defaultBinaryMessenger.handlePlatformMessage(
+ 'flutter/textinput',
+ const JSONMethodCodec().encodeMethodCall(
+ MethodCall('TextInputClient.performAction', <Object?>[
+ -1,
+ 'TextInputAction.commitContent',
+ <String, dynamic>{
+ 'mimeType': 'image/png',
+ 'data': bytes,
+ 'uri': 'content://clip/1',
+ },
+ ]),
+ ),
+ (_) {},
+ );
+ await tester.pump();
+
+ expect(received?.mimeType, 'image/png');
+ expect(received?.data, bytes);
+ },
+ variant: TargetPlatformVariant.only(TargetPlatform.android),
+ );
}
diff --git a/mobile/third_party/xterm/README.autonomous.md b/mobile/third_party/xterm/README.autonomous.md
index 75c215b60..c63917ca1 100644
--- a/mobile/third_party/xterm/README.autonomous.md
+++ b/mobile/third_party/xterm/README.autonomous.md
@@ -131,3 +131,35 @@ it if one is dropped.
`--flag` stay syntax rather than typography. Regression:
`mobile/test/terminal_ime_input_test.dart`, the only build that reaches the
mobile branch.
+
+10. **The keyboard may paste, and may hand over an image**
+ (`lib/src/ui/custom_text_edit.dart`, `lib/src/terminal_view.dart`).
+ `enableIMEPersonalizedLearning: false` reads as a privacy default and is
+ not one on Android: it sets `IME_FLAG_NO_PERSONALIZED_LEARNING`, which puts
+ Gboard in incognito mode, and incognito replaces the toolbar with the
+ incognito glasses — taking the CLIPBOARD with it. A phone has no ⌘V and no
+ readable system clipboard for anything but `text/plain`, so that one flag
+ left a pane with no way to receive text from elsewhere on the device at
+ all. The two are one switch in Gboard; the cost of turning it back on is
+ that words typed at this prompt are learned, masked password prompts
+ included, since a pty cannot tell the keyboard one is open.
+
+ Images are a second, separate refusal: Gboard rejects them in the keyboard
+ with "the current app does not allow pasting images here" whenever
+ `EditorInfo.contentMimeTypes` is empty, which is what Flutter sends while
+ `allowedMimeTypes` is unset — before any Dart runs, so `insertContent`
+ being an empty stub was never the reason. It is now forwarded through
+ `TerminalView.onContentInserted`, and the embedder decides what an image
+ means; `mobile/lib/widgets/terminal_panel.dart` sends it as the chunked
+ `TerminalSession.pasteImage` upload, which on a phone is the only path
+ there is — every pane is remote.
+
+ ⚠️ `image/png` alone, and not because other types are rare: the daemon
+ names what it receives `<uuid>.png` outright
+ (`cli/src/lib/pasteDropFiles.ts`), so a JPEG would arrive under a name that
+ lies about it. Widening this means teaching the CLI the real type first.
+
+ ⚠️ Kept out of the desktop copy deliberately. The flag is Android-only, and
+ desktop reads images through `NativeClipboard` on ⌘V instead — mirroring it
+ there would have been an inert change carrying a privacy decision made for
+ a platform that never sees it.
diff --git a/mobile/third_party/xterm/lib/src/terminal_view.dart b/mobile/third_party/xterm/lib/src/terminal_view.dart
index 2269071c6..3f1a3709f 100644
--- a/mobile/third_party/xterm/lib/src/terminal_view.dart
+++ b/mobile/third_party/xterm/lib/src/terminal_view.dart
@@ -48,6 +48,8 @@ class TerminalView extends StatefulWidget {
this.cursorType = TerminalCursorType.block,
this.alwaysShowCursor = false,
this.deleteDetection = false,
+ this.allowedMimeTypes = const <String>[],
+ this.onContentInserted,
this.shortcuts,
this.onKeyEvent,
this.readOnly = false,
@@ -141,6 +143,15 @@ class TerminalView extends StatefulWidget {
/// default.
final bool deleteDetection;
+ /// Clipboard content types this terminal accepts from a software keyboard —
+ /// see [CustomTextEdit.allowedMimeTypes]. Empty means the keyboard refuses the
+ /// paste itself, without asking.
+ final List<String> allowedMimeTypes;
+
+ /// One accepted clipboard item, handed straight to the host: a pty takes bytes,
+ /// and what to do with an image is the embedder's decision, not this widget's.
+ final void Function(KeyboardInsertedContent content)? onContentInserted;
+
/// Shortcuts for this terminal. This has higher priority than input handler
/// of the terminal If not provided, [defaultTerminalShortcuts] will be used.
final Map<ShortcutActivator, Intent>? shortcuts;
@@ -305,6 +316,8 @@ class TerminalViewState extends State<TerminalView> {
inputType: widget.keyboardType,
keyboardAppearance: widget.keyboardAppearance,
deleteDetection: widget.deleteDetection,
+ allowedMimeTypes: widget.allowedMimeTypes,
+ onContentInserted: widget.onContentInserted,
onInsert: _onInsert,
onDelete: (count) {
_scrollToBottom();
diff --git a/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dart b/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dart
index 204184186..8f6f4ef99 100644
--- a/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dart
+++ b/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dart
@@ -22,6 +22,8 @@ class CustomTextEdit extends StatefulWidget {
this.inputAction = TextInputAction.newline,
this.keyboardAppearance = Brightness.light,
this.deleteDetection = false,
+ this.allowedMimeTypes = const <String>[],
+ this.onContentInserted,
});
final Widget child;
@@ -50,6 +52,21 @@ class CustomTextEdit extends StatefulWidget {
final bool deleteDetection;
+ /// Content types this connection accepts from the keyboard's own clipboard, as
+ /// `EditorInfo.contentMimeTypes` on Android.
+ ///
+ /// ⚠️ Empty — the default — is what makes Gboard refuse a clipboard image with
+ /// "the current app does not allow pasting images here". The refusal happens in
+ /// the keyboard, before anything reaches Dart, so no amount of handling here
+ /// substitutes for declaring the type.
+ final List<String> allowedMimeTypes;
+
+ /// One item of [allowedMimeTypes] arriving from the keyboard.
+ ///
+ /// ⚠️ [KeyboardInsertedContent.data] can be null even for a type that was asked
+ /// for: the platform hands over a URI it could not read for us.
+ final void Function(KeyboardInsertedContent content)? onContentInserted;
+
@override
CustomTextEditState createState() => CustomTextEditState();
}
@@ -209,7 +226,20 @@ class CustomTextEditState extends State<CustomTextEdit> with TextInputClient {
// autocorrect on above is what would finally let iOS act on them.
smartDashesType: SmartDashesType.disabled,
smartQuotesType: SmartQuotesType.disabled,
- enableIMEPersonalizedLearning: false,
+ // ⚠️ `false` here maps to Android's IME_FLAG_NO_PERSONALIZED_LEARNING,
+ // which puts Gboard in INCOGNITO MODE — and incognito costs far more
+ // than the learning it declines. Gboard replaces its toolbar with the
+ // incognito glasses, and the clipboard goes with it: no pasting into a
+ // terminal from anywhere else on the phone. The two are one switch in
+ // Gboard, so a terminal that wants paste cannot also decline learning.
+ //
+ // Declining was the safer default and is no longer the right one: a
+ // phone has no other way to get text INTO a pane, where a desktop has
+ // ⌘V. The cost is real — Gboard now learns words typed at this prompt,
+ // masked password prompts included, since a pty gives the keyboard no
+ // way to know one is open.
+ enableIMEPersonalizedLearning: true,
+ allowedMimeTypes: widget.allowedMimeTypes,
);
_connection = TextInput.attach(this, config);
@@ -429,7 +459,9 @@ class CustomTextEditState extends State<CustomTextEdit> with TextInputClient {
) {}
@override
- void insertContent(KeyboardInsertedContent content) {}
+ void insertContent(KeyboardInsertedContent content) {
+ widget.onContentInserted?.call(content);
+ }
@override
bool onFocusReceived() => false;