The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat(mobile): let the phone's keyboard paste into a terminal, images included
bubu-dudud6fe5f85ed30
diff --git a/mobile/lib/widgets/terminal_panel.dart b/mobile/lib/widgets/terminal_panel.dartindex 1e123c3ac..dfab55f7a 100644--- a/mobile/lib/widgets/terminal_panel.dart+++ b/mobile/lib/widgets/terminal_panel.dart@@ -934,6 +934,26 @@ class _TerminalPanelState extends State<TerminalPanel>/// upload, daemon writes the far side's OS clipboard, daemon replays Ctrl+V) is/// reserved for a genuinely REMOTE pane, whose engine reads a DIFFERENT/// clipboard than this one — see `MachineState.isLocalMachine`.+ /// An image handed over by the software keyboard's own clipboard.+ ///+ /// This is the phone's ONLY way to get an image into a pane: Flutter's+ /// [Clipboard] reads `text/plain` and nothing else, and the native reader+ /// [_paste] falls back on is macOS/Linux. Gboard hands the bytes over directly,+ /// so there is no clipboard to read at all.+ ///+ /// ⚠️ A pane on a phone is ALWAYS remote — the agent runs on another machine,+ /// whose engine reads a different OS clipboard — so unlike the desktop there is+ /// no local shortcut here: every paste is the chunked upload.+ void _onContentInserted(KeyboardInsertedContent content) {+ final bytes = content.data;+ if (bytes == null || bytes.isEmpty) return;+ if (widget.readOnly || !widget.session.acceptsInput) return;+ if (bytes.length > terminalLocalImagePasteMaxPayloadBytes) return;+ final machine = widget.notifier.stateOf(widget.session.machineId);+ if (machine == null || !machine.terminalImagePasteAvailable) return;+ unawaited(widget.session.pasteImage(bytes));+ }+Future<void> _paste() async {if (widget.readOnly || !widget.session.acceptsInput) return;final text = (await Clipboard.getData(Clipboard.kTextPlain))?.text;@@ -1281,6 +1301,16 @@ class _TerminalPanelState extends State<TerminalPanel>// holds from the very first frame — no scaled first paint// and no startup resize.textScaler: TextScaler.noScaling,+ // ⚠️ PNG alone, and not because other types are rare.+ // The daemon writes what it receives to a file it names+ // `<uuid>.png` outright (`cli/src/lib/pasteDropFiles.ts`),+ // so a JPEG would arrive on the far machine under a name+ // that lies about it. Widening this means teaching the+ // CLI the real type first — and an older CLI, which+ // `terminalImagePasteAvailable` already gates on, would+ // still not know it.+ allowedMimeTypes: const ['image/png'],+ onContentInserted: _onContentInserted,onKeyEvent: _onTerminalKey,onTapDown: _onLinkTapDown,onTapUp: _onLinkTapUp,diff --git a/mobile/test/terminal_ime_input_test.dart b/mobile/test/terminal_ime_input_test.dartindex 563e0fcdd..f61a06742 100644--- a/mobile/test/terminal_ime_input_test.dart+++ b/mobile/test/terminal_ime_input_test.dart@@ -1,4 +1,5 @@import 'package:flutter/material.dart';+import 'package:flutter/services.dart';import 'package:flutter_test/flutter_test.dart';import 'package:xterm/xterm.dart';@@ -119,4 +120,85 @@ void main() {},variant: TargetPlatformVariant.only(TargetPlatform.iOS),);++ testWidgets(+ 'the keyboard keeps its clipboard, at the price of learning what is typed',+ (tester) async {+ final config = await attachedConfig(tester);++ // `false` here is IME_FLAG_NO_PERSONALIZED_LEARNING, which Gboard reads+ // as incognito — and incognito takes the toolbar, and with it the+ // CLIPBOARD. A phone has no ⌘V and no readable clipboard beyond+ // `text/plain`, so declining the learning also declined every paste.+ expect(config['enableIMEPersonalizedLearning'], isTrue);+ },+ variant: TargetPlatformVariant.all(),+ );++ testWidgets(+ 'an undeclared content type is refused before Dart sees it',+ (tester) async {+ final config = await attachedConfig(tester);++ // Empty is Flutter's default, and it is what makes Gboard answer a+ // clipboard image with "the current app does not allow pasting images+ // here" — in the keyboard, with nothing reaching `insertContent` at all.+ expect(config['contentCommitMimeTypes'], isEmpty);+ },+ variant: TargetPlatformVariant.all(),+ );++ testWidgets(+ 'a declared type is offered and its bytes reach the host',+ (tester) async {+ KeyboardInsertedContent? received;+ await tester.pumpWidget(+ MaterialApp(+ home: Scaffold(+ body: SizedBox(+ width: 400,+ height: 320,+ child: TerminalView(+ newTerminal(),+ autofocus: true,+ allowedMimeTypes: const ['image/png'],+ onContentInserted: (content) => received = content,+ ),+ ),+ ),+ ),+ );+ await tester.pump();++ expect(+ tester.testTextInput.setClientArgs!['contentCommitMimeTypes'],+ ['image/png'],+ );++ // What the engine sends once the keyboard hands an image over. Delivered+ // on the platform channel rather than through `testTextInput`, which has+ // no helper for content.+ final bytes = <int>[137, 80, 78, 71];+ await tester.binding.defaultBinaryMessenger.handlePlatformMessage(+ 'flutter/textinput',+ const JSONMethodCodec().encodeMethodCall(+ MethodCall('TextInputClient.performAction', <Object?>[+ -1,+ 'TextInputAction.commitContent',+ <String, dynamic>{+ 'mimeType': 'image/png',+ 'data': bytes,+ 'uri': 'content://clip/1',+ },+ ]),+ ),+ (_) {},+ );+ await tester.pump();++ expect(received?.mimeType, 'image/png');+ expect(received?.data, bytes);+ },+ variant: TargetPlatformVariant.only(TargetPlatform.android),+ );}diff --git a/mobile/third_party/xterm/README.autonomous.md b/mobile/third_party/xterm/README.autonomous.mdindex 75c215b60..c63917ca1 100644--- a/mobile/third_party/xterm/README.autonomous.md+++ b/mobile/third_party/xterm/README.autonomous.md@@ -131,3 +131,35 @@ it if one is dropped.`--flag` stay syntax rather than typography. Regression:`mobile/test/terminal_ime_input_test.dart`, the only build that reaches themobile branch.++10. **The keyboard may paste, and may hand over an image**+ (`lib/src/ui/custom_text_edit.dart`, `lib/src/terminal_view.dart`).+ `enableIMEPersonalizedLearning: false` reads as a privacy default and is+ not one on Android: it sets `IME_FLAG_NO_PERSONALIZED_LEARNING`, which puts+ Gboard in incognito mode, and incognito replaces the toolbar with the+ incognito glasses — taking the CLIPBOARD with it. A phone has no ⌘V and no+ readable system clipboard for anything but `text/plain`, so that one flag+ left a pane with no way to receive text from elsewhere on the device at+ all. The two are one switch in Gboard; the cost of turning it back on is+ that words typed at this prompt are learned, masked password prompts+ included, since a pty cannot tell the keyboard one is open.++ Images are a second, separate refusal: Gboard rejects them in the keyboard+ with "the current app does not allow pasting images here" whenever+ `EditorInfo.contentMimeTypes` is empty, which is what Flutter sends while+ `allowedMimeTypes` is unset — before any Dart runs, so `insertContent`+ being an empty stub was never the reason. It is now forwarded through+ `TerminalView.onContentInserted`, and the embedder decides what an image+ means; `mobile/lib/widgets/terminal_panel.dart` sends it as the chunked+ `TerminalSession.pasteImage` upload, which on a phone is the only path+ there is — every pane is remote.++ ⚠️ `image/png` alone, and not because other types are rare: the daemon+ names what it receives `<uuid>.png` outright+ (`cli/src/lib/pasteDropFiles.ts`), so a JPEG would arrive under a name that+ lies about it. Widening this means teaching the CLI the real type first.++ ⚠️ Kept out of the desktop copy deliberately. The flag is Android-only, and+ desktop reads images through `NativeClipboard` on ⌘V instead — mirroring it+ there would have been an inert change carrying a privacy decision made for+ a platform that never sees it.diff --git a/mobile/third_party/xterm/lib/src/terminal_view.dart b/mobile/third_party/xterm/lib/src/terminal_view.dartindex 2269071c6..3f1a3709f 100644--- a/mobile/third_party/xterm/lib/src/terminal_view.dart+++ b/mobile/third_party/xterm/lib/src/terminal_view.dart@@ -48,6 +48,8 @@ class TerminalView extends StatefulWidget {this.cursorType = TerminalCursorType.block,this.alwaysShowCursor = false,this.deleteDetection = false,+ this.allowedMimeTypes = const <String>[],+ this.onContentInserted,this.shortcuts,this.onKeyEvent,this.readOnly = false,@@ -141,6 +143,15 @@ class TerminalView extends StatefulWidget {/// default.final bool deleteDetection;+ /// Clipboard content types this terminal accepts from a software keyboard —+ /// see [CustomTextEdit.allowedMimeTypes]. Empty means the keyboard refuses the+ /// paste itself, without asking.+ final List<String> allowedMimeTypes;++ /// One accepted clipboard item, handed straight to the host: a pty takes bytes,+ /// and what to do with an image is the embedder's decision, not this widget's.+ final void Function(KeyboardInsertedContent content)? onContentInserted;+/// Shortcuts for this terminal. This has higher priority than input handler/// of the terminal If not provided, [defaultTerminalShortcuts] will be used.final Map<ShortcutActivator, Intent>? shortcuts;@@ -305,6 +316,8 @@ class TerminalViewState extends State<TerminalView> {inputType: widget.keyboardType,keyboardAppearance: widget.keyboardAppearance,deleteDetection: widget.deleteDetection,+ allowedMimeTypes: widget.allowedMimeTypes,+ onContentInserted: widget.onContentInserted,onInsert: _onInsert,onDelete: (count) {_scrollToBottom();diff --git a/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dart b/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dartindex 204184186..8f6f4ef99 100644--- a/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dart+++ b/mobile/third_party/xterm/lib/src/ui/custom_text_edit.dart@@ -22,6 +22,8 @@ class CustomTextEdit extends StatefulWidget {this.inputAction = TextInputAction.newline,this.keyboardAppearance = Brightness.light,this.deleteDetection = false,+ this.allowedMimeTypes = const <String>[],+ this.onContentInserted,});final Widget child;@@ -50,6 +52,21 @@ class CustomTextEdit extends StatefulWidget {final bool deleteDetection;+ /// Content types this connection accepts from the keyboard's own clipboard, as+ /// `EditorInfo.contentMimeTypes` on Android.+ ///+ /// ⚠️ Empty — the default — is what makes Gboard refuse a clipboard image with+ /// "the current app does not allow pasting images here". The refusal happens in+ /// the keyboard, before anything reaches Dart, so no amount of handling here+ /// substitutes for declaring the type.+ final List<String> allowedMimeTypes;++ /// One item of [allowedMimeTypes] arriving from the keyboard.+ ///+ /// ⚠️ [KeyboardInsertedContent.data] can be null even for a type that was asked+ /// for: the platform hands over a URI it could not read for us.+ final void Function(KeyboardInsertedContent content)? onContentInserted;+@overrideCustomTextEditState createState() => CustomTextEditState();}@@ -209,7 +226,20 @@ class CustomTextEditState extends State<CustomTextEdit> with TextInputClient {// autocorrect on above is what would finally let iOS act on them.smartDashesType: SmartDashesType.disabled,smartQuotesType: SmartQuotesType.disabled,- enableIMEPersonalizedLearning: false,+ // ⚠️ `false` here maps to Android's IME_FLAG_NO_PERSONALIZED_LEARNING,+ // which puts Gboard in INCOGNITO MODE — and incognito costs far more+ // than the learning it declines. Gboard replaces its toolbar with the+ // incognito glasses, and the clipboard goes with it: no pasting into a+ // terminal from anywhere else on the phone. The two are one switch in+ // Gboard, so a terminal that wants paste cannot also decline learning.+ //+ // Declining was the safer default and is no longer the right one: a+ // phone has no other way to get text INTO a pane, where a desktop has+ // ⌘V. The cost is real — Gboard now learns words typed at this prompt,+ // masked password prompts included, since a pty gives the keyboard no+ // way to know one is open.+ enableIMEPersonalizedLearning: true,+ allowedMimeTypes: widget.allowedMimeTypes,);_connection = TextInput.attach(this, config);@@ -429,7 +459,9 @@ class CustomTextEditState extends State<CustomTextEdit> with TextInputClient {) {}@override- void insertContent(KeyboardInsertedContent content) {}+ void insertContent(KeyboardInsertedContent content) {+ widget.onContentInserted?.call(content);+ }@overridebool onFocusReceived() => false;