The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat(mobile): send a picture to an agent from the key bar
tony504e12d732f0
diff --git a/mobile/android/app/src/main/AndroidManifest.xml b/mobile/android/app/src/main/AndroidManifest.xmlindex 1c812424a..a9f0ae0b3 100644--- a/mobile/android/app/src/main/AndroidManifest.xml+++ b/mobile/android/app/src/main/AndroidManifest.xml@@ -9,6 +9,15 @@<!-- Lets libwebrtc notice a wifi↔cellular change and restart ICE rather thanwait out the disconnect grace. Not a runtime permission. --><uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />+ <!-- Taking a picture to send to an agent. Picking an EXISTING picture needs+ no permission at all: image_picker goes through ACTION_GET_CONTENT / the+ photo picker, which hands back one item the user chose rather than+ opening the library — hence no READ_MEDIA_IMAGES here. -->+ <uses-permission android:name="android.permission.CAMERA" />+ <!-- The camera is how one feature sends a picture, not something the app+ cannot run without: a tablet or emulator with no camera must still+ install and show every other screen. -->+ <uses-feature android:name="android.hardware.camera" android:required="false" /><applicationandroid:label="harness"android:name="${applicationName}"diff --git a/mobile/ios/Runner/Info.plist b/mobile/ios/Runner/Info.plistindex 9eaf95cc6..60412a7c6 100644--- a/mobile/ios/Runner/Info.plist+++ b/mobile/ios/Runner/Info.plist@@ -26,6 +26,12 @@<string>$(FLUTTER_BUILD_NUMBER)</string><key>LSRequiresIPhoneOS</key><true/>+ <!-- Taking a picture to send to an agent. Only the camera needs a string on+ iOS 15: picking from the library goes through PHPicker, which runs out of+ process and grants nothing, so NSPhotoLibraryUsageDescription would never+ be shown and is deliberately absent. -->+ <key>NSCameraUsageDescription</key>+ <string>Take a photo to send to an agent in a terminal.</string><key>UIApplicationSceneManifest</key><dict><key>UIApplicationSupportsMultipleScenes</key>diff --git a/mobile/lib/phone/terminal_key_bar.dart b/mobile/lib/phone/terminal_key_bar.dartindex 95be7cf36..4dd694fa6 100644--- a/mobile/lib/phone/terminal_key_bar.dart+++ b/mobile/lib/phone/terminal_key_bar.dart@@ -4,6 +4,7 @@ import 'package:lucide_icons_flutter/lucide_icons.dart';import 'package:xterm/xterm.dart';import 'package:harness_mobile/shared/theme/app_theme.dart';+import 'phone_sheet.dart';/// The keys a phone keyboard does not have, in a strip above the one it does.///@@ -15,7 +16,16 @@ import 'package:harness_mobile/shared/theme/app_theme.dart';/// It appears with the keyboard and goes away with it: the terminal is short/// enough on a phone that two rows of chrome are worth their height only while/// someone is actually typing.-class TerminalKeyBar extends StatelessWidget {+///+/// ⚠️ **One row that SCROLLS, with the rest behind `»`.** The keys outgrew a+/// fixed grid the moment `enter` and `⇧tab` joined them: eleven across a narrow+/// phone leaves each one under 30px, below the 44px Apple and Android both put+/// as the floor for a touch target, and these are keys people hit repeatedly+/// while looking at the terminal rather than at their thumb. So the row keeps+/// full-size keys and runs off the edge instead, ordered by how often a key is+/// actually reached for, and `»` opens the remainder — digits included — as a+/// second row rather than shrinking the first.+class TerminalKeyBar extends StatefulWidget {const TerminalKeyBar({super.key,required this.terminal,@@ -23,6 +33,8 @@ class TerminalKeyBar extends StatelessWidget {required this.controlArmed,required this.onControlToggle,required this.onDismissKeyboard,+ this.onPickImage,+ this.onTakePhoto,});final Terminal terminal;@@ -38,57 +50,128 @@ class TerminalKeyBar extends StatelessWidget {final ValueChanged<bool> onControlToggle;final VoidCallback onDismissKeyboard;+ /// Sending a picture. Null on a pane that cannot take one — an older CLI that+ /// never advertised `terminalImagePasteAvailable` — and the buttons are then+ /// not drawn at all rather than drawn dead: a key that does nothing is worse+ /// than one that was never offered.+ final VoidCallback? onPickImage;+ final VoidCallback? onTakePhoto;++ @override+ State<TerminalKeyBar> createState() => _TerminalKeyBarState();+}++class _TerminalKeyBarState extends State<TerminalKeyBar> {+ /// Whether `»` has opened the second row.+ ///+ /// Held here rather than by the page: it is a property of this strip, it does+ /// not survive the keyboard going away, and nothing above needs to know.+ bool _expanded = false;+void _send(void Function() action) {- if (!enabled) return;+ if (!widget.enabled) return;HapticFeedback.selectionClick();action();}+ Terminal get _terminal => widget.terminal;++ /// Whether this pane can take a picture at all — see [TerminalKeyBar.onPickImage].+ bool get _canSendImage =>+ widget.onPickImage != null || widget.onTakePhoto != null;++ /// The pinned image button: asks WHICH picture, then hands off.+ ///+ /// The sheet only appears where there is a choice to make. A build with just+ /// one of the two wired goes straight there instead — a sheet with a single+ /// row is a tap spent on nothing.+ void _sendImage() {+ final pick = widget.onPickImage;+ final photo = widget.onTakePhoto;+ if (pick == null) {+ photo?.call();+ return;+ }+ if (photo == null) {+ pick();+ return;+ }+ showPhoneSheet(+ context,+ title: 'Send a picture to this agent',+ actions: [+ PhoneSheetAction(+ icon: LucideIcons.image300,+ label: 'Choose from library',+ onTap: pick,+ ),+ PhoneSheetAction(+ icon: LucideIcons.camera300,+ label: 'Take a photo',+ onTap: photo,+ ),+ ],+ );+ }+@overrideWidget build(BuildContext context) {AppTheme.watch(context);- final top = <Widget>[- _key(label: 'esc', onTap: () => terminal.keyInput(TerminalKey.escape)),- _key(label: 'tab', onTap: () => terminal.keyInput(TerminalKey.tab)),- for (final char in const ['~', '|', '/', '-'])- _key(label: char, onTap: () => terminal.textInput(char)),+ // Ordered by reach, not by keyboard layout: `esc` and `tab` drive an engine's+ // modes, `↵` ends a message, `⇧tab` cycles Claude Code's, and the arrows walk+ // history.+ //+ // ⚠️ Sending a picture is NOT in here, and that is the whole point of the+ // pinned button below. A scrolling row only shows four keys on a 393pt phone,+ // so anything past `⇧tab` is behind a swipe — measured, not guessed. The+ // image button was tenth in this list and therefore invisible on arrival,+ // which for the one action people come to this bar for is the same as absent.+ final primary = <Widget>[+ _key(label: 'esc', onTap: () => _terminal.keyInput(TerminalKey.escape)),+ _key(label: 'tab', onTap: () => _terminal.keyInput(TerminalKey.tab)),_key(- icon: LucideIcons.arrowLeft300,- semanticLabel: 'Left',- onTap: () => terminal.keyInput(TerminalKey.arrowLeft),+ icon: LucideIcons.cornerDownLeft300,+ semanticLabel: 'Enter',+ onTap: () => _terminal.keyInput(TerminalKey.enter),+ ),+ // Shift+Tab is CSI Z, and xterm builds it from the modifier rather than+ // from a key of its own — which is why this passes `shift` instead of+ // looking for a `TerminalKey.shiftTab` that does not exist.+ _key(+ label: '⇧tab',+ onTap: () => _terminal.keyInput(TerminalKey.tab, shift: true),),_key(icon: LucideIcons.arrowUp300,semanticLabel: 'Up',- onTap: () => terminal.keyInput(TerminalKey.arrowUp),+ onTap: () => _terminal.keyInput(TerminalKey.arrowUp),),_key(icon: LucideIcons.arrowDown300,semanticLabel: 'Down',- onTap: () => terminal.keyInput(TerminalKey.arrowDown),+ onTap: () => _terminal.keyInput(TerminalKey.arrowDown),),_key(- icon: LucideIcons.arrowRight300,- semanticLabel: 'Right',- onTap: () => terminal.keyInput(TerminalKey.arrowRight),+ icon: LucideIcons.arrowLeft300,+ semanticLabel: 'Left',+ onTap: () => _terminal.keyInput(TerminalKey.arrowLeft),),- // Not a key the pty ever hears about: it puts the keyboard away, which on- // a phone is the only way to read a full screen of output._key(- icon: LucideIcons.chevronDown300,- semanticLabel: 'Hide keyboard',- alwaysEnabled: true,- onTap: onDismissKeyboard,+ icon: LucideIcons.arrowRight300,+ semanticLabel: 'Right',+ onTap: () => _terminal.keyInput(TerminalKey.arrowRight),),- ];- final bottom = <Widget>[_key(label: 'ctrl',- held: controlArmed,- onTap: () => onControlToggle(!controlArmed),+ held: widget.controlArmed,+ onTap: () => widget.onControlToggle(!widget.controlArmed),),+ ];+ final secondary = <Widget>[+ for (final char in const ['~', '|', '/', '-', '_', ':', '.'])+ _key(label: char, onTap: () => _terminal.textInput(char)),for (final digit in const ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'])- _key(label: digit, onTap: () => terminal.textInput(digit)),+ _key(label: digit, onTap: () => _terminal.textInput(digit)),];return ExcludeFocus(@@ -104,11 +187,57 @@ class TerminalKeyBar extends StatelessWidget {child: Padding(padding: const EdgeInsets.all(6),// Held out of the app-wide text scale like the composer's own type:- // at a large scale eleven keys across a phone stop fitting the row.+ // the keys are sized for a thumb, and a large accessibility scale+ // would grow the labels past the boxes holding them.child: MediaQuery.withNoTextScaling(child: Column(mainAxisSize: MainAxisSize.min,- children: [_row(top), const SizedBox(height: 6), _row(bottom)],+ children: [+ Row(+ children: [+ Expanded(child: _scrollingRow(primary)),+ const SizedBox(width: 4),+ // Pinned for the same reason `»` is, and more so: this is+ // what somebody opens the bar to reach. One button rather+ // than two — the library and the camera are two answers to+ // "which picture", which is a question a sheet asks better+ // than a second key nobody can fit on the row.+ if (_canSendImage) ...[+ _key(+ icon: LucideIcons.image300,+ semanticLabel: 'Send image',+ onTap: _sendImage,+ ),+ const SizedBox(width: 4),+ ],+ // Pinned OUTSIDE the scroller, so the way to the rest of the+ // keys cannot itself be scrolled off the edge.+ _key(+ icon: _expanded+ ? LucideIcons.chevronsLeft300+ : LucideIcons.chevronsRight300,+ semanticLabel: _expanded ? 'Fewer keys' : 'More keys',+ held: _expanded,+ alwaysEnabled: true,+ onTap: () => setState(() => _expanded = !_expanded),+ ),+ const SizedBox(width: 4),+ // Not a key the pty ever hears about: it puts the keyboard+ // away, which on a phone is the only way to read a full+ // screen of output.+ _key(+ icon: LucideIcons.chevronDown300,+ semanticLabel: 'Hide keyboard',+ alwaysEnabled: true,+ onTap: widget.onDismissKeyboard,+ ),+ ],+ ),+ if (_expanded) ...[+ const SizedBox(height: 6),+ _scrollingRow(secondary),+ ],+ ],),),),@@ -116,15 +245,31 @@ class TerminalKeyBar extends StatelessWidget {);}- Widget _row(List<Widget> keys) => Row(- children: [- for (var index = 0; index < keys.length; index++) ...[- if (index > 0) const SizedBox(width: 4),- Expanded(child: keys[index]),- ],- ],+ /// One row of keys at their natural width, scrolling horizontally past the edge.+ ///+ /// `ClampingScrollPhysics` rather than the iOS default: a bouncing row above the+ /// keyboard reads as the whole strip coming loose, and the overscroll glow is+ /// what says "there is more this way" on a row with no scrollbar.+ Widget _scrollingRow(List<Widget> keys) => SizedBox(+ height: _keyHeight,+ child: ListView.separated(+ scrollDirection: Axis.horizontal,+ physics: const ClampingScrollPhysics(),+ padding: EdgeInsets.zero,+ itemCount: keys.length,+ separatorBuilder: (_, _) => const SizedBox(width: 4),+ itemBuilder: (_, index) => keys[index],+ ),);+ /// Tall enough to hit without looking — the floor both platforms put on a+ /// touch target, which the old two-row grid was under at 34.+ static const double _keyHeight = 40;++ /// Wide enough for `⇧tab` at 13px, and the same width for every key so the row+ /// reads as a keyboard rather than as a sentence of buttons.+ static const double _keyWidth = 52;+Widget _key({String? label,IconData? icon,@@ -134,7 +279,7 @@ class TerminalKeyBar extends StatelessWidget {required VoidCallback onTap,}) {assert((label == null) != (icon == null), 'a key carries one of the two');- final live = enabled || alwaysEnabled;+ final live = widget.enabled || alwaysEnabled;final foreground = held? AppPalette.accentOnSurface: live@@ -151,7 +296,8 @@ class TerminalKeyBar extends StatelessWidget {child: AnimatedContainer(duration: const Duration(milliseconds: 120),curve: Curves.easeOut,- height: 34,+ width: _keyWidth,+ height: _keyHeight,alignment: Alignment.center,decoration: BoxDecoration(color: held ? AppSurface.accentWash : AppGlass.surfaceFill,@@ -161,7 +307,7 @@ class TerminalKeyBar extends StatelessWidget {),),child: icon != null- ? Icon(icon, size: 16, color: foreground)+ ? Icon(icon, size: 17, color: foreground): Text(label!,maxLines: 1,diff --git a/mobile/lib/phone/terminal_page.dart b/mobile/lib/phone/terminal_page.dartindex d6a69b277..77cc367eb 100644--- a/mobile/lib/phone/terminal_page.dart+++ b/mobile/lib/phone/terminal_page.dart@@ -1,11 +1,17 @@import 'dart:async';import 'package:flutter/material.dart';+// `PlatformException` — a refused camera permission arrives as one, and it is+// the one picker failure with something the person can do about it.+import 'package:flutter/services.dart';+import 'package:image_picker/image_picker.dart';import 'package:lucide_icons_flutter/lucide_icons.dart';import 'package:harness_mobile/shared/theme/app_theme.dart';import 'package:harness_mobile/shared/widgets/app_icon_button.dart';import 'package:harness_mobile/state/app_state.dart';+import 'package:harness_mobile/terminal/image_transcode.dart';+import 'package:harness_mobile/terminal/terminal_session.dart';import 'package:harness_mobile/widgets/engine_identity.dart';import 'package:harness_mobile/widgets/rename_agent_dialog.dart';import 'package:harness_mobile/widgets/terminal_panel.dart';@@ -144,6 +150,68 @@ class _TerminalPageState extends State<TerminalPage>/// xterm reopens it on the next tap in the pane.void _dismissKeyboard() => FocusManager.instance.primaryFocus?.unfocus();+ /// Guards against a second picker while one is already up.+ ///+ /// The key bar stays on screen under the sheet the OS puts over it, so its+ /// button remains tappable — and `pickImage` answers a second call on iOS by+ /// throwing rather than by queueing.+ bool _picking = false;++ /// Picks a picture and sends it to the agent, re-encoded on the way.+ ///+ /// ⚠️ **The transcode is not an optimisation, it is what makes the picture+ /// arrive at all** — see `transcodeToPng`. Everything past this point names+ /// PNG: the binary kind, the file the CLI writes, and the three OS clipboard+ /// writers it hands the bytes to. A phone produces JPEG and HEIC.+ Future<void> _sendImage(TerminalSession session, ImageSource source) async {+ if (_picking || !session.acceptsInput) return;+ _picking = true;+ final messenger = ScaffoldMessenger.maybeOf(context);+ void report(String message) {+ if (mounted) messenger?.showSnackBar(SnackBar(content: Text(message)));+ }++ try {+ final XFile? picked;+ try {+ picked = await ImagePicker().pickImage(source: source);+ } on PlatformException catch (error) {+ // A refused camera permission lands here rather than as a null, and it+ // is the one failure somebody can do something about.+ report(+ error.code == 'camera_access_denied'+ ? 'Allow camera access in Settings to send a photo.'+ : 'Could not open the picker.',+ );+ return;+ }+ // Null is a CANCEL, not a failure: the person backed out of the sheet, and+ // a snackbar saying so would be noise over a deliberate act.+ if (picked == null) return;++ final result = await transcodeToPng(await picked.readAsBytes());+ switch (result) {+ case ImageTranscodeUnreadable():+ report("That file isn't an image this phone can read.");+ case ImageTranscodeTooLarge():+ report('That image is too large to send, even scaled down.');+ case ImageTranscodeOk(:final pngBytes):+ // Re-checked AFTER the picker, which the person may have had open for+ // a while: the stream can have been taken over or dropped since, and+ // `pasteImage` on a dead stream goes nowhere silently.+ if (!session.acceptsInput) {+ report('The terminal is no longer accepting input.');+ return;+ }+ if (!await session.pasteImage(pngBytes)) {+ report('The image could not be sent.');+ }+ }+ } finally {+ _picking = false;+ }+ }+/// The composer starts OPEN here, and the phone owns that answer rather than the pane.///@override@@ -293,6 +361,22 @@ class _TerminalPageState extends State<TerminalPage>controlArmed: session.controlArmed,onControlToggle: session.armControl,onDismissKeyboard: _dismissKeyboard,+ // Only where the far side can actually take one: an+ // older CLI never advertises the binary kind, so the+ // upload would go nowhere silently. Null leaves the+ // buttons undrawn rather than drawn dead.+ onPickImage:+ machine?.terminalImagePasteAvailable == true+ ? () => unawaited(+ _sendImage(session, ImageSource.gallery),+ )+ : null,+ onTakePhoto:+ machine?.terminalImagePasteAvailable == true+ ? () => unawaited(+ _sendImage(session, ImageSource.camera),+ )+ : null,),),],diff --git a/mobile/lib/terminal/image_transcode.dart b/mobile/lib/terminal/image_transcode.dartnew file mode 100644index 000000000..3b71cef4a--- /dev/null+++ b/mobile/lib/terminal/image_transcode.dart@@ -0,0 +1,160 @@+import 'dart:typed_data';+import 'dart:ui' as ui;++import 'terminal_binary.dart';++/// Turns a picture the phone picked into the PNG the terminal pipe accepts.+///+/// ⚠️ **PNG is not a preference here, it is the only format that survives the trip.** The bytes go+/// out as [TerminalBinaryKind.imagePaste], and on the far side the CLI hands them to the host's own+/// clipboard — `osascript … as «class PNGf»` on macOS, `wl-copy --type image/png` on Wayland,+/// `xclip -t image/png` on X11. All three name PNG outright, and the file the CLI writes is named+/// `<uuid>.png` whatever is inside it. A JPEG sent as-is therefore does not arrive as a smaller+/// picture — it fails the clipboard write, falls back to pasting a FILE PATH, and the agent is+/// handed a `.png` that is really a JPEG. The phone is the only place that can fix this, so it does.+///+/// A phone camera also shoots far larger than an agent needs to read: 4000px wide at 12MP, against a+/// terminal that will show it at a fraction of that. Downscaling first is what keeps the re-encode+/// from blowing through [terminalLocalImagePasteMaxPayloadBytes] — PNG is lossless, so a picture+/// that was 2MB as JPEG can be 20MB as PNG at full size.++/// The longest edge a picture is reduced to before it is encoded.+///+/// 1600px is a compromise between two real failures: smaller starts losing the text in a screenshot,+/// which is most of what anybody sends an agent; larger pushes a photographic image past the payload+/// ceiling even after the retries below.+const _preferredMaxEdge = 1600;++/// The edges tried, in order, if the first encode comes out too large.+///+/// Photographic noise is what makes a PNG big — a beach photo does not compress the way a screenshot+/// does — and halving the edge quarters the pixel count, so this converges fast. The last step is+/// small enough to be legible but is deliberately not tiny: failing is better than silently sending+/// something nobody can read.+const _fallbackMaxEdges = [1200, 800, 600];++/// The result of preparing a picture: the PNG to send, or why there is nothing to send.+sealed class ImageTranscodeResult {+ const ImageTranscodeResult();+}++/// Ready to go out over [TerminalSession.pasteImage].+class ImageTranscodeOk extends ImageTranscodeResult {+ const ImageTranscodeOk(this.pngBytes);++ final Uint8List pngBytes;+}++/// The bytes could not be read as a picture at all.+///+/// Not the same as [ImageTranscodeTooLarge], and they read differently to somebody holding a phone:+/// this one means "that file is not an image", which no amount of retrying fixes.+class ImageTranscodeUnreadable extends ImageTranscodeResult {+ const ImageTranscodeUnreadable();+}++/// A picture that is still over the ceiling at the smallest size this will try.+class ImageTranscodeTooLarge extends ImageTranscodeResult {+ const ImageTranscodeTooLarge(this.bytes);++ /// What the smallest attempt weighed, for a message that can say how far over it is.+ final int bytes;+}++/// Decodes [bytes] with the platform's own codec, downscales it, and re-encodes it as PNG.+///+/// The platform codec rather than a Dart decoder on purpose: it is the one that already knows HEIC,+/// which is what an iPhone camera produces by default, and it decodes on its own threads instead of+/// stalling the frame that is drawing the key bar. `targetWidth`/`targetHeight` are applied DURING+/// the decode, so a 12MP photo is never fully rasterised at native size just to be thrown away.+Future<ImageTranscodeResult> transcodeToPng(Uint8List bytes) async {+ if (bytes.isEmpty) return const ImageTranscodeUnreadable();++ final ui.ImageDescriptor descriptor;+ final ui.ImmutableBuffer buffer;+ try {+ buffer = await ui.ImmutableBuffer.fromUint8List(bytes);+ } catch (_) {+ return const ImageTranscodeUnreadable();+ }+ try {+ descriptor = await ui.ImageDescriptor.encoded(buffer);+ } catch (_) {+ // `fromUint8List` takes anything; this is where a PDF, a HEIC the OS cannot read, or a+ // truncated download actually fails.+ buffer.dispose();+ return const ImageTranscodeUnreadable();+ }++ final width = descriptor.width;+ final height = descriptor.height;+ if (width <= 0 || height <= 0) {+ descriptor.dispose();+ return const ImageTranscodeUnreadable();+ }++ // Every attempt re-decodes from the same descriptor rather than re-scaling an already-scaled+ // bitmap: scaling twice softens edges that were sharp, and in a screenshot those edges are text.+ var smallest = -1;+ try {+ for (final maxEdge in [_preferredMaxEdge, ..._fallbackMaxEdges]) {+ final png = await _encodePngAtMaxEdge(+ descriptor: descriptor,+ width: width,+ height: height,+ maxEdge: maxEdge,+ );+ if (png == null) return const ImageTranscodeUnreadable();+ if (png.lengthInBytes <= terminalLocalImagePasteMaxPayloadBytes) {+ return ImageTranscodeOk(png);+ }+ smallest = png.lengthInBytes;+ }+ } finally {+ descriptor.dispose();+ }+ return ImageTranscodeTooLarge(smallest);+}++/// One attempt: decode at a size whose longest edge is at most [maxEdge], then encode PNG.+///+/// Returns null if the decode itself failed, which is not the same as the result being too big.+Future<Uint8List?> _encodePngAtMaxEdge({+ required ui.ImageDescriptor descriptor,+ required int width,+ required int height,+ required int maxEdge,+}) async {+ final longest = width > height ? width : height;+ // A picture already smaller than the target is left at its own size — upscaling would add bytes+ // and no detail. The ratio is applied to BOTH edges so the aspect is kept; passing only one lets+ // the engine pick the other, and it rounds independently.+ final scale = longest <= maxEdge ? 1.0 : maxEdge / longest;+ final targetWidth = scale == 1.0 ? width : (width * scale).round().clamp(1, width);+ final targetHeight = scale == 1.0 ? height : (height * scale).round().clamp(1, height);++ ui.Codec? codec;+ ui.FrameInfo? frame;+ try {+ codec = await descriptor.instantiateCodec(+ targetWidth: targetWidth,+ targetHeight: targetHeight,+ );+ frame = await codec.getNextFrame();+ final data = await frame.image.toByteData(format: ui.ImageByteFormat.png);+ if (data == null) return null;+ // `.buffer.asUint8List()` would hand back the whole backing store, which for a view into a+ // larger buffer is more bytes than the image — and those bytes are what gets uploaded.+ return data.buffer.asUint8List(+ data.offsetInBytes,+ data.lengthInBytes,+ );+ } catch (_) {+ return null;+ } finally {+ // The frame's image holds GPU/native memory that the Dart GC does not account for, so a picture+ // per attempt adds up on a phone that is already holding a terminal's scrollback.+ frame?.image.dispose();+ codec?.dispose();+ }+}diff --git a/mobile/pubspec.lock b/mobile/pubspec.lockindex 923436ae1..04143c677 100644--- a/mobile/pubspec.lock+++ b/mobile/pubspec.lock@@ -65,6 +65,14 @@ packages:url: "https://pub.dev"source: hostedversion: "3.1.2"+ cross_file:+ dependency: transitive+ description:+ name: cross_file+ sha256: f141ea4f277af142a0356955707f6556f37b03947d39d55585981a06ca437bd6+ url: "https://pub.dev"+ source: hosted+ version: "0.3.5+5"crypto:dependency: transitivedescription:@@ -145,6 +153,38 @@ packages:url: "https://pub.dev"source: hostedversion: "0.1.2"+ file_selector_linux:+ dependency: transitive+ description:+ name: file_selector_linux+ sha256: da76400e7872ce7637ffdce12749ec24169c25f6195c28372208e65a24bcd2ab+ url: "https://pub.dev"+ source: hosted+ version: "0.9.4+1"+ file_selector_macos:+ dependency: transitive+ description:+ name: file_selector_macos+ sha256: d57c62362766b5e7ae739448650b66c6aab7a68ba7ecc65e04018652645ae0f4+ url: "https://pub.dev"+ source: hosted+ version: "0.9.5+1"+ file_selector_platform_interface:+ dependency: transitive+ description:+ name: file_selector_platform_interface+ sha256: "35e0bd61ebcdb91a3505813b055b09b79dfdc7d0aee9c09a7ba59ae4bb13dc85"+ url: "https://pub.dev"+ source: hosted+ version: "2.7.0"+ file_selector_windows:+ dependency: transitive+ description:+ name: file_selector_windows+ sha256: fbefc5fb92c6d3cbe8d284a2cd971b593bb07d2cd6da8557b81a862250b4acec+ url: "https://pub.dev"+ source: hosted+ version: "0.9.3+6"flutter:dependency: "direct main"description: flutter@@ -158,6 +198,14 @@ packages:url: "https://pub.dev"source: hostedversion: "6.0.0"+ flutter_plugin_android_lifecycle:+ dependency: transitive+ description:+ name: flutter_plugin_android_lifecycle+ sha256: "3854fe5e3bff0b113c658f260b90c95dea17c92db0f2addeac2e343dd9969785"+ url: "https://pub.dev"+ source: hosted+ version: "2.0.35"flutter_riverpod:dependency: "direct main"description:@@ -208,6 +256,70 @@ packages:url: "https://pub.dev"source: hostedversion: "4.1.2"+ image_picker:+ dependency: "direct main"+ description:+ name: image_picker+ sha256: d8402284df184bc05f4a2210c6c23983b0720f4cd87cbd05c5390a78af602667+ url: "https://pub.dev"+ source: hosted+ version: "1.2.3"+ image_picker_android:+ dependency: transitive+ description:+ name: image_picker_android+ sha256: "0a55d645a670d6ae11efa948f955ae904afa472f1344fc15a7ddacc20c1e7219"+ url: "https://pub.dev"+ source: hosted+ version: "0.8.13+23"+ image_picker_for_web:+ dependency: transitive+ description:+ name: image_picker_for_web+ sha256: "66257a3191ab360d23a55c8241c91a6e329d31e94efa7be9cf7a212e65850214"+ url: "https://pub.dev"+ source: hosted+ version: "3.1.1"+ image_picker_ios:+ dependency: transitive+ description:+ name: image_picker_ios+ sha256: ee3885b6fcd71958fbc79770dd194c63371439d536d69c47b279171a486482ae+ url: "https://pub.dev"+ source: hosted+ version: "0.8.13+7"+ image_picker_linux:+ dependency: transitive+ description:+ name: image_picker_linux+ sha256: "1f81c5f2046b9ab724f85523e4af65be1d47b038160a8c8deed909762c308ed4"+ url: "https://pub.dev"+ source: hosted+ version: "0.2.2"+ image_picker_macos:+ dependency: transitive+ description:+ name: image_picker_macos+ sha256: "86f0f15a309de7e1a552c12df9ce5b59fe927e71385329355aec4776c6a8ec91"+ url: "https://pub.dev"+ source: hosted+ version: "0.2.2+1"+ image_picker_platform_interface:+ dependency: transitive+ description:+ name: image_picker_platform_interface+ sha256: "567e056716333a1647c64bb6bd873cff7622233a5c3f694be28a583d4715690c"+ url: "https://pub.dev"+ source: hosted+ version: "2.11.1"+ image_picker_windows:+ dependency: transitive+ description:+ name: image_picker_windows+ sha256: d248c86554a72b5495a31c56f060cf73a41c7ff541689327b1a7dbccc33adfae+ url: "https://pub.dev"+ source: hosted+ version: "0.2.2"jni:dependency: transitivedescription:diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yamlindex b96312957..05030ce0d 100644--- a/mobile/pubspec.yaml+++ b/mobile/pubspec.yaml@@ -47,11 +47,18 @@ dependencies:# The phone's second wire to a machine: the `terminal-v1` data channel the# harness CLI opens with werift on the desktop's behalf (lib/p2p/). Data- # channels only — no camera or microphone, so no usage strings are needed.+ # channels only — no microphone, and the camera it never opens is opened by+ # `image_picker` below instead, which is what the usage strings are for.# Pinned exactly: a native plugin, and libwebrtc's SCTP/SDP behaviour is# what the responder's werift has to agree with.flutter_webrtc: 1.6.2+hotfix.1+ # Sending a picture to an agent: the library sheet and the camera behind the+ # key bar's image button (`phone/terminal_key_bar.dart`). The bytes it returns+ # are re-encoded to PNG before they leave — see `terminal/image_transcode.dart`+ # for why the whole pipe past this point accepts nothing else.+ image_picker: ^1.2.3+dev_dependencies:flutter_test:sdk: flutter