Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat(mobile): send a picture to an agent from the key bar

tony504e12d732f0
diff --git a/mobile/android/app/src/main/AndroidManifest.xml b/mobile/android/app/src/main/AndroidManifest.xml
index 1c812424a..a9f0ae0b3 100644
--- a/mobile/android/app/src/main/AndroidManifest.xml
+++ b/mobile/android/app/src/main/AndroidManifest.xml
@@ -9,6 +9,15 @@
<!-- Lets libwebrtc notice a wifi↔cellular change and restart ICE rather than
wait out the disconnect grace. Not a runtime permission. -->
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+ <!-- Taking a picture to send to an agent. Picking an EXISTING picture needs
+ no permission at all: image_picker goes through ACTION_GET_CONTENT / the
+ photo picker, which hands back one item the user chose rather than
+ opening the library — hence no READ_MEDIA_IMAGES here. -->
+ <uses-permission android:name="android.permission.CAMERA" />
+ <!-- The camera is how one feature sends a picture, not something the app
+ cannot run without: a tablet or emulator with no camera must still
+ install and show every other screen. -->
+ <uses-feature android:name="android.hardware.camera" android:required="false" />
<application
android:label="harness"
android:name="${applicationName}"
diff --git a/mobile/ios/Runner/Info.plist b/mobile/ios/Runner/Info.plist
index 9eaf95cc6..60412a7c6 100644
--- a/mobile/ios/Runner/Info.plist
+++ b/mobile/ios/Runner/Info.plist
@@ -26,6 +26,12 @@
<string>$(FLUTTER_BUILD_NUMBER)</string>
<key>LSRequiresIPhoneOS</key>
<true/>
+ <!-- Taking a picture to send to an agent. Only the camera needs a string on
+ iOS 15: picking from the library goes through PHPicker, which runs out of
+ process and grants nothing, so NSPhotoLibraryUsageDescription would never
+ be shown and is deliberately absent. -->
+ <key>NSCameraUsageDescription</key>
+ <string>Take a photo to send to an agent in a terminal.</string>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
diff --git a/mobile/lib/phone/terminal_key_bar.dart b/mobile/lib/phone/terminal_key_bar.dart
index 95be7cf36..4dd694fa6 100644
--- a/mobile/lib/phone/terminal_key_bar.dart
+++ b/mobile/lib/phone/terminal_key_bar.dart
@@ -4,6 +4,7 @@ import 'package:lucide_icons_flutter/lucide_icons.dart';
import 'package:xterm/xterm.dart';
import 'package:harness_mobile/shared/theme/app_theme.dart';
+import 'phone_sheet.dart';
/// The keys a phone keyboard does not have, in a strip above the one it does.
///
@@ -15,7 +16,16 @@ import 'package:harness_mobile/shared/theme/app_theme.dart';
/// It appears with the keyboard and goes away with it: the terminal is short
/// enough on a phone that two rows of chrome are worth their height only while
/// someone is actually typing.
-class TerminalKeyBar extends StatelessWidget {
+///
+/// ⚠️ **One row that SCROLLS, with the rest behind `»`.** The keys outgrew a
+/// fixed grid the moment `enter` and `⇧tab` joined them: eleven across a narrow
+/// phone leaves each one under 30px, below the 44px Apple and Android both put
+/// as the floor for a touch target, and these are keys people hit repeatedly
+/// while looking at the terminal rather than at their thumb. So the row keeps
+/// full-size keys and runs off the edge instead, ordered by how often a key is
+/// actually reached for, and `»` opens the remainder — digits included — as a
+/// second row rather than shrinking the first.
+class TerminalKeyBar extends StatefulWidget {
const TerminalKeyBar({
super.key,
required this.terminal,
@@ -23,6 +33,8 @@ class TerminalKeyBar extends StatelessWidget {
required this.controlArmed,
required this.onControlToggle,
required this.onDismissKeyboard,
+ this.onPickImage,
+ this.onTakePhoto,
});
final Terminal terminal;
@@ -38,57 +50,128 @@ class TerminalKeyBar extends StatelessWidget {
final ValueChanged<bool> onControlToggle;
final VoidCallback onDismissKeyboard;
+ /// Sending a picture. Null on a pane that cannot take one — an older CLI that
+ /// never advertised `terminalImagePasteAvailable` — and the buttons are then
+ /// not drawn at all rather than drawn dead: a key that does nothing is worse
+ /// than one that was never offered.
+ final VoidCallback? onPickImage;
+ final VoidCallback? onTakePhoto;
+
+ @override
+ State<TerminalKeyBar> createState() => _TerminalKeyBarState();
+}
+
+class _TerminalKeyBarState extends State<TerminalKeyBar> {
+ /// Whether `»` has opened the second row.
+ ///
+ /// Held here rather than by the page: it is a property of this strip, it does
+ /// not survive the keyboard going away, and nothing above needs to know.
+ bool _expanded = false;
+
void _send(void Function() action) {
- if (!enabled) return;
+ if (!widget.enabled) return;
HapticFeedback.selectionClick();
action();
}
+ Terminal get _terminal => widget.terminal;
+
+ /// Whether this pane can take a picture at all — see [TerminalKeyBar.onPickImage].
+ bool get _canSendImage =>
+ widget.onPickImage != null || widget.onTakePhoto != null;
+
+ /// The pinned image button: asks WHICH picture, then hands off.
+ ///
+ /// The sheet only appears where there is a choice to make. A build with just
+ /// one of the two wired goes straight there instead — a sheet with a single
+ /// row is a tap spent on nothing.
+ void _sendImage() {
+ final pick = widget.onPickImage;
+ final photo = widget.onTakePhoto;
+ if (pick == null) {
+ photo?.call();
+ return;
+ }
+ if (photo == null) {
+ pick();
+ return;
+ }
+ showPhoneSheet(
+ context,
+ title: 'Send a picture to this agent',
+ actions: [
+ PhoneSheetAction(
+ icon: LucideIcons.image300,
+ label: 'Choose from library',
+ onTap: pick,
+ ),
+ PhoneSheetAction(
+ icon: LucideIcons.camera300,
+ label: 'Take a photo',
+ onTap: photo,
+ ),
+ ],
+ );
+ }
+
@override
Widget build(BuildContext context) {
AppTheme.watch(context);
- final top = <Widget>[
- _key(label: 'esc', onTap: () => terminal.keyInput(TerminalKey.escape)),
- _key(label: 'tab', onTap: () => terminal.keyInput(TerminalKey.tab)),
- for (final char in const ['~', '|', '/', '-'])
- _key(label: char, onTap: () => terminal.textInput(char)),
+ // Ordered by reach, not by keyboard layout: `esc` and `tab` drive an engine's
+ // modes, `↵` ends a message, `⇧tab` cycles Claude Code's, and the arrows walk
+ // history.
+ //
+ // ⚠️ Sending a picture is NOT in here, and that is the whole point of the
+ // pinned button below. A scrolling row only shows four keys on a 393pt phone,
+ // so anything past `⇧tab` is behind a swipe — measured, not guessed. The
+ // image button was tenth in this list and therefore invisible on arrival,
+ // which for the one action people come to this bar for is the same as absent.
+ final primary = <Widget>[
+ _key(label: 'esc', onTap: () => _terminal.keyInput(TerminalKey.escape)),
+ _key(label: 'tab', onTap: () => _terminal.keyInput(TerminalKey.tab)),
_key(
- icon: LucideIcons.arrowLeft300,
- semanticLabel: 'Left',
- onTap: () => terminal.keyInput(TerminalKey.arrowLeft),
+ icon: LucideIcons.cornerDownLeft300,
+ semanticLabel: 'Enter',
+ onTap: () => _terminal.keyInput(TerminalKey.enter),
+ ),
+ // Shift+Tab is CSI Z, and xterm builds it from the modifier rather than
+ // from a key of its own — which is why this passes `shift` instead of
+ // looking for a `TerminalKey.shiftTab` that does not exist.
+ _key(
+ label: '⇧tab',
+ onTap: () => _terminal.keyInput(TerminalKey.tab, shift: true),
),
_key(
icon: LucideIcons.arrowUp300,
semanticLabel: 'Up',
- onTap: () => terminal.keyInput(TerminalKey.arrowUp),
+ onTap: () => _terminal.keyInput(TerminalKey.arrowUp),
),
_key(
icon: LucideIcons.arrowDown300,
semanticLabel: 'Down',
- onTap: () => terminal.keyInput(TerminalKey.arrowDown),
+ onTap: () => _terminal.keyInput(TerminalKey.arrowDown),
),
_key(
- icon: LucideIcons.arrowRight300,
- semanticLabel: 'Right',
- onTap: () => terminal.keyInput(TerminalKey.arrowRight),
+ icon: LucideIcons.arrowLeft300,
+ semanticLabel: 'Left',
+ onTap: () => _terminal.keyInput(TerminalKey.arrowLeft),
),
- // Not a key the pty ever hears about: it puts the keyboard away, which on
- // a phone is the only way to read a full screen of output.
_key(
- icon: LucideIcons.chevronDown300,
- semanticLabel: 'Hide keyboard',
- alwaysEnabled: true,
- onTap: onDismissKeyboard,
+ icon: LucideIcons.arrowRight300,
+ semanticLabel: 'Right',
+ onTap: () => _terminal.keyInput(TerminalKey.arrowRight),
),
- ];
- final bottom = <Widget>[
_key(
label: 'ctrl',
- held: controlArmed,
- onTap: () => onControlToggle(!controlArmed),
+ held: widget.controlArmed,
+ onTap: () => widget.onControlToggle(!widget.controlArmed),
),
+ ];
+ final secondary = <Widget>[
+ for (final char in const ['~', '|', '/', '-', '_', ':', '.'])
+ _key(label: char, onTap: () => _terminal.textInput(char)),
for (final digit in const ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'])
- _key(label: digit, onTap: () => terminal.textInput(digit)),
+ _key(label: digit, onTap: () => _terminal.textInput(digit)),
];
return ExcludeFocus(
@@ -104,11 +187,57 @@ class TerminalKeyBar extends StatelessWidget {
child: Padding(
padding: const EdgeInsets.all(6),
// Held out of the app-wide text scale like the composer's own type:
- // at a large scale eleven keys across a phone stop fitting the row.
+ // the keys are sized for a thumb, and a large accessibility scale
+ // would grow the labels past the boxes holding them.
child: MediaQuery.withNoTextScaling(
child: Column(
mainAxisSize: MainAxisSize.min,
- children: [_row(top), const SizedBox(height: 6), _row(bottom)],
+ children: [
+ Row(
+ children: [
+ Expanded(child: _scrollingRow(primary)),
+ const SizedBox(width: 4),
+ // Pinned for the same reason `»` is, and more so: this is
+ // what somebody opens the bar to reach. One button rather
+ // than two — the library and the camera are two answers to
+ // "which picture", which is a question a sheet asks better
+ // than a second key nobody can fit on the row.
+ if (_canSendImage) ...[
+ _key(
+ icon: LucideIcons.image300,
+ semanticLabel: 'Send image',
+ onTap: _sendImage,
+ ),
+ const SizedBox(width: 4),
+ ],
+ // Pinned OUTSIDE the scroller, so the way to the rest of the
+ // keys cannot itself be scrolled off the edge.
+ _key(
+ icon: _expanded
+ ? LucideIcons.chevronsLeft300
+ : LucideIcons.chevronsRight300,
+ semanticLabel: _expanded ? 'Fewer keys' : 'More keys',
+ held: _expanded,
+ alwaysEnabled: true,
+ onTap: () => setState(() => _expanded = !_expanded),
+ ),
+ const SizedBox(width: 4),
+ // Not a key the pty ever hears about: it puts the keyboard
+ // away, which on a phone is the only way to read a full
+ // screen of output.
+ _key(
+ icon: LucideIcons.chevronDown300,
+ semanticLabel: 'Hide keyboard',
+ alwaysEnabled: true,
+ onTap: widget.onDismissKeyboard,
+ ),
+ ],
+ ),
+ if (_expanded) ...[
+ const SizedBox(height: 6),
+ _scrollingRow(secondary),
+ ],
+ ],
),
),
),
@@ -116,15 +245,31 @@ class TerminalKeyBar extends StatelessWidget {
);
}
- Widget _row(List<Widget> keys) => Row(
- children: [
- for (var index = 0; index < keys.length; index++) ...[
- if (index > 0) const SizedBox(width: 4),
- Expanded(child: keys[index]),
- ],
- ],
+ /// One row of keys at their natural width, scrolling horizontally past the edge.
+ ///
+ /// `ClampingScrollPhysics` rather than the iOS default: a bouncing row above the
+ /// keyboard reads as the whole strip coming loose, and the overscroll glow is
+ /// what says "there is more this way" on a row with no scrollbar.
+ Widget _scrollingRow(List<Widget> keys) => SizedBox(
+ height: _keyHeight,
+ child: ListView.separated(
+ scrollDirection: Axis.horizontal,
+ physics: const ClampingScrollPhysics(),
+ padding: EdgeInsets.zero,
+ itemCount: keys.length,
+ separatorBuilder: (_, _) => const SizedBox(width: 4),
+ itemBuilder: (_, index) => keys[index],
+ ),
);
+ /// Tall enough to hit without looking — the floor both platforms put on a
+ /// touch target, which the old two-row grid was under at 34.
+ static const double _keyHeight = 40;
+
+ /// Wide enough for `⇧tab` at 13px, and the same width for every key so the row
+ /// reads as a keyboard rather than as a sentence of buttons.
+ static const double _keyWidth = 52;
+
Widget _key({
String? label,
IconData? icon,
@@ -134,7 +279,7 @@ class TerminalKeyBar extends StatelessWidget {
required VoidCallback onTap,
}) {
assert((label == null) != (icon == null), 'a key carries one of the two');
- final live = enabled || alwaysEnabled;
+ final live = widget.enabled || alwaysEnabled;
final foreground = held
? AppPalette.accentOnSurface
: live
@@ -151,7 +296,8 @@ class TerminalKeyBar extends StatelessWidget {
child: AnimatedContainer(
duration: const Duration(milliseconds: 120),
curve: Curves.easeOut,
- height: 34,
+ width: _keyWidth,
+ height: _keyHeight,
alignment: Alignment.center,
decoration: BoxDecoration(
color: held ? AppSurface.accentWash : AppGlass.surfaceFill,
@@ -161,7 +307,7 @@ class TerminalKeyBar extends StatelessWidget {
),
),
child: icon != null
- ? Icon(icon, size: 16, color: foreground)
+ ? Icon(icon, size: 17, color: foreground)
: Text(
label!,
maxLines: 1,
diff --git a/mobile/lib/phone/terminal_page.dart b/mobile/lib/phone/terminal_page.dart
index d6a69b277..77cc367eb 100644
--- a/mobile/lib/phone/terminal_page.dart
+++ b/mobile/lib/phone/terminal_page.dart
@@ -1,11 +1,17 @@
import 'dart:async';
import 'package:flutter/material.dart';
+// `PlatformException` — a refused camera permission arrives as one, and it is
+// the one picker failure with something the person can do about it.
+import 'package:flutter/services.dart';
+import 'package:image_picker/image_picker.dart';
import 'package:lucide_icons_flutter/lucide_icons.dart';
import 'package:harness_mobile/shared/theme/app_theme.dart';
import 'package:harness_mobile/shared/widgets/app_icon_button.dart';
import 'package:harness_mobile/state/app_state.dart';
+import 'package:harness_mobile/terminal/image_transcode.dart';
+import 'package:harness_mobile/terminal/terminal_session.dart';
import 'package:harness_mobile/widgets/engine_identity.dart';
import 'package:harness_mobile/widgets/rename_agent_dialog.dart';
import 'package:harness_mobile/widgets/terminal_panel.dart';
@@ -144,6 +150,68 @@ class _TerminalPageState extends State<TerminalPage>
/// xterm reopens it on the next tap in the pane.
void _dismissKeyboard() => FocusManager.instance.primaryFocus?.unfocus();
+ /// Guards against a second picker while one is already up.
+ ///
+ /// The key bar stays on screen under the sheet the OS puts over it, so its
+ /// button remains tappable — and `pickImage` answers a second call on iOS by
+ /// throwing rather than by queueing.
+ bool _picking = false;
+
+ /// Picks a picture and sends it to the agent, re-encoded on the way.
+ ///
+ /// ⚠️ **The transcode is not an optimisation, it is what makes the picture
+ /// arrive at all** — see `transcodeToPng`. Everything past this point names
+ /// PNG: the binary kind, the file the CLI writes, and the three OS clipboard
+ /// writers it hands the bytes to. A phone produces JPEG and HEIC.
+ Future<void> _sendImage(TerminalSession session, ImageSource source) async {
+ if (_picking || !session.acceptsInput) return;
+ _picking = true;
+ final messenger = ScaffoldMessenger.maybeOf(context);
+ void report(String message) {
+ if (mounted) messenger?.showSnackBar(SnackBar(content: Text(message)));
+ }
+
+ try {
+ final XFile? picked;
+ try {
+ picked = await ImagePicker().pickImage(source: source);
+ } on PlatformException catch (error) {
+ // A refused camera permission lands here rather than as a null, and it
+ // is the one failure somebody can do something about.
+ report(
+ error.code == 'camera_access_denied'
+ ? 'Allow camera access in Settings to send a photo.'
+ : 'Could not open the picker.',
+ );
+ return;
+ }
+ // Null is a CANCEL, not a failure: the person backed out of the sheet, and
+ // a snackbar saying so would be noise over a deliberate act.
+ if (picked == null) return;
+
+ final result = await transcodeToPng(await picked.readAsBytes());
+ switch (result) {
+ case ImageTranscodeUnreadable():
+ report("That file isn't an image this phone can read.");
+ case ImageTranscodeTooLarge():
+ report('That image is too large to send, even scaled down.');
+ case ImageTranscodeOk(:final pngBytes):
+ // Re-checked AFTER the picker, which the person may have had open for
+ // a while: the stream can have been taken over or dropped since, and
+ // `pasteImage` on a dead stream goes nowhere silently.
+ if (!session.acceptsInput) {
+ report('The terminal is no longer accepting input.');
+ return;
+ }
+ if (!await session.pasteImage(pngBytes)) {
+ report('The image could not be sent.');
+ }
+ }
+ } finally {
+ _picking = false;
+ }
+ }
+
/// The composer starts OPEN here, and the phone owns that answer rather than the pane.
///
@override
@@ -293,6 +361,22 @@ class _TerminalPageState extends State<TerminalPage>
controlArmed: session.controlArmed,
onControlToggle: session.armControl,
onDismissKeyboard: _dismissKeyboard,
+ // Only where the far side can actually take one: an
+ // older CLI never advertises the binary kind, so the
+ // upload would go nowhere silently. Null leaves the
+ // buttons undrawn rather than drawn dead.
+ onPickImage:
+ machine?.terminalImagePasteAvailable == true
+ ? () => unawaited(
+ _sendImage(session, ImageSource.gallery),
+ )
+ : null,
+ onTakePhoto:
+ machine?.terminalImagePasteAvailable == true
+ ? () => unawaited(
+ _sendImage(session, ImageSource.camera),
+ )
+ : null,
),
),
],
diff --git a/mobile/lib/terminal/image_transcode.dart b/mobile/lib/terminal/image_transcode.dart
new file mode 100644
index 000000000..3b71cef4a
--- /dev/null
+++ b/mobile/lib/terminal/image_transcode.dart
@@ -0,0 +1,160 @@
+import 'dart:typed_data';
+import 'dart:ui' as ui;
+
+import 'terminal_binary.dart';
+
+/// Turns a picture the phone picked into the PNG the terminal pipe accepts.
+///
+/// ⚠️ **PNG is not a preference here, it is the only format that survives the trip.** The bytes go
+/// out as [TerminalBinaryKind.imagePaste], and on the far side the CLI hands them to the host's own
+/// clipboard — `osascript … as «class PNGf»` on macOS, `wl-copy --type image/png` on Wayland,
+/// `xclip -t image/png` on X11. All three name PNG outright, and the file the CLI writes is named
+/// `<uuid>.png` whatever is inside it. A JPEG sent as-is therefore does not arrive as a smaller
+/// picture — it fails the clipboard write, falls back to pasting a FILE PATH, and the agent is
+/// handed a `.png` that is really a JPEG. The phone is the only place that can fix this, so it does.
+///
+/// A phone camera also shoots far larger than an agent needs to read: 4000px wide at 12MP, against a
+/// terminal that will show it at a fraction of that. Downscaling first is what keeps the re-encode
+/// from blowing through [terminalLocalImagePasteMaxPayloadBytes] — PNG is lossless, so a picture
+/// that was 2MB as JPEG can be 20MB as PNG at full size.
+
+/// The longest edge a picture is reduced to before it is encoded.
+///
+/// 1600px is a compromise between two real failures: smaller starts losing the text in a screenshot,
+/// which is most of what anybody sends an agent; larger pushes a photographic image past the payload
+/// ceiling even after the retries below.
+const _preferredMaxEdge = 1600;
+
+/// The edges tried, in order, if the first encode comes out too large.
+///
+/// Photographic noise is what makes a PNG big — a beach photo does not compress the way a screenshot
+/// does — and halving the edge quarters the pixel count, so this converges fast. The last step is
+/// small enough to be legible but is deliberately not tiny: failing is better than silently sending
+/// something nobody can read.
+const _fallbackMaxEdges = [1200, 800, 600];
+
+/// The result of preparing a picture: the PNG to send, or why there is nothing to send.
+sealed class ImageTranscodeResult {
+ const ImageTranscodeResult();
+}
+
+/// Ready to go out over [TerminalSession.pasteImage].
+class ImageTranscodeOk extends ImageTranscodeResult {
+ const ImageTranscodeOk(this.pngBytes);
+
+ final Uint8List pngBytes;
+}
+
+/// The bytes could not be read as a picture at all.
+///
+/// Not the same as [ImageTranscodeTooLarge], and they read differently to somebody holding a phone:
+/// this one means "that file is not an image", which no amount of retrying fixes.
+class ImageTranscodeUnreadable extends ImageTranscodeResult {
+ const ImageTranscodeUnreadable();
+}
+
+/// A picture that is still over the ceiling at the smallest size this will try.
+class ImageTranscodeTooLarge extends ImageTranscodeResult {
+ const ImageTranscodeTooLarge(this.bytes);
+
+ /// What the smallest attempt weighed, for a message that can say how far over it is.
+ final int bytes;
+}
+
+/// Decodes [bytes] with the platform's own codec, downscales it, and re-encodes it as PNG.
+///
+/// The platform codec rather than a Dart decoder on purpose: it is the one that already knows HEIC,
+/// which is what an iPhone camera produces by default, and it decodes on its own threads instead of
+/// stalling the frame that is drawing the key bar. `targetWidth`/`targetHeight` are applied DURING
+/// the decode, so a 12MP photo is never fully rasterised at native size just to be thrown away.
+Future<ImageTranscodeResult> transcodeToPng(Uint8List bytes) async {
+ if (bytes.isEmpty) return const ImageTranscodeUnreadable();
+
+ final ui.ImageDescriptor descriptor;
+ final ui.ImmutableBuffer buffer;
+ try {
+ buffer = await ui.ImmutableBuffer.fromUint8List(bytes);
+ } catch (_) {
+ return const ImageTranscodeUnreadable();
+ }
+ try {
+ descriptor = await ui.ImageDescriptor.encoded(buffer);
+ } catch (_) {
+ // `fromUint8List` takes anything; this is where a PDF, a HEIC the OS cannot read, or a
+ // truncated download actually fails.
+ buffer.dispose();
+ return const ImageTranscodeUnreadable();
+ }
+
+ final width = descriptor.width;
+ final height = descriptor.height;
+ if (width <= 0 || height <= 0) {
+ descriptor.dispose();
+ return const ImageTranscodeUnreadable();
+ }
+
+ // Every attempt re-decodes from the same descriptor rather than re-scaling an already-scaled
+ // bitmap: scaling twice softens edges that were sharp, and in a screenshot those edges are text.
+ var smallest = -1;
+ try {
+ for (final maxEdge in [_preferredMaxEdge, ..._fallbackMaxEdges]) {
+ final png = await _encodePngAtMaxEdge(
+ descriptor: descriptor,
+ width: width,
+ height: height,
+ maxEdge: maxEdge,
+ );
+ if (png == null) return const ImageTranscodeUnreadable();
+ if (png.lengthInBytes <= terminalLocalImagePasteMaxPayloadBytes) {
+ return ImageTranscodeOk(png);
+ }
+ smallest = png.lengthInBytes;
+ }
+ } finally {
+ descriptor.dispose();
+ }
+ return ImageTranscodeTooLarge(smallest);
+}
+
+/// One attempt: decode at a size whose longest edge is at most [maxEdge], then encode PNG.
+///
+/// Returns null if the decode itself failed, which is not the same as the result being too big.
+Future<Uint8List?> _encodePngAtMaxEdge({
+ required ui.ImageDescriptor descriptor,
+ required int width,
+ required int height,
+ required int maxEdge,
+}) async {
+ final longest = width > height ? width : height;
+ // A picture already smaller than the target is left at its own size — upscaling would add bytes
+ // and no detail. The ratio is applied to BOTH edges so the aspect is kept; passing only one lets
+ // the engine pick the other, and it rounds independently.
+ final scale = longest <= maxEdge ? 1.0 : maxEdge / longest;
+ final targetWidth = scale == 1.0 ? width : (width * scale).round().clamp(1, width);
+ final targetHeight = scale == 1.0 ? height : (height * scale).round().clamp(1, height);
+
+ ui.Codec? codec;
+ ui.FrameInfo? frame;
+ try {
+ codec = await descriptor.instantiateCodec(
+ targetWidth: targetWidth,
+ targetHeight: targetHeight,
+ );
+ frame = await codec.getNextFrame();
+ final data = await frame.image.toByteData(format: ui.ImageByteFormat.png);
+ if (data == null) return null;
+ // `.buffer.asUint8List()` would hand back the whole backing store, which for a view into a
+ // larger buffer is more bytes than the image — and those bytes are what gets uploaded.
+ return data.buffer.asUint8List(
+ data.offsetInBytes,
+ data.lengthInBytes,
+ );
+ } catch (_) {
+ return null;
+ } finally {
+ // The frame's image holds GPU/native memory that the Dart GC does not account for, so a picture
+ // per attempt adds up on a phone that is already holding a terminal's scrollback.
+ frame?.image.dispose();
+ codec?.dispose();
+ }
+}
diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock
index 923436ae1..04143c677 100644
--- a/mobile/pubspec.lock
+++ b/mobile/pubspec.lock
@@ -65,6 +65,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "3.1.2"
+ cross_file:
+ dependency: transitive
+ description:
+ name: cross_file
+ sha256: f141ea4f277af142a0356955707f6556f37b03947d39d55585981a06ca437bd6
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.3.5+5"
crypto:
dependency: transitive
description:
@@ -145,6 +153,38 @@ packages:
url: "https://pub.dev"
source: hosted
version: "0.1.2"
+ file_selector_linux:
+ dependency: transitive
+ description:
+ name: file_selector_linux
+ sha256: da76400e7872ce7637ffdce12749ec24169c25f6195c28372208e65a24bcd2ab
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.9.4+1"
+ file_selector_macos:
+ dependency: transitive
+ description:
+ name: file_selector_macos
+ sha256: d57c62362766b5e7ae739448650b66c6aab7a68ba7ecc65e04018652645ae0f4
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.9.5+1"
+ file_selector_platform_interface:
+ dependency: transitive
+ description:
+ name: file_selector_platform_interface
+ sha256: "35e0bd61ebcdb91a3505813b055b09b79dfdc7d0aee9c09a7ba59ae4bb13dc85"
+ url: "https://pub.dev"
+ source: hosted
+ version: "2.7.0"
+ file_selector_windows:
+ dependency: transitive
+ description:
+ name: file_selector_windows
+ sha256: fbefc5fb92c6d3cbe8d284a2cd971b593bb07d2cd6da8557b81a862250b4acec
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.9.3+6"
flutter:
dependency: "direct main"
description: flutter
@@ -158,6 +198,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "6.0.0"
+ flutter_plugin_android_lifecycle:
+ dependency: transitive
+ description:
+ name: flutter_plugin_android_lifecycle
+ sha256: "3854fe5e3bff0b113c658f260b90c95dea17c92db0f2addeac2e343dd9969785"
+ url: "https://pub.dev"
+ source: hosted
+ version: "2.0.35"
flutter_riverpod:
dependency: "direct main"
description:
@@ -208,6 +256,70 @@ packages:
url: "https://pub.dev"
source: hosted
version: "4.1.2"
+ image_picker:
+ dependency: "direct main"
+ description:
+ name: image_picker
+ sha256: d8402284df184bc05f4a2210c6c23983b0720f4cd87cbd05c5390a78af602667
+ url: "https://pub.dev"
+ source: hosted
+ version: "1.2.3"
+ image_picker_android:
+ dependency: transitive
+ description:
+ name: image_picker_android
+ sha256: "0a55d645a670d6ae11efa948f955ae904afa472f1344fc15a7ddacc20c1e7219"
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.8.13+23"
+ image_picker_for_web:
+ dependency: transitive
+ description:
+ name: image_picker_for_web
+ sha256: "66257a3191ab360d23a55c8241c91a6e329d31e94efa7be9cf7a212e65850214"
+ url: "https://pub.dev"
+ source: hosted
+ version: "3.1.1"
+ image_picker_ios:
+ dependency: transitive
+ description:
+ name: image_picker_ios
+ sha256: ee3885b6fcd71958fbc79770dd194c63371439d536d69c47b279171a486482ae
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.8.13+7"
+ image_picker_linux:
+ dependency: transitive
+ description:
+ name: image_picker_linux
+ sha256: "1f81c5f2046b9ab724f85523e4af65be1d47b038160a8c8deed909762c308ed4"
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.2.2"
+ image_picker_macos:
+ dependency: transitive
+ description:
+ name: image_picker_macos
+ sha256: "86f0f15a309de7e1a552c12df9ce5b59fe927e71385329355aec4776c6a8ec91"
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.2.2+1"
+ image_picker_platform_interface:
+ dependency: transitive
+ description:
+ name: image_picker_platform_interface
+ sha256: "567e056716333a1647c64bb6bd873cff7622233a5c3f694be28a583d4715690c"
+ url: "https://pub.dev"
+ source: hosted
+ version: "2.11.1"
+ image_picker_windows:
+ dependency: transitive
+ description:
+ name: image_picker_windows
+ sha256: d248c86554a72b5495a31c56f060cf73a41c7ff541689327b1a7dbccc33adfae
+ url: "https://pub.dev"
+ source: hosted
+ version: "0.2.2"
jni:
dependency: transitive
description:
diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml
index b96312957..05030ce0d 100644
--- a/mobile/pubspec.yaml
+++ b/mobile/pubspec.yaml
@@ -47,11 +47,18 @@ dependencies:
# The phone's second wire to a machine: the `terminal-v1` data channel the
# harness CLI opens with werift on the desktop's behalf (lib/p2p/). Data
- # channels only — no camera or microphone, so no usage strings are needed.
+ # channels only — no microphone, and the camera it never opens is opened by
+ # `image_picker` below instead, which is what the usage strings are for.
# Pinned exactly: a native plugin, and libwebrtc's SCTP/SDP behaviour is
# what the responder's werift has to agree with.
flutter_webrtc: 1.6.2+hotfix.1
+ # Sending a picture to an agent: the library sheet and the camera behind the
+ # key bar's image button (`phone/terminal_key_bar.dart`). The bytes it returns
+ # are re-encoded to PNG before they leave — see `terminal/image_transcode.dart`
+ # for why the whole pipe past this point accepts nothing else.
+ image_picker: ^1.2.3
+
dev_dependencies:
flutter_test:
sdk: flutter