Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

14.5 KB
Raw Blame
import 'dart:async';
import 'dart:convert';

import 'package:flutter/gestures.dart';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
// The same web/IME input adapter used by our vendored terminal.
// ignore: implementation_imports
import 'package:xterm/src/ui/custom_text_edit.dart';

import '../e2ee/bytes.dart';
import '../shared/theme/app_theme.dart' as grid;
import '../terminal/terminal_text.dart';
import '../widgets/terminal_text_action.dart';
import '../ws/ws_conn.dart';

typedef ViewerSurfaceRequest = Future<Map<String, dynamic>> Function(
  Map<String, dynamic> payload,
);

/// Pulling the next frame only after the previous reply bounds both rendering and network work.
/// Input is connection-local: a failed request is never replayed into a replacement viewer.
class InteractiveViewerSession extends ChangeNotifier {
  InteractiveViewerSession(this.request, {this.onHostAction});
  final ViewerSurfaceRequest request;
  final void Function(Map<String, dynamic>)? onHostAction;
  final String id = hexOf(secureRandomBytes(16));
  Uint8List? image;
  String? error;
  bool Function()? focusInput;
  final _events = <Map<String, dynamic>>[];
  Timer? _timer;
  bool _disposed = false, _busy = false, _reload = false, _paused = false;

  /// Bumped by every pause, so a frame asked for before one is never taken as an answer after it.
  int _pauses = 0;
  int _width = 0, _height = 0;
  bool _dark = true;

  void configure(Size size, bool dark) {
    if (_disposed || size.isEmpty) return;
    _width = size.width.round().clamp(160, 1920);
    _height = size.height.round().clamp(120, 1200);
    _dark = dark;
    if (!_busy && _timer == null && !_paused) _schedule(Duration.zero);
  }

  /// Out of sight: pull nothing, and have the machine close its renderer now rather than after its
  /// 30-second idle timeout. The rendered page runs for as long as its renderer does, and a Harness
  /// Monitor page keeps its viewer reading every machine on the account while it runs (2026-10-28).
  /// Back in sight, the next frame starts a fresh renderer; the last picture stays meanwhile.
  bool get paused => _paused;
  set paused(bool value) {
    if (_disposed || value == _paused) return;
    _paused = value;
    if (value) {
      _pauses++;
      _timer?.cancel();
      _timer = null;
      _events.clear();
      unawaited(
        request({'surfaceId': id, 'op': 'close'})
            .catchError((_) => <String, dynamic>{}),
      );
    } else if (!_busy) {
      _schedule(Duration.zero);
    }
  }

  void input(Map<String, dynamic> event) {
    if (_disposed || _paused || error != null || image == null) return;
    // Mouse motion can be coalesced, but never across a press/release or key event.
    if (event['event'] == 'mouseMoved' &&
        _events.isNotEmpty &&
        _events.last['event'] == 'mouseMoved') {
      _events[_events.length - 1] = event;
    } else if (_events.length < 64) {
      _events.add(event);
    } else {
      _events.clear();
      error = 'The connection is too slow. Reconnect the viewer and try again.';
      _timer?.cancel();
      _timer = null;
      notifyListeners();
      return;
    }
    if (!_busy) _schedule(Duration.zero);
  }

  void reload() {
    if (_disposed) return;
    error = null;
    _events.clear();
    _reload = true;
    notifyListeners();
    if (!_busy && !_paused) _schedule(Duration.zero);
  }

  void _schedule(Duration delay) {
    _timer?.cancel();
    _timer = Timer(delay, () {
      _timer = null;
      unawaited(_frame());
    });
  }

  Future<void> _frame() async {
    if (_disposed || _paused || _busy || _width == 0 || error != null) return;
    _busy = true;
    final pauses = _pauses;
    final events = List<Map<String, dynamic>>.of(_events);
    _events.clear();
    final reload = _reload;
    _reload = false;
    try {
      final reply = await request({
        'surfaceId': id,
        'op': 'frame',
        'width': _width,
        'height': _height,
        'dark': _dark,
        'reload': reload,
        'events': events,
      });
      // Paused while it was asked: the machine closed that renderer, and its answer is no news.
      if (_disposed || pauses != _pauses) return;
      if (reply['error'] != null) {
        error =
            reply['detail'] as String? ??
            'The viewer is unavailable. Try again.';
      } else if (reply['mime'] != 'image/jpeg' ||
          reply['data'] is! String ||
          (reply['data'] as String).length > 2 * 1024 * 1024) {
        error = 'The viewer sent an invalid image.';
      } else {
        image = base64Decode(reply['data'] as String);
        final actions = reply['hostActions'];
        if (actions is List && actions.length <= 8) {
          for (final action in actions) {
            if (action is Map<String, dynamic>) onHostAction?.call(action);
          }
        }
      }
    } on WsRequestTimeout {
      if (!_disposed && pauses == _pauses) {
        error =
            'Update Harness on this machine to use its viewer in the browser.';
      }
    } on WsRequestFailure catch (failure) {
      if (!_disposed && pauses == _pauses) {
        error = failure.code == 'UNSUPPORTED'
            ? 'Update Harness on this machine to use its viewer in the browser.'
            : (failure.detail?.trim().isNotEmpty == true
                  ? failure.detail!
                  : 'The viewer disconnected. Reconnect and try again.');
      }
    } catch (_) {
      if (!_disposed && pauses == _pauses) {
        error = 'The viewer disconnected. Reconnect and try again.';
      }
    } finally {
      _busy = false;
      if (!_disposed) {
        notifyListeners();
        if (error == null && !_paused) {
          _schedule(
            _events.isEmpty && !_reload
                ? const Duration(milliseconds: 160)
                : Duration.zero,
          );
        }
      }
    }
  }

  @override
  void dispose() {
    _disposed = true;
    _timer?.cancel();
    _events.clear();
    unawaited(
      request({'surfaceId': id, 'op': 'close'})
          .catchError((_) => <String, dynamic>{}),
    );
    super.dispose();
  }
}

class RemoteViewerSurface extends StatefulWidget {
  const RemoteViewerSurface({super.key, required this.session});
  final InteractiveViewerSession session;
  @override
  State<RemoteViewerSurface> createState() => _InteractiveViewerState();
}

class _InteractiveViewerState extends State<RemoteViewerSurface> {
  final _focus = FocusNode();
  final _editor = GlobalKey<CustomTextEditState>();
  Size _size = Size.zero;
  int _pressed = 0;
  int _lastPress = 0, _clicks = 0;
  Offset _lastPosition = Offset.zero;

  bool _focusInput() {
    final editor = _editor.currentState;
    if (editor == null) return false;
    editor.requestKeyboard();
    return true;
  }

  @override
  void initState() {
    super.initState();
    widget.session.focusInput = _focusInput;
  }

  @override
  void didUpdateWidget(RemoteViewerSurface oldWidget) {
    super.didUpdateWidget(oldWidget);
    if (oldWidget.session != widget.session) {
      oldWidget.session.focusInput = null;
      widget.session.focusInput = _focusInput;
    }
  }

  int get _modifiers {
    final keys = HardwareKeyboard.instance;
    return (keys.isAltPressed ? 1 : 0) |
        (keys.isControlPressed ? 2 : 0) |
        (keys.isMetaPressed ? 4 : 0) |
        (keys.isShiftPressed ? 8 : 0);
  }

  String _button(int buttons) => buttons & kSecondaryButton != 0
      ? 'right'
      : buttons & kMiddleMouseButton != 0
      ? 'middle'
      : buttons & kPrimaryButton != 0
      ? 'left'
      : 'none';

  void _pointer(PointerEvent event, String type, {int? buttons}) {
    if (_size.isEmpty) return;
    final down = buttons ?? event.buttons;
    widget.session.input({
      'type': 'pointer',
      'event': type,
      'x': (event.localPosition.dx / _size.width).clamp(0.0, 1.0),
      'y': (event.localPosition.dy / _size.height).clamp(0.0, 1.0),
      'buttons': event.buttons & 7,
      'button': _button(down),
      'modifiers': _modifiers,
      'clickCount': type == 'mouseMoved' || type == 'mouseWheel' ? 0 : _clicks,
      if (event is PointerScrollEvent) ...{
        'deltaX': event.scrollDelta.dx.clamp(-2000.0, 2000.0),
        'deltaY': event.scrollDelta.dy.clamp(-2000.0, 2000.0),
      },
    });
  }

  static final _keys = <LogicalKeyboardKey, (String, String, int)>{
    LogicalKeyboardKey.enter: ('Enter', 'Enter', 13),
    LogicalKeyboardKey.numpadEnter: ('Enter', 'NumpadEnter', 13),
    LogicalKeyboardKey.backspace: ('Backspace', 'Backspace', 8),
    LogicalKeyboardKey.delete: ('Delete', 'Delete', 46),
    LogicalKeyboardKey.tab: ('Tab', 'Tab', 9),
    LogicalKeyboardKey.escape: ('Escape', 'Escape', 27),
    LogicalKeyboardKey.arrowUp: ('ArrowUp', 'ArrowUp', 38),
    LogicalKeyboardKey.arrowDown: ('ArrowDown', 'ArrowDown', 40),
    LogicalKeyboardKey.arrowLeft: ('ArrowLeft', 'ArrowLeft', 37),
    LogicalKeyboardKey.arrowRight: ('ArrowRight', 'ArrowRight', 39),
    LogicalKeyboardKey.home: ('Home', 'Home', 36),
    LogicalKeyboardKey.end: ('End', 'End', 35),
    LogicalKeyboardKey.pageUp: ('PageUp', 'PageUp', 33),
    LogicalKeyboardKey.pageDown: ('PageDown', 'PageDown', 34),
  };

  void _key((String, String, int) key, bool down) => widget.session.input({
    'type': 'key',
    'event': down ? 'keyDown' : 'keyUp',
    'key': key.$1,
    'code': key.$2,
    'keyCode': key.$3,
    'modifiers': _modifiers,
  });

  KeyEventResult _onKey(FocusNode _, KeyEvent event) {
    // Alt/Command shortcuts remain workspace navigation, as in a terminal pane.
    if (HardwareKeyboard.instance.isAltPressed ||
        HardwareKeyboard.instance.isMetaPressed) {
      return KeyEventResult.ignored;
    }
    var key = _keys[event.logicalKey];
    if (key == null && HardwareKeyboard.instance.isControlPressed) {
      final label = event.logicalKey.keyLabel;
      if (label.length == 1) {
        key = (
          label.toLowerCase(),
          'Key${label.toUpperCase()}',
          label.toUpperCase().codeUnitAt(0),
        );
      }
    }
    if (key == null) return KeyEventResult.ignored;
    final down = event is! KeyUpEvent;
    _key(key, down);
    if (down) _editor.currentState?.resetEditingState();
    return KeyEventResult.handled;
  }

  @override
  void dispose() {
    widget.session.focusInput = null;
    _focus.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    grid.AppTheme.watch(context);
    return LayoutBuilder(
      builder: (context, constraints) {
        _size = constraints.biggest;
        widget.session.configure(
          _size,
          grid.AppTheme.brightness.value == Brightness.dark,
        );
        return ListenableBuilder(
          listenable: widget.session,
          builder: (context, _) {
            final error = widget.session.error;
            if (error != null) {
              return Center(
                child: Padding(
                  padding: const EdgeInsets.all(24),
                  child: Column(
                    mainAxisSize: MainAxisSize.min,
                    children: [
                      Text(
                        error,
                        style: terminalContentStyle(),
                        textAlign: TextAlign.center,
                      ),
                      const SizedBox(height: 16),
                      TerminalTextAction(
                        label: 'Retry',
                        onPressed: widget.session.reload,
                      ),
                    ],
                  ),
                ),
              );
            }
            final bytes = widget.session.image;
            if (bytes == null) {
              return Center(
                child: Text('Opening viewer…', style: terminalContentStyle()),
              );
            }
            return CustomTextEdit(
              key: _editor,
              focusNode: _focus,
              semanticLabel: 'Viewer input',
              onInsert: (text) =>
                  widget.session.input({'type': 'text', 'text': text}),
              onDelete: (count) {
                for (var i = 0; i < count.clamp(0, 32); i++) {
                  _key(_keys[LogicalKeyboardKey.backspace]!, true);
                  _key(_keys[LogicalKeyboardKey.backspace]!, false);
                }
              },
              onComposing: (_, _) {},
              onKeyEvent: _onKey,
              onAction: (_) {
                _key(_keys[LogicalKeyboardKey.enter]!, true);
                _key(_keys[LogicalKeyboardKey.enter]!, false);
                _editor.currentState?.resetEditingState();
              },
              child: Listener(
                behavior: HitTestBehavior.opaque,
                onPointerDown: (event) {
                  _editor.currentState?.requestKeyboard();
                  final now = DateTime.now().millisecondsSinceEpoch;
                  _clicks =
                      now - _lastPress < 400 &&
                          (event.localPosition - _lastPosition).distance < 5
                      ? 2
                      : 1;
                  _lastPress = now;
                  _lastPosition = event.localPosition;
                  _pressed = event.buttons;
                  _pointer(event, 'mousePressed');
                },
                onPointerUp: (event) {
                  _pointer(event, 'mouseReleased', buttons: _pressed);
                  _pressed = 0;
                },
                onPointerCancel: (event) {
                  _pointer(event, 'mouseReleased', buttons: _pressed);
                  _pressed = 0;
                },
                onPointerMove: (event) => _pointer(event, 'mouseMoved'),
                onPointerHover: (event) => _pointer(event, 'mouseMoved'),
                onPointerSignal: (event) {
                  if (event is PointerScrollEvent) {
                    GestureBinding.instance.pointerSignalResolver.register(
                      event,
                      (event) => _pointer(event, 'mouseWheel'),
                    );
                  }
                },
                child: Image.memory(
                  bytes,
                  // A viewer frame can be smaller than the viewport (the renderer caps its
                  // resolution). Keep painting and pointer coordinates on the same surface,
                  // including while the first image codec is still decoding.
                  width: _size.width,
                  height: _size.height,
                  fit: BoxFit.fill,
                  gaplessPlayback: true,
                  excludeFromSemantics: true,
                ),
              ),
            );
          },
        );
      },
    );
  }
}