Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

22.8 KB
Raw Blame
import 'dart:async';
import 'dart:math';

import 'package:flutter/foundation.dart';

import 'package:harness_mobile/logging/app_log.dart';

import 'voice_language_store.dart';
import 'voice_mic_mode.dart';
import 'voice_notice.dart';
import 'voice_recorder.dart';

/// Turns one WAV recording into the words in it — `ApiClient.transcribeVoice`.
typedef VoiceTranscriber = Future<String> Function(Uint8List wav, String lang);

enum VoiceInputStatus {
  /// Not recording. What was heard so far stays.
  idle,

  /// Waiting on the microphone permission prompt, or the microphone opening.
  starting,

  /// Recording.
  listening,

  /// The recording is with the backend, and its words are not back yet.
  transcribing,

  /// The microphone was refused: the keyboard is the only way in until the
  /// person allows it in Settings.
  unavailable,
}

/// Voice input for one pager of terminal pages: what is being recorded, and what
/// has been heard and not yet sent.
///
/// Record, then transcribe — the dial's shape, and the backend's: its
/// `/api/voice/stt` takes a finished recording and answers with its words, so
/// nothing appears while someone is still talking. Each take's words join the
/// transcript when it ends.
///
/// Owned by `AgentSwipeHost`, not by a page, for the reason the keyboard is one
/// screen-wide fact in `terminal_page.dart`: swiping to the next agent must not
/// drop what was said. What is SENT goes to whichever page's mic is pressed.
class VoiceInputController extends ChangeNotifier {
  VoiceInputController({
    required this.transcriber,
    VoiceRecorder? recorder,
    ValueListenable<String>? language,
  }) : _recorder = recorder ?? MicVoiceRecorder(),
       _language = language ?? voiceLanguageStore;

  /// Below this, a take is silence rather than quiet speech: digital zero, or
  /// the hiss of an input nobody is speaking into. Speech peaks in the
  /// thousands. Caught here, it is a clear notice instead of an upload that
  /// comes back empty and reads as "didn't catch that".
  static const silencePeak = 64;

  /// The longest one take runs before it ends by itself. The backend takes
  /// 25 MB; five minutes of 16 kHz mono is under 10.
  static const maxTake = Duration(minutes: 5);

  /// Under this, a take is a slip of the thumb rather than a sentence.
  ///
  /// ⚠️ **Push-to-talk needs this and tap-to-toggle never did.** Two taps are
  /// hard to do inside a third of a second; a press and release is the easiest
  /// thing in the world, and every accidental brush of the button used to go all
  /// the way to the backend and come back empty — reported as "Didn't catch
  /// that", which blames the speaking rather than the length. Caught here it is
  /// silent: nothing was said, so nothing is announced.
  ///
  /// ⚠️ **Zero in [VoiceMicMode.tapToToggle], and that is the point of reading
  /// the mode rather than a constant.** A tap mode take is bounded by two
  /// deliberate taps, so there is no brush to protect against — while a short
  /// answer that IS the message ("yes", "stop", "ok") runs well under a third of
  /// a second, and this dropped it in silence: nothing sent, nothing said about
  /// it, a mic that read as broken.
  static Duration get minTake =>
      micHoldsToTalk ? const Duration(milliseconds: 350) : Duration.zero;

  /// How long a notice stays on the row before it clears itself.
  ///
  /// These lines report something that has already finished — a take that came
  /// back empty, a send that did not land — and the row they sit in is the one
  /// that otherwise names the machine and what the mic is doing. Left up, a
  /// notice holds that space against the thing it is there for, long after the
  /// person has read it and moved on.
  ///
  /// Five seconds: long enough to read a sentence twice, short enough that the
  /// row is back to itself before the next take is spoken.
  static const noticeLinger = Duration(seconds: 5);

  final VoiceTranscriber transcriber;
  final VoiceRecorder _recorder;

  /// A code from `voiceLanguages`, read when a take is transcribed — so a language picked
  /// mid-take is the one that take is heard in.
  final ValueListenable<String> _language;

  /// What a take that reaches [maxTake] does instead of ending there: null ends it and transcribes
  /// it ([stopListening]), its words kept for the next send — the terminal's way.
  ///
  /// The Swarm composer takes it over: it puts the words in the message box, and while a system
  /// prompt covers the app it holds the take open, since nothing may be sent to be transcribed that
  /// the person cannot see.
  Future<void> Function()? onTakeLimit;

  /// Asked once a take's recording is in hand, right before it goes to [transcriber]: until it
  /// answers the take waits, and false drops it. Null sends every take — the terminal's way.
  ///
  /// The Swarm composer holds a take here while a system prompt covers the app: the microphone
  /// takes a moment to close, a prompt can come up in it, and nothing may be sent to be transcribed
  /// that the person cannot see.
  Future<bool> Function()? mayTranscribe;

  VoiceInputStatus _status = VoiceInputStatus.idle;
  String _heard = '';
  String? _notice;
  bool _isSending = false;
  bool _disposed = false;
  Timer? _takeLimit;

  /// Clears [notice] once [noticeLinger] has passed — see [_restartNoticeTimer].
  Timer? _noticeLimit;

  /// Bumped by everything that abandons a take, so a recording or a
  /// transcription still in flight from it lands nowhere — above all after
  /// [clear], where words arriving late would bring back what was dropped.
  int _take = 0;

  /// The take in progress is a pretend one ([startListening]'s `pretend`), and this is what it will
  /// be heard as.
  String? Function()? _pretending;
  final ValueNotifier<double> _pretendLevel = ValueNotifier(0);
  Timer? _pretendMeter;
  final Random _pretendRandom = Random(7);

  /// Whether the take in progress is a pretend one: listening, with no microphone open.
  bool get pretending => _pretending != null;

  bool get isSending => _isSending;
  VoiceInputStatus get status => _status;
  String get transcript => _heard;
  String? get notice => _notice;

  /// How loud the microphone is right now, 0…1 — what the waveform beside the
  /// mic draws while a take is recorded. Silence from a recorder that has no
  /// meter (see [VoiceLevelMeter]).
  ///
  /// ⚠️ **One object for the controller's life.** A widget adds its listener here as it mounts and
  /// takes it off as it goes, and the take may have changed from recorded to pretend in between:
  /// handing out whichever meter was live left the listener on the other.
  late final ValueListenable<double> level = _TakeLevel(
    recorded: switch (_recorder) {
      final VoiceLevelMeter meter => meter.level,
      _ => _silence,
    },
    pretended: _pretendLevel,
    pretending: () => _pretending != null,
  );

  static final ValueListenable<double> _silence = ValueNotifier(0);

  /// How long the take being recorded has run; zero before the first one.
  ///
  /// ⚠️ Kept here rather than by the widget that shows it: this controller is
  /// shared by every page of the pager, and a page swiped to mid-take has to
  /// show the take's time, not the time since that page appeared.
  Duration get takeLength => _takeClock?.elapsed ?? Duration.zero;

  /// Runs from the moment a take starts recording until it ends.
  Stopwatch? _takeClock;

  /// Nothing being recorded, heard or sent — the mic at rest. A refused
  /// microphone is at rest too: nothing is in flight.
  bool get isIdle =>
      (_status == VoiceInputStatus.idle ||
          _status == VoiceInputStatus.unavailable) &&
      _heard.isEmpty &&
      !_isSending;

  /// Drops the take in progress, the words held from a send that failed, and
  /// the notice. A send already on its way is not recalled.
  void clear() {
    if (_disposed) return;
    _abandonTake();
    _heard = '';
    _setStatus(VoiceInputStatus.idle);
  }

  /// Starts a take.
  ///
  /// With [pretend], and while it has something to say, the take opens no microphone and asks for no
  /// permission: it shows as listening, its meter moving as if somebody spoke, and when it ends it is
  /// heard as what [pretend] says then. The first session's mic (owner, 2026-10-23): speaking there
  /// sends the task in the prompt, whatever was said — a microphone permission prompt in the middle
  /// of the onboarding costs it more than the words are worth. [pretend] saying null — the first
  /// send has gone, or a question is open — makes it an ordinary recording.
  Future<void> startListening({String? Function()? pretend}) async {
    if (_isSending || _disposed) return;
    if (_status != VoiceInputStatus.idle &&
        _status != VoiceInputStatus.unavailable) {
      return;
    }
    final take = ++_take;
    if (pretend != null && pretend() != null) {
      _startPretending(pretend);
      return;
    }
    _setStatus(VoiceInputStatus.starting);
    if (!await _micAllowed()) {
      if (take != _take) return;
      _setStatus(VoiceInputStatus.unavailable, notice: VoiceNotice.unavailable);
      return;
    }
    // Closed, cleared or stopped while the permission prompt was up.
    if (take != _take) return;
    try {
      await _recorder.start();
    } on Exception {
      if (take != _take) return;
      _setStatus(VoiceInputStatus.idle, notice: VoiceNotice.couldNotStart);
      return;
    }
    if (take != _take) {
      unawaited(_recorder.cancel());
      return;
    }
    _takeLimit = Timer(maxTake, _reachLimit);
    _takeClock = Stopwatch()..start();
    _setStatus(VoiceInputStatus.listening);
  }

  /// [maxTake] has passed: the take ends, unless [onTakeLimit] takes it over.
  void _reachLimit() => unawaited((onTakeLimit ?? stopListening)());

  /// Ends the take and transcribes it.
  Future<void> stopListening() async {
    if (_status == VoiceInputStatus.starting) {
      _abandonTake();
      _setStatus(VoiceInputStatus.idle);
      return;
    }
    if (_status == VoiceInputStatus.listening) await _transcribeTake();
  }

  /// A push-to-talk hold has begun. Pairs with [finishHold].
  ///
  /// ⚠️ **Exists because a hold cannot be expressed as start-then-stop, and
  /// trying broke hold-to-talk outright.** [startListening] takes two awaits to
  /// reach [VoiceInputStatus.listening] — the permission check, then the
  /// microphone actually opening, which is hundreds of milliseconds of hardware
  /// on a real phone. A release landing inside that window found the status on
  /// `starting` and abandoned the take, while the `startListening` still in
  /// flight went on to open the microphone and set `listening` — a recording
  /// nobody was holding and nothing would stop, whose audio then joined the NEXT
  /// take. What came back was the tail of one sentence glued to the head of
  /// another, which transcribes to nothing and reads as "Didn't catch that".
  ///
  /// So the hold is a fact of its own, held here, and the release is applied to
  /// whatever state the start has reached by then — including a start that has
  /// not finished yet, which [_holdReleased] makes it check for itself.
  Future<void> startHold(
    Future<bool> Function(String text) deliver, {
    String? Function()? pretend,
  }) async {
    _holdReleased = false;
    await startListening(pretend: pretend);
    // Let go while the microphone was still opening: the take that just became
    // live has no thumb on it, so it is ended and sent from here — the release
    // could not do it itself, because at that moment there was no take to end.
    if (_holdReleased && _status == VoiceInputStatus.listening) {
      _holdReleased = false;
      await submit(deliver);
    }
  }

  /// A push-to-talk hold has ended, and what was said should be sent.
  ///
  /// Returns at once when the microphone is still opening — [startHold] is
  /// mid-flight and owns the take, and this call has already recorded that the
  /// thumb is up, which is what makes it finish and send as soon as there is
  /// something to send.
  Future<void> finishHold(Future<bool> Function(String text) deliver) async {
    _holdReleased = true;
    if (_status == VoiceInputStatus.starting) return;
    _holdReleased = false;
    await submit(deliver);
  }

  /// Drops a push-to-talk take without sending it — the thumb slid off the
  /// button.
  ///
  /// ⚠️ Clears the flag as well, and that is not tidying: leaving it set would
  /// let a [startHold] still in flight reach its own send branch and deliver the
  /// very take this just threw away. [clear] bumps the take counter, so the
  /// recording itself is already abandoned; this closes the other half.
  void cancelHold() {
    _holdReleased = false;
    clear();
  }

  /// Whether the thumb has come up while a hold's [startListening] is still on
  /// its way to `listening`. See [startHold].
  bool _holdReleased = false;

  /// A first-task chip takes the same send and feedback path as spoken words.
  /// Never replaces an unfinished voice take.
  Future<void> submitText(
    String text,
    Future<bool> Function(String) deliver,
  ) async {
    if (_isSending ||
        _status != VoiceInputStatus.idle ||
        transcript.isNotEmpty) {
      return;
    }
    _heard = text;
    await submit(deliver);
  }

  /// Sends everything heard, and empties the transcript once it is sent — kept
  /// when delivery fails, so nothing said has to be said twice.
  ///
  /// Send while still talking ends the take first: one tap finishes the
  /// sentence and sends it. A take that FAILED sends nothing, rather than the
  /// half of the message that was heard before it.
  ///
  /// ⚠️ A take too short to be speech is not a failure: nothing was said in it,
  /// so nothing is missing. That is the retry face held to talk — a quick tap
  /// on it has to send the words kept from the send that did not land, and it
  /// used to be dropped with the tap, leaving no way to send them but saying
  /// something new.
  Future<void> submit(Future<bool> Function(String text) deliver) async {
    if (_isSending || _status == VoiceInputStatus.transcribing) return;
    if (_status == VoiceInputStatus.listening &&
        await _transcribeTake() == _Take.failed) {
      return;
    }
    final text = transcript.trim();
    if (text.isEmpty || _isSending) return;
    _isSending = true;
    _setStatus(VoiceInputStatus.idle);
    var sent = false;
    try {
      sent = await deliver(text);
    } catch (error) {
      // Reported below exactly like a refusal: the words stay, Send stays live.
      // ⚠️ Every throwable, not only [Exception]: anything that escaped here
      // would leave [isSending] set, and the mic busy for the pager's life.
      // The type only: what failed to send is what someone said.
      appLog.warn('voice', 'send failed: ${error.runtimeType}');
    }
    _isSending = false;
    if (sent) _heard = '';
    _setStatus(
      VoiceInputStatus.idle,
      notice: sent ? null : VoiceNotice.notSent,
    );
  }

  /// Everything heard — the take still being recorded included, once it is
  /// transcribed — emptied out of here, for the keyboard to carry on from.
  ///
  /// Nothing while a send is on its way: those words are already the message,
  /// and handing them to the keyboard too would put them in the prompt twice.
  ///
  /// Nothing for a pretend take either: nothing was said in it, and what it stands for is the
  /// prompt's own text, already there.
  Future<String> takeTranscript() async {
    if (_isSending) return '';
    if (_pretending != null) {
      clear();
      return '';
    }
    if (_status == VoiceInputStatus.listening) await _transcribeTake();
    final text = transcript;
    clear();
    return text;
  }

  @override
  void dispose() {
    _abandonTake();
    _noticeLimit?.cancel();
    _pretendLevel.dispose();
    _disposed = true;
    unawaited(_recorder.dispose());
    super.dispose();
  }

  /// Ends the take, and adds what it heard to the transcript.
  Future<_Take> _transcribeTake() async {
    final take = _take;
    _takeLimit?.cancel();
    _takeClock?.stop();
    if (_pretending case final heard?) {
      final length = _takeClock?.elapsed ?? Duration.zero;
      _stopPretending();
      // A brushed button, as with a recording: nothing to send.
      if (length < minTake) {
        _setStatus(VoiceInputStatus.idle);
        return _Take.tap;
      }
      String words;
      try {
        words = (heard() ?? '').trim();
      } catch (error) {
        appLog.warn('voice', 'pretend take unread', error: error);
        words = '';
      }
      // The prompt as it stands, and nothing held from an earlier send that failed: that was the same
      // task, and joined it would be sent twice.
      _heard = words;
      _setStatus(
        VoiceInputStatus.idle,
        notice: words.isEmpty ? VoiceNotice.nothingHeard : null,
      );
      return words.isEmpty ? _Take.failed : _Take.heard;
    }
    _setStatus(VoiceInputStatus.transcribing);
    try {
      final recording = await _recorder.stop();
      if (take != _take) return _Take.failed;
      appLog.info(
        'voice',
        recording == null
            ? 'take: no audio captured'
            : 'take: ${recording.length.inMilliseconds}ms '
                  '${recording.wav.length}B peak=${recording.peak}',
      );
      // Too short to be speech — a brushed button. Silent on purpose: there is
      // no failure to report, and a notice would be the app talking back about
      // something the person did not mean to do.
      if (recording != null && recording.length < minTake) {
        _setStatus(VoiceInputStatus.idle);
        return _Take.tap;
      }
      if (recording == null || recording.peak < silencePeak) {
        _setStatus(VoiceInputStatus.idle, notice: VoiceNotice.noSound);
        return _Take.failed;
      }
      if (mayTranscribe case final mayTranscribe?) {
        final go = await mayTranscribe();
        if (take != _take) return _Take.failed;
        if (!go) {
          _setStatus(VoiceInputStatus.idle);
          return _Take.failed;
        }
      }
      final language = _language.value;
      final words = await transcriber(recording.wav, language);
      // The count, never the words: a transcript is what someone said.
      appLog.info('voice', 'stt[$language]: ${words.length} chars');
      if (take != _take) return _Take.failed;
      _heard = _joinWords(_heard, words);
      _setStatus(
        VoiceInputStatus.idle,
        notice: words.isEmpty ? VoiceNotice.nothingHeard : null,
      );
      return words.isEmpty ? _Take.failed : _Take.heard;
    } catch (error) {
      // ⚠️ Every throwable, not only [Exception]: a reply of an unexpected shape
      // fails as a TypeError, and one that escaped would leave the status on
      // `transcribing` — the mic busy, for good, until the pager is closed.
      appLog.warn('voice', 'stt[${_language.value}] failed', error: error);
      if (take == _take) {
        _setStatus(VoiceInputStatus.idle, notice: VoiceNotice.notTranscribed);
      }
      return _Take.failed;
    }
  }

  Future<bool> _micAllowed() async {
    try {
      return await _recorder.allowed();
    } on Exception {
      return false;
    }
  }

  /// A pretend take: listening at once, with no microphone to open, and a meter shaped like speech —
  /// a syllable's rise and fall, with some breath in it.
  void _startPretending(String? Function() heard) {
    _pretending = heard;
    var beats = 0;
    _pretendMeter = Timer.periodic(const Duration(milliseconds: 90), (_) {
      beats++;
      final syllable = (sin(beats * 0.9) + 1) / 2;
      _pretendLevel.value =
          0.15 + 0.6 * syllable * (0.6 + 0.4 * _pretendRandom.nextDouble());
    });
    _takeLimit = Timer(maxTake, _reachLimit);
    _takeClock = Stopwatch()..start();
    _setStatus(VoiceInputStatus.listening);
  }

  void _stopPretending() {
    _pretending = null;
    _pretendMeter?.cancel();
    _pretendMeter = null;
    _pretendLevel.value = 0;
  }

  void _abandonTake() {
    _take++;
    _takeLimit?.cancel();
    _takeClock?.stop();
    if (_pretending != null) {
      _stopPretending();
      return;
    }
    if (_status == VoiceInputStatus.starting ||
        _status == VoiceInputStatus.listening) {
      unawaited(_recorder.cancel());
    }
  }

  void _setStatus(VoiceInputStatus status, {String? notice}) {
    _status = status;
    _notice = notice;
    _restartNoticeTimer();
    _notify();
  }

  /// Starts the countdown that clears [notice], or stops it when there is none.
  ///
  /// ⚠️ **Restarted on EVERY status change, not only on the ones that carry a
  /// notice.** A timer from a previous notice would otherwise still be running,
  /// and it would clear whatever notice happens to be showing when it fires —
  /// cutting a fresh one short by however long the old one had already been up.
  ///
  /// ⚠️ **[VoiceInputStatus.unavailable] keeps its notice indefinitely.** That
  /// one is not a report of something that just happened; it is the standing
  /// reason the mic cannot be used, and the row is the only place that says so.
  /// Everything else is transient and goes.
  void _restartNoticeTimer() {
    _noticeLimit?.cancel();
    _noticeLimit = null;
    if (_disposed ||
        _notice == null ||
        _status == VoiceInputStatus.unavailable) {
      return;
    }
    _noticeLimit = Timer(noticeLinger, () {
      _noticeLimit = null;
      // Only if it is still the same notice: anything that has set another one
      // since owns the row now, and has its own timer running for it.
      if (_notice == null || _disposed) return;
      _notice = null;
      _notify();
    });
  }

  void _notify() {
    if (!_disposed) notifyListeners();
  }
}

/// [VoiceInputController.level]: the recorder's meter, or the pretend one while a pretend take is on —
/// one listenable whichever it is, so a listener added and taken off lands on the same two.
class _TakeLevel implements ValueListenable<double> {
  _TakeLevel({
    required this.recorded,
    required this.pretended,
    required this.pretending,
  });

  final ValueListenable<double> recorded;
  final ValueListenable<double> pretended;
  final bool Function() pretending;

  @override
  double get value => pretending() ? pretended.value : recorded.value;

  @override
  void addListener(VoidCallback listener) {
    recorded.addListener(listener);
    pretended.addListener(listener);
  }

  @override
  void removeListener(VoidCallback listener) {
    recorded.removeListener(listener);
    pretended.removeListener(listener);
  }
}

String _joinWords(String first, String second) =>
    [first.trim(), second.trim()].where((part) => part.isNotEmpty).join(' ');

/// How a take ended, as [VoiceInputController.submit] needs to know it.
enum _Take {
  /// Its words joined the transcript.
  heard,

  /// Too short to be speech — a tap, with nothing said and so nothing lost.
  tap,

  /// Something was meant and did not arrive: silence, nothing transcribed, an
  /// upload that failed, or a take abandoned mid-flight.
  failed,
}