Skip to content

A cross platform Audiobookshelf client for Android and iOS

Dart
25.1 KB
Raw Blame
import 'dart:math' as math;
import 'dart:async';

import 'package:flutter/widgets.dart';

import 'audio_player_service.dart';
import 'ebook_cache.dart';
import 'transcript_chunker.dart';
import 'transcript_line_store.dart';
import 'transcription_service.dart';

/// Live transcript ("lyrics") mode: while enabled, transcribes ahead of the
/// playhead in 30s chunks and exposes the line under the current position for
/// the player to display. Lines go through [TranscriptLineStore], so a
/// listened stretch is never transcribed twice - and the store persists, so
/// a book you ran this on before starts with its runway already built.
///
/// The loop builds a deep runway: it transcribes flat-out until it is
/// [_targetListeningSeconds] of listening time ahead, then idles and tops
/// back up as you listen. Lines are held back until an initial
/// [_gateListeningSeconds] of runway exists, so the transcript never starts
/// in a stutter - the overlay shows build progress until then. It always
/// yields to user-triggered transcriptions (bookmark transcribe, the find
/// features), which the deep runway absorbs without visible gaps, and stops
/// with the mode, the screen, or playback.
/// Watches for the app leaving the foreground. Transcribing is heavy enough
/// to be felt on the whole phone, and it is only worth anything while you are
/// looking at it - so backgrounding the app stops it, and it has to be turned
/// on again deliberately.
class _LyricsLifecycle extends WidgetsBindingObserver {
  @override
  void didChangeAppLifecycleState(AppLifecycleState state) {
    // `inactive` also fires for a pulled-down notification shade or a
    // permission dialog, which shouldn't count as leaving.
    if (state == AppLifecycleState.paused ||
        state == AppLifecycleState.hidden) {
      final svc = LyricsService.instance;
      if (!svc.isOn) return;
      // The transcript is only worth anything on screen, so it stops. The
      // runway built so far is on disk and is still there next time.
      debugPrint('[Lyrics] app backgrounded - stopping');
      svc.disable();
    }
  }
}

class LyricsService extends ChangeNotifier {
  LyricsService._();
  static final LyricsService instance = LyricsService._();

  final _LyricsLifecycle _lifecycle = _LyricsLifecycle();
  TranscriptChunker? _chunker;
  bool _watching = false;

  /// How much runway to build, in seconds of listening time - multiplied by
  /// the playback speed to get book-seconds. Deep on purpose: momentary
  /// stalls (model load, a bookmark transcription borrowing the engine, one
  /// slow chunk) drain a shallow buffer and put gaps on screen; a deep one
  /// absorbs them.
  static const double _targetListeningSeconds = 600;

  /// Runway required before lines are shown, in seconds of listening time.
  /// Starting to display with no slack means the first hiccup stutters;
  /// holding lines back until this much is banked means once the transcript
  /// appears, it stays. Books with saved coverage pass instantly.
  static const double _gateListeningSeconds = 60;

  static const double _chunkSeconds = 30;

  bool _on = false;
  String? _key;
  String? _epubItemId; // itemId to look for a cached EPUB under (books only)
  Timer? _tick;
  bool _working = false;

  // Stretches already transcribed that added nothing to the runway: silence,
  // music, or a first line the store dropped as overlapping the line before,
  // leaving the next one past its gap tolerance. The store keeps no empty
  // lines, so without these the loop would ask for the same window forever.
  final Map<String, List<({double start, double end})>> _holes = {};

  /// How far the runway reaches from [pos], stepping over [_holes].
  double _coveredUntil(String key, double pos) => coveredPastHoles(
        pos,
        _holes[key] ?? const [],
        (from) => TranscriptLineStore.instance.coveredUntil(key, from),
      );

  /// [coveredFrom] is the store's coverage from a point; a hole that starts
  /// where coverage ends carries it on to the hole's end and beyond.
  @visibleForTesting
  static double coveredPastHoles(
    double pos,
    List<({double start, double end})> holes,
    double Function(double from) coveredFrom,
  ) {
    var covered = coveredFrom(pos);
    var moved = true;
    while (moved) {
      moved = false;
      for (final h in holes) {
        if (h.start <= covered + 0.5 && h.end > covered) {
          covered = coveredFrom(h.end);
          moved = true;
        }
      }
    }
    return covered;
  }

  void _markHole(String key, double start, double window) {
    (_holes[key] ??= []).add((start: start, end: start + window));
    debugPrint('[Lyrics] the window at ${start.toStringAsFixed(1)}s added '
        'nothing to the runway - moving past it');
  }
  TranscriptLine? _current;
  bool _transcribingVisible = false;
  bool _pastCoverage = false;
  bool _gatePassed = false;
  double _gateProgress = 0;
  double fontSize = 16;
  int maxLines = 3;
  bool fullCover = true;
  /// Read-along coloring: the ARGB value the spoken words are painted in,
  /// and whether tracking follows single words or whole sentences.
  int readAlongColor = PlayerSettings.defaultReadAlongColor;
  String readAlongMode = 'word';
  int _wordIndex = -1;
  int _probeTick = 0;
  double? _rate;
  /// Milliseconds the transcript is held back to match what you are hearing,
  /// and which output that figure belongs to.
  int offsetMs = 0;
  bool onBluetooth = false;
  bool _routeLoaded = false;
  int _routeTick = 0;
  int _slowChunks = 0;
  bool _forceFastModel = false;
  bool _reachedEnd = false;

  bool get isOn => _on;

  /// True while the ebook reader's read along is driving the transcript.
  /// The card overlay stays out of the way then - one of them at a time -
  /// including during the reader's exit animation, before it can disable
  /// the pipeline it started.
  bool _readerOwns = false;
  bool get readerOwns => _readerOwns;
  set readerOwns(bool v) {
    if (_readerOwns == v) return;
    _readerOwns = v;
    notifyListeners();
  }
  /// The store key lyrics are currently running for, so card overlays can
  /// render only on the playing item's card.
  String? get activeKey => _key;
  TranscriptLine? get current => _current;
  /// The transcript sync offset in seconds - subtract it from the playhead to
  /// get the moment you are actually hearing.
  double get offsetSeconds => offsetMs / 1000.0;


  /// 0-1 progress toward the initial runway while lines are still held back,
  /// or null once the gate has passed and lines are showing.
  double? get gateProgress => _gatePassed ? null : _gateProgress;

  /// True when this device measurably cannot transcribe as fast as you are
  /// listening, so the transcript is going to keep running out.
  bool get cannotKeepUp {
    final rate = _rate;
    if (rate == null) return false;
    return rate < AudioPlayerService().speed * 1.2;
  }

  /// Which word of [current] is being spoken, or -1 when the line has no word
  /// timing or word tracking is off.
  int get currentWordIndex => readAlongMode == 'word' ? _wordIndex : -1;
  /// True when the transcript has taken this item's cover over, so the card
  /// should hide its artwork and let the transcript sit on the card's own
  /// background instead.
  bool coversArtFor(String key) => _on && fullCover && _key == key;

  /// True when the playhead is past coverage and a chunk is being worked on -
  /// the overlay shows a subtle "listening ahead" state instead of nothing.
  bool get buffering => _on && _current == null && _pastCoverage;

  /// The store key for what the player has loaded right now, matching the
  /// download map's keying: composite for podcast episodes, itemId for books.
  static String? playerKey() {
    final p = AudioPlayerService();
    final itemId = p.currentItemId;
    if (itemId == null) return null;
    final ep = p.currentEpisodeId;
    return ep != null ? '$itemId-$ep' : itemId;
  }

  Future<void> enableForCurrent() async {
    final key = playerKey();
    if (key == null) return;
    _key = key;
    final ep = AudioPlayerService().currentEpisodeId;
    _epubItemId = ep == null ? AudioPlayerService().currentItemId : null;
    _chunker = TranscriptChunker(key: key, epubItemId: _epubItemId);
    _on = true;
    _rate = null;
    _slowChunks = 0;
    _forceFastModel = false;
    _gatePassed = false;
    _gateProgress = 0;
    _reachedEnd = false;
    _chunker?.reset();
    _routeLoaded = false;
    if (!_watching) {
      _watching = true;
      WidgetsBinding.instance.addObserver(_lifecycle);
    }
    await _refreshRoute();
    await reloadDisplayPrefs();
    await TranscriptLineStore.instance.hydrate(key);
    _tick?.cancel();
    _tick = Timer.periodic(const Duration(milliseconds: 200), (_) => _onTick());
    notifyListeners();
    final pos = AudioPlayerService().position.inMilliseconds / 1000.0;
    final covered = _coveredUntil(key, pos);
    debugPrint('[Lyrics] enabled for $key (epub lookup: ${_epubItemId != null}, '
        'saved runway ${(covered - pos).clamp(0, double.infinity).toStringAsFixed(0)}s '
        'from pos ${pos.toStringAsFixed(0)}s)');
    unawaited(_autoCacheEpub());
  }

  bool _epubFetching = false;

  /// Correction scans the whole EPUB per chunk, so a giant one is a per-chunk
  /// CPU tax on top of an unasked-for background download - past this size the
  /// book only gets corrections once the user opens it in the reader.
  static const int _epubAutoFetchCapBytes = 50 * 1024 * 1024;

  /// Downloads the book's EPUB into the reader cache when it isn't there yet,
  /// so chunk correction has the author's words even on a device where the
  /// book was never opened in the reader. Fire-and-forget: the chunker
  /// re-checks the cache each chunk, so corrections start with the first
  /// chunk after the file lands (and the model pick flips to tiny with it).
  /// Every failure just leaves honest Whisper.
  Future<void> _autoCacheEpub() async {
    final itemId = _epubItemId;
    if (itemId == null || _epubFetching) return;
    _epubFetching = true;
    try {
      final cached = await cachedEbookFileFor(itemId);
      if (cached != null && ebookExtFromFile(cached) == '.epub') return;
      final api = AudioPlayerService().currentApi;
      if (api == null) return;
      final item = await api.getLibraryItem(itemId);
      if (item == null) {
        debugPrint('[Lyrics] epub auto-fetch skipped - item lookup failed');
        return;
      }
      final ef = resolveEbookFile(item);
      if (ef == null || ebookExtFromFile(ef) != '.epub') {
        debugPrint('[Lyrics] no epub to correct against');
        return;
      }
      final size =
          ((ef['metadata'] as Map<String, dynamic>?)?['size'] as num?)?.toInt() ?? 0;
      if (size > _epubAutoFetchCapBytes) {
        debugPrint('[Lyrics] epub too big to auto-fetch '
            '(${(size / (1024 * 1024)).toStringAsFixed(0)} MB) - open it in '
            'the reader once to get corrected lines');
        return;
      }
      final f = await fetchEbookToCache(api, itemId, ef, '');
      final mb = (await f.length()) / (1024 * 1024);
      debugPrint(
          '[Lyrics] epub cached for correction (${mb.toStringAsFixed(1)} MB)');
    } catch (e) {
      debugPrint('[Lyrics] epub auto-fetch failed: $e');
    } finally {
      _epubFetching = false;
    }
  }

  /// Re-read the overlay display settings; the settings screen calls this so
  /// changes apply live while lyrics are running.
  Future<void> reloadDisplayPrefs() async {
    fontSize = await PlayerSettings.getLyricsFontSize();
    maxLines = await PlayerSettings.getLyricsMaxLines();
    fullCover = await PlayerSettings.getLyricsFullCover();
    readAlongColor = await PlayerSettings.getReadAlongColor();
    readAlongMode = await PlayerSettings.getReadAlongMode();
    notifyListeners();
  }

  void disable() {
    _on = false;
    if (_watching) {
      _watching = false;
      WidgetsBinding.instance.removeObserver(_lifecycle);
    }
    _tick?.cancel();
    _tick = null;
    _current = null;
    _wordIndex = -1;
    _key = null;
    notifyListeners();
    debugPrint('[Lyrics] disabled');
  }

  void _onTick() {
    if (!_on) return;
    final key = _key;
    if (key == null) return;
    // The player moved to a different book/episode: follow it.
    final nowKey = playerKey();
    if (nowKey == null) {
      disable();
      return;
    }
    if (nowKey != key) {
      // A different book or episode started. Don't carry the transcript over
      // to it - it costs battery and processing, so it stays something you
      // switch on deliberately, every time.
      debugPrint('[Lyrics] the player moved to $nowKey - stopping');
      disable();
      return;
    }
    final pos = AudioPlayerService().position.inMilliseconds / 1000.0;
    if (++_routeTick % 25 == 0) _refreshRoute();
    final speed = AudioPlayerService().speed;
    final covered = _coveredUntil(key, pos);
    final lead = covered - pos;
    _pastCoverage = lead <= 0.5;

    // Hold lines back until the initial runway exists (or the book's end is
    // inside it) - showing lines with no slack means the first stall
    // stutters. Once passed, the gate stays passed for the session, even if
    // the runway later drains: switching the display off mid-listen would
    // hide lines that are still correct.
    if (!_gatePassed) {
      // Once this device's real rate is known, the gate only needs enough
      // runway to ride out a couple of chunks' worth of transcription time -
      // on a fast device that's a fraction of the full gate, so lines appear
      // in seconds instead of half a minute. Until a rate exists, or on a
      // slow device, the full gate stands.
      var gateNeed = _gateListeningSeconds * speed;
      final r = _rate;
      if (r != null && r > 0) {
        final adaptive =
            ((_chunkSeconds / r) * speed * 2.5).clamp(10.0, gateNeed);
        gateNeed = adaptive;
      }
      final progress = (lead / gateNeed).clamp(0.0, 1.0).toDouble();
      final total = AudioPlayerService().totalDuration;
      final endInReach =
          _reachedEnd || (total > 0 && covered >= total - _chunkSeconds);
      if (lead >= gateNeed || endInReach) {
        _gatePassed = true;
        debugPrint('[Lyrics] gate passed: ${lead.toStringAsFixed(0)}s of '
            'runway banked (needed ${gateNeed.toStringAsFixed(0)}s at '
            '${speed}x${endInReach ? ', end of item in reach' : ''})');
        notifyListeners();
      } else if ((progress - _gateProgress).abs() > 0.01) {
        _gateProgress = progress;
        notifyListeners();
      }
    }

    // What you are hearing right now, which on Bluetooth is a fraction of a
    // second behind the playhead. Only the display is shifted - transcribing
    // still works from the real position.
    final heard = pos - offsetSeconds;
    // Small display lead: a line arriving a beat early reads like lyrics, a
    // beat late reads like subtitles falling behind. The gap-tolerant lookup
    // holds a line through the short silences Whisper's VAD trims out.
    // The lead is in book-seconds, so scale it by playback speed to keep the
    // same wall-clock feel at 1.5x or 2x.
    // A small lead matched to the word timing and almost no pre-show: a line
    // arriving a beat early cut the last word of every sentence short. A
    // following line also waits until the current one's last word has had
    // its turn.
    var line = _gatePassed
        ? TranscriptLineStore.instance
            .lineNear(key, heard + 0.15 * speed, preShow: 0.15)
        : null;
    final cur = _current;
    if (line != null && cur != null && line.start > cur.start) {
      final lastWord = cur.wordStarts.isNotEmpty
          ? math.min(cur.end, cur.wordStarts.last + 0.35)
          : cur.end - 0.1;
      if (heard < lastWord) line = cur;
    }
    var changed = line?.start != _current?.start || line?.text != _current?.text;
    _current = line;
    // Previews have guessed timing - a moving word highlight on them would
    // look confidently wrong, so they show as a whole dimmed line instead.
    final wordIndex = line == null || line.approx
        ? -1
        : line.wordIndexAt(heard + 0.15 * speed);
    if (wordIndex != _wordIndex) {
      _wordIndex = wordIndex;
      if (readAlongMode == 'word') changed = true;
    }
    if (changed) notifyListeners();
    // The one line that tells the whole story in a shared log: where the
    // playhead is, how much runway is banked against the target, and whether
    // this device is outrunning the narration.
    if (_routeTick % 75 == 0) {
      debugPrint('[Lyrics] status pos=${pos.toStringAsFixed(0)}s '
          'lead=${lead.toStringAsFixed(0)}s/'
          '${(_targetListeningSeconds * speed).toStringAsFixed(0)}s '
          'gate=${_gatePassed ? 'open' : '${(_gateProgress * 100).round()}%'} '
          'rate=${_rate?.toStringAsFixed(2) ?? '?'}x speed=${speed}x '
          'fastModel=$_forceFastModel end=$_reachedEnd');
    }
    _probe(pos, line);
    _ensureAhead(key, pos, covered);
  }

  /// Re-read which output is in use and load that output's offset. Cheap
  /// enough to run every few seconds, which is how a headphone connection
  /// mid-listen gets picked up.
  Future<void> _refreshRoute() async {
    final bt = await AudioPlayerService.isBluetoothAudioConnected();
    if (_routeLoaded && bt == onBluetooth) return;
    onBluetooth = bt;
    _routeLoaded = true;
    offsetMs = await PlayerSettings.getTranscriptOffsetMs(bluetooth: bt);
    debugPrint('[Lyrics] output is ${bt ? 'bluetooth' : 'the device'}, '
        'sync offset ${offsetMs}ms');
    notifyListeners();
  }

  /// Shift the transcript against the audio by [deltaMs], remembered against
  /// whichever output you are on.
  Future<void> nudgeOffset(int deltaMs) async {
    offsetMs = (offsetMs + deltaMs).clamp(-1000, 2000);
    notifyListeners();
    await PlayerSettings.setTranscriptOffsetMs(offsetMs, bluetooth: onBluetooth);
  }

  Future<void> clearOffset() async {
    offsetMs = 0;
    notifyListeners();
    await PlayerSettings.setTranscriptOffsetMs(0, bluetooth: onBluetooth);
  }

  /// How fast this device transcribes, in book-seconds per wall-second,
  /// smoothed over recent chunks. Below the playback speed the transcript can
  /// never catch the narration.
  void _noteRate(double span, double wall, double speed) {
    if (wall < 0.5 || span < 1) return;
    final rate = span / wall;
    _rate = _rate == null ? rate : _rate! * 0.6 + rate * 0.4;
    final need = speed * 1.2;
    debugPrint('[Lyrics] transcribed ${span.toStringAsFixed(0)}s in '
        '${wall.toStringAsFixed(1)}s = ${rate.toStringAsFixed(2)}x '
        '(need ${need.toStringAsFixed(2)}x at ${speed}x playback)');
    if (_rate! >= need) {
      _slowChunks = 0;
      return;
    }
    _slowChunks++;
    // One badly slow chunk, or two merely slow ones - the first chunk carries
    // the model load, so a single mild miss isn't proof on its own.
    if (_forceFastModel || !(_slowChunks >= 2 || rate < speed * 0.6)) return;
    _forceFastModel = true;
    _slowChunks = 0;
    final fastest = TranscriptionService.instance.lastModelUsed ==
        TranscriptionModelSize.tiny;
    if (fastest) {
      // Already on the fastest model: nothing to switch to, and the gaps are
      // this device's limit rather than a setting to fix. Keep the measured
      // rate - aiming chunks ahead is all that is left.
      debugPrint('[Lyrics] cannot keep up at ${speed}x even on the fast '
          'model - the transcript will have gaps');
      return;
    }
    // Re-measure from scratch: the slow model's rate would otherwise aim the
    // next chunk far ahead and skip audio the fast model could have covered.
    _rate = null;
    debugPrint('[Lyrics] too slow to keep up at ${speed}x - switching to the '
        'fast model for the rest of this session');
  }

  /// Once a second, write down where the audio is and what we believe is
  /// being said there. Played back against a screen recording it shows
  /// whether a line is early, late, or simply the wrong line.
  void _probe(double pos, TranscriptLine? line) {
    if (++_probeTick % 5 != 0) return;
    if (line == null) {
      debugPrint('[LyricsSync] pos=${pos.toStringAsFixed(1)} no line '
          '(${_pastCoverage ? 'past coverage' : 'gap'})');
      return;
    }
    final words = line.words;
    final word = _wordIndex >= 0 && _wordIndex < words.length
        ? words[_wordIndex]
        : '-';
    final text = line.text.length > 70
        ? '${line.text.substring(0, 70)}...'
        : line.text;
    debugPrint('[LyricsSync] pos=${pos.toStringAsFixed(1)} '
        'line=[${line.start.toStringAsFixed(1)}-${line.end.toStringAsFixed(1)}] '
        'word=$_wordIndex/${line.wordStarts.length} "$word" '
        'exact=${line.exact} offset=${offsetMs}ms | $text');
  }

  Future<void> _ensureAhead(String key, double pos, double covered) async {
    if (_working) return;
    final speed = AudioPlayerService().speed;
    // The end of the item is inside coverage: the runway is as deep as it
    // will ever need to be.
    final dur = AudioPlayerService().totalDuration;
    if (_reachedEnd || (dur > 0 && covered >= dur - 1)) {
      if (!_reachedEnd) {
        _reachedEnd = true;
        debugPrint('[Lyrics] coverage reached the end of $key');
      }
      if (_transcribingVisible) {
        _transcribingVisible = false;
        notifyListeners();
      }
      return;
    }
    // The target is listening time, so it scales with the playback speed -
    // ten minutes of runway at 1.5x is ten minutes of runway at 1x.
    if (covered - pos >= _targetListeningSeconds * speed) {
      if (_transcribingVisible) {
        _transcribingVisible = false;
        notifyListeners();
      }
      return;
    }
    // User-triggered transcriptions (bookmark, find) always win the engine.
    if (TranscriptionService.instance.isBusy) return;
    _working = true;
    _transcribingVisible = true;
    notifyListeners();
    // Starting at or behind the playhead, begin a few seconds early: pressing
    // play auto-rewinds, and a runway that starts exactly at the playhead
    // leaves that rewind in a hole with zero lead. Overlap with anything
    // already stored is dropped by the store.
    var start = covered > pos ? covered : (pos - 5).clamp(0.0, pos);
    if (covered <= pos + 1 && _rate != null && AudioPlayerService().isPlaying) {
      // Behind while playing. Transcribing from where you are now hands back
      // audio you have already heard by the time it finishes, and it never
      // catches up - so aim at where the playhead will BE when this chunk is
      // done. The stretch in between is skipped rather than shown late.
      // Only while playing: a paused playhead never moves into the chunk, and
      // an aim past the store's gap tolerance would pin the lead at zero
      // forever - the loop re-aims at the same unreachable spot every pass.
      final wall = _chunkSeconds / _rate!;
      start = pos + wall * speed + 1.5;
      debugPrint('[Lyrics] behind the playhead, aiming '
          '${(start - pos).toStringAsFixed(1)}s ahead');
    }
    final window = _rate == null ? 12.0 : _chunkSeconds;
    try {
      // With the book's EPUB cached, every word gets replaced by the book's
      // own, so tiny is enough and its speed is what keeps the transcript
      // ahead of the narration. Without one, these words are what you read -
      // unless this device can't transcribe them fast enough to be read at
      // all, in which case the fast model beats a blank screen.
      // Sprint the session's first window: a short one puts the first line
      // (and the rate measurement the adaptive gate needs) on screen in a
      // few seconds; full windows take over from there.
      final out = await _chunker!.transcribeInto(
        start: start,
        windowSeconds: window,
        preferAccuracy: !_forceFastModel,
      );
      // A chunk can outlive its session - whisper can't be interrupted, so a
      // book switch mid-transcription lets the old item's chunk finish after
      // the new session started. Its lines landed in the right store, but its
      // timing must not steer this session's rate (a slow model's wall time
      // would skip the sprint and mis-aim the catch-up).
      if (!_on || key != _key) return;
      _noteRate(out.span, out.wall, speed);
      if (out.lines > 0) {
        debugPrint('[Lyrics] chunk at ${start.toStringAsFixed(1)}s: '
            '${out.lines} lines, exact=${out.exact}');
      }
      if (_coveredUntil(key, start) < start + 1) _markHole(key, start, window);
    } on TranscriptionException catch (e) {
      if (!_on || key != _key) return;
      if (e.kind == TranscriptionError.empty) {
        // Silence or music: mark the stretch covered so the loop moves past
        // it instead of retrying the same window forever.
        _markHole(key, start, window);
      } else if (e.kind == TranscriptionError.extractFailed &&
          dur > 0 &&
          start + _chunkSeconds >= dur - 1) {
        // The window ran off the end of the audio.
        _reachedEnd = true;
        debugPrint('[Lyrics] the last window ran past the end of $key');
      } else {
        debugPrint('[Lyrics] chunk failed: $e');
        // Setup problems (model gone, download removed) end the session.
        if (e.kind != TranscriptionError.busy) disable();
      }
    } catch (e) {
      debugPrint('[Lyrics] chunk failed: $e');
    } finally {
      _working = false;
    }
  }

}