Skip to content

A cross platform Audiobookshelf client for Android and iOS

Dart
16.6 KB
Raw Blame
import 'dart:io';
import 'dart:math';
import 'dart:typed_data';

import 'package:flutter/foundation.dart';
import 'package:path_provider/path_provider.dart';

import 'remote_audio_slice.dart';
import 'transcription_service.dart';

/// One place in the audio that might be where a chapter starts: the end of a
/// silence, with what was heard right after it and how much that sounded
/// like the chapter being edited.
class ChapterCandidate {
  /// Global book time, seconds.
  final double time;
  final double silenceSeconds;
  final String heard;
  /// 2 = the chapter's own number or title was heard, 1 = some chapter-like
  /// word ("chapter", "part", "prologue"), 0 = just a gap in the audio.
  final int score;
  /// A title read off the narration ("Chapter 2", "Chapter 2: The Storm"),
  /// when the words after the gap announced one.
  final String? suggestedTitle;
  const ChapterCandidate({
    required this.time,
    required this.silenceSeconds,
    required this.heard,
    required this.score,
    this.suggestedTitle,
  });
}

/// Pure helpers, kept apart from the audio plumbing so they can be tested.
class ChapterStartAnalysis {
  /// Runs of quiet in 16-bit mono PCM at [sampleRate], at least [minSeconds]
  /// long. Quiet is judged against the loud parts of the same clip, so a
  /// softly recorded book and a loud one both work.
  static List<({double start, double end})> findSilences(
    Int16List pcm,
    int sampleRate, {
    double minSeconds = 0.8,
    double hopSeconds = 0.05,
  }) {
    if (pcm.isEmpty || sampleRate <= 0) return const [];
    final hop = max(1, (sampleRate * hopSeconds).round());
    final frames = pcm.length ~/ hop;
    if (frames < 4) return const [];
    final rms = Float64List(frames);
    for (var f = 0; f < frames; f++) {
      var acc = 0.0;
      final base = f * hop;
      for (var i = 0; i < hop; i++) {
        final s = pcm[base + i] / 32768.0;
        acc += s * s;
      }
      rms[f] = sqrt(acc / hop);
    }
    final sorted = List<double>.from(rms)..sort();
    final loud = sorted[(sorted.length * 0.9).floor().clamp(0, sorted.length - 1)];
    final threshold = max(loud * 0.12, 0.003);
    final minFrames = max(1, (minSeconds / hopSeconds).round());
    final out = <({double start, double end})>[];
    var runStart = -1;
    for (var f = 0; f <= frames; f++) {
      final quiet = f < frames && rms[f] < threshold;
      if (quiet) {
        if (runStart < 0) runStart = f;
      } else if (runStart >= 0) {
        if (f - runStart >= minFrames) {
          out.add((start: runStart * hopSeconds, end: f * hopSeconds));
        }
        runStart = -1;
      }
    }
    return out;
  }

  /// Cut [from]..[to] seconds out of a 16-bit mono WAV and wrap it in a new
  /// header. Times are clamped to the clip.
  static Uint8List sliceWav(Uint8List wav, int sampleRate, double from, double to) {
    final pcm = pcmOf(wav);
    final a = (from * sampleRate).round().clamp(0, pcm.length);
    final b = (to * sampleRate).round().clamp(a, pcm.length);
    final slice = Int16List.sublistView(pcm, a, b);
    return wavOf(slice, sampleRate);
  }

  /// The samples of a 16-bit mono WAV, header skipped.
  static Int16List pcmOf(Uint8List wav) {
    // The extractor writes a plain 44-byte header; fall back to scanning for
    // the data chunk in case a different writer produced the file.
    var dataStart = 44;
    var dataLen = wav.length - 44;
    for (var i = 12; i + 8 <= wav.length;) {
      final id = String.fromCharCodes(wav.sublist(i, i + 4));
      final len = ByteData.sublistView(wav, i + 4, i + 8).getUint32(0, Endian.little);
      if (id == 'data') {
        dataStart = i + 8;
        dataLen = min(len, wav.length - dataStart);
        break;
      }
      i += 8 + len + (len.isOdd ? 1 : 0);
    }
    final evenLen = dataLen - (dataLen.isOdd ? 1 : 0);
    return Int16List.sublistView(wav, dataStart, dataStart + evenLen);
  }

  static Uint8List wavOf(Int16List pcm, int sampleRate) {
    final dataLen = pcm.lengthInBytes;
    final out = ByteData(44 + dataLen);
    void ascii(int offset, String s) {
      for (var i = 0; i < s.length; i++) {
        out.setUint8(offset + i, s.codeUnitAt(i));
      }
    }
    ascii(0, 'RIFF');
    out.setUint32(4, 36 + dataLen, Endian.little);
    ascii(8, 'WAVE');
    ascii(12, 'fmt ');
    out.setUint32(16, 16, Endian.little);
    out.setUint16(20, 1, Endian.little);
    out.setUint16(22, 1, Endian.little);
    out.setUint32(24, sampleRate, Endian.little);
    out.setUint32(28, sampleRate * 2, Endian.little);
    out.setUint16(32, 2, Endian.little);
    out.setUint16(34, 16, Endian.little);
    ascii(36, 'data');
    out.setUint32(40, dataLen, Endian.little);
    final bytes = out.buffer.asUint8List();
    bytes.setRange(44, 44 + dataLen, pcm.buffer.asUint8List(pcm.offsetInBytes, dataLen));
    return bytes;
  }

  static const _units = {
    'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6,
    'seven': 7, 'eight': 8, 'nine': 9, 'ten': 10, 'eleven': 11, 'twelve': 12,
    'thirteen': 13, 'fourteen': 14, 'fifteen': 15, 'sixteen': 16,
    'seventeen': 17, 'eighteen': 18, 'nineteen': 19,
  };
  static const _tens = {
    'twenty': 20, 'thirty': 30, 'forty': 40, 'fifty': 50, 'sixty': 60,
    'seventy': 70, 'eighty': 80, 'ninety': 90,
  };
  static const _ordinals = {
    'first': 1, 'second': 2, 'third': 3, 'fourth': 4, 'fifth': 5, 'sixth': 6,
    'seventh': 7, 'eighth': 8, 'ninth': 9, 'tenth': 10, 'eleventh': 11,
    'twelfth': 12, 'thirteenth': 13, 'fourteenth': 14, 'fifteenth': 15,
    'sixteenth': 16, 'seventeenth': 17, 'eighteenth': 18, 'nineteenth': 19,
    'twentieth': 20, 'thirtieth': 30, 'fortieth': 40, 'fiftieth': 50,
  };
  static const _chapterWords = {
    'chapter', 'part', 'prologue', 'epilogue', 'interlude', 'book', 'section',
    'act', 'introduction', 'preface', 'afterword',
  };

  /// Words of [s], lower-cased, with spelled-out numbers turned into digits
  /// ("twenty one" and "twenty-first" both give "21").
  static List<String> words(String s) {
    final raw = s
        .toLowerCase()
        .replaceAll(RegExp(r"[^\p{L}\p{N}' ]+", unicode: true), ' ')
        .split(RegExp(r'\s+'))
        .where((w) => w.isNotEmpty)
        .toList();
    final out = <String>[];
    for (var i = 0; i < raw.length; i++) {
      final w = raw[i];
      final tens = _tens[w];
      if (tens != null) {
        final next = i + 1 < raw.length ? raw[i + 1] : null;
        final unit = next == null ? null : (_units[next] ?? _ordinals[next]);
        if (unit != null && unit < 10) {
          out.add('${tens + unit}');
          i++;
        } else {
          out.add('$tens');
        }
        continue;
      }
      final n = _units[w] ?? _ordinals[w];
      if (n != null) {
        out.add('$n');
        continue;
      }
      if (w == 'hundred' && out.isNotEmpty && int.tryParse(out.last) != null) {
        out[out.length - 1] = '${int.parse(out.last) * 100}';
        continue;
      }
      out.add(w);
    }
    return out;
  }

  /// How much [heard] sounds like the start of chapter [number] titled
  /// [title]: 2 when the number or the title itself is spoken, 1 when any
  /// chapter-like word is, 0 otherwise.
  static int score(String heard, {required int number, required String title}) {
    final h = words(heard);
    if (h.isEmpty) return 0;
    final hasChapterWord = h.any(_chapterWords.contains);
    final numberSpoken = h.contains('$number');
    if (numberSpoken && hasChapterWord) return 2;
    final t = words(title).where((w) => w.length > 2 && !_chapterWords.contains(w)).toList();
    if (t.length >= 2) {
      final hit = t.where(h.contains).length;
      if (hit >= max(2, (t.length * 0.6).ceil())) return 2;
    } else if (t.length == 1 && h.contains(t.first) && t.first.length >= 4) {
      return 2;
    }
    if (numberSpoken && h.indexOf('$number') <= 2) return 2;
    return hasChapterWord ? 1 : 0;
  }

  static const _standaloneHeadings = {
    'prologue', 'epilogue', 'interlude', 'introduction', 'preface', 'afterword',
  };
  static const _numberedHeadings = {'chapter', 'part', 'book', 'section', 'act'};

  /// A chapter title read off what the narrator said right after the gap.
  /// "Chapter two" gives "Chapter 2"; "Chapter two, the storm. It was..."
  /// gives "Chapter 2: The Storm" when the extra words stop at a sentence
  /// break or a pause. Null when the words don't open with a heading.
  static String? suggestTitle(List<({double start, double end, String text})> segments) {
    if (segments.isEmpty) return null;
    final first = segments.first;
    final toks = words(first.text);
    if (toks.isEmpty) return null;
    final head = toks.first;
    String core;
    int consumed;
    if (_standaloneHeadings.contains(head)) {
      core = _cap(head);
      consumed = 1;
    } else if (_numberedHeadings.contains(head) && toks.length >= 2 && int.tryParse(toks[1]) != null) {
      core = '${_cap(head)} ${toks[1]}';
      consumed = 2;
    } else if (int.tryParse(head) != null && toks.length <= 8) {
      // Some narrators just say the number.
      core = 'Chapter $head';
      consumed = 1;
    } else {
      return null;
    }
    // Whatever else sits in the same breath before a sentence break is the
    // chapter's name. Whisper likes to put a full stop after the number
    // ("Chapter 3. Payback. I think..."), so a short clause right after that
    // stop counts as well, and so does a short second segment that follows
    // without a pause ("Chapter two." "The storm.").
    var rest = _clauseAfter(first.text, consumed);
    if (rest.isEmpty && consumed == 2) {
      final after = _clauseAfter(first.text, consumed, pastHeadingStop: true);
      final n = after.split(RegExp(r'\s+')).where((w) => w.isNotEmpty).length;
      if (n >= 1 && n <= 3) rest = after;
    }
    if (rest.isEmpty && consumed == 2 && segments.length > 1) {
      final next = segments[1];
      if (next.start - first.end < 0.6) {
        final nextWords = words(next.text);
        if (nextWords.isNotEmpty && nextWords.length <= 3 && _endsSentence(next.text)) {
          rest = _clauseAfter(next.text, 0);
        }
      }
    }
    final restWords = rest.split(RegExp(r'\s+')).where((w) => w.isNotEmpty).toList();
    if (restWords.isEmpty || restWords.length > 6) return core;
    return '$core: ${restWords.map(_cap).join(' ')}';
  }

  static String _cap(String w) {
    if (w.isEmpty) return w;
    final lower = w.toLowerCase();
    return lower[0].toUpperCase() + lower.substring(1);
  }

  static bool _endsSentence(String s) => RegExp(r'[.!?]\s*$').hasMatch(s.trim());

  /// The words of [raw] after its first [skipTokens] spoken tokens, up to the
  /// first sentence break, punctuation dropped. Empty when the heading
  /// itself ended the sentence ("Chapter two." and then narration), unless
  /// [pastHeadingStop] asks for the clause after that stop.
  static String _clauseAfter(String raw, int skipTokens, {bool pastHeadingStop = false}) {
    final rawWords = raw.trim().split(RegExp(r'\s+')).where((w) => w.isNotEmpty).toList();
    var i = 0;
    var consumed = 0;
    var lastHeadingWord = '';
    while (i < rawWords.length && consumed < skipTokens) {
      // "twenty two" is one token once normalized, so a pair of raw words
      // that collapse together is stepped over as one.
      final mergesWithNext = i + 1 < rawWords.length &&
          words(rawWords[i]).length == 1 &&
          words('${rawWords[i]} ${rawWords[i + 1]}').length == 1;
      if (mergesWithNext) {
        lastHeadingWord = rawWords[i + 1];
        i += 2;
      } else {
        lastHeadingWord = rawWords[i];
        i += 1;
      }
      consumed++;
    }
    if (!pastHeadingStop && skipTokens > 0 && RegExp(r'[.!?]$').hasMatch(lastHeadingWord)) {
      return '';
    }
    final out = <String>[];
    for (; i < rawWords.length; i++) {
      final w = rawWords[i];
      final clean = w.replaceAll(RegExp(r"[^\p{L}\p{N}'-]+", unicode: true), '');
      if (clean.isNotEmpty) out.add(clean);
      if (RegExp(r'[.!?;]$').hasMatch(w)) break;
    }
    return out.join(' ');
  }
}

/// Listens through a stretch of the book around a chapter marker and offers
/// the places a chapter could start: every decent silence, with the first
/// words after it, ranked by whether those words name the chapter.
class ChapterStartFinder {
  static const double afterSilenceSeconds = 6.0;
  static const int maxCandidatesPerWindow = 8;

  /// [source] is a local file or a stream URL the native extractor can open;
  /// [sourceOffset] is where that file starts in global book time and
  /// [startLocal] where the window starts inside it. With [remote] set the
  /// window's bytes are pulled down first and decoded from that slice, for
  /// platforms whose decoder can't read a URL. Candidates arrive through
  /// [onCandidate] as they are heard. Returns how many were found, or throws
  /// a [TranscriptionException] when nothing could be listened to.
  static Future<int> run({
    required String source,
    required double sourceOffset,
    required double startLocal,
    required double windowSeconds,
    required int chapterNumber,
    required String title,
    required void Function(ChapterCandidate) onCandidate,
    bool Function()? cancelled,
    RemoteTrack? remote,
  }) async {
    final svc = TranscriptionService.instance;
    var extractFrom = source;
    var extractStart = startLocal;
    AudioSlice? slice;
    if (remote != null) {
      try {
        slice = await RemoteAudioSlicer.fetch(
          remote,
          startLocal,
          windowSeconds,
          tmpDir: (await getTemporaryDirectory()).path,
        );
      } catch (e) {
        debugPrint('[ChapterFind] slice failed: $e');
        throw TranscriptionException(TranscriptionError.extractFailed, e);
      }
      extractFrom = slice.path;
      extractStart = slice.startWithin;
    }
    final String? wavPath;
    try {
      wavPath = await svc.extractWindowWav(
        source: extractFrom,
        startSeconds: extractStart,
        durationSeconds: windowSeconds,
      );
    } finally {
      if (slice != null) {
        try {
          File(slice.path).deleteSync();
        } catch (_) {}
      }
    }
    if (wavPath == null) throw TranscriptionException(TranscriptionError.extractFailed);
    var found = 0;
    try {
      final wav = await File(wavPath).readAsBytes();
      const rate = 16000;
      final pcm = ChapterStartAnalysis.pcmOf(wav);
      final clipSeconds = pcm.length / rate;
      var silences = ChapterStartAnalysis.findSilences(pcm, rate);
      // A window that opens in the middle of a silence still counts: the
      // speech that follows may be the chapter.
      if (silences.isEmpty || silences.first.start > 0.3) {
        silences = [(start: 0.0, end: 0.0), ...silences];
      }
      // Longest gaps first when there are too many, then back in time order.
      if (silences.length > maxCandidatesPerWindow) {
        silences.sort((a, b) => (b.end - b.start).compareTo(a.end - a.start));
        silences = silences.take(maxCandidatesPerWindow).toList()
          ..sort((a, b) => a.start.compareTo(b.start));
      }
      debugPrint('[ChapterFind] window ${startLocal.toStringAsFixed(1)}s+'
          '${windowSeconds.toStringAsFixed(0)}s: ${silences.length} gaps to listen after');
      for (final s in silences) {
        if (cancelled?.call() == true) break;
        final from = s.end;
        final to = min(clipSeconds, from + afterSilenceSeconds);
        if (to - from < 1.0) continue;
        final piece = ChapterStartAnalysis.sliceWav(wav, rate, from, to);
        final piecePath = '$wavPath.${(from * 10).round()}.wav';
        var heard = '';
        var segments = const <({double start, double end, String text})>[];
        try {
          await File(piecePath).writeAsBytes(piece, flush: true);
          segments = await svc.transcribeWavTimed(piecePath);
          heard = segments.map((s) => s.text).join(' ');
        } on TranscriptionException catch (e) {
          if (e.kind != TranscriptionError.empty) rethrow;
        } finally {
          try {
            File(piecePath).deleteSync();
          } catch (_) {}
        }
        final score = ChapterStartAnalysis.score(heard, number: chapterNumber, title: title);
        final candidate = ChapterCandidate(
          time: sourceOffset + startLocal + from,
          silenceSeconds: s.end - s.start,
          heard: heard,
          score: score,
          suggestedTitle: ChapterStartAnalysis.suggestTitle(segments),
        );
        debugPrint('[ChapterFind] ${candidate.time.toStringAsFixed(1)}s '
            'gap=${candidate.silenceSeconds.toStringAsFixed(1)}s score=$score "$heard"');
        found++;
        onCandidate(candidate);
      }
    } finally {
      try {
        File(wavPath).deleteSync();
      } catch (_) {}
    }
    return found;
  }
}