Skip to content

A cross platform Audiobookshelf client for Android and iOS

Dart

Remember where the audio and ebook have already been matched so finding a spot starts from a known place

Barnabas0bcd785619dc
diff --git a/lib/screens/transcription_settings_screen.dart b/lib/screens/transcription_settings_screen.dart
index ece4c6e7..71662c62 100644
--- a/lib/screens/transcription_settings_screen.dart
+++ b/lib/screens/transcription_settings_screen.dart
@@ -4,6 +4,7 @@ import '../l10n/app_localizations.dart';
import '../services/player_settings.dart';
import '../services/transcription_service.dart';
import '../services/lyrics_service.dart';
+import '../services/sync_point_store.dart';
import '../services/transcript_line_store.dart';
import '../widgets/overlay_toast.dart';
@@ -78,6 +79,7 @@ class _TranscriptionSettingsScreenState
);
if (ok != true || !mounted) return;
await TranscriptLineStore.instance.clearAll();
+ await SyncPointStore.instance.clearAll();
if (!mounted) return;
setState(() => _transcriptCacheBytes = 0);
showOverlayToast(context, l.lyricsCacheCleared,
diff --git a/lib/services/read_along_script.dart b/lib/services/read_along_script.dart
index d7a13048..a751935b 100644
--- a/lib/services/read_along_script.dart
+++ b/lib/services/read_along_script.dart
@@ -424,6 +424,7 @@ const String _body = r'''
out.visible = true;
out.dir = 0;
out.at = sStart;
+ out.si = c.sectionIndex;
out.sentence = raw.slice(sStart, Math.min(sEnd, sStart + 90)).replace(/\s+/g, ' ');
// The old sentence only goes once the new one is found, so a miss
// leaves the last sentence lit. Unwrapping merges text nodes, so the
diff --git a/lib/services/sync_point_store.dart b/lib/services/sync_point_store.dart
new file mode 100644
index 00000000..1244e7a5
--- /dev/null
+++ b/lib/services/sync_point_store.dart
@@ -0,0 +1,318 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter/foundation.dart';
+import 'package:path_provider/path_provider.dart';
+
+/// One confirmed place where the audiobook and the ebook line up: [t] seconds
+/// into the audio is [off] characters into spine section [si]. Written by
+/// every successful Find in audiobook, Find in ebook and read along line, and
+/// read back as anchors so the next find starts from a known spot instead of
+/// a chapter-title guess.
+class SyncPoint {
+ final double t;
+ final int si;
+ final double off;
+ /// 'a' = Find in audiobook, 'e' = Find in ebook, 'r' = read along.
+ final String src;
+ /// How far off [t] may be, in seconds. Find in ebook only knows the
+ /// transcript window the words came from; the other two are near exact.
+ final double err;
+ final int at;
+
+ const SyncPoint({
+ required this.t,
+ required this.si,
+ required this.off,
+ required this.src,
+ this.err = 0,
+ this.at = 0,
+ });
+
+ Map<String, dynamic> toJson() => {
+ 't': double.parse(t.toStringAsFixed(1)),
+ 'si': si,
+ 'o': off.round(),
+ 's': src,
+ if (err > 0) 'e': double.parse(err.toStringAsFixed(1)),
+ 'at': at,
+ };
+
+ factory SyncPoint.fromJson(Map<String, dynamic> j) => SyncPoint(
+ t: (j['t'] as num).toDouble(),
+ si: (j['si'] as num).toInt(),
+ off: (j['o'] as num).toDouble(),
+ src: j['s'] as String? ?? 'a',
+ err: (j['e'] as num?)?.toDouble() ?? 0,
+ at: (j['at'] as num?)?.toInt() ?? 0,
+ );
+}
+
+/// Where a finder should look first, worked out from sync points.
+class SyncEstimate {
+ final double est;
+ final double lo;
+ final double hi;
+ final double rate;
+ final String how;
+ const SyncEstimate(this.est, this.lo, this.hi, this.rate, this.how);
+}
+
+class SyncPointEstimator {
+ static const double _minRate = 0.02;
+ static const double _maxRate = 0.3;
+
+ /// Estimate for [offset] from points in the same section. Two points
+ /// either side give the section's real narration pace and a tight window;
+ /// one point paces from it at [fallbackRate] with a window that grows with
+ /// the distance. Null when [points] is empty.
+ static SyncEstimate? inSection(
+ List<SyncPoint> points, double offset, double fallbackRate) {
+ if (points.isEmpty) return null;
+ SyncPoint? before, after;
+ for (final p in points) {
+ if (p.off <= offset && (before == null || p.off > before.off)) before = p;
+ if (p.off >= offset && (after == null || p.off < after.off)) after = p;
+ }
+ if (before != null && after != null && after.off - before.off >= 50) {
+ final rate = (after.t - before.t) / (after.off - before.off);
+ if (rate >= _minRate && rate <= _maxRate) {
+ final est = before.t + (offset - before.off) * rate;
+ final span = after.t - before.t;
+ final half = (span * 0.25 + before.err + after.err).clamp(60.0, 900.0);
+ return SyncEstimate(est, est - half, est + half, rate, 'between');
+ }
+ }
+ final nearest = (before == null)
+ ? after!
+ : (after == null)
+ ? before
+ : ((offset - before.off) <= (after.off - offset) ? before : after);
+ return fromPoint(nearest, offset - nearest.off, fallbackRate);
+ }
+
+ /// Pace [chars] characters (negative = backwards) from [p] at [rate].
+ static SyncEstimate fromPoint(SyncPoint p, double chars, double rate) {
+ final est = p.t + chars * rate;
+ final half = ((chars * rate).abs() * 0.5 + 90 + p.err).clamp(90.0, 1500.0);
+ return SyncEstimate(est, est - half, est + half, rate, 'from ${p.src}');
+ }
+}
+
+/// Per-book sync points, one JSON file per library item under app support.
+/// The audio duration and spine length are stored with them so a re-encoded
+/// audiobook or a swapped epub throws the old points away instead of steering
+/// finds to the wrong place.
+class SyncPointStore {
+ SyncPointStore._();
+ static final SyncPointStore instance = SyncPointStore._();
+
+ static const int maxPoints = 600;
+
+ final Map<String, _Book> _cache = {};
+ final Set<String> _dirty = {};
+ bool _flushScheduled = false;
+
+ Future<Directory> _dir() async {
+ final base = await getApplicationSupportDirectory();
+ final dir = Directory('${base.path}/sync_points');
+ if (!dir.existsSync()) dir.createSync(recursive: true);
+ return dir;
+ }
+
+ Future<File> _fileFor(String itemId) async =>
+ File('${(await _dir()).path}/$itemId.json');
+
+ Future<_Book> _load(String itemId) async {
+ final cached = _cache[itemId];
+ if (cached != null) return cached;
+ var book = _Book();
+ try {
+ final f = await _fileFor(itemId);
+ if (f.existsSync()) {
+ final j = jsonDecode(await f.readAsString()) as Map<String, dynamic>;
+ book = _Book(
+ audioDuration: (j['dur'] as num?)?.toDouble(),
+ spineLength: (j['spine'] as num?)?.toInt(),
+ points: (j['p'] as List<dynamic>? ?? const [])
+ .map((e) => SyncPoint.fromJson(e as Map<String, dynamic>))
+ .toList(),
+ );
+ }
+ } catch (e) {
+ debugPrint('[SyncPoints] load failed for $itemId: $e');
+ }
+ _cache[itemId] = book;
+ return book;
+ }
+
+ /// Points for [itemId], dropped wholesale when the audio or the epub no
+ /// longer matches what they were recorded against.
+ Future<List<SyncPoint>> load(String itemId,
+ {double? audioDuration, int? spineLength}) async {
+ final book = await _load(itemId);
+ if (_stale(book, audioDuration, spineLength)) {
+ debugPrint('[SyncPoints] $itemId: book changed '
+ '(audio ${book.audioDuration?.toStringAsFixed(0)} -> '
+ '${audioDuration?.toStringAsFixed(0)}s, spine ${book.spineLength} -> '
+ '$spineLength) - forgetting ${book.points.length} points');
+ book.points.clear();
+ book.audioDuration = null;
+ book.spineLength = null;
+ _dirty.add(itemId);
+ _scheduleFlush();
+ }
+ _adoptShape(book, audioDuration, spineLength, itemId);
+ return List.unmodifiable(book.points);
+ }
+
+ /// What [load] last returned, without touching the disk.
+ List<SyncPoint> cached(String itemId) =>
+ List.unmodifiable(_cache[itemId]?.points ?? const <SyncPoint>[]);
+
+ Future<void> add(String itemId, SyncPoint p,
+ {double? audioDuration, int? spineLength}) async {
+ await load(itemId, audioDuration: audioDuration, spineLength: spineLength);
+ final book = _cache[itemId]!;
+ final stamped = SyncPoint(
+ t: p.t,
+ si: p.si,
+ off: p.off,
+ src: p.src,
+ err: p.err,
+ at: p.at > 0 ? p.at : DateTime.now().millisecondsSinceEpoch,
+ );
+ final merged = merge(book.points, stamped);
+ if (identical(merged, book.points)) return;
+ book.points = merged;
+ debugPrint('[SyncPoints] $itemId: +${p.src} t=${p.t.toStringAsFixed(1)} '
+ 'si=${p.si} off=${p.off.round()} (${merged.length} points)');
+ _dirty.add(itemId);
+ _scheduleFlush();
+ }
+
+ /// [existing] with [p] folded in, or [existing] itself when [p] adds
+ /// nothing: a near-duplicate keeps whichever is more exact, read along
+ /// lines only land every so often per section, and the table stays under
+ /// [maxPoints] by dropping the oldest read along points first.
+ static List<SyncPoint> merge(List<SyncPoint> existing, SyncPoint p) {
+ final out = List<SyncPoint>.from(existing);
+ for (var i = 0; i < out.length; i++) {
+ final q = out[i];
+ if (q.si != p.si) continue;
+ final nearChars = (q.off - p.off).abs();
+ final nearSecs = (q.t - p.t).abs();
+ if (nearChars < 60 && nearSecs < 8) {
+ if (p.err < q.err) {
+ out[i] = p;
+ return out;
+ }
+ return existing;
+ }
+ if (p.src == 'r' && q.src == 'r' && (nearChars < 300 || nearSecs < 20)) {
+ return existing;
+ }
+ }
+ out.add(p);
+ while (out.length > maxPoints) {
+ var victim = -1;
+ for (var i = 0; i < out.length; i++) {
+ if (out[i].src != 'r') continue;
+ if (victim < 0 || out[i].at < out[victim].at) victim = i;
+ }
+ if (victim < 0) {
+ for (var i = 0; i < out.length; i++) {
+ if (victim < 0 || out[i].at < out[victim].at) victim = i;
+ }
+ }
+ out.removeAt(victim);
+ }
+ return out;
+ }
+
+ /// Forget every book's points, memory and disk. Goes with clearing the
+ /// transcript cache: the points came from those transcripts.
+ Future<void> clearAll() async {
+ _cache.clear();
+ _dirty.clear();
+ try {
+ final dir = await _dir();
+ for (final f in dir.listSync()) {
+ if (f is! File) continue;
+ try {
+ await f.delete();
+ } catch (_) {}
+ }
+ } catch (e) {
+ debugPrint('[SyncPoints] clear failed: $e');
+ }
+ }
+
+ bool _stale(_Book book, double? audioDuration, int? spineLength) {
+ if (book.points.isEmpty) return false;
+ // A minute of slack: the session total and the metadata total of the
+ // same audiobook can disagree by that much. A different edition is off
+ // by far more.
+ if (audioDuration != null &&
+ book.audioDuration != null &&
+ (audioDuration - book.audioDuration!).abs() > 60) {
+ return true;
+ }
+ if (spineLength != null &&
+ book.spineLength != null &&
+ spineLength != book.spineLength) {
+ return true;
+ }
+ return false;
+ }
+
+ void _adoptShape(
+ _Book book, double? audioDuration, int? spineLength, String itemId) {
+ var changed = false;
+ if (audioDuration != null && audioDuration > 0 && book.audioDuration == null) {
+ book.audioDuration = audioDuration;
+ changed = true;
+ }
+ if (spineLength != null && spineLength > 0 && book.spineLength == null) {
+ book.spineLength = spineLength;
+ changed = true;
+ }
+ if (changed && book.points.isNotEmpty) {
+ _dirty.add(itemId);
+ _scheduleFlush();
+ }
+ }
+
+ void _scheduleFlush() {
+ if (_flushScheduled) return;
+ _flushScheduled = true;
+ Future.delayed(const Duration(seconds: 3), () async {
+ _flushScheduled = false;
+ final keys = _dirty.toList();
+ _dirty.clear();
+ for (final key in keys) {
+ final book = _cache[key];
+ if (book == null) continue;
+ try {
+ final f = await _fileFor(key);
+ await f.writeAsString(jsonEncode({
+ 'v': 1,
+ if (book.audioDuration != null) 'dur': book.audioDuration,
+ if (book.spineLength != null) 'spine': book.spineLength,
+ 'p': [for (final p in book.points) p.toJson()],
+ }));
+ } catch (e) {
+ debugPrint('[SyncPoints] write failed for $key: $e');
+ }
+ }
+ });
+ }
+}
+
+class _Book {
+ double? audioDuration;
+ int? spineLength;
+ List<SyncPoint> points;
+ _Book({this.audioDuration, this.spineLength, List<SyncPoint>? points})
+ : points = points ?? [];
+}
diff --git a/lib/widgets/ebook_reader_view.dart b/lib/widgets/ebook_reader_view.dart
index 104a412b..8abaf4a8 100644
--- a/lib/widgets/ebook_reader_view.dart
+++ b/lib/widgets/ebook_reader_view.dart
@@ -26,6 +26,7 @@ import '../services/scoped_prefs.dart';
import '../services/lyrics_service.dart';
import '../services/read_along_script.dart';
import '../services/screen_wake.dart';
+import '../services/sync_point_store.dart';
import '../services/transcript_line_store.dart';
import '../services/transcription_service.dart';
import '../services/volume_key_service.dart';
@@ -2248,13 +2249,16 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
showProgressDialog(context, l.findInEbookSearching);
Map<String, dynamic>? decision;
+ final pos = widget.findPositionSeconds;
+ var windowSeconds = findInEbookWindowSeconds;
try {
- decision = await _decideFindTarget(transcript, chapterHint);
+ decision = await _decideFindTarget(transcript, chapterHint,
+ nearSeconds: pos);
// A short transcript can be too generic to stand out ("So what I am.
// Am I?" ties with dialogue all over the book). Before giving up,
// listen to a longer window ending at the same spot and try once more.
- final pos = widget.findPositionSeconds;
if (decision == null && pos != null) {
+ windowSeconds = _findRetryWindowSeconds;
debugPrint('[FindEbook] not confident on the short window - '
'retrying with ${_findRetryWindowSeconds.toStringAsFixed(0)}s');
try {
@@ -2270,7 +2274,8 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
final f = File(longer.audioPath);
if (f.existsSync()) await f.delete();
} catch (_) {}
- decision = await _decideFindTarget(longer.text.trim(), chapterHint);
+ decision = await _decideFindTarget(longer.text.trim(), chapterHint,
+ nearSeconds: pos);
} on TranscriptionException catch (e) {
debugPrint('[FindEbook] retry transcription failed: $e');
}
@@ -2294,13 +2299,21 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
if (!jumped) {
showOverlayToast(context, l.findInEbookNotFound,
icon: Icons.search_off_rounded);
+ return;
+ }
+ // The transcript window ended at the pause point, so its last words are
+ // where the audio was.
+ if (pos != null && decision['e'] is num) {
+ unawaited(_rememberEbookHit(
+ decision, pos, (decision['e'] as num).toDouble(), windowSeconds));
}
}
/// Fuzzy-search the book for the transcript and apply the confidence rules.
/// Returns {href, excerpt} for a trustworthy hit, null when not confident.
Future<Map<String, dynamic>?> _decideFindTarget(
- String transcript, String? chapterHint) async {
+ String transcript, String? chapterHint,
+ {double? nearSeconds}) async {
// Whisper sometimes emits bracketed non-speech tags; they'd poison matching.
final cleaned = transcript
.replaceAll(RegExp(r'\[[^\]]*\]|\([^)]*\)'), ' ')
@@ -2313,12 +2326,15 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
// Spine sections whose TOC chapter agrees with the audio chapter get
// searched first (and an early exit on a strong hit there).
+ // Sections that earlier finds or read along pinned near this time come
+ // before the chapter-title guess: they are known, the title is inferred.
final hintBases = <String>[];
+ if (nearSeconds != null) hintBases.addAll(await _syncHintBases(nearSeconds));
if (chapterHint != null) {
for (final ch in _chapters) {
if (_chapterTitlesAgree(chapterHint, ch.title)) {
final base = ch.href.split('#').first.split('/').last.toLowerCase();
- if (base.isNotEmpty) hintBases.add(base);
+ if (base.isNotEmpty && !hintBases.contains(base)) hintBases.add(base);
}
}
}
@@ -2360,7 +2376,48 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
'second=${secondScore?.toStringAsFixed(3)} sameSpot=$sameSpot '
'confident=$confident excerpt="${best['excerpt']}"');
if (!confident) return null;
- return {'href': best['href'] as String? ?? '', 'excerpt': best['excerpt'] as String? ?? ''};
+ return {
+ 'href': best['href'] as String? ?? '',
+ 'excerpt': best['excerpt'] as String? ?? '',
+ 'idx': best['idx'],
+ 's': best['s'],
+ 'e': best['e'],
+ };
+ }
+
+ /// Spine sections holding sync points recorded within ten minutes of
+ /// [seconds], nearest first - the sections a transcript from around that
+ /// time almost certainly lives in.
+ Future<List<String>> _syncHintBases(double seconds) async {
+ final pts = await SyncPointStore.instance.load(widget.itemId);
+ if (pts.isEmpty) return const [];
+ final near = pts.where((p) => (p.t - seconds).abs() <= 600).toList()
+ ..sort((a, b) => (a.t - seconds).abs().compareTo((b.t - seconds).abs()));
+ if (near.isEmpty) return const [];
+ final hrefs = await _spineHrefs();
+ final out = <String>[];
+ for (final p in near) {
+ if (p.si < 0 || p.si >= hrefs.length) continue;
+ final base = hrefs[p.si].split('#').first.split('/').last.toLowerCase();
+ if (base.isNotEmpty && !out.contains(base)) out.add(base);
+ if (out.length >= 3) break;
+ }
+ return out;
+ }
+
+ /// A confirmed hit from a transcript taken at [seconds] becomes a sync
+ /// point at [off] characters into section [idx]; [err] is how far the
+ /// transcript window could put the words from [seconds].
+ Future<void> _rememberEbookHit(
+ Map<String, dynamic> decision, double seconds, double off, double err) async {
+ final idx = decision['idx'];
+ if (idx is! num || seconds < 0) return;
+ final hrefs = await _spineHrefs();
+ await SyncPointStore.instance.add(
+ widget.itemId,
+ SyncPoint(t: seconds, si: idx.toInt(), off: off, src: 'e', err: err),
+ spineLength: hrefs.isEmpty ? null : hrefs.length,
+ );
}
/// True when an audio chapter title and a TOC chapter title plausibly name
@@ -2745,6 +2802,19 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
debugPrint('[FindAudio] no audio chapters');
return null;
}
+ final spineAll = await _spineHrefs();
+ await SyncPointStore.instance.load(
+ widget.itemId,
+ audioDuration: audio.duration > 0 ? audio.duration : null,
+ spineLength: spineAll.isEmpty ? null : spineAll.length,
+ );
+ Future<double> remember(double t) async {
+ await SyncPointStore.instance.add(
+ widget.itemId,
+ SyncPoint(t: t, si: si, off: offset, src: 'a'),
+ );
+ return t;
+ }
// Which audio chapter is this section? That anchors both the initial
// estimate and the seconds-per-character correction rate. Books whose
@@ -2930,7 +3000,7 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
var probesLeft = _maxTotalProbes;
Future<double?> run(String label, double e, double from, double to,
- double pace, int rounds) async {
+ double pace, int rounds, {double? step}) async {
if (probesLeft <= 0) return null;
final n = rounds < probesLeft ? rounds : probesLeft;
probesLeft -= n;
@@ -2946,12 +3016,30 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
rate: pace,
duration: dur,
maxProbes: n,
- scanStepSeconds: scanStep,
+ scanStepSeconds: step ?? scanStep,
);
}
+ // A spot this book has already been matched at, in this section or the
+ // one either side, beats any chapter-title guess: it is known, and two
+ // of them give the real pace between. Probed first, in a window sized to
+ // how far the pacing had to reach.
+ final sync = await _syncPointCandidate(
+ si: si,
+ offset: offset,
+ total: total,
+ fallbackRate: rate,
+ top: dur > 0 ? dur : est + 3600);
+ if (sync != null) {
+ probed.add(sync.est);
+ final window = sync.hi - sync.lo;
+ final r = await run(sync.label, sync.est, sync.lo, sync.hi, sync.rate, 3,
+ step: (window / 3).clamp(45.0, _scanStepSeconds));
+ if (r != null) return remember(r);
+ }
+
final t = await run('anchor', est, lo, hi, rate, _maxProbes);
- if (t != null) return t;
+ if (t != null) return remember(t);
// The anchor never matched. Fallback candidates, most likely first, two
// rounds each.
@@ -3040,12 +3128,65 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
}
probed.add(c.est);
final r = await run(c.label, c.est, c.lo, c.hi, c.rate, 2);
- if (r != null) return r;
+ if (r != null) return remember(r);
}
debugPrint('[FindAudio] no candidate matched');
return null;
}
+ /// Where the sync points say [offset] in section [si] should be in the
+ /// audio. Same-section points first; failing that the last point of the
+ /// previous section or the first of the next, paced through the text in
+ /// between. Null when nothing recorded is close enough to help.
+ Future<({String label, double est, double lo, double hi, double rate})?>
+ _syncPointCandidate({
+ required int si,
+ required double offset,
+ required double total,
+ required double fallbackRate,
+ required double top,
+ }) async {
+ final pts = SyncPointStore.instance.cached(widget.itemId);
+ if (pts.isEmpty) return null;
+ final same = pts.where((p) => p.si == si).toList();
+ var e = SyncPointEstimator.inSection(same, offset, fallbackRate);
+ var label = 'sync points';
+ if (e == null) {
+ SyncPoint? prev, next;
+ for (final p in pts) {
+ if (p.si == si - 1 && (prev == null || p.off > prev.off)) prev = p;
+ if (p.si == si + 1 && (next == null || p.off < next.off)) next = p;
+ }
+ if (prev != null) {
+ final counts = await _sectionCharCounts(si - 1, si - 1);
+ if (counts.length == 1 && counts.first > 0) {
+ final chars = (counts.first - prev.off) + offset;
+ if (chars * fallbackRate <= 1800) {
+ e = SyncPointEstimator.fromPoint(prev, chars, fallbackRate);
+ label = 'sync point in the previous section';
+ }
+ }
+ }
+ if (e == null && next != null) {
+ final chars = (total - offset) + next.off;
+ if (chars * fallbackRate <= 1800) {
+ e = SyncPointEstimator.fromPoint(next, -chars, fallbackRate);
+ label = 'sync point in the next section';
+ }
+ }
+ }
+ if (e == null) return null;
+ debugPrint('[FindAudio] $label (${e.how}) from ${pts.length} points: '
+ 'est=${e.est.toStringAsFixed(1)} rate=${e.rate.toStringAsFixed(4)}');
+ return (
+ label: '$label (${e.how})',
+ est: e.est.clamp(0.0, top),
+ lo: e.lo.clamp(0.0, top),
+ hi: e.hi.clamp(0.0, top),
+ rate: e.rate,
+ );
+ }
+
/// Initial search window and seconds-per-character pacing from an audio
/// chapter anchor.
({double lo, double hi, double rate, double est}) _chapterAnchorEstimate(
@@ -3643,9 +3784,11 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
source = 'cached lines';
}
Map<String, dynamic>? decision;
+ var err = 4.0;
try {
if (text != null) {
- decision = await _decideFindTarget(text, chapterHint);
+ decision = await _decideFindTarget(text, chapterHint,
+ nearSeconds: pos);
}
// Whisper around the playhead: a short window first, a longer one if
// the words were too ordinary to pin down.
@@ -3654,7 +3797,8 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
final r = await _transcribeForJump(pos, window);
if (r == null) break;
source = '${window.toStringAsFixed(0)}s window';
- decision = await _decideFindTarget(r, chapterHint);
+ err = window / 2;
+ decision = await _decideFindTarget(r, chapterHint, nearSeconds: pos);
}
} catch (e) {
debugPrint('[ReadAlong] find place failed: $e');
@@ -3669,6 +3813,12 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
decision['href'] as String, decision['excerpt'] as String);
debugPrint('[ReadAlong] placed the audio at ${pos.toStringAsFixed(1)}s '
'via $source -> ${decision['href']} jumped=$jumped');
+ // The window sat around the playhead, so its middle is where the audio
+ // was. The lines that follow pin it exactly.
+ if (jumped && decision['s'] is num && decision['e'] is num) {
+ final mid = ((decision['s'] as num) + (decision['e'] as num)) / 2;
+ unawaited(_rememberEbookHit(decision, pos, mid, err));
+ }
}
/// A transcript of [window] seconds around [pos], or null when the engine
@@ -4102,7 +4252,10 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
}
_readAlongLocating = true;
try {
- await _paintReadAlongLine(needle, exact: needleExact, wordMode: wordMode);
+ await _paintReadAlongLine(needle,
+ exact: needleExact,
+ wordMode: wordMode,
+ lineStart: needleExact && needle == line.text.trim() ? line.start : null);
} finally {
_readAlongLocating = false;
}
@@ -4208,8 +4361,10 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
/// line that isn't in the rendered sections (chapter boundary) is found via
/// the search book when its text is ebook-exact; Whisper-flavored lines
/// that don't match are skipped - the next exact line re-anchors.
+ /// [lineStart] is the audio time the sentence starts at, when the needle is
+ /// a book line heard at a known moment; every such hit is a sync point.
Future<void> _paintReadAlongLine(String needle,
- {required bool exact, required bool wordMode}) async {
+ {required bool exact, required bool wordMode, double? lineStart}) async {
final wc = _epubController?.webViewController;
if (wc == null) return;
final watch = Stopwatch()..start();
@@ -4247,6 +4402,20 @@ class EbookReaderViewState extends State<EbookReaderView> with WidgetsBindingObs
debugPrint('[ReadAlong] "$head" -> page@$_readAlongAt '
'words=$_readAlongPageWords visible=${d['visible']} '
'${watch.elapsedMilliseconds}ms | page: "${d['sentence']}"');
+ if (lineStart != null && d['si'] is num && _readAlongAt >= 0) {
+ unawaited(SyncPointStore.instance.add(
+ widget.itemId,
+ SyncPoint(
+ t: lineStart,
+ si: (d['si'] as num).toInt(),
+ off: _readAlongAt.toDouble(),
+ src: 'r',
+ ),
+ audioDuration: AudioPlayerService().currentItemId == widget.itemId
+ ? AudioPlayerService().displayDuration
+ : null,
+ ));
+ }
if (d['visible'] == false) {
_readAlongTurn(d['cfi'], d['dir'], why: 'sentence is off the page');
}