Skip to content

A cross platform Audiobookshelf client for Android and iOS

Dart

Run Whisper on the CPU on iPhones older than the 12 so the live transcript stops crashing them, and log more around it so the crash on newer iPhones can be traced

Barnabas2cbce96a15f9
diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift
index 0a8506a6..a931b2e0 100644
--- a/ios/Runner/AppDelegate.swift
+++ b/ios/Runner/AppDelegate.swift
@@ -97,6 +97,9 @@ let flutterEngine = FlutterEngine(name: "SharedEngine", project: nil, allowHeadl
AbsorbAudioBridge.logSink = { [weak self] line in
self?.logToFlutter(line)
}
+ AudioWindowExtractor.logSink = { [weak self] line in
+ self?.logToFlutter(line)
+ }
// Register the native player core as an AppIntent dependency. The widget
// intent declares `@Dependency var core: AbsorbPlayerCoreProtocol` - that
@@ -897,18 +900,41 @@ private final class VolumeKeyWatcher {
/// conversion and downmix via its output settings, so no ffmpeg is needed.
/// Used by the opt-in bookmark transcription feature.
enum AudioWindowExtractor {
+ static var logSink: ((String) -> Void)?
+
+ private static func log(_ line: String) {
+ NSLog("%@", line)
+ logSink?(line)
+ }
+
+ private static func describe(_ track: AVAssetTrack) -> String {
+ let descriptions = track.formatDescriptions as! [CMFormatDescription]
+ guard let first = descriptions.first,
+ let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(first)?.pointee else {
+ return "unknown format"
+ }
+ let id = asbd.mFormatID
+ let bytes = [UInt8((id >> 24) & 0xff), UInt8((id >> 16) & 0xff),
+ UInt8((id >> 8) & 0xff), UInt8(id & 0xff)]
+ let codec = String(bytes: bytes, encoding: .ascii)?
+ .trimmingCharacters(in: .whitespaces) ?? "\(id)"
+ return "\(codec) \(Int(asbd.mSampleRate))Hz \(asbd.mChannelsPerFrame)ch"
+ }
+
static func extractWav(sourcePath: String, startSeconds: Double, durationSeconds: Double, outPath: String) -> Bool {
let asset = AVURLAsset(url: URL(fileURLWithPath: sourcePath))
guard let track = asset.tracks(withMediaType: .audio).first else {
- NSLog("[Transcribe] no audio track in %@", sourcePath)
+ log("[Transcribe] no audio track in \(sourcePath)")
return false
}
+ log("[Transcribe] decoding \(URL(fileURLWithPath: sourcePath).lastPathComponent): "
+ + "\(describe(track)), \(Int(startSeconds))s +\(Int(durationSeconds))s")
let reader: AVAssetReader
do {
reader = try AVAssetReader(asset: asset)
} catch {
- NSLog("[Transcribe] reader init failed: %@", error.localizedDescription)
+ log("[Transcribe] reader init failed: \(error.localizedDescription)")
return false
}
@@ -930,7 +956,7 @@ enum AudioWindowExtractor {
guard reader.canAdd(output) else { return false }
reader.add(output)
guard reader.startReading() else {
- NSLog("[Transcribe] startReading failed: %@", reader.error?.localizedDescription ?? "nil")
+ log("[Transcribe] startReading failed: \(reader.error?.localizedDescription ?? "nil")")
return false
}
@@ -953,7 +979,7 @@ enum AudioWindowExtractor {
}
if reader.status == .failed {
- NSLog("[Transcribe] reader failed: %@", reader.error?.localizedDescription ?? "nil")
+ log("[Transcribe] reader failed: \(reader.error?.localizedDescription ?? "nil")")
return false
}
if pcm.isEmpty { return false }
diff --git a/lib/services/audio_player_service.dart b/lib/services/audio_player_service.dart
index 939e5a28..b43300bb 100644
--- a/lib/services/audio_player_service.dart
+++ b/lib/services/audio_player_service.dart
@@ -14,6 +14,7 @@ import 'playback_history_service.dart' hide PlaybackEvent;
import 'progress_sync_service.dart';
import 'local_session_service.dart';
import 'sync_logic.dart';
+import 'transcription_service.dart';
import 'sleep_timer_service.dart';
import 'equalizer_service.dart';
import 'external_audio_output_types.dart';
@@ -2917,6 +2918,7 @@ class AudioPlayerService extends ChangeNotifier {
_handler!._cachedLockSeekBar = await PlayerSettings.getLockSeekBar();
debugPrint('[Player] AudioService initialized');
if (Platform.isIOS) unawaited(_reportPreviousBackgroundDeath());
+ unawaited(TranscriptionService.instance.reportInterruptedRun());
// Configure streaming cache if enabled
final cacheSizeMb = await PlayerSettings.getStreamingCacheSizeMb();
debugPrint('[Player] Streaming cache setting: $cacheSizeMb MB');
@@ -3324,7 +3326,7 @@ class AudioPlayerService extends ChangeNotifier {
/// iOS memory figures from the native side: `footprint` is what jetsam
/// judges (phys_footprint, not RSS) and `available` is how much more the
/// process may take before it is killed. Both in MB, -1 when unavailable.
- static Future<({int footprintMb, int availableMb})> _iosMemoryInfo() async {
+ static Future<({int footprintMb, int availableMb})> iosMemoryInfo() async {
try {
final info = await _eqChannelForDiag
.invokeMethod<Map<dynamic, dynamic>>('getMemoryInfo');
@@ -3346,7 +3348,7 @@ class AudioPlayerService extends ChangeNotifier {
/// came back from the background: iOS ended it there (or the user swiped
/// it away).
static Future<void> _logBackgroundMemory(int rssMb, int droppedMb) async {
- final m = await _iosMemoryInfo();
+ final m = await iosMemoryInfo();
final playing = _instance.isPlaying;
debugPrint('[Memory] Backgrounded: footprint=${m.footprintMb}MB '
'available=${m.availableMb}MB rss=${rssMb}MB playing=$playing, '
@@ -3366,7 +3368,7 @@ class AudioPlayerService extends ChangeNotifier {
final prefs = await SharedPreferences.getInstance();
await prefs.remove(_bgMarkerKey);
} catch (_) {}
- final m = await _iosMemoryInfo();
+ final m = await iosMemoryInfo();
debugPrint('[Memory] Foregrounded: footprint=${m.footprintMb}MB '
'available=${m.availableMb}MB');
}
@@ -3395,7 +3397,7 @@ class AudioPlayerService extends ChangeNotifier {
static void onMemoryPressure() {
if (!Platform.isIOS) return;
unawaited(() async {
- final m = await _iosMemoryInfo();
+ final m = await iosMemoryInfo();
debugPrint('[Memory] iOS memory warning: footprint=${m.footprintMb}MB '
'available=${m.availableMb}MB');
}());
diff --git a/lib/services/transcription_service.dart b/lib/services/transcription_service.dart
index 7178db35..37871128 100644
--- a/lib/services/transcription_service.dart
+++ b/lib/services/transcription_service.dart
@@ -8,13 +8,14 @@ import 'package:path_provider/path_provider.dart';
import 'package:whisper_ggml_plus/whisper_ggml_plus.dart';
import 'package:provider/provider.dart';
+import 'package:shared_preferences/shared_preferences.dart';
import '../main.dart' show rootNavigatorKey;
import '../providers/auth_provider.dart';
import '../providers/library_provider.dart';
import 'api_service.dart';
+import 'audio_player_service.dart';
import 'download_service.dart';
-import 'player_settings.dart';
import 'remote_audio_slice.dart';
import '../utils/whisper_language.dart';
@@ -149,13 +150,59 @@ class TranscriptionService {
// A native crash in the audio decode or in whisper takes the app down with
// nothing in the log, and the two look identical from outside. These say
- // which step was running when the log stops. Only the first few per run:
- // the live transcript does this every few seconds.
+ // which step was running when the log stops, with the memory numbers
+ // jetsam would have judged. Logged for the first few per run only (the live
+ // transcript does this every few seconds), but every step leaves a marker
+ // that the next launch reports if the run never finished.
+ static const _stepMarkerKey = 'transcribe_step_marker';
int _stepMarks = 0;
void _markStep(String step) {
- if (_stepMarks >= 6) return;
+ unawaited(_stampStep(step, log: _stepMarks < 6));
_stepMarks++;
- debugPrint('[Transcribe] step: $step');
+ }
+
+ Future<void> _stampStep(String step, {required bool log}) async {
+ final memory = await _memoryNumbers();
+ if (log) debugPrint('[Transcribe] step: $step ($memory)');
+ try {
+ final prefs = await SharedPreferences.getInstance();
+ await prefs.setString(_stepMarkerKey,
+ '${DateTime.now().millisecondsSinceEpoch}|$memory|$step');
+ } catch (_) {}
+ }
+
+ Future<void> _clearStepMarker() async {
+ try {
+ final prefs = await SharedPreferences.getInstance();
+ await prefs.remove(_stepMarkerKey);
+ } catch (_) {}
+ }
+
+ Future<String> _memoryNumbers() async {
+ if (Platform.isIOS) {
+ final m = await AudioPlayerService.iosMemoryInfo();
+ return 'footprint=${m.footprintMb}MB available=${m.availableMb}MB';
+ }
+ return 'rss=${ProcessInfo.currentRss ~/ 1048576}MB';
+ }
+
+ /// A step marker still present at launch means the last run died inside
+ /// transcription. Say which step and how much memory it held.
+ Future<void> reportInterruptedRun() async {
+ try {
+ final prefs = await SharedPreferences.getInstance();
+ final raw = prefs.getString(_stepMarkerKey);
+ if (raw == null) return;
+ await prefs.remove(_stepMarkerKey);
+ final parts = raw.split('|');
+ if (parts.length < 3) return;
+ final at = int.tryParse(parts[0]) ?? 0;
+ final ago = DateTime.now()
+ .difference(DateTime.fromMillisecondsSinceEpoch(at))
+ .inMinutes;
+ debugPrint('[Transcribe] previous run ended during "${parts.sublist(2).join('|')}" '
+ '${ago}min ago (${parts[1]})');
+ } catch (_) {}
}
/// Which model the last run actually used, after [_modelFor] had its say.
@@ -553,6 +600,7 @@ class TranscriptionService {
threads: _threads(),
);
text = (result?.transcription.text ?? '').trim();
+ _noteEngine(result);
_cacheLanguage(itemId, result?.language, text.isNotEmpty);
} catch (e) {
if (e is TranscriptionException) rethrow;
@@ -571,6 +619,7 @@ class TranscriptionService {
return out;
} finally {
_busy = false;
+ unawaited(_clearStepMarker());
if (wavPath != null) {
try {
final f = File(wavPath);
@@ -657,6 +706,7 @@ class TranscriptionService {
))
.where((s) => s.text.isNotEmpty)
.toList();
+ _noteEngine(result);
_cacheLanguage(itemId, result?.language, segments.isNotEmpty);
} catch (e) {
if (e is TranscriptionException) rethrow;
@@ -671,6 +721,7 @@ class TranscriptionService {
return segments;
} finally {
_busy = false;
+ unawaited(_clearStepMarker());
if (wavPath != null) {
try {
final f = File(wavPath);
@@ -740,6 +791,7 @@ class TranscriptionService {
threads: _threads(),
);
text = (result?.transcription.text ?? '').trim();
+ _noteEngine(result);
if (itemId != null) _cacheLanguage(itemId, result?.language, text.isNotEmpty);
} catch (e) {
if (e is TranscriptionException) rethrow;
@@ -751,6 +803,7 @@ class TranscriptionService {
return text;
} finally {
_busy = false;
+ unawaited(_clearStepMarker());
}
}
@@ -793,6 +846,7 @@ class TranscriptionService {
))
.where((s) => s.text.isNotEmpty)
.toList();
+ _noteEngine(result);
if (itemId != null) _cacheLanguage(itemId, result?.language, segments.isNotEmpty);
} catch (e) {
if (e is TranscriptionException) rethrow;
@@ -802,6 +856,7 @@ class TranscriptionService {
return segments;
} finally {
_busy = false;
+ unawaited(_clearStepMarker());
}
}
@@ -814,6 +869,25 @@ class TranscriptionService {
// stay on the big and mid cores and hold their speed unboosted.
int _threads() => (Platform.numberOfProcessors - 1).clamp(2, 4);
+ String? _loggedBackend;
+
+ /// What the engine reported about itself: which backend it is on, said
+ /// once, and whatever ggml and whisper logged, which is where a Metal
+ /// kernel that failed to build shows up.
+ void _noteEngine(dynamic result) {
+ final backend = result?.backend as String?;
+ if (backend != null && backend != _loggedBackend) {
+ _loggedBackend = backend;
+ debugPrint('[Transcribe] whisper runs on $backend');
+ }
+ final log = (result?.log as String?)?.trim();
+ if (log == null || log.isEmpty) return;
+ for (final line in log.split('\n')) {
+ final t = line.trim();
+ if (t.isNotEmpty) debugPrint('[WhisperCpp] $t');
+ }
+ }
+
/// Remember what language detection settled on, but only from a window that
/// actually produced output - a detection made on silence or music would
/// pin the whole book to a garbage language.
diff --git a/packages/whisper_ggml_plus/ios/Classes/metal_support.mm b/packages/whisper_ggml_plus/ios/Classes/metal_support.mm
new file mode 100644
index 00000000..01a420f6
--- /dev/null
+++ b/packages/whisper_ggml_plus/ios/Classes/metal_support.mm
@@ -0,0 +1,14 @@
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+
+// Whisper's Metal kernels target the Apple7 GPU family and up (A14, iPhone 12).
+// Below that a kernel can fail to build and whisper runs it anyway, which took
+// the app down on an iPhone 8. Those devices get the CPU instead.
+extern "C" bool absorb_metal_usable() {
+ id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+ if (device == nil) return false;
+ if (@available(iOS 13.0, *)) {
+ return [device supportsFamily:MTLGPUFamilyApple7];
+ }
+ return false;
+}
diff --git a/packages/whisper_ggml_plus/ios/Classes/whisper_flutter_plus.cpp b/packages/whisper_ggml_plus/ios/Classes/whisper_flutter_plus.cpp
index 5fb80924..b47e5a49 100644
--- a/packages/whisper_ggml_plus/ios/Classes/whisper_flutter_plus.cpp
+++ b/packages/whisper_ggml_plus/ios/Classes/whisper_flutter_plus.cpp
@@ -24,6 +24,7 @@
#endif
extern "C" const char* get_vad_model_path();
+extern "C" bool absorb_metal_usable();
using json = nlohmann::json;
@@ -96,9 +97,34 @@ static std::string default_vad_model_path() {
static struct whisper_context * g_ctx = nullptr;
static std::string g_model_path = "";
+static bool g_use_gpu = true;
static std::mutex g_mutex;
static std::atomic<bool> g_should_abort(false);
+// Everything ggml and whisper say, including a Metal kernel that failed to
+// build. The tail rides back to Dart with each response.
+static std::mutex g_log_mutex;
+static std::string g_log;
+static bool g_log_hooked = false;
+
+static void capture_whisper_log(enum ggml_log_level level, const char * text, void * user_data) {
+ (void) level;
+ (void) user_data;
+ fputs(text, stderr);
+ std::lock_guard<std::mutex> lock(g_log_mutex);
+ g_log += text;
+ if (g_log.size() > 8192) {
+ g_log.erase(0, g_log.size() - 8192);
+ }
+}
+
+static std::string take_whisper_log() {
+ std::lock_guard<std::mutex> lock(g_log_mutex);
+ std::string out;
+ out.swap(g_log);
+ return out;
+}
+
static void dispose_context_locked() {
if (g_ctx != nullptr) {
whisper_free(g_ctx);
@@ -162,16 +188,22 @@ json transcribe(json jsonBody)
json jsonResult;
jsonResult["@type"] = "transcribe";
- if (g_ctx == nullptr || g_model_path != params.model) {
+ if (!g_log_hooked) {
+ whisper_log_set(capture_whisper_log, nullptr);
+ g_log_hooked = true;
+ }
+ const bool use_gpu = absorb_metal_usable();
+ if (g_ctx == nullptr || g_model_path != params.model || g_use_gpu != use_gpu) {
dispose_context_locked();
-
+
whisper_context_params cparams = whisper_context_default_params();
- cparams.use_gpu = true;
+ cparams.use_gpu = use_gpu;
cparams.flash_attn = true;
g_ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
if (g_ctx != nullptr) {
g_model_path = params.model;
+ g_use_gpu = use_gpu;
}
}
@@ -311,6 +343,7 @@ json transcribe(json jsonBody)
}
jsonResult["text"] = text_result;
+ jsonResult["language"] = whisper_lang_str(whisper_full_lang_id(g_ctx));
return jsonResult;
}
@@ -330,7 +363,10 @@ extern "C"
return jsonToChar({{"@type", "dispose"}, {"message", "whisper context disposed"}});
}
if (jsonBody["@type"] == "getTextFromWavFile") {
- return jsonToChar(transcribe(jsonBody));
+ json result = transcribe(jsonBody);
+ result["backend"] = g_use_gpu ? "metal" : "cpu";
+ result["log"] = take_whisper_log();
+ return jsonToChar(result);
}
if (jsonBody["@type"] == "getVersion") {
return jsonToChar({{"@type", "version"}, {"message", "lib v1.8.3-accel"}});
diff --git a/packages/whisper_ggml_plus/lib/src/models/whisper_result.dart b/packages/whisper_ggml_plus/lib/src/models/whisper_result.dart
index 834d65e3..f6723279 100644
--- a/packages/whisper_ggml_plus/lib/src/models/whisper_result.dart
+++ b/packages/whisper_ggml_plus/lib/src/models/whisper_result.dart
@@ -5,10 +5,18 @@ class TranscribeResult {
required this.transcription,
required this.time,
this.language,
+ this.backend,
+ this.log,
});
final WhisperTranscribeResponse transcription;
final Duration time;
/// Language the native side detected (or was told), e.g. "en".
final String? language;
+
+ /// Where whisper ran this time ("metal" or "cpu"); iOS only.
+ final String? backend;
+
+ /// What ggml and whisper logged during this request; iOS only.
+ final String? log;
}
diff --git a/packages/whisper_ggml_plus/lib/src/whisper.dart b/packages/whisper_ggml_plus/lib/src/whisper.dart
index ab82431f..38c2569c 100644
--- a/packages/whisper_ggml_plus/lib/src/whisper.dart
+++ b/packages/whisper_ggml_plus/lib/src/whisper.dart
@@ -110,7 +110,7 @@ class Whisper {
/// Transcribe audio file to text. [language] is what the native side
/// detected (or was told), e.g. "en".
- Future<({WhisperTranscribeResponse response, String? language})> transcribe({
+ Future<({WhisperTranscribeResponse response, String? language, String? backend, String? log})> transcribe({
required TranscribeRequest transcribeRequest,
required String modelPath,
}) async {
@@ -124,12 +124,16 @@ class Whisper {
),
);
+ final String? log = result['log'] as String?;
if (result['text'] == null) {
- throw Exception(result['message']);
+ final String tail = (log ?? '').trim();
+ throw Exception(tail.isEmpty ? result['message'] : '${result['message']}\n$tail');
}
return (
response: WhisperTranscribeResponse.fromJson(result),
language: result['language'] as String?,
+ backend: result['backend'] as String?,
+ log: log,
);
} catch (e) {
debugPrint(e.toString());
diff --git a/packages/whisper_ggml_plus/lib/src/whisper_controller.dart b/packages/whisper_ggml_plus/lib/src/whisper_controller.dart
index 0b23b4ef..e808aa86 100644
--- a/packages/whisper_ggml_plus/lib/src/whisper_controller.dart
+++ b/packages/whisper_ggml_plus/lib/src/whisper_controller.dart
@@ -100,6 +100,8 @@ class WhisperController {
time: totalDuration,
transcription: result.response,
language: result.language,
+ backend: result.backend,
+ log: result.log,
);
} catch (e) {
debugPrint(e.toString());