Skip to content

A cross platform Audiobookshelf client for Android and iOS

Dart

Show the playing book in the Windows media controls with its cover and let media keys and headphone buttons play, pause and skip

Barnabas41a1b4b6e675
diff --git a/lib/main.dart b/lib/main.dart
index d887b24b..2349a102 100644
--- a/lib/main.dart
+++ b/lib/main.dart
@@ -38,6 +38,7 @@ import 'services/home_widget_service.dart';
import 'services/siri_service.dart';
import 'services/narrated_books.dart';
import 'services/read_listen_sync.dart';
+import 'services/windows_media_controls.dart';
import 'services/log_service.dart';
import 'services/quick_actions_service.dart';
import 'services/setup_link_service.dart';
@@ -212,6 +213,7 @@ void main() async {
if (!AppPlatform.isWeb) HttpOverrides.global = _CertOverrides();
final widgetsBinding = WidgetsFlutterBinding.ensureInitialized();
ReadListenSync.instance.start();
+ WindowsMediaControls.instance.start();
BookOrbitApiService.placeReadingSpot = NarrationService.readingSpotSeconds;
BookOrbitApiService.keptNarration = NarrationService.instance.localOverlay;
ApiService.pageReadElsewhere = NarrationService.pageReadElsewhere;
diff --git a/lib/services/windows_media_controls.dart b/lib/services/windows_media_controls.dart
new file mode 100644
index 00000000..557f62d2
--- /dev/null
+++ b/lib/services/windows_media_controls.dart
@@ -0,0 +1,145 @@
+import 'dart:async';
+import 'dart:io';
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
+import 'package:flutter_cache_manager/flutter_cache_manager.dart';
+
+import '../utils/app_platform.dart';
+import 'audio_player_service.dart';
+
+/// Windows' media overlay and media keys. Shows the playing book with its
+/// cover, and turns the keyboard media keys and headphone presses into the
+/// player's own buttons: next and previous are the skip buttons, not chapters.
+/// audio_service does this on the phones; it has no Windows side.
+class WindowsMediaControls {
+ WindowsMediaControls._();
+ static final WindowsMediaControls instance = WindowsMediaControls._();
+
+ static const _channel = MethodChannel('com.barnabas.absorb/media_controls');
+ static const _timelineEvery = Duration(seconds: 5);
+
+ bool _started = false;
+ Timer? _pending;
+ Timer? _ticker;
+ String? _lastKey;
+ DateTime? _lastTimeline;
+ String? _coverUrl;
+ String? _coverPath;
+
+ void start() {
+ if (_started || !AppPlatform.isWindows) return;
+ _started = true;
+ _channel.setMethodCallHandler(_onCall);
+ AudioPlayerService().addListener(_schedule);
+ _ticker = Timer.periodic(_timelineEvery, (_) => _schedule());
+ _schedule();
+ }
+
+ void stop() {
+ if (!_started) return;
+ _started = false;
+ _channel.setMethodCallHandler(null);
+ AudioPlayerService().removeListener(_schedule);
+ _ticker?.cancel();
+ _pending?.cancel();
+ }
+
+ Future<dynamic> _onCall(MethodCall call) async {
+ final handler = AudioPlayerService.handler;
+ if (handler == null) return null;
+ switch (call.method) {
+ case 'button':
+ switch (call.arguments as String?) {
+ case 'play':
+ await handler.play();
+ case 'pause' || 'stop':
+ await handler.pause();
+ case 'next' || 'fastForward':
+ await handler.fastForward();
+ case 'previous' || 'rewind':
+ await handler.rewind();
+ }
+ case 'seek':
+ final ms = call.arguments as int?;
+ if (ms != null) await handler.seek(Duration(milliseconds: ms));
+ }
+ return null;
+ }
+
+ // The player notifies on every position tick; one send per burst is plenty.
+ void _schedule() {
+ _pending ??= Timer(const Duration(milliseconds: 300), () {
+ _pending = null;
+ unawaited(_send());
+ });
+ }
+
+ Future<void> _send() async {
+ final player = AudioPlayerService();
+ final itemId = player.currentItemId;
+ if (itemId == null || player.currentTitle == null) {
+ if (_lastKey != null) {
+ _lastKey = null;
+ await _invoke('clear');
+ }
+ return;
+ }
+ final chapter = player.currentChapter?['title'] as String? ?? '';
+ final playing = player.isPlaying;
+ final cover = await _coverFile(player.currentCoverUrl);
+ final key =
+ '$itemId|${player.currentTitle}|${player.currentAuthor}|$chapter|$playing|$cover';
+ final now = DateTime.now();
+ final timelineDue = _lastTimeline == null ||
+ now.difference(_lastTimeline!) >= _timelineEvery;
+ if (key == _lastKey && !timelineDue) return;
+ _lastKey = key;
+ _lastTimeline = now;
+ await _invoke('update', {
+ 'title': player.currentTitle ?? '',
+ 'artist': player.currentAuthor ?? '',
+ 'album': chapter,
+ 'playing': playing,
+ 'thumbnail': cover ?? '',
+ 'position': player.position.inMilliseconds.toDouble(),
+ 'duration': player.totalDuration * 1000,
+ 'rate': player.speed,
+ });
+ }
+
+ /// The cover as a local file the system can read: downloads keep one on
+ /// disk already, server covers go through the image cache.
+ Future<String?> _coverFile(String? url) async {
+ if (url == null || url.isEmpty) return null;
+ if (url == _coverUrl) return _coverPath;
+ String? path;
+ try {
+ if (url.startsWith('file:')) {
+ path = Uri.parse(url).toFilePath(windows: true);
+ } else if (url.startsWith('http')) {
+ final headers = AudioPlayerService().currentApi?.mediaHeaders;
+ final file =
+ await DefaultCacheManager().getSingleFile(url, headers: headers);
+ path = file.path;
+ } else if (File(url).existsSync()) {
+ path = url;
+ }
+ } catch (e) {
+ debugPrint('[MediaControls] cover for the overlay failed: $e');
+ }
+ _coverUrl = url;
+ _coverPath = path;
+ return path;
+ }
+
+ Future<void> _invoke(String method, [Object? args]) async {
+ try {
+ await _channel.invokeMethod<bool>(method, args);
+ } on MissingPluginException {
+ // An older runner without the channel: nothing to show.
+ } catch (e) {
+ debugPrint('[MediaControls] $method failed: $e');
+ }
+ }
+}
diff --git a/pubspec.lock b/pubspec.lock
index 319deef1..52054fb4 100644
--- a/pubspec.lock
+++ b/pubspec.lock
@@ -357,7 +357,7 @@ packages:
source: sdk
version: "0.0.0"
flutter_cache_manager:
- dependency: transitive
+ dependency: "direct main"
description:
name: flutter_cache_manager
sha256: "400b6592f16a4409a7f2bb929a9a7e38c72cceb8ffb99ee57bbf2cb2cecf8386"
diff --git a/pubspec.yaml b/pubspec.yaml
index adc01378..f686b1b6 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -25,6 +25,8 @@ dependencies:
# Image caching
cached_network_image: ^3.3.0
+ # Covers for the Windows media overlay come out of the image cache.
+ flutter_cache_manager: ^3.4.1
# For ImageRenderMethodForWeb: web covers decode through Flutter instead of
# borrowing <img> elements the browser can empty (they drew black).
cached_network_image_platform_interface: ^4.1.1
diff --git a/windows/runner/CMakeLists.txt b/windows/runner/CMakeLists.txt
index e8be1a0f..b20ebac2 100644
--- a/windows/runner/CMakeLists.txt
+++ b/windows/runner/CMakeLists.txt
@@ -10,6 +10,7 @@ add_executable(${BINARY_NAME} WIN32
"audio_window_channel.cpp"
"flutter_window.cpp"
"main.cpp"
+ "media_controls_channel.cpp"
"utils.cpp"
"win32_window.cpp"
"${FLUTTER_MANAGED_DIR}/generated_plugin_registrant.cc"
@@ -37,6 +38,8 @@ target_link_libraries(${BINARY_NAME} PRIVATE flutter flutter_wrapper_app)
target_link_libraries(${BINARY_NAME} PRIVATE "dwmapi.lib")
# Media Foundation, for decoding audio windows for transcription.
target_link_libraries(${BINARY_NAME} PRIVATE "mfplat.lib" "mfreadwrite.lib" "mfuuid.lib")
+# WinRT, for the System Media Transport Controls.
+target_link_libraries(${BINARY_NAME} PRIVATE "windowsapp.lib")
target_include_directories(${BINARY_NAME} PRIVATE "${CMAKE_SOURCE_DIR}")
# Run the Flutter tool portions of the build. This must not be removed.
diff --git a/windows/runner/flutter_window.cpp b/windows/runner/flutter_window.cpp
index ed547abe..93e390a9 100644
--- a/windows/runner/flutter_window.cpp
+++ b/windows/runner/flutter_window.cpp
@@ -27,6 +27,8 @@ bool FlutterWindow::OnCreate() {
RegisterPlugins(flutter_controller_->engine());
audio_window_channel_ = std::make_unique<AudioWindowChannel>(
flutter_controller_->engine()->messenger(), GetHandle());
+ media_controls_channel_ = std::make_unique<MediaControlsChannel>(
+ flutter_controller_->engine()->messenger(), GetHandle());
SetChildContent(flutter_controller_->view()->GetNativeWindow());
flutter_controller_->engine()->SetNextFrameCallback([&]() {
@@ -42,6 +44,7 @@ bool FlutterWindow::OnCreate() {
}
void FlutterWindow::OnDestroy() {
+ media_controls_channel_ = nullptr;
audio_window_channel_ = nullptr;
if (flutter_controller_) {
flutter_controller_ = nullptr;
@@ -71,6 +74,15 @@ FlutterWindow::MessageHandler(HWND hwnd, UINT const message,
case AudioWindowChannel::kDoneMessage:
if (audio_window_channel_) audio_window_channel_->DrainResults();
return 0;
+ case MediaControlsChannel::kButtonMessage:
+ if (media_controls_channel_) media_controls_channel_->DispatchButton(wparam);
+ return 0;
+ case MediaControlsChannel::kSeekMessage:
+ if (media_controls_channel_) media_controls_channel_->DispatchSeek(lparam);
+ return 0;
+ case MediaControlsChannel::kThumbnailMessage:
+ if (media_controls_channel_) media_controls_channel_->ApplyThumbnail();
+ return 0;
}
return Win32Window::MessageHandler(hwnd, message, wparam, lparam);
diff --git a/windows/runner/flutter_window.h b/windows/runner/flutter_window.h
index aa04ca73..204aa4d0 100644
--- a/windows/runner/flutter_window.h
+++ b/windows/runner/flutter_window.h
@@ -7,6 +7,7 @@
#include <memory>
#include "audio_window_channel.h"
+#include "media_controls_channel.h"
#include "win32_window.h"
// A window that does nothing but host a Flutter view.
@@ -32,6 +33,7 @@ class FlutterWindow : public Win32Window {
// Decodes audio windows for transcription.
std::unique_ptr<AudioWindowChannel> audio_window_channel_;
+ std::unique_ptr<MediaControlsChannel> media_controls_channel_;
};
#endif // RUNNER_FLUTTER_WINDOW_H_
diff --git a/windows/runner/media_controls_channel.cpp b/windows/runner/media_controls_channel.cpp
new file mode 100644
index 00000000..a59df29a
--- /dev/null
+++ b/windows/runner/media_controls_channel.cpp
@@ -0,0 +1,260 @@
+#include "media_controls_channel.h"
+
+#include <flutter/standard_method_codec.h>
+#include <systemmediatransportcontrolsinterop.h>
+#include <winrt/Windows.Foundation.h>
+#include <winrt/Windows.Storage.h>
+
+#include <chrono>
+#include <thread>
+
+using winrt::Windows::Media::MediaPlaybackStatus;
+using winrt::Windows::Media::MediaPlaybackType;
+using winrt::Windows::Media::SystemMediaTransportControls;
+using winrt::Windows::Media::SystemMediaTransportControlsButton;
+using winrt::Windows::Media::SystemMediaTransportControlsTimelineProperties;
+using winrt::Windows::Storage::StorageFile;
+using winrt::Windows::Storage::Streams::RandomAccessStreamReference;
+
+namespace {
+
+std::wstring Widen(const std::string& s) {
+ if (s.empty()) return std::wstring();
+ const int n = MultiByteToWideChar(CP_UTF8, 0, s.data(),
+ static_cast<int>(s.size()), nullptr, 0);
+ std::wstring w(n, L'\0');
+ MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast<int>(s.size()),
+ w.data(), n);
+ return w;
+}
+
+std::string ReadString(const flutter::EncodableMap& args, const char* key) {
+ auto it = args.find(flutter::EncodableValue(key));
+ if (it == args.end()) return std::string();
+ const auto* s = std::get_if<std::string>(&it->second);
+ return s ? *s : std::string();
+}
+
+bool ReadBool(const flutter::EncodableMap& args, const char* key) {
+ auto it = args.find(flutter::EncodableValue(key));
+ if (it == args.end()) return false;
+ const auto* b = std::get_if<bool>(&it->second);
+ return b ? *b : false;
+}
+
+double ReadDouble(const flutter::EncodableMap& args, const char* key) {
+ auto it = args.find(flutter::EncodableValue(key));
+ if (it == args.end()) return 0.0;
+ if (const auto* d = std::get_if<double>(&it->second)) return *d;
+ if (const auto* i = std::get_if<int32_t>(&it->second)) return *i;
+ if (const auto* l = std::get_if<int64_t>(&it->second)) {
+ return static_cast<double>(*l);
+ }
+ return 0.0;
+}
+
+const char* ButtonName(SystemMediaTransportControlsButton button) {
+ switch (button) {
+ case SystemMediaTransportControlsButton::Play:
+ return "play";
+ case SystemMediaTransportControlsButton::Pause:
+ return "pause";
+ case SystemMediaTransportControlsButton::Stop:
+ return "stop";
+ case SystemMediaTransportControlsButton::FastForward:
+ return "fastForward";
+ case SystemMediaTransportControlsButton::Rewind:
+ return "rewind";
+ case SystemMediaTransportControlsButton::Next:
+ return "next";
+ case SystemMediaTransportControlsButton::Previous:
+ return "previous";
+ default:
+ return nullptr;
+ }
+}
+
+} // namespace
+
+MediaControlsChannel::MediaControlsChannel(flutter::BinaryMessenger* messenger,
+ HWND window)
+ : window_(window) {
+ channel_ = std::make_unique<flutter::MethodChannel<flutter::EncodableValue>>(
+ messenger, "com.barnabas.absorb/media_controls",
+ &flutter::StandardMethodCodec::GetInstance());
+ channel_->SetMethodCallHandler(
+ [this](const auto& call, auto result) {
+ HandleCall(call, std::move(result));
+ });
+
+ try {
+ auto interop = winrt::get_activation_factory<
+ SystemMediaTransportControls, ISystemMediaTransportControlsInterop>();
+ winrt::check_hresult(interop->GetForWindow(
+ window_, winrt::guid_of<SystemMediaTransportControls>(),
+ winrt::put_abi(smtc_)));
+ smtc_.IsEnabled(false);
+ smtc_.IsPlayEnabled(true);
+ smtc_.IsPauseEnabled(true);
+ smtc_.IsStopEnabled(true);
+ smtc_.IsNextEnabled(true);
+ smtc_.IsPreviousEnabled(true);
+ // The system raises these off the platform thread; only the window
+ // handle is captured so a press after teardown finds nothing to touch.
+ const HWND target = window_;
+ button_token_ = smtc_.ButtonPressed(
+ [target](const SystemMediaTransportControls&, const auto& args) {
+ PostMessage(target, kButtonMessage,
+ static_cast<WPARAM>(args.Button()), 0);
+ });
+ seek_token_ = smtc_.PlaybackPositionChangeRequested(
+ [target](const SystemMediaTransportControls&, const auto& args) {
+ const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ args.RequestedPlaybackPosition())
+ .count();
+ PostMessage(target, kSeekMessage, 0, static_cast<LPARAM>(ms));
+ });
+ } catch (...) {
+ smtc_ = nullptr;
+ }
+}
+
+MediaControlsChannel::~MediaControlsChannel() {
+ if (!smtc_) return;
+ try {
+ smtc_.ButtonPressed(button_token_);
+ smtc_.PlaybackPositionChangeRequested(seek_token_);
+ smtc_.IsEnabled(false);
+ } catch (...) {
+ }
+}
+
+void MediaControlsChannel::HandleCall(
+ const flutter::MethodCall<flutter::EncodableValue>& call,
+ std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
+ if (!smtc_) {
+ result->Success(flutter::EncodableValue(false));
+ return;
+ }
+ try {
+ if (call.method_name() == "update") {
+ const auto* args = std::get_if<flutter::EncodableMap>(call.arguments());
+ if (args) Update(*args);
+ result->Success(flutter::EncodableValue(true));
+ } else if (call.method_name() == "clear") {
+ Clear();
+ result->Success(flutter::EncodableValue(true));
+ } else {
+ result->NotImplemented();
+ }
+ } catch (const winrt::hresult_error& e) {
+ result->Error("smtc", winrt::to_string(e.message()));
+ } catch (...) {
+ result->Error("smtc", "media controls update failed");
+ }
+}
+
+void MediaControlsChannel::Update(const flutter::EncodableMap& args) {
+ auto updater = smtc_.DisplayUpdater();
+ updater.Type(MediaPlaybackType::Music);
+ auto music = updater.MusicProperties();
+ music.Title(Widen(ReadString(args, "title")));
+ music.Artist(Widen(ReadString(args, "artist")));
+ music.AlbumTitle(Widen(ReadString(args, "album")));
+
+ const auto thumbnail = Widen(ReadString(args, "thumbnail"));
+ if (thumbnail != thumbnail_path_) {
+ thumbnail_path_ = thumbnail;
+ if (thumbnail.empty()) {
+ updater.Thumbnail(nullptr);
+ } else {
+ LoadThumbnail(thumbnail);
+ }
+ }
+ updater.Update();
+
+ const bool playing = ReadBool(args, "playing");
+ smtc_.PlaybackStatus(playing ? MediaPlaybackStatus::Playing
+ : MediaPlaybackStatus::Paused);
+ const double rate = ReadDouble(args, "rate");
+ if (rate > 0) smtc_.PlaybackRate(rate);
+
+ const auto position = std::chrono::milliseconds(
+ static_cast<int64_t>(ReadDouble(args, "position")));
+ const auto duration = std::chrono::milliseconds(
+ static_cast<int64_t>(ReadDouble(args, "duration")));
+ SystemMediaTransportControlsTimelineProperties timeline;
+ timeline.StartTime(std::chrono::milliseconds(0));
+ timeline.MinSeekTime(std::chrono::milliseconds(0));
+ timeline.Position(position);
+ timeline.MaxSeekTime(duration);
+ timeline.EndTime(duration);
+ smtc_.UpdateTimelineProperties(timeline);
+ smtc_.IsEnabled(true);
+}
+
+void MediaControlsChannel::Clear() {
+ thumbnail_path_.clear();
+ smtc_.PlaybackStatus(MediaPlaybackStatus::Closed);
+ auto updater = smtc_.DisplayUpdater();
+ updater.ClearAll();
+ updater.Update();
+ smtc_.IsEnabled(false);
+}
+
+// Opening the file goes through an async WinRT call that must not be waited
+// on from the platform thread, so a worker does it and posts back.
+void MediaControlsChannel::LoadThumbnail(const std::wstring& path) {
+ auto shared = thumbnail_;
+ const HWND target = window_;
+ std::thread([shared, target, path]() {
+ winrt::init_apartment(winrt::apartment_type::multi_threaded);
+ RandomAccessStreamReference stream{nullptr};
+ try {
+ auto file =
+ StorageFile::GetFileFromPathAsync(winrt::hstring(path)).get();
+ stream = RandomAccessStreamReference::CreateFromFile(file);
+ } catch (...) {
+ }
+ {
+ std::lock_guard<std::mutex> lock(shared->mutex);
+ shared->path = path;
+ shared->stream = stream;
+ }
+ PostMessage(target, kThumbnailMessage, 0, 0);
+ winrt::uninit_apartment();
+ }).detach();
+}
+
+void MediaControlsChannel::ApplyThumbnail() {
+ if (!smtc_) return;
+ RandomAccessStreamReference stream{nullptr};
+ {
+ std::lock_guard<std::mutex> lock(thumbnail_->mutex);
+ // A newer cover may have been asked for while this one loaded.
+ if (thumbnail_->path != thumbnail_path_) return;
+ stream = thumbnail_->stream;
+ }
+ if (!stream) return;
+ try {
+ auto updater = smtc_.DisplayUpdater();
+ updater.Thumbnail(stream);
+ updater.Update();
+ } catch (...) {
+ }
+}
+
+void MediaControlsChannel::DispatchButton(WPARAM button) {
+ if (!channel_) return;
+ const char* name =
+ ButtonName(static_cast<SystemMediaTransportControlsButton>(button));
+ if (!name) return;
+ channel_->InvokeMethod("button",
+ std::make_unique<flutter::EncodableValue>(name));
+}
+
+void MediaControlsChannel::DispatchSeek(LPARAM position_ms) {
+ if (!channel_) return;
+ channel_->InvokeMethod("seek", std::make_unique<flutter::EncodableValue>(
+ static_cast<int64_t>(position_ms)));
+}
diff --git a/windows/runner/media_controls_channel.h b/windows/runner/media_controls_channel.h
new file mode 100644
index 00000000..3bf40438
--- /dev/null
+++ b/windows/runner/media_controls_channel.h
@@ -0,0 +1,60 @@
+#ifndef RUNNER_MEDIA_CONTROLS_CHANNEL_H_
+#define RUNNER_MEDIA_CONTROLS_CHANNEL_H_
+
+#include <flutter/binary_messenger.h>
+#include <flutter/encodable_value.h>
+#include <flutter/method_channel.h>
+#include <windows.h>
+#include <winrt/Windows.Media.h>
+#include <winrt/Windows.Storage.Streams.h>
+
+#include <memory>
+#include <mutex>
+#include <string>
+
+// Windows side of com.barnabas.absorb/media_controls: the System Media
+// Transport Controls, so the media overlay shows the playing book with its
+// cover, and the keyboard media keys and headphone buttons reach the player.
+// Android and iOS get this from audio_service, which has no Windows side.
+class MediaControlsChannel {
+ public:
+ // Posted to the window from the system's thread on a button press or a
+ // seek from the overlay, and from the thumbnail loader when the cover is
+ // ready; the window calls the matching method so the work lands on the
+ // platform thread.
+ static constexpr UINT kButtonMessage = WM_APP + 0x42;
+ static constexpr UINT kSeekMessage = WM_APP + 0x43;
+ static constexpr UINT kThumbnailMessage = WM_APP + 0x44;
+
+ MediaControlsChannel(flutter::BinaryMessenger* messenger, HWND window);
+ ~MediaControlsChannel();
+
+ void DispatchButton(WPARAM button);
+ void DispatchSeek(LPARAM position_ms);
+ void ApplyThumbnail();
+
+ private:
+ struct Thumbnail {
+ std::mutex mutex;
+ std::wstring path;
+ winrt::Windows::Storage::Streams::RandomAccessStreamReference stream{
+ nullptr};
+ };
+
+ void HandleCall(
+ const flutter::MethodCall<flutter::EncodableValue>& call,
+ std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result);
+ void Update(const flutter::EncodableMap& args);
+ void Clear();
+ void LoadThumbnail(const std::wstring& path);
+
+ HWND window_;
+ winrt::Windows::Media::SystemMediaTransportControls smtc_{nullptr};
+ winrt::event_token button_token_{};
+ winrt::event_token seek_token_{};
+ std::wstring thumbnail_path_;
+ std::shared_ptr<Thumbnail> thumbnail_ = std::make_shared<Thumbnail>();
+ std::unique_ptr<flutter::MethodChannel<flutter::EncodableValue>> channel_;
+};
+
+#endif // RUNNER_MEDIA_CONTROLS_CHANNEL_H_