There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(electron): create recording through tray (#10526)
pengx17a016630a820f
diff --git a/packages/frontend/apps/electron-renderer/package.json b/packages/frontend/apps/electron-renderer/package.jsonindex 9c25d72403..e0aa60b27a 100644--- a/packages/frontend/apps/electron-renderer/package.json+++ b/packages/frontend/apps/electron-renderer/package.json@@ -13,6 +13,7 @@"@affine/electron-api": "workspace:*","@affine/i18n": "workspace:*","@affine/nbstore": "workspace:*",+ "@blocksuite/affine": "workspace:*","@emotion/react": "^11.14.0","@sentry/react": "^9.2.0","@toeverything/infra": "workspace:*",diff --git a/packages/frontend/apps/electron-renderer/src/app.tsx b/packages/frontend/apps/electron-renderer/src/app.tsxindex 234d75d221..88079245ae 100644--- a/packages/frontend/apps/electron-renderer/src/app.tsx+++ b/packages/frontend/apps/electron-renderer/src/app.tsx@@ -1,3 +1,4 @@+import type { DocProps } from '@affine/core/blocksuite/initialization';import { AffineContext } from '@affine/core/components/context';import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls';import { AppContainer } from '@affine/core/desktop/components/app-container';@@ -38,6 +39,8 @@ import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspacimport createEmotionCache from '@affine/core/utils/create-emotion-cache';import { apis, events } from '@affine/electron-api';import { StoreManagerClient } from '@affine/nbstore/worker/client';+import type { AttachmentBlockProps } from '@blocksuite/affine/model';+import { Text } from '@blocksuite/affine/store';import { CacheProvider } from '@emotion/react';import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';import { OpClient } from '@toeverything/infra/op';@@ -174,32 +177,80 @@ events?.applicationMenu.openAboutPageInSettingModal(() => {});events?.applicationMenu.onNewPageAction(type => {- const currentWorkspace = getCurrentWorkspace();- if (!currentWorkspace) {- return;- }- const { workspace, dispose } = currentWorkspace;- const editorSettingService = frameworkProvider.get(EditorSettingService);- const docsService = workspace.scope.get(DocsService);- const editorSetting = editorSettingService.editorSetting;-- const docProps = {- note: editorSetting.get('affine:note'),- };apis?.ui.isActiveTab().then(isActive => {if (!isActive) {return;}+ const currentWorkspace = getCurrentWorkspace();+ if (!currentWorkspace) {+ return;+ }+ const { workspace, dispose } = currentWorkspace;+ const editorSettingService = frameworkProvider.get(EditorSettingService);+ const docsService = workspace.scope.get(DocsService);+ const editorSetting = editorSettingService.editorSetting;++ const docProps = {+ note: editorSetting.get('affine:note'),+ };const page = docsService.createDoc({ docProps, primaryMode: type });workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);+ dispose();}).catch(err => {console.error(err);});+});- dispose();+events?.recording.onRecordingStatusChanged(status => {+ (async () => {+ if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {+ const currentWorkspace = getCurrentWorkspace();+ if (!currentWorkspace) {+ return;+ }+ const { workspace, dispose } = currentWorkspace;+ const editorSettingService = frameworkProvider.get(EditorSettingService);+ const docsService = workspace.scope.get(DocsService);+ const editorSetting = editorSettingService.editorSetting;++ const docProps: DocProps = {+ note: editorSetting.get('affine:note'),+ page: {+ title: new Text(+ 'Recording ' ++ (status.appGroup?.name ?? 'System Audio') ++ ' ' ++ new Date(status.startTime).toISOString()+ ),+ },+ onStoreLoad: (doc, { noteId }) => {+ (async () => {+ const data = await apis?.recording.saveRecording(status.id);+ if (!data) {+ return;+ }+ const blob = new Blob([data], { type: 'audio/mp3' });+ const blobId = await doc.workspace.blobSync.set(blob);+ const attachmentProps: Partial<AttachmentBlockProps> = {+ name: 'Recording',+ size: blob.size,+ type: 'audio/mp3',+ sourceId: blobId,+ embed: true,+ };+ doc.addBlock('affine:attachment', attachmentProps, noteId);+ })().catch(console.error);+ },+ };+ const page = docsService.createDoc({ docProps, primaryMode: 'page' });+ workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);++ dispose();+ }+ })().catch(console.error);});events?.applicationMenu.onOpenJournal(() => {diff --git a/packages/frontend/apps/electron-renderer/tsconfig.json b/packages/frontend/apps/electron-renderer/tsconfig.jsonindex 19af95cc4f..422502d628 100644--- a/packages/frontend/apps/electron-renderer/tsconfig.json+++ b/packages/frontend/apps/electron-renderer/tsconfig.json@@ -13,6 +13,7 @@{ "path": "../../electron-api" },{ "path": "../../i18n" },{ "path": "../../../common/nbstore" },+ { "path": "../../../../blocksuite/affine/all" },{ "path": "../../../common/infra" },{ "path": "../../../../tools/utils" }]diff --git a/packages/frontend/apps/electron/resources/icons/monitor.png b/packages/frontend/apps/electron/resources/icons/monitor.pngnew file mode 100644index 0000000000..e4305b8093Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/monitor.png differdiff --git a/packages/frontend/apps/electron/resources/icons/pause.png b/packages/frontend/apps/electron/resources/icons/pause.pngnew file mode 100644index 0000000000..f4ff7ee83dBinary files /dev/null and b/packages/frontend/apps/electron/resources/icons/pause.png differdiff --git a/packages/frontend/apps/electron/resources/icons/stop.png b/packages/frontend/apps/electron/resources/icons/stop.pngnew file mode 100644index 0000000000..32977c4f2bBinary files /dev/null and b/packages/frontend/apps/electron/resources/icons/stop.png differdiff --git a/packages/frontend/apps/electron/resources/icons/waveform-recording.png b/packages/frontend/apps/electron/resources/icons/waveform-recording.pngnew file mode 100644index 0000000000..19e91f03f9Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/waveform-recording.png differdiff --git a/packages/frontend/apps/electron/resources/icons/waveform.png b/packages/frontend/apps/electron/resources/icons/waveform.pngnew file mode 100644index 0000000000..11a4f05d38Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/waveform.png differdiff --git a/packages/frontend/apps/electron/src/helper/main-rpc.ts b/packages/frontend/apps/electron/src/helper/main-rpc.tsindex 59be05958f..e6026f5d30 100644--- a/packages/frontend/apps/electron/src/helper/main-rpc.ts+++ b/packages/frontend/apps/electron/src/helper/main-rpc.ts@@ -2,6 +2,7 @@ import { AsyncCall } from 'async-call-rpc';import type { HelperToMain, MainToHelper } from '../shared/type';import { exposed } from './provide';+import { encodeToMp3 } from './recording/encode';const helperToMainServer: HelperToMain = {getMeta: () => {@@ -10,6 +11,8 @@ const helperToMainServer: HelperToMain = {}return exposed;},+ // allow main process encode audio samples to mp3 buffer (because it is slow and blocking)+ encodeToMp3,};export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {diff --git a/packages/frontend/apps/electron/src/helper/recording/encode.ts b/packages/frontend/apps/electron/src/helper/recording/encode.tsnew file mode 100644index 0000000000..0aa3464876--- /dev/null+++ b/packages/frontend/apps/electron/src/helper/recording/encode.ts@@ -0,0 +1,16 @@+import { Mp3Encoder } from '@affine/native';++// encode audio samples to mp3 buffer+export function encodeToMp3(+ samples: Float32Array,+ opts: {+ channels?: number;+ sampleRate?: number;+ } = {}+): Uint8Array {+ const mp3Encoder = new Mp3Encoder({+ channels: opts.channels ?? 2,+ sampleRate: opts.sampleRate ?? 44100,+ });+ return mp3Encoder.encode(samples);+}diff --git a/packages/frontend/apps/electron/src/main/events.ts b/packages/frontend/apps/electron/src/main/events.tsindex 51b5c1b07e..fec2f1dc78 100644--- a/packages/frontend/apps/electron/src/main/events.ts+++ b/packages/frontend/apps/electron/src/main/events.ts@@ -4,6 +4,7 @@ import { AFFINE_EVENT_CHANNEL_NAME } from '../shared/type';import { applicationMenuEvents } from './application-menu';import { beforeAppQuit } from './cleanup';import { logger } from './logger';+import { recordingEvents } from './recording';import { sharedStorageEvents } from './shared-storage';import { uiEvents } from './ui/events';import { updaterEvents } from './updater/event';@@ -13,6 +14,7 @@ export const allEvents = {updater: updaterEvents,ui: uiEvents,sharedStorage: sharedStorageEvents,+ recording: recordingEvents,};function getActiveWindows() {diff --git a/packages/frontend/apps/electron/src/main/handlers.ts b/packages/frontend/apps/electron/src/main/handlers.tsindex 59ef7a0d55..70d0fcf40a 100644--- a/packages/frontend/apps/electron/src/main/handlers.ts+++ b/packages/frontend/apps/electron/src/main/handlers.ts@@ -5,6 +5,7 @@ import { clipboardHandlers } from './clipboard';import { configStorageHandlers } from './config-storage';import { findInPageHandlers } from './find-in-page';import { getLogFilePath, logger, revealLogFile } from './logger';+import { recordingHandlers } from './recording';import { sharedStorageHandlers } from './shared-storage';import { uiHandlers } from './ui/handlers';import { updaterHandlers } from './updater';@@ -29,6 +30,7 @@ export const allHandlers = {findInPage: findInPageHandlers,sharedStorage: sharedStorageHandlers,worker: workerHandlers,+ recording: recordingHandlers,};export const registerHandlers = () => {diff --git a/packages/frontend/apps/electron/src/main/index.ts b/packages/frontend/apps/electron/src/main/index.tsindex e1f1d71ebb..f67e9dc9d7 100644--- a/packages/frontend/apps/electron/src/main/index.ts+++ b/packages/frontend/apps/electron/src/main/index.ts@@ -14,6 +14,8 @@ import { registerEvents } from './events';import { registerHandlers } from './handlers';import { logger } from './logger';import { registerProtocol } from './protocol';+import { setupRecording } from './recording';+import { getTrayState } from './tray';import { registerUpdater } from './updater';import { launch } from './windows-manager/launcher';import { launchStage } from './windows-manager/stage';@@ -85,7 +87,9 @@ app.then(registerHandlers).then(registerEvents).then(launch)+ .then(setupRecording).then(createApplicationMenu)+ .then(getTrayState).then(registerUpdater).catch(e => console.error('Failed create window:', e));diff --git a/packages/frontend/apps/electron/src/main/recording/index.ts b/packages/frontend/apps/electron/src/main/recording/index.tsindex abcc9bd24a..ed9dde168a 100644--- a/packages/frontend/apps/electron/src/main/recording/index.ts+++ b/packages/frontend/apps/electron/src/main/recording/index.ts@@ -1,38 +1,30 @@-import { ShareableContent, TappableApplication } from '@affine/native';-import { Notification } from 'electron';-import {- BehaviorSubject,- distinctUntilChanged,- pairwise,- startWith,-} from 'rxjs';+import { ShareableContent } from '@affine/native';+import { nativeImage, Notification } from 'electron';+import { debounce } from 'lodash-es';+import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs';import { isMacOS } from '../../shared/utils';import { beforeAppQuit } from '../cleanup';+import { ensureHelperProcess } from '../helper-process';import { logger } from '../logger';--interface TappableAppInfo {- rawInstance: TappableApplication;- isRunning: boolean;- processId: number;- processGroupId: number;- bundleIdentifier: string;- name: string;-}--interface AppGroupInfo {- processGroupId: number;- apps: TappableAppInfo[];- name: string;- icon: Buffer | undefined;- isRunning: boolean;-}+import type { NamespaceHandlers } from '../type';+import { getMainWindow } from '../windows-manager';+import type {+ AppGroupInfo,+ Recording,+ RecordingStatus,+ TappableAppInfo,+} from './types';const subscribers: Subscriber[] = [];beforeAppQuit(() => {subscribers.forEach(subscriber => {- subscriber.unsubscribe();+ try {+ subscriber.unsubscribe();+ } catch {+ // ignore unsubscribe error+ }});});@@ -41,8 +33,41 @@ let shareableContent: ShareableContent | null = null;export const applications$ = new BehaviorSubject<TappableAppInfo[]>([]);export const appGroups$ = new BehaviorSubject<AppGroupInfo[]>([]);-if (isMacOS()) {- // Update appGroups$ whenever applications$ changes+// recording id -> recording+// recordings will be saved in memory before consumed and created as an audio block to user's doc+const recordings = new Map<number, Recording>();++// there should be only one active recording at a time+export const recordingStatus$ = new BehaviorSubject<RecordingStatus | null>(+ null+);++function createAppGroup(processGroupId: number): AppGroupInfo | undefined {+ const groupProcess =+ shareableContent?.applicationWithProcessId(processGroupId);+ if (!groupProcess) {+ return;+ }+ return {+ processGroupId: processGroupId,+ apps: [], // leave it empty for now.+ name: groupProcess.name,+ bundleIdentifier: groupProcess.bundleIdentifier,+ // icon should be lazy loaded+ get icon() {+ try {+ return groupProcess.icon;+ } catch (error) {+ logger.error(`Failed to get icon for ${groupProcess.name}`, error);+ return undefined;+ }+ },+ isRunning: false,+ };+}++// pipe applications$ to appGroups$+function setupAppGroups() {subscribers.push(applications$.pipe(distinctUntilChanged()).subscribe(apps => {const appGroups: AppGroupInfo[] = [];@@ -52,69 +77,160 @@ if (isMacOS()) {);if (!appGroup) {- const groupProcess = shareableContent?.applicationWithProcessId(- app.processGroupId- );- if (!groupProcess) {- return;+ appGroup = createAppGroup(app.processGroupId);+ if (appGroup) {+ appGroups.push(appGroup);}- appGroup = {- processGroupId: app.processGroupId,- apps: [],- name: groupProcess.name,- // icon will be lazy loaded- get icon() {- try {- return groupProcess.icon;- } catch (error) {- logger.error(- `Failed to get icon for ${groupProcess.name}`,- error- );- return undefined;- }- },- get isRunning() {- return this.apps.some(app => app.rawInstance.isRunning);- },- };- appGroups.push(appGroup);}if (appGroup) {appGroup.apps.push(app);}});++ appGroups.forEach(appGroup => {+ appGroup.isRunning = appGroup.apps.some(app => app.isRunning);+ });+appGroups$.next(appGroups);}));+}+function setupNewRunningAppGroup() {+ const appGroupRunningChanged$ = appGroups$.pipe(+ mergeMap(groups => groups),+ groupBy(group => group.processGroupId),+ mergeMap(groupStream$ =>+ groupStream$.pipe(+ distinctUntilChanged((prev, curr) => prev.isRunning === curr.isRunning)+ )+ )+ );subscribers.push(- appGroups$- .pipe(startWith([] as AppGroupInfo[]), pairwise())- .subscribe(([previousGroups, currentGroups]) => {- currentGroups.forEach(currentGroup => {- const previousGroup = previousGroups.find(- group => group.processGroupId === currentGroup.processGroupId- );- if (previousGroup?.isRunning !== currentGroup.isRunning) {- console.log(- 'appgroup running changed',- currentGroup.name,- currentGroup.isRunning- );- if (currentGroup.isRunning) {- new Notification({- title: 'Recording Meeting',- body: `Recording meeting with ${currentGroup.name}`,- }).show();- }- }+ appGroupRunningChanged$.subscribe(currentGroup => {+ if (currentGroup.isRunning) {+ // TODO(@pengx17): stub impl. will be replaced with a real one later+ const notification = new Notification({+ icon: currentGroup.icon+ ? nativeImage.createFromBuffer(currentGroup.icon)+ : undefined,+ title: 'Recording Meeting',+ body: `Recording meeting with ${currentGroup.name}`,+ actions: [+ {+ type: 'button',+ text: 'Start',+ },+ ],});- })+ notification.on('action', () => {+ startRecording(currentGroup);+ });+ notification.show();+ } else {+ // if the group is not running, we should stop the recording (if it is recording)+ if (+ recordingStatus$.value?.status === 'recording' &&+ recordingStatus$.value?.appGroup?.processGroupId ===+ currentGroup.processGroupId+ ) {+ stopRecording();+ }+ }+ }));}-async function getAllApps(): Promise<TappableAppInfo[]> {+function createRecording(status: RecordingStatus) {+ const buffers: Float32Array[] = [];++ function tapAudioSamples(err: Error | null, samples: Float32Array) {+ const recordingStatus = recordingStatus$.getValue();+ if (+ !recordingStatus ||+ recordingStatus.id !== status.id ||+ recordingStatus.status === 'paused'+ ) {+ return;+ }++ if (err) {+ logger.error('failed to get audio samples', err);+ } else {+ buffers.push(new Float32Array(samples));+ }+ }++ const stream = status.app+ ? status.app.rawInstance.tapAudio(tapAudioSamples)+ : ShareableContent.tapGlobalAudio(null, tapAudioSamples);++ const recording: Recording = {+ id: status.id,+ startTime: status.startTime,+ app: status.app,+ appGroup: status.appGroup,+ buffers,+ stream,+ };++ return recording;+}++function concatBuffers(buffers: Float32Array[]): Float32Array {+ const totalSamples = buffers.reduce((acc, buf) => acc + buf.length, 0);+ const buffer = new Float32Array(totalSamples);+ let offset = 0;+ buffers.forEach(buf => {+ buffer.set(buf, offset);+ offset += buf.length;+ });+ return buffer;+}++export async function saveRecording(id: number) {+ const recording = recordings.get(id);+ if (!recording) {+ logger.error(`Recording ${id} not found`);+ return;+ }+ const { buffers } = recording;+ const helperProcessManager = await ensureHelperProcess();+ const buffer = concatBuffers(buffers);+ const mp3Buffer = await helperProcessManager.rpc?.encodeToMp3(buffer, {+ channels: recording.stream.channels,+ sampleRate: recording.stream.sampleRate,+ });++ if (!mp3Buffer) {+ logger.error('failed to encode audio samples to mp3');+ return;+ }+ recordings.delete(recording.id);+ return mp3Buffer;+}++function setupRecordingListeners() {+ subscribers.push(+ recordingStatus$.pipe(distinctUntilChanged()).subscribe(status => {+ if (status?.status === 'recording') {+ let recording = recordings.get(status.id);+ // create a recording if not exists+ if (!recording) {+ recording = createRecording(status);+ recordings.set(status.id, recording);+ }+ } else if (status?.status === 'stopped') {+ const recording = recordings.get(status.id);+ if (recording) {+ recording.stream.stop();+ }+ }+ })+ );+}++function getAllApps(): TappableAppInfo[] {if (!shareableContent) {return [];}@@ -150,62 +266,37 @@ type Subscriber = {};function setupMediaListeners() {+ applications$.next(getAllApps());subscribers.push(ShareableContent.onApplicationListChanged(() => {- getAllApps()- .then(apps => {- applications$.next(apps);- })- .catch(err => {- logger.error('failed to get apps', err);- });+ applications$.next(getAllApps());}));- getAllApps()- .then(apps => {- applications$.next(apps);- })- .catch(err => {- logger.error('failed to get apps', err);- });-let appStateSubscribers: Subscriber[] = [];subscribers.push(applications$.subscribe(apps => {appStateSubscribers.forEach(subscriber => {- subscriber.unsubscribe();+ try {+ subscriber.unsubscribe();+ } catch {+ // ignore unsubscribe error+ }});const _appStateSubscribers: Subscriber[] = [];apps.forEach(app => {try {- // Try to create a TappableApplication with a default audio object ID- // In a real implementation, you would need to get the actual audio object ID- // This is just a placeholder value that seems to work for testing- const tappableApp = TappableApplication.fromApplication(- app.rawInstance,- 1+ const tappableApp = app.rawInstance;+ const debouncedAppStateChanged = debounce(() => {+ applications$.next(getAllApps());+ }, 100);+ _appStateSubscribers.push(+ ShareableContent.onAppStateChanged(tappableApp, () => {+ debouncedAppStateChanged();+ }));-- if (tappableApp) {- _appStateSubscribers.push(- ShareableContent.onAppStateChanged(tappableApp, () => {- setTimeout(() => {- const apps = applications$.getValue();- applications$.next(- apps.map(_app => {- if (_app.processId === app.processId) {- return { ..._app, isRunning: tappableApp.isRunning };- }- return _app;- })- );- }, 10);- })- );- }} catch (error) {logger.error(`Failed to convert app ${app.name} to TappableApplication`,@@ -217,15 +308,23 @@ function setupMediaListeners() {appStateSubscribers = _appStateSubscribers;return () => {_appStateSubscribers.forEach(subscriber => {- subscriber.unsubscribe();+ try {+ subscriber.unsubscribe();+ } catch {+ // ignore unsubscribe error+ }});};}));}-export function getShareableContent() {- if (!shareableContent && isMacOS()) {+export function setupRecording() {+ if (!isMacOS()) {+ return;+ }++ if (!shareableContent) {try {shareableContent = new ShareableContent();setupMediaListeners();@@ -233,5 +332,99 @@ export function getShareableContent() {logger.error('failed to get shareable content', error);}}- return shareableContent;+ setupAppGroups();+ setupNewRunningAppGroup();+ setupRecordingListeners();+}++let recordingId = 0;++export function startRecording(+ appGroup?: AppGroupInfo+): RecordingStatus | undefined {+ if (!shareableContent) {+ return; // likely called on unsupported platform+ }++ // hmm, is it possible that there are multiple apps running (listening) in the same group?+ const appInfo = appGroup?.apps.find(app => app.isRunning);++ const recordingStatus: RecordingStatus = {+ id: recordingId++,+ status: 'recording',+ startTime: Date.now(),+ app: appInfo,+ appGroup,+ };++ recordingStatus$.next(recordingStatus);++ return recordingStatus;+}++export function pauseRecording() {+ const recordingStatus = recordingStatus$.value;+ if (!recordingStatus) {+ return;+ }++ recordingStatus$.next({+ ...recordingStatus,+ status: 'paused',+ });+}++export function resumeRecording() {+ const recordingStatus = recordingStatus$.value;+ if (!recordingStatus) {+ return;+ }++ recordingStatus$.next({+ ...recordingStatus,+ status: 'recording',+ });+}++export function stopRecording() {+ const recordingStatus = recordingStatus$.value;+ if (!recordingStatus) {+ return;+ }++ // do not remove the last recordingStatus from recordingStatus$+ recordingStatus$.next({+ ...recordingStatus,+ status: 'stopped',+ });++ // bring up the window+ getMainWindow()+ .then(mainWindow => {+ if (mainWindow) {+ mainWindow.show();+ }+ })+ .catch(err => {+ logger.error('failed to bring up the window', err);+ });}++export const recordingHandlers = {+ saveRecording: async (_, id: number) => {+ return saveRecording(id);+ },+} satisfies NamespaceHandlers;++export const recordingEvents = {+ onRecordingStatusChanged: (fn: (status: RecordingStatus | null) => void) => {+ const sub = recordingStatus$.subscribe(fn);+ return () => {+ try {+ sub.unsubscribe();+ } catch {+ // ignore unsubscribe error+ }+ };+ },+};diff --git a/packages/frontend/apps/electron/src/main/recording/types.ts b/packages/frontend/apps/electron/src/main/recording/types.tsnew file mode 100644index 0000000000..d308b5b11b--- /dev/null+++ b/packages/frontend/apps/electron/src/main/recording/types.ts@@ -0,0 +1,38 @@+import type { AudioTapStream, TappableApplication } from '@affine/native';++export interface TappableAppInfo {+ rawInstance: TappableApplication;+ isRunning: boolean;+ processId: number;+ processGroupId: number;+ bundleIdentifier: string;+ name: string;+}++export interface AppGroupInfo {+ processGroupId: number;+ apps: TappableAppInfo[];+ name: string;+ bundleIdentifier: string;+ icon: Buffer | undefined;+ isRunning: boolean;+}++export interface Recording {+ id: number;+ // the app may not be available if the user choose to record system audio+ app?: TappableAppInfo;+ appGroup?: AppGroupInfo;+ // the raw audio buffers that are already accumulated+ buffers: Float32Array[];+ stream: AudioTapStream;+ startTime: number;+}++export interface RecordingStatus {+ id: number; // corresponds to the recording id+ status: 'recording' | 'paused' | 'stopped';+ app?: TappableAppInfo;+ appGroup?: AppGroupInfo;+ startTime: number;+}diff --git a/packages/frontend/apps/electron/src/main/tray/icons.ts b/packages/frontend/apps/electron/src/main/tray/icons.tsnew file mode 100644index 0000000000..26d7a962ad--- /dev/null+++ b/packages/frontend/apps/electron/src/main/tray/icons.ts@@ -0,0 +1,13 @@+import { join } from 'node:path';++import { resourcesPath } from '../../shared/utils';++export const icons = {+ record: join(resourcesPath, 'icons/waveform.png'),+ recording: join(resourcesPath, 'icons/waveform-recording.png'),+ tray: join(resourcesPath, 'icons/tray-icon.png'),+ journal: join(resourcesPath, 'icons/journal-today.png'),+ page: join(resourcesPath, 'icons/doc-page.png'),+ edgeless: join(resourcesPath, 'icons/doc-edgeless.png'),+ monitor: join(resourcesPath, 'icons/monitor.png'),+};diff --git a/packages/frontend/apps/electron/src/main/tray/index.ts b/packages/frontend/apps/electron/src/main/tray/index.tsindex 249c441b7b..a6d8b21d0b 100644--- a/packages/frontend/apps/electron/src/main/tray/index.ts+++ b/packages/frontend/apps/electron/src/main/tray/index.ts@@ -1,25 +1,34 @@-import { join } from 'node:path';-import {app,Menu,MenuItem,+ type MenuItemConstructorOptions,type NativeImage,nativeImage,Tray,} from 'electron';-import { isMacOS, resourcesPath } from '../../shared/utils';+import { isMacOS } from '../../shared/utils';import { applicationMenuSubjects } from '../application-menu';import { beforeAppQuit } from '../cleanup';-import { appGroups$ } from '../recording';+import { logger } from '../logger';+import {+ appGroups$,+ pauseRecording,+ recordingStatus$,+ resumeRecording,+ startRecording,+ stopRecording,+} from '../recording';import { getMainWindow } from '../windows-manager';+import { icons } from './icons';export interface TrayMenuConfigItem {label: string;click?: () => void;icon?: NativeImage | string | Buffer;disabled?: boolean;+ submenu?: TrayMenuConfig;}export type TrayMenuConfig = Array<TrayMenuConfigItem | 'separator'>;@@ -35,7 +44,35 @@ function showMainWindow() {.then(w => {w.show();})- .catch(err => console.error(err));+ .catch(err => logger.error('Failed to show main window:', err));+}++function buildMenuConfig(config: TrayMenuConfig): MenuItemConstructorOptions[] {+ const menuConfig: MenuItemConstructorOptions[] = [];+ config.forEach(item => {+ if (item === 'separator') {+ menuConfig.push({ type: 'separator' });+ } else {+ const { icon, disabled, submenu, ...rest } = item;+ let nativeIcon: NativeImage | undefined;+ if (typeof icon === 'string') {+ nativeIcon = nativeImage.createFromPath(icon);+ } else if (Buffer.isBuffer(icon)) {+ nativeIcon = nativeImage.createFromBuffer(icon);+ }+ if (nativeIcon) {+ nativeIcon = nativeIcon.resize({ width: 20, height: 20 });+ }+ const submenuConfig = submenu ? buildMenuConfig(submenu) : undefined;+ menuConfig.push({+ ...rest,+ enabled: !disabled,+ icon: nativeIcon,+ submenu: submenuConfig,+ });+ }+ });+ return menuConfig;}class TrayState {@@ -43,7 +80,7 @@ class TrayState {// tray's iconicon: NativeImage = nativeImage- .createFromPath(join(resourcesPath, 'icons/tray-icon.png'))+ .createFromPath(icons.tray).resize({ width: 16, height: 16 });// tray's tooltip@@ -60,24 +97,27 @@ class TrayState {getConfig: () => [{label: 'Open Journal',- icon: join(resourcesPath, 'icons/journal-today.png'),+ icon: icons.journal,click: () => {+ logger.info('User action: Open Journal');showMainWindow();applicationMenuSubjects.openJournal$.next();},},{label: 'New Page',- icon: join(resourcesPath, 'icons/doc-page.png'),+ icon: icons.page,click: () => {+ logger.info('User action: New Page');showMainWindow();applicationMenuSubjects.newPageAction$.next('page');},},{label: 'New Edgeless',- icon: join(resourcesPath, 'icons/doc-edgeless.png'),+ icon: icons.edgeless,click: () => {+ logger.info('User action: New Edgeless');showMainWindow();applicationMenuSubjects.newPageAction$.next('edgeless');},@@ -89,20 +129,78 @@ class TrayState {getRecordingMenuProvider(): TrayMenuProvider {const appGroups = appGroups$.value;const runningAppGroups = appGroups.filter(appGroup => appGroup.isRunning);++ const recordingStatus = recordingStatus$.value;++ if (!recordingStatus || recordingStatus?.status === 'stopped') {+ return {+ key: 'recording',+ getConfig: () => [+ {+ label: 'Start Recording Meeting',+ icon: icons.record,+ submenu: [+ {+ label: 'System audio (all audio will be recorded)',+ icon: icons.monitor,+ click: () => {+ logger.info(+ 'User action: Start Recording Meeting (System audio)'+ );+ startRecording();+ },+ },+ ...runningAppGroups.map(appGroup => ({+ label: appGroup.name,+ icon: appGroup.icon || undefined,+ click: () => {+ logger.info(+ `User action: Start Recording Meeting (${appGroup.name})`+ );+ startRecording(appGroup);+ },+ })),+ ],+ },+ ],+ };+ }++ const recordingLabel = recordingStatus.appGroup?.name+ ? `Recording (${recordingStatus.appGroup?.name})`+ : 'Recording';++ // recording is either started or pausedreturn {key: 'recording',getConfig: () => [{- label: 'Start Recording Meeting',+ label: recordingLabel,+ icon: icons.recording,disabled: true,},- ...runningAppGroups.map(appGroup => ({- label: appGroup.name,- icon: appGroup.icon || undefined,+ recordingStatus.status === 'paused'+ ? {+ label: 'Resume',+ click: () => {+ logger.info('User action: Resume Recording');+ resumeRecording();+ },+ }+ : {+ label: 'Pause',+ click: () => {+ logger.info('User action: Pause Recording');+ pauseRecording();+ },+ },+ {+ label: 'Stop',click: () => {- console.log(appGroup);+ logger.info('User action: Stop Recording');+ stopRecording();},- })),+ },],};}@@ -114,12 +212,13 @@ class TrayState {{label: 'Open AFFiNE',click: () => {+ logger.info('User action: Open AFFiNE');getMainWindow().then(w => {w.show();}).catch(err => {- console.error(err);+ logger.error('Failed to open AFFiNE:', err);});},},@@ -127,6 +226,7 @@ class TrayState {{label: 'Quit AFFiNE Completely...',click: () => {+ logger.info('User action: Quit AFFiNE Completely');app.quit();},},@@ -137,32 +237,9 @@ class TrayState {buildMenu(providers: TrayMenuProvider[]) {const menu = new Menu();providers.forEach((provider, index) => {- provider.getConfig().forEach(item => {- if (item === 'separator') {- menu.append(new MenuItem({ type: 'separator' }));- } else {- const { icon, disabled, ...rest } = item;- let nativeIcon: NativeImage | undefined;- if (typeof icon === 'string') {- nativeIcon = nativeImage.createFromPath(icon);- } else if (Buffer.isBuffer(icon)) {- try {- nativeIcon = nativeImage.createFromBuffer(icon);- } catch (error) {- console.error('Failed to create icon from buffer', error);- }- }- if (nativeIcon) {- nativeIcon = nativeIcon.resize({ width: 20, height: 20 });- }- menu.append(- new MenuItem({- ...rest,- enabled: !disabled,- icon: nativeIcon,- })- );- }+ const config = provider.getConfig();+ buildMenuConfig(config).forEach(item => {+ menu.append(new MenuItem(item));});if (index !== providers.length - 1) {menu.append(new MenuItem({ type: 'separator' }));@@ -176,15 +253,22 @@ class TrayState {this.tray = new Tray(this.icon);this.tray.setToolTip(this.tooltip);const clickHandler = () => {+ logger.debug('User clicked on tray icon');this.update();if (!isMacOS()) {this.tray?.popUpContextMenu();}};this.tray.on('click', clickHandler);+ const appGroupsSubscription = appGroups$.subscribe(() => {+ logger.debug('App groups updated, refreshing tray menu');+ this.update();+ });beforeAppQuit(() => {+ logger.info('Cleaning up tray before app quit');this.tray?.off('click', clickHandler);this.tray?.destroy();+ appGroupsSubscription.unsubscribe();});}@@ -199,6 +283,7 @@ class TrayState {}init() {+ logger.info('Initializing tray');this.update();}}diff --git a/packages/frontend/apps/electron/src/shared/type.ts b/packages/frontend/apps/electron/src/shared/type.tsindex 231ee3baf1..900a561035 100644--- a/packages/frontend/apps/electron/src/shared/type.ts+++ b/packages/frontend/apps/electron/src/shared/type.ts@@ -17,6 +17,13 @@ export interface HelperToRenderer {// helper <-> mainexport interface HelperToMain {getMeta: () => ExposedMeta;+ encodeToMp3: (+ samples: Float32Array,+ opts?: {+ channels?: number;+ sampleRate?: number;+ }+ ) => Uint8Array;}export type MainToHelper = Pick<diff --git a/packages/frontend/core/src/blocksuite/initialization/index.ts b/packages/frontend/core/src/blocksuite/initialization/index.tsindex 85a4a8f969..6e79c3b753 100644--- a/packages/frontend/core/src/blocksuite/initialization/index.ts+++ b/packages/frontend/core/src/blocksuite/initialization/index.ts@@ -12,6 +12,14 @@ export interface DocProps {surface?: Partial<SurfaceBlockProps>;note?: Partial<NoteProps>;paragraph?: Partial<ParagraphProps>;+ onStoreLoad?: (+ doc: Store,+ props: {+ noteId: string;+ paragraphId: string;+ surfaceId: string;+ }+ ) => void;}export function initDocFromProps(doc: Store, props?: DocProps) {@@ -20,7 +28,11 @@ export function initDocFromProps(doc: Store, props?: DocProps) {'affine:page',props?.page || { title: new Text('') });- doc.addBlock('affine:surface' as never, props?.surface || {}, pageBlockId);+ const surfaceId = doc.addBlock(+ 'affine:surface' as never,+ props?.surface || {},+ pageBlockId+ );const noteBlockId = doc.addBlock('affine:note',{@@ -29,7 +41,16 @@ export function initDocFromProps(doc: Store, props?: DocProps) {},pageBlockId);- doc.addBlock('affine:paragraph', props?.paragraph || {}, noteBlockId);+ const paragraphBlockId = doc.addBlock(+ 'affine:paragraph',+ props?.paragraph || {},+ noteBlockId+ );+ props?.onStoreLoad?.(doc, {+ noteId: noteBlockId,+ paragraphId: paragraphBlockId,+ surfaceId,+ });doc.history.clear();});}diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.tsindex ebffc0b208..a22d6b8494 100644--- a/packages/frontend/media-capture-playground/server/main.ts+++ b/packages/frontend/media-capture-playground/server/main.ts@@ -28,12 +28,13 @@ console.log(`📁 Ensuring recordings directory exists at ${RECORDING_DIR}`);// Typesinterface Recording {- app: TappableApplication;+ app: TappableApplication | null;appGroup: Application | null;buffers: Float32Array[];stream: AudioTapStream;startTime: number;isWriting: boolean;+ isGlobal?: boolean;}interface RecordingStatus {@@ -54,6 +55,7 @@ interface RecordingMetadata {sampleRate: number;channels: number;totalSamples: number;+ isGlobal?: boolean;}interface AppInfo {@@ -118,7 +120,7 @@ app.use(async function saveRecording(recording: Recording): Promise<string | null> {try {recording.isWriting = true;- const app = recording.appGroup || recording.app;+ const app = recording.isGlobal ? null : recording.appGroup || recording.app;const totalSamples = recording.buffers.reduce((acc, buf) => acc + buf.length,@@ -133,9 +135,19 @@ async function saveRecording(recording: Recording): Promise<string | null> {const channelCount = recording.stream.channels;const expectedSamples = recordingDuration * actualSampleRate;- console.log(`💾 Saving recording for ${app.name}:`);- console.log(`- Process ID: ${app.processId}`);- console.log(`- Bundle ID: ${app.bundleIdentifier}`);+ if (recording.isGlobal) {+ console.log('💾 Saving global recording:');+ } else {+ const appName = app?.name ?? 'Unknown App';+ const processId = app?.processId ?? 0;+ const bundleId = app?.bundleIdentifier ?? 'unknown';+ console.log(`💾 Saving recording for ${appName}:`);+ if (app) {+ console.log(`- Process ID: ${processId}`);+ console.log(`- Bundle ID: ${bundleId}`);+ }+ }+console.log(`- Actual duration: ${recordingDuration.toFixed(2)}s`);console.log(`- Sample rate: ${actualSampleRate}Hz`);console.log(`- Channels: ${channelCount}`);@@ -156,7 +168,9 @@ async function saveRecording(recording: Recording): Promise<string | null> {await fs.ensureDir(RECORDING_DIR);const timestamp = Date.now();- const baseFilename = `${recording.app.bundleIdentifier}-${recording.app.processId}-${timestamp}`;+ const baseFilename = recording.isGlobal+ ? `global-recording-${timestamp}`+ : `${app?.bundleIdentifier ?? 'unknown'}-${app?.processId ?? 0}-${timestamp}`;const recordingDir = `${RECORDING_DIR}/${baseFilename}`;await fs.ensureDir(recordingDir);@@ -189,7 +203,7 @@ async function saveRecording(recording: Recording): Promise<string | null> {console.log('✅ Transcription MP3 file written successfully');// Save app icon if available- if (app.icon) {+ if (app?.icon) {console.log(`📝 Writing app icon to ${iconFilename}`);await fs.writeFile(iconFilename, app.icon);console.log('✅ App icon written successfully');@@ -198,15 +212,16 @@ async function saveRecording(recording: Recording): Promise<string | null> {console.log(`📝 Writing metadata to ${metadataFilename}`);// Save metadata with the actual sample rate from the streamconst metadata: RecordingMetadata = {- appName: app.name,- bundleIdentifier: app.bundleIdentifier,- processId: app.processId,+ appName: app?.name ?? 'Global Recording',+ bundleIdentifier: app?.bundleIdentifier ?? 'system.global',+ processId: app?.processId ?? -1,recordingStartTime: recording.startTime,recordingEndTime,recordingDuration,sampleRate: actualSampleRate,channels: channelCount,totalSamples,+ isGlobal: recording.isGlobal,};await fs.writeJson(metadataFilename, metadata, { spaces: 2 });@@ -222,8 +237,8 @@ async function saveRecording(recording: Recording): Promise<string | null> {function getRecordingStatus(): RecordingStatus[] {return Array.from(recordingMap.entries()).map(([processId, recording]) => ({processId,- bundleIdentifier: recording.app.bundleIdentifier,- name: recording.app.name,+ bundleIdentifier: recording.app?.bundleIdentifier ?? 'system.global',+ name: recording.app?.name ?? 'Global Recording',startTime: recording.startTime,duration: Date.now() - recording.startTime,}));@@ -289,8 +304,11 @@ async function stopRecording(processId: number) {}const app = recording.appGroup || recording.app;+ const appName =+ app?.name ?? (recording.isGlobal ? 'Global Recording' : 'Unknown App');+ const appPid = app?.processId ?? processId;- console.log(`⏹️ Stopping recording for ${app.name} (PID: ${app.processId})`);+ console.log(`⏹️ Stopping recording for ${appName} (PID: ${appPid})`);console.log(`⏱️ Recording duration: ${((Date.now() - recording.startTime) / 1000).toFixed(2)}s`);@@ -302,7 +320,7 @@ async function stopRecording(processId: number) {if (filename) {console.log(`✅ Recording saved successfully to ${filename}`);} else {- console.error(`❌ Failed to save recording for ${app.name}`);+ console.error(`❌ Failed to save recording for ${appName}`);}emitRecordingStatus();@@ -541,7 +559,13 @@ function listenToAppStateChanges(apps: AppInfo[]) {appsSubscriber();appsSubscriber = () => {- subscribers.forEach(subscriber => subscriber.unsubscribe());+ subscribers.forEach(subscriber => {+ try {+ subscriber.unsubscribe();+ } catch {+ // ignore unsubscribe error+ }+ });};}@@ -606,8 +630,8 @@ app.get('/apps/saved', rateLimiter, async (_req, res) => {// Utility function to validate and sanitize folder namefunction validateAndSanitizeFolderName(folderName: string): string | null {// Allow alphanumeric characters, hyphens, dots (for bundle IDs)- // Format: bundleId-processId-timestamp- if (!/^[\w.-]+-\d+-\d+$/.test(folderName)) {+ // Format: bundleId-processId-timestamp OR global-recording-timestamp+ if (!/^([\w.-]+-\d+-\d+|global-recording-\d+)$/.test(folderName)) {return null;}@@ -788,6 +812,65 @@ app.post(});+async function startGlobalRecording() {+ const GLOBAL_RECORDING_ID = -1;+ if (recordingMap.has(GLOBAL_RECORDING_ID)) {+ console.log('⚠️ Global recording already in progress');+ return;+ }++ try {+ console.log('🎙️ Starting global recording');++ const buffers: Float32Array[] = [];+ const stream = ShareableContent.tapGlobalAudio(+ null,+ (err: Error | null, samples: Float32Array) => {+ if (err) {+ console.error('❌ Global audio stream error:', err);+ return;+ }+ const recording = recordingMap.get(GLOBAL_RECORDING_ID);+ if (recording && !recording.isWriting) {+ buffers.push(new Float32Array(samples));+ }+ }+ );++ recordingMap.set(GLOBAL_RECORDING_ID, {+ app: null,+ appGroup: null,+ buffers,+ stream,+ startTime: Date.now(),+ isWriting: false,+ isGlobal: true,+ });++ console.log('✅ Global recording started successfully');+ emitRecordingStatus();+ } catch (error) {+ console.error('❌ Error starting global recording:', error);+ }+}++// Add API endpoint for global recording+app.post('/global/record', async (_req, res) => {+ try {+ await startGlobalRecording();+ res.json({ success: true });+ } catch (error) {+ console.error('❌ Error starting global recording:', error);+ res.status(500).json({ error: 'Failed to start global recording' });+ }+});++app.post('/global/stop', async (_req, res) => {+ const GLOBAL_RECORDING_ID = -1;+ await stopRecording(GLOBAL_RECORDING_ID);+ res.json({ success: true });+});+// Start serverhttpServer.listen(PORT, () => {console.log(`diff --git a/packages/frontend/media-capture-playground/web/app.tsx b/packages/frontend/media-capture-playground/web/app.tsxindex a3aaa7ff4d..7b8987f5e6 100644--- a/packages/frontend/media-capture-playground/web/app.tsx+++ b/packages/frontend/media-capture-playground/web/app.tsx@@ -1,4 +1,5 @@import { AppList } from './components/app-list';+import { GlobalRecordButton } from './components/global-record-button';import { SavedRecordings } from './components/saved-recordings';export function App() {@@ -6,11 +7,15 @@ export function App() {<div className="h-screen bg-gray-50 overflow-hidden"><div className="h-full p-4 flex gap-4 max-w-[1800px] mx-auto"><div className="flex-1 flex flex-col min-h-0">- <h1 className="text-xl font-bold text-gray-900 mb-1">- Running Applications- </h1>+ <div className="flex justify-between items-center mb-1">+ <h1 className="text-xl font-bold text-gray-900">+ Running Applications+ </h1>+ <GlobalRecordButton />+ </div><p className="text-sm text-gray-500 mb-2">- Select an application to start recording its audio+ Select an application to start recording its audio, or use global+ recording for system-wide audio</p><div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 overflow-auto"><AppList />diff --git a/packages/frontend/media-capture-playground/web/components/global-record-button.tsx b/packages/frontend/media-capture-playground/web/components/global-record-button.tsxnew file mode 100644index 0000000000..6884b8b010--- /dev/null+++ b/packages/frontend/media-capture-playground/web/components/global-record-button.tsx@@ -0,0 +1,70 @@+import { useCallback, useEffect, useState } from 'react';++import { socket } from '../utils';++export function GlobalRecordButton() {+ const [isRecording, setIsRecording] = useState(false);+ const [isLoading, setIsLoading] = useState(false);++ useEffect(() => {+ function handleRecordingStatus(data: {+ recordings: Array<{ processId: number }>;+ }) {+ // Global recording uses processId -1+ setIsRecording(data.recordings.some(r => r.processId === -1));+ }++ socket.on('apps:recording', handleRecordingStatus);+ return () => {+ socket.off('apps:recording', handleRecordingStatus);+ };+ }, []);++ const handleClick = useCallback(() => {+ setIsLoading(true);+ const endpoint = isRecording ? '/api/global/stop' : '/api/global/record';+ fetch(endpoint, { method: 'POST' })+ .then(response => {+ if (!response.ok) {+ throw new Error('Failed to toggle global recording');+ }+ })+ .catch(error => {+ console.error('Error toggling global recording:', error);+ })+ .finally(() => {+ setIsLoading(false);+ });+ }, [isRecording]);++ return (+ <button+ onClick={handleClick}+ disabled={isLoading}+ className={`+ px-4 py-2 rounded-lg font-medium text-sm+ transition-colors duration-200+ ${isLoading ? 'opacity-50 cursor-not-allowed' : ''}+ ${+ isRecording+ ? 'bg-red-100 text-red-700 hover:bg-red-200'+ : 'bg-blue-100 text-blue-700 hover:bg-blue-200'+ }+ `}+ >+ <div className="flex items-center gap-2">+ {isRecording ? (+ <>+ <div className="w-2 h-2 rounded-full bg-red-500 animate-pulse" />+ Stop Global Recording+ </>+ ) : (+ <>+ <div className="w-2 h-2 rounded-full bg-blue-500" />+ Record System Audio+ </>+ )}+ </div>+ </button>+ );+}diff --git a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsxindex 217d300d49..61da0adfd4 100644--- a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx+++ b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx@@ -378,11 +378,12 @@ function RecordingHeader({transcriptionError: string | null;}): ReactElement {const [imgError, setImgError] = React.useState(false);+ const isGlobalRecording = metadata?.isGlobal;return (<div className="flex items-start space-x-4 p-4 bg-gray-50/30"><div className="relative w-12 h-12 flex-shrink-0">- {!imgError ? (+ {!imgError && !isGlobalRecording ? (<imgsrc={`/api/recordings/${fileName}/icon.png`}alt={metadata?.appName || 'Unknown Application'}@@ -401,7 +402,12 @@ function RecordingHeader({<span className="text-gray-900 font-semibold text-base truncate">{metadata?.appName || 'Unknown Application'}</span>- <span className="text-xs px-2 py-0.5 bg-blue-50 rounded-full text-blue-600 font-medium border border-blue-100">+ {isGlobalRecording && (+ <span className="text-xs px-2 py-0.5 bg-blue-50 rounded-full text-blue-600 font-medium border border-blue-100">+ System Audio+ </span>+ )}+ <span className="text-xs px-2 py-0.5 bg-gray-50 rounded-full text-gray-600 font-medium border border-gray-100">{duration}</span></div>diff --git a/packages/frontend/media-capture-playground/web/types.ts b/packages/frontend/media-capture-playground/web/types.tsindex ced4bd4e23..c892c419a0 100644--- a/packages/frontend/media-capture-playground/web/types.ts+++ b/packages/frontend/media-capture-playground/web/types.ts@@ -17,6 +17,8 @@ export interface RecordingStatus {bundleIdentifier: string;name: string;startTime: number;+ duration: number;+ isGlobal?: boolean;}export interface RecordingMetadata {@@ -31,6 +33,7 @@ export interface RecordingMetadata {totalSamples: number;icon?: Uint8Array;mp3: string;+ isGlobal?: boolean;}export interface TranscriptionMetadata {diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rsindex 1004b7dfef..cfce74c201 100644--- a/packages/frontend/native/media_capture/src/macos/tap_audio.rs+++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs@@ -1,4 +1,4 @@-use std::{ffi::c_void, sync::Arc};+use std::{ffi::c_void, ptr, sync::Arc};use block2::{Block, RcBlock};use core_foundation::{@@ -13,12 +13,15 @@ use coreaudio::sys::{kAudioAggregateDeviceIsPrivateKey, kAudioAggregateDeviceIsStackedKey,kAudioAggregateDeviceMainSubDeviceKey, kAudioAggregateDeviceNameKey,kAudioAggregateDeviceSubDeviceListKey, kAudioAggregateDeviceTapAutoStartKey,- kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioHardwareNoError,- kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultSystemOutputDevice,- kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey, kAudioSubTapUIDKey,- AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID,- AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice,- AudioHardwareDestroyAggregateDevice, AudioObjectID, AudioTimeStamp, OSStatus,+ kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey,+ kAudioDevicePropertyAvailableNominalSampleRates, kAudioDevicePropertyNominalSampleRate,+ kAudioHardwareNoError, kAudioHardwarePropertyDefaultInputDevice,+ kAudioHardwarePropertyDefaultSystemOutputDevice, kAudioObjectPropertyElementMain,+ kAudioObjectPropertyScopeGlobal, kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey,+ kAudioSubTapUIDKey, AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID,+ AudioDeviceIOProcID, AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice,+ AudioHardwareDestroyAggregateDevice, AudioObjectGetPropertyData, AudioObjectGetPropertyDataSize,+ AudioObjectID, AudioObjectPropertyAddress, AudioObjectSetPropertyData, AudioTimeStamp, OSStatus,};use napi::{bindgen_prelude::Float32Array,@@ -30,8 +33,11 @@ use objc2::{runtime::AnyObject, Encode, Encoding, RefEncode};use crate::{audio_stream_basic_desc::read_audio_stream_basic_description,- ca_tap_description::CATapDescription, device::get_device_uid, error::CoreAudioError,- queue::create_audio_tap_queue, screen_capture_kit::TappableApplication,+ ca_tap_description::CATapDescription,+ device::{get_device_audio_id, get_device_uid},+ error::CoreAudioError,+ queue::create_audio_tap_queue,+ screen_capture_kit::TappableApplication,};extern "C" {@@ -53,6 +59,14 @@ pub struct AudioBuffer {pub mData: *mut c_void,}+// Define a struct to represent sample rate ranges+#[repr(C)]+#[allow(non_snake_case)]+struct AudioValueRange {+ mMinimum: f64,+ mMaximum: f64,+}+unsafe impl Encode for AudioBuffer {const ENCODING: Encoding = Encoding::Struct("AudioBuffer",@@ -94,6 +108,10 @@ pub struct AggregateDevice {pub tap_id: AudioObjectID,pub id: AudioObjectID,pub audio_stats: Option<AudioStats>,+ pub input_device_id: Option<AudioObjectID>,+ pub output_device_id: Option<AudioObjectID>,+ pub input_proc_id: Option<AudioDeviceIOProcID>,+ pub output_proc_id: Option<AudioDeviceIOProcID>,}impl AggregateDevice {@@ -128,6 +146,10 @@ impl AggregateDevice {tap_id,id: aggregate_device_id,audio_stats: None,+ input_device_id: None,+ output_device_id: None,+ input_proc_id: None,+ output_proc_id: None,})}@@ -160,6 +182,10 @@ impl AggregateDevice {tap_id,id: aggregate_device_id,audio_stats: None,+ input_device_id: None,+ output_device_id: None,+ input_proc_id: None,+ output_proc_id: None,})}@@ -173,6 +199,12 @@ impl AggregateDevice {return Err(CoreAudioError::CreateProcessTapFailed(status).into());}+ // Get the default input device (microphone) UID and ID+ let input_device_id = get_device_audio_id(kAudioHardwarePropertyDefaultInputDevice)?;++ // Get the default output device ID+ let output_device_id = get_device_audio_id(kAudioHardwarePropertyDefaultSystemOutputDevice)?;+let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?;let mut aggregate_device_id: AudioObjectID = 0;@@ -189,30 +221,246 @@ impl AggregateDevice {return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());}- Ok(Self {+ // Create a device with stored device IDs+ let mut device = Self {tap_id,id: aggregate_device_id,audio_stats: None,- })+ input_device_id: Some(input_device_id),+ output_device_id: Some(output_device_id),+ input_proc_id: None,+ output_proc_id: None,+ };++ // Configure the aggregate device to ensure proper handling of both input and+ // output+ device.configure_aggregate_device()?;++ // Activate both the input and output devices and store their proc IDs+ let input_proc_id = device.activate_audio_device(input_device_id)?;+ let output_proc_id = device.activate_audio_device(output_device_id)?;++ device.input_proc_id = Some(input_proc_id);+ device.output_proc_id = Some(output_proc_id);++ Ok(device)}- pub fn start(- &mut self,- audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,- ) -> Result<AudioTapStream> {- // Read and log the audio format before starting the device+ // Configures the aggregate device to ensure proper handling of both input and+ // output streams+ fn configure_aggregate_device(&self) -> Result<AudioStats> {+ // Read the current audio format to ensure it's properly configured+ let audio_format = read_audio_stream_basic_description(self.tap_id)?;++ // Create initial audio stats with the actual sample rate but always use mono+ let initial_sample_rate = audio_format.0.mSampleRate;let mut audio_stats = AudioStats {- sample_rate: 44100.0,+ sample_rate: initial_sample_rate,channels: 1, // Always set to 1 channel (mono)};- if let Ok(audio_format) = read_audio_stream_basic_description(self.tap_id) {- // Store the audio format information- audio_stats.sample_rate = audio_format.0.mSampleRate;- // Always use 1 channel regardless of what the system reports- audio_stats.channels = 1;+ // Set the preferred sample rate on the device+ // This is similar to how Screen Capture Kit allows setting the sample rate+ let preferred_sample_rate = initial_sample_rate; // Use the device's current sample rate++ // First, check if the preferred sample rate is available+ let mut is_sample_rate_available = false;+ let mut best_available_rate = preferred_sample_rate; // Default to preferred rate++ unsafe {+ // Get the available sample rates+ let address = AudioObjectPropertyAddress {+ mSelector: kAudioDevicePropertyAvailableNominalSampleRates,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };++ // Get the size of the property data+ let mut data_size: u32 = 0;+ let status = AudioObjectGetPropertyDataSize(+ self.id,+ &address as *const AudioObjectPropertyAddress,+ 0,+ std::ptr::null(),+ &mut data_size as *mut u32,+ );++ if status == 0 && data_size > 0 {+ // Calculate how many ranges we have+ let range_count = data_size as usize / std::mem::size_of::<AudioValueRange>();++ // Allocate memory for the ranges+ let mut ranges: Vec<AudioValueRange> = Vec::with_capacity(range_count);+ ranges.set_len(range_count);++ // Get the available sample rates+ let status = AudioObjectGetPropertyData(+ self.id,+ &address as *const AudioObjectPropertyAddress,+ 0,+ std::ptr::null(),+ &mut data_size as *mut u32,+ ranges.as_mut_ptr() as *mut std::ffi::c_void,+ );++ if status == 0 {+ // Check if our preferred sample rate is within any of the available ranges+ for range in &ranges {+ if preferred_sample_rate >= range.mMinimum && preferred_sample_rate <= range.mMaximum {+ is_sample_rate_available = true;+ break;+ }+ }++ // If not available, find the best available rate+ if !is_sample_rate_available && !ranges.is_empty() {+ // Common preferred sample rates in order of preference+ let common_rates = [48000.0, 44100.0, 96000.0, 88200.0, 24000.0, 22050.0];+ let mut found_common_rate = false;++ // First try to find a common rate that's available+ for &rate in &common_rates {+ for range in &ranges {+ if rate >= range.mMinimum && rate <= range.mMaximum {+ best_available_rate = rate;+ found_common_rate = true;+ break;+ }+ }+ if found_common_rate {+ break;+ }+ }++ // If no common rate is available, use the highest available rate+ if !found_common_rate {+ // Find the highest available rate+ for range in &ranges {+ // Use the maximum of the range as our best available rate+ if range.mMaximum > best_available_rate {+ best_available_rate = range.mMaximum;+ }+ }+ }+ }+ }+ }+ }++ // Set the sample rate to either the preferred rate or the best available rate+ let sample_rate_to_set = if is_sample_rate_available {+ preferred_sample_rate+ } else {+ best_available_rate+ };++ let status = unsafe {+ // Note on scope usage:+ // We use kAudioObjectPropertyScopeGlobal here because it works reliably for+ // setting the nominal sample rate on the device. While+ // kAudioObjectPropertyScopeInput or kAudioObjectPropertyScopeOutput might+ // also work in some cases (as mentioned in the comments),+ // kAudioObjectPropertyScopeGlobal is the most consistent approach.+ //+ // The CoreAudio documentation doesn't explicitly specify which scope to use+ // with kAudioDevicePropertyNominalSampleRate, but in practice,+ // kAudioObjectPropertyScopeGlobal ensures the sample rate is set for the+ // entire device, affecting both input and output.+ let address = AudioObjectPropertyAddress {+ mSelector: kAudioDevicePropertyNominalSampleRate,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };++ // Set the sample rate property+ AudioObjectSetPropertyData(+ self.id,+ &address as *const AudioObjectPropertyAddress,+ 0,+ std::ptr::null(),+ std::mem::size_of::<f64>() as u32,+ &sample_rate_to_set as *const f64 as *const std::ffi::c_void,+ )+ };++ // Update the audio_stats with the actual sample rate that was set if successful+ if status == 0 {+ audio_stats.sample_rate = sample_rate_to_set;++ // Verify the actual sample rate by reading it back+ unsafe {+ let address = AudioObjectPropertyAddress {+ mSelector: kAudioDevicePropertyNominalSampleRate,+ mScope: kAudioObjectPropertyScopeGlobal,+ mElement: kAudioObjectPropertyElementMain,+ };++ let mut actual_rate: f64 = 0.0;+ let mut data_size = std::mem::size_of::<f64>() as u32;++ let status = AudioObjectGetPropertyData(+ self.id,+ &address as *const AudioObjectPropertyAddress,+ 0,+ std::ptr::null(),+ &mut data_size as *mut u32,+ &mut actual_rate as *mut f64 as *mut std::ffi::c_void,+ );++ if status == 0 {+ // Update with the verified rate+ audio_stats.sample_rate = actual_rate;+ }+ }+ }++ Ok(audio_stats)+ }++ // Activates an audio device by creating a dummy IO proc+ fn activate_audio_device(&self, device_id: AudioObjectID) -> Result<AudioDeviceIOProcID> {+ // Create a simple no-op dummy proc+ let dummy_block = RcBlock::new(+ |_: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void| {+ // No-op function that just returns success+ kAudioHardwareNoError as i32+ },+ );++ let mut dummy_proc_id: AudioDeviceIOProcID = None;++ // Create the IO proc with our dummy block+ let status = unsafe {+ AudioDeviceCreateIOProcIDWithBlock(+ &mut dummy_proc_id,+ device_id,+ ptr::null_mut(),+ (&*dummy_block.copy() as *const Block<dyn Fn(_, _, _, _, _) -> i32>)+ .cast_mut()+ .cast(),+ )+ };++ if status != 0 {+ return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into());+ }++ // Start the device to activate it+ let status = unsafe { AudioDeviceStart(device_id, dummy_proc_id) };+ if status != 0 {+ return Err(CoreAudioError::AudioDeviceStartFailed(status).into());}+ // Return the proc ID for later cleanup+ Ok(dummy_proc_id)+ }++ pub fn start(+ &mut self,+ audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,+ ) -> Result<AudioTapStream> {+ // Configure the aggregate device and get audio stats before starting+ let audio_stats = self.configure_aggregate_device()?;self.audio_stats = Some(audio_stats);let audio_stats_clone = audio_stats;@@ -242,7 +490,7 @@ impl AggregateDevice {}] = mBuffers;// Only create slice if we have valid dataif !mData.is_null() && *mDataByteSize > 0 {- // Calculate total number of samples (accounting for interleaved stereo)+ // Calculate total number of samples (total bytes / bytes per sample)let total_samples = *mDataByteSize as usize / 4; // 4 bytes per f32// Create a slice of all samples@@ -253,27 +501,18 @@ impl AggregateDevice {let channel_count = *mNumberChannels as usize;// Process the audio based on channel count- let mut processed_samples: Vec<f32>;+ let processed_samples: Vec<f32>;if channel_count > 1 {// For stereo, samples are interleaved: [L, R, L, R, ...]// We need to average each pair to get mono- let frame_count = total_samples / channel_count;- processed_samples = Vec::with_capacity(frame_count);-- for i in 0..frame_count {- let mut frame_sum = 0.0;- for c in 0..channel_count {- frame_sum += samples[i * channel_count + c];- }- processed_samples.push(frame_sum / (channel_count as f32));- }+ processed_samples = process_mixed_audio(samples, channel_count);} else {- // Already mono, just copy the samples+ // For mono, just copy the samplesprocessed_samples = samples.to_vec();}- // Pass the processed samples to the callback+ // Send the processed audio data to JavaScriptaudio_stream_callback.call(Ok(processed_samples.into()),ThreadsafeFunctionCallMode::NonBlocking,@@ -309,7 +548,11 @@ impl AggregateDevice {device_id: self.id,in_proc_id,stop_called: false,- audio_stats: audio_stats_clone,+ audio_stats: audio_stats_clone, // Use the updated audio_stats with the actual sample rate+ input_device_id: self.input_device_id,+ output_device_id: self.output_device_id,+ input_proc_id: self.input_proc_id,+ output_proc_id: self.output_proc_id,})}@@ -325,15 +568,29 @@ impl AggregateDevice {let aggregate_device_uid_string = aggregate_device_uid.to_string();// Sub-device UID key and dictionary- let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[(- cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),- system_output_uid.as_CFType(),- )]);+ let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[+ (+ cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),+ system_output_uid.as_CFType(),+ ),+ // Explicitly mark this as an output device+ (+ CFString::new("com.apple.audio.roles").as_CFType(),+ CFString::new("output").as_CFType(),+ ),+ ]);- let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[(- cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),- default_input_uid.as_CFType(),- )]);+ let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[+ (+ cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),+ default_input_uid.as_CFType(),+ ),+ // Explicitly mark this as an input device+ (+ CFString::new("com.apple.audio.roles").as_CFType(),+ CFString::new("input").as_CFType(),+ ),+ ]);let tap_device_dict = CFDictionary::from_CFType_pairs(&[(@@ -346,6 +603,7 @@ impl AggregateDevice {),]);+ // Put input device first in the list to prioritize itlet capture_device_list = vec![sub_device_input_dict, sub_device_output_dict];// Sub-device list@@ -353,7 +611,8 @@ impl AggregateDevice {let tap_list = CFArray::from_CFTypes(&[tap_device_dict]);- // Create the aggregate device description dictionary+ // Create the aggregate device description dictionary with a balanced+ // configurationlet description_dict = CFDictionary::from_CFType_pairs(&[(cfstring_from_bytes_with_nul(kAudioAggregateDeviceNameKey).as_CFType(),@@ -365,7 +624,9 @@ impl AggregateDevice {),(cfstring_from_bytes_with_nul(kAudioAggregateDeviceMainSubDeviceKey).as_CFType(),- system_output_uid.as_CFType(),+ // Use a balanced approach that includes both input and output+ // but prioritize input for microphone capture+ default_input_uid.as_CFType(),),(cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsPrivateKey).as_CFType(),@@ -398,6 +659,10 @@ pub struct AudioTapStream {in_proc_id: AudioDeviceIOProcID,stop_called: bool,audio_stats: AudioStats,+ input_device_id: Option<AudioObjectID>,+ output_device_id: Option<AudioObjectID>,+ input_proc_id: Option<AudioDeviceIOProcID>,+ output_proc_id: Option<AudioDeviceIOProcID>,}#[napi]@@ -408,22 +673,47 @@ impl AudioTapStream {return Ok(());}self.stop_called = true;++ // Stop the main aggregate devicelet status = unsafe { AudioDeviceStop(self.device_id, self.in_proc_id) };if status != 0 {return Err(CoreAudioError::AudioDeviceStopFailed(status).into());}++ // Stop the input device if it was activated+ if let Some(input_id) = self.input_device_id {+ if let Some(proc_id) = self.input_proc_id {+ let _ = unsafe { AudioDeviceStop(input_id, proc_id) };+ let _ = unsafe { AudioDeviceDestroyIOProcID(input_id, proc_id) };+ }+ }++ // Stop the output device if it was activated+ if let Some(output_id) = self.output_device_id {+ if let Some(proc_id) = self.output_proc_id {+ let _ = unsafe { AudioDeviceStop(output_id, proc_id) };+ let _ = unsafe { AudioDeviceDestroyIOProcID(output_id, proc_id) };+ }+ }++ // Destroy the main IO proclet status = unsafe { AudioDeviceDestroyIOProcID(self.device_id, self.in_proc_id) };if status != 0 {return Err(CoreAudioError::AudioDeviceDestroyIOProcIDFailed(status).into());}++ // Destroy the aggregate devicelet status = unsafe { AudioHardwareDestroyAggregateDevice(self.device_id) };if status != 0 {return Err(CoreAudioError::AudioHardwareDestroyAggregateDeviceFailed(status).into());}++ // Destroy the process taplet status = unsafe { AudioHardwareDestroyProcessTap(self.device_id) };if status != 0 {return Err(CoreAudioError::AudioHardwareDestroyProcessTapFailed(status).into());}+Ok(())}@@ -445,3 +735,21 @@ fn cfstring_from_bytes_with_nul(bytes: &'static [u8]) -> CFString {.as_ref(),)}++// Process mixed audio from multiple channels+fn process_mixed_audio(samples: &[f32], channel_count: usize) -> Vec<f32> {+ // For stereo or multi-channel audio, we need to mix down to mono+ let samples_per_channel = samples.len() / channel_count;+ let mut mixed_samples = Vec::with_capacity(samples_per_channel);++ for i in 0..samples_per_channel {+ let mut sample_sum = 0.0;+ for c in 0..channel_count {+ sample_sum += samples[i * channel_count + c];+ }+ // Average the samples from all channels+ mixed_samples.push(sample_sum / channel_count as f32);+ }++ mixed_samples+}diff --git a/tests/affine-desktop/playwright.config.ts b/tests/affine-desktop/playwright.config.tsindex e28f551d15..323b52e5db 100644--- a/tests/affine-desktop/playwright.config.ts+++ b/tests/affine-desktop/playwright.config.ts@@ -29,7 +29,7 @@ const config: PlaywrightTestConfig = {if (process.env.CI) {config.retries = 5;- config.workers = 2;+ config.workers = 1;}if (process.env.DEV_SERVER_URL) {diff --git a/tests/kit/src/electron.ts b/tests/kit/src/electron.tsindex 64a3762d61..02bce1683a 100644--- a/tests/kit/src/electron.ts+++ b/tests/kit/src/electron.ts@@ -1,4 +1,5 @@import crypto from 'node:crypto';+import { setTimeout } from 'node:timers/promises';import { Package } from '@affine-tools/utils/workspace';import { expect, type Page } from '@playwright/test';@@ -71,7 +72,7 @@ export const test = base.extend<{return electronApp.windows().length > 1;},{- timeout: 50000,+ timeout: 10000,}).toBeTruthy();@@ -83,7 +84,7 @@ export const test = base.extend<{return !!page;},{- timeout: 50000,+ timeout: 10000,}).toBeTruthy();@@ -147,12 +148,25 @@ export const test = base.extend<{});await use(electronApp);- const pages = electronApp.windows();- for (const page of pages) {- await page.close();- }- await electronApp.close();- await removeWithRetry(clonedDist);+ const cleanup = async () => {+ const pages = electronApp.windows();+ for (const page of pages) {+ if (page.isClosed()) {+ continue;+ }+ await page.close();+ }+ await electronApp.close();+ await removeWithRetry(clonedDist);+ };+ await Promise.race([+ // cleanup may stuck and fail the test, but it should be fine.+ cleanup(),+ setTimeout(10000).then(() => {+ // kill the electron app if it is not closed after 10 seconds+ electronApp.process().kill();+ }),+ ]);} catch (error) {console.log(error);}diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.tsindex 443d9eeb74..5b6df13d07 100644--- a/tools/utils/src/workspace.gen.ts+++ b/tools/utils/src/workspace.gen.ts@@ -733,6 +733,7 @@ export const PackageList = ['packages/frontend/electron-api','packages/frontend/i18n','packages/common/nbstore',+ 'blocksuite/affine/all','packages/common/infra','tools/utils',],diff --git a/yarn.lock b/yarn.lockindex aec8ceeb57..b83142a87a 100644--- a/yarn.lock+++ b/yarn.lock@@ -496,6 +496,7 @@ __metadata:"@affine/electron-api": "workspace:*""@affine/i18n": "workspace:*""@affine/nbstore": "workspace:*"+ "@blocksuite/affine": "workspace:*""@emotion/react": "npm:^11.14.0""@sentry/react": "npm:^9.2.0""@toeverything/infra": "workspace:*"