Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

feat(electron): create recording through tray (#10526)

pengx17a016630a820f
diff --git a/packages/frontend/apps/electron-renderer/package.json b/packages/frontend/apps/electron-renderer/package.json
index 9c25d72403..e0aa60b27a 100644
--- a/packages/frontend/apps/electron-renderer/package.json
+++ b/packages/frontend/apps/electron-renderer/package.json
@@ -13,6 +13,7 @@
"@affine/electron-api": "workspace:*",
"@affine/i18n": "workspace:*",
"@affine/nbstore": "workspace:*",
+ "@blocksuite/affine": "workspace:*",
"@emotion/react": "^11.14.0",
"@sentry/react": "^9.2.0",
"@toeverything/infra": "workspace:*",
diff --git a/packages/frontend/apps/electron-renderer/src/app.tsx b/packages/frontend/apps/electron-renderer/src/app.tsx
index 234d75d221..88079245ae 100644
--- a/packages/frontend/apps/electron-renderer/src/app.tsx
+++ b/packages/frontend/apps/electron-renderer/src/app.tsx
@@ -1,3 +1,4 @@
+import type { DocProps } from '@affine/core/blocksuite/initialization';
import { AffineContext } from '@affine/core/components/context';
import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls';
import { AppContainer } from '@affine/core/desktop/components/app-container';
@@ -38,6 +39,8 @@ import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspac
import createEmotionCache from '@affine/core/utils/create-emotion-cache';
import { apis, events } from '@affine/electron-api';
import { StoreManagerClient } from '@affine/nbstore/worker/client';
+import type { AttachmentBlockProps } from '@blocksuite/affine/model';
+import { Text } from '@blocksuite/affine/store';
import { CacheProvider } from '@emotion/react';
import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra';
import { OpClient } from '@toeverything/infra/op';
@@ -174,32 +177,80 @@ events?.applicationMenu.openAboutPageInSettingModal(() => {
});
events?.applicationMenu.onNewPageAction(type => {
- const currentWorkspace = getCurrentWorkspace();
- if (!currentWorkspace) {
- return;
- }
- const { workspace, dispose } = currentWorkspace;
- const editorSettingService = frameworkProvider.get(EditorSettingService);
- const docsService = workspace.scope.get(DocsService);
- const editorSetting = editorSettingService.editorSetting;
-
- const docProps = {
- note: editorSetting.get('affine:note'),
- };
apis?.ui
.isActiveTab()
.then(isActive => {
if (!isActive) {
return;
}
+ const currentWorkspace = getCurrentWorkspace();
+ if (!currentWorkspace) {
+ return;
+ }
+ const { workspace, dispose } = currentWorkspace;
+ const editorSettingService = frameworkProvider.get(EditorSettingService);
+ const docsService = workspace.scope.get(DocsService);
+ const editorSetting = editorSettingService.editorSetting;
+
+ const docProps = {
+ note: editorSetting.get('affine:note'),
+ };
const page = docsService.createDoc({ docProps, primaryMode: type });
workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
+ dispose();
})
.catch(err => {
console.error(err);
});
+});
- dispose();
+events?.recording.onRecordingStatusChanged(status => {
+ (async () => {
+ if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') {
+ const currentWorkspace = getCurrentWorkspace();
+ if (!currentWorkspace) {
+ return;
+ }
+ const { workspace, dispose } = currentWorkspace;
+ const editorSettingService = frameworkProvider.get(EditorSettingService);
+ const docsService = workspace.scope.get(DocsService);
+ const editorSetting = editorSettingService.editorSetting;
+
+ const docProps: DocProps = {
+ note: editorSetting.get('affine:note'),
+ page: {
+ title: new Text(
+ 'Recording ' +
+ (status.appGroup?.name ?? 'System Audio') +
+ ' ' +
+ new Date(status.startTime).toISOString()
+ ),
+ },
+ onStoreLoad: (doc, { noteId }) => {
+ (async () => {
+ const data = await apis?.recording.saveRecording(status.id);
+ if (!data) {
+ return;
+ }
+ const blob = new Blob([data], { type: 'audio/mp3' });
+ const blobId = await doc.workspace.blobSync.set(blob);
+ const attachmentProps: Partial<AttachmentBlockProps> = {
+ name: 'Recording',
+ size: blob.size,
+ type: 'audio/mp3',
+ sourceId: blobId,
+ embed: true,
+ };
+ doc.addBlock('affine:attachment', attachmentProps, noteId);
+ })().catch(console.error);
+ },
+ };
+ const page = docsService.createDoc({ docProps, primaryMode: 'page' });
+ workspace.scope.get(WorkbenchService).workbench.openDoc(page.id);
+
+ dispose();
+ }
+ })().catch(console.error);
});
events?.applicationMenu.onOpenJournal(() => {
diff --git a/packages/frontend/apps/electron-renderer/tsconfig.json b/packages/frontend/apps/electron-renderer/tsconfig.json
index 19af95cc4f..422502d628 100644
--- a/packages/frontend/apps/electron-renderer/tsconfig.json
+++ b/packages/frontend/apps/electron-renderer/tsconfig.json
@@ -13,6 +13,7 @@
{ "path": "../../electron-api" },
{ "path": "../../i18n" },
{ "path": "../../../common/nbstore" },
+ { "path": "../../../../blocksuite/affine/all" },
{ "path": "../../../common/infra" },
{ "path": "../../../../tools/utils" }
]
diff --git a/packages/frontend/apps/electron/resources/icons/monitor.png b/packages/frontend/apps/electron/resources/icons/monitor.png
new file mode 100644
index 0000000000..e4305b8093
Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/monitor.png differ
diff --git a/packages/frontend/apps/electron/resources/icons/pause.png b/packages/frontend/apps/electron/resources/icons/pause.png
new file mode 100644
index 0000000000..f4ff7ee83d
Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/pause.png differ
diff --git a/packages/frontend/apps/electron/resources/icons/stop.png b/packages/frontend/apps/electron/resources/icons/stop.png
new file mode 100644
index 0000000000..32977c4f2b
Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/stop.png differ
diff --git a/packages/frontend/apps/electron/resources/icons/waveform-recording.png b/packages/frontend/apps/electron/resources/icons/waveform-recording.png
new file mode 100644
index 0000000000..19e91f03f9
Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/waveform-recording.png differ
diff --git a/packages/frontend/apps/electron/resources/icons/waveform.png b/packages/frontend/apps/electron/resources/icons/waveform.png
new file mode 100644
index 0000000000..11a4f05d38
Binary files /dev/null and b/packages/frontend/apps/electron/resources/icons/waveform.png differ
diff --git a/packages/frontend/apps/electron/src/helper/main-rpc.ts b/packages/frontend/apps/electron/src/helper/main-rpc.ts
index 59be05958f..e6026f5d30 100644
--- a/packages/frontend/apps/electron/src/helper/main-rpc.ts
+++ b/packages/frontend/apps/electron/src/helper/main-rpc.ts
@@ -2,6 +2,7 @@ import { AsyncCall } from 'async-call-rpc';
import type { HelperToMain, MainToHelper } from '../shared/type';
import { exposed } from './provide';
+import { encodeToMp3 } from './recording/encode';
const helperToMainServer: HelperToMain = {
getMeta: () => {
@@ -10,6 +11,8 @@ const helperToMainServer: HelperToMain = {
}
return exposed;
},
+ // allow main process encode audio samples to mp3 buffer (because it is slow and blocking)
+ encodeToMp3,
};
export const mainRPC = AsyncCall<MainToHelper>(helperToMainServer, {
diff --git a/packages/frontend/apps/electron/src/helper/recording/encode.ts b/packages/frontend/apps/electron/src/helper/recording/encode.ts
new file mode 100644
index 0000000000..0aa3464876
--- /dev/null
+++ b/packages/frontend/apps/electron/src/helper/recording/encode.ts
@@ -0,0 +1,16 @@
+import { Mp3Encoder } from '@affine/native';
+
+// encode audio samples to mp3 buffer
+export function encodeToMp3(
+ samples: Float32Array,
+ opts: {
+ channels?: number;
+ sampleRate?: number;
+ } = {}
+): Uint8Array {
+ const mp3Encoder = new Mp3Encoder({
+ channels: opts.channels ?? 2,
+ sampleRate: opts.sampleRate ?? 44100,
+ });
+ return mp3Encoder.encode(samples);
+}
diff --git a/packages/frontend/apps/electron/src/main/events.ts b/packages/frontend/apps/electron/src/main/events.ts
index 51b5c1b07e..fec2f1dc78 100644
--- a/packages/frontend/apps/electron/src/main/events.ts
+++ b/packages/frontend/apps/electron/src/main/events.ts
@@ -4,6 +4,7 @@ import { AFFINE_EVENT_CHANNEL_NAME } from '../shared/type';
import { applicationMenuEvents } from './application-menu';
import { beforeAppQuit } from './cleanup';
import { logger } from './logger';
+import { recordingEvents } from './recording';
import { sharedStorageEvents } from './shared-storage';
import { uiEvents } from './ui/events';
import { updaterEvents } from './updater/event';
@@ -13,6 +14,7 @@ export const allEvents = {
updater: updaterEvents,
ui: uiEvents,
sharedStorage: sharedStorageEvents,
+ recording: recordingEvents,
};
function getActiveWindows() {
diff --git a/packages/frontend/apps/electron/src/main/handlers.ts b/packages/frontend/apps/electron/src/main/handlers.ts
index 59ef7a0d55..70d0fcf40a 100644
--- a/packages/frontend/apps/electron/src/main/handlers.ts
+++ b/packages/frontend/apps/electron/src/main/handlers.ts
@@ -5,6 +5,7 @@ import { clipboardHandlers } from './clipboard';
import { configStorageHandlers } from './config-storage';
import { findInPageHandlers } from './find-in-page';
import { getLogFilePath, logger, revealLogFile } from './logger';
+import { recordingHandlers } from './recording';
import { sharedStorageHandlers } from './shared-storage';
import { uiHandlers } from './ui/handlers';
import { updaterHandlers } from './updater';
@@ -29,6 +30,7 @@ export const allHandlers = {
findInPage: findInPageHandlers,
sharedStorage: sharedStorageHandlers,
worker: workerHandlers,
+ recording: recordingHandlers,
};
export const registerHandlers = () => {
diff --git a/packages/frontend/apps/electron/src/main/index.ts b/packages/frontend/apps/electron/src/main/index.ts
index e1f1d71ebb..f67e9dc9d7 100644
--- a/packages/frontend/apps/electron/src/main/index.ts
+++ b/packages/frontend/apps/electron/src/main/index.ts
@@ -14,6 +14,8 @@ import { registerEvents } from './events';
import { registerHandlers } from './handlers';
import { logger } from './logger';
import { registerProtocol } from './protocol';
+import { setupRecording } from './recording';
+import { getTrayState } from './tray';
import { registerUpdater } from './updater';
import { launch } from './windows-manager/launcher';
import { launchStage } from './windows-manager/stage';
@@ -85,7 +87,9 @@ app
.then(registerHandlers)
.then(registerEvents)
.then(launch)
+ .then(setupRecording)
.then(createApplicationMenu)
+ .then(getTrayState)
.then(registerUpdater)
.catch(e => console.error('Failed create window:', e));
diff --git a/packages/frontend/apps/electron/src/main/recording/index.ts b/packages/frontend/apps/electron/src/main/recording/index.ts
index abcc9bd24a..ed9dde168a 100644
--- a/packages/frontend/apps/electron/src/main/recording/index.ts
+++ b/packages/frontend/apps/electron/src/main/recording/index.ts
@@ -1,38 +1,30 @@
-import { ShareableContent, TappableApplication } from '@affine/native';
-import { Notification } from 'electron';
-import {
- BehaviorSubject,
- distinctUntilChanged,
- pairwise,
- startWith,
-} from 'rxjs';
+import { ShareableContent } from '@affine/native';
+import { nativeImage, Notification } from 'electron';
+import { debounce } from 'lodash-es';
+import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs';
import { isMacOS } from '../../shared/utils';
import { beforeAppQuit } from '../cleanup';
+import { ensureHelperProcess } from '../helper-process';
import { logger } from '../logger';
-
-interface TappableAppInfo {
- rawInstance: TappableApplication;
- isRunning: boolean;
- processId: number;
- processGroupId: number;
- bundleIdentifier: string;
- name: string;
-}
-
-interface AppGroupInfo {
- processGroupId: number;
- apps: TappableAppInfo[];
- name: string;
- icon: Buffer | undefined;
- isRunning: boolean;
-}
+import type { NamespaceHandlers } from '../type';
+import { getMainWindow } from '../windows-manager';
+import type {
+ AppGroupInfo,
+ Recording,
+ RecordingStatus,
+ TappableAppInfo,
+} from './types';
const subscribers: Subscriber[] = [];
beforeAppQuit(() => {
subscribers.forEach(subscriber => {
- subscriber.unsubscribe();
+ try {
+ subscriber.unsubscribe();
+ } catch {
+ // ignore unsubscribe error
+ }
});
});
@@ -41,8 +33,41 @@ let shareableContent: ShareableContent | null = null;
export const applications$ = new BehaviorSubject<TappableAppInfo[]>([]);
export const appGroups$ = new BehaviorSubject<AppGroupInfo[]>([]);
-if (isMacOS()) {
- // Update appGroups$ whenever applications$ changes
+// recording id -> recording
+// recordings will be saved in memory before consumed and created as an audio block to user's doc
+const recordings = new Map<number, Recording>();
+
+// there should be only one active recording at a time
+export const recordingStatus$ = new BehaviorSubject<RecordingStatus | null>(
+ null
+);
+
+function createAppGroup(processGroupId: number): AppGroupInfo | undefined {
+ const groupProcess =
+ shareableContent?.applicationWithProcessId(processGroupId);
+ if (!groupProcess) {
+ return;
+ }
+ return {
+ processGroupId: processGroupId,
+ apps: [], // leave it empty for now.
+ name: groupProcess.name,
+ bundleIdentifier: groupProcess.bundleIdentifier,
+ // icon should be lazy loaded
+ get icon() {
+ try {
+ return groupProcess.icon;
+ } catch (error) {
+ logger.error(`Failed to get icon for ${groupProcess.name}`, error);
+ return undefined;
+ }
+ },
+ isRunning: false,
+ };
+}
+
+// pipe applications$ to appGroups$
+function setupAppGroups() {
subscribers.push(
applications$.pipe(distinctUntilChanged()).subscribe(apps => {
const appGroups: AppGroupInfo[] = [];
@@ -52,69 +77,160 @@ if (isMacOS()) {
);
if (!appGroup) {
- const groupProcess = shareableContent?.applicationWithProcessId(
- app.processGroupId
- );
- if (!groupProcess) {
- return;
+ appGroup = createAppGroup(app.processGroupId);
+ if (appGroup) {
+ appGroups.push(appGroup);
}
- appGroup = {
- processGroupId: app.processGroupId,
- apps: [],
- name: groupProcess.name,
- // icon will be lazy loaded
- get icon() {
- try {
- return groupProcess.icon;
- } catch (error) {
- logger.error(
- `Failed to get icon for ${groupProcess.name}`,
- error
- );
- return undefined;
- }
- },
- get isRunning() {
- return this.apps.some(app => app.rawInstance.isRunning);
- },
- };
- appGroups.push(appGroup);
}
if (appGroup) {
appGroup.apps.push(app);
}
});
+
+ appGroups.forEach(appGroup => {
+ appGroup.isRunning = appGroup.apps.some(app => app.isRunning);
+ });
+
appGroups$.next(appGroups);
})
);
+}
+function setupNewRunningAppGroup() {
+ const appGroupRunningChanged$ = appGroups$.pipe(
+ mergeMap(groups => groups),
+ groupBy(group => group.processGroupId),
+ mergeMap(groupStream$ =>
+ groupStream$.pipe(
+ distinctUntilChanged((prev, curr) => prev.isRunning === curr.isRunning)
+ )
+ )
+ );
subscribers.push(
- appGroups$
- .pipe(startWith([] as AppGroupInfo[]), pairwise())
- .subscribe(([previousGroups, currentGroups]) => {
- currentGroups.forEach(currentGroup => {
- const previousGroup = previousGroups.find(
- group => group.processGroupId === currentGroup.processGroupId
- );
- if (previousGroup?.isRunning !== currentGroup.isRunning) {
- console.log(
- 'appgroup running changed',
- currentGroup.name,
- currentGroup.isRunning
- );
- if (currentGroup.isRunning) {
- new Notification({
- title: 'Recording Meeting',
- body: `Recording meeting with ${currentGroup.name}`,
- }).show();
- }
- }
+ appGroupRunningChanged$.subscribe(currentGroup => {
+ if (currentGroup.isRunning) {
+ // TODO(@pengx17): stub impl. will be replaced with a real one later
+ const notification = new Notification({
+ icon: currentGroup.icon
+ ? nativeImage.createFromBuffer(currentGroup.icon)
+ : undefined,
+ title: 'Recording Meeting',
+ body: `Recording meeting with ${currentGroup.name}`,
+ actions: [
+ {
+ type: 'button',
+ text: 'Start',
+ },
+ ],
});
- })
+ notification.on('action', () => {
+ startRecording(currentGroup);
+ });
+ notification.show();
+ } else {
+ // if the group is not running, we should stop the recording (if it is recording)
+ if (
+ recordingStatus$.value?.status === 'recording' &&
+ recordingStatus$.value?.appGroup?.processGroupId ===
+ currentGroup.processGroupId
+ ) {
+ stopRecording();
+ }
+ }
+ })
);
}
-async function getAllApps(): Promise<TappableAppInfo[]> {
+function createRecording(status: RecordingStatus) {
+ const buffers: Float32Array[] = [];
+
+ function tapAudioSamples(err: Error | null, samples: Float32Array) {
+ const recordingStatus = recordingStatus$.getValue();
+ if (
+ !recordingStatus ||
+ recordingStatus.id !== status.id ||
+ recordingStatus.status === 'paused'
+ ) {
+ return;
+ }
+
+ if (err) {
+ logger.error('failed to get audio samples', err);
+ } else {
+ buffers.push(new Float32Array(samples));
+ }
+ }
+
+ const stream = status.app
+ ? status.app.rawInstance.tapAudio(tapAudioSamples)
+ : ShareableContent.tapGlobalAudio(null, tapAudioSamples);
+
+ const recording: Recording = {
+ id: status.id,
+ startTime: status.startTime,
+ app: status.app,
+ appGroup: status.appGroup,
+ buffers,
+ stream,
+ };
+
+ return recording;
+}
+
+function concatBuffers(buffers: Float32Array[]): Float32Array {
+ const totalSamples = buffers.reduce((acc, buf) => acc + buf.length, 0);
+ const buffer = new Float32Array(totalSamples);
+ let offset = 0;
+ buffers.forEach(buf => {
+ buffer.set(buf, offset);
+ offset += buf.length;
+ });
+ return buffer;
+}
+
+export async function saveRecording(id: number) {
+ const recording = recordings.get(id);
+ if (!recording) {
+ logger.error(`Recording ${id} not found`);
+ return;
+ }
+ const { buffers } = recording;
+ const helperProcessManager = await ensureHelperProcess();
+ const buffer = concatBuffers(buffers);
+ const mp3Buffer = await helperProcessManager.rpc?.encodeToMp3(buffer, {
+ channels: recording.stream.channels,
+ sampleRate: recording.stream.sampleRate,
+ });
+
+ if (!mp3Buffer) {
+ logger.error('failed to encode audio samples to mp3');
+ return;
+ }
+ recordings.delete(recording.id);
+ return mp3Buffer;
+}
+
+function setupRecordingListeners() {
+ subscribers.push(
+ recordingStatus$.pipe(distinctUntilChanged()).subscribe(status => {
+ if (status?.status === 'recording') {
+ let recording = recordings.get(status.id);
+ // create a recording if not exists
+ if (!recording) {
+ recording = createRecording(status);
+ recordings.set(status.id, recording);
+ }
+ } else if (status?.status === 'stopped') {
+ const recording = recordings.get(status.id);
+ if (recording) {
+ recording.stream.stop();
+ }
+ }
+ })
+ );
+}
+
+function getAllApps(): TappableAppInfo[] {
if (!shareableContent) {
return [];
}
@@ -150,62 +266,37 @@ type Subscriber = {
};
function setupMediaListeners() {
+ applications$.next(getAllApps());
subscribers.push(
ShareableContent.onApplicationListChanged(() => {
- getAllApps()
- .then(apps => {
- applications$.next(apps);
- })
- .catch(err => {
- logger.error('failed to get apps', err);
- });
+ applications$.next(getAllApps());
})
);
- getAllApps()
- .then(apps => {
- applications$.next(apps);
- })
- .catch(err => {
- logger.error('failed to get apps', err);
- });
-
let appStateSubscribers: Subscriber[] = [];
subscribers.push(
applications$.subscribe(apps => {
appStateSubscribers.forEach(subscriber => {
- subscriber.unsubscribe();
+ try {
+ subscriber.unsubscribe();
+ } catch {
+ // ignore unsubscribe error
+ }
});
const _appStateSubscribers: Subscriber[] = [];
apps.forEach(app => {
try {
- // Try to create a TappableApplication with a default audio object ID
- // In a real implementation, you would need to get the actual audio object ID
- // This is just a placeholder value that seems to work for testing
- const tappableApp = TappableApplication.fromApplication(
- app.rawInstance,
- 1
+ const tappableApp = app.rawInstance;
+ const debouncedAppStateChanged = debounce(() => {
+ applications$.next(getAllApps());
+ }, 100);
+ _appStateSubscribers.push(
+ ShareableContent.onAppStateChanged(tappableApp, () => {
+ debouncedAppStateChanged();
+ })
);
-
- if (tappableApp) {
- _appStateSubscribers.push(
- ShareableContent.onAppStateChanged(tappableApp, () => {
- setTimeout(() => {
- const apps = applications$.getValue();
- applications$.next(
- apps.map(_app => {
- if (_app.processId === app.processId) {
- return { ..._app, isRunning: tappableApp.isRunning };
- }
- return _app;
- })
- );
- }, 10);
- })
- );
- }
} catch (error) {
logger.error(
`Failed to convert app ${app.name} to TappableApplication`,
@@ -217,15 +308,23 @@ function setupMediaListeners() {
appStateSubscribers = _appStateSubscribers;
return () => {
_appStateSubscribers.forEach(subscriber => {
- subscriber.unsubscribe();
+ try {
+ subscriber.unsubscribe();
+ } catch {
+ // ignore unsubscribe error
+ }
});
};
})
);
}
-export function getShareableContent() {
- if (!shareableContent && isMacOS()) {
+export function setupRecording() {
+ if (!isMacOS()) {
+ return;
+ }
+
+ if (!shareableContent) {
try {
shareableContent = new ShareableContent();
setupMediaListeners();
@@ -233,5 +332,99 @@ export function getShareableContent() {
logger.error('failed to get shareable content', error);
}
}
- return shareableContent;
+ setupAppGroups();
+ setupNewRunningAppGroup();
+ setupRecordingListeners();
+}
+
+let recordingId = 0;
+
+export function startRecording(
+ appGroup?: AppGroupInfo
+): RecordingStatus | undefined {
+ if (!shareableContent) {
+ return; // likely called on unsupported platform
+ }
+
+ // hmm, is it possible that there are multiple apps running (listening) in the same group?
+ const appInfo = appGroup?.apps.find(app => app.isRunning);
+
+ const recordingStatus: RecordingStatus = {
+ id: recordingId++,
+ status: 'recording',
+ startTime: Date.now(),
+ app: appInfo,
+ appGroup,
+ };
+
+ recordingStatus$.next(recordingStatus);
+
+ return recordingStatus;
+}
+
+export function pauseRecording() {
+ const recordingStatus = recordingStatus$.value;
+ if (!recordingStatus) {
+ return;
+ }
+
+ recordingStatus$.next({
+ ...recordingStatus,
+ status: 'paused',
+ });
+}
+
+export function resumeRecording() {
+ const recordingStatus = recordingStatus$.value;
+ if (!recordingStatus) {
+ return;
+ }
+
+ recordingStatus$.next({
+ ...recordingStatus,
+ status: 'recording',
+ });
+}
+
+export function stopRecording() {
+ const recordingStatus = recordingStatus$.value;
+ if (!recordingStatus) {
+ return;
+ }
+
+ // do not remove the last recordingStatus from recordingStatus$
+ recordingStatus$.next({
+ ...recordingStatus,
+ status: 'stopped',
+ });
+
+ // bring up the window
+ getMainWindow()
+ .then(mainWindow => {
+ if (mainWindow) {
+ mainWindow.show();
+ }
+ })
+ .catch(err => {
+ logger.error('failed to bring up the window', err);
+ });
}
+
+export const recordingHandlers = {
+ saveRecording: async (_, id: number) => {
+ return saveRecording(id);
+ },
+} satisfies NamespaceHandlers;
+
+export const recordingEvents = {
+ onRecordingStatusChanged: (fn: (status: RecordingStatus | null) => void) => {
+ const sub = recordingStatus$.subscribe(fn);
+ return () => {
+ try {
+ sub.unsubscribe();
+ } catch {
+ // ignore unsubscribe error
+ }
+ };
+ },
+};
diff --git a/packages/frontend/apps/electron/src/main/recording/types.ts b/packages/frontend/apps/electron/src/main/recording/types.ts
new file mode 100644
index 0000000000..d308b5b11b
--- /dev/null
+++ b/packages/frontend/apps/electron/src/main/recording/types.ts
@@ -0,0 +1,38 @@
+import type { AudioTapStream, TappableApplication } from '@affine/native';
+
+export interface TappableAppInfo {
+ rawInstance: TappableApplication;
+ isRunning: boolean;
+ processId: number;
+ processGroupId: number;
+ bundleIdentifier: string;
+ name: string;
+}
+
+export interface AppGroupInfo {
+ processGroupId: number;
+ apps: TappableAppInfo[];
+ name: string;
+ bundleIdentifier: string;
+ icon: Buffer | undefined;
+ isRunning: boolean;
+}
+
+export interface Recording {
+ id: number;
+ // the app may not be available if the user choose to record system audio
+ app?: TappableAppInfo;
+ appGroup?: AppGroupInfo;
+ // the raw audio buffers that are already accumulated
+ buffers: Float32Array[];
+ stream: AudioTapStream;
+ startTime: number;
+}
+
+export interface RecordingStatus {
+ id: number; // corresponds to the recording id
+ status: 'recording' | 'paused' | 'stopped';
+ app?: TappableAppInfo;
+ appGroup?: AppGroupInfo;
+ startTime: number;
+}
diff --git a/packages/frontend/apps/electron/src/main/tray/icons.ts b/packages/frontend/apps/electron/src/main/tray/icons.ts
new file mode 100644
index 0000000000..26d7a962ad
--- /dev/null
+++ b/packages/frontend/apps/electron/src/main/tray/icons.ts
@@ -0,0 +1,13 @@
+import { join } from 'node:path';
+
+import { resourcesPath } from '../../shared/utils';
+
+export const icons = {
+ record: join(resourcesPath, 'icons/waveform.png'),
+ recording: join(resourcesPath, 'icons/waveform-recording.png'),
+ tray: join(resourcesPath, 'icons/tray-icon.png'),
+ journal: join(resourcesPath, 'icons/journal-today.png'),
+ page: join(resourcesPath, 'icons/doc-page.png'),
+ edgeless: join(resourcesPath, 'icons/doc-edgeless.png'),
+ monitor: join(resourcesPath, 'icons/monitor.png'),
+};
diff --git a/packages/frontend/apps/electron/src/main/tray/index.ts b/packages/frontend/apps/electron/src/main/tray/index.ts
index 249c441b7b..a6d8b21d0b 100644
--- a/packages/frontend/apps/electron/src/main/tray/index.ts
+++ b/packages/frontend/apps/electron/src/main/tray/index.ts
@@ -1,25 +1,34 @@
-import { join } from 'node:path';
-
import {
app,
Menu,
MenuItem,
+ type MenuItemConstructorOptions,
type NativeImage,
nativeImage,
Tray,
} from 'electron';
-import { isMacOS, resourcesPath } from '../../shared/utils';
+import { isMacOS } from '../../shared/utils';
import { applicationMenuSubjects } from '../application-menu';
import { beforeAppQuit } from '../cleanup';
-import { appGroups$ } from '../recording';
+import { logger } from '../logger';
+import {
+ appGroups$,
+ pauseRecording,
+ recordingStatus$,
+ resumeRecording,
+ startRecording,
+ stopRecording,
+} from '../recording';
import { getMainWindow } from '../windows-manager';
+import { icons } from './icons';
export interface TrayMenuConfigItem {
label: string;
click?: () => void;
icon?: NativeImage | string | Buffer;
disabled?: boolean;
+ submenu?: TrayMenuConfig;
}
export type TrayMenuConfig = Array<TrayMenuConfigItem | 'separator'>;
@@ -35,7 +44,35 @@ function showMainWindow() {
.then(w => {
w.show();
})
- .catch(err => console.error(err));
+ .catch(err => logger.error('Failed to show main window:', err));
+}
+
+function buildMenuConfig(config: TrayMenuConfig): MenuItemConstructorOptions[] {
+ const menuConfig: MenuItemConstructorOptions[] = [];
+ config.forEach(item => {
+ if (item === 'separator') {
+ menuConfig.push({ type: 'separator' });
+ } else {
+ const { icon, disabled, submenu, ...rest } = item;
+ let nativeIcon: NativeImage | undefined;
+ if (typeof icon === 'string') {
+ nativeIcon = nativeImage.createFromPath(icon);
+ } else if (Buffer.isBuffer(icon)) {
+ nativeIcon = nativeImage.createFromBuffer(icon);
+ }
+ if (nativeIcon) {
+ nativeIcon = nativeIcon.resize({ width: 20, height: 20 });
+ }
+ const submenuConfig = submenu ? buildMenuConfig(submenu) : undefined;
+ menuConfig.push({
+ ...rest,
+ enabled: !disabled,
+ icon: nativeIcon,
+ submenu: submenuConfig,
+ });
+ }
+ });
+ return menuConfig;
}
class TrayState {
@@ -43,7 +80,7 @@ class TrayState {
// tray's icon
icon: NativeImage = nativeImage
- .createFromPath(join(resourcesPath, 'icons/tray-icon.png'))
+ .createFromPath(icons.tray)
.resize({ width: 16, height: 16 });
// tray's tooltip
@@ -60,24 +97,27 @@ class TrayState {
getConfig: () => [
{
label: 'Open Journal',
- icon: join(resourcesPath, 'icons/journal-today.png'),
+ icon: icons.journal,
click: () => {
+ logger.info('User action: Open Journal');
showMainWindow();
applicationMenuSubjects.openJournal$.next();
},
},
{
label: 'New Page',
- icon: join(resourcesPath, 'icons/doc-page.png'),
+ icon: icons.page,
click: () => {
+ logger.info('User action: New Page');
showMainWindow();
applicationMenuSubjects.newPageAction$.next('page');
},
},
{
label: 'New Edgeless',
- icon: join(resourcesPath, 'icons/doc-edgeless.png'),
+ icon: icons.edgeless,
click: () => {
+ logger.info('User action: New Edgeless');
showMainWindow();
applicationMenuSubjects.newPageAction$.next('edgeless');
},
@@ -89,20 +129,78 @@ class TrayState {
getRecordingMenuProvider(): TrayMenuProvider {
const appGroups = appGroups$.value;
const runningAppGroups = appGroups.filter(appGroup => appGroup.isRunning);
+
+ const recordingStatus = recordingStatus$.value;
+
+ if (!recordingStatus || recordingStatus?.status === 'stopped') {
+ return {
+ key: 'recording',
+ getConfig: () => [
+ {
+ label: 'Start Recording Meeting',
+ icon: icons.record,
+ submenu: [
+ {
+ label: 'System audio (all audio will be recorded)',
+ icon: icons.monitor,
+ click: () => {
+ logger.info(
+ 'User action: Start Recording Meeting (System audio)'
+ );
+ startRecording();
+ },
+ },
+ ...runningAppGroups.map(appGroup => ({
+ label: appGroup.name,
+ icon: appGroup.icon || undefined,
+ click: () => {
+ logger.info(
+ `User action: Start Recording Meeting (${appGroup.name})`
+ );
+ startRecording(appGroup);
+ },
+ })),
+ ],
+ },
+ ],
+ };
+ }
+
+ const recordingLabel = recordingStatus.appGroup?.name
+ ? `Recording (${recordingStatus.appGroup?.name})`
+ : 'Recording';
+
+ // recording is either started or paused
return {
key: 'recording',
getConfig: () => [
{
- label: 'Start Recording Meeting',
+ label: recordingLabel,
+ icon: icons.recording,
disabled: true,
},
- ...runningAppGroups.map(appGroup => ({
- label: appGroup.name,
- icon: appGroup.icon || undefined,
+ recordingStatus.status === 'paused'
+ ? {
+ label: 'Resume',
+ click: () => {
+ logger.info('User action: Resume Recording');
+ resumeRecording();
+ },
+ }
+ : {
+ label: 'Pause',
+ click: () => {
+ logger.info('User action: Pause Recording');
+ pauseRecording();
+ },
+ },
+ {
+ label: 'Stop',
click: () => {
- console.log(appGroup);
+ logger.info('User action: Stop Recording');
+ stopRecording();
},
- })),
+ },
],
};
}
@@ -114,12 +212,13 @@ class TrayState {
{
label: 'Open AFFiNE',
click: () => {
+ logger.info('User action: Open AFFiNE');
getMainWindow()
.then(w => {
w.show();
})
.catch(err => {
- console.error(err);
+ logger.error('Failed to open AFFiNE:', err);
});
},
},
@@ -127,6 +226,7 @@ class TrayState {
{
label: 'Quit AFFiNE Completely...',
click: () => {
+ logger.info('User action: Quit AFFiNE Completely');
app.quit();
},
},
@@ -137,32 +237,9 @@ class TrayState {
buildMenu(providers: TrayMenuProvider[]) {
const menu = new Menu();
providers.forEach((provider, index) => {
- provider.getConfig().forEach(item => {
- if (item === 'separator') {
- menu.append(new MenuItem({ type: 'separator' }));
- } else {
- const { icon, disabled, ...rest } = item;
- let nativeIcon: NativeImage | undefined;
- if (typeof icon === 'string') {
- nativeIcon = nativeImage.createFromPath(icon);
- } else if (Buffer.isBuffer(icon)) {
- try {
- nativeIcon = nativeImage.createFromBuffer(icon);
- } catch (error) {
- console.error('Failed to create icon from buffer', error);
- }
- }
- if (nativeIcon) {
- nativeIcon = nativeIcon.resize({ width: 20, height: 20 });
- }
- menu.append(
- new MenuItem({
- ...rest,
- enabled: !disabled,
- icon: nativeIcon,
- })
- );
- }
+ const config = provider.getConfig();
+ buildMenuConfig(config).forEach(item => {
+ menu.append(new MenuItem(item));
});
if (index !== providers.length - 1) {
menu.append(new MenuItem({ type: 'separator' }));
@@ -176,15 +253,22 @@ class TrayState {
this.tray = new Tray(this.icon);
this.tray.setToolTip(this.tooltip);
const clickHandler = () => {
+ logger.debug('User clicked on tray icon');
this.update();
if (!isMacOS()) {
this.tray?.popUpContextMenu();
}
};
this.tray.on('click', clickHandler);
+ const appGroupsSubscription = appGroups$.subscribe(() => {
+ logger.debug('App groups updated, refreshing tray menu');
+ this.update();
+ });
beforeAppQuit(() => {
+ logger.info('Cleaning up tray before app quit');
this.tray?.off('click', clickHandler);
this.tray?.destroy();
+ appGroupsSubscription.unsubscribe();
});
}
@@ -199,6 +283,7 @@ class TrayState {
}
init() {
+ logger.info('Initializing tray');
this.update();
}
}
diff --git a/packages/frontend/apps/electron/src/shared/type.ts b/packages/frontend/apps/electron/src/shared/type.ts
index 231ee3baf1..900a561035 100644
--- a/packages/frontend/apps/electron/src/shared/type.ts
+++ b/packages/frontend/apps/electron/src/shared/type.ts
@@ -17,6 +17,13 @@ export interface HelperToRenderer {
// helper <-> main
export interface HelperToMain {
getMeta: () => ExposedMeta;
+ encodeToMp3: (
+ samples: Float32Array,
+ opts?: {
+ channels?: number;
+ sampleRate?: number;
+ }
+ ) => Uint8Array;
}
export type MainToHelper = Pick<
diff --git a/packages/frontend/core/src/blocksuite/initialization/index.ts b/packages/frontend/core/src/blocksuite/initialization/index.ts
index 85a4a8f969..6e79c3b753 100644
--- a/packages/frontend/core/src/blocksuite/initialization/index.ts
+++ b/packages/frontend/core/src/blocksuite/initialization/index.ts
@@ -12,6 +12,14 @@ export interface DocProps {
surface?: Partial<SurfaceBlockProps>;
note?: Partial<NoteProps>;
paragraph?: Partial<ParagraphProps>;
+ onStoreLoad?: (
+ doc: Store,
+ props: {
+ noteId: string;
+ paragraphId: string;
+ surfaceId: string;
+ }
+ ) => void;
}
export function initDocFromProps(doc: Store, props?: DocProps) {
@@ -20,7 +28,11 @@ export function initDocFromProps(doc: Store, props?: DocProps) {
'affine:page',
props?.page || { title: new Text('') }
);
- doc.addBlock('affine:surface' as never, props?.surface || {}, pageBlockId);
+ const surfaceId = doc.addBlock(
+ 'affine:surface' as never,
+ props?.surface || {},
+ pageBlockId
+ );
const noteBlockId = doc.addBlock(
'affine:note',
{
@@ -29,7 +41,16 @@ export function initDocFromProps(doc: Store, props?: DocProps) {
},
pageBlockId
);
- doc.addBlock('affine:paragraph', props?.paragraph || {}, noteBlockId);
+ const paragraphBlockId = doc.addBlock(
+ 'affine:paragraph',
+ props?.paragraph || {},
+ noteBlockId
+ );
+ props?.onStoreLoad?.(doc, {
+ noteId: noteBlockId,
+ paragraphId: paragraphBlockId,
+ surfaceId,
+ });
doc.history.clear();
});
}
diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.ts
index ebffc0b208..a22d6b8494 100644
--- a/packages/frontend/media-capture-playground/server/main.ts
+++ b/packages/frontend/media-capture-playground/server/main.ts
@@ -28,12 +28,13 @@ console.log(`📁 Ensuring recordings directory exists at ${RECORDING_DIR}`);
// Types
interface Recording {
- app: TappableApplication;
+ app: TappableApplication | null;
appGroup: Application | null;
buffers: Float32Array[];
stream: AudioTapStream;
startTime: number;
isWriting: boolean;
+ isGlobal?: boolean;
}
interface RecordingStatus {
@@ -54,6 +55,7 @@ interface RecordingMetadata {
sampleRate: number;
channels: number;
totalSamples: number;
+ isGlobal?: boolean;
}
interface AppInfo {
@@ -118,7 +120,7 @@ app.use(
async function saveRecording(recording: Recording): Promise<string | null> {
try {
recording.isWriting = true;
- const app = recording.appGroup || recording.app;
+ const app = recording.isGlobal ? null : recording.appGroup || recording.app;
const totalSamples = recording.buffers.reduce(
(acc, buf) => acc + buf.length,
@@ -133,9 +135,19 @@ async function saveRecording(recording: Recording): Promise<string | null> {
const channelCount = recording.stream.channels;
const expectedSamples = recordingDuration * actualSampleRate;
- console.log(`💾 Saving recording for ${app.name}:`);
- console.log(`- Process ID: ${app.processId}`);
- console.log(`- Bundle ID: ${app.bundleIdentifier}`);
+ if (recording.isGlobal) {
+ console.log('💾 Saving global recording:');
+ } else {
+ const appName = app?.name ?? 'Unknown App';
+ const processId = app?.processId ?? 0;
+ const bundleId = app?.bundleIdentifier ?? 'unknown';
+ console.log(`💾 Saving recording for ${appName}:`);
+ if (app) {
+ console.log(`- Process ID: ${processId}`);
+ console.log(`- Bundle ID: ${bundleId}`);
+ }
+ }
+
console.log(`- Actual duration: ${recordingDuration.toFixed(2)}s`);
console.log(`- Sample rate: ${actualSampleRate}Hz`);
console.log(`- Channels: ${channelCount}`);
@@ -156,7 +168,9 @@ async function saveRecording(recording: Recording): Promise<string | null> {
await fs.ensureDir(RECORDING_DIR);
const timestamp = Date.now();
- const baseFilename = `${recording.app.bundleIdentifier}-${recording.app.processId}-${timestamp}`;
+ const baseFilename = recording.isGlobal
+ ? `global-recording-${timestamp}`
+ : `${app?.bundleIdentifier ?? 'unknown'}-${app?.processId ?? 0}-${timestamp}`;
const recordingDir = `${RECORDING_DIR}/${baseFilename}`;
await fs.ensureDir(recordingDir);
@@ -189,7 +203,7 @@ async function saveRecording(recording: Recording): Promise<string | null> {
console.log('✅ Transcription MP3 file written successfully');
// Save app icon if available
- if (app.icon) {
+ if (app?.icon) {
console.log(`📝 Writing app icon to ${iconFilename}`);
await fs.writeFile(iconFilename, app.icon);
console.log('✅ App icon written successfully');
@@ -198,15 +212,16 @@ async function saveRecording(recording: Recording): Promise<string | null> {
console.log(`📝 Writing metadata to ${metadataFilename}`);
// Save metadata with the actual sample rate from the stream
const metadata: RecordingMetadata = {
- appName: app.name,
- bundleIdentifier: app.bundleIdentifier,
- processId: app.processId,
+ appName: app?.name ?? 'Global Recording',
+ bundleIdentifier: app?.bundleIdentifier ?? 'system.global',
+ processId: app?.processId ?? -1,
recordingStartTime: recording.startTime,
recordingEndTime,
recordingDuration,
sampleRate: actualSampleRate,
channels: channelCount,
totalSamples,
+ isGlobal: recording.isGlobal,
};
await fs.writeJson(metadataFilename, metadata, { spaces: 2 });
@@ -222,8 +237,8 @@ async function saveRecording(recording: Recording): Promise<string | null> {
function getRecordingStatus(): RecordingStatus[] {
return Array.from(recordingMap.entries()).map(([processId, recording]) => ({
processId,
- bundleIdentifier: recording.app.bundleIdentifier,
- name: recording.app.name,
+ bundleIdentifier: recording.app?.bundleIdentifier ?? 'system.global',
+ name: recording.app?.name ?? 'Global Recording',
startTime: recording.startTime,
duration: Date.now() - recording.startTime,
}));
@@ -289,8 +304,11 @@ async function stopRecording(processId: number) {
}
const app = recording.appGroup || recording.app;
+ const appName =
+ app?.name ?? (recording.isGlobal ? 'Global Recording' : 'Unknown App');
+ const appPid = app?.processId ?? processId;
- console.log(`⏹️ Stopping recording for ${app.name} (PID: ${app.processId})`);
+ console.log(`⏹️ Stopping recording for ${appName} (PID: ${appPid})`);
console.log(
`⏱️ Recording duration: ${((Date.now() - recording.startTime) / 1000).toFixed(2)}s`
);
@@ -302,7 +320,7 @@ async function stopRecording(processId: number) {
if (filename) {
console.log(`✅ Recording saved successfully to ${filename}`);
} else {
- console.error(`❌ Failed to save recording for ${app.name}`);
+ console.error(`❌ Failed to save recording for ${appName}`);
}
emitRecordingStatus();
@@ -541,7 +559,13 @@ function listenToAppStateChanges(apps: AppInfo[]) {
appsSubscriber();
appsSubscriber = () => {
- subscribers.forEach(subscriber => subscriber.unsubscribe());
+ subscribers.forEach(subscriber => {
+ try {
+ subscriber.unsubscribe();
+ } catch {
+ // ignore unsubscribe error
+ }
+ });
};
}
@@ -606,8 +630,8 @@ app.get('/apps/saved', rateLimiter, async (_req, res) => {
// Utility function to validate and sanitize folder name
function validateAndSanitizeFolderName(folderName: string): string | null {
// Allow alphanumeric characters, hyphens, dots (for bundle IDs)
- // Format: bundleId-processId-timestamp
- if (!/^[\w.-]+-\d+-\d+$/.test(folderName)) {
+ // Format: bundleId-processId-timestamp OR global-recording-timestamp
+ if (!/^([\w.-]+-\d+-\d+|global-recording-\d+)$/.test(folderName)) {
return null;
}
@@ -788,6 +812,65 @@ app.post(
}
);
+async function startGlobalRecording() {
+ const GLOBAL_RECORDING_ID = -1;
+ if (recordingMap.has(GLOBAL_RECORDING_ID)) {
+ console.log('⚠️ Global recording already in progress');
+ return;
+ }
+
+ try {
+ console.log('🎙️ Starting global recording');
+
+ const buffers: Float32Array[] = [];
+ const stream = ShareableContent.tapGlobalAudio(
+ null,
+ (err: Error | null, samples: Float32Array) => {
+ if (err) {
+ console.error('❌ Global audio stream error:', err);
+ return;
+ }
+ const recording = recordingMap.get(GLOBAL_RECORDING_ID);
+ if (recording && !recording.isWriting) {
+ buffers.push(new Float32Array(samples));
+ }
+ }
+ );
+
+ recordingMap.set(GLOBAL_RECORDING_ID, {
+ app: null,
+ appGroup: null,
+ buffers,
+ stream,
+ startTime: Date.now(),
+ isWriting: false,
+ isGlobal: true,
+ });
+
+ console.log('✅ Global recording started successfully');
+ emitRecordingStatus();
+ } catch (error) {
+ console.error('❌ Error starting global recording:', error);
+ }
+}
+
+// Add API endpoint for global recording
+app.post('/global/record', async (_req, res) => {
+ try {
+ await startGlobalRecording();
+ res.json({ success: true });
+ } catch (error) {
+ console.error('❌ Error starting global recording:', error);
+ res.status(500).json({ error: 'Failed to start global recording' });
+ }
+});
+
+app.post('/global/stop', async (_req, res) => {
+ const GLOBAL_RECORDING_ID = -1;
+ await stopRecording(GLOBAL_RECORDING_ID);
+ res.json({ success: true });
+});
+
// Start server
httpServer.listen(PORT, () => {
console.log(`
diff --git a/packages/frontend/media-capture-playground/web/app.tsx b/packages/frontend/media-capture-playground/web/app.tsx
index a3aaa7ff4d..7b8987f5e6 100644
--- a/packages/frontend/media-capture-playground/web/app.tsx
+++ b/packages/frontend/media-capture-playground/web/app.tsx
@@ -1,4 +1,5 @@
import { AppList } from './components/app-list';
+import { GlobalRecordButton } from './components/global-record-button';
import { SavedRecordings } from './components/saved-recordings';
export function App() {
@@ -6,11 +7,15 @@ export function App() {
<div className="h-screen bg-gray-50 overflow-hidden">
<div className="h-full p-4 flex gap-4 max-w-[1800px] mx-auto">
<div className="flex-1 flex flex-col min-h-0">
- <h1 className="text-xl font-bold text-gray-900 mb-1">
- Running Applications
- </h1>
+ <div className="flex justify-between items-center mb-1">
+ <h1 className="text-xl font-bold text-gray-900">
+ Running Applications
+ </h1>
+ <GlobalRecordButton />
+ </div>
<p className="text-sm text-gray-500 mb-2">
- Select an application to start recording its audio
+ Select an application to start recording its audio, or use global
+ recording for system-wide audio
</p>
<div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 overflow-auto">
<AppList />
diff --git a/packages/frontend/media-capture-playground/web/components/global-record-button.tsx b/packages/frontend/media-capture-playground/web/components/global-record-button.tsx
new file mode 100644
index 0000000000..6884b8b010
--- /dev/null
+++ b/packages/frontend/media-capture-playground/web/components/global-record-button.tsx
@@ -0,0 +1,70 @@
+import { useCallback, useEffect, useState } from 'react';
+
+import { socket } from '../utils';
+
+export function GlobalRecordButton() {
+ const [isRecording, setIsRecording] = useState(false);
+ const [isLoading, setIsLoading] = useState(false);
+
+ useEffect(() => {
+ function handleRecordingStatus(data: {
+ recordings: Array<{ processId: number }>;
+ }) {
+ // Global recording uses processId -1
+ setIsRecording(data.recordings.some(r => r.processId === -1));
+ }
+
+ socket.on('apps:recording', handleRecordingStatus);
+ return () => {
+ socket.off('apps:recording', handleRecordingStatus);
+ };
+ }, []);
+
+ const handleClick = useCallback(() => {
+ setIsLoading(true);
+ const endpoint = isRecording ? '/api/global/stop' : '/api/global/record';
+ fetch(endpoint, { method: 'POST' })
+ .then(response => {
+ if (!response.ok) {
+ throw new Error('Failed to toggle global recording');
+ }
+ })
+ .catch(error => {
+ console.error('Error toggling global recording:', error);
+ })
+ .finally(() => {
+ setIsLoading(false);
+ });
+ }, [isRecording]);
+
+ return (
+ <button
+ onClick={handleClick}
+ disabled={isLoading}
+ className={`
+ px-4 py-2 rounded-lg font-medium text-sm
+ transition-colors duration-200
+ ${isLoading ? 'opacity-50 cursor-not-allowed' : ''}
+ ${
+ isRecording
+ ? 'bg-red-100 text-red-700 hover:bg-red-200'
+ : 'bg-blue-100 text-blue-700 hover:bg-blue-200'
+ }
+ `}
+ >
+ <div className="flex items-center gap-2">
+ {isRecording ? (
+ <>
+ <div className="w-2 h-2 rounded-full bg-red-500 animate-pulse" />
+ Stop Global Recording
+ </>
+ ) : (
+ <>
+ <div className="w-2 h-2 rounded-full bg-blue-500" />
+ Record System Audio
+ </>
+ )}
+ </div>
+ </button>
+ );
+}
diff --git a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx
index 217d300d49..61da0adfd4 100644
--- a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx
+++ b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx
@@ -378,11 +378,12 @@ function RecordingHeader({
transcriptionError: string | null;
}): ReactElement {
const [imgError, setImgError] = React.useState(false);
+ const isGlobalRecording = metadata?.isGlobal;
return (
<div className="flex items-start space-x-4 p-4 bg-gray-50/30">
<div className="relative w-12 h-12 flex-shrink-0">
- {!imgError ? (
+ {!imgError && !isGlobalRecording ? (
<img
src={`/api/recordings/${fileName}/icon.png`}
alt={metadata?.appName || 'Unknown Application'}
@@ -401,7 +402,12 @@ function RecordingHeader({
<span className="text-gray-900 font-semibold text-base truncate">
{metadata?.appName || 'Unknown Application'}
</span>
- <span className="text-xs px-2 py-0.5 bg-blue-50 rounded-full text-blue-600 font-medium border border-blue-100">
+ {isGlobalRecording && (
+ <span className="text-xs px-2 py-0.5 bg-blue-50 rounded-full text-blue-600 font-medium border border-blue-100">
+ System Audio
+ </span>
+ )}
+ <span className="text-xs px-2 py-0.5 bg-gray-50 rounded-full text-gray-600 font-medium border border-gray-100">
{duration}
</span>
</div>
diff --git a/packages/frontend/media-capture-playground/web/types.ts b/packages/frontend/media-capture-playground/web/types.ts
index ced4bd4e23..c892c419a0 100644
--- a/packages/frontend/media-capture-playground/web/types.ts
+++ b/packages/frontend/media-capture-playground/web/types.ts
@@ -17,6 +17,8 @@ export interface RecordingStatus {
bundleIdentifier: string;
name: string;
startTime: number;
+ duration: number;
+ isGlobal?: boolean;
}
export interface RecordingMetadata {
@@ -31,6 +33,7 @@ export interface RecordingMetadata {
totalSamples: number;
icon?: Uint8Array;
mp3: string;
+ isGlobal?: boolean;
}
export interface TranscriptionMetadata {
diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rs
index 1004b7dfef..cfce74c201 100644
--- a/packages/frontend/native/media_capture/src/macos/tap_audio.rs
+++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs
@@ -1,4 +1,4 @@
-use std::{ffi::c_void, sync::Arc};
+use std::{ffi::c_void, ptr, sync::Arc};
use block2::{Block, RcBlock};
use core_foundation::{
@@ -13,12 +13,15 @@ use coreaudio::sys::{
kAudioAggregateDeviceIsPrivateKey, kAudioAggregateDeviceIsStackedKey,
kAudioAggregateDeviceMainSubDeviceKey, kAudioAggregateDeviceNameKey,
kAudioAggregateDeviceSubDeviceListKey, kAudioAggregateDeviceTapAutoStartKey,
- kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey, kAudioHardwareNoError,
- kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultSystemOutputDevice,
- kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey, kAudioSubTapUIDKey,
- AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID, AudioDeviceIOProcID,
- AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice,
- AudioHardwareDestroyAggregateDevice, AudioObjectID, AudioTimeStamp, OSStatus,
+ kAudioAggregateDeviceTapListKey, kAudioAggregateDeviceUIDKey,
+ kAudioDevicePropertyAvailableNominalSampleRates, kAudioDevicePropertyNominalSampleRate,
+ kAudioHardwareNoError, kAudioHardwarePropertyDefaultInputDevice,
+ kAudioHardwarePropertyDefaultSystemOutputDevice, kAudioObjectPropertyElementMain,
+ kAudioObjectPropertyScopeGlobal, kAudioSubDeviceUIDKey, kAudioSubTapDriftCompensationKey,
+ kAudioSubTapUIDKey, AudioDeviceCreateIOProcIDWithBlock, AudioDeviceDestroyIOProcID,
+ AudioDeviceIOProcID, AudioDeviceStart, AudioDeviceStop, AudioHardwareCreateAggregateDevice,
+ AudioHardwareDestroyAggregateDevice, AudioObjectGetPropertyData, AudioObjectGetPropertyDataSize,
+ AudioObjectID, AudioObjectPropertyAddress, AudioObjectSetPropertyData, AudioTimeStamp, OSStatus,
};
use napi::{
bindgen_prelude::Float32Array,
@@ -30,8 +33,11 @@ use objc2::{runtime::AnyObject, Encode, Encoding, RefEncode};
use crate::{
audio_stream_basic_desc::read_audio_stream_basic_description,
- ca_tap_description::CATapDescription, device::get_device_uid, error::CoreAudioError,
- queue::create_audio_tap_queue, screen_capture_kit::TappableApplication,
+ ca_tap_description::CATapDescription,
+ device::{get_device_audio_id, get_device_uid},
+ error::CoreAudioError,
+ queue::create_audio_tap_queue,
+ screen_capture_kit::TappableApplication,
};
extern "C" {
@@ -53,6 +59,14 @@ pub struct AudioBuffer {
pub mData: *mut c_void,
}
+// Define a struct to represent sample rate ranges
+#[repr(C)]
+#[allow(non_snake_case)]
+struct AudioValueRange {
+ mMinimum: f64,
+ mMaximum: f64,
+}
+
unsafe impl Encode for AudioBuffer {
const ENCODING: Encoding = Encoding::Struct(
"AudioBuffer",
@@ -94,6 +108,10 @@ pub struct AggregateDevice {
pub tap_id: AudioObjectID,
pub id: AudioObjectID,
pub audio_stats: Option<AudioStats>,
+ pub input_device_id: Option<AudioObjectID>,
+ pub output_device_id: Option<AudioObjectID>,
+ pub input_proc_id: Option<AudioDeviceIOProcID>,
+ pub output_proc_id: Option<AudioDeviceIOProcID>,
}
impl AggregateDevice {
@@ -128,6 +146,10 @@ impl AggregateDevice {
tap_id,
id: aggregate_device_id,
audio_stats: None,
+ input_device_id: None,
+ output_device_id: None,
+ input_proc_id: None,
+ output_proc_id: None,
})
}
@@ -160,6 +182,10 @@ impl AggregateDevice {
tap_id,
id: aggregate_device_id,
audio_stats: None,
+ input_device_id: None,
+ output_device_id: None,
+ input_proc_id: None,
+ output_proc_id: None,
})
}
@@ -173,6 +199,12 @@ impl AggregateDevice {
return Err(CoreAudioError::CreateProcessTapFailed(status).into());
}
+ // Get the default input device (microphone) UID and ID
+ let input_device_id = get_device_audio_id(kAudioHardwarePropertyDefaultInputDevice)?;
+
+ // Get the default output device ID
+ let output_device_id = get_device_audio_id(kAudioHardwarePropertyDefaultSystemOutputDevice)?;
+
let description_dict = Self::create_aggregate_description(tap_id, tap_description.get_uuid()?)?;
let mut aggregate_device_id: AudioObjectID = 0;
@@ -189,30 +221,246 @@ impl AggregateDevice {
return Err(CoreAudioError::CreateAggregateDeviceFailed(status).into());
}
- Ok(Self {
+ // Create a device with stored device IDs
+ let mut device = Self {
tap_id,
id: aggregate_device_id,
audio_stats: None,
- })
+ input_device_id: Some(input_device_id),
+ output_device_id: Some(output_device_id),
+ input_proc_id: None,
+ output_proc_id: None,
+ };
+
+ // Configure the aggregate device to ensure proper handling of both input and
+ // output
+ device.configure_aggregate_device()?;
+
+ // Activate both the input and output devices and store their proc IDs
+ let input_proc_id = device.activate_audio_device(input_device_id)?;
+ let output_proc_id = device.activate_audio_device(output_device_id)?;
+
+ device.input_proc_id = Some(input_proc_id);
+ device.output_proc_id = Some(output_proc_id);
+
+ Ok(device)
}
- pub fn start(
- &mut self,
- audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,
- ) -> Result<AudioTapStream> {
- // Read and log the audio format before starting the device
+ // Configures the aggregate device to ensure proper handling of both input and
+ // output streams
+ fn configure_aggregate_device(&self) -> Result<AudioStats> {
+ // Read the current audio format to ensure it's properly configured
+ let audio_format = read_audio_stream_basic_description(self.tap_id)?;
+
+ // Create initial audio stats with the actual sample rate but always use mono
+ let initial_sample_rate = audio_format.0.mSampleRate;
let mut audio_stats = AudioStats {
- sample_rate: 44100.0,
+ sample_rate: initial_sample_rate,
channels: 1, // Always set to 1 channel (mono)
};
- if let Ok(audio_format) = read_audio_stream_basic_description(self.tap_id) {
- // Store the audio format information
- audio_stats.sample_rate = audio_format.0.mSampleRate;
- // Always use 1 channel regardless of what the system reports
- audio_stats.channels = 1;
+ // Set the preferred sample rate on the device
+ // This is similar to how Screen Capture Kit allows setting the sample rate
+ let preferred_sample_rate = initial_sample_rate; // Use the device's current sample rate
+
+ // First, check if the preferred sample rate is available
+ let mut is_sample_rate_available = false;
+ let mut best_available_rate = preferred_sample_rate; // Default to preferred rate
+
+ unsafe {
+ // Get the available sample rates
+ let address = AudioObjectPropertyAddress {
+ mSelector: kAudioDevicePropertyAvailableNominalSampleRates,
+ mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain,
+ };
+
+ // Get the size of the property data
+ let mut data_size: u32 = 0;
+ let status = AudioObjectGetPropertyDataSize(
+ self.id,
+ &address as *const AudioObjectPropertyAddress,
+ 0,
+ std::ptr::null(),
+ &mut data_size as *mut u32,
+ );
+
+ if status == 0 && data_size > 0 {
+ // Calculate how many ranges we have
+ let range_count = data_size as usize / std::mem::size_of::<AudioValueRange>();
+
+ // Allocate memory for the ranges
+ let mut ranges: Vec<AudioValueRange> = Vec::with_capacity(range_count);
+ ranges.set_len(range_count);
+
+ // Get the available sample rates
+ let status = AudioObjectGetPropertyData(
+ self.id,
+ &address as *const AudioObjectPropertyAddress,
+ 0,
+ std::ptr::null(),
+ &mut data_size as *mut u32,
+ ranges.as_mut_ptr() as *mut std::ffi::c_void,
+ );
+
+ if status == 0 {
+ // Check if our preferred sample rate is within any of the available ranges
+ for range in &ranges {
+ if preferred_sample_rate >= range.mMinimum && preferred_sample_rate <= range.mMaximum {
+ is_sample_rate_available = true;
+ break;
+ }
+ }
+
+ // If not available, find the best available rate
+ if !is_sample_rate_available && !ranges.is_empty() {
+ // Common preferred sample rates in order of preference
+ let common_rates = [48000.0, 44100.0, 96000.0, 88200.0, 24000.0, 22050.0];
+ let mut found_common_rate = false;
+
+ // First try to find a common rate that's available
+ for &rate in &common_rates {
+ for range in &ranges {
+ if rate >= range.mMinimum && rate <= range.mMaximum {
+ best_available_rate = rate;
+ found_common_rate = true;
+ break;
+ }
+ }
+ if found_common_rate {
+ break;
+ }
+ }
+
+ // If no common rate is available, use the highest available rate
+ if !found_common_rate {
+ // Find the highest available rate
+ for range in &ranges {
+ // Use the maximum of the range as our best available rate
+ if range.mMaximum > best_available_rate {
+ best_available_rate = range.mMaximum;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Set the sample rate to either the preferred rate or the best available rate
+ let sample_rate_to_set = if is_sample_rate_available {
+ preferred_sample_rate
+ } else {
+ best_available_rate
+ };
+
+ let status = unsafe {
+ // Note on scope usage:
+ // We use kAudioObjectPropertyScopeGlobal here because it works reliably for
+ // setting the nominal sample rate on the device. While
+ // kAudioObjectPropertyScopeInput or kAudioObjectPropertyScopeOutput might
+ // also work in some cases (as mentioned in the comments),
+ // kAudioObjectPropertyScopeGlobal is the most consistent approach.
+ //
+ // The CoreAudio documentation doesn't explicitly specify which scope to use
+ // with kAudioDevicePropertyNominalSampleRate, but in practice,
+ // kAudioObjectPropertyScopeGlobal ensures the sample rate is set for the
+ // entire device, affecting both input and output.
+ let address = AudioObjectPropertyAddress {
+ mSelector: kAudioDevicePropertyNominalSampleRate,
+ mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain,
+ };
+
+ // Set the sample rate property
+ AudioObjectSetPropertyData(
+ self.id,
+ &address as *const AudioObjectPropertyAddress,
+ 0,
+ std::ptr::null(),
+ std::mem::size_of::<f64>() as u32,
+ &sample_rate_to_set as *const f64 as *const std::ffi::c_void,
+ )
+ };
+
+ // Update the audio_stats with the actual sample rate that was set if successful
+ if status == 0 {
+ audio_stats.sample_rate = sample_rate_to_set;
+
+ // Verify the actual sample rate by reading it back
+ unsafe {
+ let address = AudioObjectPropertyAddress {
+ mSelector: kAudioDevicePropertyNominalSampleRate,
+ mScope: kAudioObjectPropertyScopeGlobal,
+ mElement: kAudioObjectPropertyElementMain,
+ };
+
+ let mut actual_rate: f64 = 0.0;
+ let mut data_size = std::mem::size_of::<f64>() as u32;
+
+ let status = AudioObjectGetPropertyData(
+ self.id,
+ &address as *const AudioObjectPropertyAddress,
+ 0,
+ std::ptr::null(),
+ &mut data_size as *mut u32,
+ &mut actual_rate as *mut f64 as *mut std::ffi::c_void,
+ );
+
+ if status == 0 {
+ // Update with the verified rate
+ audio_stats.sample_rate = actual_rate;
+ }
+ }
+ }
+
+ Ok(audio_stats)
+ }
+
+ // Activates an audio device by creating a dummy IO proc
+ fn activate_audio_device(&self, device_id: AudioObjectID) -> Result<AudioDeviceIOProcID> {
+ // Create a simple no-op dummy proc
+ let dummy_block = RcBlock::new(
+ |_: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void, _: *mut c_void| {
+ // No-op function that just returns success
+ kAudioHardwareNoError as i32
+ },
+ );
+
+ let mut dummy_proc_id: AudioDeviceIOProcID = None;
+
+ // Create the IO proc with our dummy block
+ let status = unsafe {
+ AudioDeviceCreateIOProcIDWithBlock(
+ &mut dummy_proc_id,
+ device_id,
+ ptr::null_mut(),
+ (&*dummy_block.copy() as *const Block<dyn Fn(_, _, _, _, _) -> i32>)
+ .cast_mut()
+ .cast(),
+ )
+ };
+
+ if status != 0 {
+ return Err(CoreAudioError::CreateIOProcIDWithBlockFailed(status).into());
+ }
+
+ // Start the device to activate it
+ let status = unsafe { AudioDeviceStart(device_id, dummy_proc_id) };
+ if status != 0 {
+ return Err(CoreAudioError::AudioDeviceStartFailed(status).into());
}
+ // Return the proc ID for later cleanup
+ Ok(dummy_proc_id)
+ }
+
+ pub fn start(
+ &mut self,
+ audio_stream_callback: Arc<ThreadsafeFunction<Float32Array, (), Float32Array, true>>,
+ ) -> Result<AudioTapStream> {
+ // Configure the aggregate device and get audio stats before starting
+ let audio_stats = self.configure_aggregate_device()?;
self.audio_stats = Some(audio_stats);
let audio_stats_clone = audio_stats;
@@ -242,7 +490,7 @@ impl AggregateDevice {
}] = mBuffers;
// Only create slice if we have valid data
if !mData.is_null() && *mDataByteSize > 0 {
- // Calculate total number of samples (accounting for interleaved stereo)
+ // Calculate total number of samples (total bytes / bytes per sample)
let total_samples = *mDataByteSize as usize / 4; // 4 bytes per f32
// Create a slice of all samples
@@ -253,27 +501,18 @@ impl AggregateDevice {
let channel_count = *mNumberChannels as usize;
// Process the audio based on channel count
- let mut processed_samples: Vec<f32>;
+ let processed_samples: Vec<f32>;
if channel_count > 1 {
// For stereo, samples are interleaved: [L, R, L, R, ...]
// We need to average each pair to get mono
- let frame_count = total_samples / channel_count;
- processed_samples = Vec::with_capacity(frame_count);
-
- for i in 0..frame_count {
- let mut frame_sum = 0.0;
- for c in 0..channel_count {
- frame_sum += samples[i * channel_count + c];
- }
- processed_samples.push(frame_sum / (channel_count as f32));
- }
+ processed_samples = process_mixed_audio(samples, channel_count);
} else {
- // Already mono, just copy the samples
+ // For mono, just copy the samples
processed_samples = samples.to_vec();
}
- // Pass the processed samples to the callback
+ // Send the processed audio data to JavaScript
audio_stream_callback.call(
Ok(processed_samples.into()),
ThreadsafeFunctionCallMode::NonBlocking,
@@ -309,7 +548,11 @@ impl AggregateDevice {
device_id: self.id,
in_proc_id,
stop_called: false,
- audio_stats: audio_stats_clone,
+ audio_stats: audio_stats_clone, // Use the updated audio_stats with the actual sample rate
+ input_device_id: self.input_device_id,
+ output_device_id: self.output_device_id,
+ input_proc_id: self.input_proc_id,
+ output_proc_id: self.output_proc_id,
})
}
@@ -325,15 +568,29 @@ impl AggregateDevice {
let aggregate_device_uid_string = aggregate_device_uid.to_string();
// Sub-device UID key and dictionary
- let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[(
- cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),
- system_output_uid.as_CFType(),
- )]);
+ let sub_device_output_dict = CFDictionary::from_CFType_pairs(&[
+ (
+ cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),
+ system_output_uid.as_CFType(),
+ ),
+ // Explicitly mark this as an output device
+ (
+ CFString::new("com.apple.audio.roles").as_CFType(),
+ CFString::new("output").as_CFType(),
+ ),
+ ]);
- let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[(
- cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),
- default_input_uid.as_CFType(),
- )]);
+ let sub_device_input_dict = CFDictionary::from_CFType_pairs(&[
+ (
+ cfstring_from_bytes_with_nul(kAudioSubDeviceUIDKey).as_CFType(),
+ default_input_uid.as_CFType(),
+ ),
+ // Explicitly mark this as an input device
+ (
+ CFString::new("com.apple.audio.roles").as_CFType(),
+ CFString::new("input").as_CFType(),
+ ),
+ ]);
let tap_device_dict = CFDictionary::from_CFType_pairs(&[
(
@@ -346,6 +603,7 @@ impl AggregateDevice {
),
]);
+ // Put input device first in the list to prioritize it
let capture_device_list = vec![sub_device_input_dict, sub_device_output_dict];
// Sub-device list
@@ -353,7 +611,8 @@ impl AggregateDevice {
let tap_list = CFArray::from_CFTypes(&[tap_device_dict]);
- // Create the aggregate device description dictionary
+ // Create the aggregate device description dictionary with a balanced
+ // configuration
let description_dict = CFDictionary::from_CFType_pairs(&[
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceNameKey).as_CFType(),
@@ -365,7 +624,9 @@ impl AggregateDevice {
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceMainSubDeviceKey).as_CFType(),
- system_output_uid.as_CFType(),
+ // Use a balanced approach that includes both input and output
+ // but prioritize input for microphone capture
+ default_input_uid.as_CFType(),
),
(
cfstring_from_bytes_with_nul(kAudioAggregateDeviceIsPrivateKey).as_CFType(),
@@ -398,6 +659,10 @@ pub struct AudioTapStream {
in_proc_id: AudioDeviceIOProcID,
stop_called: bool,
audio_stats: AudioStats,
+ input_device_id: Option<AudioObjectID>,
+ output_device_id: Option<AudioObjectID>,
+ input_proc_id: Option<AudioDeviceIOProcID>,
+ output_proc_id: Option<AudioDeviceIOProcID>,
}
#[napi]
@@ -408,22 +673,47 @@ impl AudioTapStream {
return Ok(());
}
self.stop_called = true;
+
+ // Stop the main aggregate device
let status = unsafe { AudioDeviceStop(self.device_id, self.in_proc_id) };
if status != 0 {
return Err(CoreAudioError::AudioDeviceStopFailed(status).into());
}
+
+ // Stop the input device if it was activated
+ if let Some(input_id) = self.input_device_id {
+ if let Some(proc_id) = self.input_proc_id {
+ let _ = unsafe { AudioDeviceStop(input_id, proc_id) };
+ let _ = unsafe { AudioDeviceDestroyIOProcID(input_id, proc_id) };
+ }
+ }
+
+ // Stop the output device if it was activated
+ if let Some(output_id) = self.output_device_id {
+ if let Some(proc_id) = self.output_proc_id {
+ let _ = unsafe { AudioDeviceStop(output_id, proc_id) };
+ let _ = unsafe { AudioDeviceDestroyIOProcID(output_id, proc_id) };
+ }
+ }
+
+ // Destroy the main IO proc
let status = unsafe { AudioDeviceDestroyIOProcID(self.device_id, self.in_proc_id) };
if status != 0 {
return Err(CoreAudioError::AudioDeviceDestroyIOProcIDFailed(status).into());
}
+
+ // Destroy the aggregate device
let status = unsafe { AudioHardwareDestroyAggregateDevice(self.device_id) };
if status != 0 {
return Err(CoreAudioError::AudioHardwareDestroyAggregateDeviceFailed(status).into());
}
+
+ // Destroy the process tap
let status = unsafe { AudioHardwareDestroyProcessTap(self.device_id) };
if status != 0 {
return Err(CoreAudioError::AudioHardwareDestroyProcessTapFailed(status).into());
}
+
Ok(())
}
@@ -445,3 +735,21 @@ fn cfstring_from_bytes_with_nul(bytes: &'static [u8]) -> CFString {
.as_ref(),
)
}
+
+// Process mixed audio from multiple channels
+fn process_mixed_audio(samples: &[f32], channel_count: usize) -> Vec<f32> {
+ // For stereo or multi-channel audio, we need to mix down to mono
+ let samples_per_channel = samples.len() / channel_count;
+ let mut mixed_samples = Vec::with_capacity(samples_per_channel);
+
+ for i in 0..samples_per_channel {
+ let mut sample_sum = 0.0;
+ for c in 0..channel_count {
+ sample_sum += samples[i * channel_count + c];
+ }
+ // Average the samples from all channels
+ mixed_samples.push(sample_sum / channel_count as f32);
+ }
+
+ mixed_samples
+}
diff --git a/tests/affine-desktop/playwright.config.ts b/tests/affine-desktop/playwright.config.ts
index e28f551d15..323b52e5db 100644
--- a/tests/affine-desktop/playwright.config.ts
+++ b/tests/affine-desktop/playwright.config.ts
@@ -29,7 +29,7 @@ const config: PlaywrightTestConfig = {
if (process.env.CI) {
config.retries = 5;
- config.workers = 2;
+ config.workers = 1;
}
if (process.env.DEV_SERVER_URL) {
diff --git a/tests/kit/src/electron.ts b/tests/kit/src/electron.ts
index 64a3762d61..02bce1683a 100644
--- a/tests/kit/src/electron.ts
+++ b/tests/kit/src/electron.ts
@@ -1,4 +1,5 @@
import crypto from 'node:crypto';
+import { setTimeout } from 'node:timers/promises';
import { Package } from '@affine-tools/utils/workspace';
import { expect, type Page } from '@playwright/test';
@@ -71,7 +72,7 @@ export const test = base.extend<{
return electronApp.windows().length > 1;
},
{
- timeout: 50000,
+ timeout: 10000,
}
)
.toBeTruthy();
@@ -83,7 +84,7 @@ export const test = base.extend<{
return !!page;
},
{
- timeout: 50000,
+ timeout: 10000,
}
)
.toBeTruthy();
@@ -147,12 +148,25 @@ export const test = base.extend<{
});
await use(electronApp);
- const pages = electronApp.windows();
- for (const page of pages) {
- await page.close();
- }
- await electronApp.close();
- await removeWithRetry(clonedDist);
+ const cleanup = async () => {
+ const pages = electronApp.windows();
+ for (const page of pages) {
+ if (page.isClosed()) {
+ continue;
+ }
+ await page.close();
+ }
+ await electronApp.close();
+ await removeWithRetry(clonedDist);
+ };
+ await Promise.race([
+ // cleanup may stuck and fail the test, but it should be fine.
+ cleanup(),
+ setTimeout(10000).then(() => {
+ // kill the electron app if it is not closed after 10 seconds
+ electronApp.process().kill();
+ }),
+ ]);
} catch (error) {
console.log(error);
}
diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts
index 443d9eeb74..5b6df13d07 100644
--- a/tools/utils/src/workspace.gen.ts
+++ b/tools/utils/src/workspace.gen.ts
@@ -733,6 +733,7 @@ export const PackageList = [
'packages/frontend/electron-api',
'packages/frontend/i18n',
'packages/common/nbstore',
+ 'blocksuite/affine/all',
'packages/common/infra',
'tools/utils',
],
diff --git a/yarn.lock b/yarn.lock
index aec8ceeb57..b83142a87a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -496,6 +496,7 @@ __metadata:
"@affine/electron-api": "workspace:*"
"@affine/i18n": "workspace:*"
"@affine/nbstore": "workspace:*"
+ "@blocksuite/affine": "workspace:*"
"@emotion/react": "npm:^11.14.0"
"@sentry/react": "npm:^9.2.0"
"@toeverything/infra": "workspace:*"