Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(sdk-recorder): retain microphone audio when sharing tab sound
Akshar Patelcbe18d66bd04
diff --git a/packages/sdk-recorder/__tests__/audio-mixing.test.ts b/packages/sdk-recorder/__tests__/audio-mixing.test.tsnew file mode 100644index 000000000..cb582c48e--- /dev/null+++ b/packages/sdk-recorder/__tests__/audio-mixing.test.ts@@ -0,0 +1,205 @@+import { afterEach, beforeEach, expect, mock, test } from "bun:test";+import {+ buildCompositeStream,+ type ManagedStreams,+} from "../src/core/stream-manager";+import { CapRecorder } from "../src/index";++class Track {+ stop = mock(() => {});++ constructor(readonly kind: "audio" | "video") {}+}++class Stream {+ constructor(private readonly tracks: Track[]) {}++ getTracks() {+ return this.tracks;+ }++ getAudioTracks() {+ return this.tracks.filter((track) => track.kind === "audio");+ }++ getVideoTracks() {+ return this.tracks.filter((track) => track.kind === "video");+ }+}++class Mixer {+ static instances: Mixer[] = [];+ readonly inputs: Track[] = [];+ readonly output = new Track("audio");+ readonly destination = { stream: new Stream([this.output]) };+ resume = mock(async () => {});+ close = mock(async () => {});++ constructor() {+ Mixer.instances.push(this);+ }++ createMediaStreamDestination() {+ return this.destination;+ }++ createMediaStreamSource(stream: Stream) {+ return {+ connect: () => {+ this.inputs.push(...stream.getAudioTracks());+ },+ };+ }+}++const globals = ["MediaStream", "AudioContext", "MediaRecorder", "navigator"];+const descriptors = globals.map(+ (name) => [name, Object.getOwnPropertyDescriptor(globalThis, name)] as const,+);++const setGlobal = (name: string, value: unknown) => {+ Object.defineProperty(globalThis, name, { configurable: true, value });+};++const capture = (systemAudio: boolean, microphone: boolean) => {+ const video = new Track("video");+ const system = new Track("audio");+ const mic = new Track("audio");+ const display = new Stream(systemAudio ? [video, system] : [video]);+ const micStream = microphone ? new Stream([mic]) : null;+ const streams = {+ display,+ camera: null,+ microphone: micStream,+ } as unknown as ManagedStreams;+ setGlobal("navigator", {+ mediaDevices: {+ getDisplayMedia: async () => display,+ getUserMedia: async () => micStream,+ },+ });+ return { streams, video, system, mic };+};++beforeEach(() => {+ Mixer.instances = [];+ setGlobal("MediaStream", Stream);+ setGlobal("AudioContext", Mixer);+});++afterEach(() => {+ for (const [name, descriptor] of descriptors) {+ if (descriptor) Object.defineProperty(globalThis, name, descriptor);+ else Reflect.deleteProperty(globalThis, name);+ }+});++test("records both audio sources through one track and releases the mixer", async () => {+ const { video, system, mic } = capture(true, true);+ let input: MediaStream | undefined;+ setGlobal(+ "MediaRecorder",+ class {+ static isTypeSupported() {+ return true;+ }++ state = "recording";++ constructor(stream: MediaStream) {+ input = stream;+ }++ start() {}+ stop() {+ this.state = "inactive";+ }+ },+ );+ const recorder = new CapRecorder({+ publicKey: "test-public-key",+ mode: "tab",+ camera: { enabled: false },+ systemAudio: true,+ });+ try {+ await recorder.start();+ const mixer = Mixer.instances[0];+ expect(recorder.phase).toBe("recording");+ expect(input?.getVideoTracks()).toEqual([video]);+ expect(input?.getAudioTracks()).toEqual([mixer?.output]);+ expect(mixer?.inputs).toEqual([system, mic]);+ } finally {+ recorder.destroy();+ }+ const mixer = Mixer.instances[0];+ expect(video.stop).toHaveBeenCalled();+ expect(system.stop).toHaveBeenCalled();+ expect(mic.stop).toHaveBeenCalled();+ expect(mixer?.output.stop).toHaveBeenCalled();+ expect(mixer?.close).toHaveBeenCalledTimes(1);+});++test.each([+ [false, false],+ [true, false],+ [false, true],+])(+ "preserves direct audio tracks for system=%s microphone=%s",+ async (systemAudio, microphone) => {+ const { streams, video, system, mic } = capture(systemAudio, microphone);+ const composite = await buildCompositeStream(streams);+ expect(composite.getVideoTracks()).toEqual([video]);+ expect(composite.getAudioTracks()).toEqual(+ systemAudio ? [system] : microphone ? [mic] : [],+ );+ expect(Mixer.instances).toHaveLength(0);+ },+);++test.each(["encoder", "audio"] as const)(+ "cleans up both audio sources when %s initialization fails",+ async (failure) => {+ const { video, system, mic } = capture(true, true);+ if (failure === "audio") {+ setGlobal(+ "AudioContext",+ class extends Mixer {+ resume = mock(async () => {+ throw new Error("Audio initialization failed");+ });+ },+ );+ }+ setGlobal(+ "MediaRecorder",+ class {+ static isTypeSupported() {+ return true;+ }++ constructor() {+ throw new Error("Encoder initialization failed");+ }+ },+ );+ const recorder = new CapRecorder({+ publicKey: "test-public-key",+ mode: "tab",+ camera: { enabled: false },+ systemAudio: true,+ });+ await expect(recorder.start()).rejects.toThrow(+ failure === "audio"+ ? "Audio initialization failed"+ : "Encoder initialization failed",+ );+ expect(recorder.phase).toBe("error");+ expect(video.stop).toHaveBeenCalled();+ expect(system.stop).toHaveBeenCalled();+ expect(mic.stop).toHaveBeenCalled();+ const mixer = Mixer.instances[0];+ expect(mixer?.output.stop).toHaveBeenCalled();+ expect(mixer?.close).toHaveBeenCalledTimes(1);+ },+);diff --git a/packages/sdk-recorder/package.json b/packages/sdk-recorder/package.jsonindex 7fef27857..d318a0e45 100644--- a/packages/sdk-recorder/package.json+++ b/packages/sdk-recorder/package.json@@ -20,6 +20,7 @@],"scripts": {"build": "tsup",+ "test": "bun test","typecheck": "tsc --noEmit"},"dependencies": {},diff --git a/packages/sdk-recorder/src/core/stream-manager.ts b/packages/sdk-recorder/src/core/stream-manager.tsindex 7c39f6b3a..f6a0d1690 100644--- a/packages/sdk-recorder/src/core/stream-manager.ts+++ b/packages/sdk-recorder/src/core/stream-manager.ts@@ -11,6 +11,7 @@ export interface ManagedStreams {display: MediaStream | null;camera: MediaStream | null;microphone: MediaStream | null;+ audioMix?: { context: AudioContext; stream: MediaStream };}export async function acquireStreams(@@ -69,16 +70,30 @@ export async function acquireStreams(}export function releaseStreams(streams: ManagedStreams) {- for (const stream of Object.values(streams)) {+ const { audioMix } = streams;+ for (const stream of [+ streams.display,+ streams.camera,+ streams.microphone,+ audioMix?.stream,+ ]) {if (stream) {for (const track of stream.getTracks()) {track.stop();}}}+ if (audioMix) {+ void audioMix.context.close().catch((error) => {+ console.error("CapRecorder: failed to close audio mixer:", error);+ });+ streams.audioMix = undefined;+ }}-export function buildCompositeStream(streams: ManagedStreams): MediaStream {+export async function buildCompositeStream(+ streams: ManagedStreams,+): Promise<MediaStream> {const tracks: MediaStreamTrack[] = [];if (streams.display) {@@ -102,5 +117,21 @@ export function buildCompositeStream(streams: ManagedStreams): MediaStream {}}- return new MediaStream(tracks);+ const audioTracks = tracks.filter((track) => track.kind === "audio");+ if (audioTracks.length <= 1) return new MediaStream(tracks);++ const context = new AudioContext();+ const destination = context.createMediaStreamDestination();+ streams.audioMix = { context, stream: destination.stream };+ for (const track of audioTracks) {+ context+ .createMediaStreamSource(new MediaStream([track]))+ .connect(destination);+ }+ await context.resume();++ return new MediaStream([+ ...tracks.filter((track) => track.kind !== "audio"),+ ...destination.stream.getAudioTracks(),+ ]);}diff --git a/packages/sdk-recorder/src/index.ts b/packages/sdk-recorder/src/index.tsindex 0f7eb10e1..193882099 100644--- a/packages/sdk-recorder/src/index.ts+++ b/packages/sdk-recorder/src/index.ts@@ -14,10 +14,10 @@ import type {import { MultipartClient } from "./upload/multipart-client";export type {+ RecorderEventMap,RecorderOptions,RecorderPhase,RecordingResult,- RecorderEventMap,};const DEFAULT_API_BASE = "https://cap.so";@@ -124,7 +124,7 @@ export class CapRecorder {systemAudio: this.options.systemAudio,});- const composite = buildCompositeStream(this.streams);+ const composite = await buildCompositeStream(this.streams);if (composite.getTracks().length === 0) {throw new Error("No media tracks available");}