Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(web): support TTS playback in Safari and Firefox (#39444)
eux00fe96325d18
diff --git a/oxlint-suppressions.json b/oxlint-suppressions.jsonindex fc02eaf6753..fb2c46efaf5 100644--- a/oxlint-suppressions.json+++ b/oxlint-suppressions.json@@ -835,14 +835,6 @@"count": 1}},- "web/app/components/base/audio-btn/audio.ts": {- "node-js/prefer-global/buffer": {- "count": 1- },- "typescript/no-explicit-any": {- "count": 3- }- },"web/app/components/base/audio-gallery/AudioPlayer.tsx": {"jsx_a11y/media-has-caption": {"count": 1diff --git a/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts b/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.tsindex a93e7e50383..60023038339 100644--- a/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts+++ b/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts@@ -12,14 +12,8 @@ type AudioPlayerCtorArgs = [type MockAudioPlayerInstance = {setCallback: ReturnType<typeof vi.fn>- pauseAudio: ReturnType<typeof vi.fn>+ destroy: ReturnType<typeof vi.fn>resetMsgId: ReturnType<typeof vi.fn>- cacheBuffers: Array<ArrayBuffer>- sourceBuffer:- | {- abort: ReturnType<typeof vi.fn>- }- | undefined}const mockState = vi.hoisted(() => ({@@ -31,10 +25,8 @@ const mockAudioPlayerConstructor = vi.hoisted(() => vi.fn())const MockAudioPlayer = vi.hoisted(() => {return class MockAudioPlayerClass {setCallback = vi.fn()- pauseAudio = vi.fn()+ destroy = vi.fn()resetMsgId = vi.fn()- cacheBuffers = [new ArrayBuffer(1)]- sourceBuffer = { abort: vi.fn() }constructor(...args: AudioPlayerCtorArgs) {mockAudioPlayerConstructor(...args)@@ -132,9 +124,7 @@ describe('AudioPlayerManager', () => {callback,)- expect(previous!.pauseAudio).toHaveBeenCalledTimes(1)- expect(previous!.cacheBuffers).toEqual([])- expect(previous!.sourceBuffer?.abort).toHaveBeenCalledTimes(1)+ expect(previous!.destroy).toHaveBeenCalledTimes(1)expect(mockAudioPlayerConstructor).toHaveBeenCalledTimes(2)expect(next).toBe(mockState.instances[1])})@@ -144,7 +134,7 @@ describe('AudioPlayerManager', () => {const callback = vi.fn()manager.getAudioPlayer('/text-to-audio', false, 'msg-1', 'hello', 'en-US', callback)const previous = mockState.instances[0]- previous!.pauseAudio.mockImplementation(() => {+ previous!.destroy.mockImplementation(() => {throw new Error('cleanup failure')})@@ -152,7 +142,7 @@ describe('AudioPlayerManager', () => {manager.getAudioPlayer('/apps/1/text-to-audio', false, 'msg-2', 'world', 'en-US', callback)}).not.toThrow()- expect(previous!.pauseAudio).toHaveBeenCalledTimes(1)+ expect(previous!.destroy).toHaveBeenCalledTimes(1)expect(mockAudioPlayerConstructor).toHaveBeenCalledTimes(2)})})diff --git a/web/app/components/base/audio-btn/__tests__/audio.spec.ts b/web/app/components/base/audio-btn/__tests__/audio.spec.tsindex 95cd0ef97c9..ac69433dd5b 100644--- a/web/app/components/base/audio-btn/__tests__/audio.spec.ts+++ b/web/app/components/base/audio-btn/__tests__/audio.spec.ts@@ -3,15 +3,8 @@ import { waitFor } from '@testing-library/react'import { AppSourceType } from '@/service/share'import AudioPlayer from '../audio'-const mockToastNotify = vi.hoisted(() => vi.fn())const mockTextToAudioStream = vi.hoisted(() => vi.fn())-vi.mock('@langgenius/dify-ui/toast', () => ({- toast: {- error: (message: string) => mockToastNotify({ type: 'error', message }),- },-}))-vi.mock('@/service/share', () => ({AppSourceType: {webApp: 'webApp',@@ -22,7 +15,7 @@ vi.mock('@/service/share', () => ({type AudioEventName =| 'ended'- | 'paused'+ | 'pause'| 'loaded'| 'play'| 'timeupdate'@@ -30,6 +23,7 @@ type AudioEventName =| 'canplay'| 'error'| 'sourceopen'+ | 'updateend'type AudioEventListener = () => void@@ -51,12 +45,31 @@ type AudioResponse = {class MockSourceBuffer {updating = false+ private listeners: Partial<Record<AudioEventName, AudioEventListener[]>> = {}++ addEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => {+ const listeners = this.listeners[event] || []+ listeners.push(listener)+ this.listeners[event] = listeners+ })++ removeEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => {+ this.listeners[event] = (this.listeners[event] || []).filter((item) => item !== listener)+ })+appendBuffer = vi.fn((_buffer: ArrayBuffer) => undefined)abort = vi.fn(() => undefined)++ emit(event: AudioEventName) {+ const listeners = this.listeners[event] || []+ listeners.forEach((listener) => {+ listener()+ })+ }}class MockMediaSource {- readyState: 'open' | 'closed' = 'open'+ readyState: 'open' | 'closed' | 'ended' = 'closed'sourceBuffer = new MockSourceBuffer()private listeners: Partial<Record<AudioEventName, AudioEventListener[]>> = {}@@ -66,10 +79,15 @@ class MockMediaSource {this.listeners[event] = listeners})+ removeEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => {+ this.listeners[event] = (this.listeners[event] || []).filter((item) => item !== listener)+ })+addSourceBuffer = vi.fn((_contentType: string) => this.sourceBuffer)endOfStream = vi.fn(() => undefined)emit(event: AudioEventName) {+ if (event === 'sourceopen') this.readyState = 'open'const listeners = this.listeners[event] || []listeners.forEach((listener) => {listener()@@ -110,7 +128,7 @@ class MockAudio {}class MockAudioContext {- state: 'running' | 'suspended' = 'running'+ state: 'interrupted' | 'running' | 'suspended' = 'running'destination = {}connect = vi.fn(() => undefined)createMediaElementSource = vi.fn((_audio: MockAudio) => ({@@ -121,9 +139,11 @@ class MockAudioContext {this.state = 'running'})- suspend = vi.fn(() => {+ suspend = vi.fn(async () => {this.state = 'suspended'})++ close = vi.fn(async () => undefined)}const testState = {@@ -133,6 +153,8 @@ const testState = {}class MockMediaSourceCtor extends MockMediaSource {+ static isTypeSupported = vi.fn(() => true)+constructor() {super()testState.mediaSources.push(this)@@ -156,6 +178,7 @@ class MockAudioContextCtor extends MockAudioContext {const originalAudio = globalThis.Audioconst originalAudioContext = globalThis.AudioContextconst originalCreateObjectURL = globalThis.URL.createObjectURL+const originalRevokeObjectURL = globalThis.URL.revokeObjectURLconst originalMediaSource = window.MediaSourceconst originalManagedMediaSource = window.ManagedMediaSource@@ -192,6 +215,7 @@ describe('AudioPlayer', () => {testState.mediaSources = []testState.audios = []testState.audioContexts = []+ MockMediaSourceCtor.isTypeSupported.mockReturnValue(true)Object.defineProperty(globalThis, 'Audio', {configurable: true,@@ -208,6 +232,11 @@ describe('AudioPlayer', () => {writable: true,value: vi.fn(() => 'blob:mock-url'),})+ Object.defineProperty(globalThis.URL, 'revokeObjectURL', {+ configurable: true,+ writable: true,+ value: vi.fn(),+ })setMediaSourceSupport({ mediaSource: true, managedMediaSource: false })})@@ -228,6 +257,11 @@ describe('AudioPlayer', () => {writable: true,value: originalCreateObjectURL,})+ Object.defineProperty(globalThis.URL, 'revokeObjectURL', {+ configurable: true,+ writable: true,+ value: originalRevokeObjectURL,+ })Object.defineProperty(window, 'MediaSource', {configurable: true,writable: true,@@ -256,7 +290,7 @@ describe('AudioPlayer', () => {expect(audioContext!.connect).toHaveBeenCalledTimes(1)})- it('should notify unsupported browser when no MediaSource implementation exists', () => {+ it('should use complete-audio fallback when no MediaSource implementation exists', () => {setMediaSourceSupport({ mediaSource: false, managedMediaSource: false })const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)@@ -264,12 +298,22 @@ describe('AudioPlayer', () => {expect(player.mediaSource).toBeNull()expect(audio!.src).toBe('')- expect(mockToastNotify).toHaveBeenCalledTimes(1)- expect(mockToastNotify).toHaveBeenCalledWith(- expect.objectContaining({- type: 'error',- }),- )+ expect(audio!.autoplay).toBe(false)+ expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled()+ })++ it('should use complete-audio fallback when MP3 MediaSource is unsupported', () => {+ MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)++ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)+ const audio = testState.audios[0]++ expect(MockMediaSourceCtor.isTypeSupported).toHaveBeenCalledWith('audio/mpeg')+ expect(player.mediaSource).toBeNull()+ expect(testState.mediaSources).toHaveLength(0)+ expect(audio!.src).toBe('')+ expect(audio!.autoplay).toBe(false)+ expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled()})it('should configure fallback audio controls when ManagedMediaSource is used', () => {@@ -283,6 +327,17 @@ describe('AudioPlayer', () => {expect(audio!.disableRemotePlayback).toBe(true)expect(audio!.controls).toBe(true)})++ it('should configure ManagedMediaSource when both media source implementations exist', () => {+ setMediaSourceSupport({ mediaSource: true, managedMediaSource: true })++ const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, vi.fn())+ const audio = testState.audios[0]++ expect(player.mediaSource).not.toBeNull()+ expect(audio!.disableRemotePlayback).toBe(true)+ expect(audio!.controls).toBe(true)+ })})describe('event wiring', () => {@@ -294,7 +349,7 @@ describe('AudioPlayer', () => {audio!.emit('play')audio!.emit('ended')audio!.emit('error')- audio!.emit('paused')+ audio!.emit('pause')audio!.emit('loaded')audio!.emit('timeupdate')audio!.emit('loadeddate')@@ -354,6 +409,7 @@ describe('AudioPlayer', () => {})it('should emit error callback and reset load flag when stream response status is not 200', async () => {+ MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)const callback = vi.fn()mockTextToAudioStream.mockResolvedValue(makeAudioResponse(500, [{ value: new Uint8Array([1]), done: true }]),@@ -366,25 +422,171 @@ describe('AudioPlayer', () => {expect(callback).toHaveBeenCalledWith('error')})expect(player.isLoadData).toBe(false)+ expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled()+ expect(testState.audios[0]!.play).not.toHaveBeenCalled()})- it('should resume and play immediately when playAudio is called in suspended loaded state', async () => {+ it('should play a complete MP3 blob when MediaSource does not support audio/mpeg', async () => {+ MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)+ const callback = vi.fn()+ mockTextToAudioStream.mockResolvedValue(+ makeAudioResponse(200, [+ { value: new Uint8Array([1, 2]), done: false },+ { value: new Uint8Array([3, 4]), done: true },+ ]),+ )++ const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)+ const audio = testState.audios[0]++ player.playAudio()++ await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))+ expect(player.mediaSource).toBeNull()+ expect(player.cacheBuffers).toHaveLength(0)+ expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(1)+ const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[0]![0] as Blob+ expect(audioBlob).toBeInstanceOf(Blob)+ expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 4 })+ expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3, 4]))+ expect(audio!.src).toBe('blob:mock-url')+ expect(callback).toHaveBeenCalledWith('play')+ })++ it('should wait for the complete MP3 before retrying playback without MediaSource', async () => {+ MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)+ let resolveResponse: ((response: AudioResponse) => void) | undefined+ mockTextToAudioStream.mockImplementationOnce(+ () =>+ new Promise<AudioResponse>((resolve) => {+ resolveResponse = resolve+ }),+ )++ const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, vi.fn())+ const audio = testState.audios[0]++ player.playAudio()+ player.playAudio()++ expect(audio!.play).not.toHaveBeenCalled()++ resolveResponse?.(makeAudioResponse(200, [{ value: new Uint8Array([1, 2]), done: true }]))+ await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))+ })++ it.each(['suspended', 'interrupted'] as const)(+ 'should resume and play immediately when playAudio is called in %s loaded state',+ async (audioContextState) => {+ const callback = vi.fn()+ const player = new AudioPlayer(+ '/text-to-audio',+ false,+ 'msg-1',+ 'hello',+ undefined,+ callback,+ )+ const audio = testState.audios[0]+ const audioContext = testState.audioContexts[0]++ player.isLoadData = true+ audioContext!.state = audioContextState+ player.playAudio()++ await waitFor(() => {+ expect(audioContext!.resume).toHaveBeenCalledTimes(1)+ expect(audio!.play).toHaveBeenCalledTimes(1)+ expect(callback).toHaveBeenCalledWith('play')+ })+ },+ )++ it('should request media playback before a suspended audio context finishes resuming', async () => {const callback = vi.fn()const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)const audio = testState.audios[0]const audioContext = testState.audioContexts[0]+ let resolveResume: (() => void) | undefinedplayer.isLoadData = trueaudioContext!.state = 'suspended'+ audioContext!.resume.mockImplementationOnce(+ () =>+ new Promise<void>((resolve) => {+ resolveResume = () => {+ audioContext!.state = 'running'+ resolve()+ }+ }),+ )+player.playAudio()- await Promise.resolve()expect(audioContext!.resume).toHaveBeenCalledTimes(1)expect(audio!.play).toHaveBeenCalledTimes(1)- expect(callback).toHaveBeenCalledWith('play')+ expect(callback).not.toHaveBeenCalledWith('play')++ resolveResume?.()+ await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))})- it('should play ended audio when data is already loaded', () => {+ it.each(['suspended', 'interrupted'] as const)(+ 'should resume a %s audio context when the media element is still playing',+ async (audioContextState) => {+ const callback = vi.fn()+ const player = new AudioPlayer(+ '/text-to-audio',+ false,+ 'msg-1',+ 'hello',+ undefined,+ callback,+ )+ const audio = testState.audios[0]+ const audioContext = testState.audioContexts[0]++ player.isLoadData = true+ audio!.paused = false+ audioContext!.state = audioContextState+ player.playAudio()++ await waitFor(() => {+ expect(audioContext!.resume).toHaveBeenCalledTimes(1)+ expect(callback).toHaveBeenCalledWith('play')+ })+ expect(audio!.play).not.toHaveBeenCalled()+ },+ )++ it('should report an error when the audio context remains interrupted and allow retry', async () => {+ const callback = vi.fn()+ const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)+ const audio = testState.audios[0]+ const audioContext = testState.audioContexts[0]++ player.isLoadData = true+ audio!.paused = false+ audioContext!.state = 'suspended'+ audioContext!.resume.mockImplementationOnce(async () => {+ audioContext!.state = 'interrupted'+ })+ player.playAudio()++ await waitFor(() => expect(callback).toHaveBeenCalledWith('error'))+ expect(callback).not.toHaveBeenCalledWith('play')++ audioContext!.resume.mockImplementationOnce(async () => {+ audioContext!.state = 'running'+ })+ player.playAudio()++ await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))+ expect(audioContext!.resume).toHaveBeenCalledTimes(2)+ expect(audio!.play).not.toHaveBeenCalled()+ })++ it('should play ended audio when data is already loaded', async () => {const callback = vi.fn()const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)const audio = testState.audios[0]@@ -395,11 +597,13 @@ describe('AudioPlayer', () => {audio!.ended = trueplayer.playAudio()- expect(audio!.play).toHaveBeenCalledTimes(1)- expect(callback).toHaveBeenCalledWith('play')+ await waitFor(() => {+ expect(audio!.play).toHaveBeenCalledTimes(1)+ expect(callback).toHaveBeenCalledWith('play')+ })})- it('should only emit play callback without replaying when loaded audio is already playing', () => {+ it('should report loaded audio that is already playing without replaying it', () => {const callback = vi.fn()const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)const audio = testState.audios[0]@@ -407,6 +611,7 @@ describe('AudioPlayer', () => {player.isLoadData = trueaudioContext!.state = 'running'+ audio!.paused = falseaudio!.ended = falseplayer.playAudio()@@ -451,22 +656,20 @@ describe('AudioPlayer', () => {})it('should end stream without playback when playAudioWithAudio receives empty content', async () => {- vi.useFakeTimers()- try {- const callback = vi.fn()- const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)- const mediaSource = testState.mediaSources[0]+ const callback = vi.fn()+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)+ const mediaSource = testState.mediaSources[0]++ await player.playAudioWithAudio('', true)- await player.playAudioWithAudio('', true)- await vi.advanceTimersByTimeAsync(40)+ expect(player.isLoadData).toBe(false)+ expect(player.cacheBuffers).toHaveLength(0)+ expect(mediaSource!.endOfStream).not.toHaveBeenCalled()- expect(player.isLoadData).toBe(false)- expect(player.cacheBuffers).toHaveLength(0)- expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)- expect(callback).not.toHaveBeenCalledWith('play')- } finally {- vi.useRealTimers()- }+ mediaSource!.emit('sourceopen')++ expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)+ expect(callback).not.toHaveBeenCalledWith('play')})it('should decode base64 and start playback when playAudioWithAudio is called with playable content', async () => {@@ -479,8 +682,8 @@ describe('AudioPlayer', () => {mediaSource!.emit('sourceopen')audio!.paused = true+ audioContext!.state = 'suspended'await player.playAudioWithAudio(audioBase64, true)- await Promise.resolve()expect(player.isLoadData).toBe(true)expect(player.cacheBuffers).toHaveLength(0)@@ -488,9 +691,11 @@ describe('AudioPlayer', () => {const appendedAudioData = mediaSource!.sourceBuffer.appendBuffer.mock.calls[0]![0]expect(appendedAudioData).toBeInstanceOf(ArrayBuffer)expect(appendedAudioData.byteLength).toBeGreaterThan(0)- expect(audioContext!.resume).toHaveBeenCalledTimes(1)- expect(audio!.play).toHaveBeenCalledTimes(1)- expect(callback).toHaveBeenCalledWith('play')+ await waitFor(() => {+ expect(audioContext!.resume).toHaveBeenCalledTimes(1)+ expect(audio!.play).toHaveBeenCalledTimes(1)+ expect(callback).toHaveBeenCalledWith('play')+ })})it('should skip playback when playAudioWithAudio is called with play=false', async () => {@@ -507,6 +712,88 @@ describe('AudioPlayer', () => {expect(callback).not.toHaveBeenCalledWith('play')})+ it('should combine automatic TTS chunks into a playable MP3 blob without MediaSource', async () => {+ MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)+ const callback = vi.fn()+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)+ const audio = testState.audios[0]++ await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)+ await player.playAudioWithAudio(Buffer.from([3, 4]).toString('base64'), true)++ expect(audio!.play).not.toHaveBeenCalled()+ expect(player.cacheBuffers).toHaveLength(2)++ await player.playAudioWithAudio('', false)++ await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))+ expect(player.cacheBuffers).toHaveLength(0)+ expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(1)+ const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[0]![0] as Blob+ expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 4 })+ expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3, 4]))+ expect(audio!.src).toBe('blob:mock-url')+ expect(callback).toHaveBeenCalledWith('play')+ })++ it('should not start fallback playback after it is paused while buffering', async () => {+ MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', vi.fn())+ const audio = testState.audios[0]++ await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)+ player.pauseAudio()+ await player.playAudioWithAudio('', false)++ expect(audio!.autoplay).toBe(false)+ expect(audio!.play).not.toHaveBeenCalled()+ expect(audio!.src).toBe('blob:mock-url')+ })++ it('should fall back to a complete MP3 when addSourceBuffer throws', async () => {+ const callback = vi.fn()+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)+ const mediaSource = testState.mediaSources[0]+ const audio = testState.audios[0]+ mediaSource!.addSourceBuffer.mockImplementationOnce(() => {+ throw new DOMException('Unsupported type', 'NotSupportedError')+ })++ mediaSource!.emit('sourceopen')+ await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)+ await player.playAudioWithAudio('', false)++ await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))+ expect(player.mediaSource).toBeNull()+ expect(audio!.autoplay).toBe(false)+ expect(globalThis.URL.revokeObjectURL).toHaveBeenCalledWith('blob:mock-url')+ expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(2)+ })++ it('should complete buffered fallback when addSourceBuffer throws after stream end', async () => {+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', vi.fn())+ const mediaSource = testState.mediaSources[0]+ const audio = testState.audios[0]+ mediaSource!.addSourceBuffer.mockImplementationOnce(() => {+ throw new DOMException('Unsupported type', 'NotSupportedError')+ })++ await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)+ await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))+ await player.playAudioWithAudio('', false)+ audio!.paused = true++ mediaSource!.emit('sourceopen')++ await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(2))+ expect(player.mediaSource).toBeNull()+ expect(player.cacheBuffers).toHaveLength(0)+ expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(2)+ const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[1]![0] as Blob+ expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 2 })+ expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2]))+ })+it('should play immediately for ended audio in playAudioWithAudio', async () => {const callback = vi.fn()const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)@@ -517,7 +804,7 @@ describe('AudioPlayer', () => {await player.playAudioWithAudio(Buffer.from('hello').toString('base64'), true)expect(audio!.play).toHaveBeenCalledTimes(1)- expect(callback).toHaveBeenCalledWith('play')+ await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))})it('should not replay when played list exists in playAudioWithAudio', async () => {@@ -534,18 +821,63 @@ describe('AudioPlayer', () => {expect(callback).not.toHaveBeenCalledWith('play')})- it('should replay when paused is false and played list is empty in playAudioWithAudio', async () => {+ it('should report a play failure and retry without requesting audio again', async () => {const callback = vi.fn()const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)const audio = testState.audios[0]+ const mediaSource = testState.mediaSources[0]+ mockTextToAudioStream.mockResolvedValue(+ makeAudioResponse(200, [{ value: undefined, done: true }]),+ )+ audio!.play.mockRejectedValueOnce(new DOMException('Playback aborted', 'AbortError'))- audio!.paused = false- audio!.ended = false- audio!.played = null- await player.playAudioWithAudio(Buffer.from('hello').toString('base64'), true)+ mediaSource!.emit('sourceopen')+ player.playAudio()++ await waitFor(() => expect(callback).toHaveBeenCalledWith('error'))+ expect(callback).not.toHaveBeenCalledWith('play')+ expect(player.isLoadData).toBe(true)++ audio!.play.mockImplementationOnce(async () => {+ audio!.paused = false+ })+ player.playAudio()++ await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))+ expect(audio!.play).toHaveBeenCalledTimes(2)+ expect(mockTextToAudioStream).toHaveBeenCalledTimes(1)+ })++ it('should report a resume failure and allow playback to be retried', async () => {+ const callback = vi.fn()+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)+ const audio = testState.audios[0]+ const audioContext = testState.audioContexts[0]+ const mediaSource = testState.mediaSources[0]+ mockTextToAudioStream.mockResolvedValue(+ makeAudioResponse(200, [{ value: undefined, done: true }]),+ )+ audioContext!.state = 'suspended'+ audioContext!.resume.mockRejectedValueOnce(new DOMException('Not allowed', 'NotAllowedError'))++ mediaSource!.emit('sourceopen')+ player.playAudio()++ await waitFor(() => expect(callback).toHaveBeenCalledWith('error'))expect(audio!.play).toHaveBeenCalledTimes(1)- expect(callback).toHaveBeenCalledWith('play')+ expect(player.isLoadData).toBe(true)++ audioContext!.resume.mockImplementationOnce(async () => {+ audioContext!.state = 'running'+ })+ player.playAudio()++ await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))++ expect(audioContext!.resume).toHaveBeenCalledTimes(2)+ expect(audio!.play).toHaveBeenCalledTimes(1)+ expect(mockTextToAudioStream).toHaveBeenCalledTimes(1)})})@@ -562,7 +894,7 @@ describe('AudioPlayer', () => {expect(finishStream).toHaveBeenCalledTimes(1)})- it('should finish stream when receiveAudioData gets empty bytes while source is open', () => {+ it('should finish stream when receiveAudioData gets empty bytes', () => {const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)const finishStream = vi.spyOn(player as unknown as { finishStream: () => void }, 'finishStream')@@ -586,6 +918,52 @@ describe('AudioPlayer', () => {expect(player.cacheBuffers.length).toBe(1)})+ it('should preserve audio received before sourceopen and append it once ready', () => {+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)+ const mediaSource = testState.mediaSources[0]++ ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(+ new Uint8Array([1, 2, 3]),+ )++ expect(player.cacheBuffers).toHaveLength(1)+ expect(mediaSource!.sourceBuffer.appendBuffer).not.toHaveBeenCalled()++ mediaSource!.emit('sourceopen')++ expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1)+ expect(player.cacheBuffers).toHaveLength(0)+ })++ it('should append queued buffers in order after updateend', () => {+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)+ const mediaSource = testState.mediaSources[0]+ mediaSource!.emit('sourceopen')+ mediaSource!.sourceBuffer.updating = true++ const first = new Uint8Array([1])+ const second = new Uint8Array([2])+ ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(+ first,+ )+ ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(+ second,+ )++ mediaSource!.sourceBuffer.updating = false+ mediaSource!.sourceBuffer.emit('updateend')+ expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1)+ expect(new Uint8Array(mediaSource!.sourceBuffer.appendBuffer.mock.calls[0]![0])).toEqual(+ first,+ )++ mediaSource!.sourceBuffer.emit('updateend')+ expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(2)+ expect(new Uint8Array(mediaSource!.sourceBuffer.appendBuffer.mock.calls[1]![0])).toEqual(+ second,+ )+ })+it('should append previously queued buffer before new one when source buffer is idle', () => {const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)const mediaSource = testState.mediaSources[0]@@ -603,19 +981,68 @@ describe('AudioPlayer', () => {expect(player.cacheBuffers.length).toBe(1)})- it('should append cache chunks and end stream when finishStream drains buffers', () => {- vi.useFakeTimers()+ it('should end the stream only after the final queued buffer is appended', () => {const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)const mediaSource = testState.mediaSources[0]mediaSource!.emit('sourceopen')- mediaSource!.sourceBuffer.updating = false+ mediaSource!.sourceBuffer.updating = trueplayer.cacheBuffers = [new ArrayBuffer(3)]+;(player as unknown as { finishStream: () => void }).finishStream()- vi.advanceTimersByTime(50)+ expect(mediaSource!.endOfStream).not.toHaveBeenCalled()++ mediaSource!.sourceBuffer.updating = false+ mediaSource!.sourceBuffer.emit('updateend')expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1)+ expect(mediaSource!.endOfStream).not.toHaveBeenCalled()++ mediaSource!.sourceBuffer.emit('updateend')+ expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)+ })++ it('should end an open stream at most once', () => {+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)+ const mediaSource = testState.mediaSources[0]+ mediaSource!.emit('sourceopen')++ ;(player as unknown as { finishStream: () => void }).finishStream()+ ;(player as unknown as { finishStream: () => void }).finishStream()+expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)- vi.useRealTimers()+ })++ it.each(['closed', 'ended'] as const)('should not end a %s media source', (readyState) => {+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)+ const mediaSource = testState.mediaSources[0]+ mediaSource!.emit('sourceopen')+ mediaSource!.readyState = readyState++ ;(player as unknown as { finishStream: () => void }).finishStream()++ expect(mediaSource!.endOfStream).not.toHaveBeenCalled()+ })++ it('should stop buffering and release browser resources after destroy', async () => {+ const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)+ const mediaSource = testState.mediaSources[0]+ const audio = testState.audios[0]+ const audioContext = testState.audioContexts[0]+ mediaSource!.emit('sourceopen')++ player.destroy()+ ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(+ new Uint8Array([1]),+ )+ ;(player as unknown as { finishStream: () => void }).finishStream()+ mediaSource!.sourceBuffer.emit('updateend')+ await Promise.resolve()++ expect(mediaSource!.sourceBuffer.appendBuffer).not.toHaveBeenCalled()+ expect(mediaSource!.endOfStream).not.toHaveBeenCalled()+ expect(audio!.pause).toHaveBeenCalledTimes(1)+ expect(audioContext!.close).toHaveBeenCalledTimes(1)+ expect(globalThis.URL.revokeObjectURL).toHaveBeenCalledWith('blob:mock-url')})})})diff --git a/web/app/components/base/audio-btn/audio.player.manager.ts b/web/app/components/base/audio-btn/audio.player.manager.tsindex 59fb1faee34..fefeac7ca82 100644--- a/web/app/components/base/audio-btn/audio.player.manager.ts+++ b/web/app/components/base/audio-btn/audio.player.manager.ts@@ -35,9 +35,7 @@ export class AudioPlayerManager {} else {if (this.audioPlayers) {try {- this.audioPlayers.pauseAudio()- this.audioPlayers.cacheBuffers = []- this.audioPlayers.sourceBuffer?.abort()+ this.audioPlayers.destroy()} catch {}}diff --git a/web/app/components/base/audio-btn/audio.ts b/web/app/components/base/audio-btn/audio.tsindex be115023d9f..123c768f1f1 100644--- a/web/app/components/base/audio-btn/audio.ts+++ b/web/app/components/base/audio-btn/audio.ts@@ -1,19 +1,19 @@-import { toast } from '@langgenius/dify-ui/toast'import { AppSourceType, textToAudioStream } from '@/service/share'+const AUDIO_CONTENT_TYPE = 'audio/mpeg'+declare global {// oxlint-disable-next-line typescript/consistent-type-definitionsinterface Window {- ManagedMediaSource: any+ ManagedMediaSource?: typeof MediaSource}}export default class AudioPlayer {mediaSource: MediaSource | nullaudio: HTMLAudioElementaudioContext: AudioContext- sourceBuffer?: any+ sourceBuffer?: SourceBuffercacheBuffers: ArrayBuffer[] = []- pauseTimer: number | null = nullmsgId: string | undefinedmsgContent: string | null | undefined = nullvoice: string | undefined = undefined@@ -21,6 +21,13 @@ export default class AudioPlayer {url: stringisPublic: booleancallback: ((event: string) => void) | null+ private objectUrl = ''+ private streamEnded = false+ private endOfStreamCalled = false+ private destroyed = false+ private playbackPending = false+ private playWhenReady = false+ private sourceOpenListener?: () => voidconstructor(streamUrl: string,isPublic: boolean,@@ -37,25 +44,26 @@ export default class AudioPlayer {this.voice = voicethis.callback = callback// Compatible with iphone ios17 ManagedMediaSource- const MediaSource = window.ManagedMediaSource || window.MediaSource- if (!MediaSource) {- toast.error(- 'Your browser does not support audio streaming, if you are using an iPhone, please update to iOS 17.1 or later.',- )- }- this.mediaSource = MediaSource ? new MediaSource() : null+ const MediaSourceConstructor = window.ManagedMediaSource || window.MediaSource+ const isManagedMediaSource = Boolean(+ window.ManagedMediaSource && MediaSourceConstructor === window.ManagedMediaSource,+ )+ const supportsStreaming = Boolean(MediaSourceConstructor?.isTypeSupported?.(AUDIO_CONTENT_TYPE))+ this.mediaSource =+ supportsStreaming && MediaSourceConstructor ? new MediaSourceConstructor() : nullthis.audio = new Audio()this.setCallback(callback)- if (!window.MediaSource) {+ if (this.mediaSource && isManagedMediaSource) {// if use ManagedMediaSourcethis.audio.disableRemotePlayback = truethis.audio.controls = true}- this.audio.src = this.mediaSource ? URL.createObjectURL(this.mediaSource) : ''- this.audio.autoplay = true+ this.listenMediaSource(AUDIO_CONTENT_TYPE)+ this.objectUrl = this.mediaSource ? URL.createObjectURL(this.mediaSource) : ''+ this.audio.src = this.objectUrl+ this.audio.autoplay = Boolean(this.mediaSource)const source = this.audioContext.createMediaElementSource(this.audio)source.connect(this.audioContext.destination)- this.listenMediaSource('audio/mpeg')}public resetMsgId(msgId: string) {@@ -63,10 +71,77 @@ export default class AudioPlayer {}private listenMediaSource(contentType: string) {- this.mediaSource?.addEventListener('sourceopen', () => {- if (this.sourceBuffer) return- this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType)- })+ this.sourceOpenListener = () => {+ if (this.destroyed || this.sourceBuffer) return+ try {+ this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType)+ this.sourceBuffer?.addEventListener('updateend', this.flushBuffers)+ this.flushBuffers()+ } catch {+ this.mediaSource = null+ this.audio.autoplay = false+ this.releaseObjectUrl()+ if (this.streamEnded) this.finishBlobAudio()+ }+ }+ this.mediaSource?.addEventListener('sourceopen', this.sourceOpenListener)+ }++ private flushBuffers = () => {+ if (+ this.destroyed ||+ !this.sourceBuffer ||+ this.sourceBuffer.updating ||+ this.mediaSource?.readyState !== 'open'+ )+ return++ const nextBuffer = this.cacheBuffers.shift()+ if (nextBuffer) {+ this.sourceBuffer.appendBuffer(nextBuffer)+ return+ }++ if (this.streamEnded && !this.endOfStreamCalled) {+ this.endOfStreamCalled = true+ this.mediaSource.endOfStream()+ }+ }++ private requestPlayback(reportIfPlaying = false) {+ if (this.destroyed || this.playbackPending) return+ if (!this.isAudioContextPaused() && !this.audio.paused && !this.audio.ended) {+ if (reportIfPlaying) this.callback?.('play')+ return+ }++ this.playbackPending = true+ void this.resumeAndPlay()+ }++ private isAudioContextPaused() {+ return this.audioContext.state === 'suspended' || this.audioContext.state === 'interrupted'+ }++ private async resumeAndPlay() {+ try {+ const pendingOperations: Promise<unknown>[] = []+ if (this.isAudioContextPaused()) pendingOperations.push(this.audioContext.resume())+ if (this.audio.paused || this.audio.ended) pendingOperations.push(this.audio.play())++ await Promise.all(pendingOperations)+ if (this.destroyed) return+ if (this.isAudioContextPaused()) {+ this.callback?.('error')+ return+ }++ if (!this.destroyed) this.callback?.('play')+ } catch {+ if (!this.destroyed) this.callback?.('error')+ } finally {+ this.playbackPending = false+ }}public setCallback(callback: ((event: string) => void) | null) {@@ -80,7 +155,7 @@ export default class AudioPlayer {false,)this.audio.addEventListener(- 'paused',+ 'pause',() => {callback('paused')},@@ -133,7 +208,7 @@ export default class AudioPlayer {private async loadAudio() {try {- const audioResponse: any = await textToAudioStream(+ const audioResponse = (await textToAudioStream(this.url,this.isPublic ? AppSourceType.webApp : AppSourceType.installedApp,{ content_type: 'audio/mpeg' },@@ -143,19 +218,21 @@ export default class AudioPlayer {voice: this.voice,text: this.msgContent,},- )+ )) as Responseif (audioResponse.status !== 200) {this.isLoadData = false- if (this.callback) this.callback('error')+ this.callback?.('error')+ return}+ if (!audioResponse.body) throw new Error('Audio response body is missing')const reader = audioResponse.body.getReader()while (true) {const { value, done } = await reader.read()+ if (value?.byteLength) this.receiveAudioData(value)if (done) {- this.receiveAudioData(value)+ this.finishStream()break}- this.receiveAudioData(value)}} catch {this.isLoadData = false@@ -166,46 +243,29 @@ export default class AudioPlayer {// play audiopublic playAudio() {if (this.isLoadData) {- if (this.audioContext.state === 'suspended') {- this.audioContext.resume().then((_) => {- this.audio.play()- this.callback?.('play')- })- } else if (this.audio.ended) {- this.audio.play()- this.callback?.('play')+ if (!this.mediaSource && !this.objectUrl) {+ this.playWhenReady = true+ return}- this.callback?.('play')+ this.requestPlayback(true)} else {this.isLoadData = true- this.audioContext.resume().then((_) => {- this.audio.play()- this.callback?.('play')- })+ this.playWhenReady = true+ if (this.mediaSource) this.requestPlayback(true)+ else if (this.isAudioContextPaused()) void this.audioContext.resume().catch(() => {})this.loadAudio()}}- private theEndOfStream() {- const endTimer = setInterval(() => {- if (!this.sourceBuffer?.updating) {- this.mediaSource?.endOfStream()- clearInterval(endTimer)- }- }, 10)- }-private finishStream() {- const timer = setInterval(() => {- if (!this.cacheBuffers.length) {- this.theEndOfStream()- clearInterval(timer)- }- if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {- const arrayBuffer = this.cacheBuffers.shift()!- this.sourceBuffer?.appendBuffer(arrayBuffer)- }- }, 10)+ if (this.destroyed) return+ this.streamEnded = true+ if (this.mediaSource) {+ this.flushBuffers()+ return+ }++ this.finishBlobAudio()}public async playAudioWithAudio(audio: string, play = true) {@@ -213,54 +273,82 @@ export default class AudioPlayer {this.finishStream()return}- const audioContent = Buffer.from(audio, 'base64')- this.receiveAudioData(new Uint8Array(audioContent))+ const audioContent = Uint8Array.from(atob(audio), (char) => char.charCodeAt(0))+ this.receiveAudioData(audioContent)if (play) {this.isLoadData = true- if (this.audio.paused) {- this.audioContext.resume().then((_) => {- this.audio.play()- this.callback?.('play')- })- } else if (this.audio.ended) {- this.audio.play()- this.callback?.('play')- } else if (this.audio.played) {- /* empty */- } else {- this.audio.play()- this.callback?.('play')- }+ this.playWhenReady = true+ if (this.mediaSource) this.requestPlayback()}}public pauseAudio() {+ this.playWhenReady = falsethis.callback?.('paused')this.audio.pause()- this.audioContext.suspend()+ void this.audioContext.suspend().catch(() => {})}- private receiveAudioData(unit8Array: Uint8Array) {+ public destroy() {+ if (this.destroyed) return++ this.destroyed = true+ this.cacheBuffers = []+ this.callback?.('paused')+ this.audio.pause()++ if (this.sourceOpenListener)+ this.mediaSource?.removeEventListener('sourceopen', this.sourceOpenListener)++ if (this.sourceBuffer) {+ this.sourceBuffer.removeEventListener('updateend', this.flushBuffers)+ if (this.mediaSource?.readyState === 'open') {+ try {+ this.sourceBuffer.abort()+ } catch {}+ }+ }++ void this.audioContext.close().catch(() => {})+ this.releaseObjectUrl()+ }++ private receiveAudioData(unit8Array: Uint8Array | undefined) {+ if (this.destroyed || this.streamEnded) returnif (!unit8Array) {this.finishStream()return}const audioData = this.byteArrayToArrayBuffer(unit8Array)if (!audioData.byteLength) {- if (this.mediaSource?.readyState === 'open') this.finishStream()+ this.finishStream()return}- if (this.sourceBuffer?.updating) {- this.cacheBuffers.push(audioData)- } else {- if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {- this.cacheBuffers.push(audioData)- const cacheBuffer = this.cacheBuffers.shift()!- this.sourceBuffer?.appendBuffer(cacheBuffer)- } else {- this.sourceBuffer?.appendBuffer(audioData)- }+ this.cacheBuffers.push(audioData)+ this.flushBuffers()+ }++ private finishBlobAudio() {+ if (!this.cacheBuffers.length) {+ if (!this.objectUrl) this.isLoadData = false+ return}++ const audioBlob = new Blob(this.cacheBuffers, { type: AUDIO_CONTENT_TYPE })+ this.cacheBuffers = []+ this.releaseObjectUrl()+ this.objectUrl = URL.createObjectURL(audioBlob)+ this.audio.src = this.objectUrl+ this.isLoadData = true+ if (this.playWhenReady) this.requestPlayback()+ }++ private releaseObjectUrl() {+ if (!this.objectUrl) return++ URL.revokeObjectURL(this.objectUrl)+ this.objectUrl = ''+ this.audio.src = ''}private byteArrayToArrayBuffer(byteArray: Uint8Array): ArrayBuffer {