Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
feat(tts): add multi-provider dubbing support
wulien3746b41287d7
diff --git a/cmd/cli/main.go b/cmd/cli/main.goindex 20bef207..9c6a9116 100644--- a/cmd/cli/main.go+++ b/cmd/cli/main.go@@ -29,9 +29,8 @@ func main() {return}if cmd.Name == "voices" {- if cmd.Voices.Provider == "" {- _ = config.LoadConfig()- }+ _ = config.LoadConfig()+ _ = config.CheckBaseConfig()writeAndExit(output, cli.Execute(context.Background(), nil, cmd))return}@@ -58,6 +57,13 @@ func main() {if err := config.CheckBaseConfig(); err != nil {writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))}+ if cmd.Name == "speech" {+ if err := config.ValidateTTSConfig(); err != nil {+ writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))+ }+ writeAndExit(output, cli.Execute(context.Background(), nil, cmd))+ return+ }if requiresTranscriptionAtStart(cmd) {if err := config.ValidateTranscriptionConfig(); err != nil {writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))@@ -72,6 +78,9 @@ func main() {}}if cmd.Name == "tts" {+ if err := config.ValidateTTSConfig(); err != nil {+ writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))+ }if err := deps.CheckTTSDependency(); err != nil {writeAndExit(output, errorResponse(err, pipeline.ErrorKindDependency))}@@ -105,20 +114,27 @@ func (w *jsonLineWriter) Write(value any) error {}func configureOpenCreatorProgress(cmd *cli.Command, output *jsonLineWriter) {- if os.Getenv("OPENCREATOR_KRILLINAI_CLI") != "1" || cmd.Name != "subtitle" {+ if os.Getenv("OPENCREATOR_KRILLINAI_CLI") != "1" {return}- previous := cmd.Subtitle.ReportProgress- cmd.Subtitle.ReportProgress = func(phase string, percent int, message string) {- if previous != nil {- previous(phase, percent, message)+ writeProgress := func(previous func(string, int, string)) func(string, int, string) {+ return func(phase string, percent int, message string) {+ if previous != nil {+ previous(phase, percent, message)+ }+ _ = output.Write(progressFrame{+ Type: "progress",+ Phase: phase,+ Percent: percent,+ Message: message,+ })}- _ = output.Write(progressFrame{- Type: "progress",- Phase: phase,- Percent: percent,- Message: message,- })+ }+ switch cmd.Name {+ case "subtitle":+ cmd.Subtitle.ReportProgress = writeProgress(cmd.Subtitle.ReportProgress)+ case "tts":+ cmd.TTS.ReportProgress = writeProgress(cmd.TTS.ReportProgress)}}diff --git a/cmd/cli/main_test.go b/cmd/cli/main_test.goindex 14296fed..b67c2e47 100644--- a/cmd/cli/main_test.go+++ b/cmd/cli/main_test.go@@ -81,6 +81,28 @@ func TestConfigureOpenCreatorProgressWritesJSONLines(t *testing.T) {}}+func TestConfigureOpenCreatorTTSProgressWritesJSONLines(t *testing.T) {+ t.Setenv("OPENCREATOR_KRILLINAI_CLI", "1")+ var buffer bytes.Buffer+ output := &jsonLineWriter{output: &buffer}+ cmd := cli.Command{Name: "tts"}++ configureOpenCreatorProgress(&cmd, output)+ if cmd.TTS.ReportProgress == nil {+ t.Fatal("TTS ReportProgress is nil")+ }+ cmd.TTS.ReportProgress("generating_voice", 64, "working")++ line := strings.TrimSpace(buffer.String())+ var frame progressFrame+ if err := json.Unmarshal([]byte(line), &frame); err != nil {+ t.Fatalf("invalid progress JSON %q: %v", line, err)+ }+ if frame.Type != "progress" || frame.Phase != "generating_voice" || frame.Percent != 64 || frame.Message != "working" {+ t.Fatalf("unexpected progress frame: %+v", frame)+ }+}+func TestConfigureOpenCreatorProgressStaysDisabledByDefault(t *testing.T) {t.Setenv("OPENCREATOR_KRILLINAI_CLI", "")cmd := cli.Command{Name: "subtitle"}diff --git a/config/config-example.toml b/config/config-example.tomlindex 7a6f1e25..2e20f223 100644--- a/config/config-example.toml+++ b/config/config-example.toml@@ -56,15 +56,10 @@base_url = "" # 留空默认海外版 https://api.minimax.io,国内可填 https://api.minimaxi.comapi_key = "" # MiniMax API密钥model = "" # 留空默认 speech-2.8-hd,可选 speech-2.8-turbo, speech-2.6-hd, speech-2.6-turbo- [tts.aliyun] # provider选aliyun这块就都要填- [tts.aliyun.oss]- access_key_id = ""- access_key_secret = ""- bucket = ""- [tts.aliyun.speech]- access_key_id = ""- access_key_secret = ""- app_key= ""+ [tts.aliyun] # 阿里云百炼语音合成,仅 API Key 必填+ base_url = "https://dashscope.aliyuncs.com/api/v1"+ api_key = ""+ model = "qwen3-tts-flash"[dubbing]min_subtitle_duration = 2.5 # 最短配音字幕时长,短句会优先合并diff --git a/config/config.go b/config/config.goindex 16a0a557..f02960c9 100644--- a/config/config.go+++ b/config/config.go@@ -73,8 +73,11 @@ type Transcribe struct {}type AliyunTtsConfig struct {- Oss AliyunOssConfig `toml:"oss"`- Speech AliyunSpeechConfig `toml:"speech"`+ BaseUrl string `toml:"base_url"`+ ApiKey string `toml:"api_key"`+ Model string `toml:"model"`+ Oss AliyunOssConfig `toml:"oss"`+ Speech AliyunSpeechConfig `toml:"speech"`}type Tts struct {@@ -155,6 +158,14 @@ var Conf = Config{Openai: OpenaiCompatibleConfig{Model: "gpt-4o-mini-tts",},+ Aliyun: AliyunTtsConfig{+ BaseUrl: "https://dashscope.aliyuncs.com/api/v1",+ Model: "qwen3-tts-flash",+ },+ Minimax: OpenaiCompatibleConfig{+ BaseUrl: "https://api.minimax.io",+ Model: "speech-2.8-hd",+ },},Dubbing: Dubbing{MinSubtitleDuration: 2.5,@@ -215,6 +226,28 @@ func ValidateTranscriptionConfig() error {return nil}+func ValidateTTSConfig() error {+ switch Conf.Tts.Provider {+ case "openai":+ if Conf.Tts.Openai.ApiKey == "" {+ return errors.New("使用 OpenAI 配音服务需要配置 API Key")+ }+ case "aliyun":+ if Conf.Tts.Aliyun.ApiKey == "" {+ return errors.New("使用阿里云百炼配音服务需要配置 API Key")+ }+ case "minimax":+ if Conf.Tts.Minimax.ApiKey == "" {+ return errors.New("使用 MiniMax 配音服务需要配置 API Key")+ }+ case "edge-tts":+ return nil+ default:+ return errors.New("不支持的配音提供商")+ }+ return nil+}+func LoadConfig() bool {var err errorconfigPath := "./config/config.toml"diff --git a/internal/cli/commands.go b/internal/cli/commands.goindex 14e11a86..6ec16bff 100644--- a/internal/cli/commands.go+++ b/internal/cli/commands.go@@ -9,6 +9,8 @@ import ("krillin-ai/config""krillin-ai/internal/pipeline"subtitlestyle "krillin-ai/internal/subtitle_style"+ "krillin-ai/internal/ttsprovider"+ "krillin-ai/internal/types""krillin-ai/internal/updater""krillin-ai/internal/voices""os"@@ -50,6 +52,7 @@ type Command struct {SubtitleStyleFile stringSubtitle pipeline.SubtitleRequestTTS pipeline.TTSRequest+ Speech SpeechRequestRender pipeline.RenderRequestCover pipeline.CoverRequestPipeline pipeline.PipelineRequest@@ -68,6 +71,17 @@ type VoicesRequest struct {Provider string}+type SpeechRequest struct {+ Text string+ TextFile string+ Output string+ Provider string+ Voice string+ Format string+ Speed float64+ Instructions string+}+func Parse(args []string) (Command, error) {if len(args) == 0 {return Command{}, errors.New("missing command")@@ -81,6 +95,8 @@ func Parse(args []string) (Command, error) {return parseSubtitle(name, args[1:])case "tts":return parseTTS(name, args[1:])+ case "speech":+ return parseSpeech(name, args[1:])case "render-horizontal":return parseRender(name, args[1:], true)case "render-vertical":@@ -136,6 +152,22 @@ Flags:--voice-clone-source <source> Optional voice clone source--dry-run Validate and write manifest without external calls-h, --help Show this help+`+ case "speech":+ return `Usage:+ krillinai-cli speech (--text <text> | --text-file <file>) --output <file> [flags]++Flags:+ --text <text> Text to synthesize+ --text-file <file> UTF-8 text file to synthesize+ --output <file> Output audio file+ --provider <name> aliyun, openai, or minimax; default current config+ --voice <voice> Provider-specific voice+ --format <format> wav or mp3; defaults from output extension+ --speed <value> Speaking rate from 0.5 to 2.0 (default 1)+ --instructions <text> Delivery instructions when supported by the provider+ --dry-run Validate command without external calls+ -h, --help Show this help`case "render-horizontal":return `Usage:@@ -225,6 +257,7 @@ Status query is a reserved/planned CLI surface in the current implementation.Commands:subtitle Generate source, target, bilingual, and short vertical subtitlestts Generate target-language dubbing from SRT subtitles+ speech Generate audio from plain textrender-horizontal Render landscape subtitle or dubbed videosrender-vertical Render portrait subtitle or dubbed videospipeline Plan or run multi-stage workflows when supported@@ -254,6 +287,8 @@ func Execute(ctx context.Context, svc pipeline.StageService, cmd Command) pipelicase "tts":resp, err := pipeline.GenerateTTS(ctx, svc, cmd.TTS)return responseWithError(resp, err)+ case "speech":+ return executeSpeech(ctx, cmd.Speech)case "render-horizontal", "render-vertical":style, err := loadSubtitleStyleForCLI(cmd.SubtitleStyleFile)if err != nil {@@ -308,7 +343,7 @@ func executeVoices(req VoicesRequest) pipeline.Response {if provider == "" {provider = currentTTSProvider()}- list, err := voices.List(provider)+ list, err := voices.List(context.Background(), provider)if err != nil {return pipeline.Response{OK: false,@@ -333,6 +368,112 @@ func executeVoices(req VoicesRequest) pipeline.Response {}}+func parseSpeech(name string, args []string) (Command, error) {+ if hasHelpArg(args) {+ return Command{Name: name, Help: true}, nil+ }+ fs := newFlagSet(name)+ text := fs.String("text", "", "text")+ textFile := fs.String("text-file", "", "text file")+ output := fs.String("output", "", "output audio")+ provider := fs.String("provider", "", "tts provider")+ voice := fs.String("voice", "", "voice")+ format := fs.String("format", "", "audio format")+ speed := fs.Float64("speed", 1, "speaking rate")+ instructions := fs.String("instructions", "", "delivery instructions")+ dryRun := fs.Bool("dry-run", false, "validate command without running external services")+ if err := fs.Parse(args); err != nil {+ return Command{}, err+ }+ if fs.NArg() != 0 {+ return Command{}, errors.New("speech does not accept positional arguments")+ }+ if (strings.TrimSpace(*text) == "") == (strings.TrimSpace(*textFile) == "") {+ return Command{}, errors.New("speech requires exactly one of --text or --text-file")+ }+ if strings.TrimSpace(*output) == "" {+ return Command{}, errors.New("speech requires --output")+ }+ if *format != "" && *format != "wav" && *format != "mp3" {+ return Command{}, errors.New("speech --format must be wav or mp3")+ }+ if *speed < 0.5 || *speed > 2 {+ return Command{}, errors.New("speech --speed must be between 0.5 and 2")+ }+ return Command{+ Name: name,+ DryRun: *dryRun,+ Speech: SpeechRequest{+ Text: *text,+ TextFile: *textFile,+ Output: *output,+ Provider: *provider,+ Voice: *voice,+ Format: *format,+ Speed: *speed,+ Instructions: *instructions,+ },+ }, nil+}++func executeSpeech(ctx context.Context, req SpeechRequest) pipeline.Response {+ provider := strings.TrimSpace(req.Provider)+ if provider == "" {+ provider = currentTTSProvider()+ }+ text := req.Text+ if req.TextFile != "" {+ data, err := os.ReadFile(req.TextFile)+ if err != nil {+ return speechFailure(provider, req.Output, "speech_input_failed", err)+ }+ text = string(data)+ }+ if strings.TrimSpace(text) == "" {+ return speechFailure(provider, req.Output, "speech_input_empty", errors.New("speech text is empty"))+ }+ client, err := ttsprovider.New(provider)+ if err != nil {+ return speechFailure(provider, req.Output, "speech_provider_unsupported", err)+ }+ if err := client.Synthesize(ctx, types.TTSSpeechOptions{+ Text: text,+ Voice: req.Voice,+ OutputFile: req.Output,+ Format: req.Format,+ Speed: req.Speed,+ Instructions: req.Instructions,+ }); err != nil {+ return speechFailure(provider, req.Output, "speech_synthesis_failed", err)+ }+ return pipeline.Response{+ OK: true,+ Stage: pipeline.StageSpeech,+ Inputs: map[string]string{+ "provider": provider,+ "voice": req.Voice,+ },+ Outputs: pipeline.Outputs{TTSAudio: req.Output},+ }+}++func speechFailure(provider, output, code string, err error) pipeline.Response {+ return pipeline.Response{+ OK: false,+ Stage: pipeline.StageSpeech,+ Inputs: map[string]string{+ "provider": provider,+ "output": output,+ },+ Error: &pipeline.Error{+ Kind: pipeline.ErrorKindRetryable,+ Code: code,+ Message: err.Error(),+ Retryable: true,+ },+ }+}+func parseUpdate(name string, args []string) (Command, error) {if hasHelpArg(args) {return Command{Name: name, Help: true}, nil@@ -582,6 +723,17 @@ func dryRun(cmd Command) pipeline.Response {return dryRunResponse(pipeline.StageSubtitle, cmd.Subtitle.Workdir, cmd.Subtitle.TaskID)case "tts":return dryRunManifest(cmd.TTS.Workdir, cmd.TTS.TaskID, pipeline.StageTTS, nil)+ case "speech":+ return pipeline.Response{+ OK: true,+ Stage: pipeline.StageSpeech,+ Inputs: map[string]string{+ "provider": cmd.Speech.Provider,+ "voice": cmd.Speech.Voice,+ "output": cmd.Speech.Output,+ "format": cmd.Speech.Format,+ },+ }case "render-horizontal":if _, err := loadSubtitleStyleForCLI(cmd.SubtitleStyleFile); err != nil {return styleLoadFailure(pipeline.StageRenderHorizontal, cmd.Render.Workdir, cmd.Render.TaskID, err)diff --git a/internal/cli/commands_test.go b/internal/cli/commands_test.goindex 0f8c1335..fe0d29c5 100644--- a/internal/cli/commands_test.go+++ b/internal/cli/commands_test.go@@ -316,8 +316,8 @@ func TestExecuteDryRunVoicesReturnsAliyunVoiceList(t *testing.T) {if len(resp.Voices) == 0 {t.Fatal("Voices is empty, want aliyun voice list")}- if !containsVoiceCode(resp.Voices, "longxiaochun_v2") {- t.Fatalf("Voices = %#v, want aliyun voice code longxiaochun_v2", resp.Voices)+ if !containsVoiceCode(resp.Voices, "Cherry") {+ t.Fatalf("Voices = %#v, want aliyun voice code Cherry", resp.Voices)}}diff --git a/internal/pipeline/subtitle_test.go b/internal/pipeline/subtitle_test.goindex dbb82347..3fe01383 100644--- a/internal/pipeline/subtitle_test.go+++ b/internal/pipeline/subtitle_test.go@@ -23,6 +23,7 @@ type fakeStageService struct {coverImageB64 stringpreparedVideoPath stringpreparedAudioPath string+ audioProgress []uint8}func (f *fakeStageService) PrepareMedia(_ context.Context, p *types.SubtitleTaskStepParam) error {@@ -39,8 +40,11 @@ func (f *fakeStageService) PrepareMedia(_ context.Context, p *types.SubtitleTaskreturn nil}-func (f *fakeStageService) GenerateSubtitlesFromAudio(context.Context, *types.SubtitleTaskStepParam) error {+func (f *fakeStageService) GenerateSubtitlesFromAudio(_ context.Context, p *types.SubtitleTaskStepParam) error {f.calls = append(f.calls, "audio")+ for _, percent := range f.audioProgress {+ p.TaskPtr.SetProgress(percent)+ }return nil}@@ -161,6 +165,45 @@ func TestGenerateSubtitlesReportsPreparedLocalMediaOutputs(t *testing.T) {}}+func TestGenerateSubtitlesMapsDetailedAudioProgress(t *testing.T) {+ dir := t.TempDir()+ fake := &fakeStageService{+ audioProgress: []uint8{22, 32, 42, 52, 65, 77, 90},+ }+ var reported []int+ req := SubtitleRequest{+ Input: "local:demo.mp4",+ Workdir: dir,+ TaskID: "demo",+ OriginLang: "zh_cn",+ TargetLang: "en",+ CaptionSource: CaptionSourceWhisper,+ ReportProgress: func(_ string, percent int, _ string) {+ reported = append(reported, percent)+ },+ }++ if _, err := GenerateSubtitles(context.Background(), fake, req); err != nil {+ t.Fatal(err)+ }++ want := []int{39, 45, 52, 58, 67, 75, 83}+ for _, percent := range want {+ if !containsInt(reported, percent) {+ t.Fatalf("reported progress %v does not contain %d", reported, percent)+ }+ }+}++func containsInt(values []int, target int) bool {+ for _, value := range values {+ if value == target {+ return true+ }+ }+ return false+}+func TestGenerateSubtitlesManualDoesNotFallback(t *testing.T) {dir := t.TempDir()fake := &fakeStageService{downloadErr: errors.New("no captions")}diff --git a/internal/pipeline/tts.go b/internal/pipeline/tts.goindex 7b63c650..71d0c019 100644--- a/internal/pipeline/tts.go+++ b/internal/pipeline/tts.go@@ -18,12 +18,14 @@ type TTSRequest struct {Video stringVoice stringVoiceCloneSource string+ ReportProgress func(phase string, percent int, message string)}func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (Response, error) {if req.LineMode == "" {req.LineMode = LineModeTargetOnly}+ reportTTSProgress(req, "preparing_voice", 10, "正在准备配音")manifest, err := ttsManifest(req)if err != nil {return ttsFailureResponse(req, nil, ErrorKindInternal, "load_manifest_failed", err), err@@ -65,6 +67,14 @@ func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (ResponsVideoWithTtsFilePath: manifest.Outputs.VideoWithTTS,TargetLanguage: types.StandardLanguageCode(manifest.TargetLanguage),}+ stepParam.TaskPtr.SetProgressReporter(func(percent uint8) {+ if percent >= 95 {+ reportTTSProgress(req, "collecting_outputs", int(percent), "正在整理配音产物")+ return+ }+ reportTTSProgress(req, "generating_voice", int(percent), "正在生成配音")+ })+ reportTTSProgress(req, "generating_voice", 20, "正在生成配音")if err := svc.GenerateSpeechFromSRT(ctx, stepParam); err != nil {return failTTSStage(req, manifest, "generate_speech_failed", err)}@@ -78,9 +88,17 @@ func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (Responsif err := manifest.Save(); err != nil {return ttsFailureResponse(req, manifest, ErrorKindInternal, "save_manifest_failed", err), err}+ reportTTSProgress(req, "collecting_outputs", 99, "正在整理配音产物")return ttsResponse(true, req, manifest, nil), nil}+func reportTTSProgress(req TTSRequest, phase string, percent int, message string) {+ if req.ReportProgress == nil {+ return+ }+ req.ReportProgress(phase, percent, message)+}+func ttsManifest(req TTSRequest) (*Manifest, error) {manifest, err := LoadManifest(req.Workdir)if err == nil {diff --git a/internal/pipeline/tts_test.go b/internal/pipeline/tts_test.goindex fe9aeb05..b1e49d67 100644--- a/internal/pipeline/tts_test.go+++ b/internal/pipeline/tts_test.go@@ -102,3 +102,39 @@ func TestGenerateTTSUsesManifestTargetSRTWhenInputEmpty(t *testing.T) {t.Fatalf("FailedIndexes = %v, want [2]", resp.FailedIndexes)}}++func TestGenerateTTSReportsVoiceGenerationProgress(t *testing.T) {+ dir := t.TempDir()+ input := filepath.Join(dir, "target.srt")+ if err := os.WriteFile(input, []byte("1\n00:00:00,000 --> 00:00:01,000\n你好\n\n"), 0644); err != nil {+ t.Fatal(err)+ }+ fake := &fakeStageService{}+ type update struct {+ phase string+ percent int+ }+ var updates []update+ req := TTSRequest{+ Workdir: dir,+ TaskID: "demo",+ InputSRT: input,+ ReportProgress: func(phase string, percent int, _ string) {+ updates = append(updates, update{phase: phase, percent: percent})+ },+ }++ if _, err := GenerateTTS(context.Background(), fake, req); err != nil {+ t.Fatalf("GenerateTTS() error = %v", err)+ }+ if len(updates) < 3 {+ t.Fatalf("progress updates = %+v, want preparing, generating, and collecting", updates)+ }+ if updates[0] != (update{phase: "preparing_voice", percent: 10}) {+ t.Fatalf("first progress = %+v, want preparing_voice/10", updates[0])+ }+ last := updates[len(updates)-1]+ if last != (update{phase: "collecting_outputs", percent: 99}) {+ t.Fatalf("last progress = %+v, want collecting_outputs/99", last)+ }+}diff --git a/internal/pipeline/types.go b/internal/pipeline/types.goindex e794328b..c74f69ee 100644--- a/internal/pipeline/types.go+++ b/internal/pipeline/types.go@@ -8,6 +8,7 @@ const (StageDownload Stage = "download"StageSubtitle Stage = "subtitle"StageTTS Stage = "tts"+ StageSpeech Stage = "speech"StageRenderHorizontal Stage = "render-horizontal"StageRenderVertical Stage = "render-vertical"StageCover Stage = "cover"@@ -94,12 +95,15 @@ type Outputs struct {}type Voice struct {- Code string `json:"code"`- Name string `json:"name,omitempty"`- Language string `json:"language,omitempty"`- Gender string `json:"gender,omitempty"`- Provider string `json:"provider,omitempty"`- Scenario string `json:"scenario,omitempty"`+ Code string `json:"code"`+ Name string `json:"name,omitempty"`+ Language string `json:"language,omitempty"`+ Gender string `json:"gender,omitempty"`+ Provider string `json:"provider,omitempty"`+ Scenario string `json:"scenario,omitempty"`+ Kind string `json:"kind,omitempty"`+ SupportedModels []string `json:"supported_models,omitempty"`+ Recommended bool `json:"recommended,omitempty"`}type Response struct {diff --git a/internal/service/audio2subtitle.go b/internal/service/audio2subtitle.goindex 388d0b98..ae258035 100644--- a/internal/service/audio2subtitle.go+++ b/internal/service/audio2subtitle.go@@ -51,8 +51,7 @@ type AudioSegment struct {}type TranslationProgress struct {- Completed int- Total int+ Fraction float64}type contextChatCompleter interface {@@ -120,7 +119,13 @@ func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleT// return nil//}-func (s Service) transcribeAudio(id int, audioFilePath string, language string, taskBasePath string) (transcriptionData *types.TranscriptionData, err error) {+func (s Service) transcribeAudio(+ id int,+ audioFilePath string,+ language string,+ taskBasePath string,+ reportProgress func(percent int),+) (transcriptionData *types.TranscriptionData, err error) {defer func() {if r := recover(); r != nil {err = fmt.Errorf("audioToSubtitle transcribeAudio panic recovered: %v", r)@@ -130,7 +135,16 @@ func (s Service) transcribeAudio(id int, audioFilePath string, language string,if language == "zh_cn" {language = "zh" // 切换一下}- transcriptionData, err = s.Transcriber.Transcription(audioFilePath, language, taskBasePath)+ if transcriber, ok := s.Transcriber.(types.ProgressTranscriber); ok {+ transcriptionData, err = transcriber.TranscriptionWithProgress(+ audioFilePath,+ language,+ taskBasePath,+ reportProgress,+ )+ } else {+ transcriptionData, err = s.Transcriber.Transcription(audioFilePath, language, taskBasePath)+ }if err != nil {return nil, fmt.Errorf("audioToSubtitle transcribeAudio Transcription err: %w", err)@@ -417,6 +431,7 @@ func (s Service) processAudioSegments(ctx context.Context, stepParam *types.SubtsplitResultQueue := make(chan DataWithId[string], segmentNum)pendingTranscriptionQueue := make(chan DataWithId[string], segmentNum)transcribedQueue := make(chan DataWithId[*types.TranscriptionData], segmentNum)+ transcriptionProgressQueue := make(chan DataWithId[float64], segmentNum*2)pendingTranslationQueue := make(chan DataWithId[string], segmentNum)translationProgressQueue := make(chan DataWithId[TranslationProgress], segmentNum*2)translatedQueue := make(chan DataWithId[[]*TranslatedItem], segmentNum)@@ -438,14 +453,21 @@ func (s Service) processAudioSegments(ctx context.Context, stepParam *types.Subts.startSplitWorkers(ctx, eg, stepParam, pendingSplitQueue, splitResultQueue)// 启动转录协程- s.startTranscribeWorkers(ctx, eg, stepParam, pendingTranscriptionQueue, transcribedQueue)+ s.startTranscribeWorkers(+ ctx,+ eg,+ stepParam,+ pendingTranscriptionQueue,+ transcriptionProgressQueue,+ transcribedQueue,+ )// 启动翻译协程s.startTranslateWorker(ctx, eg, stepParam, pendingTranslationQueue, translationProgressQueue, translatedQueue)// 处理结果协程s.startResultHandler(ctx, eg, stepParam, segmentNum, timePoints, audioSegments,- splitResultQueue, pendingTranscriptionQueue, transcribedQueue,+ splitResultQueue, pendingTranscriptionQueue, transcriptionProgressQueue, transcribedQueue,pendingTranslationQueue, translationProgressQueue, translatedQueue, pendingSplitQueue)if err := eg.Wait(); err != nil {@@ -492,7 +514,9 @@ func (s Service) startSplitWorkers(ctx context.Context, eg *errgroup.Group, step// 启动转录工作协程func (s Service) startTranscribeWorkers(ctx context.Context, eg *errgroup.Group, stepParam *types.SubtitleTaskStepParam,- pendingTranscriptionQueue chan DataWithId[string], transcribedQueue chan DataWithId[*types.TranscriptionData]) {+ pendingTranscriptionQueue chan DataWithId[string],+ transcriptionProgressQueue chan DataWithId[float64],+ transcribedQueue chan DataWithId[*types.TranscriptionData]) {for range config.Conf.App.TranscribeParallelNum {eg.Go(func() error {@@ -511,7 +535,21 @@ func (s Service) startTranscribeWorkers(ctx context.Context, eg *errgroup.Group,log.GetLogger().Info("Begin transcribe", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", audioFileItem.Id))// 语音转文字for range config.Conf.App.TranscribeMaxAttempts {- transcriptionData, err = s.transcribeAudio(audioFileItem.Id, audioFileItem.Data, string(stepParam.OriginLanguage), stepParam.TaskBasePath)+ transcriptionData, err = s.transcribeAudio(+ audioFileItem.Id,+ audioFileItem.Data,+ string(stepParam.OriginLanguage),+ stepParam.TaskBasePath,+ func(percent int) {+ select {+ case transcriptionProgressQueue <- DataWithId[float64]{+ Data: float64(percent) / 100,+ Id: audioFileItem.Id,+ }:+ case <-ctx.Done():+ }+ },+ )if err == nil {break}@@ -537,6 +575,8 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, spendingTranslationQueue chan DataWithId[string], translationProgressQueue chan DataWithId[TranslationProgress],translatedQueue chan DataWithId[[]*TranslatedItem]) {+ const primaryTranslationProgressShare = 0.35+eg.Go(func() error {for {select {@@ -555,9 +595,10 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, sattempts = 1}reportProgress := func(completed, total int) error {+ fraction := scaledProgress(completed, total, 0, primaryTranslationProgressShare)select {case translationProgressQueue <- DataWithId[TranslationProgress]{- Data: TranslationProgress{Completed: completed, Total: total},+ Data: TranslationProgress{Fraction: fraction},Id: translateItem.Id,}:return nil@@ -576,8 +617,21 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, s}_ = util.SaveToDisk(translatedResults, filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskTranslationDataPersistenceFileNamePattern, translateItem.Id)))log.GetLogger().Info("Translate completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))+ reportTranslationProgress(+ ctx,+ translationProgressQueue,+ translateItem.Id,+ primaryTranslationProgressShare,+ )// 二次分割长句- splitResults, err := s.splitTranslateItem(translatedResults)+ splitResults, err := s.splitTranslateItem(translatedResults, func(completed, total int) {+ reportTranslationProgress(+ ctx,+ translationProgressQueue,+ translateItem.Id,+ scaledProgress(completed, total, primaryTranslationProgressShare, 1),+ )+ })if err != nil {// 不中断log.GetLogger().Error("audioToSubtitle audioToSrt splitTranslateItem err", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id), zap.Error(err))@@ -600,6 +654,7 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, sfunc (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, stepParam *types.SubtitleTaskStepParam,segmentNum int, timePoints []float64, audioSegments []AudioSegment,splitResultQueue chan DataWithId[string], pendingTranscriptionQueue chan DataWithId[string],+ transcriptionProgressQueue chan DataWithId[float64],transcribedQueue chan DataWithId[*types.TranscriptionData], pendingTranslationQueue chan DataWithId[string],translationProgressQueue chan DataWithId[TranslationProgress], translatedQueue chan DataWithId[[]*TranslatedItem],pendingSplitQueue chan DataWithId[[2]float64]) {@@ -612,18 +667,18 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, steTRANSLATE_WEIGHT = 0.5)splitCompleted := 0- transcriptionCompleted := 0+ transcriptionProgress := make(map[int]float64, segmentNum)translationProgress := make(map[int]float64, segmentNum)updateProgress := func() {- translationTotal := 0.0- for _, progress := range translationProgress {- translationTotal += progress- }- processPct := 15.0 +- float64(splitCompleted)/float64(segmentNum)*75*SPLIT_WEIGHT +- float64(transcriptionCompleted)/float64(segmentNum)*75*TRANSCRIBE_WEIGHT +- translationTotal/float64(segmentNum)*75*TRANSLATE_WEIGHT- stepParam.TaskPtr.SetProgress(uint8(processPct))+ stepParam.TaskPtr.SetProgress(audioProcessPercent(+ segmentNum,+ splitCompleted,+ transcriptionProgress,+ translationProgress,+ SPLIT_WEIGHT,+ TRANSCRIBE_WEIGHT,+ TRANSLATE_WEIGHT,+ ))}// 完成的任务数量completedTasks := 0@@ -641,8 +696,14 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, steData: splitResultItem.Data,Id: splitResultItem.Id,}+ case progressItem := <-transcriptionProgressQueue:+ fraction := clampFraction(progressItem.Data)+ if fraction > transcriptionProgress[progressItem.Id] {+ transcriptionProgress[progressItem.Id] = fraction+ updateProgress()+ }case transcribedItem := <-transcribedQueue:- transcriptionCompleted+++ transcriptionProgress[transcribedItem.Id] = 1updateProgress()// 处理转录结果audioSegments[transcribedItem.Id].TranscriptionData = transcribedItem.Data@@ -652,15 +713,10 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, steId: transcribedItem.Id,}case progressItem := <-translationProgressQueue:- if progressItem.Data.Total > 0 {- fraction := float64(progressItem.Data.Completed) / float64(progressItem.Data.Total)- if fraction > 1 {- fraction = 1- }- if fraction > translationProgress[progressItem.Id] {- translationProgress[progressItem.Id] = fraction- updateProgress()- }+ fraction := clampFraction(progressItem.Data.Fraction)+ if fraction > translationProgress[progressItem.Id] {+ translationProgress[progressItem.Id] = fraction+ updateProgress()}case translatedItems := <-translatedQueue:translationProgress[translatedItems.Id] = 1@@ -711,6 +767,7 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, steclose(pendingSplitQueue)close(splitResultQueue)close(pendingTranscriptionQueue)+ close(transcriptionProgressQueue)close(transcribedQueue)close(pendingTranslationQueue)close(translationProgressQueue)@@ -722,6 +779,65 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, ste})}+func reportTranslationProgress(+ ctx context.Context,+ queue chan DataWithId[TranslationProgress],+ id int,+ fraction float64,+) {+ select {+ case queue <- DataWithId[TranslationProgress]{+ Data: TranslationProgress{Fraction: clampFraction(fraction)},+ Id: id,+ }:+ case <-ctx.Done():+ }+}++func scaledProgress(completed, total int, start, end float64) float64 {+ if total <= 0 {+ return end+ }+ return start + (end-start)*clampFraction(float64(completed)/float64(total))+}++func clampFraction(value float64) float64 {+ if value < 0 {+ return 0+ }+ if value > 1 {+ return 1+ }+ return value+}++func audioProcessPercent(+ segmentNum int,+ splitCompleted int,+ transcriptionProgress map[int]float64,+ translationProgress map[int]float64,+ splitWeight float64,+ transcribeWeight float64,+ translateWeight float64,+) uint8 {+ if segmentNum <= 0 {+ return 15+ }+ transcriptionTotal := 0.0+ for _, progress := range transcriptionProgress {+ transcriptionTotal += clampFraction(progress)+ }+ translationTotal := 0.0+ for _, progress := range translationProgress {+ translationTotal += clampFraction(progress)+ }+ processPct := 15.0 ++ float64(splitCompleted)/float64(segmentNum)*75*splitWeight ++ transcriptionTotal/float64(segmentNum)*75*transcribeWeight ++ translationTotal/float64(segmentNum)*75*translateWeight+ return uint8(processPct)+}+// 合并字幕文件func (s Service) mergeSubtitleFiles(stepParam *types.SubtitleTaskStepParam, audioSegments []AudioSegment, segmentNum int) error {// 合并文件@@ -1459,9 +1575,22 @@ func calcLength(text string) float64 {}// splitTranslateItem 根据字符权重和最大长度分割长句-func (s Service) splitTranslateItem(items []*TranslatedItem) ([]*TranslatedItem, error) {+func (s Service) splitTranslateItem(+ items []*TranslatedItem,+ reportProgress func(completed, total int),+) ([]*TranslatedItem, error) {var result []*TranslatedItemmaxLength := config.Conf.App.MaxSentenceLength + 30+ longSentenceTotal := 0+ for _, item := range items {+ if calcLength(item.OriginText) > float64(maxLength) || calcLength(item.TranslatedText) > float64(maxLength) {+ longSentenceTotal+++ }+ }+ if reportProgress != nil {+ reportProgress(0, longSentenceTotal)+ }+ longSentenceCompleted := 0for _, item := range items {// 计算翻译文本的加权长度@@ -1478,8 +1607,15 @@ func (s Service) splitTranslateItem(items []*TranslatedItem) ([]*TranslatedItem,return nil, fmt.Errorf("split long sentence error: %w", err)}result = append(result, splitItems...)+ longSentenceCompleted+++ if reportProgress != nil {+ reportProgress(longSentenceCompleted, longSentenceTotal)+ }}+ if longSentenceTotal == 0 && reportProgress != nil {+ reportProgress(1, 1)+ }return result, nil}diff --git a/internal/service/audio2subtitle_test.go b/internal/service/audio2subtitle_test.goindex 791ea78c..9ee637c4 100644--- a/internal/service/audio2subtitle_test.go+++ b/internal/service/audio2subtitle_test.go@@ -93,6 +93,97 @@ func Test_splitOriginLongSentence(t *testing.T) {}}+type progressTranscriber struct {+ progress []int+}++func (t progressTranscriber) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {+ return t.TranscriptionWithProgress(audioFile, language, workDir, nil)+}++func (t progressTranscriber) TranscriptionWithProgress(+ _, _, _ string,+ reportProgress func(percent int),+) (*types.TranscriptionData, error) {+ for _, percent := range t.progress {+ if reportProgress != nil {+ reportProgress(percent)+ }+ }+ return &types.TranscriptionData{Text: "transcribed"}, nil+}++func TestTranscribeAudioForwardsProviderProgress(t *testing.T) {+ service := Service{Transcriber: progressTranscriber{progress: []int{0, 33, 66, 100}}}+ var reported []int++ result, err := service.transcribeAudio(+ 0,+ "sample.mp3",+ "zh_cn",+ t.TempDir(),+ func(percent int) {+ reported = append(reported, percent)+ },+ )+ if err != nil {+ t.Fatal(err)+ }+ if result.Text != "transcribed" {+ t.Fatalf("transcription text = %q", result.Text)+ }+ if got := fmt.Sprint(reported); got != "[0 33 66 100]" {+ t.Fatalf("reported progress = %s", got)+ }+}++func TestAudioProcessPercentUsesDetailedTranscriptionAndTranslationProgress(t *testing.T) {+ const (+ splitWeight = 0.1+ transcribeWeight = 0.4+ translateWeight = 0.5+ )+ tests := []struct {+ name string+ split int+ transcription float64+ translation float64+ want uint8+ }{+ {name: "audio split", split: 1, want: 22},+ {name: "transcription one third", split: 1, transcription: 0.33, want: 32},+ {name: "transcription two thirds", split: 1, transcription: 0.66, want: 42},+ {name: "transcription complete", split: 1, transcription: 1, want: 52},+ {name: "primary translation complete", split: 1, transcription: 1, translation: 0.35, want: 65},+ {name: "long sentence halfway", split: 1, transcription: 1, translation: 0.675, want: 77},+ {name: "subtitle processing complete", split: 1, transcription: 1, translation: 1, want: 90},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ transcription := map[int]float64{}+ if test.transcription > 0 {+ transcription[0] = test.transcription+ }+ translation := map[int]float64{}+ if test.translation > 0 {+ translation[0] = test.translation+ }+ got := audioProcessPercent(+ 1,+ test.split,+ transcription,+ translation,+ splitWeight,+ transcribeWeight,+ translateWeight,+ )+ if got != test.want {+ t.Fatalf("audioProcessPercent() = %d, want %d", got, test.want)+ }+ })+ }+}+type echoBatchCompleter struct {batchSizes []int}@@ -236,3 +327,37 @@ func TestParseTranslationBatchAcceptsFencedJSON(t *testing.T) {t.Fatalf("translations = %v", translations)}}++func TestSplitTranslateItemReportsLongSentenceProgress(t *testing.T) {+ previousMaxSentenceLength := config.Conf.App.MaxSentenceLength+ config.Conf.App.MaxSentenceLength = 10+ t.Cleanup(func() { config.Conf.App.MaxSentenceLength = previousMaxSentenceLength })++ completer := &scriptedCompleter{responses: []string{+ `{"align":[{"origin_part":"first origin part","translated_part":"first translated part"}]}`,+ `{"align":[{"origin_part":"second origin part","translated_part":"second translated part"}]}`,+ }}+ service := Service{ChatCompleter: completer}+ items := []*TranslatedItem{+ {OriginText: "short", TranslatedText: "short"},+ {+ OriginText: strings.Repeat("first origin ", 8),+ TranslatedText: strings.Repeat("first translated ", 8),+ },+ {+ OriginText: strings.Repeat("second origin ", 8),+ TranslatedText: strings.Repeat("second translated ", 8),+ },+ }+ var progress []string++ _, err := service.splitTranslateItem(items, func(completed, total int) {+ progress = append(progress, fmt.Sprintf("%d/%d", completed, total))+ })+ if err != nil {+ t.Fatal(err)+ }+ if got := fmt.Sprint(progress); got != "[0/2 1/2 2/2]" {+ t.Fatalf("progress = %s", got)+ }+}diff --git a/internal/service/dubbing/audio.go b/internal/service/dubbing/audio.goindex 929336fd..52ccb72e 100644--- a/internal/service/dubbing/audio.go+++ b/internal/service/dubbing/audio.go@@ -10,6 +10,8 @@ import ("strings")+const timelineComparisonEpsilon = 1e-6+func defaultFFmpegRunner(args []string) error {cmd := exec.Command(storage.FfmpegPath, args...)output, err := cmd.CombinedOutput()@@ -250,7 +252,7 @@ func validateAssemblePlan(plan []PlanItem, segmentsDir string) ([]string, error)if item.NewEnd <= item.NewStart {return nil, fmt.Errorf("plan item %d new end must be greater than new start: start %.3f end %.3f", item.Index, item.NewStart, item.NewEnd)}- if item.NewStart < lastEnd {+ if item.NewStart+timelineComparisonEpsilon < lastEnd {return nil, fmt.Errorf("plan item %d starts before previous end: start %.3f lastEnd %.3f", item.Index, item.NewStart, lastEnd)}@@ -284,7 +286,7 @@ func validateAssembleChunkPlan(plan []PlanItem, chunks []Chunk, segmentsDir striif len(chunk.Items) == 0 {return nil, fmt.Errorf("chunk %d has no items", chunk.ID)}- if chunk.Start < lastEnd {+ if chunk.Start+timelineComparisonEpsilon < lastEnd {return nil, fmt.Errorf("chunk %d starts before previous end: start %.3f lastEnd %.3f", chunk.ID, chunk.Start, lastEnd)}end := chunkFittedEnd(plan, chunk)diff --git a/internal/service/dubbing/audio_assemble_test.go b/internal/service/dubbing/audio_assemble_test.goindex 187f1046..a1f856d8 100644--- a/internal/service/dubbing/audio_assemble_test.go+++ b/internal/service/dubbing/audio_assemble_test.go@@ -237,3 +237,31 @@ func TestAssembleChunkAudioPreservesGapAfterShortChunkAudio(t *testing.T) {t.Fatalf("silence before second chunk = %q, want 4.000", silenceDuration)}}++func TestAssembleChunkAudioAllowsFloatingPointBoundaryNoise(t *testing.T) {+ dir := t.TempDir()+ rawDir := filepath.Join(dir, "raw")+ if err := os.MkdirAll(rawDir, 0755); err != nil {+ t.Fatal(err)+ }+ for _, name := range []string{"chunk_16.wav", "chunk_17.wav"} {+ if err := os.WriteFile(filepath.Join(rawDir, name), []byte("raw"), 0644); err != nil {+ t.Fatal(err)+ }+ }+ plan := []PlanItem{+ {Index: 42, NewStart: 96.84, NewEnd: 111.00000000000001, SpeedFactor: 1.028, ChunkID: 16},+ {Index: 43, NewStart: 111, NewEnd: 114.08, SpeedFactor: 1, ChunkID: 17},+ }+ chunks := []Chunk{+ {ID: 16, Items: []int{0}, Start: 96.84, End: 111, ActualDuration: 14.56, SpeedFactor: 1.028},+ {ID: 17, Items: []int{1}, Start: 111, End: 114.08, ActualDuration: 3.08, SpeedFactor: 1},+ }++ err := AssembleChunkAudio(plan, chunks, dir, filepath.Join(dir, "out.wav"), func(args []string) error {+ return os.WriteFile(args[len(args)-1], []byte("media"), 0644)+ })+ if err != nil {+ t.Fatalf("AssembleChunkAudio() error = %v, want floating-point boundary accepted", err)+ }+}diff --git a/internal/service/dubbing/fit.go b/internal/service/dubbing/fit.goindex 9b2f2619..9d3c99db 100644--- a/internal/service/dubbing/fit.go+++ b/internal/service/dubbing/fit.go@@ -40,9 +40,6 @@ func FitTimeline(plan []PlanItem, chunks []Chunk, cfg Config) ([]PlanItem, []Chureport.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d speed %.2f exceeds max %.2f", chunk.ID, speed, cfg.SpeedMax))}appliedSpeed := speed- if appliedSpeed > cfg.SpeedMax {- appliedSpeed = cfg.SpeedMax- }if appliedSpeed < cfg.SpeedMin {appliedSpeed = cfg.SpeedMin}diff --git a/internal/service/dubbing/fit_test.go b/internal/service/dubbing/fit_test.goindex 53a6ff87..4f2185d5 100644--- a/internal/service/dubbing/fit_test.go+++ b/internal/service/dubbing/fit_test.go@@ -50,7 +50,7 @@ func TestFitTimelineUsesChunkAudioAndEstimatedWeights(t *testing.T) {}}-func TestFitTimelineClampsAppliedSpeedToMaxButReportsRequiredSpeed(t *testing.T) {+func TestFitTimelineUsesRequiredSpeedBeyondQualityLimitWithoutOverflow(t *testing.T) {cfg := DefaultConfig()cfg.SpeedMax = 1.25cfg.SpeedAccept = 1.1@@ -65,14 +65,42 @@ func TestFitTimelineClampsAppliedSpeedToMaxButReportsRequiredSpeed(t *testing.T)if report.MaxSpeedFactor != 2 {t.Fatalf("MaxSpeedFactor = %v, want raw required speed 2", report.MaxSpeedFactor)}- if got[0].SpeedFactor != cfg.SpeedMax {- t.Fatalf("SpeedFactor = %v, want clamped max %v", got[0].SpeedFactor, cfg.SpeedMax)+ if got[0].SpeedFactor != 2 {+ t.Fatalf("SpeedFactor = %v, want required speed 2", got[0].SpeedFactor)}- if got[0].NewEnd <= chunks[0].End {- t.Fatalf("NewEnd = %v, want overflow beyond chunk end %v", got[0].NewEnd, chunks[0].End)+ if got[0].NewEnd != chunks[0].End {+ t.Fatalf("NewEnd = %v, want chunk end %v", got[0].NewEnd, chunks[0].End)}- if !warningsContain(report.Warnings, "exceeds max") || !warningsContain(report.Warnings, "overflows") {- t.Fatalf("warnings = %+v, want max and overflow warnings", report.Warnings)+ if !warningsContain(report.Warnings, "exceeds max") || warningsContain(report.Warnings, "overflows") {+ t.Fatalf("warnings = %+v, want max warning without overflow", report.Warnings)+ }+}++func TestFitTimelineKeepsOverlongChunkBeforeNextChunkBoundary(t *testing.T) {+ cfg := DefaultConfig()+ plan := []PlanItem{+ {Index: 33, EstimatedDuration: 4.8, ChunkID: 13},+ {Index: 34, EstimatedDuration: 3.6, ChunkID: 13},+ {Index: 35, EstimatedDuration: 4.9, ChunkID: 13},+ {Index: 36, EstimatedDuration: 5.1, ChunkID: 14},+ }+ chunks := []Chunk{+ {ID: 13, Items: []int{0, 1, 2}, Start: 82.8, End: 90, ActualDuration: 11.04},+ {ID: 14, Items: []int{3}, Start: 90, End: 93.28, ActualDuration: 3.28},+ }++ got, gotChunks, report, err := FitTimeline(plan, chunks, cfg)+ if err != nil {+ t.Fatalf("FitTimeline() error = %v", err)+ }+ if got[2].NewEnd > got[3].NewStart {+ t.Fatalf("chunk boundary overlaps: previous end %.3f next start %.3f", got[2].NewEnd, got[3].NewStart)+ }+ if gotChunks[0].SpeedFactor <= cfg.SpeedMax {+ t.Fatalf("chunk 13 speed = %.3f, want fallback above max %.3f", gotChunks[0].SpeedFactor, cfg.SpeedMax)+ }+ if warningsContain(report.Warnings, "overflows") {+ t.Fatalf("warnings = %+v, want no overflow warning", report.Warnings)}}diff --git a/internal/service/dubbing/runner.go b/internal/service/dubbing/runner.goindex b0f78201..b5fbb358 100644--- a/internal/service/dubbing/runner.go+++ b/internal/service/dubbing/runner.go@@ -76,7 +76,17 @@ func (r *Runner) Run(ctx context.Context) (Result, error) {return Result{}, err}- plan, chunks, err = GenerateRawChunkSegments(ctx, r.deps.TTS, plan, chunks, r.deps.Voice, segmentsDir, r.deps.FFmpeg, r.deps.Duration)+ plan, chunks, err = GenerateRawChunkSegments(+ ctx,+ r.deps.TTS,+ plan,+ chunks,+ r.deps.Voice,+ segmentsDir,+ r.deps.FFmpeg,+ r.deps.Duration,+ r.deps.ReportProgress,+ )if err != nil {return Result{}, err}diff --git a/internal/service/dubbing/tts.go b/internal/service/dubbing/tts.goindex 428926b2..2a83d47b 100644--- a/internal/service/dubbing/tts.go+++ b/internal/service/dubbing/tts.go@@ -55,7 +55,17 @@ func GenerateRawSegments(ctx context.Context, tts types.Ttser, plan []PlanItem,return plan, nil}-func GenerateRawChunkSegments(ctx context.Context, tts types.Ttser, plan []PlanItem, chunks []Chunk, voice, dir string, run CommandRunner, duration DurationProbe) ([]PlanItem, []Chunk, error) {+func GenerateRawChunkSegments(+ ctx context.Context,+ tts types.Ttser,+ plan []PlanItem,+ chunks []Chunk,+ voice,+ dir string,+ run CommandRunner,+ duration DurationProbe,+ reportProgress func(completed, total int),+) ([]PlanItem, []Chunk, error) {if ctx == nil {ctx = context.Background()}@@ -101,6 +111,9 @@ func GenerateRawChunkSegments(ctx context.Context, tts types.Ttser, plan []PlanIreturn nil, nil, fmt.Errorf("measure chunk %d duration failed for %s: %w", outChunks[i].ID, output, err)}outChunks[i].ActualDuration = dur+ if reportProgress != nil {+ reportProgress(i+1, len(outChunks))+ }}return outPlan, outChunks, nildiff --git a/internal/service/dubbing/tts_test.go b/internal/service/dubbing/tts_test.goindex 109c1994..a4b6bcb9 100644--- a/internal/service/dubbing/tts_test.go+++ b/internal/service/dubbing/tts_test.go@@ -102,12 +102,25 @@ func TestGenerateRawChunkSegmentsCallsTTSOncePerChunk(t *testing.T) {{ID: 2, Items: []int{2}, Start: 6, End: 8},}- gotPlan, gotChunks, err := GenerateRawChunkSegments(context.Background(), tts, plan, chunks, "voice", dir, nil, func(path string) (float64, error) {- if strings.Contains(path, "chunk_1.wav") {- return 3.2, nil- }- return 1.1, nil- })+ var progress [][2]int+ gotPlan, gotChunks, err := GenerateRawChunkSegments(+ context.Background(),+ tts,+ plan,+ chunks,+ "voice",+ dir,+ nil,+ func(path string) (float64, error) {+ if strings.Contains(path, "chunk_1.wav") {+ return 3.2, nil+ }+ return 1.1, nil+ },+ func(completed, total int) {+ progress = append(progress, [2]int{completed, total})+ },+ )if err != nil {t.Fatalf("GenerateRawChunkSegments() error = %v", err)}@@ -123,6 +136,9 @@ func TestGenerateRawChunkSegmentsCallsTTSOncePerChunk(t *testing.T) {if gotPlan[0].ActualDuration != 0 || gotPlan[1].ActualDuration != 0 {t.Fatalf("item actual durations should remain chunk-derived later: %+v", gotPlan)}+ if len(progress) != 2 || progress[0] != [2]int{1, 2} || progress[1] != [2]int{2, 2} {+ t.Fatalf("progress = %+v, want [[1 2] [2 2]]", progress)+ }if _, err := os.Stat(filepath.Join(dir, "raw", "chunk_1.wav")); err != nil {t.Fatalf("chunk raw file missing: %v", err)}diff --git a/internal/service/dubbing/types.go b/internal/service/dubbing/types.goindex d66083b6..1987a165 100644--- a/internal/service/dubbing/types.go+++ b/internal/service/dubbing/types.go@@ -88,18 +88,19 @@ type CommandRunner func(args []string) errortype DurationProbe func(path string) (float64, error)type Dependencies struct {- TTS types.Ttser- Chat types.ChatCompleter- Language types.StandardLanguageCode- Voice string- Workdir string- InputSRT string- InputVideo string- OutputAudio string- OutputVideo string- Config Config- FFmpeg CommandRunner- Duration DurationProbe+ TTS types.Ttser+ Chat types.ChatCompleter+ Language types.StandardLanguageCode+ Voice string+ Workdir string+ InputSRT string+ InputVideo string+ OutputAudio string+ OutputVideo string+ Config Config+ FFmpeg CommandRunner+ Duration DurationProbe+ ReportProgress func(completed, total int)}type TextOptimizer interface {diff --git a/internal/service/init.go b/internal/service/init.goindex 03cbe8f0..273c5f56 100644--- a/internal/service/init.go+++ b/internal/service/init.go@@ -2,13 +2,13 @@ package serviceimport ("krillin-ai/config"+ "krillin-ai/internal/ttsprovider""krillin-ai/internal/types""krillin-ai/log""krillin-ai/pkg/aliyun""krillin-ai/pkg/fasterwhisper"pkgimage "krillin-ai/pkg/image""krillin-ai/pkg/localtts"- "krillin-ai/pkg/minimax""krillin-ai/pkg/openai""krillin-ai/pkg/whisper""krillin-ai/pkg/whispercpp"@@ -55,13 +55,13 @@ func NewService() *Service {switch config.Conf.Tts.Provider {case "openai":- ttsClient = openai.NewClient(config.Conf.Tts.Openai.BaseUrl, config.Conf.Tts.Openai.ApiKey, config.Conf.App.Proxy)+ ttsClient, _ = ttsprovider.New("openai")case "aliyun":- ttsClient = aliyun.NewTtsClient(config.Conf.Tts.Aliyun.Speech.AccessKeyId, config.Conf.Tts.Aliyun.Speech.AccessKeySecret, config.Conf.Tts.Aliyun.Speech.AppKey)+ ttsClient, _ = ttsprovider.New("aliyun")case "edge-tts":ttsClient = localtts.NewEdgeTtsClient()case "minimax":- ttsClient = minimax.NewTtsClient(config.Conf.Tts.Minimax.BaseUrl, config.Conf.Tts.Minimax.ApiKey, config.Conf.Tts.Minimax.Model)+ ttsClient, _ = ttsprovider.New("minimax")}s := &Service{diff --git a/internal/service/srt2speech.go b/internal/service/srt2speech.goindex bfed2cd7..58db4530 100644--- a/internal/service/srt2speech.go+++ b/internal/service/srt2speech.go@@ -69,6 +69,16 @@ func (s Service) srtFileToSpeech(ctx context.Context, stepParam *types.SubtitleTInputVideo: stepParam.InputVideoPath,OutputAudio: outputAudio,OutputVideo: outputVideo,+ ReportProgress: func(completed, total int) {+ if stepParam.TaskPtr == nil || total <= 0 {+ return+ }+ percent := 20 + int(float64(completed)/float64(total)*70)+ if percent > 90 {+ percent = 90+ }+ stepParam.TaskPtr.SetProgress(uint8(percent))+ },Config: dubbing.Config{MinSubtitleDuration: config.Conf.Dubbing.MinSubtitleDuration,MaxChunkSize: config.Conf.Dubbing.MaxChunkSize,diff --git a/internal/ttsprovider/provider.go b/internal/ttsprovider/provider.gonew file mode 100644index 00000000..eae21648--- /dev/null+++ b/internal/ttsprovider/provider.go@@ -0,0 +1,38 @@+package ttsprovider++import (+ "fmt"++ "krillin-ai/config"+ "krillin-ai/internal/types"+ "krillin-ai/pkg/aliyun"+ "krillin-ai/pkg/minimax"+ "krillin-ai/pkg/openai"+)++func New(provider string) (types.TTSProvider, error) {+ switch provider {+ case "openai":+ return openai.NewTtsClient(+ config.Conf.Tts.Openai.BaseUrl,+ config.Conf.Tts.Openai.ApiKey,+ config.Conf.Tts.Openai.Model,+ config.Conf.App.Proxy,+ ), nil+ case "aliyun":+ return aliyun.NewTtsClient(+ config.Conf.Tts.Aliyun.BaseUrl,+ config.Conf.Tts.Aliyun.ApiKey,+ config.Conf.Tts.Aliyun.Model,+ config.Conf.App.Proxy,+ ), nil+ case "minimax":+ return minimax.NewTtsClient(+ config.Conf.Tts.Minimax.BaseUrl,+ config.Conf.Tts.Minimax.ApiKey,+ config.Conf.Tts.Minimax.Model,+ ), nil+ default:+ return nil, fmt.Errorf("unsupported tts provider: %s", provider)+ }+}diff --git a/internal/types/interface.go b/internal/types/interface.goindex 203c0a33..994fdf4f 100644--- a/internal/types/interface.go+++ b/internal/types/interface.go@@ -8,6 +8,13 @@ type Transcriber interface {Transcription(audioFile, language, wordDir string) (*TranscriptionData, error)}+type ProgressTranscriber interface {+ TranscriptionWithProgress(+ audioFile, language, workDir string,+ reportProgress func(percent int),+ ) (*TranscriptionData, error)+}+type Ttser interface {Text2Speech(text string, voice string, outputFile string) error}diff --git a/internal/types/tts_provider.go b/internal/types/tts_provider.gonew file mode 100644index 00000000..7baf8fe9--- /dev/null+++ b/internal/types/tts_provider.go@@ -0,0 +1,30 @@+package types++import "context"++type TTSSpeechOptions struct {+ Text string+ Voice string+ OutputFile string+ Format string+ Speed float64+ Instructions string+}++type TTSVoice struct {+ Code string+ Name string+ Language string+ Gender string+ Provider string+ Scenario string+ Kind string+ SupportedModels []string+ Recommended bool+}++type TTSProvider interface {+ Ttser+ ListVoices(context.Context) ([]TTSVoice, error)+ Synthesize(context.Context, TTSSpeechOptions) error+}diff --git a/internal/voices/voices.go b/internal/voices/voices.goindex f3f922e4..b99df45b 100644--- a/internal/voices/voices.go+++ b/internal/voices/voices.go@@ -1,82 +1,57 @@package voicesimport (+ "context""fmt""sort""strings""krillin-ai/internal/pipeline"+ "krillin-ai/internal/ttsprovider")const (ProviderAliyun = "aliyun"ProviderOpenAI = "openai"- ProviderEdge = "edge-tts"Minimax = "minimax")-func List(provider string) ([]pipeline.Voice, error) {+func List(ctx context.Context, provider string) ([]pipeline.Voice, error) {provider = strings.TrimSpace(strings.ToLower(provider))- switch provider {- case ProviderAliyun:- return cloneVoices(aliyunVoices), nil- case ProviderOpenAI:- return cloneVoices(openaiVoices), nil- case Minimax:- return cloneVoices(minimaxVoices), nil- case ProviderEdge:- return nil, fmt.Errorf("edge-tts voice listing is not supported yet; use edge-tts --list-voices")- default:- return nil, fmt.Errorf("unsupported tts provider: %s", provider)+ client, err := ttsprovider.New(provider)+ if err != nil {+ return nil, err}-}--func Providers() []string {- return []string{ProviderAliyun, ProviderOpenAI, Minimax, ProviderEdge}-}--func cloneVoices(in []pipeline.Voice) []pipeline.Voice {- out := append([]pipeline.Voice(nil), in...)- sort.SliceStable(out, func(i, j int) bool {- if out[i].Provider != out[j].Provider {- return out[i].Provider < out[j].Provider+ values, err := client.ListVoices(ctx)+ if err != nil {+ return nil, fmt.Errorf("list %s voices: %w", provider, err)+ }+ result := make([]pipeline.Voice, 0, len(values))+ for _, voice := range values {+ result = append(result, pipeline.Voice{+ Code: voice.Code,+ Name: voice.Name,+ Language: voice.Language,+ Gender: voice.Gender,+ Provider: voice.Provider,+ Scenario: voice.Scenario,+ Kind: voice.Kind,+ SupportedModels: append([]string(nil), voice.SupportedModels...),+ Recommended: voice.Recommended,+ })+ }+ sort.SliceStable(result, func(i, j int) bool {+ if result[i].Kind != result[j].Kind {+ return result[i].Kind < result[j].Kind}- return out[i].Code < out[j].Code+ if result[i].Recommended != result[j].Recommended {+ return result[i].Recommended+ }+ return result[i].Code < result[j].Code})- return out-}--var aliyunVoices = []pipeline.Voice{- {Provider: ProviderAliyun, Code: "longxiaochun_v2", Name: "龙小淳", Language: "zh-CN", Gender: "female", Scenario: "通用女声"},- {Provider: ProviderAliyun, Code: "longxiaoxia_v2", Name: "龙小夏", Language: "zh-CN", Gender: "female", Scenario: "自然女声"},- {Provider: ProviderAliyun, Code: "longxiaocheng_v2", Name: "龙小诚", Language: "zh-CN", Gender: "male", Scenario: "通用男声"},- {Provider: ProviderAliyun, Code: "longxiaobai_v2", Name: "龙小白", Language: "zh-CN", Gender: "female", Scenario: "甜美女声"},- {Provider: ProviderAliyun, Code: "longlaotie_v2", Name: "龙老铁", Language: "zh-CN", Gender: "male", Scenario: "东北口音"},- {Provider: ProviderAliyun, Code: "longshu_v2", Name: "龙叔", Language: "zh-CN", Gender: "male", Scenario: "沉稳男声"},- {Provider: ProviderAliyun, Code: "longshuo_v2", Name: "龙硕", Language: "zh-CN", Gender: "male", Scenario: "朗读男声"},- {Provider: ProviderAliyun, Code: "longjing_v2", Name: "龙婧", Language: "zh-CN", Gender: "female", Scenario: "新闻女声"},- {Provider: ProviderAliyun, Code: "longmiao_v2", Name: "龙妙", Language: "zh-CN", Gender: "female", Scenario: "客服女声"},- {Provider: ProviderAliyun, Code: "longyue_v2", Name: "龙悦", Language: "zh-CN", Gender: "female", Scenario: "温柔女声"},+ return result, nil}-var openaiVoices = []pipeline.Voice{- {Provider: ProviderOpenAI, Code: "alloy", Language: "multi", Scenario: "balanced"},- {Provider: ProviderOpenAI, Code: "ash", Language: "multi", Scenario: "calm"},- {Provider: ProviderOpenAI, Code: "ballad", Language: "multi", Scenario: "expressive"},- {Provider: ProviderOpenAI, Code: "coral", Language: "multi", Scenario: "warm"},- {Provider: ProviderOpenAI, Code: "echo", Language: "multi", Scenario: "clear"},- {Provider: ProviderOpenAI, Code: "fable", Language: "multi", Scenario: "narration"},- {Provider: ProviderOpenAI, Code: "nova", Language: "multi", Scenario: "bright"},- {Provider: ProviderOpenAI, Code: "onyx", Language: "multi", Scenario: "deep"},- {Provider: ProviderOpenAI, Code: "sage", Language: "multi", Scenario: "neutral"},- {Provider: ProviderOpenAI, Code: "shimmer", Language: "multi", Scenario: "soft"},-}--var minimaxVoices = []pipeline.Voice{- {Provider: Minimax, Code: "English_Graceful_Lady", Name: "Graceful Lady", Language: "en", Gender: "female", Scenario: "优雅女声"},- {Provider: Minimax, Code: "English_radiant_girl", Name: "Radiant Girl", Language: "en", Gender: "female", Scenario: "活泼女声"},- {Provider: Minimax, Code: "English_Insightful_Speaker", Name: "Insightful Speaker", Language: "en", Gender: "male", Scenario: "沉稳男声"},- {Provider: Minimax, Code: "English_Persuasive_Man", Name: "Persuasive Man", Language: "en", Gender: "male", Scenario: "有说服力男声"},- {Provider: Minimax, Code: "English_expressive_narrator", Name: "Expressive Narrator", Language: "en", Gender: "male", Scenario: "旁白"},- {Provider: Minimax, Code: "English_Lucky_Robot", Name: "Lucky Robot", Language: "en", Gender: "neutral", Scenario: "机器人"},+func Providers() []string {+ return []string{ProviderAliyun, ProviderOpenAI, Minimax}}diff --git a/internal/voices/voices_test.go b/internal/voices/voices_test.goindex 06a90fa1..c07ee05a 100644--- a/internal/voices/voices_test.go+++ b/internal/voices/voices_test.go@@ -1,26 +1,27 @@package voicesimport (+ "context""krillin-ai/internal/pipeline""strings""testing")-func TestListAliyunVoicesIncludesCosyVoiceCodes(t *testing.T) {- got, err := List(ProviderAliyun)+func TestListAliyunVoicesIncludesQwenVoiceCodes(t *testing.T) {+ got, err := List(context.Background(), ProviderAliyun)if err != nil {t.Fatalf("List() error = %v", err)}- if !hasVoice(got, "longxiaochun_v2") {- t.Fatalf("aliyun voices = %#v, want longxiaochun_v2", got)+ if !hasVoice(got, "Cherry") {+ t.Fatalf("aliyun voices = %#v, want Cherry", got)}- if !hasVoice(got, "longxiaocheng_v2") {- t.Fatalf("aliyun voices = %#v, want longxiaocheng_v2", got)+ if !hasVoice(got, "Ethan") {+ t.Fatalf("aliyun voices = %#v, want Ethan", got)}}func TestListRejectsUnsupportedProvider(t *testing.T) {- _, err := List("unknown")+ _, err := List(context.Background(), "unknown")if err == nil {t.Fatal("List() error = nil, want unsupported provider error")}@@ -30,7 +31,7 @@ func TestListRejectsUnsupportedProvider(t *testing.T) {}func TestListMinimaxVoices(t *testing.T) {- got, err := List(Minimax)+ got, err := List(context.Background(), Minimax)if err != nil {t.Fatalf("List() error = %v", err)}diff --git a/pkg/aliyun/tts.go b/pkg/aliyun/tts.goindex 923ea352..28cfdf33 100644--- a/pkg/aliyun/tts.go+++ b/pkg/aliyun/tts.go@@ -1,206 +1,341 @@package aliyunimport (+ "bytes"+ "context""encoding/json""fmt"- "github.com/gorilla/websocket"- "go.uber.org/zap"- "krillin-ai/config"- "krillin-ai/log"- "krillin-ai/pkg/util"+ "io""net/http"+ "net/url""os"+ "os/exec"+ "path/filepath"+ "runtime"+ "strconv"+ "strings""time"++ "krillin-ai/internal/resourcepath"+ "krillin-ai/internal/types")-type TtsClient struct {- AccessKeyID string- AccessKeySecret string- Appkey string-}+const (+ DefaultTTSBaseURL = "https://dashscope.aliyuncs.com/api/v1"+ DefaultTTSModel = "qwen3-tts-flash"+ DefaultTTSVoice = "Cherry"+)-type TtsHeader struct {- Appkey string `json:"appkey"`- MessageID string `json:"message_id"`- TaskID string `json:"task_id"`- Namespace string `json:"namespace"`- Name string `json:"name"`+type TtsClient struct {+ BaseURL string+ APIKey string+ Model string+ httpClient *http.Client}-type StartSynthesisPayload struct {- Voice string `json:"voice,omitempty"`- Format string `json:"format,omitempty"`- SampleRate int `json:"sample_rate,omitempty"`- Volume int `json:"volume,omitempty"`- SpeechRate int `json:"speech_rate,omitempty"`- PitchRate int `json:"pitch_rate,omitempty"`- EnableSubtitle bool `json:"enable_subtitle,omitempty"`- EnablePhonemeTimestamp bool `json:"enable_phoneme_timestamp,omitempty"`+type ttsRequest struct {+ Model string `json:"model"`+ Input ttsInput `json:"input"`}-type RunSynthesisPayload struct {- Text string `json:"text"`+type ttsInput struct {+ Text string `json:"text"`+ Voice string `json:"voice"`+ LanguageType string `json:"language_type"`}-type Message struct {- Header TtsHeader `json:"header"`- Payload interface{} `json:"payload,omitempty"`+type ttsResponse struct {+ Output struct {+ Audio struct {+ URL string `json:"url"`+ } `json:"audio"`+ } `json:"output"`+ Code string `json:"code"`+ Message string `json:"message"`+ RequestID string `json:"request_id"`}-func NewTtsClient(accessKeyId, accessKeySecret, appkey string) *TtsClient {+func NewTtsClient(baseURL, apiKey, model, proxy string) *TtsClient {+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")+ if baseURL == "" {+ baseURL = DefaultTTSBaseURL+ }+ if strings.TrimSpace(model) == "" {+ model = DefaultTTSModel+ }+ transport := http.DefaultTransport.(*http.Transport).Clone()+ if proxyURL, err := url.Parse(strings.TrimSpace(proxy)); err == nil && proxyURL.Scheme != "" {+ transport.Proxy = http.ProxyURL(proxyURL)+ }return &TtsClient{- AccessKeyID: accessKeyId,- AccessKeySecret: accessKeySecret,- Appkey: appkey,+ BaseURL: baseURL,+ APIKey: strings.TrimSpace(apiKey),+ Model: strings.TrimSpace(model),+ httpClient: &http.Client{+ Transport: transport,+ Timeout: 90 * time.Second,+ },}}func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {- file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0666)+ return c.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: text,+ Voice: voice,+ OutputFile: outputFile,+ })+}++func (c *TtsClient) Synthesize(ctx context.Context, options types.TTSSpeechOptions) error {+ if c.APIKey == "" {+ return fmt.Errorf("aliyun bailian tts api key is empty")+ }+ voice := strings.TrimSpace(options.Voice)+ if voice == "" {+ voice = DefaultTTSVoice+ }+ format := speechFormat(options.Format, options.OutputFile)+ speed := options.Speed+ if speed <= 0 {+ speed = 1+ }+ body, err := json.Marshal(ttsRequest{+ Model: c.Model,+ Input: ttsInput{+ Text: options.Text,+ Voice: voice,+ LanguageType: "Auto",+ },+ })if err != nil {- return fmt.Errorf("failed to create file: %w", err)+ return fmt.Errorf("aliyun bailian tts encode request: %w", err)}- defer file.Close()- var conn *websocket.Conn- token, _ := CreateToken(c.AccessKeyID, c.AccessKeySecret)- fullURL := "wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1?token=" + token- dialer := websocket.DefaultDialer- if config.Conf.App.Proxy != "" {- dialer.Proxy = http.ProxyURL(config.Conf.App.ParsedProxy)- }- dialer.HandshakeTimeout = 10 * time.Second- conn, _, err = dialer.Dial(fullURL, nil)+ ctx, cancel := context.WithTimeout(ctx, 90*time.Second)+ defer cancel()+ req, err := http.NewRequestWithContext(+ ctx,+ http.MethodPost,+ c.generationEndpoint(),+ bytes.NewReader(body),+ )if err != nil {return err}- _ = conn.SetReadDeadline(time.Now().Add(time.Second * 60))- defer c.Close(conn)-- onTextMessage := func(message string) {- log.GetLogger().Info("Received text message", zap.String("Message", message))+ req.Header.Set("Authorization", "Bearer "+c.APIKey)+ req.Header.Set("Content-Type", "application/json")+ response, err := c.httpClient.Do(req)+ if err != nil {+ return fmt.Errorf("aliyun bailian tts request failed: %w", err)}-- onBinaryMessage := func(data []byte) {- if _, err := file.Write(data); err != nil {- log.GetLogger().Error("Failed to write data to file", zap.Error(err))- }+ defer response.Body.Close()+ responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))+ if err != nil {+ return err}-- var (- synthesisStarted = make(chan struct{})- synthesisComplete = make(chan struct{})- )-- startPayload := StartSynthesisPayload{- Voice: voice,- Format: "wav",- SampleRate: 44100,- Volume: 50,- SpeechRate: 0,- PitchRate: 0,+ if response.StatusCode != http.StatusOK {+ return fmt.Errorf("aliyun bailian tts returned HTTP %d: %s", response.StatusCode, boundedMessage(responseBody))}-- go c.receiveMessages(conn, onTextMessage, onBinaryMessage, synthesisStarted, synthesisComplete)-- taskId := util.GenerateID()- log.GetLogger().Info("SpeechClient StartSynthesis", zap.String("taskId", taskId), zap.Any("payload", startPayload))- if err := c.StartSynthesis(conn, taskId, startPayload, synthesisStarted); err != nil {- return fmt.Errorf("failed to start synthesis: %w", err)+ var decoded ttsResponse+ if err := json.Unmarshal(responseBody, &decoded); err != nil {+ return fmt.Errorf("aliyun bailian tts decode response: %w", err)}-- if err := c.RunSynthesis(conn, taskId, text); err != nil {- return fmt.Errorf("failed to run synthesis: %w", err)+ if decoded.Code != "" {+ return fmt.Errorf("aliyun bailian tts error %s: %s", decoded.Code, decoded.Message)}-- if err := c.StopSynthesis(conn, taskId, synthesisComplete); err != nil {- return fmt.Errorf("failed to stop synthesis: %w", err)+ if decoded.Output.Audio.URL == "" {+ return fmt.Errorf("aliyun bailian tts response did not include an audio URL")}+ return c.downloadAndProcessAudio(+ ctx,+ decoded.Output.Audio.URL,+ options.OutputFile,+ format,+ speed,+ )+}- return nil+func (c *TtsClient) ListVoices(context.Context) ([]types.TTSVoice, error) {+ return append([]types.TTSVoice(nil), aliyunBuiltinVoices...), nil}-func (c *TtsClient) sendMessage(conn *websocket.Conn, taskId, name string, payload interface{}) error {- message := Message{- Header: TtsHeader{- Appkey: c.Appkey,- MessageID: util.GenerateID(),- TaskID: taskId,- Namespace: "FlowingSpeechSynthesizer",- Name: name,- },- Payload: payload,+func (c *TtsClient) generationEndpoint() string {+ if strings.HasSuffix(c.BaseURL, "/services/aigc/multimodal-generation/generation") {+ return c.BaseURL}- jsonData, _ := json.Marshal(message)- log.GetLogger().Debug("SpeechClient sendMessage", zap.String("message", string(jsonData)))- return conn.WriteJSON(message)+ return c.BaseURL + "/services/aigc/multimodal-generation/generation"}-func (c *TtsClient) StartSynthesis(conn *websocket.Conn, taskId string, payload StartSynthesisPayload, synthesisStarted chan struct{}) error {- err := c.sendMessage(conn, taskId, "StartSynthesis", payload)+func (c *TtsClient) downloadAudio(ctx context.Context, source, outputFile string) error {+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)if err != nil {return err}-- // 阻塞等待 SynthesisStarted 事件- <-synthesisStarted-+ response, err := c.httpClient.Do(req)+ if err != nil {+ return fmt.Errorf("aliyun bailian tts download audio: %w", err)+ }+ defer response.Body.Close()+ if response.StatusCode != http.StatusOK {+ return fmt.Errorf("aliyun bailian audio download returned HTTP %d", response.StatusCode)+ }+ if err := os.MkdirAll(filepath.Dir(outputFile), 0755); err != nil {+ return err+ }+ file, err := os.Create(outputFile)+ if err != nil {+ return err+ }+ defer file.Close()+ if _, err := io.Copy(file, io.LimitReader(response.Body, 100<<20)); err != nil {+ return err+ }return nil}-func (c *TtsClient) RunSynthesis(conn *websocket.Conn, taskId, text string) error {- return c.sendMessage(conn, taskId, "RunSynthesis", RunSynthesisPayload{Text: text})+func boundedMessage(value []byte) string {+ const limit = 300+ message := strings.TrimSpace(string(value))+ if len(message) <= limit {+ return message+ }+ return message[:limit]}-func (c *TtsClient) StopSynthesis(conn *websocket.Conn, taskId string, synthesisComplete chan struct{}) error {- err := c.sendMessage(conn, taskId, "StopSynthesis", nil)+func (c *TtsClient) downloadAndProcessAudio(+ ctx context.Context,+ source string,+ outputFile string,+ format string,+ speed float64,+) error {+ outputDir := filepath.Dir(outputFile)+ if err := os.MkdirAll(outputDir, 0755); err != nil {+ return err+ }+ native, err := os.CreateTemp(outputDir, ".aliyun-tts-*.wav")if err != nil {return err}-- // 阻塞等待 SynthesisCompleted 事件- <-synthesisComplete-- return nil+ nativePath := native.Name()+ if err := native.Close(); err != nil {+ return err+ }+ defer os.Remove(nativePath)+ if err := c.downloadAudio(ctx, source, nativePath); err != nil {+ return err+ }+ if format == "wav" && speed == 1 {+ if err := os.Remove(outputFile); err != nil && !os.IsNotExist(err) {+ return err+ }+ return os.Rename(nativePath, outputFile)+ }+ return transcodeAudio(ctx, nativePath, outputFile, format, speed)}-func (c *TtsClient) Close(conn *websocket.Conn) error {- err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))+func transcodeAudio(ctx context.Context, input, output, format string, speed float64) error {+ ffmpeg, err := ffmpegPath()if err != nil {return err}- return conn.Close()+ args := []string{"-y", "-i", input}+ if speed != 1 {+ args = append(args, "-filter:a", "atempo="+strconv.FormatFloat(speed, 'f', 3, 64))+ }+ args = append(args, "-ar", "44100", "-ac", "1")+ if format == "mp3" {+ args = append(args, "-c:a", "libmp3lame", "-b:a", "192k")+ } else {+ args = append(args, "-c:a", "pcm_s16le")+ }+ args = append(args, output)+ command := exec.CommandContext(ctx, ffmpeg, args...)+ if result, err := command.CombinedOutput(); err != nil {+ return fmt.Errorf("aliyun bailian tts audio conversion failed: %w, output: %s", err, boundedMessage(result))+ }+ return nil}-func (c *TtsClient) receiveMessages(conn *websocket.Conn, onTextMessage func(string), onBinaryMessage func([]byte), synthesisStarted, synthesisComplete chan struct{}) {- defer close(synthesisComplete)- for {- messageType, message, err := conn.ReadMessage()+func ffmpegPath() (string, error) {+ name := "ffmpeg"+ if runtime.GOOS == "windows" {+ name += ".exe"+ }+ if root, configured, err := resourcepath.Root(); err != nil {+ return "", err+ } else if configured {+ path, err := resourcepath.RequireFile("bin", name)if err != nil {- if websocket.IsCloseError(err, websocket.CloseNormalClosure) {- log.GetLogger().Error("SpeechClient receiveMessages websocket非正常关闭", zap.Error(err))- return- }- return- }- if messageType == websocket.TextMessage {- var msg Message- if err := json.Unmarshal(message, &msg); err != nil {- log.GetLogger().Error("SpeechClient receiveMessages json解析失败", zap.Error(err))- return- }- if msg.Header.Name == "SynthesisCompleted" {- log.GetLogger().Info("SynthesisCompleted event received")- // 收到结束消息退出- break- } else if msg.Header.Name == "SynthesisStarted" {- log.GetLogger().Info("SynthesisStarted event received")- close(synthesisStarted)- } else {- onTextMessage(string(message))- }- } else if messageType == websocket.BinaryMessage {- onBinaryMessage(message)+ return "", fmt.Errorf("aliyun bailian tts requires packaged ffmpeg under %s: %w", root, err)}+ return path, nil+ }+ path, err := exec.LookPath("ffmpeg")+ if err != nil {+ return "", fmt.Errorf("aliyun bailian tts requires ffmpeg for MP3 or speed conversion: %w", err)+ }+ return path, nil+}++var aliyunBuiltinVoices = []types.TTSVoice{+ {Code: "Cherry", Name: "芊悦", Language: "zh-CN", Gender: "female", Provider: "aliyun", Scenario: "阳光积极", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}, Recommended: true},+ {Code: "Serena", Name: "苏瑶", Language: "zh-CN", Gender: "female", Provider: "aliyun", Scenario: "温柔叙事", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Ethan", Name: "晨煦", Language: "zh-CN", Gender: "male", Provider: "aliyun", Scenario: "阳光男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Chelsie", Name: "千雪", Language: "zh-CN", Gender: "female", Provider: "aliyun", Scenario: "二次元", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Momo", Name: "茉兔", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "撒娇搞怪", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Vivian", Name: "十三", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "可爱小暴躁", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Moon", Name: "月白", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "率性帅气", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Maia", Name: "四月", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "知性温柔", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Kai", Name: "凯", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "舒缓自然", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Nofish", Name: "不吃鱼", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "自然设计师", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Bella", Name: "萌宝", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "活泼萝莉", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Jennifer", Name: "詹妮弗", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "电影质感美语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Ryan", Name: "甜茶", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "戏感张力", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Katerina", Name: "卡捷琳娜", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "成熟御姐", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Aiden", Name: "艾登", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "美语青年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Eldric Sage", Name: "沧明子", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "沉稳睿智老者", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Mia", Name: "乖小妹", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "温顺乖巧", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Mochi", Name: "沙小弥", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "聪慧少年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Bellona", Name: "燕铮莺", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "洪亮热血", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Vincent", Name: "田叔", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "沙哑豪迈", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Bunny", Name: "萌小姬", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "萌系萝莉", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Neil", Name: "阿闻", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "新闻主持", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Elias", Name: "墨讲师", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "知识讲解", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Arthur", Name: "徐大爷", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "乡土故事", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Nini", Name: "邻家妹妹", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "甜软亲切", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Seren", Name: "小婉", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "舒缓助眠", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Pip", Name: "顽屁小孩", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "调皮童声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Stella", Name: "少女阿月", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "甜美少女", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Bodega", Name: "博德加", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "西班牙男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Sonrisa", Name: "索尼莎", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "拉美女声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Alek", Name: "阿列克", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "俄语男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Dolce", Name: "多尔切", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "意大利男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Sohee", Name: "素熙", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "韩语女声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Ono Anna", Name: "小野杏", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "日语少女", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Lenn", Name: "莱恩", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "德语青年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Emilien", Name: "埃米尔安", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "法语青年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Andre", Name: "安德雷", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "磁性沉稳", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Radio Gol", Name: "拉迪奥·戈尔", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "足球解说", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Jada", Name: "上海-阿珍", Language: "zh-shanghai", Gender: "female", Provider: "aliyun", Scenario: "上海话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Dylan", Name: "北京-晓东", Language: "zh-beijing", Gender: "male", Provider: "aliyun", Scenario: "北京话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Li", Name: "南京-老李", Language: "zh-nanjing", Gender: "male", Provider: "aliyun", Scenario: "南京话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Marcus", Name: "陕西-秦川", Language: "zh-shaanxi", Gender: "male", Provider: "aliyun", Scenario: "陕西话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Roy", Name: "闽南-阿杰", Language: "zh-minnan", Gender: "male", Provider: "aliyun", Scenario: "闽南语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Peter", Name: "天津-李彼得", Language: "zh-tianjin", Gender: "male", Provider: "aliyun", Scenario: "天津话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Sunny", Name: "四川-晴儿", Language: "zh-sichuan", Gender: "female", Provider: "aliyun", Scenario: "四川话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Eric", Name: "四川-程川", Language: "zh-sichuan", Gender: "male", Provider: "aliyun", Scenario: "四川话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Rocky", Name: "粤语-阿强", Language: "zh-yue", Gender: "male", Provider: "aliyun", Scenario: "粤语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+ {Code: "Kiki", Name: "粤语-阿清", Language: "zh-yue", Gender: "female", Provider: "aliyun", Scenario: "粤语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},+}++func speechFormat(explicit, outputFile string) string {+ format := strings.ToLower(strings.TrimSpace(explicit))+ if format == "mp3" || format == "wav" {+ return format+ }+ if strings.EqualFold(filepath.Ext(outputFile), ".mp3") {+ return "mp3"}+ return "wav"}diff --git a/pkg/aliyun/tts_test.go b/pkg/aliyun/tts_test.gonew file mode 100644index 00000000..7841c225--- /dev/null+++ b/pkg/aliyun/tts_test.go@@ -0,0 +1,151 @@+package aliyun++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "runtime"+ "strings"+ "testing"++ "krillin-ai/internal/resourcepath"+ "krillin-ai/internal/types"+)++func TestSynthesizeUsesQwenTTSRequestAndDownloadsAudio(t *testing.T) {+ var server *httptest.Server+ server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/services/aigc/multimodal-generation/generation":+ if r.Method != http.MethodPost {+ t.Fatalf("method = %s, want POST", r.Method)+ }+ if r.Header.Get("Authorization") != "Bearer dashscope-key" {+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))+ }+ var request map[string]any+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Fatalf("decode request: %v", err)+ }+ if request["model"] != "qwen3-tts-flash" {+ t.Fatalf("model = %#v", request["model"])+ }+ input, ok := request["input"].(map[string]any)+ if !ok {+ t.Fatalf("input = %#v", request["input"])+ }+ if input["text"] != "你好,\"百炼\"\n下一行" {+ t.Fatalf("text = %#v", input["text"])+ }+ if input["voice"] != "Cherry" {+ t.Fatalf("voice = %#v", input["voice"])+ }+ if input["language_type"] != "Auto" {+ t.Fatalf("language_type = %#v", input["language_type"])+ }+ if _, exists := request["parameters"]; exists {+ t.Fatalf("parameters = %#v, Qwen-TTS does not accept CosyVoice parameters", request["parameters"])+ }+ _ = json.NewEncoder(w).Encode(map[string]any{+ "output": map[string]any{+ "audio": map[string]string{"url": server.URL + "/audio.wav"},+ },+ "request_id": "request-1",+ })+ case "/audio.wav":+ _, _ = w.Write([]byte("RIFF-test-audio"))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ output := filepath.Join(t.TempDir(), "speech.wav")+ client := NewTtsClient(server.URL, "dashscope-key", "qwen3-tts-flash", "")+ if err := client.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: "你好,\"百炼\"\n下一行",+ Voice: "Cherry",+ OutputFile: output,+ Format: "wav",+ Speed: 1,+ }); err != nil {+ t.Fatalf("Synthesize() error = %v", err)+ }+ content, err := os.ReadFile(output)+ if err != nil {+ t.Fatal(err)+ }+ if string(content) != "RIFF-test-audio" {+ t.Fatalf("audio = %q", content)+ }+}++func TestListVoicesUsesBuiltinQwenCatalogWithoutCredentials(t *testing.T) {+ client := NewTtsClient("http://127.0.0.1:1", "unused-key", "", "")+ voices, err := client.ListVoices(context.Background())+ if err != nil {+ t.Fatalf("ListVoices() error = %v", err)+ }+ for _, code := range []string{"Cherry", "Nofish", "Jennifer", "Jada", "Kiki"} {+ if !containsVoice(voices, code) {+ t.Fatalf("ListVoices() missing %q", code)+ }+ }+}++func TestTranscodeAudioUsesPackagedFFmpegForFormatAndSpeed(t *testing.T) {+ if runtime.GOOS == "windows" {+ t.Skip("shell fixture is POSIX-only")+ }+ root := t.TempDir()+ binDir := filepath.Join(root, "bin")+ if err := os.MkdirAll(binDir, 0755); err != nil {+ t.Fatal(err)+ }+ argsPath := filepath.Join(root, "ffmpeg-args.txt")+ ffmpeg := filepath.Join(binDir, "ffmpeg")+ script := "#!/bin/sh\n" ++ "printf '%s\\n' \"$@\" > \"" + argsPath + "\"\n" ++ "input=''\noutput=''\nprevious=''\n" ++ "for value in \"$@\"; do\n" ++ " if [ \"$previous\" = '-i' ]; then input=\"$value\"; fi\n" ++ " previous=\"$value\"\n" ++ " output=\"$value\"\n" ++ "done\n" ++ "cp \"$input\" \"$output\"\n"+ if err := os.WriteFile(ffmpeg, []byte(script), 0755); err != nil {+ t.Fatal(err)+ }+ t.Setenv(resourcepath.RootEnv, root)++ input := filepath.Join(root, "input.wav")+ output := filepath.Join(root, "output.mp3")+ if err := os.WriteFile(input, []byte("audio"), 0644); err != nil {+ t.Fatal(err)+ }+ if err := transcodeAudio(context.Background(), input, output, "mp3", 1.25); err != nil {+ t.Fatalf("transcodeAudio() error = %v", err)+ }+ args, err := os.ReadFile(argsPath)+ if err != nil {+ t.Fatal(err)+ }+ joined := string(args)+ for _, expected := range []string{"atempo=1.250", "libmp3lame", "192k", output} {+ if !strings.Contains(joined, expected) {+ t.Fatalf("ffmpeg args = %q, want %q", joined, expected)+ }+ }+}++func containsVoice(voices []types.TTSVoice, code string) bool {+ for _, voice := range voices {+ if voice.Code == code {+ return true+ }+ }+ return false+}diff --git a/pkg/minimax/tts.go b/pkg/minimax/tts.goindex 62e51373..5787e661 100644--- a/pkg/minimax/tts.go+++ b/pkg/minimax/tts.go@@ -14,6 +14,7 @@ import ("time""krillin-ai/config"+ "krillin-ai/internal/types""krillin-ai/log""go.uber.org/zap"@@ -96,23 +97,32 @@ type t2aResponse struct {// buildRequestBody 组装非流式 T2A v2 请求体,输出 wav 以匹配下游配音流程。func (c *TtsClient) buildRequestBody(text, voice string) ([]byte, error) {+ return c.buildSpeechRequest(types.TTSSpeechOptions{Text: text, Voice: voice})+}++func (c *TtsClient) buildSpeechRequest(options types.TTSSpeechOptions) ([]byte, error) {+ voice := strings.TrimSpace(options.Voice)voice = strings.TrimSpace(voice)if voice == "" {voice = DefaultVoice}+ speed := options.Speed+ if speed <= 0 {+ speed = 1+ }reqBody := t2aRequest{Model: c.Model,- Text: text,+ Text: options.Text,Stream: false,VoiceSetting: voiceSetting{VoiceID: voice,- Speed: 1,+ Speed: speed,Vol: 1,Pitch: 0,},AudioSetting: audioSetting{SampleRate: 44100,- Format: "wav",+ Format: speechFormat(options.Format, options.OutputFile),Channel: 1,},}@@ -141,16 +151,24 @@ func decodeAudio(respBody []byte) ([]byte, error) {// Text2Speech 将文本合成为语音并写入 outputFile(wav)。func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {+ return c.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: text,+ Voice: voice,+ OutputFile: outputFile,+ })+}++func (c *TtsClient) Synthesize(ctx context.Context, options types.TTSSpeechOptions) error {if c.ApiKey == "" {return fmt.Errorf("minimax tts api key is empty")}- body, err := c.buildRequestBody(text, voice)+ body, err := c.buildSpeechRequest(options)if err != nil {return fmt.Errorf("minimax tts build request failed: %w", err)}- ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ ctx, cancel := context.WithTimeout(ctx, 60*time.Second)defer cancel()url := c.BaseUrl + "/v1/t2a_v2"@@ -183,13 +201,118 @@ func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {return err}- outputDir := filepath.Dir(outputFile)+ outputDir := filepath.Dir(options.OutputFile)if err = os.MkdirAll(outputDir, 0755); err != nil {return fmt.Errorf("minimax tts create output dir failed: %w", err)}- if err = os.WriteFile(outputFile, audio, 0644); err != nil {+ if err = os.WriteFile(options.OutputFile, audio, 0644); err != nil {return fmt.Errorf("minimax tts write output file failed: %w", err)}return nil}++func speechFormat(explicit, outputFile string) string {+ format := strings.ToLower(strings.TrimSpace(explicit))+ if format == "mp3" || format == "wav" {+ return format+ }+ if strings.EqualFold(filepath.Ext(outputFile), ".mp3") {+ return "mp3"+ }+ return "wav"+}++func (c *TtsClient) ListVoices(ctx context.Context) ([]types.TTSVoice, error) {+ if c.ApiKey == "" {+ return append([]types.TTSVoice(nil), minimaxFallbackVoices...), nil+ }+ body, err := json.Marshal(map[string]string{"voice_type": "all"})+ if err != nil {+ return nil, err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseUrl+"/v1/get_voice", bytes.NewReader(body))+ if err != nil {+ return nil, err+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", "Bearer "+c.ApiKey)+ response, err := c.httpClient.Do(req)+ if err != nil {+ return nil, err+ }+ defer response.Body.Close()+ responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))+ if err != nil {+ return nil, err+ }+ if response.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("minimax get_voice returned HTTP %d", response.StatusCode)+ }+ var payload map[string]any+ if err := json.Unmarshal(responseBody, &payload); err != nil {+ return nil, err+ }+ voices := extractMiniMaxVoices(payload)+ if len(voices) == 0 {+ return append([]types.TTSVoice(nil), minimaxFallbackVoices...), nil+ }+ return voices, nil+}++func extractMiniMaxVoices(payload map[string]any) []types.TTSVoice {+ groups := []struct {+ key string+ kind string+ }{+ {key: "system_voice", kind: "builtin"},+ {key: "voice_cloning", kind: "custom"},+ {key: "voice_generation", kind: "designed"},+ }+ var voices []types.TTSVoice+ for _, group := range groups {+ values, ok := payload[group.key].([]any)+ if !ok {+ continue+ }+ for _, value := range values {+ item, ok := value.(map[string]any)+ if !ok {+ continue+ }+ code := miniMaxString(item, "voice_id", "voiceId")+ if code == "" {+ continue+ }+ name := miniMaxString(item, "voice_name", "voiceName")+ if name == "" {+ name = code+ }+ voices = append(voices, types.TTSVoice{+ Code: code,+ Name: name,+ Provider: "minimax",+ Kind: group.kind,+ })+ }+ }+ return voices+}++func miniMaxString(value map[string]any, keys ...string) string {+ for _, key := range keys {+ if candidate, ok := value[key].(string); ok && strings.TrimSpace(candidate) != "" {+ return strings.TrimSpace(candidate)+ }+ }+ return ""+}++var minimaxFallbackVoices = []types.TTSVoice{+ {Code: "English_Graceful_Lady", Name: "Graceful Lady", Language: "en", Gender: "female", Provider: "minimax", Scenario: "优雅女声", Kind: "builtin", Recommended: true},+ {Code: "English_radiant_girl", Name: "Radiant Girl", Language: "en", Gender: "female", Provider: "minimax", Scenario: "活泼女声", Kind: "builtin"},+ {Code: "English_Insightful_Speaker", Name: "Insightful Speaker", Language: "en", Gender: "male", Provider: "minimax", Scenario: "沉稳男声", Kind: "builtin"},+ {Code: "English_Persuasive_Man", Name: "Persuasive Man", Language: "en", Gender: "male", Provider: "minimax", Scenario: "说服力", Kind: "builtin"},+ {Code: "English_expressive_narrator", Name: "Expressive Narrator", Language: "en", Gender: "male", Provider: "minimax", Scenario: "旁白", Kind: "builtin"},+ {Code: "English_Lucky_Robot", Name: "Lucky Robot", Language: "en", Gender: "neutral", Provider: "minimax", Scenario: "机器人", Kind: "builtin"},+}diff --git a/pkg/minimax/tts_test.go b/pkg/minimax/tts_test.goindex c5275f7f..ef513cb4 100644--- a/pkg/minimax/tts_test.go+++ b/pkg/minimax/tts_test.go@@ -1,11 +1,16 @@package minimaximport (+ "context""encoding/hex""encoding/json"+ "net/http"+ "net/http/httptest""os""path/filepath""testing"++ "krillin-ai/internal/types")func TestNewTtsClientDefaults(t *testing.T) {@@ -110,3 +115,113 @@ func TestText2SpeechRequiresApiKey(t *testing.T) {t.Fatal("output file written despite missing api key")}}++func TestListVoicesRequestsAndParsesAllVoiceGroups(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/get_voice" {+ http.NotFound(w, r)+ return+ }+ if r.Header.Get("Authorization") != "Bearer minimax-key" {+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))+ }+ var request map[string]string+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Fatalf("decode request: %v", err)+ }+ if request["voice_type"] != "all" {+ t.Fatalf("voice_type = %q", request["voice_type"])+ }+ _ = json.NewEncoder(w).Encode(map[string]any{+ "system_voice": []map[string]string{{+ "voice_id": "system-1", "voice_name": "System Voice",+ }},+ "voice_cloning": []map[string]string{{+ "voice_id": "clone-1", "voice_name": "Cloned Voice",+ }},+ "voice_generation": []map[string]string{{+ "voice_id": "design-1", "voice_name": "Designed Voice",+ }},+ })+ }))+ defer server.Close()++ client := NewTtsClient(server.URL, "minimax-key", "speech-2.8-hd")+ voices, err := client.ListVoices(context.Background())+ if err != nil {+ t.Fatalf("ListVoices() error = %v", err)+ }+ if len(voices) != 3 {+ t.Fatalf("voices = %#v", voices)+ }+ if voices[0].Code != "system-1" || voices[0].Kind != "builtin" {+ t.Fatalf("system voice = %#v", voices[0])+ }+ if voices[1].Code != "clone-1" || voices[1].Kind != "custom" {+ t.Fatalf("cloned voice = %#v", voices[1])+ }+ if voices[2].Code != "design-1" || voices[2].Kind != "designed" {+ t.Fatalf("designed voice = %#v", voices[2])+ }+}++func TestSynthesizeSendsMiniMaxFormatAndSpeed(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/t2a_v2" {+ http.NotFound(w, r)+ return+ }+ if r.Header.Get("Authorization") != "Bearer minimax-key" {+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))+ }+ var request t2aRequest+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Fatalf("decode request: %v", err)+ }+ if request.Model != "speech-2.8-hd" {+ t.Fatalf("model = %q", request.Model)+ }+ if request.Text != "MiniMax test" {+ t.Fatalf("text = %q", request.Text)+ }+ if request.VoiceSetting.VoiceID != "English_Graceful_Lady" {+ t.Fatalf("voice_id = %q", request.VoiceSetting.VoiceID)+ }+ if request.VoiceSetting.Speed != 1.2 {+ t.Fatalf("speed = %v", request.VoiceSetting.Speed)+ }+ if request.AudioSetting.Format != "mp3" {+ t.Fatalf("format = %q", request.AudioSetting.Format)+ }+ _ = json.NewEncoder(w).Encode(map[string]any{+ "data": map[string]any{+ "audio": hex.EncodeToString([]byte("ID3-minimax-audio")),+ "status": 2,+ },+ "base_resp": map[string]any{+ "status_code": 0,+ "status_msg": "success",+ },+ })+ }))+ defer server.Close()++ output := filepath.Join(t.TempDir(), "speech.mp3")+ client := NewTtsClient(server.URL, "minimax-key", "speech-2.8-hd")+ if err := client.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: "MiniMax test",+ Voice: "English_Graceful_Lady",+ OutputFile: output,+ Format: "mp3",+ Speed: 1.2,+ }); err != nil {+ t.Fatalf("Synthesize() error = %v", err)+ }+ content, err := os.ReadFile(output)+ if err != nil {+ t.Fatal(err)+ }+ if string(content) != "ID3-minimax-audio" {+ t.Fatalf("audio = %q", content)+ }+}diff --git a/pkg/openai/openai.go b/pkg/openai/openai.goindex 0d6ea915..b60e753c 100644--- a/pkg/openai/openai.go+++ b/pkg/openai/openai.go@@ -10,8 +10,6 @@ import ("io""krillin-ai/config""krillin-ai/log"- "net/http"- "os""strings")@@ -84,53 +82,12 @@ func normalizeChatCompletionError(err error) error {}func (c *Client) Text2Speech(text, voice string, outputFile string) error {- baseUrl := config.Conf.Tts.Openai.BaseUrl- if baseUrl == "" {- baseUrl = "https://api.openai.com/v1"- }- url := baseUrl + "/audio/speech"-- // 创建HTTP请求- reqBody := fmt.Sprintf(`{- "model": "tts-1",- "input": "%s",- "voice":"%s",- "response_format": "wav"- }`, text, voice)- req, err := http.NewRequest("POST", url, strings.NewReader(reqBody))- if err != nil {- return err- }-- req.Header.Set("Content-Type", "application/json")- req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", config.Conf.Tts.Openai.ApiKey))-- // 发送HTTP请求- client := &http.Client{}- resp, err := client.Do(req)- if err != nil {- return err- }- defer resp.Body.Close()-- if resp.StatusCode != http.StatusOK {- body, _ := io.ReadAll(resp.Body)- log.GetLogger().Error("openai tts failed", zap.Int("status_code", resp.StatusCode), zap.String("body", string(body)))- return fmt.Errorf("openai tts none-200 status code: %d", resp.StatusCode)- }-- file, err := os.Create(outputFile)- if err != nil {- return err- }- defer file.Close()-- _, err = io.Copy(file, resp.Body)- if err != nil {- return err- }-- return nil+ return NewTtsClient(+ config.Conf.Tts.Openai.BaseUrl,+ config.Conf.Tts.Openai.ApiKey,+ config.Conf.Tts.Openai.Model,+ config.Conf.App.Proxy,+ ).Text2Speech(text, voice, outputFile)}func parseJSONResponse(jsonStr string) (string, error) {diff --git a/pkg/openai/tts.go b/pkg/openai/tts.gonew file mode 100644index 00000000..5e973518--- /dev/null+++ b/pkg/openai/tts.go@@ -0,0 +1,148 @@+package openai++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "os"+ "path/filepath"+ "strings"+ "time"++ "krillin-ai/internal/types"+)++const (+ DefaultTTSBaseURL = "https://api.openai.com/v1"+ DefaultTTSModel = "gpt-4o-mini-tts"+ DefaultTTSVoice = "marin"+)++type TtsClient struct {+ BaseURL string+ APIKey string+ Model string+ httpClient *http.Client+}++func NewTtsClient(baseURL, apiKey, model, proxy string) *TtsClient {+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")+ if baseURL == "" {+ baseURL = DefaultTTSBaseURL+ }+ if strings.TrimSpace(model) == "" {+ model = DefaultTTSModel+ }+ transport := http.DefaultTransport.(*http.Transport).Clone()+ if proxyURL, err := url.Parse(strings.TrimSpace(proxy)); err == nil && proxyURL.Scheme != "" {+ transport.Proxy = http.ProxyURL(proxyURL)+ }+ return &TtsClient{+ BaseURL: baseURL,+ APIKey: strings.TrimSpace(apiKey),+ Model: strings.TrimSpace(model),+ httpClient: &http.Client{+ Transport: transport,+ Timeout: 90 * time.Second,+ },+ }+}++func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {+ return c.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: text,+ Voice: voice,+ OutputFile: outputFile,+ })+}++func (c *TtsClient) Synthesize(ctx context.Context, options types.TTSSpeechOptions) error {+ if c.APIKey == "" {+ return fmt.Errorf("openai tts api key is empty")+ }+ voice := strings.TrimSpace(options.Voice)+ if voice == "" {+ voice = DefaultTTSVoice+ }+ format := speechFormat(options.Format, options.OutputFile)+ speed := options.Speed+ if speed <= 0 {+ speed = 1+ }+ requestBody := map[string]any{+ "model": c.Model,+ "input": options.Text,+ "voice": voice,+ "response_format": format,+ "speed": speed,+ }+ if strings.TrimSpace(options.Instructions) != "" && strings.Contains(strings.ToLower(c.Model), "gpt-4o") {+ requestBody["instructions"] = strings.TrimSpace(options.Instructions)+ }+ body, err := json.Marshal(requestBody)+ if err != nil {+ return err+ }+ ctx, cancel := context.WithTimeout(ctx, 90*time.Second)+ defer cancel()+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/audio/speech", bytes.NewReader(body))+ if err != nil {+ return err+ }+ req.Header.Set("Authorization", "Bearer "+c.APIKey)+ req.Header.Set("Content-Type", "application/json")+ response, err := c.httpClient.Do(req)+ if err != nil {+ return err+ }+ defer response.Body.Close()+ if response.StatusCode != http.StatusOK {+ message, _ := io.ReadAll(io.LimitReader(response.Body, 2048))+ return fmt.Errorf("openai tts returned HTTP %d: %s", response.StatusCode, strings.TrimSpace(string(message)))+ }+ if err := os.MkdirAll(filepath.Dir(options.OutputFile), 0755); err != nil {+ return err+ }+ file, err := os.Create(options.OutputFile)+ if err != nil {+ return err+ }+ defer file.Close()+ _, err = io.Copy(file, io.LimitReader(response.Body, 100<<20))+ return err+}++func speechFormat(explicit, outputFile string) string {+ format := strings.ToLower(strings.TrimSpace(explicit))+ if format == "mp3" || format == "wav" {+ return format+ }+ if strings.EqualFold(filepath.Ext(outputFile), ".mp3") {+ return "mp3"+ }+ return "wav"+}++func (c *TtsClient) ListVoices(context.Context) ([]types.TTSVoice, error) {+ return append([]types.TTSVoice(nil), openAIBuiltinVoices...), nil+}++var openAIBuiltinVoices = []types.TTSVoice{+ {Code: "alloy", Name: "Alloy", Language: "multi", Provider: "openai", Scenario: "均衡", Kind: "builtin"},+ {Code: "ash", Name: "Ash", Language: "multi", Provider: "openai", Scenario: "沉稳", Kind: "builtin"},+ {Code: "ballad", Name: "Ballad", Language: "multi", Provider: "openai", Scenario: "表现力", Kind: "builtin"},+ {Code: "cedar", Name: "Cedar", Language: "multi", Provider: "openai", Scenario: "自然", Kind: "builtin", Recommended: true},+ {Code: "coral", Name: "Coral", Language: "multi", Provider: "openai", Scenario: "温暖", Kind: "builtin"},+ {Code: "echo", Name: "Echo", Language: "multi", Provider: "openai", Scenario: "清晰", Kind: "builtin"},+ {Code: "fable", Name: "Fable", Language: "multi", Provider: "openai", Scenario: "叙事", Kind: "builtin"},+ {Code: "marin", Name: "Marin", Language: "multi", Provider: "openai", Scenario: "自然", Kind: "builtin", Recommended: true},+ {Code: "nova", Name: "Nova", Language: "multi", Provider: "openai", Scenario: "明快", Kind: "builtin"},+ {Code: "onyx", Name: "Onyx", Language: "multi", Provider: "openai", Scenario: "低沉", Kind: "builtin"},+ {Code: "sage", Name: "Sage", Language: "multi", Provider: "openai", Scenario: "中性", Kind: "builtin"},+ {Code: "shimmer", Name: "Shimmer", Language: "multi", Provider: "openai", Scenario: "柔和", Kind: "builtin"},+ {Code: "verse", Name: "Verse", Language: "multi", Provider: "openai", Scenario: "表达", Kind: "builtin"},+}diff --git a/pkg/openai/tts_test.go b/pkg/openai/tts_test.gonew file mode 100644index 00000000..cf2c4222--- /dev/null+++ b/pkg/openai/tts_test.go@@ -0,0 +1,77 @@+package openai++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "testing"++ "krillin-ai/internal/types"+)++func TestSynthesizeSendsOpenAITTSControlsAndWritesAudio(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/audio/speech" {+ http.NotFound(w, r)+ return+ }+ if r.Method != http.MethodPost {+ t.Fatalf("method = %s, want POST", r.Method)+ }+ if r.Header.Get("Authorization") != "Bearer openai-key" {+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))+ }+ var request map[string]any+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {+ t.Fatalf("decode request: %v", err)+ }+ if request["model"] != "gpt-4o-mini-tts" {+ t.Fatalf("model = %#v", request["model"])+ }+ if request["input"] != "say \"hello\"\nnext" {+ t.Fatalf("input = %#v", request["input"])+ }+ if request["voice"] != "marin" {+ t.Fatalf("voice = %#v", request["voice"])+ }+ if request["response_format"] != "mp3" {+ t.Fatalf("response_format = %#v", request["response_format"])+ }+ if request["speed"] != 1.15 {+ t.Fatalf("speed = %#v", request["speed"])+ }+ if request["instructions"] != "Speak warmly." {+ t.Fatalf("instructions = %#v", request["instructions"])+ }+ _, _ = w.Write([]byte("ID3-test-audio"))+ }))+ defer server.Close()++ output := filepath.Join(t.TempDir(), "speech.mp3")+ client := NewTtsClient(server.URL, "openai-key", "gpt-4o-mini-tts", "")+ if err := client.Synthesize(context.Background(), types.TTSSpeechOptions{+ Text: "say \"hello\"\nnext",+ OutputFile: output,+ Format: "mp3",+ Speed: 1.15,+ Instructions: " Speak warmly. ",+ }); err != nil {+ t.Fatalf("Synthesize() error = %v", err)+ }+ content, err := os.ReadFile(output)+ if err != nil {+ t.Fatal(err)+ }+ if string(content) != "ID3-test-audio" {+ t.Fatalf("audio = %q", content)+ }+}++func TestDefaultVoiceMatchesOpenCreator(t *testing.T) {+ if DefaultTTSVoice != "marin" {+ t.Fatalf("DefaultTTSVoice = %q, want marin", DefaultTTSVoice)+ }+}diff --git a/pkg/whisperkit/transcription.go b/pkg/whisperkit/transcription.goindex 7dd8236c..94b1472a 100644--- a/pkg/whisperkit/transcription.go+++ b/pkg/whisperkit/transcription.go@@ -1,6 +1,7 @@package whisperkitimport (+ "bytes""encoding/json""krillin-ai/internal/storage""krillin-ai/internal/types"@@ -8,12 +9,22 @@ import ("krillin-ai/pkg/util""os""os/exec"+ "regexp"+ "strconv""strings"+ "sync""go.uber.org/zap")func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {+ return c.TranscriptionWithProgress(audioFile, language, workDir, nil)+}++func (c *WhisperKitProcessor) TranscriptionWithProgress(+ audioFile, language, workDir string,+ reportProgress func(percent int),+) (*types.TranscriptionData, error) {cmdArgs := []string{"transcribe","--model-path", "./models/whisperkit/openai_whisper-large-v2",@@ -26,11 +37,17 @@ func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string)"--skip-special-tokens","--audio-path", audioFile,}+ if reportProgress != nil {+ cmdArgs = append(cmdArgs, "--verbose")+ }cmd := exec.Command(storage.WhisperKitPath, cmdArgs...)log.GetLogger().Info("WhisperKitProcessor转录开始", zap.String("cmd", cmd.String()))- output, err := cmd.CombinedOutput()+ output := newProgressOutput(reportProgress)+ cmd.Stdout = output+ cmd.Stderr = output+ err := cmd.Run()if err != nil {- log.GetLogger().Error("WhisperKitProcessor cmd 执行失败", zap.String("output", string(output)), zap.Error(err))+ log.GetLogger().Error("WhisperKitProcessor cmd 执行失败", zap.String("output", output.String()), zap.Error(err))return nil, err}log.GetLogger().Info("WhisperKitProcessor转录json生成完毕", zap.String("audio file", audioFile))@@ -88,3 +105,55 @@ func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string)log.GetLogger().Info("WhisperKitProcessor转录成功")return &transcriptionData, nil}++var whisperKitProgressPattern = regexp.MustCompile(`\]\s*([0-9]{1,3})%\s*\|`)++type progressOutput struct {+ mu sync.Mutex+ output bytes.Buffer+ pending string+ lastProgress int+ reportProgress func(percent int)+}++func newProgressOutput(reportProgress func(percent int)) *progressOutput {+ return &progressOutput{+ lastProgress: -1,+ reportProgress: reportProgress,+ }+}++func (w *progressOutput) Write(data []byte) (int, error) {+ w.mu.Lock()+ _, _ = w.output.Write(data)+ value := w.pending + string(data)+ progressValues := make([]int, 0)+ for _, match := range whisperKitProgressPattern.FindAllStringSubmatch(value, -1) {+ percent, err := strconv.Atoi(match[1])+ if err != nil || percent < 0 || percent > 100 || percent <= w.lastProgress {+ continue+ }+ w.lastProgress = percent+ progressValues = append(progressValues, percent)+ }+ const pendingLimit = 128+ if len(value) > pendingLimit {+ w.pending = value[len(value)-pendingLimit:]+ } else {+ w.pending = value+ }+ w.mu.Unlock()++ if w.reportProgress != nil {+ for _, percent := range progressValues {+ w.reportProgress(percent)+ }+ }+ return len(data), nil+}++func (w *progressOutput) String() string {+ w.mu.Lock()+ defer w.mu.Unlock()+ return w.output.String()+}diff --git a/pkg/whisperkit/transcription_test.go b/pkg/whisperkit/transcription_test.gonew file mode 100644index 00000000..8171e0b2--- /dev/null+++ b/pkg/whisperkit/transcription_test.go@@ -0,0 +1,34 @@+package whisperkit++import (+ "fmt"+ "testing"+)++func TestProgressOutputReportsMonotonicFragmentedPercentages(t *testing.T) {+ var reported []int+ output := newProgressOutput(func(percent int) {+ reported = append(reported, percent)+ })++ chunks := []string{+ "\x1b[K[ ] 0% | Elapsed Time: 0.00 s\r",+ "\x1b[K[=== ] 3",+ "3% | Elapsed Time: 1.00 s\r",+ "\x1b[K[====== ] 66% | Elapsed Time: 2.00 s\r",+ "\x1b[K[=== ] 33% | repeated\r",+ "\x1b[K[==========] 100% | Elapsed Time: 3.00 s\n",+ }+ for _, chunk := range chunks {+ if _, err := output.Write([]byte(chunk)); err != nil {+ t.Fatal(err)+ }+ }++ if got := fmt.Sprint(reported); got != "[0 33 66 100]" {+ t.Fatalf("reported progress = %s", got)+ }+ if output.String() == "" {+ t.Fatal("captured output is empty")+ }+}