Skip to content

Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.

feat(tts): add multi-provider dubbing support

wulien3746b41287d7
diff --git a/cmd/cli/main.go b/cmd/cli/main.go
index 20bef207..9c6a9116 100644
--- a/cmd/cli/main.go
+++ b/cmd/cli/main.go
@@ -29,9 +29,8 @@ func main() {
return
}
if cmd.Name == "voices" {
- if cmd.Voices.Provider == "" {
- _ = config.LoadConfig()
- }
+ _ = config.LoadConfig()
+ _ = config.CheckBaseConfig()
writeAndExit(output, cli.Execute(context.Background(), nil, cmd))
return
}
@@ -58,6 +57,13 @@ func main() {
if err := config.CheckBaseConfig(); err != nil {
writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))
}
+ if cmd.Name == "speech" {
+ if err := config.ValidateTTSConfig(); err != nil {
+ writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))
+ }
+ writeAndExit(output, cli.Execute(context.Background(), nil, cmd))
+ return
+ }
if requiresTranscriptionAtStart(cmd) {
if err := config.ValidateTranscriptionConfig(); err != nil {
writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))
@@ -72,6 +78,9 @@ func main() {
}
}
if cmd.Name == "tts" {
+ if err := config.ValidateTTSConfig(); err != nil {
+ writeAndExit(output, errorResponse(err, pipeline.ErrorKindUsage))
+ }
if err := deps.CheckTTSDependency(); err != nil {
writeAndExit(output, errorResponse(err, pipeline.ErrorKindDependency))
}
@@ -105,20 +114,27 @@ func (w *jsonLineWriter) Write(value any) error {
}
func configureOpenCreatorProgress(cmd *cli.Command, output *jsonLineWriter) {
- if os.Getenv("OPENCREATOR_KRILLINAI_CLI") != "1" || cmd.Name != "subtitle" {
+ if os.Getenv("OPENCREATOR_KRILLINAI_CLI") != "1" {
return
}
- previous := cmd.Subtitle.ReportProgress
- cmd.Subtitle.ReportProgress = func(phase string, percent int, message string) {
- if previous != nil {
- previous(phase, percent, message)
+ writeProgress := func(previous func(string, int, string)) func(string, int, string) {
+ return func(phase string, percent int, message string) {
+ if previous != nil {
+ previous(phase, percent, message)
+ }
+ _ = output.Write(progressFrame{
+ Type: "progress",
+ Phase: phase,
+ Percent: percent,
+ Message: message,
+ })
}
- _ = output.Write(progressFrame{
- Type: "progress",
- Phase: phase,
- Percent: percent,
- Message: message,
- })
+ }
+ switch cmd.Name {
+ case "subtitle":
+ cmd.Subtitle.ReportProgress = writeProgress(cmd.Subtitle.ReportProgress)
+ case "tts":
+ cmd.TTS.ReportProgress = writeProgress(cmd.TTS.ReportProgress)
}
}
diff --git a/cmd/cli/main_test.go b/cmd/cli/main_test.go
index 14296fed..b67c2e47 100644
--- a/cmd/cli/main_test.go
+++ b/cmd/cli/main_test.go
@@ -81,6 +81,28 @@ func TestConfigureOpenCreatorProgressWritesJSONLines(t *testing.T) {
}
}
+func TestConfigureOpenCreatorTTSProgressWritesJSONLines(t *testing.T) {
+ t.Setenv("OPENCREATOR_KRILLINAI_CLI", "1")
+ var buffer bytes.Buffer
+ output := &jsonLineWriter{output: &buffer}
+ cmd := cli.Command{Name: "tts"}
+
+ configureOpenCreatorProgress(&cmd, output)
+ if cmd.TTS.ReportProgress == nil {
+ t.Fatal("TTS ReportProgress is nil")
+ }
+ cmd.TTS.ReportProgress("generating_voice", 64, "working")
+
+ line := strings.TrimSpace(buffer.String())
+ var frame progressFrame
+ if err := json.Unmarshal([]byte(line), &frame); err != nil {
+ t.Fatalf("invalid progress JSON %q: %v", line, err)
+ }
+ if frame.Type != "progress" || frame.Phase != "generating_voice" || frame.Percent != 64 || frame.Message != "working" {
+ t.Fatalf("unexpected progress frame: %+v", frame)
+ }
+}
+
func TestConfigureOpenCreatorProgressStaysDisabledByDefault(t *testing.T) {
t.Setenv("OPENCREATOR_KRILLINAI_CLI", "")
cmd := cli.Command{Name: "subtitle"}
diff --git a/config/config-example.toml b/config/config-example.toml
index 7a6f1e25..2e20f223 100644
--- a/config/config-example.toml
+++ b/config/config-example.toml
@@ -56,15 +56,10 @@
base_url = "" # 留空默认海外版 https://api.minimax.io,国内可填 https://api.minimaxi.com
api_key = "" # MiniMax API密钥
model = "" # 留空默认 speech-2.8-hd,可选 speech-2.8-turbo, speech-2.6-hd, speech-2.6-turbo
- [tts.aliyun] # provider选aliyun这块就都要填
- [tts.aliyun.oss]
- access_key_id = ""
- access_key_secret = ""
- bucket = ""
- [tts.aliyun.speech]
- access_key_id = ""
- access_key_secret = ""
- app_key= ""
+ [tts.aliyun] # 阿里云百炼语音合成,仅 API Key 必填
+ base_url = "https://dashscope.aliyuncs.com/api/v1"
+ api_key = ""
+ model = "qwen3-tts-flash"
[dubbing]
min_subtitle_duration = 2.5 # 最短配音字幕时长,短句会优先合并
diff --git a/config/config.go b/config/config.go
index 16a0a557..f02960c9 100644
--- a/config/config.go
+++ b/config/config.go
@@ -73,8 +73,11 @@ type Transcribe struct {
}
type AliyunTtsConfig struct {
- Oss AliyunOssConfig `toml:"oss"`
- Speech AliyunSpeechConfig `toml:"speech"`
+ BaseUrl string `toml:"base_url"`
+ ApiKey string `toml:"api_key"`
+ Model string `toml:"model"`
+ Oss AliyunOssConfig `toml:"oss"`
+ Speech AliyunSpeechConfig `toml:"speech"`
}
type Tts struct {
@@ -155,6 +158,14 @@ var Conf = Config{
Openai: OpenaiCompatibleConfig{
Model: "gpt-4o-mini-tts",
},
+ Aliyun: AliyunTtsConfig{
+ BaseUrl: "https://dashscope.aliyuncs.com/api/v1",
+ Model: "qwen3-tts-flash",
+ },
+ Minimax: OpenaiCompatibleConfig{
+ BaseUrl: "https://api.minimax.io",
+ Model: "speech-2.8-hd",
+ },
},
Dubbing: Dubbing{
MinSubtitleDuration: 2.5,
@@ -215,6 +226,28 @@ func ValidateTranscriptionConfig() error {
return nil
}
+func ValidateTTSConfig() error {
+ switch Conf.Tts.Provider {
+ case "openai":
+ if Conf.Tts.Openai.ApiKey == "" {
+ return errors.New("使用 OpenAI 配音服务需要配置 API Key")
+ }
+ case "aliyun":
+ if Conf.Tts.Aliyun.ApiKey == "" {
+ return errors.New("使用阿里云百炼配音服务需要配置 API Key")
+ }
+ case "minimax":
+ if Conf.Tts.Minimax.ApiKey == "" {
+ return errors.New("使用 MiniMax 配音服务需要配置 API Key")
+ }
+ case "edge-tts":
+ return nil
+ default:
+ return errors.New("不支持的配音提供商")
+ }
+ return nil
+}
+
func LoadConfig() bool {
var err error
configPath := "./config/config.toml"
diff --git a/internal/cli/commands.go b/internal/cli/commands.go
index 14e11a86..6ec16bff 100644
--- a/internal/cli/commands.go
+++ b/internal/cli/commands.go
@@ -9,6 +9,8 @@ import (
"krillin-ai/config"
"krillin-ai/internal/pipeline"
subtitlestyle "krillin-ai/internal/subtitle_style"
+ "krillin-ai/internal/ttsprovider"
+ "krillin-ai/internal/types"
"krillin-ai/internal/updater"
"krillin-ai/internal/voices"
"os"
@@ -50,6 +52,7 @@ type Command struct {
SubtitleStyleFile string
Subtitle pipeline.SubtitleRequest
TTS pipeline.TTSRequest
+ Speech SpeechRequest
Render pipeline.RenderRequest
Cover pipeline.CoverRequest
Pipeline pipeline.PipelineRequest
@@ -68,6 +71,17 @@ type VoicesRequest struct {
Provider string
}
+type SpeechRequest struct {
+ Text string
+ TextFile string
+ Output string
+ Provider string
+ Voice string
+ Format string
+ Speed float64
+ Instructions string
+}
+
func Parse(args []string) (Command, error) {
if len(args) == 0 {
return Command{}, errors.New("missing command")
@@ -81,6 +95,8 @@ func Parse(args []string) (Command, error) {
return parseSubtitle(name, args[1:])
case "tts":
return parseTTS(name, args[1:])
+ case "speech":
+ return parseSpeech(name, args[1:])
case "render-horizontal":
return parseRender(name, args[1:], true)
case "render-vertical":
@@ -136,6 +152,22 @@ Flags:
--voice-clone-source <source> Optional voice clone source
--dry-run Validate and write manifest without external calls
-h, --help Show this help
+`
+ case "speech":
+ return `Usage:
+ krillinai-cli speech (--text <text> | --text-file <file>) --output <file> [flags]
+
+Flags:
+ --text <text> Text to synthesize
+ --text-file <file> UTF-8 text file to synthesize
+ --output <file> Output audio file
+ --provider <name> aliyun, openai, or minimax; default current config
+ --voice <voice> Provider-specific voice
+ --format <format> wav or mp3; defaults from output extension
+ --speed <value> Speaking rate from 0.5 to 2.0 (default 1)
+ --instructions <text> Delivery instructions when supported by the provider
+ --dry-run Validate command without external calls
+ -h, --help Show this help
`
case "render-horizontal":
return `Usage:
@@ -225,6 +257,7 @@ Status query is a reserved/planned CLI surface in the current implementation.
Commands:
subtitle Generate source, target, bilingual, and short vertical subtitles
tts Generate target-language dubbing from SRT subtitles
+ speech Generate audio from plain text
render-horizontal Render landscape subtitle or dubbed videos
render-vertical Render portrait subtitle or dubbed videos
pipeline Plan or run multi-stage workflows when supported
@@ -254,6 +287,8 @@ func Execute(ctx context.Context, svc pipeline.StageService, cmd Command) pipeli
case "tts":
resp, err := pipeline.GenerateTTS(ctx, svc, cmd.TTS)
return responseWithError(resp, err)
+ case "speech":
+ return executeSpeech(ctx, cmd.Speech)
case "render-horizontal", "render-vertical":
style, err := loadSubtitleStyleForCLI(cmd.SubtitleStyleFile)
if err != nil {
@@ -308,7 +343,7 @@ func executeVoices(req VoicesRequest) pipeline.Response {
if provider == "" {
provider = currentTTSProvider()
}
- list, err := voices.List(provider)
+ list, err := voices.List(context.Background(), provider)
if err != nil {
return pipeline.Response{
OK: false,
@@ -333,6 +368,112 @@ func executeVoices(req VoicesRequest) pipeline.Response {
}
}
+func parseSpeech(name string, args []string) (Command, error) {
+ if hasHelpArg(args) {
+ return Command{Name: name, Help: true}, nil
+ }
+ fs := newFlagSet(name)
+ text := fs.String("text", "", "text")
+ textFile := fs.String("text-file", "", "text file")
+ output := fs.String("output", "", "output audio")
+ provider := fs.String("provider", "", "tts provider")
+ voice := fs.String("voice", "", "voice")
+ format := fs.String("format", "", "audio format")
+ speed := fs.Float64("speed", 1, "speaking rate")
+ instructions := fs.String("instructions", "", "delivery instructions")
+ dryRun := fs.Bool("dry-run", false, "validate command without running external services")
+ if err := fs.Parse(args); err != nil {
+ return Command{}, err
+ }
+ if fs.NArg() != 0 {
+ return Command{}, errors.New("speech does not accept positional arguments")
+ }
+ if (strings.TrimSpace(*text) == "") == (strings.TrimSpace(*textFile) == "") {
+ return Command{}, errors.New("speech requires exactly one of --text or --text-file")
+ }
+ if strings.TrimSpace(*output) == "" {
+ return Command{}, errors.New("speech requires --output")
+ }
+ if *format != "" && *format != "wav" && *format != "mp3" {
+ return Command{}, errors.New("speech --format must be wav or mp3")
+ }
+ if *speed < 0.5 || *speed > 2 {
+ return Command{}, errors.New("speech --speed must be between 0.5 and 2")
+ }
+ return Command{
+ Name: name,
+ DryRun: *dryRun,
+ Speech: SpeechRequest{
+ Text: *text,
+ TextFile: *textFile,
+ Output: *output,
+ Provider: *provider,
+ Voice: *voice,
+ Format: *format,
+ Speed: *speed,
+ Instructions: *instructions,
+ },
+ }, nil
+}
+
+func executeSpeech(ctx context.Context, req SpeechRequest) pipeline.Response {
+ provider := strings.TrimSpace(req.Provider)
+ if provider == "" {
+ provider = currentTTSProvider()
+ }
+ text := req.Text
+ if req.TextFile != "" {
+ data, err := os.ReadFile(req.TextFile)
+ if err != nil {
+ return speechFailure(provider, req.Output, "speech_input_failed", err)
+ }
+ text = string(data)
+ }
+ if strings.TrimSpace(text) == "" {
+ return speechFailure(provider, req.Output, "speech_input_empty", errors.New("speech text is empty"))
+ }
+ client, err := ttsprovider.New(provider)
+ if err != nil {
+ return speechFailure(provider, req.Output, "speech_provider_unsupported", err)
+ }
+ if err := client.Synthesize(ctx, types.TTSSpeechOptions{
+ Text: text,
+ Voice: req.Voice,
+ OutputFile: req.Output,
+ Format: req.Format,
+ Speed: req.Speed,
+ Instructions: req.Instructions,
+ }); err != nil {
+ return speechFailure(provider, req.Output, "speech_synthesis_failed", err)
+ }
+ return pipeline.Response{
+ OK: true,
+ Stage: pipeline.StageSpeech,
+ Inputs: map[string]string{
+ "provider": provider,
+ "voice": req.Voice,
+ },
+ Outputs: pipeline.Outputs{TTSAudio: req.Output},
+ }
+}
+
+func speechFailure(provider, output, code string, err error) pipeline.Response {
+ return pipeline.Response{
+ OK: false,
+ Stage: pipeline.StageSpeech,
+ Inputs: map[string]string{
+ "provider": provider,
+ "output": output,
+ },
+ Error: &pipeline.Error{
+ Kind: pipeline.ErrorKindRetryable,
+ Code: code,
+ Message: err.Error(),
+ Retryable: true,
+ },
+ }
+}
+
func parseUpdate(name string, args []string) (Command, error) {
if hasHelpArg(args) {
return Command{Name: name, Help: true}, nil
@@ -582,6 +723,17 @@ func dryRun(cmd Command) pipeline.Response {
return dryRunResponse(pipeline.StageSubtitle, cmd.Subtitle.Workdir, cmd.Subtitle.TaskID)
case "tts":
return dryRunManifest(cmd.TTS.Workdir, cmd.TTS.TaskID, pipeline.StageTTS, nil)
+ case "speech":
+ return pipeline.Response{
+ OK: true,
+ Stage: pipeline.StageSpeech,
+ Inputs: map[string]string{
+ "provider": cmd.Speech.Provider,
+ "voice": cmd.Speech.Voice,
+ "output": cmd.Speech.Output,
+ "format": cmd.Speech.Format,
+ },
+ }
case "render-horizontal":
if _, err := loadSubtitleStyleForCLI(cmd.SubtitleStyleFile); err != nil {
return styleLoadFailure(pipeline.StageRenderHorizontal, cmd.Render.Workdir, cmd.Render.TaskID, err)
diff --git a/internal/cli/commands_test.go b/internal/cli/commands_test.go
index 0f8c1335..fe0d29c5 100644
--- a/internal/cli/commands_test.go
+++ b/internal/cli/commands_test.go
@@ -316,8 +316,8 @@ func TestExecuteDryRunVoicesReturnsAliyunVoiceList(t *testing.T) {
if len(resp.Voices) == 0 {
t.Fatal("Voices is empty, want aliyun voice list")
}
- if !containsVoiceCode(resp.Voices, "longxiaochun_v2") {
- t.Fatalf("Voices = %#v, want aliyun voice code longxiaochun_v2", resp.Voices)
+ if !containsVoiceCode(resp.Voices, "Cherry") {
+ t.Fatalf("Voices = %#v, want aliyun voice code Cherry", resp.Voices)
}
}
diff --git a/internal/pipeline/subtitle_test.go b/internal/pipeline/subtitle_test.go
index dbb82347..3fe01383 100644
--- a/internal/pipeline/subtitle_test.go
+++ b/internal/pipeline/subtitle_test.go
@@ -23,6 +23,7 @@ type fakeStageService struct {
coverImageB64 string
preparedVideoPath string
preparedAudioPath string
+ audioProgress []uint8
}
func (f *fakeStageService) PrepareMedia(_ context.Context, p *types.SubtitleTaskStepParam) error {
@@ -39,8 +40,11 @@ func (f *fakeStageService) PrepareMedia(_ context.Context, p *types.SubtitleTask
return nil
}
-func (f *fakeStageService) GenerateSubtitlesFromAudio(context.Context, *types.SubtitleTaskStepParam) error {
+func (f *fakeStageService) GenerateSubtitlesFromAudio(_ context.Context, p *types.SubtitleTaskStepParam) error {
f.calls = append(f.calls, "audio")
+ for _, percent := range f.audioProgress {
+ p.TaskPtr.SetProgress(percent)
+ }
return nil
}
@@ -161,6 +165,45 @@ func TestGenerateSubtitlesReportsPreparedLocalMediaOutputs(t *testing.T) {
}
}
+func TestGenerateSubtitlesMapsDetailedAudioProgress(t *testing.T) {
+ dir := t.TempDir()
+ fake := &fakeStageService{
+ audioProgress: []uint8{22, 32, 42, 52, 65, 77, 90},
+ }
+ var reported []int
+ req := SubtitleRequest{
+ Input: "local:demo.mp4",
+ Workdir: dir,
+ TaskID: "demo",
+ OriginLang: "zh_cn",
+ TargetLang: "en",
+ CaptionSource: CaptionSourceWhisper,
+ ReportProgress: func(_ string, percent int, _ string) {
+ reported = append(reported, percent)
+ },
+ }
+
+ if _, err := GenerateSubtitles(context.Background(), fake, req); err != nil {
+ t.Fatal(err)
+ }
+
+ want := []int{39, 45, 52, 58, 67, 75, 83}
+ for _, percent := range want {
+ if !containsInt(reported, percent) {
+ t.Fatalf("reported progress %v does not contain %d", reported, percent)
+ }
+ }
+}
+
+func containsInt(values []int, target int) bool {
+ for _, value := range values {
+ if value == target {
+ return true
+ }
+ }
+ return false
+}
+
func TestGenerateSubtitlesManualDoesNotFallback(t *testing.T) {
dir := t.TempDir()
fake := &fakeStageService{downloadErr: errors.New("no captions")}
diff --git a/internal/pipeline/tts.go b/internal/pipeline/tts.go
index 7b63c650..71d0c019 100644
--- a/internal/pipeline/tts.go
+++ b/internal/pipeline/tts.go
@@ -18,12 +18,14 @@ type TTSRequest struct {
Video string
Voice string
VoiceCloneSource string
+ ReportProgress func(phase string, percent int, message string)
}
func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (Response, error) {
if req.LineMode == "" {
req.LineMode = LineModeTargetOnly
}
+ reportTTSProgress(req, "preparing_voice", 10, "正在准备配音")
manifest, err := ttsManifest(req)
if err != nil {
return ttsFailureResponse(req, nil, ErrorKindInternal, "load_manifest_failed", err), err
@@ -65,6 +67,14 @@ func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (Respons
VideoWithTtsFilePath: manifest.Outputs.VideoWithTTS,
TargetLanguage: types.StandardLanguageCode(manifest.TargetLanguage),
}
+ stepParam.TaskPtr.SetProgressReporter(func(percent uint8) {
+ if percent >= 95 {
+ reportTTSProgress(req, "collecting_outputs", int(percent), "正在整理配音产物")
+ return
+ }
+ reportTTSProgress(req, "generating_voice", int(percent), "正在生成配音")
+ })
+ reportTTSProgress(req, "generating_voice", 20, "正在生成配音")
if err := svc.GenerateSpeechFromSRT(ctx, stepParam); err != nil {
return failTTSStage(req, manifest, "generate_speech_failed", err)
}
@@ -78,9 +88,17 @@ func GenerateTTS(ctx context.Context, svc StageService, req TTSRequest) (Respons
if err := manifest.Save(); err != nil {
return ttsFailureResponse(req, manifest, ErrorKindInternal, "save_manifest_failed", err), err
}
+ reportTTSProgress(req, "collecting_outputs", 99, "正在整理配音产物")
return ttsResponse(true, req, manifest, nil), nil
}
+func reportTTSProgress(req TTSRequest, phase string, percent int, message string) {
+ if req.ReportProgress == nil {
+ return
+ }
+ req.ReportProgress(phase, percent, message)
+}
+
func ttsManifest(req TTSRequest) (*Manifest, error) {
manifest, err := LoadManifest(req.Workdir)
if err == nil {
diff --git a/internal/pipeline/tts_test.go b/internal/pipeline/tts_test.go
index fe9aeb05..b1e49d67 100644
--- a/internal/pipeline/tts_test.go
+++ b/internal/pipeline/tts_test.go
@@ -102,3 +102,39 @@ func TestGenerateTTSUsesManifestTargetSRTWhenInputEmpty(t *testing.T) {
t.Fatalf("FailedIndexes = %v, want [2]", resp.FailedIndexes)
}
}
+
+func TestGenerateTTSReportsVoiceGenerationProgress(t *testing.T) {
+ dir := t.TempDir()
+ input := filepath.Join(dir, "target.srt")
+ if err := os.WriteFile(input, []byte("1\n00:00:00,000 --> 00:00:01,000\n你好\n\n"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ fake := &fakeStageService{}
+ type update struct {
+ phase string
+ percent int
+ }
+ var updates []update
+ req := TTSRequest{
+ Workdir: dir,
+ TaskID: "demo",
+ InputSRT: input,
+ ReportProgress: func(phase string, percent int, _ string) {
+ updates = append(updates, update{phase: phase, percent: percent})
+ },
+ }
+
+ if _, err := GenerateTTS(context.Background(), fake, req); err != nil {
+ t.Fatalf("GenerateTTS() error = %v", err)
+ }
+ if len(updates) < 3 {
+ t.Fatalf("progress updates = %+v, want preparing, generating, and collecting", updates)
+ }
+ if updates[0] != (update{phase: "preparing_voice", percent: 10}) {
+ t.Fatalf("first progress = %+v, want preparing_voice/10", updates[0])
+ }
+ last := updates[len(updates)-1]
+ if last != (update{phase: "collecting_outputs", percent: 99}) {
+ t.Fatalf("last progress = %+v, want collecting_outputs/99", last)
+ }
+}
diff --git a/internal/pipeline/types.go b/internal/pipeline/types.go
index e794328b..c74f69ee 100644
--- a/internal/pipeline/types.go
+++ b/internal/pipeline/types.go
@@ -8,6 +8,7 @@ const (
StageDownload Stage = "download"
StageSubtitle Stage = "subtitle"
StageTTS Stage = "tts"
+ StageSpeech Stage = "speech"
StageRenderHorizontal Stage = "render-horizontal"
StageRenderVertical Stage = "render-vertical"
StageCover Stage = "cover"
@@ -94,12 +95,15 @@ type Outputs struct {
}
type Voice struct {
- Code string `json:"code"`
- Name string `json:"name,omitempty"`
- Language string `json:"language,omitempty"`
- Gender string `json:"gender,omitempty"`
- Provider string `json:"provider,omitempty"`
- Scenario string `json:"scenario,omitempty"`
+ Code string `json:"code"`
+ Name string `json:"name,omitempty"`
+ Language string `json:"language,omitempty"`
+ Gender string `json:"gender,omitempty"`
+ Provider string `json:"provider,omitempty"`
+ Scenario string `json:"scenario,omitempty"`
+ Kind string `json:"kind,omitempty"`
+ SupportedModels []string `json:"supported_models,omitempty"`
+ Recommended bool `json:"recommended,omitempty"`
}
type Response struct {
diff --git a/internal/service/audio2subtitle.go b/internal/service/audio2subtitle.go
index 388d0b98..ae258035 100644
--- a/internal/service/audio2subtitle.go
+++ b/internal/service/audio2subtitle.go
@@ -51,8 +51,7 @@ type AudioSegment struct {
}
type TranslationProgress struct {
- Completed int
- Total int
+ Fraction float64
}
type contextChatCompleter interface {
@@ -120,7 +119,13 @@ func (s Service) audioToSubtitle(ctx context.Context, stepParam *types.SubtitleT
// return nil
//}
-func (s Service) transcribeAudio(id int, audioFilePath string, language string, taskBasePath string) (transcriptionData *types.TranscriptionData, err error) {
+func (s Service) transcribeAudio(
+ id int,
+ audioFilePath string,
+ language string,
+ taskBasePath string,
+ reportProgress func(percent int),
+) (transcriptionData *types.TranscriptionData, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("audioToSubtitle transcribeAudio panic recovered: %v", r)
@@ -130,7 +135,16 @@ func (s Service) transcribeAudio(id int, audioFilePath string, language string,
if language == "zh_cn" {
language = "zh" // 切换一下
}
- transcriptionData, err = s.Transcriber.Transcription(audioFilePath, language, taskBasePath)
+ if transcriber, ok := s.Transcriber.(types.ProgressTranscriber); ok {
+ transcriptionData, err = transcriber.TranscriptionWithProgress(
+ audioFilePath,
+ language,
+ taskBasePath,
+ reportProgress,
+ )
+ } else {
+ transcriptionData, err = s.Transcriber.Transcription(audioFilePath, language, taskBasePath)
+ }
if err != nil {
return nil, fmt.Errorf("audioToSubtitle transcribeAudio Transcription err: %w", err)
@@ -417,6 +431,7 @@ func (s Service) processAudioSegments(ctx context.Context, stepParam *types.Subt
splitResultQueue := make(chan DataWithId[string], segmentNum)
pendingTranscriptionQueue := make(chan DataWithId[string], segmentNum)
transcribedQueue := make(chan DataWithId[*types.TranscriptionData], segmentNum)
+ transcriptionProgressQueue := make(chan DataWithId[float64], segmentNum*2)
pendingTranslationQueue := make(chan DataWithId[string], segmentNum)
translationProgressQueue := make(chan DataWithId[TranslationProgress], segmentNum*2)
translatedQueue := make(chan DataWithId[[]*TranslatedItem], segmentNum)
@@ -438,14 +453,21 @@ func (s Service) processAudioSegments(ctx context.Context, stepParam *types.Subt
s.startSplitWorkers(ctx, eg, stepParam, pendingSplitQueue, splitResultQueue)
// 启动转录协程
- s.startTranscribeWorkers(ctx, eg, stepParam, pendingTranscriptionQueue, transcribedQueue)
+ s.startTranscribeWorkers(
+ ctx,
+ eg,
+ stepParam,
+ pendingTranscriptionQueue,
+ transcriptionProgressQueue,
+ transcribedQueue,
+ )
// 启动翻译协程
s.startTranslateWorker(ctx, eg, stepParam, pendingTranslationQueue, translationProgressQueue, translatedQueue)
// 处理结果协程
s.startResultHandler(ctx, eg, stepParam, segmentNum, timePoints, audioSegments,
- splitResultQueue, pendingTranscriptionQueue, transcribedQueue,
+ splitResultQueue, pendingTranscriptionQueue, transcriptionProgressQueue, transcribedQueue,
pendingTranslationQueue, translationProgressQueue, translatedQueue, pendingSplitQueue)
if err := eg.Wait(); err != nil {
@@ -492,7 +514,9 @@ func (s Service) startSplitWorkers(ctx context.Context, eg *errgroup.Group, step
// 启动转录工作协程
func (s Service) startTranscribeWorkers(ctx context.Context, eg *errgroup.Group, stepParam *types.SubtitleTaskStepParam,
- pendingTranscriptionQueue chan DataWithId[string], transcribedQueue chan DataWithId[*types.TranscriptionData]) {
+ pendingTranscriptionQueue chan DataWithId[string],
+ transcriptionProgressQueue chan DataWithId[float64],
+ transcribedQueue chan DataWithId[*types.TranscriptionData]) {
for range config.Conf.App.TranscribeParallelNum {
eg.Go(func() error {
@@ -511,7 +535,21 @@ func (s Service) startTranscribeWorkers(ctx context.Context, eg *errgroup.Group,
log.GetLogger().Info("Begin transcribe", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", audioFileItem.Id))
// 语音转文字
for range config.Conf.App.TranscribeMaxAttempts {
- transcriptionData, err = s.transcribeAudio(audioFileItem.Id, audioFileItem.Data, string(stepParam.OriginLanguage), stepParam.TaskBasePath)
+ transcriptionData, err = s.transcribeAudio(
+ audioFileItem.Id,
+ audioFileItem.Data,
+ string(stepParam.OriginLanguage),
+ stepParam.TaskBasePath,
+ func(percent int) {
+ select {
+ case transcriptionProgressQueue <- DataWithId[float64]{
+ Data: float64(percent) / 100,
+ Id: audioFileItem.Id,
+ }:
+ case <-ctx.Done():
+ }
+ },
+ )
if err == nil {
break
}
@@ -537,6 +575,8 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, s
pendingTranslationQueue chan DataWithId[string], translationProgressQueue chan DataWithId[TranslationProgress],
translatedQueue chan DataWithId[[]*TranslatedItem]) {
+ const primaryTranslationProgressShare = 0.35
+
eg.Go(func() error {
for {
select {
@@ -555,9 +595,10 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, s
attempts = 1
}
reportProgress := func(completed, total int) error {
+ fraction := scaledProgress(completed, total, 0, primaryTranslationProgressShare)
select {
case translationProgressQueue <- DataWithId[TranslationProgress]{
- Data: TranslationProgress{Completed: completed, Total: total},
+ Data: TranslationProgress{Fraction: fraction},
Id: translateItem.Id,
}:
return nil
@@ -576,8 +617,21 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, s
}
_ = util.SaveToDisk(translatedResults, filepath.Join(stepParam.TaskBasePath, fmt.Sprintf(types.SubtitleTaskTranslationDataPersistenceFileNamePattern, translateItem.Id)))
log.GetLogger().Info("Translate completed", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id))
+ reportTranslationProgress(
+ ctx,
+ translationProgressQueue,
+ translateItem.Id,
+ primaryTranslationProgressShare,
+ )
// 二次分割长句
- splitResults, err := s.splitTranslateItem(translatedResults)
+ splitResults, err := s.splitTranslateItem(translatedResults, func(completed, total int) {
+ reportTranslationProgress(
+ ctx,
+ translationProgressQueue,
+ translateItem.Id,
+ scaledProgress(completed, total, primaryTranslationProgressShare, 1),
+ )
+ })
if err != nil {
// 不中断
log.GetLogger().Error("audioToSubtitle audioToSrt splitTranslateItem err", zap.Any("taskId", stepParam.TaskId), zap.Any("splitId", translateItem.Id), zap.Error(err))
@@ -600,6 +654,7 @@ func (s Service) startTranslateWorker(ctx context.Context, eg *errgroup.Group, s
func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, stepParam *types.SubtitleTaskStepParam,
segmentNum int, timePoints []float64, audioSegments []AudioSegment,
splitResultQueue chan DataWithId[string], pendingTranscriptionQueue chan DataWithId[string],
+ transcriptionProgressQueue chan DataWithId[float64],
transcribedQueue chan DataWithId[*types.TranscriptionData], pendingTranslationQueue chan DataWithId[string],
translationProgressQueue chan DataWithId[TranslationProgress], translatedQueue chan DataWithId[[]*TranslatedItem],
pendingSplitQueue chan DataWithId[[2]float64]) {
@@ -612,18 +667,18 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, ste
TRANSLATE_WEIGHT = 0.5
)
splitCompleted := 0
- transcriptionCompleted := 0
+ transcriptionProgress := make(map[int]float64, segmentNum)
translationProgress := make(map[int]float64, segmentNum)
updateProgress := func() {
- translationTotal := 0.0
- for _, progress := range translationProgress {
- translationTotal += progress
- }
- processPct := 15.0 +
- float64(splitCompleted)/float64(segmentNum)*75*SPLIT_WEIGHT +
- float64(transcriptionCompleted)/float64(segmentNum)*75*TRANSCRIBE_WEIGHT +
- translationTotal/float64(segmentNum)*75*TRANSLATE_WEIGHT
- stepParam.TaskPtr.SetProgress(uint8(processPct))
+ stepParam.TaskPtr.SetProgress(audioProcessPercent(
+ segmentNum,
+ splitCompleted,
+ transcriptionProgress,
+ translationProgress,
+ SPLIT_WEIGHT,
+ TRANSCRIBE_WEIGHT,
+ TRANSLATE_WEIGHT,
+ ))
}
// 完成的任务数量
completedTasks := 0
@@ -641,8 +696,14 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, ste
Data: splitResultItem.Data,
Id: splitResultItem.Id,
}
+ case progressItem := <-transcriptionProgressQueue:
+ fraction := clampFraction(progressItem.Data)
+ if fraction > transcriptionProgress[progressItem.Id] {
+ transcriptionProgress[progressItem.Id] = fraction
+ updateProgress()
+ }
case transcribedItem := <-transcribedQueue:
- transcriptionCompleted++
+ transcriptionProgress[transcribedItem.Id] = 1
updateProgress()
// 处理转录结果
audioSegments[transcribedItem.Id].TranscriptionData = transcribedItem.Data
@@ -652,15 +713,10 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, ste
Id: transcribedItem.Id,
}
case progressItem := <-translationProgressQueue:
- if progressItem.Data.Total > 0 {
- fraction := float64(progressItem.Data.Completed) / float64(progressItem.Data.Total)
- if fraction > 1 {
- fraction = 1
- }
- if fraction > translationProgress[progressItem.Id] {
- translationProgress[progressItem.Id] = fraction
- updateProgress()
- }
+ fraction := clampFraction(progressItem.Data.Fraction)
+ if fraction > translationProgress[progressItem.Id] {
+ translationProgress[progressItem.Id] = fraction
+ updateProgress()
}
case translatedItems := <-translatedQueue:
translationProgress[translatedItems.Id] = 1
@@ -711,6 +767,7 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, ste
close(pendingSplitQueue)
close(splitResultQueue)
close(pendingTranscriptionQueue)
+ close(transcriptionProgressQueue)
close(transcribedQueue)
close(pendingTranslationQueue)
close(translationProgressQueue)
@@ -722,6 +779,65 @@ func (s Service) startResultHandler(ctx context.Context, eg *errgroup.Group, ste
})
}
+func reportTranslationProgress(
+ ctx context.Context,
+ queue chan DataWithId[TranslationProgress],
+ id int,
+ fraction float64,
+) {
+ select {
+ case queue <- DataWithId[TranslationProgress]{
+ Data: TranslationProgress{Fraction: clampFraction(fraction)},
+ Id: id,
+ }:
+ case <-ctx.Done():
+ }
+}
+
+func scaledProgress(completed, total int, start, end float64) float64 {
+ if total <= 0 {
+ return end
+ }
+ return start + (end-start)*clampFraction(float64(completed)/float64(total))
+}
+
+func clampFraction(value float64) float64 {
+ if value < 0 {
+ return 0
+ }
+ if value > 1 {
+ return 1
+ }
+ return value
+}
+
+func audioProcessPercent(
+ segmentNum int,
+ splitCompleted int,
+ transcriptionProgress map[int]float64,
+ translationProgress map[int]float64,
+ splitWeight float64,
+ transcribeWeight float64,
+ translateWeight float64,
+) uint8 {
+ if segmentNum <= 0 {
+ return 15
+ }
+ transcriptionTotal := 0.0
+ for _, progress := range transcriptionProgress {
+ transcriptionTotal += clampFraction(progress)
+ }
+ translationTotal := 0.0
+ for _, progress := range translationProgress {
+ translationTotal += clampFraction(progress)
+ }
+ processPct := 15.0 +
+ float64(splitCompleted)/float64(segmentNum)*75*splitWeight +
+ transcriptionTotal/float64(segmentNum)*75*transcribeWeight +
+ translationTotal/float64(segmentNum)*75*translateWeight
+ return uint8(processPct)
+}
+
// 合并字幕文件
func (s Service) mergeSubtitleFiles(stepParam *types.SubtitleTaskStepParam, audioSegments []AudioSegment, segmentNum int) error {
// 合并文件
@@ -1459,9 +1575,22 @@ func calcLength(text string) float64 {
}
// splitTranslateItem 根据字符权重和最大长度分割长句
-func (s Service) splitTranslateItem(items []*TranslatedItem) ([]*TranslatedItem, error) {
+func (s Service) splitTranslateItem(
+ items []*TranslatedItem,
+ reportProgress func(completed, total int),
+) ([]*TranslatedItem, error) {
var result []*TranslatedItem
maxLength := config.Conf.App.MaxSentenceLength + 30
+ longSentenceTotal := 0
+ for _, item := range items {
+ if calcLength(item.OriginText) > float64(maxLength) || calcLength(item.TranslatedText) > float64(maxLength) {
+ longSentenceTotal++
+ }
+ }
+ if reportProgress != nil {
+ reportProgress(0, longSentenceTotal)
+ }
+ longSentenceCompleted := 0
for _, item := range items {
// 计算翻译文本的加权长度
@@ -1478,8 +1607,15 @@ func (s Service) splitTranslateItem(items []*TranslatedItem) ([]*TranslatedItem,
return nil, fmt.Errorf("split long sentence error: %w", err)
}
result = append(result, splitItems...)
+ longSentenceCompleted++
+ if reportProgress != nil {
+ reportProgress(longSentenceCompleted, longSentenceTotal)
+ }
}
+ if longSentenceTotal == 0 && reportProgress != nil {
+ reportProgress(1, 1)
+ }
return result, nil
}
diff --git a/internal/service/audio2subtitle_test.go b/internal/service/audio2subtitle_test.go
index 791ea78c..9ee637c4 100644
--- a/internal/service/audio2subtitle_test.go
+++ b/internal/service/audio2subtitle_test.go
@@ -93,6 +93,97 @@ func Test_splitOriginLongSentence(t *testing.T) {
}
}
+type progressTranscriber struct {
+ progress []int
+}
+
+func (t progressTranscriber) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
+ return t.TranscriptionWithProgress(audioFile, language, workDir, nil)
+}
+
+func (t progressTranscriber) TranscriptionWithProgress(
+ _, _, _ string,
+ reportProgress func(percent int),
+) (*types.TranscriptionData, error) {
+ for _, percent := range t.progress {
+ if reportProgress != nil {
+ reportProgress(percent)
+ }
+ }
+ return &types.TranscriptionData{Text: "transcribed"}, nil
+}
+
+func TestTranscribeAudioForwardsProviderProgress(t *testing.T) {
+ service := Service{Transcriber: progressTranscriber{progress: []int{0, 33, 66, 100}}}
+ var reported []int
+
+ result, err := service.transcribeAudio(
+ 0,
+ "sample.mp3",
+ "zh_cn",
+ t.TempDir(),
+ func(percent int) {
+ reported = append(reported, percent)
+ },
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.Text != "transcribed" {
+ t.Fatalf("transcription text = %q", result.Text)
+ }
+ if got := fmt.Sprint(reported); got != "[0 33 66 100]" {
+ t.Fatalf("reported progress = %s", got)
+ }
+}
+
+func TestAudioProcessPercentUsesDetailedTranscriptionAndTranslationProgress(t *testing.T) {
+ const (
+ splitWeight = 0.1
+ transcribeWeight = 0.4
+ translateWeight = 0.5
+ )
+ tests := []struct {
+ name string
+ split int
+ transcription float64
+ translation float64
+ want uint8
+ }{
+ {name: "audio split", split: 1, want: 22},
+ {name: "transcription one third", split: 1, transcription: 0.33, want: 32},
+ {name: "transcription two thirds", split: 1, transcription: 0.66, want: 42},
+ {name: "transcription complete", split: 1, transcription: 1, want: 52},
+ {name: "primary translation complete", split: 1, transcription: 1, translation: 0.35, want: 65},
+ {name: "long sentence halfway", split: 1, transcription: 1, translation: 0.675, want: 77},
+ {name: "subtitle processing complete", split: 1, transcription: 1, translation: 1, want: 90},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ transcription := map[int]float64{}
+ if test.transcription > 0 {
+ transcription[0] = test.transcription
+ }
+ translation := map[int]float64{}
+ if test.translation > 0 {
+ translation[0] = test.translation
+ }
+ got := audioProcessPercent(
+ 1,
+ test.split,
+ transcription,
+ translation,
+ splitWeight,
+ transcribeWeight,
+ translateWeight,
+ )
+ if got != test.want {
+ t.Fatalf("audioProcessPercent() = %d, want %d", got, test.want)
+ }
+ })
+ }
+}
+
type echoBatchCompleter struct {
batchSizes []int
}
@@ -236,3 +327,37 @@ func TestParseTranslationBatchAcceptsFencedJSON(t *testing.T) {
t.Fatalf("translations = %v", translations)
}
}
+
+func TestSplitTranslateItemReportsLongSentenceProgress(t *testing.T) {
+ previousMaxSentenceLength := config.Conf.App.MaxSentenceLength
+ config.Conf.App.MaxSentenceLength = 10
+ t.Cleanup(func() { config.Conf.App.MaxSentenceLength = previousMaxSentenceLength })
+
+ completer := &scriptedCompleter{responses: []string{
+ `{"align":[{"origin_part":"first origin part","translated_part":"first translated part"}]}`,
+ `{"align":[{"origin_part":"second origin part","translated_part":"second translated part"}]}`,
+ }}
+ service := Service{ChatCompleter: completer}
+ items := []*TranslatedItem{
+ {OriginText: "short", TranslatedText: "short"},
+ {
+ OriginText: strings.Repeat("first origin ", 8),
+ TranslatedText: strings.Repeat("first translated ", 8),
+ },
+ {
+ OriginText: strings.Repeat("second origin ", 8),
+ TranslatedText: strings.Repeat("second translated ", 8),
+ },
+ }
+ var progress []string
+
+ _, err := service.splitTranslateItem(items, func(completed, total int) {
+ progress = append(progress, fmt.Sprintf("%d/%d", completed, total))
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := fmt.Sprint(progress); got != "[0/2 1/2 2/2]" {
+ t.Fatalf("progress = %s", got)
+ }
+}
diff --git a/internal/service/dubbing/audio.go b/internal/service/dubbing/audio.go
index 929336fd..52ccb72e 100644
--- a/internal/service/dubbing/audio.go
+++ b/internal/service/dubbing/audio.go
@@ -10,6 +10,8 @@ import (
"strings"
)
+const timelineComparisonEpsilon = 1e-6
+
func defaultFFmpegRunner(args []string) error {
cmd := exec.Command(storage.FfmpegPath, args...)
output, err := cmd.CombinedOutput()
@@ -250,7 +252,7 @@ func validateAssemblePlan(plan []PlanItem, segmentsDir string) ([]string, error)
if item.NewEnd <= item.NewStart {
return nil, fmt.Errorf("plan item %d new end must be greater than new start: start %.3f end %.3f", item.Index, item.NewStart, item.NewEnd)
}
- if item.NewStart < lastEnd {
+ if item.NewStart+timelineComparisonEpsilon < lastEnd {
return nil, fmt.Errorf("plan item %d starts before previous end: start %.3f lastEnd %.3f", item.Index, item.NewStart, lastEnd)
}
@@ -284,7 +286,7 @@ func validateAssembleChunkPlan(plan []PlanItem, chunks []Chunk, segmentsDir stri
if len(chunk.Items) == 0 {
return nil, fmt.Errorf("chunk %d has no items", chunk.ID)
}
- if chunk.Start < lastEnd {
+ if chunk.Start+timelineComparisonEpsilon < lastEnd {
return nil, fmt.Errorf("chunk %d starts before previous end: start %.3f lastEnd %.3f", chunk.ID, chunk.Start, lastEnd)
}
end := chunkFittedEnd(plan, chunk)
diff --git a/internal/service/dubbing/audio_assemble_test.go b/internal/service/dubbing/audio_assemble_test.go
index 187f1046..a1f856d8 100644
--- a/internal/service/dubbing/audio_assemble_test.go
+++ b/internal/service/dubbing/audio_assemble_test.go
@@ -237,3 +237,31 @@ func TestAssembleChunkAudioPreservesGapAfterShortChunkAudio(t *testing.T) {
t.Fatalf("silence before second chunk = %q, want 4.000", silenceDuration)
}
}
+
+func TestAssembleChunkAudioAllowsFloatingPointBoundaryNoise(t *testing.T) {
+ dir := t.TempDir()
+ rawDir := filepath.Join(dir, "raw")
+ if err := os.MkdirAll(rawDir, 0755); err != nil {
+ t.Fatal(err)
+ }
+ for _, name := range []string{"chunk_16.wav", "chunk_17.wav"} {
+ if err := os.WriteFile(filepath.Join(rawDir, name), []byte("raw"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ plan := []PlanItem{
+ {Index: 42, NewStart: 96.84, NewEnd: 111.00000000000001, SpeedFactor: 1.028, ChunkID: 16},
+ {Index: 43, NewStart: 111, NewEnd: 114.08, SpeedFactor: 1, ChunkID: 17},
+ }
+ chunks := []Chunk{
+ {ID: 16, Items: []int{0}, Start: 96.84, End: 111, ActualDuration: 14.56, SpeedFactor: 1.028},
+ {ID: 17, Items: []int{1}, Start: 111, End: 114.08, ActualDuration: 3.08, SpeedFactor: 1},
+ }
+
+ err := AssembleChunkAudio(plan, chunks, dir, filepath.Join(dir, "out.wav"), func(args []string) error {
+ return os.WriteFile(args[len(args)-1], []byte("media"), 0644)
+ })
+ if err != nil {
+ t.Fatalf("AssembleChunkAudio() error = %v, want floating-point boundary accepted", err)
+ }
+}
diff --git a/internal/service/dubbing/fit.go b/internal/service/dubbing/fit.go
index 9b2f2619..9d3c99db 100644
--- a/internal/service/dubbing/fit.go
+++ b/internal/service/dubbing/fit.go
@@ -40,9 +40,6 @@ func FitTimeline(plan []PlanItem, chunks []Chunk, cfg Config) ([]PlanItem, []Chu
report.Warnings = append(report.Warnings, fmt.Sprintf("chunk %d speed %.2f exceeds max %.2f", chunk.ID, speed, cfg.SpeedMax))
}
appliedSpeed := speed
- if appliedSpeed > cfg.SpeedMax {
- appliedSpeed = cfg.SpeedMax
- }
if appliedSpeed < cfg.SpeedMin {
appliedSpeed = cfg.SpeedMin
}
diff --git a/internal/service/dubbing/fit_test.go b/internal/service/dubbing/fit_test.go
index 53a6ff87..4f2185d5 100644
--- a/internal/service/dubbing/fit_test.go
+++ b/internal/service/dubbing/fit_test.go
@@ -50,7 +50,7 @@ func TestFitTimelineUsesChunkAudioAndEstimatedWeights(t *testing.T) {
}
}
-func TestFitTimelineClampsAppliedSpeedToMaxButReportsRequiredSpeed(t *testing.T) {
+func TestFitTimelineUsesRequiredSpeedBeyondQualityLimitWithoutOverflow(t *testing.T) {
cfg := DefaultConfig()
cfg.SpeedMax = 1.25
cfg.SpeedAccept = 1.1
@@ -65,14 +65,42 @@ func TestFitTimelineClampsAppliedSpeedToMaxButReportsRequiredSpeed(t *testing.T)
if report.MaxSpeedFactor != 2 {
t.Fatalf("MaxSpeedFactor = %v, want raw required speed 2", report.MaxSpeedFactor)
}
- if got[0].SpeedFactor != cfg.SpeedMax {
- t.Fatalf("SpeedFactor = %v, want clamped max %v", got[0].SpeedFactor, cfg.SpeedMax)
+ if got[0].SpeedFactor != 2 {
+ t.Fatalf("SpeedFactor = %v, want required speed 2", got[0].SpeedFactor)
}
- if got[0].NewEnd <= chunks[0].End {
- t.Fatalf("NewEnd = %v, want overflow beyond chunk end %v", got[0].NewEnd, chunks[0].End)
+ if got[0].NewEnd != chunks[0].End {
+ t.Fatalf("NewEnd = %v, want chunk end %v", got[0].NewEnd, chunks[0].End)
}
- if !warningsContain(report.Warnings, "exceeds max") || !warningsContain(report.Warnings, "overflows") {
- t.Fatalf("warnings = %+v, want max and overflow warnings", report.Warnings)
+ if !warningsContain(report.Warnings, "exceeds max") || warningsContain(report.Warnings, "overflows") {
+ t.Fatalf("warnings = %+v, want max warning without overflow", report.Warnings)
+ }
+}
+
+func TestFitTimelineKeepsOverlongChunkBeforeNextChunkBoundary(t *testing.T) {
+ cfg := DefaultConfig()
+ plan := []PlanItem{
+ {Index: 33, EstimatedDuration: 4.8, ChunkID: 13},
+ {Index: 34, EstimatedDuration: 3.6, ChunkID: 13},
+ {Index: 35, EstimatedDuration: 4.9, ChunkID: 13},
+ {Index: 36, EstimatedDuration: 5.1, ChunkID: 14},
+ }
+ chunks := []Chunk{
+ {ID: 13, Items: []int{0, 1, 2}, Start: 82.8, End: 90, ActualDuration: 11.04},
+ {ID: 14, Items: []int{3}, Start: 90, End: 93.28, ActualDuration: 3.28},
+ }
+
+ got, gotChunks, report, err := FitTimeline(plan, chunks, cfg)
+ if err != nil {
+ t.Fatalf("FitTimeline() error = %v", err)
+ }
+ if got[2].NewEnd > got[3].NewStart {
+ t.Fatalf("chunk boundary overlaps: previous end %.3f next start %.3f", got[2].NewEnd, got[3].NewStart)
+ }
+ if gotChunks[0].SpeedFactor <= cfg.SpeedMax {
+ t.Fatalf("chunk 13 speed = %.3f, want fallback above max %.3f", gotChunks[0].SpeedFactor, cfg.SpeedMax)
+ }
+ if warningsContain(report.Warnings, "overflows") {
+ t.Fatalf("warnings = %+v, want no overflow warning", report.Warnings)
}
}
diff --git a/internal/service/dubbing/runner.go b/internal/service/dubbing/runner.go
index b0f78201..b5fbb358 100644
--- a/internal/service/dubbing/runner.go
+++ b/internal/service/dubbing/runner.go
@@ -76,7 +76,17 @@ func (r *Runner) Run(ctx context.Context) (Result, error) {
return Result{}, err
}
- plan, chunks, err = GenerateRawChunkSegments(ctx, r.deps.TTS, plan, chunks, r.deps.Voice, segmentsDir, r.deps.FFmpeg, r.deps.Duration)
+ plan, chunks, err = GenerateRawChunkSegments(
+ ctx,
+ r.deps.TTS,
+ plan,
+ chunks,
+ r.deps.Voice,
+ segmentsDir,
+ r.deps.FFmpeg,
+ r.deps.Duration,
+ r.deps.ReportProgress,
+ )
if err != nil {
return Result{}, err
}
diff --git a/internal/service/dubbing/tts.go b/internal/service/dubbing/tts.go
index 428926b2..2a83d47b 100644
--- a/internal/service/dubbing/tts.go
+++ b/internal/service/dubbing/tts.go
@@ -55,7 +55,17 @@ func GenerateRawSegments(ctx context.Context, tts types.Ttser, plan []PlanItem,
return plan, nil
}
-func GenerateRawChunkSegments(ctx context.Context, tts types.Ttser, plan []PlanItem, chunks []Chunk, voice, dir string, run CommandRunner, duration DurationProbe) ([]PlanItem, []Chunk, error) {
+func GenerateRawChunkSegments(
+ ctx context.Context,
+ tts types.Ttser,
+ plan []PlanItem,
+ chunks []Chunk,
+ voice,
+ dir string,
+ run CommandRunner,
+ duration DurationProbe,
+ reportProgress func(completed, total int),
+) ([]PlanItem, []Chunk, error) {
if ctx == nil {
ctx = context.Background()
}
@@ -101,6 +111,9 @@ func GenerateRawChunkSegments(ctx context.Context, tts types.Ttser, plan []PlanI
return nil, nil, fmt.Errorf("measure chunk %d duration failed for %s: %w", outChunks[i].ID, output, err)
}
outChunks[i].ActualDuration = dur
+ if reportProgress != nil {
+ reportProgress(i+1, len(outChunks))
+ }
}
return outPlan, outChunks, nil
diff --git a/internal/service/dubbing/tts_test.go b/internal/service/dubbing/tts_test.go
index 109c1994..a4b6bcb9 100644
--- a/internal/service/dubbing/tts_test.go
+++ b/internal/service/dubbing/tts_test.go
@@ -102,12 +102,25 @@ func TestGenerateRawChunkSegmentsCallsTTSOncePerChunk(t *testing.T) {
{ID: 2, Items: []int{2}, Start: 6, End: 8},
}
- gotPlan, gotChunks, err := GenerateRawChunkSegments(context.Background(), tts, plan, chunks, "voice", dir, nil, func(path string) (float64, error) {
- if strings.Contains(path, "chunk_1.wav") {
- return 3.2, nil
- }
- return 1.1, nil
- })
+ var progress [][2]int
+ gotPlan, gotChunks, err := GenerateRawChunkSegments(
+ context.Background(),
+ tts,
+ plan,
+ chunks,
+ "voice",
+ dir,
+ nil,
+ func(path string) (float64, error) {
+ if strings.Contains(path, "chunk_1.wav") {
+ return 3.2, nil
+ }
+ return 1.1, nil
+ },
+ func(completed, total int) {
+ progress = append(progress, [2]int{completed, total})
+ },
+ )
if err != nil {
t.Fatalf("GenerateRawChunkSegments() error = %v", err)
}
@@ -123,6 +136,9 @@ func TestGenerateRawChunkSegmentsCallsTTSOncePerChunk(t *testing.T) {
if gotPlan[0].ActualDuration != 0 || gotPlan[1].ActualDuration != 0 {
t.Fatalf("item actual durations should remain chunk-derived later: %+v", gotPlan)
}
+ if len(progress) != 2 || progress[0] != [2]int{1, 2} || progress[1] != [2]int{2, 2} {
+ t.Fatalf("progress = %+v, want [[1 2] [2 2]]", progress)
+ }
if _, err := os.Stat(filepath.Join(dir, "raw", "chunk_1.wav")); err != nil {
t.Fatalf("chunk raw file missing: %v", err)
}
diff --git a/internal/service/dubbing/types.go b/internal/service/dubbing/types.go
index d66083b6..1987a165 100644
--- a/internal/service/dubbing/types.go
+++ b/internal/service/dubbing/types.go
@@ -88,18 +88,19 @@ type CommandRunner func(args []string) error
type DurationProbe func(path string) (float64, error)
type Dependencies struct {
- TTS types.Ttser
- Chat types.ChatCompleter
- Language types.StandardLanguageCode
- Voice string
- Workdir string
- InputSRT string
- InputVideo string
- OutputAudio string
- OutputVideo string
- Config Config
- FFmpeg CommandRunner
- Duration DurationProbe
+ TTS types.Ttser
+ Chat types.ChatCompleter
+ Language types.StandardLanguageCode
+ Voice string
+ Workdir string
+ InputSRT string
+ InputVideo string
+ OutputAudio string
+ OutputVideo string
+ Config Config
+ FFmpeg CommandRunner
+ Duration DurationProbe
+ ReportProgress func(completed, total int)
}
type TextOptimizer interface {
diff --git a/internal/service/init.go b/internal/service/init.go
index 03cbe8f0..273c5f56 100644
--- a/internal/service/init.go
+++ b/internal/service/init.go
@@ -2,13 +2,13 @@ package service
import (
"krillin-ai/config"
+ "krillin-ai/internal/ttsprovider"
"krillin-ai/internal/types"
"krillin-ai/log"
"krillin-ai/pkg/aliyun"
"krillin-ai/pkg/fasterwhisper"
pkgimage "krillin-ai/pkg/image"
"krillin-ai/pkg/localtts"
- "krillin-ai/pkg/minimax"
"krillin-ai/pkg/openai"
"krillin-ai/pkg/whisper"
"krillin-ai/pkg/whispercpp"
@@ -55,13 +55,13 @@ func NewService() *Service {
switch config.Conf.Tts.Provider {
case "openai":
- ttsClient = openai.NewClient(config.Conf.Tts.Openai.BaseUrl, config.Conf.Tts.Openai.ApiKey, config.Conf.App.Proxy)
+ ttsClient, _ = ttsprovider.New("openai")
case "aliyun":
- ttsClient = aliyun.NewTtsClient(config.Conf.Tts.Aliyun.Speech.AccessKeyId, config.Conf.Tts.Aliyun.Speech.AccessKeySecret, config.Conf.Tts.Aliyun.Speech.AppKey)
+ ttsClient, _ = ttsprovider.New("aliyun")
case "edge-tts":
ttsClient = localtts.NewEdgeTtsClient()
case "minimax":
- ttsClient = minimax.NewTtsClient(config.Conf.Tts.Minimax.BaseUrl, config.Conf.Tts.Minimax.ApiKey, config.Conf.Tts.Minimax.Model)
+ ttsClient, _ = ttsprovider.New("minimax")
}
s := &Service{
diff --git a/internal/service/srt2speech.go b/internal/service/srt2speech.go
index bfed2cd7..58db4530 100644
--- a/internal/service/srt2speech.go
+++ b/internal/service/srt2speech.go
@@ -69,6 +69,16 @@ func (s Service) srtFileToSpeech(ctx context.Context, stepParam *types.SubtitleT
InputVideo: stepParam.InputVideoPath,
OutputAudio: outputAudio,
OutputVideo: outputVideo,
+ ReportProgress: func(completed, total int) {
+ if stepParam.TaskPtr == nil || total <= 0 {
+ return
+ }
+ percent := 20 + int(float64(completed)/float64(total)*70)
+ if percent > 90 {
+ percent = 90
+ }
+ stepParam.TaskPtr.SetProgress(uint8(percent))
+ },
Config: dubbing.Config{
MinSubtitleDuration: config.Conf.Dubbing.MinSubtitleDuration,
MaxChunkSize: config.Conf.Dubbing.MaxChunkSize,
diff --git a/internal/ttsprovider/provider.go b/internal/ttsprovider/provider.go
new file mode 100644
index 00000000..eae21648
--- /dev/null
+++ b/internal/ttsprovider/provider.go
@@ -0,0 +1,38 @@
+package ttsprovider
+
+import (
+ "fmt"
+
+ "krillin-ai/config"
+ "krillin-ai/internal/types"
+ "krillin-ai/pkg/aliyun"
+ "krillin-ai/pkg/minimax"
+ "krillin-ai/pkg/openai"
+)
+
+func New(provider string) (types.TTSProvider, error) {
+ switch provider {
+ case "openai":
+ return openai.NewTtsClient(
+ config.Conf.Tts.Openai.BaseUrl,
+ config.Conf.Tts.Openai.ApiKey,
+ config.Conf.Tts.Openai.Model,
+ config.Conf.App.Proxy,
+ ), nil
+ case "aliyun":
+ return aliyun.NewTtsClient(
+ config.Conf.Tts.Aliyun.BaseUrl,
+ config.Conf.Tts.Aliyun.ApiKey,
+ config.Conf.Tts.Aliyun.Model,
+ config.Conf.App.Proxy,
+ ), nil
+ case "minimax":
+ return minimax.NewTtsClient(
+ config.Conf.Tts.Minimax.BaseUrl,
+ config.Conf.Tts.Minimax.ApiKey,
+ config.Conf.Tts.Minimax.Model,
+ ), nil
+ default:
+ return nil, fmt.Errorf("unsupported tts provider: %s", provider)
+ }
+}
diff --git a/internal/types/interface.go b/internal/types/interface.go
index 203c0a33..994fdf4f 100644
--- a/internal/types/interface.go
+++ b/internal/types/interface.go
@@ -8,6 +8,13 @@ type Transcriber interface {
Transcription(audioFile, language, wordDir string) (*TranscriptionData, error)
}
+type ProgressTranscriber interface {
+ TranscriptionWithProgress(
+ audioFile, language, workDir string,
+ reportProgress func(percent int),
+ ) (*TranscriptionData, error)
+}
+
type Ttser interface {
Text2Speech(text string, voice string, outputFile string) error
}
diff --git a/internal/types/tts_provider.go b/internal/types/tts_provider.go
new file mode 100644
index 00000000..7baf8fe9
--- /dev/null
+++ b/internal/types/tts_provider.go
@@ -0,0 +1,30 @@
+package types
+
+import "context"
+
+type TTSSpeechOptions struct {
+ Text string
+ Voice string
+ OutputFile string
+ Format string
+ Speed float64
+ Instructions string
+}
+
+type TTSVoice struct {
+ Code string
+ Name string
+ Language string
+ Gender string
+ Provider string
+ Scenario string
+ Kind string
+ SupportedModels []string
+ Recommended bool
+}
+
+type TTSProvider interface {
+ Ttser
+ ListVoices(context.Context) ([]TTSVoice, error)
+ Synthesize(context.Context, TTSSpeechOptions) error
+}
diff --git a/internal/voices/voices.go b/internal/voices/voices.go
index f3f922e4..b99df45b 100644
--- a/internal/voices/voices.go
+++ b/internal/voices/voices.go
@@ -1,82 +1,57 @@
package voices
import (
+ "context"
"fmt"
"sort"
"strings"
"krillin-ai/internal/pipeline"
+ "krillin-ai/internal/ttsprovider"
)
const (
ProviderAliyun = "aliyun"
ProviderOpenAI = "openai"
- ProviderEdge = "edge-tts"
Minimax = "minimax"
)
-func List(provider string) ([]pipeline.Voice, error) {
+func List(ctx context.Context, provider string) ([]pipeline.Voice, error) {
provider = strings.TrimSpace(strings.ToLower(provider))
- switch provider {
- case ProviderAliyun:
- return cloneVoices(aliyunVoices), nil
- case ProviderOpenAI:
- return cloneVoices(openaiVoices), nil
- case Minimax:
- return cloneVoices(minimaxVoices), nil
- case ProviderEdge:
- return nil, fmt.Errorf("edge-tts voice listing is not supported yet; use edge-tts --list-voices")
- default:
- return nil, fmt.Errorf("unsupported tts provider: %s", provider)
+ client, err := ttsprovider.New(provider)
+ if err != nil {
+ return nil, err
}
-}
-
-func Providers() []string {
- return []string{ProviderAliyun, ProviderOpenAI, Minimax, ProviderEdge}
-}
-
-func cloneVoices(in []pipeline.Voice) []pipeline.Voice {
- out := append([]pipeline.Voice(nil), in...)
- sort.SliceStable(out, func(i, j int) bool {
- if out[i].Provider != out[j].Provider {
- return out[i].Provider < out[j].Provider
+ values, err := client.ListVoices(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("list %s voices: %w", provider, err)
+ }
+ result := make([]pipeline.Voice, 0, len(values))
+ for _, voice := range values {
+ result = append(result, pipeline.Voice{
+ Code: voice.Code,
+ Name: voice.Name,
+ Language: voice.Language,
+ Gender: voice.Gender,
+ Provider: voice.Provider,
+ Scenario: voice.Scenario,
+ Kind: voice.Kind,
+ SupportedModels: append([]string(nil), voice.SupportedModels...),
+ Recommended: voice.Recommended,
+ })
+ }
+ sort.SliceStable(result, func(i, j int) bool {
+ if result[i].Kind != result[j].Kind {
+ return result[i].Kind < result[j].Kind
}
- return out[i].Code < out[j].Code
+ if result[i].Recommended != result[j].Recommended {
+ return result[i].Recommended
+ }
+ return result[i].Code < result[j].Code
})
- return out
-}
-
-var aliyunVoices = []pipeline.Voice{
- {Provider: ProviderAliyun, Code: "longxiaochun_v2", Name: "龙小淳", Language: "zh-CN", Gender: "female", Scenario: "通用女声"},
- {Provider: ProviderAliyun, Code: "longxiaoxia_v2", Name: "龙小夏", Language: "zh-CN", Gender: "female", Scenario: "自然女声"},
- {Provider: ProviderAliyun, Code: "longxiaocheng_v2", Name: "龙小诚", Language: "zh-CN", Gender: "male", Scenario: "通用男声"},
- {Provider: ProviderAliyun, Code: "longxiaobai_v2", Name: "龙小白", Language: "zh-CN", Gender: "female", Scenario: "甜美女声"},
- {Provider: ProviderAliyun, Code: "longlaotie_v2", Name: "龙老铁", Language: "zh-CN", Gender: "male", Scenario: "东北口音"},
- {Provider: ProviderAliyun, Code: "longshu_v2", Name: "龙叔", Language: "zh-CN", Gender: "male", Scenario: "沉稳男声"},
- {Provider: ProviderAliyun, Code: "longshuo_v2", Name: "龙硕", Language: "zh-CN", Gender: "male", Scenario: "朗读男声"},
- {Provider: ProviderAliyun, Code: "longjing_v2", Name: "龙婧", Language: "zh-CN", Gender: "female", Scenario: "新闻女声"},
- {Provider: ProviderAliyun, Code: "longmiao_v2", Name: "龙妙", Language: "zh-CN", Gender: "female", Scenario: "客服女声"},
- {Provider: ProviderAliyun, Code: "longyue_v2", Name: "龙悦", Language: "zh-CN", Gender: "female", Scenario: "温柔女声"},
+ return result, nil
}
-var openaiVoices = []pipeline.Voice{
- {Provider: ProviderOpenAI, Code: "alloy", Language: "multi", Scenario: "balanced"},
- {Provider: ProviderOpenAI, Code: "ash", Language: "multi", Scenario: "calm"},
- {Provider: ProviderOpenAI, Code: "ballad", Language: "multi", Scenario: "expressive"},
- {Provider: ProviderOpenAI, Code: "coral", Language: "multi", Scenario: "warm"},
- {Provider: ProviderOpenAI, Code: "echo", Language: "multi", Scenario: "clear"},
- {Provider: ProviderOpenAI, Code: "fable", Language: "multi", Scenario: "narration"},
- {Provider: ProviderOpenAI, Code: "nova", Language: "multi", Scenario: "bright"},
- {Provider: ProviderOpenAI, Code: "onyx", Language: "multi", Scenario: "deep"},
- {Provider: ProviderOpenAI, Code: "sage", Language: "multi", Scenario: "neutral"},
- {Provider: ProviderOpenAI, Code: "shimmer", Language: "multi", Scenario: "soft"},
-}
-
-var minimaxVoices = []pipeline.Voice{
- {Provider: Minimax, Code: "English_Graceful_Lady", Name: "Graceful Lady", Language: "en", Gender: "female", Scenario: "优雅女声"},
- {Provider: Minimax, Code: "English_radiant_girl", Name: "Radiant Girl", Language: "en", Gender: "female", Scenario: "活泼女声"},
- {Provider: Minimax, Code: "English_Insightful_Speaker", Name: "Insightful Speaker", Language: "en", Gender: "male", Scenario: "沉稳男声"},
- {Provider: Minimax, Code: "English_Persuasive_Man", Name: "Persuasive Man", Language: "en", Gender: "male", Scenario: "有说服力男声"},
- {Provider: Minimax, Code: "English_expressive_narrator", Name: "Expressive Narrator", Language: "en", Gender: "male", Scenario: "旁白"},
- {Provider: Minimax, Code: "English_Lucky_Robot", Name: "Lucky Robot", Language: "en", Gender: "neutral", Scenario: "机器人"},
+func Providers() []string {
+ return []string{ProviderAliyun, ProviderOpenAI, Minimax}
}
diff --git a/internal/voices/voices_test.go b/internal/voices/voices_test.go
index 06a90fa1..c07ee05a 100644
--- a/internal/voices/voices_test.go
+++ b/internal/voices/voices_test.go
@@ -1,26 +1,27 @@
package voices
import (
+ "context"
"krillin-ai/internal/pipeline"
"strings"
"testing"
)
-func TestListAliyunVoicesIncludesCosyVoiceCodes(t *testing.T) {
- got, err := List(ProviderAliyun)
+func TestListAliyunVoicesIncludesQwenVoiceCodes(t *testing.T) {
+ got, err := List(context.Background(), ProviderAliyun)
if err != nil {
t.Fatalf("List() error = %v", err)
}
- if !hasVoice(got, "longxiaochun_v2") {
- t.Fatalf("aliyun voices = %#v, want longxiaochun_v2", got)
+ if !hasVoice(got, "Cherry") {
+ t.Fatalf("aliyun voices = %#v, want Cherry", got)
}
- if !hasVoice(got, "longxiaocheng_v2") {
- t.Fatalf("aliyun voices = %#v, want longxiaocheng_v2", got)
+ if !hasVoice(got, "Ethan") {
+ t.Fatalf("aliyun voices = %#v, want Ethan", got)
}
}
func TestListRejectsUnsupportedProvider(t *testing.T) {
- _, err := List("unknown")
+ _, err := List(context.Background(), "unknown")
if err == nil {
t.Fatal("List() error = nil, want unsupported provider error")
}
@@ -30,7 +31,7 @@ func TestListRejectsUnsupportedProvider(t *testing.T) {
}
func TestListMinimaxVoices(t *testing.T) {
- got, err := List(Minimax)
+ got, err := List(context.Background(), Minimax)
if err != nil {
t.Fatalf("List() error = %v", err)
}
diff --git a/pkg/aliyun/tts.go b/pkg/aliyun/tts.go
index 923ea352..28cfdf33 100644
--- a/pkg/aliyun/tts.go
+++ b/pkg/aliyun/tts.go
@@ -1,206 +1,341 @@
package aliyun
import (
+ "bytes"
+ "context"
"encoding/json"
"fmt"
- "github.com/gorilla/websocket"
- "go.uber.org/zap"
- "krillin-ai/config"
- "krillin-ai/log"
- "krillin-ai/pkg/util"
+ "io"
"net/http"
+ "net/url"
"os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "strings"
"time"
+
+ "krillin-ai/internal/resourcepath"
+ "krillin-ai/internal/types"
)
-type TtsClient struct {
- AccessKeyID string
- AccessKeySecret string
- Appkey string
-}
+const (
+ DefaultTTSBaseURL = "https://dashscope.aliyuncs.com/api/v1"
+ DefaultTTSModel = "qwen3-tts-flash"
+ DefaultTTSVoice = "Cherry"
+)
-type TtsHeader struct {
- Appkey string `json:"appkey"`
- MessageID string `json:"message_id"`
- TaskID string `json:"task_id"`
- Namespace string `json:"namespace"`
- Name string `json:"name"`
+type TtsClient struct {
+ BaseURL string
+ APIKey string
+ Model string
+ httpClient *http.Client
}
-type StartSynthesisPayload struct {
- Voice string `json:"voice,omitempty"`
- Format string `json:"format,omitempty"`
- SampleRate int `json:"sample_rate,omitempty"`
- Volume int `json:"volume,omitempty"`
- SpeechRate int `json:"speech_rate,omitempty"`
- PitchRate int `json:"pitch_rate,omitempty"`
- EnableSubtitle bool `json:"enable_subtitle,omitempty"`
- EnablePhonemeTimestamp bool `json:"enable_phoneme_timestamp,omitempty"`
+type ttsRequest struct {
+ Model string `json:"model"`
+ Input ttsInput `json:"input"`
}
-type RunSynthesisPayload struct {
- Text string `json:"text"`
+type ttsInput struct {
+ Text string `json:"text"`
+ Voice string `json:"voice"`
+ LanguageType string `json:"language_type"`
}
-type Message struct {
- Header TtsHeader `json:"header"`
- Payload interface{} `json:"payload,omitempty"`
+type ttsResponse struct {
+ Output struct {
+ Audio struct {
+ URL string `json:"url"`
+ } `json:"audio"`
+ } `json:"output"`
+ Code string `json:"code"`
+ Message string `json:"message"`
+ RequestID string `json:"request_id"`
}
-func NewTtsClient(accessKeyId, accessKeySecret, appkey string) *TtsClient {
+func NewTtsClient(baseURL, apiKey, model, proxy string) *TtsClient {
+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+ if baseURL == "" {
+ baseURL = DefaultTTSBaseURL
+ }
+ if strings.TrimSpace(model) == "" {
+ model = DefaultTTSModel
+ }
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ if proxyURL, err := url.Parse(strings.TrimSpace(proxy)); err == nil && proxyURL.Scheme != "" {
+ transport.Proxy = http.ProxyURL(proxyURL)
+ }
return &TtsClient{
- AccessKeyID: accessKeyId,
- AccessKeySecret: accessKeySecret,
- Appkey: appkey,
+ BaseURL: baseURL,
+ APIKey: strings.TrimSpace(apiKey),
+ Model: strings.TrimSpace(model),
+ httpClient: &http.Client{
+ Transport: transport,
+ Timeout: 90 * time.Second,
+ },
}
}
func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {
- file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0666)
+ return c.Synthesize(context.Background(), types.TTSSpeechOptions{
+ Text: text,
+ Voice: voice,
+ OutputFile: outputFile,
+ })
+}
+
+func (c *TtsClient) Synthesize(ctx context.Context, options types.TTSSpeechOptions) error {
+ if c.APIKey == "" {
+ return fmt.Errorf("aliyun bailian tts api key is empty")
+ }
+ voice := strings.TrimSpace(options.Voice)
+ if voice == "" {
+ voice = DefaultTTSVoice
+ }
+ format := speechFormat(options.Format, options.OutputFile)
+ speed := options.Speed
+ if speed <= 0 {
+ speed = 1
+ }
+ body, err := json.Marshal(ttsRequest{
+ Model: c.Model,
+ Input: ttsInput{
+ Text: options.Text,
+ Voice: voice,
+ LanguageType: "Auto",
+ },
+ })
if err != nil {
- return fmt.Errorf("failed to create file: %w", err)
+ return fmt.Errorf("aliyun bailian tts encode request: %w", err)
}
- defer file.Close()
- var conn *websocket.Conn
- token, _ := CreateToken(c.AccessKeyID, c.AccessKeySecret)
- fullURL := "wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1?token=" + token
- dialer := websocket.DefaultDialer
- if config.Conf.App.Proxy != "" {
- dialer.Proxy = http.ProxyURL(config.Conf.App.ParsedProxy)
- }
- dialer.HandshakeTimeout = 10 * time.Second
- conn, _, err = dialer.Dial(fullURL, nil)
+ ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
+ defer cancel()
+ req, err := http.NewRequestWithContext(
+ ctx,
+ http.MethodPost,
+ c.generationEndpoint(),
+ bytes.NewReader(body),
+ )
if err != nil {
return err
}
- _ = conn.SetReadDeadline(time.Now().Add(time.Second * 60))
- defer c.Close(conn)
-
- onTextMessage := func(message string) {
- log.GetLogger().Info("Received text message", zap.String("Message", message))
+ req.Header.Set("Authorization", "Bearer "+c.APIKey)
+ req.Header.Set("Content-Type", "application/json")
+ response, err := c.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("aliyun bailian tts request failed: %w", err)
}
-
- onBinaryMessage := func(data []byte) {
- if _, err := file.Write(data); err != nil {
- log.GetLogger().Error("Failed to write data to file", zap.Error(err))
- }
+ defer response.Body.Close()
+ responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
+ if err != nil {
+ return err
}
-
- var (
- synthesisStarted = make(chan struct{})
- synthesisComplete = make(chan struct{})
- )
-
- startPayload := StartSynthesisPayload{
- Voice: voice,
- Format: "wav",
- SampleRate: 44100,
- Volume: 50,
- SpeechRate: 0,
- PitchRate: 0,
+ if response.StatusCode != http.StatusOK {
+ return fmt.Errorf("aliyun bailian tts returned HTTP %d: %s", response.StatusCode, boundedMessage(responseBody))
}
-
- go c.receiveMessages(conn, onTextMessage, onBinaryMessage, synthesisStarted, synthesisComplete)
-
- taskId := util.GenerateID()
- log.GetLogger().Info("SpeechClient StartSynthesis", zap.String("taskId", taskId), zap.Any("payload", startPayload))
- if err := c.StartSynthesis(conn, taskId, startPayload, synthesisStarted); err != nil {
- return fmt.Errorf("failed to start synthesis: %w", err)
+ var decoded ttsResponse
+ if err := json.Unmarshal(responseBody, &decoded); err != nil {
+ return fmt.Errorf("aliyun bailian tts decode response: %w", err)
}
-
- if err := c.RunSynthesis(conn, taskId, text); err != nil {
- return fmt.Errorf("failed to run synthesis: %w", err)
+ if decoded.Code != "" {
+ return fmt.Errorf("aliyun bailian tts error %s: %s", decoded.Code, decoded.Message)
}
-
- if err := c.StopSynthesis(conn, taskId, synthesisComplete); err != nil {
- return fmt.Errorf("failed to stop synthesis: %w", err)
+ if decoded.Output.Audio.URL == "" {
+ return fmt.Errorf("aliyun bailian tts response did not include an audio URL")
}
+ return c.downloadAndProcessAudio(
+ ctx,
+ decoded.Output.Audio.URL,
+ options.OutputFile,
+ format,
+ speed,
+ )
+}
- return nil
+func (c *TtsClient) ListVoices(context.Context) ([]types.TTSVoice, error) {
+ return append([]types.TTSVoice(nil), aliyunBuiltinVoices...), nil
}
-func (c *TtsClient) sendMessage(conn *websocket.Conn, taskId, name string, payload interface{}) error {
- message := Message{
- Header: TtsHeader{
- Appkey: c.Appkey,
- MessageID: util.GenerateID(),
- TaskID: taskId,
- Namespace: "FlowingSpeechSynthesizer",
- Name: name,
- },
- Payload: payload,
+func (c *TtsClient) generationEndpoint() string {
+ if strings.HasSuffix(c.BaseURL, "/services/aigc/multimodal-generation/generation") {
+ return c.BaseURL
}
- jsonData, _ := json.Marshal(message)
- log.GetLogger().Debug("SpeechClient sendMessage", zap.String("message", string(jsonData)))
- return conn.WriteJSON(message)
+ return c.BaseURL + "/services/aigc/multimodal-generation/generation"
}
-func (c *TtsClient) StartSynthesis(conn *websocket.Conn, taskId string, payload StartSynthesisPayload, synthesisStarted chan struct{}) error {
- err := c.sendMessage(conn, taskId, "StartSynthesis", payload)
+func (c *TtsClient) downloadAudio(ctx context.Context, source, outputFile string) error {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
if err != nil {
return err
}
-
- // 阻塞等待 SynthesisStarted 事件
- <-synthesisStarted
-
+ response, err := c.httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("aliyun bailian tts download audio: %w", err)
+ }
+ defer response.Body.Close()
+ if response.StatusCode != http.StatusOK {
+ return fmt.Errorf("aliyun bailian audio download returned HTTP %d", response.StatusCode)
+ }
+ if err := os.MkdirAll(filepath.Dir(outputFile), 0755); err != nil {
+ return err
+ }
+ file, err := os.Create(outputFile)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+ if _, err := io.Copy(file, io.LimitReader(response.Body, 100<<20)); err != nil {
+ return err
+ }
return nil
}
-func (c *TtsClient) RunSynthesis(conn *websocket.Conn, taskId, text string) error {
- return c.sendMessage(conn, taskId, "RunSynthesis", RunSynthesisPayload{Text: text})
+func boundedMessage(value []byte) string {
+ const limit = 300
+ message := strings.TrimSpace(string(value))
+ if len(message) <= limit {
+ return message
+ }
+ return message[:limit]
}
-func (c *TtsClient) StopSynthesis(conn *websocket.Conn, taskId string, synthesisComplete chan struct{}) error {
- err := c.sendMessage(conn, taskId, "StopSynthesis", nil)
+func (c *TtsClient) downloadAndProcessAudio(
+ ctx context.Context,
+ source string,
+ outputFile string,
+ format string,
+ speed float64,
+) error {
+ outputDir := filepath.Dir(outputFile)
+ if err := os.MkdirAll(outputDir, 0755); err != nil {
+ return err
+ }
+ native, err := os.CreateTemp(outputDir, ".aliyun-tts-*.wav")
if err != nil {
return err
}
-
- // 阻塞等待 SynthesisCompleted 事件
- <-synthesisComplete
-
- return nil
+ nativePath := native.Name()
+ if err := native.Close(); err != nil {
+ return err
+ }
+ defer os.Remove(nativePath)
+ if err := c.downloadAudio(ctx, source, nativePath); err != nil {
+ return err
+ }
+ if format == "wav" && speed == 1 {
+ if err := os.Remove(outputFile); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ return os.Rename(nativePath, outputFile)
+ }
+ return transcodeAudio(ctx, nativePath, outputFile, format, speed)
}
-func (c *TtsClient) Close(conn *websocket.Conn) error {
- err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
+func transcodeAudio(ctx context.Context, input, output, format string, speed float64) error {
+ ffmpeg, err := ffmpegPath()
if err != nil {
return err
}
- return conn.Close()
+ args := []string{"-y", "-i", input}
+ if speed != 1 {
+ args = append(args, "-filter:a", "atempo="+strconv.FormatFloat(speed, 'f', 3, 64))
+ }
+ args = append(args, "-ar", "44100", "-ac", "1")
+ if format == "mp3" {
+ args = append(args, "-c:a", "libmp3lame", "-b:a", "192k")
+ } else {
+ args = append(args, "-c:a", "pcm_s16le")
+ }
+ args = append(args, output)
+ command := exec.CommandContext(ctx, ffmpeg, args...)
+ if result, err := command.CombinedOutput(); err != nil {
+ return fmt.Errorf("aliyun bailian tts audio conversion failed: %w, output: %s", err, boundedMessage(result))
+ }
+ return nil
}
-func (c *TtsClient) receiveMessages(conn *websocket.Conn, onTextMessage func(string), onBinaryMessage func([]byte), synthesisStarted, synthesisComplete chan struct{}) {
- defer close(synthesisComplete)
- for {
- messageType, message, err := conn.ReadMessage()
+func ffmpegPath() (string, error) {
+ name := "ffmpeg"
+ if runtime.GOOS == "windows" {
+ name += ".exe"
+ }
+ if root, configured, err := resourcepath.Root(); err != nil {
+ return "", err
+ } else if configured {
+ path, err := resourcepath.RequireFile("bin", name)
if err != nil {
- if websocket.IsCloseError(err, websocket.CloseNormalClosure) {
- log.GetLogger().Error("SpeechClient receiveMessages websocket非正常关闭", zap.Error(err))
- return
- }
- return
- }
- if messageType == websocket.TextMessage {
- var msg Message
- if err := json.Unmarshal(message, &msg); err != nil {
- log.GetLogger().Error("SpeechClient receiveMessages json解析失败", zap.Error(err))
- return
- }
- if msg.Header.Name == "SynthesisCompleted" {
- log.GetLogger().Info("SynthesisCompleted event received")
- // 收到结束消息退出
- break
- } else if msg.Header.Name == "SynthesisStarted" {
- log.GetLogger().Info("SynthesisStarted event received")
- close(synthesisStarted)
- } else {
- onTextMessage(string(message))
- }
- } else if messageType == websocket.BinaryMessage {
- onBinaryMessage(message)
+ return "", fmt.Errorf("aliyun bailian tts requires packaged ffmpeg under %s: %w", root, err)
}
+ return path, nil
+ }
+ path, err := exec.LookPath("ffmpeg")
+ if err != nil {
+ return "", fmt.Errorf("aliyun bailian tts requires ffmpeg for MP3 or speed conversion: %w", err)
+ }
+ return path, nil
+}
+
+var aliyunBuiltinVoices = []types.TTSVoice{
+ {Code: "Cherry", Name: "芊悦", Language: "zh-CN", Gender: "female", Provider: "aliyun", Scenario: "阳光积极", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}, Recommended: true},
+ {Code: "Serena", Name: "苏瑶", Language: "zh-CN", Gender: "female", Provider: "aliyun", Scenario: "温柔叙事", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Ethan", Name: "晨煦", Language: "zh-CN", Gender: "male", Provider: "aliyun", Scenario: "阳光男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Chelsie", Name: "千雪", Language: "zh-CN", Gender: "female", Provider: "aliyun", Scenario: "二次元", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Momo", Name: "茉兔", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "撒娇搞怪", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Vivian", Name: "十三", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "可爱小暴躁", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Moon", Name: "月白", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "率性帅气", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Maia", Name: "四月", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "知性温柔", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Kai", Name: "凯", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "舒缓自然", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Nofish", Name: "不吃鱼", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "自然设计师", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Bella", Name: "萌宝", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "活泼萝莉", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Jennifer", Name: "詹妮弗", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "电影质感美语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Ryan", Name: "甜茶", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "戏感张力", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Katerina", Name: "卡捷琳娜", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "成熟御姐", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Aiden", Name: "艾登", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "美语青年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Eldric Sage", Name: "沧明子", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "沉稳睿智老者", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Mia", Name: "乖小妹", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "温顺乖巧", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Mochi", Name: "沙小弥", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "聪慧少年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Bellona", Name: "燕铮莺", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "洪亮热血", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Vincent", Name: "田叔", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "沙哑豪迈", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Bunny", Name: "萌小姬", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "萌系萝莉", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Neil", Name: "阿闻", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "新闻主持", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Elias", Name: "墨讲师", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "知识讲解", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Arthur", Name: "徐大爷", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "乡土故事", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Nini", Name: "邻家妹妹", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "甜软亲切", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Seren", Name: "小婉", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "舒缓助眠", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Pip", Name: "顽屁小孩", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "调皮童声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Stella", Name: "少女阿月", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "甜美少女", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Bodega", Name: "博德加", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "西班牙男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Sonrisa", Name: "索尼莎", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "拉美女声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Alek", Name: "阿列克", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "俄语男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Dolce", Name: "多尔切", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "意大利男声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Sohee", Name: "素熙", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "韩语女声", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Ono Anna", Name: "小野杏", Language: "multi", Gender: "female", Provider: "aliyun", Scenario: "日语少女", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Lenn", Name: "莱恩", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "德语青年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Emilien", Name: "埃米尔安", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "法语青年", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Andre", Name: "安德雷", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "磁性沉稳", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Radio Gol", Name: "拉迪奥·戈尔", Language: "multi", Gender: "male", Provider: "aliyun", Scenario: "足球解说", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Jada", Name: "上海-阿珍", Language: "zh-shanghai", Gender: "female", Provider: "aliyun", Scenario: "上海话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Dylan", Name: "北京-晓东", Language: "zh-beijing", Gender: "male", Provider: "aliyun", Scenario: "北京话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Li", Name: "南京-老李", Language: "zh-nanjing", Gender: "male", Provider: "aliyun", Scenario: "南京话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Marcus", Name: "陕西-秦川", Language: "zh-shaanxi", Gender: "male", Provider: "aliyun", Scenario: "陕西话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Roy", Name: "闽南-阿杰", Language: "zh-minnan", Gender: "male", Provider: "aliyun", Scenario: "闽南语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Peter", Name: "天津-李彼得", Language: "zh-tianjin", Gender: "male", Provider: "aliyun", Scenario: "天津话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Sunny", Name: "四川-晴儿", Language: "zh-sichuan", Gender: "female", Provider: "aliyun", Scenario: "四川话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Eric", Name: "四川-程川", Language: "zh-sichuan", Gender: "male", Provider: "aliyun", Scenario: "四川话", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Rocky", Name: "粤语-阿强", Language: "zh-yue", Gender: "male", Provider: "aliyun", Scenario: "粤语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+ {Code: "Kiki", Name: "粤语-阿清", Language: "zh-yue", Gender: "female", Provider: "aliyun", Scenario: "粤语", Kind: "builtin", SupportedModels: []string{"qwen3-tts-flash"}},
+}
+
+func speechFormat(explicit, outputFile string) string {
+ format := strings.ToLower(strings.TrimSpace(explicit))
+ if format == "mp3" || format == "wav" {
+ return format
+ }
+ if strings.EqualFold(filepath.Ext(outputFile), ".mp3") {
+ return "mp3"
}
+ return "wav"
}
diff --git a/pkg/aliyun/tts_test.go b/pkg/aliyun/tts_test.go
new file mode 100644
index 00000000..7841c225
--- /dev/null
+++ b/pkg/aliyun/tts_test.go
@@ -0,0 +1,151 @@
+package aliyun
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+
+ "krillin-ai/internal/resourcepath"
+ "krillin-ai/internal/types"
+)
+
+func TestSynthesizeUsesQwenTTSRequestAndDownloadsAudio(t *testing.T) {
+ var server *httptest.Server
+ server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/services/aigc/multimodal-generation/generation":
+ if r.Method != http.MethodPost {
+ t.Fatalf("method = %s, want POST", r.Method)
+ }
+ if r.Header.Get("Authorization") != "Bearer dashscope-key" {
+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))
+ }
+ var request map[string]any
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Fatalf("decode request: %v", err)
+ }
+ if request["model"] != "qwen3-tts-flash" {
+ t.Fatalf("model = %#v", request["model"])
+ }
+ input, ok := request["input"].(map[string]any)
+ if !ok {
+ t.Fatalf("input = %#v", request["input"])
+ }
+ if input["text"] != "你好,\"百炼\"\n下一行" {
+ t.Fatalf("text = %#v", input["text"])
+ }
+ if input["voice"] != "Cherry" {
+ t.Fatalf("voice = %#v", input["voice"])
+ }
+ if input["language_type"] != "Auto" {
+ t.Fatalf("language_type = %#v", input["language_type"])
+ }
+ if _, exists := request["parameters"]; exists {
+ t.Fatalf("parameters = %#v, Qwen-TTS does not accept CosyVoice parameters", request["parameters"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "output": map[string]any{
+ "audio": map[string]string{"url": server.URL + "/audio.wav"},
+ },
+ "request_id": "request-1",
+ })
+ case "/audio.wav":
+ _, _ = w.Write([]byte("RIFF-test-audio"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ output := filepath.Join(t.TempDir(), "speech.wav")
+ client := NewTtsClient(server.URL, "dashscope-key", "qwen3-tts-flash", "")
+ if err := client.Synthesize(context.Background(), types.TTSSpeechOptions{
+ Text: "你好,\"百炼\"\n下一行",
+ Voice: "Cherry",
+ OutputFile: output,
+ Format: "wav",
+ Speed: 1,
+ }); err != nil {
+ t.Fatalf("Synthesize() error = %v", err)
+ }
+ content, err := os.ReadFile(output)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(content) != "RIFF-test-audio" {
+ t.Fatalf("audio = %q", content)
+ }
+}
+
+func TestListVoicesUsesBuiltinQwenCatalogWithoutCredentials(t *testing.T) {
+ client := NewTtsClient("http://127.0.0.1:1", "unused-key", "", "")
+ voices, err := client.ListVoices(context.Background())
+ if err != nil {
+ t.Fatalf("ListVoices() error = %v", err)
+ }
+ for _, code := range []string{"Cherry", "Nofish", "Jennifer", "Jada", "Kiki"} {
+ if !containsVoice(voices, code) {
+ t.Fatalf("ListVoices() missing %q", code)
+ }
+ }
+}
+
+func TestTranscodeAudioUsesPackagedFFmpegForFormatAndSpeed(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("shell fixture is POSIX-only")
+ }
+ root := t.TempDir()
+ binDir := filepath.Join(root, "bin")
+ if err := os.MkdirAll(binDir, 0755); err != nil {
+ t.Fatal(err)
+ }
+ argsPath := filepath.Join(root, "ffmpeg-args.txt")
+ ffmpeg := filepath.Join(binDir, "ffmpeg")
+ script := "#!/bin/sh\n" +
+ "printf '%s\\n' \"$@\" > \"" + argsPath + "\"\n" +
+ "input=''\noutput=''\nprevious=''\n" +
+ "for value in \"$@\"; do\n" +
+ " if [ \"$previous\" = '-i' ]; then input=\"$value\"; fi\n" +
+ " previous=\"$value\"\n" +
+ " output=\"$value\"\n" +
+ "done\n" +
+ "cp \"$input\" \"$output\"\n"
+ if err := os.WriteFile(ffmpeg, []byte(script), 0755); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv(resourcepath.RootEnv, root)
+
+ input := filepath.Join(root, "input.wav")
+ output := filepath.Join(root, "output.mp3")
+ if err := os.WriteFile(input, []byte("audio"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err := transcodeAudio(context.Background(), input, output, "mp3", 1.25); err != nil {
+ t.Fatalf("transcodeAudio() error = %v", err)
+ }
+ args, err := os.ReadFile(argsPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ joined := string(args)
+ for _, expected := range []string{"atempo=1.250", "libmp3lame", "192k", output} {
+ if !strings.Contains(joined, expected) {
+ t.Fatalf("ffmpeg args = %q, want %q", joined, expected)
+ }
+ }
+}
+
+func containsVoice(voices []types.TTSVoice, code string) bool {
+ for _, voice := range voices {
+ if voice.Code == code {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkg/minimax/tts.go b/pkg/minimax/tts.go
index 62e51373..5787e661 100644
--- a/pkg/minimax/tts.go
+++ b/pkg/minimax/tts.go
@@ -14,6 +14,7 @@ import (
"time"
"krillin-ai/config"
+ "krillin-ai/internal/types"
"krillin-ai/log"
"go.uber.org/zap"
@@ -96,23 +97,32 @@ type t2aResponse struct {
// buildRequestBody 组装非流式 T2A v2 请求体,输出 wav 以匹配下游配音流程。
func (c *TtsClient) buildRequestBody(text, voice string) ([]byte, error) {
+ return c.buildSpeechRequest(types.TTSSpeechOptions{Text: text, Voice: voice})
+}
+
+func (c *TtsClient) buildSpeechRequest(options types.TTSSpeechOptions) ([]byte, error) {
+ voice := strings.TrimSpace(options.Voice)
voice = strings.TrimSpace(voice)
if voice == "" {
voice = DefaultVoice
}
+ speed := options.Speed
+ if speed <= 0 {
+ speed = 1
+ }
reqBody := t2aRequest{
Model: c.Model,
- Text: text,
+ Text: options.Text,
Stream: false,
VoiceSetting: voiceSetting{
VoiceID: voice,
- Speed: 1,
+ Speed: speed,
Vol: 1,
Pitch: 0,
},
AudioSetting: audioSetting{
SampleRate: 44100,
- Format: "wav",
+ Format: speechFormat(options.Format, options.OutputFile),
Channel: 1,
},
}
@@ -141,16 +151,24 @@ func decodeAudio(respBody []byte) ([]byte, error) {
// Text2Speech 将文本合成为语音并写入 outputFile(wav)。
func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {
+ return c.Synthesize(context.Background(), types.TTSSpeechOptions{
+ Text: text,
+ Voice: voice,
+ OutputFile: outputFile,
+ })
+}
+
+func (c *TtsClient) Synthesize(ctx context.Context, options types.TTSSpeechOptions) error {
if c.ApiKey == "" {
return fmt.Errorf("minimax tts api key is empty")
}
- body, err := c.buildRequestBody(text, voice)
+ body, err := c.buildSpeechRequest(options)
if err != nil {
return fmt.Errorf("minimax tts build request failed: %w", err)
}
- ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
url := c.BaseUrl + "/v1/t2a_v2"
@@ -183,13 +201,118 @@ func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {
return err
}
- outputDir := filepath.Dir(outputFile)
+ outputDir := filepath.Dir(options.OutputFile)
if err = os.MkdirAll(outputDir, 0755); err != nil {
return fmt.Errorf("minimax tts create output dir failed: %w", err)
}
- if err = os.WriteFile(outputFile, audio, 0644); err != nil {
+ if err = os.WriteFile(options.OutputFile, audio, 0644); err != nil {
return fmt.Errorf("minimax tts write output file failed: %w", err)
}
return nil
}
+
+func speechFormat(explicit, outputFile string) string {
+ format := strings.ToLower(strings.TrimSpace(explicit))
+ if format == "mp3" || format == "wav" {
+ return format
+ }
+ if strings.EqualFold(filepath.Ext(outputFile), ".mp3") {
+ return "mp3"
+ }
+ return "wav"
+}
+
+func (c *TtsClient) ListVoices(ctx context.Context) ([]types.TTSVoice, error) {
+ if c.ApiKey == "" {
+ return append([]types.TTSVoice(nil), minimaxFallbackVoices...), nil
+ }
+ body, err := json.Marshal(map[string]string{"voice_type": "all"})
+ if err != nil {
+ return nil, err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseUrl+"/v1/get_voice", bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Bearer "+c.ApiKey)
+ response, err := c.httpClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+ responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
+ if err != nil {
+ return nil, err
+ }
+ if response.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("minimax get_voice returned HTTP %d", response.StatusCode)
+ }
+ var payload map[string]any
+ if err := json.Unmarshal(responseBody, &payload); err != nil {
+ return nil, err
+ }
+ voices := extractMiniMaxVoices(payload)
+ if len(voices) == 0 {
+ return append([]types.TTSVoice(nil), minimaxFallbackVoices...), nil
+ }
+ return voices, nil
+}
+
+func extractMiniMaxVoices(payload map[string]any) []types.TTSVoice {
+ groups := []struct {
+ key string
+ kind string
+ }{
+ {key: "system_voice", kind: "builtin"},
+ {key: "voice_cloning", kind: "custom"},
+ {key: "voice_generation", kind: "designed"},
+ }
+ var voices []types.TTSVoice
+ for _, group := range groups {
+ values, ok := payload[group.key].([]any)
+ if !ok {
+ continue
+ }
+ for _, value := range values {
+ item, ok := value.(map[string]any)
+ if !ok {
+ continue
+ }
+ code := miniMaxString(item, "voice_id", "voiceId")
+ if code == "" {
+ continue
+ }
+ name := miniMaxString(item, "voice_name", "voiceName")
+ if name == "" {
+ name = code
+ }
+ voices = append(voices, types.TTSVoice{
+ Code: code,
+ Name: name,
+ Provider: "minimax",
+ Kind: group.kind,
+ })
+ }
+ }
+ return voices
+}
+
+func miniMaxString(value map[string]any, keys ...string) string {
+ for _, key := range keys {
+ if candidate, ok := value[key].(string); ok && strings.TrimSpace(candidate) != "" {
+ return strings.TrimSpace(candidate)
+ }
+ }
+ return ""
+}
+
+var minimaxFallbackVoices = []types.TTSVoice{
+ {Code: "English_Graceful_Lady", Name: "Graceful Lady", Language: "en", Gender: "female", Provider: "minimax", Scenario: "优雅女声", Kind: "builtin", Recommended: true},
+ {Code: "English_radiant_girl", Name: "Radiant Girl", Language: "en", Gender: "female", Provider: "minimax", Scenario: "活泼女声", Kind: "builtin"},
+ {Code: "English_Insightful_Speaker", Name: "Insightful Speaker", Language: "en", Gender: "male", Provider: "minimax", Scenario: "沉稳男声", Kind: "builtin"},
+ {Code: "English_Persuasive_Man", Name: "Persuasive Man", Language: "en", Gender: "male", Provider: "minimax", Scenario: "说服力", Kind: "builtin"},
+ {Code: "English_expressive_narrator", Name: "Expressive Narrator", Language: "en", Gender: "male", Provider: "minimax", Scenario: "旁白", Kind: "builtin"},
+ {Code: "English_Lucky_Robot", Name: "Lucky Robot", Language: "en", Gender: "neutral", Provider: "minimax", Scenario: "机器人", Kind: "builtin"},
+}
diff --git a/pkg/minimax/tts_test.go b/pkg/minimax/tts_test.go
index c5275f7f..ef513cb4 100644
--- a/pkg/minimax/tts_test.go
+++ b/pkg/minimax/tts_test.go
@@ -1,11 +1,16 @@
package minimax
import (
+ "context"
"encoding/hex"
"encoding/json"
+ "net/http"
+ "net/http/httptest"
"os"
"path/filepath"
"testing"
+
+ "krillin-ai/internal/types"
)
func TestNewTtsClientDefaults(t *testing.T) {
@@ -110,3 +115,113 @@ func TestText2SpeechRequiresApiKey(t *testing.T) {
t.Fatal("output file written despite missing api key")
}
}
+
+func TestListVoicesRequestsAndParsesAllVoiceGroups(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/get_voice" {
+ http.NotFound(w, r)
+ return
+ }
+ if r.Header.Get("Authorization") != "Bearer minimax-key" {
+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))
+ }
+ var request map[string]string
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Fatalf("decode request: %v", err)
+ }
+ if request["voice_type"] != "all" {
+ t.Fatalf("voice_type = %q", request["voice_type"])
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "system_voice": []map[string]string{{
+ "voice_id": "system-1", "voice_name": "System Voice",
+ }},
+ "voice_cloning": []map[string]string{{
+ "voice_id": "clone-1", "voice_name": "Cloned Voice",
+ }},
+ "voice_generation": []map[string]string{{
+ "voice_id": "design-1", "voice_name": "Designed Voice",
+ }},
+ })
+ }))
+ defer server.Close()
+
+ client := NewTtsClient(server.URL, "minimax-key", "speech-2.8-hd")
+ voices, err := client.ListVoices(context.Background())
+ if err != nil {
+ t.Fatalf("ListVoices() error = %v", err)
+ }
+ if len(voices) != 3 {
+ t.Fatalf("voices = %#v", voices)
+ }
+ if voices[0].Code != "system-1" || voices[0].Kind != "builtin" {
+ t.Fatalf("system voice = %#v", voices[0])
+ }
+ if voices[1].Code != "clone-1" || voices[1].Kind != "custom" {
+ t.Fatalf("cloned voice = %#v", voices[1])
+ }
+ if voices[2].Code != "design-1" || voices[2].Kind != "designed" {
+ t.Fatalf("designed voice = %#v", voices[2])
+ }
+}
+
+func TestSynthesizeSendsMiniMaxFormatAndSpeed(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/t2a_v2" {
+ http.NotFound(w, r)
+ return
+ }
+ if r.Header.Get("Authorization") != "Bearer minimax-key" {
+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))
+ }
+ var request t2aRequest
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Fatalf("decode request: %v", err)
+ }
+ if request.Model != "speech-2.8-hd" {
+ t.Fatalf("model = %q", request.Model)
+ }
+ if request.Text != "MiniMax test" {
+ t.Fatalf("text = %q", request.Text)
+ }
+ if request.VoiceSetting.VoiceID != "English_Graceful_Lady" {
+ t.Fatalf("voice_id = %q", request.VoiceSetting.VoiceID)
+ }
+ if request.VoiceSetting.Speed != 1.2 {
+ t.Fatalf("speed = %v", request.VoiceSetting.Speed)
+ }
+ if request.AudioSetting.Format != "mp3" {
+ t.Fatalf("format = %q", request.AudioSetting.Format)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "data": map[string]any{
+ "audio": hex.EncodeToString([]byte("ID3-minimax-audio")),
+ "status": 2,
+ },
+ "base_resp": map[string]any{
+ "status_code": 0,
+ "status_msg": "success",
+ },
+ })
+ }))
+ defer server.Close()
+
+ output := filepath.Join(t.TempDir(), "speech.mp3")
+ client := NewTtsClient(server.URL, "minimax-key", "speech-2.8-hd")
+ if err := client.Synthesize(context.Background(), types.TTSSpeechOptions{
+ Text: "MiniMax test",
+ Voice: "English_Graceful_Lady",
+ OutputFile: output,
+ Format: "mp3",
+ Speed: 1.2,
+ }); err != nil {
+ t.Fatalf("Synthesize() error = %v", err)
+ }
+ content, err := os.ReadFile(output)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(content) != "ID3-minimax-audio" {
+ t.Fatalf("audio = %q", content)
+ }
+}
diff --git a/pkg/openai/openai.go b/pkg/openai/openai.go
index 0d6ea915..b60e753c 100644
--- a/pkg/openai/openai.go
+++ b/pkg/openai/openai.go
@@ -10,8 +10,6 @@ import (
"io"
"krillin-ai/config"
"krillin-ai/log"
- "net/http"
- "os"
"strings"
)
@@ -84,53 +82,12 @@ func normalizeChatCompletionError(err error) error {
}
func (c *Client) Text2Speech(text, voice string, outputFile string) error {
- baseUrl := config.Conf.Tts.Openai.BaseUrl
- if baseUrl == "" {
- baseUrl = "https://api.openai.com/v1"
- }
- url := baseUrl + "/audio/speech"
-
- // 创建HTTP请求
- reqBody := fmt.Sprintf(`{
- "model": "tts-1",
- "input": "%s",
- "voice":"%s",
- "response_format": "wav"
- }`, text, voice)
- req, err := http.NewRequest("POST", url, strings.NewReader(reqBody))
- if err != nil {
- return err
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", config.Conf.Tts.Openai.ApiKey))
-
- // 发送HTTP请求
- client := &http.Client{}
- resp, err := client.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- log.GetLogger().Error("openai tts failed", zap.Int("status_code", resp.StatusCode), zap.String("body", string(body)))
- return fmt.Errorf("openai tts none-200 status code: %d", resp.StatusCode)
- }
-
- file, err := os.Create(outputFile)
- if err != nil {
- return err
- }
- defer file.Close()
-
- _, err = io.Copy(file, resp.Body)
- if err != nil {
- return err
- }
-
- return nil
+ return NewTtsClient(
+ config.Conf.Tts.Openai.BaseUrl,
+ config.Conf.Tts.Openai.ApiKey,
+ config.Conf.Tts.Openai.Model,
+ config.Conf.App.Proxy,
+ ).Text2Speech(text, voice, outputFile)
}
func parseJSONResponse(jsonStr string) (string, error) {
diff --git a/pkg/openai/tts.go b/pkg/openai/tts.go
new file mode 100644
index 00000000..5e973518
--- /dev/null
+++ b/pkg/openai/tts.go
@@ -0,0 +1,148 @@
+package openai
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "krillin-ai/internal/types"
+)
+
+const (
+ DefaultTTSBaseURL = "https://api.openai.com/v1"
+ DefaultTTSModel = "gpt-4o-mini-tts"
+ DefaultTTSVoice = "marin"
+)
+
+type TtsClient struct {
+ BaseURL string
+ APIKey string
+ Model string
+ httpClient *http.Client
+}
+
+func NewTtsClient(baseURL, apiKey, model, proxy string) *TtsClient {
+ baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+ if baseURL == "" {
+ baseURL = DefaultTTSBaseURL
+ }
+ if strings.TrimSpace(model) == "" {
+ model = DefaultTTSModel
+ }
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ if proxyURL, err := url.Parse(strings.TrimSpace(proxy)); err == nil && proxyURL.Scheme != "" {
+ transport.Proxy = http.ProxyURL(proxyURL)
+ }
+ return &TtsClient{
+ BaseURL: baseURL,
+ APIKey: strings.TrimSpace(apiKey),
+ Model: strings.TrimSpace(model),
+ httpClient: &http.Client{
+ Transport: transport,
+ Timeout: 90 * time.Second,
+ },
+ }
+}
+
+func (c *TtsClient) Text2Speech(text, voice, outputFile string) error {
+ return c.Synthesize(context.Background(), types.TTSSpeechOptions{
+ Text: text,
+ Voice: voice,
+ OutputFile: outputFile,
+ })
+}
+
+func (c *TtsClient) Synthesize(ctx context.Context, options types.TTSSpeechOptions) error {
+ if c.APIKey == "" {
+ return fmt.Errorf("openai tts api key is empty")
+ }
+ voice := strings.TrimSpace(options.Voice)
+ if voice == "" {
+ voice = DefaultTTSVoice
+ }
+ format := speechFormat(options.Format, options.OutputFile)
+ speed := options.Speed
+ if speed <= 0 {
+ speed = 1
+ }
+ requestBody := map[string]any{
+ "model": c.Model,
+ "input": options.Text,
+ "voice": voice,
+ "response_format": format,
+ "speed": speed,
+ }
+ if strings.TrimSpace(options.Instructions) != "" && strings.Contains(strings.ToLower(c.Model), "gpt-4o") {
+ requestBody["instructions"] = strings.TrimSpace(options.Instructions)
+ }
+ body, err := json.Marshal(requestBody)
+ if err != nil {
+ return err
+ }
+ ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/audio/speech", bytes.NewReader(body))
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Authorization", "Bearer "+c.APIKey)
+ req.Header.Set("Content-Type", "application/json")
+ response, err := c.httpClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer response.Body.Close()
+ if response.StatusCode != http.StatusOK {
+ message, _ := io.ReadAll(io.LimitReader(response.Body, 2048))
+ return fmt.Errorf("openai tts returned HTTP %d: %s", response.StatusCode, strings.TrimSpace(string(message)))
+ }
+ if err := os.MkdirAll(filepath.Dir(options.OutputFile), 0755); err != nil {
+ return err
+ }
+ file, err := os.Create(options.OutputFile)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+ _, err = io.Copy(file, io.LimitReader(response.Body, 100<<20))
+ return err
+}
+
+func speechFormat(explicit, outputFile string) string {
+ format := strings.ToLower(strings.TrimSpace(explicit))
+ if format == "mp3" || format == "wav" {
+ return format
+ }
+ if strings.EqualFold(filepath.Ext(outputFile), ".mp3") {
+ return "mp3"
+ }
+ return "wav"
+}
+
+func (c *TtsClient) ListVoices(context.Context) ([]types.TTSVoice, error) {
+ return append([]types.TTSVoice(nil), openAIBuiltinVoices...), nil
+}
+
+var openAIBuiltinVoices = []types.TTSVoice{
+ {Code: "alloy", Name: "Alloy", Language: "multi", Provider: "openai", Scenario: "均衡", Kind: "builtin"},
+ {Code: "ash", Name: "Ash", Language: "multi", Provider: "openai", Scenario: "沉稳", Kind: "builtin"},
+ {Code: "ballad", Name: "Ballad", Language: "multi", Provider: "openai", Scenario: "表现力", Kind: "builtin"},
+ {Code: "cedar", Name: "Cedar", Language: "multi", Provider: "openai", Scenario: "自然", Kind: "builtin", Recommended: true},
+ {Code: "coral", Name: "Coral", Language: "multi", Provider: "openai", Scenario: "温暖", Kind: "builtin"},
+ {Code: "echo", Name: "Echo", Language: "multi", Provider: "openai", Scenario: "清晰", Kind: "builtin"},
+ {Code: "fable", Name: "Fable", Language: "multi", Provider: "openai", Scenario: "叙事", Kind: "builtin"},
+ {Code: "marin", Name: "Marin", Language: "multi", Provider: "openai", Scenario: "自然", Kind: "builtin", Recommended: true},
+ {Code: "nova", Name: "Nova", Language: "multi", Provider: "openai", Scenario: "明快", Kind: "builtin"},
+ {Code: "onyx", Name: "Onyx", Language: "multi", Provider: "openai", Scenario: "低沉", Kind: "builtin"},
+ {Code: "sage", Name: "Sage", Language: "multi", Provider: "openai", Scenario: "中性", Kind: "builtin"},
+ {Code: "shimmer", Name: "Shimmer", Language: "multi", Provider: "openai", Scenario: "柔和", Kind: "builtin"},
+ {Code: "verse", Name: "Verse", Language: "multi", Provider: "openai", Scenario: "表达", Kind: "builtin"},
+}
diff --git a/pkg/openai/tts_test.go b/pkg/openai/tts_test.go
new file mode 100644
index 00000000..cf2c4222
--- /dev/null
+++ b/pkg/openai/tts_test.go
@@ -0,0 +1,77 @@
+package openai
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "krillin-ai/internal/types"
+)
+
+func TestSynthesizeSendsOpenAITTSControlsAndWritesAudio(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/audio/speech" {
+ http.NotFound(w, r)
+ return
+ }
+ if r.Method != http.MethodPost {
+ t.Fatalf("method = %s, want POST", r.Method)
+ }
+ if r.Header.Get("Authorization") != "Bearer openai-key" {
+ t.Fatalf("authorization = %q", r.Header.Get("Authorization"))
+ }
+ var request map[string]any
+ if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+ t.Fatalf("decode request: %v", err)
+ }
+ if request["model"] != "gpt-4o-mini-tts" {
+ t.Fatalf("model = %#v", request["model"])
+ }
+ if request["input"] != "say \"hello\"\nnext" {
+ t.Fatalf("input = %#v", request["input"])
+ }
+ if request["voice"] != "marin" {
+ t.Fatalf("voice = %#v", request["voice"])
+ }
+ if request["response_format"] != "mp3" {
+ t.Fatalf("response_format = %#v", request["response_format"])
+ }
+ if request["speed"] != 1.15 {
+ t.Fatalf("speed = %#v", request["speed"])
+ }
+ if request["instructions"] != "Speak warmly." {
+ t.Fatalf("instructions = %#v", request["instructions"])
+ }
+ _, _ = w.Write([]byte("ID3-test-audio"))
+ }))
+ defer server.Close()
+
+ output := filepath.Join(t.TempDir(), "speech.mp3")
+ client := NewTtsClient(server.URL, "openai-key", "gpt-4o-mini-tts", "")
+ if err := client.Synthesize(context.Background(), types.TTSSpeechOptions{
+ Text: "say \"hello\"\nnext",
+ OutputFile: output,
+ Format: "mp3",
+ Speed: 1.15,
+ Instructions: " Speak warmly. ",
+ }); err != nil {
+ t.Fatalf("Synthesize() error = %v", err)
+ }
+ content, err := os.ReadFile(output)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(content) != "ID3-test-audio" {
+ t.Fatalf("audio = %q", content)
+ }
+}
+
+func TestDefaultVoiceMatchesOpenCreator(t *testing.T) {
+ if DefaultTTSVoice != "marin" {
+ t.Fatalf("DefaultTTSVoice = %q, want marin", DefaultTTSVoice)
+ }
+}
diff --git a/pkg/whisperkit/transcription.go b/pkg/whisperkit/transcription.go
index 7dd8236c..94b1472a 100644
--- a/pkg/whisperkit/transcription.go
+++ b/pkg/whisperkit/transcription.go
@@ -1,6 +1,7 @@
package whisperkit
import (
+ "bytes"
"encoding/json"
"krillin-ai/internal/storage"
"krillin-ai/internal/types"
@@ -8,12 +9,22 @@ import (
"krillin-ai/pkg/util"
"os"
"os/exec"
+ "regexp"
+ "strconv"
"strings"
+ "sync"
"go.uber.org/zap"
)
func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string) (*types.TranscriptionData, error) {
+ return c.TranscriptionWithProgress(audioFile, language, workDir, nil)
+}
+
+func (c *WhisperKitProcessor) TranscriptionWithProgress(
+ audioFile, language, workDir string,
+ reportProgress func(percent int),
+) (*types.TranscriptionData, error) {
cmdArgs := []string{
"transcribe",
"--model-path", "./models/whisperkit/openai_whisper-large-v2",
@@ -26,11 +37,17 @@ func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string)
"--skip-special-tokens",
"--audio-path", audioFile,
}
+ if reportProgress != nil {
+ cmdArgs = append(cmdArgs, "--verbose")
+ }
cmd := exec.Command(storage.WhisperKitPath, cmdArgs...)
log.GetLogger().Info("WhisperKitProcessor转录开始", zap.String("cmd", cmd.String()))
- output, err := cmd.CombinedOutput()
+ output := newProgressOutput(reportProgress)
+ cmd.Stdout = output
+ cmd.Stderr = output
+ err := cmd.Run()
if err != nil {
- log.GetLogger().Error("WhisperKitProcessor cmd 执行失败", zap.String("output", string(output)), zap.Error(err))
+ log.GetLogger().Error("WhisperKitProcessor cmd 执行失败", zap.String("output", output.String()), zap.Error(err))
return nil, err
}
log.GetLogger().Info("WhisperKitProcessor转录json生成完毕", zap.String("audio file", audioFile))
@@ -88,3 +105,55 @@ func (c *WhisperKitProcessor) Transcription(audioFile, language, workDir string)
log.GetLogger().Info("WhisperKitProcessor转录成功")
return &transcriptionData, nil
}
+
+var whisperKitProgressPattern = regexp.MustCompile(`\]\s*([0-9]{1,3})%\s*\|`)
+
+type progressOutput struct {
+ mu sync.Mutex
+ output bytes.Buffer
+ pending string
+ lastProgress int
+ reportProgress func(percent int)
+}
+
+func newProgressOutput(reportProgress func(percent int)) *progressOutput {
+ return &progressOutput{
+ lastProgress: -1,
+ reportProgress: reportProgress,
+ }
+}
+
+func (w *progressOutput) Write(data []byte) (int, error) {
+ w.mu.Lock()
+ _, _ = w.output.Write(data)
+ value := w.pending + string(data)
+ progressValues := make([]int, 0)
+ for _, match := range whisperKitProgressPattern.FindAllStringSubmatch(value, -1) {
+ percent, err := strconv.Atoi(match[1])
+ if err != nil || percent < 0 || percent > 100 || percent <= w.lastProgress {
+ continue
+ }
+ w.lastProgress = percent
+ progressValues = append(progressValues, percent)
+ }
+ const pendingLimit = 128
+ if len(value) > pendingLimit {
+ w.pending = value[len(value)-pendingLimit:]
+ } else {
+ w.pending = value
+ }
+ w.mu.Unlock()
+
+ if w.reportProgress != nil {
+ for _, percent := range progressValues {
+ w.reportProgress(percent)
+ }
+ }
+ return len(data), nil
+}
+
+func (w *progressOutput) String() string {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ return w.output.String()
+}
diff --git a/pkg/whisperkit/transcription_test.go b/pkg/whisperkit/transcription_test.go
new file mode 100644
index 00000000..8171e0b2
--- /dev/null
+++ b/pkg/whisperkit/transcription_test.go
@@ -0,0 +1,34 @@
+package whisperkit
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestProgressOutputReportsMonotonicFragmentedPercentages(t *testing.T) {
+ var reported []int
+ output := newProgressOutput(func(percent int) {
+ reported = append(reported, percent)
+ })
+
+ chunks := []string{
+ "\x1b[K[ ] 0% | Elapsed Time: 0.00 s\r",
+ "\x1b[K[=== ] 3",
+ "3% | Elapsed Time: 1.00 s\r",
+ "\x1b[K[====== ] 66% | Elapsed Time: 2.00 s\r",
+ "\x1b[K[=== ] 33% | repeated\r",
+ "\x1b[K[==========] 100% | Elapsed Time: 3.00 s\n",
+ }
+ for _, chunk := range chunks {
+ if _, err := output.Write([]byte(chunk)); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if got := fmt.Sprint(reported); got != "[0 33 66 100]" {
+ t.Fatalf("reported progress = %s", got)
+ }
+ if output.String() == "" {
+ t.Fatal("captured output is empty")
+ }
+}