Formerly KrillinAI, VideoTranslation and Dubbing Tool for 120+ Languages. Open-source AI workspace for creators, powered by Codex. Video downloader, video translation, create videos, images, audios, and edits with Agents in one place. For your Social Media like YouTube, TikTok, Instagram, etc.
TypeScript agent-skillscodex-clideskop-appdubbingimage-generationlocalizationskillsttsvideo-editingvideo-generationvideo-transcriptionvideo-translation
Improve dubbing and configuration
沈浩9b9dc60da7c5
diff --git a/README.md b/README.mdindex 3f0f812c..7ac05666 100644--- a/README.md+++ b/README.md@@ -49,7 +49,7 @@ Krillin AI 是一个专为追求高质量视频处理的用户和开发者设计1. 下载release中的可执行文件,放入空文件夹2. 在文件夹内创建config.toml文件,复制源代码config目录下的config-example.toml文件的内容,并对照填写你的配置信息。3. 双击可执行文件,启动服务-4. 打开浏览器,输入http://127.0.0.1:8888,开始使用 (8888替换成你在配置文件中填写的端口)+4. 打开浏览器,输入`http://127.0.0.1:8888`,开始使用 (8888替换成你在配置文件中填写的端口)### Cookie配置说明diff --git a/config/config-example.toml b/config/config-example.tomlindex 30f1cab0..92e6f94f 100644--- a/config/config-example.toml+++ b/config/config-example.toml@@ -7,4 +7,10 @@port = 8888[openai]- api_key = "sk-XXX"\ No newline at end of file+ api_key = "sk-XXX"++[aliyun]+ access_key_id = ""+ access_key_secret = ""+ app_key = ""+ cosy_voice_ws_addr = ""\ No newline at end of filediff --git a/config/config.go b/config/config.goindex 9d6fff15..27e3c0ef 100644--- a/config/config.go+++ b/config/config.go@@ -18,10 +18,18 @@ type Openai struct {ApiKey string `toml:"api_key"`}+type Aliyun struct {+ AccessKeyId string `toml:"access_key_id"`+ AccessKeySecret string `toml:"access_key_secret"`+ AppKey string `toml:"app_key"`+ CosyVoiceWsAddr string `toml:"cosy_voice_ws_addr"`+}+type Config struct {App App `toml:"app"`- Openai Openai `toml:"openai"`Server Server `toml:"server"`+ Openai Openai `toml:"openai"`+ Aliyun Aliyun `toml:"aliyun"`}var Conf Configdiff --git a/go.mod b/go.modindex 54f5fb10..0db30be5 100644--- a/go.mod+++ b/go.mod@@ -6,7 +6,11 @@ toolchain go1.22.10require (github.com/BurntSushi/toml v1.3.2+ github.com/aliyun/alibaba-cloud-sdk-go v1.63.72github.com/gin-gonic/gin v1.10.0+ github.com/go-resty/resty/v2 v2.7.0+ github.com/google/uuid v1.3.0+ github.com/gorilla/websocket v1.5.0github.com/samber/lo v1.38.1github.com/sashabaranov/go-openai v1.36.0github.com/u2takey/ffmpeg-go v0.5.0@@ -42,6 +46,7 @@ require (github.com/mitchellh/mapstructure v1.5.0 // indirectgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirectgithub.com/modern-go/reflect2 v1.0.2 // indirect+ github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirectgithub.com/pelletier/go-toml/v2 v2.2.2 // indirectgithub.com/pkg/errors v0.9.1 // indirectgithub.com/prometheus/client_golang v1.16.0 // indirect@@ -61,5 +66,6 @@ require (golang.org/x/sys v0.20.0 // indirectgolang.org/x/text v0.20.0 // indirectgoogle.golang.org/protobuf v1.34.1 // indirect+ gopkg.in/ini.v1 v1.67.0 // indirectgopkg.in/yaml.v3 v3.0.1 // indirect)diff --git a/go.sum b/go.sumindex 183f407b..0fdd47e6 100644--- a/go.sum+++ b/go.sum@@ -1,5 +1,11 @@+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=+github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=+github.com/aliyun/alibaba-cloud-sdk-go v1.63.72 h1:HvFZUzEbNvfe8F2Mg0wBGv90bPhWDxgVtDHR5zoBOU0=+github.com/aliyun/alibaba-cloud-sdk-go v1.63.72/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ=github.com/aws/aws-sdk-go v1.38.20 h1:QbzNx/tdfATbdKfubBpkt84OM6oBkxQZRw6+bW2GyeA=github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=@@ -16,12 +22,14 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=@@ -31,6 +39,7 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qEgithub.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=@@ -40,19 +49,28 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJngithub.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=+github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=+github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=+github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=@@ -61,6 +79,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCgithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=@@ -69,6 +88,8 @@ github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYgithub.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=@@ -89,6 +110,9 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNgithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=+github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A=+github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=@@ -136,10 +160,17 @@ github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4ukgithub.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc=github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys=github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs=+github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=+github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=+github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=+github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=github.com/wulien/jupiter v0.11.13 h1:hI3Oo+2PBRDJBytpPRPDUEJk9+JzPEc8NAF0aPuPLM4=github.com/wulien/jupiter v0.11.13/go.mod h1:+8CS93tYmUk3TTDEErMxzeHU2aOn3W8OZkdfnVSwyCA=+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=+go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=@@ -151,50 +182,82 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUugolang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=+golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=+gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=diff --git a/internal/dto/subtitle_task.go b/internal/dto/subtitle_task.goindex 541fc0dd..23028eba 100644--- a/internal/dto/subtitle_task.go+++ b/internal/dto/subtitle_task.go@@ -9,6 +9,7 @@ type StartVideoSubtitleTaskReq struct {TranslationSubtitlePos uint8 `json:"translation_subtitle_pos"`ModalFilter uint8 `json:"modal_filter"`Tts uint8 `json:"tts"`+ TtsVoiceCode uint8 `json:"tts_voice_code"`Replace []string `json:"replace"`Language string `json:"language"`}diff --git a/internal/service/init.go b/internal/service/init.goindex 81c32098..8d514301 100644--- a/internal/service/init.go+++ b/internal/service/init.go@@ -1,13 +1,18 @@package service-import "krillin-ai/pkg/openai"+import (+ "krillin-ai/pkg/aliyun"+ "krillin-ai/pkg/openai"+)type Service struct {- OpenaiClient *openai.Client+ OpenaiClient *openai.Client+ CosyCloneClient *aliyun.CosyCloneClient}func NewService() *Service {return &Service{- OpenaiClient: openai.NewClient(),+ OpenaiClient: openai.NewClient(),+ CosyCloneClient: aliyun.NewCosyCloneClient(),}}diff --git a/internal/service/subtitle_service.go b/internal/service/subtitle_service.goindex 3f213d09..17ad9a34 100644--- a/internal/service/subtitle_service.go+++ b/internal/service/subtitle_service.go@@ -7,8 +7,10 @@ import ("fmt""github.com/samber/lo"ffmpeg "github.com/u2takey/ffmpeg-go"+ "github.com/wulien/jupiter/pkg/xlog""go.uber.org/zap""golang.org/x/sync/errgroup"+ "io""krillin-ai/config""krillin-ai/internal/dto""krillin-ai/internal/storage"@@ -19,9 +21,12 @@ import ("os""os/exec""path/filepath"+ "regexp""runtime"+ "strconv""strings""sync"+ "time")func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.StartVideoSubtitleTaskResData, error) {@@ -86,17 +91,24 @@ func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.StarVideoSrc: req.Url,Status: types.SubtitleTaskStatusProcessing,}+ var ttsVoiceCode string+ if req.TtsVoiceCode == types.SubtitleTaskTtsVoiceCodeSijia {+ ttsVoiceCode = "sijia"+ } else {+ ttsVoiceCode = "sicheng"+ }stepParam := types.SubtitleTaskStepParam{TaskId: taskId,TaskBasePath: taskBasePath,Link: req.Url,SubtitleResultType: resultType,EnableModalFilter: req.ModalFilter == types.SubtitleTaskModalFilterYes,- //EnableTts: req.Tts,- ReplaceWordsMap: replaceWordsMap,- OriginLanguage: types.StandardLanguageName(req.OriginLanguage),- TargetLanguage: types.StandardLanguageName(req.TargetLang),- UserUILanguage: types.StandardLanguageName(req.Language),+ EnableTts: req.Tts == types.SubtitleTaskTtsYes,+ TtsVoiceCode: ttsVoiceCode,+ ReplaceWordsMap: replaceWordsMap,+ OriginLanguage: types.StandardLanguageName(req.OriginLanguage),+ TargetLanguage: types.StandardLanguageName(req.TargetLang),+ UserUILanguage: types.StandardLanguageName(req.Language),}go func() {defer func() {@@ -130,13 +142,13 @@ func (s Service) StartSubtitleTask(req dto.StartVideoSubtitleTaskReq) (*dto.Starstorage.SubtitleTasks[stepParam.TaskId].FailReason = "audio to subtitle error"return}- //err = s.srtFileToSpeech(ctx, &stepParam)- //if err != nil {- // //zap.Default().Error("StartVideoSubtitleTask srtFileToSpeech err", zap.Any("req", req), zap.FieldErr(err))- // storage.SubtitleTasks[stepParam.TaskId].Status = types.SubtitleTaskStatusFailed- // storage.SubtitleTasks[stepParam.TaskId].FailReason = "srt file to speech error"- // return- //}+ err = s.srtFileToSpeech(ctx, &stepParam)+ if err != nil {+ log.GetLogger().Error("StartVideoSubtitleTask srtFileToSpeech err", zap.Any("req", req), zap.Error(err))+ storage.SubtitleTasks[stepParam.TaskId].Status = types.SubtitleTaskStatusFailed+ storage.SubtitleTasks[stepParam.TaskId].FailReason = "srt file to speech error"+ return+ }err = s.uploadSubtitles(ctx, &stepParam)if err != nil {log.GetLogger().Error("StartVideoSubtitleTask uploadSubtitles err", zap.Any("req", req), zap.Error(err))@@ -918,3 +930,285 @@ func (s Service) splitTextAndTranslate(taskId, baseTaskPath string, targetLanguareturn nil}++// 配音部分++// 输入中文字幕,生成配音+func (s Service) srtFileToSpeech(ctx context.Context, stepParam *types.SubtitleTaskStepParam) error {+ if !stepParam.EnableTts {+ return nil+ }+ // Step 1: 解析字幕文件+ subtitles, err := parseSRT(stepParam.TtsSourceFilePath)+ if err != nil {+ log.GetLogger().Error("audioToSubtitle.parseSRT err", xlog.Any("stepParam", stepParam), xlog.FieldErr(err))+ return err+ }++ // 创建临时目录存储音频文件+ aliyunClient := s.CosyCloneClient+ var audioFiles []string+ var currentTime time.Time++ // 创建文件记录音频的开始和结束时间+ durationDetailFile, err := os.Create(filepath.Join(stepParam.TaskBasePath, types.TtsAudioDurationDetailsFileName))+ if err != nil {+ log.GetLogger().Error("generateAudioSubtitles.srtFileToSpeech.os.Create err", zap.Any("stepParam", stepParam), zap.Error(err))+ return err+ }+ defer durationDetailFile.Close()++ // Step 2: 使用 Google TTS 生成音频+ for i, sub := range subtitles {+ outputFile := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf("subtitle_%d.wav", i+1))+ err = aliyunClient.Text2Speech(sub.Text, stepParam.TtsVoiceCode, outputFile)+ if err != nil {+ log.GetLogger().Error("generateAudioSubtitles.srtFileToSpeech.Text2Speech err", zap.Any("taskId", stepParam.TaskId), zap.Any("num", i+1), zap.Error(err))+ return err+ }++ // Step 3: 调整音频时长+ startTime, err := time.Parse("15:04:05,000", sub.Start)+ if err != nil {+ log.GetLogger().Error("generateAudioSubtitles.adjustAudioDuration.time.Parse err", zap.Any("taskId", stepParam.TaskId), zap.Any("num", i+1), zap.Error(err))+ return err+ }+ endTime, err := time.Parse("15:04:05,000", sub.End)+ if err != nil {+ xlog.Default().Error("audioToSubtitle.time.Parse err", xlog.Any("stepParam", stepParam), xlog.Any("num", i+1), xlog.FieldErr(err))+ return err+ }+ if i == 0 {+ // 如果第一条字幕不是从00:00开始,增加静音帧+ if startTime.Second() > 0 {+ silenceDurationMs := startTime.Sub(time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC)).Milliseconds()+ silenceFilePath := filepath.Join(stepParam.TaskBasePath, "silence_0.wav")+ err := newGenerateSilence(silenceFilePath, float64(silenceDurationMs)/1000)+ if err != nil {+ log.GetLogger().Error("generateAudioSubtitles.newGenerateSilence.ChatCompletion err", zap.Any("taskId", stepParam.TaskId), zap.Error(err))+ return err+ }+ audioFiles = append(audioFiles, silenceFilePath)++ // 计算静音帧的结束时间+ silenceEndTime := currentTime.Add(time.Duration(silenceDurationMs) * time.Millisecond)+ durationDetailFile.WriteString(fmt.Sprintf("Silence: start=%s, end=%s\n", currentTime.Format("15:04:05,000"), silenceEndTime.Format("15:04:05,000")))+ currentTime = silenceEndTime+ }+ }++ duration := endTime.Sub(startTime).Seconds()+ if i < len(subtitles)-1 {+ // 如果不是最后一条字幕,增加静音帧时长+ nextStartTime, err := time.Parse("15:04:05,000", subtitles[i+1].Start)+ if err != nil {+ xlog.Default().Error("audioToSubtitle.time.Parse err", xlog.Any("stepParam", stepParam), xlog.Any("num", i+2), xlog.FieldErr(err))+ return err+ }+ if endTime.Before(nextStartTime) {+ duration = nextStartTime.Sub(startTime).Seconds()+ }+ }++ adjustedFile := filepath.Join(stepParam.TaskBasePath, fmt.Sprintf("adjusted_%d.wav", i+1))+ err = adjustAudioDuration(outputFile, adjustedFile, stepParam.TaskBasePath, duration)+ if err != nil {+ log.GetLogger().Error("audioToSubtitle.adjustAudioDuration err", zap.Any("stepParam", stepParam), zap.Any("num", i+1), zap.Error(err))+ return err+ }++ audioFiles = append(audioFiles, adjustedFile)++ // 计算音频的实际时长+ audioDuration, err := getAudioDuration(adjustedFile)+ if err != nil {+ log.GetLogger().Error("audioToSubtitle.getAudioDuration err", zap.Any("stepParam", stepParam), zap.Any("num", i+1), zap.Error(err))+ return err+ }++ // 计算音频的结束时间+ audioEndTime := currentTime.Add(time.Duration(audioDuration*1000) * time.Millisecond)+ // 写入文件+ durationDetailFile.WriteString(fmt.Sprintf("Audio %d: start=%s, end=%s\n", i+1, currentTime.Format("15:04:05,000"), audioEndTime.Format("15:04:05,000")))+ currentTime = audioEndTime+ }++ // Step 6: 拼接所有音频文件+ finalOutput := filepath.Join(stepParam.TaskBasePath, types.TtsResultAudioFileName)+ err = concatenateAudioFiles(audioFiles, finalOutput, stepParam.TaskBasePath)+ if err != nil {+ log.GetLogger().Error("audioToSubtitle.concatenateAudioFiles err", zap.Any("stepParam", stepParam), zap.Error(err))+ return err+ }+ stepParam.TtsResultFilePath = finalOutput+ // 更新字幕任务信息+ storage.SubtitleTasks[stepParam.TaskId].ProcessPct = 98+ log.GetLogger().Info("srtFileToSpeech success", zap.String("task id", stepParam.TaskId))+ return nil+}++// 解析 SRT 字幕文件+func parseSRT(filePath string) ([]types.SrtSentenceWithStrTime, error) {+ data, err := os.ReadFile(filePath)+ if err != nil {+ return nil, fmt.Errorf("读取文件失败: %v", err)+ }++ var subtitles []types.SrtSentenceWithStrTime+ re := regexp.MustCompile(`(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\s+(.+?)\n`)+ matches := re.FindAllStringSubmatch(string(data), -1)++ for _, match := range matches {+ subtitles = append(subtitles, types.SrtSentenceWithStrTime{+ Start: match[1],+ End: match[2],+ Text: strings.Replace(match[3], "\n", " ", -1), // 去除换行+ })+ }++ return subtitles, nil+}++func newGenerateSilence(outputAudio string, duration float64) error {+ // 生成 PCM 格式的静音文件+ cmd := exec.Command("ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=channel_layout=mono:sample_rate=44100", "-t",+ fmt.Sprintf("%.3f", duration), "-ar", "44100", "-ac", "1", "-c:a", "pcm_s16le", outputAudio)+ cmd.Stderr = os.Stderr+ err := cmd.Run()+ if err != nil {+ return fmt.Errorf("failed to generate PCM silence: %w", err)+ }++ return nil+}++// adjustAudioDuration 调整音频时长,确保音频与字幕时长一致+func adjustAudioDuration(inputFile, outputFile, taskBasePath string, subtitleDuration float64) error {+ // 获取音频时长+ audioDuration, err := getAudioDuration(inputFile)+ if err != nil {+ return err+ }++ // 如果音频时长短于字幕时长,插入静音延长音频+ if audioDuration < subtitleDuration {+ // 计算需要插入的静音时长+ silenceDuration := subtitleDuration - audioDuration++ // 生成静音音频+ silenceFile := filepath.Join(taskBasePath, "silence.wav")+ err := newGenerateSilence(silenceFile, silenceDuration)+ if err != nil {+ return fmt.Errorf("error generating silence: %v", err)+ }++ silenceAudioDuration, _ := getAudioDuration(silenceFile)+ fmt.Println("silenceDuration:", silenceAudioDuration)++ // 拼接音频和静音+ concatFile := filepath.Join(taskBasePath, "concat.txt")+ f, err := os.Create(concatFile)+ if err != nil {+ return fmt.Errorf("error creating concat file: %v", err)+ }+ defer os.Remove(concatFile)++ _, err = f.WriteString(fmt.Sprintf("file '%s'\nfile '%s'\n", filepath.Base(inputFile), filepath.Base(silenceFile)))+ if err != nil {+ return fmt.Errorf("error writing to concat file: %v", err)+ }+ f.Close()++ cmd := exec.Command("ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concatFile, "-c", "copy", outputFile)+ xlog.Default().Info("AiCapabilityGrpcServer adjustAudioDuration", xlog.Any("inputFile", inputFile), xlog.Any("outputFile", outputFile), xlog.String("run command", cmd.String()))+ cmd.Stderr = os.Stderr+ err = cmd.Run()+ if err != nil {+ return fmt.Errorf("error concatenating audio and silence: %v", err)+ }++ concatFileDuration, _ := getAudioDuration(outputFile)+ fmt.Println("concatFileDuration:", concatFileDuration)+ return nil+ }++ // 如果音频时长长于字幕时长,缩放音频的播放速率+ if audioDuration > subtitleDuration {+ // 计算播放速率+ speed := audioDuration / subtitleDuration+ //if speed < 0.5 || speed > 2.0 {+ // // 速率在 FFmpeg 支持的范围内一般是 [0.5, 2.0]+ // return fmt.Errorf("speed factor %.2f is out of range (0.5 to 2.0)", speed)+ //}++ // 使用 atempo 滤镜调整音频播放速率+ cmd := exec.Command("ffmpeg", "-y", "-i", inputFile, "-filter:a", fmt.Sprintf("atempo=%.2f", speed), outputFile)+ cmd.Stderr = os.Stderr+ return cmd.Run()+ }++ // 如果音频时长和字幕时长相同,则直接复制文件+ return copyFile(inputFile, outputFile)+}++// copyFile 复制文件+func copyFile(src, dst string) error {+ sourceFile, err := os.Open(src)+ if err != nil {+ return err+ }+ defer sourceFile.Close()++ destinationFile, err := os.Create(dst)+ if err != nil {+ return err+ }+ defer destinationFile.Close()++ _, err = io.Copy(destinationFile, sourceFile)+ if err != nil {+ return err+ }++ return destinationFile.Sync()+}++func getAudioDuration(inputFile string) (float64, error) {+ // 使用 ffprobe 获取精确时长+ cmd := exec.Command("ffprobe", "-i", inputFile, "-show_entries", "format=duration", "-v", "quiet", "-of", "csv=p=0")+ cmdOutput, err := cmd.Output()+ if err != nil {+ return 0, fmt.Errorf("failed to get audio duration: %w", err)+ }++ // 解析时长+ duration, err := strconv.ParseFloat(strings.TrimSpace(string(cmdOutput)), 64)+ if err != nil {+ return 0, fmt.Errorf("failed to parse audio duration: %w", err)+ }++ return duration, nil+}++// concatenateAudioFiles 拼接音频文件+func concatenateAudioFiles(audioFiles []string, outputFile, taskBasePath string) error {+ // 创建一个临时文件保存音频文件列表+ listFile := filepath.Join(taskBasePath, "audio_list.txt")+ f, err := os.Create(listFile)+ if err != nil {+ return err+ }+ defer os.Remove(listFile)++ for _, file := range audioFiles {+ _, err := f.WriteString(fmt.Sprintf("file '%s'\n", filepath.Base(file)))+ if err != nil {+ return err+ }+ }+ f.Close()++ cmd := exec.Command("ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", listFile, "-c", "copy", outputFile)+ cmd.Stderr = os.Stderr+ return cmd.Run()+}diff --git a/internal/types/subtitle_task.go b/internal/types/subtitle_task.goindex 73a0d0c2..a5be8ee7 100644--- a/internal/types/subtitle_task.go+++ b/internal/types/subtitle_task.go@@ -105,6 +105,16 @@ const (SubtitleTaskModalFilterNo)+const (+ SubtitleTaskTtsYes uint8 = iota + 1+ SubtitleTaskTtsNo+)++const (+ SubtitleTaskTtsVoiceCodeSijia uint8 = iota + 1+ SubtitleTaskTtsVoiceCodeSicheng+)+const (SubtitleTaskStatusProcessing uint8 = iota + 1SubtitleTaskStatusSuccess@@ -126,6 +136,11 @@ const (SubtitleTaskStepParamGobPersistenceFileName = "step_param.gob")+const (+ TtsAudioDurationDetailsFileName = "audio_duration_details.txt"+ TtsResultAudioFileName = "tts_final_audio.wav"+)+type StandardLanguageName stringconst (@@ -224,6 +239,7 @@ type SubtitleTaskStepParam struct {SubtitleResultType SubtitleResultTypeEnableModalFilter boolEnableTts bool+ TtsVoiceCode string // 人声语音编码ReplaceWordsMap map[string]stringOriginLanguage StandardLanguageName // 视频源语言TargetLanguage StandardLanguageName // 用户希望的目标翻译语言diff --git a/pkg/aliyun/speech.go b/pkg/aliyun/speech.gonew file mode 100644index 00000000..09e1f9bf--- /dev/null+++ b/pkg/aliyun/speech.go@@ -0,0 +1,195 @@+package aliyun++import (+ "crypto/hmac"+ "crypto/sha1"+ "encoding/base64"+ "fmt"+ "krillin-ai/config"+ "net/url"+ "os"+ "sort"+ "strings"+ "time"++ "github.com/go-resty/resty/v2"+ "github.com/google/uuid"+)++// type CosyClone struct{}++// _encodeText URL-编码文本,保证符合规范+func _encodeText(text string) string {+ encoded := url.QueryEscape(text)+ // 根据规范替换特殊字符+ return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(encoded, "+", "%20"), "*", "%2A"), "%7E", "~")+}++// _encodeDict URL-编码字典(map)为查询字符串+func _encodeDict(dic map[string]string) string {+ var keys []string+ for key := range dic {+ keys = append(keys, key)+ }+ sort.Strings(keys)+ values := url.Values{}++ for _, k := range keys {+ values.Add(k, dic[k])+ }+ encodedText := values.Encode()+ // 对整个查询字符串进行编码+ return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(encodedText, "+", "%20"), "*", "%2A"), "%7E", "~")+}++// 生成签名+func GenerateSignature(secret, stringToSign string) string {+ key := []byte(secret + "&")+ data := []byte(stringToSign)+ hash := hmac.New(sha1.New, key)+ hash.Write(data)+ signature := base64.StdEncoding.EncodeToString(hash.Sum(nil))+ // 对签名进行URL编码+ return _encodeText(signature)+}++type CosyCloneClient struct {+ restyClient *resty.Client+ accessKeyID string+ accessKeySecret string+ appkey string+ wsUrl string+}++func NewCosyCloneClient() *CosyCloneClient {+ cli := resty.New()+ return &CosyCloneClient{+ restyClient: cli,+ accessKeyID: config.Conf.Aliyun.AccessKeyId,+ accessKeySecret: config.Conf.Aliyun.AccessKeySecret,+ appkey: config.Conf.Aliyun.AppKey,+ wsUrl: config.Conf.Aliyun.CosyVoiceWsAddr,+ }+}++func (client *CosyCloneClient) CosyVoiceClone(voicePrefix, audioURL string) {+ parameters := map[string]string{+ "AccessKeyId": client.accessKeyID,+ "Action": "CosyVoiceClone",+ "Format": "JSON",+ "RegionId": "cn-shanghai",+ "SignatureMethod": "HMAC-SHA1",+ "SignatureNonce": uuid.New().String(),+ "SignatureVersion": "1.0",+ "Timestamp": time.Now().UTC().Format("2006-01-02T15:04:05Z"),+ "Version": "2019-08-19",+ "VoicePrefix": voicePrefix,+ "Url": audioURL,+ }++ queryString := _encodeDict(parameters)+ fmt.Println("规范化的请求字符串:", queryString)+ stringToSign := "POST" + "&" + _encodeText("/") + "&" + _encodeText(queryString)+ fmt.Println("待签名的字符串:", stringToSign)+ signature := GenerateSignature(client.accessKeySecret, stringToSign)+ fmt.Println("URL编码后的签名:", signature)+ fullURL := fmt.Sprintf("https://nls-slp.cn-shanghai.aliyuncs.com/?Signature=%s&%s", signature, queryString)+ fmt.Printf("url: %s\n", fullURL)++ // Make the POST request using resty+ values := url.Values{}+ for key, value := range parameters {+ values.Add(key, value)+ }+ // resp, err := client.restyClient.R().SetQueryParam("Signature", signature).SetQueryParamsFromValues(values).Post(fullURL)+ resp, err := client.restyClient.R().Post(fullURL)+ if err != nil {+ fmt.Println("Error:", err)+ return+ }+ fmt.Println("Response:", resp.String())+}++func (client *CosyCloneClient) CosyCloneList(voicePrefix string, pageIndex, pageSize int) {+ parameters := map[string]string{+ "AccessKeyId": client.accessKeyID,+ "Action": "ListCosyVoice",+ "Format": "JSON",+ "RegionId": "cn-shanghai",+ "SignatureMethod": "HMAC-SHA1",+ "SignatureNonce": uuid.New().String(),+ "SignatureVersion": "1.0",+ "Timestamp": time.Now().UTC().Format("2006-01-02T15:04:05Z"),+ "Version": "2019-08-19",+ "VoicePrefix": voicePrefix,+ "PageIndex": fmt.Sprintf("%d", pageIndex),+ "PageSize": fmt.Sprintf("%d", pageSize),+ }++ queryString := _encodeDict(parameters)+ fmt.Println("规范化的请求字符串:", queryString)+ stringToSign := "POST" + "&" + _encodeText("/") + "&" + _encodeText(queryString)+ fmt.Println("待签名的字符串:", stringToSign)+ signature := GenerateSignature(client.accessKeySecret, stringToSign)+ fmt.Println("URL编码后的签名:", signature)+ fullURL := fmt.Sprintf("https://nls-slp.cn-shanghai.aliyuncs.com/?Signature=%s&%s", signature, queryString)+ fmt.Printf("url: %s\n", fullURL)++ // Make the POST request using resty+ values := url.Values{}+ for key, value := range parameters {+ values.Add(key, value)+ }+ resp, err := client.restyClient.R().Post(fullURL)+ if err != nil {+ fmt.Println("Error:", err)+ return+ }+ fmt.Println("Response:", resp.String())+}++func (client *CosyCloneClient) Text2Speech(text, voice, outputFile string) error {+ file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY, 0666)+ if err != nil {+ return fmt.Errorf("failed to create file: %w", err)+ }+ defer file.Close()++ onTextMessage := func(message string) {+ fmt.Println("Received text message:", message)+ }++ onBinaryMessage := func(data []byte) {+ if _, err := file.Write(data); err != nil {+ fmt.Println("Failed to write data to file:", err)+ }+ }++ speechClient, err := NewSpeechClient(client.wsUrl, client.appkey, onTextMessage, onBinaryMessage)+ if err != nil {+ return fmt.Errorf("failed to create speech client: %w", err)+ }+ defer speechClient.Close()++ startPayload := StartSynthesisPayload{+ Voice: voice,+ Format: "wav",+ SampleRate: 44100,+ Volume: 50,+ SpeechRate: 0,+ PitchRate: 0,+ }+ if err := speechClient.StartSynthesis(startPayload); err != nil {+ return fmt.Errorf("failed to start synthesis: %w", err)+ }++ if err := speechClient.RunSynthesis(text); err != nil {+ return fmt.Errorf("failed to run synthesis: %w", err)+ }++ if err := speechClient.StopSynthesis(); err != nil {+ return fmt.Errorf("failed to stop synthesis: %w", err)+ }++ return nil+}diff --git a/pkg/aliyun/speech_client.go b/pkg/aliyun/speech_client.gonew file mode 100644index 00000000..e9f427d4--- /dev/null+++ b/pkg/aliyun/speech_client.go@@ -0,0 +1,164 @@+package aliyun++import (+ "encoding/json"+ "fmt"+ "log"+ "strings"+ "time"++ "github.com/google/uuid"+ "github.com/gorilla/websocket"+)++type SpeechClient struct {+ conn *websocket.Conn+ appkey string+ taskID string+ synthesisStarted chan struct{}+ synthesisComplete chan struct{}+}++type Header struct {+ Appkey string `json:"appkey"`+ MessageID string `json:"message_id"`+ TaskID string `json:"task_id"`+ Namespace string `json:"namespace"`+ Name string `json:"name"`+}++type StartSynthesisPayload struct {+ Voice string `json:"voice,omitempty"`+ Format string `json:"format,omitempty"`+ SampleRate int `json:"sample_rate,omitempty"`+ Volume int `json:"volume,omitempty"`+ SpeechRate int `json:"speech_rate,omitempty"`+ PitchRate int `json:"pitch_rate,omitempty"`+ EnableSubtitle bool `json:"enable_subtitle,omitempty"`+ EnablePhonemeTimestamp bool `json:"enable_phoneme_timestamp,omitempty"`+}++type RunSynthesisPayload struct {+ Text string `json:"text"`+}++type Message struct {+ Header Header `json:"header"`+ Payload interface{} `json:"payload,omitempty"`+}++func NewSpeechClient(url, appkey string, onTextMessage func(string), onBinaryMessage func([]byte)) (*SpeechClient, error) {+ token, _ := CreateToken()+ fullURL := "wss://nls-gateway-cn-beijing.aliyuncs.com/ws/v1?token=" + token+ dialer := websocket.DefaultDialer+ dialer.HandshakeTimeout = 10 * time.Second+ conn, _, err := dialer.Dial(fullURL, nil)+ if err != nil {+ return nil, err+ }+ conn.SetReadDeadline(time.Now().Add(60 * time.Second))+ client := &SpeechClient{+ conn: conn,+ appkey: appkey,+ taskID: generateID(),+ synthesisComplete: make(chan struct{}),+ synthesisStarted: make(chan struct{}),+ }++ go client.receiveMessages(onTextMessage, onBinaryMessage)++ return client, nil+}++func generateID() string {+ return strings.ReplaceAll(uuid.New().String(), "-", "")+}++func (c *SpeechClient) sendMessage(name string, payload interface{}) error {+ message := Message{+ Header: Header{+ Appkey: c.appkey,+ MessageID: generateID(),+ TaskID: c.taskID,+ Namespace: "FlowingSpeechSynthesizer",+ Name: name,+ },+ Payload: payload,+ }+ //打印message,转成json字符串格式+ jsonData, _ := json.Marshal(message)+ // 打印 JSON 字符串+ fmt.Println(string(jsonData))+ return c.conn.WriteJSON(message)+}++func (c *SpeechClient) StartSynthesis(payload StartSynthesisPayload) error {+ err := c.sendMessage("StartSynthesis", payload)+ if err != nil {+ return err+ }++ // 阻塞等待 SynthesisStarted 事件+ <-c.synthesisStarted++ return nil+}++func (c *SpeechClient) RunSynthesis(text string) error {+ return c.sendMessage("RunSynthesis", RunSynthesisPayload{Text: text})+}++func (c *SpeechClient) StopSynthesis() error {+ err := c.sendMessage("StopSynthesis", nil)+ if err != nil {+ return err+ }++ // 阻塞等待 SynthesisCompleted 事件+ <-c.synthesisComplete++ return nil+}++func (c *SpeechClient) Close() error {+ err := c.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))+ if err != nil {+ log.Println("write close:", err)+ return err+ }+ return c.conn.Close()+}++func (c *SpeechClient) receiveMessages(onTextMessage func(string), onBinaryMessage func([]byte)) {+ defer close(c.synthesisComplete)+ for {+ messageType, message, err := c.conn.ReadMessage()+ if err != nil {+ if websocket.IsCloseError(err, websocket.CloseNormalClosure) {+ log.Println("connection closed normally")+ return+ }+ fmt.Println("read:", err)+ return+ }+ if messageType == websocket.TextMessage {+ var msg Message+ if err := json.Unmarshal(message, &msg); err != nil {+ fmt.Println("failed to unmarshal message:", err)+ return+ }+ if msg.Header.Name == "SynthesisCompleted" {+ fmt.Println("SynthesisCompleted event received")+ // 收到结束消息退出+ break+ } else if msg.Header.Name == "SynthesisStarted" {+ fmt.Println("SynthesisStarted event received")+ close(c.synthesisStarted)+ } else {+ onTextMessage(string(message))+ }+ } else if messageType == websocket.BinaryMessage {+ onBinaryMessage(message)+ }+ }+}diff --git a/pkg/aliyun/token.go b/pkg/aliyun/token.gonew file mode 100644index 00000000..1b287c71--- /dev/null+++ b/pkg/aliyun/token.go@@ -0,0 +1,43 @@+package aliyun++import (+ "encoding/json"+ "github.com/aliyun/alibaba-cloud-sdk-go/sdk"+ "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"+ "github.com/wulien/jupiter/pkg/xlog"+ "krillin-ai/config"+)++type TokenResult struct {+ ErrMsg string+ Token struct {+ UserId string+ Id string+ ExpireTime int64+ }+}++func CreateToken() (string, error) {+ client, err := sdk.NewClientWithAccessKey("cn-shanghai", config.Conf.Aliyun.AccessKeyId, config.Conf.Aliyun.AccessKeySecret)+ if err != nil {+ return "", err+ }+ request := requests.NewCommonRequest()+ request.Method = "POST"+ request.Domain = "nls-meta.cn-shanghai.aliyuncs.com"+ request.ApiName = "CreateToken"+ request.Version = "2019-02-28"+ response, err := client.ProcessCommonRequest(request)+ if err != nil {+ xlog.Default().Error("aliyun sdk create token request error:", xlog.Any("err", err))+ return "", err+ }++ var tr TokenResult+ err = json.Unmarshal([]byte(response.GetHttpContentString()), &tr)+ if err != nil {+ xlog.Default().Error("aliyun sdk json unmarshal error:", xlog.Any("err", err))+ return "", err+ }+ return tr.Token.Id, nil+}