SOTA Open Source TTS
Data server is deprecated
Lengyue3c8912e4eb2c
diff --git a/data_server/Cargo.lock b/data_server/Cargo.lockdeleted file mode 100644index 9b37bf1..0000000--- a/data_server/Cargo.lock+++ /dev/null@@ -1,1403 +0,0 @@-# This file is automatically @generated by Cargo.-# It is not intended for manual editing.-version = 3--[[package]]-name = "addr2line"-version = "0.21.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"-dependencies = [- "gimli",-]--[[package]]-name = "adler"-version = "1.0.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"--[[package]]-name = "aho-corasick"-version = "1.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"-dependencies = [- "memchr",-]--[[package]]-name = "anstream"-version = "0.6.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"-dependencies = [- "anstyle",- "anstyle-parse",- "anstyle-query",- "anstyle-wincon",- "colorchoice",- "utf8parse",-]--[[package]]-name = "anstyle"-version = "1.0.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"--[[package]]-name = "anstyle-parse"-version = "0.2.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"-dependencies = [- "utf8parse",-]--[[package]]-name = "anstyle-query"-version = "1.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748"-dependencies = [- "windows-sys 0.52.0",-]--[[package]]-name = "anstyle-wincon"-version = "3.0.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"-dependencies = [- "anstyle",- "windows-sys 0.52.0",-]--[[package]]-name = "anyhow"-version = "1.0.75"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"--[[package]]-name = "async-stream"-version = "0.3.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"-dependencies = [- "async-stream-impl",- "futures-core",- "pin-project-lite",-]--[[package]]-name = "async-stream-impl"-version = "0.3.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "async-trait"-version = "0.1.74"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "autocfg"-version = "1.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"--[[package]]-name = "axum"-version = "0.6.20"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"-dependencies = [- "async-trait",- "axum-core",- "bitflags 1.3.2",- "bytes",- "futures-util",- "http",- "http-body",- "hyper",- "itoa",- "matchit",- "memchr",- "mime",- "percent-encoding",- "pin-project-lite",- "rustversion",- "serde",- "sync_wrapper",- "tower",- "tower-layer",- "tower-service",-]--[[package]]-name = "axum-core"-version = "0.3.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"-dependencies = [- "async-trait",- "bytes",- "futures-util",- "http",- "http-body",- "mime",- "rustversion",- "tower-layer",- "tower-service",-]--[[package]]-name = "backtrace"-version = "0.3.69"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"-dependencies = [- "addr2line",- "cc",- "cfg-if",- "libc",- "miniz_oxide",- "object",- "rustc-demangle",-]--[[package]]-name = "base64"-version = "0.21.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"--[[package]]-name = "bitflags"-version = "1.3.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"--[[package]]-name = "bitflags"-version = "2.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"--[[package]]-name = "bytes"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"--[[package]]-name = "cc"-version = "1.0.83"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"-dependencies = [- "libc",-]--[[package]]-name = "cfg-if"-version = "1.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"--[[package]]-name = "clap"-version = "4.4.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2"-dependencies = [- "clap_builder",- "clap_derive",-]--[[package]]-name = "clap_builder"-version = "4.4.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb"-dependencies = [- "anstream",- "anstyle",- "clap_lex",- "strsim",-]--[[package]]-name = "clap_derive"-version = "4.4.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"-dependencies = [- "heck",- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "clap_lex"-version = "0.6.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"--[[package]]-name = "colorchoice"-version = "1.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"--[[package]]-name = "data_server"-version = "0.1.0"-dependencies = [- "bytes",- "clap",- "env_logger",- "log",- "prost",- "rand",- "tokio",- "tonic",- "tonic-build",-]--[[package]]-name = "either"-version = "1.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"--[[package]]-name = "env_logger"-version = "0.10.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece"-dependencies = [- "humantime",- "is-terminal",- "log",- "regex",- "termcolor",-]--[[package]]-name = "equivalent"-version = "1.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"--[[package]]-name = "errno"-version = "0.3.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"-dependencies = [- "libc",- "windows-sys 0.52.0",-]--[[package]]-name = "fastrand"-version = "2.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"--[[package]]-name = "fixedbitset"-version = "0.4.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"--[[package]]-name = "fnv"-version = "1.0.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"--[[package]]-name = "futures-channel"-version = "0.3.29"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"-dependencies = [- "futures-core",-]--[[package]]-name = "futures-core"-version = "0.3.29"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"--[[package]]-name = "futures-sink"-version = "0.3.29"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"--[[package]]-name = "futures-task"-version = "0.3.29"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"--[[package]]-name = "futures-util"-version = "0.3.29"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"-dependencies = [- "futures-core",- "futures-task",- "pin-project-lite",- "pin-utils",-]--[[package]]-name = "getrandom"-version = "0.2.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"-dependencies = [- "cfg-if",- "libc",- "wasi",-]--[[package]]-name = "gimli"-version = "0.28.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"--[[package]]-name = "h2"-version = "0.3.22"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"-dependencies = [- "bytes",- "fnv",- "futures-core",- "futures-sink",- "futures-util",- "http",- "indexmap 2.1.0",- "slab",- "tokio",- "tokio-util",- "tracing",-]--[[package]]-name = "hashbrown"-version = "0.12.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"--[[package]]-name = "hashbrown"-version = "0.14.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"--[[package]]-name = "heck"-version = "0.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"--[[package]]-name = "hermit-abi"-version = "0.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"--[[package]]-name = "home"-version = "0.5.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb"-dependencies = [- "windows-sys 0.48.0",-]--[[package]]-name = "http"-version = "0.2.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"-dependencies = [- "bytes",- "fnv",- "itoa",-]--[[package]]-name = "http-body"-version = "0.4.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"-dependencies = [- "bytes",- "http",- "pin-project-lite",-]--[[package]]-name = "httparse"-version = "1.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"--[[package]]-name = "httpdate"-version = "1.0.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"--[[package]]-name = "humantime"-version = "2.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"--[[package]]-name = "hyper"-version = "0.14.27"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"-dependencies = [- "bytes",- "futures-channel",- "futures-core",- "futures-util",- "h2",- "http",- "http-body",- "httparse",- "httpdate",- "itoa",- "pin-project-lite",- "socket2 0.4.10",- "tokio",- "tower-service",- "tracing",- "want",-]--[[package]]-name = "hyper-timeout"-version = "0.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"-dependencies = [- "hyper",- "pin-project-lite",- "tokio",- "tokio-io-timeout",-]--[[package]]-name = "indexmap"-version = "1.9.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"-dependencies = [- "autocfg",- "hashbrown 0.12.3",-]--[[package]]-name = "indexmap"-version = "2.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"-dependencies = [- "equivalent",- "hashbrown 0.14.3",-]--[[package]]-name = "is-terminal"-version = "0.4.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"-dependencies = [- "hermit-abi",- "rustix",- "windows-sys 0.48.0",-]--[[package]]-name = "itertools"-version = "0.11.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"-dependencies = [- "either",-]--[[package]]-name = "itoa"-version = "1.0.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"--[[package]]-name = "libc"-version = "0.2.150"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"--[[package]]-name = "linux-raw-sys"-version = "0.4.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"--[[package]]-name = "lock_api"-version = "0.4.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"-dependencies = [- "autocfg",- "scopeguard",-]--[[package]]-name = "log"-version = "0.4.20"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"--[[package]]-name = "matchit"-version = "0.7.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"--[[package]]-name = "memchr"-version = "2.6.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"--[[package]]-name = "mime"-version = "0.3.17"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"--[[package]]-name = "miniz_oxide"-version = "0.7.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"-dependencies = [- "adler",-]--[[package]]-name = "mio"-version = "0.8.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"-dependencies = [- "libc",- "wasi",- "windows-sys 0.48.0",-]--[[package]]-name = "multimap"-version = "0.8.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"--[[package]]-name = "num_cpus"-version = "1.16.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"-dependencies = [- "hermit-abi",- "libc",-]--[[package]]-name = "object"-version = "0.32.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"-dependencies = [- "memchr",-]--[[package]]-name = "once_cell"-version = "1.19.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"--[[package]]-name = "parking_lot"-version = "0.12.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"-dependencies = [- "lock_api",- "parking_lot_core",-]--[[package]]-name = "parking_lot_core"-version = "0.9.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"-dependencies = [- "cfg-if",- "libc",- "redox_syscall",- "smallvec",- "windows-targets 0.48.5",-]--[[package]]-name = "percent-encoding"-version = "2.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"--[[package]]-name = "petgraph"-version = "0.6.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"-dependencies = [- "fixedbitset",- "indexmap 2.1.0",-]--[[package]]-name = "pin-project"-version = "1.1.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"-dependencies = [- "pin-project-internal",-]--[[package]]-name = "pin-project-internal"-version = "1.1.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "pin-project-lite"-version = "0.2.13"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"--[[package]]-name = "pin-utils"-version = "0.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"--[[package]]-name = "ppv-lite86"-version = "0.2.17"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"--[[package]]-name = "prettyplease"-version = "0.2.15"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"-dependencies = [- "proc-macro2",- "syn",-]--[[package]]-name = "proc-macro2"-version = "1.0.70"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"-dependencies = [- "unicode-ident",-]--[[package]]-name = "prost"-version = "0.12.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"-dependencies = [- "bytes",- "prost-derive",-]--[[package]]-name = "prost-build"-version = "0.12.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2"-dependencies = [- "bytes",- "heck",- "itertools",- "log",- "multimap",- "once_cell",- "petgraph",- "prettyplease",- "prost",- "prost-types",- "regex",- "syn",- "tempfile",- "which",-]--[[package]]-name = "prost-derive"-version = "0.12.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"-dependencies = [- "anyhow",- "itertools",- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "prost-types"-version = "0.12.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e"-dependencies = [- "prost",-]--[[package]]-name = "quote"-version = "1.0.33"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"-dependencies = [- "proc-macro2",-]--[[package]]-name = "rand"-version = "0.8.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"-dependencies = [- "libc",- "rand_chacha",- "rand_core",-]--[[package]]-name = "rand_chacha"-version = "0.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"-dependencies = [- "ppv-lite86",- "rand_core",-]--[[package]]-name = "rand_core"-version = "0.6.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"-dependencies = [- "getrandom",-]--[[package]]-name = "redox_syscall"-version = "0.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"-dependencies = [- "bitflags 1.3.2",-]--[[package]]-name = "regex"-version = "1.10.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"-dependencies = [- "aho-corasick",- "memchr",- "regex-automata",- "regex-syntax",-]--[[package]]-name = "regex-automata"-version = "0.4.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"-dependencies = [- "aho-corasick",- "memchr",- "regex-syntax",-]--[[package]]-name = "regex-syntax"-version = "0.8.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"--[[package]]-name = "rustc-demangle"-version = "0.1.23"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"--[[package]]-name = "rustix"-version = "0.38.27"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bfeae074e687625746172d639330f1de242a178bf3189b51e35a7a21573513ac"-dependencies = [- "bitflags 2.4.1",- "errno",- "libc",- "linux-raw-sys",- "windows-sys 0.52.0",-]--[[package]]-name = "rustversion"-version = "1.0.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"--[[package]]-name = "scopeguard"-version = "1.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"--[[package]]-name = "serde"-version = "1.0.193"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"-dependencies = [- "serde_derive",-]--[[package]]-name = "serde_derive"-version = "1.0.193"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "signal-hook-registry"-version = "1.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"-dependencies = [- "libc",-]--[[package]]-name = "slab"-version = "0.4.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"-dependencies = [- "autocfg",-]--[[package]]-name = "smallvec"-version = "1.11.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"--[[package]]-name = "socket2"-version = "0.4.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d"-dependencies = [- "libc",- "winapi",-]--[[package]]-name = "socket2"-version = "0.5.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"-dependencies = [- "libc",- "windows-sys 0.48.0",-]--[[package]]-name = "strsim"-version = "0.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"--[[package]]-name = "syn"-version = "2.0.39"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"-dependencies = [- "proc-macro2",- "quote",- "unicode-ident",-]--[[package]]-name = "sync_wrapper"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"--[[package]]-name = "tempfile"-version = "3.8.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"-dependencies = [- "cfg-if",- "fastrand",- "redox_syscall",- "rustix",- "windows-sys 0.48.0",-]--[[package]]-name = "termcolor"-version = "1.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"-dependencies = [- "winapi-util",-]--[[package]]-name = "tokio"-version = "1.34.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"-dependencies = [- "backtrace",- "bytes",- "libc",- "mio",- "num_cpus",- "parking_lot",- "pin-project-lite",- "signal-hook-registry",- "socket2 0.5.5",- "tokio-macros",- "windows-sys 0.48.0",-]--[[package]]-name = "tokio-io-timeout"-version = "1.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"-dependencies = [- "pin-project-lite",- "tokio",-]--[[package]]-name = "tokio-macros"-version = "2.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "tokio-stream"-version = "0.1.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"-dependencies = [- "futures-core",- "pin-project-lite",- "tokio",-]--[[package]]-name = "tokio-util"-version = "0.7.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"-dependencies = [- "bytes",- "futures-core",- "futures-sink",- "pin-project-lite",- "tokio",- "tracing",-]--[[package]]-name = "tonic"-version = "0.10.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e"-dependencies = [- "async-stream",- "async-trait",- "axum",- "base64",- "bytes",- "h2",- "http",- "http-body",- "hyper",- "hyper-timeout",- "percent-encoding",- "pin-project",- "prost",- "tokio",- "tokio-stream",- "tower",- "tower-layer",- "tower-service",- "tracing",-]--[[package]]-name = "tonic-build"-version = "0.10.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889"-dependencies = [- "prettyplease",- "proc-macro2",- "prost-build",- "quote",- "syn",-]--[[package]]-name = "tower"-version = "0.4.13"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"-dependencies = [- "futures-core",- "futures-util",- "indexmap 1.9.3",- "pin-project",- "pin-project-lite",- "rand",- "slab",- "tokio",- "tokio-util",- "tower-layer",- "tower-service",- "tracing",-]--[[package]]-name = "tower-layer"-version = "0.3.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"--[[package]]-name = "tower-service"-version = "0.3.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"--[[package]]-name = "tracing"-version = "0.1.40"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"-dependencies = [- "pin-project-lite",- "tracing-attributes",- "tracing-core",-]--[[package]]-name = "tracing-attributes"-version = "0.1.27"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "tracing-core"-version = "0.1.32"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"-dependencies = [- "once_cell",-]--[[package]]-name = "try-lock"-version = "0.2.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"--[[package]]-name = "unicode-ident"-version = "1.0.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"--[[package]]-name = "utf8parse"-version = "0.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"--[[package]]-name = "want"-version = "0.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"-dependencies = [- "try-lock",-]--[[package]]-name = "wasi"-version = "0.11.0+wasi-snapshot-preview1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"--[[package]]-name = "which"-version = "4.4.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"-dependencies = [- "either",- "home",- "once_cell",- "rustix",-]--[[package]]-name = "winapi"-version = "0.3.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"-dependencies = [- "winapi-i686-pc-windows-gnu",- "winapi-x86_64-pc-windows-gnu",-]--[[package]]-name = "winapi-i686-pc-windows-gnu"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"--[[package]]-name = "winapi-util"-version = "0.1.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"-dependencies = [- "winapi",-]--[[package]]-name = "winapi-x86_64-pc-windows-gnu"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"--[[package]]-name = "windows-sys"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"-dependencies = [- "windows-targets 0.48.5",-]--[[package]]-name = "windows-sys"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"-dependencies = [- "windows-targets 0.52.0",-]--[[package]]-name = "windows-targets"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"-dependencies = [- "windows_aarch64_gnullvm 0.48.5",- "windows_aarch64_msvc 0.48.5",- "windows_i686_gnu 0.48.5",- "windows_i686_msvc 0.48.5",- "windows_x86_64_gnu 0.48.5",- "windows_x86_64_gnullvm 0.48.5",- "windows_x86_64_msvc 0.48.5",-]--[[package]]-name = "windows-targets"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"-dependencies = [- "windows_aarch64_gnullvm 0.52.0",- "windows_aarch64_msvc 0.52.0",- "windows_i686_gnu 0.52.0",- "windows_i686_msvc 0.52.0",- "windows_x86_64_gnu 0.52.0",- "windows_x86_64_gnullvm 0.52.0",- "windows_x86_64_msvc 0.52.0",-]--[[package]]-name = "windows_aarch64_gnullvm"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"--[[package]]-name = "windows_aarch64_gnullvm"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"--[[package]]-name = "windows_aarch64_msvc"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"--[[package]]-name = "windows_aarch64_msvc"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"--[[package]]-name = "windows_i686_gnu"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"--[[package]]-name = "windows_i686_gnu"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"--[[package]]-name = "windows_i686_msvc"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"--[[package]]-name = "windows_i686_msvc"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"--[[package]]-name = "windows_x86_64_gnu"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"--[[package]]-name = "windows_x86_64_gnu"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"--[[package]]-name = "windows_x86_64_gnullvm"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"--[[package]]-name = "windows_x86_64_gnullvm"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"--[[package]]-name = "windows_x86_64_msvc"-version = "0.48.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"--[[package]]-name = "windows_x86_64_msvc"-version = "0.52.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"diff --git a/data_server/Cargo.toml b/data_server/Cargo.tomldeleted file mode 100644index 94f2f95..0000000--- a/data_server/Cargo.toml+++ /dev/null@@ -1,19 +0,0 @@-[package]-name = "data_server"-version = "0.1.0"-edition = "2021"--# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html--[dependencies]-bytes = "1.5.0"-clap = { version = "4.4.11", features = ["derive"] }-env_logger = "0.10.1"-log = "0.4.20"-prost = "0.12.3"-rand = "0.8.5"-tokio = { version = "1.34.0", features = ["full"] }-tonic = "0.10.2"--[build-dependencies]-tonic-build = "0.10.2"diff --git a/data_server/build.rs b/data_server/build.rsdeleted file mode 100644index c094d10..0000000--- a/data_server/build.rs+++ /dev/null@@ -1,4 +0,0 @@-fn main() -> Result<(), Box<dyn std::error::Error>> {- tonic_build::compile_protos("../fish_speech/datasets/protos/text-data.proto")?;- Ok(())-}diff --git a/data_server/src/main.rs b/data_server/src/main.rsdeleted file mode 100644index ef3e682..0000000--- a/data_server/src/main.rs+++ /dev/null@@ -1,280 +0,0 @@-use clap::Parser;-use log::info;-use prost::Message;-use rand::seq::SliceRandom;-use rand::{thread_rng, Rng};-use std::fs::{self, File};-use std::io::{self, BufReader, Read, Result as IoResult};-use std::path::{Path, PathBuf};-use std::{env, vec};-use tonic::{transport::Server, Request, Response, Status};--pub mod text_data {- tonic::include_proto!("text_data");-}--use text_data::{- data_service_server::{DataService, DataServiceServer},- SampleDataRequest, SampledData, Sentence, TextData,-};--#[derive(Default, Debug, Clone)]-pub struct RSSentence {- text: String,- phones: Vec<String>,- semantics: Vec<Vec<u32>>,-}--#[derive(Default, Debug, Clone)]-pub struct RSTextData {- source: String,- name: String,- languages: Vec<String>,- sentences: Vec<RSSentence>,-}--#[derive(Default)]-pub struct MyDataService {- groups: Vec<RSTextData>,- causual_sampling: bool,- weights: Vec<f32>,-}--async fn read_pb_stream<R: Read>(mut reader: BufReader<R>) -> io::Result<Vec<RSTextData>> {- let mut text_data_list = Vec::new();- let mut index = 0;- let mut total_vq_frames = 0;-- loop {- let mut size_buf = [0u8; 4];- match reader.read_exact(&mut size_buf) {- Ok(()) => (),- Err(ref e) if e.kind() == io::ErrorKind::UnexpectedEof => break, // End of file- Err(e) => return Err(e),- }-- let size = u32::from_le_bytes(size_buf) as usize;-- let mut message_buf = vec![0u8; size];- reader.read_exact(&mut message_buf)?;-- let text_data = TextData::decode(&message_buf[..])- .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;-- text_data.sentences.iter().for_each(|sentence| {- total_vq_frames += sentence.semantics[0].values.len();- });-- text_data_list.push(RSTextData {- source: text_data.source.clone(),- name: text_data.name.clone(),- languages: text_data.languages.clone(),- sentences: text_data- .sentences- .iter()- .map(|sentence| RSSentence {- text: sentence.text.clone(),- phones: sentence.phones.clone(),- semantics: sentence- .semantics- .iter()- .map(|semantics| semantics.values.iter().map(|v| *v).collect())- .collect(),- })- .collect(),- });-- index += 1;-- if index % 1000 == 0 {- info!("Loaded {} groups, total vq frames: {}", index, total_vq_frames);- }- }-- info!("Worker loaded {} groups, total vq frames: {}", index, total_vq_frames);-- Ok(text_data_list)-}--fn list_files(path: PathBuf) -> Vec<String> {- let mut files = Vec::new();-- for entry in fs::read_dir(path).unwrap() {- let entry = entry.unwrap();- let path = entry.path();- if path.is_dir() {- files.extend(list_files(path));- } else {- files.push(path.to_str().unwrap().to_string());- }- }-- files-}--impl MyDataService {- pub async fn new(files: Vec<String>, causual_sampling: bool) -> IoResult<Self> {- let mut groups = Vec::new();- let mut weights = Vec::new();- let mut handles = Vec::new();- let start_time = std::time::Instant::now();-- // Expand files if some are directories- let mut new_files = Vec::new();- for filename in files.iter() {- let path = Path::new(filename);- if path.is_dir() {- // run recursively on all files in the directory- for entry in list_files(path.to_path_buf()) {- if entry.ends_with(".protos") {- new_files.push(entry);- }- }- } else {- new_files.push(filename.clone());- }- }-- log::info!("Loading files: {:?}", new_files.len());-- for filename in new_files.iter() {- // Tokio launch multiple tasks to read the files in parallel- let copied_filename = filename.clone();- let handle = tokio::spawn(async move {- let file = File::open(copied_filename)?;- let reader = BufReader::new(file);- read_pb_stream(reader).await- });- handles.push(handle);- }-- for handle in handles {- let text_data_list = handle.await??;- for text_data in text_data_list {- weights.push(text_data.sentences.len() as f32);- groups.push(text_data);- }- }-- log::info!("All workers finished, total groups: {}, used time: {:?}", groups.len(), start_time.elapsed());-- Ok(MyDataService {- groups,- weights,- causual_sampling,- })- }-}--fn cast_rs_sentence(sentence: &RSSentence) -> Sentence {- Sentence {- text: sentence.text.clone(),- phones: sentence.phones.clone(),- semantics: sentence- .semantics- .iter()- .map(|semantics| text_data::Semantics {- values: semantics.iter().map(|v| *v as u32).collect(),- })- .collect(),- }-}--#[tonic::async_trait]-impl DataService for MyDataService {- async fn sample_data(- &self,- request: Request<SampleDataRequest>,- ) -> Result<Response<SampledData>, Status> {- let mut num_samples = request.into_inner().num_samples as usize;- let mut rng = thread_rng();-- let group = self- .groups- .choose_weighted(&mut rng, |item| item.sentences.len() as f32);-- if group.is_err() {- return Err(Status::internal("Failed to select a group"));- }-- let group = group.unwrap();-- if self.causual_sampling {- if num_samples > group.sentences.len() {- num_samples = group.sentences.len();- }-- // Random number between 0 and group.sentences.len() - num_samples- let max = group.sentences.len() - num_samples;- if max <= 0 {- return Ok(Response::new(SampledData {- name: group.name.clone(),- source: group.source.clone(),- samples: (&group.sentences).into_iter().map(cast_rs_sentence).collect(),- }));- }-- let start = rng.gen_range(0..max);- Ok(Response::new(SampledData {- name: group.name.clone(),- source: group.source.clone(),- samples: group.sentences[start..start + num_samples].iter().map(cast_rs_sentence).collect(),- }))- } else {- let sentences_ref = group.sentences.choose_multiple(&mut rng, num_samples);-- let sentences: Vec<Sentence> = sentences_ref- .into_iter()- .map(cast_rs_sentence)- .collect();-- Ok(Response::new(SampledData {- name: group.name.clone(),- source: group.source.clone(),- samples: sentences,- }))- }- }-}--/// My Data Service Application-#[derive(Parser, Debug)]-#[clap(author, version, about, long_about = None)]-struct Args {- /// Files to process- #[clap(short, long, value_name = "FILE", required = true)]- files: Vec<String>,-- /// Causual sampling- #[clap(short, long, default_value = "true")]- causal: bool,-- /// Address to bind to- #[clap(short, long, default_value = "127.0.0.1:50051")]- address: String,-}--#[tokio::main(flavor = "multi_thread", worker_threads = 8)]-async fn main() -> Result<(), Box<dyn std::error::Error>> {- if env::var("RUST_LOG").is_err() {- env::set_var("RUST_LOG", "info")- }-- env_logger::init();-- // Parse command-line arguments- let args = Args::parse();- info!("Arguments: {:?}", args);-- let addr = args.address.parse()?;- let data_service = MyDataService::new(args.files, args.causal).await?;-- info!("Starting server at {}", addr);-- Server::builder()- .add_service(DataServiceServer::new(data_service))- .serve(addr)- .await?;-- Ok(())-}diff --git a/fish_speech/configs/model/dual_ar_8_codebook_small.yaml b/fish_speech/configs/model/dual_ar_8_codebook_small.yamlindex 2d7868e..19d9e0e 100644--- a/fish_speech/configs/model/dual_ar_8_codebook_small.yaml+++ b/fish_speech/configs/model/dual_ar_8_codebook_small.yaml@@ -9,5 +9,5 @@ config:dim: 768rope_base: 10000norm_eps: 1e-5- num_codebooks: 8 # input/output codebook size- codebook_size: 264 # codebook size 256 + 2 special tokens+ num_codebooks: 2 # input/output codebook size+ codebook_size: 1032 # codebook size 1024 + 2 special tokensdiff --git a/fish_speech/configs/text2semantic_pretrain.yaml b/fish_speech/configs/text2semantic_pretrain.yamlindex 3f793dc..f4a15af 100644--- a/fish_speech/configs/text2semantic_pretrain.yaml+++ b/fish_speech/configs/text2semantic_pretrain.yaml@@ -23,6 +23,9 @@ tokenizer:# Dataset Configurationtrain_dataset:_target_: fish_speech.datasets.text.AutoAugTextDataset+ proto_files:+ - data/protos/gigaspeech+ - data/protos/libirtts-traintokenizer: ${tokenizer}max_length: ${max_length}num_codebooks: ${model.model.config.num_codebooks}@@ -32,6 +35,8 @@ train_dataset:val_dataset:_target_: fish_speech.datasets.text.AutoAugTextDataset+ proto_files:+ - data/protos/libirtts-testtokenizer: ${tokenizer}max_length: ${max_length}num_codebooks: ${model.model.config.num_codebooks}diff --git a/fish_speech/configs/vqgan_pretrain.yaml b/fish_speech/configs/vqgan_pretrain.yamlindex 1fb568e..9538921 100644--- a/fish_speech/configs/vqgan_pretrain.yaml+++ b/fish_speech/configs/vqgan_pretrain.yaml@@ -24,7 +24,7 @@ train_dataset:_target_: torch.utils.data.ConcatDatasetdatasets:- _target_: fish_speech.datasets.vqgan.VQGANDataset- filelist: data/libri-light-pack/filelist.txt+ filelist: data/gigaspeech/vq_train_filelist.txtsample_rate: ${sample_rate}hop_length: ${hop_length}slice_frames: 512@@ -53,7 +53,7 @@ model:_target_: fish_speech.models.vqgan.VQGANsampling_rate: ${sample_rate}- weight_adv: 0.5+ weight_adv: 0.2weight_vq: 1.0weight_mel: 1.0freeze_encoder: false@@ -69,7 +69,7 @@ model:_target_: fish_speech.models.vqgan.modules.fsq.DownsampleFiniteScalarQuantizeinput_dim: 768n_codebooks: 1- n_groups: 8+ n_groups: 2levels: [8, 5, 5, 5]decoder:diff --git a/fish_speech/datasets/text.py b/fish_speech/datasets/text.pyindex b99130f..8ac6ed9 100644--- a/fish_speech/datasets/text.py+++ b/fish_speech/datasets/text.py@@ -1,6 +1,7 @@import randomfrom dataclasses import dataclassfrom itertools import chain+from pathlib import Pathfrom random import Randomfrom typing import Optional, Union@@ -185,7 +186,7 @@ class AutoAugTextDataset(IterableDataset):def __init__(self,- server: str = "localhost:50051",+ proto_files: list[str],seed: int = 42,phones_prob: float = 0.3,repetition_prob: float = 0.0,@@ -193,8 +194,6 @@ class AutoAugTextDataset(IterableDataset):max_length: int = 1024,tokenizer: AutoTokenizer = None,use_speaker: bool = True,- use_data_server: bool = True,- proto_files: Optional[list[str]] = None,causual: bool = True,mix_text_phone_prob: float = 0.5,use_negative_samples: bool = False,@@ -202,7 +201,7 @@ class AutoAugTextDataset(IterableDataset):):"""Args:- server: gRPC server address+ proto_files: proto buf files if using local dataseed: random seedphones_prob: probability to use phonesrepetition_prob: probability to repeat the same sentence@@ -210,8 +209,6 @@ class AutoAugTextDataset(IterableDataset):max_length: max length of the texttokenizer: tokenizeruse_speaker: include speaker information in the prompt- use_data_server: use data server or local data- proto_files: proto buf files if using local datacausual: use causual sampling when using local data, disable will lead to random samplingmix_text_phone_prob: probability to mix text and phones, if this is 0, then it will be pure text or pure phonesuse_negative_samples: generate negative samples@@ -232,7 +229,6 @@ class AutoAugTextDataset(IterableDataset):self.repetition_prob = repetition_probself.interactive_prob = interactive_probself.use_speaker = use_speaker- self.use_data_server = use_data_serverself.proto_files = proto_filesself.causual = causualself.mix_text_phone_prob = mix_text_phone_prob@@ -240,25 +236,41 @@ class AutoAugTextDataset(IterableDataset):self.num_codebooks = num_codebooksself.semantic_token_id = self.tokenizer.convert_tokens_to_ids("<s:0>")-- if use_data_server is True:- self.channel = grpc.insecure_channel(server)- self.stub = DataServiceStub(self.channel)- else:- self.init_mock_data_server()+ self.groups = Nonedef init_mock_data_server(self):+ if self.groups is not None:+ return++ # Expand the proto files+ expanded_proto_files = []+ for filename in self.proto_files:+ for i in braceexpand(filename):+ i = Path(i)+ if i.is_file():+ expanded_proto_files.append(i)+ elif i.is_dir():+ expanded_proto_files.extend(i.rglob("*.proto"))+ expanded_proto_files.extend(i.rglob("*.protos"))+ else:+ raise ValueError(f"{i} is not a file or directory")++ expanded_proto_files = sorted(expanded_proto_files)+ Random(self.seed).shuffle(expanded_proto_files)+self.groups = []+ shard_proto_files = split_by_rank_worker(expanded_proto_files)+ log.info(+ f"Reading {len(shard_proto_files)} / {len(expanded_proto_files)} files"+ )+count = 0- for filename in self.proto_files:+ for filename in shard_proto_files:with open(filename, "rb") as f:for text_data in read_pb_stream(f):self.groups.append(text_data)count += 1- if count % 1000 == 0:- log.info(f"Read {count} groups of data")-log.info(f"Read total {count} groups of data")# Shuffle the lines@@ -280,6 +292,7 @@ class AutoAugTextDataset(IterableDataset):)else:sentence = clean_text(sentence)+ sentence = " ".join([f"<s:{i:d}>" for i in sentence.encode("utf-8")])tokens = self.tokenizer.encode(f"{sentence}",@@ -290,13 +303,12 @@ class AutoAugTextDataset(IterableDataset):return sentence, len(tokens)def sample_data(self):+ if self.groups is None:+ self.init_mock_data_server()+# Shuffle unique lines, estimate that each sample is at least 20 tokensnum_samples = self.max_length // 20- if self.use_data_server:- request = SampleDataRequest(num_samples=num_samples)- return self.stub.SampleData(request)-# choice group based on their number of samplesgroup = random.choices(self.groups, weights=[len(i.sentences) for i in self.groups], k=1@@ -691,14 +703,6 @@ if __name__ == "__main__":num_codebooks=4,)- # ds = AutoAugTextDataset(- # tokenizer=AutoTokenizer.from_pretrained("fishaudio/speech-lm-v1"),- # use_speaker=True,- # interactive_prob=1.0,- # use_data_server=False,- # proto_files=["data/wenet-speech.protos"],- # )-dm = TextDataModule(train_dataset=ds,val_dataset=ds,diff --git a/tools/llama/generate.py b/tools/llama/generate.pyindex 4498404..59450b8 100644--- a/tools/llama/generate.py+++ b/tools/llama/generate.py@@ -290,6 +290,7 @@ def encode_tokens(string = " ".join(prompt)else:string = clean_text(string)+ string = " ".join([f"<s:{i:d}>" for i in string.encode("utf-8")])if speaker is not None:string = f"[SPK: {speaker}] {string}"@@ -413,8 +414,8 @@ def split_text(text, min_length):@click.option("--num-samples", type=int, default=1)@click.option("--max-new-tokens", type=int, default=0)@click.option("--top-k", type=int, default=None)[email protected]("--top-p", type=float, default=0.9)[email protected]("--repetition-penalty", type=float, default=1.2)[email protected]("--top-p", type=float, default=0.7)[email protected]("--repetition-penalty", type=float, default=1.5)@click.option("--temperature", type=float, default=0.7)@click.option("--checkpoint-path",diff --git a/tools/vqgan/inference.py b/tools/vqgan/inference.pyindex 4d4cb65..6c73ef5 100644--- a/tools/vqgan/inference.py+++ b/tools/vqgan/inference.py@@ -95,7 +95,7 @@ def main(input_path, output_path, config_name, checkpoint_path):# Save audiofake_audio = fake_audios[0, 0].cpu().numpy().astype(np.float32)- sf.write("fake.wav", fake_audio, model.sampling_rate)+ sf.write(output_path, fake_audio, model.sampling_rate)logger.info(f"Saved audio to {output_path}")