A high-performance inference engine for AI models
open server
eugene96c355fbf75d
diff --git a/Cargo.lock b/Cargo.lockindex 956fe0d03..1f75496d3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -325,6 +325,28 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "async-stream"+version = "0.3.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"+dependencies = [+ "async-stream-impl",+ "futures-core",+ "pin-project-lite",+]++[[package]]+name = "async-stream-impl"+version = "0.3.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.117",+]+[[package]]name = "async-trait"version = "0.1.89"@@ -336,6 +358,12 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "atomic"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba"+[[package]]name = "atomic"version = "0.6.1"@@ -667,6 +695,12 @@ dependencies = ["sysinfo",]+[[package]]+name = "binascii"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"+[[package]]name = "bindgen"version = "0.72.1"@@ -1117,8 +1151,12 @@ dependencies = ["clap","comfy-table","indicatif",+ "rocket",+ "serde","serde_json","tokio",+ "tokio-stream",+ "uuid","uzu",]@@ -1156,7 +1194,7 @@ dependencies = ["serde_json","syn 2.0.117","toml 1.1.2+spec-1.1.0",- "toml_edit",+ "toml_edit 0.25.12+spec-1.1.0",][[package]]@@ -1304,6 +1342,17 @@ dependencies = ["unicode-segmentation",]+[[package]]+name = "cookie"+version = "0.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"+dependencies = [+ "percent-encoding",+ "time",+ "version_check",+]+[[package]]name = "core-foundation"version = "0.9.4"@@ -1371,7 +1420,7 @@ dependencies = ["wasm-bindgen","wasm-bindgen-futures","web-sys",- "windows",+ "windows 0.62.2",][[package]]@@ -1829,6 +1878,39 @@ dependencies = ["unicode-xid",]+[[package]]+name = "devise"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1d90b0c4c777a2cad215e3c7be59ac7c15adf45cf76317009b7d096d46f651d"+dependencies = [+ "devise_codegen",+ "devise_core",+]++[[package]]+name = "devise_codegen"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "71b28680d8be17a570a2334922518be6adc3f58ecc880cbb404eaeb8624fd867"+dependencies = [+ "devise_core",+ "quote",+]++[[package]]+name = "devise_core"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b035a542cf7abf01f2e3c4d5a7acbaebfefe120ae4efc7bde3df98186e4b8af7"+dependencies = [+ "bitflags 2.11.1",+ "proc-macro2",+ "proc-macro2-diagnostics",+ "quote",+ "syn 2.0.117",+]+[[package]]name = "digest"version = "0.10.7"@@ -2153,6 +2235,20 @@ dependencies = ["simd-adler32",]+[[package]]+name = "figment"+version = "0.10.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3"+dependencies = [+ "atomic 0.6.1",+ "pear",+ "serde",+ "toml 0.8.23",+ "uncased",+ "version_check",+]+[[package]]name = "filedescriptor"version = "0.8.3"@@ -2358,6 +2454,19 @@ dependencies = ["parking_lot",]+[[package]]+name = "generator"+version = "0.7.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e"+dependencies = [+ "cc",+ "libc",+ "log",+ "rustversion",+ "windows 0.48.0",+]+[[package]]name = "generic-array"version = "0.14.7"@@ -2477,6 +2586,25 @@ version = "0.14.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "be136d9dacc2a13cc70bb6c8f902b414fb2641f8db1314637c6b7933411a8f82"+[[package]]+name = "h2"+version = "0.3.27"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d"+dependencies = [+ "bytes",+ "fnv",+ "futures-core",+ "futures-sink",+ "futures-util",+ "http 0.2.12",+ "indexmap 2.14.0",+ "slab",+ "tokio",+ "tokio-util",+ "tracing",+]+[[package]]name = "h2"version = "0.4.14"@@ -2488,7 +2616,7 @@ dependencies = ["fnv","futures-core","futures-sink",- "http",+ "http 1.4.1","indexmap 2.14.0","slab","tokio",@@ -2603,6 +2731,17 @@ version = "3.5.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"+[[package]]+name = "http"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"+dependencies = [+ "bytes",+ "fnv",+ "itoa",+]+[[package]]name = "http"version = "1.4.1"@@ -2613,6 +2752,17 @@ dependencies = ["itoa",]+[[package]]+name = "http-body"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"+dependencies = [+ "bytes",+ "http 0.2.12",+ "pin-project-lite",+]+[[package]]name = "http-body"version = "1.0.1"@@ -2620,7 +2770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"dependencies = ["bytes",- "http",+ "http 1.4.1",][[package]]@@ -2631,8 +2781,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"dependencies = ["bytes","futures-core",- "http",- "http-body",+ "http 1.4.1",+ "http-body 1.0.1","pin-project-lite",]@@ -2654,6 +2804,30 @@ version = "2.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"+[[package]]+name = "hyper"+version = "0.14.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7"+dependencies = [+ "bytes",+ "futures-channel",+ "futures-core",+ "futures-util",+ "h2 0.3.27",+ "http 0.2.12",+ "http-body 0.4.6",+ "httparse",+ "httpdate",+ "itoa",+ "pin-project-lite",+ "socket2 0.5.10",+ "tokio",+ "tower-service",+ "tracing",+ "want",+]+[[package]]name = "hyper"version = "1.10.1"@@ -2664,9 +2838,9 @@ dependencies = ["bytes","futures-channel","futures-core",- "h2",- "http",- "http-body",+ "h2 0.4.14",+ "http 1.4.1",+ "http-body 1.0.1","httparse","httpdate","itoa",@@ -2682,8 +2856,8 @@ version = "0.27.9"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"dependencies = [- "http",- "hyper",+ "http 1.4.1",+ "hyper 1.10.1","hyper-util","rustls","rustls-native-certs",@@ -2703,14 +2877,14 @@ dependencies = ["bytes","futures-channel","futures-util",- "http",- "http-body",- "hyper",+ "http 1.4.1",+ "http-body 1.0.1",+ "hyper 1.10.1","ipnet","libc","percent-encoding","pin-project-lite",- "socket2",+ "socket2 0.6.4","system-configuration","tokio","tower-service",@@ -2964,6 +3138,12 @@ dependencies = ["rustversion",]+[[package]]+name = "inlinable_string"+version = "0.1.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"+[[package]]name = "instability"version = "0.3.12"@@ -3051,6 +3231,17 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "is-terminal"+version = "0.4.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"+dependencies = [+ "hermit-abi 0.5.2",+ "libc",+ "windows-sys 0.61.2",+]+[[package]]name = "is_close"version = "0.1.3"@@ -3461,6 +3652,21 @@ dependencies = ["logos-codegen",]+[[package]]+name = "loom"+version = "0.5.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"+dependencies = [+ "cfg-if",+ "generator",+ "scoped-tls",+ "serde",+ "serde_json",+ "tracing",+ "tracing-subscriber",+]+[[package]]name = "loop9"version = "0.1.5"@@ -3711,7 +3917,7 @@ dependencies = ["bitflags 2.11.1","bytesize","crc32c",- "http",+ "http 1.4.1","httparse","serde_json","shoji",@@ -3778,6 +3984,25 @@ dependencies = ["objc2-io-surface",]+[[package]]+name = "multer"+version = "3.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"+dependencies = [+ "bytes",+ "encoding_rs",+ "futures-util",+ "http 1.4.1",+ "httparse",+ "memchr",+ "mime",+ "spin",+ "tokio",+ "tokio-util",+ "version_check",+]+[[package]]name = "nagare"version = "0.5.8"@@ -4432,6 +4657,29 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"+[[package]]+name = "pear"+version = "0.2.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467"+dependencies = [+ "inlinable_string",+ "pear_codegen",+ "yansi",+]++[[package]]+name = "pear_codegen"+version = "0.2.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147"+dependencies = [+ "proc-macro2",+ "proc-macro2-diagnostics",+ "quote",+ "syn 2.0.117",+]+[[package]]name = "percent-encoding"version = "2.3.2"@@ -4653,7 +4901,7 @@ version = "3.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"dependencies = [- "toml_edit",+ "toml_edit 0.25.12+spec-1.1.0",][[package]]@@ -4689,6 +4937,19 @@ dependencies = ["unicode-ident",]+[[package]]+name = "proc-macro2-diagnostics"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.117",+ "version_check",+ "yansi",+]+[[package]]name = "proc-macros"version = "0.5.8"@@ -4889,7 +5150,7 @@ dependencies = ["quinn-udp","rustc-hash 2.1.2","rustls",- "socket2",+ "socket2 0.6.4","thiserror 2.0.18","tokio","tracing",@@ -4927,7 +5188,7 @@ dependencies = ["cfg_aliases","libc","once_cell",- "socket2",+ "socket2 0.6.4","tracing","windows-sys 0.60.2",]@@ -5385,10 +5646,10 @@ dependencies = ["futures-channel","futures-core","futures-util",- "http",- "http-body",+ "http 1.4.1",+ "http-body 1.0.1","http-body-util",- "hyper",+ "hyper 1.10.1","hyper-rustls","hyper-util","js-sys",@@ -5430,11 +5691,11 @@ dependencies = ["futures-channel","futures-core","futures-util",- "h2",- "http",- "http-body",+ "h2 0.4.14",+ "http 1.4.1",+ "http-body 1.0.1","http-body-util",- "hyper",+ "hyper 1.10.1","hyper-rustls","hyper-util","js-sys",@@ -5527,6 +5788,88 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "rocket"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a516907296a31df7dc04310e7043b61d71954d703b603cc6867a026d7e72d73f"+dependencies = [+ "async-stream",+ "async-trait",+ "atomic 0.5.3",+ "binascii",+ "bytes",+ "either",+ "figment",+ "futures",+ "indexmap 2.14.0",+ "log",+ "memchr",+ "multer",+ "num_cpus",+ "parking_lot",+ "pin-project-lite",+ "rand 0.8.6",+ "ref-cast",+ "rocket_codegen",+ "rocket_http",+ "serde",+ "serde_json",+ "state",+ "tempfile",+ "time",+ "tokio",+ "tokio-stream",+ "tokio-util",+ "ubyte",+ "version_check",+ "yansi",+]++[[package]]+name = "rocket_codegen"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "575d32d7ec1a9770108c879fc7c47815a80073f96ca07ff9525a94fcede1dd46"+dependencies = [+ "devise",+ "glob",+ "indexmap 2.14.0",+ "proc-macro2",+ "quote",+ "rocket_http",+ "syn 2.0.117",+ "unicode-xid",+ "version_check",+]++[[package]]+name = "rocket_http"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e274915a20ee3065f611c044bd63c40757396b6dbc057d6046aec27f14f882b9"+dependencies = [+ "cookie",+ "either",+ "futures",+ "http 0.2.12",+ "hyper 0.14.32",+ "indexmap 2.14.0",+ "log",+ "memchr",+ "pear",+ "percent-encoding",+ "pin-project-lite",+ "ref-cast",+ "serde",+ "smallvec",+ "stable-pattern",+ "state",+ "time",+ "tokio",+ "uncased",+]+[[package]]name = "rodio"version = "0.22.2"@@ -5828,6 +6171,12 @@ dependencies = ["syn 2.0.117",]+[[package]]+name = "scoped-tls"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"+[[package]]name = "scopeguard"version = "1.2.0"@@ -5980,6 +6329,15 @@ dependencies = ["zmij",]+[[package]]+name = "serde_spanned"+version = "0.6.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"+dependencies = [+ "serde",+]+[[package]]name = "serde_spanned"version = "1.1.1"@@ -6216,6 +6574,16 @@ version = "0.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"+[[package]]+name = "socket2"+version = "0.5.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"+dependencies = [+ "libc",+ "windows-sys 0.52.0",+]+[[package]]name = "socket2"version = "0.6.4"@@ -6226,6 +6594,12 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "spin"+version = "0.9.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"+[[package]]name = "spm_precompiled"version = "0.1.4"@@ -6238,12 +6612,30 @@ dependencies = ["unicode-segmentation",]+[[package]]+name = "stable-pattern"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045"+dependencies = [+ "memchr",+]+[[package]]name = "stable_deref_trait"version = "1.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"+[[package]]+name = "state"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b8c4a4445d81357df8b1a650d0d0d6fbbbfe99d064aa5e02f3e4022061476d8"+dependencies = [+ "loom",+]+[[package]]name = "static_assertions"version = "1.1.0"@@ -6414,7 +6806,7 @@ dependencies = ["ntapi","objc2-core-foundation","objc2-io-kit",- "windows",+ "windows 0.62.2",][[package]]@@ -6783,7 +7175,7 @@ dependencies = ["parking_lot","pin-project-lite","signal-hook-registry",- "socket2",+ "socket2 0.6.4","tokio-macros","windows-sys 0.61.2",]@@ -6835,6 +7227,18 @@ dependencies = ["tokio",]+[[package]]+name = "toml"+version = "0.8.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"+dependencies = [+ "serde",+ "serde_spanned 0.6.9",+ "toml_datetime 0.6.11",+ "toml_edit 0.22.27",+]+[[package]]name = "toml"version = "0.9.12+spec-1.1.0"@@ -6843,7 +7247,7 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"dependencies = ["indexmap 2.14.0","serde_core",- "serde_spanned",+ "serde_spanned 1.1.1","toml_datetime 0.7.5+spec-1.1.0","toml_parser","toml_writer",@@ -6858,13 +7262,22 @@ checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee"dependencies = ["indexmap 2.14.0","serde_core",- "serde_spanned",+ "serde_spanned 1.1.1","toml_datetime 1.1.1+spec-1.1.0","toml_parser","toml_writer","winnow 1.0.3",]+[[package]]+name = "toml_datetime"+version = "0.6.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"+dependencies = [+ "serde",+]+[[package]]name = "toml_datetime"version = "0.7.5+spec-1.1.0"@@ -6883,6 +7296,20 @@ dependencies = ["serde_core",]+[[package]]+name = "toml_edit"+version = "0.22.27"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"+dependencies = [+ "indexmap 2.14.0",+ "serde",+ "serde_spanned 0.6.9",+ "toml_datetime 0.6.11",+ "toml_write",+ "winnow 0.7.15",+]+[[package]]name = "toml_edit"version = "0.25.12+spec-1.1.0"@@ -6905,6 +7332,12 @@ dependencies = ["winnow 1.0.3",]+[[package]]+name = "toml_write"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"+[[package]]name = "toml_writer"version = "1.1.1+spec-1.1.0"@@ -6935,8 +7368,8 @@ dependencies = ["bitflags 2.11.1","bytes","futures-util",- "http",- "http-body",+ "http 1.4.1",+ "http-body 1.0.1","pin-project-lite","tower","tower-layer",@@ -7080,6 +7513,15 @@ version = "1.20.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"+[[package]]+name = "ubyte"+version = "0.10.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"+dependencies = [+ "serde",+]+[[package]]name = "ucd-trie"version = "0.1.7"@@ -7092,6 +7534,16 @@ version = "0.1.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"+[[package]]+name = "uncased"+version = "0.9.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697"+dependencies = [+ "serde",+ "version_check",+]+[[package]]name = "unic-char-property"version = "0.9.0"@@ -7396,7 +7848,7 @@ version = "1.23.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"dependencies = [- "atomic",+ "atomic 0.6.1","getrandom 0.4.2","js-sys","serde_core",@@ -7841,6 +8293,15 @@ version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"+[[package]]+name = "windows"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"+dependencies = [+ "windows-targets 0.48.5",+]+[[package]]name = "windows"version = "0.62.2"@@ -8013,6 +8474,21 @@ dependencies = ["windows_x86_64_msvc 0.42.2",]+[[package]]+name = "windows-targets"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"+dependencies = [+ "windows_aarch64_gnullvm 0.48.5",+ "windows_aarch64_msvc 0.48.5",+ "windows_i686_gnu 0.48.5",+ "windows_i686_msvc 0.48.5",+ "windows_x86_64_gnu 0.48.5",+ "windows_x86_64_gnullvm 0.48.5",+ "windows_x86_64_msvc 0.48.5",+]+[[package]]name = "windows-targets"version = "0.52.6"@@ -8061,6 +8537,12 @@ version = "0.42.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"+[[package]]+name = "windows_aarch64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"+[[package]]name = "windows_aarch64_gnullvm"version = "0.52.6"@@ -8079,6 +8561,12 @@ version = "0.42.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"+[[package]]+name = "windows_aarch64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"+[[package]]name = "windows_aarch64_msvc"version = "0.52.6"@@ -8097,6 +8585,12 @@ version = "0.42.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"+[[package]]+name = "windows_i686_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"+[[package]]name = "windows_i686_gnu"version = "0.52.6"@@ -8127,6 +8621,12 @@ version = "0.42.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"+[[package]]+name = "windows_i686_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"+[[package]]name = "windows_i686_msvc"version = "0.52.6"@@ -8145,6 +8645,12 @@ version = "0.42.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"+[[package]]+name = "windows_x86_64_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"+[[package]]name = "windows_x86_64_gnu"version = "0.52.6"@@ -8163,6 +8669,12 @@ version = "0.42.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"+[[package]]+name = "windows_x86_64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"+[[package]]name = "windows_x86_64_gnullvm"version = "0.52.6"@@ -8181,6 +8693,12 @@ version = "0.42.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"+[[package]]+name = "windows_x86_64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"+[[package]]name = "windows_x86_64_msvc"version = "0.52.6"@@ -8221,9 +8739,9 @@ dependencies = ["base64 0.22.1","deadpool","futures",- "http",+ "http 1.4.1","http-body-util",- "hyper",+ "hyper 1.10.1","hyper-util","log","once_cell",@@ -8361,6 +8879,15 @@ version = "0.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448"+[[package]]+name = "yansi"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"+dependencies = [+ "is-terminal",+]+[[package]]name = "yoke"version = "0.8.2"diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.tomlindex b6d58de13..278350560 100644--- a/crates/cli/Cargo.toml+++ b/crates/cli/Cargo.toml@@ -28,7 +28,11 @@ anyhow.workspace = trueclap.workspace = truecomfy-table.workspace = trueindicatif.workspace = true+serde.workspace = trueserde_json.workspace = true+rocket.workspace = true+uuid.workspace = true+tokio-stream.workspace = true[lints]workspace = truediff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rsindex 0f40e8625..0a4d19213 100644--- a/crates/cli/src/main.rs+++ b/crates/cli/src/main.rs@@ -2,6 +2,7 @@ use anyhow::Result;use clap::{Parser, Subcommand};mod bench;+mod server;#[derive(Parser)]#[command(name = "cli", bin_name = "cli")]@@ -20,6 +21,14 @@ enum Commands {task_path: String,output_path: String,},+ Server {+ #[arg(long, value_name = "MODEL")]+ model: String,+ #[arg(long, default_value_t = 8000)]+ port: u16,+ #[arg(long, default_value = "127.0.0.1")]+ host: String,+ },}#[tokio::main]@@ -32,6 +41,11 @@ async fn main() -> Result<()> {task_path,output_path,}) => bench::run_bench(model_path, task_path, output_path)?,+ Some(Commands::Server {+ model,+ port,+ host,+ }) => server::run_server(model, host, port).await?,None => run_interactive(cli.model).await?,}diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rsnew file mode 100644index 000000000..44a3ff4fa--- /dev/null+++ b/crates/cli/src/server/chat_completions.rs@@ -0,0 +1,405 @@+use std::{+ pin::Pin,+ str::FromStr,+ sync::Arc,+ time::{SystemTime, UNIX_EPOCH},+};++use rocket::{+ Request, State,+ futures::Stream,+ post,+ response::{+ Responder,+ stream::{Event, EventStream},+ },+ serde::json::Json,+};+use serde::{Deserialize, Serialize};+use tokio::sync::{Mutex, mpsc};+use tokio_stream::wrappers::UnboundedReceiverStream;+use uuid::Uuid;+use uzu::{+ session::chat::{ChatSession, ChatSessionStreamChunk},+ types::{+ basic::SamplingMethod,+ session::chat::{ChatMessage, ChatReplyConfig, ChatReplyFinishReason, ChatReplyStats, ChatRole},+ },+};++use crate::server::ServerState;++#[derive(Serialize, Deserialize, Clone)]+pub struct OaiMessage {+ pub role: String,+ #[serde(default)]+ pub content: String,+}++#[derive(Deserialize)]+pub struct ChatCompletionRequest {+ pub messages: Vec<OaiMessage>,+ #[serde(default)]+ pub stream: Option<bool>,+ #[serde(default)]+ pub max_tokens: Option<u32>,+ #[serde(default)]+ pub max_completion_tokens: Option<u32>,+ #[serde(default)]+ pub temperature: Option<f64>,+ #[serde(default)]+ pub top_p: Option<f64>,+ #[serde(default)]+ pub top_k: Option<i64>,+ #[serde(default)]+ #[allow(dead_code)]+ pub model: Option<String>,+}++#[derive(Serialize, Clone)]+pub struct ChatCompletionChoice {+ pub index: u32,+ pub message: OaiMessage,+ pub finish_reason: String,+}++#[derive(Serialize, Clone, Default)]+pub struct ChatCompletionUsage {+ pub prompt_tokens: u32,+ pub completion_tokens: u32,+ pub total_tokens: u32,+}++#[derive(Serialize, Clone)]+pub struct ChatCompletionResponse {+ pub id: String,+ pub object: String,+ pub created: i64,+ pub model: String,+ pub choices: Vec<ChatCompletionChoice>,+ pub usage: ChatCompletionUsage,+}++#[derive(Serialize)]+struct StreamDelta {+ #[serde(skip_serializing_if = "Option::is_none")]+ role: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ content: Option<String>,+}++#[derive(Serialize)]+struct StreamChoice {+ index: u32,+ delta: StreamDelta,+ finish_reason: Option<String>,+}++#[derive(Serialize)]+struct ChatCompletionChunk {+ id: String,+ object: String,+ created: i64,+ model: String,+ choices: Vec<StreamChoice>,+ #[serde(skip_serializing_if = "Option::is_none")]+ usage: Option<ChatCompletionUsage>,+}++pub enum ChatCompletionResult {+ Json(Json<ChatCompletionResponse>),+ Stream(EventStream<Pin<Box<dyn Stream<Item = Event> + Send>>>),+}++impl<'r> Responder<'r, 'r> for ChatCompletionResult {+ fn respond_to(+ self,+ request: &'r Request<'_>,+ ) -> rocket::response::Result<'r> {+ match self {+ ChatCompletionResult::Json(json) => json.respond_to(request),+ ChatCompletionResult::Stream(stream) => stream.respond_to(request),+ }+ }+}++fn now_unix() -> i64 {+ SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)+}++fn to_chat_messages(messages: &[OaiMessage]) -> Vec<ChatMessage> {+ messages+ .iter()+ .map(|message| {+ let role = ChatRole::from_str(&message.role).unwrap_or(ChatRole::User {});+ ChatMessage::for_role(role).with_text(message.content.clone())+ })+ .collect()+}++fn build_reply_config(request: &ChatCompletionRequest) -> ChatReplyConfig {+ let token_limit = request.max_completion_tokens.or(request.max_tokens);+ let config = ChatReplyConfig::default().with_token_limit(token_limit);++ if request.temperature.is_some_and(|temperature| temperature <= 0.0) {+ return config.with_sampling_method(SamplingMethod::Greedy {});+ }++ if request.temperature.is_some() || request.top_p.is_some() || request.top_k.is_some() {+ return config.with_sampling_method(SamplingMethod::Stochastic {+ temperature: request.temperature,+ top_k: request.top_k,+ top_p: request.top_p,+ min_p: None,+ });+ }++ config+}++fn map_finish_reason(finish_reason: &ChatReplyFinishReason) -> String {+ match finish_reason {+ ChatReplyFinishReason::Stop | ChatReplyFinishReason::Cancelled => "stop",+ ChatReplyFinishReason::Length | ChatReplyFinishReason::ContextLimitReached => "length",+ ChatReplyFinishReason::ToolCalls => "tool_calls",+ ChatReplyFinishReason::Rejected => "content_filter",+ }+ .to_string()+}++fn usage_from_stats(stats: &ChatReplyStats) -> ChatCompletionUsage {+ let prompt_tokens = stats.tokens_count_input.unwrap_or(0);+ let completion_tokens = stats.tokens_count_output.unwrap_or(0);+ ChatCompletionUsage {+ prompt_tokens,+ completion_tokens,+ total_tokens: prompt_tokens + completion_tokens,+ }+}++fn error_response(+ id: String,+ model: String,+ created: i64,+ message: &str,+) -> ChatCompletionResponse {+ ChatCompletionResponse {+ id,+ object: "chat.completion".to_string(),+ created,+ model,+ choices: vec![ChatCompletionChoice {+ index: 0,+ message: OaiMessage {+ role: "assistant".to_string(),+ content: format!("Error: {message}"),+ },+ finish_reason: "stop".to_string(),+ }],+ usage: ChatCompletionUsage::default(),+ }+}++fn chunk_json(+ id: &str,+ model: &str,+ created: i64,+ delta: StreamDelta,+ finish_reason: Option<String>,+ usage: Option<ChatCompletionUsage>,+) -> String {+ let chunk = ChatCompletionChunk {+ id: id.to_string(),+ object: "chat.completion.chunk".to_string(),+ created,+ model: model.to_string(),+ choices: vec![StreamChoice {+ index: 0,+ delta,+ finish_reason,+ }],+ usage,+ };+ serde_json::to_string(&chunk).unwrap_or_default()+}++async fn run_blocking(+ session: Arc<Mutex<ChatSession>>,+ messages: Vec<ChatMessage>,+ config: ChatReplyConfig,+ id: String,+ model: String,+ created: i64,+) -> ChatCompletionResponse {+ let session = session.lock().await;+ if let Err(error) = session.reset().await {+ return error_response(id, model, created, &error.to_string());+ }++ match session.reply(messages, config).await {+ Ok(replies) => match replies.last() {+ Some(reply) => ChatCompletionResponse {+ id,+ object: "chat.completion".to_string(),+ created,+ model,+ choices: vec![ChatCompletionChoice {+ index: 0,+ message: OaiMessage {+ role: "assistant".to_string(),+ content: reply.message.text().unwrap_or_default(),+ },+ finish_reason: reply+ .finish_reason+ .as_ref()+ .map(map_finish_reason)+ .unwrap_or_else(|| "stop".to_string()),+ }],+ usage: usage_from_stats(&reply.stats),+ },+ None => error_response(id, model, created, "No response generated"),+ },+ Err(error) => error_response(id, model, created, &error.to_string()),+ }+}++async fn run_stream(+ session: Arc<Mutex<ChatSession>>,+ messages: Vec<ChatMessage>,+ config: ChatReplyConfig,+ id: String,+ model: String,+ created: i64,+ sender: mpsc::UnboundedSender<Event>,+) {+ let session = session.lock().await;+ if let Err(error) = session.reset().await {+ let _ = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ role: None,+ content: Some(format!("Error: {error}")),+ },+ Some("stop".to_string()),+ None,+ )));+ let _ = sender.send(Event::data("[DONE]"));+ return;+ }++ let _ = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ role: Some("assistant".to_string()),+ content: None,+ },+ None,+ None,+ )));++ let stream = session.reply_with_stream(messages, config).await;+ let mut emitted = 0usize;+ let mut finish_reason = "stop".to_string();+ let mut usage = ChatCompletionUsage::default();+ let mut errored = false;++ while let Some(chunk) = stream.next().await {+ match chunk {+ ChatSessionStreamChunk::Replies {+ replies,+ } => {+ let Some(reply) = replies.last() else {+ continue;+ };+ let text = reply.message.text().unwrap_or_default();+ let start = (emitted..=text.len()).find(|&index| text.is_char_boundary(index)).unwrap_or(text.len());+ if text.len() > start {+ let delta = text[start..].to_string();+ emitted = text.len();+ let sent = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ role: None,+ content: Some(delta),+ },+ None,+ None,+ )));+ if sent.is_err() {+ return;+ }+ }+ if let Some(reason) = &reply.finish_reason {+ finish_reason = map_finish_reason(reason);+ }+ usage = usage_from_stats(&reply.stats);+ },+ ChatSessionStreamChunk::Error {+ error,+ } => {+ errored = true;+ let _ = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ role: None,+ content: Some(format!("Error: {error}")),+ },+ Some("stop".to_string()),+ None,+ )));+ break;+ },+ }+ }++ if !errored {+ let _ = sender.send(Event::data(chunk_json(+ &id,+ &model,+ created,+ StreamDelta {+ role: None,+ content: None,+ },+ Some(finish_reason),+ Some(usage),+ )));+ }+ let _ = sender.send(Event::data("[DONE]"));+}++#[allow(private_interfaces)]+#[post("/chat/completions", format = "json", data = "<request>")]+pub async fn handle_chat_completions(+ request: Json<ChatCompletionRequest>,+ state: &State<ServerState>,+) -> ChatCompletionResult {+ let request = request.into_inner();+ let id = format!("chatcmpl-{}", Uuid::new_v4().simple());+ let created = now_unix();+ let model = state.model_name.clone();+ let is_stream = request.stream.unwrap_or(false);++ let messages = to_chat_messages(&request.messages);+ let config = build_reply_config(&request);++ if is_stream {+ let session = Arc::clone(&state.session);+ let (sender, receiver) = mpsc::unbounded_channel::<Event>();+ rocket::tokio::spawn(run_stream(session, messages, config, id, model, created, sender));+ let body: Pin<Box<dyn Stream<Item = Event> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));+ ChatCompletionResult::Stream(EventStream::from(body))+ } else {+ let session = Arc::clone(&state.session);+ let response = run_blocking(session, messages, config, id, model, created).await;+ ChatCompletionResult::Json(Json(response))+ }+}diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rsnew file mode 100644index 000000000..3b2e32f04--- /dev/null+++ b/crates/cli/src/server/mod.rs@@ -0,0 +1,9 @@+pub mod chat_completions;+pub mod models;+pub mod runner;+pub mod state;++pub use chat_completions::handle_chat_completions;+pub use models::handle_models;+pub use runner::run_server;+pub use state::ServerState;diff --git a/crates/cli/src/server/models.rs b/crates/cli/src/server/models.rsnew file mode 100644index 000000000..1db75c6b3--- /dev/null+++ b/crates/cli/src/server/models.rs@@ -0,0 +1,31 @@+use rocket::{State, get, serde::json::Json};+use serde::{Deserialize, Serialize};++use crate::server::ServerState;++#[derive(Serialize, Deserialize)]+pub struct Model {+ pub id: String,+ pub object: String,+ pub created: u64,+ pub owned_by: String,+}++#[derive(Serialize, Deserialize)]+pub struct ModelsResponse {+ pub object: String,+ pub data: Vec<Model>,+}++#[get("/models")]+pub fn handle_models(state: &State<ServerState>) -> Json<ModelsResponse> {+ Json(ModelsResponse {+ object: "list".to_string(),+ data: vec![Model {+ id: state.model_name.clone(),+ object: "model".to_string(),+ created: 0,+ owned_by: "uzu".to_string(),+ }],+ })+}diff --git a/crates/cli/src/server/runner.rs b/crates/cli/src/server/runner.rsnew file mode 100644index 000000000..333d90996--- /dev/null+++ b/crates/cli/src/server/runner.rs@@ -0,0 +1,69 @@+use std::{net::IpAddr, sync::Arc, time::Duration};++use anyhow::{Context, Result, bail};+use indicatif::{ProgressBar, ProgressStyle};+use rocket::{Config, config::LogLevel, routes};+use tokio::sync::Mutex;+use uzu::{+ engine::{Engine, EngineConfig},+ types::session::chat::ChatConfig,+};++use crate::server::{ServerState, handle_chat_completions, handle_models};++pub async fn run_server(+ model: String,+ host: String,+ port: u16,+) -> Result<()> {+ let engine_config = EngineConfig::default().with_application_identifier("com.trymirai.cli".to_string());+ let engine = Engine::new(engine_config).await.context("Failed to create engine")?;++ let resolved = match engine.model(model.clone()).await? {+ Some(model) => model,+ None => engine.model_by_path(model.clone()).await?.with_context(|| format!("Model not found: {model}"))?,+ };++ let spinner = ProgressBar::new_spinner();+ spinner.enable_steady_tick(Duration::from_millis(100));+ spinner.set_style(ProgressStyle::default_spinner().template("{spinner:.green} {msg}").unwrap());+ spinner.set_message(format!("Preparing model: {}", resolved.identifier));+ let downloader = engine.download(&resolved).await.context("Failed to start model download")?;+ while let Some(update) = downloader.next().await {+ spinner.set_message(format!("Downloading {}: {:.0}%", resolved.identifier, update.progress() * 100.0));+ }+ spinner.finish_with_message(format!("Loaded: {}", resolved.identifier));++ let session =+ engine.chat(resolved.clone(), ChatConfig::default()).await.context("Failed to create chat session")?;+ let model_name = resolved.identifier.clone();+ let state = ServerState {+ model_name: model_name.clone(),+ session: Arc::new(Mutex::new(session)),+ };++ let address: IpAddr = host.parse().with_context(|| format!("Invalid host: {host}"))?;+ let config = Config {+ port,+ address,+ log_level: LogLevel::Off,+ ..Config::default()+ };++ println!("🚀 OpenAI-compatible server for model: {model_name}");+ println!("🌐 Available at: http://{host}:{port}");+ println!("📝 Endpoints:");+ println!(" POST /v1/chat/completions (or /chat/completions)");+ println!(" GET /v1/models (or /models)");++ let rocket = rocket::custom(config)+ .manage(state)+ .mount("/", routes![handle_chat_completions, handle_models])+ .mount("/v1", routes![handle_chat_completions, handle_models]);++ if let Err(error) = rocket.launch().await {+ bail!("Server error: {error}");+ }++ Ok(())+}diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rsnew file mode 100644index 000000000..4ce7dced8--- /dev/null+++ b/crates/cli/src/server/state.rs@@ -0,0 +1,9 @@+use std::sync::Arc;++use tokio::sync::Mutex;+use uzu::session::chat::ChatSession;++pub struct ServerState {+ pub model_name: String,+ pub session: Arc<Mutex<ChatSession>>,+}