Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

open server

eugene96c355fbf75d
diff --git a/Cargo.lock b/Cargo.lock
index 956fe0d03..1f75496d3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -325,6 +325,28 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "async-stream"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "async-trait"
version = "0.1.89"
@@ -336,6 +358,12 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "atomic"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba"
+
[[package]]
name = "atomic"
version = "0.6.1"
@@ -667,6 +695,12 @@ dependencies = [
"sysinfo",
]
+[[package]]
+name = "binascii"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"
+
[[package]]
name = "bindgen"
version = "0.72.1"
@@ -1117,8 +1151,12 @@ dependencies = [
"clap",
"comfy-table",
"indicatif",
+ "rocket",
+ "serde",
"serde_json",
"tokio",
+ "tokio-stream",
+ "uuid",
"uzu",
]
@@ -1156,7 +1194,7 @@ dependencies = [
"serde_json",
"syn 2.0.117",
"toml 1.1.2+spec-1.1.0",
- "toml_edit",
+ "toml_edit 0.25.12+spec-1.1.0",
]
[[package]]
@@ -1304,6 +1342,17 @@ dependencies = [
"unicode-segmentation",
]
+[[package]]
+name = "cookie"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
+dependencies = [
+ "percent-encoding",
+ "time",
+ "version_check",
+]
+
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -1371,7 +1420,7 @@ dependencies = [
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
- "windows",
+ "windows 0.62.2",
]
[[package]]
@@ -1829,6 +1878,39 @@ dependencies = [
"unicode-xid",
]
+[[package]]
+name = "devise"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1d90b0c4c777a2cad215e3c7be59ac7c15adf45cf76317009b7d096d46f651d"
+dependencies = [
+ "devise_codegen",
+ "devise_core",
+]
+
+[[package]]
+name = "devise_codegen"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71b28680d8be17a570a2334922518be6adc3f58ecc880cbb404eaeb8624fd867"
+dependencies = [
+ "devise_core",
+ "quote",
+]
+
+[[package]]
+name = "devise_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b035a542cf7abf01f2e3c4d5a7acbaebfefe120ae4efc7bde3df98186e4b8af7"
+dependencies = [
+ "bitflags 2.11.1",
+ "proc-macro2",
+ "proc-macro2-diagnostics",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "digest"
version = "0.10.7"
@@ -2153,6 +2235,20 @@ dependencies = [
"simd-adler32",
]
+[[package]]
+name = "figment"
+version = "0.10.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3"
+dependencies = [
+ "atomic 0.6.1",
+ "pear",
+ "serde",
+ "toml 0.8.23",
+ "uncased",
+ "version_check",
+]
+
[[package]]
name = "filedescriptor"
version = "0.8.3"
@@ -2358,6 +2454,19 @@ dependencies = [
"parking_lot",
]
+[[package]]
+name = "generator"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e"
+dependencies = [
+ "cc",
+ "libc",
+ "log",
+ "rustversion",
+ "windows 0.48.0",
+]
+
[[package]]
name = "generic-array"
version = "0.14.7"
@@ -2477,6 +2586,25 @@ version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be136d9dacc2a13cc70bb6c8f902b414fb2641f8db1314637c6b7933411a8f82"
+[[package]]
+name = "h2"
+version = "0.3.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d"
+dependencies = [
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "http 0.2.12",
+ "indexmap 2.14.0",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
[[package]]
name = "h2"
version = "0.4.14"
@@ -2488,7 +2616,7 @@ dependencies = [
"fnv",
"futures-core",
"futures-sink",
- "http",
+ "http 1.4.1",
"indexmap 2.14.0",
"slab",
"tokio",
@@ -2603,6 +2731,17 @@ version = "3.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
+[[package]]
+name = "http"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
[[package]]
name = "http"
version = "1.4.1"
@@ -2613,6 +2752,17 @@ dependencies = [
"itoa",
]
+[[package]]
+name = "http-body"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
+dependencies = [
+ "bytes",
+ "http 0.2.12",
+ "pin-project-lite",
+]
+
[[package]]
name = "http-body"
version = "1.0.1"
@@ -2620,7 +2770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
- "http",
+ "http 1.4.1",
]
[[package]]
@@ -2631,8 +2781,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
dependencies = [
"bytes",
"futures-core",
- "http",
- "http-body",
+ "http 1.4.1",
+ "http-body 1.0.1",
"pin-project-lite",
]
@@ -2654,6 +2804,30 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+[[package]]
+name = "hyper"
+version = "0.14.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2 0.3.27",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2 0.5.10",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
[[package]]
name = "hyper"
version = "1.10.1"
@@ -2664,9 +2838,9 @@ dependencies = [
"bytes",
"futures-channel",
"futures-core",
- "h2",
- "http",
- "http-body",
+ "h2 0.4.14",
+ "http 1.4.1",
+ "http-body 1.0.1",
"httparse",
"httpdate",
"itoa",
@@ -2682,8 +2856,8 @@ version = "0.27.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
dependencies = [
- "http",
- "hyper",
+ "http 1.4.1",
+ "hyper 1.10.1",
"hyper-util",
"rustls",
"rustls-native-certs",
@@ -2703,14 +2877,14 @@ dependencies = [
"bytes",
"futures-channel",
"futures-util",
- "http",
- "http-body",
- "hyper",
+ "http 1.4.1",
+ "http-body 1.0.1",
+ "hyper 1.10.1",
"ipnet",
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2",
+ "socket2 0.6.4",
"system-configuration",
"tokio",
"tower-service",
@@ -2964,6 +3138,12 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "inlinable_string"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"
+
[[package]]
name = "instability"
version = "0.3.12"
@@ -3051,6 +3231,17 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "is-terminal"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
+dependencies = [
+ "hermit-abi 0.5.2",
+ "libc",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "is_close"
version = "0.1.3"
@@ -3461,6 +3652,21 @@ dependencies = [
"logos-codegen",
]
+[[package]]
+name = "loom"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"
+dependencies = [
+ "cfg-if",
+ "generator",
+ "scoped-tls",
+ "serde",
+ "serde_json",
+ "tracing",
+ "tracing-subscriber",
+]
+
[[package]]
name = "loop9"
version = "0.1.5"
@@ -3711,7 +3917,7 @@ dependencies = [
"bitflags 2.11.1",
"bytesize",
"crc32c",
- "http",
+ "http 1.4.1",
"httparse",
"serde_json",
"shoji",
@@ -3778,6 +3984,25 @@ dependencies = [
"objc2-io-surface",
]
+[[package]]
+name = "multer"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
+dependencies = [
+ "bytes",
+ "encoding_rs",
+ "futures-util",
+ "http 1.4.1",
+ "httparse",
+ "memchr",
+ "mime",
+ "spin",
+ "tokio",
+ "tokio-util",
+ "version_check",
+]
+
[[package]]
name = "nagare"
version = "0.5.8"
@@ -4432,6 +4657,29 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
+[[package]]
+name = "pear"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467"
+dependencies = [
+ "inlinable_string",
+ "pear_codegen",
+ "yansi",
+]
+
+[[package]]
+name = "pear_codegen"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147"
+dependencies = [
+ "proc-macro2",
+ "proc-macro2-diagnostics",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -4653,7 +4901,7 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
dependencies = [
- "toml_edit",
+ "toml_edit 0.25.12+spec-1.1.0",
]
[[package]]
@@ -4689,6 +4937,19 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "proc-macro2-diagnostics"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+ "version_check",
+ "yansi",
+]
+
[[package]]
name = "proc-macros"
version = "0.5.8"
@@ -4889,7 +5150,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.2",
"rustls",
- "socket2",
+ "socket2 0.6.4",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -4927,7 +5188,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
- "socket2",
+ "socket2 0.6.4",
"tracing",
"windows-sys 0.60.2",
]
@@ -5385,10 +5646,10 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
- "http",
- "http-body",
+ "http 1.4.1",
+ "http-body 1.0.1",
"http-body-util",
- "hyper",
+ "hyper 1.10.1",
"hyper-rustls",
"hyper-util",
"js-sys",
@@ -5430,11 +5691,11 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
- "h2",
- "http",
- "http-body",
+ "h2 0.4.14",
+ "http 1.4.1",
+ "http-body 1.0.1",
"http-body-util",
- "hyper",
+ "hyper 1.10.1",
"hyper-rustls",
"hyper-util",
"js-sys",
@@ -5527,6 +5788,88 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "rocket"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a516907296a31df7dc04310e7043b61d71954d703b603cc6867a026d7e72d73f"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "atomic 0.5.3",
+ "binascii",
+ "bytes",
+ "either",
+ "figment",
+ "futures",
+ "indexmap 2.14.0",
+ "log",
+ "memchr",
+ "multer",
+ "num_cpus",
+ "parking_lot",
+ "pin-project-lite",
+ "rand 0.8.6",
+ "ref-cast",
+ "rocket_codegen",
+ "rocket_http",
+ "serde",
+ "serde_json",
+ "state",
+ "tempfile",
+ "time",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "ubyte",
+ "version_check",
+ "yansi",
+]
+
+[[package]]
+name = "rocket_codegen"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "575d32d7ec1a9770108c879fc7c47815a80073f96ca07ff9525a94fcede1dd46"
+dependencies = [
+ "devise",
+ "glob",
+ "indexmap 2.14.0",
+ "proc-macro2",
+ "quote",
+ "rocket_http",
+ "syn 2.0.117",
+ "unicode-xid",
+ "version_check",
+]
+
+[[package]]
+name = "rocket_http"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e274915a20ee3065f611c044bd63c40757396b6dbc057d6046aec27f14f882b9"
+dependencies = [
+ "cookie",
+ "either",
+ "futures",
+ "http 0.2.12",
+ "hyper 0.14.32",
+ "indexmap 2.14.0",
+ "log",
+ "memchr",
+ "pear",
+ "percent-encoding",
+ "pin-project-lite",
+ "ref-cast",
+ "serde",
+ "smallvec",
+ "stable-pattern",
+ "state",
+ "time",
+ "tokio",
+ "uncased",
+]
+
[[package]]
name = "rodio"
version = "0.22.2"
@@ -5828,6 +6171,12 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "scoped-tls"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
+
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -5980,6 +6329,15 @@ dependencies = [
"zmij",
]
+[[package]]
+name = "serde_spanned"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "serde_spanned"
version = "1.1.1"
@@ -6216,6 +6574,16 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
+[[package]]
+name = "socket2"
+version = "0.5.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
[[package]]
name = "socket2"
version = "0.6.4"
@@ -6226,6 +6594,12 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
[[package]]
name = "spm_precompiled"
version = "0.1.4"
@@ -6238,12 +6612,30 @@ dependencies = [
"unicode-segmentation",
]
+[[package]]
+name = "stable-pattern"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+[[package]]
+name = "state"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b8c4a4445d81357df8b1a650d0d0d6fbbbfe99d064aa5e02f3e4022061476d8"
+dependencies = [
+ "loom",
+]
+
[[package]]
name = "static_assertions"
version = "1.1.0"
@@ -6414,7 +6806,7 @@ dependencies = [
"ntapi",
"objc2-core-foundation",
"objc2-io-kit",
- "windows",
+ "windows 0.62.2",
]
[[package]]
@@ -6783,7 +7175,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
- "socket2",
+ "socket2 0.6.4",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -6835,6 +7227,18 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "toml"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
+dependencies = [
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_edit 0.22.27",
+]
+
[[package]]
name = "toml"
version = "0.9.12+spec-1.1.0"
@@ -6843,7 +7247,7 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
dependencies = [
"indexmap 2.14.0",
"serde_core",
- "serde_spanned",
+ "serde_spanned 1.1.1",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"toml_writer",
@@ -6858,13 +7262,22 @@ checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee"
dependencies = [
"indexmap 2.14.0",
"serde_core",
- "serde_spanned",
+ "serde_spanned 1.1.1",
"toml_datetime 1.1.1+spec-1.1.0",
"toml_parser",
"toml_writer",
"winnow 1.0.3",
]
+[[package]]
+name = "toml_datetime"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "toml_datetime"
version = "0.7.5+spec-1.1.0"
@@ -6883,6 +7296,20 @@ dependencies = [
"serde_core",
]
+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap 2.14.0",
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_write",
+ "winnow 0.7.15",
+]
+
[[package]]
name = "toml_edit"
version = "0.25.12+spec-1.1.0"
@@ -6905,6 +7332,12 @@ dependencies = [
"winnow 1.0.3",
]
+[[package]]
+name = "toml_write"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
+
[[package]]
name = "toml_writer"
version = "1.1.1+spec-1.1.0"
@@ -6935,8 +7368,8 @@ dependencies = [
"bitflags 2.11.1",
"bytes",
"futures-util",
- "http",
- "http-body",
+ "http 1.4.1",
+ "http-body 1.0.1",
"pin-project-lite",
"tower",
"tower-layer",
@@ -7080,6 +7513,15 @@ version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
+[[package]]
+name = "ubyte"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "ucd-trie"
version = "0.1.7"
@@ -7092,6 +7534,16 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+[[package]]
+name = "uncased"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697"
+dependencies = [
+ "serde",
+ "version_check",
+]
+
[[package]]
name = "unic-char-property"
version = "0.9.0"
@@ -7396,7 +7848,7 @@ version = "1.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
dependencies = [
- "atomic",
+ "atomic 0.6.1",
"getrandom 0.4.2",
"js-sys",
"serde_core",
@@ -7841,6 +8293,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+[[package]]
+name = "windows"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
[[package]]
name = "windows"
version = "0.62.2"
@@ -8013,6 +8474,21 @@ dependencies = [
"windows_x86_64_msvc 0.42.2",
]
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
[[package]]
name = "windows-targets"
version = "0.52.6"
@@ -8061,6 +8537,12 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
@@ -8079,6 +8561,12 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
@@ -8097,6 +8585,12 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
@@ -8127,6 +8621,12 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
@@ -8145,6 +8645,12 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
@@ -8163,6 +8669,12 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
@@ -8181,6 +8693,12 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
@@ -8221,9 +8739,9 @@ dependencies = [
"base64 0.22.1",
"deadpool",
"futures",
- "http",
+ "http 1.4.1",
"http-body-util",
- "hyper",
+ "hyper 1.10.1",
"hyper-util",
"log",
"once_cell",
@@ -8361,6 +8879,15 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448"
+[[package]]
+name = "yansi"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
+dependencies = [
+ "is-terminal",
+]
+
[[package]]
name = "yoke"
version = "0.8.2"
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
index b6d58de13..278350560 100644
--- a/crates/cli/Cargo.toml
+++ b/crates/cli/Cargo.toml
@@ -28,7 +28,11 @@ anyhow.workspace = true
clap.workspace = true
comfy-table.workspace = true
indicatif.workspace = true
+serde.workspace = true
serde_json.workspace = true
+rocket.workspace = true
+uuid.workspace = true
+tokio-stream.workspace = true
[lints]
workspace = true
diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs
index 0f40e8625..0a4d19213 100644
--- a/crates/cli/src/main.rs
+++ b/crates/cli/src/main.rs
@@ -2,6 +2,7 @@ use anyhow::Result;
use clap::{Parser, Subcommand};
mod bench;
+mod server;
#[derive(Parser)]
#[command(name = "cli", bin_name = "cli")]
@@ -20,6 +21,14 @@ enum Commands {
task_path: String,
output_path: String,
},
+ Server {
+ #[arg(long, value_name = "MODEL")]
+ model: String,
+ #[arg(long, default_value_t = 8000)]
+ port: u16,
+ #[arg(long, default_value = "127.0.0.1")]
+ host: String,
+ },
}
#[tokio::main]
@@ -32,6 +41,11 @@ async fn main() -> Result<()> {
task_path,
output_path,
}) => bench::run_bench(model_path, task_path, output_path)?,
+ Some(Commands::Server {
+ model,
+ port,
+ host,
+ }) => server::run_server(model, host, port).await?,
None => run_interactive(cli.model).await?,
}
diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rs
new file mode 100644
index 000000000..44a3ff4fa
--- /dev/null
+++ b/crates/cli/src/server/chat_completions.rs
@@ -0,0 +1,405 @@
+use std::{
+ pin::Pin,
+ str::FromStr,
+ sync::Arc,
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+use rocket::{
+ Request, State,
+ futures::Stream,
+ post,
+ response::{
+ Responder,
+ stream::{Event, EventStream},
+ },
+ serde::json::Json,
+};
+use serde::{Deserialize, Serialize};
+use tokio::sync::{Mutex, mpsc};
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use uuid::Uuid;
+use uzu::{
+ session::chat::{ChatSession, ChatSessionStreamChunk},
+ types::{
+ basic::SamplingMethod,
+ session::chat::{ChatMessage, ChatReplyConfig, ChatReplyFinishReason, ChatReplyStats, ChatRole},
+ },
+};
+
+use crate::server::ServerState;
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct OaiMessage {
+ pub role: String,
+ #[serde(default)]
+ pub content: String,
+}
+
+#[derive(Deserialize)]
+pub struct ChatCompletionRequest {
+ pub messages: Vec<OaiMessage>,
+ #[serde(default)]
+ pub stream: Option<bool>,
+ #[serde(default)]
+ pub max_tokens: Option<u32>,
+ #[serde(default)]
+ pub max_completion_tokens: Option<u32>,
+ #[serde(default)]
+ pub temperature: Option<f64>,
+ #[serde(default)]
+ pub top_p: Option<f64>,
+ #[serde(default)]
+ pub top_k: Option<i64>,
+ #[serde(default)]
+ #[allow(dead_code)]
+ pub model: Option<String>,
+}
+
+#[derive(Serialize, Clone)]
+pub struct ChatCompletionChoice {
+ pub index: u32,
+ pub message: OaiMessage,
+ pub finish_reason: String,
+}
+
+#[derive(Serialize, Clone, Default)]
+pub struct ChatCompletionUsage {
+ pub prompt_tokens: u32,
+ pub completion_tokens: u32,
+ pub total_tokens: u32,
+}
+
+#[derive(Serialize, Clone)]
+pub struct ChatCompletionResponse {
+ pub id: String,
+ pub object: String,
+ pub created: i64,
+ pub model: String,
+ pub choices: Vec<ChatCompletionChoice>,
+ pub usage: ChatCompletionUsage,
+}
+
+#[derive(Serialize)]
+struct StreamDelta {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ role: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ content: Option<String>,
+}
+
+#[derive(Serialize)]
+struct StreamChoice {
+ index: u32,
+ delta: StreamDelta,
+ finish_reason: Option<String>,
+}
+
+#[derive(Serialize)]
+struct ChatCompletionChunk {
+ id: String,
+ object: String,
+ created: i64,
+ model: String,
+ choices: Vec<StreamChoice>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ usage: Option<ChatCompletionUsage>,
+}
+
+pub enum ChatCompletionResult {
+ Json(Json<ChatCompletionResponse>),
+ Stream(EventStream<Pin<Box<dyn Stream<Item = Event> + Send>>>),
+}
+
+impl<'r> Responder<'r, 'r> for ChatCompletionResult {
+ fn respond_to(
+ self,
+ request: &'r Request<'_>,
+ ) -> rocket::response::Result<'r> {
+ match self {
+ ChatCompletionResult::Json(json) => json.respond_to(request),
+ ChatCompletionResult::Stream(stream) => stream.respond_to(request),
+ }
+ }
+}
+
+fn now_unix() -> i64 {
+ SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)
+}
+
+fn to_chat_messages(messages: &[OaiMessage]) -> Vec<ChatMessage> {
+ messages
+ .iter()
+ .map(|message| {
+ let role = ChatRole::from_str(&message.role).unwrap_or(ChatRole::User {});
+ ChatMessage::for_role(role).with_text(message.content.clone())
+ })
+ .collect()
+}
+
+fn build_reply_config(request: &ChatCompletionRequest) -> ChatReplyConfig {
+ let token_limit = request.max_completion_tokens.or(request.max_tokens);
+ let config = ChatReplyConfig::default().with_token_limit(token_limit);
+
+ if request.temperature.is_some_and(|temperature| temperature <= 0.0) {
+ return config.with_sampling_method(SamplingMethod::Greedy {});
+ }
+
+ if request.temperature.is_some() || request.top_p.is_some() || request.top_k.is_some() {
+ return config.with_sampling_method(SamplingMethod::Stochastic {
+ temperature: request.temperature,
+ top_k: request.top_k,
+ top_p: request.top_p,
+ min_p: None,
+ });
+ }
+
+ config
+}
+
+fn map_finish_reason(finish_reason: &ChatReplyFinishReason) -> String {
+ match finish_reason {
+ ChatReplyFinishReason::Stop | ChatReplyFinishReason::Cancelled => "stop",
+ ChatReplyFinishReason::Length | ChatReplyFinishReason::ContextLimitReached => "length",
+ ChatReplyFinishReason::ToolCalls => "tool_calls",
+ ChatReplyFinishReason::Rejected => "content_filter",
+ }
+ .to_string()
+}
+
+fn usage_from_stats(stats: &ChatReplyStats) -> ChatCompletionUsage {
+ let prompt_tokens = stats.tokens_count_input.unwrap_or(0);
+ let completion_tokens = stats.tokens_count_output.unwrap_or(0);
+ ChatCompletionUsage {
+ prompt_tokens,
+ completion_tokens,
+ total_tokens: prompt_tokens + completion_tokens,
+ }
+}
+
+fn error_response(
+ id: String,
+ model: String,
+ created: i64,
+ message: &str,
+) -> ChatCompletionResponse {
+ ChatCompletionResponse {
+ id,
+ object: "chat.completion".to_string(),
+ created,
+ model,
+ choices: vec![ChatCompletionChoice {
+ index: 0,
+ message: OaiMessage {
+ role: "assistant".to_string(),
+ content: format!("Error: {message}"),
+ },
+ finish_reason: "stop".to_string(),
+ }],
+ usage: ChatCompletionUsage::default(),
+ }
+}
+
+fn chunk_json(
+ id: &str,
+ model: &str,
+ created: i64,
+ delta: StreamDelta,
+ finish_reason: Option<String>,
+ usage: Option<ChatCompletionUsage>,
+) -> String {
+ let chunk = ChatCompletionChunk {
+ id: id.to_string(),
+ object: "chat.completion.chunk".to_string(),
+ created,
+ model: model.to_string(),
+ choices: vec![StreamChoice {
+ index: 0,
+ delta,
+ finish_reason,
+ }],
+ usage,
+ };
+ serde_json::to_string(&chunk).unwrap_or_default()
+}
+
+async fn run_blocking(
+ session: Arc<Mutex<ChatSession>>,
+ messages: Vec<ChatMessage>,
+ config: ChatReplyConfig,
+ id: String,
+ model: String,
+ created: i64,
+) -> ChatCompletionResponse {
+ let session = session.lock().await;
+ if let Err(error) = session.reset().await {
+ return error_response(id, model, created, &error.to_string());
+ }
+
+ match session.reply(messages, config).await {
+ Ok(replies) => match replies.last() {
+ Some(reply) => ChatCompletionResponse {
+ id,
+ object: "chat.completion".to_string(),
+ created,
+ model,
+ choices: vec![ChatCompletionChoice {
+ index: 0,
+ message: OaiMessage {
+ role: "assistant".to_string(),
+ content: reply.message.text().unwrap_or_default(),
+ },
+ finish_reason: reply
+ .finish_reason
+ .as_ref()
+ .map(map_finish_reason)
+ .unwrap_or_else(|| "stop".to_string()),
+ }],
+ usage: usage_from_stats(&reply.stats),
+ },
+ None => error_response(id, model, created, "No response generated"),
+ },
+ Err(error) => error_response(id, model, created, &error.to_string()),
+ }
+}
+
+async fn run_stream(
+ session: Arc<Mutex<ChatSession>>,
+ messages: Vec<ChatMessage>,
+ config: ChatReplyConfig,
+ id: String,
+ model: String,
+ created: i64,
+ sender: mpsc::UnboundedSender<Event>,
+) {
+ let session = session.lock().await;
+ if let Err(error) = session.reset().await {
+ let _ = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ role: None,
+ content: Some(format!("Error: {error}")),
+ },
+ Some("stop".to_string()),
+ None,
+ )));
+ let _ = sender.send(Event::data("[DONE]"));
+ return;
+ }
+
+ let _ = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ role: Some("assistant".to_string()),
+ content: None,
+ },
+ None,
+ None,
+ )));
+
+ let stream = session.reply_with_stream(messages, config).await;
+ let mut emitted = 0usize;
+ let mut finish_reason = "stop".to_string();
+ let mut usage = ChatCompletionUsage::default();
+ let mut errored = false;
+
+ while let Some(chunk) = stream.next().await {
+ match chunk {
+ ChatSessionStreamChunk::Replies {
+ replies,
+ } => {
+ let Some(reply) = replies.last() else {
+ continue;
+ };
+ let text = reply.message.text().unwrap_or_default();
+ let start = (emitted..=text.len()).find(|&index| text.is_char_boundary(index)).unwrap_or(text.len());
+ if text.len() > start {
+ let delta = text[start..].to_string();
+ emitted = text.len();
+ let sent = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ role: None,
+ content: Some(delta),
+ },
+ None,
+ None,
+ )));
+ if sent.is_err() {
+ return;
+ }
+ }
+ if let Some(reason) = &reply.finish_reason {
+ finish_reason = map_finish_reason(reason);
+ }
+ usage = usage_from_stats(&reply.stats);
+ },
+ ChatSessionStreamChunk::Error {
+ error,
+ } => {
+ errored = true;
+ let _ = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ role: None,
+ content: Some(format!("Error: {error}")),
+ },
+ Some("stop".to_string()),
+ None,
+ )));
+ break;
+ },
+ }
+ }
+
+ if !errored {
+ let _ = sender.send(Event::data(chunk_json(
+ &id,
+ &model,
+ created,
+ StreamDelta {
+ role: None,
+ content: None,
+ },
+ Some(finish_reason),
+ Some(usage),
+ )));
+ }
+ let _ = sender.send(Event::data("[DONE]"));
+}
+
+#[allow(private_interfaces)]
+#[post("/chat/completions", format = "json", data = "<request>")]
+pub async fn handle_chat_completions(
+ request: Json<ChatCompletionRequest>,
+ state: &State<ServerState>,
+) -> ChatCompletionResult {
+ let request = request.into_inner();
+ let id = format!("chatcmpl-{}", Uuid::new_v4().simple());
+ let created = now_unix();
+ let model = state.model_name.clone();
+ let is_stream = request.stream.unwrap_or(false);
+
+ let messages = to_chat_messages(&request.messages);
+ let config = build_reply_config(&request);
+
+ if is_stream {
+ let session = Arc::clone(&state.session);
+ let (sender, receiver) = mpsc::unbounded_channel::<Event>();
+ rocket::tokio::spawn(run_stream(session, messages, config, id, model, created, sender));
+ let body: Pin<Box<dyn Stream<Item = Event> + Send>> = Box::pin(UnboundedReceiverStream::new(receiver));
+ ChatCompletionResult::Stream(EventStream::from(body))
+ } else {
+ let session = Arc::clone(&state.session);
+ let response = run_blocking(session, messages, config, id, model, created).await;
+ ChatCompletionResult::Json(Json(response))
+ }
+}
diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs
new file mode 100644
index 000000000..3b2e32f04
--- /dev/null
+++ b/crates/cli/src/server/mod.rs
@@ -0,0 +1,9 @@
+pub mod chat_completions;
+pub mod models;
+pub mod runner;
+pub mod state;
+
+pub use chat_completions::handle_chat_completions;
+pub use models::handle_models;
+pub use runner::run_server;
+pub use state::ServerState;
diff --git a/crates/cli/src/server/models.rs b/crates/cli/src/server/models.rs
new file mode 100644
index 000000000..1db75c6b3
--- /dev/null
+++ b/crates/cli/src/server/models.rs
@@ -0,0 +1,31 @@
+use rocket::{State, get, serde::json::Json};
+use serde::{Deserialize, Serialize};
+
+use crate::server::ServerState;
+
+#[derive(Serialize, Deserialize)]
+pub struct Model {
+ pub id: String,
+ pub object: String,
+ pub created: u64,
+ pub owned_by: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ModelsResponse {
+ pub object: String,
+ pub data: Vec<Model>,
+}
+
+#[get("/models")]
+pub fn handle_models(state: &State<ServerState>) -> Json<ModelsResponse> {
+ Json(ModelsResponse {
+ object: "list".to_string(),
+ data: vec![Model {
+ id: state.model_name.clone(),
+ object: "model".to_string(),
+ created: 0,
+ owned_by: "uzu".to_string(),
+ }],
+ })
+}
diff --git a/crates/cli/src/server/runner.rs b/crates/cli/src/server/runner.rs
new file mode 100644
index 000000000..333d90996
--- /dev/null
+++ b/crates/cli/src/server/runner.rs
@@ -0,0 +1,69 @@
+use std::{net::IpAddr, sync::Arc, time::Duration};
+
+use anyhow::{Context, Result, bail};
+use indicatif::{ProgressBar, ProgressStyle};
+use rocket::{Config, config::LogLevel, routes};
+use tokio::sync::Mutex;
+use uzu::{
+ engine::{Engine, EngineConfig},
+ types::session::chat::ChatConfig,
+};
+
+use crate::server::{ServerState, handle_chat_completions, handle_models};
+
+pub async fn run_server(
+ model: String,
+ host: String,
+ port: u16,
+) -> Result<()> {
+ let engine_config = EngineConfig::default().with_application_identifier("com.trymirai.cli".to_string());
+ let engine = Engine::new(engine_config).await.context("Failed to create engine")?;
+
+ let resolved = match engine.model(model.clone()).await? {
+ Some(model) => model,
+ None => engine.model_by_path(model.clone()).await?.with_context(|| format!("Model not found: {model}"))?,
+ };
+
+ let spinner = ProgressBar::new_spinner();
+ spinner.enable_steady_tick(Duration::from_millis(100));
+ spinner.set_style(ProgressStyle::default_spinner().template("{spinner:.green} {msg}").unwrap());
+ spinner.set_message(format!("Preparing model: {}", resolved.identifier));
+ let downloader = engine.download(&resolved).await.context("Failed to start model download")?;
+ while let Some(update) = downloader.next().await {
+ spinner.set_message(format!("Downloading {}: {:.0}%", resolved.identifier, update.progress() * 100.0));
+ }
+ spinner.finish_with_message(format!("Loaded: {}", resolved.identifier));
+
+ let session =
+ engine.chat(resolved.clone(), ChatConfig::default()).await.context("Failed to create chat session")?;
+ let model_name = resolved.identifier.clone();
+ let state = ServerState {
+ model_name: model_name.clone(),
+ session: Arc::new(Mutex::new(session)),
+ };
+
+ let address: IpAddr = host.parse().with_context(|| format!("Invalid host: {host}"))?;
+ let config = Config {
+ port,
+ address,
+ log_level: LogLevel::Off,
+ ..Config::default()
+ };
+
+ println!("🚀 OpenAI-compatible server for model: {model_name}");
+ println!("🌐 Available at: http://{host}:{port}");
+ println!("📝 Endpoints:");
+ println!(" POST /v1/chat/completions (or /chat/completions)");
+ println!(" GET /v1/models (or /models)");
+
+ let rocket = rocket::custom(config)
+ .manage(state)
+ .mount("/", routes![handle_chat_completions, handle_models])
+ .mount("/v1", routes![handle_chat_completions, handle_models]);
+
+ if let Err(error) = rocket.launch().await {
+ bail!("Server error: {error}");
+ }
+
+ Ok(())
+}
diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rs
new file mode 100644
index 000000000..4ce7dced8
--- /dev/null
+++ b/crates/cli/src/server/state.rs
@@ -0,0 +1,9 @@
+use std::sync::Arc;
+
+use tokio::sync::Mutex;
+use uzu::session::chat::ChatSession;
+
+pub struct ServerState {
+ pub model_name: String,
+ pub session: Arc<Mutex<ChatSession>>,
+}