Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Improve API (#33)

Aleksei Savin43fb2e91a2c7
diff --git a/.gitignore b/.gitignore
index ee2c89ed3..bd22a0711 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,6 @@ target/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
*.DS_Store
+
+# Models
+models/
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index f5cd86c23..4cb11608b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -317,7 +317,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
[[package]]
name = "cli"
-version = "0.1.3"
+version = "0.1.4"
dependencies = [
"clap",
"console 0.16.0",
@@ -2640,7 +2640,7 @@ dependencies = [
[[package]]
name = "uzu"
-version = "0.1.3"
+version = "0.1.4"
dependencies = [
"bitflags 2.9.4",
"bytemuck",
diff --git a/Cargo.toml b/Cargo.toml
index 780e676fe..c69d8505e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,7 +3,7 @@ members = ["crates/cli", "crates/uzu"]
resolver = "3"
[workspace.package]
-version = "0.1.3"
+version = "0.1.4"
edition = "2024"
license = "MIT"
authors = ["Mirai Tech <https://trymirai.com>"]
diff --git a/README.md b/README.md
index f0c165c56..bd05bfcff 100644
--- a/README.md
+++ b/README.md
@@ -35,13 +35,13 @@ uv run lalamo list-models
Then, export the specific one:
```bash
-uv run lalamo convert meta-llama/Llama-3.2-1B-Instruct --precision float16
+uv run lalamo convert meta-llama/Llama-3.2-1B-Instruct
```
Alternatively, you can download a prepared model using the sample script:
```bash
-./scripts/download_test_model.sh $MODEL_PATH
+./scripts/download_test_model.sh
```
### Bindings
@@ -80,32 +80,25 @@ Then, create an inference `Session` with a specific model and configuration:
```rust
use std::path::PathBuf;
-use uzu::{
- backends::metal::sampling_config::SamplingConfig,
- session::{
- session::Session, session_config::SessionConfig,
- session_input::SessionInput, session_output::SessionOutput,
- session_run_config::SessionRunConfig,
- },
+
+use uzu::session::{
+ config::{DecodingConfig, RunConfig},
+ session::Session,
+ types::{Input, Output},
};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let model_path = PathBuf::from("MODEL_PATH");
-
- let mut session = Session::new(model_path.clone())?;
- session.load_with_session_config(SessionConfig::default())?;
-
- let input = SessionInput::Text("Tell about London".to_string());
-
- let tokens_limit = 128;
- let run_config = SessionRunConfig::new_with_sampling_config(
- tokens_limit,
- Some(SamplingConfig::default())
- );
-
- let output = session.run(input, run_config, Some(|_: SessionOutput| {
- return true;
- }))?;
+ let mut session = Session::new(model_path, DecodingConfig::default())?;
+
+ let input = Input::Text(String::from("Tell about London"));
+ let output = session.run(
+ input,
+ RunConfig::default().tokens_limit(128),
+ Some(|_: Output| {
+ return true;
+ }),
+ )?;
println!("{}", output.text);
Ok(())
}
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
index f77ad4f8c..d65e5724f 100644
--- a/crates/cli/Cargo.toml
+++ b/crates/cli/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "cli"
-version = "0.1.3"
+version = "0.1.4"
edition = "2024"
[[bin]]
diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rs
index 147afc884..da117fa2e 100644
--- a/crates/cli/src/handlers/run.rs
+++ b/crates/cli/src/handlers/run.rs
@@ -7,13 +7,14 @@ use console::Style;
use indicatif::{ProgressBar, ProgressStyle};
use inquire::Text;
use uzu::session::{
- session_input::SessionInput, session_output::SessionOutput,
- session_run_config::SessionRunConfig,
+ config::RunConfig,
+ parameter::SamplingPolicy,
+ types::{Input, Output},
};
use crate::server::load_session;
-fn format_output(output: SessionOutput) -> String {
+fn format_output(output: Output) -> String {
let stats = &output.stats;
let tokens_per_second = if let Some(generate_stats) = &stats.generate_stats
{
@@ -72,7 +73,7 @@ pub fn handle_run(
let progress_bar_for_progress = progress_bar.clone();
let is_model_running_for_progress = is_model_running.clone();
- let session_progress = move |output: SessionOutput| {
+ let session_progress = move |output: Output| {
if !is_model_running_for_progress.load(Ordering::SeqCst) {
return false;
}
@@ -90,8 +91,8 @@ pub fn handle_run(
};
let session_output = match session.run(
- SessionInput::Text(input.to_string()),
- SessionRunConfig::new(tokens_limit as u64),
+ Input::Text(input.to_string()),
+ RunConfig::new(tokens_limit as u64, true, SamplingPolicy::Default),
Some(session_progress),
) {
Ok(output) => output,
diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rs
index 38313b8e8..5054dc427 100644
--- a/crates/cli/src/server/chat_completions.rs
+++ b/crates/cli/src/server/chat_completions.rs
@@ -7,13 +7,12 @@ use rocket::{State, post, serde::json::Json};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use uzu::session::{
- session_input::SessionInput,
- session_message::{SessionMessage, SessionMessageRole},
- session_output::{
- SessionOutput, SessionOutputFinishReason, SessionOutputRunStats,
- SessionOutputStats, SessionOutputStepStats, SessionOutputTotalStats,
+ config::RunConfig,
+ parameter::SamplingPolicy,
+ types::{
+ FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats,
+ TotalStats,
},
- session_run_config::SessionRunConfig,
};
use crate::server::SessionState;
@@ -21,7 +20,7 @@ use crate::server::SessionState;
#[derive(Serialize, Deserialize, Clone)]
pub struct ChatMessage {
pub content: String,
- pub role: SessionMessageRole,
+ pub role: Role,
}
#[derive(Serialize, Deserialize)]
@@ -45,7 +44,7 @@ pub struct ChatCompletionResponse {
pub created: i64,
pub model: String,
pub choices: Vec<ChatCompletionChoice>,
- pub stats: SessionOutputStats,
+ pub stats: Stats,
}
#[post("/chat/completions", format = "json", data = "<request>")]
@@ -76,16 +75,17 @@ pub fn handle_chat_completions(
let model_name = state.model_name.clone();
let system_prompt_key = request.system_prompt_key;
let tokens_limit = request.max_completion_tokens.unwrap_or(2048);
- let messages: Vec<SessionMessage> = request
+ let messages: Vec<Message> = request
.messages
.into_iter()
- .map(|m| SessionMessage {
+ .map(|m| Message {
content: m.content,
role: m.role,
})
.collect();
- let input = SessionInput::Messages(messages);
- let run_config = SessionRunConfig::new(tokens_limit);
+ let input = Input::Messages(messages);
+ let run_config =
+ RunConfig::new(tokens_limit, true, SamplingPolicy::Default);
let start_time = std::time::Instant::now();
let mut session = state.session_wrapper.lock();
@@ -116,24 +116,24 @@ pub fn handle_chat_completions(
index: 0,
message: ChatMessage {
content: format!("Error: {}", e),
- role: SessionMessageRole::Assistant,
+ role: Role::Assistant,
},
finish_reason: "error".to_string(),
}],
- stats: SessionOutputStats {
- prefill_stats: SessionOutputStepStats {
+ stats: Stats {
+ prefill_stats: StepStats {
duration: 0.0,
suffix_length: 0,
tokens_count: 0,
tokens_per_second: 0.0,
- model_run: SessionOutputRunStats {
+ model_run: RunStats {
count: 0,
average_duration: 0.0,
},
run: None,
},
generate_stats: None,
- total_stats: SessionOutputTotalStats {
+ total_stats: TotalStats {
duration: 0.0,
tokens_count_input: 0,
tokens_count_output: 0,
@@ -161,24 +161,24 @@ pub fn handle_chat_completions(
index: 0,
message: ChatMessage {
content: format!("Error: {}", e),
- role: SessionMessageRole::Assistant,
+ role: Role::Assistant,
},
finish_reason: "error".to_string(),
}],
- stats: SessionOutputStats {
- prefill_stats: SessionOutputStepStats {
+ stats: Stats {
+ prefill_stats: StepStats {
duration: 0.0,
suffix_length: 0,
tokens_count: 0,
tokens_per_second: 0.0,
- model_run: SessionOutputRunStats {
+ model_run: RunStats {
count: 0,
average_duration: 0.0,
},
run: None,
},
generate_stats: None,
- total_stats: SessionOutputTotalStats {
+ total_stats: TotalStats {
duration: 0.0,
tokens_count_input: 0,
tokens_count_output: 0,
@@ -194,15 +194,14 @@ pub fn handle_chat_completions(
println!("output: {:?}", &output);
- let SessionOutput {
+ let Output {
text,
stats,
finish_reason,
..
} = output;
- let finish_reason_val =
- finish_reason.unwrap_or(SessionOutputFinishReason::Cancelled);
+ let finish_reason_val = finish_reason.unwrap_or(FinishReason::Cancelled);
println!("📤 [{}] Sending response:", id);
println!(" Response length: {} chars", text.len());
@@ -236,7 +235,7 @@ pub fn handle_chat_completions(
index: 0,
message: ChatMessage {
content: text,
- role: SessionMessageRole::Assistant,
+ role: Role::Assistant,
},
finish_reason: format!("{:?}", finish_reason_val).to_lowercase(),
}],
diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rs
index c88f1db23..e92854f43 100644
--- a/crates/cli/src/server/state.rs
+++ b/crates/cli/src/server/state.rs
@@ -8,12 +8,11 @@ use std::{
use console::Style;
use indicatif::{ProgressBar, ProgressStyle};
use uzu::session::{
- session::Session, session_config::SessionConfig,
- session_context::SessionContext,
+ config::DecodingConfig, helpers::Context, session::Session,
};
pub struct ContextCache {
- pub map: HashMap<String, Rc<SessionContext>>,
+ pub map: HashMap<String, Rc<Context>>,
pub order: VecDeque<String>,
pub capacity: usize,
}
@@ -30,8 +29,8 @@ impl ContextCache {
pub fn insert(
&mut self,
key: String,
- context: Rc<SessionContext>,
- ) -> Option<(String, Rc<SessionContext>)> {
+ context: Rc<Context>,
+ ) -> Option<(String, Rc<Context>)> {
if self.map.contains_key(&key) {
return None;
}
@@ -51,7 +50,7 @@ impl ContextCache {
pub fn get(
&self,
key: &str,
- ) -> Option<Rc<SessionContext>> {
+ ) -> Option<Rc<Context>> {
self.map.get(key).cloned()
}
}
@@ -97,12 +96,8 @@ pub fn load_session(model_path: String) -> Session {
);
progress_bar.set_message(model_name.clone());
- let mut session =
- Session::new(model_path_buf).expect("Failed to create session");
-
- session
- .load_with_session_config(SessionConfig::default())
- .expect("Failed to load session");
+ let session = Session::new(model_path_buf, DecodingConfig::default())
+ .expect("Failed to create session");
progress_bar.set_style(
ProgressStyle::default_spinner().template("Loaded: {msg}").unwrap(),
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index c50a238c4..ce96d709c 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "uzu"
-version = "0.1.3"
+version = "0.1.4"
edition = "2024"
build = "build.rs"
diff --git a/crates/uzu/build.rs b/crates/uzu/build.rs
index a937909d7..f3c06e90f 100644
--- a/crates/uzu/build.rs
+++ b/crates/uzu/build.rs
@@ -107,7 +107,6 @@ fn compile_metal_shaders() {
for file in sorted_files {
println!("cargo:rerun-if-changed={}", file.display());
}
- println!("cargo:rerun-if-changed=src/ffi.rs");
println!("cargo:rerun-if-changed=build.rs");
}
diff --git a/crates/uzu/examples/dequantize_qkv_layer0.rs b/crates/uzu/examples/dequantize_qkv_layer0.rs
index 7a06a70ce..79412f7eb 100644
--- a/crates/uzu/examples/dequantize_qkv_layer0.rs
+++ b/crates/uzu/examples/dequantize_qkv_layer0.rs
@@ -23,7 +23,7 @@ use uzu::{
graph::common::{GraphConstructionError, load_constant},
},
parameters::{ParameterLoader, ParameterLoaderError},
- storage::{NSSearchPathDirectory, root_dir},
+ utils::{NSSearchPathDirectory, root_dir},
};
#[derive(Debug, Error)]
diff --git a/crates/uzu/examples/quick_start.rs b/crates/uzu/examples/quick_start.rs
new file mode 100644
index 000000000..809aae564
--- /dev/null
+++ b/crates/uzu/examples/quick_start.rs
@@ -0,0 +1,23 @@
+use std::path::PathBuf;
+
+use uzu::session::{
+ config::{DecodingConfig, RunConfig},
+ session::Session,
+ types::{Input, Output},
+};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+ let model_path = PathBuf::from("MODEL_PATH");
+ let mut session = Session::new(model_path, DecodingConfig::default())?;
+
+ let input = Input::Text(String::from("Tell about London"));
+ let output = session.run(
+ input,
+ RunConfig::default().tokens_limit(128),
+ Some(|_: Output| {
+ return true;
+ }),
+ )?;
+ println!("{}", output.text);
+ Ok(())
+}
diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rs
index 56331e285..7997291da 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs
@@ -11,12 +11,10 @@ use super::{
};
use crate::{
DataType, DeviceContext,
- backends::metal::{
- forward_pass::traces::DecoderActivationTrace,
- sampling_config::SamplingConfig,
- },
+ backends::metal::forward_pass::traces::DecoderActivationTrace,
config::{DecoderConfig, EmbeddingConfig},
parameters::ParameterTree,
+ session::parameter::SamplingMethod,
};
type ArrayCell = RefCell<MetalArray>;
@@ -333,7 +331,7 @@ pub struct ForwardPassState {
/// [suffix_length] - u32 sampling output buffer
pub sampling_output: Option<ArrayCell>,
/// Current sampling configuration for this forward pass
- pub sampling_config: Option<SamplingConfig>,
+ pub sampling_method: Option<SamplingMethod>,
pub traces: Option<Rc<RefCell<DecoderActivationTrace>>>,
}
@@ -473,7 +471,7 @@ impl ForwardPassState {
shared_buffers,
aux_buffers,
sampling_output: Some(sampling_output),
- sampling_config: None,
+ sampling_method: None,
traces,
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
index 708569170..e1dcabe10 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
@@ -8,13 +8,15 @@ use metal::{
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use thiserror::Error;
-use crate::backends::metal::{
- KernelDataType, MTLContext, MTLError,
- forward_pass::{
- ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
+use crate::{
+ backends::metal::{
+ KernelDataType, MTLContext, MTLError,
+ forward_pass::{
+ ForwardPassState,
+ encodable_with_state::{EncodableWithState, EncodingParameters},
+ },
},
- sampling_config::SamplingConfig,
+ session::parameter::SamplingMethod,
};
const BLOCK_SIZE: usize = 1024;
@@ -270,7 +272,7 @@ impl SamplingKernel {
pub fn encode(
&self,
- sampling_config: &SamplingConfig,
+ sampling_method: &SamplingMethod,
logits_buffer: &MTLBuffer,
sampled_tokens_buffer: &MTLBuffer,
batch_size: usize,
@@ -279,7 +281,7 @@ impl SamplingKernel {
) -> Result<(), SamplingError> {
let compute_encoder = command_buffer.new_compute_command_encoder();
self.encode_with_encoder(
- sampling_config,
+ sampling_method,
logits_buffer,
sampled_tokens_buffer,
batch_size,
@@ -292,7 +294,7 @@ impl SamplingKernel {
pub fn encode_with_encoder(
&self,
- sampling_config: &SamplingConfig,
+ sampling_method: &SamplingMethod,
logits_buffer: &MTLBuffer,
sampled_tokens_buffer: &MTLBuffer,
batch_size: usize,
@@ -321,8 +323,8 @@ impl SamplingKernel {
let threads_per_threadgroup = MTLSize::new(BLOCK_SIZE as u64, 1, 1);
- match sampling_config {
- SamplingConfig::Argmax => self.encode_argmax(
+ match sampling_method {
+ SamplingMethod::Greedy => self.encode_argmax(
logits_buffer,
sampled_tokens_buffer,
batch_size_u32,
@@ -331,7 +333,7 @@ impl SamplingKernel {
threads_per_threadgroup,
compute_encoder,
),
- SamplingConfig::TopP {
+ SamplingMethod::TopP {
top_p,
} => self.encode_top_p(
*top_p,
@@ -343,7 +345,7 @@ impl SamplingKernel {
threads_per_threadgroup,
compute_encoder,
),
- SamplingConfig::Categorical {
+ SamplingMethod::Temperature {
temperature,
} => self.encode_categorical(
*temperature,
@@ -801,7 +803,7 @@ impl EncodableWithState for SamplingKernelEncodable {
command_buffer: &MPSCommandBuffer,
parameters: &EncodingParameters,
) {
- let sampling_config = state.sampling_config.unwrap_or_default();
+ let sampling_method = state.sampling_method.unwrap_or_default();
let logits_binding = state
.arrays(&[crate::backends::metal::forward_pass::ArrayId::Logits]);
@@ -829,7 +831,7 @@ impl EncodableWithState for SamplingKernelEncodable {
let root_command_buffer =
command_buffer.root_command_buffer().to_owned();
if let Err(e) = self.kernel.encode(
- &sampling_config,
+ &sampling_method,
unsafe { &logits.mtl_buffer() },
unsafe { &output_buffer_ref.mtl_buffer() },
batch_size,
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index a62eabc78..f6ea05d42 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -28,4 +28,3 @@ pub use media_utils::{
MetalImagePreprocessor,
};
pub use placement_analysis::PlacementAnalysis;
-pub mod sampling_config;
diff --git a/crates/uzu/src/backends/metal/sampling_config.rs b/crates/uzu/src/backends/metal/sampling_config.rs
deleted file mode 100644
index 81662b667..000000000
--- a/crates/uzu/src/backends/metal/sampling_config.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum SamplingConfig {
- Argmax,
- TopP {
- top_p: f32,
- },
- Categorical {
- temperature: f32,
- },
-}
-
-impl SamplingConfig {
- pub fn argmax() -> Self {
- Self::Argmax
- }
-
- pub fn top_p(top_p: f32) -> Self {
- Self::TopP {
- top_p,
- }
- }
-
- pub fn categorical(temperature: f32) -> Self {
- Self::Categorical {
- temperature,
- }
- }
-}
-
-impl Default for SamplingConfig {
- fn default() -> Self {
- Self::Argmax
- }
-}
diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/device/array.rs
similarity index 100%
rename from crates/uzu/src/array.rs
rename to crates/uzu/src/device/array.rs
diff --git a/crates/uzu/src/data_type.rs b/crates/uzu/src/device/data_type.rs
similarity index 100%
rename from crates/uzu/src/data_type.rs
rename to crates/uzu/src/device/data_type.rs
diff --git a/crates/uzu/src/device_context.rs b/crates/uzu/src/device/device_context.rs
similarity index 100%
rename from crates/uzu/src/device_context.rs
rename to crates/uzu/src/device/device_context.rs
diff --git a/crates/uzu/src/device/mod.rs b/crates/uzu/src/device/mod.rs
new file mode 100644
index 000000000..52ae2e87a
--- /dev/null
+++ b/crates/uzu/src/device/mod.rs
@@ -0,0 +1,6 @@
+pub mod array;
+pub use array::*;
+pub mod data_type;
+pub use data_type::*;
+pub mod device_context;
+pub use device_context::*;
diff --git a/crates/uzu/src/generator/config.rs b/crates/uzu/src/generator/config.rs
deleted file mode 100644
index 15756770e..000000000
--- a/crates/uzu/src/generator/config.rs
+++ /dev/null
@@ -1,105 +0,0 @@
-use std::sync::Arc;
-
-use rand::prelude::*;
-use tokenizers::Tokenizer;
-
-use crate::speculators::{
- empty_speculator::EmptySpeculator, speculator::Speculator,
-};
-
-#[derive(Clone)]
-pub struct SpeculatorConfig {
- pub number_of_speculated_tokens: usize,
- pub speculator: Arc<dyn Speculator>,
-}
-
-impl Default for SpeculatorConfig {
- fn default() -> Self {
- Self {
- number_of_speculated_tokens: 0,
- speculator: Arc::new(EmptySpeculator {}),
- }
- }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub enum ContextLength {
- // 8192 is the default max prefix length
- Default,
- // Custom max prefix length
- Custom(u64),
-}
-
-impl Default for ContextLength {
- fn default() -> Self {
- ContextLength::Default
- }
-}
-
-impl ContextLength {
- pub fn get_value(&self) -> u64 {
- match self {
- ContextLength::Default => 8192,
- ContextLength::Custom(length) => *length,
- }
- }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub enum SamplingSeed {
- Default,
- Custom(u64),
-}
-
-impl Default for SamplingSeed {
- fn default() -> Self {
- SamplingSeed::Default
- }
-}
-
-impl SamplingSeed {
- pub fn get_value(&self) -> u64 {
- match self {
- SamplingSeed::Default => rand::rng().random::<u64>(),
- SamplingSeed::Custom(seed) => *seed,
- }
- }
-}
-
-pub trait GeneratorConfigProvider {
- fn generator_config(
- &self,
- tokenizer: &Tokenizer,
- ) -> GeneratorConfig;
-}
-
-#[derive(Clone)]
-pub struct GeneratorConfig {
- pub prefill_step_size: usize,
- pub speculator_config: SpeculatorConfig,
- pub allow_pre_encode: bool,
- pub sampling_seed: u64,
- pub context_length: usize,
-}
-
-impl GeneratorConfig {
- pub fn new(
- prefill_step_size: usize,
- speculator_config: SpeculatorConfig,
- allow_pre_encode: bool,
- sampling_seed: SamplingSeed,
- context_length: ContextLength,
- ) -> Self {
- Self {
- prefill_step_size,
- speculator_config,
- allow_pre_encode,
- sampling_seed: sampling_seed.get_value(),
- context_length: context_length.get_value() as usize,
- }
- }
-
- pub fn generate_suffix_length(&self) -> usize {
- self.speculator_config.number_of_speculated_tokens + 1
- }
-}
diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/generator/context.rs
index bea5a510d..b44d65b94 100644
--- a/crates/uzu/src/generator/context.rs
+++ b/crates/uzu/src/generator/context.rs
@@ -4,7 +4,6 @@ use metal::SharedEvent;
use mpsgraph::CommandBuffer as MPSCommandBuffer;
use objc2::rc::Retained;
-use super::{config::GeneratorConfig, error::GeneratorError};
use crate::{
DataType,
backends::metal::{
@@ -14,8 +13,13 @@ use crate::{
forward_pass::{ForwardPassBuffers, SharedBuffers},
kernel::SamplingKernelEncodable,
},
- config::ModelMetadata,
+ config::{LanguageModelConfig, ModelMetadata},
parameters::ParameterLoader,
+ session::{
+ config::DecodingConfig,
+ parameter::{ConfigResolvableValue, ResolvableValue},
+ types::Error,
+ },
};
pub struct GeneratorContext {
@@ -28,6 +32,7 @@ pub struct GeneratorContext {
pub shared_buffers: Rc<RefCell<SharedBuffers>>,
pub scratch_buffers: ForwardPassBuffers,
+ pub model_config: LanguageModelConfig,
pub model_shape: ModelShape,
pub executables: DecoderExecutables,
pub kv_cache_update: Box<KVCacheUpdate>,
@@ -37,10 +42,10 @@ pub struct GeneratorContext {
impl GeneratorContext {
pub fn new(
model_path: &Path,
- config: &GeneratorConfig,
- ) -> Result<Self, GeneratorError> {
+ decoding_config: &DecodingConfig,
+ ) -> Result<Self, Error> {
let mtl_device = metal::Device::system_default()
- .ok_or(GeneratorError::UnableToCreateMetalContext)?;
+ .ok_or(Error::UnableToCreateMetalContext)?;
let mtl_command_queue =
mtl_device.new_command_queue_with_max_command_buffer_count(1024);
@@ -49,38 +54,41 @@ impl GeneratorContext {
let config_path = model_path.join("config.json");
if !config_path.exists() {
- return Err(GeneratorError::UnableToLoadConfig);
+ return Err(Error::UnableToLoadConfig);
}
- let config_file = File::open(&config_path)
- .map_err(|_| GeneratorError::UnableToLoadConfig)?;
+ let config_file =
+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
let model_metadata: ModelMetadata =
serde_json::from_reader(BufReader::new(config_file))
- .map_err(|_| GeneratorError::UnableToLoadConfig)?;
+ .map_err(|_| Error::UnableToLoadConfig)?;
let decoder_config =
Rc::new(model_metadata.model_config.decoder_config.clone());
let model_shape = ModelShape::from_decoder_config(&decoder_config);
- let prefill_step_size = config.prefill_step_size;
- let generate_suffix_length = config.generate_suffix_length();
- let max_prefix_length: usize =
- std::cmp::min(config.context_length, decoder_config.context_length);
+ let prefill_step_size = decoding_config
+ .prefill_step_size
+ .resolve(&model_metadata.model_config);
+ let generate_suffix_length = decoding_config.generate_suffix_length();
+ let max_prefix_length: usize = decoding_config
+ .context_length
+ .resolve(&model_metadata.model_config);
let max_suffix_length: usize =
std::cmp::max(prefill_step_size, generate_suffix_length);
let mtl_context = Rc::new(
MTLContext::new(mtl_device, mtl_command_queue)
- .map_err(|_| GeneratorError::UnableToCreateMetalContext)?,
+ .map_err(|_| Error::UnableToCreateMetalContext)?,
);
let compilation_config = Rc::new(CompilationConfig::default());
let weights_path = model_path.join("model.safetensors");
if !weights_path.exists() {
- return Err(GeneratorError::UnableToLoadWeights);
+ return Err(Error::UnableToLoadWeights);
}
let weights_file = File::open(&weights_path)
- .map_err(|_| GeneratorError::UnableToLoadWeights)?;
+ .map_err(|_| Error::UnableToLoadWeights)?;
let loader = ParameterLoader::new(&weights_file, &mtl_context)
- .map_err(|_| GeneratorError::UnableToLoadWeights)?;
+ .map_err(|_| Error::UnableToLoadWeights)?;
let root_loader_view = loader.tree();
let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(
@@ -120,7 +128,7 @@ impl GeneratorContext {
kernel_data_type,
max_prefix_length,
)
- .unwrap(),
+ .map_err(|_| Error::UnableToCreateMetalContext)?,
);
let gpu_sampler = SamplingKernelEncodable::new(
@@ -128,9 +136,9 @@ impl GeneratorContext {
kernel_data_type,
max_suffix_length,
decoder_config.vocab_size,
- config.sampling_seed,
+ decoding_config.sampling_seed.resolve(),
)
- .map_err(|_| GeneratorError::UnableToCreateMetalContext)?;
+ .map_err(|_| Error::UnableToCreateMetalContext)?;
let kv_update_event = mtl_context.device.new_shared_event();
let kv_update_signal = 1;
@@ -143,6 +151,7 @@ impl GeneratorContext {
kv_cache,
shared_buffers,
scratch_buffers,
+ model_config: model_metadata.model_config.clone(),
model_shape,
executables,
kv_cache_update,
diff --git a/crates/uzu/src/generator/error.rs b/crates/uzu/src/generator/error.rs
deleted file mode 100644
index 80d8fb323..000000000
--- a/crates/uzu/src/generator/error.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-#[derive(Debug)]
-pub enum GeneratorError {
- UnableToCreateMetalContext,
- UnableToLoadConfig,
- UnableToLoadWeights,
-}
diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rs
index dc5208574..ba9234cc5 100644
--- a/crates/uzu/src/generator/generator.rs
+++ b/crates/uzu/src/generator/generator.rs
@@ -3,28 +3,28 @@ use std::{collections::HashMap, path::Path, time::Instant};
use mpsgraph::CommandBuffer;
use super::{
- config::GeneratorConfig,
context::GeneratorContext,
result::{GenerateResult, PrefillResult},
tasks::{GeneratorEncodedTask, GeneratorRunTask},
};
use crate::{
Array,
- backends::metal::{
- forward_pass::{
- ForwardPassState,
- encodable_with_state::{EncodableWithState, EncodingParameters},
- kv_cache::INVALID_POSITION,
- },
- sampling_config::SamplingConfig,
+ backends::metal::forward_pass::{
+ ForwardPassState,
+ encodable_with_state::{EncodableWithState, EncodingParameters},
+ kv_cache::INVALID_POSITION,
},
- env_utils::MetalEnvVar,
- generator::error::GeneratorError,
linearizer::trie::TokenTrie,
+ session::{
+ config::DecodingConfig,
+ parameter::{ConfigResolvableValue, SamplingMethod},
+ types::Error,
+ },
+ utils::env_utils::MetalEnvVar,
};
pub struct Generator {
- pub config: GeneratorConfig,
+ pub decoding_config: DecodingConfig,
pub tokens: Vec<u64>,
pub context: GeneratorContext,
@@ -36,15 +36,16 @@ pub struct Generator {
impl Generator {
pub fn new(
model_path: &Path,
- config: GeneratorConfig,
- ) -> Result<Self, GeneratorError> {
- let context = GeneratorContext::new(model_path, &config)?;
+ decoding_config: DecodingConfig,
+ ) -> Result<Self, Error> {
+ let context = GeneratorContext::new(model_path, &decoding_config)?;
- let prefill_step_size = config.prefill_step_size;
- let generate_suffix_length = config.generate_suffix_length();
+ let prefill_step_size =
+ decoding_config.prefill_step_size.resolve(&context.model_config);
+ let generate_suffix_length = decoding_config.generate_suffix_length();
let mut generator = Self {
- config: config,
+ decoding_config: decoding_config,
tokens: Vec::new(),
context,
encoded_tasks: HashMap::new(),
@@ -62,22 +63,26 @@ impl Generator {
pub fn prefill(
&mut self,
tokens: Vec<u64>,
- sampling_config: SamplingConfig,
+ sampling_method: SamplingMethod,
prefix_offset: usize,
- ) -> PrefillResult {
+ ) -> Result<PrefillResult, Error> {
assert!(!tokens.is_empty());
self.tokens.extend(tokens.clone());
+ let prefill_step_size = self
+ .decoding_config
+ .prefill_step_size
+ .resolve(&self.context.model_config);
+
let tokens_length = tokens.len();
- let number_of_prefill_steps = (tokens_length as f32
- / self.config.prefill_step_size as f32)
- .ceil() as usize;
+ let number_of_prefill_steps =
+ (tokens_length as f32 / prefill_step_size as f32).ceil() as usize;
let total_prefill_tokens_count =
- number_of_prefill_steps * self.config.prefill_step_size;
+ number_of_prefill_steps * prefill_step_size;
let unused_tokens_count = total_prefill_tokens_count - tokens_length;
- let speculator = &self.config.speculator_config.speculator;
+ let speculator = &self.decoding_config.speculator_config.speculator;
let proposals = speculator.generate_proposals(&tokens);
let speculated_suffix = TokenTrie::from_sequences(&proposals)
.linearize(0, unused_tokens_count);
@@ -110,9 +115,8 @@ impl Generator {
// Process each prefill step and update the KV cache.
for step in 0..number_of_prefill_steps {
- let tokens_start_index = step * self.config.prefill_step_size;
- let tokens_end_index =
- tokens_start_index + self.config.prefill_step_size;
+ let tokens_start_index = step * prefill_step_size;
+ let tokens_end_index = tokens_start_index + prefill_step_size;
let tokens_for_step =
&padded_tokens[tokens_start_index..tokens_end_index];
let positions_for_step =
@@ -125,14 +129,14 @@ impl Generator {
let task = GeneratorRunTask {
token_ids: tokens_for_step.to_vec(),
token_positions: positions_for_step.to_vec(),
- expected_amount_of_new_tokens: self.config.prefill_step_size,
+ expected_number_of_new_tokens: prefill_step_size,
};
let (mut state, run_time) = self.run_model(
task,
false,
self.allow_pre_encode(),
- Some(sampling_config.clone()),
+ sampling_method,
);
// Register the *accepted* real tokens from this step (exclude the
@@ -175,18 +179,17 @@ impl Generator {
run_times.push(run_time);
}
- let mut final_state = last_state.unwrap();
- let argmax_tokens = self.gpu_sample(&mut final_state);
+ let mut final_state = last_state.ok_or(Error::PrefillFailed)?;
+ let sampled_tokens = self.sample(&mut final_state)?;
let mut accepted_token_indices: Vec<usize> = Vec::new();
let mut accepted_tokens: Vec<u64> = Vec::new();
let mut current_token_index: isize = -1;
loop {
- let current_index_in_window = ((current_token_index
- + tokens_length as isize)
- % self.config.prefill_step_size as isize)
- as usize;
- let new_token = argmax_tokens[current_index_in_window];
+ let current_index_in_window =
+ ((current_token_index + tokens_length as isize)
+ % prefill_step_size as isize) as usize;
+ let new_token = sampled_tokens[current_index_in_window];
accepted_tokens.push(new_token);
if let Some(map) =
@@ -209,19 +212,19 @@ impl Generator {
self.sync_prefix();
- PrefillResult {
+ Ok(PrefillResult {
tokens: accepted_tokens,
forwardpass_durations: run_times,
- }
+ })
}
pub fn generate(
&mut self,
- sampling_config: SamplingConfig,
- ) -> GenerateResult {
- let last_token = self.tokens.last().unwrap();
+ sampling_method: SamplingMethod,
+ ) -> Result<GenerateResult, Error> {
+ let last_token = self.tokens.last().ok_or(Error::GenerateFailed)?;
- let speculator = &self.config.speculator_config.speculator;
+ let speculator = &self.decoding_config.speculator_config.speculator;
let mut proposals: Vec<Vec<u64>> = speculator
.generate_proposals(&self.tokens)
.into_iter()
@@ -232,9 +235,10 @@ impl Generator {
}
let speculated_suffix = TokenTrie::from_sequences(&proposals)
- .linearize(0, self.config.generate_suffix_length());
+ .linearize(0, self.decoding_config.generate_suffix_length());
- let expected_suffix_length = self.config.generate_suffix_length();
+ let expected_suffix_length =
+ self.decoding_config.generate_suffix_length();
let unused_tokens_count =
expected_suffix_length - speculated_suffix.tokens.len();
let zero_padding_tokens: Vec<u64> = vec![0; unused_tokens_count];
@@ -255,23 +259,23 @@ impl Generator {
let task = GeneratorRunTask {
token_ids: padded_tokens,
token_positions: padded_positions,
- expected_amount_of_new_tokens: 1,
+ expected_number_of_new_tokens: 1,
};
let (mut state, run_time) = self.run_model(
task,
false,
self.allow_pre_encode(),
- Some(sampling_config),
+ sampling_method,
);
- let argmax_tokens = self.gpu_sample(&mut state);
+ let sampled_tokens = self.sample(&mut state)?;
let mut accepted_token_indices = Vec::new();
let mut accepted_tokens = Vec::new();
let mut current_token_index: isize = 0;
loop {
- let new_token = argmax_tokens[current_token_index as usize];
+ let new_token = sampled_tokens[current_token_index as usize];
accepted_tokens.push(new_token);
if let Some(map) =
speculated_suffix.transition_map.get(&(current_token_index))
@@ -292,10 +296,10 @@ impl Generator {
self.tokens.extend(accepted_tokens.clone());
self.sync_prefix();
- GenerateResult {
+ Ok(GenerateResult {
tokens: accepted_tokens,
forwardpass_duration: run_time,
- }
+ })
}
pub fn clear_cache(&mut self) {
@@ -323,10 +327,10 @@ impl Generator {
let task = GeneratorRunTask {
token_ids: vec![0; suffix_length],
token_positions: (0..suffix_length).collect::<Vec<usize>>(),
- expected_amount_of_new_tokens: suffix_length,
+ expected_number_of_new_tokens: suffix_length,
};
- let (_, _) = self.run_model(task, true, false, None);
+ let (_, _) = self.run_model(task, true, false, SamplingMethod::Greedy);
}
fn run_model(
@@ -334,13 +338,13 @@ impl Generator {
task: GeneratorRunTask,
warmup: bool,
allow_pre_encode: bool,
- sampling_config: Option<SamplingConfig>,
+ sampling_method: SamplingMethod,
) -> (ForwardPassState, f64) {
objc2::rc::autoreleasepool(|_pool| {
let run_start = Instant::now();
let mut state = task.create_state(&mut self.context, None);
- state.sampling_config = sampling_config;
+ state.sampling_method = Some(sampling_method);
let encoded_task_key = task.encoded_task_key(self.tokens.len());
@@ -400,15 +404,17 @@ impl Generator {
})
}
- fn gpu_sample(
+ fn sample(
&mut self,
state: &mut ForwardPassState,
- ) -> Vec<u64> {
+ ) -> Result<Vec<u64>, Error> {
let sampling_output = state.sampling_output.as_ref()
.expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");
let output_buffer = sampling_output.borrow();
- let output_view = output_buffer.as_view::<u32>().unwrap();
+ let output_view = output_buffer
+ .as_view::<u32>()
+ .map_err(|_| Error::SamplingFailed)?;
let batch_size = output_buffer.shape()[0];
let mut result = Vec::with_capacity(batch_size);
@@ -416,7 +422,7 @@ impl Generator {
result.push(output_view[[i]] as u64);
}
- result
+ Ok(result)
}
fn update_kv_cache(
@@ -450,7 +456,8 @@ impl Generator {
fn allow_pre_encode(&self) -> bool {
let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();
- let result = self.config.allow_pre_encode && !metal_debug_active;
+ let result =
+ self.decoding_config.allow_pre_encode && !metal_debug_active;
result
}
diff --git a/crates/uzu/src/generator/mod.rs b/crates/uzu/src/generator/mod.rs
index 4462d420e..c6e6e1c90 100644
--- a/crates/uzu/src/generator/mod.rs
+++ b/crates/uzu/src/generator/mod.rs
@@ -1,6 +1,4 @@
-pub mod config;
pub mod context;
-pub mod error;
pub mod generator;
pub mod result;
pub mod sampler;
diff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/generator/tasks.rs
index a89e3ec86..728398089 100644
--- a/crates/uzu/src/generator/tasks.rs
+++ b/crates/uzu/src/generator/tasks.rs
@@ -14,7 +14,7 @@ pub struct GeneratorEncodedTask {
pub struct GeneratorRunTask {
pub token_ids: Vec<u64>,
pub token_positions: Vec<usize>,
- pub expected_amount_of_new_tokens: usize,
+ pub expected_number_of_new_tokens: usize,
}
impl GeneratorRunTask {
@@ -22,7 +22,7 @@ impl GeneratorRunTask {
GeneratorRunTask {
token_ids: self.token_ids.clone(),
token_positions: self.token_positions.clone(),
- expected_amount_of_new_tokens: self.expected_amount_of_new_tokens,
+ expected_number_of_new_tokens: self.expected_number_of_new_tokens,
}
}
@@ -32,7 +32,7 @@ impl GeneratorRunTask {
) -> String {
format!(
"tokens:{}_suffix:{}",
- tokens_count, self.expected_amount_of_new_tokens
+ tokens_count, self.expected_number_of_new_tokens
)
}
diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rs
index 2bc3b16e8..8a9f80912 100644
--- a/crates/uzu/src/lib.rs
+++ b/crates/uzu/src/lib.rs
@@ -1,21 +1,14 @@
-pub mod array;
pub mod backends;
pub mod config;
-pub mod data_type;
-pub use data_type::{ArrayElement, DataType};
-pub mod decoder_runner;
-pub mod device_context;
-pub use array::Array;
-pub use device_context::DeviceContext;
-pub mod env_utils;
pub mod generator;
pub mod linearizer;
pub mod parameters;
-pub mod quant_matmul;
pub mod session;
pub mod speculators;
-pub mod storage;
pub mod tracer;
-pub mod version;
-pub use quant_matmul::*;
-pub use version::VERSION;
+pub mod utils;
+pub use utils::*;
+pub mod device;
+pub use device::*;
+pub mod runners;
+pub use runners::*;
diff --git a/crates/uzu/src/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rs
similarity index 100%
rename from crates/uzu/src/decoder_runner.rs
rename to crates/uzu/src/runners/decoder_runner.rs
diff --git a/crates/uzu/src/runners/mod.rs b/crates/uzu/src/runners/mod.rs
new file mode 100644
index 000000000..1ad5471a3
--- /dev/null
+++ b/crates/uzu/src/runners/mod.rs
@@ -0,0 +1,4 @@
+pub mod decoder_runner;
+pub use decoder_runner::*;
+pub mod quant_matmul;
+pub use quant_matmul::*;
diff --git a/crates/uzu/src/quant_matmul.rs b/crates/uzu/src/runners/quant_matmul.rs
similarity index 99%
rename from crates/uzu/src/quant_matmul.rs
rename to crates/uzu/src/runners/quant_matmul.rs
index 8601e2bf1..60350f043 100644
--- a/crates/uzu/src/quant_matmul.rs
+++ b/crates/uzu/src/runners/quant_matmul.rs
@@ -27,7 +27,7 @@ use crate::{
MTLContext, MetalArray,
compilation_parameters::{BlockDevice, make_compilation_descriptor},
},
- storage::{NSSearchPathDirectory, root_dir},
+ utils::storage::{NSSearchPathDirectory, root_dir},
};
// ================== Public error type =====================
diff --git a/crates/uzu/src/session/config/decoding_config.rs b/crates/uzu/src/session/config/decoding_config.rs
new file mode 100644
index 000000000..227ce5301
--- /dev/null
+++ b/crates/uzu/src/session/config/decoding_config.rs
@@ -0,0 +1,47 @@
+use crate::session::{
+ config::SpeculatorConfig,
+ parameter::{ContextLength, PrefillStepSize, SamplingSeed},
+};
+
+#[derive(Clone)]
+pub struct DecodingConfig {
+ pub prefill_step_size: PrefillStepSize,
+ pub context_length: ContextLength,
+ pub speculator_config: SpeculatorConfig,
+ pub sampling_seed: SamplingSeed,
+ pub allow_pre_encode: bool,
+}
+
+impl DecodingConfig {
+ pub fn new(
+ prefill_step_size: PrefillStepSize,
+ context_length: ContextLength,
+ speculator_config: SpeculatorConfig,
+ sampling_seed: SamplingSeed,
+ allow_pre_encode: bool,
+ ) -> Self {
+ Self {
+ prefill_step_size,
+ context_length,
+ speculator_config,
+ sampling_seed,
+ allow_pre_encode,
+ }
+ }
+
+ pub fn generate_suffix_length(&self) -> usize {
+ self.speculator_config.number_of_speculated_tokens + 1
+ }
+}
+
+impl Default for DecodingConfig {
+ fn default() -> Self {
+ Self {
+ prefill_step_size: PrefillStepSize::default(),
+ context_length: ContextLength::default(),
+ speculator_config: SpeculatorConfig::default(),
+ sampling_seed: SamplingSeed::default(),
+ allow_pre_encode: true,
+ }
+ }
+}
diff --git a/crates/uzu/src/session/config/mod.rs b/crates/uzu/src/session/config/mod.rs
new file mode 100644
index 000000000..89c713ed1
--- /dev/null
+++ b/crates/uzu/src/session/config/mod.rs
@@ -0,0 +1,6 @@
+mod decoding_config;
+mod run_config;
+mod speculator_config;
+pub use decoding_config::DecodingConfig;
+pub use run_config::RunConfig;
+pub use speculator_config::SpeculatorConfig;
diff --git a/crates/uzu/src/session/config/run_config.rs b/crates/uzu/src/session/config/run_config.rs
new file mode 100644
index 000000000..aa07e06d7
--- /dev/null
+++ b/crates/uzu/src/session/config/run_config.rs
@@ -0,0 +1,52 @@
+use crate::session::parameter::SamplingPolicy;
+
+#[derive(Debug, Clone)]
+pub struct RunConfig {
+ pub tokens_limit: u64,
+ pub enable_thinking: bool,
+ pub sampling_policy: SamplingPolicy,
+}
+
+impl RunConfig {
+ pub fn new(
+ tokens_limit: u64,
+ enable_thinking: bool,
+ sampling_policy: SamplingPolicy,
+ ) -> Self {
+ Self {
+ tokens_limit,
+ enable_thinking,
+ sampling_policy,
+ }
+ }
+
+ pub fn tokens_limit(
+ mut self,
+ tokens_limit: u64,
+ ) -> Self {
+ self.tokens_limit = tokens_limit;
+ self
+ }
+
+ pub fn enable_thinking(
+ mut self,
+ enable_thinking: bool,
+ ) -> Self {
+ self.enable_thinking = enable_thinking;
+ self
+ }
+
+ pub fn sampling_policy(
+ mut self,
+ sampling_policy: SamplingPolicy,
+ ) -> Self {
+ self.sampling_policy = sampling_policy;
+ self
+ }
+}
+
+impl Default for RunConfig {
+ fn default() -> Self {
+ Self::new(1024, true, SamplingPolicy::Default)
+ }
+}
diff --git a/crates/uzu/src/session/config/speculator_config.rs b/crates/uzu/src/session/config/speculator_config.rs
new file mode 100644
index 000000000..5ba59db53
--- /dev/null
+++ b/crates/uzu/src/session/config/speculator_config.rs
@@ -0,0 +1,32 @@
+use std::sync::Arc;
+
+use crate::speculators::{
+ empty_speculator::EmptySpeculator, speculator::Speculator,
+};
+
+#[derive(Clone)]
+pub struct SpeculatorConfig {
+ pub number_of_speculated_tokens: usize,
+ pub speculator: Arc<dyn Speculator>,
+}
+
+impl SpeculatorConfig {
+ pub fn new(
+ number_of_speculated_tokens: usize,
+ speculator: Arc<dyn Speculator>,
+ ) -> Self {
+ Self {
+ number_of_speculated_tokens,
+ speculator,
+ }
+ }
+}
+
+impl Default for SpeculatorConfig {
+ fn default() -> Self {
+ Self {
+ number_of_speculated_tokens: 0,
+ speculator: Arc::new(EmptySpeculator {}),
+ }
+ }
+}
diff --git a/crates/uzu/src/session/session_context.rs b/crates/uzu/src/session/helpers/context.rs
similarity index 60%
rename from crates/uzu/src/session/session_context.rs
rename to crates/uzu/src/session/helpers/context.rs
index 57e838e40..cb2b91ba4 100644
--- a/crates/uzu/src/session/session_context.rs
+++ b/crates/uzu/src/session/helpers/context.rs
@@ -1,24 +1,24 @@
use crate::{
backends::metal::forward_pass::kv_cache::KVCache,
- generator::config::GeneratorConfig,
+ session::config::DecodingConfig,
};
-pub struct SessionContext {
+pub struct Context {
pub tokens: Vec<u64>,
pub kv_cache: KVCache,
- pub config: GeneratorConfig,
+ pub decoding_config: DecodingConfig,
}
-impl SessionContext {
+impl Context {
pub fn new(
tokens: Vec<u64>,
kv_cache: KVCache,
- config: GeneratorConfig,
+ decoding_config: DecodingConfig,
) -> Self {
Self {
tokens,
kv_cache,
- config,
+ decoding_config,
}
}
}
diff --git a/crates/uzu/src/session/helpers/input_processor.rs b/crates/uzu/src/session/helpers/input_processor.rs
new file mode 100644
index 000000000..e951b2e1f
--- /dev/null
+++ b/crates/uzu/src/session/helpers/input_processor.rs
@@ -0,0 +1,65 @@
+use minijinja::{Environment, context};
+use minijinja_contrib::pycompat::unknown_method_callback;
+
+use crate::{
+ config::MessageProcessorConfig,
+ session::{
+ parameter::ConfigResolvableValue,
+ types::{Error, Input},
+ },
+};
+
+pub trait InputProcessor: Send + Sync {
+ fn process(
+ &self,
+ input: &Input,
+ enable_thinking: bool,
+ ) -> Result<String, Error>;
+}
+
+pub struct InputProcessorDefault {
+ message_processing_config: MessageProcessorConfig,
+}
+
+impl InputProcessorDefault {
+ pub fn new(message_processing_config: MessageProcessorConfig) -> Self {
+ Self {
+ message_processing_config,
+ }
+ }
+}
+
+impl InputProcessor for InputProcessorDefault {
+ fn process(
+ &self,
+ input: &Input,
+ enable_thinking: bool,
+ ) -> Result<String, Error> {
+ let messages = input
+ .get_messages()
+ .into_iter()
+ .map(|message| message.resolve(&self.message_processing_config))
+ .collect::<Vec<_>>();
+ let template = self.message_processing_config.prompt_template.clone();
+ let bos_token = self.message_processing_config.bos_token.clone();
+
+ let template_name = "chat_template";
+ let mut environment = Environment::new();
+ environment.set_unknown_method_callback(unknown_method_callback);
+ environment
+ .add_template(template_name, template.as_str())
+ .map_err(|_| Error::UnableToLoadPromptTemplate)?;
+ let template = environment
+ .get_template(template_name)
+ .map_err(|_| Error::UnableToLoadPromptTemplate)?;
+ let result = template
+ .render(context!(
+ messages => messages,
+ add_generation_prompt => true,
+ bos_token => bos_token,
+ enable_thinking => enable_thinking
+ ))
+ .map_err(|_| Error::UnableToRenderPromptTemplate)?;
+ Ok(result)
+ }
+}
diff --git a/crates/uzu/src/session/helpers/memory_checker.rs b/crates/uzu/src/session/helpers/memory_checker.rs
new file mode 100644
index 000000000..3da918246
--- /dev/null
+++ b/crates/uzu/src/session/helpers/memory_checker.rs
@@ -0,0 +1,27 @@
+use std::path::Path;
+
+fn get_directory_size(path: &Path) -> std::io::Result<u64> {
+ let mut size = 0u64;
+ for entry_result in std::fs::read_dir(path)? {
+ let entry = entry_result?;
+ let metadata = entry.metadata()?;
+ if metadata.is_dir() {
+ size += get_directory_size(&entry.path())?;
+ } else {
+ size += metadata.len();
+ }
+ }
+ Ok(size)
+}
+
+pub fn is_directory_fits_ram(path: &Path) -> bool {
+ use sysinfo::System;
+
+ let model_size_bytes = get_directory_size(path).unwrap_or(0);
+
+ let mut sys = System::new();
+ sys.refresh_memory();
+
+ let allowed_bytes = sys.total_memory() * 60 / 100;
+ return model_size_bytes <= allowed_bytes;
+}
diff --git a/crates/uzu/src/session/helpers/mod.rs b/crates/uzu/src/session/helpers/mod.rs
new file mode 100644
index 000000000..720c4f950
--- /dev/null
+++ b/crates/uzu/src/session/helpers/mod.rs
@@ -0,0 +1,6 @@
+mod context;
+mod input_processor;
+mod memory_checker;
+pub use context::Context;
+pub use input_processor::{InputProcessor, InputProcessorDefault};
+pub use memory_checker::is_directory_fits_ram;
diff --git a/crates/uzu/src/session/mod.rs b/crates/uzu/src/session/mod.rs
index 8de03ebcf..7fa66d369 100644
--- a/crates/uzu/src/session/mod.rs
+++ b/crates/uzu/src/session/mod.rs
@@ -1,12 +1,5 @@
+pub mod config;
+pub mod helpers;
+pub mod parameter;
pub mod session;
-pub mod session_classification_feature;
-pub mod session_config;
-pub mod session_error;
-pub mod session_input;
-pub mod session_message;
-pub mod session_output;
-pub mod session_run_config;
-pub mod session_tokenizer_config;
-pub use session_context::*;
-
-pub mod session_context;
+pub mod types;
diff --git a/crates/uzu/src/session/parameter/context_length.rs b/crates/uzu/src/session/parameter/context_length.rs
new file mode 100644
index 000000000..800a88545
--- /dev/null
+++ b/crates/uzu/src/session/parameter/context_length.rs
@@ -0,0 +1,37 @@
+use crate::{
+ config::LanguageModelConfig, session::parameter::ConfigResolvableValue,
+};
+
+#[derive(Debug, Clone, Copy)]
+pub enum ContextLength {
+ Default,
+ Maximal,
+ Custom(usize),
+}
+
+impl Default for ContextLength {
+ fn default() -> Self {
+ ContextLength::Default
+ }
+}
+
+impl ConfigResolvableValue<LanguageModelConfig, usize> for ContextLength {
+ fn resolve(
+ &self,
+ config: &LanguageModelConfig,
+ ) -> usize {
+ let model_context_length = config.decoder_config.context_length;
+ let proposed_value = match self {
+ ContextLength::Default => {
+ if cfg!(target_os = "ios") {
+ return 8192;
+ } else {
+ return 16384;
+ }
+ },
+ ContextLength::Maximal => model_context_length,
+ ContextLength::Custom(value) => *value,
+ };
+ std::cmp::min(proposed_value, model_context_length)
+ }
+}
diff --git a/crates/uzu/src/session/parameter/mod.rs b/crates/uzu/src/session/parameter/mod.rs
new file mode 100644
index 000000000..6bd57ab0b
--- /dev/null
+++ b/crates/uzu/src/session/parameter/mod.rs
@@ -0,0 +1,10 @@
+mod context_length;
+mod prefill_step_size;
+mod resolvable_value;
+mod sampling_policy;
+mod sampling_seed;
+pub use context_length::ContextLength;
+pub use prefill_step_size::PrefillStepSize;
+pub use resolvable_value::{ConfigResolvableValue, ResolvableValue};
+pub use sampling_policy::{SamplingMethod, SamplingPolicy};
+pub use sampling_seed::SamplingSeed;
diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rs
new file mode 100644
index 000000000..ae130a646
--- /dev/null
+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs
@@ -0,0 +1,54 @@
+use crate::{
+ config::LanguageModelConfig, session::parameter::ConfigResolvableValue,
+};
+
+#[derive(Debug, Clone, Copy)]
+pub enum PrefillStepSize {
+ Default,
+ Maximal,
+ Custom(usize),
+}
+
+impl Default for PrefillStepSize {
+ fn default() -> Self {
+ PrefillStepSize::Default
+ }
+}
+
+impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
+ fn resolve(
+ &self,
+ config: &LanguageModelConfig,
+ ) -> usize {
+ let default_limit: usize = 1024;
+ let model_context_length = config.decoder_config.context_length;
+ let minimal_sliding_window_size = config
+ .decoder_config
+ .sliding_window_sizes
+ .as_deref()
+ .into_iter()
+ .flatten()
+ .flatten()
+ .copied()
+ .min()
+ .unwrap_or(usize::MAX);
+ let maximal_value =
+ [model_context_length, minimal_sliding_window_size, default_limit]
+ .into_iter()
+ .min()
+ .unwrap_or(default_limit);
+
+ let proposed_value = match self {
+ PrefillStepSize::Default => {
+ if cfg!(target_os = "ios") {
+ return 64;
+ } else {
+ return 256;
+ }
+ },
+ PrefillStepSize::Maximal => maximal_value,
+ PrefillStepSize::Custom(value) => *value,
+ };
+ std::cmp::min(proposed_value, maximal_value)
+ }
+}
diff --git a/crates/uzu/src/session/parameter/resolvable_value.rs b/crates/uzu/src/session/parameter/resolvable_value.rs
new file mode 100644
index 000000000..05f156979
--- /dev/null
+++ b/crates/uzu/src/session/parameter/resolvable_value.rs
@@ -0,0 +1,10 @@
+pub trait ResolvableValue<Value> {
+ fn resolve(&self) -> Value;
+}
+
+pub trait ConfigResolvableValue<Config, Value> {
+ fn resolve(
+ &self,
+ config: &Config,
+ ) -> Value;
+}
diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rs
new file mode 100644
index 000000000..89a1e41d9
--- /dev/null
+++ b/crates/uzu/src/session/parameter/sampling_policy.rs
@@ -0,0 +1,63 @@
+use crate::{
+ config::LanguageModelConfig, session::parameter::ConfigResolvableValue,
+};
+
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum SamplingMethod {
+ Greedy,
+ Temperature {
+ temperature: f32,
+ },
+ TopP {
+ top_p: f32,
+ },
+}
+
+impl Default for SamplingMethod {
+ fn default() -> Self {
+ SamplingMethod::Greedy
+ }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum SamplingPolicy {
+ Default,
+ Custom {
+ value: SamplingMethod,
+ },
+}
+
+impl Default for SamplingPolicy {
+ fn default() -> Self {
+ SamplingPolicy::Default
+ }
+}
+
+impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod>
+ for SamplingPolicy
+{
+ fn resolve(
+ &self,
+ config: &LanguageModelConfig,
+ ) -> SamplingMethod {
+ let generation_config = &config.generation_config;
+ match self {
+ SamplingPolicy::Default => {
+ if let Some(top_p) = generation_config.top_p {
+ return SamplingMethod::TopP {
+ top_p: top_p,
+ };
+ }
+ if let Some(temperature) = generation_config.temperature {
+ return SamplingMethod::Temperature {
+ temperature: temperature,
+ };
+ }
+ return SamplingMethod::Greedy;
+ },
+ SamplingPolicy::Custom {
+ value,
+ } => *value,
+ }
+ }
+}
diff --git a/crates/uzu/src/session/parameter/sampling_seed.rs b/crates/uzu/src/session/parameter/sampling_seed.rs
new file mode 100644
index 000000000..77098926e
--- /dev/null
+++ b/crates/uzu/src/session/parameter/sampling_seed.rs
@@ -0,0 +1,24 @@
+use rand::prelude::*;
+
+use crate::session::parameter::ResolvableValue;
+
+#[derive(Debug, Clone, Copy)]
+pub enum SamplingSeed {
+ Default,
+ Custom(u64),
+}
+
+impl Default for SamplingSeed {
+ fn default() -> Self {
+ SamplingSeed::Default
+ }
+}
+
+impl ResolvableValue<u64> for SamplingSeed {
+ fn resolve(&self) -> u64 {
+ match self {
+ SamplingSeed::Default => rand::rng().random::<u64>(),
+ SamplingSeed::Custom(seed) => *seed,
+ }
+ }
+}
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rs
index e8881663c..9905e6421 100644
--- a/crates/uzu/src/session/session.rs
+++ b/crates/uzu/src/session/session.rs
@@ -1,260 +1,223 @@
-#[cfg(target_os = "ios")]
-use std::path::Path;
use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};
use objc2::rc::autoreleasepool;
use tokenizers::Tokenizer;
-use super::{
- session_config::SessionConfig,
- session_context::SessionContext,
- session_input::{
- SessionInput, SessionInputProcessor, SessionInputProcessorDefault,
- },
- session_output::{
- SessionOutput, SessionOutputFinishReason, SessionOutputRunStats,
- SessionOutputStats, SessionOutputStepStats, SessionOutputTotalStats,
- },
- session_run_config::SessionRunConfig,
-};
use crate::{
config::ModelMetadata,
generator::{
- config::GeneratorConfigProvider,
generator::Generator,
result::{GenerateResult, PrefillResult},
},
session::{
- session_error::SessionError,
- session_tokenizer_config::SessionTokenizerConfig,
+ config::{DecodingConfig, RunConfig},
+ helpers::{
+ Context, InputProcessor, InputProcessorDefault,
+ is_directory_fits_ram,
+ },
+ parameter::ConfigResolvableValue,
+ types::{
+ Error, FinishReason, Input, Output, RunStats, Stats, StepStats,
+ TotalStats,
+ },
},
};
pub struct Session {
- model_path: PathBuf,
+ pub model_path: PathBuf,
+ pub model_metadata: ModelMetadata,
+
tokenizer: Tokenizer,
- tokenizer_config: SessionTokenizerConfig,
- input_processor: Box<dyn SessionInputProcessor>,
+ input_processor: Box<dyn InputProcessor>,
generator: Option<Generator>,
}
impl Session {
- #[cfg(target_os = "ios")]
- fn directory_size(path: &Path) -> std::io::Result<u64> {
- let mut size = 0u64;
- for entry_result in std::fs::read_dir(path)? {
- let entry = entry_result?;
- let metadata = entry.metadata()?;
- if metadata.is_dir() {
- size += Self::directory_size(&entry.path())?;
- } else {
- size += metadata.len();
- }
+ pub fn new(
+ model_path: PathBuf,
+ decoding_config: DecodingConfig,
+ ) -> Result<Self, Error> {
+ if !model_path.exists() {
+ return Err(Error::ModelFolderNotFound);
}
- Ok(size)
- }
- #[cfg(target_os = "ios")]
- fn assert_model_fits_ram(model_path: &Path) -> Result<(), SessionError> {
- use sysinfo::System;
-
- let model_size_bytes = Self::directory_size(model_path).unwrap_or(0);
-
- let mut sys = System::new();
- sys.refresh_memory();
-
- let allowed_bytes = sys.total_memory() * 60 / 100;
-
- if model_size_bytes > allowed_bytes {
- Err(SessionError::UnsupportedModel)
- } else {
- Ok(())
+ if !is_directory_fits_ram(&model_path) {
+ return Err(Error::NotEnoughMemory);
}
- }
-
- pub fn new(model_path: PathBuf) -> Result<Self, SessionError> {
- #[cfg(target_os = "ios")]
- Self::assert_model_fits_ram(&model_path)?;
let config_path = model_path.join("config.json");
if !config_path.exists() {
- return Err(SessionError::UnableToLoadConfig);
+ return Err(Error::UnableToLoadConfig);
}
- let config_file = File::open(&config_path)
- .map_err(|_| SessionError::UnableToLoadConfig)?;
+ let config_file =
+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
let model_metadata: ModelMetadata =
serde_json::from_reader(BufReader::new(config_file))
- .map_err(|_| SessionError::UnableToLoadConfig)?;
+ .map_err(|_| Error::UnableToLoadConfig)?;
let tokenizer_path = model_path.join("tokenizer.json");
+ if !tokenizer_path.exists() {
+ return Err(Error::UnableToLoadTokenizer);
+ }
let tokenizer = Tokenizer::from_file(&tokenizer_path)
- .map_err(|_| SessionError::UnableToLoadTokenizer)?;
+ .map_err(|_| Error::UnableToLoadTokenizer)?;
- let tokenizer_config = SessionTokenizerConfig::load(
- &model_metadata.model_config,
- &tokenizer,
- )
- .ok_or(SessionError::UnableToLoadTokenizerConfig)?;
+ let input_processor = InputProcessorDefault::new(
+ model_metadata.model_config.message_processor_config.clone(),
+ );
- let input_processor =
- SessionInputProcessorDefault::new(tokenizer_config.clone());
+ let generator = Generator::new(&model_path, decoding_config)
+ .map_err(Error::from)?;
Ok(Self {
model_path,
+ model_metadata,
tokenizer,
- tokenizer_config,
input_processor: Box::new(input_processor),
- generator: None,
+ generator: Some(generator),
})
}
- pub fn load(
- &mut self,
- generator_config_provider: Box<dyn GeneratorConfigProvider>,
- ) -> Result<(), SessionError> {
- let generator_config =
- generator_config_provider.generator_config(&self.tokenizer);
- let generator = Generator::new(&self.model_path, generator_config)
- .map_err(SessionError::from)?;
-
- self.generator = Some(generator);
- Ok(())
- }
-
- pub fn load_with_session_config(
- &mut self,
- config: SessionConfig,
- ) -> Result<(), SessionError> {
- self.load(Box::new(config))
- }
-
- pub fn extend(
- &mut self,
- input: SessionInput,
- context: Option<&SessionContext>,
- config: SessionRunConfig,
- ) -> Result<(SessionOutput, SessionContext), SessionError> {
- self.reconfigure_generator(context)?;
- let output = self.run_internal(
- input,
- config,
- None::<fn(SessionOutput) -> bool>,
- )?;
- let new_context = self.build_context_from_generator()?;
- let generator =
- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;
- generator.reset_state();
- Ok((output, new_context))
- }
-
pub fn run<F>(
&mut self,
- input: SessionInput,
- config: SessionRunConfig,
+ input: Input,
+ config: RunConfig,
progress: Option<F>,
- ) -> Result<SessionOutput, SessionError>
+ ) -> Result<Output, Error>
where
- F: Fn(SessionOutput) -> bool,
+ F: Fn(Output) -> bool,
{
let generator =
- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
generator.reset_state();
let output = self.run_internal(input, config, progress)?;
let generator =
- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
generator.reset_state();
Ok(output)
}
+ pub fn extend(
+ &mut self,
+ input: Input,
+ context: Option<&Context>,
+ config: RunConfig,
+ ) -> Result<(Output, Context), Error> {
+ self.reconfigure_generator(context)?;
+ let output =
+ self.run_internal(input, config, None::<fn(Output) -> bool>)?;
+ let new_context = self.build_context_from_generator()?;
+ let generator =
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
+ generator.reset_state();
+ Ok((output, new_context))
+ }
+
pub fn run_with_context(
&mut self,
- input: SessionInput,
- context: Option<&SessionContext>,
- config: SessionRunConfig,
- ) -> Result<SessionOutput, SessionError> {
+ input: Input,
+ context: Option<&Context>,
+ config: RunConfig,
+ ) -> Result<Output, Error> {
self.reconfigure_generator(context)?;
- let output = self.run_internal(
- input,
- config,
- None::<fn(SessionOutput) -> bool>,
- )?;
+ let output =
+ self.run_internal(input, config, None::<fn(Output) -> bool>)?;
let generator =
- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
generator.reset_state();
Ok(output)
}
fn run_internal<F>(
&mut self,
- input: SessionInput,
- config: SessionRunConfig,
+ input: Input,
+ config: RunConfig,
progress: Option<F>,
- ) -> Result<SessionOutput, SessionError>
+ ) -> Result<Output, Error>
where
- F: Fn(SessionOutput) -> bool,
+ F: Fn(Output) -> bool,
{
let generator =
- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
let run_start = Instant::now();
- let text = self.input_processor.process(&input);
+ let text =
+ self.input_processor.process(&input, config.enable_thinking)?;
let tokens: Vec<u64> = self
.tokenizer
.encode(text.as_str(), false)
- .unwrap()
+ .map_err(|_| Error::UnableToEncodeText)?
.get_ids()
.iter()
.map(|&id| id as u64)
.collect();
+ let context_length = generator
+ .decoding_config
+ .context_length
+ .resolve(&self.model_metadata.model_config);
+ if tokens.len() >= context_length {
+ return Err(Error::ContextLengthExceeded);
+ }
+
let prefix_len_before = generator.prefix_len();
- let eos_tokens = self
- .tokenizer_config
- .eos_tokens
+ let eos_tokens: Vec<u64> = self
+ .model_metadata
+ .model_config
+ .generation_config
+ .stop_token_ids
.iter()
- .map(|token| {
- self.tokenizer.token_to_id(token.as_str()).unwrap() as u64
- })
- .collect::<Vec<_>>();
+ .map(|&x| x as u64)
+ .collect();
let finish_reason = |generator: &Generator,
tokens_new: Vec<u64>|
- -> Option<SessionOutputFinishReason> {
+ -> Option<FinishReason> {
let start_idx = prefix_len_before + tokens.len();
+ let total_tokens = generator.tokens.len();
let total_new_tokens = generator.tokens[start_idx..].len();
let has_eos_token =
tokens_new.iter().any(|token| eos_tokens.contains(token));
+ let context_limit_reached = total_tokens >= context_length;
if has_eos_token {
- Some(SessionOutputFinishReason::Stop)
+ Some(FinishReason::Stop)
} else if total_new_tokens >= config.tokens_limit as usize {
- Some(SessionOutputFinishReason::Length)
+ Some(FinishReason::Length)
+ } else if context_limit_reached {
+ Some(FinishReason::ContextLimitReached)
} else {
None
}
};
- let build_generated_text =
- |generator: &Generator, tokenizer: &Tokenizer| -> String {
- let start_idx = prefix_len_before + tokens.len();
- let generated_tokens: Vec<u32> = generator.tokens[start_idx..]
- .to_vec()
- .iter()
- .map(|value| *value as u32)
- .collect();
- tokenizer.decode(&generated_tokens, true).unwrap()
- };
+ let build_generated_text = |generator: &Generator,
+ tokenizer: &Tokenizer|
+ -> Result<String, Error> {
+ let start_idx = prefix_len_before + tokens.len();
+ let generated_tokens: Vec<u32> = generator.tokens[start_idx..]
+ .to_vec()
+ .iter()
+ .map(|value| *value as u32)
+ .collect();
+ let generated_text = tokenizer
+ .decode(&generated_tokens, true)
+ .map_err(|_| Error::UnableToDecodeText)?;
+ Ok(generated_text)
+ };
- let sampling_config = config
- .sampling_config
- .unwrap_or(self.tokenizer_config.sampling_config);
+ let sampling_method =
+ config.sampling_policy.resolve(&self.model_metadata.model_config);
let prefill_start = Instant::now();
let prefix_offset = generator.tokens.len();
- let prefill_result =
- generator.prefill(tokens.clone(), sampling_config, prefix_offset);
+ let prefill_result = generator.prefill(
+ tokens.clone(),
+ sampling_method,
+ prefix_offset,
+ )?;
let prefill_tokens = prefill_result.tokens.clone();
let prefill_duration = prefill_start.elapsed().as_secs_f64();
generator.clear_cache();
@@ -262,17 +225,21 @@ impl Session {
let prefill_finish_reason =
finish_reason(generator, prefill_tokens.clone());
let prefill_generated_text =
- build_generated_text(generator, &self.tokenizer);
+ build_generated_text(generator, &self.tokenizer)?;
- let prefill_output = SessionOutput {
+ let prefill_suffix_length = generator
+ .decoding_config
+ .prefill_step_size
+ .resolve(&self.model_metadata.model_config);
+ let prefill_output = Output {
text: prefill_generated_text,
stats: Self::build_stats(
prefill_result.clone(),
prefill_duration,
- generator.config.prefill_step_size,
+ prefill_suffix_length,
Vec::new(),
Vec::new(),
- generator.config.generate_suffix_length(),
+ generator.decoding_config.generate_suffix_length(),
run_start.elapsed().as_secs_f64(),
tokens.len(),
generator.tokens[prefix_len_before + tokens.len()..].len(),
@@ -290,9 +257,8 @@ impl Session {
if prefill_should_continue {
return Ok(prefill_output);
} else {
- return Ok(prefill_output.clone_with_finish_reason(Some(
- SessionOutputFinishReason::Cancelled,
- )));
+ return Ok(prefill_output
+ .clone_with_finish_reason(Some(FinishReason::Cancelled)));
}
}
@@ -300,7 +266,7 @@ impl Session {
let mut generate_durations: Vec<f64> = Vec::new();
let generate_output = loop {
let generate_start = Instant::now();
- let generate_result = generator.generate(sampling_config);
+ let generate_result = generator.generate(sampling_method)?;
let generate_tokens = generate_result.tokens.clone();
let generate_duration = generate_start.elapsed().as_secs_f64();
generate_results.push(generate_result);
@@ -309,17 +275,17 @@ impl Session {
let generate_finish_reason =
finish_reason(generator, generate_tokens);
let generate_generated_text =
- build_generated_text(generator, &self.tokenizer);
+ build_generated_text(generator, &self.tokenizer)?;
- let generate_output = SessionOutput {
+ let generate_output = Output {
text: generate_generated_text,
stats: Self::build_stats(
prefill_result.clone(),
prefill_duration,
- generator.config.prefill_step_size,
+ prefill_suffix_length,
generate_results.clone(),
generate_durations.clone(),
- generator.config.generate_suffix_length(),
+ generator.decoding_config.generate_suffix_length(),
run_start.elapsed().as_secs_f64(),
tokens.len(),
generator.tokens[prefix_len_before + tokens.len()..].len(),
@@ -338,7 +304,7 @@ impl Session {
break generate_output;
} else {
break generate_output.clone_with_finish_reason(Some(
- SessionOutputFinishReason::Cancelled,
+ FinishReason::Cancelled,
));
}
}
@@ -351,10 +317,10 @@ impl Session {
fn reconfigure_generator(
&mut self,
- context: Option<&SessionContext>,
- ) -> Result<(), SessionError> {
+ context: Option<&Context>,
+ ) -> Result<(), Error> {
let generator =
- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;
+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;
generator.reset_state();
if let Some(ctx) = context {
let mut generator_cache = generator.context.kv_cache.borrow_mut();
@@ -387,21 +353,19 @@ impl Session {
Ok(())
}
- fn build_context_from_generator(
- &self
- ) -> Result<SessionContext, SessionError> {
+ fn build_context_from_generator(&self) -> Result<Context, Error> {
let generator =
- self.generator.as_ref().ok_or(SessionError::GeneratorNotLoaded)?;
+ self.generator.as_ref().ok_or(Error::GeneratorNotLoaded)?;
let prefix_len = generator.prefix_len();
let kv_cache = generator
.context
.kv_cache
.borrow()
.clone_with_prefix_len(&generator.context.mtl_context, prefix_len);
- let context = SessionContext::new(
+ let context = Context::new(
generator.tokens.clone(),
kv_cache,
- generator.config.clone(),
+ generator.decoding_config.clone(),
);
Ok(context)
}
@@ -418,7 +382,7 @@ impl Session {
total_duration: f64,
tokens_count_input: usize,
tokens_count_output: usize,
- ) -> SessionOutputStats {
+ ) -> Stats {
let prefill_stats = {
let tokens_count = prefill_result.tokens.len();
let tokens_per_second = tokens_count as f64 / prefill_duration;
@@ -428,12 +392,12 @@ impl Session {
prefill_result.forwardpass_durations.iter().sum::<f64>()
/ model_run_count as f64;
- SessionOutputStepStats {
+ StepStats {
duration: prefill_duration,
suffix_length: prefill_suffix_length as u64,
tokens_count: tokens_count as u64,
tokens_per_second,
- model_run: SessionOutputRunStats {
+ model_run: RunStats {
count: model_run_count as u64,
average_duration: model_run_average_duration,
},
@@ -441,7 +405,7 @@ impl Session {
}
};
- let generate_stats: Option<SessionOutputStepStats>;
+ let generate_stats: Option<StepStats>;
if generate_results.len() != 0 {
let duration = generate_durations.iter().sum::<f64>();
let tokens_count = generate_results
@@ -462,16 +426,16 @@ impl Session {
let run_average_duration =
generate_durations.iter().sum::<f64>() / run_count as f64;
- generate_stats = Some(SessionOutputStepStats {
+ generate_stats = Some(StepStats {
duration,
suffix_length: generate_suffix_length as u64,
tokens_count: tokens_count as u64,
tokens_per_second,
- model_run: SessionOutputRunStats {
+ model_run: RunStats {
count: model_run_count as u64,
average_duration: model_run_average_duration,
},
- run: Some(SessionOutputRunStats {
+ run: Some(RunStats {
count: run_count as u64,
average_duration: run_average_duration,
}),
@@ -480,13 +444,13 @@ impl Session {
generate_stats = None;
}
- let total_stats = SessionOutputTotalStats {
+ let total_stats = TotalStats {
duration: total_duration,
tokens_count_input: tokens_count_input as u64,
tokens_count_output: tokens_count_output as u64,
};
- let stats = SessionOutputStats {
+ let stats = Stats {
prefill_stats,
generate_stats,
total_stats,
diff --git a/crates/uzu/src/session/session_classification_feature.rs b/crates/uzu/src/session/session_classification_feature.rs
deleted file mode 100644
index c8c3948d3..000000000
--- a/crates/uzu/src/session/session_classification_feature.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-#[derive(Clone, Debug)]
-pub struct SessionClassificationFeature {
- pub name: String,
- pub values: Vec<String>,
-}
diff --git a/crates/uzu/src/session/session_config.rs b/crates/uzu/src/session/session_config.rs
deleted file mode 100644
index dbd4e6250..000000000
--- a/crates/uzu/src/session/session_config.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-use tokenizers::Tokenizer;
-
-use crate::generator::config::{
- ContextLength, GeneratorConfig, GeneratorConfigProvider, SamplingSeed,
- SpeculatorConfig,
-};
-
-#[derive(Clone)]
-pub struct SessionConfig {
- pub prefill_step_size: usize,
- pub speculator_config: SpeculatorConfig,
- pub allow_pre_encode: bool,
- pub sampling_seed: SamplingSeed,
- pub context_length: ContextLength,
-}
-
-impl SessionConfig {
- pub fn new(
- prefill_step_size: usize,
- speculator_config: SpeculatorConfig,
- allow_pre_encode: bool,
- sampling_seed: SamplingSeed,
- context_length: ContextLength,
- ) -> Self {
- Self {
- prefill_step_size,
- speculator_config,
- allow_pre_encode,
- sampling_seed,
- context_length,
- }
- }
-
- pub fn to_generator_config(&self) -> GeneratorConfig {
- GeneratorConfig::new(
- self.prefill_step_size,
- self.speculator_config.clone(),
- self.allow_pre_encode,
- self.sampling_seed,
- self.context_length,
- )
- }
-}
-
-impl Default for SessionConfig {
- fn default() -> Self {
- Self::new(
- 8,
- SpeculatorConfig::default(),
- true,
- SamplingSeed::default(),
- ContextLength::default(),
- )
- }
-}
-
-impl GeneratorConfigProvider for SessionConfig {
- fn generator_config(
- &self,
- _tokenizer: &Tokenizer,
- ) -> GeneratorConfig {
- self.to_generator_config()
- }
-}
diff --git a/crates/uzu/src/session/session_error.rs b/crates/uzu/src/session/session_error.rs
deleted file mode 100644
index b00df81f4..000000000
--- a/crates/uzu/src/session/session_error.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-use crate::generator::error::GeneratorError;
-
-#[derive(Debug, thiserror::Error)]
-pub enum SessionError {
- #[error("Unable to create Metal context")]
- UnableToCreateMetalContext,
- #[error("Unable to load model configuration")]
- UnableToLoadConfig,
- #[error("Unable to load model weights")]
- UnableToLoadWeights,
- #[error("Unable to load tokenizer")]
- UnableToLoadTokenizer,
- #[error("Unable to load tokenizer configuration")]
- UnableToLoadTokenizerConfig,
- #[error("Model is too large to fit into available RAM")]
- UnsupportedModel,
- #[error("Generator not loaded. Call load() first")]
- GeneratorNotLoaded,
-}
-
-impl From<GeneratorError> for SessionError {
- fn from(value: GeneratorError) -> Self {
- match value {
- GeneratorError::UnableToCreateMetalContext => {
- Self::UnableToCreateMetalContext
- },
- GeneratorError::UnableToLoadConfig => Self::UnableToLoadConfig,
- GeneratorError::UnableToLoadWeights => Self::UnableToLoadWeights,
- }
- }
-}
diff --git a/crates/uzu/src/session/session_input.rs b/crates/uzu/src/session/session_input.rs
deleted file mode 100644
index ba269e506..000000000
--- a/crates/uzu/src/session/session_input.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-use minijinja::{Environment, context};
-use minijinja_contrib::pycompat::unknown_method_callback;
-
-use super::session_message::{SessionMessage, SessionMessageRole};
-use crate::session::session_tokenizer_config::SessionTokenizerConfig;
-
-#[derive(Debug)]
-pub enum SessionInput {
- Text(String),
- Messages(Vec<SessionMessage>),
-}
-
-impl SessionInput {
- fn get_messages(&self) -> Vec<SessionMessage> {
- match self {
- SessionInput::Text(content) => vec![SessionMessage {
- role: SessionMessageRole::User,
- content: content.clone(),
- }],
- SessionInput::Messages(messages) => messages.clone(),
- }
- }
-}
-
-pub trait SessionInputProcessor: Send + Sync {
- fn process(
- &self,
- input: &SessionInput,
- ) -> String;
-}
-
-pub struct SessionInputProcessorDefault {
- tokenizer_config: SessionTokenizerConfig,
-}
-
-impl SessionInputProcessorDefault {
- pub fn new(tokenizer_config: SessionTokenizerConfig) -> Self {
- Self {
- tokenizer_config,
- }
- }
-}
-
-impl SessionInputProcessor for SessionInputProcessorDefault {
- fn process(
- &self,
- input: &SessionInput,
- ) -> String {
- let messages = input.get_messages();
- let template = self.tokenizer_config.chat_template.clone();
- let bos_token = self.tokenizer_config.bos_token.clone();
-
- let template_name = "chat_template";
- let mut environment = Environment::new();
- environment.set_unknown_method_callback(unknown_method_callback);
- environment.add_template(template_name, template.as_str()).unwrap();
- let template = environment.get_template(template_name).unwrap();
- let result = template
- .render(context!(
- messages => messages,
- add_generation_prompt => true,
- bos_token => bos_token
- ))
- .unwrap();
- result
- }
-}
diff --git a/crates/uzu/src/session/session_message.rs b/crates/uzu/src/session/session_message.rs
deleted file mode 100644
index e93e00608..000000000
--- a/crates/uzu/src/session/session_message.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "lowercase")]
-pub enum SessionMessageRole {
- System,
- User,
- Assistant,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct SessionMessage {
- pub role: SessionMessageRole,
- pub content: String,
-}
diff --git a/crates/uzu/src/session/session_output.rs b/crates/uzu/src/session/session_output.rs
deleted file mode 100644
index cc1125c3a..000000000
--- a/crates/uzu/src/session/session_output.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct SessionOutputRunStats {
- pub count: u64,
- pub average_duration: f64,
-}
-
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct SessionOutputStepStats {
- pub duration: f64,
- pub suffix_length: u64,
- pub tokens_count: u64,
- pub tokens_per_second: f64,
- pub model_run: SessionOutputRunStats,
- pub run: Option<SessionOutputRunStats>,
-}
-
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct SessionOutputTotalStats {
- pub duration: f64,
- pub tokens_count_input: u64,
- pub tokens_count_output: u64,
-}
-
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct SessionOutputStats {
- pub prefill_stats: SessionOutputStepStats,
- pub generate_stats: Option<SessionOutputStepStats>,
- pub total_stats: SessionOutputTotalStats,
-}
-
-#[derive(Debug, Clone, Deserialize, Serialize)]
-#[serde(rename_all = "lowercase")]
-pub enum SessionOutputFinishReason {
- Stop,
- Length,
- Cancelled,
-}
-
-#[derive(Debug, Clone)]
-pub struct SessionOutput {
- pub text: String,
- pub stats: SessionOutputStats,
- pub finish_reason: Option<SessionOutputFinishReason>,
-}
-
-impl SessionOutput {
- pub fn clone_with_finish_reason(
- &self,
- finish_reason: Option<SessionOutputFinishReason>,
- ) -> Self {
- Self {
- text: self.text.clone(),
- stats: self.stats.clone(),
- finish_reason: finish_reason,
- }
- }
-
- pub fn clone_with_duration(
- &self,
- duration: f64,
- ) -> Self {
- Self {
- text: self.text.clone(),
- stats: SessionOutputStats {
- prefill_stats: self.stats.prefill_stats.clone(),
- generate_stats: self.stats.generate_stats.clone(),
- total_stats: SessionOutputTotalStats {
- duration: duration,
- ..self.stats.total_stats.clone()
- },
- },
- finish_reason: self.finish_reason.clone(),
- }
- }
-}
diff --git a/crates/uzu/src/session/session_run_config.rs b/crates/uzu/src/session/session_run_config.rs
deleted file mode 100644
index 1ceb4801d..000000000
--- a/crates/uzu/src/session/session_run_config.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-use crate::backends::metal::sampling_config::SamplingConfig;
-
-#[derive(Debug)]
-pub struct SessionRunConfig {
- pub tokens_limit: u64,
- pub sampling_config: Option<SamplingConfig>,
-}
-
-impl SessionRunConfig {
- pub fn new(tokens_limit: u64) -> Self {
- Self {
- tokens_limit: tokens_limit,
- sampling_config: None,
- }
- }
-
- pub fn new_with_sampling_config(
- tokens_limit: u64,
- sampling_config: Option<SamplingConfig>,
- ) -> Self {
- Self {
- tokens_limit: tokens_limit,
- sampling_config: sampling_config,
- }
- }
-}
diff --git a/crates/uzu/src/session/session_tokenizer_config.rs b/crates/uzu/src/session/session_tokenizer_config.rs
deleted file mode 100644
index 5fe3241f2..000000000
--- a/crates/uzu/src/session/session_tokenizer_config.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-use tokenizers::Tokenizer;
-
-use crate::{
- backends::metal::sampling_config::SamplingConfig,
- config::{GenerationConfig, LanguageModelConfig},
-};
-
-#[derive(Clone, Debug)]
-pub struct SessionTokenizerConfig {
- pub eos_tokens: Vec<String>,
- pub bos_token: Option<String>,
- pub chat_template: String,
- pub sampling_config: SamplingConfig,
-}
-
-impl SessionTokenizerConfig {
- pub fn load(
- model_config: &LanguageModelConfig,
- tokenizer: &Tokenizer,
- ) -> Option<Self> {
- let eos_tokens = model_config
- .generation_config
- .stop_token_ids
- .iter()
- .flat_map(|token_id| tokenizer.id_to_token(*token_id))
- .collect::<Vec<_>>();
- if eos_tokens.is_empty() {
- return None;
- }
-
- let bos_token = model_config.message_processor_config.bos_token.clone();
-
- let chat_template =
- model_config.message_processor_config.prompt_template.clone();
-
- let sampling_config =
- Self::build_sampling_config(&model_config.generation_config);
-
- Some(Self {
- eos_tokens,
- bos_token,
- chat_template,
- sampling_config,
- })
- }
-
- fn build_sampling_config(
- generation_config: &GenerationConfig
- ) -> SamplingConfig {
- if let Some(top_p) = &generation_config.top_p {
- return SamplingConfig::TopP {
- top_p: *top_p,
- };
- }
- if let Some(temperature) = &generation_config.temperature {
- return SamplingConfig::Categorical {
- temperature: *temperature,
- };
- }
- return SamplingConfig::Argmax;
- }
-}
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
new file mode 100644
index 000000000..215dce323
--- /dev/null
+++ b/crates/uzu/src/session/types/error.rs
@@ -0,0 +1,33 @@
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("Model folder not found")]
+ ModelFolderNotFound,
+ #[error("Unable to create Metal context")]
+ UnableToCreateMetalContext,
+ #[error("Unable to load model configuration")]
+ UnableToLoadConfig,
+ #[error("Unable to load model weights")]
+ UnableToLoadWeights,
+ #[error("Unable to load tokenizer")]
+ UnableToLoadTokenizer,
+ #[error("Model is too large to fit into available RAM")]
+ NotEnoughMemory,
+ #[error("Generator not loaded")]
+ GeneratorNotLoaded,
+ #[error("Unable to load prompt template")]
+ UnableToLoadPromptTemplate,
+ #[error("Unable to render prompt template")]
+ UnableToRenderPromptTemplate,
+ #[error("Unable to encode text")]
+ UnableToEncodeText,
+ #[error("Unable to decode text")]
+ UnableToDecodeText,
+ #[error("Context length exceeded")]
+ ContextLengthExceeded,
+ #[error("Prefill failed")]
+ PrefillFailed,
+ #[error("Generate failed")]
+ GenerateFailed,
+ #[error("Sampling failed")]
+ SamplingFailed,
+}
diff --git a/crates/uzu/src/session/types/input.rs b/crates/uzu/src/session/types/input.rs
new file mode 100644
index 000000000..17b15f488
--- /dev/null
+++ b/crates/uzu/src/session/types/input.rs
@@ -0,0 +1,19 @@
+use crate::session::types::{Message, Role};
+
+#[derive(Debug)]
+pub enum Input {
+ Text(String),
+ Messages(Vec<Message>),
+}
+
+impl Input {
+ pub fn get_messages(&self) -> Vec<Message> {
+ match self {
+ Input::Text(content) => vec![Message {
+ role: Role::User,
+ content: content.clone(),
+ }],
+ Input::Messages(messages) => messages.clone(),
+ }
+ }
+}
diff --git a/crates/uzu/src/session/types/message.rs b/crates/uzu/src/session/types/message.rs
new file mode 100644
index 000000000..e44981d72
--- /dev/null
+++ b/crates/uzu/src/session/types/message.rs
@@ -0,0 +1,34 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ config::MessageProcessorConfig,
+ session::{parameter::ConfigResolvableValue, types::Role},
+};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Message {
+ pub role: Role,
+ pub content: String,
+}
+
+impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>>
+ for Message
+{
+ fn resolve(
+ &self,
+ config: &MessageProcessorConfig,
+ ) -> HashMap<String, String> {
+ let role = match self.role {
+ Role::System => config.system_role_name.clone(),
+ Role::User => config.user_role_name.clone(),
+ Role::Assistant => config.assistant_role_name.clone(),
+ };
+ let content = self.content.clone();
+ HashMap::from([
+ (String::from("role"), role),
+ (String::from("content"), content),
+ ])
+ }
+}
diff --git a/crates/uzu/src/session/types/mod.rs b/crates/uzu/src/session/types/mod.rs
new file mode 100644
index 000000000..01ba25346
--- /dev/null
+++ b/crates/uzu/src/session/types/mod.rs
@@ -0,0 +1,12 @@
+mod error;
+mod input;
+mod message;
+mod output;
+mod role;
+mod stats;
+pub use error::Error;
+pub use input::Input;
+pub use message::Message;
+pub use output::{FinishReason, Output};
+pub use role::Role;
+pub use stats::{RunStats, Stats, StepStats, TotalStats};
diff --git a/crates/uzu/src/session/types/output.rs b/crates/uzu/src/session/types/output.rs
new file mode 100644
index 000000000..0c4647afe
--- /dev/null
+++ b/crates/uzu/src/session/types/output.rs
@@ -0,0 +1,50 @@
+use serde::{Deserialize, Serialize};
+
+use crate::session::types::{Stats, TotalStats};
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum FinishReason {
+ Stop,
+ Length,
+ Cancelled,
+ ContextLimitReached,
+}
+
+#[derive(Debug, Clone)]
+pub struct Output {
+ pub text: String,
+ pub stats: Stats,
+ pub finish_reason: Option<FinishReason>,
+}
+
+impl Output {
+ pub fn clone_with_finish_reason(
+ &self,
+ finish_reason: Option<FinishReason>,
+ ) -> Self {
+ Self {
+ text: self.text.clone(),
+ stats: self.stats.clone(),
+ finish_reason: finish_reason,
+ }
+ }
+
+ pub fn clone_with_duration(
+ &self,
+ duration: f64,
+ ) -> Self {
+ Self {
+ text: self.text.clone(),
+ stats: Stats {
+ prefill_stats: self.stats.prefill_stats.clone(),
+ generate_stats: self.stats.generate_stats.clone(),
+ total_stats: TotalStats {
+ duration: duration,
+ ..self.stats.total_stats.clone()
+ },
+ },
+ finish_reason: self.finish_reason.clone(),
+ }
+ }
+}
diff --git a/crates/uzu/src/session/types/role.rs b/crates/uzu/src/session/types/role.rs
new file mode 100644
index 000000000..b86a520d3
--- /dev/null
+++ b/crates/uzu/src/session/types/role.rs
@@ -0,0 +1,9 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Role {
+ System,
+ User,
+ Assistant,
+}
diff --git a/crates/uzu/src/session/types/stats.rs b/crates/uzu/src/session/types/stats.rs
new file mode 100644
index 000000000..74d75b7b8
--- /dev/null
+++ b/crates/uzu/src/session/types/stats.rs
@@ -0,0 +1,31 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct RunStats {
+ pub count: u64,
+ pub average_duration: f64,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct StepStats {
+ pub duration: f64,
+ pub suffix_length: u64,
+ pub tokens_count: u64,
+ pub tokens_per_second: f64,
+ pub model_run: RunStats,
+ pub run: Option<RunStats>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct TotalStats {
+ pub duration: f64,
+ pub tokens_count_input: u64,
+ pub tokens_count_output: u64,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct Stats {
+ pub prefill_stats: StepStats,
+ pub generate_stats: Option<StepStats>,
+ pub total_stats: TotalStats,
+}
diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rs
index a0fb050f8..2d44cfc21 100644
--- a/crates/uzu/src/tracer/tracer.rs
+++ b/crates/uzu/src/tracer/tracer.rs
@@ -3,7 +3,6 @@ use std::{
fs::File,
path::{Path, PathBuf},
rc::Rc,
- sync::Arc,
};
use half::{bf16, f16};
@@ -21,14 +20,14 @@ use crate::{
},
},
generator::{
- config::{
- ContextLength, GeneratorConfig, SamplingSeed, SpeculatorConfig,
- },
context::GeneratorContext,
sampler::{ArgmaxSampler, LogitsSampler},
},
parameters::{ParameterLoader, ParameterTree},
- speculators::empty_speculator::EmptySpeculator,
+ session::{
+ config::{DecodingConfig, SpeculatorConfig},
+ parameter::{ContextLength, PrefillStepSize, SamplingSeed},
+ },
};
enum Transform {
@@ -141,18 +140,15 @@ pub struct Tracer {
impl Tracer {
pub fn new(model_path: &Path) -> Self {
- let generator_config = GeneratorConfig::new(
- 1024,
- SpeculatorConfig {
- number_of_speculated_tokens: 0,
- speculator: Arc::new(EmptySpeculator {}),
- },
+ let decoding_config = DecodingConfig::new(
+ PrefillStepSize::Custom(1024),
+ ContextLength::default(),
+ SpeculatorConfig::default(),
+ SamplingSeed::default(),
false,
- SamplingSeed::Default,
- ContextLength::Default,
);
let generator_context =
- GeneratorContext::new(model_path, &generator_config).unwrap();
+ GeneratorContext::new(model_path, &decoding_config).unwrap();
Self {
model_path: model_path.to_path_buf(),
@@ -165,18 +161,15 @@ impl Tracer {
sampling_seed: SamplingSeed,
max_prefix_length: ContextLength,
) -> Self {
- let generator_config = GeneratorConfig::new(
- 1024,
- SpeculatorConfig {
- number_of_speculated_tokens: 0,
- speculator: Arc::new(EmptySpeculator {}),
- },
- false,
- sampling_seed,
+ let decoding_config = DecodingConfig::new(
+ PrefillStepSize::Custom(1024),
max_prefix_length,
+ SpeculatorConfig::default(),
+ sampling_seed,
+ false,
);
let generator_context =
- GeneratorContext::new(model_path, &generator_config).unwrap();
+ GeneratorContext::new(model_path, &decoding_config).unwrap();
Self {
model_path: model_path.to_path_buf(),
@@ -206,19 +199,6 @@ impl Tracer {
&traces_view,
"activation_trace.token_positions".to_string(),
);
- let mask: Vec<Vec<bool>> = Self::load_array_as_vec::<i8, i8>(
- &traces_view,
- "activation_trace.mask".to_string(),
- )
- .iter()
- .map(|x| *x != 0)
- .collect::<Vec<bool>>()
- .chunks(token_ids.len())
- .map(|chunk| chunk.to_vec())
- .collect();
-
- let external_bias_fn =
- |row: usize, col: usize| -> bool { !mask[row][col] };
let mut state = ForwardPassState::new(
self.generator_context.mtl_context.clone(),
@@ -229,7 +209,7 @@ impl Tracer {
&token_ids,
&token_positions,
true,
- Some(&external_bias_fn),
+ None,
);
let root_command_buffer = self
diff --git a/crates/uzu/src/env_utils.rs b/crates/uzu/src/utils/env_utils.rs
similarity index 100%
rename from crates/uzu/src/env_utils.rs
rename to crates/uzu/src/utils/env_utils.rs
diff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rs
new file mode 100644
index 000000000..1c4996f25
--- /dev/null
+++ b/crates/uzu/src/utils/mod.rs
@@ -0,0 +1,6 @@
+pub mod storage;
+pub use storage::*;
+pub mod version;
+pub use version::VERSION;
+pub mod env_utils;
+pub use env_utils::*;
diff --git a/crates/uzu/src/storage.rs b/crates/uzu/src/utils/storage.rs
similarity index 100%
rename from crates/uzu/src/storage.rs
rename to crates/uzu/src/utils/storage.rs
diff --git a/crates/uzu/src/version.rs b/crates/uzu/src/utils/version.rs
similarity index 100%
rename from crates/uzu/src/version.rs
rename to crates/uzu/src/utils/version.rs
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index 609cc3c89..1c1f9b325 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -1,7 +1,9 @@
#![allow(dead_code)]
use std::path::PathBuf;
-pub const MODEL_DIR_NAME: &str = "Meta-Llama-3.2-1B-Instruct-float16";
+use uzu::VERSION;
+
+pub const MODEL_DIR_NAME: &str = "Llama-3.2-1B-Instruct";
pub const MODEL_FILE_NAME: &str = "model.safetensors";
pub const TRACES_FILE_NAME: &str = "traces.safetensors";
@@ -9,12 +11,12 @@ pub fn get_test_model_path() -> PathBuf {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
- .join("..")
- .join("mirai_models")
+ .join("models")
+ .join(VERSION)
.join(MODEL_DIR_NAME);
if !path.exists() {
panic!(
- "Test model not found at {:?}. Run `bash scripts/download_test_model.sh` to fetch it.",
+ "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",
path
);
}
diff --git a/crates/uzu/tests/context_kv_cache_test.rs b/crates/uzu/tests/context_kv_cache_test.rs
index e2ceb509f..02d1c05d6 100644
--- a/crates/uzu/tests/context_kv_cache_test.rs
+++ b/crates/uzu/tests/context_kv_cache_test.rs
@@ -1,12 +1,11 @@
use std::{path::PathBuf, rc::Rc};
-use uzu::{
- backends::metal::sampling_config::SamplingConfig,
- session::{
- session::Session, session_config::SessionConfig,
- session_context::SessionContext, session_input::SessionInput,
- session_output::SessionOutput, session_run_config::SessionRunConfig,
- },
+use uzu::session::{
+ config::{DecodingConfig, RunConfig},
+ helpers::Context,
+ parameter::{SamplingMethod, SamplingPolicy},
+ session::Session,
+ types::{Input, Output},
};
mod common;
@@ -75,11 +74,15 @@ fn test_context_extension() {
// Extend the context with new information
let (_, extended_context) = session
.extend(
- SessionInput::Text(
- "Update: Name: Dave. Occupation: Doctor.".to_string(),
- ),
+ Input::Text("Update: Name: Dave. Occupation: Doctor.".to_string()),
Some(initial_context.as_ref()),
- SessionRunConfig::new(1),
+ RunConfig::new(
+ 1,
+ true,
+ SamplingPolicy::Custom {
+ value: SamplingMethod::Greedy,
+ },
+ ),
)
.unwrap();
@@ -221,24 +224,27 @@ fn test_performance_cached_vs_plain() {
}
fn create_session() -> Session {
- let mut session = Session::new(model_path()).unwrap();
- session
- .load_with_session_config(SessionConfig::default())
+ let session = Session::new(model_path(), DecodingConfig::default())
.expect("Failed to load session");
-
session
}
fn build_context(
session: &mut Session,
prompt: &str,
-) -> Rc<SessionContext> {
+) -> Rc<Context> {
// Run the prompt once to build the context.
let (_, context) = session
.extend(
- SessionInput::Text(prompt.to_string()),
+ Input::Text(prompt.to_string()),
None,
- SessionRunConfig::new(1),
+ RunConfig::new(
+ 1,
+ true,
+ SamplingPolicy::Custom {
+ value: SamplingMethod::Greedy,
+ },
+ ),
)
.unwrap();
Rc::new(context)
@@ -246,16 +252,19 @@ fn build_context(
fn ask_with_context(
session: &mut Session,
- context: Option<Rc<SessionContext>>,
+ context: Option<Rc<Context>>,
question: &str,
) -> String {
session
.run_with_context(
- SessionInput::Text(format!("{} /no_think", question)),
+ Input::Text(format!("{} /no_think", question)),
context.as_deref(),
- SessionRunConfig::new_with_sampling_config(
+ RunConfig::new(
96,
- Some(SamplingConfig::Argmax),
+ true,
+ SamplingPolicy::Custom {
+ value: SamplingMethod::Greedy,
+ },
),
)
.unwrap()
@@ -268,12 +277,15 @@ fn ask_without_context(
) -> String {
session
.run(
- SessionInput::Text(format!("{} /no_think", question)),
- SessionRunConfig::new_with_sampling_config(
+ Input::Text(format!("{} /no_think", question)),
+ RunConfig::new(
96,
- Some(SamplingConfig::Argmax),
+ true,
+ SamplingPolicy::Custom {
+ value: SamplingMethod::Greedy,
+ },
),
- None::<fn(SessionOutput) -> bool>,
+ None::<fn(Output) -> bool>,
)
.unwrap()
.text
diff --git a/crates/uzu/tests/generator_test.rs b/crates/uzu/tests/generator_test.rs
deleted file mode 100644
index 4c22492e8..000000000
--- a/crates/uzu/tests/generator_test.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-mod common;
-use std::path::PathBuf;
-
-use uzu::{
- backends::metal::sampling_config::SamplingConfig,
- generator::config::{ContextLength, SamplingSeed, SpeculatorConfig},
- session::{
- session::Session,
- session_config::SessionConfig,
- session_input::SessionInput,
- session_message::{SessionMessage, SessionMessageRole},
- session_output::SessionOutput,
- session_run_config::SessionRunConfig,
- },
-};
-
-fn build_model_path() -> PathBuf {
- common::get_test_model_path()
-}
-
-#[test]
-fn test_generation_base() {
- let text = String::from("Tell about London");
- let config = SessionConfig::new(
- 8,
- SpeculatorConfig::default(),
- true,
- SamplingSeed::Custom(42),
- ContextLength::Default,
- );
- run(text, config, 128);
-}
-
-fn run(
- text: String,
- config: SessionConfig,
- tokens_limit: u64,
-) {
- let mut session = Session::new(build_model_path()).unwrap();
- session.load_with_session_config(config).unwrap();
-
- let input = SessionInput::Text(text);
- let output = session
- .run(
- input,
- SessionRunConfig::new_with_sampling_config(
- tokens_limit,
- Some(SamplingConfig::Argmax),
- ),
- Some(|_: SessionOutput| {
- return true;
- }),
- )
- .unwrap();
-
- println!("-------------------------");
- println!("{}", output.text);
- println!("-------------------------");
- println!("{:#?}", output.stats);
- println!("-------------------------");
- println!("Finish reason: {:?}", output.finish_reason);
- println!("-------------------------");
-}
-
-#[test]
-fn test_generation_scenario() {
- let system_prompt = String::from("You are a helpful assistant.");
- let messages = vec![
- String::from("Tell about London"),
- String::from("Compare with New York"),
- ];
- run_scenario(Some(system_prompt), messages);
-}
-
-fn run_scenario(
- system_prompt: Option<String>,
- user_prompts: Vec<String>,
-) {
- let config = SessionConfig::new(
- 64,
- SpeculatorConfig::default(),
- true,
- SamplingSeed::Custom(42),
- ContextLength::Default,
- );
- let mut session = Session::new(build_model_path()).unwrap();
- session.load_with_session_config(config).unwrap();
-
- let mut messages: Vec<SessionMessage> = vec![];
- if let Some(system_prompt) = system_prompt {
- messages.push(SessionMessage {
- role: SessionMessageRole::System,
- content: system_prompt.clone(),
- });
- println!("System > {}", system_prompt.clone());
- }
-
- for user_prompt in user_prompts {
- messages.push(SessionMessage {
- role: SessionMessageRole::User,
- content: user_prompt.clone(),
- });
- println!("User > {}", user_prompt.clone());
-
- let input = SessionInput::Messages(messages.clone());
- let output = session
- .run(
- input,
- SessionRunConfig::new_with_sampling_config(2048, None),
- Some(|_: SessionOutput| {
- return true;
- }),
- )
- .unwrap();
- messages.push(SessionMessage {
- role: SessionMessageRole::Assistant,
- content: output.text.clone(),
- });
- println!("Assistant > {}", output.text.clone());
- }
-}
diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rs
index 36a10606b..30178eae1 100644
--- a/crates/uzu/tests/parameters_loader_test.rs
+++ b/crates/uzu/tests/parameters_loader_test.rs
@@ -1,7 +1,7 @@
mod common;
// New integration test for ParameterLoader
-use half::f16;
+use half::bf16;
use is_close::is_close;
use uzu::{Array, backends::cpu::CPUContext, parameters::ParameterLoader};
@@ -15,11 +15,11 @@ fn test_parameter_loader_basic() {
let loader = ParameterLoader::new(&file, &context).expect("create loader");
let embeddings =
loader.get("embedding.weights").expect("weights embeddings");
- let view = embeddings.as_view::<f16>().unwrap();
- assert!(is_close!(view[[5usize, 3usize]], f16::from_f32(-0.01819)));
+ let view = embeddings.as_view::<bf16>().unwrap();
+ assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));
// tree API check
let subtree = loader.tree().subtree("embedding").unwrap();
let same = subtree.leaf("weights").unwrap();
- assert_eq!(view, same.as_view::<f16>().unwrap());
+ assert_eq!(view, same.as_view::<bf16>().unwrap());
}
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index a90899cc4..2e9e8e9ab 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -1,14 +1,16 @@
mod common;
use metal::{Device, MTLResourceOptions};
use rand::seq::SliceRandom; // for Vec::shuffle
-use uzu::backends::metal::{
- KernelDataType, MTLContext,
- kernel::{
- SamplingKernel,
- sampling::{ArgmaxStrategy, CategoricalStrategy},
+use uzu::{
+ backends::metal::{
+ KernelDataType, MTLContext,
+ kernel::{
+ SamplingKernel,
+ sampling::{ArgmaxStrategy, CategoricalStrategy},
+ },
+ metal_extensions::command_buffer_extensions::CommandBufferTimingAccess,
},
- metal_extensions::command_buffer_extensions::CommandBufferTimingAccess,
- sampling_config::SamplingConfig,
+ session::parameter::SamplingMethod,
};
// Constant seed for reproducible test results
@@ -103,7 +105,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {
// Run sampling
kernel
.encode(
- &SamplingConfig::argmax(),
+ &SamplingMethod::Greedy,
&logits_buffer,
&output_buffer,
batch_size,
@@ -236,7 +238,9 @@ fn test_topp_sampling_from_prob_exact_match(
let cb = cb_ref.to_owned();
kernel
.encode(
- &SamplingConfig::top_p(p),
+ &SamplingMethod::TopP {
+ top_p: p,
+ },
&logits_buf,
&output_buf,
batch_size,
@@ -357,7 +361,9 @@ fn test_topp_sampling_statistical_large() {
kernel
.encode(
- &SamplingConfig::top_p(TOP_P),
+ &SamplingMethod::TopP {
+ top_p: TOP_P,
+ },
&logits_buf,
&output_buf,
BATCH,
@@ -459,7 +465,9 @@ fn perf_topp_128k_vocab() {
kernel
.encode(
- &SamplingConfig::top_p(TOP_P),
+ &SamplingMethod::TopP {
+ top_p: TOP_P,
+ },
&logits_buf,
&output_buf,
BATCH,
@@ -554,7 +562,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
kernel
.encode(
- &SamplingConfig::argmax(),
+ &SamplingMethod::Greedy,
&logits_buf,
&output_buf,
BATCH,
@@ -668,7 +676,9 @@ fn test_categorical_sampling() {
kernel
.encode(
- &SamplingConfig::categorical(1.0),
+ &SamplingMethod::Temperature {
+ temperature: 1.0,
+ },
&logits_buffer,
&output_buffer,
batch_size,
@@ -804,7 +814,9 @@ fn test_categorical_sampling_statistical() {
kernel
.encode(
- &SamplingConfig::categorical(1.0),
+ &SamplingMethod::Temperature {
+ temperature: 1.0,
+ },
&logits_buffer,
&output_buffer,
BATCH,
@@ -904,7 +916,9 @@ fn perf_categorical_128k_vocab() {
kernel
.encode(
- &SamplingConfig::categorical(1.0),
+ &SamplingMethod::Temperature {
+ temperature: 1.0,
+ },
&logits_buf,
&output_buf,
BATCH,
@@ -1028,7 +1042,9 @@ fn test_categorical_1pass_vs_2pass_equivalence() {
kernel_1pass
.encode(
- &SamplingConfig::categorical(1.0),
+ &SamplingMethod::Temperature {
+ temperature: 1.0,
+ },
&logits_buffer,
&output_buffer_1pass,
batch_size,
@@ -1054,7 +1070,9 @@ fn test_categorical_1pass_vs_2pass_equivalence() {
kernel_2pass
.encode(
- &SamplingConfig::categorical(1.0),
+ &SamplingMethod::Temperature {
+ temperature: 1.0,
+ },
&logits_buffer,
&output_buffer_2pass,
batch_size,
@@ -1225,7 +1243,9 @@ fn test_categorical_sampling_with_strategy(strategy: CategoricalStrategy) {
kernel
.encode(
- &SamplingConfig::categorical(1.0),
+ &SamplingMethod::Temperature {
+ temperature: 1.0,
+ },
&logits_buffer,
&output_buffer,
batch_size,
@@ -1334,7 +1354,9 @@ fn perf_categorical_128k_vocab_with_strategy(strategy: CategoricalStrategy) {
kernel
.encode(
- &SamplingConfig::categorical(1.0),
+ &SamplingMethod::Temperature {
+ temperature: 1.0,
+ },
&logits_buf,
&output_buf,
BATCH,
diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rs
new file mode 100644
index 000000000..4507aeaf5
--- /dev/null
+++ b/crates/uzu/tests/text_session_test.rs
@@ -0,0 +1,116 @@
+mod common;
+use std::path::PathBuf;
+
+use uzu::session::{
+ config::{DecodingConfig, RunConfig, SpeculatorConfig},
+ parameter::{
+ ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy,
+ SamplingSeed,
+ },
+ session::Session,
+ types::{Input, Message, Output, Role},
+};
+
+fn build_model_path() -> PathBuf {
+ common::get_test_model_path()
+}
+
+fn build_decoding_config() -> DecodingConfig {
+ DecodingConfig::new(
+ PrefillStepSize::default(),
+ ContextLength::default(),
+ SpeculatorConfig::default(),
+ SamplingSeed::Custom(42),
+ true,
+ )
+}
+
+#[test]
+fn test_text_session_base() {
+ let text = String::from("Tell about London");
+ run(text, build_decoding_config(), 128);
+}
+
+#[test]
+fn test_text_session_scenario() {
+ let system_prompt = String::from("You are a helpful assistant.");
+ let user_prompts = vec![
+ String::from("Tell about London"),
+ String::from("Compare with New York"),
+ ];
+ run_scenario(Some(system_prompt), user_prompts);
+}
+
+fn run(
+ text: String,
+ decoding_config: DecodingConfig,
+ tokens_limit: u64,
+) {
+ let mut session =
+ Session::new(build_model_path(), decoding_config).unwrap();
+ let input = Input::Text(text);
+ let output = session
+ .run(
+ input,
+ RunConfig::new(
+ tokens_limit,
+ true,
+ SamplingPolicy::Custom {
+ value: SamplingMethod::Greedy,
+ },
+ ),
+ Some(|_: Output| {
+ return true;
+ }),
+ )
+ .unwrap();
+
+ println!("-------------------------");
+ println!("{}", output.text);
+ println!("-------------------------");
+ println!("{:#?}", output.stats);
+ println!("-------------------------");
+ println!("Finish reason: {:?}", output.finish_reason);
+ println!("-------------------------");
+}
+
+fn run_scenario(
+ system_prompt: Option<String>,
+ user_prompts: Vec<String>,
+) {
+ let mut session =
+ Session::new(build_model_path(), build_decoding_config()).unwrap();
+
+ let mut messages: Vec<Message> = vec![];
+ if let Some(system_prompt) = system_prompt {
+ messages.push(Message {
+ role: Role::System,
+ content: system_prompt.clone(),
+ });
+ println!("System > {}", system_prompt.clone());
+ }
+
+ for user_prompt in user_prompts {
+ messages.push(Message {
+ role: Role::User,
+ content: user_prompt.clone(),
+ });
+ println!("User > {}", user_prompt.clone());
+
+ let input = Input::Messages(messages.clone());
+ let output = session
+ .run(
+ input,
+ RunConfig::default(),
+ Some(|_: Output| {
+ return true;
+ }),
+ )
+ .unwrap();
+ messages.push(Message {
+ role: Role::Assistant,
+ content: output.text.clone(),
+ });
+ println!("Assistant > {}", output.text.clone());
+ }
+}
diff --git a/scripts/download_test_model.sh b/scripts/download_test_model.sh
index c0a521358..ea1425f1d 100755
--- a/scripts/download_test_model.sh
+++ b/scripts/download_test_model.sh
@@ -1,59 +1,15 @@
#!/usr/bin/env bash
-# download_test_model.sh
-#
+
set -euo pipefail
-# Exit early on non-macOS platform …
if [[ "$(uname)" != "Darwin" ]]; then
echo "Non-macOS platform detected (\"$(uname)\"); skipping test-model download."
exit 0
fi
-# Optional first argument specifies the root directory where the model
-# should be stored. Defaults to <project-root>/../mirai_models
-# Usage: bash download_test_model.sh [ROOT_DIR]
-
-PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-ROOT_DIR="${1:-$PROJECT_ROOT/../mirai_models}"
-
-MODEL_DIR="$ROOT_DIR/Meta-Llama-3.2-1B-Instruct-float16"
-
-mkdir -p "$MODEL_DIR"
-
-echo "Model directory: $MODEL_DIR"
-
-HF_BASE="https://huggingface.co/trymirai/Llama-3.2-1B-Instruct-float16/resolve/main"
-FILES=(
- "config.json ${HF_BASE}/config.json"
- "model.safetensors ${HF_BASE}/model.safetensors"
- "tokenizer.json ${HF_BASE}/tokenizer.json"
- "tokenizer_config.json ${HF_BASE}/tokenizer_config.json"
- "traces.safetensors ${HF_BASE}/traces.safetensors"
-)
-
-for ITEM in "${FILES[@]}"; do
- NAME="${ITEM%% *}"
- URL="${ITEM#* }"
- DEST="$MODEL_DIR/$NAME"
- PART="$DEST.part"
-
- if [[ -f "$DEST" ]]; then
- echo "✓ $NAME already present — skipping download"
- continue
- fi
-
- # If a partial download exists, resume it
- if [[ -f "$PART" ]]; then
- echo "↻ Resuming ${NAME} download…"
- else
- echo "↓ Downloading ${NAME}…"
- fi
-
- # Use --continue-at - (alias -C -) to resume; always write to .part file first
- curl -L --fail --progress-bar -C - "$URL" -o "$PART"
-
- # On successful download, rename to final file name
- mv -f "$PART" "$DEST"
-done
+SCRIPTS_PATH="$(cd "$(dirname "$0")" && pwd)"
+TOOLS_PATH="$SCRIPTS_PATH/tools"
-echo "✔ All required artifacts are available in $MODEL_DIR"
\ No newline at end of file
+cd $TOOLS_PATH
+uv sync
+uv run main.py download-model meta-llama/Llama-3.2-1B-Instruct
\ No newline at end of file
diff --git a/scripts/tools/.gitignore b/scripts/tools/.gitignore
new file mode 100644
index 000000000..9cefeace5
--- /dev/null
+++ b/scripts/tools/.gitignore
@@ -0,0 +1,37 @@
+.DS_Store
+
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Ruff
+.ruff_cache/
+
+# Virtual environments
+.venv
+
+# Pyright
+.mypy_cache/
+.pytest_cache/
+.pyright/
+
+# Notebooks
+notebooks/
+
+# Downloads
+downloads/
+
+# IDE
+.idea/
+
+sync.bb
+.clj-kondo
+.lsp
+
+# Profiler
+memray-*.bin
+memray-*.html
diff --git a/scripts/tools/.python-version b/scripts/tools/.python-version
new file mode 100644
index 000000000..e4fba2183
--- /dev/null
+++ b/scripts/tools/.python-version
@@ -0,0 +1 @@
+3.12
diff --git a/scripts/tools/main.py b/scripts/tools/main.py
new file mode 100644
index 000000000..5f5587580
--- /dev/null
+++ b/scripts/tools/main.py
@@ -0,0 +1,151 @@
+from pathlib import Path
+from typing import Annotated
+
+import requests
+import tomllib
+from model import Registry
+from rich.console import Console
+from rich.progress import (
+ BarColumn,
+ DownloadColumn,
+ Progress,
+ SpinnerColumn,
+ TextColumn,
+ TimeRemainingColumn,
+ TransferSpeedColumn,
+)
+from rich.table import Table
+from typer import Argument, Typer
+from utils import download_file_with_resume
+
+ROOT_PATH = Path(__file__).parent.parent.parent
+CARGO_TOML_PATH = ROOT_PATH / "Cargo.toml"
+MODELS_PATH = ROOT_PATH / "models"
+
+
+console = Console()
+err_console = Console(stderr=True)
+app = Typer(
+ rich_markup_mode="rich",
+ add_completion=False,
+ pretty_exceptions_show_locals=False,
+)
+
+
+def get_uzu_version() -> str:
+ with open(CARGO_TOML_PATH, "rb") as file:
+ cargo_toml = tomllib.load(file)
+ return cargo_toml["workspace"]["package"]["version"]
+
+
+def load_registry() -> Registry:
+ url = f"https://sdk.trymirai.com/api/v1/models/list/uzu/{get_uzu_version()}"
+ response = requests.get(url)
+ response.raise_for_status()
+ data = response.json()
+ registry = Registry.from_dict(data)
+ return registry
+
+
[email protected](help="List models")
+def list_models():
+ try:
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ console=console,
+ transient=True,
+ ) as progress:
+ task = progress.add_task("Loading registry...", total=None)
+ registry = load_registry()
+ progress.update(task, completed=True)
+
+ table = Table()
+ table.add_column("Repo", style="cyan")
+ table.add_column("Name", style="green")
+ for model in registry.models:
+ table.add_row(model.repod_id, model.name)
+ console.print(table)
+
+ except Exception as e:
+ err_console.print(f"[red]Error: {e}[/red]")
+
+
[email protected](help="Download model")
+def download_model(
+ model_repo: Annotated[
+ str,
+ Argument(
+ help=(
+ "Hugging Face model repo. Example: [cyan]'meta-llama/Llama-3.2-1B-Instruct'[/cyan]."
+ ),
+ ),
+ ],
+):
+ try:
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ console=console,
+ transient=True,
+ ) as progress:
+ task = progress.add_task("Loading registry...", total=None)
+ registry = load_registry()
+ progress.update(task, completed=True)
+
+ model = None
+ for current_model in registry.models:
+ if current_model.repod_id == model_repo:
+ model = current_model
+ break
+ if not model:
+ err_console.print(
+ f"[red]Error: Model '{model_repo}' not found in registry[/red]"
+ )
+ return
+
+ engine_version = get_uzu_version()
+ engine_path = MODELS_PATH / engine_version
+ model_path = engine_path / model.name
+ model_path.mkdir(parents=True, exist_ok=True)
+
+ files_to_download = []
+ for file in model.files:
+ file_path = model_path / file.name
+ if file_path.exists():
+ console.print(f"[green]✓[/green] {file.name} already exists")
+ else:
+ files_to_download.append((file, file_path))
+ if not files_to_download:
+ console.print("[green]All files already downloaded![/green]")
+ return
+
+ console.print(
+ f"Downloading {len(files_to_download)} file(s) for [cyan]{model.name}[/cyan]..."
+ )
+ with Progress(
+ TextColumn("[progress.description]{task.description}"),
+ BarColumn(),
+ DownloadColumn(),
+ TransferSpeedColumn(),
+ TimeRemainingColumn(),
+ console=console,
+ ) as progress:
+ for file, file_path in files_to_download:
+ task = progress.add_task(f"[cyan]{file.name}[/cyan]", total=None)
+
+ try:
+ download_file_with_resume(file.url, file_path, progress, task)
+ progress.update(task, description=f"[green]✓[/green] {file.name}")
+ except Exception as e:
+ progress.update(task, description=f"[red]✗[/red] {file.name}")
+ raise e
+
+ console.print(f"[green]Successfully downloaded model: {model.name}[/green]")
+
+ except Exception as e:
+ err_console.print(f"[red]Error: {e}[/red]")
+
+
+if __name__ == "__main__":
+ app()
diff --git a/scripts/tools/model.py b/scripts/tools/model.py
new file mode 100644
index 000000000..3bb39f5f3
--- /dev/null
+++ b/scripts/tools/model.py
@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass(frozen=True)
+class File:
+ name: str
+ url: str
+
+ @classmethod
+ def from_dict(cls, data: dict) -> "File":
+ return cls(name=data["name"], url=data["url"])
+
+
+@dataclass(frozen=True)
+class Model:
+ id: str
+ toolchain_version: str
+ repod_id: str
+ name: str
+ files: List[File]
+
+ @classmethod
+ def from_dict(cls, data: dict) -> "Model":
+ return cls(
+ id=data["id"],
+ toolchain_version=data["toolchainVersion"],
+ repod_id=data["repoId"],
+ name=data["name"],
+ files=[File.from_dict(file) for file in data["files"]],
+ )
+
+
+@dataclass(frozen=True)
+class Registry:
+ models: List[Model]
+
+ @classmethod
+ def from_dict(cls, data: dict) -> "Registry":
+ return cls(models=[Model.from_dict(model) for model in data["models"]])
diff --git a/scripts/tools/pyproject.toml b/scripts/tools/pyproject.toml
new file mode 100644
index 000000000..b0a18ce64
--- /dev/null
+++ b/scripts/tools/pyproject.toml
@@ -0,0 +1,9 @@
+[project]
+name = "tools"
+version = "0.1.0"
+requires-python = ">=3.12"
+dependencies = [
+ "requests>=2.32.5",
+ "rich>=14.1.0",
+ "typer>=0.19.2",
+]
diff --git a/scripts/tools/utils.py b/scripts/tools/utils.py
new file mode 100644
index 000000000..e4f70bea7
--- /dev/null
+++ b/scripts/tools/utils.py
@@ -0,0 +1,37 @@
+from pathlib import Path
+
+import requests
+from rich.progress import Progress
+
+
+def download_file_with_resume(
+ url: str, file_path: Path, progress: Progress, task_id
+) -> None:
+ part_file_path = file_path.with_suffix(file_path.suffix + ".part")
+
+ resume_position = 0
+ mode = "wb"
+ if part_file_path.exists():
+ resume_position = part_file_path.stat().st_size
+ mode = "ab"
+
+ headers = {}
+ if resume_position > 0:
+ headers["Range"] = f"bytes={resume_position}-"
+
+ response = requests.get(url, headers=headers, stream=True)
+ response.raise_for_status()
+
+ total_size = int(response.headers.get("content-length", 0))
+ if resume_position > 0 and response.status_code == 206:
+ total_size += resume_position
+
+ progress.update(task_id, total=total_size, completed=resume_position)
+
+ with open(part_file_path, mode) as file:
+ for chunk in response.iter_content(chunk_size=8192):
+ if chunk:
+ file.write(chunk)
+ progress.update(task_id, advance=len(chunk))
+
+ part_file_path.rename(file_path)
diff --git a/scripts/tools/uv.lock b/scripts/tools/uv.lock
new file mode 100644
index 000000000..632477377
--- /dev/null
+++ b/scripts/tools/uv.lock
@@ -0,0 +1,201 @@
+version = 1
+revision = 2
+requires-python = ">=3.12"
+
+[[package]]
+name = "certifi"
+version = "2025.8.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" },
+]
+
+[[package]]
+name = "charset-normalizer"
+version = "3.4.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" },
+ { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" },
+ { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" },
+ { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" },
+ { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" },
+ { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" },
+ { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" },
+ { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" },
+ { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" },
+ { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" },
+ { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" },
+ { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" },
+ { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" },
+ { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" },
+ { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" },
+ { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" },
+ { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" },
+ { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" },
+]
+
+[[package]]
+name = "click"
+version = "8.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" },
+]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
+[[package]]
+name = "idna"
+version = "3.10"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },
+]
+
+[[package]]
+name = "markdown-it-py"
+version = "4.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "mdurl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },
+]
+
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.19.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
+]
+
+[[package]]
+name = "requests"
+version = "2.32.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "certifi" },
+ { name = "charset-normalizer" },
+ { name = "idna" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" },
+]
+
+[[package]]
+name = "rich"
+version = "14.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "markdown-it-py" },
+ { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" },
+]
+
+[[package]]
+name = "shellingham"
+version = "1.5.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
+]
+
+[[package]]
+name = "tools"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+ { name = "requests" },
+ { name = "rich" },
+ { name = "typer" },
+]
+
+[package.metadata]
+requires-dist = [
+ { name = "requests", specifier = ">=2.32.5" },
+ { name = "rich", specifier = ">=14.1.0" },
+ { name = "typer", specifier = ">=0.19.2" },
+]
+
+[[package]]
+name = "typer"
+version = "0.19.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "click" },
+ { name = "rich" },
+ { name = "shellingham" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/21/ca/950278884e2ca20547ff3eb109478c6baf6b8cf219318e6bc4f666fad8e8/typer-0.19.2.tar.gz", hash = "sha256:9ad824308ded0ad06cc716434705f691d4ee0bfd0fb081839d2e426860e7fdca", size = 104755, upload-time = "2025-09-23T09:47:48.256Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl", hash = "sha256:755e7e19670ffad8283db353267cb81ef252f595aa6834a0d1ca9312d9326cb9", size = 46748, upload-time = "2025-09-23T09:47:46.777Z" },
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.15.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
+]
+
+[[package]]
+name = "urllib3"
+version = "2.5.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
+]