A high-performance inference engine for AI models
Improve API (#33)
Aleksei Savin43fb2e91a2c7
diff --git a/.gitignore b/.gitignoreindex ee2c89ed3..bd22a0711 100644--- a/.gitignore+++ b/.gitignore@@ -20,3 +20,6 @@ target/# option (not recommended) you can uncomment the following to ignore the entire idea folder.#.idea/*.DS_Store++# Models+models/\ No newline at end of filediff --git a/Cargo.lock b/Cargo.lockindex f5cd86c23..4cb11608b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -317,7 +317,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"[[package]]name = "cli"-version = "0.1.3"+version = "0.1.4"dependencies = ["clap","console 0.16.0",@@ -2640,7 +2640,7 @@ dependencies = [[[package]]name = "uzu"-version = "0.1.3"+version = "0.1.4"dependencies = ["bitflags 2.9.4","bytemuck",diff --git a/Cargo.toml b/Cargo.tomlindex 780e676fe..c69d8505e 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -3,7 +3,7 @@ members = ["crates/cli", "crates/uzu"]resolver = "3"[workspace.package]-version = "0.1.3"+version = "0.1.4"edition = "2024"license = "MIT"authors = ["Mirai Tech <https://trymirai.com>"]diff --git a/README.md b/README.mdindex f0c165c56..bd05bfcff 100644--- a/README.md+++ b/README.md@@ -35,13 +35,13 @@ uv run lalamo list-modelsThen, export the specific one:```bash-uv run lalamo convert meta-llama/Llama-3.2-1B-Instruct --precision float16+uv run lalamo convert meta-llama/Llama-3.2-1B-Instruct```Alternatively, you can download a prepared model using the sample script:```bash-./scripts/download_test_model.sh $MODEL_PATH+./scripts/download_test_model.sh```### Bindings@@ -80,32 +80,25 @@ Then, create an inference `Session` with a specific model and configuration:```rustuse std::path::PathBuf;-use uzu::{- backends::metal::sampling_config::SamplingConfig,- session::{- session::Session, session_config::SessionConfig,- session_input::SessionInput, session_output::SessionOutput,- session_run_config::SessionRunConfig,- },++use uzu::session::{+ config::{DecodingConfig, RunConfig},+ session::Session,+ types::{Input, Output},};fn main() -> Result<(), Box<dyn std::error::Error>> {let model_path = PathBuf::from("MODEL_PATH");-- let mut session = Session::new(model_path.clone())?;- session.load_with_session_config(SessionConfig::default())?;-- let input = SessionInput::Text("Tell about London".to_string());-- let tokens_limit = 128;- let run_config = SessionRunConfig::new_with_sampling_config(- tokens_limit,- Some(SamplingConfig::default())- );-- let output = session.run(input, run_config, Some(|_: SessionOutput| {- return true;- }))?;+ let mut session = Session::new(model_path, DecodingConfig::default())?;++ let input = Input::Text(String::from("Tell about London"));+ let output = session.run(+ input,+ RunConfig::default().tokens_limit(128),+ Some(|_: Output| {+ return true;+ }),+ )?;println!("{}", output.text);Ok(())}diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.tomlindex f77ad4f8c..d65e5724f 100644--- a/crates/cli/Cargo.toml+++ b/crates/cli/Cargo.toml@@ -1,6 +1,6 @@[package]name = "cli"-version = "0.1.3"+version = "0.1.4"edition = "2024"[[bin]]diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rsindex 147afc884..da117fa2e 100644--- a/crates/cli/src/handlers/run.rs+++ b/crates/cli/src/handlers/run.rs@@ -7,13 +7,14 @@ use console::Style;use indicatif::{ProgressBar, ProgressStyle};use inquire::Text;use uzu::session::{- session_input::SessionInput, session_output::SessionOutput,- session_run_config::SessionRunConfig,+ config::RunConfig,+ parameter::SamplingPolicy,+ types::{Input, Output},};use crate::server::load_session;-fn format_output(output: SessionOutput) -> String {+fn format_output(output: Output) -> String {let stats = &output.stats;let tokens_per_second = if let Some(generate_stats) = &stats.generate_stats{@@ -72,7 +73,7 @@ pub fn handle_run(let progress_bar_for_progress = progress_bar.clone();let is_model_running_for_progress = is_model_running.clone();- let session_progress = move |output: SessionOutput| {+ let session_progress = move |output: Output| {if !is_model_running_for_progress.load(Ordering::SeqCst) {return false;}@@ -90,8 +91,8 @@ pub fn handle_run(};let session_output = match session.run(- SessionInput::Text(input.to_string()),- SessionRunConfig::new(tokens_limit as u64),+ Input::Text(input.to_string()),+ RunConfig::new(tokens_limit as u64, true, SamplingPolicy::Default),Some(session_progress),) {Ok(output) => output,diff --git a/crates/cli/src/server/chat_completions.rs b/crates/cli/src/server/chat_completions.rsindex 38313b8e8..5054dc427 100644--- a/crates/cli/src/server/chat_completions.rs+++ b/crates/cli/src/server/chat_completions.rs@@ -7,13 +7,12 @@ use rocket::{State, post, serde::json::Json};use serde::{Deserialize, Serialize};use uuid::Uuid;use uzu::session::{- session_input::SessionInput,- session_message::{SessionMessage, SessionMessageRole},- session_output::{- SessionOutput, SessionOutputFinishReason, SessionOutputRunStats,- SessionOutputStats, SessionOutputStepStats, SessionOutputTotalStats,+ config::RunConfig,+ parameter::SamplingPolicy,+ types::{+ FinishReason, Input, Message, Output, Role, RunStats, Stats, StepStats,+ TotalStats,},- session_run_config::SessionRunConfig,};use crate::server::SessionState;@@ -21,7 +20,7 @@ use crate::server::SessionState;#[derive(Serialize, Deserialize, Clone)]pub struct ChatMessage {pub content: String,- pub role: SessionMessageRole,+ pub role: Role,}#[derive(Serialize, Deserialize)]@@ -45,7 +44,7 @@ pub struct ChatCompletionResponse {pub created: i64,pub model: String,pub choices: Vec<ChatCompletionChoice>,- pub stats: SessionOutputStats,+ pub stats: Stats,}#[post("/chat/completions", format = "json", data = "<request>")]@@ -76,16 +75,17 @@ pub fn handle_chat_completions(let model_name = state.model_name.clone();let system_prompt_key = request.system_prompt_key;let tokens_limit = request.max_completion_tokens.unwrap_or(2048);- let messages: Vec<SessionMessage> = request+ let messages: Vec<Message> = request.messages.into_iter()- .map(|m| SessionMessage {+ .map(|m| Message {content: m.content,role: m.role,}).collect();- let input = SessionInput::Messages(messages);- let run_config = SessionRunConfig::new(tokens_limit);+ let input = Input::Messages(messages);+ let run_config =+ RunConfig::new(tokens_limit, true, SamplingPolicy::Default);let start_time = std::time::Instant::now();let mut session = state.session_wrapper.lock();@@ -116,24 +116,24 @@ pub fn handle_chat_completions(index: 0,message: ChatMessage {content: format!("Error: {}", e),- role: SessionMessageRole::Assistant,+ role: Role::Assistant,},finish_reason: "error".to_string(),}],- stats: SessionOutputStats {- prefill_stats: SessionOutputStepStats {+ stats: Stats {+ prefill_stats: StepStats {duration: 0.0,suffix_length: 0,tokens_count: 0,tokens_per_second: 0.0,- model_run: SessionOutputRunStats {+ model_run: RunStats {count: 0,average_duration: 0.0,},run: None,},generate_stats: None,- total_stats: SessionOutputTotalStats {+ total_stats: TotalStats {duration: 0.0,tokens_count_input: 0,tokens_count_output: 0,@@ -161,24 +161,24 @@ pub fn handle_chat_completions(index: 0,message: ChatMessage {content: format!("Error: {}", e),- role: SessionMessageRole::Assistant,+ role: Role::Assistant,},finish_reason: "error".to_string(),}],- stats: SessionOutputStats {- prefill_stats: SessionOutputStepStats {+ stats: Stats {+ prefill_stats: StepStats {duration: 0.0,suffix_length: 0,tokens_count: 0,tokens_per_second: 0.0,- model_run: SessionOutputRunStats {+ model_run: RunStats {count: 0,average_duration: 0.0,},run: None,},generate_stats: None,- total_stats: SessionOutputTotalStats {+ total_stats: TotalStats {duration: 0.0,tokens_count_input: 0,tokens_count_output: 0,@@ -194,15 +194,14 @@ pub fn handle_chat_completions(println!("output: {:?}", &output);- let SessionOutput {+ let Output {text,stats,finish_reason,..} = output;- let finish_reason_val =- finish_reason.unwrap_or(SessionOutputFinishReason::Cancelled);+ let finish_reason_val = finish_reason.unwrap_or(FinishReason::Cancelled);println!("📤 [{}] Sending response:", id);println!(" Response length: {} chars", text.len());@@ -236,7 +235,7 @@ pub fn handle_chat_completions(index: 0,message: ChatMessage {content: text,- role: SessionMessageRole::Assistant,+ role: Role::Assistant,},finish_reason: format!("{:?}", finish_reason_val).to_lowercase(),}],diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rsindex c88f1db23..e92854f43 100644--- a/crates/cli/src/server/state.rs+++ b/crates/cli/src/server/state.rs@@ -8,12 +8,11 @@ use std::{use console::Style;use indicatif::{ProgressBar, ProgressStyle};use uzu::session::{- session::Session, session_config::SessionConfig,- session_context::SessionContext,+ config::DecodingConfig, helpers::Context, session::Session,};pub struct ContextCache {- pub map: HashMap<String, Rc<SessionContext>>,+ pub map: HashMap<String, Rc<Context>>,pub order: VecDeque<String>,pub capacity: usize,}@@ -30,8 +29,8 @@ impl ContextCache {pub fn insert(&mut self,key: String,- context: Rc<SessionContext>,- ) -> Option<(String, Rc<SessionContext>)> {+ context: Rc<Context>,+ ) -> Option<(String, Rc<Context>)> {if self.map.contains_key(&key) {return None;}@@ -51,7 +50,7 @@ impl ContextCache {pub fn get(&self,key: &str,- ) -> Option<Rc<SessionContext>> {+ ) -> Option<Rc<Context>> {self.map.get(key).cloned()}}@@ -97,12 +96,8 @@ pub fn load_session(model_path: String) -> Session {);progress_bar.set_message(model_name.clone());- let mut session =- Session::new(model_path_buf).expect("Failed to create session");-- session- .load_with_session_config(SessionConfig::default())- .expect("Failed to load session");+ let session = Session::new(model_path_buf, DecodingConfig::default())+ .expect("Failed to create session");progress_bar.set_style(ProgressStyle::default_spinner().template("Loaded: {msg}").unwrap(),diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex c50a238c4..ce96d709c 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -1,6 +1,6 @@[package]name = "uzu"-version = "0.1.3"+version = "0.1.4"edition = "2024"build = "build.rs"diff --git a/crates/uzu/build.rs b/crates/uzu/build.rsindex a937909d7..f3c06e90f 100644--- a/crates/uzu/build.rs+++ b/crates/uzu/build.rs@@ -107,7 +107,6 @@ fn compile_metal_shaders() {for file in sorted_files {println!("cargo:rerun-if-changed={}", file.display());}- println!("cargo:rerun-if-changed=src/ffi.rs");println!("cargo:rerun-if-changed=build.rs");}diff --git a/crates/uzu/examples/dequantize_qkv_layer0.rs b/crates/uzu/examples/dequantize_qkv_layer0.rsindex 7a06a70ce..79412f7eb 100644--- a/crates/uzu/examples/dequantize_qkv_layer0.rs+++ b/crates/uzu/examples/dequantize_qkv_layer0.rs@@ -23,7 +23,7 @@ use uzu::{graph::common::{GraphConstructionError, load_constant},},parameters::{ParameterLoader, ParameterLoaderError},- storage::{NSSearchPathDirectory, root_dir},+ utils::{NSSearchPathDirectory, root_dir},};#[derive(Debug, Error)]diff --git a/crates/uzu/examples/quick_start.rs b/crates/uzu/examples/quick_start.rsnew file mode 100644index 000000000..809aae564--- /dev/null+++ b/crates/uzu/examples/quick_start.rs@@ -0,0 +1,23 @@+use std::path::PathBuf;++use uzu::session::{+ config::{DecodingConfig, RunConfig},+ session::Session,+ types::{Input, Output},+};++fn main() -> Result<(), Box<dyn std::error::Error>> {+ let model_path = PathBuf::from("MODEL_PATH");+ let mut session = Session::new(model_path, DecodingConfig::default())?;++ let input = Input::Text(String::from("Tell about London"));+ let output = session.run(+ input,+ RunConfig::default().tokens_limit(128),+ Some(|_: Output| {+ return true;+ }),+ )?;+ println!("{}", output.text);+ Ok(())+}diff --git a/crates/uzu/src/backends/metal/forward_pass/state.rs b/crates/uzu/src/backends/metal/forward_pass/state.rsindex 56331e285..7997291da 100644--- a/crates/uzu/src/backends/metal/forward_pass/state.rs+++ b/crates/uzu/src/backends/metal/forward_pass/state.rs@@ -11,12 +11,10 @@ use super::{};use crate::{DataType, DeviceContext,- backends::metal::{- forward_pass::traces::DecoderActivationTrace,- sampling_config::SamplingConfig,- },+ backends::metal::forward_pass::traces::DecoderActivationTrace,config::{DecoderConfig, EmbeddingConfig},parameters::ParameterTree,+ session::parameter::SamplingMethod,};type ArrayCell = RefCell<MetalArray>;@@ -333,7 +331,7 @@ pub struct ForwardPassState {/// [suffix_length] - u32 sampling output bufferpub sampling_output: Option<ArrayCell>,/// Current sampling configuration for this forward pass- pub sampling_config: Option<SamplingConfig>,+ pub sampling_method: Option<SamplingMethod>,pub traces: Option<Rc<RefCell<DecoderActivationTrace>>>,}@@ -473,7 +471,7 @@ impl ForwardPassState {shared_buffers,aux_buffers,sampling_output: Some(sampling_output),- sampling_config: None,+ sampling_method: None,traces,}}diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rsindex 708569170..e1dcabe10 100644--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs@@ -8,13 +8,15 @@ use metal::{use mpsgraph::CommandBuffer as MPSCommandBuffer;use thiserror::Error;-use crate::backends::metal::{- KernelDataType, MTLContext, MTLError,- forward_pass::{- ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},+use crate::{+ backends::metal::{+ KernelDataType, MTLContext, MTLError,+ forward_pass::{+ ForwardPassState,+ encodable_with_state::{EncodableWithState, EncodingParameters},+ },},- sampling_config::SamplingConfig,+ session::parameter::SamplingMethod,};const BLOCK_SIZE: usize = 1024;@@ -270,7 +272,7 @@ impl SamplingKernel {pub fn encode(&self,- sampling_config: &SamplingConfig,+ sampling_method: &SamplingMethod,logits_buffer: &MTLBuffer,sampled_tokens_buffer: &MTLBuffer,batch_size: usize,@@ -279,7 +281,7 @@ impl SamplingKernel {) -> Result<(), SamplingError> {let compute_encoder = command_buffer.new_compute_command_encoder();self.encode_with_encoder(- sampling_config,+ sampling_method,logits_buffer,sampled_tokens_buffer,batch_size,@@ -292,7 +294,7 @@ impl SamplingKernel {pub fn encode_with_encoder(&self,- sampling_config: &SamplingConfig,+ sampling_method: &SamplingMethod,logits_buffer: &MTLBuffer,sampled_tokens_buffer: &MTLBuffer,batch_size: usize,@@ -321,8 +323,8 @@ impl SamplingKernel {let threads_per_threadgroup = MTLSize::new(BLOCK_SIZE as u64, 1, 1);- match sampling_config {- SamplingConfig::Argmax => self.encode_argmax(+ match sampling_method {+ SamplingMethod::Greedy => self.encode_argmax(logits_buffer,sampled_tokens_buffer,batch_size_u32,@@ -331,7 +333,7 @@ impl SamplingKernel {threads_per_threadgroup,compute_encoder,),- SamplingConfig::TopP {+ SamplingMethod::TopP {top_p,} => self.encode_top_p(*top_p,@@ -343,7 +345,7 @@ impl SamplingKernel {threads_per_threadgroup,compute_encoder,),- SamplingConfig::Categorical {+ SamplingMethod::Temperature {temperature,} => self.encode_categorical(*temperature,@@ -801,7 +803,7 @@ impl EncodableWithState for SamplingKernelEncodable {command_buffer: &MPSCommandBuffer,parameters: &EncodingParameters,) {- let sampling_config = state.sampling_config.unwrap_or_default();+ let sampling_method = state.sampling_method.unwrap_or_default();let logits_binding = state.arrays(&[crate::backends::metal::forward_pass::ArrayId::Logits]);@@ -829,7 +831,7 @@ impl EncodableWithState for SamplingKernelEncodable {let root_command_buffer =command_buffer.root_command_buffer().to_owned();if let Err(e) = self.kernel.encode(- &sampling_config,+ &sampling_method,unsafe { &logits.mtl_buffer() },unsafe { &output_buffer_ref.mtl_buffer() },batch_size,diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rsindex a62eabc78..f6ea05d42 100644--- a/crates/uzu/src/backends/metal/mod.rs+++ b/crates/uzu/src/backends/metal/mod.rs@@ -28,4 +28,3 @@ pub use media_utils::{MetalImagePreprocessor,};pub use placement_analysis::PlacementAnalysis;-pub mod sampling_config;diff --git a/crates/uzu/src/backends/metal/sampling_config.rs b/crates/uzu/src/backends/metal/sampling_config.rsdeleted file mode 100644index 81662b667..000000000--- a/crates/uzu/src/backends/metal/sampling_config.rs+++ /dev/null@@ -1,34 +0,0 @@-#[derive(Debug, Clone, Copy, PartialEq)]-pub enum SamplingConfig {- Argmax,- TopP {- top_p: f32,- },- Categorical {- temperature: f32,- },-}--impl SamplingConfig {- pub fn argmax() -> Self {- Self::Argmax- }-- pub fn top_p(top_p: f32) -> Self {- Self::TopP {- top_p,- }- }-- pub fn categorical(temperature: f32) -> Self {- Self::Categorical {- temperature,- }- }-}--impl Default for SamplingConfig {- fn default() -> Self {- Self::Argmax- }-}diff --git a/crates/uzu/src/array.rs b/crates/uzu/src/device/array.rssimilarity index 100%rename from crates/uzu/src/array.rsrename to crates/uzu/src/device/array.rsdiff --git a/crates/uzu/src/data_type.rs b/crates/uzu/src/device/data_type.rssimilarity index 100%rename from crates/uzu/src/data_type.rsrename to crates/uzu/src/device/data_type.rsdiff --git a/crates/uzu/src/device_context.rs b/crates/uzu/src/device/device_context.rssimilarity index 100%rename from crates/uzu/src/device_context.rsrename to crates/uzu/src/device/device_context.rsdiff --git a/crates/uzu/src/device/mod.rs b/crates/uzu/src/device/mod.rsnew file mode 100644index 000000000..52ae2e87a--- /dev/null+++ b/crates/uzu/src/device/mod.rs@@ -0,0 +1,6 @@+pub mod array;+pub use array::*;+pub mod data_type;+pub use data_type::*;+pub mod device_context;+pub use device_context::*;diff --git a/crates/uzu/src/generator/config.rs b/crates/uzu/src/generator/config.rsdeleted file mode 100644index 15756770e..000000000--- a/crates/uzu/src/generator/config.rs+++ /dev/null@@ -1,105 +0,0 @@-use std::sync::Arc;--use rand::prelude::*;-use tokenizers::Tokenizer;--use crate::speculators::{- empty_speculator::EmptySpeculator, speculator::Speculator,-};--#[derive(Clone)]-pub struct SpeculatorConfig {- pub number_of_speculated_tokens: usize,- pub speculator: Arc<dyn Speculator>,-}--impl Default for SpeculatorConfig {- fn default() -> Self {- Self {- number_of_speculated_tokens: 0,- speculator: Arc::new(EmptySpeculator {}),- }- }-}--#[derive(Debug, Clone, Copy)]-pub enum ContextLength {- // 8192 is the default max prefix length- Default,- // Custom max prefix length- Custom(u64),-}--impl Default for ContextLength {- fn default() -> Self {- ContextLength::Default- }-}--impl ContextLength {- pub fn get_value(&self) -> u64 {- match self {- ContextLength::Default => 8192,- ContextLength::Custom(length) => *length,- }- }-}--#[derive(Debug, Clone, Copy)]-pub enum SamplingSeed {- Default,- Custom(u64),-}--impl Default for SamplingSeed {- fn default() -> Self {- SamplingSeed::Default- }-}--impl SamplingSeed {- pub fn get_value(&self) -> u64 {- match self {- SamplingSeed::Default => rand::rng().random::<u64>(),- SamplingSeed::Custom(seed) => *seed,- }- }-}--pub trait GeneratorConfigProvider {- fn generator_config(- &self,- tokenizer: &Tokenizer,- ) -> GeneratorConfig;-}--#[derive(Clone)]-pub struct GeneratorConfig {- pub prefill_step_size: usize,- pub speculator_config: SpeculatorConfig,- pub allow_pre_encode: bool,- pub sampling_seed: u64,- pub context_length: usize,-}--impl GeneratorConfig {- pub fn new(- prefill_step_size: usize,- speculator_config: SpeculatorConfig,- allow_pre_encode: bool,- sampling_seed: SamplingSeed,- context_length: ContextLength,- ) -> Self {- Self {- prefill_step_size,- speculator_config,- allow_pre_encode,- sampling_seed: sampling_seed.get_value(),- context_length: context_length.get_value() as usize,- }- }-- pub fn generate_suffix_length(&self) -> usize {- self.speculator_config.number_of_speculated_tokens + 1- }-}diff --git a/crates/uzu/src/generator/context.rs b/crates/uzu/src/generator/context.rsindex bea5a510d..b44d65b94 100644--- a/crates/uzu/src/generator/context.rs+++ b/crates/uzu/src/generator/context.rs@@ -4,7 +4,6 @@ use metal::SharedEvent;use mpsgraph::CommandBuffer as MPSCommandBuffer;use objc2::rc::Retained;-use super::{config::GeneratorConfig, error::GeneratorError};use crate::{DataType,backends::metal::{@@ -14,8 +13,13 @@ use crate::{forward_pass::{ForwardPassBuffers, SharedBuffers},kernel::SamplingKernelEncodable,},- config::ModelMetadata,+ config::{LanguageModelConfig, ModelMetadata},parameters::ParameterLoader,+ session::{+ config::DecodingConfig,+ parameter::{ConfigResolvableValue, ResolvableValue},+ types::Error,+ },};pub struct GeneratorContext {@@ -28,6 +32,7 @@ pub struct GeneratorContext {pub shared_buffers: Rc<RefCell<SharedBuffers>>,pub scratch_buffers: ForwardPassBuffers,+ pub model_config: LanguageModelConfig,pub model_shape: ModelShape,pub executables: DecoderExecutables,pub kv_cache_update: Box<KVCacheUpdate>,@@ -37,10 +42,10 @@ pub struct GeneratorContext {impl GeneratorContext {pub fn new(model_path: &Path,- config: &GeneratorConfig,- ) -> Result<Self, GeneratorError> {+ decoding_config: &DecodingConfig,+ ) -> Result<Self, Error> {let mtl_device = metal::Device::system_default()- .ok_or(GeneratorError::UnableToCreateMetalContext)?;+ .ok_or(Error::UnableToCreateMetalContext)?;let mtl_command_queue =mtl_device.new_command_queue_with_max_command_buffer_count(1024);@@ -49,38 +54,41 @@ impl GeneratorContext {let config_path = model_path.join("config.json");if !config_path.exists() {- return Err(GeneratorError::UnableToLoadConfig);+ return Err(Error::UnableToLoadConfig);}- let config_file = File::open(&config_path)- .map_err(|_| GeneratorError::UnableToLoadConfig)?;+ let config_file =+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;let model_metadata: ModelMetadata =serde_json::from_reader(BufReader::new(config_file))- .map_err(|_| GeneratorError::UnableToLoadConfig)?;+ .map_err(|_| Error::UnableToLoadConfig)?;let decoder_config =Rc::new(model_metadata.model_config.decoder_config.clone());let model_shape = ModelShape::from_decoder_config(&decoder_config);- let prefill_step_size = config.prefill_step_size;- let generate_suffix_length = config.generate_suffix_length();- let max_prefix_length: usize =- std::cmp::min(config.context_length, decoder_config.context_length);+ let prefill_step_size = decoding_config+ .prefill_step_size+ .resolve(&model_metadata.model_config);+ let generate_suffix_length = decoding_config.generate_suffix_length();+ let max_prefix_length: usize = decoding_config+ .context_length+ .resolve(&model_metadata.model_config);let max_suffix_length: usize =std::cmp::max(prefill_step_size, generate_suffix_length);let mtl_context = Rc::new(MTLContext::new(mtl_device, mtl_command_queue)- .map_err(|_| GeneratorError::UnableToCreateMetalContext)?,+ .map_err(|_| Error::UnableToCreateMetalContext)?,);let compilation_config = Rc::new(CompilationConfig::default());let weights_path = model_path.join("model.safetensors");if !weights_path.exists() {- return Err(GeneratorError::UnableToLoadWeights);+ return Err(Error::UnableToLoadWeights);}let weights_file = File::open(&weights_path)- .map_err(|_| GeneratorError::UnableToLoadWeights)?;+ .map_err(|_| Error::UnableToLoadWeights)?;let loader = ParameterLoader::new(&weights_file, &mtl_context)- .map_err(|_| GeneratorError::UnableToLoadWeights)?;+ .map_err(|_| Error::UnableToLoadWeights)?;let root_loader_view = loader.tree();let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(@@ -120,7 +128,7 @@ impl GeneratorContext {kernel_data_type,max_prefix_length,)- .unwrap(),+ .map_err(|_| Error::UnableToCreateMetalContext)?,);let gpu_sampler = SamplingKernelEncodable::new(@@ -128,9 +136,9 @@ impl GeneratorContext {kernel_data_type,max_suffix_length,decoder_config.vocab_size,- config.sampling_seed,+ decoding_config.sampling_seed.resolve(),)- .map_err(|_| GeneratorError::UnableToCreateMetalContext)?;+ .map_err(|_| Error::UnableToCreateMetalContext)?;let kv_update_event = mtl_context.device.new_shared_event();let kv_update_signal = 1;@@ -143,6 +151,7 @@ impl GeneratorContext {kv_cache,shared_buffers,scratch_buffers,+ model_config: model_metadata.model_config.clone(),model_shape,executables,kv_cache_update,diff --git a/crates/uzu/src/generator/error.rs b/crates/uzu/src/generator/error.rsdeleted file mode 100644index 80d8fb323..000000000--- a/crates/uzu/src/generator/error.rs+++ /dev/null@@ -1,6 +0,0 @@-#[derive(Debug)]-pub enum GeneratorError {- UnableToCreateMetalContext,- UnableToLoadConfig,- UnableToLoadWeights,-}diff --git a/crates/uzu/src/generator/generator.rs b/crates/uzu/src/generator/generator.rsindex dc5208574..ba9234cc5 100644--- a/crates/uzu/src/generator/generator.rs+++ b/crates/uzu/src/generator/generator.rs@@ -3,28 +3,28 @@ use std::{collections::HashMap, path::Path, time::Instant};use mpsgraph::CommandBuffer;use super::{- config::GeneratorConfig,context::GeneratorContext,result::{GenerateResult, PrefillResult},tasks::{GeneratorEncodedTask, GeneratorRunTask},};use crate::{Array,- backends::metal::{- forward_pass::{- ForwardPassState,- encodable_with_state::{EncodableWithState, EncodingParameters},- kv_cache::INVALID_POSITION,- },- sampling_config::SamplingConfig,+ backends::metal::forward_pass::{+ ForwardPassState,+ encodable_with_state::{EncodableWithState, EncodingParameters},+ kv_cache::INVALID_POSITION,},- env_utils::MetalEnvVar,- generator::error::GeneratorError,linearizer::trie::TokenTrie,+ session::{+ config::DecodingConfig,+ parameter::{ConfigResolvableValue, SamplingMethod},+ types::Error,+ },+ utils::env_utils::MetalEnvVar,};pub struct Generator {- pub config: GeneratorConfig,+ pub decoding_config: DecodingConfig,pub tokens: Vec<u64>,pub context: GeneratorContext,@@ -36,15 +36,16 @@ pub struct Generator {impl Generator {pub fn new(model_path: &Path,- config: GeneratorConfig,- ) -> Result<Self, GeneratorError> {- let context = GeneratorContext::new(model_path, &config)?;+ decoding_config: DecodingConfig,+ ) -> Result<Self, Error> {+ let context = GeneratorContext::new(model_path, &decoding_config)?;- let prefill_step_size = config.prefill_step_size;- let generate_suffix_length = config.generate_suffix_length();+ let prefill_step_size =+ decoding_config.prefill_step_size.resolve(&context.model_config);+ let generate_suffix_length = decoding_config.generate_suffix_length();let mut generator = Self {- config: config,+ decoding_config: decoding_config,tokens: Vec::new(),context,encoded_tasks: HashMap::new(),@@ -62,22 +63,26 @@ impl Generator {pub fn prefill(&mut self,tokens: Vec<u64>,- sampling_config: SamplingConfig,+ sampling_method: SamplingMethod,prefix_offset: usize,- ) -> PrefillResult {+ ) -> Result<PrefillResult, Error> {assert!(!tokens.is_empty());self.tokens.extend(tokens.clone());+ let prefill_step_size = self+ .decoding_config+ .prefill_step_size+ .resolve(&self.context.model_config);+let tokens_length = tokens.len();- let number_of_prefill_steps = (tokens_length as f32- / self.config.prefill_step_size as f32)- .ceil() as usize;+ let number_of_prefill_steps =+ (tokens_length as f32 / prefill_step_size as f32).ceil() as usize;let total_prefill_tokens_count =- number_of_prefill_steps * self.config.prefill_step_size;+ number_of_prefill_steps * prefill_step_size;let unused_tokens_count = total_prefill_tokens_count - tokens_length;- let speculator = &self.config.speculator_config.speculator;+ let speculator = &self.decoding_config.speculator_config.speculator;let proposals = speculator.generate_proposals(&tokens);let speculated_suffix = TokenTrie::from_sequences(&proposals).linearize(0, unused_tokens_count);@@ -110,9 +115,8 @@ impl Generator {// Process each prefill step and update the KV cache.for step in 0..number_of_prefill_steps {- let tokens_start_index = step * self.config.prefill_step_size;- let tokens_end_index =- tokens_start_index + self.config.prefill_step_size;+ let tokens_start_index = step * prefill_step_size;+ let tokens_end_index = tokens_start_index + prefill_step_size;let tokens_for_step =&padded_tokens[tokens_start_index..tokens_end_index];let positions_for_step =@@ -125,14 +129,14 @@ impl Generator {let task = GeneratorRunTask {token_ids: tokens_for_step.to_vec(),token_positions: positions_for_step.to_vec(),- expected_amount_of_new_tokens: self.config.prefill_step_size,+ expected_number_of_new_tokens: prefill_step_size,};let (mut state, run_time) = self.run_model(task,false,self.allow_pre_encode(),- Some(sampling_config.clone()),+ sampling_method,);// Register the *accepted* real tokens from this step (exclude the@@ -175,18 +179,17 @@ impl Generator {run_times.push(run_time);}- let mut final_state = last_state.unwrap();- let argmax_tokens = self.gpu_sample(&mut final_state);+ let mut final_state = last_state.ok_or(Error::PrefillFailed)?;+ let sampled_tokens = self.sample(&mut final_state)?;let mut accepted_token_indices: Vec<usize> = Vec::new();let mut accepted_tokens: Vec<u64> = Vec::new();let mut current_token_index: isize = -1;loop {- let current_index_in_window = ((current_token_index- + tokens_length as isize)- % self.config.prefill_step_size as isize)- as usize;- let new_token = argmax_tokens[current_index_in_window];+ let current_index_in_window =+ ((current_token_index + tokens_length as isize)+ % prefill_step_size as isize) as usize;+ let new_token = sampled_tokens[current_index_in_window];accepted_tokens.push(new_token);if let Some(map) =@@ -209,19 +212,19 @@ impl Generator {self.sync_prefix();- PrefillResult {+ Ok(PrefillResult {tokens: accepted_tokens,forwardpass_durations: run_times,- }+ })}pub fn generate(&mut self,- sampling_config: SamplingConfig,- ) -> GenerateResult {- let last_token = self.tokens.last().unwrap();+ sampling_method: SamplingMethod,+ ) -> Result<GenerateResult, Error> {+ let last_token = self.tokens.last().ok_or(Error::GenerateFailed)?;- let speculator = &self.config.speculator_config.speculator;+ let speculator = &self.decoding_config.speculator_config.speculator;let mut proposals: Vec<Vec<u64>> = speculator.generate_proposals(&self.tokens).into_iter()@@ -232,9 +235,10 @@ impl Generator {}let speculated_suffix = TokenTrie::from_sequences(&proposals)- .linearize(0, self.config.generate_suffix_length());+ .linearize(0, self.decoding_config.generate_suffix_length());- let expected_suffix_length = self.config.generate_suffix_length();+ let expected_suffix_length =+ self.decoding_config.generate_suffix_length();let unused_tokens_count =expected_suffix_length - speculated_suffix.tokens.len();let zero_padding_tokens: Vec<u64> = vec![0; unused_tokens_count];@@ -255,23 +259,23 @@ impl Generator {let task = GeneratorRunTask {token_ids: padded_tokens,token_positions: padded_positions,- expected_amount_of_new_tokens: 1,+ expected_number_of_new_tokens: 1,};let (mut state, run_time) = self.run_model(task,false,self.allow_pre_encode(),- Some(sampling_config),+ sampling_method,);- let argmax_tokens = self.gpu_sample(&mut state);+ let sampled_tokens = self.sample(&mut state)?;let mut accepted_token_indices = Vec::new();let mut accepted_tokens = Vec::new();let mut current_token_index: isize = 0;loop {- let new_token = argmax_tokens[current_token_index as usize];+ let new_token = sampled_tokens[current_token_index as usize];accepted_tokens.push(new_token);if let Some(map) =speculated_suffix.transition_map.get(&(current_token_index))@@ -292,10 +296,10 @@ impl Generator {self.tokens.extend(accepted_tokens.clone());self.sync_prefix();- GenerateResult {+ Ok(GenerateResult {tokens: accepted_tokens,forwardpass_duration: run_time,- }+ })}pub fn clear_cache(&mut self) {@@ -323,10 +327,10 @@ impl Generator {let task = GeneratorRunTask {token_ids: vec![0; suffix_length],token_positions: (0..suffix_length).collect::<Vec<usize>>(),- expected_amount_of_new_tokens: suffix_length,+ expected_number_of_new_tokens: suffix_length,};- let (_, _) = self.run_model(task, true, false, None);+ let (_, _) = self.run_model(task, true, false, SamplingMethod::Greedy);}fn run_model(@@ -334,13 +338,13 @@ impl Generator {task: GeneratorRunTask,warmup: bool,allow_pre_encode: bool,- sampling_config: Option<SamplingConfig>,+ sampling_method: SamplingMethod,) -> (ForwardPassState, f64) {objc2::rc::autoreleasepool(|_pool| {let run_start = Instant::now();let mut state = task.create_state(&mut self.context, None);- state.sampling_config = sampling_config;+ state.sampling_method = Some(sampling_method);let encoded_task_key = task.encoded_task_key(self.tokens.len());@@ -400,15 +404,17 @@ impl Generator {})}- fn gpu_sample(+ fn sample(&mut self,state: &mut ForwardPassState,- ) -> Vec<u64> {+ ) -> Result<Vec<u64>, Error> {let sampling_output = state.sampling_output.as_ref().expect("Sampling output buffer not found - ensure sampling was encoded during forward pass");let output_buffer = sampling_output.borrow();- let output_view = output_buffer.as_view::<u32>().unwrap();+ let output_view = output_buffer+ .as_view::<u32>()+ .map_err(|_| Error::SamplingFailed)?;let batch_size = output_buffer.shape()[0];let mut result = Vec::with_capacity(batch_size);@@ -416,7 +422,7 @@ impl Generator {result.push(output_view[[i]] as u64);}- result+ Ok(result)}fn update_kv_cache(@@ -450,7 +456,8 @@ impl Generator {fn allow_pre_encode(&self) -> bool {let metal_debug_active = MetalEnvVar::DeviceWrapperType.is_enabled();- let result = self.config.allow_pre_encode && !metal_debug_active;+ let result =+ self.decoding_config.allow_pre_encode && !metal_debug_active;result}diff --git a/crates/uzu/src/generator/mod.rs b/crates/uzu/src/generator/mod.rsindex 4462d420e..c6e6e1c90 100644--- a/crates/uzu/src/generator/mod.rs+++ b/crates/uzu/src/generator/mod.rs@@ -1,6 +1,4 @@-pub mod config;pub mod context;-pub mod error;pub mod generator;pub mod result;pub mod sampler;diff --git a/crates/uzu/src/generator/tasks.rs b/crates/uzu/src/generator/tasks.rsindex a89e3ec86..728398089 100644--- a/crates/uzu/src/generator/tasks.rs+++ b/crates/uzu/src/generator/tasks.rs@@ -14,7 +14,7 @@ pub struct GeneratorEncodedTask {pub struct GeneratorRunTask {pub token_ids: Vec<u64>,pub token_positions: Vec<usize>,- pub expected_amount_of_new_tokens: usize,+ pub expected_number_of_new_tokens: usize,}impl GeneratorRunTask {@@ -22,7 +22,7 @@ impl GeneratorRunTask {GeneratorRunTask {token_ids: self.token_ids.clone(),token_positions: self.token_positions.clone(),- expected_amount_of_new_tokens: self.expected_amount_of_new_tokens,+ expected_number_of_new_tokens: self.expected_number_of_new_tokens,}}@@ -32,7 +32,7 @@ impl GeneratorRunTask {) -> String {format!("tokens:{}_suffix:{}",- tokens_count, self.expected_amount_of_new_tokens+ tokens_count, self.expected_number_of_new_tokens)}diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rsindex 2bc3b16e8..8a9f80912 100644--- a/crates/uzu/src/lib.rs+++ b/crates/uzu/src/lib.rs@@ -1,21 +1,14 @@-pub mod array;pub mod backends;pub mod config;-pub mod data_type;-pub use data_type::{ArrayElement, DataType};-pub mod decoder_runner;-pub mod device_context;-pub use array::Array;-pub use device_context::DeviceContext;-pub mod env_utils;pub mod generator;pub mod linearizer;pub mod parameters;-pub mod quant_matmul;pub mod session;pub mod speculators;-pub mod storage;pub mod tracer;-pub mod version;-pub use quant_matmul::*;-pub use version::VERSION;+pub mod utils;+pub use utils::*;+pub mod device;+pub use device::*;+pub mod runners;+pub use runners::*;diff --git a/crates/uzu/src/decoder_runner.rs b/crates/uzu/src/runners/decoder_runner.rssimilarity index 100%rename from crates/uzu/src/decoder_runner.rsrename to crates/uzu/src/runners/decoder_runner.rsdiff --git a/crates/uzu/src/runners/mod.rs b/crates/uzu/src/runners/mod.rsnew file mode 100644index 000000000..1ad5471a3--- /dev/null+++ b/crates/uzu/src/runners/mod.rs@@ -0,0 +1,4 @@+pub mod decoder_runner;+pub use decoder_runner::*;+pub mod quant_matmul;+pub use quant_matmul::*;diff --git a/crates/uzu/src/quant_matmul.rs b/crates/uzu/src/runners/quant_matmul.rssimilarity index 99%rename from crates/uzu/src/quant_matmul.rsrename to crates/uzu/src/runners/quant_matmul.rsindex 8601e2bf1..60350f043 100644--- a/crates/uzu/src/quant_matmul.rs+++ b/crates/uzu/src/runners/quant_matmul.rs@@ -27,7 +27,7 @@ use crate::{MTLContext, MetalArray,compilation_parameters::{BlockDevice, make_compilation_descriptor},},- storage::{NSSearchPathDirectory, root_dir},+ utils::storage::{NSSearchPathDirectory, root_dir},};// ================== Public error type =====================diff --git a/crates/uzu/src/session/config/decoding_config.rs b/crates/uzu/src/session/config/decoding_config.rsnew file mode 100644index 000000000..227ce5301--- /dev/null+++ b/crates/uzu/src/session/config/decoding_config.rs@@ -0,0 +1,47 @@+use crate::session::{+ config::SpeculatorConfig,+ parameter::{ContextLength, PrefillStepSize, SamplingSeed},+};++#[derive(Clone)]+pub struct DecodingConfig {+ pub prefill_step_size: PrefillStepSize,+ pub context_length: ContextLength,+ pub speculator_config: SpeculatorConfig,+ pub sampling_seed: SamplingSeed,+ pub allow_pre_encode: bool,+}++impl DecodingConfig {+ pub fn new(+ prefill_step_size: PrefillStepSize,+ context_length: ContextLength,+ speculator_config: SpeculatorConfig,+ sampling_seed: SamplingSeed,+ allow_pre_encode: bool,+ ) -> Self {+ Self {+ prefill_step_size,+ context_length,+ speculator_config,+ sampling_seed,+ allow_pre_encode,+ }+ }++ pub fn generate_suffix_length(&self) -> usize {+ self.speculator_config.number_of_speculated_tokens + 1+ }+}++impl Default for DecodingConfig {+ fn default() -> Self {+ Self {+ prefill_step_size: PrefillStepSize::default(),+ context_length: ContextLength::default(),+ speculator_config: SpeculatorConfig::default(),+ sampling_seed: SamplingSeed::default(),+ allow_pre_encode: true,+ }+ }+}diff --git a/crates/uzu/src/session/config/mod.rs b/crates/uzu/src/session/config/mod.rsnew file mode 100644index 000000000..89c713ed1--- /dev/null+++ b/crates/uzu/src/session/config/mod.rs@@ -0,0 +1,6 @@+mod decoding_config;+mod run_config;+mod speculator_config;+pub use decoding_config::DecodingConfig;+pub use run_config::RunConfig;+pub use speculator_config::SpeculatorConfig;diff --git a/crates/uzu/src/session/config/run_config.rs b/crates/uzu/src/session/config/run_config.rsnew file mode 100644index 000000000..aa07e06d7--- /dev/null+++ b/crates/uzu/src/session/config/run_config.rs@@ -0,0 +1,52 @@+use crate::session::parameter::SamplingPolicy;++#[derive(Debug, Clone)]+pub struct RunConfig {+ pub tokens_limit: u64,+ pub enable_thinking: bool,+ pub sampling_policy: SamplingPolicy,+}++impl RunConfig {+ pub fn new(+ tokens_limit: u64,+ enable_thinking: bool,+ sampling_policy: SamplingPolicy,+ ) -> Self {+ Self {+ tokens_limit,+ enable_thinking,+ sampling_policy,+ }+ }++ pub fn tokens_limit(+ mut self,+ tokens_limit: u64,+ ) -> Self {+ self.tokens_limit = tokens_limit;+ self+ }++ pub fn enable_thinking(+ mut self,+ enable_thinking: bool,+ ) -> Self {+ self.enable_thinking = enable_thinking;+ self+ }++ pub fn sampling_policy(+ mut self,+ sampling_policy: SamplingPolicy,+ ) -> Self {+ self.sampling_policy = sampling_policy;+ self+ }+}++impl Default for RunConfig {+ fn default() -> Self {+ Self::new(1024, true, SamplingPolicy::Default)+ }+}diff --git a/crates/uzu/src/session/config/speculator_config.rs b/crates/uzu/src/session/config/speculator_config.rsnew file mode 100644index 000000000..5ba59db53--- /dev/null+++ b/crates/uzu/src/session/config/speculator_config.rs@@ -0,0 +1,32 @@+use std::sync::Arc;++use crate::speculators::{+ empty_speculator::EmptySpeculator, speculator::Speculator,+};++#[derive(Clone)]+pub struct SpeculatorConfig {+ pub number_of_speculated_tokens: usize,+ pub speculator: Arc<dyn Speculator>,+}++impl SpeculatorConfig {+ pub fn new(+ number_of_speculated_tokens: usize,+ speculator: Arc<dyn Speculator>,+ ) -> Self {+ Self {+ number_of_speculated_tokens,+ speculator,+ }+ }+}++impl Default for SpeculatorConfig {+ fn default() -> Self {+ Self {+ number_of_speculated_tokens: 0,+ speculator: Arc::new(EmptySpeculator {}),+ }+ }+}diff --git a/crates/uzu/src/session/session_context.rs b/crates/uzu/src/session/helpers/context.rssimilarity index 60%rename from crates/uzu/src/session/session_context.rsrename to crates/uzu/src/session/helpers/context.rsindex 57e838e40..cb2b91ba4 100644--- a/crates/uzu/src/session/session_context.rs+++ b/crates/uzu/src/session/helpers/context.rs@@ -1,24 +1,24 @@use crate::{backends::metal::forward_pass::kv_cache::KVCache,- generator::config::GeneratorConfig,+ session::config::DecodingConfig,};-pub struct SessionContext {+pub struct Context {pub tokens: Vec<u64>,pub kv_cache: KVCache,- pub config: GeneratorConfig,+ pub decoding_config: DecodingConfig,}-impl SessionContext {+impl Context {pub fn new(tokens: Vec<u64>,kv_cache: KVCache,- config: GeneratorConfig,+ decoding_config: DecodingConfig,) -> Self {Self {tokens,kv_cache,- config,+ decoding_config,}}}diff --git a/crates/uzu/src/session/helpers/input_processor.rs b/crates/uzu/src/session/helpers/input_processor.rsnew file mode 100644index 000000000..e951b2e1f--- /dev/null+++ b/crates/uzu/src/session/helpers/input_processor.rs@@ -0,0 +1,65 @@+use minijinja::{Environment, context};+use minijinja_contrib::pycompat::unknown_method_callback;++use crate::{+ config::MessageProcessorConfig,+ session::{+ parameter::ConfigResolvableValue,+ types::{Error, Input},+ },+};++pub trait InputProcessor: Send + Sync {+ fn process(+ &self,+ input: &Input,+ enable_thinking: bool,+ ) -> Result<String, Error>;+}++pub struct InputProcessorDefault {+ message_processing_config: MessageProcessorConfig,+}++impl InputProcessorDefault {+ pub fn new(message_processing_config: MessageProcessorConfig) -> Self {+ Self {+ message_processing_config,+ }+ }+}++impl InputProcessor for InputProcessorDefault {+ fn process(+ &self,+ input: &Input,+ enable_thinking: bool,+ ) -> Result<String, Error> {+ let messages = input+ .get_messages()+ .into_iter()+ .map(|message| message.resolve(&self.message_processing_config))+ .collect::<Vec<_>>();+ let template = self.message_processing_config.prompt_template.clone();+ let bos_token = self.message_processing_config.bos_token.clone();++ let template_name = "chat_template";+ let mut environment = Environment::new();+ environment.set_unknown_method_callback(unknown_method_callback);+ environment+ .add_template(template_name, template.as_str())+ .map_err(|_| Error::UnableToLoadPromptTemplate)?;+ let template = environment+ .get_template(template_name)+ .map_err(|_| Error::UnableToLoadPromptTemplate)?;+ let result = template+ .render(context!(+ messages => messages,+ add_generation_prompt => true,+ bos_token => bos_token,+ enable_thinking => enable_thinking+ ))+ .map_err(|_| Error::UnableToRenderPromptTemplate)?;+ Ok(result)+ }+}diff --git a/crates/uzu/src/session/helpers/memory_checker.rs b/crates/uzu/src/session/helpers/memory_checker.rsnew file mode 100644index 000000000..3da918246--- /dev/null+++ b/crates/uzu/src/session/helpers/memory_checker.rs@@ -0,0 +1,27 @@+use std::path::Path;++fn get_directory_size(path: &Path) -> std::io::Result<u64> {+ let mut size = 0u64;+ for entry_result in std::fs::read_dir(path)? {+ let entry = entry_result?;+ let metadata = entry.metadata()?;+ if metadata.is_dir() {+ size += get_directory_size(&entry.path())?;+ } else {+ size += metadata.len();+ }+ }+ Ok(size)+}++pub fn is_directory_fits_ram(path: &Path) -> bool {+ use sysinfo::System;++ let model_size_bytes = get_directory_size(path).unwrap_or(0);++ let mut sys = System::new();+ sys.refresh_memory();++ let allowed_bytes = sys.total_memory() * 60 / 100;+ return model_size_bytes <= allowed_bytes;+}diff --git a/crates/uzu/src/session/helpers/mod.rs b/crates/uzu/src/session/helpers/mod.rsnew file mode 100644index 000000000..720c4f950--- /dev/null+++ b/crates/uzu/src/session/helpers/mod.rs@@ -0,0 +1,6 @@+mod context;+mod input_processor;+mod memory_checker;+pub use context::Context;+pub use input_processor::{InputProcessor, InputProcessorDefault};+pub use memory_checker::is_directory_fits_ram;diff --git a/crates/uzu/src/session/mod.rs b/crates/uzu/src/session/mod.rsindex 8de03ebcf..7fa66d369 100644--- a/crates/uzu/src/session/mod.rs+++ b/crates/uzu/src/session/mod.rs@@ -1,12 +1,5 @@+pub mod config;+pub mod helpers;+pub mod parameter;pub mod session;-pub mod session_classification_feature;-pub mod session_config;-pub mod session_error;-pub mod session_input;-pub mod session_message;-pub mod session_output;-pub mod session_run_config;-pub mod session_tokenizer_config;-pub use session_context::*;--pub mod session_context;+pub mod types;diff --git a/crates/uzu/src/session/parameter/context_length.rs b/crates/uzu/src/session/parameter/context_length.rsnew file mode 100644index 000000000..800a88545--- /dev/null+++ b/crates/uzu/src/session/parameter/context_length.rs@@ -0,0 +1,37 @@+use crate::{+ config::LanguageModelConfig, session::parameter::ConfigResolvableValue,+};++#[derive(Debug, Clone, Copy)]+pub enum ContextLength {+ Default,+ Maximal,+ Custom(usize),+}++impl Default for ContextLength {+ fn default() -> Self {+ ContextLength::Default+ }+}++impl ConfigResolvableValue<LanguageModelConfig, usize> for ContextLength {+ fn resolve(+ &self,+ config: &LanguageModelConfig,+ ) -> usize {+ let model_context_length = config.decoder_config.context_length;+ let proposed_value = match self {+ ContextLength::Default => {+ if cfg!(target_os = "ios") {+ return 8192;+ } else {+ return 16384;+ }+ },+ ContextLength::Maximal => model_context_length,+ ContextLength::Custom(value) => *value,+ };+ std::cmp::min(proposed_value, model_context_length)+ }+}diff --git a/crates/uzu/src/session/parameter/mod.rs b/crates/uzu/src/session/parameter/mod.rsnew file mode 100644index 000000000..6bd57ab0b--- /dev/null+++ b/crates/uzu/src/session/parameter/mod.rs@@ -0,0 +1,10 @@+mod context_length;+mod prefill_step_size;+mod resolvable_value;+mod sampling_policy;+mod sampling_seed;+pub use context_length::ContextLength;+pub use prefill_step_size::PrefillStepSize;+pub use resolvable_value::{ConfigResolvableValue, ResolvableValue};+pub use sampling_policy::{SamplingMethod, SamplingPolicy};+pub use sampling_seed::SamplingSeed;diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rsnew file mode 100644index 000000000..ae130a646--- /dev/null+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs@@ -0,0 +1,54 @@+use crate::{+ config::LanguageModelConfig, session::parameter::ConfigResolvableValue,+};++#[derive(Debug, Clone, Copy)]+pub enum PrefillStepSize {+ Default,+ Maximal,+ Custom(usize),+}++impl Default for PrefillStepSize {+ fn default() -> Self {+ PrefillStepSize::Default+ }+}++impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {+ fn resolve(+ &self,+ config: &LanguageModelConfig,+ ) -> usize {+ let default_limit: usize = 1024;+ let model_context_length = config.decoder_config.context_length;+ let minimal_sliding_window_size = config+ .decoder_config+ .sliding_window_sizes+ .as_deref()+ .into_iter()+ .flatten()+ .flatten()+ .copied()+ .min()+ .unwrap_or(usize::MAX);+ let maximal_value =+ [model_context_length, minimal_sliding_window_size, default_limit]+ .into_iter()+ .min()+ .unwrap_or(default_limit);++ let proposed_value = match self {+ PrefillStepSize::Default => {+ if cfg!(target_os = "ios") {+ return 64;+ } else {+ return 256;+ }+ },+ PrefillStepSize::Maximal => maximal_value,+ PrefillStepSize::Custom(value) => *value,+ };+ std::cmp::min(proposed_value, maximal_value)+ }+}diff --git a/crates/uzu/src/session/parameter/resolvable_value.rs b/crates/uzu/src/session/parameter/resolvable_value.rsnew file mode 100644index 000000000..05f156979--- /dev/null+++ b/crates/uzu/src/session/parameter/resolvable_value.rs@@ -0,0 +1,10 @@+pub trait ResolvableValue<Value> {+ fn resolve(&self) -> Value;+}++pub trait ConfigResolvableValue<Config, Value> {+ fn resolve(+ &self,+ config: &Config,+ ) -> Value;+}diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rsnew file mode 100644index 000000000..89a1e41d9--- /dev/null+++ b/crates/uzu/src/session/parameter/sampling_policy.rs@@ -0,0 +1,63 @@+use crate::{+ config::LanguageModelConfig, session::parameter::ConfigResolvableValue,+};++#[derive(Debug, Copy, Clone, PartialEq)]+pub enum SamplingMethod {+ Greedy,+ Temperature {+ temperature: f32,+ },+ TopP {+ top_p: f32,+ },+}++impl Default for SamplingMethod {+ fn default() -> Self {+ SamplingMethod::Greedy+ }+}++#[derive(Debug, Clone, PartialEq)]+pub enum SamplingPolicy {+ Default,+ Custom {+ value: SamplingMethod,+ },+}++impl Default for SamplingPolicy {+ fn default() -> Self {+ SamplingPolicy::Default+ }+}++impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod>+ for SamplingPolicy+{+ fn resolve(+ &self,+ config: &LanguageModelConfig,+ ) -> SamplingMethod {+ let generation_config = &config.generation_config;+ match self {+ SamplingPolicy::Default => {+ if let Some(top_p) = generation_config.top_p {+ return SamplingMethod::TopP {+ top_p: top_p,+ };+ }+ if let Some(temperature) = generation_config.temperature {+ return SamplingMethod::Temperature {+ temperature: temperature,+ };+ }+ return SamplingMethod::Greedy;+ },+ SamplingPolicy::Custom {+ value,+ } => *value,+ }+ }+}diff --git a/crates/uzu/src/session/parameter/sampling_seed.rs b/crates/uzu/src/session/parameter/sampling_seed.rsnew file mode 100644index 000000000..77098926e--- /dev/null+++ b/crates/uzu/src/session/parameter/sampling_seed.rs@@ -0,0 +1,24 @@+use rand::prelude::*;++use crate::session::parameter::ResolvableValue;++#[derive(Debug, Clone, Copy)]+pub enum SamplingSeed {+ Default,+ Custom(u64),+}++impl Default for SamplingSeed {+ fn default() -> Self {+ SamplingSeed::Default+ }+}++impl ResolvableValue<u64> for SamplingSeed {+ fn resolve(&self) -> u64 {+ match self {+ SamplingSeed::Default => rand::rng().random::<u64>(),+ SamplingSeed::Custom(seed) => *seed,+ }+ }+}diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rsindex e8881663c..9905e6421 100644--- a/crates/uzu/src/session/session.rs+++ b/crates/uzu/src/session/session.rs@@ -1,260 +1,223 @@-#[cfg(target_os = "ios")]-use std::path::Path;use std::{fs::File, io::BufReader, path::PathBuf, time::Instant};use objc2::rc::autoreleasepool;use tokenizers::Tokenizer;-use super::{- session_config::SessionConfig,- session_context::SessionContext,- session_input::{- SessionInput, SessionInputProcessor, SessionInputProcessorDefault,- },- session_output::{- SessionOutput, SessionOutputFinishReason, SessionOutputRunStats,- SessionOutputStats, SessionOutputStepStats, SessionOutputTotalStats,- },- session_run_config::SessionRunConfig,-};use crate::{config::ModelMetadata,generator::{- config::GeneratorConfigProvider,generator::Generator,result::{GenerateResult, PrefillResult},},session::{- session_error::SessionError,- session_tokenizer_config::SessionTokenizerConfig,+ config::{DecodingConfig, RunConfig},+ helpers::{+ Context, InputProcessor, InputProcessorDefault,+ is_directory_fits_ram,+ },+ parameter::ConfigResolvableValue,+ types::{+ Error, FinishReason, Input, Output, RunStats, Stats, StepStats,+ TotalStats,+ },},};pub struct Session {- model_path: PathBuf,+ pub model_path: PathBuf,+ pub model_metadata: ModelMetadata,+tokenizer: Tokenizer,- tokenizer_config: SessionTokenizerConfig,- input_processor: Box<dyn SessionInputProcessor>,+ input_processor: Box<dyn InputProcessor>,generator: Option<Generator>,}impl Session {- #[cfg(target_os = "ios")]- fn directory_size(path: &Path) -> std::io::Result<u64> {- let mut size = 0u64;- for entry_result in std::fs::read_dir(path)? {- let entry = entry_result?;- let metadata = entry.metadata()?;- if metadata.is_dir() {- size += Self::directory_size(&entry.path())?;- } else {- size += metadata.len();- }+ pub fn new(+ model_path: PathBuf,+ decoding_config: DecodingConfig,+ ) -> Result<Self, Error> {+ if !model_path.exists() {+ return Err(Error::ModelFolderNotFound);}- Ok(size)- }- #[cfg(target_os = "ios")]- fn assert_model_fits_ram(model_path: &Path) -> Result<(), SessionError> {- use sysinfo::System;-- let model_size_bytes = Self::directory_size(model_path).unwrap_or(0);-- let mut sys = System::new();- sys.refresh_memory();-- let allowed_bytes = sys.total_memory() * 60 / 100;-- if model_size_bytes > allowed_bytes {- Err(SessionError::UnsupportedModel)- } else {- Ok(())+ if !is_directory_fits_ram(&model_path) {+ return Err(Error::NotEnoughMemory);}- }-- pub fn new(model_path: PathBuf) -> Result<Self, SessionError> {- #[cfg(target_os = "ios")]- Self::assert_model_fits_ram(&model_path)?;let config_path = model_path.join("config.json");if !config_path.exists() {- return Err(SessionError::UnableToLoadConfig);+ return Err(Error::UnableToLoadConfig);}- let config_file = File::open(&config_path)- .map_err(|_| SessionError::UnableToLoadConfig)?;+ let config_file =+ File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;let model_metadata: ModelMetadata =serde_json::from_reader(BufReader::new(config_file))- .map_err(|_| SessionError::UnableToLoadConfig)?;+ .map_err(|_| Error::UnableToLoadConfig)?;let tokenizer_path = model_path.join("tokenizer.json");+ if !tokenizer_path.exists() {+ return Err(Error::UnableToLoadTokenizer);+ }let tokenizer = Tokenizer::from_file(&tokenizer_path)- .map_err(|_| SessionError::UnableToLoadTokenizer)?;+ .map_err(|_| Error::UnableToLoadTokenizer)?;- let tokenizer_config = SessionTokenizerConfig::load(- &model_metadata.model_config,- &tokenizer,- )- .ok_or(SessionError::UnableToLoadTokenizerConfig)?;+ let input_processor = InputProcessorDefault::new(+ model_metadata.model_config.message_processor_config.clone(),+ );- let input_processor =- SessionInputProcessorDefault::new(tokenizer_config.clone());+ let generator = Generator::new(&model_path, decoding_config)+ .map_err(Error::from)?;Ok(Self {model_path,+ model_metadata,tokenizer,- tokenizer_config,input_processor: Box::new(input_processor),- generator: None,+ generator: Some(generator),})}- pub fn load(- &mut self,- generator_config_provider: Box<dyn GeneratorConfigProvider>,- ) -> Result<(), SessionError> {- let generator_config =- generator_config_provider.generator_config(&self.tokenizer);- let generator = Generator::new(&self.model_path, generator_config)- .map_err(SessionError::from)?;-- self.generator = Some(generator);- Ok(())- }-- pub fn load_with_session_config(- &mut self,- config: SessionConfig,- ) -> Result<(), SessionError> {- self.load(Box::new(config))- }-- pub fn extend(- &mut self,- input: SessionInput,- context: Option<&SessionContext>,- config: SessionRunConfig,- ) -> Result<(SessionOutput, SessionContext), SessionError> {- self.reconfigure_generator(context)?;- let output = self.run_internal(- input,- config,- None::<fn(SessionOutput) -> bool>,- )?;- let new_context = self.build_context_from_generator()?;- let generator =- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;- generator.reset_state();- Ok((output, new_context))- }-pub fn run<F>(&mut self,- input: SessionInput,- config: SessionRunConfig,+ input: Input,+ config: RunConfig,progress: Option<F>,- ) -> Result<SessionOutput, SessionError>+ ) -> Result<Output, Error>where- F: Fn(SessionOutput) -> bool,+ F: Fn(Output) -> bool,{let generator =- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;generator.reset_state();let output = self.run_internal(input, config, progress)?;let generator =- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;generator.reset_state();Ok(output)}+ pub fn extend(+ &mut self,+ input: Input,+ context: Option<&Context>,+ config: RunConfig,+ ) -> Result<(Output, Context), Error> {+ self.reconfigure_generator(context)?;+ let output =+ self.run_internal(input, config, None::<fn(Output) -> bool>)?;+ let new_context = self.build_context_from_generator()?;+ let generator =+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;+ generator.reset_state();+ Ok((output, new_context))+ }+pub fn run_with_context(&mut self,- input: SessionInput,- context: Option<&SessionContext>,- config: SessionRunConfig,- ) -> Result<SessionOutput, SessionError> {+ input: Input,+ context: Option<&Context>,+ config: RunConfig,+ ) -> Result<Output, Error> {self.reconfigure_generator(context)?;- let output = self.run_internal(- input,- config,- None::<fn(SessionOutput) -> bool>,- )?;+ let output =+ self.run_internal(input, config, None::<fn(Output) -> bool>)?;let generator =- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;generator.reset_state();Ok(output)}fn run_internal<F>(&mut self,- input: SessionInput,- config: SessionRunConfig,+ input: Input,+ config: RunConfig,progress: Option<F>,- ) -> Result<SessionOutput, SessionError>+ ) -> Result<Output, Error>where- F: Fn(SessionOutput) -> bool,+ F: Fn(Output) -> bool,{let generator =- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;let run_start = Instant::now();- let text = self.input_processor.process(&input);+ let text =+ self.input_processor.process(&input, config.enable_thinking)?;let tokens: Vec<u64> = self.tokenizer.encode(text.as_str(), false)- .unwrap()+ .map_err(|_| Error::UnableToEncodeText)?.get_ids().iter().map(|&id| id as u64).collect();+ let context_length = generator+ .decoding_config+ .context_length+ .resolve(&self.model_metadata.model_config);+ if tokens.len() >= context_length {+ return Err(Error::ContextLengthExceeded);+ }+let prefix_len_before = generator.prefix_len();- let eos_tokens = self- .tokenizer_config- .eos_tokens+ let eos_tokens: Vec<u64> = self+ .model_metadata+ .model_config+ .generation_config+ .stop_token_ids.iter()- .map(|token| {- self.tokenizer.token_to_id(token.as_str()).unwrap() as u64- })- .collect::<Vec<_>>();+ .map(|&x| x as u64)+ .collect();let finish_reason = |generator: &Generator,tokens_new: Vec<u64>|- -> Option<SessionOutputFinishReason> {+ -> Option<FinishReason> {let start_idx = prefix_len_before + tokens.len();+ let total_tokens = generator.tokens.len();let total_new_tokens = generator.tokens[start_idx..].len();let has_eos_token =tokens_new.iter().any(|token| eos_tokens.contains(token));+ let context_limit_reached = total_tokens >= context_length;if has_eos_token {- Some(SessionOutputFinishReason::Stop)+ Some(FinishReason::Stop)} else if total_new_tokens >= config.tokens_limit as usize {- Some(SessionOutputFinishReason::Length)+ Some(FinishReason::Length)+ } else if context_limit_reached {+ Some(FinishReason::ContextLimitReached)} else {None}};- let build_generated_text =- |generator: &Generator, tokenizer: &Tokenizer| -> String {- let start_idx = prefix_len_before + tokens.len();- let generated_tokens: Vec<u32> = generator.tokens[start_idx..]- .to_vec()- .iter()- .map(|value| *value as u32)- .collect();- tokenizer.decode(&generated_tokens, true).unwrap()- };+ let build_generated_text = |generator: &Generator,+ tokenizer: &Tokenizer|+ -> Result<String, Error> {+ let start_idx = prefix_len_before + tokens.len();+ let generated_tokens: Vec<u32> = generator.tokens[start_idx..]+ .to_vec()+ .iter()+ .map(|value| *value as u32)+ .collect();+ let generated_text = tokenizer+ .decode(&generated_tokens, true)+ .map_err(|_| Error::UnableToDecodeText)?;+ Ok(generated_text)+ };- let sampling_config = config- .sampling_config- .unwrap_or(self.tokenizer_config.sampling_config);+ let sampling_method =+ config.sampling_policy.resolve(&self.model_metadata.model_config);let prefill_start = Instant::now();let prefix_offset = generator.tokens.len();- let prefill_result =- generator.prefill(tokens.clone(), sampling_config, prefix_offset);+ let prefill_result = generator.prefill(+ tokens.clone(),+ sampling_method,+ prefix_offset,+ )?;let prefill_tokens = prefill_result.tokens.clone();let prefill_duration = prefill_start.elapsed().as_secs_f64();generator.clear_cache();@@ -262,17 +225,21 @@ impl Session {let prefill_finish_reason =finish_reason(generator, prefill_tokens.clone());let prefill_generated_text =- build_generated_text(generator, &self.tokenizer);+ build_generated_text(generator, &self.tokenizer)?;- let prefill_output = SessionOutput {+ let prefill_suffix_length = generator+ .decoding_config+ .prefill_step_size+ .resolve(&self.model_metadata.model_config);+ let prefill_output = Output {text: prefill_generated_text,stats: Self::build_stats(prefill_result.clone(),prefill_duration,- generator.config.prefill_step_size,+ prefill_suffix_length,Vec::new(),Vec::new(),- generator.config.generate_suffix_length(),+ generator.decoding_config.generate_suffix_length(),run_start.elapsed().as_secs_f64(),tokens.len(),generator.tokens[prefix_len_before + tokens.len()..].len(),@@ -290,9 +257,8 @@ impl Session {if prefill_should_continue {return Ok(prefill_output);} else {- return Ok(prefill_output.clone_with_finish_reason(Some(- SessionOutputFinishReason::Cancelled,- )));+ return Ok(prefill_output+ .clone_with_finish_reason(Some(FinishReason::Cancelled)));}}@@ -300,7 +266,7 @@ impl Session {let mut generate_durations: Vec<f64> = Vec::new();let generate_output = loop {let generate_start = Instant::now();- let generate_result = generator.generate(sampling_config);+ let generate_result = generator.generate(sampling_method)?;let generate_tokens = generate_result.tokens.clone();let generate_duration = generate_start.elapsed().as_secs_f64();generate_results.push(generate_result);@@ -309,17 +275,17 @@ impl Session {let generate_finish_reason =finish_reason(generator, generate_tokens);let generate_generated_text =- build_generated_text(generator, &self.tokenizer);+ build_generated_text(generator, &self.tokenizer)?;- let generate_output = SessionOutput {+ let generate_output = Output {text: generate_generated_text,stats: Self::build_stats(prefill_result.clone(),prefill_duration,- generator.config.prefill_step_size,+ prefill_suffix_length,generate_results.clone(),generate_durations.clone(),- generator.config.generate_suffix_length(),+ generator.decoding_config.generate_suffix_length(),run_start.elapsed().as_secs_f64(),tokens.len(),generator.tokens[prefix_len_before + tokens.len()..].len(),@@ -338,7 +304,7 @@ impl Session {break generate_output;} else {break generate_output.clone_with_finish_reason(Some(- SessionOutputFinishReason::Cancelled,+ FinishReason::Cancelled,));}}@@ -351,10 +317,10 @@ impl Session {fn reconfigure_generator(&mut self,- context: Option<&SessionContext>,- ) -> Result<(), SessionError> {+ context: Option<&Context>,+ ) -> Result<(), Error> {let generator =- self.generator.as_mut().ok_or(SessionError::GeneratorNotLoaded)?;+ self.generator.as_mut().ok_or(Error::GeneratorNotLoaded)?;generator.reset_state();if let Some(ctx) = context {let mut generator_cache = generator.context.kv_cache.borrow_mut();@@ -387,21 +353,19 @@ impl Session {Ok(())}- fn build_context_from_generator(- &self- ) -> Result<SessionContext, SessionError> {+ fn build_context_from_generator(&self) -> Result<Context, Error> {let generator =- self.generator.as_ref().ok_or(SessionError::GeneratorNotLoaded)?;+ self.generator.as_ref().ok_or(Error::GeneratorNotLoaded)?;let prefix_len = generator.prefix_len();let kv_cache = generator.context.kv_cache.borrow().clone_with_prefix_len(&generator.context.mtl_context, prefix_len);- let context = SessionContext::new(+ let context = Context::new(generator.tokens.clone(),kv_cache,- generator.config.clone(),+ generator.decoding_config.clone(),);Ok(context)}@@ -418,7 +382,7 @@ impl Session {total_duration: f64,tokens_count_input: usize,tokens_count_output: usize,- ) -> SessionOutputStats {+ ) -> Stats {let prefill_stats = {let tokens_count = prefill_result.tokens.len();let tokens_per_second = tokens_count as f64 / prefill_duration;@@ -428,12 +392,12 @@ impl Session {prefill_result.forwardpass_durations.iter().sum::<f64>()/ model_run_count as f64;- SessionOutputStepStats {+ StepStats {duration: prefill_duration,suffix_length: prefill_suffix_length as u64,tokens_count: tokens_count as u64,tokens_per_second,- model_run: SessionOutputRunStats {+ model_run: RunStats {count: model_run_count as u64,average_duration: model_run_average_duration,},@@ -441,7 +405,7 @@ impl Session {}};- let generate_stats: Option<SessionOutputStepStats>;+ let generate_stats: Option<StepStats>;if generate_results.len() != 0 {let duration = generate_durations.iter().sum::<f64>();let tokens_count = generate_results@@ -462,16 +426,16 @@ impl Session {let run_average_duration =generate_durations.iter().sum::<f64>() / run_count as f64;- generate_stats = Some(SessionOutputStepStats {+ generate_stats = Some(StepStats {duration,suffix_length: generate_suffix_length as u64,tokens_count: tokens_count as u64,tokens_per_second,- model_run: SessionOutputRunStats {+ model_run: RunStats {count: model_run_count as u64,average_duration: model_run_average_duration,},- run: Some(SessionOutputRunStats {+ run: Some(RunStats {count: run_count as u64,average_duration: run_average_duration,}),@@ -480,13 +444,13 @@ impl Session {generate_stats = None;}- let total_stats = SessionOutputTotalStats {+ let total_stats = TotalStats {duration: total_duration,tokens_count_input: tokens_count_input as u64,tokens_count_output: tokens_count_output as u64,};- let stats = SessionOutputStats {+ let stats = Stats {prefill_stats,generate_stats,total_stats,diff --git a/crates/uzu/src/session/session_classification_feature.rs b/crates/uzu/src/session/session_classification_feature.rsdeleted file mode 100644index c8c3948d3..000000000--- a/crates/uzu/src/session/session_classification_feature.rs+++ /dev/null@@ -1,5 +0,0 @@-#[derive(Clone, Debug)]-pub struct SessionClassificationFeature {- pub name: String,- pub values: Vec<String>,-}diff --git a/crates/uzu/src/session/session_config.rs b/crates/uzu/src/session/session_config.rsdeleted file mode 100644index dbd4e6250..000000000--- a/crates/uzu/src/session/session_config.rs+++ /dev/null@@ -1,64 +0,0 @@-use tokenizers::Tokenizer;--use crate::generator::config::{- ContextLength, GeneratorConfig, GeneratorConfigProvider, SamplingSeed,- SpeculatorConfig,-};--#[derive(Clone)]-pub struct SessionConfig {- pub prefill_step_size: usize,- pub speculator_config: SpeculatorConfig,- pub allow_pre_encode: bool,- pub sampling_seed: SamplingSeed,- pub context_length: ContextLength,-}--impl SessionConfig {- pub fn new(- prefill_step_size: usize,- speculator_config: SpeculatorConfig,- allow_pre_encode: bool,- sampling_seed: SamplingSeed,- context_length: ContextLength,- ) -> Self {- Self {- prefill_step_size,- speculator_config,- allow_pre_encode,- sampling_seed,- context_length,- }- }-- pub fn to_generator_config(&self) -> GeneratorConfig {- GeneratorConfig::new(- self.prefill_step_size,- self.speculator_config.clone(),- self.allow_pre_encode,- self.sampling_seed,- self.context_length,- )- }-}--impl Default for SessionConfig {- fn default() -> Self {- Self::new(- 8,- SpeculatorConfig::default(),- true,- SamplingSeed::default(),- ContextLength::default(),- )- }-}--impl GeneratorConfigProvider for SessionConfig {- fn generator_config(- &self,- _tokenizer: &Tokenizer,- ) -> GeneratorConfig {- self.to_generator_config()- }-}diff --git a/crates/uzu/src/session/session_error.rs b/crates/uzu/src/session/session_error.rsdeleted file mode 100644index b00df81f4..000000000--- a/crates/uzu/src/session/session_error.rs+++ /dev/null@@ -1,31 +0,0 @@-use crate::generator::error::GeneratorError;--#[derive(Debug, thiserror::Error)]-pub enum SessionError {- #[error("Unable to create Metal context")]- UnableToCreateMetalContext,- #[error("Unable to load model configuration")]- UnableToLoadConfig,- #[error("Unable to load model weights")]- UnableToLoadWeights,- #[error("Unable to load tokenizer")]- UnableToLoadTokenizer,- #[error("Unable to load tokenizer configuration")]- UnableToLoadTokenizerConfig,- #[error("Model is too large to fit into available RAM")]- UnsupportedModel,- #[error("Generator not loaded. Call load() first")]- GeneratorNotLoaded,-}--impl From<GeneratorError> for SessionError {- fn from(value: GeneratorError) -> Self {- match value {- GeneratorError::UnableToCreateMetalContext => {- Self::UnableToCreateMetalContext- },- GeneratorError::UnableToLoadConfig => Self::UnableToLoadConfig,- GeneratorError::UnableToLoadWeights => Self::UnableToLoadWeights,- }- }-}diff --git a/crates/uzu/src/session/session_input.rs b/crates/uzu/src/session/session_input.rsdeleted file mode 100644index ba269e506..000000000--- a/crates/uzu/src/session/session_input.rs+++ /dev/null@@ -1,67 +0,0 @@-use minijinja::{Environment, context};-use minijinja_contrib::pycompat::unknown_method_callback;--use super::session_message::{SessionMessage, SessionMessageRole};-use crate::session::session_tokenizer_config::SessionTokenizerConfig;--#[derive(Debug)]-pub enum SessionInput {- Text(String),- Messages(Vec<SessionMessage>),-}--impl SessionInput {- fn get_messages(&self) -> Vec<SessionMessage> {- match self {- SessionInput::Text(content) => vec![SessionMessage {- role: SessionMessageRole::User,- content: content.clone(),- }],- SessionInput::Messages(messages) => messages.clone(),- }- }-}--pub trait SessionInputProcessor: Send + Sync {- fn process(- &self,- input: &SessionInput,- ) -> String;-}--pub struct SessionInputProcessorDefault {- tokenizer_config: SessionTokenizerConfig,-}--impl SessionInputProcessorDefault {- pub fn new(tokenizer_config: SessionTokenizerConfig) -> Self {- Self {- tokenizer_config,- }- }-}--impl SessionInputProcessor for SessionInputProcessorDefault {- fn process(- &self,- input: &SessionInput,- ) -> String {- let messages = input.get_messages();- let template = self.tokenizer_config.chat_template.clone();- let bos_token = self.tokenizer_config.bos_token.clone();-- let template_name = "chat_template";- let mut environment = Environment::new();- environment.set_unknown_method_callback(unknown_method_callback);- environment.add_template(template_name, template.as_str()).unwrap();- let template = environment.get_template(template_name).unwrap();- let result = template- .render(context!(- messages => messages,- add_generation_prompt => true,- bos_token => bos_token- ))- .unwrap();- result- }-}diff --git a/crates/uzu/src/session/session_message.rs b/crates/uzu/src/session/session_message.rsdeleted file mode 100644index e93e00608..000000000--- a/crates/uzu/src/session/session_message.rs+++ /dev/null@@ -1,15 +0,0 @@-use serde::{Deserialize, Serialize};--#[derive(Clone, Debug, Deserialize, Serialize)]-#[serde(rename_all = "lowercase")]-pub enum SessionMessageRole {- System,- User,- Assistant,-}--#[derive(Clone, Debug, Deserialize, Serialize)]-pub struct SessionMessage {- pub role: SessionMessageRole,- pub content: String,-}diff --git a/crates/uzu/src/session/session_output.rs b/crates/uzu/src/session/session_output.rsdeleted file mode 100644index cc1125c3a..000000000--- a/crates/uzu/src/session/session_output.rs+++ /dev/null@@ -1,77 +0,0 @@-use serde::{Deserialize, Serialize};--#[derive(Debug, Clone, Deserialize, Serialize)]-pub struct SessionOutputRunStats {- pub count: u64,- pub average_duration: f64,-}--#[derive(Debug, Clone, Deserialize, Serialize)]-pub struct SessionOutputStepStats {- pub duration: f64,- pub suffix_length: u64,- pub tokens_count: u64,- pub tokens_per_second: f64,- pub model_run: SessionOutputRunStats,- pub run: Option<SessionOutputRunStats>,-}--#[derive(Debug, Clone, Deserialize, Serialize)]-pub struct SessionOutputTotalStats {- pub duration: f64,- pub tokens_count_input: u64,- pub tokens_count_output: u64,-}--#[derive(Debug, Clone, Deserialize, Serialize)]-pub struct SessionOutputStats {- pub prefill_stats: SessionOutputStepStats,- pub generate_stats: Option<SessionOutputStepStats>,- pub total_stats: SessionOutputTotalStats,-}--#[derive(Debug, Clone, Deserialize, Serialize)]-#[serde(rename_all = "lowercase")]-pub enum SessionOutputFinishReason {- Stop,- Length,- Cancelled,-}--#[derive(Debug, Clone)]-pub struct SessionOutput {- pub text: String,- pub stats: SessionOutputStats,- pub finish_reason: Option<SessionOutputFinishReason>,-}--impl SessionOutput {- pub fn clone_with_finish_reason(- &self,- finish_reason: Option<SessionOutputFinishReason>,- ) -> Self {- Self {- text: self.text.clone(),- stats: self.stats.clone(),- finish_reason: finish_reason,- }- }-- pub fn clone_with_duration(- &self,- duration: f64,- ) -> Self {- Self {- text: self.text.clone(),- stats: SessionOutputStats {- prefill_stats: self.stats.prefill_stats.clone(),- generate_stats: self.stats.generate_stats.clone(),- total_stats: SessionOutputTotalStats {- duration: duration,- ..self.stats.total_stats.clone()- },- },- finish_reason: self.finish_reason.clone(),- }- }-}diff --git a/crates/uzu/src/session/session_run_config.rs b/crates/uzu/src/session/session_run_config.rsdeleted file mode 100644index 1ceb4801d..000000000--- a/crates/uzu/src/session/session_run_config.rs+++ /dev/null@@ -1,26 +0,0 @@-use crate::backends::metal::sampling_config::SamplingConfig;--#[derive(Debug)]-pub struct SessionRunConfig {- pub tokens_limit: u64,- pub sampling_config: Option<SamplingConfig>,-}--impl SessionRunConfig {- pub fn new(tokens_limit: u64) -> Self {- Self {- tokens_limit: tokens_limit,- sampling_config: None,- }- }-- pub fn new_with_sampling_config(- tokens_limit: u64,- sampling_config: Option<SamplingConfig>,- ) -> Self {- Self {- tokens_limit: tokens_limit,- sampling_config: sampling_config,- }- }-}diff --git a/crates/uzu/src/session/session_tokenizer_config.rs b/crates/uzu/src/session/session_tokenizer_config.rsdeleted file mode 100644index 5fe3241f2..000000000--- a/crates/uzu/src/session/session_tokenizer_config.rs+++ /dev/null@@ -1,62 +0,0 @@-use tokenizers::Tokenizer;--use crate::{- backends::metal::sampling_config::SamplingConfig,- config::{GenerationConfig, LanguageModelConfig},-};--#[derive(Clone, Debug)]-pub struct SessionTokenizerConfig {- pub eos_tokens: Vec<String>,- pub bos_token: Option<String>,- pub chat_template: String,- pub sampling_config: SamplingConfig,-}--impl SessionTokenizerConfig {- pub fn load(- model_config: &LanguageModelConfig,- tokenizer: &Tokenizer,- ) -> Option<Self> {- let eos_tokens = model_config- .generation_config- .stop_token_ids- .iter()- .flat_map(|token_id| tokenizer.id_to_token(*token_id))- .collect::<Vec<_>>();- if eos_tokens.is_empty() {- return None;- }-- let bos_token = model_config.message_processor_config.bos_token.clone();-- let chat_template =- model_config.message_processor_config.prompt_template.clone();-- let sampling_config =- Self::build_sampling_config(&model_config.generation_config);-- Some(Self {- eos_tokens,- bos_token,- chat_template,- sampling_config,- })- }-- fn build_sampling_config(- generation_config: &GenerationConfig- ) -> SamplingConfig {- if let Some(top_p) = &generation_config.top_p {- return SamplingConfig::TopP {- top_p: *top_p,- };- }- if let Some(temperature) = &generation_config.temperature {- return SamplingConfig::Categorical {- temperature: *temperature,- };- }- return SamplingConfig::Argmax;- }-}diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsnew file mode 100644index 000000000..215dce323--- /dev/null+++ b/crates/uzu/src/session/types/error.rs@@ -0,0 +1,33 @@+#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("Model folder not found")]+ ModelFolderNotFound,+ #[error("Unable to create Metal context")]+ UnableToCreateMetalContext,+ #[error("Unable to load model configuration")]+ UnableToLoadConfig,+ #[error("Unable to load model weights")]+ UnableToLoadWeights,+ #[error("Unable to load tokenizer")]+ UnableToLoadTokenizer,+ #[error("Model is too large to fit into available RAM")]+ NotEnoughMemory,+ #[error("Generator not loaded")]+ GeneratorNotLoaded,+ #[error("Unable to load prompt template")]+ UnableToLoadPromptTemplate,+ #[error("Unable to render prompt template")]+ UnableToRenderPromptTemplate,+ #[error("Unable to encode text")]+ UnableToEncodeText,+ #[error("Unable to decode text")]+ UnableToDecodeText,+ #[error("Context length exceeded")]+ ContextLengthExceeded,+ #[error("Prefill failed")]+ PrefillFailed,+ #[error("Generate failed")]+ GenerateFailed,+ #[error("Sampling failed")]+ SamplingFailed,+}diff --git a/crates/uzu/src/session/types/input.rs b/crates/uzu/src/session/types/input.rsnew file mode 100644index 000000000..17b15f488--- /dev/null+++ b/crates/uzu/src/session/types/input.rs@@ -0,0 +1,19 @@+use crate::session::types::{Message, Role};++#[derive(Debug)]+pub enum Input {+ Text(String),+ Messages(Vec<Message>),+}++impl Input {+ pub fn get_messages(&self) -> Vec<Message> {+ match self {+ Input::Text(content) => vec![Message {+ role: Role::User,+ content: content.clone(),+ }],+ Input::Messages(messages) => messages.clone(),+ }+ }+}diff --git a/crates/uzu/src/session/types/message.rs b/crates/uzu/src/session/types/message.rsnew file mode 100644index 000000000..e44981d72--- /dev/null+++ b/crates/uzu/src/session/types/message.rs@@ -0,0 +1,34 @@+use std::collections::HashMap;++use serde::{Deserialize, Serialize};++use crate::{+ config::MessageProcessorConfig,+ session::{parameter::ConfigResolvableValue, types::Role},+};++#[derive(Clone, Debug, Deserialize, Serialize)]+pub struct Message {+ pub role: Role,+ pub content: String,+}++impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>>+ for Message+{+ fn resolve(+ &self,+ config: &MessageProcessorConfig,+ ) -> HashMap<String, String> {+ let role = match self.role {+ Role::System => config.system_role_name.clone(),+ Role::User => config.user_role_name.clone(),+ Role::Assistant => config.assistant_role_name.clone(),+ };+ let content = self.content.clone();+ HashMap::from([+ (String::from("role"), role),+ (String::from("content"), content),+ ])+ }+}diff --git a/crates/uzu/src/session/types/mod.rs b/crates/uzu/src/session/types/mod.rsnew file mode 100644index 000000000..01ba25346--- /dev/null+++ b/crates/uzu/src/session/types/mod.rs@@ -0,0 +1,12 @@+mod error;+mod input;+mod message;+mod output;+mod role;+mod stats;+pub use error::Error;+pub use input::Input;+pub use message::Message;+pub use output::{FinishReason, Output};+pub use role::Role;+pub use stats::{RunStats, Stats, StepStats, TotalStats};diff --git a/crates/uzu/src/session/types/output.rs b/crates/uzu/src/session/types/output.rsnew file mode 100644index 000000000..0c4647afe--- /dev/null+++ b/crates/uzu/src/session/types/output.rs@@ -0,0 +1,50 @@+use serde::{Deserialize, Serialize};++use crate::session::types::{Stats, TotalStats};++#[derive(Debug, Clone, Deserialize, Serialize)]+#[serde(rename_all = "lowercase")]+pub enum FinishReason {+ Stop,+ Length,+ Cancelled,+ ContextLimitReached,+}++#[derive(Debug, Clone)]+pub struct Output {+ pub text: String,+ pub stats: Stats,+ pub finish_reason: Option<FinishReason>,+}++impl Output {+ pub fn clone_with_finish_reason(+ &self,+ finish_reason: Option<FinishReason>,+ ) -> Self {+ Self {+ text: self.text.clone(),+ stats: self.stats.clone(),+ finish_reason: finish_reason,+ }+ }++ pub fn clone_with_duration(+ &self,+ duration: f64,+ ) -> Self {+ Self {+ text: self.text.clone(),+ stats: Stats {+ prefill_stats: self.stats.prefill_stats.clone(),+ generate_stats: self.stats.generate_stats.clone(),+ total_stats: TotalStats {+ duration: duration,+ ..self.stats.total_stats.clone()+ },+ },+ finish_reason: self.finish_reason.clone(),+ }+ }+}diff --git a/crates/uzu/src/session/types/role.rs b/crates/uzu/src/session/types/role.rsnew file mode 100644index 000000000..b86a520d3--- /dev/null+++ b/crates/uzu/src/session/types/role.rs@@ -0,0 +1,9 @@+use serde::{Deserialize, Serialize};++#[derive(Clone, Debug, Deserialize, Serialize)]+#[serde(rename_all = "lowercase")]+pub enum Role {+ System,+ User,+ Assistant,+}diff --git a/crates/uzu/src/session/types/stats.rs b/crates/uzu/src/session/types/stats.rsnew file mode 100644index 000000000..74d75b7b8--- /dev/null+++ b/crates/uzu/src/session/types/stats.rs@@ -0,0 +1,31 @@+use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, Deserialize, Serialize)]+pub struct RunStats {+ pub count: u64,+ pub average_duration: f64,+}++#[derive(Debug, Clone, Deserialize, Serialize)]+pub struct StepStats {+ pub duration: f64,+ pub suffix_length: u64,+ pub tokens_count: u64,+ pub tokens_per_second: f64,+ pub model_run: RunStats,+ pub run: Option<RunStats>,+}++#[derive(Debug, Clone, Deserialize, Serialize)]+pub struct TotalStats {+ pub duration: f64,+ pub tokens_count_input: u64,+ pub tokens_count_output: u64,+}++#[derive(Debug, Clone, Deserialize, Serialize)]+pub struct Stats {+ pub prefill_stats: StepStats,+ pub generate_stats: Option<StepStats>,+ pub total_stats: TotalStats,+}diff --git a/crates/uzu/src/tracer/tracer.rs b/crates/uzu/src/tracer/tracer.rsindex a0fb050f8..2d44cfc21 100644--- a/crates/uzu/src/tracer/tracer.rs+++ b/crates/uzu/src/tracer/tracer.rs@@ -3,7 +3,6 @@ use std::{fs::File,path::{Path, PathBuf},rc::Rc,- sync::Arc,};use half::{bf16, f16};@@ -21,14 +20,14 @@ use crate::{},},generator::{- config::{- ContextLength, GeneratorConfig, SamplingSeed, SpeculatorConfig,- },context::GeneratorContext,sampler::{ArgmaxSampler, LogitsSampler},},parameters::{ParameterLoader, ParameterTree},- speculators::empty_speculator::EmptySpeculator,+ session::{+ config::{DecodingConfig, SpeculatorConfig},+ parameter::{ContextLength, PrefillStepSize, SamplingSeed},+ },};enum Transform {@@ -141,18 +140,15 @@ pub struct Tracer {impl Tracer {pub fn new(model_path: &Path) -> Self {- let generator_config = GeneratorConfig::new(- 1024,- SpeculatorConfig {- number_of_speculated_tokens: 0,- speculator: Arc::new(EmptySpeculator {}),- },+ let decoding_config = DecodingConfig::new(+ PrefillStepSize::Custom(1024),+ ContextLength::default(),+ SpeculatorConfig::default(),+ SamplingSeed::default(),false,- SamplingSeed::Default,- ContextLength::Default,);let generator_context =- GeneratorContext::new(model_path, &generator_config).unwrap();+ GeneratorContext::new(model_path, &decoding_config).unwrap();Self {model_path: model_path.to_path_buf(),@@ -165,18 +161,15 @@ impl Tracer {sampling_seed: SamplingSeed,max_prefix_length: ContextLength,) -> Self {- let generator_config = GeneratorConfig::new(- 1024,- SpeculatorConfig {- number_of_speculated_tokens: 0,- speculator: Arc::new(EmptySpeculator {}),- },- false,- sampling_seed,+ let decoding_config = DecodingConfig::new(+ PrefillStepSize::Custom(1024),max_prefix_length,+ SpeculatorConfig::default(),+ sampling_seed,+ false,);let generator_context =- GeneratorContext::new(model_path, &generator_config).unwrap();+ GeneratorContext::new(model_path, &decoding_config).unwrap();Self {model_path: model_path.to_path_buf(),@@ -206,19 +199,6 @@ impl Tracer {&traces_view,"activation_trace.token_positions".to_string(),);- let mask: Vec<Vec<bool>> = Self::load_array_as_vec::<i8, i8>(- &traces_view,- "activation_trace.mask".to_string(),- )- .iter()- .map(|x| *x != 0)- .collect::<Vec<bool>>()- .chunks(token_ids.len())- .map(|chunk| chunk.to_vec())- .collect();-- let external_bias_fn =- |row: usize, col: usize| -> bool { !mask[row][col] };let mut state = ForwardPassState::new(self.generator_context.mtl_context.clone(),@@ -229,7 +209,7 @@ impl Tracer {&token_ids,&token_positions,true,- Some(&external_bias_fn),+ None,);let root_command_buffer = selfdiff --git a/crates/uzu/src/env_utils.rs b/crates/uzu/src/utils/env_utils.rssimilarity index 100%rename from crates/uzu/src/env_utils.rsrename to crates/uzu/src/utils/env_utils.rsdiff --git a/crates/uzu/src/utils/mod.rs b/crates/uzu/src/utils/mod.rsnew file mode 100644index 000000000..1c4996f25--- /dev/null+++ b/crates/uzu/src/utils/mod.rs@@ -0,0 +1,6 @@+pub mod storage;+pub use storage::*;+pub mod version;+pub use version::VERSION;+pub mod env_utils;+pub use env_utils::*;diff --git a/crates/uzu/src/storage.rs b/crates/uzu/src/utils/storage.rssimilarity index 100%rename from crates/uzu/src/storage.rsrename to crates/uzu/src/utils/storage.rsdiff --git a/crates/uzu/src/version.rs b/crates/uzu/src/utils/version.rssimilarity index 100%rename from crates/uzu/src/version.rsrename to crates/uzu/src/utils/version.rsdiff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 609cc3c89..1c1f9b325 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -1,7 +1,9 @@#![allow(dead_code)]use std::path::PathBuf;-pub const MODEL_DIR_NAME: &str = "Meta-Llama-3.2-1B-Instruct-float16";+use uzu::VERSION;++pub const MODEL_DIR_NAME: &str = "Llama-3.2-1B-Instruct";pub const MODEL_FILE_NAME: &str = "model.safetensors";pub const TRACES_FILE_NAME: &str = "traces.safetensors";@@ -9,12 +11,12 @@ pub fn get_test_model_path() -> PathBuf {let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..").join("..")- .join("..")- .join("mirai_models")+ .join("models")+ .join(VERSION).join(MODEL_DIR_NAME);if !path.exists() {panic!(- "Test model not found at {:?}. Run `bash scripts/download_test_model.sh` to fetch it.",+ "Test model not found at {:?}. Run `./scripts/download_test_model.sh` to fetch it.",path);}diff --git a/crates/uzu/tests/context_kv_cache_test.rs b/crates/uzu/tests/context_kv_cache_test.rsindex e2ceb509f..02d1c05d6 100644--- a/crates/uzu/tests/context_kv_cache_test.rs+++ b/crates/uzu/tests/context_kv_cache_test.rs@@ -1,12 +1,11 @@use std::{path::PathBuf, rc::Rc};-use uzu::{- backends::metal::sampling_config::SamplingConfig,- session::{- session::Session, session_config::SessionConfig,- session_context::SessionContext, session_input::SessionInput,- session_output::SessionOutput, session_run_config::SessionRunConfig,- },+use uzu::session::{+ config::{DecodingConfig, RunConfig},+ helpers::Context,+ parameter::{SamplingMethod, SamplingPolicy},+ session::Session,+ types::{Input, Output},};mod common;@@ -75,11 +74,15 @@ fn test_context_extension() {// Extend the context with new informationlet (_, extended_context) = session.extend(- SessionInput::Text(- "Update: Name: Dave. Occupation: Doctor.".to_string(),- ),+ Input::Text("Update: Name: Dave. Occupation: Doctor.".to_string()),Some(initial_context.as_ref()),- SessionRunConfig::new(1),+ RunConfig::new(+ 1,+ true,+ SamplingPolicy::Custom {+ value: SamplingMethod::Greedy,+ },+ ),).unwrap();@@ -221,24 +224,27 @@ fn test_performance_cached_vs_plain() {}fn create_session() -> Session {- let mut session = Session::new(model_path()).unwrap();- session- .load_with_session_config(SessionConfig::default())+ let session = Session::new(model_path(), DecodingConfig::default()).expect("Failed to load session");-session}fn build_context(session: &mut Session,prompt: &str,-) -> Rc<SessionContext> {+) -> Rc<Context> {// Run the prompt once to build the context.let (_, context) = session.extend(- SessionInput::Text(prompt.to_string()),+ Input::Text(prompt.to_string()),None,- SessionRunConfig::new(1),+ RunConfig::new(+ 1,+ true,+ SamplingPolicy::Custom {+ value: SamplingMethod::Greedy,+ },+ ),).unwrap();Rc::new(context)@@ -246,16 +252,19 @@ fn build_context(fn ask_with_context(session: &mut Session,- context: Option<Rc<SessionContext>>,+ context: Option<Rc<Context>>,question: &str,) -> String {session.run_with_context(- SessionInput::Text(format!("{} /no_think", question)),+ Input::Text(format!("{} /no_think", question)),context.as_deref(),- SessionRunConfig::new_with_sampling_config(+ RunConfig::new(96,- Some(SamplingConfig::Argmax),+ true,+ SamplingPolicy::Custom {+ value: SamplingMethod::Greedy,+ },),).unwrap()@@ -268,12 +277,15 @@ fn ask_without_context() -> String {session.run(- SessionInput::Text(format!("{} /no_think", question)),- SessionRunConfig::new_with_sampling_config(+ Input::Text(format!("{} /no_think", question)),+ RunConfig::new(96,- Some(SamplingConfig::Argmax),+ true,+ SamplingPolicy::Custom {+ value: SamplingMethod::Greedy,+ },),- None::<fn(SessionOutput) -> bool>,+ None::<fn(Output) -> bool>,).unwrap().textdiff --git a/crates/uzu/tests/generator_test.rs b/crates/uzu/tests/generator_test.rsdeleted file mode 100644index 4c22492e8..000000000--- a/crates/uzu/tests/generator_test.rs+++ /dev/null@@ -1,121 +0,0 @@-mod common;-use std::path::PathBuf;--use uzu::{- backends::metal::sampling_config::SamplingConfig,- generator::config::{ContextLength, SamplingSeed, SpeculatorConfig},- session::{- session::Session,- session_config::SessionConfig,- session_input::SessionInput,- session_message::{SessionMessage, SessionMessageRole},- session_output::SessionOutput,- session_run_config::SessionRunConfig,- },-};--fn build_model_path() -> PathBuf {- common::get_test_model_path()-}--#[test]-fn test_generation_base() {- let text = String::from("Tell about London");- let config = SessionConfig::new(- 8,- SpeculatorConfig::default(),- true,- SamplingSeed::Custom(42),- ContextLength::Default,- );- run(text, config, 128);-}--fn run(- text: String,- config: SessionConfig,- tokens_limit: u64,-) {- let mut session = Session::new(build_model_path()).unwrap();- session.load_with_session_config(config).unwrap();-- let input = SessionInput::Text(text);- let output = session- .run(- input,- SessionRunConfig::new_with_sampling_config(- tokens_limit,- Some(SamplingConfig::Argmax),- ),- Some(|_: SessionOutput| {- return true;- }),- )- .unwrap();-- println!("-------------------------");- println!("{}", output.text);- println!("-------------------------");- println!("{:#?}", output.stats);- println!("-------------------------");- println!("Finish reason: {:?}", output.finish_reason);- println!("-------------------------");-}--#[test]-fn test_generation_scenario() {- let system_prompt = String::from("You are a helpful assistant.");- let messages = vec![- String::from("Tell about London"),- String::from("Compare with New York"),- ];- run_scenario(Some(system_prompt), messages);-}--fn run_scenario(- system_prompt: Option<String>,- user_prompts: Vec<String>,-) {- let config = SessionConfig::new(- 64,- SpeculatorConfig::default(),- true,- SamplingSeed::Custom(42),- ContextLength::Default,- );- let mut session = Session::new(build_model_path()).unwrap();- session.load_with_session_config(config).unwrap();-- let mut messages: Vec<SessionMessage> = vec![];- if let Some(system_prompt) = system_prompt {- messages.push(SessionMessage {- role: SessionMessageRole::System,- content: system_prompt.clone(),- });- println!("System > {}", system_prompt.clone());- }-- for user_prompt in user_prompts {- messages.push(SessionMessage {- role: SessionMessageRole::User,- content: user_prompt.clone(),- });- println!("User > {}", user_prompt.clone());-- let input = SessionInput::Messages(messages.clone());- let output = session- .run(- input,- SessionRunConfig::new_with_sampling_config(2048, None),- Some(|_: SessionOutput| {- return true;- }),- )- .unwrap();- messages.push(SessionMessage {- role: SessionMessageRole::Assistant,- content: output.text.clone(),- });- println!("Assistant > {}", output.text.clone());- }-}diff --git a/crates/uzu/tests/parameters_loader_test.rs b/crates/uzu/tests/parameters_loader_test.rsindex 36a10606b..30178eae1 100644--- a/crates/uzu/tests/parameters_loader_test.rs+++ b/crates/uzu/tests/parameters_loader_test.rs@@ -1,7 +1,7 @@mod common;// New integration test for ParameterLoader-use half::f16;+use half::bf16;use is_close::is_close;use uzu::{Array, backends::cpu::CPUContext, parameters::ParameterLoader};@@ -15,11 +15,11 @@ fn test_parameter_loader_basic() {let loader = ParameterLoader::new(&file, &context).expect("create loader");let embeddings =loader.get("embedding.weights").expect("weights embeddings");- let view = embeddings.as_view::<f16>().unwrap();- assert!(is_close!(view[[5usize, 3usize]], f16::from_f32(-0.01819)));+ let view = embeddings.as_view::<bf16>().unwrap();+ assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));// tree API checklet subtree = loader.tree().subtree("embedding").unwrap();let same = subtree.leaf("weights").unwrap();- assert_eq!(view, same.as_view::<f16>().unwrap());+ assert_eq!(view, same.as_view::<bf16>().unwrap());}diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rsindex a90899cc4..2e9e8e9ab 100644--- a/crates/uzu/tests/sampling_test.rs+++ b/crates/uzu/tests/sampling_test.rs@@ -1,14 +1,16 @@mod common;use metal::{Device, MTLResourceOptions};use rand::seq::SliceRandom; // for Vec::shuffle-use uzu::backends::metal::{- KernelDataType, MTLContext,- kernel::{- SamplingKernel,- sampling::{ArgmaxStrategy, CategoricalStrategy},+use uzu::{+ backends::metal::{+ KernelDataType, MTLContext,+ kernel::{+ SamplingKernel,+ sampling::{ArgmaxStrategy, CategoricalStrategy},+ },+ metal_extensions::command_buffer_extensions::CommandBufferTimingAccess,},- metal_extensions::command_buffer_extensions::CommandBufferTimingAccess,- sampling_config::SamplingConfig,+ session::parameter::SamplingMethod,};// Constant seed for reproducible test results@@ -103,7 +105,7 @@ fn test_argmax_sampling_with_strategy(strategy: ArgmaxStrategy) {// Run samplingkernel.encode(- &SamplingConfig::argmax(),+ &SamplingMethod::Greedy,&logits_buffer,&output_buffer,batch_size,@@ -236,7 +238,9 @@ fn test_topp_sampling_from_prob_exact_match(let cb = cb_ref.to_owned();kernel.encode(- &SamplingConfig::top_p(p),+ &SamplingMethod::TopP {+ top_p: p,+ },&logits_buf,&output_buf,batch_size,@@ -357,7 +361,9 @@ fn test_topp_sampling_statistical_large() {kernel.encode(- &SamplingConfig::top_p(TOP_P),+ &SamplingMethod::TopP {+ top_p: TOP_P,+ },&logits_buf,&output_buf,BATCH,@@ -459,7 +465,9 @@ fn perf_topp_128k_vocab() {kernel.encode(- &SamplingConfig::top_p(TOP_P),+ &SamplingMethod::TopP {+ top_p: TOP_P,+ },&logits_buf,&output_buf,BATCH,@@ -554,7 +562,7 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {kernel.encode(- &SamplingConfig::argmax(),+ &SamplingMethod::Greedy,&logits_buf,&output_buf,BATCH,@@ -668,7 +676,9 @@ fn test_categorical_sampling() {kernel.encode(- &SamplingConfig::categorical(1.0),+ &SamplingMethod::Temperature {+ temperature: 1.0,+ },&logits_buffer,&output_buffer,batch_size,@@ -804,7 +814,9 @@ fn test_categorical_sampling_statistical() {kernel.encode(- &SamplingConfig::categorical(1.0),+ &SamplingMethod::Temperature {+ temperature: 1.0,+ },&logits_buffer,&output_buffer,BATCH,@@ -904,7 +916,9 @@ fn perf_categorical_128k_vocab() {kernel.encode(- &SamplingConfig::categorical(1.0),+ &SamplingMethod::Temperature {+ temperature: 1.0,+ },&logits_buf,&output_buf,BATCH,@@ -1028,7 +1042,9 @@ fn test_categorical_1pass_vs_2pass_equivalence() {kernel_1pass.encode(- &SamplingConfig::categorical(1.0),+ &SamplingMethod::Temperature {+ temperature: 1.0,+ },&logits_buffer,&output_buffer_1pass,batch_size,@@ -1054,7 +1070,9 @@ fn test_categorical_1pass_vs_2pass_equivalence() {kernel_2pass.encode(- &SamplingConfig::categorical(1.0),+ &SamplingMethod::Temperature {+ temperature: 1.0,+ },&logits_buffer,&output_buffer_2pass,batch_size,@@ -1225,7 +1243,9 @@ fn test_categorical_sampling_with_strategy(strategy: CategoricalStrategy) {kernel.encode(- &SamplingConfig::categorical(1.0),+ &SamplingMethod::Temperature {+ temperature: 1.0,+ },&logits_buffer,&output_buffer,batch_size,@@ -1334,7 +1354,9 @@ fn perf_categorical_128k_vocab_with_strategy(strategy: CategoricalStrategy) {kernel.encode(- &SamplingConfig::categorical(1.0),+ &SamplingMethod::Temperature {+ temperature: 1.0,+ },&logits_buf,&output_buf,BATCH,diff --git a/crates/uzu/tests/text_session_test.rs b/crates/uzu/tests/text_session_test.rsnew file mode 100644index 000000000..4507aeaf5--- /dev/null+++ b/crates/uzu/tests/text_session_test.rs@@ -0,0 +1,116 @@+mod common;+use std::path::PathBuf;++use uzu::session::{+ config::{DecodingConfig, RunConfig, SpeculatorConfig},+ parameter::{+ ContextLength, PrefillStepSize, SamplingMethod, SamplingPolicy,+ SamplingSeed,+ },+ session::Session,+ types::{Input, Message, Output, Role},+};++fn build_model_path() -> PathBuf {+ common::get_test_model_path()+}++fn build_decoding_config() -> DecodingConfig {+ DecodingConfig::new(+ PrefillStepSize::default(),+ ContextLength::default(),+ SpeculatorConfig::default(),+ SamplingSeed::Custom(42),+ true,+ )+}++#[test]+fn test_text_session_base() {+ let text = String::from("Tell about London");+ run(text, build_decoding_config(), 128);+}++#[test]+fn test_text_session_scenario() {+ let system_prompt = String::from("You are a helpful assistant.");+ let user_prompts = vec![+ String::from("Tell about London"),+ String::from("Compare with New York"),+ ];+ run_scenario(Some(system_prompt), user_prompts);+}++fn run(+ text: String,+ decoding_config: DecodingConfig,+ tokens_limit: u64,+) {+ let mut session =+ Session::new(build_model_path(), decoding_config).unwrap();+ let input = Input::Text(text);+ let output = session+ .run(+ input,+ RunConfig::new(+ tokens_limit,+ true,+ SamplingPolicy::Custom {+ value: SamplingMethod::Greedy,+ },+ ),+ Some(|_: Output| {+ return true;+ }),+ )+ .unwrap();++ println!("-------------------------");+ println!("{}", output.text);+ println!("-------------------------");+ println!("{:#?}", output.stats);+ println!("-------------------------");+ println!("Finish reason: {:?}", output.finish_reason);+ println!("-------------------------");+}++fn run_scenario(+ system_prompt: Option<String>,+ user_prompts: Vec<String>,+) {+ let mut session =+ Session::new(build_model_path(), build_decoding_config()).unwrap();++ let mut messages: Vec<Message> = vec![];+ if let Some(system_prompt) = system_prompt {+ messages.push(Message {+ role: Role::System,+ content: system_prompt.clone(),+ });+ println!("System > {}", system_prompt.clone());+ }++ for user_prompt in user_prompts {+ messages.push(Message {+ role: Role::User,+ content: user_prompt.clone(),+ });+ println!("User > {}", user_prompt.clone());++ let input = Input::Messages(messages.clone());+ let output = session+ .run(+ input,+ RunConfig::default(),+ Some(|_: Output| {+ return true;+ }),+ )+ .unwrap();+ messages.push(Message {+ role: Role::Assistant,+ content: output.text.clone(),+ });+ println!("Assistant > {}", output.text.clone());+ }+}diff --git a/scripts/download_test_model.sh b/scripts/download_test_model.shindex c0a521358..ea1425f1d 100755--- a/scripts/download_test_model.sh+++ b/scripts/download_test_model.sh@@ -1,59 +1,15 @@#!/usr/bin/env bash-# download_test_model.sh-#+set -euo pipefail-# Exit early on non-macOS platform …if [[ "$(uname)" != "Darwin" ]]; thenecho "Non-macOS platform detected (\"$(uname)\"); skipping test-model download."exit 0fi-# Optional first argument specifies the root directory where the model-# should be stored. Defaults to <project-root>/../mirai_models-# Usage: bash download_test_model.sh [ROOT_DIR]--PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"-ROOT_DIR="${1:-$PROJECT_ROOT/../mirai_models}"--MODEL_DIR="$ROOT_DIR/Meta-Llama-3.2-1B-Instruct-float16"--mkdir -p "$MODEL_DIR"--echo "Model directory: $MODEL_DIR"--HF_BASE="https://huggingface.co/trymirai/Llama-3.2-1B-Instruct-float16/resolve/main"-FILES=(- "config.json ${HF_BASE}/config.json"- "model.safetensors ${HF_BASE}/model.safetensors"- "tokenizer.json ${HF_BASE}/tokenizer.json"- "tokenizer_config.json ${HF_BASE}/tokenizer_config.json"- "traces.safetensors ${HF_BASE}/traces.safetensors"-)--for ITEM in "${FILES[@]}"; do- NAME="${ITEM%% *}"- URL="${ITEM#* }"- DEST="$MODEL_DIR/$NAME"- PART="$DEST.part"-- if [[ -f "$DEST" ]]; then- echo "✓ $NAME already present — skipping download"- continue- fi-- # If a partial download exists, resume it- if [[ -f "$PART" ]]; then- echo "↻ Resuming ${NAME} download…"- else- echo "↓ Downloading ${NAME}…"- fi-- # Use --continue-at - (alias -C -) to resume; always write to .part file first- curl -L --fail --progress-bar -C - "$URL" -o "$PART"-- # On successful download, rename to final file name- mv -f "$PART" "$DEST"-done+SCRIPTS_PATH="$(cd "$(dirname "$0")" && pwd)"+TOOLS_PATH="$SCRIPTS_PATH/tools"-echo "✔ All required artifacts are available in $MODEL_DIR"\ No newline at end of file+cd $TOOLS_PATH+uv sync+uv run main.py download-model meta-llama/Llama-3.2-1B-Instruct\ No newline at end of filediff --git a/scripts/tools/.gitignore b/scripts/tools/.gitignorenew file mode 100644index 000000000..9cefeace5--- /dev/null+++ b/scripts/tools/.gitignore@@ -0,0 +1,37 @@+.DS_Store++# Python-generated files+__pycache__/+*.py[oc]+build/+dist/+wheels/+*.egg-info++# Ruff+.ruff_cache/++# Virtual environments+.venv++# Pyright+.mypy_cache/+.pytest_cache/+.pyright/++# Notebooks+notebooks/++# Downloads+downloads/++# IDE+.idea/++sync.bb+.clj-kondo+.lsp++# Profiler+memray-*.bin+memray-*.htmldiff --git a/scripts/tools/.python-version b/scripts/tools/.python-versionnew file mode 100644index 000000000..e4fba2183--- /dev/null+++ b/scripts/tools/.python-version@@ -0,0 +1 @@+3.12diff --git a/scripts/tools/main.py b/scripts/tools/main.pynew file mode 100644index 000000000..5f5587580--- /dev/null+++ b/scripts/tools/main.py@@ -0,0 +1,151 @@+from pathlib import Path+from typing import Annotated++import requests+import tomllib+from model import Registry+from rich.console import Console+from rich.progress import (+ BarColumn,+ DownloadColumn,+ Progress,+ SpinnerColumn,+ TextColumn,+ TimeRemainingColumn,+ TransferSpeedColumn,+)+from rich.table import Table+from typer import Argument, Typer+from utils import download_file_with_resume++ROOT_PATH = Path(__file__).parent.parent.parent+CARGO_TOML_PATH = ROOT_PATH / "Cargo.toml"+MODELS_PATH = ROOT_PATH / "models"+++console = Console()+err_console = Console(stderr=True)+app = Typer(+ rich_markup_mode="rich",+ add_completion=False,+ pretty_exceptions_show_locals=False,+)+++def get_uzu_version() -> str:+ with open(CARGO_TOML_PATH, "rb") as file:+ cargo_toml = tomllib.load(file)+ return cargo_toml["workspace"]["package"]["version"]+++def load_registry() -> Registry:+ url = f"https://sdk.trymirai.com/api/v1/models/list/uzu/{get_uzu_version()}"+ response = requests.get(url)+ response.raise_for_status()+ data = response.json()+ registry = Registry.from_dict(data)+ return registry++[email protected](help="List models")+def list_models():+ try:+ with Progress(+ SpinnerColumn(),+ TextColumn("[progress.description]{task.description}"),+ console=console,+ transient=True,+ ) as progress:+ task = progress.add_task("Loading registry...", total=None)+ registry = load_registry()+ progress.update(task, completed=True)++ table = Table()+ table.add_column("Repo", style="cyan")+ table.add_column("Name", style="green")+ for model in registry.models:+ table.add_row(model.repod_id, model.name)+ console.print(table)++ except Exception as e:+ err_console.print(f"[red]Error: {e}[/red]")++[email protected](help="Download model")+def download_model(+ model_repo: Annotated[+ str,+ Argument(+ help=(+ "Hugging Face model repo. Example: [cyan]'meta-llama/Llama-3.2-1B-Instruct'[/cyan]."+ ),+ ),+ ],+):+ try:+ with Progress(+ SpinnerColumn(),+ TextColumn("[progress.description]{task.description}"),+ console=console,+ transient=True,+ ) as progress:+ task = progress.add_task("Loading registry...", total=None)+ registry = load_registry()+ progress.update(task, completed=True)++ model = None+ for current_model in registry.models:+ if current_model.repod_id == model_repo:+ model = current_model+ break+ if not model:+ err_console.print(+ f"[red]Error: Model '{model_repo}' not found in registry[/red]"+ )+ return++ engine_version = get_uzu_version()+ engine_path = MODELS_PATH / engine_version+ model_path = engine_path / model.name+ model_path.mkdir(parents=True, exist_ok=True)++ files_to_download = []+ for file in model.files:+ file_path = model_path / file.name+ if file_path.exists():+ console.print(f"[green]✓[/green] {file.name} already exists")+ else:+ files_to_download.append((file, file_path))+ if not files_to_download:+ console.print("[green]All files already downloaded![/green]")+ return++ console.print(+ f"Downloading {len(files_to_download)} file(s) for [cyan]{model.name}[/cyan]..."+ )+ with Progress(+ TextColumn("[progress.description]{task.description}"),+ BarColumn(),+ DownloadColumn(),+ TransferSpeedColumn(),+ TimeRemainingColumn(),+ console=console,+ ) as progress:+ for file, file_path in files_to_download:+ task = progress.add_task(f"[cyan]{file.name}[/cyan]", total=None)++ try:+ download_file_with_resume(file.url, file_path, progress, task)+ progress.update(task, description=f"[green]✓[/green] {file.name}")+ except Exception as e:+ progress.update(task, description=f"[red]✗[/red] {file.name}")+ raise e++ console.print(f"[green]Successfully downloaded model: {model.name}[/green]")++ except Exception as e:+ err_console.print(f"[red]Error: {e}[/red]")+++if __name__ == "__main__":+ app()diff --git a/scripts/tools/model.py b/scripts/tools/model.pynew file mode 100644index 000000000..3bb39f5f3--- /dev/null+++ b/scripts/tools/model.py@@ -0,0 +1,40 @@+from dataclasses import dataclass+from typing import List+++@dataclass(frozen=True)+class File:+ name: str+ url: str++ @classmethod+ def from_dict(cls, data: dict) -> "File":+ return cls(name=data["name"], url=data["url"])+++@dataclass(frozen=True)+class Model:+ id: str+ toolchain_version: str+ repod_id: str+ name: str+ files: List[File]++ @classmethod+ def from_dict(cls, data: dict) -> "Model":+ return cls(+ id=data["id"],+ toolchain_version=data["toolchainVersion"],+ repod_id=data["repoId"],+ name=data["name"],+ files=[File.from_dict(file) for file in data["files"]],+ )+++@dataclass(frozen=True)+class Registry:+ models: List[Model]++ @classmethod+ def from_dict(cls, data: dict) -> "Registry":+ return cls(models=[Model.from_dict(model) for model in data["models"]])diff --git a/scripts/tools/pyproject.toml b/scripts/tools/pyproject.tomlnew file mode 100644index 000000000..b0a18ce64--- /dev/null+++ b/scripts/tools/pyproject.toml@@ -0,0 +1,9 @@+[project]+name = "tools"+version = "0.1.0"+requires-python = ">=3.12"+dependencies = [+ "requests>=2.32.5",+ "rich>=14.1.0",+ "typer>=0.19.2",+]diff --git a/scripts/tools/utils.py b/scripts/tools/utils.pynew file mode 100644index 000000000..e4f70bea7--- /dev/null+++ b/scripts/tools/utils.py@@ -0,0 +1,37 @@+from pathlib import Path++import requests+from rich.progress import Progress+++def download_file_with_resume(+ url: str, file_path: Path, progress: Progress, task_id+) -> None:+ part_file_path = file_path.with_suffix(file_path.suffix + ".part")++ resume_position = 0+ mode = "wb"+ if part_file_path.exists():+ resume_position = part_file_path.stat().st_size+ mode = "ab"++ headers = {}+ if resume_position > 0:+ headers["Range"] = f"bytes={resume_position}-"++ response = requests.get(url, headers=headers, stream=True)+ response.raise_for_status()++ total_size = int(response.headers.get("content-length", 0))+ if resume_position > 0 and response.status_code == 206:+ total_size += resume_position++ progress.update(task_id, total=total_size, completed=resume_position)++ with open(part_file_path, mode) as file:+ for chunk in response.iter_content(chunk_size=8192):+ if chunk:+ file.write(chunk)+ progress.update(task_id, advance=len(chunk))++ part_file_path.rename(file_path)diff --git a/scripts/tools/uv.lock b/scripts/tools/uv.locknew file mode 100644index 000000000..632477377--- /dev/null+++ b/scripts/tools/uv.lock@@ -0,0 +1,201 @@+version = 1+revision = 2+requires-python = ">=3.12"++[[package]]+name = "certifi"+version = "2025.8.3"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" },+]++[[package]]+name = "charset-normalizer"+version = "3.4.3"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" },+ { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" },+ { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" },+ { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" },+ { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" },+ { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" },+ { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" },+ { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" },+ { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" },+ { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" },+ { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" },+ { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" },+ { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" },+ { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" },+ { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" },+ { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" },+ { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" },+ { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" },+ { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" },+ { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" },+ { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" },+ { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" },+ { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" },+ { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" },+ { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" },+ { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" },+ { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" },+ { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" },+ { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" },+ { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" },+ { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" },+ { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" },+ { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" },+ { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" },+]++[[package]]+name = "click"+version = "8.3.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "colorama", marker = "sys_platform == 'win32'" },+]+sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" },+]++[[package]]+name = "colorama"+version = "0.4.6"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },+]++[[package]]+name = "idna"+version = "3.10"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },+]++[[package]]+name = "markdown-it-py"+version = "4.0.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "mdurl" },+]+sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },+]++[[package]]+name = "mdurl"+version = "0.1.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },+]++[[package]]+name = "pygments"+version = "2.19.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },+]++[[package]]+name = "requests"+version = "2.32.5"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "certifi" },+ { name = "charset-normalizer" },+ { name = "idna" },+ { name = "urllib3" },+]+sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" },+]++[[package]]+name = "rich"+version = "14.1.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "markdown-it-py" },+ { name = "pygments" },+]+sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" },+]++[[package]]+name = "shellingham"+version = "1.5.4"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },+]++[[package]]+name = "tools"+version = "0.1.0"+source = { virtual = "." }+dependencies = [+ { name = "requests" },+ { name = "rich" },+ { name = "typer" },+]++[package.metadata]+requires-dist = [+ { name = "requests", specifier = ">=2.32.5" },+ { name = "rich", specifier = ">=14.1.0" },+ { name = "typer", specifier = ">=0.19.2" },+]++[[package]]+name = "typer"+version = "0.19.2"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "click" },+ { name = "rich" },+ { name = "shellingham" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/21/ca/950278884e2ca20547ff3eb109478c6baf6b8cf219318e6bc4f666fad8e8/typer-0.19.2.tar.gz", hash = "sha256:9ad824308ded0ad06cc716434705f691d4ee0bfd0fb081839d2e426860e7fdca", size = 104755, upload-time = "2025-09-23T09:47:48.256Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/00/22/35617eee79080a5d071d0f14ad698d325ee6b3bf824fc0467c03b30e7fa8/typer-0.19.2-py3-none-any.whl", hash = "sha256:755e7e19670ffad8283db353267cb81ef252f595aa6834a0d1ca9312d9326cb9", size = 46748, upload-time = "2025-09-23T09:47:46.777Z" },+]++[[package]]+name = "typing-extensions"+version = "4.15.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },+]++[[package]]+name = "urllib3"+version = "2.5.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },+]