A high-performance inference engine for AI models
Decision model benchmark metrics (#1053)
Aleksandr Golokoza262d2fa9269
diff --git a/crates/legacy/cli/src/bench/decision.rs b/crates/legacy/cli/src/bench/decision.rsnew file mode 100644index 000000000..7c0220d5d--- /dev/null+++ b/crates/legacy/cli/src/bench/decision.rs@@ -0,0 +1,160 @@+use std::{+ ops::Add,+ path::Path,+ time::{SystemTime, UNIX_EPOCH},+};++use anyhow::{Context, Result, bail};+use comfy_table::{CellAlignment, ContentArrangement, Table, presets::UTF8_FULL};+use indexmap::IndexMap;+use indicatif::ProgressBar;+use serde::{Deserialize, Serialize};+use tokio::fs;+use uzu::{+ engine::{Engine, EngineConfig},+ types::session::{+ chat::ReplyEnergy,+ decision::{DecisionInput, DecisionResult},+ },+};+use uzu_engine::VERSION;++use crate::{+ bench::{model::BenchDevice, stat::calculate_metric},+ common::decision::DecisionRequestQuestion,+};++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct DecisionBenchTask {+ pub identifier: String,+ pub repo_id: String,+ pub number_of_runs: u64,+ pub state: String,+ // Keyed by question id, as in the System One request.+ pub questions: IndexMap<String, DecisionRequestQuestion>,+}++// Totals over all questions of one run.+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct DecisionBenchResult {+ pub task: DecisionBenchTask,+ pub device: BenchDevice,+ pub engine_version: String,+ pub timestamp: u64,+ pub tokens_count: u64,+ pub prefill_duration: f64,+ pub total_duration: f64,+ pub prefill_tokens_per_second: Option<f64>,+ pub energy: Option<ReplyEnergy>,+ pub joules_per_token: Option<f64>,+ pub results: IndexMap<String, DecisionResult>,+}++pub async fn run(+ task: DecisionBenchTask,+ model_path: String,+ output_path: String,+) -> Result<()> {+ println!("Model: {}", task.repo_id);++ let model_path = Path::new(&model_path)+ .canonicalize()+ .with_context(|| format!("Can not open model path: {model_path}"))?+ .to_string_lossy()+ .into_owned();+ let engine = Engine::new(EngineConfig::default()).await.context("Can not create engine")?;+ let model = engine+ .model_by_path(model_path.clone())+ .await?+ .with_context(|| format!("Model not found at path: {model_path}"))?;+ let session = engine.decision(model).await.context("Can not create decision session")?;++ let input = DecisionInput {+ state: task.state.clone(),+ questions: task.questions.values().cloned().map(Into::into).collect(),+ };+ let device = BenchDevice::current();++ // The first decision pays for kernel setup.+ session.decide(input.clone()).await?;++ let progress_bar = ProgressBar::new(task.number_of_runs);+ let mut results = Vec::<DecisionBenchResult>::new();+ for _ in 0..task.number_of_runs {+ let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();+ let output = session.decide(input.clone()).await?;++ let tokens_count = output.results.iter().map(|result| result.stats.tokens_count as u64).sum::<u64>();+ let prefill_duration = output.results.iter().map(|result| result.stats.prefill_duration).sum::<f64>();+ let total_duration = output.results.iter().map(|result| result.stats.total_duration).sum::<f64>();+ let prefill_tokens_per_second = (prefill_duration > 0.0).then(|| tokens_count as f64 / prefill_duration);+ // A partial sum would understate the run, so energy is reported only when every question has it.+ let energy = output+ .results+ .iter()+ .map(|result| result.stats.energy.clone())+ .collect::<Option<Vec<_>>>()+ .and_then(|energy| energy.into_iter().reduce(ReplyEnergy::add));+ let joules_per_token =+ energy.as_ref().and_then(|energy| (tokens_count > 0).then(|| energy.total() / tokens_count as f64));++ results.push(DecisionBenchResult {+ task: task.clone(),+ device: device.clone(),+ engine_version: VERSION.to_string(),+ timestamp,+ tokens_count,+ prefill_duration,+ total_duration,+ prefill_tokens_per_second,+ energy,+ joules_per_token,+ results: task.questions.keys().cloned().zip(output.results).collect(),+ });+ progress_bar.inc(1);+ }+ progress_bar.finish();++ let results_data = serde_json::to_string_pretty(&results).context("Failed to serialize benchmark results")?;+ fs::write(&output_path, results_data)+ .await+ .with_context(|| format!("Failed to write output file: {output_path}"))?;++ let Some(first_result) = results.first() else {+ bail!("Benchmark produced no results");+ };+ println!("Questions count: {}", task.questions.len());+ println!("Tokens count: {}", first_result.tokens_count);++ let prefill_duration_list = results.iter().map(|result| result.prefill_duration).collect::<Vec<f64>>();+ let total_duration_list = results.iter().map(|result| result.total_duration).collect::<Vec<f64>>();+ let prefill_tokens_per_second_list =+ results.iter().filter_map(|result| result.prefill_tokens_per_second).collect::<Vec<f64>>();+ let energy_joules_list =+ results.iter().filter_map(|result| result.energy.as_ref().map(ReplyEnergy::total)).collect::<Vec<f64>>();+ let joules_per_token_list = results.iter().filter_map(|result| result.joules_per_token).collect::<Vec<f64>>();++ let prefill_duration_metric = calculate_metric(&prefill_duration_list);+ let total_duration_metric = calculate_metric(&total_duration_list);+ let prefill_tokens_per_second_metric = calculate_metric(&prefill_tokens_per_second_list);+ let energy_joules_metric = calculate_metric(&energy_joules_list);+ let joules_per_token_metric = calculate_metric(&joules_per_token_list);++ let mut table = Table::new();+ table+ .load_style(UTF8_FULL.with_rounded_corners())+ .set_content_arrangement(ContentArrangement::Dynamic)+ .set_header(vec!["Metric", "Value"])+ .add_row(vec!["Prefill, s", prefill_duration_metric.as_str()])+ .add_row(vec!["Total, s", total_duration_metric.as_str()])+ .add_row(vec!["Prefill, t/s", prefill_tokens_per_second_metric.as_str()])+ .add_row(vec!["Total energy, J", energy_joules_metric.as_str()])+ .add_row(vec!["Energy per token, J/tok", joules_per_token_metric.as_str()]);+ let Some(column) = table.column_mut(1) else {+ bail!("Benchmark summary table value column is missing");+ };+ column.set_cell_alignment(CellAlignment::Right);+ println!("{table}");++ Ok(())+}diff --git a/crates/legacy/cli/src/bench/mod.rs b/crates/legacy/cli/src/bench/mod.rsindex a9ecfd45b..2c9132f6b 100644--- a/crates/legacy/cli/src/bench/mod.rs+++ b/crates/legacy/cli/src/bench/mod.rs@@ -1,3 +1,4 @@+mod decision;mod model;mod runner;mod stat;@@ -7,7 +8,7 @@ use comfy_table::{CellAlignment, ContentArrangement, Table, presets::UTF8_FULL};use indicatif::ProgressBar;use tokio::fs;-use crate::bench::{model::BenchTask, runner::BenchRunner, stat::calculate_metric};+use crate::bench::{decision::DecisionBenchTask, model::BenchTask, runner::BenchRunner, stat::calculate_metric};pub async fn run_bench(model_path: String,@@ -16,8 +17,16 @@ pub async fn run_bench() -> Result<()> {let task_content =fs::read_to_string(&task_path).await.with_context(|| format!("Failed to read task file: {task_path}"))?;- let task: BenchTask =+ let task_value: serde_json::Value =serde_json::from_str(&task_content).with_context(|| format!("Failed to parse task file: {task_path}"))?;+ if task_value.get("questions").is_some() {+ let task: DecisionBenchTask = serde_json::from_value(task_value)+ .with_context(|| format!("Failed to parse decision task file: {task_path}"))?;+ return decision::run(task, model_path, output_path).await;+ }++ let task: BenchTask =+ serde_json::from_value(task_value).with_context(|| format!("Failed to parse task file: {task_path}"))?;println!("Model: {}", task.repo_id);let progress_bar = ProgressBar::new(task.number_of_runs);diff --git a/crates/legacy/cli/src/bench/model.rs b/crates/legacy/cli/src/bench/model.rsindex afbb05bb4..f6fb0a90d 100644--- a/crates/legacy/cli/src/bench/model.rs+++ b/crates/legacy/cli/src/bench/model.rs@@ -1,8 +1,9 @@use anyhow::{Result, anyhow, ensure};use rocket::serde::{Deserialize, Serialize};+use sysinfo::System;use uzu::types::{basic::ReasoningEffort,- session::chat::{ChatMessage, ChatReply, ChatReplyEnergy, ChatRole},+ session::chat::{ChatMessage, ChatReply, ChatRole, ReplyEnergy},};use crate::{@@ -98,6 +99,23 @@ pub struct BenchDevice {pub memory_total: u64,}+impl BenchDevice {+ pub fn current() -> Self {+ let mut system_info = System::new_all();+ system_info.refresh_all();++ let os_name = System::long_os_version();+ let cpu_name = system_info.cpus().first().map(|cpu| cpu.brand().to_string());+ let memory_total = system_info.total_memory();++ Self {+ os_name,+ cpu_name,+ memory_total,+ }+ }+}+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct BenchResult {pub task: BenchTask,@@ -111,8 +129,8 @@ pub struct BenchResult {pub time_to_first_token: f64,pub prompt_tokens_per_second: f64,pub generate_tokens_per_second: Option<f64>,- pub input_energy: Option<ChatReplyEnergy>,- pub output_energy: Option<ChatReplyEnergy>,+ pub input_energy: Option<ReplyEnergy>,+ pub output_energy: Option<ReplyEnergy>,pub joules_per_token: Option<f64>,pub text: String,#[serde(default, skip_serializing_if = "Option::is_none")]diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rsindex 767cff52d..c03818e75 100644--- a/crates/legacy/cli/src/bench/runner.rs+++ b/crates/legacy/cli/src/bench/runner.rs@@ -5,13 +5,12 @@ use std::{use anyhow::{Context, Result};use shoji::types::model::{ModelFamily, ModelSource};-use sysinfo::System;use uzu::{engine::{Engine, EngineConfig},types::{basic::SamplingMethod,model::ModelAccessibility,- session::chat::{ChatConfig, ChatReplyConfig, ChatReplyEnergy},+ session::chat::{ChatConfig, ChatReplyConfig, ReplyEnergy},},};use uzu_engine::VERSION;@@ -64,7 +63,7 @@ impl BenchRunner {}let messages = self.task.to_chat_messages(ThinkingSupport::for_model(&model))?;- let device = self.get_device_info();+ let device = BenchDevice::current();let session_config = ChatConfig::default();let session = engine.chat(model, session_config).await?;@@ -144,21 +143,6 @@ impl BenchRunner {Ok(results)}- fn get_device_info(&self) -> BenchDevice {- let mut system_info = System::new_all();- system_info.refresh_all();-- let os_name = System::long_os_version();- let cpu_name = system_info.cpus().first().map(|cpu| cpu.brand().to_string());- let memory_total = system_info.total_memory();-- BenchDevice {- os_name,- cpu_name,- memory_total,- }- }-async fn get_remote_model_family(&self,model_path: &Path,@@ -199,6 +183,6 @@ impl BenchRunner {}}-fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ChatReplyEnergy>) -> Option<ChatReplyEnergy> {+fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ReplyEnergy>) -> Option<ReplyEnergy> {energy.into_iter().cloned().reduce(|total, energy| total + energy)}diff --git a/crates/legacy/cli/src/common/decision.rs b/crates/legacy/cli/src/common/decision.rsnew file mode 100644index 000000000..2b8a4daa5--- /dev/null+++ b/crates/legacy/cli/src/common/decision.rs@@ -0,0 +1,70 @@+use indexmap::IndexMap;+use serde::{Deserialize, Serialize};+use shoji::types::session::decision::{DecisionOption, DecisionQuestion};++// A question of the System One request of vLLM Semantic Router, shared by the server and the bench.+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]+pub enum DecisionRequestQuestion {+ Choice {+ instructions: Option<String>,+ // Option order is answer-code order, so it must survive parsing.+ criteria: IndexMap<String, Option<String>>,+ },+ Noul {+ instructions: Option<String>,+ criteria: Option<NoulCriteria>,+ },+ Score {+ instructions: Option<String>,+ criteria: Vec<String>,+ },+}++#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct NoulCriteria {+ #[serde(rename = "false")]+ no: Option<String>,+ #[serde(rename = "true")]+ yes: Option<String>,+}++impl From<DecisionRequestQuestion> for DecisionQuestion {+ fn from(question: DecisionRequestQuestion) -> Self {+ match question {+ DecisionRequestQuestion::Choice {+ instructions,+ criteria,+ } => DecisionQuestion::Choice {+ instructions,+ options: criteria+ .into_iter()+ .map(|(key, description)| DecisionOption {+ key,+ description,+ })+ .collect(),+ },+ DecisionRequestQuestion::Noul {+ instructions,+ criteria,+ } => {+ let (no_description, yes_description) =+ criteria.map_or((None, None), |criteria| (criteria.no, criteria.yes));+ DecisionQuestion::YesNo {+ instructions,+ no_description,+ yes_description,+ }+ },+ DecisionRequestQuestion::Score {+ instructions,+ criteria,+ } => DecisionQuestion::Score {+ instructions,+ levels: criteria,+ },+ }+ }+}diff --git a/crates/legacy/cli/src/common/mod.rs b/crates/legacy/cli/src/common/mod.rsindex 9602a9e2c..c4997e405 100644--- a/crates/legacy/cli/src/common/mod.rs+++ b/crates/legacy/cli/src/common/mod.rs@@ -1 +1,2 @@+pub mod decision;pub mod thinking;diff --git a/crates/legacy/cli/src/server/decisions/request.rs b/crates/legacy/cli/src/server/decisions/request.rsindex 6043aad04..9ee844eaf 100644--- a/crates/legacy/cli/src/server/decisions/request.rs+++ b/crates/legacy/cli/src/server/decisions/request.rs@@ -1,6 +1,8 @@use indexmap::IndexMap;use serde::Deserialize;-use shoji::types::session::decision::{DecisionInput, DecisionOption, DecisionQuestion};+use shoji::types::session::decision::DecisionInput;++use crate::common::decision::DecisionRequestQuestion;// The System One request of vLLM Semantic Router and d3's `d3_server.py`, text only:// `state` must be a string and there are no `images` or `videos`.@@ -17,72 +19,6 @@ pub struct DecisionRequest {pub questions: IndexMap<String, serde_json::Value>,}-#[derive(Deserialize)]-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]-enum DecisionRequestQuestion {- Choice {- instructions: Option<String>,- // Option order is answer-code order, so it must survive parsing.- criteria: IndexMap<String, Option<String>>,- },- Noul {- instructions: Option<String>,- criteria: Option<NoulCriteria>,- },- Score {- instructions: Option<String>,- criteria: Vec<String>,- },-}--#[derive(Deserialize)]-#[serde(deny_unknown_fields)]-struct NoulCriteria {- #[serde(rename = "false")]- no: Option<String>,- #[serde(rename = "true")]- yes: Option<String>,-}--impl From<DecisionRequestQuestion> for DecisionQuestion {- fn from(question: DecisionRequestQuestion) -> Self {- match question {- DecisionRequestQuestion::Choice {- instructions,- criteria,- } => DecisionQuestion::Choice {- instructions,- options: criteria- .into_iter()- .map(|(key, description)| DecisionOption {- key,- description,- })- .collect(),- },- DecisionRequestQuestion::Noul {- instructions,- criteria,- } => {- let (no_description, yes_description) =- criteria.map_or((None, None), |criteria| (criteria.no, criteria.yes));- DecisionQuestion::YesNo {- instructions,- no_description,- yes_description,- }- },- DecisionRequestQuestion::Score {- instructions,- criteria,- } => DecisionQuestion::Score {- instructions,- levels: criteria,- },- }- }-}-pub struct InvalidQuestion {pub id: String,pub message: String,diff --git a/crates/legacy/nagare/src/chat/mod.rs b/crates/legacy/nagare/src/chat/mod.rsindex 13eba202f..ae4fcbb65 100644--- a/crates/legacy/nagare/src/chat/mod.rs+++ b/crates/legacy/nagare/src/chat/mod.rs@@ -28,8 +28,8 @@ use shoji::{basic::{CancelToken, SamplingParameters, ToolCall, ToolDescription, ToolNamespace, Value, parse_lenient_json},model::Model,session::chat::{- ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyEnergy,- ChatReplyFinishReason, ChatReplySpeculatorStats, ChatReplyStats, ChatRole,+ ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,+ ChatReplySpeculatorStats, ChatReplyStats, ChatRole, ReplyEnergy,},},};@@ -804,7 +804,7 @@ fn sum_optional_u32(stats.iter().try_fold(0_u32, |sum, stats| sum.checked_add(value(stats)?))}-fn aggregate_input_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy> {+fn aggregate_input_energy(stats: &[&ChatReplyStats]) -> Option<ReplyEnergy> {let mut phases = Vec::with_capacity(stats.len());for stats in stats {phases.push(stats.input_energy.as_ref()?);@@ -812,7 +812,7 @@ fn aggregate_input_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy>aggregate_energy(phases)}-fn aggregate_output_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy> {+fn aggregate_output_energy(stats: &[&ChatReplyStats]) -> Option<ReplyEnergy> {let mut phases = Vec::with_capacity(stats.len());for stats in stats {let tokens = stats.tokens_count_output?;@@ -824,7 +824,7 @@ fn aggregate_output_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy>aggregate_energy(phases)}-fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ChatReplyEnergy>) -> Option<ChatReplyEnergy> {+fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ReplyEnergy>) -> Option<ReplyEnergy> {energy.into_iter().cloned().reduce(|total, energy| total + energy)}diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rsindex d426217bf..7c777548f 100644--- a/crates/legacy/nagare/src/chat/token.rs+++ b/crates/legacy/nagare/src/chat/token.rs@@ -23,8 +23,8 @@ use shoji::{basic::{SamplingParameters, Token, TokenId},model::Model,session::chat::{- ChatConfig, ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyEnergy, ChatReplyFinishReason,- ChatReplySpeculatorStats, ChatReplyStats,+ ChatConfig, ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyFinishReason,+ ChatReplySpeculatorStats, ChatReplyStats, ReplyEnergy,},},};@@ -432,7 +432,7 @@ struct StreamingState<'a> {time_last_token: Option<Instant>,time_prefill_start: Instant,time_first_token: Option<Instant>,- input_energy: Option<ChatReplyEnergy>,+ input_energy: Option<ReplyEnergy>,total_tokens_input: usize,cached_tokens_input: usize,total_tokens_output: usize,diff --git a/crates/legacy/nagare/src/decision/error.rs b/crates/legacy/nagare/src/decision/error.rsindex ff8c0cd59..cc6b302ef 100644--- a/crates/legacy/nagare/src/decision/error.rs+++ b/crates/legacy/nagare/src/decision/error.rs@@ -1,3 +1,6 @@+#[cfg(any(target_os = "macos", target_os = "ios"))]+use crate::util::power::Error as EnergyError;+#[bindings::export(Error)]#[derive(Debug, Clone, PartialEq, thiserror::Error)]#[non_exhaustive]@@ -24,3 +27,12 @@ pub enum DecisionSessionError {#[error("No response")]NoResponse {},}++#[cfg(any(target_os = "macos", target_os = "ios"))]+impl From<EnergyError> for DecisionSessionError {+ fn from(error: EnergyError) -> Self {+ Self::Backend {+ message: error.to_string(),+ }+ }+}diff --git a/crates/legacy/nagare/src/decision/session.rs b/crates/legacy/nagare/src/decision/session.rsindex a5cbea080..a7cb787a4 100644--- a/crates/legacy/nagare/src/decision/session.rs+++ b/crates/legacy/nagare/src/decision/session.rs@@ -1,6 +1,8 @@use std::{sync::Arc, time::Instant};use hanashi::{DecisionEncoding, DecisionEncodingError};+#[cfg(any(target_os = "macos", target_os = "ios"))]+use keisoku::KeisokuError;use serde::{Deserialize, Serialize};use shoji::{traits::{Backend, backend::decision::DecisionInstance},@@ -14,6 +16,8 @@ use shoji::{use tokio::sync::{Mutex, mpsc};pub use crate::decision::error::DecisionSessionError;+#[cfg(any(target_os = "macos", target_os = "ios"))]+use crate::util::power::{EnergyRecorder, Error as EnergyError};#[bindings::export(Enumeration)]#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]@@ -25,6 +29,8 @@ pub enum DecisionSessionState {struct DecisionInstanceHolder {instance: Box<dyn DecisionInstance>,encoding: DecisionEncoding,+ #[cfg(any(target_os = "macos", target_os = "ios"))]+ energy_recorder: EnergyRecorder,}#[bindings::export(Class)]@@ -54,9 +60,11 @@ impl DecisionSession {let encoding = DecisionEncoding::new(instance.token_codec_config(), instance.tokenizer()).map_err(|_| DecisionSessionError::UnsupportedModel {})?;- Ok(DecisionInstanceHolder {+ Ok::<_, DecisionSessionError>(DecisionInstanceHolder {instance,encoding,+ #[cfg(any(target_os = "macos", target_os = "ios"))]+ energy_recorder: EnergyRecorder::new(),})}).await@@ -71,7 +79,7 @@ impl DecisionSession {}async fn execute(- holder: &DecisionInstanceHolder,+ holder: &mut DecisionInstanceHolder,input: &DecisionInput,) -> Result<Vec<DecisionResult>, DecisionSessionError> {if input.questions.is_empty() {@@ -107,17 +115,39 @@ impl DecisionSession {let time_start = Instant::now();let tokens_count = prompt.len();let tokens = prompt.into_iter().map(u64::from).collect::<Vec<_>>();- let probabilities =- holder.instance.probabilities(&tokens, option_count(question)).await.map_err(|error| {- DecisionSessionError::Backend {- message: error.to_string(),- }- })?;++ #[cfg(any(target_os = "macos", target_os = "ios"))]+ holder.energy_recorder.begin()?;++ let output = holder.instance.probabilities(&tokens, option_count(question)).await.map_err(|error| {+ DecisionSessionError::Backend {+ message: error.to_string(),+ }+ });++ // Stop sampling before propagating an inference error.+ #[cfg(any(target_os = "macos", target_os = "ios"))]+ let energy = holder.energy_recorder.finish();++ let output = output?;++ #[cfg(any(target_os = "macos", target_os = "ios"))]+ let energy = match energy {+ Ok(energy) => Some(energy),+ Err(EnergyError::Keisoku(KeisokuError::PowerReadingUnavailable)) => None,+ Err(error) => return Err(error.into()),+ };++ #[cfg(not(any(target_os = "macos", target_os = "ios")))]+ let energy = None;+results.push(DecisionResult {- answer: answer(question, probabilities),+ answer: answer(question, output.probabilities),stats: DecisionStats {tokens_count: tokens_count as i64,+ prefill_duration: output.prefill_duration.as_secs_f64(),total_duration: (encoding_duration + time_start.elapsed()).as_secs_f64(),+ energy,},});}@@ -156,8 +186,8 @@ impl DecisionSession {}let result = {- let holder = holder.lock().await;- Self::execute(&holder, &input).await+ let mut holder = holder.lock().await;+ Self::execute(&mut holder, &input).await};let output = result.map(|results| DecisionOutput {results,diff --git a/crates/legacy/nagare/src/util/power/mod.rs b/crates/legacy/nagare/src/util/power/mod.rsindex a6da8a858..3a487d37f 100644--- a/crates/legacy/nagare/src/util/power/mod.rs+++ b/crates/legacy/nagare/src/util/power/mod.rs@@ -2,7 +2,7 @@ mod error;pub use error::Error;use keisoku::{PowerMeter, PowerReading};-use shoji::types::session::chat::ChatReplyEnergy;+use shoji::types::session::chat::ReplyEnergy;pub struct EnergyRecorder {meter: PowerMeter,@@ -20,20 +20,20 @@ impl EnergyRecorder {Ok(())}- pub fn split(&mut self) -> Result<ChatReplyEnergy, Error> {+ pub fn split(&mut self) -> Result<ReplyEnergy, Error> {Ok(energy(self.meter.split()?))}- pub fn finish(&mut self) -> Result<ChatReplyEnergy, Error> {+ pub fn finish(&mut self) -> Result<ReplyEnergy, Error> {Ok(energy(self.meter.stop()?))}}-fn energy(reading: PowerReading) -> ChatReplyEnergy {+fn energy(reading: PowerReading) -> ReplyEnergy {match reading {PowerReading::Total {total,- } => ChatReplyEnergy::Total {+ } => ReplyEnergy::Total {total: total.value() as f64,},PowerReading::Components {@@ -41,7 +41,7 @@ fn energy(reading: PowerReading) -> ChatReplyEnergy {gpu,ane,ram,- } => ChatReplyEnergy::Components {+ } => ReplyEnergy::Components {cpu: cpu.value() as f64,gpu: gpu.value() as f64,ane: ane.value() as f64,diff --git a/crates/legacy/shoji/src/traits/backend/decision.rs b/crates/legacy/shoji/src/traits/backend/decision.rsindex 5bc810daf..9f9307d34 100644--- a/crates/legacy/shoji/src/traits/backend/decision.rs+++ b/crates/legacy/shoji/src/traits/backend/decision.rs@@ -1,9 +1,15 @@-use std::{pin::Pin, sync::Arc};+use std::{pin::Pin, sync::Arc, time::Duration};use tokenizers::Tokenizer;use crate::{traits::backend::Error, types::session::classification::TokenCodecConfig};+pub struct DecisionSessionOutput {+ pub probabilities: Vec<f32>,+ pub prefill_duration: Duration,+ pub memory_used_bytes: Option<usize>,+}+pub type Config = ();pub trait DecisionBackend: Send + Sync {@@ -27,5 +33,5 @@ pub trait DecisionInstance: Send + Sync {&'a self,input: &'a [u64],option_count: usize,- ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>, Error>> + Send + 'a>>;+ ) -> Pin<Box<dyn Future<Output = Result<DecisionSessionOutput, Error>> + Send + 'a>>;}diff --git a/crates/legacy/shoji/src/types/session/chat/mod.rs b/crates/legacy/shoji/src/types/session/chat/mod.rsindex 9083c674a..be9008b94 100644--- a/crates/legacy/shoji/src/types/session/chat/mod.rs+++ b/crates/legacy/shoji/src/types/session/chat/mod.rs@@ -10,7 +10,7 @@ pub use config::ChatConfig;pub use content_block::{ChatContentBlock, ChatContentBlockType};pub use message::{ChatMessage, ChatMessageList, ChatMessageMetadata};pub use reply::{- ChatReply, ChatReplyConfig, ChatReplyEnergy, ChatReplyFinishReason, ChatReplyJoulesPerToken,- ChatReplySpeculatorStats, ChatReplyStats,+ ChatReply, ChatReplyConfig, ChatReplyFinishReason, ChatReplyJoulesPerToken, ChatReplySpeculatorStats,+ ChatReplyStats, ReplyEnergy,};pub use role::ChatRole;diff --git a/crates/legacy/shoji/src/types/session/chat/reply/mod.rs b/crates/legacy/shoji/src/types/session/chat/reply/mod.rsindex 4fbdadf15..b7869fe96 100644--- a/crates/legacy/shoji/src/types/session/chat/reply/mod.rs+++ b/crates/legacy/shoji/src/types/session/chat/reply/mod.rs@@ -5,7 +5,7 @@ mod stats;pub use config::ChatReplyConfig;pub use finish_reason::ChatReplyFinishReason;use serde::{Deserialize, Serialize};-pub use stats::{ChatReplyEnergy, ChatReplyJoulesPerToken, ChatReplySpeculatorStats, ChatReplyStats};+pub use stats::{ChatReplyJoulesPerToken, ChatReplySpeculatorStats, ChatReplyStats, ReplyEnergy};use crate::types::session::chat::ChatMessage;diff --git a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs b/crates/legacy/shoji/src/types/session/chat/reply/stats.rsindex 33a4c0c27..9193b4f05 100644--- a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs+++ b/crates/legacy/shoji/src/types/session/chat/reply/stats.rs@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};#[bindings::export(Enumeration)]#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]#[serde(tag = "type", rename_all = "snake_case")]-pub enum ChatReplyEnergy {+pub enum ReplyEnergy {Total {total: f64,},@@ -21,7 +21,7 @@ pub enum ChatReplyEnergy {}#[cfg_attr(feature = "bindings-uniffi", uniffi::export)]-impl ChatReplyEnergy {+impl ReplyEnergy {pub fn total(&self) -> f64 {match self {Self::Total {@@ -37,7 +37,7 @@ impl ChatReplyEnergy {}}-impl ChatReplyEnergy {+impl ReplyEnergy {fn per_token(&self,tokens_count: u32,@@ -68,7 +68,7 @@ impl ChatReplyEnergy {}}-impl Add for ChatReplyEnergy {+impl Add for ReplyEnergy {type Output = Self;fn add(@@ -172,8 +172,8 @@ pub struct ChatReplyStats {pub tokens_count_output: Option<u32>,pub memory_used_bytes: Option<i64>,pub speculator_stats: Option<ChatReplySpeculatorStats>,- pub input_energy: Option<ChatReplyEnergy>,- pub output_energy: Option<ChatReplyEnergy>,+ pub input_energy: Option<ReplyEnergy>,+ pub output_energy: Option<ReplyEnergy>,}#[bindings::export(Implementation)]@@ -185,7 +185,7 @@ impl ChatReplyStats {#[bindings::export(Method(Getter))]pub fn total_joules(&self) -> Option<f64> {- self.input_energy.iter().chain(self.output_energy.iter()).map(ChatReplyEnergy::total).reduce(f64::add)+ self.input_energy.iter().chain(self.output_energy.iter()).map(ReplyEnergy::total).reduce(f64::add)}#[bindings::export(Method(Getter))]diff --git a/crates/legacy/shoji/src/types/session/decision/mod.rs b/crates/legacy/shoji/src/types/session/decision/mod.rsindex ad1a5c68e..c5fd8bd31 100644--- a/crates/legacy/shoji/src/types/session/decision/mod.rs+++ b/crates/legacy/shoji/src/types/session/decision/mod.rs@@ -1,5 +1,7 @@use serde::{Deserialize, Serialize};+use crate::types::session::chat::ReplyEnergy;+#[bindings::export(Structure(Class))]#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]pub struct DecisionOption {@@ -67,7 +69,9 @@ pub enum DecisionAnswer {#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]pub struct DecisionStats {pub tokens_count: i64,+ pub prefill_duration: f64,pub total_duration: f64,+ pub energy: Option<ReplyEnergy>,}#[bindings::export(Structure(Class))]diff --git a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi b/crates/legacy/uzu/bindings/python/uzu/__init__.pyiindex 092079d2c..0da35d2a6 100644--- a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi+++ b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi@@ -19,7 +19,6 @@ __all__ = ["ChatModelInfo","ChatReply","ChatReplyConfig",- "ChatReplyEnergy","ChatReplyFinishReason","ChatReplyJoulesPerToken","ChatReplySpeculatorStats",@@ -79,6 +78,7 @@ __all__ = ["PcmBatch","Player","ReasoningEffort",+ "ReplyEnergy","Repository","SamplingMethod","SamplingParameters",@@ -379,29 +379,6 @@ class ChatReplyConfig:def with_tool_turn_limit(self, tool_turn_limit: typing.Optional[builtins.int]) -> ChatReplyConfig: ...def with_grammar(self, grammar: typing.Optional[Grammar]) -> ChatReplyConfig: ...-class ChatReplyEnergy:- @typing.final- class Total(ChatReplyEnergy):- __match_args__ = ("total",)- @property- def total(self) -> builtins.float: ...- def __new__(cls, total: builtins.float) -> ChatReplyEnergy.Total: ...-- @typing.final- class Components(ChatReplyEnergy):- __match_args__ = ("cpu", "gpu", "ane", "dram",)- @property- def cpu(self) -> builtins.float: ...- @property- def gpu(self) -> builtins.float: ...- @property- def ane(self) -> builtins.float: ...- @property- def dram(self) -> builtins.float: ...- def __new__(cls, cpu: builtins.float, gpu: builtins.float, ane: builtins.float, dram: builtins.float) -> ChatReplyEnergy.Components: ...-- ...-class ChatReplyJoulesPerToken:@typing.finalclass Total(ChatReplyJoulesPerToken):@@ -454,9 +431,9 @@ class ChatReplyStats:@propertydef speculator_stats(self) -> typing.Optional[ChatReplySpeculatorStats]: ...@property- def input_energy(self) -> typing.Optional[ChatReplyEnergy]: ...+ def input_energy(self) -> typing.Optional[ReplyEnergy]: ...@property- def output_energy(self) -> typing.Optional[ChatReplyEnergy]: ...+ def output_energy(self) -> typing.Optional[ReplyEnergy]: ...@propertydef tokens_count(self) -> typing.Optional[builtins.int]: ...@property@@ -465,7 +442,7 @@ class ChatReplyStats:def input_joules_per_token(self) -> typing.Optional[ChatReplyJoulesPerToken]: ...@propertydef output_joules_per_token(self) -> typing.Optional[ChatReplyJoulesPerToken]: ...- def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_input_cached: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], input_energy: typing.Optional[ChatReplyEnergy], output_energy: typing.Optional[ChatReplyEnergy]) -> ChatReplyStats: ...+ def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_input_cached: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], input_energy: typing.Optional[ReplyEnergy], output_energy: typing.Optional[ReplyEnergy]) -> ChatReplyStats: ...class ChatRole:@typing.final@@ -736,8 +713,12 @@ class DecisionStats:@propertydef tokens_count(self) -> builtins.int: ...@property+ def prefill_duration(self) -> builtins.float: ...+ @propertydef total_duration(self) -> builtins.float: ...- def __new__(cls, tokens_count: builtins.int, total_duration: builtins.float) -> DecisionStats: ...+ @property+ def energy(self) -> typing.Optional[ReplyEnergy]: ...+ def __new__(cls, tokens_count: builtins.int, prefill_duration: builtins.float, total_duration: builtins.float, energy: typing.Optional[ReplyEnergy]) -> DecisionStats: ...@typing.finalclass Device:@@ -1225,6 +1206,29 @@ class Player:def append_pcm_batch(self, batch: PcmBatch) -> None: ...def stop(self) -> None: ...+class ReplyEnergy:+ @typing.final+ class Total(ReplyEnergy):+ __match_args__ = ("total",)+ @property+ def total(self) -> builtins.float: ...+ def __new__(cls, total: builtins.float) -> ReplyEnergy.Total: ...++ @typing.final+ class Components(ReplyEnergy):+ __match_args__ = ("cpu", "gpu", "ane", "dram",)+ @property+ def cpu(self) -> builtins.float: ...+ @property+ def gpu(self) -> builtins.float: ...+ @property+ def ane(self) -> builtins.float: ...+ @property+ def dram(self) -> builtins.float: ...+ def __new__(cls, cpu: builtins.float, gpu: builtins.float, ane: builtins.float, dram: builtins.float) -> ReplyEnergy.Components: ...++ ...+@typing.finalclass Repository:@propertydiff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swiftindex 9705ac209..af97bd94c 100644--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift@@ -1245,12 +1245,12 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {public var tokensCountOutput: UInt32?public var memoryUsedBytes: Int64?public var speculatorStats: ChatReplySpeculatorStats?- public var inputEnergy: ChatReplyEnergy?- public var outputEnergy: ChatReplyEnergy?+ public var inputEnergy: ReplyEnergy?+ public var outputEnergy: ReplyEnergy?// Default memberwise initializers are never public by default, so we// declare one manually.- public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountInputCached: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, inputEnergy: ChatReplyEnergy?, outputEnergy: ChatReplyEnergy?) {+ public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountInputCached: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, inputEnergy: ReplyEnergy?, outputEnergy: ReplyEnergy?) {self.duration = durationself.timeToFirstToken = timeToFirstTokenself.prefillTokensPerSecond = prefillTokensPerSecond@@ -1321,8 +1321,8 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {tokensCountOutput: FfiConverterOptionUInt32.read(from: &buf),memoryUsedBytes: FfiConverterOptionInt64.read(from: &buf),speculatorStats: FfiConverterOptionTypeChatReplySpeculatorStats.read(from: &buf),- inputEnergy: FfiConverterOptionTypeChatReplyEnergy.read(from: &buf),- outputEnergy: FfiConverterOptionTypeChatReplyEnergy.read(from: &buf)+ inputEnergy: FfiConverterOptionTypeReplyEnergy.read(from: &buf),+ outputEnergy: FfiConverterOptionTypeReplyEnergy.read(from: &buf))}@@ -1336,8 +1336,8 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {FfiConverterOptionUInt32.write(value.tokensCountOutput, into: &buf)FfiConverterOptionInt64.write(value.memoryUsedBytes, into: &buf)FfiConverterOptionTypeChatReplySpeculatorStats.write(value.speculatorStats, into: &buf)- FfiConverterOptionTypeChatReplyEnergy.write(value.inputEnergy, into: &buf)- FfiConverterOptionTypeChatReplyEnergy.write(value.outputEnergy, into: &buf)+ FfiConverterOptionTypeReplyEnergy.write(value.inputEnergy, into: &buf)+ FfiConverterOptionTypeReplyEnergy.write(value.outputEnergy, into: &buf)}}@@ -1877,13 +1877,17 @@ public func FfiConverterTypeDecisionResult_lower(_ value: DecisionResult) -> Ruspublic struct DecisionStats: Equatable, Hashable, Codable {public var tokensCount: Int64+ public var prefillDuration: Doublepublic var totalDuration: Double+ public var energy: ReplyEnergy?// Default memberwise initializers are never public by default, so we// declare one manually.- public init(tokensCount: Int64, totalDuration: Double) {+ public init(tokensCount: Int64, prefillDuration: Double, totalDuration: Double, energy: ReplyEnergy?) {self.tokensCount = tokensCount+ self.prefillDuration = prefillDurationself.totalDuration = totalDuration+ self.energy = energy}@@ -1903,13 +1907,17 @@ public struct FfiConverterTypeDecisionStats: FfiConverterRustBuffer {returntry DecisionStats(tokensCount: FfiConverterInt64.read(from: &buf),- totalDuration: FfiConverterDouble.read(from: &buf)+ prefillDuration: FfiConverterDouble.read(from: &buf),+ totalDuration: FfiConverterDouble.read(from: &buf),+ energy: FfiConverterOptionTypeReplyEnergy.read(from: &buf))}public static func write(_ value: DecisionStats, into buf: inout [UInt8]) {FfiConverterInt64.write(value.tokensCount, into: &buf)+ FfiConverterDouble.write(value.prefillDuration, into: &buf)FfiConverterDouble.write(value.totalDuration, into: &buf)+ FfiConverterOptionTypeReplyEnergy.write(value.energy, into: &buf)}}@@ -3863,90 +3871,6 @@ public func FfiConverterTypeChatContentBlockType_lower(_ value: ChatContentBlock}-// Note that we don't yet support `indirect` for enums.-// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.--public enum ChatReplyEnergy: Equatable, Hashable, Codable {-- case total(total: Double- )- case components(cpu: Double, gpu: Double, ane: Double, dram: Double- )----public func total() -> Double {- return try! FfiConverterDouble.lift(try! rustCall() {- uniffi_shoji_fn_method_chatreplyenergy_total(- FfiConverterTypeChatReplyEnergy_lower(self),$0- )-})-}----}--#if compiler(>=6)-extension ChatReplyEnergy: Sendable {}-#endif--#if swift(>=5.8)-@_documentation(visibility: private)-#endif-public struct FfiConverterTypeChatReplyEnergy: FfiConverterRustBuffer {- typealias SwiftType = ChatReplyEnergy-- public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatReplyEnergy {- let variant: Int32 = try readInt(&buf)- switch variant {-- case 1: return .total(total: try FfiConverterDouble.read(from: &buf)- )-- case 2: return .components(cpu: try FfiConverterDouble.read(from: &buf), gpu: try FfiConverterDouble.read(from: &buf), ane: try FfiConverterDouble.read(from: &buf), dram: try FfiConverterDouble.read(from: &buf)- )-- default: throw UniffiInternalError.unexpectedEnumCase- }- }-- public static func write(_ value: ChatReplyEnergy, into buf: inout [UInt8]) {- switch value {--- case let .total(total):- writeInt(&buf, Int32(1))- FfiConverterDouble.write(total, into: &buf)--- case let .components(cpu,gpu,ane,dram):- writeInt(&buf, Int32(2))- FfiConverterDouble.write(cpu, into: &buf)- FfiConverterDouble.write(gpu, into: &buf)- FfiConverterDouble.write(ane, into: &buf)- FfiConverterDouble.write(dram, into: &buf)-- }- }-}---#if swift(>=5.8)-@_documentation(visibility: private)-#endif-public func FfiConverterTypeChatReplyEnergy_lift(_ buf: RustBuffer) throws -> ChatReplyEnergy {- return try FfiConverterTypeChatReplyEnergy.lift(buf)-}--#if swift(>=5.8)-@_documentation(visibility: private)-#endif-public func FfiConverterTypeChatReplyEnergy_lower(_ value: ChatReplyEnergy) -> RustBuffer {- return FfiConverterTypeChatReplyEnergy.lower(value)-}--// Note that we don't yet support `indirect` for enums.// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.@@ -5249,6 +5173,90 @@ public func FfiConverterTypeReasoningEffort_lower(_ value: ReasoningEffort) -> R}+// Note that we don't yet support `indirect` for enums.+// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.++public enum ReplyEnergy: Equatable, Hashable, Codable {++ case total(total: Double+ )+ case components(cpu: Double, gpu: Double, ane: Double, dram: Double+ )++++public func total() -> Double {+ return try! FfiConverterDouble.lift(try! rustCall() {+ uniffi_shoji_fn_method_replyenergy_total(+ FfiConverterTypeReplyEnergy_lower(self),$0+ )+})+}++++}++#if compiler(>=6)+extension ReplyEnergy: Sendable {}+#endif++#if swift(>=5.8)+@_documentation(visibility: private)+#endif+public struct FfiConverterTypeReplyEnergy: FfiConverterRustBuffer {+ typealias SwiftType = ReplyEnergy++ public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ReplyEnergy {+ let variant: Int32 = try readInt(&buf)+ switch variant {++ case 1: return .total(total: try FfiConverterDouble.read(from: &buf)+ )++ case 2: return .components(cpu: try FfiConverterDouble.read(from: &buf), gpu: try FfiConverterDouble.read(from: &buf), ane: try FfiConverterDouble.read(from: &buf), dram: try FfiConverterDouble.read(from: &buf)+ )++ default: throw UniffiInternalError.unexpectedEnumCase+ }+ }++ public static func write(_ value: ReplyEnergy, into buf: inout [UInt8]) {+ switch value {+++ case let .total(total):+ writeInt(&buf, Int32(1))+ FfiConverterDouble.write(total, into: &buf)+++ case let .components(cpu,gpu,ane,dram):+ writeInt(&buf, Int32(2))+ FfiConverterDouble.write(cpu, into: &buf)+ FfiConverterDouble.write(gpu, into: &buf)+ FfiConverterDouble.write(ane, into: &buf)+ FfiConverterDouble.write(dram, into: &buf)++ }+ }+}+++#if swift(>=5.8)+@_documentation(visibility: private)+#endif+public func FfiConverterTypeReplyEnergy_lift(_ buf: RustBuffer) throws -> ReplyEnergy {+ return try FfiConverterTypeReplyEnergy.lift(buf)+}++#if swift(>=5.8)+@_documentation(visibility: private)+#endif+public func FfiConverterTypeReplyEnergy_lower(_ value: ReplyEnergy) -> RustBuffer {+ return FfiConverterTypeReplyEnergy.lower(value)+}++// Note that we don't yet support `indirect` for enums.// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.@@ -5843,8 +5851,8 @@ fileprivate struct FfiConverterOptionTypeValue: FfiConverterRustBuffer {#if swift(>=5.8)@_documentation(visibility: private)#endif-fileprivate struct FfiConverterOptionTypeChatReplyEnergy: FfiConverterRustBuffer {- typealias SwiftType = ChatReplyEnergy?+fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRustBuffer {+ typealias SwiftType = ChatReplyFinishReason?public static func write(_ value: SwiftType, into buf: inout [UInt8]) {guard let value = value else {@@ -5852,13 +5860,13 @@ fileprivate struct FfiConverterOptionTypeChatReplyEnergy: FfiConverterRustBufferreturn}writeInt(&buf, Int8(1))- FfiConverterTypeChatReplyEnergy.write(value, into: &buf)+ FfiConverterTypeChatReplyFinishReason.write(value, into: &buf)}public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {switch try readInt(&buf) as Int8 {case 0: return nil- case 1: return try FfiConverterTypeChatReplyEnergy.read(from: &buf)+ case 1: return try FfiConverterTypeChatReplyFinishReason.read(from: &buf)default: throw UniffiInternalError.unexpectedOptionalTag}}@@ -5867,8 +5875,8 @@ fileprivate struct FfiConverterOptionTypeChatReplyEnergy: FfiConverterRustBuffer#if swift(>=5.8)@_documentation(visibility: private)#endif-fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRustBuffer {- typealias SwiftType = ChatReplyFinishReason?+fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRustBuffer {+ typealias SwiftType = ChatReplyJoulesPerToken?public static func write(_ value: SwiftType, into buf: inout [UInt8]) {guard let value = value else {@@ -5876,13 +5884,13 @@ fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRustreturn}writeInt(&buf, Int8(1))- FfiConverterTypeChatReplyFinishReason.write(value, into: &buf)+ FfiConverterTypeChatReplyJoulesPerToken.write(value, into: &buf)}public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {switch try readInt(&buf) as Int8 {case 0: return nil- case 1: return try FfiConverterTypeChatReplyFinishReason.read(from: &buf)+ case 1: return try FfiConverterTypeChatReplyJoulesPerToken.read(from: &buf)default: throw UniffiInternalError.unexpectedOptionalTag}}@@ -5891,8 +5899,8 @@ fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRust#if swift(>=5.8)@_documentation(visibility: private)#endif-fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRustBuffer {- typealias SwiftType = ChatReplyJoulesPerToken?+fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {+ typealias SwiftType = Grammar?public static func write(_ value: SwiftType, into buf: inout [UInt8]) {guard let value = value else {@@ -5900,13 +5908,13 @@ fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRureturn}writeInt(&buf, Int8(1))- FfiConverterTypeChatReplyJoulesPerToken.write(value, into: &buf)+ FfiConverterTypeGrammar.write(value, into: &buf)}public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {switch try readInt(&buf) as Int8 {case 0: return nil- case 1: return try FfiConverterTypeChatReplyJoulesPerToken.read(from: &buf)+ case 1: return try FfiConverterTypeGrammar.read(from: &buf)default: throw UniffiInternalError.unexpectedOptionalTag}}@@ -5915,8 +5923,8 @@ fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRu#if swift(>=5.8)@_documentation(visibility: private)#endif-fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {- typealias SwiftType = Grammar?+fileprivate struct FfiConverterOptionTypeReasoningEffort: FfiConverterRustBuffer {+ typealias SwiftType = ReasoningEffort?public static func write(_ value: SwiftType, into buf: inout [UInt8]) {guard let value = value else {@@ -5924,13 +5932,13 @@ fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {return}writeInt(&buf, Int8(1))- FfiConverterTypeGrammar.write(value, into: &buf)+ FfiConverterTypeReasoningEffort.write(value, into: &buf)}public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {switch try readInt(&buf) as Int8 {case 0: return nil- case 1: return try FfiConverterTypeGrammar.read(from: &buf)+ case 1: return try FfiConverterTypeReasoningEffort.read(from: &buf)default: throw UniffiInternalError.unexpectedOptionalTag}}@@ -5939,8 +5947,8 @@ fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {#if swift(>=5.8)@_documentation(visibility: private)#endif-fileprivate struct FfiConverterOptionTypeReasoningEffort: FfiConverterRustBuffer {- typealias SwiftType = ReasoningEffort?+fileprivate struct FfiConverterOptionTypeReplyEnergy: FfiConverterRustBuffer {+ typealias SwiftType = ReplyEnergy?public static func write(_ value: SwiftType, into buf: inout [UInt8]) {guard let value = value else {@@ -5948,13 +5956,13 @@ fileprivate struct FfiConverterOptionTypeReasoningEffort: FfiConverterRustBufferreturn}writeInt(&buf, Int8(1))- FfiConverterTypeReasoningEffort.write(value, into: &buf)+ FfiConverterTypeReplyEnergy.write(value, into: &buf)}public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {switch try readInt(&buf) as Int8 {case 0: return nil- case 1: return try FfiConverterTypeReasoningEffort.read(from: &buf)+ case 1: return try FfiConverterTypeReplyEnergy.read(from: &buf)default: throw UniffiInternalError.unexpectedOptionalTag}}diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mtsindex 82cce3f98..c95ac4238 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts@@ -376,19 +376,6 @@ export declare class ChatReplyConfig {withGrammar(grammar?: Grammar | undefined | null): ChatReplyConfig}-export declare class ChatReplyEnergyComponents {- cpu: number- gpu: number- ane: number- dram: number- constructor(cpu: number, gpu: number, ane: number, dram: number)-}--export declare class ChatReplyEnergyTotal {- total: number- constructor(total: number)-}-export declare class ChatReplyJoulesPerTokenComponents {cpu: numbergpu: number@@ -418,9 +405,9 @@ export declare class ChatReplyStats {tokensCountOutput?: numbermemoryUsedBytes?: numberspeculatorStats?: ChatReplySpeculatorStats- inputEnergy?: ChatReplyEnergy- outputEnergy?: ChatReplyEnergy- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ChatReplyEnergy, outputEnergy?: ChatReplyEnergy)+ inputEnergy?: ReplyEnergy+ outputEnergy?: ReplyEnergy+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ReplyEnergy, outputEnergy?: ReplyEnergy)get tokensCount(): number | nullget totalJoules(): number | nullget inputJoulesPerToken(): ChatReplyJoulesPerToken | null@@ -564,8 +551,10 @@ export declare class DecisionResult {export declare class DecisionStats {tokensCount: number+ prefillDuration: numbertotalDuration: number- constructor(tokensCount: number, totalDuration: number)+ energy?: ReplyEnergy+ constructor(tokensCount: number, prefillDuration: number, totalDuration: number, energy?: ReplyEnergy)}export declare class Feature {@@ -761,6 +750,19 @@ export declare class PcmBatch {saveAsWav(path: string): void}+export declare class ReplyEnergyComponents {+ cpu: number+ gpu: number+ ane: number+ dram: number+ constructor(cpu: number, gpu: number, ane: number, dram: number)+}++export declare class ReplyEnergyTotal {+ total: number+ constructor(total: number)+}+export declare class Repository {identifier: stringcommitHash?: string@@ -900,9 +902,6 @@ export interface ChatMessageMetadata {values: Record<string, Value>}-export type ChatReplyEnergy =- ChatReplyEnergyTotal | ChatReplyEnergyComponents-export declare const enum ChatReplyFinishReason {Stop = 'Stop',Length = 'Length',@@ -980,6 +979,9 @@ export declare const enum ReasoningEffort {XHigh = 'XHigh'}+export type ReplyEnergy =+ ReplyEnergyTotal | ReplyEnergyComponents+export type SamplingMethod =SamplingMethodGreedy | SamplingMethodStochasticdiff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.tsindex 82cce3f98..c95ac4238 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts@@ -376,19 +376,6 @@ export declare class ChatReplyConfig {withGrammar(grammar?: Grammar | undefined | null): ChatReplyConfig}-export declare class ChatReplyEnergyComponents {- cpu: number- gpu: number- ane: number- dram: number- constructor(cpu: number, gpu: number, ane: number, dram: number)-}--export declare class ChatReplyEnergyTotal {- total: number- constructor(total: number)-}-export declare class ChatReplyJoulesPerTokenComponents {cpu: numbergpu: number@@ -418,9 +405,9 @@ export declare class ChatReplyStats {tokensCountOutput?: numbermemoryUsedBytes?: numberspeculatorStats?: ChatReplySpeculatorStats- inputEnergy?: ChatReplyEnergy- outputEnergy?: ChatReplyEnergy- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ChatReplyEnergy, outputEnergy?: ChatReplyEnergy)+ inputEnergy?: ReplyEnergy+ outputEnergy?: ReplyEnergy+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ReplyEnergy, outputEnergy?: ReplyEnergy)get tokensCount(): number | nullget totalJoules(): number | nullget inputJoulesPerToken(): ChatReplyJoulesPerToken | null@@ -564,8 +551,10 @@ export declare class DecisionResult {export declare class DecisionStats {tokensCount: number+ prefillDuration: numbertotalDuration: number- constructor(tokensCount: number, totalDuration: number)+ energy?: ReplyEnergy+ constructor(tokensCount: number, prefillDuration: number, totalDuration: number, energy?: ReplyEnergy)}export declare class Feature {@@ -761,6 +750,19 @@ export declare class PcmBatch {saveAsWav(path: string): void}+export declare class ReplyEnergyComponents {+ cpu: number+ gpu: number+ ane: number+ dram: number+ constructor(cpu: number, gpu: number, ane: number, dram: number)+}++export declare class ReplyEnergyTotal {+ total: number+ constructor(total: number)+}+export declare class Repository {identifier: stringcommitHash?: string@@ -900,9 +902,6 @@ export interface ChatMessageMetadata {values: Record<string, Value>}-export type ChatReplyEnergy =- ChatReplyEnergyTotal | ChatReplyEnergyComponents-export declare const enum ChatReplyFinishReason {Stop = 'Stop',Length = 'Length',@@ -980,6 +979,9 @@ export declare const enum ReasoningEffort {XHigh = 'XHigh'}+export type ReplyEnergy =+ ReplyEnergyTotal | ReplyEnergyComponents+export type SamplingMethod =SamplingMethodGreedy | SamplingMethodStochasticdiff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.js b/crates/legacy/uzu/bindings/typescript/src/napi/index.jsindex a9d3f5008..56899d3ba 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.js+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.js@@ -559,8 +559,6 @@ module.exports.ChatMessage = nativeBinding.ChatMessagemodule.exports.ChatModelCapabilities = nativeBinding.ChatModelCapabilitiesmodule.exports.ChatReply = nativeBinding.ChatReplymodule.exports.ChatReplyConfig = nativeBinding.ChatReplyConfig-module.exports.ChatReplyEnergyComponents = nativeBinding.ChatReplyEnergyComponents-module.exports.ChatReplyEnergyTotal = nativeBinding.ChatReplyEnergyTotalmodule.exports.ChatReplyJoulesPerTokenComponents = nativeBinding.ChatReplyJoulesPerTokenComponentsmodule.exports.ChatReplyJoulesPerTokenTotal = nativeBinding.ChatReplyJoulesPerTokenTotalmodule.exports.ChatReplySpeculatorStats = nativeBinding.ChatReplySpeculatorStats@@ -615,6 +613,8 @@ module.exports.ModelSpecializationTextToSpeech = nativeBinding.ModelSpecializatimodule.exports.ModelSpecializationTranslation = nativeBinding.ModelSpecializationTranslationmodule.exports.ModelVendor = nativeBinding.ModelVendormodule.exports.PcmBatch = nativeBinding.PcmBatch+module.exports.ReplyEnergyComponents = nativeBinding.ReplyEnergyComponents+module.exports.ReplyEnergyTotal = nativeBinding.ReplyEnergyTotalmodule.exports.Repository = nativeBinding.Repositorymodule.exports.SamplingMethodGreedy = nativeBinding.SamplingMethodGreedymodule.exports.SamplingMethodStochastic = nativeBinding.SamplingMethodStochasticdiff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjsindex 8d63f1a86..081f6024a 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs@@ -56,8 +56,6 @@ export const ChatMessage = cjs.ChatMessage;export const ChatModelCapabilities = cjs.ChatModelCapabilities;export const ChatReply = cjs.ChatReply;export const ChatReplyConfig = cjs.ChatReplyConfig;-export const ChatReplyEnergyComponents = cjs.ChatReplyEnergyComponents;-export const ChatReplyEnergyTotal = cjs.ChatReplyEnergyTotal;export const ChatReplyJoulesPerTokenComponents = cjs.ChatReplyJoulesPerTokenComponents;export const ChatReplyJoulesPerTokenTotal = cjs.ChatReplyJoulesPerTokenTotal;export const ChatReplySpeculatorStats = cjs.ChatReplySpeculatorStats;@@ -112,6 +110,8 @@ export const ModelSpecializationTextToSpeech = cjs.ModelSpecializationTextToSpeeexport const ModelSpecializationTranslation = cjs.ModelSpecializationTranslation;export const ModelVendor = cjs.ModelVendor;export const PcmBatch = cjs.PcmBatch;+export const ReplyEnergyComponents = cjs.ReplyEnergyComponents;+export const ReplyEnergyTotal = cjs.ReplyEnergyTotal;export const Repository = cjs.Repository;export const SamplingMethodGreedy = cjs.SamplingMethodGreedy;export const SamplingMethodStochastic = cjs.SamplingMethodStochastic;diff --git a/crates/legacy/uzu/src/engine/bridge/decision_backend.rs b/crates/legacy/uzu/src/engine/bridge/decision_backend.rsindex b8fc8558e..ffb4ea0fd 100644--- a/crates/legacy/uzu/src/engine/bridge/decision_backend.rs+++ b/crates/legacy/uzu/src/engine/bridge/decision_backend.rs@@ -1,7 +1,10 @@use std::{path::PathBuf, pin::Pin, sync::Arc};use shoji::{- traits::backend::{Error as BackendError, decision::DecisionInstance},+ traits::backend::{+ Error as BackendError,+ decision::{DecisionInstance, DecisionSessionOutput},+ },types::session::classification::TokenCodecConfig,};use tokenizers::Tokenizer;@@ -42,7 +45,7 @@ impl<B: Backend> DecisionInstance for UzuDecisionBackendInstance<B> {&'a self,input: &'a [u64],option_count: usize,- ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>, BackendError>> + Send + 'a>> {+ ) -> Pin<Box<dyn Future<Output = Result<DecisionSessionOutput, BackendError>> + Send + 'a>> {Box::pin(async move {self.model.probabilities(input, option_count).await.map_err(|err| BackendError::from(err.to_string()))})diff --git a/crates/uzu-engine/src/engine/decision_model.rs b/crates/uzu-engine/src/engine/decision_model.rsindex ccc91d362..c3a38c6ec 100644--- a/crates/uzu-engine/src/engine/decision_model.rs+++ b/crates/uzu-engine/src/engine/decision_model.rs@@ -1,6 +1,6 @@-use std::{io, path::Path, sync::Arc};+use std::{io, path::Path, sync::Arc, time::Instant};-use shoji::types::session::classification::TokenCodecConfig;+use shoji::{traits::backend::decision::DecisionSessionOutput, types::session::classification::TokenCodecConfig};use thiserror::Error;use tokenizers::Tokenizer;@@ -67,7 +67,7 @@ impl<B: Backend> DecisionModel<B> {&self,input: &[u64],option_count: usize,- ) -> Result<Vec<f32>, DecisionModelError<B>> {+ ) -> Result<DecisionSessionOutput, DecisionModelError<B>> {if input.is_empty() {return Err(DecisionModelError::EmptyInput);}@@ -88,13 +88,14 @@ impl<B: Backend> DecisionModel<B> {});}- let mut command_buffer = self.context.create_command_buffer(None, None).map_err(DecisionModelError::Backend)?;+ let prefill_start = Instant::now();+ let mut command_buffer = self.context.create_command_buffer(None, None).map_err(DecisionModelError::Backend)?;let token_ids = command_buffer.allocate_constant_from_slice(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>()).map_err(DecisionModelError::Backend)?;- let logits = self.decision.encode(&token_ids, input.len() as u32, &mut command_buffer)?;+ let logits = self.decision.encode(&token_ids, input.len() as u32, &mut command_buffer)?;let mut output_buffer = self.context.create_buffer_gpu(answer_count * DataType::F32.size_in_bytes())@@ -105,14 +106,20 @@ impl<B: Backend> DecisionModel<B> {command_buffer.end_encoding().submit().wait_until_completed().await.map_err(DecisionModelError::Backend)?;+ let prefill_duration = prefill_start.elapsed();+let temperature = self.decision_config.temperature;let logits = self.context.copyout::<f32>(&output_buffer).await.map_err(DecisionModelError::Backend)?;let scaled = logits[..option_count].iter().map(|logit| logit / temperature).collect::<Vec<_>>();let max = scaled.iter().copied().fold(f32::NEG_INFINITY, f32::max);let exps = scaled.iter().map(|logit| (logit - max).exp()).collect::<Vec<_>>();let sum = exps.iter().sum::<f32>();- let result = exps.into_iter().map(|value| value / sum).collect();- Ok(result)+ let probabilities = exps.into_iter().map(|value| value / sum).collect();+ Ok(DecisionSessionOutput {+ probabilities,+ prefill_duration,+ memory_used_bytes: self.context.peak_memory_usage(),+ })}}