Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Decision model benchmark metrics (#1053)

Aleksandr Golokoza262d2fa9269
diff --git a/crates/legacy/cli/src/bench/decision.rs b/crates/legacy/cli/src/bench/decision.rs
new file mode 100644
index 000000000..7c0220d5d
--- /dev/null
+++ b/crates/legacy/cli/src/bench/decision.rs
@@ -0,0 +1,160 @@
+use std::{
+ ops::Add,
+ path::Path,
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+use anyhow::{Context, Result, bail};
+use comfy_table::{CellAlignment, ContentArrangement, Table, presets::UTF8_FULL};
+use indexmap::IndexMap;
+use indicatif::ProgressBar;
+use serde::{Deserialize, Serialize};
+use tokio::fs;
+use uzu::{
+ engine::{Engine, EngineConfig},
+ types::session::{
+ chat::ReplyEnergy,
+ decision::{DecisionInput, DecisionResult},
+ },
+};
+use uzu_engine::VERSION;
+
+use crate::{
+ bench::{model::BenchDevice, stat::calculate_metric},
+ common::decision::DecisionRequestQuestion,
+};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DecisionBenchTask {
+ pub identifier: String,
+ pub repo_id: String,
+ pub number_of_runs: u64,
+ pub state: String,
+ // Keyed by question id, as in the System One request.
+ pub questions: IndexMap<String, DecisionRequestQuestion>,
+}
+
+// Totals over all questions of one run.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DecisionBenchResult {
+ pub task: DecisionBenchTask,
+ pub device: BenchDevice,
+ pub engine_version: String,
+ pub timestamp: u64,
+ pub tokens_count: u64,
+ pub prefill_duration: f64,
+ pub total_duration: f64,
+ pub prefill_tokens_per_second: Option<f64>,
+ pub energy: Option<ReplyEnergy>,
+ pub joules_per_token: Option<f64>,
+ pub results: IndexMap<String, DecisionResult>,
+}
+
+pub async fn run(
+ task: DecisionBenchTask,
+ model_path: String,
+ output_path: String,
+) -> Result<()> {
+ println!("Model: {}", task.repo_id);
+
+ let model_path = Path::new(&model_path)
+ .canonicalize()
+ .with_context(|| format!("Can not open model path: {model_path}"))?
+ .to_string_lossy()
+ .into_owned();
+ let engine = Engine::new(EngineConfig::default()).await.context("Can not create engine")?;
+ let model = engine
+ .model_by_path(model_path.clone())
+ .await?
+ .with_context(|| format!("Model not found at path: {model_path}"))?;
+ let session = engine.decision(model).await.context("Can not create decision session")?;
+
+ let input = DecisionInput {
+ state: task.state.clone(),
+ questions: task.questions.values().cloned().map(Into::into).collect(),
+ };
+ let device = BenchDevice::current();
+
+ // The first decision pays for kernel setup.
+ session.decide(input.clone()).await?;
+
+ let progress_bar = ProgressBar::new(task.number_of_runs);
+ let mut results = Vec::<DecisionBenchResult>::new();
+ for _ in 0..task.number_of_runs {
+ let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
+ let output = session.decide(input.clone()).await?;
+
+ let tokens_count = output.results.iter().map(|result| result.stats.tokens_count as u64).sum::<u64>();
+ let prefill_duration = output.results.iter().map(|result| result.stats.prefill_duration).sum::<f64>();
+ let total_duration = output.results.iter().map(|result| result.stats.total_duration).sum::<f64>();
+ let prefill_tokens_per_second = (prefill_duration > 0.0).then(|| tokens_count as f64 / prefill_duration);
+ // A partial sum would understate the run, so energy is reported only when every question has it.
+ let energy = output
+ .results
+ .iter()
+ .map(|result| result.stats.energy.clone())
+ .collect::<Option<Vec<_>>>()
+ .and_then(|energy| energy.into_iter().reduce(ReplyEnergy::add));
+ let joules_per_token =
+ energy.as_ref().and_then(|energy| (tokens_count > 0).then(|| energy.total() / tokens_count as f64));
+
+ results.push(DecisionBenchResult {
+ task: task.clone(),
+ device: device.clone(),
+ engine_version: VERSION.to_string(),
+ timestamp,
+ tokens_count,
+ prefill_duration,
+ total_duration,
+ prefill_tokens_per_second,
+ energy,
+ joules_per_token,
+ results: task.questions.keys().cloned().zip(output.results).collect(),
+ });
+ progress_bar.inc(1);
+ }
+ progress_bar.finish();
+
+ let results_data = serde_json::to_string_pretty(&results).context("Failed to serialize benchmark results")?;
+ fs::write(&output_path, results_data)
+ .await
+ .with_context(|| format!("Failed to write output file: {output_path}"))?;
+
+ let Some(first_result) = results.first() else {
+ bail!("Benchmark produced no results");
+ };
+ println!("Questions count: {}", task.questions.len());
+ println!("Tokens count: {}", first_result.tokens_count);
+
+ let prefill_duration_list = results.iter().map(|result| result.prefill_duration).collect::<Vec<f64>>();
+ let total_duration_list = results.iter().map(|result| result.total_duration).collect::<Vec<f64>>();
+ let prefill_tokens_per_second_list =
+ results.iter().filter_map(|result| result.prefill_tokens_per_second).collect::<Vec<f64>>();
+ let energy_joules_list =
+ results.iter().filter_map(|result| result.energy.as_ref().map(ReplyEnergy::total)).collect::<Vec<f64>>();
+ let joules_per_token_list = results.iter().filter_map(|result| result.joules_per_token).collect::<Vec<f64>>();
+
+ let prefill_duration_metric = calculate_metric(&prefill_duration_list);
+ let total_duration_metric = calculate_metric(&total_duration_list);
+ let prefill_tokens_per_second_metric = calculate_metric(&prefill_tokens_per_second_list);
+ let energy_joules_metric = calculate_metric(&energy_joules_list);
+ let joules_per_token_metric = calculate_metric(&joules_per_token_list);
+
+ let mut table = Table::new();
+ table
+ .load_style(UTF8_FULL.with_rounded_corners())
+ .set_content_arrangement(ContentArrangement::Dynamic)
+ .set_header(vec!["Metric", "Value"])
+ .add_row(vec!["Prefill, s", prefill_duration_metric.as_str()])
+ .add_row(vec!["Total, s", total_duration_metric.as_str()])
+ .add_row(vec!["Prefill, t/s", prefill_tokens_per_second_metric.as_str()])
+ .add_row(vec!["Total energy, J", energy_joules_metric.as_str()])
+ .add_row(vec!["Energy per token, J/tok", joules_per_token_metric.as_str()]);
+ let Some(column) = table.column_mut(1) else {
+ bail!("Benchmark summary table value column is missing");
+ };
+ column.set_cell_alignment(CellAlignment::Right);
+ println!("{table}");
+
+ Ok(())
+}
diff --git a/crates/legacy/cli/src/bench/mod.rs b/crates/legacy/cli/src/bench/mod.rs
index a9ecfd45b..2c9132f6b 100644
--- a/crates/legacy/cli/src/bench/mod.rs
+++ b/crates/legacy/cli/src/bench/mod.rs
@@ -1,3 +1,4 @@
+mod decision;
mod model;
mod runner;
mod stat;
@@ -7,7 +8,7 @@ use comfy_table::{CellAlignment, ContentArrangement, Table, presets::UTF8_FULL};
use indicatif::ProgressBar;
use tokio::fs;
-use crate::bench::{model::BenchTask, runner::BenchRunner, stat::calculate_metric};
+use crate::bench::{decision::DecisionBenchTask, model::BenchTask, runner::BenchRunner, stat::calculate_metric};
pub async fn run_bench(
model_path: String,
@@ -16,8 +17,16 @@ pub async fn run_bench(
) -> Result<()> {
let task_content =
fs::read_to_string(&task_path).await.with_context(|| format!("Failed to read task file: {task_path}"))?;
- let task: BenchTask =
+ let task_value: serde_json::Value =
serde_json::from_str(&task_content).with_context(|| format!("Failed to parse task file: {task_path}"))?;
+ if task_value.get("questions").is_some() {
+ let task: DecisionBenchTask = serde_json::from_value(task_value)
+ .with_context(|| format!("Failed to parse decision task file: {task_path}"))?;
+ return decision::run(task, model_path, output_path).await;
+ }
+
+ let task: BenchTask =
+ serde_json::from_value(task_value).with_context(|| format!("Failed to parse task file: {task_path}"))?;
println!("Model: {}", task.repo_id);
let progress_bar = ProgressBar::new(task.number_of_runs);
diff --git a/crates/legacy/cli/src/bench/model.rs b/crates/legacy/cli/src/bench/model.rs
index afbb05bb4..f6fb0a90d 100644
--- a/crates/legacy/cli/src/bench/model.rs
+++ b/crates/legacy/cli/src/bench/model.rs
@@ -1,8 +1,9 @@
use anyhow::{Result, anyhow, ensure};
use rocket::serde::{Deserialize, Serialize};
+use sysinfo::System;
use uzu::types::{
basic::ReasoningEffort,
- session::chat::{ChatMessage, ChatReply, ChatReplyEnergy, ChatRole},
+ session::chat::{ChatMessage, ChatReply, ChatRole, ReplyEnergy},
};
use crate::{
@@ -98,6 +99,23 @@ pub struct BenchDevice {
pub memory_total: u64,
}
+impl BenchDevice {
+ pub fn current() -> Self {
+ let mut system_info = System::new_all();
+ system_info.refresh_all();
+
+ let os_name = System::long_os_version();
+ let cpu_name = system_info.cpus().first().map(|cpu| cpu.brand().to_string());
+ let memory_total = system_info.total_memory();
+
+ Self {
+ os_name,
+ cpu_name,
+ memory_total,
+ }
+ }
+}
+
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchResult {
pub task: BenchTask,
@@ -111,8 +129,8 @@ pub struct BenchResult {
pub time_to_first_token: f64,
pub prompt_tokens_per_second: f64,
pub generate_tokens_per_second: Option<f64>,
- pub input_energy: Option<ChatReplyEnergy>,
- pub output_energy: Option<ChatReplyEnergy>,
+ pub input_energy: Option<ReplyEnergy>,
+ pub output_energy: Option<ReplyEnergy>,
pub joules_per_token: Option<f64>,
pub text: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
diff --git a/crates/legacy/cli/src/bench/runner.rs b/crates/legacy/cli/src/bench/runner.rs
index 767cff52d..c03818e75 100644
--- a/crates/legacy/cli/src/bench/runner.rs
+++ b/crates/legacy/cli/src/bench/runner.rs
@@ -5,13 +5,12 @@ use std::{
use anyhow::{Context, Result};
use shoji::types::model::{ModelFamily, ModelSource};
-use sysinfo::System;
use uzu::{
engine::{Engine, EngineConfig},
types::{
basic::SamplingMethod,
model::ModelAccessibility,
- session::chat::{ChatConfig, ChatReplyConfig, ChatReplyEnergy},
+ session::chat::{ChatConfig, ChatReplyConfig, ReplyEnergy},
},
};
use uzu_engine::VERSION;
@@ -64,7 +63,7 @@ impl BenchRunner {
}
let messages = self.task.to_chat_messages(ThinkingSupport::for_model(&model))?;
- let device = self.get_device_info();
+ let device = BenchDevice::current();
let session_config = ChatConfig::default();
let session = engine.chat(model, session_config).await?;
@@ -144,21 +143,6 @@ impl BenchRunner {
Ok(results)
}
- fn get_device_info(&self) -> BenchDevice {
- let mut system_info = System::new_all();
- system_info.refresh_all();
-
- let os_name = System::long_os_version();
- let cpu_name = system_info.cpus().first().map(|cpu| cpu.brand().to_string());
- let memory_total = system_info.total_memory();
-
- BenchDevice {
- os_name,
- cpu_name,
- memory_total,
- }
- }
-
async fn get_remote_model_family(
&self,
model_path: &Path,
@@ -199,6 +183,6 @@ impl BenchRunner {
}
}
-fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ChatReplyEnergy>) -> Option<ChatReplyEnergy> {
+fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ReplyEnergy>) -> Option<ReplyEnergy> {
energy.into_iter().cloned().reduce(|total, energy| total + energy)
}
diff --git a/crates/legacy/cli/src/common/decision.rs b/crates/legacy/cli/src/common/decision.rs
new file mode 100644
index 000000000..2b8a4daa5
--- /dev/null
+++ b/crates/legacy/cli/src/common/decision.rs
@@ -0,0 +1,70 @@
+use indexmap::IndexMap;
+use serde::{Deserialize, Serialize};
+use shoji::types::session::decision::{DecisionOption, DecisionQuestion};
+
+// A question of the System One request of vLLM Semantic Router, shared by the server and the bench.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum DecisionRequestQuestion {
+ Choice {
+ instructions: Option<String>,
+ // Option order is answer-code order, so it must survive parsing.
+ criteria: IndexMap<String, Option<String>>,
+ },
+ Noul {
+ instructions: Option<String>,
+ criteria: Option<NoulCriteria>,
+ },
+ Score {
+ instructions: Option<String>,
+ criteria: Vec<String>,
+ },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct NoulCriteria {
+ #[serde(rename = "false")]
+ no: Option<String>,
+ #[serde(rename = "true")]
+ yes: Option<String>,
+}
+
+impl From<DecisionRequestQuestion> for DecisionQuestion {
+ fn from(question: DecisionRequestQuestion) -> Self {
+ match question {
+ DecisionRequestQuestion::Choice {
+ instructions,
+ criteria,
+ } => DecisionQuestion::Choice {
+ instructions,
+ options: criteria
+ .into_iter()
+ .map(|(key, description)| DecisionOption {
+ key,
+ description,
+ })
+ .collect(),
+ },
+ DecisionRequestQuestion::Noul {
+ instructions,
+ criteria,
+ } => {
+ let (no_description, yes_description) =
+ criteria.map_or((None, None), |criteria| (criteria.no, criteria.yes));
+ DecisionQuestion::YesNo {
+ instructions,
+ no_description,
+ yes_description,
+ }
+ },
+ DecisionRequestQuestion::Score {
+ instructions,
+ criteria,
+ } => DecisionQuestion::Score {
+ instructions,
+ levels: criteria,
+ },
+ }
+ }
+}
diff --git a/crates/legacy/cli/src/common/mod.rs b/crates/legacy/cli/src/common/mod.rs
index 9602a9e2c..c4997e405 100644
--- a/crates/legacy/cli/src/common/mod.rs
+++ b/crates/legacy/cli/src/common/mod.rs
@@ -1 +1,2 @@
+pub mod decision;
pub mod thinking;
diff --git a/crates/legacy/cli/src/server/decisions/request.rs b/crates/legacy/cli/src/server/decisions/request.rs
index 6043aad04..9ee844eaf 100644
--- a/crates/legacy/cli/src/server/decisions/request.rs
+++ b/crates/legacy/cli/src/server/decisions/request.rs
@@ -1,6 +1,8 @@
use indexmap::IndexMap;
use serde::Deserialize;
-use shoji::types::session::decision::{DecisionInput, DecisionOption, DecisionQuestion};
+use shoji::types::session::decision::DecisionInput;
+
+use crate::common::decision::DecisionRequestQuestion;
// The System One request of vLLM Semantic Router and d3's `d3_server.py`, text only:
// `state` must be a string and there are no `images` or `videos`.
@@ -17,72 +19,6 @@ pub struct DecisionRequest {
pub questions: IndexMap<String, serde_json::Value>,
}
-#[derive(Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
-enum DecisionRequestQuestion {
- Choice {
- instructions: Option<String>,
- // Option order is answer-code order, so it must survive parsing.
- criteria: IndexMap<String, Option<String>>,
- },
- Noul {
- instructions: Option<String>,
- criteria: Option<NoulCriteria>,
- },
- Score {
- instructions: Option<String>,
- criteria: Vec<String>,
- },
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct NoulCriteria {
- #[serde(rename = "false")]
- no: Option<String>,
- #[serde(rename = "true")]
- yes: Option<String>,
-}
-
-impl From<DecisionRequestQuestion> for DecisionQuestion {
- fn from(question: DecisionRequestQuestion) -> Self {
- match question {
- DecisionRequestQuestion::Choice {
- instructions,
- criteria,
- } => DecisionQuestion::Choice {
- instructions,
- options: criteria
- .into_iter()
- .map(|(key, description)| DecisionOption {
- key,
- description,
- })
- .collect(),
- },
- DecisionRequestQuestion::Noul {
- instructions,
- criteria,
- } => {
- let (no_description, yes_description) =
- criteria.map_or((None, None), |criteria| (criteria.no, criteria.yes));
- DecisionQuestion::YesNo {
- instructions,
- no_description,
- yes_description,
- }
- },
- DecisionRequestQuestion::Score {
- instructions,
- criteria,
- } => DecisionQuestion::Score {
- instructions,
- levels: criteria,
- },
- }
- }
-}
-
pub struct InvalidQuestion {
pub id: String,
pub message: String,
diff --git a/crates/legacy/nagare/src/chat/mod.rs b/crates/legacy/nagare/src/chat/mod.rs
index 13eba202f..ae4fcbb65 100644
--- a/crates/legacy/nagare/src/chat/mod.rs
+++ b/crates/legacy/nagare/src/chat/mod.rs
@@ -28,8 +28,8 @@ use shoji::{
basic::{CancelToken, SamplingParameters, ToolCall, ToolDescription, ToolNamespace, Value, parse_lenient_json},
model::Model,
session::chat::{
- ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyEnergy,
- ChatReplyFinishReason, ChatReplySpeculatorStats, ChatReplyStats, ChatRole,
+ ChatConfig, ChatContentBlock, ChatMessage, ChatReply, ChatReplyConfig, ChatReplyFinishReason,
+ ChatReplySpeculatorStats, ChatReplyStats, ChatRole, ReplyEnergy,
},
},
};
@@ -804,7 +804,7 @@ fn sum_optional_u32(
stats.iter().try_fold(0_u32, |sum, stats| sum.checked_add(value(stats)?))
}
-fn aggregate_input_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy> {
+fn aggregate_input_energy(stats: &[&ChatReplyStats]) -> Option<ReplyEnergy> {
let mut phases = Vec::with_capacity(stats.len());
for stats in stats {
phases.push(stats.input_energy.as_ref()?);
@@ -812,7 +812,7 @@ fn aggregate_input_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy>
aggregate_energy(phases)
}
-fn aggregate_output_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy> {
+fn aggregate_output_energy(stats: &[&ChatReplyStats]) -> Option<ReplyEnergy> {
let mut phases = Vec::with_capacity(stats.len());
for stats in stats {
let tokens = stats.tokens_count_output?;
@@ -824,7 +824,7 @@ fn aggregate_output_energy(stats: &[&ChatReplyStats]) -> Option<ChatReplyEnergy>
aggregate_energy(phases)
}
-fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ChatReplyEnergy>) -> Option<ChatReplyEnergy> {
+fn aggregate_energy<'a>(energy: impl IntoIterator<Item = &'a ReplyEnergy>) -> Option<ReplyEnergy> {
energy.into_iter().cloned().reduce(|total, energy| total + energy)
}
diff --git a/crates/legacy/nagare/src/chat/token.rs b/crates/legacy/nagare/src/chat/token.rs
index d426217bf..7c777548f 100644
--- a/crates/legacy/nagare/src/chat/token.rs
+++ b/crates/legacy/nagare/src/chat/token.rs
@@ -23,8 +23,8 @@ use shoji::{
basic::{SamplingParameters, Token, TokenId},
model::Model,
session::chat::{
- ChatConfig, ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyEnergy, ChatReplyFinishReason,
- ChatReplySpeculatorStats, ChatReplyStats,
+ ChatConfig, ChatContentBlock, ChatMessage, ChatReplyConfig, ChatReplyFinishReason,
+ ChatReplySpeculatorStats, ChatReplyStats, ReplyEnergy,
},
},
};
@@ -432,7 +432,7 @@ struct StreamingState<'a> {
time_last_token: Option<Instant>,
time_prefill_start: Instant,
time_first_token: Option<Instant>,
- input_energy: Option<ChatReplyEnergy>,
+ input_energy: Option<ReplyEnergy>,
total_tokens_input: usize,
cached_tokens_input: usize,
total_tokens_output: usize,
diff --git a/crates/legacy/nagare/src/decision/error.rs b/crates/legacy/nagare/src/decision/error.rs
index ff8c0cd59..cc6b302ef 100644
--- a/crates/legacy/nagare/src/decision/error.rs
+++ b/crates/legacy/nagare/src/decision/error.rs
@@ -1,3 +1,6 @@
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+use crate::util::power::Error as EnergyError;
+
#[bindings::export(Error)]
#[derive(Debug, Clone, PartialEq, thiserror::Error)]
#[non_exhaustive]
@@ -24,3 +27,12 @@ pub enum DecisionSessionError {
#[error("No response")]
NoResponse {},
}
+
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+impl From<EnergyError> for DecisionSessionError {
+ fn from(error: EnergyError) -> Self {
+ Self::Backend {
+ message: error.to_string(),
+ }
+ }
+}
diff --git a/crates/legacy/nagare/src/decision/session.rs b/crates/legacy/nagare/src/decision/session.rs
index a5cbea080..a7cb787a4 100644
--- a/crates/legacy/nagare/src/decision/session.rs
+++ b/crates/legacy/nagare/src/decision/session.rs
@@ -1,6 +1,8 @@
use std::{sync::Arc, time::Instant};
use hanashi::{DecisionEncoding, DecisionEncodingError};
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+use keisoku::KeisokuError;
use serde::{Deserialize, Serialize};
use shoji::{
traits::{Backend, backend::decision::DecisionInstance},
@@ -14,6 +16,8 @@ use shoji::{
use tokio::sync::{Mutex, mpsc};
pub use crate::decision::error::DecisionSessionError;
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+use crate::util::power::{EnergyRecorder, Error as EnergyError};
#[bindings::export(Enumeration)]
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -25,6 +29,8 @@ pub enum DecisionSessionState {
struct DecisionInstanceHolder {
instance: Box<dyn DecisionInstance>,
encoding: DecisionEncoding,
+ #[cfg(any(target_os = "macos", target_os = "ios"))]
+ energy_recorder: EnergyRecorder,
}
#[bindings::export(Class)]
@@ -54,9 +60,11 @@ impl DecisionSession {
let encoding = DecisionEncoding::new(instance.token_codec_config(), instance.tokenizer())
.map_err(|_| DecisionSessionError::UnsupportedModel {})?;
- Ok(DecisionInstanceHolder {
+ Ok::<_, DecisionSessionError>(DecisionInstanceHolder {
instance,
encoding,
+ #[cfg(any(target_os = "macos", target_os = "ios"))]
+ energy_recorder: EnergyRecorder::new(),
})
})
.await
@@ -71,7 +79,7 @@ impl DecisionSession {
}
async fn execute(
- holder: &DecisionInstanceHolder,
+ holder: &mut DecisionInstanceHolder,
input: &DecisionInput,
) -> Result<Vec<DecisionResult>, DecisionSessionError> {
if input.questions.is_empty() {
@@ -107,17 +115,39 @@ impl DecisionSession {
let time_start = Instant::now();
let tokens_count = prompt.len();
let tokens = prompt.into_iter().map(u64::from).collect::<Vec<_>>();
- let probabilities =
- holder.instance.probabilities(&tokens, option_count(question)).await.map_err(|error| {
- DecisionSessionError::Backend {
- message: error.to_string(),
- }
- })?;
+
+ #[cfg(any(target_os = "macos", target_os = "ios"))]
+ holder.energy_recorder.begin()?;
+
+ let output = holder.instance.probabilities(&tokens, option_count(question)).await.map_err(|error| {
+ DecisionSessionError::Backend {
+ message: error.to_string(),
+ }
+ });
+
+ // Stop sampling before propagating an inference error.
+ #[cfg(any(target_os = "macos", target_os = "ios"))]
+ let energy = holder.energy_recorder.finish();
+
+ let output = output?;
+
+ #[cfg(any(target_os = "macos", target_os = "ios"))]
+ let energy = match energy {
+ Ok(energy) => Some(energy),
+ Err(EnergyError::Keisoku(KeisokuError::PowerReadingUnavailable)) => None,
+ Err(error) => return Err(error.into()),
+ };
+
+ #[cfg(not(any(target_os = "macos", target_os = "ios")))]
+ let energy = None;
+
results.push(DecisionResult {
- answer: answer(question, probabilities),
+ answer: answer(question, output.probabilities),
stats: DecisionStats {
tokens_count: tokens_count as i64,
+ prefill_duration: output.prefill_duration.as_secs_f64(),
total_duration: (encoding_duration + time_start.elapsed()).as_secs_f64(),
+ energy,
},
});
}
@@ -156,8 +186,8 @@ impl DecisionSession {
}
let result = {
- let holder = holder.lock().await;
- Self::execute(&holder, &input).await
+ let mut holder = holder.lock().await;
+ Self::execute(&mut holder, &input).await
};
let output = result.map(|results| DecisionOutput {
results,
diff --git a/crates/legacy/nagare/src/util/power/mod.rs b/crates/legacy/nagare/src/util/power/mod.rs
index a6da8a858..3a487d37f 100644
--- a/crates/legacy/nagare/src/util/power/mod.rs
+++ b/crates/legacy/nagare/src/util/power/mod.rs
@@ -2,7 +2,7 @@ mod error;
pub use error::Error;
use keisoku::{PowerMeter, PowerReading};
-use shoji::types::session::chat::ChatReplyEnergy;
+use shoji::types::session::chat::ReplyEnergy;
pub struct EnergyRecorder {
meter: PowerMeter,
@@ -20,20 +20,20 @@ impl EnergyRecorder {
Ok(())
}
- pub fn split(&mut self) -> Result<ChatReplyEnergy, Error> {
+ pub fn split(&mut self) -> Result<ReplyEnergy, Error> {
Ok(energy(self.meter.split()?))
}
- pub fn finish(&mut self) -> Result<ChatReplyEnergy, Error> {
+ pub fn finish(&mut self) -> Result<ReplyEnergy, Error> {
Ok(energy(self.meter.stop()?))
}
}
-fn energy(reading: PowerReading) -> ChatReplyEnergy {
+fn energy(reading: PowerReading) -> ReplyEnergy {
match reading {
PowerReading::Total {
total,
- } => ChatReplyEnergy::Total {
+ } => ReplyEnergy::Total {
total: total.value() as f64,
},
PowerReading::Components {
@@ -41,7 +41,7 @@ fn energy(reading: PowerReading) -> ChatReplyEnergy {
gpu,
ane,
ram,
- } => ChatReplyEnergy::Components {
+ } => ReplyEnergy::Components {
cpu: cpu.value() as f64,
gpu: gpu.value() as f64,
ane: ane.value() as f64,
diff --git a/crates/legacy/shoji/src/traits/backend/decision.rs b/crates/legacy/shoji/src/traits/backend/decision.rs
index 5bc810daf..9f9307d34 100644
--- a/crates/legacy/shoji/src/traits/backend/decision.rs
+++ b/crates/legacy/shoji/src/traits/backend/decision.rs
@@ -1,9 +1,15 @@
-use std::{pin::Pin, sync::Arc};
+use std::{pin::Pin, sync::Arc, time::Duration};
use tokenizers::Tokenizer;
use crate::{traits::backend::Error, types::session::classification::TokenCodecConfig};
+pub struct DecisionSessionOutput {
+ pub probabilities: Vec<f32>,
+ pub prefill_duration: Duration,
+ pub memory_used_bytes: Option<usize>,
+}
+
pub type Config = ();
pub trait DecisionBackend: Send + Sync {
@@ -27,5 +33,5 @@ pub trait DecisionInstance: Send + Sync {
&'a self,
input: &'a [u64],
option_count: usize,
- ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>, Error>> + Send + 'a>>;
+ ) -> Pin<Box<dyn Future<Output = Result<DecisionSessionOutput, Error>> + Send + 'a>>;
}
diff --git a/crates/legacy/shoji/src/types/session/chat/mod.rs b/crates/legacy/shoji/src/types/session/chat/mod.rs
index 9083c674a..be9008b94 100644
--- a/crates/legacy/shoji/src/types/session/chat/mod.rs
+++ b/crates/legacy/shoji/src/types/session/chat/mod.rs
@@ -10,7 +10,7 @@ pub use config::ChatConfig;
pub use content_block::{ChatContentBlock, ChatContentBlockType};
pub use message::{ChatMessage, ChatMessageList, ChatMessageMetadata};
pub use reply::{
- ChatReply, ChatReplyConfig, ChatReplyEnergy, ChatReplyFinishReason, ChatReplyJoulesPerToken,
- ChatReplySpeculatorStats, ChatReplyStats,
+ ChatReply, ChatReplyConfig, ChatReplyFinishReason, ChatReplyJoulesPerToken, ChatReplySpeculatorStats,
+ ChatReplyStats, ReplyEnergy,
};
pub use role::ChatRole;
diff --git a/crates/legacy/shoji/src/types/session/chat/reply/mod.rs b/crates/legacy/shoji/src/types/session/chat/reply/mod.rs
index 4fbdadf15..b7869fe96 100644
--- a/crates/legacy/shoji/src/types/session/chat/reply/mod.rs
+++ b/crates/legacy/shoji/src/types/session/chat/reply/mod.rs
@@ -5,7 +5,7 @@ mod stats;
pub use config::ChatReplyConfig;
pub use finish_reason::ChatReplyFinishReason;
use serde::{Deserialize, Serialize};
-pub use stats::{ChatReplyEnergy, ChatReplyJoulesPerToken, ChatReplySpeculatorStats, ChatReplyStats};
+pub use stats::{ChatReplyJoulesPerToken, ChatReplySpeculatorStats, ChatReplyStats, ReplyEnergy};
use crate::types::session::chat::ChatMessage;
diff --git a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs b/crates/legacy/shoji/src/types/session/chat/reply/stats.rs
index 33a4c0c27..9193b4f05 100644
--- a/crates/legacy/shoji/src/types/session/chat/reply/stats.rs
+++ b/crates/legacy/shoji/src/types/session/chat/reply/stats.rs
@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
#[bindings::export(Enumeration)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
-pub enum ChatReplyEnergy {
+pub enum ReplyEnergy {
Total {
total: f64,
},
@@ -21,7 +21,7 @@ pub enum ChatReplyEnergy {
}
#[cfg_attr(feature = "bindings-uniffi", uniffi::export)]
-impl ChatReplyEnergy {
+impl ReplyEnergy {
pub fn total(&self) -> f64 {
match self {
Self::Total {
@@ -37,7 +37,7 @@ impl ChatReplyEnergy {
}
}
-impl ChatReplyEnergy {
+impl ReplyEnergy {
fn per_token(
&self,
tokens_count: u32,
@@ -68,7 +68,7 @@ impl ChatReplyEnergy {
}
}
-impl Add for ChatReplyEnergy {
+impl Add for ReplyEnergy {
type Output = Self;
fn add(
@@ -172,8 +172,8 @@ pub struct ChatReplyStats {
pub tokens_count_output: Option<u32>,
pub memory_used_bytes: Option<i64>,
pub speculator_stats: Option<ChatReplySpeculatorStats>,
- pub input_energy: Option<ChatReplyEnergy>,
- pub output_energy: Option<ChatReplyEnergy>,
+ pub input_energy: Option<ReplyEnergy>,
+ pub output_energy: Option<ReplyEnergy>,
}
#[bindings::export(Implementation)]
@@ -185,7 +185,7 @@ impl ChatReplyStats {
#[bindings::export(Method(Getter))]
pub fn total_joules(&self) -> Option<f64> {
- self.input_energy.iter().chain(self.output_energy.iter()).map(ChatReplyEnergy::total).reduce(f64::add)
+ self.input_energy.iter().chain(self.output_energy.iter()).map(ReplyEnergy::total).reduce(f64::add)
}
#[bindings::export(Method(Getter))]
diff --git a/crates/legacy/shoji/src/types/session/decision/mod.rs b/crates/legacy/shoji/src/types/session/decision/mod.rs
index ad1a5c68e..c5fd8bd31 100644
--- a/crates/legacy/shoji/src/types/session/decision/mod.rs
+++ b/crates/legacy/shoji/src/types/session/decision/mod.rs
@@ -1,5 +1,7 @@
use serde::{Deserialize, Serialize};
+use crate::types::session::chat::ReplyEnergy;
+
#[bindings::export(Structure(Class))]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DecisionOption {
@@ -67,7 +69,9 @@ pub enum DecisionAnswer {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct DecisionStats {
pub tokens_count: i64,
+ pub prefill_duration: f64,
pub total_duration: f64,
+ pub energy: Option<ReplyEnergy>,
}
#[bindings::export(Structure(Class))]
diff --git a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
index 092079d2c..0da35d2a6 100644
--- a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
+++ b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
@@ -19,7 +19,6 @@ __all__ = [
"ChatModelInfo",
"ChatReply",
"ChatReplyConfig",
- "ChatReplyEnergy",
"ChatReplyFinishReason",
"ChatReplyJoulesPerToken",
"ChatReplySpeculatorStats",
@@ -79,6 +78,7 @@ __all__ = [
"PcmBatch",
"Player",
"ReasoningEffort",
+ "ReplyEnergy",
"Repository",
"SamplingMethod",
"SamplingParameters",
@@ -379,29 +379,6 @@ class ChatReplyConfig:
def with_tool_turn_limit(self, tool_turn_limit: typing.Optional[builtins.int]) -> ChatReplyConfig: ...
def with_grammar(self, grammar: typing.Optional[Grammar]) -> ChatReplyConfig: ...
-class ChatReplyEnergy:
- @typing.final
- class Total(ChatReplyEnergy):
- __match_args__ = ("total",)
- @property
- def total(self) -> builtins.float: ...
- def __new__(cls, total: builtins.float) -> ChatReplyEnergy.Total: ...
-
- @typing.final
- class Components(ChatReplyEnergy):
- __match_args__ = ("cpu", "gpu", "ane", "dram",)
- @property
- def cpu(self) -> builtins.float: ...
- @property
- def gpu(self) -> builtins.float: ...
- @property
- def ane(self) -> builtins.float: ...
- @property
- def dram(self) -> builtins.float: ...
- def __new__(cls, cpu: builtins.float, gpu: builtins.float, ane: builtins.float, dram: builtins.float) -> ChatReplyEnergy.Components: ...
-
- ...
-
class ChatReplyJoulesPerToken:
@typing.final
class Total(ChatReplyJoulesPerToken):
@@ -454,9 +431,9 @@ class ChatReplyStats:
@property
def speculator_stats(self) -> typing.Optional[ChatReplySpeculatorStats]: ...
@property
- def input_energy(self) -> typing.Optional[ChatReplyEnergy]: ...
+ def input_energy(self) -> typing.Optional[ReplyEnergy]: ...
@property
- def output_energy(self) -> typing.Optional[ChatReplyEnergy]: ...
+ def output_energy(self) -> typing.Optional[ReplyEnergy]: ...
@property
def tokens_count(self) -> typing.Optional[builtins.int]: ...
@property
@@ -465,7 +442,7 @@ class ChatReplyStats:
def input_joules_per_token(self) -> typing.Optional[ChatReplyJoulesPerToken]: ...
@property
def output_joules_per_token(self) -> typing.Optional[ChatReplyJoulesPerToken]: ...
- def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_input_cached: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], input_energy: typing.Optional[ChatReplyEnergy], output_energy: typing.Optional[ChatReplyEnergy]) -> ChatReplyStats: ...
+ def __new__(cls, duration: builtins.float, time_to_first_token: typing.Optional[builtins.float], prefill_tokens_per_second: typing.Optional[builtins.float], generate_tokens_per_second: typing.Optional[builtins.float], tokens_count_input: typing.Optional[builtins.int], tokens_count_input_cached: typing.Optional[builtins.int], tokens_count_output: typing.Optional[builtins.int], memory_used_bytes: typing.Optional[builtins.int], speculator_stats: typing.Optional[ChatReplySpeculatorStats], input_energy: typing.Optional[ReplyEnergy], output_energy: typing.Optional[ReplyEnergy]) -> ChatReplyStats: ...
class ChatRole:
@typing.final
@@ -736,8 +713,12 @@ class DecisionStats:
@property
def tokens_count(self) -> builtins.int: ...
@property
+ def prefill_duration(self) -> builtins.float: ...
+ @property
def total_duration(self) -> builtins.float: ...
- def __new__(cls, tokens_count: builtins.int, total_duration: builtins.float) -> DecisionStats: ...
+ @property
+ def energy(self) -> typing.Optional[ReplyEnergy]: ...
+ def __new__(cls, tokens_count: builtins.int, prefill_duration: builtins.float, total_duration: builtins.float, energy: typing.Optional[ReplyEnergy]) -> DecisionStats: ...
@typing.final
class Device:
@@ -1225,6 +1206,29 @@ class Player:
def append_pcm_batch(self, batch: PcmBatch) -> None: ...
def stop(self) -> None: ...
+class ReplyEnergy:
+ @typing.final
+ class Total(ReplyEnergy):
+ __match_args__ = ("total",)
+ @property
+ def total(self) -> builtins.float: ...
+ def __new__(cls, total: builtins.float) -> ReplyEnergy.Total: ...
+
+ @typing.final
+ class Components(ReplyEnergy):
+ __match_args__ = ("cpu", "gpu", "ane", "dram",)
+ @property
+ def cpu(self) -> builtins.float: ...
+ @property
+ def gpu(self) -> builtins.float: ...
+ @property
+ def ane(self) -> builtins.float: ...
+ @property
+ def dram(self) -> builtins.float: ...
+ def __new__(cls, cpu: builtins.float, gpu: builtins.float, ane: builtins.float, dram: builtins.float) -> ReplyEnergy.Components: ...
+
+ ...
+
@typing.final
class Repository:
@property
diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index 9705ac209..af97bd94c 100644
--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -1245,12 +1245,12 @@ public struct ChatReplyStats: Equatable, Hashable, Codable {
public var tokensCountOutput: UInt32?
public var memoryUsedBytes: Int64?
public var speculatorStats: ChatReplySpeculatorStats?
- public var inputEnergy: ChatReplyEnergy?
- public var outputEnergy: ChatReplyEnergy?
+ public var inputEnergy: ReplyEnergy?
+ public var outputEnergy: ReplyEnergy?
// Default memberwise initializers are never public by default, so we
// declare one manually.
- public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountInputCached: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, inputEnergy: ChatReplyEnergy?, outputEnergy: ChatReplyEnergy?) {
+ public init(duration: Double, timeToFirstToken: Double?, prefillTokensPerSecond: Double?, generateTokensPerSecond: Double?, tokensCountInput: UInt32?, tokensCountInputCached: UInt32?, tokensCountOutput: UInt32?, memoryUsedBytes: Int64?, speculatorStats: ChatReplySpeculatorStats?, inputEnergy: ReplyEnergy?, outputEnergy: ReplyEnergy?) {
self.duration = duration
self.timeToFirstToken = timeToFirstToken
self.prefillTokensPerSecond = prefillTokensPerSecond
@@ -1321,8 +1321,8 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {
tokensCountOutput: FfiConverterOptionUInt32.read(from: &buf),
memoryUsedBytes: FfiConverterOptionInt64.read(from: &buf),
speculatorStats: FfiConverterOptionTypeChatReplySpeculatorStats.read(from: &buf),
- inputEnergy: FfiConverterOptionTypeChatReplyEnergy.read(from: &buf),
- outputEnergy: FfiConverterOptionTypeChatReplyEnergy.read(from: &buf)
+ inputEnergy: FfiConverterOptionTypeReplyEnergy.read(from: &buf),
+ outputEnergy: FfiConverterOptionTypeReplyEnergy.read(from: &buf)
)
}
@@ -1336,8 +1336,8 @@ public struct FfiConverterTypeChatReplyStats: FfiConverterRustBuffer {
FfiConverterOptionUInt32.write(value.tokensCountOutput, into: &buf)
FfiConverterOptionInt64.write(value.memoryUsedBytes, into: &buf)
FfiConverterOptionTypeChatReplySpeculatorStats.write(value.speculatorStats, into: &buf)
- FfiConverterOptionTypeChatReplyEnergy.write(value.inputEnergy, into: &buf)
- FfiConverterOptionTypeChatReplyEnergy.write(value.outputEnergy, into: &buf)
+ FfiConverterOptionTypeReplyEnergy.write(value.inputEnergy, into: &buf)
+ FfiConverterOptionTypeReplyEnergy.write(value.outputEnergy, into: &buf)
}
}
@@ -1877,13 +1877,17 @@ public func FfiConverterTypeDecisionResult_lower(_ value: DecisionResult) -> Rus
public struct DecisionStats: Equatable, Hashable, Codable {
public var tokensCount: Int64
+ public var prefillDuration: Double
public var totalDuration: Double
+ public var energy: ReplyEnergy?
// Default memberwise initializers are never public by default, so we
// declare one manually.
- public init(tokensCount: Int64, totalDuration: Double) {
+ public init(tokensCount: Int64, prefillDuration: Double, totalDuration: Double, energy: ReplyEnergy?) {
self.tokensCount = tokensCount
+ self.prefillDuration = prefillDuration
self.totalDuration = totalDuration
+ self.energy = energy
}
@@ -1903,13 +1907,17 @@ public struct FfiConverterTypeDecisionStats: FfiConverterRustBuffer {
return
try DecisionStats(
tokensCount: FfiConverterInt64.read(from: &buf),
- totalDuration: FfiConverterDouble.read(from: &buf)
+ prefillDuration: FfiConverterDouble.read(from: &buf),
+ totalDuration: FfiConverterDouble.read(from: &buf),
+ energy: FfiConverterOptionTypeReplyEnergy.read(from: &buf)
)
}
public static func write(_ value: DecisionStats, into buf: inout [UInt8]) {
FfiConverterInt64.write(value.tokensCount, into: &buf)
+ FfiConverterDouble.write(value.prefillDuration, into: &buf)
FfiConverterDouble.write(value.totalDuration, into: &buf)
+ FfiConverterOptionTypeReplyEnergy.write(value.energy, into: &buf)
}
}
@@ -3863,90 +3871,6 @@ public func FfiConverterTypeChatContentBlockType_lower(_ value: ChatContentBlock
}
-// Note that we don't yet support `indirect` for enums.
-// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.
-
-public enum ChatReplyEnergy: Equatable, Hashable, Codable {
-
- case total(total: Double
- )
- case components(cpu: Double, gpu: Double, ane: Double, dram: Double
- )
-
-
-
-public func total() -> Double {
- return try! FfiConverterDouble.lift(try! rustCall() {
- uniffi_shoji_fn_method_chatreplyenergy_total(
- FfiConverterTypeChatReplyEnergy_lower(self),$0
- )
-})
-}
-
-
-
-}
-
-#if compiler(>=6)
-extension ChatReplyEnergy: Sendable {}
-#endif
-
-#if swift(>=5.8)
-@_documentation(visibility: private)
-#endif
-public struct FfiConverterTypeChatReplyEnergy: FfiConverterRustBuffer {
- typealias SwiftType = ChatReplyEnergy
-
- public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ChatReplyEnergy {
- let variant: Int32 = try readInt(&buf)
- switch variant {
-
- case 1: return .total(total: try FfiConverterDouble.read(from: &buf)
- )
-
- case 2: return .components(cpu: try FfiConverterDouble.read(from: &buf), gpu: try FfiConverterDouble.read(from: &buf), ane: try FfiConverterDouble.read(from: &buf), dram: try FfiConverterDouble.read(from: &buf)
- )
-
- default: throw UniffiInternalError.unexpectedEnumCase
- }
- }
-
- public static func write(_ value: ChatReplyEnergy, into buf: inout [UInt8]) {
- switch value {
-
-
- case let .total(total):
- writeInt(&buf, Int32(1))
- FfiConverterDouble.write(total, into: &buf)
-
-
- case let .components(cpu,gpu,ane,dram):
- writeInt(&buf, Int32(2))
- FfiConverterDouble.write(cpu, into: &buf)
- FfiConverterDouble.write(gpu, into: &buf)
- FfiConverterDouble.write(ane, into: &buf)
- FfiConverterDouble.write(dram, into: &buf)
-
- }
- }
-}
-
-
-#if swift(>=5.8)
-@_documentation(visibility: private)
-#endif
-public func FfiConverterTypeChatReplyEnergy_lift(_ buf: RustBuffer) throws -> ChatReplyEnergy {
- return try FfiConverterTypeChatReplyEnergy.lift(buf)
-}
-
-#if swift(>=5.8)
-@_documentation(visibility: private)
-#endif
-public func FfiConverterTypeChatReplyEnergy_lower(_ value: ChatReplyEnergy) -> RustBuffer {
- return FfiConverterTypeChatReplyEnergy.lower(value)
-}
-
-
// Note that we don't yet support `indirect` for enums.
// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.
@@ -5249,6 +5173,90 @@ public func FfiConverterTypeReasoningEffort_lower(_ value: ReasoningEffort) -> R
}
+// Note that we don't yet support `indirect` for enums.
+// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.
+
+public enum ReplyEnergy: Equatable, Hashable, Codable {
+
+ case total(total: Double
+ )
+ case components(cpu: Double, gpu: Double, ane: Double, dram: Double
+ )
+
+
+
+public func total() -> Double {
+ return try! FfiConverterDouble.lift(try! rustCall() {
+ uniffi_shoji_fn_method_replyenergy_total(
+ FfiConverterTypeReplyEnergy_lower(self),$0
+ )
+})
+}
+
+
+
+}
+
+#if compiler(>=6)
+extension ReplyEnergy: Sendable {}
+#endif
+
+#if swift(>=5.8)
+@_documentation(visibility: private)
+#endif
+public struct FfiConverterTypeReplyEnergy: FfiConverterRustBuffer {
+ typealias SwiftType = ReplyEnergy
+
+ public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ReplyEnergy {
+ let variant: Int32 = try readInt(&buf)
+ switch variant {
+
+ case 1: return .total(total: try FfiConverterDouble.read(from: &buf)
+ )
+
+ case 2: return .components(cpu: try FfiConverterDouble.read(from: &buf), gpu: try FfiConverterDouble.read(from: &buf), ane: try FfiConverterDouble.read(from: &buf), dram: try FfiConverterDouble.read(from: &buf)
+ )
+
+ default: throw UniffiInternalError.unexpectedEnumCase
+ }
+ }
+
+ public static func write(_ value: ReplyEnergy, into buf: inout [UInt8]) {
+ switch value {
+
+
+ case let .total(total):
+ writeInt(&buf, Int32(1))
+ FfiConverterDouble.write(total, into: &buf)
+
+
+ case let .components(cpu,gpu,ane,dram):
+ writeInt(&buf, Int32(2))
+ FfiConverterDouble.write(cpu, into: &buf)
+ FfiConverterDouble.write(gpu, into: &buf)
+ FfiConverterDouble.write(ane, into: &buf)
+ FfiConverterDouble.write(dram, into: &buf)
+
+ }
+ }
+}
+
+
+#if swift(>=5.8)
+@_documentation(visibility: private)
+#endif
+public func FfiConverterTypeReplyEnergy_lift(_ buf: RustBuffer) throws -> ReplyEnergy {
+ return try FfiConverterTypeReplyEnergy.lift(buf)
+}
+
+#if swift(>=5.8)
+@_documentation(visibility: private)
+#endif
+public func FfiConverterTypeReplyEnergy_lower(_ value: ReplyEnergy) -> RustBuffer {
+ return FfiConverterTypeReplyEnergy.lower(value)
+}
+
+
// Note that we don't yet support `indirect` for enums.
// See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion.
@@ -5843,8 +5851,8 @@ fileprivate struct FfiConverterOptionTypeValue: FfiConverterRustBuffer {
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-fileprivate struct FfiConverterOptionTypeChatReplyEnergy: FfiConverterRustBuffer {
- typealias SwiftType = ChatReplyEnergy?
+fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRustBuffer {
+ typealias SwiftType = ChatReplyFinishReason?
public static func write(_ value: SwiftType, into buf: inout [UInt8]) {
guard let value = value else {
@@ -5852,13 +5860,13 @@ fileprivate struct FfiConverterOptionTypeChatReplyEnergy: FfiConverterRustBuffer
return
}
writeInt(&buf, Int8(1))
- FfiConverterTypeChatReplyEnergy.write(value, into: &buf)
+ FfiConverterTypeChatReplyFinishReason.write(value, into: &buf)
}
public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {
switch try readInt(&buf) as Int8 {
case 0: return nil
- case 1: return try FfiConverterTypeChatReplyEnergy.read(from: &buf)
+ case 1: return try FfiConverterTypeChatReplyFinishReason.read(from: &buf)
default: throw UniffiInternalError.unexpectedOptionalTag
}
}
@@ -5867,8 +5875,8 @@ fileprivate struct FfiConverterOptionTypeChatReplyEnergy: FfiConverterRustBuffer
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRustBuffer {
- typealias SwiftType = ChatReplyFinishReason?
+fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRustBuffer {
+ typealias SwiftType = ChatReplyJoulesPerToken?
public static func write(_ value: SwiftType, into buf: inout [UInt8]) {
guard let value = value else {
@@ -5876,13 +5884,13 @@ fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRust
return
}
writeInt(&buf, Int8(1))
- FfiConverterTypeChatReplyFinishReason.write(value, into: &buf)
+ FfiConverterTypeChatReplyJoulesPerToken.write(value, into: &buf)
}
public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {
switch try readInt(&buf) as Int8 {
case 0: return nil
- case 1: return try FfiConverterTypeChatReplyFinishReason.read(from: &buf)
+ case 1: return try FfiConverterTypeChatReplyJoulesPerToken.read(from: &buf)
default: throw UniffiInternalError.unexpectedOptionalTag
}
}
@@ -5891,8 +5899,8 @@ fileprivate struct FfiConverterOptionTypeChatReplyFinishReason: FfiConverterRust
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRustBuffer {
- typealias SwiftType = ChatReplyJoulesPerToken?
+fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {
+ typealias SwiftType = Grammar?
public static func write(_ value: SwiftType, into buf: inout [UInt8]) {
guard let value = value else {
@@ -5900,13 +5908,13 @@ fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRu
return
}
writeInt(&buf, Int8(1))
- FfiConverterTypeChatReplyJoulesPerToken.write(value, into: &buf)
+ FfiConverterTypeGrammar.write(value, into: &buf)
}
public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {
switch try readInt(&buf) as Int8 {
case 0: return nil
- case 1: return try FfiConverterTypeChatReplyJoulesPerToken.read(from: &buf)
+ case 1: return try FfiConverterTypeGrammar.read(from: &buf)
default: throw UniffiInternalError.unexpectedOptionalTag
}
}
@@ -5915,8 +5923,8 @@ fileprivate struct FfiConverterOptionTypeChatReplyJoulesPerToken: FfiConverterRu
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {
- typealias SwiftType = Grammar?
+fileprivate struct FfiConverterOptionTypeReasoningEffort: FfiConverterRustBuffer {
+ typealias SwiftType = ReasoningEffort?
public static func write(_ value: SwiftType, into buf: inout [UInt8]) {
guard let value = value else {
@@ -5924,13 +5932,13 @@ fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {
return
}
writeInt(&buf, Int8(1))
- FfiConverterTypeGrammar.write(value, into: &buf)
+ FfiConverterTypeReasoningEffort.write(value, into: &buf)
}
public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {
switch try readInt(&buf) as Int8 {
case 0: return nil
- case 1: return try FfiConverterTypeGrammar.read(from: &buf)
+ case 1: return try FfiConverterTypeReasoningEffort.read(from: &buf)
default: throw UniffiInternalError.unexpectedOptionalTag
}
}
@@ -5939,8 +5947,8 @@ fileprivate struct FfiConverterOptionTypeGrammar: FfiConverterRustBuffer {
#if swift(>=5.8)
@_documentation(visibility: private)
#endif
-fileprivate struct FfiConverterOptionTypeReasoningEffort: FfiConverterRustBuffer {
- typealias SwiftType = ReasoningEffort?
+fileprivate struct FfiConverterOptionTypeReplyEnergy: FfiConverterRustBuffer {
+ typealias SwiftType = ReplyEnergy?
public static func write(_ value: SwiftType, into buf: inout [UInt8]) {
guard let value = value else {
@@ -5948,13 +5956,13 @@ fileprivate struct FfiConverterOptionTypeReasoningEffort: FfiConverterRustBuffer
return
}
writeInt(&buf, Int8(1))
- FfiConverterTypeReasoningEffort.write(value, into: &buf)
+ FfiConverterTypeReplyEnergy.write(value, into: &buf)
}
public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType {
switch try readInt(&buf) as Int8 {
case 0: return nil
- case 1: return try FfiConverterTypeReasoningEffort.read(from: &buf)
+ case 1: return try FfiConverterTypeReplyEnergy.read(from: &buf)
default: throw UniffiInternalError.unexpectedOptionalTag
}
}
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
index 82cce3f98..c95ac4238 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
@@ -376,19 +376,6 @@ export declare class ChatReplyConfig {
withGrammar(grammar?: Grammar | undefined | null): ChatReplyConfig
}
-export declare class ChatReplyEnergyComponents {
- cpu: number
- gpu: number
- ane: number
- dram: number
- constructor(cpu: number, gpu: number, ane: number, dram: number)
-}
-
-export declare class ChatReplyEnergyTotal {
- total: number
- constructor(total: number)
-}
-
export declare class ChatReplyJoulesPerTokenComponents {
cpu: number
gpu: number
@@ -418,9 +405,9 @@ export declare class ChatReplyStats {
tokensCountOutput?: number
memoryUsedBytes?: number
speculatorStats?: ChatReplySpeculatorStats
- inputEnergy?: ChatReplyEnergy
- outputEnergy?: ChatReplyEnergy
- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ChatReplyEnergy, outputEnergy?: ChatReplyEnergy)
+ inputEnergy?: ReplyEnergy
+ outputEnergy?: ReplyEnergy
+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ReplyEnergy, outputEnergy?: ReplyEnergy)
get tokensCount(): number | null
get totalJoules(): number | null
get inputJoulesPerToken(): ChatReplyJoulesPerToken | null
@@ -564,8 +551,10 @@ export declare class DecisionResult {
export declare class DecisionStats {
tokensCount: number
+ prefillDuration: number
totalDuration: number
- constructor(tokensCount: number, totalDuration: number)
+ energy?: ReplyEnergy
+ constructor(tokensCount: number, prefillDuration: number, totalDuration: number, energy?: ReplyEnergy)
}
export declare class Feature {
@@ -761,6 +750,19 @@ export declare class PcmBatch {
saveAsWav(path: string): void
}
+export declare class ReplyEnergyComponents {
+ cpu: number
+ gpu: number
+ ane: number
+ dram: number
+ constructor(cpu: number, gpu: number, ane: number, dram: number)
+}
+
+export declare class ReplyEnergyTotal {
+ total: number
+ constructor(total: number)
+}
+
export declare class Repository {
identifier: string
commitHash?: string
@@ -900,9 +902,6 @@ export interface ChatMessageMetadata {
values: Record<string, Value>
}
-export type ChatReplyEnergy =
- ChatReplyEnergyTotal | ChatReplyEnergyComponents
-
export declare const enum ChatReplyFinishReason {
Stop = 'Stop',
Length = 'Length',
@@ -980,6 +979,9 @@ export declare const enum ReasoningEffort {
XHigh = 'XHigh'
}
+export type ReplyEnergy =
+ ReplyEnergyTotal | ReplyEnergyComponents
+
export type SamplingMethod =
SamplingMethodGreedy | SamplingMethodStochastic
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
index 82cce3f98..c95ac4238 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
@@ -376,19 +376,6 @@ export declare class ChatReplyConfig {
withGrammar(grammar?: Grammar | undefined | null): ChatReplyConfig
}
-export declare class ChatReplyEnergyComponents {
- cpu: number
- gpu: number
- ane: number
- dram: number
- constructor(cpu: number, gpu: number, ane: number, dram: number)
-}
-
-export declare class ChatReplyEnergyTotal {
- total: number
- constructor(total: number)
-}
-
export declare class ChatReplyJoulesPerTokenComponents {
cpu: number
gpu: number
@@ -418,9 +405,9 @@ export declare class ChatReplyStats {
tokensCountOutput?: number
memoryUsedBytes?: number
speculatorStats?: ChatReplySpeculatorStats
- inputEnergy?: ChatReplyEnergy
- outputEnergy?: ChatReplyEnergy
- constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ChatReplyEnergy, outputEnergy?: ChatReplyEnergy)
+ inputEnergy?: ReplyEnergy
+ outputEnergy?: ReplyEnergy
+ constructor(duration: number, timeToFirstToken?: number, prefillTokensPerSecond?: number, generateTokensPerSecond?: number, tokensCountInput?: number, tokensCountInputCached?: number, tokensCountOutput?: number, memoryUsedBytes?: number, speculatorStats?: ChatReplySpeculatorStats, inputEnergy?: ReplyEnergy, outputEnergy?: ReplyEnergy)
get tokensCount(): number | null
get totalJoules(): number | null
get inputJoulesPerToken(): ChatReplyJoulesPerToken | null
@@ -564,8 +551,10 @@ export declare class DecisionResult {
export declare class DecisionStats {
tokensCount: number
+ prefillDuration: number
totalDuration: number
- constructor(tokensCount: number, totalDuration: number)
+ energy?: ReplyEnergy
+ constructor(tokensCount: number, prefillDuration: number, totalDuration: number, energy?: ReplyEnergy)
}
export declare class Feature {
@@ -761,6 +750,19 @@ export declare class PcmBatch {
saveAsWav(path: string): void
}
+export declare class ReplyEnergyComponents {
+ cpu: number
+ gpu: number
+ ane: number
+ dram: number
+ constructor(cpu: number, gpu: number, ane: number, dram: number)
+}
+
+export declare class ReplyEnergyTotal {
+ total: number
+ constructor(total: number)
+}
+
export declare class Repository {
identifier: string
commitHash?: string
@@ -900,9 +902,6 @@ export interface ChatMessageMetadata {
values: Record<string, Value>
}
-export type ChatReplyEnergy =
- ChatReplyEnergyTotal | ChatReplyEnergyComponents
-
export declare const enum ChatReplyFinishReason {
Stop = 'Stop',
Length = 'Length',
@@ -980,6 +979,9 @@ export declare const enum ReasoningEffort {
XHigh = 'XHigh'
}
+export type ReplyEnergy =
+ ReplyEnergyTotal | ReplyEnergyComponents
+
export type SamplingMethod =
SamplingMethodGreedy | SamplingMethodStochastic
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.js b/crates/legacy/uzu/bindings/typescript/src/napi/index.js
index a9d3f5008..56899d3ba 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.js
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.js
@@ -559,8 +559,6 @@ module.exports.ChatMessage = nativeBinding.ChatMessage
module.exports.ChatModelCapabilities = nativeBinding.ChatModelCapabilities
module.exports.ChatReply = nativeBinding.ChatReply
module.exports.ChatReplyConfig = nativeBinding.ChatReplyConfig
-module.exports.ChatReplyEnergyComponents = nativeBinding.ChatReplyEnergyComponents
-module.exports.ChatReplyEnergyTotal = nativeBinding.ChatReplyEnergyTotal
module.exports.ChatReplyJoulesPerTokenComponents = nativeBinding.ChatReplyJoulesPerTokenComponents
module.exports.ChatReplyJoulesPerTokenTotal = nativeBinding.ChatReplyJoulesPerTokenTotal
module.exports.ChatReplySpeculatorStats = nativeBinding.ChatReplySpeculatorStats
@@ -615,6 +613,8 @@ module.exports.ModelSpecializationTextToSpeech = nativeBinding.ModelSpecializati
module.exports.ModelSpecializationTranslation = nativeBinding.ModelSpecializationTranslation
module.exports.ModelVendor = nativeBinding.ModelVendor
module.exports.PcmBatch = nativeBinding.PcmBatch
+module.exports.ReplyEnergyComponents = nativeBinding.ReplyEnergyComponents
+module.exports.ReplyEnergyTotal = nativeBinding.ReplyEnergyTotal
module.exports.Repository = nativeBinding.Repository
module.exports.SamplingMethodGreedy = nativeBinding.SamplingMethodGreedy
module.exports.SamplingMethodStochastic = nativeBinding.SamplingMethodStochastic
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
index 8d63f1a86..081f6024a 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
@@ -56,8 +56,6 @@ export const ChatMessage = cjs.ChatMessage;
export const ChatModelCapabilities = cjs.ChatModelCapabilities;
export const ChatReply = cjs.ChatReply;
export const ChatReplyConfig = cjs.ChatReplyConfig;
-export const ChatReplyEnergyComponents = cjs.ChatReplyEnergyComponents;
-export const ChatReplyEnergyTotal = cjs.ChatReplyEnergyTotal;
export const ChatReplyJoulesPerTokenComponents = cjs.ChatReplyJoulesPerTokenComponents;
export const ChatReplyJoulesPerTokenTotal = cjs.ChatReplyJoulesPerTokenTotal;
export const ChatReplySpeculatorStats = cjs.ChatReplySpeculatorStats;
@@ -112,6 +110,8 @@ export const ModelSpecializationTextToSpeech = cjs.ModelSpecializationTextToSpee
export const ModelSpecializationTranslation = cjs.ModelSpecializationTranslation;
export const ModelVendor = cjs.ModelVendor;
export const PcmBatch = cjs.PcmBatch;
+export const ReplyEnergyComponents = cjs.ReplyEnergyComponents;
+export const ReplyEnergyTotal = cjs.ReplyEnergyTotal;
export const Repository = cjs.Repository;
export const SamplingMethodGreedy = cjs.SamplingMethodGreedy;
export const SamplingMethodStochastic = cjs.SamplingMethodStochastic;
diff --git a/crates/legacy/uzu/src/engine/bridge/decision_backend.rs b/crates/legacy/uzu/src/engine/bridge/decision_backend.rs
index b8fc8558e..ffb4ea0fd 100644
--- a/crates/legacy/uzu/src/engine/bridge/decision_backend.rs
+++ b/crates/legacy/uzu/src/engine/bridge/decision_backend.rs
@@ -1,7 +1,10 @@
use std::{path::PathBuf, pin::Pin, sync::Arc};
use shoji::{
- traits::backend::{Error as BackendError, decision::DecisionInstance},
+ traits::backend::{
+ Error as BackendError,
+ decision::{DecisionInstance, DecisionSessionOutput},
+ },
types::session::classification::TokenCodecConfig,
};
use tokenizers::Tokenizer;
@@ -42,7 +45,7 @@ impl<B: Backend> DecisionInstance for UzuDecisionBackendInstance<B> {
&'a self,
input: &'a [u64],
option_count: usize,
- ) -> Pin<Box<dyn Future<Output = Result<Vec<f32>, BackendError>> + Send + 'a>> {
+ ) -> Pin<Box<dyn Future<Output = Result<DecisionSessionOutput, BackendError>> + Send + 'a>> {
Box::pin(async move {
self.model.probabilities(input, option_count).await.map_err(|err| BackendError::from(err.to_string()))
})
diff --git a/crates/uzu-engine/src/engine/decision_model.rs b/crates/uzu-engine/src/engine/decision_model.rs
index ccc91d362..c3a38c6ec 100644
--- a/crates/uzu-engine/src/engine/decision_model.rs
+++ b/crates/uzu-engine/src/engine/decision_model.rs
@@ -1,6 +1,6 @@
-use std::{io, path::Path, sync::Arc};
+use std::{io, path::Path, sync::Arc, time::Instant};
-use shoji::types::session::classification::TokenCodecConfig;
+use shoji::{traits::backend::decision::DecisionSessionOutput, types::session::classification::TokenCodecConfig};
use thiserror::Error;
use tokenizers::Tokenizer;
@@ -67,7 +67,7 @@ impl<B: Backend> DecisionModel<B> {
&self,
input: &[u64],
option_count: usize,
- ) -> Result<Vec<f32>, DecisionModelError<B>> {
+ ) -> Result<DecisionSessionOutput, DecisionModelError<B>> {
if input.is_empty() {
return Err(DecisionModelError::EmptyInput);
}
@@ -88,13 +88,14 @@ impl<B: Backend> DecisionModel<B> {
});
}
- let mut command_buffer = self.context.create_command_buffer(None, None).map_err(DecisionModelError::Backend)?;
+ let prefill_start = Instant::now();
+ let mut command_buffer = self.context.create_command_buffer(None, None).map_err(DecisionModelError::Backend)?;
let token_ids = command_buffer
.allocate_constant_from_slice(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>())
.map_err(DecisionModelError::Backend)?;
- let logits = self.decision.encode(&token_ids, input.len() as u32, &mut command_buffer)?;
+ let logits = self.decision.encode(&token_ids, input.len() as u32, &mut command_buffer)?;
let mut output_buffer = self
.context
.create_buffer_gpu(answer_count * DataType::F32.size_in_bytes())
@@ -105,14 +106,20 @@ impl<B: Backend> DecisionModel<B> {
command_buffer.end_encoding().submit().wait_until_completed().await.map_err(DecisionModelError::Backend)?;
+ let prefill_duration = prefill_start.elapsed();
+
let temperature = self.decision_config.temperature;
let logits = self.context.copyout::<f32>(&output_buffer).await.map_err(DecisionModelError::Backend)?;
let scaled = logits[..option_count].iter().map(|logit| logit / temperature).collect::<Vec<_>>();
let max = scaled.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let exps = scaled.iter().map(|logit| (logit - max).exp()).collect::<Vec<_>>();
let sum = exps.iter().sum::<f32>();
- let result = exps.into_iter().map(|value| value / sum).collect();
- Ok(result)
+ let probabilities = exps.into_iter().map(|value| value / sum).collect();
+ Ok(DecisionSessionOutput {
+ probabilities,
+ prefill_duration,
+ memory_used_bytes: self.context.peak_memory_usage(),
+ })
}
}