A high-performance inference engine for AI models
Benchmarks
Aleksei Savin3d4a66b03e4d
diff --git a/Cargo.lock b/Cargo.lockindex 09d40fe79..ccfe4808b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -305,6 +305,18 @@ version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "benchmarks"+version = "0.1.7"+dependencies = [+ "libc",+ "mach2",+ "serde",+ "serde_json",+ "sysinfo",+ "uzu",+]+[[package]]name = "binascii"version = "0.1.4"@@ -479,13 +491,16 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"name = "cli"version = "0.1.7"dependencies = [+ "benchmarks","clap",+ "comfy-table","console 0.16.0","ctrlc","indicatif 0.18.0","inquire","rocket","serde",+ "serde_json","tokio","uuid","uzu",@@ -517,6 +532,17 @@ version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"+[[package]]+name = "comfy-table"+version = "7.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b"+dependencies = [+ "crossterm 0.29.0",+ "unicode-segmentation",+ "unicode-width 0.2.1",+]+[[package]]name = "compact_str"version = "0.9.0"@@ -665,6 +691,20 @@ dependencies = ["winapi",]+[[package]]+name = "crossterm"+version = "0.29.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b"+dependencies = [+ "bitflags 2.9.4",+ "crossterm_winapi",+ "document-features",+ "parking_lot",+ "rustix",+ "winapi",+]+[[package]]name = "crossterm_winapi"version = "0.9.1"@@ -944,6 +984,15 @@ dependencies = ["syn 2.0.102",]+[[package]]+name = "document-features"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"+dependencies = [+ "litrs",+]+[[package]]name = "dyn-clone"version = "1.0.19"@@ -1727,7 +1776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0fddf93031af70e75410a2511ec04d49e758ed2f26dad3404a934e0fb45cc12a"dependencies = ["bitflags 2.9.4",- "crossterm",+ "crossterm 0.25.0","dyn-clone","fuzzy-matcher","fxhash",@@ -1839,9 +1888,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"[[package]]name = "libc"-version = "0.2.175"+version = "0.2.178"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"+checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"[[package]]name = "libloading"@@ -1890,6 +1939,12 @@ version = "0.8.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"+[[package]]+name = "litrs"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"+[[package]]name = "lock_api"version = "0.4.13"@@ -1921,6 +1976,12 @@ dependencies = ["tracing-subscriber",]+[[package]]+name = "mach2"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b"+[[package]]name = "macro_rules_attribute"version = "0.2.2"@@ -3451,9 +3512,9 @@ dependencies = [[[package]]name = "sysinfo"-version = "0.37.0"+version = "0.37.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753"+checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f"dependencies = ["libc","memchr",diff --git a/Cargo.toml b/Cargo.tomlindex dcfab2552..78ce593fb 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,5 @@[workspace]-members = ["crates/cli", "crates/uzu"]+members = ["crates/cli", "crates/uzu", "crates/benchmarks"]resolver = "3"[workspace.package]diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.tomlnew file mode 100644index 000000000..0f7293282--- /dev/null+++ b/crates/benchmarks/Cargo.toml@@ -0,0 +1,12 @@+[package]+name = "benchmarks"+version.workspace = true+edition = "2024"++[dependencies]+uzu = { path = "../uzu" }+serde = { version = "1.0.219", features = ["derive"] }+serde_json = "1.0.143"+sysinfo = "0.37.2"+libc = "0.2.178"+mach2 = "0.6.0"diff --git a/crates/benchmarks/src/lib.rs b/crates/benchmarks/src/lib.rsnew file mode 100644index 000000000..7a7c54c23--- /dev/null+++ b/crates/benchmarks/src/lib.rs@@ -0,0 +1,3 @@+pub mod runner;++pub use runner::*;diff --git a/crates/benchmarks/src/runner/helpers/memory_usage.rs b/crates/benchmarks/src/runner/helpers/memory_usage.rsnew file mode 100644index 000000000..751f9e9e2--- /dev/null+++ b/crates/benchmarks/src/runner/helpers/memory_usage.rs@@ -0,0 +1,41 @@+use std::mem;++use mach2::{+ kern_return::KERN_SUCCESS,+ mach_types::task_t,+ message::mach_msg_type_number_t,+ task::task_info,+ task_info::{TASK_BASIC_INFO, task_basic_info},+ traps::mach_task_self,+};++pub fn get_memory_usage() -> Option<u64> {+ unsafe {+ let task: task_t = mach_task_self();++ let mut info = task_basic_info {+ virtual_size: 0,+ resident_size: 0,+ user_time: mem::zeroed(),+ system_time: mem::zeroed(),+ policy: 0,+ suspend_count: 0,+ };++ let mut count: mach_msg_type_number_t =+ (mem::size_of::<task_basic_info>() / mem::size_of::<u32>()) as u32;++ let result = task_info(+ task,+ TASK_BASIC_INFO,+ &mut info as *mut _ as *mut i32,+ &mut count,+ );++ if result == KERN_SUCCESS {+ Some(info.resident_size as u64)+ } else {+ None+ }+ }+}diff --git a/crates/benchmarks/src/runner/helpers/mod.rs b/crates/benchmarks/src/runner/helpers/mod.rsnew file mode 100644index 000000000..30437bc54--- /dev/null+++ b/crates/benchmarks/src/runner/helpers/mod.rs@@ -0,0 +1,3 @@+mod memory_usage;++pub use memory_usage::get_memory_usage;diff --git a/crates/benchmarks/src/runner/mod.rs b/crates/benchmarks/src/runner/mod.rsnew file mode 100644index 000000000..2cdb5af45--- /dev/null+++ b/crates/benchmarks/src/runner/mod.rs@@ -0,0 +1,5 @@+mod helpers;+mod runner;+pub mod types;++pub use runner::Runner;diff --git a/crates/benchmarks/src/runner/runner.rs b/crates/benchmarks/src/runner/runner.rsnew file mode 100644index 000000000..6791a9b0f--- /dev/null+++ b/crates/benchmarks/src/runner/runner.rs@@ -0,0 +1,133 @@+use std::{+ path::PathBuf,+ time::{SystemTime, UNIX_EPOCH},+};++use sysinfo::System;+use uzu::{+ VERSION,+ session::{+ ChatSession,+ config::{DecodingConfig, RunConfig},+ types::{Input, Output},+ },+};++use crate::runner::{+ helpers::get_memory_usage,+ types::{Device, Result as TaskResult, Task},+};++pub struct Runner {+ pub task: Task,+ pub model_path: String,+}++impl Runner {+ pub fn new(+ task: Task,+ model_path: String,+ ) -> Self {+ Self {+ task,+ model_path,+ }+ }++ pub fn run<F>(+ &self,+ mut progress: Option<F>,+ ) -> Result<Vec<TaskResult>, Box<dyn std::error::Error>>+ where+ F: FnMut(f64),+ {+ let mut session = ChatSession::new(+ PathBuf::from(self.model_path.clone()),+ DecodingConfig::default(),+ )?;++ let precision =+ session.model_metadata.model_config.as_language_model().map(+ |config| {+ config+ .model_config+ .transformer_config+ .output_norm_config+ .scale_precision+ },+ );++ let device = self.get_device_info();++ let mut results: Vec<TaskResult> = Vec::new();+ for run_index in 0..self.task.number_of_runs {+ let timestamp =+ SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();++ let run_config =+ RunConfig::default().tokens_limit(self.task.tokens_limit);+ let output = session.run(+ Input::Messages(self.task.messages.clone()),+ run_config,+ Some(|_: Output| {+ return true;+ }),+ )?;++ let memory_used = get_memory_usage();++ let result = TaskResult {+ task: self.task.clone(),+ device: device.clone(),+ engine_version: VERSION.to_string(),+ timestamp,+ precision,+ memory_used,+ tokens_count_input: output.stats.total_stats.tokens_count_input,+ tokens_count_output: output+ .stats+ .total_stats+ .tokens_count_output,+ time_to_first_token: output.stats.prefill_stats.duration,+ prompt_tokens_per_second: output+ .stats+ .prefill_stats+ .processed_tokens_per_second,+ generate_tokens_per_second: output+ .stats+ .generate_stats+ .map(|stats| stats.tokens_per_second),+ text: output.text.original,+ };+ results.push(result);++ if let Some(progress) = progress.as_mut() {+ progress(+ (run_index + 1) as f64 / self.task.number_of_runs as f64,+ );+ }+ }++ Ok(results)+ }+}++impl Runner {+ fn get_device_info(&self) -> Device {+ let mut system_info = System::new_all();+ system_info.refresh_all();++ let os_name = System::long_os_version();+ let cpu_name =+ system_info.cpus().first().map(|cpu| cpu.brand().to_string());+ let memory_total = system_info.total_memory();++ let device = Device {+ os_name,+ cpu_name,+ memory_total,+ };++ device+ }+}diff --git a/crates/benchmarks/src/runner/types/device.rs b/crates/benchmarks/src/runner/types/device.rsnew file mode 100644index 000000000..ba267435d--- /dev/null+++ b/crates/benchmarks/src/runner/types/device.rs@@ -0,0 +1,8 @@+use serde::{Deserialize, Serialize};++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct Device {+ pub os_name: Option<String>,+ pub cpu_name: Option<String>,+ pub memory_total: u64,+}diff --git a/crates/benchmarks/src/runner/types/mod.rs b/crates/benchmarks/src/runner/types/mod.rsnew file mode 100644index 000000000..034ae4765--- /dev/null+++ b/crates/benchmarks/src/runner/types/mod.rs@@ -0,0 +1,7 @@+mod device;+mod result;+mod task;++pub use device::Device;+pub use result::Result;+pub use task::Task;diff --git a/crates/benchmarks/src/runner/types/result.rs b/crates/benchmarks/src/runner/types/result.rsnew file mode 100644index 000000000..5d7125552--- /dev/null+++ b/crates/benchmarks/src/runner/types/result.rs@@ -0,0 +1,20 @@+use serde::{Deserialize, Serialize};+use uzu::config::ConfigDataType;++use crate::runner::types::{Device, Task};++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct Result {+ pub task: Task,+ pub device: Device,+ pub engine_version: String,+ pub timestamp: u64,+ pub precision: Option<ConfigDataType>,+ pub memory_used: Option<u64>,+ pub tokens_count_input: u64,+ pub tokens_count_output: u64,+ pub time_to_first_token: f64,+ pub prompt_tokens_per_second: f64,+ pub generate_tokens_per_second: Option<f64>,+ pub text: String,+}diff --git a/crates/benchmarks/src/runner/types/task.rs b/crates/benchmarks/src/runner/types/task.rsnew file mode 100644index 000000000..072a63ef5--- /dev/null+++ b/crates/benchmarks/src/runner/types/task.rs@@ -0,0 +1,11 @@+use serde::{Deserialize, Serialize};+use uzu::session::types::Message;++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct Task {+ pub identifier: String,+ pub repo_id: String,+ pub number_of_runs: u64,+ pub tokens_limit: u64,+ pub messages: Vec<Message>,+}diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.tomlindex 55f9c5d4f..041809478 100644--- a/crates/cli/Cargo.toml+++ b/crates/cli/Cargo.toml@@ -10,6 +10,7 @@ path = "src/main.rs"[dependencies]clap = { version = "4.5.39", features = ["derive"] }uzu = { path = "../uzu" }+benchmarks = { path = "../benchmarks" }ctrlc = "3.4.1"indicatif = "0.18.0"inquire = "0.7.5"@@ -17,4 +18,6 @@ console = "0.16.0"rocket = { version = "0.5.1", features = ["json"] }tokio = { version = "1.45.1", features = ["full"] }serde = { version = "1.0.219", features = ["derive"] }+serde_json = "1.0.143"uuid = { version = "1.17.0", features = ["v4"] }+comfy-table = "7.2.1"diff --git a/crates/cli/src/handlers/bench.rs b/crates/cli/src/handlers/bench.rsnew file mode 100644index 000000000..7aff71622--- /dev/null+++ b/crates/cli/src/handlers/bench.rs@@ -0,0 +1,113 @@+use std::fs;++use benchmarks::{runner::Runner, types::Task};+use comfy_table::{+ CellAlignment, ContentArrangement, Table, modifiers::UTF8_ROUND_CORNERS,+ presets::UTF8_FULL,+};+use indicatif::ProgressBar;++pub fn handle_bench(+ model_path: String,+ task_path: String,+ output_path: String,+) -> Result<(), Box<dyn std::error::Error>> {+ let task_data = fs::read_to_string(task_path)?;+ let task: Task = serde_json::from_str(&task_data)?;++ println!("Model: {}", task.repo_id);+ let progress_bar = ProgressBar::new(task.number_of_runs);+ progress_bar.set_position(0);+ let runner = Runner::new(task.clone(), model_path);+ let results = runner.run(Some(|_: f64| {+ progress_bar.inc(1);+ }))?;+ progress_bar.finish();++ let results_data = serde_json::to_string_pretty(&results)?;+ fs::write(output_path, results_data)?;++ let tokens_count_input =+ results.first().expect("No results").tokens_count_input;+ println!("Input tokens count: {}", tokens_count_input);++ let memory_used_list = results+ .iter()+ .filter_map(|result| result.memory_used)+ .map(|memory_used| (memory_used as f64) / 1024.0 / 1024.0 / 1024.0)+ .collect::<Vec<f64>>();+ let time_to_first_token_list = results+ .iter()+ .map(|result| result.time_to_first_token)+ .collect::<Vec<f64>>();+ let prompt_tokens_per_second_list = results+ .iter()+ .map(|result| result.prompt_tokens_per_second)+ .collect::<Vec<f64>>();+ let generate_tokens_per_second_list = results+ .iter()+ .filter_map(|result| result.generate_tokens_per_second)+ .collect::<Vec<f64>>();++ let memory_used_metric = calculate_metric(memory_used_list);+ let time_to_first_token_metric = calculate_metric(time_to_first_token_list);+ let prompt_tokens_per_second_metric =+ calculate_metric(prompt_tokens_per_second_list);+ let generate_tokens_per_second_metric =+ calculate_metric(generate_tokens_per_second_list);++ let mut table = Table::new();+ table+ .load_preset(UTF8_FULL)+ .apply_modifier(UTF8_ROUND_CORNERS)+ .set_content_arrangement(ContentArrangement::Dynamic)+ .set_header(vec!["Metric", "Value"])+ .add_row(vec!["Used memory, GB", memory_used_metric.as_str()])+ .add_row(vec!["TTFT, s", time_to_first_token_metric.as_str()])+ .add_row(vec!["Prompt, t/s", prompt_tokens_per_second_metric.as_str()])+ .add_row(vec![+ "Generate, t/s",+ generate_tokens_per_second_metric.as_str(),+ ]);+ let column = table.column_mut(1).expect("Column not found");+ column.set_cell_alignment(CellAlignment::Right);+ println!("{table}");++ Ok(())+}++fn calculate_metric(data: Vec<f64>) -> String {+ if let (Some(mean), Some(std_dev)) = (mean(&data), std_dev(&data)) {+ format!("{:.3} ± {:.3}", mean, std_dev)+ } else {+ "-".to_string()+ }+}++fn mean(data: &[f64]) -> Option<f64> {+ if data.is_empty() {+ return None;+ }++ Some(data.iter().sum::<f64>() / data.len() as f64)+}++fn std_dev(data: &[f64]) -> Option<f64> {+ let n = data.len();+ if n < 2 {+ return None;+ }++ let mean = mean(data)?;++ let variance = data+ .iter()+ .map(|x| {+ let diff = x - mean;+ diff * diff+ })+ .sum::<f64>()+ / (n as f64 - 1.0);++ Some(variance.sqrt())+}diff --git a/crates/cli/src/handlers/mod.rs b/crates/cli/src/handlers/mod.rsindex 620ef1876..0ce67a8dc 100644--- a/crates/cli/src/handlers/mod.rs+++ b/crates/cli/src/handlers/mod.rs@@ -2,3 +2,5 @@ pub mod run;pub use run::handle_run;pub mod serve;pub use serve::handle_serve;+pub mod bench;+pub use bench::handle_bench;diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rsindex 9e74854ed..63a44015d 100644--- a/crates/cli/src/main.rs+++ b/crates/cli/src/main.rs@@ -1,5 +1,5 @@use clap::{CommandFactory, Parser, Subcommand};-use cli::handlers::{handle_run, handle_serve};+use cli::handlers::{handle_bench, handle_run, handle_serve};#[derive(Parser)]struct Cli {@@ -23,6 +23,15 @@ enum Commands {/// Prefill step sizeprefill_step_size: Option<usize>,},+ /// Run benchmarks for the specified model+ Bench {+ /// Folder with model's files+ model_path: String,+ /// Path to the task file+ task_path: String,+ /// Path to the output file+ output_path: String,+ },}fn main() {@@ -41,6 +50,13 @@ fn main() {}) => {handle_serve(model_path, prefill_step_size);},+ Some(Commands::Bench {+ model_path,+ task_path,+ output_path,+ }) => {+ let _ = handle_bench(model_path, task_path, output_path);+ },None => {let mut cmd = Cli::command();cmd.print_help().unwrap();diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex c08574262..42d33594b 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -26,7 +26,7 @@ minijinja = "2.12.0"minijinja-contrib = { version = "2.12.0", features = ["pycompat"] }clap = { version = "4.5.47", features = ["derive"] }rand = "0.9.2"-sysinfo = "0.37.0"+sysinfo = "0.37.2"itertools = "0.14.0"schemars = "1.0.5"diff --git a/scripts/tools/main.py b/scripts/tools/main.pyindex f67d29ed5..30e2c3a91 100644--- a/scripts/tools/main.py+++ b/scripts/tools/main.py@@ -1,9 +1,10 @@from pathlib import Pathfrom typing import Annotated+import jsonimport requestsimport tomllib-from model import Registry+from model import Registry, Model, BenchmarkTask, Message, Rolefrom rich.console import Consolefrom rich.progress import (BarColumn,@@ -16,6 +17,7 @@ from rich.progress import ()from rich.table import Tablefrom typer import Argument, Typer+from dataclasses import asdictfrom utils import download_file_with_resumeROOT_PATH = Path(__file__).parent.parent.parent@@ -109,6 +111,14 @@ def download_model(model_path = engine_path / model.namemodel_path.mkdir(parents=True, exist_ok=True)+ benchmark_task = generate_benchmark_task(model)+ benchmark_task_data = asdict(benchmark_task)+ benchmark_task_name = "benchmark_task.json"+ benchmark_task_path = model_path / benchmark_task_name+ with open(benchmark_task_path, "w") as file:+ json.dump(benchmark_task_data, file, indent=4, sort_keys=True)+ console.print(f"[green]✓[/green] {benchmark_task_name} generated")+files_to_download = []for file in model.files:file_path = model_path / file.name@@ -147,5 +157,21 @@ def download_model(err_console.print(f"[red]Error: {e}[/red]")+def generate_benchmark_task(model: Model) -> BenchmarkTask:+ messages: [Message] = [+ Message(role=Role.SYSTEM, content="Summarize user's input"),+ Message(role=Role.USER, content="Large language models, commonly referred to as LLMs, are a class of artificial intelligence systems designed to understand, generate, and manipulate human language at scale. They are built using deep learning techniques, most notably transformer architectures, and are trained on vast collections of text data drawn from books, articles, websites, and other written sources. Through this training process, LLMs learn statistical patterns in language, allowing them to predict likely sequences of words and produce coherent, contextually appropriate responses to prompts. At their core, LLMs operate by representing words or subword units as numerical vectors in a high-dimensional space. These representations capture semantic and syntactic relationships, such that words with similar meanings or grammatical roles tend to have similar vector representations. The transformer architecture enables the model to process entire sequences of text simultaneously rather than sequentially, using mechanisms such as self-attention to determine which parts of the input are most relevant at any given moment. This allows LLMs to handle long-range dependencies in text, such as references made many sentences earlier, more effectively than earlier generations of language models. Training an LLM typically involves two main phases. The first is pretraining, during which the model learns general language patterns by predicting missing or next tokens in large, mostly uncurated text corpora. This phase gives the model broad linguistic competence and general world knowledge as reflected in its training data. The second phase often involves fine-tuning, where the model is further trained on more specific datasets, such as question-answer pairs, instructional content, or conversational examples. Fine-tuning helps align the model’s behavior with particular tasks or desired styles of interaction. One of the most notable characteristics of LLMs is their versatility. A single model can perform a wide range of language-related tasks, including text generation, summarization, translation, classification, question answering, and code generation, often without task-specific retraining. This flexibility arises from the model’s general-purpose training and its ability to condition its outputs on the instructions or context provided in the prompt. As a result, LLMs are increasingly used as foundational models that can be adapted to many applications across different domains. Despite their impressive capabilities, LLMs do not possess true understanding or consciousness. They do not have beliefs, intentions, or awareness in the human sense. Instead, they generate responses based on learned correlations in data. This limitation can lead to errors such as producing confident-sounding but incorrect information, sometimes referred to as hallucinations. Because LLMs rely on patterns in their training data, they may also reflect biases, inaccuracies, or gaps present in that data. Addressing these issues is an ongoing area of research and development. The computational cost of training and running LLMs is another significant consideration. Training state-of-the-art models can require enormous amounts of computing power, energy, and financial investment. This has implications for environmental sustainability and for who can realistically develop and deploy such models. As a result, there is growing interest in techniques that improve efficiency, such as model compression, distillation, sparse architectures, and more efficient training algorithms, as well as in smaller models that can perform well on specific tasks. LLMs also raise important ethical and social questions. Their ability to generate human-like text can be beneficial in areas such as education, accessibility, and productivity, but it can also be misused for purposes like misinformation, plagiarism, or automated spam. Ensuring responsible use involves a combination of technical safeguards, policy decisions, and user education. Researchers and developers are actively exploring methods for making LLMs more transparent, controllable, and aligned with human values. As research continues, LLMs are likely to become more capable, more efficient, and more integrated into everyday tools and workflows. Future developments may involve better reasoning abilities, improved factual reliability, stronger multimodal integration with images, audio, and video, and more personalized interactions that adapt to individual users while respecting privacy. While LLMs are not a replacement for human judgment or creativity, they represent a powerful technology that, when used thoughtfully, can augment human capabilities and transform how people interact with information and with machines."),+ ]++ task = BenchmarkTask(+ identifier=model.name.lower(),+ repo_id=model.repod_id,+ number_of_runs=15,+ tokens_limit=512,+ messages=messages,+ )+ return task++if __name__ == "__main__":app()diff --git a/scripts/tools/model.py b/scripts/tools/model.pyindex 3bb39f5f3..c38c09b70 100644--- a/scripts/tools/model.py+++ b/scripts/tools/model.py@@ -1,5 +1,6 @@from dataclasses import dataclassfrom typing import List+from enum import Enum@dataclass(frozen=True)@@ -38,3 +39,24 @@ class Registry:@classmethoddef from_dict(cls, data: dict) -> "Registry":return cls(models=[Model.from_dict(model) for model in data["models"]])+++class Role(str, Enum):+ SYSTEM = "system"+ USER = "user"+ ASSISTANT = "assistant"+++@dataclass(frozen=True)+class Message:+ role: Role+ content: str+++@dataclass(frozen=True)+class BenchmarkTask:+ identifier: str+ repo_id: str+ number_of_runs: int+ tokens_limit: int+ messages: List[Message]