Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Capture gpu trace (#659)

uuuvn95de8b22d724
diff --git a/Cargo.toml b/Cargo.toml
index 21a27e8c6..7aef02c1d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -229,8 +229,14 @@ forward-cfgs = ["backend", "grammar"]
uzu_test = "dinghy_apple_runner_macros::test_case"
uzu_bench = "dinghy_apple_runner_macros::bench_case"
+[profile.dev]
+opt-level = 1
+
+[profile.dev.package."*"]
+opt-level = 2
+
[profile.dev.build-override]
-opt-level = 3
+opt-level = 2
[profile.release]
lto = "thin"
diff --git a/crates/backend-uzu/build/metal/toolchain.rs b/crates/backend-uzu/build/metal/toolchain.rs
index c9201c4a9..aa89bcf54 100644
--- a/crates/backend-uzu/build/metal/toolchain.rs
+++ b/crates/backend-uzu/build/metal/toolchain.rs
@@ -102,19 +102,22 @@ impl MetalToolchain {
let sdk = MetalSdk::from_env().context("cannot get sdk")?;
let std = MetalStd::Metal4_0;
- let opt_flags = match env::var("OPT_LEVEL").context("missing OPT_LEVEL")?.as_str() {
- "0" => {
- [
- OsString::from("-O1"), // matmul kernels compiled with -O0 are broken and require a reboot to unfreeze the os
- OsString::from("-gline-tables-only"), // debug with line tables only
- OsString::from("-frecord-sources"), // include source code
- ]
- .into()
- },
- "1" => [OsString::from("-O1")].into(),
- _ => [OsString::from("-O2")].into(), // treat levels 2,3,s,z as O2 for metal
+ let opt_level_flags = match env::var("OPT_LEVEL").context("missing OPT_LEVEL")?.as_str() {
+ "0" => vec![OsString::from("-O1")], // matmul kernels compiled with -O0 are broken and require a reboot to unfreeze the os
+ _ => vec![OsString::from("-O2")], // treat levels everything else (1,2,3,s,z) as O2 for metal
};
+ let debug_flags = match env::var("DEBUG").context("missing DEBUG")?.as_str() {
+ "false" => vec![],
+ "true" => vec![
+ OsString::from("-gline-tables-only"), // debug with line tables only
+ OsString::from("-frecord-sources"), // include source code
+ ],
+ debug => bail!("Unknown DEBUG value {debug}"),
+ };
+
+ let opt_flags = [opt_level_flags, debug_flags].concat().into_boxed_slice();
+
let extra_options = Box::new([OsString::from(format!("-m{}-version-min={}", sdk.os(), std.min_os()))]);
let include_dirs = include_dir.into_iter().collect();
diff --git a/crates/backend-uzu/src/backends/common/backend.rs b/crates/backend-uzu/src/backends/common/backend.rs
index 94f344e19..f29bfc6b1 100644
--- a/crates/backend-uzu/src/backends/common/backend.rs
+++ b/crates/backend-uzu/src/backends/common/backend.rs
@@ -10,6 +10,7 @@ pub trait Backend: Debug + Clone + Send + Sync + 'static {
type Kernels: Kernels<Backend = Self>;
type Error: Error + Debug;
+ const NAME: &'static str;
const MIN_ALLOCATION_ALIGNMENT: usize;
const MAX_ALLOCATION_ALIGNMENT: usize;
const ALLOCATION_GRANULARITY: usize;
diff --git a/crates/backend-uzu/src/backends/cpu/backend.rs b/crates/backend-uzu/src/backends/cpu/backend.rs
index 9da2a5495..b039f13f3 100644
--- a/crates/backend-uzu/src/backends/cpu/backend.rs
+++ b/crates/backend-uzu/src/backends/cpu/backend.rs
@@ -15,6 +15,7 @@ impl Backend for Cpu {
type Kernels = CpuKernels;
type Error = CpuError;
+ const NAME: &'static str = "cpu";
const MIN_ALLOCATION_ALIGNMENT: usize = 4;
const MAX_ALLOCATION_ALIGNMENT: usize = 64;
const ALLOCATION_GRANULARITY: usize = 8 * 1024 * 1024;
diff --git a/crates/backend-uzu/src/backends/metal/backend.rs b/crates/backend-uzu/src/backends/metal/backend.rs
index fe777ae3f..a394a7c1a 100644
--- a/crates/backend-uzu/src/backends/metal/backend.rs
+++ b/crates/backend-uzu/src/backends/metal/backend.rs
@@ -15,6 +15,7 @@ impl Backend for Metal {
type Kernels = MetalKernels;
type Error = MetalError;
+ const NAME: &'static str = "metal";
const MIN_ALLOCATION_ALIGNMENT: usize = 4;
const MAX_ALLOCATION_ALIGNMENT: usize = 64;
const ALLOCATION_GRANULARITY: usize = 8 * 1024 * 1024;
diff --git a/crates/backend-uzu/src/backends/metal/context.rs b/crates/backend-uzu/src/backends/metal/context.rs
index 81bf19bc6..8b683a71f 100644
--- a/crates/backend-uzu/src/backends/metal/context.rs
+++ b/crates/backend-uzu/src/backends/metal/context.rs
@@ -216,7 +216,7 @@ impl Context for MetalContext {
let capture_manager = MTLCaptureManager::shared_capture_manager();
let capture_descriptor = MTLCaptureDescriptor::new();
capture_descriptor.set_destination(MTLCaptureDestination::GPUTraceDocument);
- capture_descriptor.set_output_path(Some(trace_path));
+ capture_descriptor.set_output_path(Some(&trace_path.with_added_extension("gputrace")));
self.command_queue.set_label(Some("uzu_command_queue"));
capture_descriptor.set_capture_object(Some(self.command_queue.as_ref()));
diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rs
index 3d9a09300..5c60f58ae 100644
--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs
+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs
@@ -44,8 +44,8 @@ use crate::{
};
pub struct UzuChatTokenBackendInstance<B: Backend> {
- engine: Arc<Mutex<Engine<B>>>,
- model: Arc<Mutex<LanguageModel<B>>>,
+ engine: Arc<Engine<B>>,
+ model: Arc<LanguageModel<B>>,
config: ChatConfig,
#[cfg(grammar)]
tokenizer: Arc<Tokenizer>,
@@ -67,8 +67,8 @@ impl<B: Backend> UzuChatTokenBackendInstance<B> {
let max_context_length = get_max_context_length(&model, config.context_length.clone());
Ok(Self {
- engine: Arc::new(Mutex::new(engine)),
- model: Arc::new(Mutex::new(model)),
+ engine,
+ model: Arc::new(model),
config,
#[cfg(grammar)]
tokenizer,
@@ -86,9 +86,8 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn State>, BackendError>> + Send + '_>> {
Box::pin(async move {
- let model = self.model.lock();
- let max_context_length = get_max_context_length(&model, self.config.context_length.clone());
- model
+ let max_context_length = get_max_context_length(&self.model, self.config.context_length.clone());
+ self.model
.create_empty_state(max_context_length)
.map_err(|err| BackendError::from(err.to_string()))
.map(|state| Box::new(UzuChatTokenBackendInstanceState::new(state)) as Box<dyn State>)
@@ -109,7 +108,6 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
>,
> {
let model = self.model.clone();
- let model_guard = Box::pin(model.lock());
let state =
(state as &mut dyn Any).downcast_mut::<UzuChatTokenBackendInstanceState<B>>().unwrap().value.clone();
@@ -134,12 +132,12 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
}
let options = LanguageModelStreamOptions {
- sampling_method: get_sampling_method::<B>(&model_guard, &config.sampling_policy),
+ sampling_method: get_sampling_method::<B>(&self.model, &config.sampling_policy),
#[cfg(grammar)]
grammar,
};
- let stream = match model_guard.stream(input, &mut state_guard, options) {
+ let stream = match self.model.stream(input, &mut state_guard, options) {
Ok(iter) => iter,
Err(err) => {
return Box::pin(NoMetricsStream::new(error_stream(err.to_string())));
@@ -158,18 +156,12 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
>(state_guard)
},
_state: state,
- _model_guard: unsafe {
- std::mem::transmute::<
- Pin<Box<MutexGuard<'_, LanguageModel<B>>>>,
- Pin<Box<MutexGuard<'a, LanguageModel<B>>>>,
- >(model_guard)
- },
_model: model,
})
}
fn peak_memory_usage(&self) -> Option<usize> {
- self.engine.lock().peak_memory_usage()
+ self.engine.peak_memory_usage()
}
}
@@ -192,8 +184,7 @@ struct UzuChatTokenStream<'a, B: Backend> {
token_limit: Option<usize>,
_state_guard: Pin<Box<MutexGuard<'a, LanguageModelState<B>>>>,
_state: Arc<Mutex<LanguageModelState<B>>>,
- _model_guard: Pin<Box<MutexGuard<'a, LanguageModel<B>>>>,
- _model: Arc<Mutex<LanguageModel<B>>>,
+ _model: Arc<LanguageModel<B>>,
}
impl<'a, B: Backend> UzuChatTokenStream<'a, B> {
diff --git a/crates/backend-uzu/src/engine/capture.rs b/crates/backend-uzu/src/engine/capture.rs
new file mode 100644
index 000000000..93713fea3
--- /dev/null
+++ b/crates/backend-uzu/src/engine/capture.rs
@@ -0,0 +1,98 @@
+use std::{
+ path::PathBuf,
+ sync::{
+ Arc, LazyLock,
+ atomic::{AtomicBool, Ordering},
+ },
+};
+
+use crate::backends::common::{Backend, Context};
+
+static CAPTURE_FIRST_PREFILL: LazyLock<AtomicBool> =
+ LazyLock::new(|| AtomicBool::new(std::env::var("UZU_CAPTURE_FIRST_PREFILL").is_ok()));
+static CAPTURE_FIRST_DECODE: LazyLock<AtomicBool> =
+ LazyLock::new(|| AtomicBool::new(std::env::var("UZU_CAPTURE_FIRST_DECODE").is_ok()));
+
+pub struct CaptureRequest<B: Backend> {
+ context: Arc<B::Context>,
+ path: PathBuf,
+}
+
+impl<B: Backend> CaptureRequest<B> {
+ fn new(
+ context: Arc<B::Context>,
+ path: PathBuf,
+ ) -> Self {
+ Self {
+ context,
+ path,
+ }
+ }
+
+ pub fn start(self) -> Result<CaptureSpan<B>, B::Error> {
+ self.context.start_capture(&self.path)?;
+ eprintln!("Started capturing to {:?}", self.path);
+
+ Ok(CaptureSpan {
+ context: self.context,
+ path: self.path,
+ })
+ }
+}
+
+pub struct CaptureSpan<B: Backend> {
+ context: Arc<B::Context>,
+ path: PathBuf,
+}
+
+impl<B: Backend> Drop for CaptureSpan<B> {
+ fn drop(&mut self) {
+ match self.context.stop_capture() {
+ Ok(()) => {
+ eprintln!("Successfully saved capture to {:?}", self.path);
+ },
+ Err(err) => {
+ eprintln!("Error when saving capture to {:?}: {:?}", self.path, err);
+ },
+ }
+ }
+}
+
+pub struct CaptureManager<B: Backend> {
+ context: Arc<B::Context>,
+}
+
+impl<B: Backend> CaptureManager<B> {
+ pub fn pre_load_enable() -> bool {
+ let capture_enabled =
+ CAPTURE_FIRST_PREFILL.load(Ordering::Relaxed) || CAPTURE_FIRST_DECODE.load(Ordering::Relaxed);
+
+ if capture_enabled {
+ <B::Context as Context>::enable_capture();
+ }
+
+ capture_enabled
+ }
+
+ pub fn new(context: Arc<B::Context>) -> Self {
+ Self {
+ context,
+ }
+ }
+
+ fn capture_path(suffix: &str) -> PathBuf {
+ PathBuf::from("/tmp").join(format!("uzu-capture-{}-{}", B::NAME, suffix))
+ }
+
+ pub fn maybe_capture_prefill_step(&self) -> Option<CaptureRequest<B>> {
+ CAPTURE_FIRST_PREFILL
+ .swap(false, Ordering::Relaxed)
+ .then(|| CaptureRequest::new(self.context.clone(), Self::capture_path("prefill")))
+ }
+
+ pub fn maybe_capture_decode_step(&self) -> Option<CaptureRequest<B>> {
+ CAPTURE_FIRST_DECODE
+ .swap(false, Ordering::Relaxed)
+ .then(|| CaptureRequest::new(self.context.clone(), Self::capture_path("decode")))
+ }
+}
diff --git a/crates/backend-uzu/src/engine/language_model/mod.rs b/crates/backend-uzu/src/engine/language_model/mod.rs
index 7343cbcaf..bac953aee 100644
--- a/crates/backend-uzu/src/engine/language_model/mod.rs
+++ b/crates/backend-uzu/src/engine/language_model/mod.rs
@@ -22,7 +22,7 @@ pub mod stream;
pub mod grammar;
pub struct LanguageModel<B: Backend> {
- context: Arc<B::Context>,
+ engine: Arc<Engine<B>>,
decoder: Decoder<B>,
speculator: Option<DFlashSpeculator<B>>,
sampling: Sampling<B>,
@@ -52,7 +52,7 @@ pub enum EngineLoadLanguageModelError<B: Backend> {
impl<B: Backend> Engine<B> {
pub fn load_language_model(
- &self,
+ self: &Arc<Self>,
model_path: &Path,
) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {
let config: LanguageModelConfig =
@@ -70,7 +70,7 @@ impl<B: Backend> Engine<B> {
// TODO: better design
pub fn load_language_model_random(
- &self,
+ self: &Arc<Self>,
config_path: &Path,
header_path: &Path,
seed: u64,
@@ -84,17 +84,15 @@ impl<B: Backend> Engine<B> {
}
fn build_language_model(
- &self,
+ self: &Arc<Self>,
config: LanguageModelConfig,
weight_loader: &ParameterLoader<B>,
speculator_path: Option<&Path>,
) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {
- let context = self.context.clone();
-
let data_type = DataType::BF16;
let decoder = Decoder::new(
- context.as_ref(),
+ self.context.as_ref(),
&config.decoder_config,
&weight_loader.tree().subtree("decoder")?,
data_type,
@@ -106,12 +104,12 @@ impl<B: Backend> Engine<B> {
);
let speculator = speculator_path
- .map(|speculator_path| DFlashSpeculator::load(speculator_path, context.clone()))
+ .map(|speculator_path| DFlashSpeculator::load(speculator_path, self.context.clone()))
.transpose()?;
let sampling = Sampling::new(data_type, config.decoder_config.vocab_size);
- let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&context)
+ let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)
.map_err(EngineLoadLanguageModelError::Backend)?;
weight_loader.tree().assert_all_tensors_validated()?;
@@ -122,7 +120,7 @@ impl<B: Backend> Engine<B> {
let vocab_size = config.decoder_config.vocab_size;
Ok(LanguageModel {
- context,
+ engine: self.clone(),
decoder,
speculator,
sampling,
@@ -143,7 +141,7 @@ impl<B: Backend> LanguageModel<B> {
let max_context_length = self.max_context_length();
// TODO: This is not the correct way to do it, there should be a real memory model
- if self.context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS) {
+ if self.engine.context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS) {
// We just assume that all mixers use sparse if it's available to make max context free until it's actually used
// Currenlty true for all mixers in uzu:
// - full attention uses sparse if it's available to make max context free until it's actually used
diff --git a/crates/backend-uzu/src/engine/language_model/state.rs b/crates/backend-uzu/src/engine/language_model/state.rs
index d796779c0..86f1c545a 100644
--- a/crates/backend-uzu/src/engine/language_model/state.rs
+++ b/crates/backend-uzu/src/engine/language_model/state.rs
@@ -39,7 +39,7 @@ impl<B: Backend> LanguageModel<B> {
let transformer_state = self
.decoder
- .create_empty_state(max_context_length, &self.context)
+ .create_empty_state(max_context_length, &self.engine.context)
.map_err(LanguageModelCreateEmptyStateError::Backend)?;
let speculator_state = self
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index eaa6ad021..b7d01ad76 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -15,10 +15,13 @@ use crate::{
},
data_type::DataType,
encodable_block::{batch_topology::BatchTopology, sampling::SamplingMethod},
- engine::language_model::{
- LanguageModel,
- state::LanguageModelState,
- stream::{LanguageModelStreamError, LanguageModelStreamOptions},
+ engine::{
+ capture::CaptureSpan,
+ language_model::{
+ LanguageModel,
+ state::LanguageModelState,
+ stream::{LanguageModelStreamError, LanguageModelStreamOptions},
+ },
},
speculators::dflash_speculator::DFlashTreeOptions,
trie::TrieNode,
@@ -78,6 +81,7 @@ struct DecodingStatePending<B: Backend> {
input_trie: TrieNode,
full_accept: bool,
pending: Box<[Pending<B>]>,
+ _capture_span: Option<CaptureSpan<B>>,
hidden_features: Option<Box<[Allocation<B>]>>,
output_norm: Option<Allocation<B>>,
output_tokens: Allocation<B>,
@@ -141,11 +145,20 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
return Err(LanguageModelStreamError::ContextOverflow);
}
- let allocation_pool = Arc::new(model.context.create_allocation_pool(false));
+ let capture_span = if let Some(capture_manager) = &model.engine.capture_manager
+ && let Some(capture_request) = capture_manager.maybe_capture_prefill_step()
+ {
+ Some(capture_request.start().map_err(LanguageModelStreamError::Backend)?)
+ } else {
+ None
+ };
+
+ let allocation_pool = Arc::new(model.engine.context.create_allocation_pool(false));
let mut context_ring =
if let Some(suffix_repetition_length) = options.sampling_method.suffix_repetition_length() {
let mut context_ring = model
+ .engine
.context
.create_allocation(
(2 + suffix_repetition_length) * DataType::U32.size_in_bytes(),
@@ -184,11 +197,11 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
.prepare(
model_state.transformer_state.context_length() + (number_of_batches - 1) * max_batch_size,
usize::min(max_batch_size, input.len()),
- &model.context,
+ &model.engine.context,
)
.map_err(LanguageModelStreamError::Backend)?;
- let mut encoder = Encoder::<B>::new_with_pool(&model.context, allocation_pool.clone())
+ let mut encoder = Encoder::<B>::new_with_pool(&model.engine.context, allocation_pool.clone())
.map_err(LanguageModelStreamError::Backend)?;
let mut output_tokens = None;
@@ -320,6 +333,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
input_trie: TrieNode::new(0, 0),
full_accept: true,
pending,
+ _capture_span: capture_span,
hidden_features: None,
output_norm,
output_tokens: output_tokens.unwrap(),
@@ -419,7 +433,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
let accepted_input_token_ids = full.iter().map(|(_, t, _)| *t).collect::<Box<[u64]>>();
let accepted_output_token_ids = full.iter().map(|(_, _, t)| *t).collect::<Box<[u64]>>();
let mut encoder =
- Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone())
+ Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
.map_err(LanguageModelStreamError::Backend)?;
self.model_state
.transformer_state
@@ -492,11 +506,24 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
));
}
+ let capture_span = if let Some(capture_manager) = &self.model.engine.capture_manager
+ && let Some(capture_request) = capture_manager.maybe_capture_decode_step()
+ {
+ prev_output.resolve(
+ &mut self.model_state.tokens,
+ #[cfg(grammar)]
+ self.options.grammar.as_mut(),
+ )?;
+ Some(capture_request.start().map_err(LanguageModelStreamError::Backend)?)
+ } else {
+ None
+ };
+
let mut pending = Vec::new();
let mut encoder = if let Some(encoder) = encoder {
encoder
} else {
- Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone())
+ Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
.map_err(LanguageModelStreamError::Backend)?
};
@@ -513,7 +540,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.options.grammar.as_mut(),
)? {
pending.push(encoder.end_encoding().submit());
- encoder = Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone())
+ encoder = Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
.map_err(LanguageModelStreamError::Backend)?;
let trie = speculator.propose_tree(
self.model_state.speculator_state.as_mut().unwrap(),
@@ -560,7 +587,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
self.model_state
.transformer_state
- .prepare(self.model_state.transformer_state.context_length(), batch_dim.size(), &self.model.context)
+ .prepare(self.model_state.transformer_state.context_length(), batch_dim.size(), &self.model.engine.context)
.map_err(LanguageModelStreamError::Backend)?;
let hidden_feature_layer_indices =
@@ -581,7 +608,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
if chain_copy.is_some() {
pending.push(encoder.end_encoding().submit());
- let mut encoder = Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone())
+ let mut encoder = Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone())
.map_err(LanguageModelStreamError::Backend)?;
let mut bitmask = encoder
@@ -690,6 +717,7 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
input_trie,
full_accept,
pending: pending.into_boxed_slice(),
+ _capture_span: capture_span,
hidden_features: if full_accept {
None
} else {
@@ -736,7 +764,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
if !in_flight.full_accept {
let mut encoder =
- Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone()).unwrap();
+ Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone()).unwrap();
self.model_state.transformer_state.encode_accept(&[0], &mut encoder).unwrap();
if let Some(speculator) = self.model.speculator.as_ref() {
speculator
@@ -762,7 +790,7 @@ impl<'a, B: Backend> Drop for LanguageModelStream<'a, B> {
assert!(num_accepted > 0 && num_accepted < full.len());
let mut encoder =
- Encoder::<B>::new_with_pool(&self.model.context, self.allocation_pool.clone()).unwrap();
+ Encoder::<B>::new_with_pool(&self.model.engine.context, self.allocation_pool.clone()).unwrap();
let accepted_token_indicies =
full.iter().take(num_accepted + 1).map(|(i, _, _)| *i).collect::<Box<[usize]>>();
self.model_state.transformer_state.encode_accept(&accepted_token_indicies, &mut encoder).unwrap();
diff --git a/crates/backend-uzu/src/engine/mod.rs b/crates/backend-uzu/src/engine/mod.rs
index e6c733ad6..4c8cb641b 100644
--- a/crates/backend-uzu/src/engine/mod.rs
+++ b/crates/backend-uzu/src/engine/mod.rs
@@ -2,13 +2,19 @@ use std::sync::Arc;
use thiserror::Error;
-use crate::backends::common::{Backend, Context};
+use crate::{
+ backends::common::{Backend, Context},
+ engine::capture::CaptureManager,
+};
pub mod classifier_model;
pub mod language_model;
+mod capture;
+
pub struct Engine<B: Backend> {
context: Arc<B::Context>,
+ capture_manager: Option<CaptureManager<B>>,
}
#[derive(Debug, Error)]
@@ -18,12 +24,17 @@ pub enum EngineNewError<B: Backend> {
}
impl<B: Backend> Engine<B> {
- pub fn new() -> Result<Self, EngineNewError<B>> {
+ pub fn new() -> Result<Arc<Self>, EngineNewError<B>> {
+ let capture_enabled = CaptureManager::<B>::pre_load_enable();
+
let context = <B::Context as Context>::new().map_err(EngineNewError::Backend)?;
- Ok(Self {
+ let capture_manager = capture_enabled.then(|| CaptureManager::new(context.clone()));
+
+ Ok(Arc::new(Self {
context,
- })
+ capture_manager,
+ }))
}
pub fn peak_memory_usage(&self) -> Option<usize> {