Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Muse glimmer dflash (astra slop) (#839)

uuuvn17ce26ab3353
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index 4f73f9249..6ae7f489b 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -334,7 +334,7 @@ impl<B: Backend> DFlash<B> {
let mut lookahead_hidden =
encoder.allocate_scratch(lookahead_rows.len()).map_err(DFlashEncodeError::Backend)?;
encoder.encode_copy(&draft_hidden, lookahead_rows, &mut lookahead_hidden, ..);
- let logits = target_embedding.encode_readout(batch_size - 1, &lookahead_hidden, DataType::F32, encoder)?;
+ let logits = target_embedding.encode_readout_raw(batch_size - 1, &lookahead_hidden, DataType::F32, encoder)?;
encoder.pop_debug_group();
diff --git a/crates/uzu-engine/src/encodable_block/embedding.rs b/crates/uzu-engine/src/encodable_block/embedding.rs
index 7b352fff4..c210bf60c 100644
--- a/crates/uzu-engine/src/encodable_block/embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/embedding.rs
@@ -78,15 +78,15 @@ struct LogitTransform<B: Backend> {
}
impl<B: Backend> Embedding<B> {
- pub(crate) fn data_type(&self) -> DataType {
+ pub fn data_type(&self) -> DataType {
self.data_type
}
- pub(crate) fn vocab_size(&self) -> u32 {
+ pub fn vocab_size(&self) -> u32 {
self.vocab_size
}
- pub(crate) fn model_dim(&self) -> u32 {
+ pub fn model_dim(&self) -> u32 {
self.model_dim
}
@@ -377,6 +377,36 @@ impl<B: Backend> Embedding<B> {
input_allocation: &Allocation<B>,
output_data_type: DataType,
encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, EmbeddingError<B>> {
+ let mut output_allocation = self.encode_readout_raw(batch_dim, input_allocation, output_data_type, encoder)?;
+ let native_output = output_data_type == self.data_type;
+
+ if let Some(logit_transform) = &self.logit_transform {
+ let length = batch_dim * self.vocab_size;
+ let kernel = if native_output {
+ &logit_transform.kernel
+ } else {
+ assert_eq!(output_data_type, DataType::F32, "unsupported readout output data type");
+ logit_transform.widened_kernel.as_ref().expect("widened logit transform kernel is missing")
+ };
+ kernel.encode(
+ &mut output_allocation,
+ length,
+ logit_transform.scale,
+ logit_transform.soft_cap.unwrap_or(0.0),
+ encoder,
+ );
+ }
+
+ Ok(output_allocation)
+ }
+
+ pub fn encode_readout_raw(
+ &self,
+ batch_dim: u32,
+ input_allocation: &Allocation<B>,
+ output_data_type: DataType,
+ encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, EmbeddingError<B>> {
encoder.push_debug_group("embedding readout");
@@ -433,31 +463,12 @@ impl<B: Backend> Embedding<B> {
widened.encode(arguments, encoder).map_err(EmbeddingError::BackendError)?;
}
- if let Some(logit_transform) = &self.logit_transform {
- let length = batch_dim * self.vocab_size;
- let kernel = if native_output {
- &logit_transform.kernel
- } else {
- assert_eq!(output_data_type, DataType::F32, "unsupported readout output data type");
- logit_transform.widened_kernel.as_ref().expect("widened logit transform kernel is missing")
- };
- kernel.encode(
- &mut output_allocation,
- length,
- logit_transform.scale,
- logit_transform.soft_cap.unwrap_or(0.0),
- encoder,
- );
- }
-
encoder.pop_debug_group();
Ok(output_allocation)
}
- /// Per-row candidate readout via the GEMV B-row gather: `out[r][j] == dense[r][token_ids[r][j]]`,
- /// soft-capped when configured, one dispatch. Caller guarantees `token_ids < vocab_size`.
- pub(crate) fn encode_readout_sparse(
+ pub fn encode_readout_sparse_raw(
&self,
input: &Allocation<B>,
token_ids: &Allocation<B>,
@@ -493,11 +504,6 @@ impl<B: Backend> Embedding<B> {
None => input,
};
- let fuse_soft_cap = match &self.logit_transform {
- Some(logit_transform) if logit_transform.scale != 1.0 => None,
- Some(logit_transform) => logit_transform.soft_cap,
- None => None,
- };
readout
.lock()
.encode(
@@ -510,10 +516,7 @@ impl<B: Backend> Embedding<B> {
b_leading_dimension: None,
b_transpose: true,
d: &mut output,
- d_transform: MatmulDOps {
- soft_cap: fuse_soft_cap,
- ..MatmulDOps::none()
- },
+ d_transform: MatmulDOps::none(),
gather_indices: Some(token_ids),
m: rows,
n: ids_per_row,
@@ -523,19 +526,6 @@ impl<B: Backend> Embedding<B> {
)
.map_err(EmbeddingError::BackendError)?;
- if let Some(logit_transform) = &self.logit_transform
- && logit_transform.scale != 1.0
- {
- let length = rows * ids_per_row;
- logit_transform.kernel.encode(
- &mut output,
- length,
- logit_transform.scale,
- logit_transform.soft_cap.unwrap_or(0.0),
- encoder,
- );
- }
-
encoder.pop_debug_group();
Ok(output)
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index 07bcfa28e..427821f4f 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -474,7 +474,7 @@ impl<B: Backend> Weaver<B> {
.readout_query_projection
.encode(normalized_output, batch_node_count, encoder)
.map_err(WeaverEncodeError::Backend)?;
- let logit_residuals = target_embedding.encode_readout_sparse(
+ let logit_residuals = target_embedding.encode_readout_sparse_raw(
&query,
batch_candidate_ids,
batch_node_count,