A high-performance inference engine for AI models
webgpu stuff (#1055)
uuuvnde3a3418e211
diff --git a/Cargo.lock b/Cargo.lockindex 43bdcb31c..0f27b88b8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -6171,12 +6171,6 @@ dependencies = ["windows-sys 0.61.2",]-[[package]]-name = "pollster"-version = "1.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bc6355899e1c9462875b6757c79f3caa011a1fdae12bbb1a2e72dd1f234f8336"-[[package]]name = "portable-atomic"version = "1.15.0"@@ -10087,7 +10081,6 @@ dependencies = ["objc2","objc2-foundation","parking_lot",- "pollster","prettyplease 0.3.0","proc-macro2","proptest",diff --git a/Cargo.toml b/Cargo.tomlindex 0e03875fb..e1f8f9f55 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -119,7 +119,6 @@ futures = "0.3.34"tokio = "1.53.3"tokio-stream = { version = "0.1.19", features = ["sync"] }tokio-util = { version = "0.7.20", default-features = false, features = ["rt"] }-pollster = "1.0.1"blake3 = "1.8.7"quote = "1.0.47"proc-macro2 = "1.0.107"diff --git a/crates/uzu-engine/Cargo.toml b/crates/uzu-engine/Cargo.tomlindex d3281e81b..54a229daf 100644--- a/crates/uzu-engine/Cargo.toml+++ b/crates/uzu-engine/Cargo.toml@@ -37,7 +37,6 @@ webgpu = ["dep:regex","dep:naga","dep:wgpu",- "dep:pollster",]cpu = [] # TODO: move libm here@@ -88,7 +87,6 @@ futures.workspace = trueitertools.workspace = trueparking_lot.workspace = truewgpu = { workspace = true, optional = true }-pollster = { workspace = true, optional = true }[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "visionos"))'.dependencies]objc2 = { workspace = true, optional = true }diff --git a/crates/uzu-engine/src/backends/slang/kernel/activation/activation.slang b/crates/uzu-engine/src/backends/slang/kernel/activation/activation.slangindex 25eb0f8c3..7c2d74686 100644--- a/crates/uzu-engine/src/backends/slang/kernel/activation/activation.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/activation/activation.slang@@ -6,10 +6,10 @@ import generated.activation_type;[[Public]] [[Kernel]] func Activation<T>([[Optional("!in_place")]] StructuredBuffer<T> input,RWStructuredBuffer<T> output,- uint n,+ uint32_t n,ActivationType act_type,[[Specialize]] bool in_place,- [[Axis("n", "256")]] uint tid,+ [[Axis("n", "256")]] uint32_t tid,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slangindex f1f147bc7..b45a250b8 100644--- a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/activation_transform.slang@@ -6,24 +6,24 @@ import generated.activation_transform;[[Variants("BiasT", "float, BFloat16")]][[Public]] [[Kernel]] func ActivationTransform<T, BiasT>([[Optional("!in_place")]] StructuredBuffer<T> input,- [[Optional("(ops == ActivationTransformOp::InputRht || ops == ActivationTransformOp::OutputRht)")]] RWStructuredBuffer<T> fp_out,+ [[Optional("!(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums) && !(ops == ActivationTransformOp::PackHalf)")]] RWStructuredBuffer<T> fp_out,[[Optional("has_bias")]] StructuredBuffer<BiasT> bias,[[Optional("(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int8_t> q_out,[[Optional("(ops == ActivationTransformOp::Quantize || ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<float> scales_out,- [[Optional("(ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int> group_sums_out,- StructuredBuffer<int> rht_factors,- uint batch_size,- uint element_count,- [[Optional("(ops == ActivationTransformOp::PackHalf)")]] RWStructuredBuffer<uint> packed_half_out,+ [[Optional("(ops == ActivationTransformOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int32_t> group_sums_out,+ StructuredBuffer<int32_t> rht_factors,+ uint32_t batch_size,+ uint32_t element_count,+ [[Optional("(ops == ActivationTransformOp::PackHalf)")]] RWStructuredBuffer<uint32_t> packed_half_out,[[Specialize]] ActivationTransformOp ops,[[Specialize]] bool grouped_by_weight_nibble,[[Specialize]] bool in_place,- [[Specialize]] uint activation_scale_group_size,- [[Specialize]] uint sum_group_size,+ [[Specialize]] uint32_t activation_scale_group_size,+ [[Specialize]] uint32_t sum_group_size,[[Specialize]] bool has_bias,- [[Groups("element_count.div_ceil(4* 32* 4)")]] uint activation_tile_index,- [[Groups("batch_size")]] uint batch_index,- [[Threads("4* 32")]] uint thread_index,+ [[Groups("element_count.div_ceil(4* 32* 4)")]] uint32_t activation_tile_index,+ [[Groups("batch_size")]] uint32_t batch_index,+ [[Threads("4* 32")]] uint32_t thread_index,ThreadContext thread_context,) {// TODOdiff --git a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/trellis.slang b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/trellis.slangindex 1c7499f65..9dc601a85 100644--- a/crates/uzu-engine/src/backends/slang/kernel/activation_transform/trellis.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/activation_transform/trellis.slang@@ -8,10 +8,10 @@ import definitions;RWStructuredBuffer<int8_t> activations,RWStructuredBuffer<vector<float, 4>> column_group_sums,RWStructuredBuffer<float> scales,- uint batch,- [[Specialize]] uint dimension,- [[Groups("batch")]] uint token,- [[Threads("512")]] uint thread_index,+ uint32_t batch,+ [[Specialize]] uint32_t dimension,+ [[Groups("batch")]] uint32_t token,+ [[Threads("512")]] uint32_t thread_index,ThreadContext thread_context,) {// TODOdiff --git a/crates/uzu-engine/src/backends/slang/kernel/attention/ancestor_attention.slang b/crates/uzu-engine/src/backends/slang/kernel/attention/ancestor_attention.slangindex eee29c2a4..e6ffafab0 100644--- a/crates/uzu-engine/src/backends/slang/kernel/attention/ancestor_attention.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/attention/ancestor_attention.slang@@ -2,27 +2,27 @@import definitions;[[Variants("HEAD_DIM", "128")]]-[[Public]] [[Kernel]] func AncestorAttention<let HEAD_DIM: uint>(+[[Public]] [[Kernel]] func AncestorAttention<let HEAD_DIM: uint32_t>(StructuredBuffer<BFloat16> prefix_kv,RWStructuredBuffer<BFloat16> node_kv,StructuredBuffer<BFloat16> current_qkv,StructuredBuffer<float> cosines,StructuredBuffer<float> sines,- StructuredBuffer<uint> node_metadata,- StructuredBuffer<uint> ancestor_indices,- StructuredBuffer<uint> ancestor_counts,- StructuredBuffer<uint> node_indices,+ StructuredBuffer<uint32_t> node_metadata,+ StructuredBuffer<uint32_t> ancestor_indices,+ StructuredBuffer<uint32_t> ancestor_counts,+ StructuredBuffer<uint32_t> node_indices,RWStructuredBuffer<BFloat16> output,- uint rows,- uint prefix_length,- uint ancestor_stride,- uint node_capacity,- uint max_depth,+ uint32_t rows,+ uint32_t prefix_length,+ uint32_t ancestor_stride,+ uint32_t node_capacity,+ uint32_t max_depth,float scale,- [[Specialize]] uint num_heads,+ [[Specialize]] uint32_t num_heads,ThreadContext thread_context,- [[Groups("(rows * num_heads).div_ceil(4)")]] uint group,- [[Threads("(4 * 32)")]] uint lid,+ [[Groups("(rows * num_heads).div_ceil(4)")]] uint32_t group,+ [[Threads("(4 * 32)")]] uint32_t lid,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/attention/attention_prepare.slang b/crates/uzu-engine/src/backends/slang/kernel/attention/attention_prepare.slangindex 87495c4ae..8f72ac54a 100644--- a/crates/uzu-engine/src/backends/slang/kernel/attention/attention_prepare.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/attention/attention_prepare.slang@@ -10,18 +10,18 @@ import definitions;[[Optional("has_kv")]] RWStructuredBuffer<ElementT> values,[[Optional("has_rope")]] StructuredBuffer<RopeT> cosines,[[Optional("has_rope")]] StructuredBuffer<RopeT> sines,- uint num_q_heads,- [[Optional("has_kv")]] uint num_kv_heads,- uint head_dim,- [[Optional("has_rope")]] uint rope_dim,- [[Optional("has_kv")]] uint kv_token_offset,- uint input_row_stride,- uint batch_dim,+ uint32_t num_q_heads,+ [[Optional("has_kv")]] uint32_t num_kv_heads,+ uint32_t head_dim,+ [[Optional("has_rope")]] uint32_t rope_dim,+ [[Optional("has_kv")]] uint32_t kv_token_offset,+ uint32_t input_row_stride,+ uint32_t batch_dim,[[Specialize]] bool has_kv,[[Specialize]] bool has_rope,- [[Axis("head_dim", "128")]] uint head_dim_idx,- [[Axis("num_q_heads + num_kv_heads.unwrap_or(0) * 2", "1")]] uint head_idx,- [[Axis("batch_dim", "1")]] uint batch_idx,+ [[Axis("head_dim", "128")]] uint32_t head_dim_idx,+ [[Axis("num_q_heads + num_kv_heads.unwrap_or(0) * 2", "1")]] uint32_t head_idx,+ [[Axis("batch_dim", "1")]] uint32_t batch_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/attention/attention_single_pass.slang b/crates/uzu-engine/src/backends/slang/kernel/attention/attention_single_pass.slangindex be760d281..c1a89f702 100644--- a/crates/uzu-engine/src/backends/slang/kernel/attention/attention_single_pass.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/attention/attention_single_pass.slang@@ -5,33 +5,33 @@ import generated.trie;[[Variants("T", "BFloat16")]][[Variants("HEAD_DIM", "64, 128, 256, 512")]]-[[Public]] [[Kernel]] func AttentionSinglePass<T, let HEAD_DIM: uint>(+[[Public]] [[Kernel]] func AttentionSinglePass<T, let HEAD_DIM: uint32_t>(StructuredBuffer<T> queries,StructuredBuffer<T> keys,StructuredBuffer<T> values,RWStructuredBuffer<T> out,- uint gqa_factor,- uint sequence_length,- uint k_head_stride,- uint k_seq_stride,- uint v_head_stride,- uint v_seq_stride,+ uint32_t gqa_factor,+ uint32_t sequence_length,+ uint32_t k_head_stride,+ uint32_t k_seq_stride,+ uint32_t v_head_stride,+ uint32_t v_seq_stride,[[Optional("is_kv_cache_ring")]] RingParams ring_params,float scale,[[Optional("is_trie")]] StructuredBuffer<TrieNode> trie,- [[Optional("is_sliding_window")]] uint sliding_window_size,+ [[Optional("is_sliding_window")]] uint32_t sliding_window_size,[[Optional("has_sinks")]] StructuredBuffer<T> sinks,- uint num_heads,- uint suffix_length,+ uint32_t num_heads,+ uint32_t suffix_length,[[Specialize]] bool has_sinks,[[Specialize]] bool is_kv_cache_ring,[[Specialize]] bool is_causal,[[Specialize]] bool is_trie,[[Specialize]] bool is_sliding_window,ThreadContext thread_context,- [[Groups("num_heads")]] uint head_idx,- [[Groups("suffix_length")]] uint q_seq_idx,- [[Threads("1024")]] uint tid,+ [[Groups("num_heads")]] uint32_t head_idx,+ [[Groups("suffix_length")]] uint32_t q_seq_idx,+ [[Threads("1024")]] uint32_t tid,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/attention/kv_cache_update.slang b/crates/uzu-engine/src/backends/slang/kernel/attention/kv_cache_update.slangindex 08f70936e..e4f4585f2 100644--- a/crates/uzu-engine/src/backends/slang/kernel/attention/kv_cache_update.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/attention/kv_cache_update.slang@@ -7,9 +7,9 @@ import generated.kv_cache_update;RWStructuredBuffer<T> in_place_keys,RWStructuredBuffer<T> in_place_values,[[Constant]] StructuredBuffer<Copy> copies,- uint copy_count,- uint element_dim,- [[Axis("element_dim", "256")]] uint element_idx,+ uint32_t copy_count,+ uint32_t element_dim,+ [[Axis("element_dim", "256")]] uint32_t element_idx,) {// TODO}@@ -20,10 +20,10 @@ import generated.kv_cache_update;RWStructuredBuffer<BFloat16> in_place_k_scales,RWStructuredBuffer<BFloat16> in_place_v_scales,[[Constant]] StructuredBuffer<Copy> copies,- uint copy_count,- [[Specialize]] uint head_dim,- [[Specialize]] uint num_kv_heads,- [[Axis("num_kv_heads* head_dim", "256")]] uint element,+ uint32_t copy_count,+ [[Specialize]] uint32_t head_dim,+ [[Specialize]] uint32_t num_kv_heads,+ [[Axis("num_kv_heads* head_dim", "256")]] uint32_t element,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/attention/prepare_int8.slang b/crates/uzu-engine/src/backends/slang/kernel/attention/prepare_int8.slangindex f51596adf..7eb8f8626 100644--- a/crates/uzu-engine/src/backends/slang/kernel/attention/prepare_int8.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/attention/prepare_int8.slang@@ -4,7 +4,7 @@ import definitions;[[Variants("ElementT", "BFloat16")]][[Variants("RopeT", "float")]][[Variants("HEAD_DIM", "128, 256")]]-[[Public]] [[Kernel]] func AttentionPrepareInt8<ElementT, RopeT, let HEAD_DIM: uint>(+[[Public]] [[Kernel]] func AttentionPrepareInt8<ElementT, RopeT, let HEAD_DIM: uint32_t>(StructuredBuffer<ElementT> qkvg,[[Optional("!quantize_queries")]] RWStructuredBuffer<ElementT> queries,[[Optional("quantize_queries")]] RWStructuredBuffer<int8_t> q_codes,@@ -15,19 +15,19 @@ import definitions;[[Optional("append_kv")]] RWStructuredBuffer<ElementT> v_scales,[[Optional("has_rope")]] StructuredBuffer<RopeT> cosines,[[Optional("has_rope")]] StructuredBuffer<RopeT> sines,- uint num_q_heads,- [[Optional("has_rope")]] uint rope_dim,- uint kv_token_offset,- uint input_row_stride,- uint suffix_length,- uint query_rows,+ uint32_t num_q_heads,+ [[Optional("has_rope")]] uint32_t rope_dim,+ uint32_t kv_token_offset,+ uint32_t input_row_stride,+ uint32_t suffix_length,+ uint32_t query_rows,[[Specialize]] bool has_rope,[[Specialize]] bool quantize_queries,- [[Specialize]] uint num_kv_heads,+ [[Specialize]] uint32_t num_kv_heads,[[Specialize]] bool append_kv,- [[Groups("suffix_length")]] uint token_idx,- [[Groups("num_q_heads + num_kv_heads * 2")]] uint head_idx,- [[Threads("32")]] uint lane,+ [[Groups("suffix_length")]] uint32_t token_idx,+ [[Groups("num_q_heads + num_kv_heads * 2")]] uint32_t head_idx,+ [[Threads("32")]] uint32_t lane,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/attention/qkv_norm.slang b/crates/uzu-engine/src/backends/slang/kernel/attention/qkv_norm.slangindex 77cba1ab9..92bd585ce 100644--- a/crates/uzu-engine/src/backends/slang/kernel/attention/qkv_norm.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/attention/qkv_norm.slang@@ -9,17 +9,17 @@ import definitions;[[Optional("!in_place")]] StructuredBuffer<InputT> qkvg_input,[[Optional("has_scales")]] StructuredBuffer<ScaleT> scales,RWStructuredBuffer<OutputT> qkvg_output,- uint batch_size,- uint input_row_stride,- uint head_dim,+ uint32_t batch_size,+ uint32_t input_row_stride,+ uint32_t head_dim,float epsilon,float scale_offset,- uint head_offset,- uint head_count,+ uint32_t head_offset,+ uint32_t head_count,bool full_layer,- [[Groups("batch_size")]] uint batch_idx,- [[Groups("head_count")]] uint head_idx,- [[Threads("32")]] uint lane_id,+ [[Groups("batch_size")]] uint32_t batch_idx,+ [[Groups("head_count")]] uint32_t head_idx,+ [[Threads("32")]] uint32_t lane_id,[[Specialize]] bool in_place,[[Specialize]] bool has_scales,) {diff --git a/crates/uzu-engine/src/backends/slang/kernel/attention/sigmoid_gate.slang b/crates/uzu-engine/src/backends/slang/kernel/attention/sigmoid_gate.slangindex 3cb02704b..4cdac646e 100644--- a/crates/uzu-engine/src/backends/slang/kernel/attention/sigmoid_gate.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/attention/sigmoid_gate.slang@@ -5,11 +5,11 @@ import definitions;[[Public]] [[Kernel]] func SigmoidGate<T>(StructuredBuffer<T> gate,RWStructuredBuffer<T> output,- uint gate_dim,- uint batch_dim,- uint gate_row_stride,- [[Axis("gate_dim", "256")]] uint gate_idx,- [[Axis("batch_dim", "1")]] uint batch_idx,+ uint32_t gate_dim,+ uint32_t batch_dim,+ uint32_t gate_row_stride,+ [[Axis("gate_dim", "256")]] uint32_t gate_idx,+ [[Axis("batch_dim", "1")]] uint32_t batch_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/convolution/separable_causal_conv.slang b/crates/uzu-engine/src/backends/slang/kernel/convolution/separable_causal_conv.slangindex 43062ff42..0bd16de3f 100644--- a/crates/uzu-engine/src/backends/slang/kernel/convolution/separable_causal_conv.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/convolution/separable_causal_conv.slang@@ -8,14 +8,14 @@ import definitions;StructuredBuffer<T> weights,[[Optional("has_bias")]] StructuredBuffer<T> bias,RWStructuredBuffer<T> output,- uint sequence_length,- uint coefficient_row_stride,- [[Specialize]] uint model_dim,- [[Specialize]] uint kernel_size,- [[Specialize]] uint group_size,+ uint32_t sequence_length,+ uint32_t coefficient_row_stride,+ [[Specialize]] uint32_t model_dim,+ [[Specialize]] uint32_t kernel_size,+ [[Specialize]] uint32_t group_size,[[Specialize]] bool has_bias,- [[Axis("sequence_length", "1")]] uint token,- [[Axis("model_dim.div_ceil(4)", "128")]] uint channel_block_index,+ [[Axis("sequence_length", "1")]] uint32_t token,+ [[Axis("model_dim.div_ceil(4)", "128")]] uint32_t channel_block_index,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/embedding/input_embedding_lookup.slang b/crates/uzu-engine/src/backends/slang/kernel/embedding/input_embedding_lookup.slangindex b830f3fd7..476169e14 100644--- a/crates/uzu-engine/src/backends/slang/kernel/embedding/input_embedding_lookup.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/embedding/input_embedding_lookup.slang@@ -6,27 +6,27 @@ import generated.quantization_method;[[Variants("T", "float, BFloat16")]][[Public]] [[Kernel]] func InputEmbeddingLookup<T>(- StructuredBuffer<uint> token_ids,+ StructuredBuffer<uint32_t> token_ids,StructuredBuffer<uint8_t> values,[[Optional("table_kind != EmbeddingTableKind::Dense")]] StructuredBuffer<T> scales,[[Optional("table_kind == EmbeddingTableKind::Quantized && quantization_method == QuantizationMethod::ScaleZeroPoint")]] StructuredBuffer<uint8_t> zero_points,[[Optional("table_kind == EmbeddingTableKind::Quantized && quantization_method == QuantizationMethod::ScaleBias")]] StructuredBuffer<T> biases,- [[Optional("use_hadamard")]] StructuredBuffer<int> hadamard_factors,+ [[Optional("use_hadamard")]] StructuredBuffer<int32_t> hadamard_factors,[[Optional("table_kind == EmbeddingTableKind::D4S4")]] StructuredBuffer<uint8_t> ladder_indices,[[Optional("table_kind == EmbeddingTableKind::D4S4")]] StructuredBuffer<half> ladder,[[Optional("table_kind == EmbeddingTableKind::D4S4")]] StructuredBuffer<vector<int8_t, 4>> codebook,RWStructuredBuffer<T> output,- uint batch_size,- uint vocab_size,- uint model_dim,+ uint32_t batch_size,+ uint32_t vocab_size,+ uint32_t model_dim,float input_scale,[[Specialize]] EmbeddingTableKind table_kind,- [[Specialize]] [[Optional("table_kind == EmbeddingTableKind::Quantized")]] uint group_size,+ [[Specialize]] [[Optional("table_kind == EmbeddingTableKind::Quantized")]] uint32_t group_size,[[Specialize]] [[Optional("table_kind == EmbeddingTableKind::Quantized")]] QuantizationMode quantization_mode,[[Specialize]] [[Optional("table_kind == EmbeddingTableKind::Quantized")]] QuantizationMethod quantization_method,[[Specialize]] bool use_hadamard,- [[Axis("model_dim", "256")]] uint dim_idx,- [[Axis("batch_size", "1")]] uint batch_idx,+ [[Axis("model_dim", "256")]] uint32_t dim_idx,+ [[Axis("batch_size", "1")]] uint32_t batch_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang b/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slangindex 3f12193ef..afc64b6ee 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gated_act_mul/gated_act_mul.slang@@ -7,34 +7,34 @@ import generated.gated_act_mul;[[Public]] [[Kernel]] func GatedActMul<T>(StructuredBuffer<T> act_operand,[[Optional("!interleaved")]] StructuredBuffer<T> value_operand,- [[Optional("(ops == GatedActMulOp::FullPrecision)")]] RWStructuredBuffer<T> fp_out,+ [[Optional("!(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums) && !(ops == GatedActMulOp::PackHalf)")]] RWStructuredBuffer<T> fp_out,[[Optional("(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int8_t> q_out,[[Optional("(ops == GatedActMulOp::Quantize || ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<float> scales_out,- [[Optional("(ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int> group_sums_out,- [[Optional("use_hadamard")]] StructuredBuffer<int> hadamard_factors,- uint gated_dim,- uint batch_dim,- uint value_offset,- uint value_row_stride,+ [[Optional("(ops == GatedActMulOp::QuantizeWithGroupSums)")]] RWStructuredBuffer<int32_t> group_sums_out,+ [[Optional("use_hadamard")]] StructuredBuffer<int32_t> hadamard_factors,+ uint32_t gated_dim,+ uint32_t batch_dim,+ uint32_t value_offset,+ uint32_t value_row_stride,ActivationType act_type,[[Optional("custom_activation_alpha")]] float activation_alpha,[[Optional("clip_gate")]] float gate_clip_min,[[Optional("clip_gate")]] float gate_clip_max,[[Optional("clip_value")]] float value_clip_min,[[Optional("clip_value")]] float value_clip_max,- [[Optional("(ops == GatedActMulOp::PackHalf)")]] RWStructuredBuffer<uint> packed_half_out,+ [[Optional("(ops == GatedActMulOp::PackHalf)")]] RWStructuredBuffer<uint32_t> packed_half_out,[[Specialize]] GatedActMulOp ops,[[Specialize]] bool grouped_by_weight_nibble,[[Specialize]] bool interleaved,[[Specialize]] bool use_hadamard,- [[Specialize]] uint activation_scale_group_size,- [[Specialize]] uint sum_group_size,+ [[Specialize]] uint32_t activation_scale_group_size,+ [[Specialize]] uint32_t sum_group_size,[[Specialize]] bool custom_activation_alpha,[[Specialize]] bool clip_gate,[[Specialize]] bool clip_value,- [[Groups("gated_dim.div_ceil(4* 32* 4)")]] uint activation_tile_index,- [[Groups("batch_dim")]] uint batch_idx,- [[Threads("4* 32")]] uint thread_index,+ [[Groups("gated_dim.div_ceil(4* 32* 4)")]] uint32_t activation_tile_index,+ [[Groups("batch_dim")]] uint32_t batch_idx,+ [[Threads("4* 32")]] uint32_t thread_index,ThreadContext thread_context,) {// TODOdiff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_pack.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_pack.slangindex beacb6103..bb24bfe06 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_pack.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_pack.slang@@ -3,16 +3,16 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Variants("KERNEL_SIZE", "4")]]-[[Public]] [[Kernel]] func DeltaNetConvPack<T, let KERNEL_SIZE: uint>(+[[Public]] [[Kernel]] func DeltaNetConvPack<T, let KERNEL_SIZE: uint32_t>(StructuredBuffer<float> state_in,StructuredBuffer<T> x,RWStructuredBuffer<float> packed,- uint suffix_len,- uint tile_tokens,- uint row_stride,- uint conv_dim,- [[Axis("conv_dim", "128")]] uint channel_idx,- [[Axis("suffix_len.div_ceil(tile_tokens)", "1")]] uint tile_idx,+ uint32_t suffix_len,+ uint32_t tile_tokens,+ uint32_t row_stride,+ uint32_t conv_dim,+ [[Axis("conv_dim", "128")]] uint32_t channel_idx,+ [[Axis("suffix_len.div_ceil(tile_tokens)", "1")]] uint32_t tile_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_scan.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_scan.slangindex 3932e9071..caecf3b0f 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_scan.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_scan.slang@@ -3,19 +3,19 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Variants("KERNEL_SIZE", "4")]]-[[Public]] [[Kernel]] func DeltaNetConvScan<T, let KERNEL_SIZE: uint>(+[[Public]] [[Kernel]] func DeltaNetConvScan<T, let KERNEL_SIZE: uint32_t>(StructuredBuffer<float> packed,StructuredBuffer<float> conv_weight,[[Optional("has_bias")]] StructuredBuffer<float> bias,RWStructuredBuffer<T> in_proj,RWStructuredBuffer<float> state_out,- uint suffix_len,- uint tile_tokens,- uint conv_dim,- uint out_stride,+ uint32_t suffix_len,+ uint32_t tile_tokens,+ uint32_t conv_dim,+ uint32_t out_stride,[[Specialize]] bool has_bias,- [[Axis("conv_dim / 8", "32")]] uint channel_group,- [[Axis("suffix_len.div_ceil(tile_tokens)", "4")]] uint tile_idx,+ [[Axis("conv_dim / 8", "32")]] uint32_t channel_group,+ [[Axis("suffix_len.div_ceil(tile_tokens)", "4")]] uint32_t tile_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_update.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_update.slangindex 17cc7067c..e5158ddb5 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_update.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/conv_update.slang@@ -7,11 +7,11 @@ import definitions;[[Optional("has_bias")]] StructuredBuffer<float> bias,RWStructuredBuffer<T> in_out,RWStructuredBuffer<float> state,- uint kernel_size,- uint conv_dim,- uint state_stride,+ uint32_t kernel_size,+ uint32_t conv_dim,+ uint32_t state_stride,[[Specialize]] bool has_bias,- [[Axis("conv_dim", "256")]] uint channel_idx,+ [[Axis("conv_dim", "256")]] uint32_t channel_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/norm_gate.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/norm_gate.slangindex feb36ccfb..6d28c3046 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/norm_gate.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/norm_gate.slang@@ -6,17 +6,17 @@ import definitions;RWStructuredBuffer<T> in_out,StructuredBuffer<T> in_proj,StructuredBuffer<float> norm_weight,- uint num_v_heads,- uint head_v_dim,- uint value_dim,- uint conv_dim,- uint total_proj_dim,+ uint32_t num_v_heads,+ uint32_t head_v_dim,+ uint32_t value_dim,+ uint32_t conv_dim,+ uint32_t total_proj_dim,float norm_epsilon,- uint suffix_len,+ uint32_t suffix_len,ThreadContext thread_context,- [[Groups("suffix_len")]] uint token_idx,- [[Groups("num_v_heads")]] uint hv_idx,- [[Threads("128")]] uint tid,+ [[Groups("suffix_len")]] uint32_t token_idx,+ [[Groups("num_v_heads")]] uint32_t hv_idx,+ [[Threads("128")]] uint32_t tid,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill.slangindex 74ac76467..281e07651 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill.slang@@ -3,7 +3,7 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Variants("HEAD_K_DIM", "128")]]-[[Public]] [[Kernel]] func DeltaNetPrefill<T, let HEAD_K_DIM: uint>(+[[Public]] [[Kernel]] func DeltaNetPrefill<T, let HEAD_K_DIM: uint32_t>(StructuredBuffer<float> q_norm,StructuredBuffer<float> k_norm,StructuredBuffer<float> beta_buf,@@ -11,16 +11,16 @@ import definitions;StructuredBuffer<T> in_proj,RWStructuredBuffer<float> state,RWStructuredBuffer<T> out,- uint num_v_heads,- uint num_k_heads,- uint head_v_dim,- uint key_dim,- uint value_dim,- uint suffix_len,- uint num_dv_groups,- [[Groups("num_v_heads")]] uint hv_idx,- [[Groups("num_dv_groups")]] uint dv_group,- [[Threads("128")]] uint tid,+ uint32_t num_v_heads,+ uint32_t num_k_heads,+ uint32_t head_v_dim,+ uint32_t key_dim,+ uint32_t value_dim,+ uint32_t suffix_len,+ uint32_t num_dv_groups,+ [[Groups("num_v_heads")]] uint32_t hv_idx,+ [[Groups("num_dv_groups")]] uint32_t dv_group,+ [[Threads("128")]] uint32_t tid,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill_prep.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill_prep.slangindex 634cd6ccf..ac31a1054 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill_prep.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/prefill_prep.slang@@ -4,7 +4,7 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Variants("QKT", "float, BFloat16")]][[Variants("HEAD_K_DIM", "128")]]-[[Public]] [[Kernel]] func DeltaNetPrefillPrep<T, QKT, let HEAD_K_DIM: uint>(+[[Public]] [[Kernel]] func DeltaNetPrefillPrep<T, QKT, let HEAD_K_DIM: uint32_t>(StructuredBuffer<T> in_proj,StructuredBuffer<float> a_log,StructuredBuffer<float> dt_bias,@@ -13,16 +13,16 @@ import definitions;[[Optional("write_compact_v")]] RWStructuredBuffer<T> compact_v_out,RWStructuredBuffer<float> beta_out,RWStructuredBuffer<float> decay_out,- uint num_v_heads,- uint num_k_heads,- uint key_dim,- uint value_dim,- uint suffix_len,- [[Groups("suffix_len")]] uint token_idx,- [[Groups("num_k_heads")]] uint hk_idx,+ uint32_t num_v_heads,+ uint32_t num_k_heads,+ uint32_t key_dim,+ uint32_t value_dim,+ uint32_t suffix_len,+ [[Groups("suffix_len")]] uint32_t token_idx,+ [[Groups("num_k_heads")]] uint32_t hk_idx,[[Specialize]] bool write_log_decay,[[Specialize]] bool write_compact_v,- [[Threads("32")]] uint lane,+ [[Threads("32")]] uint32_t lane,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/conv_scan.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/conv_scan.slangindex ff4ac6727..c0aa07c2a 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/conv_scan.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/conv_scan.slang@@ -7,16 +7,16 @@ import definitions;StructuredBuffer<float> conv_weight,[[Optional("has_bias")]] StructuredBuffer<float> bias,StructuredBuffer<float> base_state,- StructuredBuffer<int> parents,+ StructuredBuffer<int32_t> parents,RWStructuredBuffer<T> out_proj,RWStructuredBuffer<float> suffix_state,- uint suffix_len,- [[Specialize]] uint kernel_size,- uint total_proj_dim,- uint conv_dim,+ uint32_t suffix_len,+ [[Specialize]] uint32_t kernel_size,+ uint32_t total_proj_dim,+ uint32_t conv_dim,[[Specialize]] bool has_bias,- [[Axis("suffix_len", "1")]] uint node_idx,- [[Axis("total_proj_dim", "256")]] uint channel_idx,+ [[Axis("suffix_len", "1")]] uint32_t node_idx,+ [[Axis("total_proj_dim", "256")]] uint32_t channel_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/state_advance.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/state_advance.slangindex bf855e2e8..4305fd35d 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/state_advance.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/state_advance.slang@@ -3,20 +3,20 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Variants("HEAD_K_DIM", "128")]]-[[Public]] [[Kernel]] func StateAdvance<T, let HEAD_K_DIM: uint>(+[[Public]] [[Kernel]] func StateAdvance<T, let HEAD_K_DIM: uint32_t>(StructuredBuffer<T> k_norm,StructuredBuffer<T> v,StructuredBuffer<float> log_decay_buf,StructuredBuffer<float> beta_buf,- StructuredBuffer<uint> accepted_indices,+ StructuredBuffer<uint32_t> accepted_indices,RWStructuredBuffer<float> state,- uint accepted_len,- [[Specialize]] uint num_v_heads,- [[Specialize]] uint num_k_heads,+ uint32_t accepted_len,+ [[Specialize]] uint32_t num_v_heads,+ [[Specialize]] uint32_t num_k_heads,ThreadContext thread_context,- [[Groups("num_v_heads")]] uint hv_idx,- [[Groups("8")]] uint dv_group_idx,- [[Threads("128")]] uint thread_idx,+ [[Groups("num_v_heads")]] uint32_t hv_idx,+ [[Groups("8")]] uint32_t dv_group_idx,+ [[Threads("128")]] uint32_t thread_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/tree_verify.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/tree_verify.slangindex 5e5a0724d..937439ee9 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/tree_verify.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/tree_verify/tree_verify.slang@@ -4,7 +4,7 @@ import generated.trie;[[Variants("T", "float, BFloat16")]][[Variants("NB", "1, 2")]]-[[Public]] [[Kernel]] func TreeVerify<T, let NB: uint>(+[[Public]] [[Kernel]] func TreeVerify<T, let NB: uint32_t>(StructuredBuffer<T> q,StructuredBuffer<T> k,StructuredBuffer<T> v,@@ -17,18 +17,18 @@ import generated.trie;RWStructuredBuffer<T> o,float scale,float norm_epsilon,- uint batch_size,- uint tree_size,- uint num_k_heads,- uint num_v_heads,- uint head_k_dim,- uint head_v_dim,- uint conv_dim,- uint total_proj_dim,+ uint32_t batch_size,+ uint32_t tree_size,+ uint32_t num_k_heads,+ uint32_t num_v_heads,+ uint32_t head_k_dim,+ uint32_t head_v_dim,+ uint32_t conv_dim,+ uint32_t total_proj_dim,ThreadContext thread_context,- [[Groups("batch_size")]] uint batch_idx,- [[Groups("num_v_heads")]] uint hv_idx,- [[Threads("(8 + NB) * 32")]] uint thread_idx,+ [[Groups("batch_size")]] uint32_t batch_idx,+ [[Groups("num_v_heads")]] uint32_t hv_idx,+ [[Threads("(8 + NB) * 32")]] uint32_t thread_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/gdn/update.slang b/crates/uzu-engine/src/backends/slang/kernel/gdn/update.slangindex 068d0fd3a..4b4284314 100644--- a/crates/uzu-engine/src/backends/slang/kernel/gdn/update.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/gdn/update.slang@@ -3,22 +3,22 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Variants("HEAD_K_DIM", "128")]]-[[Public]] [[Kernel]] func DeltaNetUpdate<T, let HEAD_K_DIM: uint>(+[[Public]] [[Kernel]] func DeltaNetUpdate<T, let HEAD_K_DIM: uint32_t>(StructuredBuffer<T> in_proj,StructuredBuffer<float> a_log,StructuredBuffer<float> dt_bias,StructuredBuffer<float> norm_weight,RWStructuredBuffer<float> state,RWStructuredBuffer<T> out,- uint num_v_heads,- uint num_k_heads,- uint head_v_dim,- uint key_dim,- uint value_dim,+ uint32_t num_v_heads,+ uint32_t num_k_heads,+ uint32_t head_v_dim,+ uint32_t key_dim,+ uint32_t value_dim,float norm_epsilon,ThreadContext thread_context,- [[Groups("num_v_heads")]] uint hv_idx,- [[Threads("512")]] uint tid,+ [[Groups("num_v_heads")]] uint32_t hv_idx,+ [[Threads("512")]] uint32_t tid,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/logit_transform/logit_transform.slang b/crates/uzu-engine/src/backends/slang/kernel/logit_transform/logit_transform.slangindex 777afd865..1f603fb10 100644--- a/crates/uzu-engine/src/backends/slang/kernel/logit_transform/logit_transform.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/logit_transform/logit_transform.slang@@ -4,10 +4,10 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Public]] [[Kernel]] func LogitTransform<T>(RWStructuredBuffer<T> logits,- uint length,+ uint32_t length,float scale,float soft_cap,- [[Axis("length", "256")]] uint position,+ [[Axis("length", "256")]] uint32_t position,[[Specialize]] bool has_soft_cap,) {// TODOdiff --git a/crates/uzu-engine/src/backends/slang/kernel/moe/combine.slang b/crates/uzu-engine/src/backends/slang/kernel/moe/combine.slangindex 297f0dcfd..eb3c79dff 100644--- a/crates/uzu-engine/src/backends/slang/kernel/moe/combine.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/moe/combine.slang@@ -3,17 +3,17 @@ import definitions;[[Variants("T", "BFloat16")]][[Variants("BD", "256")]]-[[Public]] [[Kernel]] func MoeCombine<T, let BD: uint>(+[[Public]] [[Kernel]] func MoeCombine<T, let BD: uint32_t>(StructuredBuffer<T> expert_outputs,StructuredBuffer<float> expert_weights,StructuredBuffer<T> shared_output,StructuredBuffer<T> shared_gate,RWStructuredBuffer<T> output,- uint model_dim,- uint top_k,- uint batch_size,- [[Axis("model_dim", "BD")]] uint dim_idx,- [[Axis("batch_size", "1")]] uint token_idx,+ uint32_t model_dim,+ uint32_t top_k,+ uint32_t batch_size,+ [[Axis("model_dim", "BD")]] uint32_t dim_idx,+ [[Axis("batch_size", "1")]] uint32_t token_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/moe/gemv.slang b/crates/uzu-engine/src/backends/slang/kernel/moe/gemv.slangindex d625df6df..60dbcf5d9 100644--- a/crates/uzu-engine/src/backends/slang/kernel/moe/gemv.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/moe/gemv.slang@@ -4,31 +4,31 @@ import definitions;[[Variants("T", "BFloat16")]][[Variants("GROUP_SIZE", "64")]][[Variants("BITS", "4, 8")]]-[[Public]] [[Kernel]] func MoeGemv<T, let GROUP_SIZE: uint, let BITS: uint>(+[[Public]] [[Kernel]] func MoeGemv<T, let GROUP_SIZE: uint32_t, let BITS: uint32_t>(StructuredBuffer<uint8_t> b,StructuredBuffer<T> scales,[[Optional("BITS == 4")]] StructuredBuffer<uint8_t> zero_points,StructuredBuffer<T> a,- StructuredBuffer<uint> expert_ids,+ StructuredBuffer<uint32_t> expert_ids,RWStructuredBuffer<T> d,StructuredBuffer<int32_t> hadamard_factors,- uint in_vec_size,- uint out_vec_size,- uint num_routes,- uint active_experts_per_token,- uint group_count_x,- uint b_expert_stride_bytes,- uint scale_expert_stride,- uint zero_point_expert_stride,- uint scale_output_stride,- uint scale_group_stride,- uint zero_point_output_stride,- uint zero_point_group_stride,+ uint32_t in_vec_size,+ uint32_t out_vec_size,+ uint32_t num_routes,+ uint32_t active_experts_per_token,+ uint32_t group_count_x,+ uint32_t b_expert_stride_bytes,+ uint32_t scale_expert_stride,+ uint32_t zero_point_expert_stride,+ uint32_t scale_output_stride,+ uint32_t scale_group_stride,+ uint32_t zero_point_output_stride,+ uint32_t zero_point_group_stride,[[Specialize]] bool a_per_token,- [[Groups("num_routes")]] uint route_idx,- [[Groups("group_count_x")]] uint output_tile_idx,- [[Threads("32")]] uint simd_lane,- [[Threads("8")]] uint simd_group,+ [[Groups("num_routes")]] uint32_t route_idx,+ [[Groups("group_count_x")]] uint32_t output_tile_idx,+ [[Threads("32")]] uint32_t simd_lane,+ [[Threads("8")]] uint32_t simd_group,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/moe/router_top_k.slang b/crates/uzu-engine/src/backends/slang/kernel/moe/router_top_k.slangindex b41671c91..5ab53f897 100644--- a/crates/uzu-engine/src/backends/slang/kernel/moe/router_top_k.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/moe/router_top_k.slang@@ -3,15 +3,15 @@ import definitions;[[Variants("T", "float")]][[Variants("MAX_TOP_K", "8")]]-[[Public]] [[Kernel]] func MoeRouterTopK<T, let MAX_TOP_K: uint>(+[[Public]] [[Kernel]] func MoeRouterTopK<T, let MAX_TOP_K: uint32_t>(StructuredBuffer<T> logits,- RWStructuredBuffer<uint> expert_ids,+ RWStructuredBuffer<uint32_t> expert_ids,RWStructuredBuffer<float> expert_weights,- uint num_experts,- uint top_k,- uint batch_size,- [[Groups("batch_size")]] uint token_idx,- [[Threads("32")]] uint lane,+ uint32_t num_experts,+ uint32_t top_k,+ uint32_t batch_size,+ [[Groups("batch_size")]] uint32_t token_idx,+ [[Threads("32")]] uint32_t lane,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/normalization/normalization.slang b/crates/uzu-engine/src/backends/slang/kernel/normalization/normalization.slangindex d1f9261cb..0bcfc518f 100644--- a/crates/uzu-engine/src/backends/slang/kernel/normalization/normalization.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/normalization/normalization.slang@@ -11,9 +11,9 @@ import definitions;[[Optional("has_biases")]] StructuredBuffer<AffineT> biases,RWStructuredBuffer<OutputT> output,[[Optional("copy_to_shortcut")]] RWStructuredBuffer<InputT> shortcut,- [[Optional("use_hadamard")]] StructuredBuffer<int> hadamard_factors,- uint batch_size,- uint element_count,+ [[Optional("use_hadamard")]] StructuredBuffer<int32_t> hadamard_factors,+ uint32_t batch_size,+ uint32_t element_count,float epsilon,float scale_offset,float post_layer_scalar,@@ -28,8 +28,8 @@ import definitions;[[Specialize]] bool has_biases,[[Specialize]] bool has_scales,ThreadContext thread_context,- [[Groups("batch_size")]] uint batch_idx,- [[Threads("1024")]] uint thread_in_row,+ [[Groups("batch_size")]] uint32_t batch_idx,+ [[Threads("1024")]] uint32_t thread_in_row,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/pooling/pooling.slang b/crates/uzu-engine/src/backends/slang/kernel/pooling/pooling.slangindex 1cdbea91b..df337abed 100644--- a/crates/uzu-engine/src/backends/slang/kernel/pooling/pooling.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/pooling/pooling.slang@@ -5,11 +5,11 @@ import definitions;[[Public]] [[Kernel]] func PoolingMean<T>(StructuredBuffer<T> input,RWStructuredBuffer<T> output,- uint seq_len,- uint hidden_dim,- uint batch_size,- [[Axis("hidden_dim", "16")]] uint dim_idx,- [[Axis("batch_size", "16")]] uint batch_idx,+ uint32_t seq_len,+ uint32_t hidden_dim,+ uint32_t batch_size,+ [[Axis("hidden_dim", "16")]] uint32_t dim_idx,+ [[Axis("batch_size", "16")]] uint32_t batch_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/sampling/context_ring_update.slang b/crates/uzu-engine/src/backends/slang/kernel/sampling/context_ring_update.slangindex c12bbb84f..638264969 100644--- a/crates/uzu-engine/src/backends/slang/kernel/sampling/context_ring_update.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/sampling/context_ring_update.slang@@ -2,11 +2,11 @@import definitions;[[Public]] [[Kernel]] func ContextRingUpdate(- StructuredBuffer<uint> input,- RWStructuredBuffer<uint> context_ring,- uint suffix_repetition_length,- uint input_length,- [[Threads("256")]] uint thread_index,+ StructuredBuffer<uint32_t> input,+ RWStructuredBuffer<uint32_t> context_ring,+ uint32_t suffix_repetition_length,+ uint32_t input_length,+ [[Threads("256")]] uint32_t thread_index,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/sampling/repetition_penalty.slang b/crates/uzu-engine/src/backends/slang/kernel/sampling/repetition_penalty.slangindex 830b9d227..caa39a278 100644--- a/crates/uzu-engine/src/backends/slang/kernel/sampling/repetition_penalty.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/sampling/repetition_penalty.slang@@ -5,14 +5,14 @@ import definitions;[[Public]] [[Kernel]] func RepetitionPenalty<T>(StructuredBuffer<T> original_logits,RWStructuredBuffer<T> logits_copy,- StructuredBuffer<uint> context_ring,- StructuredBuffer<uint> token_ids,+ StructuredBuffer<uint32_t> context_ring,+ StructuredBuffer<uint32_t> token_ids,float repetition_penalty,- uint suffix_repetition_length,- uint vocab_size,- uint sampling_start,- uint sampling_length,- [[Axis("sampling_length * suffix_repetition_length", "256")]] uint position,+ uint32_t suffix_repetition_length,+ uint32_t vocab_size,+ uint32_t sampling_start,+ uint32_t sampling_length,+ [[Axis("sampling_length * suffix_repetition_length", "256")]] uint32_t position,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/sampling/unified_sampling.slang b/crates/uzu-engine/src/backends/slang/kernel/sampling/unified_sampling.slangindex 97fc9bb4c..471286c96 100644--- a/crates/uzu-engine/src/backends/slang/kernel/sampling/unified_sampling.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/sampling/unified_sampling.slang@@ -4,15 +4,15 @@ import definitions;[[Variants("T", "float, BFloat16")]][[Public]] [[Kernel]] func UnifiedSampling<T>(StructuredBuffer<T> logits,- RWStructuredBuffer<uint> output,+ RWStructuredBuffer<uint32_t> output,[[Optional("is_stochastic")]] StructuredBuffer<uint64_t> seeds,- [[Optional("has_bitmask")]] StructuredBuffer<uint> bitmask,+ [[Optional("has_bitmask")]] StructuredBuffer<uint32_t> bitmask,[[Optional("has_temperature")]] float temperature,- [[Optional("has_top_k")]] uint top_k,+ [[Optional("has_top_k")]] uint32_t top_k,[[Optional("has_top_p")]] float top_p,[[Optional("has_min_p")]] float min_p,- uint vocab_size,- uint batch_size,+ uint32_t vocab_size,+ uint32_t batch_size,[[Specialize]] bool is_stochastic,[[Specialize]] bool has_bitmask,[[Specialize]] bool has_temperature,@@ -20,8 +20,8 @@ import definitions;[[Specialize]] bool has_top_p,[[Specialize]] bool has_min_p,ThreadContext thread_context,- [[Groups("batch_size")]] uint batch_idx,- [[Threads("1024")]] uint thread_idx,+ [[Groups("batch_size")]] uint32_t batch_idx,+ [[Threads("1024")]] uint32_t thread_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/short_conv/short_conv.slang b/crates/uzu-engine/src/backends/slang/kernel/short_conv/short_conv.slangindex e3e0034f4..859da25d1 100644--- a/crates/uzu-engine/src/backends/slang/kernel/short_conv/short_conv.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/short_conv/short_conv.slang@@ -6,12 +6,12 @@ import definitions;StructuredBuffer<T> state_in,StructuredBuffer<T> in_proj,RWStructuredBuffer<T> padded,- uint state_stride,- uint suffix_len,- uint in_proj_stride,- uint model_dim,- [[Axis("model_dim", "256")]] uint channel_idx,- [[Axis("state_stride + suffix_len", "1")]] uint row_idx,+ uint32_t state_stride,+ uint32_t suffix_len,+ uint32_t in_proj_stride,+ uint32_t model_dim,+ [[Axis("model_dim", "256")]] uint32_t channel_idx,+ [[Axis("state_stride + suffix_len", "1")]] uint32_t row_idx,) {// TODO}@@ -25,14 +25,14 @@ import definitions;[[Optional("has_bias")]] StructuredBuffer<WeightT> b,RWStructuredBuffer<T> out,RWStructuredBuffer<T> state_out,- uint suffix_len,- uint kernel_size,- uint in_proj_stride,- uint state_stride,- uint model_dim,+ uint32_t suffix_len,+ uint32_t kernel_size,+ uint32_t in_proj_stride,+ uint32_t state_stride,+ uint32_t model_dim,[[Specialize]] bool has_bias,- [[Axis("suffix_len + kernel_size.saturating_sub(1)", "1")]] uint token_idx,- [[Axis("model_dim", "256")]] uint channel_idx,+ [[Axis("suffix_len + kernel_size.saturating_sub(1)", "1")]] uint32_t token_idx,+ [[Axis("model_dim", "256")]] uint32_t channel_idx,) {// TODO}@@ -46,15 +46,15 @@ import definitions;[[Optional("!state_in_place")]] StructuredBuffer<T> state,RWStructuredBuffer<T> out,RWStructuredBuffer<T> next_state,- uint suffix_len,- uint kernel_size,- uint in_proj_stride,- uint state_stride,- uint model_dim,+ uint32_t suffix_len,+ uint32_t kernel_size,+ uint32_t in_proj_stride,+ uint32_t state_stride,+ uint32_t model_dim,[[Specialize]] bool has_bias,[[Specialize]] bool state_in_place,- [[Axis("suffix_len", "1")]] uint token_idx,- [[Axis("model_dim", "256")]] uint channel_idx,+ [[Axis("suffix_len", "1")]] uint32_t token_idx,+ [[Axis("model_dim", "256")]] uint32_t channel_idx,) {// TODO}@@ -66,16 +66,16 @@ import definitions;StructuredBuffer<WeightT> w,[[Optional("has_bias")]] StructuredBuffer<WeightT> b,StructuredBuffer<T> base_state,- StructuredBuffer<int> parents,+ StructuredBuffer<int32_t> parents,RWStructuredBuffer<T> out,RWStructuredBuffer<T> suffix_state,- uint suffix_len,- uint kernel_size,- uint in_proj_stride,- uint state_stride,- uint model_dim,+ uint32_t suffix_len,+ uint32_t kernel_size,+ uint32_t in_proj_stride,+ uint32_t state_stride,+ uint32_t model_dim,[[Specialize]] bool has_bias,- [[Axis("model_dim", "256")]] uint channel_idx,+ [[Axis("model_dim", "256")]] uint32_t channel_idx,) {// TODO}diff --git a/crates/uzu-engine/src/backends/slang/kernel/tensor_add_bias/tensor_add_bias.slang b/crates/uzu-engine/src/backends/slang/kernel/tensor_add_bias/tensor_add_bias.slangindex a2fbd1aef..d28ba7902 100644--- a/crates/uzu-engine/src/backends/slang/kernel/tensor_add_bias/tensor_add_bias.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/tensor_add_bias/tensor_add_bias.slang@@ -7,9 +7,9 @@ import definitions;[[Optional("!in_place")]] StructuredBuffer<T> input,StructuredBuffer<BiasT> bias,RWStructuredBuffer<T> output,- uint num_cols,- uint length,- [[Axis("length", "32")]] uint position,+ uint32_t num_cols,+ uint32_t length,+ [[Axis("length", "32")]] uint32_t position,[[Specialize]] bool in_place,) {// TODOdiff --git a/crates/uzu-engine/src/backends/slang/kernel/tensor_add_scale/tensor_add_scale.slang b/crates/uzu-engine/src/backends/slang/kernel/tensor_add_scale/tensor_add_scale.slangindex 80971b51c..674140bb8 100644--- a/crates/uzu-engine/src/backends/slang/kernel/tensor_add_scale/tensor_add_scale.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/tensor_add_scale/tensor_add_scale.slang@@ -6,10 +6,10 @@ import definitions;[[Optional("!in_place")]] StructuredBuffer<T> input,StructuredBuffer<T> bias,RWStructuredBuffer<T> output,- uint num_cols,- uint length,+ uint32_t num_cols,+ uint32_t length,float scale,- [[Axis("length", "32")]] uint position,+ [[Axis("length", "32")]] uint32_t position,[[Specialize]] bool in_place,) {// TODOdiff --git a/crates/uzu-engine/src/backends/slang/kernel/weaver/weaver_selection.slang b/crates/uzu-engine/src/backends/slang/kernel/weaver/weaver_selection.slangindex 576454b3a..7c3c824fb 100644--- a/crates/uzu-engine/src/backends/slang/kernel/weaver/weaver_selection.slang+++ b/crates/uzu-engine/src/backends/slang/kernel/weaver/weaver_selection.slang@@ -3,38 +3,38 @@ import definitions;import generated.weaver;[[Variants("BLOCK_THREADS", "256, 1024")]]-[[Public]] [[Kernel]] func WeaverSelect<let BLOCK_THREADS: uint>(+[[Public]] [[Kernel]] func WeaverSelect<let BLOCK_THREADS: uint32_t>([[Optional("mode != WeaverSelectionMode::Frontier")]] StructuredBuffer<BFloat16> residual_logits,[[Optional("mode != WeaverSelectionMode::Frontier")]] StructuredBuffer<uint64_t> depth_seeds,- RWStructuredBuffer<uint> frontier,- RWStructuredBuffer<uint> packed_tree,- [[Optional("mode != WeaverSelectionMode::Children")]] RWStructuredBuffer<uint> slot_ancestors,- [[Optional("mode != WeaverSelectionMode::Children")]] RWStructuredBuffer<uint> node_token_ids,- RWStructuredBuffer<uint> node_metadata,- [[Optional("mode != WeaverSelectionMode::Children")]] RWStructuredBuffer<uint> node_ancestor_indices,- RWStructuredBuffer<uint> node_valid,- StructuredBuffer<uint> candidate_pool_ids,+ RWStructuredBuffer<uint32_t> frontier,+ RWStructuredBuffer<uint32_t> packed_tree,+ [[Optional("mode != WeaverSelectionMode::Children")]] RWStructuredBuffer<uint32_t> slot_ancestors,+ [[Optional("mode != WeaverSelectionMode::Children")]] RWStructuredBuffer<uint32_t> node_token_ids,+ RWStructuredBuffer<uint32_t> node_metadata,+ [[Optional("mode != WeaverSelectionMode::Children")]] RWStructuredBuffer<uint32_t> node_ancestor_indices,+ RWStructuredBuffer<uint32_t> node_valid,+ StructuredBuffer<uint32_t> candidate_pool_ids,StructuredBuffer<float> candidate_pool_logits,- RWStructuredBuffer<uint> node_candidate_ids,+ RWStructuredBuffer<uint32_t> node_candidate_ids,RWStructuredBuffer<float> node_candidate_logits,- uint children_rows,- uint children_from_pool,- uint candidates,- uint expand_width,- uint vocab_size,- uint frontier_capacity,- uint tree_slot_count,- uint node_count,- uint batch_start_slot,- uint ancestor_stride,- uint lookahead_count,- uint candidate_depth_count,+ uint32_t children_rows,+ uint32_t children_from_pool,+ uint32_t candidates,+ uint32_t expand_width,+ uint32_t vocab_size,+ uint32_t frontier_capacity,+ uint32_t tree_slot_count,+ uint32_t node_count,+ uint32_t batch_start_slot,+ uint32_t ancestor_stride,+ uint32_t lookahead_count,+ uint32_t candidate_depth_count,[[Optional("has_prune_noise")]] float prune_noise_scale,[[Specialize]] WeaverSelectionMode mode,[[Specialize]] bool has_prune_noise,ThreadContext global_context,- [[Groups("if mode == WeaverSelectionMode::Children { children_rows } else {1}")]] uint group_index,- [[Threads("BLOCK_THREADS")]] uint global_lid,+ [[Groups("if mode == WeaverSelectionMode::Children { children_rows } else {1}")]] uint32_t group_index,+ [[Threads("BLOCK_THREADS")]] uint32_t global_lid,) {// TODO}diff --git a/scripts/autogen_slang_stubs.py b/scripts/autogen_slang_stubs.pynew file mode 100755index 000000000..3bc919037--- /dev/null+++ b/scripts/autogen_slang_stubs.py@@ -0,0 +1,275 @@+#!/usr/bin/env python3++import json+import re+import subprocess+import sys+import tempfile+from pathlib import Path++ROOT = Path(__file__).resolve().parent.parent+METAL = ROOT / "crates/uzu-engine/src/backends/metal/kernel"+SLANG = ROOT / "crates/uzu-engine/src/backends/slang/kernel"+++def annotations(node):+ def strings(child):+ if child["kind"] == "StringLiteral":+ yield json.loads(child["value"])+ for nested in child.get("inner", []):+ yield from strings(nested)++ return [+ list(strings(child))+ for child in node.get("inner", [])+ if child["kind"] == "AnnotateAttr"+ ]+++def generate(path, cache):+ source = path.read_bytes()+ result = subprocess.run(+ [+ "xcrun",+ "-sdk",+ "macosx",+ "metal",+ "-x",+ "metal",+ "-std=metal4.0",+ "-mmacosx-version-min=26.4",+ f"-fmodules-cache-path={cache}",+ "-I",+ str(METAL / "generated"),+ "-DDSL_ANALYZE",+ str(path),+ "-fsyntax-only",+ "-Xclang",+ "-ast-dump=json",+ ],+ check=True,+ capture_output=True,+ text=True,+ )+ ast = json.loads(result.stdout)+ imports = set()+ defines = {}+ visited = set()++ def read_constants(header):+ header = header.resolve()+ if header in visited:+ return+ visited.add(header)+ text = header.read_text(encoding="utf-8")+ for include in re.findall(r'^\s*#include "([^"\n]+)"', text, re.M):+ dependency = header.parent / include+ if not dependency.exists():+ dependency = METAL / "generated" / include+ if dependency.exists():+ read_constants(dependency)+ defines.update(re.findall(r"^\s*#define[ \t]+(\w+)[ \t]+([^\n]+)", text, re.M))+ defines.update(+ re.findall(+ r"(?:static constant constexpr|UZU_CONST)\s+\w+\s+(\w+)\s*=\s*([^;]+);",+ text,+ )+ )++ read_constants(path)++ def expression(value):+ for _ in range(len(defines) + 1):+ expanded = re.sub(r"\b\w+\b", lambda m: defines.get(m[0], m[0]), value)+ if expanded == value:+ return re.sub(r"\b(\d+)[uU]\b", r"\1", value)+ value = expanded+ raise ValueError(f"{path}: recursive macro in {value}")++ def slang_type(ty, generics=()):+ if ty in generics:+ return ty+ if match := re.fullmatch(+ r"(char|uchar|short|ushort|int|uint|float|half)([234])", ty+ ):+ return f"vector<{slang_type(match[1])}, {match[2]}>"+ if ty.startswith("uzu::"):+ _, module, name = ty.split("::")+ imports.add(f"import generated.{module};")+ return name+ types = {+ "float": "float",+ "half": "half",+ "bfloat": "BFloat16",+ "bfloat16_t": "BFloat16",+ "bool": "bool",+ "char": "int8_t",+ "signed char": "int8_t",+ "short": "int16_t",+ "int8_t": "int8_t",+ "uchar": "uint8_t",+ "unsigned char": "uint8_t",+ "uint8_t": "uint8_t",+ "ushort": "uint16_t",+ "uint16_t": "uint16_t",+ "int": "int32_t",+ "int32_t": "int32_t",+ "uint": "uint32_t",+ "uint32_t": "uint32_t",+ "unsigned int": "uint32_t",+ "uint64_t": "uint64_t",+ "unsigned long": "uint64_t",+ "_atomic<uint>": "uint32_t",+ "ThreadContext": "ThreadContext",+ }+ if ty not in types:+ raise ValueError(f"unsupported Metal type {ty!r}")+ return types[ty]++ kernels = []+ for node in ast["inner"]:+ template = []+ if node["kind"] == "FunctionTemplateDecl":+ template = [+ n+ for n in node["inner"]+ if n["kind"] in ("TemplateTypeParmDecl", "NonTypeTemplateParmDecl")+ ]+ node = next(n for n in node["inner"] if n["kind"] == "FunctionDecl")+ if node["kind"] != "FunctionDecl":+ continue+ attrs = annotations(node)+ if ["dsl.public"] not in attrs or ["dsl.kernel"] not in attrs:+ continue+ parameters = [n for n in node["inner"] if n["kind"] == "ParmVarDecl"]+ generic_names = {n["name"] for n in template}+ generics, lines, arguments = [], [], []+ variants = {a[1]: a[2] for a in attrs if a[0] == "dsl.variants"}+ for param in template:+ name = param["name"]+ is_type = param["kind"] == "TemplateTypeParmDecl"+ generics.append(+ name+ if is_type+ else f"let {name}: {slang_type(param['type']['qualType'])}"+ )+ if name not in variants:+ raise ValueError(f"{node['name']}: missing VARIANTS for {name}")+ values = [v.strip() for v in variants[name].split(",")]+ if is_type:+ values = [slang_type(v) for v in values]+ lines.append(+ f"[[Variants({json.dumps(name)}, {json.dumps(', '.join(values))})]]"+ )+ for param in parameters:+ name = param["name"]+ ty = param["type"].get("desugaredQualType", param["type"]["qualType"])+ if "threadgroup" in ty.split():+ # Scratch storage belongs to the implementation, not the public API.+ continue+ attrs = annotations(param)+ attr = attrs[0] if attrs else []+ if attr and attr[0] in ("dsl.axis", "dsl.groups", "dsl.threads"):+ annotation = attr[0].removeprefix("dsl.").capitalize()+ values = ", ".join(json.dumps(expression(v)) for v in attr[1:])+ arguments.append(f" [[{annotation}({values})]] uint32_t {name},")+ continue+ element = re.sub(+ r"\b(const|device|constant|threadgroup)\b|[&*()]|\[[^]]*\]", "", ty+ ).strip()+ element = " ".join(element.split())+ element = slang_type(element, generic_names)+ begin, end = param["range"]["begin"], param["range"]["end"]+ begin, end = begin.get("spellingLoc", begin), end.get("spellingLoc", end)+ declaration = source[+ begin["offset"] : end["offset"] + end["tokLen"]+ ].decode("utf-8")+ array = re.search(r"\[([^]]+)\]", declaration)+ if "device" in ty.split():+ access = (+ "StructuredBuffer"+ if "const" in ty.split()+ else "RWStructuredBuffer"+ )+ ty = f"{access}<{element}>"+ elif "constant" in ty.split() and "*" in ty:+ ty = (+ f"{element}[{expression(array[1])}]"+ if array+ else f"[[Constant]] StructuredBuffer<{element}>"+ )+ else:+ ty = element+ prefix = ""+ if attr:+ if attr[0] == "dsl.optional":+ prefix = f"[[Optional({json.dumps(expression(attr[1]))})]] "+ elif attr[0] in ("dsl.specialize", "dsl.specialize_if"):+ prefix = "[[Specialize]] "+ if attr[0] == "dsl.specialize_if":+ prefix += f"[[Optional({json.dumps(expression(attr[1]))})]] "+ else:+ raise ValueError(f"{path}: unsupported annotation {attr}")+ arguments.append(f" {prefix}{ty} {name},")+ for attr in annotations(node):+ if attr[0] == "dsl.constraint":+ lines.append(f"[[Constraint({json.dumps(expression(attr[1]))})]]")+ generic = f"<{', '.join(generics)}>" if generics else ""+ lines.extend(+ [+ f"[[Public]] [[Kernel]] func {node['name']}{generic}(",+ *arguments,+ ") {",+ " // TODO",+ "}",+ ]+ )+ kernels.append("\n".join(lines))+ if not kernels:+ return None+ return "\n".join(+ [+ "// Autogenerated stub",+ "import definitions;",+ *sorted(imports),+ "",+ "\n\n".join(kernels),+ "",+ ]+ )+++def main():+ generated = {}+ skipped = 0+ with tempfile.TemporaryDirectory(prefix="uzu-slang-autogen-") as cache:+ for path in sorted(METAL.rglob("*.metal")):+ if not re.search(r"\bPUBLIC\b", path.read_text(encoding="utf-8")):+ continue+ destination = SLANG / path.relative_to(METAL).with_suffix(".slang")+ if destination.exists() and not destination.read_text(+ encoding="utf-8"+ ).startswith("// Autogenerated stub"):+ skipped += 1+ continue+ try:+ content = generate(path, cache)+ except subprocess.CalledProcessError as error:+ sys.exit(f"{path}: Metal analysis failed:\n{error.stderr}")+ except (OSError, ValueError) as error:+ sys.exit(f"{path}: {error}")+ if content:+ generated[destination] = content+ for path, content in generated.items():+ path.parent.mkdir(parents=True, exist_ok=True)+ if not path.exists() or path.read_text(encoding="utf-8") != content:+ path.write_text(content, encoding="utf-8")+ count = sum(s.count("[[Public]]") for s in generated.values())+ print(+ f"Generated {count} public kernel stubs in {len(generated)} files; "+ f"skipped {skipped} existing files."+ )+++if __name__ == "__main__":+ main()