Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

1.7 KB
Raw Blame
diff --git a/src/engines/mlx/dispatch.zig b/src/engines/mlx/dispatch.zig
index 8caa18d..2cc951a 100644
--- a/src/engines/mlx/dispatch.zig
+++ b/src/engines/mlx/dispatch.zig
@@ -127,6 +127,7 @@ pub fn forward(self: *Transformer, token_ids: mlx.mlx_array) !mlx.mlx_array {
 }

 pub fn forwardWith(self: *Transformer, ctx: *ForwardCtx, token_ids: mlx.mlx_array) !mlx.mlx_array {
+    if (self.benchmark_target_forwards) |count| count.* += 1;
     if (ctx.batch_slots == null) try self.ssmGroupRelease(ctx);
     if (self.plugin) |b| return plugin_driver.forward(self, b, ctx, token_ids);
     if (self.mlx_stream_ext) |m| return forwardMlxStream(self, ctx, token_ids, m);
diff --git a/src/engines/mlx/generate/init.zig b/src/engines/mlx/generate/init.zig
index 438d8b9..094b9ce 100644
--- a/src/engines/mlx/generate/init.zig
+++ b/src/engines/mlx/generate/init.zig
@@ -544,6 +544,7 @@ pub fn initWithOptions(
                 }
                 _ = mlx.mlx_eval(eval_vec);
             }
+            try @import("../../../bench_memory.zig").sampleActive();
             _ = mlx.mlx_clear_cache();
             if (trace_enabled) eval_ns += prefill_sw.read() - eval_start_ns;

diff --git a/src/engines/mlx/transformer.zig b/src/engines/mlx/transformer.zig
index 5fb3034..ba95991 100644
--- a/src/engines/mlx/transformer.zig
+++ b/src/engines/mlx/transformer.zig
@@ -351,6 +351,8 @@ const kda_recurrence = @import("../../sdk/blocks/kda/recurrence.zig");
 const Weights = model_mod.Weights;

 pub const Transformer = struct {
+    // Benchmark-only logical target-forward counter; no GPU synchronization.
+    benchmark_target_forwards: ?*usize = null,
     config: ModelConfig,
     cache: KVCache,
     s: mlx.mlx_stream,