A high-performance inference engine for AI models
diff --git a/src/engines/mlx/dispatch.zig b/src/engines/mlx/dispatch.zig
index 8caa18d..2cc951a 100644
--- a/src/engines/mlx/dispatch.zig
+++ b/src/engines/mlx/dispatch.zig
@@ -127,6 +127,7 @@ pub fn forward(self: *Transformer, token_ids: mlx.mlx_array) !mlx.mlx_array {
}
pub fn forwardWith(self: *Transformer, ctx: *ForwardCtx, token_ids: mlx.mlx_array) !mlx.mlx_array {
+ if (self.benchmark_target_forwards) |count| count.* += 1;
if (ctx.batch_slots == null) try self.ssmGroupRelease(ctx);
if (self.plugin) |b| return plugin_driver.forward(self, b, ctx, token_ids);
if (self.mlx_stream_ext) |m| return forwardMlxStream(self, ctx, token_ids, m);
diff --git a/src/engines/mlx/generate/init.zig b/src/engines/mlx/generate/init.zig
index 438d8b9..094b9ce 100644
--- a/src/engines/mlx/generate/init.zig
+++ b/src/engines/mlx/generate/init.zig
@@ -544,6 +544,7 @@ pub fn initWithOptions(
}
_ = mlx.mlx_eval(eval_vec);
}
+ try @import("../../../bench_memory.zig").sampleActive();
_ = mlx.mlx_clear_cache();
if (trace_enabled) eval_ns += prefill_sw.read() - eval_start_ns;
diff --git a/src/engines/mlx/transformer.zig b/src/engines/mlx/transformer.zig
index 5fb3034..ba95991 100644
--- a/src/engines/mlx/transformer.zig
+++ b/src/engines/mlx/transformer.zig
@@ -351,6 +351,8 @@ const kda_recurrence = @import("../../sdk/blocks/kda/recurrence.zig");
const Weights = model_mod.Weights;
pub const Transformer = struct {
+ // Benchmark-only logical target-forward counter; no GPU synchronization.
+ benchmark_target_forwards: ?*usize = null,
config: ModelConfig,
cache: KVCache,
s: mlx.mlx_stream,