Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

6.8 KB
Raw Blame
diff --git a/.bazelrc b/.bazelrc
index 4b8bfa8..8154a16 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -2,7 +2,9 @@
 build --jobs 128
 build --define='absl=1'  # for gtest
 build --enable_platform_specific_config
-build --cxxopt=-std=c++17
+build --cxxopt=-std=c++23
+# Stripped Rust proc-macros have a misaligned string table that dyld rejects.
+build --@rules_rust//rust/settings:extra_exec_rustc_flags=-Cstrip=none
 
 # Avoid XNNPACK using `-mavxvnniint8` (needs clang-16+/gcc-13+)
 build --define=xnn_enable_avxvnniint8=false
diff --git a/MODULE.bazel b/MODULE.bazel
index b12d23c..9d9a911 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -214,7 +214,10 @@ bazel_dep(
 )
 
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
-rust.toolchain(edition = "2021")
+rust.toolchain(
+    edition = "2021",
+    extra_target_triples = ["aarch64-apple-ios"],
+)
 use_repo(rust, "rust_toolchains")
 
 register_toolchains("@rust_toolchains//:all")
@@ -235,7 +238,12 @@ crate.spec(
     package = "serde_json",
     version = "1.0",
 )
-crate.from_specs()
+crate.from_specs(
+    supported_platform_triples = [
+        "aarch64-apple-darwin",
+        "aarch64-apple-ios",
+    ],
+)
 use_repo(crate, crate_index = "crates")
 
 http_archive(
@@ -263,3 +271,16 @@ http_archive(
     strip_prefix = "tokenizers-cpp-0.1.1",
     url = "https://github.com/mlc-ai/tokenizers-cpp/archive/refs/tags/v0.1.1.tar.gz",
 )
+
+http_archive(
+    name = "glaze",
+    build_file_content = """cc_library(
+    name = "glaze",
+    hdrs = glob(["include/**/*.hpp"]),
+    includes = ["include"],
+    visibility = ["//visibility:public"],
+)""",
+    sha256 = "55f8a675397c505f9bee6c160e1c26b0085868f185d04d7a7727531fdbff4bd4",
+    strip_prefix = "glaze-9.1.0",
+    url = "https://github.com/stephenberry/glaze/archive/refs/tags/v9.1.0.tar.gz",
+)
diff --git a/ml_drift/samples/llm/BUILD b/ml_drift/samples/llm/BUILD
index 8c63b10..b36c8fb 100644
--- a/ml_drift/samples/llm/BUILD
+++ b/ml_drift/samples/llm/BUILD
@@ -20,7 +20,10 @@ load(
     "pytype_strict_library",
 )
 
-package(licenses = ["notice"])
+package(
+    default_visibility = ["//benchmarks:__pkg__"],
+    licenses = ["notice"],
+)
 
 # Extracts weights from a HuggingFace model (safetensors).
 # bazel run //ml_drift/samples/llm:extract_weights_hf -- --model_path=/tmp/model.safetensors --output_dir=/tmp/extracted/
@@ -372,8 +375,7 @@ cc_test(
 )
 
 cc_library(
-    name = "llm_runner",
-    srcs = ["llm_runner.cc"],
+    name = "llm_runner_interface",
     hdrs = ["llm_runner.h"],
     deps = [
         ":llm_config",
@@ -391,6 +393,12 @@ cc_library(
         "@sentencepiece//:sentencepiece_processor",
         "@tokenizers_cpp//:huggingface_tokenizer",
     ],
+)
+
+cc_library(
+    name = "llm_runner",
+    srcs = ["llm_runner.cc"],
+    deps = [":llm_runner_interface"],
     alwayslink = 1,
 )
 
@@ -501,7 +509,7 @@ objc_library(
         ":gemma4_model_builder",
         ":llm_config",
         ":llm_file_tensor_loader",
-        ":llm_runner",
+        ":llm_runner_interface",
         ":llm_tensor_loader",
         ":qwen3_model_builder",
         "//ml_drift/common:data_type",
diff --git a/ml_drift/samples/llm/llm_config.h b/ml_drift/samples/llm/llm_config.h
index 8839d72..f2c3481 100644
--- a/ml_drift/samples/llm/llm_config.h
+++ b/ml_drift/samples/llm/llm_config.h
@@ -196,6 +196,22 @@ inline LlmConfig GetQwen3ModelConfig8B() {
   };
 }
 
+inline LlmConfig GetQwen3ModelConfig14B() {
+  return LlmConfig{
+      .batch_size = 1,
+      .sequence_size = 1,
+      .model_dimension = 5120,
+      .hidden_dimension = 17408,
+      .head_dimension = 128,
+      .number_of_heads = 40,
+      .number_of_kv_heads = 8,  // GQA
+      .vocabulary_size = 151936,
+      .stack_size = 40,
+      .num_local_layers_per_global = 0,
+      .sliding_window_size = 0,
+  };
+}
+
 struct ModelInfo {
   ModelType type;
   LlmConfig config;
@@ -240,6 +256,11 @@ inline absl::StatusOr<ModelInfo> GetModelInfo(absl::string_view model_name) {
         .type = ModelType::kQwen3,
         .config = GetQwen3ModelConfig8B(),
     };
+  } else if (model_name == "qwen3:14b") {
+    return ModelInfo{
+        .type = ModelType::kQwen3,
+        .config = GetQwen3ModelConfig14B(),
+    };
   }
   return absl::InvalidArgumentError(
       absl::StrCat("Unknown model: ", model_name));
diff --git a/ml_drift/samples/llm/llm_metal.cc b/ml_drift/samples/llm/llm_metal.cc
index 47e06d2..510a1b1 100644
--- a/ml_drift/samples/llm/llm_metal.cc
+++ b/ml_drift/samples/llm/llm_metal.cc
@@ -168,6 +168,7 @@ class MetalLlmModelRunner : public LlmModelRunner {
   GpuModelBuilder::TensorHandle greedy_input_logits_handle_;
   GpuModelBuilder::TensorHandle greedy_output_token_handle_;
   std::unique_ptr<MetalSpatialTensor> greedy_output_token_;
+  absl::Time decode_end_;
 };
 
 absl::Status MetalLlmModelRunner::Init(const std::string& weights_path,
@@ -626,6 +627,8 @@ absl::Status MetalLlmModelRunner::Decode(const DecodeOptions& options) {
     }
   }
 
+  decode_end_ = absl::Now();
+
   // Drain any remaining GPU work.
   metal::WaitUntilCompleted(command_queue_);
 
@@ -641,10 +644,15 @@ absl::Status MetalLlmModelRunner::PostProcessGreedy(
     int num_gen_tokens, std::vector<int>& output_tokens_vec) {
   // 1. Prefill Step: Process all prompt input tokens on the GPU.
   ABSL_LOG(INFO) << "Running prefill...";
+  stats_ = {};
   int first_token = 0;
+  const absl::Time prefill_start = absl::Now();
   ABSL_RETURN_IF_ERROR(
       Prefill(input_tokens_vec, /*download_token=*/true, &first_token))
       << "Error during prefill";
+  const absl::Time first_token_time = absl::Now();
+  stats_.prefill_seconds =
+      absl::ToDoubleSeconds(first_token_time - prefill_start);
   output_tokens_vec.push_back(first_token);
 
   if (num_gen_tokens <= 1) {
@@ -666,6 +674,8 @@ absl::Status MetalLlmModelRunner::PostProcessGreedy(
   options.tokenizer = &tokenizer;
   options.output_tokens = &output_tokens_vec;
   ABSL_RETURN_IF_ERROR(Decode(options));
+  stats_.decode_seconds =
+      absl::ToDoubleSeconds(decode_end_ - first_token_time);
   std::cout << std::endl;
 
   return absl::OkStatus();
diff --git a/ml_drift/samples/llm/llm_runner.h b/ml_drift/samples/llm/llm_runner.h
index 130c493..a360343 100644
--- a/ml_drift/samples/llm/llm_runner.h
+++ b/ml_drift/samples/llm/llm_runner.h
@@ -192,6 +192,11 @@ class Tokenizer {
   int end_of_turn_id_ = -1;
 };
 
+struct GenerationStats {
+  double prefill_seconds = 0.0;
+  double decode_seconds = 0.0;
+};
+
 // Abstract base class for the backend-specific LLM runner.
 class LlmModelRunner {
  public:
@@ -212,6 +217,11 @@ class LlmModelRunner {
     return absl::UnimplementedError(
         "PostProcessGreedyBench is not implemented.");
   }
+
+  const GenerationStats& stats() const { return stats_; }
+
+ protected:
+  GenerationStats stats_;
 };
 
 // Factory function implemented by each backend.