A high-performance inference engine for AI models
`MXU` refactor (#673)
CCYeh8e28784d4793
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.hdeleted file mode 100644index 643b9a7b6..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.h+++ /dev/null@@ -1,53 +0,0 @@-template <typename RightElement, class OutputFragment, class LeftFragment>-METAL_FUNC static void fragment_matmul_int8_device_weights(- thread OutputFragment& output,- thread LeftFragment& left,- const device uchar* right_signed_codes,- const int right_row_stride_bytes-) {- static_assert(RELAXED, "device weight tensors require the relaxed MXU layout");- static_assert(LeftFragment::COL_FRAGMENTS == 2, "device weight tensors expect K tiled as two fragments");- static_assert(OutputFragment::COL_FRAGMENTS % 2 == 0, "device weight tensors require even N fragments");- static_assert(LeftFragment::ROW_FRAGMENTS == OutputFragment::ROW_FRAGMENTS, "M tiles must match");-- constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;- constexpr ushort cols = OutputFragment::COL_FRAGMENTS;- constexpr int tile_k = int(2 * FRAGMENT_COLS);- constexpr int tile_n = int(2 * FRAGMENT_COLS);- constexpr int element_bits = metal::is_same_v<RightElement, metal::int4b_format> ? 4 : 8;- constexpr int elements_per_byte = 8 / element_bits;- using RightTensor = tensor<device RightElement, extents<int, tile_k, tile_n>, tensor_inline>;- using RightPointer =- metal::conditional_t<metal::is_same_v<RightElement, metal::int4b_format>, device uchar*, device RightElement*>;-- constexpr auto descriptor =- mpp::tensor_ops::matmul2d_descriptor(FRAGMENT_ROWS, tile_n, tile_k, false, true, RELAXED, MatmulMode::multiply);- mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;-- const array<int, 2> right_strides = {1, right_row_stride_bytes * elements_per_byte};-- METAL_PRAGMA_UNROLL- for (ushort row = 0; row < rows; ++row) {- METAL_PRAGMA_UNROLL- for (ushort col = 0; col < cols; col += 2) {- auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<int8_t, RightElement, int>();- load_paired_vectors(cooperative_left, left.fragment_at(row, 0), left.fragment_at(row, 1));-- RightTensor right_tensor(- // MPP rejects const-qualified tensor element types.- reinterpret_cast<RightPointer>(- const_cast<device uchar*>(right_signed_codes + int(col * FRAGMENT_COLS) * right_row_stride_bytes)- ),- extents<int, tile_k, tile_n>{},- right_strides- );-- auto cooperative_output =- matmul_op.template get_destination_cooperative_tensor<decltype(cooperative_left), RightTensor, int>();-- matmul_op.run(cooperative_left, right_tensor, cooperative_output);-- store_paired_vectors(cooperative_output, output.fragment_at(row, col), output.fragment_at(row, col + 1));- }- }-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.hindex b0e441729..77e4f7a3d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h@@ -1,7 +1,51 @@+// Fragment storage is paired into the 16x32 cooperative tile used here.template <- bool ACCUMULATE,- bool transpose_a,- bool transpose_b,+ MatmulMode OUTPUT_MODE,+ typename OutputType,+ typename LeftType,+ typename RightType,+ bool transpose_left,+ bool transpose_right,+ typename MarshalInputs>+METAL_FUNC static void matmul(+ thread ThreadVector<OutputType>& output_0,+ thread ThreadVector<OutputType>& output_1,+ MarshalInputs marshal_inputs+) {+ constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(+ FRAGMENT_ROWS,+ 2 * FRAGMENT_COLS,+ FRAGMENT_COLS,+ transpose_left,+ transpose_right,+ RELAXED,+ OUTPUT_MODE+ );++ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;++ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<LeftType, RightType, OutputType>();+ auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<LeftType, RightType, OutputType>();+ auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<+ decltype(cooperative_left),+ decltype(cooperative_right),+ OutputType>();++ marshal_inputs(cooperative_left, cooperative_right);++ if constexpr (OUTPUT_MODE == MatmulMode::multiply_accumulate) {+ load_paired_vectors(cooperative_output, output_0, output_1);+ }++ matmul_op.run(cooperative_left, cooperative_right, cooperative_output);++ store_paired_vectors(cooperative_output, output_0, output_1);+}++template <+ MatmulMode OUTPUT_MODE,+ bool transpose_left,+ bool transpose_right,class OutputFragment,class LeftFragment,class RightFragment>@@ -10,16 +54,16 @@ METAL_FUNC static void fragment_matmul(thread LeftFragment& left,thread RightFragment& right) {- constexpr ushort left_rows = transpose_a ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;+ constexpr ushort left_rows = transpose_left ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;static_assert(left_rows == rows, "fragment matmul: M dimensions do not match");- constexpr ushort right_cols = transpose_b ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;+ constexpr ushort right_cols = transpose_right ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;constexpr ushort cols = OutputFragment::COL_FRAGMENTS;static_assert(right_cols == cols, "fragment matmul: N dimensions do not match");- constexpr ushort left_depth = transpose_a ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;- constexpr ushort depth = transpose_b ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;+ constexpr ushort left_depth = transpose_left ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;+ constexpr ushort depth = transpose_right ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;static_assert(left_depth == depth, "fragment matmul: K dimensions do not match");static_assert(@@ -27,43 +71,54 @@ METAL_FUNC static void fragment_matmul("MXU fragment_mma requires even N, or N==1 with even M (MPP pairing)");- constexpr auto transpose_left = metal::bool_constant<transpose_a>{};- constexpr auto transpose_right = metal::bool_constant<transpose_b>{};+ constexpr auto left_transpose = metal::bool_constant<transpose_left>{};+ constexpr auto right_transpose = metal::bool_constant<transpose_right>{};constexpr bool pair_output_rows = (cols == 1 && rows % 2 == 0);- auto matmul_paired_outputs = [&](ushort row, ushort col, ushort depth_index, auto use_multiply_accumulate) {- constexpr bool matmul_accumulate = decltype(use_multiply_accumulate)::value;+ auto matmul_paired_outputs = [&](ushort row, ushort col, ushort depth_index, auto output_mode) {+ constexpr MatmulMode mode = decltype(output_mode)::value;if constexpr (pair_output_rows) {+ static_assert(RELAXED, "strict MXU row-pairing is not implemented");+ const thread auto& left_row_0 = left.fragment_at(row, depth_index, left_transpose);+ const thread auto& left_row_1 = left.fragment_at(row + 1, depth_index, left_transpose);+ const thread auto& right_operand = right.fragment_at(depth_index, col, right_transpose);matmul<- matmul_accumulate,+ mode,typename OutputFragment::ElementType,typename LeftFragment::ElementType,typename RightFragment::ElementType,- transpose_a,- transpose_b>(+ transpose_left,+ transpose_right>(output.fragment_at(row, col),output.fragment_at(row + 1, col),- left.fragment_at(row, depth_index, transpose_left),- left.fragment_at(row + 1, depth_index, transpose_left),- transpose_left,- right.fragment_at(depth_index, col, transpose_right),- transpose_right+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {+ load_paired_vectors(cooperative_left, left_row_0, left_row_1);+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_right[i] = right_operand[i];+ }+ });} else {+ const thread auto& left_operand = left.fragment_at(row, depth_index, left_transpose);+ const thread auto& right_col_0 = right.fragment_at(depth_index, col, right_transpose);+ const thread auto& right_col_1 = right.fragment_at(depth_index, col + 1, right_transpose);matmul<- matmul_accumulate,+ mode,typename OutputFragment::ElementType,typename LeftFragment::ElementType,typename RightFragment::ElementType,- transpose_a,- transpose_b>(+ transpose_left,+ transpose_right>(output.fragment_at(row, col),output.fragment_at(row, col + 1),- left.fragment_at(row, depth_index, transpose_left),- transpose_left,- right.fragment_at(depth_index, col, transpose_right),- right.fragment_at(depth_index, col + 1, transpose_right),- transpose_right+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_left[i] = left_operand[i];+ }+ load_paired_vectors(cooperative_right, right_col_0, right_col_1);+ });}};@@ -76,32 +131,38 @@ METAL_FUNC static void fragment_matmul(for (ushort row = 0; row < rows; row += output_row_step) {METAL_PRAGMA_UNROLLfor (ushort col = 0; col < output_col_count; col += output_col_step) {- if constexpr (!ACCUMULATE) {- matmul_paired_outputs(row, col, 0, metal::bool_constant<false>{});+ if constexpr (OUTPUT_MODE == MatmulMode::multiply) {+ matmul_paired_outputs(row, col, 0, uzu::integral_constant<MatmulMode, MatmulMode::multiply>{});}METAL_PRAGMA_UNROLL- for (ushort depth_index = ACCUMULATE ? 0 : 1; depth_index < depth; ++depth_index) {- matmul_paired_outputs(row, col, depth_index, metal::bool_constant<true>{});+ for (ushort depth_index = OUTPUT_MODE == MatmulMode::multiply_accumulate ? 0 : 1; depth_index < depth;+ ++depth_index) {+ matmul_paired_outputs(+ row,+ col,+ depth_index,+ uzu::integral_constant<MatmulMode, MatmulMode::multiply_accumulate>{}+ );}}}}-template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, class RightFragment>METAL_FUNC static void fragment_mma(thread OutputFragment& output,thread LeftFragment& left,thread RightFragment& right) {- fragment_matmul<true, transpose_a, transpose_b>(output, left, right);+ fragment_matmul<MatmulMode::multiply_accumulate, transpose_left, transpose_right>(output, left, right);}-template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>+template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, class RightFragment>METAL_FUNC static void fragment_mm(thread OutputFragment& output,thread LeftFragment& left,thread RightFragment& right) {// MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.- fragment_matmul<false, transpose_a, transpose_b>(output, left, right);+ fragment_matmul<MatmulMode::multiply, transpose_left, transpose_right>(output, left, right);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/integer_formats.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/integer_formats.hnew file mode 100644index 000000000..53ea74127--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/integer_formats.h@@ -0,0 +1,41 @@+#pragma once++#include <metal_stdlib>++#include "../../../common/defines.h"++using namespace metal;++namespace uzu {+namespace matmul {++enum class Signedness { Signed, Unsigned };++template <ushort BITS, Signedness SIGNEDNESS>+struct IntegerFormat;++template <Signedness SIGNEDNESS>+struct IntegerFormat<4, SIGNEDNESS> {+ using StorageElement = uchar;+ using UnpackedElement = metal::conditional_t<SIGNEDNESS == Signedness::Signed, int8_t, uint8_t>;+ using TensorElement =+ metal::conditional_t<SIGNEDNESS == Signedness::Signed, metal::int4b_format, metal::uint4b_format>;+ using DevicePointer = const device uchar*;+ using MutableDevicePointer = device uchar*;+ UZU_CONST ushort BITS = 4;+ UZU_CONST ushort ELEMENTS_PER_BYTE = 2;+};++template <Signedness SIGNEDNESS>+struct IntegerFormat<8, SIGNEDNESS> {+ using StorageElement = metal::conditional_t<SIGNEDNESS == Signedness::Signed, int8_t, uint8_t>;+ using UnpackedElement = StorageElement;+ using TensorElement = StorageElement;+ using DevicePointer = const device StorageElement*;+ using MutableDevicePointer = device StorageElement*;+ UZU_CONST ushort BITS = 8;+ UZU_CONST ushort ELEMENTS_PER_BYTE = 1;+};++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.hindex 88619860a..a4f2385f0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h@@ -7,6 +7,7 @@#include "../defines.h"#include "../loader.h"+#include "tensor_operand.h"#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>@@ -25,9 +26,8 @@ struct MxuFragmentOps {#include "layout.h"#include "cooperative_vectors.h"-#include "tile_matmul.h"#include "fragment_matmul.h"-#include "device_weight_matmul.h"+#include "tensor_matmul.h"};using MxuStrictFragmentOps = MxuFragmentOps<false>;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_matmul.hnew file mode 100644index 000000000..0c53c8140--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_matmul.h@@ -0,0 +1,68 @@+template <typename Format>+METAL_FUNC static auto make_tensor_view(+ const typename Format::DevicePointer tile_base,+ const int right_row_stride_bytes+) {+ constexpr int tile_k = int(2 * FRAGMENT_COLS);+ constexpr int tile_n = int(2 * FRAGMENT_COLS);+ using Element = typename Format::TensorElement;+ using RightTensor = tensor<device Element, extents<int, tile_k, tile_n>, tensor_inline>;++ const array<int, 2> right_strides = {1, right_row_stride_bytes * Format::ELEMENTS_PER_BYTE};++ return RightTensor(+ const_cast<typename Format::MutableDevicePointer>(tile_base),+ extents<int, tile_k, tile_n>{},+ right_strides+ );+}++template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, typename Format>+METAL_FUNC static void fragment_mm(+ thread OutputFragment& output,+ thread LeftFragment& left,+ const DeviceTensorOperand<Format> right+) {+ static_assert(!transpose_left && transpose_right, "device-tensor leaf: only left x right^T is implemented");+ static_assert(+ metal::is_same_v<typename LeftFragment::ElementType, typename Format::UnpackedElement>,+ "device-tensor leaf: integer operand signedness must match"+ );+ static_assert(RELAXED, "device-tensor operands require the relaxed MXU layout");+ static_assert(LeftFragment::COL_FRAGMENTS == 2, "device-tensor operands expect K tiled as two fragments");+ static_assert(OutputFragment::COL_FRAGMENTS % 2 == 0, "device-tensor operands require even N fragments");+ static_assert(LeftFragment::ROW_FRAGMENTS == OutputFragment::ROW_FRAGMENTS, "M tiles must match");++ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;+ constexpr int tile_k = int(2 * FRAGMENT_COLS);+ constexpr int tile_n = int(2 * FRAGMENT_COLS);+ constexpr auto descriptor =+ mpp::tensor_ops::matmul2d_descriptor(FRAGMENT_ROWS, tile_n, tile_k, false, true, RELAXED, MatmulMode::multiply);+ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;++ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < rows; ++row) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < cols; col += 2) {+ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<+ typename Format::UnpackedElement,+ typename Format::TensorElement,+ int>();+ load_paired_vectors(cooperative_left, left.fragment_at(row, 0), left.fragment_at(row, 1));++ auto right_tensor = make_tensor_view<Format>(+ right.base + int(col * FRAGMENT_COLS) * right.row_stride_bytes,+ right.row_stride_bytes+ );++ auto cooperative_output =+ matmul_op+ .template get_destination_cooperative_tensor<decltype(cooperative_left), decltype(right_tensor), int>();++ matmul_op.run(cooperative_left, right_tensor, cooperative_output);++ store_paired_vectors(cooperative_output, output.fragment_at(row, col), output.fragment_at(row, col + 1));+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_operand.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_operand.hnew file mode 100644index 000000000..e1e34a955--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_operand.h@@ -0,0 +1,22 @@+#pragma once++#include <metal_stdlib>++#include "integer_formats.h"++using namespace metal;++namespace uzu {+namespace matmul {++template <typename Format>+struct DeviceTensorOperand {+ using Element = typename Format::TensorElement;+ using Pointer = typename Format::DevicePointer;++ Pointer base;+ int row_stride_bytes;+};++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.hdeleted file mode 100644index e54ad95aa..000000000--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.h+++ /dev/null@@ -1,102 +0,0 @@-// MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.-template <- bool ACCUMULATE,- typename CType,- typename AType,- typename BType,- bool transpose_a,- bool transpose_b,- typename MarshalInputs>-METAL_FUNC static void mma_impl(- thread ThreadVector<CType>& output_0,- thread ThreadVector<CType>& output_1,- MarshalInputs marshal_inputs-) {- constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(- FRAGMENT_ROWS,- 2 * FRAGMENT_COLS,- FRAGMENT_COLS,- transpose_a,- transpose_b,- RELAXED,- ACCUMULATE ? MatmulMode::multiply_accumulate : MatmulMode::multiply- );-- mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;-- auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<AType, BType, CType>();- auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<AType, BType, CType>();- auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<- decltype(cooperative_left),- decltype(cooperative_right),- CType>();-- marshal_inputs(cooperative_left, cooperative_right);-- if constexpr (ACCUMULATE) {- load_paired_vectors(cooperative_output, output_0, output_1);- }-- matmul_op.run(cooperative_left, cooperative_right, cooperative_output);-- store_paired_vectors(cooperative_output, output_0, output_1);-}--template <- bool ACCUMULATE,- typename CType,- typename AType,- typename BType,- bool transpose_a = false,- bool transpose_b = false>-METAL_FUNC static void matmul(- thread ThreadVector<CType>& output_col_0,- thread ThreadVector<CType>& output_col_1,- const thread ThreadVector<AType>& left,- metal::bool_constant<transpose_a>,- const thread ThreadVector<BType>& right_col_0,- const thread ThreadVector<BType>& right_col_1,- metal::bool_constant<transpose_b>-) {- mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(- output_col_0,- output_col_1,- [&](thread auto& cooperative_left, thread auto& cooperative_right) {- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- cooperative_left[i] = left[i];- }- load_paired_vectors(cooperative_right, right_col_0, right_col_1);- }- );-}--template <- bool ACCUMULATE,- typename CType,- typename AType,- typename BType,- bool transpose_a = false,- bool transpose_b = false>-METAL_FUNC static void matmul(- thread ThreadVector<CType>& output_row_0,- thread ThreadVector<CType>& output_row_1,- const thread ThreadVector<AType>& left_row_0,- const thread ThreadVector<AType>& left_row_1,- metal::bool_constant<transpose_a>,- const thread ThreadVector<BType>& right,- metal::bool_constant<transpose_b>-) {- static_assert(RELAXED, "strict MXU row-pairing is not implemented");- mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(- output_row_0,- output_row_1,- [&](thread auto& cooperative_left, thread auto& cooperative_right) {- load_paired_vectors(cooperative_left, left_row_0, left_row_1);- METAL_PRAGMA_UNROLL- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {- cooperative_right[i] = right[i];- }- }- );-}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/integer_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/integer_source.hnew file mode 100644index 000000000..b50d82535--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/integer_source.h@@ -0,0 +1,28 @@+#pragma once++#include "../../common/fragment.h"+#include "../../common/mxu_fragment/ops.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <typename Element, typename FragmentOps, bool ALIGNED_M, ushort TILES_M, ushort TILES_K>+static METAL_FUNC uzu::matmul::Fragment<Element, TILES_M, TILES_K, FragmentOps> load_integer_tile(+ const device Element* activation,+ const int leading_dimension,+ const short simdgroup_limit_m,+ const ushort simd_lane_id+) {+ uzu::matmul::Fragment<Element, TILES_M, TILES_K, FragmentOps> tile;+ auto source = uzu::matmul::fragment_source(activation, leading_dimension);+ if constexpr (!ALIGNED_M) {+ source = source.bounded(simdgroup_limit_m, TILES_K * FragmentOps::FRAGMENT_ROWS);+ }+ tile.load_from(simd_lane_id, source);+ return tile;+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 61eac5a2d..460e0a0ed 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -6,15 +6,14 @@#include "../../common/fragment.h"#include "gemm_rht.h"#include "../../common/mxu_fragment/ops.h"-#include "../../common/mxu_gemm_loop.h"#include "../../../generated/matmul.h"#include "../generated/gemm.h"#include "block_geometry.h"#include "gemm_tiling.h"-#include "../../common/quant_pack.h"-#include "quant_scale_bias.h"-#include "quant_scale_zero_point.h"-#include "../../common/quant_unpack.h"+#include "operands.h"+#include "schedules/selector.h"+#include "schedules/staged.h"+#include "schedules/tile_context.h"using namespace metal;@@ -22,16 +21,22 @@ namespace uzu {namespace gemm {template <- typename AT,- typename BT,- typename DT,+ typename OutputElementType_,GemmTiling GEMM_TILING,bool TRANSPOSE_B,- GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,- int BITS = 0,- int GROUP_SIZE = 0,- GemmAPrologueKind A_PROLOGUE = GemmAPrologueKind::FullPrecision>+ typename LeftOperand,+ typename RightOperand_>struct MxuMmaCore {+ using OutputElementType = OutputElementType_;+ using RightOperand = RightOperand_;+ using Left = operands::LeftBinding<LeftOperand, RightOperand>;+ using Right = operands::RightBinding<RightOperand>;+ using LeftElementType = typename Left::ElementType;+ using RightElementType = typename Right::ElementType;+ using LeftArgs = typename Left::Args;+ using RightArgs = typename Right::Args;+ using FragmentOps = uzu::matmul::MxuFragmentOps<>;+ using Schedule = typename schedules::ScheduleFor<LeftOperand, RightOperand>::type;UZU_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);UZU_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);UZU_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);@@ -39,6 +44,7 @@ struct MxuMmaCore {UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);+ UZU_CONST bool TRANSPOSE_RIGHT = TRANSPOSE_B;UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);static_assert(THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,@@ -48,447 +54,27 @@ struct MxuMmaCore {UZU_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;UZU_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;- UZU_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;- UZU_CONST ushort PADDING_B = 16 / sizeof(BT);- UZU_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;+ using OuterBlock = schedules::OuterBlockK<RightOperand, THREADGROUP_BLOCK_K_FP, SIMDGROUP_BLOCK_K>;+ UZU_CONST ushort THREADGROUP_BLOCK_K = OuterBlock::VALUE;+ UZU_CONST ushort SHARED_STRIDE_B = THREADGROUP_BLOCK_K + 16 / sizeof(RightElementType);+ UZU_CONST ushort STAGING_ELEMENTS =+ metal::is_same<Schedule, schedules::StagedSchedule>::value ? THREADGROUP_BLOCK_N * SHARED_STRIDE_B : 1;UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;- static_assert(- B_PROLOGUE == GemmBPrologueKind::FullPrecision || QUANT_BK % SIMDGROUP_BLOCK_K == 0,- "QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K"- );- static_assert(- B_PROLOGUE == GemmBPrologueKind::FullPrecision || THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,- "Tile block_k must be a multiple of QUANT_BK"- );using AccumulatorType = float;- using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<- BT,- THREADGROUP_BLOCK_N,- (QUANT_BK > 0) ? QUANT_BK : 1,- SHARED_STRIDE_B,- 1,- THREADGROUP_THREADS,- (GROUP_SIZE > 0) ? GROUP_SIZE : 1,- (BITS > 0) ? BITS : 4>;- using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<- BT,- THREADGROUP_BLOCK_N,- (QUANT_BK > 0) ? QUANT_BK : 1,- SHARED_STRIDE_B,- 1,- THREADGROUP_THREADS,- (GROUP_SIZE > 0) ? GROUP_SIZE : 1,- (BITS > 0) ? BITS : 4>;- using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<- BT,- THREADGROUP_BLOCK_N,- (QUANT_BK > 0) ? QUANT_BK : 1,- SHARED_STRIDE_B,- 1,- THREADGROUP_THREADS,- (GROUP_SIZE > 0) ? GROUP_SIZE : 1,- (BITS > 0) ? BITS : 4,- true>;-- using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>>;-- struct QuantizedWeightAddressing {- int row_stride_bytes;- int groups_per_row;- int k_offset_groups;- const device uint8_t* block;- };-- static METAL_FUNC QuantizedWeightAddressing- quantized_weight_addressing(const device BT* b, const size_t block_col, const uint k_offset, const int k_elements) {- constexpr int pack_factor = get_pack_factor<(BITS > 0) ? BITS : 4, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<(BITS > 0) ? BITS : 4>();- const int row_stride_bytes = k_elements * bytes_per_pack / pack_factor;- return QuantizedWeightAddressing{- row_stride_bytes,- (k_elements + int(GROUP_SIZE) - 1) / int(GROUP_SIZE),- int(k_offset) / int(GROUP_SIZE),- reinterpret_cast<const device uint8_t*>(b) + block_col * row_stride_bytes +- int(k_offset) * bytes_per_pack / pack_factor,- };- }-- template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>- static METAL_FUNC AccumFragment quant_k_loop(- const device AT* a_simdgroup,- threadgroup BT* b_shared,- const int leading_dimension_a,- const int aligned_k_iterations,- const short simdgroup_limit_m,- const short simdgroup_limit_n,- const ushort tile_col_offset,- const ushort tile_block_cols,- thread Loader& loader_b,- const thread ThreadContext& thread_context- ) {- AccumFragment accumulator;- accumulator.clear();-- threadgroup BT* b_shared_simdgroup = b_shared + tile_col_offset * SHARED_STRIDE_B;- const short2 tile_dimensions_b = short2(QUANT_BK, tile_block_cols);-- METAL_PRAGMA_NO_UNROLL- for (int outer_k = 0; outer_k < aligned_k_iterations; ++outer_k) {- threadgroup_barrier(mem_flags::mem_threadgroup);- if constexpr (ALIGNED_N) {- loader_b.load_unsafe();- } else {- loader_b.load_safe(tile_dimensions_b);- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- METAL_PRAGMA_NO_UNROLL- for (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> left_tile;- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps<>> right_tile;-- const int left_offset = inner_k;- auto left_src = uzu::matmul::fragment_source(a_simdgroup + left_offset, leading_dimension_a);- if constexpr (!ALIGNED_M) {- left_src = left_src.bounded(simdgroup_limit_m, SIMDGROUP_BLOCK_K);- }- left_tile.load_from(thread_context.simd_lane_id, left_src);-- right_tile.load_from(- thread_context.simd_lane_id,- uzu::matmul::fragment_source(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B))- );-- uzu::matmul::MxuFragmentOps<>::template fragment_mma<false, true>(accumulator, left_tile, right_tile);- }-- a_simdgroup += QUANT_BK;- loader_b.next();- }-- return accumulator;- }-- template <bool ALIGNED_M>- static METAL_FUNC uzu::matmul::Fragment<int8_t, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> load_int8_left_tile(- const device int8_t* a_int8_simdgroup,- const int leading_dimension_a,- const short simdgroup_limit_m,- const ushort simd_lane_id- ) {- uzu::matmul::Fragment<int8_t, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> left_tile;- auto left_src = uzu::matmul::fragment_source(a_int8_simdgroup, leading_dimension_a);- if constexpr (!ALIGNED_M) {- left_src = left_src.bounded(simdgroup_limit_m, SIMDGROUP_BLOCK_K);- }- left_tile.load_from(simd_lane_id, left_src);- return left_tile;- }-- template <bool ALIGNED_N>- static METAL_FUNC uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps<>> load_int8_weight_tile(- const device uint8_t* b_packed_simdgroup,- const int k_element_offset,- const int b_row_stride_bytes,- const short simdgroup_limit_n,- const short2 position,- const ushort simd_lane_id- ) {- using Ops = uzu::matmul::MxuFragmentOps<>;- uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops> right_tile;- if constexpr (BITS == 4) {- METAL_PRAGMA_UNROLL- for (ushort tile_n = 0; tile_n < TILES_N; ++tile_n) {- METAL_PRAGMA_UNROLL- for (ushort tile_k = 0; tile_k < TILES_K; ++tile_k) {- thread auto& weight_vector = right_tile.fragment_at(tile_n, tile_k);- METAL_PRAGMA_UNROLL- for (ushort thread_row = 0; thread_row < Ops::THREAD_ELEMENT_ROWS; ++thread_row) {- const short row =- short(tile_n * Ops::FRAGMENT_ROWS) + position.y + short(thread_row * Ops::THREAD_ELEMENT_ROW_STRIDE);- const ushort element_base = thread_row * Ops::THREAD_ELEMENT_COLS;- char4 codes = char4(0);- if (ALIGNED_N || row < simdgroup_limit_n) {- const int k_base = k_element_offset + int(tile_k * Ops::FRAGMENT_COLS) + int(position.x);- const ushort packed = *reinterpret_cast<const device ushort*>(- b_packed_simdgroup + int(row) * b_row_stride_bytes + (k_base >> 1)- );- codes = unpack_signed_nibbles_to_int8(uint(packed));- }- weight_vector[element_base + 0] = codes.x;- weight_vector[element_base + 1] = codes.y;- weight_vector[element_base + 2] = codes.z;- weight_vector[element_base + 3] = codes.w;- }- }- }- } else {- static_assert(BITS == 8, "symmetric int8 activations only support 4-bit or 8-bit weights");- auto right_src = uzu::matmul::fragment_source(- reinterpret_cast<const device int8_t*>(b_packed_simdgroup) + k_element_offset,- b_row_stride_bytes- );- if constexpr (!ALIGNED_N) {- right_src = right_src.bounded(simdgroup_limit_n, SIMDGROUP_BLOCK_K);- }- right_tile.load_from(simd_lane_id, right_src);- }- return right_tile;- }-- // Per-thread cache holding one value per distinct fragment row (or column) line- // a thread owns, addressable either by slot or by offset from the thread origin.- template <ushort TILES, ushort SLOTS_PER_TILE, ushort FRAGMENT_EXTENT, ushort SLOT_STRIDE>- struct FragmentLineCache {- float values[TILES * SLOTS_PER_TILE];-- METAL_FUNC thread float& slot(const ushort tile, const ushort index) thread {- return values[tile * SLOTS_PER_TILE + index];- }-- METAL_FUNC void clear() thread {- METAL_PRAGMA_UNROLL- for (ushort index = 0; index < TILES * SLOTS_PER_TILE; ++index) {- values[index] = 0.0f;- }- }-- METAL_FUNC float at(const short offset) const thread {- const ushort tile = ushort(offset) / FRAGMENT_EXTENT;- const ushort index = (ushort(offset) % FRAGMENT_EXTENT) / SLOT_STRIDE;- return values[tile * SLOTS_PER_TILE + index];- }-- template <bool ALIGNED, typename Sink>- static METAL_FUNC void for_each_line(- const short origin,- const short limit,- const uint abs_base,- const uint groups_per_row,- const uint group_index,- Sink sink- ) {- METAL_PRAGMA_UNROLL- for (ushort tile = 0; tile < TILES; ++tile) {- METAL_PRAGMA_UNROLL- for (ushort index = 0; index < SLOTS_PER_TILE; ++index) {- const short coord = origin + tile * FRAGMENT_EXTENT + index * SLOT_STRIDE;- if (ALIGNED || coord < limit) {- const uint line_index = abs_base + uint(coord);- sink(tile, index, line_index * groups_per_row + group_index, line_index);- }- }- }- }- };-- using FragmentOps = uzu::matmul::MxuFragmentOps<>;- using ActivationLineCache = FragmentLineCache<- TILES_M,- FragmentOps::THREAD_ELEMENT_ROWS,- FragmentOps::FRAGMENT_ROWS,- FragmentOps::THREAD_ELEMENT_ROW_STRIDE>;- using WeightLineCache = FragmentLineCache<TILES_N, FragmentOps::THREAD_ELEMENT_COLS, FragmentOps::FRAGMENT_COLS, 1>;-- static_assert(- A_PROLOGUE != GemmAPrologueKind::Int8Symmetric || GROUP_SIZE % int(SIMDGROUP_BLOCK_K) == 0,- "A8 weight group size must be a multiple of the 32-wide activation chunk"- );- UZU_CONST bool int8_activation_needs_weight_correction =- A_PROLOGUE == GemmAPrologueKind::Int8Symmetric && B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant;- UZU_CONST float int8_weight_midpoint = float(symmetric_zero_point<ushort((BITS > 0) ? BITS : 4)>());-- static METAL_FUNC float int8_weight_dequant_bias(- const float scale,- const uint scale_index,- const uint weight_column,- const uint weight_group,- const uint weight_groups_per_row,- const device BT* biases,- const device uint8_t* zero_points- ) {- if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {- return static_cast<float>(biases[scale_index]);- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {- if constexpr (BITS == 8) {- return -scale * float(zero_points[scale_index]);- } else {- const device uint8_t* zero_points_row =- zero_points + weight_column * zero_point_row_stride<ushort(BITS)>(weight_groups_per_row);- return -scale * float(decode_zero_point<ushort(BITS)>(zero_points_row, weight_group));- }- } else {- return 0.0f;- }- }-- template <bool ALIGNED_M, bool ALIGNED_N>- static METAL_FUNC AccumFragment int8_activation_k_loop(- const device int8_t* a_int8_simdgroup,- const device uint8_t* b_packed_simdgroup,- const device float* a_scales,- const device int32_t* a_group_sums,- const device BT* b_scales,- const device BT* biases,- const device uint8_t* zero_points,- const int leading_dimension_a,- const int b_row_stride_bytes,- const int weight_group_iterations,- const short simdgroup_limit_m,- const short simdgroup_limit_n,- const uint abs_row_base,- const uint abs_col_base,- const uint k_offset_weight_groups,- const uint k_offset_act_groups,- const uint weight_groups_per_row,- const uint act_groups_per_row,- const thread ThreadContext& thread_context- ) {- using Ops = uzu::matmul::MxuFragmentOps<>;- AccumFragment accumulator;- accumulator.clear();-- const short2 position = Ops::get_position(thread_context.simd_lane_id);- constexpr int k_bytes_per_weight_group = (BITS == 4) ? (int(GROUP_SIZE) / 2) : int(GROUP_SIZE);- constexpr int act_chunks_per_weight_group = int(GROUP_SIZE) / int(SIMDGROUP_BLOCK_K);-- METAL_PRAGMA_NO_UNROLL- for (int weight_group = 0; weight_group < weight_group_iterations; ++weight_group) {- const uint weight_group_index = k_offset_weight_groups + uint(weight_group);-- WeightLineCache weight_scales;- WeightLineCache weight_corrections;- WeightLineCache::template for_each_line<ALIGNED_N>(- position.x,- simdgroup_limit_n,- abs_col_base,- weight_groups_per_row,- weight_group_index,- [&](ushort tile_col, ushort thread_col, uint scale_index, uint weight_column) {- const float scale = static_cast<float>(b_scales[scale_index]);- weight_scales.slot(tile_col, thread_col) = scale;- if constexpr (int8_activation_needs_weight_correction) {- weight_corrections.slot(tile_col, thread_col) = scale * int8_weight_midpoint + int8_weight_dequant_bias(- scale,- scale_index,- weight_column,- weight_group_index,- weight_groups_per_row,- biases,- zero_points- );- }- }- );-- ActivationLineCache activation_corrections;- if constexpr (int8_activation_needs_weight_correction) {- activation_corrections.clear();- }-- METAL_PRAGMA_NO_UNROLL- for (int act_chunk = 0; act_chunk < act_chunks_per_weight_group; ++act_chunk) {- const int k_element_offset = act_chunk * int(SIMDGROUP_BLOCK_K);- auto activation_tile = load_int8_left_tile<ALIGNED_M>(- a_int8_simdgroup + k_element_offset,- leading_dimension_a,- simdgroup_limit_m,- thread_context.simd_lane_id- );-- uzu::matmul::Fragment<int, TILES_M, TILES_N, Ops> chunk_products;- if constexpr (BITS == 4 && ALIGNED_N) {- Ops::template fragment_matmul_int8_device_weights<metal::int4b_format>(- chunk_products,- activation_tile,- b_packed_simdgroup + (k_element_offset >> 1),- b_row_stride_bytes- );- } else {- auto right_tile = load_int8_weight_tile<ALIGNED_N>(- b_packed_simdgroup,- k_element_offset,- b_row_stride_bytes,- simdgroup_limit_n,- position,- thread_context.simd_lane_id- );- Ops::template fragment_mm<false, true>(chunk_products, activation_tile, right_tile);- }-- const uint act_group_index = k_offset_act_groups + uint(weight_group * act_chunks_per_weight_group + act_chunk);- ActivationLineCache activation_scales;- ActivationLineCache::template for_each_line<ALIGNED_M>(- position.y,- simdgroup_limit_m,- abs_row_base,- act_groups_per_row,- act_group_index,- [&](ushort tile_row, ushort thread_row, uint scale_index, uint) {- const float activation_scale = a_scales[scale_index];- activation_scales.slot(tile_row, thread_row) = activation_scale;- if constexpr (int8_activation_needs_weight_correction) {- activation_corrections.slot(tile_row, thread_row) +=- activation_scale * float(a_group_sums[scale_index]);- }- }- );-- AccumFragment::zip_for_each_coord(- thread_context.simd_lane_id,- [&](short row, short col, thread float& accumulated, thread int& products) {- if (!ALIGNED_M && row >= simdgroup_limit_m) {- return;- }- if (!ALIGNED_N && col >= simdgroup_limit_n) {- return;- }- accumulated +=- activation_scales.at(row - position.y) * weight_scales.at(col - position.x) * float(products);- },- accumulator,- chunk_products- );- }-- if constexpr (int8_activation_needs_weight_correction) {- accumulator.map_coords(thread_context.simd_lane_id, [&](short row, short col, float value) {- if (!ALIGNED_M && row >= simdgroup_limit_m) {- return value;- }- if (!ALIGNED_N && col >= simdgroup_limit_n) {- return value;- }- return value + weight_corrections.at(col - position.x) * activation_corrections.at(row - position.y);- });- }-- a_int8_simdgroup += GROUP_SIZE;- b_packed_simdgroup += k_bytes_per_weight_group;- }-- return accumulator;- }+ using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, FragmentOps>;static METAL_FUNC void run(- const device AT* a,- const device BT* b,- device DT* d,+ LeftArgs left,+ RightArgs right,+ device OutputElementType* d,const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,GemmDTransform output_transform,- const bool signed_codes,- const device BT* scales,- const device BT* biases,- const device uint8_t* zero_points,- const device BT* output_bias,+ const device RightElementType* output_bias,const device int32_t* rht_factors,- const device int8_t* a_int8,- const device float* a_scales,- const device int32_t* a_group_sums,- threadgroup BT* b_shared,+ threadgroup RightElementType* b_shared,const thread ThreadContext& thread_context) {const uint partition = thread_context.threadgroup_position.z;@@ -503,19 +89,14 @@ struct MxuMmaCore {const size_t block_row = size_t(geometry.block_row_start);const size_t block_col = size_t(geometry.block_col_start);- const uint k_offset_per_block =- (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? uint(THREADGROUP_BLOCK_K_FP) : uint(QUANT_BK);- const uint k_offset = partition * params->aligned_inner_iterations * k_offset_per_block;-- const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +- (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));+ const uint k_offset = partition * params->aligned_inner_iterations * uint(THREADGROUP_BLOCK_K);const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);- device DT* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) +- block_row * params->leading_dimension_d + block_col +- tile_row_offset * params->leading_dimension_d + tile_col_offset;+ device OutputElementType* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) ++ block_row * params->leading_dimension_d + block_col ++ tile_row_offset * params->leading_dimension_d + tile_col_offset;const short simdgroup_limit_m =alignment.contains(GemmAlignment::M)@@ -526,209 +107,88 @@ struct MxuMmaCore {? SIMDGROUP_BLOCK_N: short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));- const device AT* a_simdgroup = a;- if constexpr (A_PROLOGUE == GemmAPrologueKind::FullPrecision) {- a_simdgroup +=- block_row * params->leading_dimension_a + k_offset + size_t(tile_row_offset) * params->leading_dimension_a;- }- const device BT* b_simdgroup_fp =- b_block_fp +- (TRANSPOSE_B ? size_t(tile_col_offset) * int(params->leading_dimension_b) : size_t(tile_col_offset));+ left.seek_rows(block_row + tile_row_offset, params->leading_dimension_a, k_offset);- const ushort tile_block_cols =+ schedules::TileContext tile_context;+ tile_context.simdgroup_limit_m = simdgroup_limit_m;+ tile_context.simdgroup_limit_n = simdgroup_limit_n;+ tile_context.block_col = block_col;+ tile_context.tile_col_offset = tile_col_offset;+ tile_context.tile_block_cols =ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start)));+ tile_context.k_offset = k_offset;+ tile_context.abs_row_base = uint(geometry.block_row_start) + tile_row_offset;const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);- const device BT* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);-- auto dispatch_aligned_k = [&](auto body) {- if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {- dispatch_bool(alignment.contains(GemmAlignment::K), body);- } else {- body(true_type{});- }- };- dispatch_aligned_k([&](auto aligned_k) {- dispatch_bool(- alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),- [&](auto aligned_m) {- dispatch_bool(- alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),- [&](auto aligned_n) {- auto accumulator_tile = [&]() {- if constexpr (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric) {- const auto quantized_weights =- quantized_weight_addressing(b, block_col, k_offset, int(params->K));- const device int8_t* a_int8_simdgroup = a_int8 + block_row * params->leading_dimension_a +- k_offset +- size_t(tile_row_offset) * params->leading_dimension_a;- const device uint8_t* b_packed_simdgroup =- quantized_weights.block + size_t(tile_col_offset) * quantized_weights.row_stride_bytes;- return int8_activation_k_loop<aligned_m.value, aligned_n.value>(- a_int8_simdgroup,- b_packed_simdgroup,- a_scales,- a_group_sums,- scales,- biases,- zero_points,- int(params->leading_dimension_a),- quantized_weights.row_stride_bytes,- int(params->aligned_inner_iterations),- simdgroup_limit_m,- simdgroup_limit_n,- uint(geometry.block_row_start) + tile_row_offset,- uint(geometry.block_col_start) + tile_col_offset,- uint(quantized_weights.k_offset_groups),- k_offset / uint(SIMDGROUP_BLOCK_K),- uint(quantized_weights.groups_per_row),- uint(params->K) / uint(SIMDGROUP_BLOCK_K),- thread_context- );- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {- const int aligned_k_iterations_fp = int(params->aligned_inner_iterations);- return uzu::matmul::gemm_loop<- AT,- BT,- SIMDGROUP_BLOCK_M,- SIMDGROUP_BLOCK_N,- SIMDGROUP_BLOCK_K,- THREADGROUP_BLOCK_K_FP,- false,- TRANSPOSE_B,- aligned_m.value,- aligned_n.value,- aligned_k.value,- AccumulatorType>(- a_simdgroup,- b_simdgroup_fp,- int(params->leading_dimension_a),- int(params->leading_dimension_b),- int(params->K),- aligned_k_iterations_fp,- simdgroup_limit_m,- simdgroup_limit_n,- thread_context- );- } else {- const int aligned_k_iterations_q = int(params->aligned_inner_iterations);- const int k_elements = int(params->K);- const auto quantized_weights = quantized_weight_addressing(b, block_col, k_offset, k_elements);- const int groups_per_row = quantized_weights.groups_per_row;- const int k_offset_groups = quantized_weights.k_offset_groups;- const device uint8_t* weights_block = quantized_weights.block;- const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;-- auto loader_b = [&]() {- if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {- const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;- return BLoaderScaleBias(- weights_block,- scales_offset,- biases_offset,- signed_codes,- k_elements,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id- );- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {- const int zero_point_stride_per_row = zero_point_row_stride<ushort(BITS)>(groups_per_row);- const device uint8_t* zero_points_row_start =- zero_points + block_col * zero_point_stride_per_row +- ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);- return BLoaderScaleZeroPoint(- weights_block,- scales_offset,- zero_points_row_start,- signed_codes,- k_elements,- groups_per_row,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id- );- } else {- return BLoaderScaleSymmetric(- weights_block,- scales_offset,- nullptr,- signed_codes,- k_elements,- groups_per_row,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id- );- }- }();-- return quant_k_loop<aligned_m.value, aligned_n.value>(- a_simdgroup,- b_shared,- int(params->leading_dimension_a),- aligned_k_iterations_q,- simdgroup_limit_m,- simdgroup_limit_n,- tile_col_offset,- tile_block_cols,- loader_b,- thread_context- );- }- }();-- if (apply_scale) {- const AccumulatorType scale = AccumulatorType(params->ab_scale);- accumulator_tile.map([&](auto value) { return value * scale; });- }+ const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);++ dispatch_bool(+ alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),+ [&](auto aligned_m) {+ dispatch_bool(+ alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),+ [&](auto aligned_n) {+ auto accumulator_tile = Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(+ left,+ right,+ b_shared,+ params,+ tile_context,+ alignment,+ thread_context+ );++ if (apply_scale) {+ const AccumulatorType scale = AccumulatorType(params->ab_scale);+ accumulator_tile.map([&](auto value) { return value * scale; });+ }- if (apply_accumulate) {- uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>> existing_output;- auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));- if constexpr (!(aligned_m.value && aligned_n.value)) {- output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);- }- existing_output.load_from(thread_context.simd_lane_id, output_src);- thread DT* existing_data = existing_output.elements();- accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });+ if (apply_accumulate) {+ uzu::matmul::Fragment<OutputElementType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>>+ existing_output;+ auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));+ if constexpr (!(aligned_m.value && aligned_n.value)) {+ output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);}+ existing_output.load_from(thread_context.simd_lane_id, output_src);+ thread OutputElementType* existing_data = existing_output.elements();+ accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });+ }- if (apply_bias) {- accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {- if constexpr (aligned_n.value) {+ if (apply_bias) {+ accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {+ if constexpr (aligned_n.value) {+ return value + AccumulatorType(bias_simdgroup[col]);+ } else {+ if (col < simdgroup_limit_n) {return value + AccumulatorType(bias_simdgroup[col]);- } else {- if (col < simdgroup_limit_n) {- return value + AccumulatorType(bias_simdgroup[col]);- }- return value;}- });- }+ return value;+ }+ });+ }- if constexpr (aligned_m.value && aligned_n.value) {- accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));- } else {- accumulator_tile.store_safe(- thread_context.simd_lane_id,- d_simdgroup,- int(params->leading_dimension_d),- short2(simdgroup_limit_n, simdgroup_limit_m)- );- }+ if constexpr (aligned_m.value && aligned_n.value) {+ accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));+ } else {+ accumulator_tile.store_safe(+ thread_context.simd_lane_id,+ d_simdgroup,+ int(params->leading_dimension_d),+ short2(simdgroup_limit_n, simdgroup_limit_m)+ );}- );- }- );- });+ }+ );+ }+ );if (output_transform.contains(GemmDTransform::RHT)) {threadgroup_barrier(mem_flags::mem_device);- device DT* d_block = d + block_row * params->leading_dimension_d + block_col;+ device OutputElementType* d_block = d + block_row * params->leading_dimension_d + block_col;const ushort tile_block_rows = ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row)));const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col)));apply_output_random_hadamard_transform(diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/operands.hnew file mode 100644index 000000000..039ba4644--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/operands.h@@ -0,0 +1,283 @@+#pragma once++#include <metal_stdlib>++#include "../../../generated/gemm.h"+#include "../../common/mxu_fragment/integer_formats.h"+#include "../../common/quant_pack.h"+#include "../../common/quant_unpack.h"++using namespace metal;++namespace uzu {+namespace gemm {+namespace operands {++namespace detail {++template <bool WITH_GROUP_SUMS>+struct QuantizedLeftArgs;++template <>+struct QuantizedLeftArgs<false> {+ const device int8_t* values;+ const device float* scales;++ METAL_FUNC void seek_rows(const size_t row, const int leading_dimension, const uint k_offset) thread {+ values += row * leading_dimension + k_offset;+ }++ METAL_FUNC void advance_k(const ushort elements) thread { values += elements; }+};++template <>+struct QuantizedLeftArgs<true> {+ const device int8_t* values;+ const device float* scales;+ const device int32_t* group_sums;++ METAL_FUNC void seek_rows(const size_t row, const int leading_dimension, const uint k_offset) thread {+ values += row * leading_dimension + k_offset;+ }++ METAL_FUNC void advance_k(const ushort elements) thread { values += elements; }+};++template <GemmBPrologueKind SCHEME, ushort BITS, typename ElementType>+struct QuantizedRightArgs;++template <ushort BITS>+struct QuantizedStorage {+ const device uint8_t* values;+ bool signed_codes;++ QuantizedStorage(const device uint8_t* values_, const bool signed_codes_)+ : values(values_), signed_codes(signed_codes_) {}++ METAL_FUNC void seek_block(const size_t block_column, const uint k_offset, const int row_stride_bytes) thread {+ values += block_column * row_stride_bytes + int(k_offset) * get_bytes_per_pack<BITS>() / get_pack_factor<BITS>();+ }++ METAL_FUNC void seek_columns(const ushort tile_column_offset, const int row_stride_bytes) thread {+ values += size_t(tile_column_offset) * row_stride_bytes;+ }++ METAL_FUNC void advance_k(const int bytes) thread { values += bytes; }+};++// Constructors keep the specialized signedness flag mandatory; aggregate+// initialization would silently default an omitted trailing flag to false.++template <ushort BITS, typename ElementType>+struct QuantizedRightArgs<GemmBPrologueKind::ScaleSymmetricDequant, BITS, ElementType> {+ QuantizedStorage<BITS> storage;+ const device ElementType* scales;++ QuantizedRightArgs(const device uint8_t* values_, const device ElementType* scales_, const bool signed_codes_)+ : storage(values_, signed_codes_), scales(scales_) {}+};++template <ushort BITS, typename ElementType>+struct QuantizedRightArgs<GemmBPrologueKind::ScaleBiasDequant, BITS, ElementType> {+ QuantizedStorage<BITS> storage;+ const device ElementType* scales;+ const device ElementType* biases;++ QuantizedRightArgs(+ const device uint8_t* values_,+ const device ElementType* scales_,+ const device ElementType* biases_,+ const bool signed_codes_+ )+ : storage(values_, signed_codes_), scales(scales_), biases(biases_) {}+};++template <ushort BITS, typename ElementType>+struct QuantizedRightArgs<GemmBPrologueKind::ScaleZeroPointDequant, BITS, ElementType> {+ QuantizedStorage<BITS> storage;+ const device ElementType* scales;+ const device uint8_t* zero_points;++ QuantizedRightArgs(+ const device uint8_t* values_,+ const device ElementType* scales_,+ const device uint8_t* zero_points_,+ const bool signed_codes_+ )+ : storage(values_, signed_codes_), scales(scales_), zero_points(zero_points_) {}+};++} // namespace detail++template <GemmBPrologueKind SCHEME, ushort BITS, typename ElementType>+static METAL_FUNC void seek_quantized_metadata(+ thread detail::QuantizedRightArgs<SCHEME, BITS, ElementType>& args,+ const size_t block_column,+ const int groups_per_row,+ const int first_group+) {+ args.scales += block_column * groups_per_row + first_group;+ if constexpr (SCHEME == GemmBPrologueKind::ScaleBiasDequant) {+ args.biases += block_column * groups_per_row + first_group;+ } else if constexpr (SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {+ args.zero_points +=+ block_column * zero_point_row_stride<BITS>(groups_per_row) + ((BITS == 4) ? (first_group / 2) : first_group);+ }+}++template <typename ElementType_>+struct Dense {+ using ElementType = ElementType_;+};++template <typename Format_, ushort GROUP_SIZE_, GemmBPrologueKind SCHEME_, typename ScaleElementType_>+struct Quantized {+ using Format = Format_;+ using ScaleElementType = ScaleElementType_;+ UZU_CONST ushort BITS = Format::BITS;+ UZU_CONST ushort GROUP_SIZE = GROUP_SIZE_;+ UZU_CONST GemmBPrologueKind SCHEME = SCHEME_;++ static_assert(BITS == 4 || BITS == 8, "quantized integer bits must be 4 or 8");+ static_assert(GROUP_SIZE > 0, "quantized groups must be non-empty");+ static_assert(SCHEME != GemmBPrologueKind::FullPrecision, "quantized operands need a quantization scheme");+};++template <typename ElementType>+struct DenseArgs {+ const device ElementType* values;++ METAL_FUNC void seek_rows(const size_t row, const int leading_dimension, const uint k_offset) thread {+ values += row * leading_dimension + k_offset;+ }++ template <bool TRANSPOSED>+ METAL_FUNC void seek_columns(const size_t column, const uint k_offset, const int leading_dimension) thread {+ values += (TRANSPOSED ? column * leading_dimension : column) ++ (TRANSPOSED ? k_offset : k_offset * uint(leading_dimension));+ }++ METAL_FUNC void advance_k(const ushort elements) thread { values += elements; }+};++template <typename Operand>+struct RightBinding;++template <typename Element>+struct RightBinding<Dense<Element>> {+ using ElementType = Element;+ using Args = DenseArgs<Element>;+ UZU_CONST bool NEEDS_CORRECTION = false;+};++template <typename Format, ushort GROUP_SIZE, GemmBPrologueKind SCHEME, typename ScaleElementType>+struct RightBinding<Quantized<Format, GROUP_SIZE, SCHEME, ScaleElementType>> {+ using ElementType = ScaleElementType;+ using Args = detail::QuantizedRightArgs<SCHEME, Format::BITS, ScaleElementType>;+ UZU_CONST bool NEEDS_CORRECTION = SCHEME != GemmBPrologueKind::ScaleSymmetricDequant;+};++template <typename LeftOperand, typename RightOperand>+struct LeftBinding;++template <typename Element, typename RightOperand>+struct LeftBinding<Dense<Element>, RightOperand> {+ using ElementType = Element;+ using Args = DenseArgs<Element>;+};++template <typename Format, ushort GROUP_SIZE, typename ScaleElementType, typename RightOperand>+struct LeftBinding<+ Quantized<Format, GROUP_SIZE, GemmBPrologueKind::ScaleSymmetricDequant, ScaleElementType>,+ RightOperand> {+ static_assert(Format::BITS == 8, "integer MMA left operands must be 8-bit");+ static_assert(+ metal::is_same<typename Format::UnpackedElement, int8_t>::value,+ "integer MMA left operands must be signed"+ );+ using ElementType = typename Format::UnpackedElement;+ using Args = detail::QuantizedLeftArgs<RightBinding<RightOperand>::NEEDS_CORRECTION>;+};++template <GemmAPrologueKind A_PROLOGUE, typename ElementType>+struct LeftOperandFor;++template <typename ElementType>+struct LeftOperandFor<GemmAPrologueKind::FullPrecision, ElementType> {+ using type = Dense<ElementType>;+};++template <typename ElementType>+struct LeftOperandFor<GemmAPrologueKind::Int8Symmetric, ElementType> {+ using type = Quantized<+ uzu::matmul::IntegerFormat<8, uzu::matmul::Signedness::Signed>,+ 32,+ GemmBPrologueKind::ScaleSymmetricDequant,+ float>;+};++template <GemmBPrologueKind B_PROLOGUE, ushort BITS, ushort GROUP_SIZE, typename ElementType>+struct RightOperandFor {+ static_assert(+ B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant || B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant ||+ B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant,+ "unsupported quantized right operand scheme"+ );+ using type =+ Quantized<uzu::matmul::IntegerFormat<BITS, uzu::matmul::Signedness::Signed>, GROUP_SIZE, B_PROLOGUE, ElementType>;+};++template <ushort BITS, ushort GROUP_SIZE, typename ElementType>+struct RightOperandFor<GemmBPrologueKind::FullPrecision, BITS, GROUP_SIZE, ElementType> {+ using type = Dense<ElementType>;+};++template <typename LeftOperand, typename RightOperand, typename ElementType>+static METAL_FUNC typename LeftBinding<LeftOperand, RightOperand>::Args make_left_args(+ const device ElementType* a,+ const device int8_t* a_int8,+ const device float* a_scales,+ const device int32_t* a_group_sums+) {+ if constexpr (metal::is_same<LeftOperand, Dense<ElementType>>::value) {+ return {a};+ } else if constexpr (RightBinding<RightOperand>::NEEDS_CORRECTION) {+ return {a_int8, a_scales, a_group_sums};+ } else {+ return {a_int8, a_scales};+ }+}++template <typename RightOperand, typename ElementType>+static METAL_FUNC typename RightBinding<RightOperand>::Args make_right_args(+ const device ElementType* b,+ const device ElementType* scales,+ const device ElementType* biases,+ const device uint8_t* zero_points,+ const bool signed_codes+) {+ using Binding = RightBinding<RightOperand>;+ if constexpr (metal::is_same<RightOperand, Dense<ElementType>>::value) {+ return {b};+ } else {+ const device uint8_t* storage = reinterpret_cast<const device uint8_t*>(b);+ if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant) {+ return typename Binding::Args(storage, scales, biases, signed_codes);+ } else if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {+ return typename Binding::Args(storage, scales, zero_points, signed_codes);+ } else if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleSymmetricDequant) {+ return typename Binding::Args(storage, scales, signed_codes);+ } else {+ static_assert(+ RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant ||+ RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ||+ RightOperand::SCHEME == GemmBPrologueKind::ScaleSymmetricDequant,+ "unsupported right operand scheme"+ );+ }+ }+}++} // namespace operands+} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.hindex a15c132a2..30b669cf4 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h@@ -87,6 +87,30 @@ struct QuantizedBlockLoaderScaleZeroPoint {),signed_codes(signed_codes_) {}+ QuantizedBlockLoaderScaleZeroPoint(+ const device uint8_t* src_,+ const device T* scales_,+ const bool signed_codes_,+ const int src_leading_dim_,+ const int groups_per_row_,+ threadgroup T* dst_,+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],+ ushort simd_lane_id [[thread_index_in_simdgroup]]+ )+ : QuantizedBlockLoaderScaleZeroPoint(+ src_,+ scales_,+ static_cast<const device uint8_t*>(nullptr),+ signed_codes_,+ src_leading_dim_,+ groups_per_row_,+ dst_,+ simd_group_id,+ simd_lane_id+ ) {+ static_assert(SCALE_SYMMETRIC, "zero-point-free loader construction requires symmetric quantization");+ }+inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {uint zero_point_value;T scale_value;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/loader.hnew file mode 100644index 000000000..f10293622--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/loader.h@@ -0,0 +1,107 @@+#pragma once++#include <metal_stdlib>++#include "../../../../generated/gemm.h"+#include "../quant_scale_bias.h"+#include "../quant_scale_zero_point.h"++using namespace metal;++namespace uzu {+namespace gemm {++template <+ typename T,+ short THREADGROUP_TILE_ROWS,+ short THREADGROUP_TILE_COLS,+ short DESTINATION_LEADING_DIMENSION,+ short REDUCTION_DIMENSION,+ short THREADGROUP_SIZE,+ short GROUP_SIZE,+ short BITS>+struct QuantizedLoaders {+ using ScaleBias = QuantizedBlockLoaderScaleBias<+ T,+ THREADGROUP_TILE_ROWS,+ THREADGROUP_TILE_COLS,+ DESTINATION_LEADING_DIMENSION,+ REDUCTION_DIMENSION,+ THREADGROUP_SIZE,+ GROUP_SIZE,+ BITS>;+ using ScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<+ T,+ THREADGROUP_TILE_ROWS,+ THREADGROUP_TILE_COLS,+ DESTINATION_LEADING_DIMENSION,+ REDUCTION_DIMENSION,+ THREADGROUP_SIZE,+ GROUP_SIZE,+ BITS>;+ using ScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<+ T,+ THREADGROUP_TILE_ROWS,+ THREADGROUP_TILE_COLS,+ DESTINATION_LEADING_DIMENSION,+ REDUCTION_DIMENSION,+ THREADGROUP_SIZE,+ GROUP_SIZE,+ BITS,+ true>;+};++template <GemmBPrologueKind SCHEME, typename Loaders, typename RightArgs, typename ElementType>+static METAL_FUNC auto make_quantized_loader(+ const thread RightArgs& right,+ const int k_elements,+ const int groups_per_row,+ threadgroup ElementType* shared,+ const ushort simdgroup_index,+ const ushort simd_lane_id+) {+ if constexpr (SCHEME == GemmBPrologueKind::ScaleBiasDequant) {+ return typename Loaders::ScaleBias(+ right.storage.values,+ right.scales,+ right.biases,+ right.storage.signed_codes,+ k_elements,+ shared,+ simdgroup_index,+ simd_lane_id+ );+ } else if constexpr (SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {+ return typename Loaders::ScaleZeroPoint(+ right.storage.values,+ right.scales,+ right.zero_points,+ right.storage.signed_codes,+ k_elements,+ groups_per_row,+ shared,+ simdgroup_index,+ simd_lane_id+ );+ } else if constexpr (SCHEME == GemmBPrologueKind::ScaleSymmetricDequant) {+ return typename Loaders::ScaleSymmetric(+ right.storage.values,+ right.scales,+ right.storage.signed_codes,+ k_elements,+ groups_per_row,+ shared,+ simdgroup_index,+ simd_lane_id+ );+ } else {+ static_assert(+ SCHEME == GemmBPrologueKind::ScaleBiasDequant || SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ||+ SCHEME == GemmBPrologueKind::ScaleSymmetricDequant,+ "unsupported quantized loader scheme"+ );+ }+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/metadata.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/metadata.hnew file mode 100644index 000000000..1a26d12fd--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/metadata.h@@ -0,0 +1,94 @@+#pragma once++#include <metal_stdlib>++#include "../../../../generated/gemm.h"+#include "../../../common/quant_unpack.h"++using namespace metal;++namespace uzu {+namespace gemm {++// Per-thread replicated metadata. Compact row storage must consume each row+// sum in the first split-K partition.+template <ushort TILES, ushort SLOTS_PER_TILE, ushort FRAGMENT_EXTENT, ushort SLOT_STRIDE>+struct QuantizationLineCache {+ float values[TILES * SLOTS_PER_TILE];++ METAL_FUNC thread float& slot(const ushort tile, const ushort index) thread {+ return values[tile * SLOTS_PER_TILE + index];+ }++ METAL_FUNC void clear() thread {+ METAL_PRAGMA_UNROLL+ for (ushort index = 0; index < TILES * SLOTS_PER_TILE; ++index) {+ values[index] = 0.0f;+ }+ }++ METAL_FUNC float at(const short offset) const thread {+ const ushort tile = ushort(offset) / FRAGMENT_EXTENT;+ const ushort index = (ushort(offset) % FRAGMENT_EXTENT) / SLOT_STRIDE;+ return values[tile * SLOTS_PER_TILE + index];+ }++ template <bool ALIGNED, typename Sink>+ static METAL_FUNC void for_each_line(+ const short origin,+ const short limit,+ const uint abs_base,+ const uint groups_per_row,+ const uint group_index,+ Sink sink+ ) {+ METAL_PRAGMA_UNROLL+ for (ushort tile = 0; tile < TILES; ++tile) {+ METAL_PRAGMA_UNROLL+ for (ushort index = 0; index < SLOTS_PER_TILE; ++index) {+ const short coord = origin + tile * FRAGMENT_EXTENT + index * SLOT_STRIDE;+ if (ALIGNED || coord < limit) {+ const uint line_index = abs_base + uint(coord);+ sink(tile, index, line_index * groups_per_row + group_index, line_index);+ }+ }+ }+ }+};++// Symmetric formats omit correction storage entirely.+struct NoCorrectionCache {};++template <typename RightOperand>+struct Correction {+ using Format = typename RightOperand::Format;++ static METAL_FUNC float midpoint() { return float(symmetric_zero_point<Format::BITS>()); }++ template <typename RightArgs>+ static METAL_FUNC float offset(+ const float scale,+ const uint scale_index,+ const uint column,+ const uint group,+ const uint groups_per_row,+ const thread RightArgs& right+ ) {+ if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant) {+ return static_cast<float>(right.biases[scale_index]);+ } else if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {+ if constexpr (Format::BITS == 8) {+ return -scale * float(right.zero_points[scale_index]);+ } else {+ const device uint8_t* zero_points_row =+ right.zero_points + column * zero_point_row_stride<Format::BITS>(groups_per_row);+ return -scale * float(decode_zero_point<Format::BITS>(zero_points_row, group));+ }+ } else {+ return 0.0f;+ }+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/slice.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/slice.hnew file mode 100644index 000000000..7b787d79d--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/slice.h@@ -0,0 +1,30 @@+#pragma once++#include <metal_stdlib>++#include "../../../common/quant_pack.h"++using namespace metal;++namespace uzu {+namespace gemm {++struct QuantizedSlice {+ int row_stride_bytes;+ int groups_per_row;+ int first_group;+};++template <typename RightOperand>+static METAL_FUNC QuantizedSlice make_quantized_slice(const uint k_offset, const int k_elements) {+ constexpr int pack_factor = get_pack_factor<RightOperand::BITS, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<RightOperand::BITS>();+ return QuantizedSlice{+ k_elements * bytes_per_pack / pack_factor,+ (k_elements + RightOperand::GROUP_SIZE - 1) / RightOperand::GROUP_SIZE,+ int(k_offset) / RightOperand::GROUP_SIZE,+ };+}++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/source.hnew file mode 100644index 000000000..f63ad7040--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/source.h@@ -0,0 +1,107 @@+#pragma once++#include <metal_stdlib>++#include "../../../common/fragment.h"+#include "../../../common/mxu_fragment/ops.h"+#include "../../../common/quant_unpack.h"++using namespace metal;++namespace uzu {+namespace gemm {++using uzu::matmul::DeviceTensorOperand;+using uzu::matmul::IntegerFormat;+using uzu::matmul::Signedness;++using SignedInt4Format = IntegerFormat<4, Signedness::Signed>;+using SignedInt8Format = IntegerFormat<8, Signedness::Signed>;++template <typename Format, bool ALIGNED_N, ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>+struct QuantizedSource;++template <ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>+struct QuantizedSource<SignedInt4Format, true, TILES_N, TILES_K, SIMDGROUP_BLOCK_K> {+ static METAL_FUNC DeviceTensorOperand<SignedInt4Format> make(+ const device uint8_t* storage,+ const int k_element_offset,+ const int row_stride_bytes,+ const short,+ const short2,+ const ushort+ ) {+ return DeviceTensorOperand<SignedInt4Format>{storage + (k_element_offset >> 1), row_stride_bytes};+ }+};++template <ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>+struct QuantizedSource<SignedInt4Format, false, TILES_N, TILES_K, SIMDGROUP_BLOCK_K> {+ using Ops = uzu::matmul::MxuFragmentOps<>;+ using RegisterTile = uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops>;++ static METAL_FUNC RegisterTile make(+ const device uint8_t* storage,+ const int k_element_offset,+ const int row_stride_bytes,+ const short simdgroup_limit_n,+ const short2 position,+ const ushort+ ) {+ RegisterTile tile;+ METAL_PRAGMA_UNROLL+ for (ushort tile_n = 0; tile_n < TILES_N; ++tile_n) {+ METAL_PRAGMA_UNROLL+ for (ushort tile_k = 0; tile_k < TILES_K; ++tile_k) {+ thread auto& vector = tile.fragment_at(tile_n, tile_k);+ METAL_PRAGMA_UNROLL+ for (ushort thread_row = 0; thread_row < Ops::THREAD_ELEMENT_ROWS; ++thread_row) {+ const short row =+ short(tile_n * Ops::FRAGMENT_ROWS) + position.y + short(thread_row * Ops::THREAD_ELEMENT_ROW_STRIDE);+ const ushort element_base = thread_row * Ops::THREAD_ELEMENT_COLS;+ char4 codes = char4(0);+ if (row < simdgroup_limit_n) {+ const int k_base = k_element_offset + int(tile_k * Ops::FRAGMENT_COLS) + int(position.x);+ const ushort packed =+ *reinterpret_cast<const device ushort*>(storage + int(row) * row_stride_bytes + (k_base >> 1));+ codes = unpack_signed_nibbles_to_int8(uint(packed));+ }+ vector[element_base + 0] = codes.x;+ vector[element_base + 1] = codes.y;+ vector[element_base + 2] = codes.z;+ vector[element_base + 3] = codes.w;+ }+ }+ }+ return tile;+ }+};++template <bool ALIGNED_N, ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>+struct QuantizedSource<SignedInt8Format, ALIGNED_N, TILES_N, TILES_K, SIMDGROUP_BLOCK_K> {+ using Ops = uzu::matmul::MxuFragmentOps<>;+ using RegisterTile = uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops>;++ static METAL_FUNC RegisterTile make(+ const device uint8_t* storage,+ const int k_element_offset,+ const int row_stride_bytes,+ const short simdgroup_limit_n,+ const short2,+ const ushort simd_lane_id+ ) {+ RegisterTile tile;+ auto source = uzu::matmul::fragment_source(+ reinterpret_cast<const device int8_t*>(storage) + k_element_offset,+ row_stride_bytes+ );+ if constexpr (!ALIGNED_N) {+ source = source.bounded(simdgroup_limit_n, SIMDGROUP_BLOCK_K);+ }+ tile.load_from(simd_lane_id, source);+ return tile;+ }+};++} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/dense.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/dense.hnew file mode 100644index 000000000..2c62cafe9--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/dense.h@@ -0,0 +1,66 @@+#pragma once++#include "../../../../common/integral_constant.h"+#include "../../../../common/thread_context.h"+#include "../../../../generated/gemm.h"+#include "../../../common/fragment.h"+#include "../../../common/mxu_gemm_loop.h"+#include "../gemm_alignment.h"+#include "tile_context.h"++using namespace metal;++namespace uzu {+namespace gemm {+namespace schedules {++struct DenseSchedule {+ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>+ static METAL_FUNC typename Core::AccumFragment launch(+ typename Core::LeftArgs left,+ typename Core::RightArgs right,+ threadgroup typename Core::RightElementType*,+ const constant uzu::matmul::GemmParams* params,+ const TileContext tile,+ const GemmAlignment alignment,+ const thread ThreadContext& thread_context+ ) {+ right.template seek_columns<Core::TRANSPOSE_RIGHT>(+ tile.block_col + tile.tile_col_offset,+ tile.k_offset,+ params->leading_dimension_b+ );++ typename Core::AccumFragment accumulator;+ dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {+ accumulator = uzu::matmul::gemm_loop<+ typename Core::LeftElementType,+ typename Core::RightElementType,+ Core::SIMDGROUP_BLOCK_M,+ Core::SIMDGROUP_BLOCK_N,+ Core::SIMDGROUP_BLOCK_K,+ Core::THREADGROUP_BLOCK_K_FP,+ false,+ Core::TRANSPOSE_RIGHT,+ ALIGNED_M,+ ALIGNED_N,+ aligned_k.value,+ typename Core::AccumulatorType>(+ left.values,+ right.values,+ int(params->leading_dimension_a),+ int(params->leading_dimension_b),+ int(params->K),+ int(params->aligned_inner_iterations),+ tile.simdgroup_limit_m,+ tile.simdgroup_limit_n,+ thread_context+ );+ });+ return accumulator;+ }+};++} // namespace schedules+} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.hnew file mode 100644index 000000000..25a57d7ca--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h@@ -0,0 +1,187 @@+#pragma once++#include <metal_stdlib>++#include "../../../../common/thread_context.h"+#include "../../../../generated/gemm.h"+#include "../../../common/fragment.h"+#include "../../../common/mxu_fragment/ops.h"+#include "../gemm_alignment.h"+#include "../integer_source.h"+#include "../quantized/metadata.h"+#include "../quantized/slice.h"+#include "../quantized/source.h"+#include "tile_context.h"++using namespace metal;++namespace uzu {+namespace gemm {+namespace schedules {++template <typename LeftOperand, typename RightOperand>+struct IntegerSchedule {+ using RightFormat = typename RightOperand::Format;+ using RightBinding = operands::RightBinding<RightOperand>;+ UZU_CONST bool needs_correction = RightBinding::NEEDS_CORRECTION;++ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>+ static METAL_FUNC typename Core::AccumFragment launch(+ typename Core::LeftArgs left,+ typename Core::RightArgs right,+ threadgroup typename Core::RightElementType*,+ const constant uzu::matmul::GemmParams* params,+ const TileContext tile,+ const GemmAlignment,+ const thread ThreadContext& thread_context+ ) {+ using Ops = typename Core::FragmentOps;+ using AccumFragment = typename Core::AccumFragment;+ using LeftScaleCache = QuantizationLineCache<+ Core::TILES_M,+ Ops::THREAD_ELEMENT_ROWS,+ Ops::FRAGMENT_ROWS,+ Ops::THREAD_ELEMENT_ROW_STRIDE>;+ using RightScaleCache = QuantizationLineCache<Core::TILES_N, Ops::THREAD_ELEMENT_COLS, Ops::FRAGMENT_COLS, 1>;+ using CorrectionPolicy = Correction<RightOperand>;+ using ColumnCorrectionCache = metal::conditional_t<needs_correction, RightScaleCache, NoCorrectionCache>;+ using RowCorrectionCache = metal::conditional_t<needs_correction, LeftScaleCache, NoCorrectionCache>;++ static_assert(RightOperand::GROUP_SIZE % Core::SIMDGROUP_BLOCK_K == 0, "right groups must contain MMA chunks");++ const auto slice = make_quantized_slice<RightOperand>(tile.k_offset, int(params->K));+ right.storage.seek_block(tile.block_col, tile.k_offset, slice.row_stride_bytes);+ right.storage.seek_columns(tile.tile_col_offset, slice.row_stride_bytes);++ const int leading_dimension_a = int(params->leading_dimension_a);+ const int b_row_stride_bytes = slice.row_stride_bytes;+ const int right_group_iterations = int(params->aligned_inner_iterations);+ const short simdgroup_limit_m = tile.simdgroup_limit_m;+ const short simdgroup_limit_n = tile.simdgroup_limit_n;+ const uint abs_row_base = tile.abs_row_base;+ const uint abs_col_base = tile.absolute_column_base();+ const uint k_offset_right_groups = uint(slice.first_group);+ const uint k_offset_left_scale_groups = tile.k_offset / uint(LeftOperand::GROUP_SIZE);+ const uint right_groups_per_row = uint(slice.groups_per_row);+ const uint left_scale_groups_per_row = uint(params->K) / uint(LeftOperand::GROUP_SIZE);++ AccumFragment accumulator;+ accumulator.clear();++ const short2 position = Ops::get_position(thread_context.simd_lane_id);+ constexpr int k_bytes_per_right_group = int(RightOperand::GROUP_SIZE) * RightOperand::BITS / 8;+ constexpr int left_groups_per_right_group = int(RightOperand::GROUP_SIZE) / int(LeftOperand::GROUP_SIZE);++ METAL_PRAGMA_NO_UNROLL+ for (int right_group = 0; right_group < right_group_iterations; ++right_group) {+ const uint right_group_index = k_offset_right_groups + uint(right_group);++ RightScaleCache right_scales;+ ColumnCorrectionCache column_corrections;+ RightScaleCache::template for_each_line<ALIGNED_N>(+ position.x,+ simdgroup_limit_n,+ abs_col_base,+ right_groups_per_row,+ right_group_index,+ [&](ushort tile_col, ushort thread_col, uint right_scale_index, uint right_column) {+ const float scale = static_cast<float>(right.scales[right_scale_index]);+ right_scales.slot(tile_col, thread_col) = scale;+ if constexpr (needs_correction) {+ column_corrections.slot(tile_col, thread_col) =+ scale * CorrectionPolicy::midpoint() + CorrectionPolicy::offset(+ scale,+ right_scale_index,+ right_column,+ right_group_index,+ right_groups_per_row,+ right+ );+ }+ }+ );++ RowCorrectionCache row_corrections;+ if constexpr (needs_correction) {+ row_corrections.clear();+ }++ METAL_PRAGMA_NO_UNROLL+ for (int left_group = 0; left_group < left_groups_per_right_group; ++left_group) {+ const int k_element_offset = left_group * int(LeftOperand::GROUP_SIZE);+ auto left_tile = load_integer_tile<int8_t, Ops, ALIGNED_M, Core::TILES_M, Core::TILES_K>(+ left.values + k_element_offset,+ leading_dimension_a,+ simdgroup_limit_m,+ thread_context.simd_lane_id+ );++ uzu::matmul::Fragment<int, Core::TILES_M, Core::TILES_N, Ops> chunk_products;+ auto right_tile =+ QuantizedSource<RightFormat, ALIGNED_N, Core::TILES_N, Core::TILES_K, Core::SIMDGROUP_BLOCK_K>::make(+ right.storage.values,+ k_element_offset,+ b_row_stride_bytes,+ simdgroup_limit_n,+ position,+ thread_context.simd_lane_id+ );+ Ops::template fragment_mm<false, true>(chunk_products, left_tile, right_tile);++ const uint left_scale_group_index =+ k_offset_left_scale_groups + uint(right_group * left_groups_per_right_group + left_group);+ LeftScaleCache left_scales;+ LeftScaleCache::template for_each_line<ALIGNED_M>(+ position.y,+ simdgroup_limit_m,+ abs_row_base,+ left_scale_groups_per_row,+ left_scale_group_index,+ [&](ushort tile_row, ushort thread_row, uint left_scale_index, uint) {+ const float left_scale = left.scales[left_scale_index];+ left_scales.slot(tile_row, thread_row) = left_scale;+ if constexpr (needs_correction) {+ row_corrections.slot(tile_row, thread_row) += left_scale * float(left.group_sums[left_scale_index]);+ }+ }+ );++ AccumFragment::zip_for_each_coord(+ thread_context.simd_lane_id,+ [&](short row, short col, thread float& accumulated, thread int& products) {+ if (!ALIGNED_M && row >= simdgroup_limit_m) {+ return;+ }+ if (!ALIGNED_N && col >= simdgroup_limit_n) {+ return;+ }+ accumulated += left_scales.at(row - position.y) * right_scales.at(col - position.x) * float(products);+ },+ accumulator,+ chunk_products+ );+ }++ if constexpr (needs_correction) {+ accumulator.map_coords(thread_context.simd_lane_id, [&](short row, short col, float value) {+ if (!ALIGNED_M && row >= simdgroup_limit_m) {+ return value;+ }+ if (!ALIGNED_N && col >= simdgroup_limit_n) {+ return value;+ }+ return value + column_corrections.at(col - position.x) * row_corrections.at(row - position.y);+ });+ }++ left.advance_k(RightOperand::GROUP_SIZE);+ right.storage.advance_k(k_bytes_per_right_group);+ }++ return accumulator;+ }+};++} // namespace schedules+} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/selector.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/selector.hnew file mode 100644index 000000000..e88ee2584--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/selector.h@@ -0,0 +1,43 @@+#pragma once++#include "../../../../generated/gemm.h"+#include "../operands.h"+#include "dense.h"+#include "integer.h"+#include "staged.h"++namespace uzu {+namespace gemm {+namespace schedules {++template <typename LeftOperand, typename RightOperand>+struct ScheduleFor;++template <typename LeftElementType, typename RightElementType>+struct ScheduleFor<operands::Dense<LeftElementType>, operands::Dense<RightElementType>> {+ using type = DenseSchedule;+};++template <typename LeftElementType, typename Format, ushort GROUP_SIZE, GemmBPrologueKind SCHEME, typename ElementType>+struct ScheduleFor<operands::Dense<LeftElementType>, operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>> {+ using type = StagedSchedule;+};++template <+ typename LeftFormat,+ ushort LEFT_GROUP_SIZE,+ typename RightFormat,+ ushort GROUP_SIZE,+ GemmBPrologueKind SCHEME,+ typename ElementType>+struct ScheduleFor<+ operands::Quantized<LeftFormat, LEFT_GROUP_SIZE, GemmBPrologueKind::ScaleSymmetricDequant, float>,+ operands::Quantized<RightFormat, GROUP_SIZE, SCHEME, ElementType>> {+ using type = IntegerSchedule<+ operands::Quantized<LeftFormat, LEFT_GROUP_SIZE, GemmBPrologueKind::ScaleSymmetricDequant, float>,+ operands::Quantized<RightFormat, GROUP_SIZE, SCHEME, ElementType>>;+};++} // namespace schedules+} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.hnew file mode 100644index 000000000..fc2a4e0e8--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h@@ -0,0 +1,172 @@+#pragma once++#include <metal_stdlib>++#include "../../../../common/thread_context.h"+#include "../../../../generated/gemm.h"+#include "../../../common/fragment.h"+#include "../../../common/mxu_fragment/ops.h"+#include "../../../common/mxu_gemm_loop.h"+#include "../gemm_alignment.h"+#include "../integer_source.h"+#include "../operands.h"+#include "../quantized/loader.h"+#include "../quantized/slice.h"+#include "tile_context.h"++using namespace metal;++namespace uzu {+namespace gemm {+namespace schedules {++template <typename RightOperand, ushort DENSE_BLOCK_K, ushort SIMDGROUP_BLOCK_K>+struct OuterBlockK;++template <typename ElementType, ushort DENSE_BLOCK_K, ushort SIMDGROUP_BLOCK_K>+struct OuterBlockK<operands::Dense<ElementType>, DENSE_BLOCK_K, SIMDGROUP_BLOCK_K> {+ UZU_CONST ushort VALUE = DENSE_BLOCK_K;+};++template <+ typename Format,+ ushort GROUP_SIZE,+ GemmBPrologueKind SCHEME,+ typename ElementType,+ ushort DENSE_BLOCK_K,+ ushort SIMDGROUP_BLOCK_K>+struct OuterBlockK<operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>, DENSE_BLOCK_K, SIMDGROUP_BLOCK_K> {+ UZU_CONST ushort VALUE = GROUP_SIZE;++ static_assert(VALUE % SIMDGROUP_BLOCK_K == 0, "quantized group size must contain complete MMA chunks");+ static_assert(DENSE_BLOCK_K % VALUE == 0, "tile block K must contain complete quantized groups");+};++template <typename RightOperand>+struct StagedRightLoader;++template <typename ElementType>+struct StagedRightLoader<operands::Dense<ElementType>> {+ template <typename Core>+ static METAL_FUNC typename Core::FullPrecisionRightLoader make(+ typename Core::RightArgs right,+ const constant uzu::matmul::GemmParams* params,+ const size_t block_col,+ const uint k_offset,+ threadgroup typename Core::RightElementType* staging,+ const thread ThreadContext& thread_context+ ) {+ right.template seek_columns<Core::TRANSPOSE_RIGHT>(block_col, k_offset, params->leading_dimension_b);+ return typename Core::FullPrecisionRightLoader(right.values, params->leading_dimension_b, staging, thread_context);+ }+};++template <typename Format, ushort GROUP_SIZE, GemmBPrologueKind SCHEME, typename ElementType>+struct StagedRightLoader<operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>> {+ template <typename Core>+ static METAL_FUNC auto make(+ typename Core::RightArgs right,+ const constant uzu::matmul::GemmParams* params,+ const size_t block_col,+ const uint k_offset,+ threadgroup typename Core::RightElementType* staging,+ const thread ThreadContext& thread_context+ ) {+ const int k_elements = int(params->K);+ const auto slice =+ make_quantized_slice<operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>>(k_offset, k_elements);+ right.storage.seek_block(block_col, k_offset, slice.row_stride_bytes);+ operands::seek_quantized_metadata(right, block_col, slice.groups_per_row, slice.first_group);++ using Loaders = QuantizedLoaders<+ typename Core::RightElementType,+ Core::THREADGROUP_BLOCK_N,+ Core::THREADGROUP_BLOCK_K,+ Core::SHARED_STRIDE_B,+ 1,+ Core::THREADGROUP_THREADS,+ GROUP_SIZE,+ Format::BITS>;++ return make_quantized_loader<SCHEME, Loaders>(+ right,+ k_elements,+ slice.groups_per_row,+ staging,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ }+};++// Bit width and group size come from the right operand descriptor.+struct StagedSchedule {+ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>+ static METAL_FUNC typename Core::AccumFragment launch(+ typename Core::LeftArgs left,+ typename Core::RightArgs right,+ threadgroup typename Core::RightElementType* staging,+ const constant uzu::matmul::GemmParams* params,+ const TileContext tile,+ const GemmAlignment,+ const thread ThreadContext& thread_context+ ) {+ using FragmentOps = typename Core::FragmentOps;+ using RightOperand = typename Core::RightOperand;++ auto loader_b = StagedRightLoader<RightOperand>::template make<Core>(+ right,+ params,+ tile.block_col,+ tile.k_offset,+ staging,+ thread_context+ );++ typename Core::AccumFragment accumulator;+ accumulator.clear();++ threadgroup typename Core::RightElementType* staging_simdgroup =+ staging + tile.tile_col_offset * Core::SHARED_STRIDE_B;+ const short2 tile_dimensions_b = short2(Core::THREADGROUP_BLOCK_K, tile.tile_block_cols);++ METAL_PRAGMA_NO_UNROLL+ for (int outer_k = 0; outer_k < int(params->aligned_inner_iterations); ++outer_k) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if constexpr (ALIGNED_N) {+ loader_b.load_unsafe();+ } else {+ loader_b.load_safe(tile_dimensions_b);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ METAL_PRAGMA_NO_UNROLL+ for (int inner_k = 0; inner_k < Core::THREADGROUP_BLOCK_K; inner_k += Core::SIMDGROUP_BLOCK_K) {+ uzu::matmul::Fragment<typename Core::LeftElementType, Core::TILES_M, Core::TILES_K, FragmentOps> left_tile;+ uzu::matmul::Fragment<typename Core::RightElementType, Core::TILES_N, Core::TILES_K, FragmentOps> right_tile;++ auto left_src = uzu::matmul::fragment_source(left.values + inner_k, int(params->leading_dimension_a));+ if constexpr (!ALIGNED_M) {+ left_src = left_src.bounded(tile.simdgroup_limit_m, Core::SIMDGROUP_BLOCK_K);+ }+ left_tile.load_from(thread_context.simd_lane_id, left_src);++ right_tile.load_from(+ thread_context.simd_lane_id,+ uzu::matmul::fragment_source(staging_simdgroup + inner_k, int(Core::SHARED_STRIDE_B))+ );++ FragmentOps::template fragment_mma<false, true>(accumulator, left_tile, right_tile);+ }++ left.advance_k(Core::THREADGROUP_BLOCK_K);+ loader_b.next();+ }++ return accumulator;+ }+};++} // namespace schedules+} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/tile_context.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/tile_context.hnew file mode 100644index 000000000..a0af83f6d--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/tile_context.h@@ -0,0 +1,25 @@+#pragma once++#include <metal_stdlib>++using namespace metal;++namespace uzu {+namespace gemm {+namespace schedules {++struct TileContext {+ short simdgroup_limit_m;+ short simdgroup_limit_n;+ size_t block_col;+ ushort tile_col_offset;+ ushort tile_block_cols;+ uint k_offset;+ uint abs_row_base;++ METAL_FUNC uint absolute_column_base() const { return uint(block_col) + uint(tile_col_offset); }+};++} // namespace schedules+} // namespace gemm+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 86c405de5..814211298 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -14,6 +14,8 @@#include "gemm_tiling.h"#include "quant_scale_bias.h"#include "quant_scale_zero_point.h"+#include "operands.h"+#include "schedules/staged.h"using namespace metal;@@ -21,67 +23,55 @@ namespace uzu {namespace gemm {template <- typename AT,- typename BT,- typename DT,+ typename OutputElementType_,GemmTiling GEMM_TILING,bool TRANSPOSE_B,- GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,- int BITS = 0,- int GROUP_SIZE = 0>+ typename LeftOperand,+ typename RightOperand>struct SimdgroupMmaCore {+ using OutputElementType = OutputElementType_;+ using Left = operands::LeftBinding<LeftOperand, RightOperand>;+ using Right = operands::RightBinding<RightOperand>;+ using LeftElementType = typename Left::ElementType;+ using RightElementType = typename Right::ElementType;+ using LeftArgs = typename Left::Args;+ using RightArgs = typename Right::Args;+ static_assert(+ metal::is_same<LeftOperand, operands::Dense<LeftElementType>>::value,+ "simdgroup MMA stages full-precision left operands only"+ );+ UZU_CONST bool TRANSPOSE_RIGHT = TRANSPOSE_B;UZU_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);UZU_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);UZU_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);UZU_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);UZU_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);- UZU_CONST ushort PADDING_A = 16 / sizeof(AT);- UZU_CONST ushort PADDING_B = 16 / sizeof(BT);+ UZU_CONST ushort PADDING_A = 16 / sizeof(LeftElementType);+ UZU_CONST ushort PADDING_B = 16 / sizeof(RightElementType);UZU_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;UZU_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;+ UZU_CONST ushort STAGING_ELEMENTS =+ TRANSPOSE_B ? THREADGROUP_BLOCK_N * SHARED_STRIDE_B : THREADGROUP_BLOCK_K * SHARED_STRIDE_B;UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;- using ALoader = uzu::matmul::- ThreadgroupLoader<AT, THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_K, SHARED_STRIDE_A, true, THREADGROUP_THREADS>;- using BLoaderFp = uzu::matmul::ThreadgroupLoader<- BT,+ using ALoader = uzu::matmul::ThreadgroupLoader<+ LeftElementType,+ THREADGROUP_BLOCK_M,+ THREADGROUP_BLOCK_K,+ SHARED_STRIDE_A,+ true,+ THREADGROUP_THREADS>;+ using FullPrecisionRightLoader = uzu::matmul::ThreadgroupLoader<+ RightElementType,TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,SHARED_STRIDE_B,TRANSPOSE_B,THREADGROUP_THREADS>;- using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<- BT,- THREADGROUP_BLOCK_N,- THREADGROUP_BLOCK_K,- SHARED_STRIDE_B,- 1,- THREADGROUP_THREADS,- GROUP_SIZE,- BITS>;- using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<- BT,- THREADGROUP_BLOCK_N,- THREADGROUP_BLOCK_K,- SHARED_STRIDE_B,- 1,- THREADGROUP_THREADS,- GROUP_SIZE,- BITS>;- using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<- BT,- THREADGROUP_BLOCK_N,- THREADGROUP_BLOCK_K,- SHARED_STRIDE_B,- 1,- THREADGROUP_THREADS,- GROUP_SIZE,- BITS,- true>;using TileAccumulator = uzu::matmul::ThreadgroupTile<- AT,- BT,- DT,+ LeftElementType,+ RightElementType,+ OutputElementType,THREADGROUP_BLOCK_M,THREADGROUP_BLOCK_N,THREADGROUP_BLOCK_K,@@ -92,12 +82,12 @@ struct SimdgroupMmaCore {SHARED_STRIDE_A,SHARED_STRIDE_B,float,- uzu::matmul::TransformNone<DT, float>>;+ uzu::matmul::TransformNone<OutputElementType, float>>;template <uint GEMM_ALIGNMENT_RAW, typename BLoader>static METAL_FUNC void k_loop(- threadgroup AT* a_shared,- threadgroup BT* b_shared,+ threadgroup LeftElementType* a_shared,+ threadgroup RightElementType* b_shared,const int aligned_k_iterations,thread ALoader& loader_a,thread BLoader& loader_b,@@ -149,13 +139,13 @@ struct SimdgroupMmaCore {template <uint GEMM_ALIGNMENT_RAW>static METAL_FUNC void finalize(thread TileAccumulator& accumulator,- device DT* d,+ device OutputElementType* d,const constant uzu::matmul::GemmParams* params,const thread ushort& tile_block_rows,const thread ushort& tile_block_cols,const bool needs_epilogue,const thread uzu::matmul::TransformScaleAccumulate<float, float>& epilogue,- const device BT* bias_block,+ const device RightElementType* bias_block,const bool needs_bias,const device int32_t* rht_factors_block,const bool needs_rht,@@ -196,20 +186,16 @@ struct SimdgroupMmaCore {}static METAL_FUNC void run(- const device AT* a,- const device BT* b,- device DT* d,+ LeftArgs left,+ RightArgs right,+ device OutputElementType* d,const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,GemmDTransform output_transform,- const bool signed_codes,- const device BT* scales,- const device BT* biases,- const device uint8_t* zero_points,- const device BT* output_bias,+ const device RightElementType* output_bias,const device int32_t* rht_factors,- threadgroup AT* a_shared,- threadgroup BT* b_shared,+ threadgroup LeftElementType* a_shared,+ threadgroup RightElementType* b_shared,const thread ThreadContext& thread_context) {const uint partition = thread_context.threadgroup_position.z;@@ -227,11 +213,11 @@ struct SimdgroupMmaCore {const size_t block_row = size_t(geometry.block_row_start);const size_t block_col = size_t(geometry.block_col_start);- a += block_row * params->leading_dimension_a + k_offset;+ left.seek_rows(block_row, params->leading_dimension_a, k_offset);d +=size_t(partition) * size_t(params->M) * size_t(params->N) + block_row * params->leading_dimension_d + block_col;- thread ALoader loader_a(a, params->leading_dimension_a, a_shared, thread_context);+ thread ALoader loader_a(left.values, params->leading_dimension_a, a_shared, thread_context);thread TileAccumulator accumulator(thread_context);const ushort tile_block_rows =@@ -248,68 +234,17 @@ struct SimdgroupMmaCore {const float alpha = needs_scale ? params->ab_scale : 1.0f;const float beta = needs_accumulate ? 1.0f : 0.0f;uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);- const device BT* bias_block = output_bias + block_col;+ const device RightElementType* bias_block = output_bias + block_col;const device int32_t* rht_factors_block = rht_factors + block_col;- auto loader_b = [&]() {- if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {- const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +- (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);- return BLoaderFp(b_block_fp, params->leading_dimension_b, b_shared, thread_context);- } else {- constexpr int pack_factor = get_pack_factor<BITS, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();- const int k_elements = int(params->K);- const int weights_row_stride_bytes = k_elements * bytes_per_pack / pack_factor;- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;- const int k_offset_groups = int(k_offset) / GROUP_SIZE;- const device uint8_t* weights_block = reinterpret_cast<const device uint8_t*>(b) +- block_col * weights_row_stride_bytes +- int(k_offset) * bytes_per_pack / pack_factor;- const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;-- if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {- const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;- return BLoaderScaleBias(- weights_block,- scales_offset,- biases_offset,- signed_codes,- k_elements,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id- );- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {- const int zero_point_stride_per_row = (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;- const device uint8_t* zero_points_row_start = zero_points + block_col * zero_point_stride_per_row +- ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);- return BLoaderScaleZeroPoint(- weights_block,- scales_offset,- zero_points_row_start,- signed_codes,- k_elements,- groups_per_row,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id- );- } else {- return BLoaderScaleSymmetric(- weights_block,- scales_offset,- nullptr,- signed_codes,- k_elements,- groups_per_row,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id- );- }- }- }();+ auto loader_b = schedules::StagedRightLoader<RightOperand>::template make<SimdgroupMmaCore>(+ right,+ params,+ block_col,+ k_offset,+ b_shared,+ thread_context+ );const bool all_aligned = ((alignment.contains(GemmAlignment::M)) || (tile_block_rows == THREADGROUP_BLOCK_M)) &&((alignment.contains(GemmAlignment::N)) || (tile_block_cols == THREADGROUP_BLOCK_N)) &&diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex c8da2bb8e..ffc214532 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -140,38 +140,40 @@ KERNEL(Gemm)((void)thread_y;(void)thread_z;+ using LeftOperand = typename operands::LeftOperandFor<A_PROLOGUE, AT>::type;+ using RightOperand = typename operands::RightOperandFor<B_PROLOGUE, ushort(BITS), ushort(GROUP_SIZE), BT>::type;+ static_assert(+ NEEDS_ASYMMETRIC_WEIGHT_CORRECTION == (A_IS_INT8 && operands::RightBinding<RightOperand>::NEEDS_CORRECTION),+ "kernel bindings and operand correction policy must agree"+ );+ const auto left_args = operands::make_left_args<LeftOperand, RightOperand>(a, a_int8, a_scales, a_group_sums);+ const auto right_args = operands::make_right_args<RightOperand>(b, scales, biases, zero_points, signed_codes);+if constexpr (USE_MXU) {- MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE, A_PROLOGUE>::run(- a,- b,+ using Core = MxuMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;+ static_assert(GEMM_TGB_ELEMENTS >= Core::STAGING_ELEMENTS, "threadgroup right staging is undersized");+ Core::run(+ left_args,+ right_args,d,params,alignment,output_transform,- signed_codes,- scales,- biases,- zero_points,output_bias,rht_factors,- a_int8,- a_scales,- a_group_sums,b_shared,thread_context);} else {- SimdgroupMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE>::run(- a,- b,+ using Core = SimdgroupMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;+ static_assert(GEMM_TGB_ELEMENTS >= Core::STAGING_ELEMENTS, "threadgroup right staging is undersized");+ Core::run(+ left_args,+ right_args,d,params,alignment,output_transform,- signed_codes,- scales,- biases,- zero_points,output_bias,rht_factors,a_shared,