Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

`MXU` refactor (#673)

CCYeh8e28784d4793
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.h
deleted file mode 100644
index 643b9a7b6..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/device_weight_matmul.h
+++ /dev/null
@@ -1,53 +0,0 @@
-template <typename RightElement, class OutputFragment, class LeftFragment>
-METAL_FUNC static void fragment_matmul_int8_device_weights(
- thread OutputFragment& output,
- thread LeftFragment& left,
- const device uchar* right_signed_codes,
- const int right_row_stride_bytes
-) {
- static_assert(RELAXED, "device weight tensors require the relaxed MXU layout");
- static_assert(LeftFragment::COL_FRAGMENTS == 2, "device weight tensors expect K tiled as two fragments");
- static_assert(OutputFragment::COL_FRAGMENTS % 2 == 0, "device weight tensors require even N fragments");
- static_assert(LeftFragment::ROW_FRAGMENTS == OutputFragment::ROW_FRAGMENTS, "M tiles must match");
-
- constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;
- constexpr ushort cols = OutputFragment::COL_FRAGMENTS;
- constexpr int tile_k = int(2 * FRAGMENT_COLS);
- constexpr int tile_n = int(2 * FRAGMENT_COLS);
- constexpr int element_bits = metal::is_same_v<RightElement, metal::int4b_format> ? 4 : 8;
- constexpr int elements_per_byte = 8 / element_bits;
- using RightTensor = tensor<device RightElement, extents<int, tile_k, tile_n>, tensor_inline>;
- using RightPointer =
- metal::conditional_t<metal::is_same_v<RightElement, metal::int4b_format>, device uchar*, device RightElement*>;
-
- constexpr auto descriptor =
- mpp::tensor_ops::matmul2d_descriptor(FRAGMENT_ROWS, tile_n, tile_k, false, true, RELAXED, MatmulMode::multiply);
- mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;
-
- const array<int, 2> right_strides = {1, right_row_stride_bytes * elements_per_byte};
-
- METAL_PRAGMA_UNROLL
- for (ushort row = 0; row < rows; ++row) {
- METAL_PRAGMA_UNROLL
- for (ushort col = 0; col < cols; col += 2) {
- auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<int8_t, RightElement, int>();
- load_paired_vectors(cooperative_left, left.fragment_at(row, 0), left.fragment_at(row, 1));
-
- RightTensor right_tensor(
- // MPP rejects const-qualified tensor element types.
- reinterpret_cast<RightPointer>(
- const_cast<device uchar*>(right_signed_codes + int(col * FRAGMENT_COLS) * right_row_stride_bytes)
- ),
- extents<int, tile_k, tile_n>{},
- right_strides
- );
-
- auto cooperative_output =
- matmul_op.template get_destination_cooperative_tensor<decltype(cooperative_left), RightTensor, int>();
-
- matmul_op.run(cooperative_left, right_tensor, cooperative_output);
-
- store_paired_vectors(cooperative_output, output.fragment_at(row, col), output.fragment_at(row, col + 1));
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
index b0e441729..77e4f7a3d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/fragment_matmul.h
@@ -1,7 +1,51 @@
+// Fragment storage is paired into the 16x32 cooperative tile used here.
template <
- bool ACCUMULATE,
- bool transpose_a,
- bool transpose_b,
+ MatmulMode OUTPUT_MODE,
+ typename OutputType,
+ typename LeftType,
+ typename RightType,
+ bool transpose_left,
+ bool transpose_right,
+ typename MarshalInputs>
+METAL_FUNC static void matmul(
+ thread ThreadVector<OutputType>& output_0,
+ thread ThreadVector<OutputType>& output_1,
+ MarshalInputs marshal_inputs
+) {
+ constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(
+ FRAGMENT_ROWS,
+ 2 * FRAGMENT_COLS,
+ FRAGMENT_COLS,
+ transpose_left,
+ transpose_right,
+ RELAXED,
+ OUTPUT_MODE
+ );
+
+ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;
+
+ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<LeftType, RightType, OutputType>();
+ auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<LeftType, RightType, OutputType>();
+ auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<
+ decltype(cooperative_left),
+ decltype(cooperative_right),
+ OutputType>();
+
+ marshal_inputs(cooperative_left, cooperative_right);
+
+ if constexpr (OUTPUT_MODE == MatmulMode::multiply_accumulate) {
+ load_paired_vectors(cooperative_output, output_0, output_1);
+ }
+
+ matmul_op.run(cooperative_left, cooperative_right, cooperative_output);
+
+ store_paired_vectors(cooperative_output, output_0, output_1);
+}
+
+template <
+ MatmulMode OUTPUT_MODE,
+ bool transpose_left,
+ bool transpose_right,
class OutputFragment,
class LeftFragment,
class RightFragment>
@@ -10,16 +54,16 @@ METAL_FUNC static void fragment_matmul(
thread LeftFragment& left,
thread RightFragment& right
) {
- constexpr ushort left_rows = transpose_a ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;
+ constexpr ushort left_rows = transpose_left ? LeftFragment::COL_FRAGMENTS : LeftFragment::ROW_FRAGMENTS;
constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;
static_assert(left_rows == rows, "fragment matmul: M dimensions do not match");
- constexpr ushort right_cols = transpose_b ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;
+ constexpr ushort right_cols = transpose_right ? RightFragment::ROW_FRAGMENTS : RightFragment::COL_FRAGMENTS;
constexpr ushort cols = OutputFragment::COL_FRAGMENTS;
static_assert(right_cols == cols, "fragment matmul: N dimensions do not match");
- constexpr ushort left_depth = transpose_a ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;
- constexpr ushort depth = transpose_b ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;
+ constexpr ushort left_depth = transpose_left ? LeftFragment::ROW_FRAGMENTS : LeftFragment::COL_FRAGMENTS;
+ constexpr ushort depth = transpose_right ? RightFragment::COL_FRAGMENTS : RightFragment::ROW_FRAGMENTS;
static_assert(left_depth == depth, "fragment matmul: K dimensions do not match");
static_assert(
@@ -27,43 +71,54 @@ METAL_FUNC static void fragment_matmul(
"MXU fragment_mma requires even N, or N==1 with even M (MPP pairing)"
);
- constexpr auto transpose_left = metal::bool_constant<transpose_a>{};
- constexpr auto transpose_right = metal::bool_constant<transpose_b>{};
+ constexpr auto left_transpose = metal::bool_constant<transpose_left>{};
+ constexpr auto right_transpose = metal::bool_constant<transpose_right>{};
constexpr bool pair_output_rows = (cols == 1 && rows % 2 == 0);
- auto matmul_paired_outputs = [&](ushort row, ushort col, ushort depth_index, auto use_multiply_accumulate) {
- constexpr bool matmul_accumulate = decltype(use_multiply_accumulate)::value;
+ auto matmul_paired_outputs = [&](ushort row, ushort col, ushort depth_index, auto output_mode) {
+ constexpr MatmulMode mode = decltype(output_mode)::value;
if constexpr (pair_output_rows) {
+ static_assert(RELAXED, "strict MXU row-pairing is not implemented");
+ const thread auto& left_row_0 = left.fragment_at(row, depth_index, left_transpose);
+ const thread auto& left_row_1 = left.fragment_at(row + 1, depth_index, left_transpose);
+ const thread auto& right_operand = right.fragment_at(depth_index, col, right_transpose);
matmul<
- matmul_accumulate,
+ mode,
typename OutputFragment::ElementType,
typename LeftFragment::ElementType,
typename RightFragment::ElementType,
- transpose_a,
- transpose_b>(
+ transpose_left,
+ transpose_right>(
output.fragment_at(row, col),
output.fragment_at(row + 1, col),
- left.fragment_at(row, depth_index, transpose_left),
- left.fragment_at(row + 1, depth_index, transpose_left),
- transpose_left,
- right.fragment_at(depth_index, col, transpose_right),
- transpose_right
+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {
+ load_paired_vectors(cooperative_left, left_row_0, left_row_1);
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ cooperative_right[i] = right_operand[i];
+ }
+ }
);
} else {
+ const thread auto& left_operand = left.fragment_at(row, depth_index, left_transpose);
+ const thread auto& right_col_0 = right.fragment_at(depth_index, col, right_transpose);
+ const thread auto& right_col_1 = right.fragment_at(depth_index, col + 1, right_transpose);
matmul<
- matmul_accumulate,
+ mode,
typename OutputFragment::ElementType,
typename LeftFragment::ElementType,
typename RightFragment::ElementType,
- transpose_a,
- transpose_b>(
+ transpose_left,
+ transpose_right>(
output.fragment_at(row, col),
output.fragment_at(row, col + 1),
- left.fragment_at(row, depth_index, transpose_left),
- transpose_left,
- right.fragment_at(depth_index, col, transpose_right),
- right.fragment_at(depth_index, col + 1, transpose_right),
- transpose_right
+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ cooperative_left[i] = left_operand[i];
+ }
+ load_paired_vectors(cooperative_right, right_col_0, right_col_1);
+ }
);
}
};
@@ -76,32 +131,38 @@ METAL_FUNC static void fragment_matmul(
for (ushort row = 0; row < rows; row += output_row_step) {
METAL_PRAGMA_UNROLL
for (ushort col = 0; col < output_col_count; col += output_col_step) {
- if constexpr (!ACCUMULATE) {
- matmul_paired_outputs(row, col, 0, metal::bool_constant<false>{});
+ if constexpr (OUTPUT_MODE == MatmulMode::multiply) {
+ matmul_paired_outputs(row, col, 0, uzu::integral_constant<MatmulMode, MatmulMode::multiply>{});
}
METAL_PRAGMA_UNROLL
- for (ushort depth_index = ACCUMULATE ? 0 : 1; depth_index < depth; ++depth_index) {
- matmul_paired_outputs(row, col, depth_index, metal::bool_constant<true>{});
+ for (ushort depth_index = OUTPUT_MODE == MatmulMode::multiply_accumulate ? 0 : 1; depth_index < depth;
+ ++depth_index) {
+ matmul_paired_outputs(
+ row,
+ col,
+ depth_index,
+ uzu::integral_constant<MatmulMode, MatmulMode::multiply_accumulate>{}
+ );
}
}
}
}
-template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
+template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, class RightFragment>
METAL_FUNC static void fragment_mma(
thread OutputFragment& output,
thread LeftFragment& left,
thread RightFragment& right
) {
- fragment_matmul<true, transpose_a, transpose_b>(output, left, right);
+ fragment_matmul<MatmulMode::multiply_accumulate, transpose_left, transpose_right>(output, left, right);
}
-template <bool transpose_a, bool transpose_b, class OutputFragment, class LeftFragment, class RightFragment>
+template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, class RightFragment>
METAL_FUNC static void fragment_mm(
thread OutputFragment& output,
thread LeftFragment& left,
thread RightFragment& right
) {
// MXU relaxed multiply is slightly faster than multiply_accumulate for pure matmul.
- fragment_matmul<false, transpose_a, transpose_b>(output, left, right);
+ fragment_matmul<MatmulMode::multiply, transpose_left, transpose_right>(output, left, right);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/integer_formats.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/integer_formats.h
new file mode 100644
index 000000000..53ea74127
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/integer_formats.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../common/defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+enum class Signedness { Signed, Unsigned };
+
+template <ushort BITS, Signedness SIGNEDNESS>
+struct IntegerFormat;
+
+template <Signedness SIGNEDNESS>
+struct IntegerFormat<4, SIGNEDNESS> {
+ using StorageElement = uchar;
+ using UnpackedElement = metal::conditional_t<SIGNEDNESS == Signedness::Signed, int8_t, uint8_t>;
+ using TensorElement =
+ metal::conditional_t<SIGNEDNESS == Signedness::Signed, metal::int4b_format, metal::uint4b_format>;
+ using DevicePointer = const device uchar*;
+ using MutableDevicePointer = device uchar*;
+ UZU_CONST ushort BITS = 4;
+ UZU_CONST ushort ELEMENTS_PER_BYTE = 2;
+};
+
+template <Signedness SIGNEDNESS>
+struct IntegerFormat<8, SIGNEDNESS> {
+ using StorageElement = metal::conditional_t<SIGNEDNESS == Signedness::Signed, int8_t, uint8_t>;
+ using UnpackedElement = StorageElement;
+ using TensorElement = StorageElement;
+ using DevicePointer = const device StorageElement*;
+ using MutableDevicePointer = device StorageElement*;
+ UZU_CONST ushort BITS = 8;
+ UZU_CONST ushort ELEMENTS_PER_BYTE = 1;
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h
index 88619860a..a4f2385f0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/ops.h
@@ -7,6 +7,7 @@
#include "../defines.h"
#include "../loader.h"
+#include "tensor_operand.h"
#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>
@@ -25,9 +26,8 @@ struct MxuFragmentOps {
#include "layout.h"
#include "cooperative_vectors.h"
-#include "tile_matmul.h"
#include "fragment_matmul.h"
-#include "device_weight_matmul.h"
+#include "tensor_matmul.h"
};
using MxuStrictFragmentOps = MxuFragmentOps<false>;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_matmul.h
new file mode 100644
index 000000000..0c53c8140
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_matmul.h
@@ -0,0 +1,68 @@
+template <typename Format>
+METAL_FUNC static auto make_tensor_view(
+ const typename Format::DevicePointer tile_base,
+ const int right_row_stride_bytes
+) {
+ constexpr int tile_k = int(2 * FRAGMENT_COLS);
+ constexpr int tile_n = int(2 * FRAGMENT_COLS);
+ using Element = typename Format::TensorElement;
+ using RightTensor = tensor<device Element, extents<int, tile_k, tile_n>, tensor_inline>;
+
+ const array<int, 2> right_strides = {1, right_row_stride_bytes * Format::ELEMENTS_PER_BYTE};
+
+ return RightTensor(
+ const_cast<typename Format::MutableDevicePointer>(tile_base),
+ extents<int, tile_k, tile_n>{},
+ right_strides
+ );
+}
+
+template <bool transpose_left, bool transpose_right, class OutputFragment, class LeftFragment, typename Format>
+METAL_FUNC static void fragment_mm(
+ thread OutputFragment& output,
+ thread LeftFragment& left,
+ const DeviceTensorOperand<Format> right
+) {
+ static_assert(!transpose_left && transpose_right, "device-tensor leaf: only left x right^T is implemented");
+ static_assert(
+ metal::is_same_v<typename LeftFragment::ElementType, typename Format::UnpackedElement>,
+ "device-tensor leaf: integer operand signedness must match"
+ );
+ static_assert(RELAXED, "device-tensor operands require the relaxed MXU layout");
+ static_assert(LeftFragment::COL_FRAGMENTS == 2, "device-tensor operands expect K tiled as two fragments");
+ static_assert(OutputFragment::COL_FRAGMENTS % 2 == 0, "device-tensor operands require even N fragments");
+ static_assert(LeftFragment::ROW_FRAGMENTS == OutputFragment::ROW_FRAGMENTS, "M tiles must match");
+
+ constexpr ushort rows = OutputFragment::ROW_FRAGMENTS;
+ constexpr ushort cols = OutputFragment::COL_FRAGMENTS;
+ constexpr int tile_k = int(2 * FRAGMENT_COLS);
+ constexpr int tile_n = int(2 * FRAGMENT_COLS);
+ constexpr auto descriptor =
+ mpp::tensor_ops::matmul2d_descriptor(FRAGMENT_ROWS, tile_n, tile_k, false, true, RELAXED, MatmulMode::multiply);
+ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;
+
+ METAL_PRAGMA_UNROLL
+ for (ushort row = 0; row < rows; ++row) {
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < cols; col += 2) {
+ auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<
+ typename Format::UnpackedElement,
+ typename Format::TensorElement,
+ int>();
+ load_paired_vectors(cooperative_left, left.fragment_at(row, 0), left.fragment_at(row, 1));
+
+ auto right_tensor = make_tensor_view<Format>(
+ right.base + int(col * FRAGMENT_COLS) * right.row_stride_bytes,
+ right.row_stride_bytes
+ );
+
+ auto cooperative_output =
+ matmul_op
+ .template get_destination_cooperative_tensor<decltype(cooperative_left), decltype(right_tensor), int>();
+
+ matmul_op.run(cooperative_left, right_tensor, cooperative_output);
+
+ store_paired_vectors(cooperative_output, output.fragment_at(row, col), output.fragment_at(row, col + 1));
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_operand.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_operand.h
new file mode 100644
index 000000000..e1e34a955
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tensor_operand.h
@@ -0,0 +1,22 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "integer_formats.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+template <typename Format>
+struct DeviceTensorOperand {
+ using Element = typename Format::TensorElement;
+ using Pointer = typename Format::DevicePointer;
+
+ Pointer base;
+ int row_stride_bytes;
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.h
deleted file mode 100644
index e54ad95aa..000000000
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment/tile_matmul.h
+++ /dev/null
@@ -1,102 +0,0 @@
-// MPP has no valid 16x16x16 op; fragment_mma pairs fragments into 16x32.
-template <
- bool ACCUMULATE,
- typename CType,
- typename AType,
- typename BType,
- bool transpose_a,
- bool transpose_b,
- typename MarshalInputs>
-METAL_FUNC static void mma_impl(
- thread ThreadVector<CType>& output_0,
- thread ThreadVector<CType>& output_1,
- MarshalInputs marshal_inputs
-) {
- constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(
- FRAGMENT_ROWS,
- 2 * FRAGMENT_COLS,
- FRAGMENT_COLS,
- transpose_a,
- transpose_b,
- RELAXED,
- ACCUMULATE ? MatmulMode::multiply_accumulate : MatmulMode::multiply
- );
-
- mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;
-
- auto cooperative_left = matmul_op.template get_left_input_cooperative_tensor<AType, BType, CType>();
- auto cooperative_right = matmul_op.template get_right_input_cooperative_tensor<AType, BType, CType>();
- auto cooperative_output = matmul_op.template get_destination_cooperative_tensor<
- decltype(cooperative_left),
- decltype(cooperative_right),
- CType>();
-
- marshal_inputs(cooperative_left, cooperative_right);
-
- if constexpr (ACCUMULATE) {
- load_paired_vectors(cooperative_output, output_0, output_1);
- }
-
- matmul_op.run(cooperative_left, cooperative_right, cooperative_output);
-
- store_paired_vectors(cooperative_output, output_0, output_1);
-}
-
-template <
- bool ACCUMULATE,
- typename CType,
- typename AType,
- typename BType,
- bool transpose_a = false,
- bool transpose_b = false>
-METAL_FUNC static void matmul(
- thread ThreadVector<CType>& output_col_0,
- thread ThreadVector<CType>& output_col_1,
- const thread ThreadVector<AType>& left,
- metal::bool_constant<transpose_a>,
- const thread ThreadVector<BType>& right_col_0,
- const thread ThreadVector<BType>& right_col_1,
- metal::bool_constant<transpose_b>
-) {
- mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(
- output_col_0,
- output_col_1,
- [&](thread auto& cooperative_left, thread auto& cooperative_right) {
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
- cooperative_left[i] = left[i];
- }
- load_paired_vectors(cooperative_right, right_col_0, right_col_1);
- }
- );
-}
-
-template <
- bool ACCUMULATE,
- typename CType,
- typename AType,
- typename BType,
- bool transpose_a = false,
- bool transpose_b = false>
-METAL_FUNC static void matmul(
- thread ThreadVector<CType>& output_row_0,
- thread ThreadVector<CType>& output_row_1,
- const thread ThreadVector<AType>& left_row_0,
- const thread ThreadVector<AType>& left_row_1,
- metal::bool_constant<transpose_a>,
- const thread ThreadVector<BType>& right,
- metal::bool_constant<transpose_b>
-) {
- static_assert(RELAXED, "strict MXU row-pairing is not implemented");
- mma_impl<ACCUMULATE, CType, AType, BType, transpose_a, transpose_b>(
- output_row_0,
- output_row_1,
- [&](thread auto& cooperative_left, thread auto& cooperative_right) {
- load_paired_vectors(cooperative_left, left_row_0, left_row_1);
- METAL_PRAGMA_UNROLL
- for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
- cooperative_right[i] = right[i];
- }
- }
- );
-}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/integer_source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/integer_source.h
new file mode 100644
index 000000000..b50d82535
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/integer_source.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "../../common/fragment.h"
+#include "../../common/mxu_fragment/ops.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <typename Element, typename FragmentOps, bool ALIGNED_M, ushort TILES_M, ushort TILES_K>
+static METAL_FUNC uzu::matmul::Fragment<Element, TILES_M, TILES_K, FragmentOps> load_integer_tile(
+ const device Element* activation,
+ const int leading_dimension,
+ const short simdgroup_limit_m,
+ const ushort simd_lane_id
+) {
+ uzu::matmul::Fragment<Element, TILES_M, TILES_K, FragmentOps> tile;
+ auto source = uzu::matmul::fragment_source(activation, leading_dimension);
+ if constexpr (!ALIGNED_M) {
+ source = source.bounded(simdgroup_limit_m, TILES_K * FragmentOps::FRAGMENT_ROWS);
+ }
+ tile.load_from(simd_lane_id, source);
+ return tile;
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 61eac5a2d..460e0a0ed 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -6,15 +6,14 @@
#include "../../common/fragment.h"
#include "gemm_rht.h"
#include "../../common/mxu_fragment/ops.h"
-#include "../../common/mxu_gemm_loop.h"
#include "../../../generated/matmul.h"
#include "../generated/gemm.h"
#include "block_geometry.h"
#include "gemm_tiling.h"
-#include "../../common/quant_pack.h"
-#include "quant_scale_bias.h"
-#include "quant_scale_zero_point.h"
-#include "../../common/quant_unpack.h"
+#include "operands.h"
+#include "schedules/selector.h"
+#include "schedules/staged.h"
+#include "schedules/tile_context.h"
using namespace metal;
@@ -22,16 +21,22 @@ namespace uzu {
namespace gemm {
template <
- typename AT,
- typename BT,
- typename DT,
+ typename OutputElementType_,
GemmTiling GEMM_TILING,
bool TRANSPOSE_B,
- GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,
- int BITS = 0,
- int GROUP_SIZE = 0,
- GemmAPrologueKind A_PROLOGUE = GemmAPrologueKind::FullPrecision>
+ typename LeftOperand,
+ typename RightOperand_>
struct MxuMmaCore {
+ using OutputElementType = OutputElementType_;
+ using RightOperand = RightOperand_;
+ using Left = operands::LeftBinding<LeftOperand, RightOperand>;
+ using Right = operands::RightBinding<RightOperand>;
+ using LeftElementType = typename Left::ElementType;
+ using RightElementType = typename Right::ElementType;
+ using LeftArgs = typename Left::Args;
+ using RightArgs = typename Right::Args;
+ using FragmentOps = uzu::matmul::MxuFragmentOps<>;
+ using Schedule = typename schedules::ScheduleFor<LeftOperand, RightOperand>::type;
UZU_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
UZU_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
UZU_CONST ushort SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
@@ -39,6 +44,7 @@ struct MxuMmaCore {
UZU_CONST ushort SIMDGROUP_BLOCK_M = THREADGROUP_BLOCK_M / SIMDGROUPS_PER_ROW;
UZU_CONST ushort SIMDGROUP_BLOCK_N = THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
UZU_CONST ushort SIMDGROUP_BLOCK_K = static_cast<ushort>(MXU_SIMDGROUP_BLOCK_K);
+ UZU_CONST bool TRANSPOSE_RIGHT = TRANSPOSE_B;
UZU_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);
static_assert(
THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,
@@ -48,447 +54,27 @@ struct MxuMmaCore {
UZU_CONST ushort TILES_N = SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps<>::FRAGMENT_COLS;
UZU_CONST ushort TILES_K = SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps<>::FRAGMENT_ROWS;
- UZU_CONST ushort QUANT_BK = (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;
- UZU_CONST ushort PADDING_B = 16 / sizeof(BT);
- UZU_CONST ushort SHARED_STRIDE_B = (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;
+ using OuterBlock = schedules::OuterBlockK<RightOperand, THREADGROUP_BLOCK_K_FP, SIMDGROUP_BLOCK_K>;
+ UZU_CONST ushort THREADGROUP_BLOCK_K = OuterBlock::VALUE;
+ UZU_CONST ushort SHARED_STRIDE_B = THREADGROUP_BLOCK_K + 16 / sizeof(RightElementType);
+ UZU_CONST ushort STAGING_ELEMENTS =
+ metal::is_same<Schedule, schedules::StagedSchedule>::value ? THREADGROUP_BLOCK_N * SHARED_STRIDE_B : 1;
UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
- static_assert(
- B_PROLOGUE == GemmBPrologueKind::FullPrecision || QUANT_BK % SIMDGROUP_BLOCK_K == 0,
- "QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K"
- );
- static_assert(
- B_PROLOGUE == GemmBPrologueKind::FullPrecision || THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,
- "Tile block_k must be a multiple of QUANT_BK"
- );
using AccumulatorType = float;
- using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<
- BT,
- THREADGROUP_BLOCK_N,
- (QUANT_BK > 0) ? QUANT_BK : 1,
- SHARED_STRIDE_B,
- 1,
- THREADGROUP_THREADS,
- (GROUP_SIZE > 0) ? GROUP_SIZE : 1,
- (BITS > 0) ? BITS : 4>;
- using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<
- BT,
- THREADGROUP_BLOCK_N,
- (QUANT_BK > 0) ? QUANT_BK : 1,
- SHARED_STRIDE_B,
- 1,
- THREADGROUP_THREADS,
- (GROUP_SIZE > 0) ? GROUP_SIZE : 1,
- (BITS > 0) ? BITS : 4>;
- using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<
- BT,
- THREADGROUP_BLOCK_N,
- (QUANT_BK > 0) ? QUANT_BK : 1,
- SHARED_STRIDE_B,
- 1,
- THREADGROUP_THREADS,
- (GROUP_SIZE > 0) ? GROUP_SIZE : 1,
- (BITS > 0) ? BITS : 4,
- true>;
-
- using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>>;
-
- struct QuantizedWeightAddressing {
- int row_stride_bytes;
- int groups_per_row;
- int k_offset_groups;
- const device uint8_t* block;
- };
-
- static METAL_FUNC QuantizedWeightAddressing
- quantized_weight_addressing(const device BT* b, const size_t block_col, const uint k_offset, const int k_elements) {
- constexpr int pack_factor = get_pack_factor<(BITS > 0) ? BITS : 4, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<(BITS > 0) ? BITS : 4>();
- const int row_stride_bytes = k_elements * bytes_per_pack / pack_factor;
- return QuantizedWeightAddressing{
- row_stride_bytes,
- (k_elements + int(GROUP_SIZE) - 1) / int(GROUP_SIZE),
- int(k_offset) / int(GROUP_SIZE),
- reinterpret_cast<const device uint8_t*>(b) + block_col * row_stride_bytes +
- int(k_offset) * bytes_per_pack / pack_factor,
- };
- }
-
- template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>
- static METAL_FUNC AccumFragment quant_k_loop(
- const device AT* a_simdgroup,
- threadgroup BT* b_shared,
- const int leading_dimension_a,
- const int aligned_k_iterations,
- const short simdgroup_limit_m,
- const short simdgroup_limit_n,
- const ushort tile_col_offset,
- const ushort tile_block_cols,
- thread Loader& loader_b,
- const thread ThreadContext& thread_context
- ) {
- AccumFragment accumulator;
- accumulator.clear();
-
- threadgroup BT* b_shared_simdgroup = b_shared + tile_col_offset * SHARED_STRIDE_B;
- const short2 tile_dimensions_b = short2(QUANT_BK, tile_block_cols);
-
- METAL_PRAGMA_NO_UNROLL
- for (int outer_k = 0; outer_k < aligned_k_iterations; ++outer_k) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if constexpr (ALIGNED_N) {
- loader_b.load_unsafe();
- } else {
- loader_b.load_safe(tile_dimensions_b);
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- METAL_PRAGMA_NO_UNROLL
- for (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {
- uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> left_tile;
- uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps<>> right_tile;
-
- const int left_offset = inner_k;
- auto left_src = uzu::matmul::fragment_source(a_simdgroup + left_offset, leading_dimension_a);
- if constexpr (!ALIGNED_M) {
- left_src = left_src.bounded(simdgroup_limit_m, SIMDGROUP_BLOCK_K);
- }
- left_tile.load_from(thread_context.simd_lane_id, left_src);
-
- right_tile.load_from(
- thread_context.simd_lane_id,
- uzu::matmul::fragment_source(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B))
- );
-
- uzu::matmul::MxuFragmentOps<>::template fragment_mma<false, true>(accumulator, left_tile, right_tile);
- }
-
- a_simdgroup += QUANT_BK;
- loader_b.next();
- }
-
- return accumulator;
- }
-
- template <bool ALIGNED_M>
- static METAL_FUNC uzu::matmul::Fragment<int8_t, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> load_int8_left_tile(
- const device int8_t* a_int8_simdgroup,
- const int leading_dimension_a,
- const short simdgroup_limit_m,
- const ushort simd_lane_id
- ) {
- uzu::matmul::Fragment<int8_t, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps<>> left_tile;
- auto left_src = uzu::matmul::fragment_source(a_int8_simdgroup, leading_dimension_a);
- if constexpr (!ALIGNED_M) {
- left_src = left_src.bounded(simdgroup_limit_m, SIMDGROUP_BLOCK_K);
- }
- left_tile.load_from(simd_lane_id, left_src);
- return left_tile;
- }
-
- template <bool ALIGNED_N>
- static METAL_FUNC uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps<>> load_int8_weight_tile(
- const device uint8_t* b_packed_simdgroup,
- const int k_element_offset,
- const int b_row_stride_bytes,
- const short simdgroup_limit_n,
- const short2 position,
- const ushort simd_lane_id
- ) {
- using Ops = uzu::matmul::MxuFragmentOps<>;
- uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops> right_tile;
- if constexpr (BITS == 4) {
- METAL_PRAGMA_UNROLL
- for (ushort tile_n = 0; tile_n < TILES_N; ++tile_n) {
- METAL_PRAGMA_UNROLL
- for (ushort tile_k = 0; tile_k < TILES_K; ++tile_k) {
- thread auto& weight_vector = right_tile.fragment_at(tile_n, tile_k);
- METAL_PRAGMA_UNROLL
- for (ushort thread_row = 0; thread_row < Ops::THREAD_ELEMENT_ROWS; ++thread_row) {
- const short row =
- short(tile_n * Ops::FRAGMENT_ROWS) + position.y + short(thread_row * Ops::THREAD_ELEMENT_ROW_STRIDE);
- const ushort element_base = thread_row * Ops::THREAD_ELEMENT_COLS;
- char4 codes = char4(0);
- if (ALIGNED_N || row < simdgroup_limit_n) {
- const int k_base = k_element_offset + int(tile_k * Ops::FRAGMENT_COLS) + int(position.x);
- const ushort packed = *reinterpret_cast<const device ushort*>(
- b_packed_simdgroup + int(row) * b_row_stride_bytes + (k_base >> 1)
- );
- codes = unpack_signed_nibbles_to_int8(uint(packed));
- }
- weight_vector[element_base + 0] = codes.x;
- weight_vector[element_base + 1] = codes.y;
- weight_vector[element_base + 2] = codes.z;
- weight_vector[element_base + 3] = codes.w;
- }
- }
- }
- } else {
- static_assert(BITS == 8, "symmetric int8 activations only support 4-bit or 8-bit weights");
- auto right_src = uzu::matmul::fragment_source(
- reinterpret_cast<const device int8_t*>(b_packed_simdgroup) + k_element_offset,
- b_row_stride_bytes
- );
- if constexpr (!ALIGNED_N) {
- right_src = right_src.bounded(simdgroup_limit_n, SIMDGROUP_BLOCK_K);
- }
- right_tile.load_from(simd_lane_id, right_src);
- }
- return right_tile;
- }
-
- // Per-thread cache holding one value per distinct fragment row (or column) line
- // a thread owns, addressable either by slot or by offset from the thread origin.
- template <ushort TILES, ushort SLOTS_PER_TILE, ushort FRAGMENT_EXTENT, ushort SLOT_STRIDE>
- struct FragmentLineCache {
- float values[TILES * SLOTS_PER_TILE];
-
- METAL_FUNC thread float& slot(const ushort tile, const ushort index) thread {
- return values[tile * SLOTS_PER_TILE + index];
- }
-
- METAL_FUNC void clear() thread {
- METAL_PRAGMA_UNROLL
- for (ushort index = 0; index < TILES * SLOTS_PER_TILE; ++index) {
- values[index] = 0.0f;
- }
- }
-
- METAL_FUNC float at(const short offset) const thread {
- const ushort tile = ushort(offset) / FRAGMENT_EXTENT;
- const ushort index = (ushort(offset) % FRAGMENT_EXTENT) / SLOT_STRIDE;
- return values[tile * SLOTS_PER_TILE + index];
- }
-
- template <bool ALIGNED, typename Sink>
- static METAL_FUNC void for_each_line(
- const short origin,
- const short limit,
- const uint abs_base,
- const uint groups_per_row,
- const uint group_index,
- Sink sink
- ) {
- METAL_PRAGMA_UNROLL
- for (ushort tile = 0; tile < TILES; ++tile) {
- METAL_PRAGMA_UNROLL
- for (ushort index = 0; index < SLOTS_PER_TILE; ++index) {
- const short coord = origin + tile * FRAGMENT_EXTENT + index * SLOT_STRIDE;
- if (ALIGNED || coord < limit) {
- const uint line_index = abs_base + uint(coord);
- sink(tile, index, line_index * groups_per_row + group_index, line_index);
- }
- }
- }
- }
- };
-
- using FragmentOps = uzu::matmul::MxuFragmentOps<>;
- using ActivationLineCache = FragmentLineCache<
- TILES_M,
- FragmentOps::THREAD_ELEMENT_ROWS,
- FragmentOps::FRAGMENT_ROWS,
- FragmentOps::THREAD_ELEMENT_ROW_STRIDE>;
- using WeightLineCache = FragmentLineCache<TILES_N, FragmentOps::THREAD_ELEMENT_COLS, FragmentOps::FRAGMENT_COLS, 1>;
-
- static_assert(
- A_PROLOGUE != GemmAPrologueKind::Int8Symmetric || GROUP_SIZE % int(SIMDGROUP_BLOCK_K) == 0,
- "A8 weight group size must be a multiple of the 32-wide activation chunk"
- );
- UZU_CONST bool int8_activation_needs_weight_correction =
- A_PROLOGUE == GemmAPrologueKind::Int8Symmetric && B_PROLOGUE != GemmBPrologueKind::ScaleSymmetricDequant;
- UZU_CONST float int8_weight_midpoint = float(symmetric_zero_point<ushort((BITS > 0) ? BITS : 4)>());
-
- static METAL_FUNC float int8_weight_dequant_bias(
- const float scale,
- const uint scale_index,
- const uint weight_column,
- const uint weight_group,
- const uint weight_groups_per_row,
- const device BT* biases,
- const device uint8_t* zero_points
- ) {
- if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
- return static_cast<float>(biases[scale_index]);
- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
- if constexpr (BITS == 8) {
- return -scale * float(zero_points[scale_index]);
- } else {
- const device uint8_t* zero_points_row =
- zero_points + weight_column * zero_point_row_stride<ushort(BITS)>(weight_groups_per_row);
- return -scale * float(decode_zero_point<ushort(BITS)>(zero_points_row, weight_group));
- }
- } else {
- return 0.0f;
- }
- }
-
- template <bool ALIGNED_M, bool ALIGNED_N>
- static METAL_FUNC AccumFragment int8_activation_k_loop(
- const device int8_t* a_int8_simdgroup,
- const device uint8_t* b_packed_simdgroup,
- const device float* a_scales,
- const device int32_t* a_group_sums,
- const device BT* b_scales,
- const device BT* biases,
- const device uint8_t* zero_points,
- const int leading_dimension_a,
- const int b_row_stride_bytes,
- const int weight_group_iterations,
- const short simdgroup_limit_m,
- const short simdgroup_limit_n,
- const uint abs_row_base,
- const uint abs_col_base,
- const uint k_offset_weight_groups,
- const uint k_offset_act_groups,
- const uint weight_groups_per_row,
- const uint act_groups_per_row,
- const thread ThreadContext& thread_context
- ) {
- using Ops = uzu::matmul::MxuFragmentOps<>;
- AccumFragment accumulator;
- accumulator.clear();
-
- const short2 position = Ops::get_position(thread_context.simd_lane_id);
- constexpr int k_bytes_per_weight_group = (BITS == 4) ? (int(GROUP_SIZE) / 2) : int(GROUP_SIZE);
- constexpr int act_chunks_per_weight_group = int(GROUP_SIZE) / int(SIMDGROUP_BLOCK_K);
-
- METAL_PRAGMA_NO_UNROLL
- for (int weight_group = 0; weight_group < weight_group_iterations; ++weight_group) {
- const uint weight_group_index = k_offset_weight_groups + uint(weight_group);
-
- WeightLineCache weight_scales;
- WeightLineCache weight_corrections;
- WeightLineCache::template for_each_line<ALIGNED_N>(
- position.x,
- simdgroup_limit_n,
- abs_col_base,
- weight_groups_per_row,
- weight_group_index,
- [&](ushort tile_col, ushort thread_col, uint scale_index, uint weight_column) {
- const float scale = static_cast<float>(b_scales[scale_index]);
- weight_scales.slot(tile_col, thread_col) = scale;
- if constexpr (int8_activation_needs_weight_correction) {
- weight_corrections.slot(tile_col, thread_col) = scale * int8_weight_midpoint + int8_weight_dequant_bias(
- scale,
- scale_index,
- weight_column,
- weight_group_index,
- weight_groups_per_row,
- biases,
- zero_points
- );
- }
- }
- );
-
- ActivationLineCache activation_corrections;
- if constexpr (int8_activation_needs_weight_correction) {
- activation_corrections.clear();
- }
-
- METAL_PRAGMA_NO_UNROLL
- for (int act_chunk = 0; act_chunk < act_chunks_per_weight_group; ++act_chunk) {
- const int k_element_offset = act_chunk * int(SIMDGROUP_BLOCK_K);
- auto activation_tile = load_int8_left_tile<ALIGNED_M>(
- a_int8_simdgroup + k_element_offset,
- leading_dimension_a,
- simdgroup_limit_m,
- thread_context.simd_lane_id
- );
-
- uzu::matmul::Fragment<int, TILES_M, TILES_N, Ops> chunk_products;
- if constexpr (BITS == 4 && ALIGNED_N) {
- Ops::template fragment_matmul_int8_device_weights<metal::int4b_format>(
- chunk_products,
- activation_tile,
- b_packed_simdgroup + (k_element_offset >> 1),
- b_row_stride_bytes
- );
- } else {
- auto right_tile = load_int8_weight_tile<ALIGNED_N>(
- b_packed_simdgroup,
- k_element_offset,
- b_row_stride_bytes,
- simdgroup_limit_n,
- position,
- thread_context.simd_lane_id
- );
- Ops::template fragment_mm<false, true>(chunk_products, activation_tile, right_tile);
- }
-
- const uint act_group_index = k_offset_act_groups + uint(weight_group * act_chunks_per_weight_group + act_chunk);
- ActivationLineCache activation_scales;
- ActivationLineCache::template for_each_line<ALIGNED_M>(
- position.y,
- simdgroup_limit_m,
- abs_row_base,
- act_groups_per_row,
- act_group_index,
- [&](ushort tile_row, ushort thread_row, uint scale_index, uint) {
- const float activation_scale = a_scales[scale_index];
- activation_scales.slot(tile_row, thread_row) = activation_scale;
- if constexpr (int8_activation_needs_weight_correction) {
- activation_corrections.slot(tile_row, thread_row) +=
- activation_scale * float(a_group_sums[scale_index]);
- }
- }
- );
-
- AccumFragment::zip_for_each_coord(
- thread_context.simd_lane_id,
- [&](short row, short col, thread float& accumulated, thread int& products) {
- if (!ALIGNED_M && row >= simdgroup_limit_m) {
- return;
- }
- if (!ALIGNED_N && col >= simdgroup_limit_n) {
- return;
- }
- accumulated +=
- activation_scales.at(row - position.y) * weight_scales.at(col - position.x) * float(products);
- },
- accumulator,
- chunk_products
- );
- }
-
- if constexpr (int8_activation_needs_weight_correction) {
- accumulator.map_coords(thread_context.simd_lane_id, [&](short row, short col, float value) {
- if (!ALIGNED_M && row >= simdgroup_limit_m) {
- return value;
- }
- if (!ALIGNED_N && col >= simdgroup_limit_n) {
- return value;
- }
- return value + weight_corrections.at(col - position.x) * activation_corrections.at(row - position.y);
- });
- }
-
- a_int8_simdgroup += GROUP_SIZE;
- b_packed_simdgroup += k_bytes_per_weight_group;
- }
-
- return accumulator;
- }
+ using AccumFragment = uzu::matmul::Fragment<AccumulatorType, TILES_M, TILES_N, FragmentOps>;
static METAL_FUNC void run(
- const device AT* a,
- const device BT* b,
- device DT* d,
+ LeftArgs left,
+ RightArgs right,
+ device OutputElementType* d,
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
GemmDTransform output_transform,
- const bool signed_codes,
- const device BT* scales,
- const device BT* biases,
- const device uint8_t* zero_points,
- const device BT* output_bias,
+ const device RightElementType* output_bias,
const device int32_t* rht_factors,
- const device int8_t* a_int8,
- const device float* a_scales,
- const device int32_t* a_group_sums,
- threadgroup BT* b_shared,
+ threadgroup RightElementType* b_shared,
const thread ThreadContext& thread_context
) {
const uint partition = thread_context.threadgroup_position.z;
@@ -503,19 +89,14 @@ struct MxuMmaCore {
const size_t block_row = size_t(geometry.block_row_start);
const size_t block_col = size_t(geometry.block_col_start);
- const uint k_offset_per_block =
- (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? uint(THREADGROUP_BLOCK_K_FP) : uint(QUANT_BK);
- const uint k_offset = partition * params->aligned_inner_iterations * k_offset_per_block;
-
- const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +
- (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));
+ const uint k_offset = partition * params->aligned_inner_iterations * uint(THREADGROUP_BLOCK_K);
const ushort tile_row_offset = SIMDGROUP_BLOCK_M * (thread_context.simdgroup_index / SIMDGROUPS_PER_COLUMN);
const ushort tile_col_offset = SIMDGROUP_BLOCK_N * (thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
- device DT* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) +
- block_row * params->leading_dimension_d + block_col +
- tile_row_offset * params->leading_dimension_d + tile_col_offset;
+ device OutputElementType* d_simdgroup = d + size_t(partition) * size_t(params->M) * size_t(params->N) +
+ block_row * params->leading_dimension_d + block_col +
+ tile_row_offset * params->leading_dimension_d + tile_col_offset;
const short simdgroup_limit_m =
alignment.contains(GemmAlignment::M)
@@ -526,209 +107,88 @@ struct MxuMmaCore {
? SIMDGROUP_BLOCK_N
: short(min(int(SIMDGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start + tile_col_offset)));
- const device AT* a_simdgroup = a;
- if constexpr (A_PROLOGUE == GemmAPrologueKind::FullPrecision) {
- a_simdgroup +=
- block_row * params->leading_dimension_a + k_offset + size_t(tile_row_offset) * params->leading_dimension_a;
- }
- const device BT* b_simdgroup_fp =
- b_block_fp +
- (TRANSPOSE_B ? size_t(tile_col_offset) * int(params->leading_dimension_b) : size_t(tile_col_offset));
+ left.seek_rows(block_row + tile_row_offset, params->leading_dimension_a, k_offset);
- const ushort tile_block_cols =
+ schedules::TileContext tile_context;
+ tile_context.simdgroup_limit_m = simdgroup_limit_m;
+ tile_context.simdgroup_limit_n = simdgroup_limit_n;
+ tile_context.block_col = block_col;
+ tile_context.tile_col_offset = tile_col_offset;
+ tile_context.tile_block_cols =
ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(geometry.block_col_start)));
+ tile_context.k_offset = k_offset;
+ tile_context.abs_row_base = uint(geometry.block_row_start) + tile_row_offset;
const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);
const bool apply_accumulate = output_transform.contains(GemmDTransform::ACCUMULATE);
const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);
- const device BT* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);
-
- auto dispatch_aligned_k = [&](auto body) {
- if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
- dispatch_bool(alignment.contains(GemmAlignment::K), body);
- } else {
- body(true_type{});
- }
- };
- dispatch_aligned_k([&](auto aligned_k) {
- dispatch_bool(
- alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
- [&](auto aligned_m) {
- dispatch_bool(
- alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
- [&](auto aligned_n) {
- auto accumulator_tile = [&]() {
- if constexpr (A_PROLOGUE == GemmAPrologueKind::Int8Symmetric) {
- const auto quantized_weights =
- quantized_weight_addressing(b, block_col, k_offset, int(params->K));
- const device int8_t* a_int8_simdgroup = a_int8 + block_row * params->leading_dimension_a +
- k_offset +
- size_t(tile_row_offset) * params->leading_dimension_a;
- const device uint8_t* b_packed_simdgroup =
- quantized_weights.block + size_t(tile_col_offset) * quantized_weights.row_stride_bytes;
- return int8_activation_k_loop<aligned_m.value, aligned_n.value>(
- a_int8_simdgroup,
- b_packed_simdgroup,
- a_scales,
- a_group_sums,
- scales,
- biases,
- zero_points,
- int(params->leading_dimension_a),
- quantized_weights.row_stride_bytes,
- int(params->aligned_inner_iterations),
- simdgroup_limit_m,
- simdgroup_limit_n,
- uint(geometry.block_row_start) + tile_row_offset,
- uint(geometry.block_col_start) + tile_col_offset,
- uint(quantized_weights.k_offset_groups),
- k_offset / uint(SIMDGROUP_BLOCK_K),
- uint(quantized_weights.groups_per_row),
- uint(params->K) / uint(SIMDGROUP_BLOCK_K),
- thread_context
- );
- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
- const int aligned_k_iterations_fp = int(params->aligned_inner_iterations);
- return uzu::matmul::gemm_loop<
- AT,
- BT,
- SIMDGROUP_BLOCK_M,
- SIMDGROUP_BLOCK_N,
- SIMDGROUP_BLOCK_K,
- THREADGROUP_BLOCK_K_FP,
- false,
- TRANSPOSE_B,
- aligned_m.value,
- aligned_n.value,
- aligned_k.value,
- AccumulatorType>(
- a_simdgroup,
- b_simdgroup_fp,
- int(params->leading_dimension_a),
- int(params->leading_dimension_b),
- int(params->K),
- aligned_k_iterations_fp,
- simdgroup_limit_m,
- simdgroup_limit_n,
- thread_context
- );
- } else {
- const int aligned_k_iterations_q = int(params->aligned_inner_iterations);
- const int k_elements = int(params->K);
- const auto quantized_weights = quantized_weight_addressing(b, block_col, k_offset, k_elements);
- const int groups_per_row = quantized_weights.groups_per_row;
- const int k_offset_groups = quantized_weights.k_offset_groups;
- const device uint8_t* weights_block = quantized_weights.block;
- const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;
-
- auto loader_b = [&]() {
- if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
- const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;
- return BLoaderScaleBias(
- weights_block,
- scales_offset,
- biases_offset,
- signed_codes,
- k_elements,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
- );
- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
- const int zero_point_stride_per_row = zero_point_row_stride<ushort(BITS)>(groups_per_row);
- const device uint8_t* zero_points_row_start =
- zero_points + block_col * zero_point_stride_per_row +
- ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);
- return BLoaderScaleZeroPoint(
- weights_block,
- scales_offset,
- zero_points_row_start,
- signed_codes,
- k_elements,
- groups_per_row,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
- );
- } else {
- return BLoaderScaleSymmetric(
- weights_block,
- scales_offset,
- nullptr,
- signed_codes,
- k_elements,
- groups_per_row,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
- );
- }
- }();
-
- return quant_k_loop<aligned_m.value, aligned_n.value>(
- a_simdgroup,
- b_shared,
- int(params->leading_dimension_a),
- aligned_k_iterations_q,
- simdgroup_limit_m,
- simdgroup_limit_n,
- tile_col_offset,
- tile_block_cols,
- loader_b,
- thread_context
- );
- }
- }();
-
- if (apply_scale) {
- const AccumulatorType scale = AccumulatorType(params->ab_scale);
- accumulator_tile.map([&](auto value) { return value * scale; });
- }
+ const device RightElementType* bias_simdgroup = output_bias + size_t(block_col) + size_t(tile_col_offset);
+
+ dispatch_bool(
+ alignment.contains(GemmAlignment::M) || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
+ [&](auto aligned_m) {
+ dispatch_bool(
+ alignment.contains(GemmAlignment::N) || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
+ [&](auto aligned_n) {
+ auto accumulator_tile = Schedule::template launch<MxuMmaCore, aligned_m.value, aligned_n.value>(
+ left,
+ right,
+ b_shared,
+ params,
+ tile_context,
+ alignment,
+ thread_context
+ );
+
+ if (apply_scale) {
+ const AccumulatorType scale = AccumulatorType(params->ab_scale);
+ accumulator_tile.map([&](auto value) { return value * scale; });
+ }
- if (apply_accumulate) {
- uzu::matmul::Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>> existing_output;
- auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));
- if constexpr (!(aligned_m.value && aligned_n.value)) {
- output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);
- }
- existing_output.load_from(thread_context.simd_lane_id, output_src);
- thread DT* existing_data = existing_output.elements();
- accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });
+ if (apply_accumulate) {
+ uzu::matmul::Fragment<OutputElementType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps<>>
+ existing_output;
+ auto output_src = uzu::matmul::fragment_source(d_simdgroup, int(params->leading_dimension_d));
+ if constexpr (!(aligned_m.value && aligned_n.value)) {
+ output_src = output_src.bounded(simdgroup_limit_m, simdgroup_limit_n);
}
+ existing_output.load_from(thread_context.simd_lane_id, output_src);
+ thread OutputElementType* existing_data = existing_output.elements();
+ accumulator_tile.map([&](auto value) { return value + AccumulatorType(*(existing_data++)); });
+ }
- if (apply_bias) {
- accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {
- if constexpr (aligned_n.value) {
+ if (apply_bias) {
+ accumulator_tile.map_coords(thread_context.simd_lane_id, [&](short, short col, auto value) {
+ if constexpr (aligned_n.value) {
+ return value + AccumulatorType(bias_simdgroup[col]);
+ } else {
+ if (col < simdgroup_limit_n) {
return value + AccumulatorType(bias_simdgroup[col]);
- } else {
- if (col < simdgroup_limit_n) {
- return value + AccumulatorType(bias_simdgroup[col]);
- }
- return value;
}
- });
- }
+ return value;
+ }
+ });
+ }
- if constexpr (aligned_m.value && aligned_n.value) {
- accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));
- } else {
- accumulator_tile.store_safe(
- thread_context.simd_lane_id,
- d_simdgroup,
- int(params->leading_dimension_d),
- short2(simdgroup_limit_n, simdgroup_limit_m)
- );
- }
+ if constexpr (aligned_m.value && aligned_n.value) {
+ accumulator_tile.store(thread_context.simd_lane_id, d_simdgroup, int(params->leading_dimension_d));
+ } else {
+ accumulator_tile.store_safe(
+ thread_context.simd_lane_id,
+ d_simdgroup,
+ int(params->leading_dimension_d),
+ short2(simdgroup_limit_n, simdgroup_limit_m)
+ );
}
- );
- }
- );
- });
+ }
+ );
+ }
+ );
if (output_transform.contains(GemmDTransform::RHT)) {
threadgroup_barrier(mem_flags::mem_device);
- device DT* d_block = d + block_row * params->leading_dimension_d + block_col;
+ device OutputElementType* d_block = d + block_row * params->leading_dimension_d + block_col;
const ushort tile_block_rows = ushort(min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row)));
const ushort tile_block_cols = ushort(min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col)));
apply_output_random_hadamard_transform(
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/operands.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/operands.h
new file mode 100644
index 000000000..039ba4644
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/operands.h
@@ -0,0 +1,283 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../generated/gemm.h"
+#include "../../common/mxu_fragment/integer_formats.h"
+#include "../../common/quant_pack.h"
+#include "../../common/quant_unpack.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+namespace operands {
+
+namespace detail {
+
+template <bool WITH_GROUP_SUMS>
+struct QuantizedLeftArgs;
+
+template <>
+struct QuantizedLeftArgs<false> {
+ const device int8_t* values;
+ const device float* scales;
+
+ METAL_FUNC void seek_rows(const size_t row, const int leading_dimension, const uint k_offset) thread {
+ values += row * leading_dimension + k_offset;
+ }
+
+ METAL_FUNC void advance_k(const ushort elements) thread { values += elements; }
+};
+
+template <>
+struct QuantizedLeftArgs<true> {
+ const device int8_t* values;
+ const device float* scales;
+ const device int32_t* group_sums;
+
+ METAL_FUNC void seek_rows(const size_t row, const int leading_dimension, const uint k_offset) thread {
+ values += row * leading_dimension + k_offset;
+ }
+
+ METAL_FUNC void advance_k(const ushort elements) thread { values += elements; }
+};
+
+template <GemmBPrologueKind SCHEME, ushort BITS, typename ElementType>
+struct QuantizedRightArgs;
+
+template <ushort BITS>
+struct QuantizedStorage {
+ const device uint8_t* values;
+ bool signed_codes;
+
+ QuantizedStorage(const device uint8_t* values_, const bool signed_codes_)
+ : values(values_), signed_codes(signed_codes_) {}
+
+ METAL_FUNC void seek_block(const size_t block_column, const uint k_offset, const int row_stride_bytes) thread {
+ values += block_column * row_stride_bytes + int(k_offset) * get_bytes_per_pack<BITS>() / get_pack_factor<BITS>();
+ }
+
+ METAL_FUNC void seek_columns(const ushort tile_column_offset, const int row_stride_bytes) thread {
+ values += size_t(tile_column_offset) * row_stride_bytes;
+ }
+
+ METAL_FUNC void advance_k(const int bytes) thread { values += bytes; }
+};
+
+// Constructors keep the specialized signedness flag mandatory; aggregate
+// initialization would silently default an omitted trailing flag to false.
+
+template <ushort BITS, typename ElementType>
+struct QuantizedRightArgs<GemmBPrologueKind::ScaleSymmetricDequant, BITS, ElementType> {
+ QuantizedStorage<BITS> storage;
+ const device ElementType* scales;
+
+ QuantizedRightArgs(const device uint8_t* values_, const device ElementType* scales_, const bool signed_codes_)
+ : storage(values_, signed_codes_), scales(scales_) {}
+};
+
+template <ushort BITS, typename ElementType>
+struct QuantizedRightArgs<GemmBPrologueKind::ScaleBiasDequant, BITS, ElementType> {
+ QuantizedStorage<BITS> storage;
+ const device ElementType* scales;
+ const device ElementType* biases;
+
+ QuantizedRightArgs(
+ const device uint8_t* values_,
+ const device ElementType* scales_,
+ const device ElementType* biases_,
+ const bool signed_codes_
+ )
+ : storage(values_, signed_codes_), scales(scales_), biases(biases_) {}
+};
+
+template <ushort BITS, typename ElementType>
+struct QuantizedRightArgs<GemmBPrologueKind::ScaleZeroPointDequant, BITS, ElementType> {
+ QuantizedStorage<BITS> storage;
+ const device ElementType* scales;
+ const device uint8_t* zero_points;
+
+ QuantizedRightArgs(
+ const device uint8_t* values_,
+ const device ElementType* scales_,
+ const device uint8_t* zero_points_,
+ const bool signed_codes_
+ )
+ : storage(values_, signed_codes_), scales(scales_), zero_points(zero_points_) {}
+};
+
+} // namespace detail
+
+template <GemmBPrologueKind SCHEME, ushort BITS, typename ElementType>
+static METAL_FUNC void seek_quantized_metadata(
+ thread detail::QuantizedRightArgs<SCHEME, BITS, ElementType>& args,
+ const size_t block_column,
+ const int groups_per_row,
+ const int first_group
+) {
+ args.scales += block_column * groups_per_row + first_group;
+ if constexpr (SCHEME == GemmBPrologueKind::ScaleBiasDequant) {
+ args.biases += block_column * groups_per_row + first_group;
+ } else if constexpr (SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {
+ args.zero_points +=
+ block_column * zero_point_row_stride<BITS>(groups_per_row) + ((BITS == 4) ? (first_group / 2) : first_group);
+ }
+}
+
+template <typename ElementType_>
+struct Dense {
+ using ElementType = ElementType_;
+};
+
+template <typename Format_, ushort GROUP_SIZE_, GemmBPrologueKind SCHEME_, typename ScaleElementType_>
+struct Quantized {
+ using Format = Format_;
+ using ScaleElementType = ScaleElementType_;
+ UZU_CONST ushort BITS = Format::BITS;
+ UZU_CONST ushort GROUP_SIZE = GROUP_SIZE_;
+ UZU_CONST GemmBPrologueKind SCHEME = SCHEME_;
+
+ static_assert(BITS == 4 || BITS == 8, "quantized integer bits must be 4 or 8");
+ static_assert(GROUP_SIZE > 0, "quantized groups must be non-empty");
+ static_assert(SCHEME != GemmBPrologueKind::FullPrecision, "quantized operands need a quantization scheme");
+};
+
+template <typename ElementType>
+struct DenseArgs {
+ const device ElementType* values;
+
+ METAL_FUNC void seek_rows(const size_t row, const int leading_dimension, const uint k_offset) thread {
+ values += row * leading_dimension + k_offset;
+ }
+
+ template <bool TRANSPOSED>
+ METAL_FUNC void seek_columns(const size_t column, const uint k_offset, const int leading_dimension) thread {
+ values += (TRANSPOSED ? column * leading_dimension : column) +
+ (TRANSPOSED ? k_offset : k_offset * uint(leading_dimension));
+ }
+
+ METAL_FUNC void advance_k(const ushort elements) thread { values += elements; }
+};
+
+template <typename Operand>
+struct RightBinding;
+
+template <typename Element>
+struct RightBinding<Dense<Element>> {
+ using ElementType = Element;
+ using Args = DenseArgs<Element>;
+ UZU_CONST bool NEEDS_CORRECTION = false;
+};
+
+template <typename Format, ushort GROUP_SIZE, GemmBPrologueKind SCHEME, typename ScaleElementType>
+struct RightBinding<Quantized<Format, GROUP_SIZE, SCHEME, ScaleElementType>> {
+ using ElementType = ScaleElementType;
+ using Args = detail::QuantizedRightArgs<SCHEME, Format::BITS, ScaleElementType>;
+ UZU_CONST bool NEEDS_CORRECTION = SCHEME != GemmBPrologueKind::ScaleSymmetricDequant;
+};
+
+template <typename LeftOperand, typename RightOperand>
+struct LeftBinding;
+
+template <typename Element, typename RightOperand>
+struct LeftBinding<Dense<Element>, RightOperand> {
+ using ElementType = Element;
+ using Args = DenseArgs<Element>;
+};
+
+template <typename Format, ushort GROUP_SIZE, typename ScaleElementType, typename RightOperand>
+struct LeftBinding<
+ Quantized<Format, GROUP_SIZE, GemmBPrologueKind::ScaleSymmetricDequant, ScaleElementType>,
+ RightOperand> {
+ static_assert(Format::BITS == 8, "integer MMA left operands must be 8-bit");
+ static_assert(
+ metal::is_same<typename Format::UnpackedElement, int8_t>::value,
+ "integer MMA left operands must be signed"
+ );
+ using ElementType = typename Format::UnpackedElement;
+ using Args = detail::QuantizedLeftArgs<RightBinding<RightOperand>::NEEDS_CORRECTION>;
+};
+
+template <GemmAPrologueKind A_PROLOGUE, typename ElementType>
+struct LeftOperandFor;
+
+template <typename ElementType>
+struct LeftOperandFor<GemmAPrologueKind::FullPrecision, ElementType> {
+ using type = Dense<ElementType>;
+};
+
+template <typename ElementType>
+struct LeftOperandFor<GemmAPrologueKind::Int8Symmetric, ElementType> {
+ using type = Quantized<
+ uzu::matmul::IntegerFormat<8, uzu::matmul::Signedness::Signed>,
+ 32,
+ GemmBPrologueKind::ScaleSymmetricDequant,
+ float>;
+};
+
+template <GemmBPrologueKind B_PROLOGUE, ushort BITS, ushort GROUP_SIZE, typename ElementType>
+struct RightOperandFor {
+ static_assert(
+ B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant || B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant ||
+ B_PROLOGUE == GemmBPrologueKind::ScaleSymmetricDequant,
+ "unsupported quantized right operand scheme"
+ );
+ using type =
+ Quantized<uzu::matmul::IntegerFormat<BITS, uzu::matmul::Signedness::Signed>, GROUP_SIZE, B_PROLOGUE, ElementType>;
+};
+
+template <ushort BITS, ushort GROUP_SIZE, typename ElementType>
+struct RightOperandFor<GemmBPrologueKind::FullPrecision, BITS, GROUP_SIZE, ElementType> {
+ using type = Dense<ElementType>;
+};
+
+template <typename LeftOperand, typename RightOperand, typename ElementType>
+static METAL_FUNC typename LeftBinding<LeftOperand, RightOperand>::Args make_left_args(
+ const device ElementType* a,
+ const device int8_t* a_int8,
+ const device float* a_scales,
+ const device int32_t* a_group_sums
+) {
+ if constexpr (metal::is_same<LeftOperand, Dense<ElementType>>::value) {
+ return {a};
+ } else if constexpr (RightBinding<RightOperand>::NEEDS_CORRECTION) {
+ return {a_int8, a_scales, a_group_sums};
+ } else {
+ return {a_int8, a_scales};
+ }
+}
+
+template <typename RightOperand, typename ElementType>
+static METAL_FUNC typename RightBinding<RightOperand>::Args make_right_args(
+ const device ElementType* b,
+ const device ElementType* scales,
+ const device ElementType* biases,
+ const device uint8_t* zero_points,
+ const bool signed_codes
+) {
+ using Binding = RightBinding<RightOperand>;
+ if constexpr (metal::is_same<RightOperand, Dense<ElementType>>::value) {
+ return {b};
+ } else {
+ const device uint8_t* storage = reinterpret_cast<const device uint8_t*>(b);
+ if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant) {
+ return typename Binding::Args(storage, scales, biases, signed_codes);
+ } else if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {
+ return typename Binding::Args(storage, scales, zero_points, signed_codes);
+ } else if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleSymmetricDequant) {
+ return typename Binding::Args(storage, scales, signed_codes);
+ } else {
+ static_assert(
+ RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant ||
+ RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ||
+ RightOperand::SCHEME == GemmBPrologueKind::ScaleSymmetricDequant,
+ "unsupported right operand scheme"
+ );
+ }
+ }
+}
+
+} // namespace operands
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
index a15c132a2..30b669cf4 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_scale_zero_point.h
@@ -87,6 +87,30 @@ struct QuantizedBlockLoaderScaleZeroPoint {
),
signed_codes(signed_codes_) {}
+ QuantizedBlockLoaderScaleZeroPoint(
+ const device uint8_t* src_,
+ const device T* scales_,
+ const bool signed_codes_,
+ const int src_leading_dim_,
+ const int groups_per_row_,
+ threadgroup T* dst_,
+ ushort simd_group_id [[simdgroup_index_in_threadgroup]],
+ ushort simd_lane_id [[thread_index_in_simdgroup]]
+ )
+ : QuantizedBlockLoaderScaleZeroPoint(
+ src_,
+ scales_,
+ static_cast<const device uint8_t*>(nullptr),
+ signed_codes_,
+ src_leading_dim_,
+ groups_per_row_,
+ dst_,
+ simd_group_id,
+ simd_lane_id
+ ) {
+ static_assert(SCALE_SYMMETRIC, "zero-point-free loader construction requires symmetric quantization");
+ }
+
inline void current_scale_bias(thread T& out_scale, thread T& out_bias) const {
uint zero_point_value;
T scale_value;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/loader.h
new file mode 100644
index 000000000..f10293622
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/loader.h
@@ -0,0 +1,107 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../../generated/gemm.h"
+#include "../quant_scale_bias.h"
+#include "../quant_scale_zero_point.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+template <
+ typename T,
+ short THREADGROUP_TILE_ROWS,
+ short THREADGROUP_TILE_COLS,
+ short DESTINATION_LEADING_DIMENSION,
+ short REDUCTION_DIMENSION,
+ short THREADGROUP_SIZE,
+ short GROUP_SIZE,
+ short BITS>
+struct QuantizedLoaders {
+ using ScaleBias = QuantizedBlockLoaderScaleBias<
+ T,
+ THREADGROUP_TILE_ROWS,
+ THREADGROUP_TILE_COLS,
+ DESTINATION_LEADING_DIMENSION,
+ REDUCTION_DIMENSION,
+ THREADGROUP_SIZE,
+ GROUP_SIZE,
+ BITS>;
+ using ScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<
+ T,
+ THREADGROUP_TILE_ROWS,
+ THREADGROUP_TILE_COLS,
+ DESTINATION_LEADING_DIMENSION,
+ REDUCTION_DIMENSION,
+ THREADGROUP_SIZE,
+ GROUP_SIZE,
+ BITS>;
+ using ScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<
+ T,
+ THREADGROUP_TILE_ROWS,
+ THREADGROUP_TILE_COLS,
+ DESTINATION_LEADING_DIMENSION,
+ REDUCTION_DIMENSION,
+ THREADGROUP_SIZE,
+ GROUP_SIZE,
+ BITS,
+ true>;
+};
+
+template <GemmBPrologueKind SCHEME, typename Loaders, typename RightArgs, typename ElementType>
+static METAL_FUNC auto make_quantized_loader(
+ const thread RightArgs& right,
+ const int k_elements,
+ const int groups_per_row,
+ threadgroup ElementType* shared,
+ const ushort simdgroup_index,
+ const ushort simd_lane_id
+) {
+ if constexpr (SCHEME == GemmBPrologueKind::ScaleBiasDequant) {
+ return typename Loaders::ScaleBias(
+ right.storage.values,
+ right.scales,
+ right.biases,
+ right.storage.signed_codes,
+ k_elements,
+ shared,
+ simdgroup_index,
+ simd_lane_id
+ );
+ } else if constexpr (SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {
+ return typename Loaders::ScaleZeroPoint(
+ right.storage.values,
+ right.scales,
+ right.zero_points,
+ right.storage.signed_codes,
+ k_elements,
+ groups_per_row,
+ shared,
+ simdgroup_index,
+ simd_lane_id
+ );
+ } else if constexpr (SCHEME == GemmBPrologueKind::ScaleSymmetricDequant) {
+ return typename Loaders::ScaleSymmetric(
+ right.storage.values,
+ right.scales,
+ right.storage.signed_codes,
+ k_elements,
+ groups_per_row,
+ shared,
+ simdgroup_index,
+ simd_lane_id
+ );
+ } else {
+ static_assert(
+ SCHEME == GemmBPrologueKind::ScaleBiasDequant || SCHEME == GemmBPrologueKind::ScaleZeroPointDequant ||
+ SCHEME == GemmBPrologueKind::ScaleSymmetricDequant,
+ "unsupported quantized loader scheme"
+ );
+ }
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/metadata.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/metadata.h
new file mode 100644
index 000000000..1a26d12fd
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/metadata.h
@@ -0,0 +1,94 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../../generated/gemm.h"
+#include "../../../common/quant_unpack.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+// Per-thread replicated metadata. Compact row storage must consume each row
+// sum in the first split-K partition.
+template <ushort TILES, ushort SLOTS_PER_TILE, ushort FRAGMENT_EXTENT, ushort SLOT_STRIDE>
+struct QuantizationLineCache {
+ float values[TILES * SLOTS_PER_TILE];
+
+ METAL_FUNC thread float& slot(const ushort tile, const ushort index) thread {
+ return values[tile * SLOTS_PER_TILE + index];
+ }
+
+ METAL_FUNC void clear() thread {
+ METAL_PRAGMA_UNROLL
+ for (ushort index = 0; index < TILES * SLOTS_PER_TILE; ++index) {
+ values[index] = 0.0f;
+ }
+ }
+
+ METAL_FUNC float at(const short offset) const thread {
+ const ushort tile = ushort(offset) / FRAGMENT_EXTENT;
+ const ushort index = (ushort(offset) % FRAGMENT_EXTENT) / SLOT_STRIDE;
+ return values[tile * SLOTS_PER_TILE + index];
+ }
+
+ template <bool ALIGNED, typename Sink>
+ static METAL_FUNC void for_each_line(
+ const short origin,
+ const short limit,
+ const uint abs_base,
+ const uint groups_per_row,
+ const uint group_index,
+ Sink sink
+ ) {
+ METAL_PRAGMA_UNROLL
+ for (ushort tile = 0; tile < TILES; ++tile) {
+ METAL_PRAGMA_UNROLL
+ for (ushort index = 0; index < SLOTS_PER_TILE; ++index) {
+ const short coord = origin + tile * FRAGMENT_EXTENT + index * SLOT_STRIDE;
+ if (ALIGNED || coord < limit) {
+ const uint line_index = abs_base + uint(coord);
+ sink(tile, index, line_index * groups_per_row + group_index, line_index);
+ }
+ }
+ }
+ }
+};
+
+// Symmetric formats omit correction storage entirely.
+struct NoCorrectionCache {};
+
+template <typename RightOperand>
+struct Correction {
+ using Format = typename RightOperand::Format;
+
+ static METAL_FUNC float midpoint() { return float(symmetric_zero_point<Format::BITS>()); }
+
+ template <typename RightArgs>
+ static METAL_FUNC float offset(
+ const float scale,
+ const uint scale_index,
+ const uint column,
+ const uint group,
+ const uint groups_per_row,
+ const thread RightArgs& right
+ ) {
+ if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleBiasDequant) {
+ return static_cast<float>(right.biases[scale_index]);
+ } else if constexpr (RightOperand::SCHEME == GemmBPrologueKind::ScaleZeroPointDequant) {
+ if constexpr (Format::BITS == 8) {
+ return -scale * float(right.zero_points[scale_index]);
+ } else {
+ const device uint8_t* zero_points_row =
+ right.zero_points + column * zero_point_row_stride<Format::BITS>(groups_per_row);
+ return -scale * float(decode_zero_point<Format::BITS>(zero_points_row, group));
+ }
+ } else {
+ return 0.0f;
+ }
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/slice.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/slice.h
new file mode 100644
index 000000000..7b787d79d
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/slice.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../common/quant_pack.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+struct QuantizedSlice {
+ int row_stride_bytes;
+ int groups_per_row;
+ int first_group;
+};
+
+template <typename RightOperand>
+static METAL_FUNC QuantizedSlice make_quantized_slice(const uint k_offset, const int k_elements) {
+ constexpr int pack_factor = get_pack_factor<RightOperand::BITS, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<RightOperand::BITS>();
+ return QuantizedSlice{
+ k_elements * bytes_per_pack / pack_factor,
+ (k_elements + RightOperand::GROUP_SIZE - 1) / RightOperand::GROUP_SIZE,
+ int(k_offset) / RightOperand::GROUP_SIZE,
+ };
+}
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/source.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/source.h
new file mode 100644
index 000000000..f63ad7040
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quantized/source.h
@@ -0,0 +1,107 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../common/fragment.h"
+#include "../../../common/mxu_fragment/ops.h"
+#include "../../../common/quant_unpack.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+
+using uzu::matmul::DeviceTensorOperand;
+using uzu::matmul::IntegerFormat;
+using uzu::matmul::Signedness;
+
+using SignedInt4Format = IntegerFormat<4, Signedness::Signed>;
+using SignedInt8Format = IntegerFormat<8, Signedness::Signed>;
+
+template <typename Format, bool ALIGNED_N, ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>
+struct QuantizedSource;
+
+template <ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>
+struct QuantizedSource<SignedInt4Format, true, TILES_N, TILES_K, SIMDGROUP_BLOCK_K> {
+ static METAL_FUNC DeviceTensorOperand<SignedInt4Format> make(
+ const device uint8_t* storage,
+ const int k_element_offset,
+ const int row_stride_bytes,
+ const short,
+ const short2,
+ const ushort
+ ) {
+ return DeviceTensorOperand<SignedInt4Format>{storage + (k_element_offset >> 1), row_stride_bytes};
+ }
+};
+
+template <ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>
+struct QuantizedSource<SignedInt4Format, false, TILES_N, TILES_K, SIMDGROUP_BLOCK_K> {
+ using Ops = uzu::matmul::MxuFragmentOps<>;
+ using RegisterTile = uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops>;
+
+ static METAL_FUNC RegisterTile make(
+ const device uint8_t* storage,
+ const int k_element_offset,
+ const int row_stride_bytes,
+ const short simdgroup_limit_n,
+ const short2 position,
+ const ushort
+ ) {
+ RegisterTile tile;
+ METAL_PRAGMA_UNROLL
+ for (ushort tile_n = 0; tile_n < TILES_N; ++tile_n) {
+ METAL_PRAGMA_UNROLL
+ for (ushort tile_k = 0; tile_k < TILES_K; ++tile_k) {
+ thread auto& vector = tile.fragment_at(tile_n, tile_k);
+ METAL_PRAGMA_UNROLL
+ for (ushort thread_row = 0; thread_row < Ops::THREAD_ELEMENT_ROWS; ++thread_row) {
+ const short row =
+ short(tile_n * Ops::FRAGMENT_ROWS) + position.y + short(thread_row * Ops::THREAD_ELEMENT_ROW_STRIDE);
+ const ushort element_base = thread_row * Ops::THREAD_ELEMENT_COLS;
+ char4 codes = char4(0);
+ if (row < simdgroup_limit_n) {
+ const int k_base = k_element_offset + int(tile_k * Ops::FRAGMENT_COLS) + int(position.x);
+ const ushort packed =
+ *reinterpret_cast<const device ushort*>(storage + int(row) * row_stride_bytes + (k_base >> 1));
+ codes = unpack_signed_nibbles_to_int8(uint(packed));
+ }
+ vector[element_base + 0] = codes.x;
+ vector[element_base + 1] = codes.y;
+ vector[element_base + 2] = codes.z;
+ vector[element_base + 3] = codes.w;
+ }
+ }
+ }
+ return tile;
+ }
+};
+
+template <bool ALIGNED_N, ushort TILES_N, ushort TILES_K, ushort SIMDGROUP_BLOCK_K>
+struct QuantizedSource<SignedInt8Format, ALIGNED_N, TILES_N, TILES_K, SIMDGROUP_BLOCK_K> {
+ using Ops = uzu::matmul::MxuFragmentOps<>;
+ using RegisterTile = uzu::matmul::Fragment<int8_t, TILES_N, TILES_K, Ops>;
+
+ static METAL_FUNC RegisterTile make(
+ const device uint8_t* storage,
+ const int k_element_offset,
+ const int row_stride_bytes,
+ const short simdgroup_limit_n,
+ const short2,
+ const ushort simd_lane_id
+ ) {
+ RegisterTile tile;
+ auto source = uzu::matmul::fragment_source(
+ reinterpret_cast<const device int8_t*>(storage) + k_element_offset,
+ row_stride_bytes
+ );
+ if constexpr (!ALIGNED_N) {
+ source = source.bounded(simdgroup_limit_n, SIMDGROUP_BLOCK_K);
+ }
+ tile.load_from(simd_lane_id, source);
+ return tile;
+ }
+};
+
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/dense.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/dense.h
new file mode 100644
index 000000000..2c62cafe9
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/dense.h
@@ -0,0 +1,66 @@
+#pragma once
+
+#include "../../../../common/integral_constant.h"
+#include "../../../../common/thread_context.h"
+#include "../../../../generated/gemm.h"
+#include "../../../common/fragment.h"
+#include "../../../common/mxu_gemm_loop.h"
+#include "../gemm_alignment.h"
+#include "tile_context.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+namespace schedules {
+
+struct DenseSchedule {
+ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>
+ static METAL_FUNC typename Core::AccumFragment launch(
+ typename Core::LeftArgs left,
+ typename Core::RightArgs right,
+ threadgroup typename Core::RightElementType*,
+ const constant uzu::matmul::GemmParams* params,
+ const TileContext tile,
+ const GemmAlignment alignment,
+ const thread ThreadContext& thread_context
+ ) {
+ right.template seek_columns<Core::TRANSPOSE_RIGHT>(
+ tile.block_col + tile.tile_col_offset,
+ tile.k_offset,
+ params->leading_dimension_b
+ );
+
+ typename Core::AccumFragment accumulator;
+ dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {
+ accumulator = uzu::matmul::gemm_loop<
+ typename Core::LeftElementType,
+ typename Core::RightElementType,
+ Core::SIMDGROUP_BLOCK_M,
+ Core::SIMDGROUP_BLOCK_N,
+ Core::SIMDGROUP_BLOCK_K,
+ Core::THREADGROUP_BLOCK_K_FP,
+ false,
+ Core::TRANSPOSE_RIGHT,
+ ALIGNED_M,
+ ALIGNED_N,
+ aligned_k.value,
+ typename Core::AccumulatorType>(
+ left.values,
+ right.values,
+ int(params->leading_dimension_a),
+ int(params->leading_dimension_b),
+ int(params->K),
+ int(params->aligned_inner_iterations),
+ tile.simdgroup_limit_m,
+ tile.simdgroup_limit_n,
+ thread_context
+ );
+ });
+ return accumulator;
+ }
+};
+
+} // namespace schedules
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
new file mode 100644
index 000000000..25a57d7ca
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
@@ -0,0 +1,187 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../../common/thread_context.h"
+#include "../../../../generated/gemm.h"
+#include "../../../common/fragment.h"
+#include "../../../common/mxu_fragment/ops.h"
+#include "../gemm_alignment.h"
+#include "../integer_source.h"
+#include "../quantized/metadata.h"
+#include "../quantized/slice.h"
+#include "../quantized/source.h"
+#include "tile_context.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+namespace schedules {
+
+template <typename LeftOperand, typename RightOperand>
+struct IntegerSchedule {
+ using RightFormat = typename RightOperand::Format;
+ using RightBinding = operands::RightBinding<RightOperand>;
+ UZU_CONST bool needs_correction = RightBinding::NEEDS_CORRECTION;
+
+ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>
+ static METAL_FUNC typename Core::AccumFragment launch(
+ typename Core::LeftArgs left,
+ typename Core::RightArgs right,
+ threadgroup typename Core::RightElementType*,
+ const constant uzu::matmul::GemmParams* params,
+ const TileContext tile,
+ const GemmAlignment,
+ const thread ThreadContext& thread_context
+ ) {
+ using Ops = typename Core::FragmentOps;
+ using AccumFragment = typename Core::AccumFragment;
+ using LeftScaleCache = QuantizationLineCache<
+ Core::TILES_M,
+ Ops::THREAD_ELEMENT_ROWS,
+ Ops::FRAGMENT_ROWS,
+ Ops::THREAD_ELEMENT_ROW_STRIDE>;
+ using RightScaleCache = QuantizationLineCache<Core::TILES_N, Ops::THREAD_ELEMENT_COLS, Ops::FRAGMENT_COLS, 1>;
+ using CorrectionPolicy = Correction<RightOperand>;
+ using ColumnCorrectionCache = metal::conditional_t<needs_correction, RightScaleCache, NoCorrectionCache>;
+ using RowCorrectionCache = metal::conditional_t<needs_correction, LeftScaleCache, NoCorrectionCache>;
+
+ static_assert(RightOperand::GROUP_SIZE % Core::SIMDGROUP_BLOCK_K == 0, "right groups must contain MMA chunks");
+
+ const auto slice = make_quantized_slice<RightOperand>(tile.k_offset, int(params->K));
+ right.storage.seek_block(tile.block_col, tile.k_offset, slice.row_stride_bytes);
+ right.storage.seek_columns(tile.tile_col_offset, slice.row_stride_bytes);
+
+ const int leading_dimension_a = int(params->leading_dimension_a);
+ const int b_row_stride_bytes = slice.row_stride_bytes;
+ const int right_group_iterations = int(params->aligned_inner_iterations);
+ const short simdgroup_limit_m = tile.simdgroup_limit_m;
+ const short simdgroup_limit_n = tile.simdgroup_limit_n;
+ const uint abs_row_base = tile.abs_row_base;
+ const uint abs_col_base = tile.absolute_column_base();
+ const uint k_offset_right_groups = uint(slice.first_group);
+ const uint k_offset_left_scale_groups = tile.k_offset / uint(LeftOperand::GROUP_SIZE);
+ const uint right_groups_per_row = uint(slice.groups_per_row);
+ const uint left_scale_groups_per_row = uint(params->K) / uint(LeftOperand::GROUP_SIZE);
+
+ AccumFragment accumulator;
+ accumulator.clear();
+
+ const short2 position = Ops::get_position(thread_context.simd_lane_id);
+ constexpr int k_bytes_per_right_group = int(RightOperand::GROUP_SIZE) * RightOperand::BITS / 8;
+ constexpr int left_groups_per_right_group = int(RightOperand::GROUP_SIZE) / int(LeftOperand::GROUP_SIZE);
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int right_group = 0; right_group < right_group_iterations; ++right_group) {
+ const uint right_group_index = k_offset_right_groups + uint(right_group);
+
+ RightScaleCache right_scales;
+ ColumnCorrectionCache column_corrections;
+ RightScaleCache::template for_each_line<ALIGNED_N>(
+ position.x,
+ simdgroup_limit_n,
+ abs_col_base,
+ right_groups_per_row,
+ right_group_index,
+ [&](ushort tile_col, ushort thread_col, uint right_scale_index, uint right_column) {
+ const float scale = static_cast<float>(right.scales[right_scale_index]);
+ right_scales.slot(tile_col, thread_col) = scale;
+ if constexpr (needs_correction) {
+ column_corrections.slot(tile_col, thread_col) =
+ scale * CorrectionPolicy::midpoint() + CorrectionPolicy::offset(
+ scale,
+ right_scale_index,
+ right_column,
+ right_group_index,
+ right_groups_per_row,
+ right
+ );
+ }
+ }
+ );
+
+ RowCorrectionCache row_corrections;
+ if constexpr (needs_correction) {
+ row_corrections.clear();
+ }
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int left_group = 0; left_group < left_groups_per_right_group; ++left_group) {
+ const int k_element_offset = left_group * int(LeftOperand::GROUP_SIZE);
+ auto left_tile = load_integer_tile<int8_t, Ops, ALIGNED_M, Core::TILES_M, Core::TILES_K>(
+ left.values + k_element_offset,
+ leading_dimension_a,
+ simdgroup_limit_m,
+ thread_context.simd_lane_id
+ );
+
+ uzu::matmul::Fragment<int, Core::TILES_M, Core::TILES_N, Ops> chunk_products;
+ auto right_tile =
+ QuantizedSource<RightFormat, ALIGNED_N, Core::TILES_N, Core::TILES_K, Core::SIMDGROUP_BLOCK_K>::make(
+ right.storage.values,
+ k_element_offset,
+ b_row_stride_bytes,
+ simdgroup_limit_n,
+ position,
+ thread_context.simd_lane_id
+ );
+ Ops::template fragment_mm<false, true>(chunk_products, left_tile, right_tile);
+
+ const uint left_scale_group_index =
+ k_offset_left_scale_groups + uint(right_group * left_groups_per_right_group + left_group);
+ LeftScaleCache left_scales;
+ LeftScaleCache::template for_each_line<ALIGNED_M>(
+ position.y,
+ simdgroup_limit_m,
+ abs_row_base,
+ left_scale_groups_per_row,
+ left_scale_group_index,
+ [&](ushort tile_row, ushort thread_row, uint left_scale_index, uint) {
+ const float left_scale = left.scales[left_scale_index];
+ left_scales.slot(tile_row, thread_row) = left_scale;
+ if constexpr (needs_correction) {
+ row_corrections.slot(tile_row, thread_row) += left_scale * float(left.group_sums[left_scale_index]);
+ }
+ }
+ );
+
+ AccumFragment::zip_for_each_coord(
+ thread_context.simd_lane_id,
+ [&](short row, short col, thread float& accumulated, thread int& products) {
+ if (!ALIGNED_M && row >= simdgroup_limit_m) {
+ return;
+ }
+ if (!ALIGNED_N && col >= simdgroup_limit_n) {
+ return;
+ }
+ accumulated += left_scales.at(row - position.y) * right_scales.at(col - position.x) * float(products);
+ },
+ accumulator,
+ chunk_products
+ );
+ }
+
+ if constexpr (needs_correction) {
+ accumulator.map_coords(thread_context.simd_lane_id, [&](short row, short col, float value) {
+ if (!ALIGNED_M && row >= simdgroup_limit_m) {
+ return value;
+ }
+ if (!ALIGNED_N && col >= simdgroup_limit_n) {
+ return value;
+ }
+ return value + column_corrections.at(col - position.x) * row_corrections.at(row - position.y);
+ });
+ }
+
+ left.advance_k(RightOperand::GROUP_SIZE);
+ right.storage.advance_k(k_bytes_per_right_group);
+ }
+
+ return accumulator;
+ }
+};
+
+} // namespace schedules
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/selector.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/selector.h
new file mode 100644
index 000000000..e88ee2584
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/selector.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include "../../../../generated/gemm.h"
+#include "../operands.h"
+#include "dense.h"
+#include "integer.h"
+#include "staged.h"
+
+namespace uzu {
+namespace gemm {
+namespace schedules {
+
+template <typename LeftOperand, typename RightOperand>
+struct ScheduleFor;
+
+template <typename LeftElementType, typename RightElementType>
+struct ScheduleFor<operands::Dense<LeftElementType>, operands::Dense<RightElementType>> {
+ using type = DenseSchedule;
+};
+
+template <typename LeftElementType, typename Format, ushort GROUP_SIZE, GemmBPrologueKind SCHEME, typename ElementType>
+struct ScheduleFor<operands::Dense<LeftElementType>, operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>> {
+ using type = StagedSchedule;
+};
+
+template <
+ typename LeftFormat,
+ ushort LEFT_GROUP_SIZE,
+ typename RightFormat,
+ ushort GROUP_SIZE,
+ GemmBPrologueKind SCHEME,
+ typename ElementType>
+struct ScheduleFor<
+ operands::Quantized<LeftFormat, LEFT_GROUP_SIZE, GemmBPrologueKind::ScaleSymmetricDequant, float>,
+ operands::Quantized<RightFormat, GROUP_SIZE, SCHEME, ElementType>> {
+ using type = IntegerSchedule<
+ operands::Quantized<LeftFormat, LEFT_GROUP_SIZE, GemmBPrologueKind::ScaleSymmetricDequant, float>,
+ operands::Quantized<RightFormat, GROUP_SIZE, SCHEME, ElementType>>;
+};
+
+} // namespace schedules
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
new file mode 100644
index 000000000..fc2a4e0e8
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/staged.h
@@ -0,0 +1,172 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../../../common/thread_context.h"
+#include "../../../../generated/gemm.h"
+#include "../../../common/fragment.h"
+#include "../../../common/mxu_fragment/ops.h"
+#include "../../../common/mxu_gemm_loop.h"
+#include "../gemm_alignment.h"
+#include "../integer_source.h"
+#include "../operands.h"
+#include "../quantized/loader.h"
+#include "../quantized/slice.h"
+#include "tile_context.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+namespace schedules {
+
+template <typename RightOperand, ushort DENSE_BLOCK_K, ushort SIMDGROUP_BLOCK_K>
+struct OuterBlockK;
+
+template <typename ElementType, ushort DENSE_BLOCK_K, ushort SIMDGROUP_BLOCK_K>
+struct OuterBlockK<operands::Dense<ElementType>, DENSE_BLOCK_K, SIMDGROUP_BLOCK_K> {
+ UZU_CONST ushort VALUE = DENSE_BLOCK_K;
+};
+
+template <
+ typename Format,
+ ushort GROUP_SIZE,
+ GemmBPrologueKind SCHEME,
+ typename ElementType,
+ ushort DENSE_BLOCK_K,
+ ushort SIMDGROUP_BLOCK_K>
+struct OuterBlockK<operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>, DENSE_BLOCK_K, SIMDGROUP_BLOCK_K> {
+ UZU_CONST ushort VALUE = GROUP_SIZE;
+
+ static_assert(VALUE % SIMDGROUP_BLOCK_K == 0, "quantized group size must contain complete MMA chunks");
+ static_assert(DENSE_BLOCK_K % VALUE == 0, "tile block K must contain complete quantized groups");
+};
+
+template <typename RightOperand>
+struct StagedRightLoader;
+
+template <typename ElementType>
+struct StagedRightLoader<operands::Dense<ElementType>> {
+ template <typename Core>
+ static METAL_FUNC typename Core::FullPrecisionRightLoader make(
+ typename Core::RightArgs right,
+ const constant uzu::matmul::GemmParams* params,
+ const size_t block_col,
+ const uint k_offset,
+ threadgroup typename Core::RightElementType* staging,
+ const thread ThreadContext& thread_context
+ ) {
+ right.template seek_columns<Core::TRANSPOSE_RIGHT>(block_col, k_offset, params->leading_dimension_b);
+ return typename Core::FullPrecisionRightLoader(right.values, params->leading_dimension_b, staging, thread_context);
+ }
+};
+
+template <typename Format, ushort GROUP_SIZE, GemmBPrologueKind SCHEME, typename ElementType>
+struct StagedRightLoader<operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>> {
+ template <typename Core>
+ static METAL_FUNC auto make(
+ typename Core::RightArgs right,
+ const constant uzu::matmul::GemmParams* params,
+ const size_t block_col,
+ const uint k_offset,
+ threadgroup typename Core::RightElementType* staging,
+ const thread ThreadContext& thread_context
+ ) {
+ const int k_elements = int(params->K);
+ const auto slice =
+ make_quantized_slice<operands::Quantized<Format, GROUP_SIZE, SCHEME, ElementType>>(k_offset, k_elements);
+ right.storage.seek_block(block_col, k_offset, slice.row_stride_bytes);
+ operands::seek_quantized_metadata(right, block_col, slice.groups_per_row, slice.first_group);
+
+ using Loaders = QuantizedLoaders<
+ typename Core::RightElementType,
+ Core::THREADGROUP_BLOCK_N,
+ Core::THREADGROUP_BLOCK_K,
+ Core::SHARED_STRIDE_B,
+ 1,
+ Core::THREADGROUP_THREADS,
+ GROUP_SIZE,
+ Format::BITS>;
+
+ return make_quantized_loader<SCHEME, Loaders>(
+ right,
+ k_elements,
+ slice.groups_per_row,
+ staging,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ }
+};
+
+// Bit width and group size come from the right operand descriptor.
+struct StagedSchedule {
+ template <typename Core, bool ALIGNED_M, bool ALIGNED_N>
+ static METAL_FUNC typename Core::AccumFragment launch(
+ typename Core::LeftArgs left,
+ typename Core::RightArgs right,
+ threadgroup typename Core::RightElementType* staging,
+ const constant uzu::matmul::GemmParams* params,
+ const TileContext tile,
+ const GemmAlignment,
+ const thread ThreadContext& thread_context
+ ) {
+ using FragmentOps = typename Core::FragmentOps;
+ using RightOperand = typename Core::RightOperand;
+
+ auto loader_b = StagedRightLoader<RightOperand>::template make<Core>(
+ right,
+ params,
+ tile.block_col,
+ tile.k_offset,
+ staging,
+ thread_context
+ );
+
+ typename Core::AccumFragment accumulator;
+ accumulator.clear();
+
+ threadgroup typename Core::RightElementType* staging_simdgroup =
+ staging + tile.tile_col_offset * Core::SHARED_STRIDE_B;
+ const short2 tile_dimensions_b = short2(Core::THREADGROUP_BLOCK_K, tile.tile_block_cols);
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int outer_k = 0; outer_k < int(params->aligned_inner_iterations); ++outer_k) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if constexpr (ALIGNED_N) {
+ loader_b.load_unsafe();
+ } else {
+ loader_b.load_safe(tile_dimensions_b);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int inner_k = 0; inner_k < Core::THREADGROUP_BLOCK_K; inner_k += Core::SIMDGROUP_BLOCK_K) {
+ uzu::matmul::Fragment<typename Core::LeftElementType, Core::TILES_M, Core::TILES_K, FragmentOps> left_tile;
+ uzu::matmul::Fragment<typename Core::RightElementType, Core::TILES_N, Core::TILES_K, FragmentOps> right_tile;
+
+ auto left_src = uzu::matmul::fragment_source(left.values + inner_k, int(params->leading_dimension_a));
+ if constexpr (!ALIGNED_M) {
+ left_src = left_src.bounded(tile.simdgroup_limit_m, Core::SIMDGROUP_BLOCK_K);
+ }
+ left_tile.load_from(thread_context.simd_lane_id, left_src);
+
+ right_tile.load_from(
+ thread_context.simd_lane_id,
+ uzu::matmul::fragment_source(staging_simdgroup + inner_k, int(Core::SHARED_STRIDE_B))
+ );
+
+ FragmentOps::template fragment_mma<false, true>(accumulator, left_tile, right_tile);
+ }
+
+ left.advance_k(Core::THREADGROUP_BLOCK_K);
+ loader_b.next();
+ }
+
+ return accumulator;
+ }
+};
+
+} // namespace schedules
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/tile_context.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/tile_context.h
new file mode 100644
index 000000000..a0af83f6d
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/schedules/tile_context.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#include <metal_stdlib>
+
+using namespace metal;
+
+namespace uzu {
+namespace gemm {
+namespace schedules {
+
+struct TileContext {
+ short simdgroup_limit_m;
+ short simdgroup_limit_n;
+ size_t block_col;
+ ushort tile_col_offset;
+ ushort tile_block_cols;
+ uint k_offset;
+ uint abs_row_base;
+
+ METAL_FUNC uint absolute_column_base() const { return uint(block_col) + uint(tile_col_offset); }
+};
+
+} // namespace schedules
+} // namespace gemm
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 86c405de5..814211298 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -14,6 +14,8 @@
#include "gemm_tiling.h"
#include "quant_scale_bias.h"
#include "quant_scale_zero_point.h"
+#include "operands.h"
+#include "schedules/staged.h"
using namespace metal;
@@ -21,67 +23,55 @@ namespace uzu {
namespace gemm {
template <
- typename AT,
- typename BT,
- typename DT,
+ typename OutputElementType_,
GemmTiling GEMM_TILING,
bool TRANSPOSE_B,
- GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,
- int BITS = 0,
- int GROUP_SIZE = 0>
+ typename LeftOperand,
+ typename RightOperand>
struct SimdgroupMmaCore {
+ using OutputElementType = OutputElementType_;
+ using Left = operands::LeftBinding<LeftOperand, RightOperand>;
+ using Right = operands::RightBinding<RightOperand>;
+ using LeftElementType = typename Left::ElementType;
+ using RightElementType = typename Right::ElementType;
+ using LeftArgs = typename Left::Args;
+ using RightArgs = typename Right::Args;
+ static_assert(
+ metal::is_same<LeftOperand, operands::Dense<LeftElementType>>::value,
+ "simdgroup MMA stages full-precision left operands only"
+ );
+ UZU_CONST bool TRANSPOSE_RIGHT = TRANSPOSE_B;
UZU_CONST int THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
UZU_CONST int THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
UZU_CONST int THREADGROUP_BLOCK_K = gemm_tiling_block_k(GEMM_TILING);
UZU_CONST int SIMDGROUPS_PER_ROW = gemm_tiling_simdgroups_per_row(GEMM_TILING);
UZU_CONST int SIMDGROUPS_PER_COLUMN = gemm_tiling_simdgroups_per_column(GEMM_TILING);
- UZU_CONST ushort PADDING_A = 16 / sizeof(AT);
- UZU_CONST ushort PADDING_B = 16 / sizeof(BT);
+ UZU_CONST ushort PADDING_A = 16 / sizeof(LeftElementType);
+ UZU_CONST ushort PADDING_B = 16 / sizeof(RightElementType);
UZU_CONST ushort SHARED_STRIDE_A = THREADGROUP_BLOCK_K + PADDING_A;
UZU_CONST ushort SHARED_STRIDE_B = (TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N) + PADDING_B;
+ UZU_CONST ushort STAGING_ELEMENTS =
+ TRANSPOSE_B ? THREADGROUP_BLOCK_N * SHARED_STRIDE_B : THREADGROUP_BLOCK_K * SHARED_STRIDE_B;
UZU_CONST ushort THREADGROUP_THREADS = SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
- using ALoader = uzu::matmul::
- ThreadgroupLoader<AT, THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_K, SHARED_STRIDE_A, true, THREADGROUP_THREADS>;
- using BLoaderFp = uzu::matmul::ThreadgroupLoader<
- BT,
+ using ALoader = uzu::matmul::ThreadgroupLoader<
+ LeftElementType,
+ THREADGROUP_BLOCK_M,
+ THREADGROUP_BLOCK_K,
+ SHARED_STRIDE_A,
+ true,
+ THREADGROUP_THREADS>;
+ using FullPrecisionRightLoader = uzu::matmul::ThreadgroupLoader<
+ RightElementType,
TRANSPOSE_B ? THREADGROUP_BLOCK_N : THREADGROUP_BLOCK_K,
TRANSPOSE_B ? THREADGROUP_BLOCK_K : THREADGROUP_BLOCK_N,
SHARED_STRIDE_B,
TRANSPOSE_B,
THREADGROUP_THREADS>;
- using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<
- BT,
- THREADGROUP_BLOCK_N,
- THREADGROUP_BLOCK_K,
- SHARED_STRIDE_B,
- 1,
- THREADGROUP_THREADS,
- GROUP_SIZE,
- BITS>;
- using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<
- BT,
- THREADGROUP_BLOCK_N,
- THREADGROUP_BLOCK_K,
- SHARED_STRIDE_B,
- 1,
- THREADGROUP_THREADS,
- GROUP_SIZE,
- BITS>;
- using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<
- BT,
- THREADGROUP_BLOCK_N,
- THREADGROUP_BLOCK_K,
- SHARED_STRIDE_B,
- 1,
- THREADGROUP_THREADS,
- GROUP_SIZE,
- BITS,
- true>;
using TileAccumulator = uzu::matmul::ThreadgroupTile<
- AT,
- BT,
- DT,
+ LeftElementType,
+ RightElementType,
+ OutputElementType,
THREADGROUP_BLOCK_M,
THREADGROUP_BLOCK_N,
THREADGROUP_BLOCK_K,
@@ -92,12 +82,12 @@ struct SimdgroupMmaCore {
SHARED_STRIDE_A,
SHARED_STRIDE_B,
float,
- uzu::matmul::TransformNone<DT, float>>;
+ uzu::matmul::TransformNone<OutputElementType, float>>;
template <uint GEMM_ALIGNMENT_RAW, typename BLoader>
static METAL_FUNC void k_loop(
- threadgroup AT* a_shared,
- threadgroup BT* b_shared,
+ threadgroup LeftElementType* a_shared,
+ threadgroup RightElementType* b_shared,
const int aligned_k_iterations,
thread ALoader& loader_a,
thread BLoader& loader_b,
@@ -149,13 +139,13 @@ struct SimdgroupMmaCore {
template <uint GEMM_ALIGNMENT_RAW>
static METAL_FUNC void finalize(
thread TileAccumulator& accumulator,
- device DT* d,
+ device OutputElementType* d,
const constant uzu::matmul::GemmParams* params,
const thread ushort& tile_block_rows,
const thread ushort& tile_block_cols,
const bool needs_epilogue,
const thread uzu::matmul::TransformScaleAccumulate<float, float>& epilogue,
- const device BT* bias_block,
+ const device RightElementType* bias_block,
const bool needs_bias,
const device int32_t* rht_factors_block,
const bool needs_rht,
@@ -196,20 +186,16 @@ struct SimdgroupMmaCore {
}
static METAL_FUNC void run(
- const device AT* a,
- const device BT* b,
- device DT* d,
+ LeftArgs left,
+ RightArgs right,
+ device OutputElementType* d,
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
GemmDTransform output_transform,
- const bool signed_codes,
- const device BT* scales,
- const device BT* biases,
- const device uint8_t* zero_points,
- const device BT* output_bias,
+ const device RightElementType* output_bias,
const device int32_t* rht_factors,
- threadgroup AT* a_shared,
- threadgroup BT* b_shared,
+ threadgroup LeftElementType* a_shared,
+ threadgroup RightElementType* b_shared,
const thread ThreadContext& thread_context
) {
const uint partition = thread_context.threadgroup_position.z;
@@ -227,11 +213,11 @@ struct SimdgroupMmaCore {
const size_t block_row = size_t(geometry.block_row_start);
const size_t block_col = size_t(geometry.block_col_start);
- a += block_row * params->leading_dimension_a + k_offset;
+ left.seek_rows(block_row, params->leading_dimension_a, k_offset);
d +=
size_t(partition) * size_t(params->M) * size_t(params->N) + block_row * params->leading_dimension_d + block_col;
- thread ALoader loader_a(a, params->leading_dimension_a, a_shared, thread_context);
+ thread ALoader loader_a(left.values, params->leading_dimension_a, a_shared, thread_context);
thread TileAccumulator accumulator(thread_context);
const ushort tile_block_rows =
@@ -248,68 +234,17 @@ struct SimdgroupMmaCore {
const float alpha = needs_scale ? params->ab_scale : 1.0f;
const float beta = needs_accumulate ? 1.0f : 0.0f;
uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);
- const device BT* bias_block = output_bias + block_col;
+ const device RightElementType* bias_block = output_bias + block_col;
const device int32_t* rht_factors_block = rht_factors + block_col;
- auto loader_b = [&]() {
- if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
- const device BT* b_block_fp = b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +
- (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);
- return BLoaderFp(b_block_fp, params->leading_dimension_b, b_shared, thread_context);
- } else {
- constexpr int pack_factor = get_pack_factor<BITS, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
- const int k_elements = int(params->K);
- const int weights_row_stride_bytes = k_elements * bytes_per_pack / pack_factor;
- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
- const int k_offset_groups = int(k_offset) / GROUP_SIZE;
- const device uint8_t* weights_block = reinterpret_cast<const device uint8_t*>(b) +
- block_col * weights_row_stride_bytes +
- int(k_offset) * bytes_per_pack / pack_factor;
- const device BT* scales_offset = scales + block_col * groups_per_row + k_offset_groups;
-
- if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
- const device BT* biases_offset = biases + block_col * groups_per_row + k_offset_groups;
- return BLoaderScaleBias(
- weights_block,
- scales_offset,
- biases_offset,
- signed_codes,
- k_elements,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
- );
- } else if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant) {
- const int zero_point_stride_per_row = (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
- const device uint8_t* zero_points_row_start = zero_points + block_col * zero_point_stride_per_row +
- ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);
- return BLoaderScaleZeroPoint(
- weights_block,
- scales_offset,
- zero_points_row_start,
- signed_codes,
- k_elements,
- groups_per_row,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
- );
- } else {
- return BLoaderScaleSymmetric(
- weights_block,
- scales_offset,
- nullptr,
- signed_codes,
- k_elements,
- groups_per_row,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
- );
- }
- }
- }();
+ auto loader_b = schedules::StagedRightLoader<RightOperand>::template make<SimdgroupMmaCore>(
+ right,
+ params,
+ block_col,
+ k_offset,
+ b_shared,
+ thread_context
+ );
const bool all_aligned = ((alignment.contains(GemmAlignment::M)) || (tile_block_rows == THREADGROUP_BLOCK_M)) &&
((alignment.contains(GemmAlignment::N)) || (tile_block_cols == THREADGROUP_BLOCK_N)) &&
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index c8da2bb8e..ffc214532 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -140,38 +140,40 @@ KERNEL(Gemm)(
(void)thread_y;
(void)thread_z;
+ using LeftOperand = typename operands::LeftOperandFor<A_PROLOGUE, AT>::type;
+ using RightOperand = typename operands::RightOperandFor<B_PROLOGUE, ushort(BITS), ushort(GROUP_SIZE), BT>::type;
+ static_assert(
+ NEEDS_ASYMMETRIC_WEIGHT_CORRECTION == (A_IS_INT8 && operands::RightBinding<RightOperand>::NEEDS_CORRECTION),
+ "kernel bindings and operand correction policy must agree"
+ );
+ const auto left_args = operands::make_left_args<LeftOperand, RightOperand>(a, a_int8, a_scales, a_group_sums);
+ const auto right_args = operands::make_right_args<RightOperand>(b, scales, biases, zero_points, signed_codes);
+
if constexpr (USE_MXU) {
- MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE, A_PROLOGUE>::run(
- a,
- b,
+ using Core = MxuMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;
+ static_assert(GEMM_TGB_ELEMENTS >= Core::STAGING_ELEMENTS, "threadgroup right staging is undersized");
+ Core::run(
+ left_args,
+ right_args,
d,
params,
alignment,
output_transform,
- signed_codes,
- scales,
- biases,
- zero_points,
output_bias,
rht_factors,
- a_int8,
- a_scales,
- a_group_sums,
b_shared,
thread_context
);
} else {
- SimdgroupMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B, B_PROLOGUE, BITS, GROUP_SIZE>::run(
- a,
- b,
+ using Core = SimdgroupMmaCore<DT, GEMM_TILING, TRANSPOSE_B, LeftOperand, RightOperand>;
+ static_assert(GEMM_TGB_ELEMENTS >= Core::STAGING_ELEMENTS, "threadgroup right staging is undersized");
+ Core::run(
+ left_args,
+ right_args,
d,
params,
alignment,
output_transform,
- signed_codes,
- scales,
- biases,
- zero_points,
output_bias,
rht_factors,
a_shared,