Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

faster suffix=16 A8W4 (#911)

CCYeh3c06bed9651b
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
index 56e1140c1..4fe454596 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/quantized/cursor.h
@@ -68,6 +68,8 @@ struct Int8Cursor {
using Ops = typename Fragment::FragmentOpsType;
UZU_CONST short BLOCK_K = short(Fragment::COL_FRAGMENTS * Ops::FRAGMENT_ROWS);
+ struct PackedChunk {};
+
const device int8_t* origin;
const device int8_t* address;
int row_stride;
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
index 487a90a54..de9f50c10 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemm/common/schedules/integer.h
@@ -196,27 +196,47 @@ struct IntegerSchedule {
template <typename Core>
using GroupProducts = uzu::matmul::Fragment<int, Core::TILES_M, Core::TILES_N, typename Core::FragmentOps>;
- template <typename Core, typename LeftCodes, typename RightCodes>
- static METAL_FUNC GroupProducts<Core> multiply_group(thread LeftCodes& left_codes, thread RightCodes& right_codes) {
- constexpr int chunks_per_group = int(RIGHT_GROUP_SIZE) / int(Core::SIMDGROUP_BLOCK_K);
- constexpr bool PREFETCH_INT4_CHUNKS = Core::TILES_M == 1 && chunks_per_group > 1 && RightOperand::BITS == 4;
+ template <typename Core>
+ static constexpr int chunks_per_k_group() {
+ return int(RIGHT_GROUP_SIZE) / int(Core::SIMDGROUP_BLOCK_K);
+ }
+
+ template <typename Core>
+ static constexpr bool prefetches_int4_chunks() {
+ return Core::TILES_M == 1 && chunks_per_k_group<Core>() > 1 && RightOperand::BITS == 4;
+ }
+
+ template <typename RightCodes, int K_GROUP_COUNT, int CHUNKS_PER_K_GROUP>
+ static METAL_FUNC void fetch_k_groups(
+ const thread RightCodes& right_codes,
+ thread typename RightCodes::PackedChunk (&packed_right_chunks)[K_GROUP_COUNT][CHUNKS_PER_K_GROUP]
+ ) {
+ for_each_static_index<K_GROUP_COUNT>([&](const ushort k_group) {
+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
+ packed_right_chunks[k_group][chunk] = right_codes.fetch(uint(k_group * CHUNKS_PER_K_GROUP + chunk));
+ });
+ });
+ }
+
+ template <typename Core, typename LeftCodes, typename RightCodes, int CHUNKS_PER_K_GROUP>
+ static METAL_FUNC GroupProducts<Core> multiply_k_group(
+ thread LeftCodes& left_codes,
+ thread RightCodes& right_codes,
+ const thread typename RightCodes::PackedChunk (&packed_right_chunks)[CHUNKS_PER_K_GROUP]
+ ) {
GroupProducts<Core> group_product;
group_product.clear();
- if constexpr (PREFETCH_INT4_CHUNKS) {
- typename RightCodes::PackedChunk packed_chunks[chunks_per_group];
- for_each_static_index<chunks_per_group>([&](const ushort chunk) {
- packed_chunks[chunk] = right_codes.fetch(uint(chunk));
- });
- for_each_static_index<chunks_per_group>([&](const ushort chunk) {
+ if constexpr (prefetches_int4_chunks<Core>()) {
+ for_each_static_index<CHUNKS_PER_K_GROUP>([&](const ushort chunk) {
auto left_tile = left_codes.load(uint(chunk));
- auto right_tile = right_codes.decode(packed_chunks[chunk]);
+ auto right_tile = right_codes.decode(packed_right_chunks[chunk]);
uzu::matmul::fragment_mma(group_product, left_tile, right_tile);
left_codes.advance();
right_codes.advance();
});
} else {
METAL_PRAGMA_NO_UNROLL
- for (int chunk = 0; chunk < chunks_per_group; ++chunk) {
+ for (int chunk = 0; chunk < CHUNKS_PER_K_GROUP; ++chunk) {
auto left_tile = left_codes.load(uint(chunk));
auto right_tile = right_codes.load(uint(chunk));
uzu::matmul::fragment_mma(group_product, left_tile, right_tile);
@@ -317,15 +337,47 @@ struct IntegerSchedule {
accumulator.clear();
const int right_group_count = int(params->aligned_inner_iterations);
+ constexpr int CHUNKS_PER_K_GROUP = chunks_per_k_group<Core>();
+ constexpr bool PREFETCH_INT4_CHUNKS = prefetches_int4_chunks<Core>();
+ constexpr bool IS_ALIGNED_M_INT4_TILE16X32 =
+ PREFETCH_INT4_CHUNKS && ALIGNED_M && Core::TILING == GemmTiling::Tile16x32x256_Simdgroups1x1;
+ constexpr int K_GROUPS_PER_FETCH = IS_ALIGNED_M_INT4_TILE16X32 ? 4 : 1;
+ constexpr int MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH = 16;
+ using PackedRightChunk = typename decltype(right_codes)::PackedChunk;
+
+ int right_group_index = 0;
+ if constexpr (K_GROUPS_PER_FETCH > 1) {
+ const int batched_fetch_right_group_limit =
+ right_group_count >= MIN_K_GROUPS_PER_THREADGROUP_FOR_BATCHED_FETCH ? right_group_count : 0;
+ METAL_PRAGMA_NO_UNROLL
+ for (; right_group_index + K_GROUPS_PER_FETCH <= batched_fetch_right_group_limit;
+ right_group_index += K_GROUPS_PER_FETCH) {
+ PackedRightChunk packed_right_chunks[K_GROUPS_PER_FETCH][CHUNKS_PER_K_GROUP];
+ fetch_k_groups(right_codes, packed_right_chunks);
+ for_each_static_index<K_GROUPS_PER_FETCH>([&](const ushort k_group) {
+ const int fetched_right_group_index = right_group_index + int(k_group);
+ const uint right_group_offset = uint(fetched_right_group_index) * RIGHT_GROUP_SIZE;
+ left_codes.begin_k_group(right_group_offset);
+ right_codes.begin_k_group(right_group_offset);
+ auto group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks[k_group]);
+ left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);
+ right_metadata.load(right_storage, metadata_context, first_right_group + uint(fetched_right_group_index));
+ accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);
+ });
+ }
+ }
METAL_PRAGMA_NO_UNROLL
- for (int right_group_index = 0; right_group_index < right_group_count; ++right_group_index) {
- const uint absolute_right_group = first_right_group + uint(right_group_index);
- const uint right_group_offset = uint(right_group_index * int(RIGHT_GROUP_SIZE));
+ for (; right_group_index < right_group_count; ++right_group_index) {
+ PackedRightChunk packed_right_chunks[1][CHUNKS_PER_K_GROUP];
+ if constexpr (PREFETCH_INT4_CHUNKS) {
+ fetch_k_groups(right_codes, packed_right_chunks);
+ }
+ const uint right_group_offset = uint(right_group_index) * RIGHT_GROUP_SIZE;
left_codes.begin_k_group(right_group_offset);
right_codes.begin_k_group(right_group_offset);
- auto group_product = multiply_group<Core>(left_codes, right_codes);
+ auto group_product = multiply_k_group<Core>(left_codes, right_codes, packed_right_chunks[0]);
left_metadata.load(left_storage, metadata_context, tile.k_offset + right_group_offset);
- right_metadata.load(right_storage, metadata_context, absolute_right_group);
+ right_metadata.load(right_storage, metadata_context, first_right_group + uint(right_group_index));
accumulate_group<Core, ALIGNED_M, ALIGNED_N>(accumulator, group_product, left_metadata, right_metadata);
}