PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix SIGSEGV in pack-weight unpack/GEMM on malformed blob (#1928)
Luo Yu495e25399ad2
diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex b7b80dc6d..63e83af96 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -294,6 +294,7 @@ def woqgemm(scale_type,asym,):+ _validate_packed_blob(B, n, k, groupsize, compute_type, weight_type, scale_type, asym)m = A.shape[0]lib = get_lib(A)ct = cvtstr_dtype(compute_type)@@ -316,10 +317,73 @@ def woqgemm(wt,st,asym,+ B.numel(),)return C+def _validate_packed_blob(+ blob: torch.Tensor,+ n: int,+ k: int,+ groupsize: int,+ compute_type: str,+ weight_type: str,+ scale_type: str,+ asym: bool,+) -> None:+ """Validate a packed-weight blob before passing it to native code.++ Raises ``TypeError`` or ``ValueError`` if any check fails, preventing+ out-of-bounds memory access in the native ``unpack_weight`` /+ ``woqgemm`` path when the blob is malformed or the parameters are+ inconsistent with the blob contents.+ """+ if not isinstance(blob, torch.Tensor):+ raise TypeError(f"blob must be a torch.Tensor, got {type(blob).__name__}")++ if blob.device.type not in ("cpu", "xpu"):+ raise ValueError(f"blob must reside on cpu or xpu, got {blob.device.type}")++ if blob.dtype != torch.int8:+ raise ValueError(f"blob must have dtype torch.int8, got {blob.dtype}")++ if blob.dim() != 1:+ raise ValueError(f"blob must be a 1-D tensor, got {blob.dim()}-D")++ if not isinstance(n, (int,)) or n <= 0:+ raise ValueError(f"n must be a positive integer, got {n!r}")+ if not isinstance(k, (int,)) or k <= 0:+ raise ValueError(f"k must be a positive integer, got {k!r}")+ if not isinstance(groupsize, (int,)) or groupsize <= 0:+ raise ValueError(f"groupsize must be a positive integer, got {groupsize!r}")+ if not isinstance(asym, bool):+ raise TypeError(f"asym must be a bool, got {type(asym).__name__}")++ valid_types = {+ "fp32",+ "fp16",+ "bf16",+ "int8",+ "int4",+ "int2",+ "int3",+ "int5",+ "int6",+ "int7",+ "fp8_e4m3",+ "fp8_e5m2",+ "fp8_e8m0",+ }+ valid_compute_types = valid_types | {"auto"}+ if compute_type not in valid_compute_types:+ raise ValueError(f"compute_type must be one of {valid_compute_types}, got {compute_type!r}")+ if weight_type not in valid_types:+ raise ValueError(f"weight_type must be one of {valid_types}, got {weight_type!r}")+ if scale_type not in valid_types:+ raise ValueError(f"scale_type must be one of {valid_types}, got {scale_type!r}")++# QB: k*n:int8, scaleB: k/blocksize*n:DT# return: blob:BS:int8def _repack_quantized_weight_core(@@ -332,6 +396,12 @@ def _repack_quantized_weight_core(scale_type,asym,):+ if not isinstance(QB, torch.Tensor) or QB.dim() != 2:+ raise ValueError(f"QB must be a 2-D tensor, got shape {QB.shape!r}")+ if not isinstance(scaleB, torch.Tensor) or scaleB.dim() != 2:+ raise ValueError(f"scaleB must be a 2-D tensor, got shape {scaleB.shape!r}")+ if not isinstance(zp, torch.Tensor):+ raise TypeError(f"zp must be a torch.Tensor, got {type(zp).__name__}")k = QB.shape[0]n = QB.shape[1]lib = get_lib(QB)@@ -371,6 +441,7 @@ def _unpack_weight_core(scale_type,asym,):+ _validate_packed_blob(blob, n, k, groupsize, compute_type, weight_type, scale_type, asym)lib = get_lib(blob)stream = get_stream(blob)ct = cvtstr_dtype(compute_type)@@ -391,6 +462,7 @@ def _unpack_weight_core(wt,st,asym,+ blob.numel(),)if blob.device.type == "cpu":return out.Tdiff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex dbf0eac16..34e115c1e 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -55,13 +55,15 @@ static void woqgemm_s8(torch_ptr stream, int m, int n, int k, torch_ptr A, int A}static void woqgemm(torch_ptr stream, int m, int n, int k, torch_ptr A, int ACdt, torch_ptr BlobB, torch_ptr C,- torch_ptr bias, int blocksize, int compute_type, int weight_type, int scale_type, bool asym) {+ torch_ptr bias, int blocksize, int compute_type, int weight_type, int scale_type, bool asym,+ int blob_numel = 0) {QuantParam param{n, k, blocksize, compute_type, weight_type, scale_type, asym};+ size_t bc = static_cast<size_t>(blob_numel);#ifdef ARK_XPUXpuWrapper::woq_gemm(m, (void*)A, (void*)BlobB, (void*)C, (void*)bias, (BTLA_DTYPE)ACdt, ¶m,- (sycl::queue*)stream);+ (sycl::queue*)stream, bc);#else- CpuWrapper::woq_gemm(m, (void*)A, (void*)BlobB, (void*)C, (void*)bias, (BTLA_DTYPE)ACdt, ¶m);+ CpuWrapper::woq_gemm(m, (void*)A, (void*)BlobB, (void*)C, (void*)bias, (BTLA_DTYPE)ACdt, ¶m, bc);#endif}@@ -77,12 +79,13 @@ static void repack_quantized_weight(torch_ptr stream, torch_ptr raws8, torch_ptr}static void unpack_weight(torch_ptr stream, torch_ptr blob, torch_ptr output, int out_type, int n, int k, int blocksize,- int compute_type, int weight_type, int scale_type, bool asym) {+ int compute_type, int weight_type, int scale_type, bool asym, int blob_numel = 0) {QuantParam param{n, k, blocksize, compute_type, weight_type, scale_type, asym};+ size_t bc = static_cast<size_t>(blob_numel);#ifdef ARK_XPU- XpuWrapper::unpackq((BTLA_DTYPE)out_type, (int8_t*)blob, (void*)output, ¶m, (sycl::queue*)stream);+ XpuWrapper::unpackq((BTLA_DTYPE)out_type, (int8_t*)blob, (void*)output, ¶m, (sycl::queue*)stream, bc);#else- CpuWrapper::unpackq((BTLA_DTYPE)out_type, (int8_t*)blob, (void*)output, ¶m);+ CpuWrapper::unpackq((BTLA_DTYPE)out_type, (int8_t*)blob, (void*)output, ¶m, bc);#endif}diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hppindex b39e6a216..91f842bb5 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp@@ -291,7 +291,71 @@ static inline void BTLAGemmPackQB(void* PackedBuf, const int8_t* QB, const float}}+/// Validate a deserialized StorageWeightNInt to guard against corrupted metadata.+/// Throws std::runtime_error if any field is inconsistent or out of bounds.+static inline void _validate_storage_weight(const storage::gemm::StorageWeightNInt& stor,+ size_t blob_count) {+ using namespace bestla;+ // --- dimension sanity --------------------------------------------------+ if (stor.info_.n_ <= 0) {+ throw std::runtime_error("Corrupt packed weight: n must be positive");+ }+ if (stor.info_.k_ <= 0) {+ throw std::runtime_error("Corrupt packed weight: k must be positive");+ }+ constexpr int MAX_DIM = 1 << 24; // 16M — far beyond any realistic matrix+ if (stor.info_.n_ > MAX_DIM || stor.info_.k_ > MAX_DIM) {+ throw std::runtime_error("Corrupt packed weight: dimensions exceed sane maximum");+ }+ if (stor.info_.npad_ < stor.info_.n_) {+ throw std::runtime_error("Corrupt packed weight: npad < n");+ }+ if (stor.info_.kpad_ < stor.info_.k_) {+ throw std::runtime_error("Corrupt packed weight: kpad < k");+ }+ if (stor.info_.npad_ > MAX_DIM || stor.info_.kpad_ > MAX_DIM) {+ throw std::runtime_error("Corrupt packed weight: padded dimensions exceed sane maximum");+ }++ // --- block size ---------------------------------------------------------+ if (stor.block_ <= 0) {+ throw std::runtime_error("Corrupt packed weight: block size must be positive");+ }+ if (stor.block_ > stor.info_.k_) {+ throw std::runtime_error("Corrupt packed weight: block size > k");+ }+ int expected_nblk = (stor.info_.kpad_ + stor.block_ - 1) / stor.block_;+ if (stor.n_block_ != expected_nblk) {+ throw std::runtime_error("Corrupt packed weight: n_block inconsistent with kpad/block");+ }++ // --- layout -------------------------------------------------------------+ if (stor.info_.layout_ != storage::gemm::Layout::NPack) {+ throw std::runtime_error("Corrupt packed weight: expected NPack layout");+ }++ // --- buffer sizes against available data --------------------------------+ if (blob_count > 0) {+ // The serialised header sits at the start of the blob; data_ptr points+ // right after the header (offset = misc_size_ bytes from start).+ // After set_buffers the data occupies stor.buf_size_ bytes.+ auto off = stor.misc_size_;+ if (off + stor.buf_size_ > blob_count) {+ throw std::runtime_error("Corrupt packed weight: declared data size exceeds blob size (" ++ std::to_string(off) + " + " + std::to_string(stor.buf_size_) + " > " ++ std::to_string(blob_count) + ")");+ }+ }+}++// Forward declaration for the 4-arg overload used by the 3-arg compatibility wrapper.+static inline void BTLAGemmUnpack(void* B, float* DQB, void* ThreadPool, size_t blob_count);+static inline void BTLAGemmUnpack(void* B, float* DQB, void* ThreadPool) {+ BTLAGemmUnpack(B, DQB, ThreadPool, 0);+}++static inline void BTLAGemmUnpack(void* B, float* DQB, void* ThreadPool, size_t blob_count) {GetCPUDevice();using namespace bestla;auto prologue = storage::CollectionParser::parse_prologue_id(B);@@ -300,6 +364,8 @@ static inline void BTLAGemmUnpack(void* B, float* DQB, void* ThreadPool) {auto stor_ptr = (int8_t*)(B);storage::gemm::StorageWeightNInt stor;auto data_ptr = stor.deserialize(stor_ptr);+ // Validate deserialised metadata BEFORE calling set_buffers().+ _validate_storage_weight(stor, blob_count);stor.set_buffers(data_ptr);auto& id = stor.info_.core_id_;if (id == tAVX512_VNNI::ID) {@@ -436,7 +502,8 @@ static inline size_t BTLAGemmNPackBSize(int N, int K, BTLA_DTYPE dt) {return 0;}-static inline size_t BTLAWOQGemmForwardWorkspace(void* B, int M, int N, int K, int lda, int ldc) {+static inline size_t BTLAWOQGemmForwardWorkspace(void* B, int M, int N, int K, int lda, int ldc,+ size_t blob_count = 0) {GetCPUDevice();using namespace bestla;auto prologue = storage::CollectionParser::parse_prologue_id(B);@@ -445,6 +512,7 @@ static inline size_t BTLAWOQGemmForwardWorkspace(void* B, int M, int N, int K, istorage::gemm::StorageWeightNInt stor;auto stor_ptr = (int8_t*)(B);auto data_ptr = stor.deserialize(stor_ptr);+ _validate_storage_weight(stor, blob_count);stor.set_buffers(data_ptr);auto& id = stor.info_.core_id_;if (id == tAVX512_VNNI_KBlock::ID) {@@ -478,7 +546,8 @@ static inline size_t BTLAWOQGemmForwardWorkspace(void* B, int M, int N, int K, i}static inline void BTLAWOQGemmFp32Forward(void* B, const float* A, float* C, const float* bias, int M, int N, int K,- int lda, int ldc, void* ws_ptr, parallel::IThreading* ThreadPool) {+ int lda, int ldc, void* ws_ptr, parallel::IThreading* ThreadPool,+ size_t blob_count = 0) {GetCPUDevice();using namespace bestla;auto prologue = storage::CollectionParser::parse_prologue_id(B);@@ -487,6 +556,7 @@ static inline void BTLAWOQGemmFp32Forward(void* B, const float* A, float* C, conauto stor_ptr = (int8_t*)(B);storage::gemm::StorageWeightNInt stor;auto data_ptr = stor.deserialize(stor_ptr);+ _validate_storage_weight(stor, blob_count);stor.set_buffers(data_ptr);auto& id = stor.info_.core_id_;if (id == tAVX512_VNNI::ID) {@@ -536,9 +606,9 @@ class CpuWrapper {bestla::BTLAGemmPackQB(blob, raws8, scaleptr, p->asym ? zp : nullptr, coreid, p, get_threading());}- static void unpackq(BTLA_DTYPE outt, int8_t* blob, void* optr, QuantParam* p) {+ static void unpackq(BTLA_DTYPE outt, int8_t* blob, void* optr, QuantParam* p, size_t blob_count = 0) {assert(outt == BTLA_DTYPE::F32);- bestla::BTLAGemmUnpack(blob, (float*)optr, get_threading());+ bestla::BTLAGemmUnpack(blob, (float*)optr, get_threading(), blob_count);}static void gemm(int m, int n, int k, const void* A, BTLA_DTYPE dt, const void* B, bool BT, float* C,@@ -558,11 +628,12 @@ class CpuWrapper {bestla::BTLAGemmNPackBForward(dt, A, bptr, C, bias, m, n, k, k, n, packwptr, thd);}- static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p) {- auto wssize = bestla::BTLAWOQGemmForwardWorkspace((void*)b, m, p->n, p->k, p->k, p->n);+ static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p,+ size_t blob_count = 0) {+ auto wssize = bestla::BTLAWOQGemmForwardWorkspace((void*)b, m, p->n, p->k, p->k, p->n, blob_count);auto ptr = DnnlContext::Instance()->get_scratch_mem(wssize, 0, nullptr);bestla::BTLAWOQGemmFp32Forward((void*)b, (const float*)a, (float*)c, (const float*)bias, m, p->n, p->k, p->k, p->n,- ptr, get_threading());+ ptr, get_threading(), blob_count);}};diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppindex 2cc82ff19..cba3f7020 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -343,7 +343,15 @@ class XpuWrapper {packzp(zpptr, blob, p, q);}- static void unpackq(BTLA_DTYPE outt, int8_t* blob, void* optr, QuantParam* p, sycl::queue* q) {+ static void unpackq(BTLA_DTYPE outt, int8_t* blob, void* optr, QuantParam* p, sycl::queue* q,+ size_t blob_count = 0) {+ if (blob_count > 0) {+ auto expected = get_packw_size(p);+ if (blob_count < expected) {+ throw std::runtime_error("Corrupt packed weight: blob size (" + std::to_string(blob_count) ++ ") less than expected (" + std::to_string(expected) + ")");+ }+ }using namespace bestla;using namespace bestla::sycl_prologue_b;auto qptr = (uint8_t*)blob;@@ -678,7 +686,14 @@ class XpuWrapper {}static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p,- sycl::queue* q) {+ sycl::queue* q, size_t blob_count = 0) {+ if (blob_count > 0) {+ auto expected = get_packw_size(p);+ if (blob_count < expected) {+ throw std::runtime_error("Corrupt packed weight: blob size (" + std::to_string(blob_count) ++ ") less than expected (" + std::to_string(expected) + ")");+ }+ }auto ret = woq_gemv(q, m, p, a, b, c, bias, acdt);if (ret) {auto dnnl_dt = to_dt(acdt);