PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
XPU oneDNN: bind matmul execution to live SYCL queue (graph-capture safe) (#2280)
Copilot79516ff3026f
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex 94fe51ada..02273e27a 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -45,7 +45,7 @@ steps:- task: DownloadPipelineArtifact@2condition: and(succeeded(), eq(variables['BUILD_ARK_WHEEL'], 'true'))inputs:- artifact: auto-round-lib-wheel-${{ replace(parameters.imageTag, 'py', 'cp') }}+ artifact: auto-round-lib-wheel-cp314patterns: '*.whl'path: $(Build.SourcesDirectory)/ark_wheeldisplayName: "Download ark wheel"diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppindex aed72f4ca..affb8b5bb 100755--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -27,6 +27,7 @@ typedef uintptr_t torch_ptr;#include <sycl/sycl.hpp>#include "xpu_wrapper.hpp"#include "sycl_s8_wrapper.hpp"+#include "utils.hpp"#if ARK_SYCL_TLA#include "sycl_tla_common.hpp"#endif@@ -145,6 +146,33 @@ static size_t packed_weight_size(torch_ptr stream, int n, int k, int blocksize,#endif}+#if ARK_XPU+static size_t debug_xpu_device_context_key(torch_ptr stream) {+ return DeviceMemoryPool::Instance()->get_device_context_key(reinterpret_cast<sycl::queue*>(stream));+}++static size_t debug_xpu_device_queue_key(torch_ptr stream) {+ return DeviceMemoryPool::Instance()->get_device_key(reinterpret_cast<sycl::queue*>(stream));+}++static size_t debug_xpu_pool_scratch_ptr(torch_ptr stream, size_t bytes, size_t slot) {+ auto* ptr = DeviceMemoryPool::Instance()->get_scratch_mem(bytes, slot, reinterpret_cast<sycl::queue*>(stream));+ return reinterpret_cast<size_t>(ptr);+}++#if ARK_DNNL+static size_t debug_xpu_dnnl_engine_ptr(torch_ptr stream) {+ auto* eng = DnnlContext::Instance()->get_eng(reinterpret_cast<sycl::queue*>(stream));+ return reinterpret_cast<size_t>(eng);+}++static size_t debug_xpu_dnnl_scratch_ptr(torch_ptr stream, size_t bytes, size_t slot) {+ auto* ptr = DnnlContext::Instance()->get_scratch_mem(bytes, slot, reinterpret_cast<sycl::queue*>(stream));+ return reinterpret_cast<size_t>(ptr);+}+#endif+#endif+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)static void matmul_sycl_tla(torch_ptr stream, int m, int n, int k, torch_ptr A, int Adt, torch_ptr B,@@ -1322,6 +1350,15 @@ PYBIND11_MODULE(PY_NAME, m) {m.def("packed_weight_size", &ark::packed_weight_size);m.def("repack_quantized_weight", &ark::repack_quantized_weight);m.def("unpack_weight", &ark::unpack_weight);+#if defined(ARK_XPU)+ m.def("_debug_xpu_device_context_key", &ark::debug_xpu_device_context_key);+ m.def("_debug_xpu_device_queue_key", &ark::debug_xpu_device_queue_key);+ m.def("_debug_xpu_pool_scratch_ptr", &ark::debug_xpu_pool_scratch_ptr);+#if ARK_DNNL+ m.def("_debug_xpu_dnnl_engine_ptr", &ark::debug_xpu_dnnl_engine_ptr);+ m.def("_debug_xpu_dnnl_scratch_ptr", &ark::debug_xpu_dnnl_scratch_ptr);+#endif+#endif#if (defined(ARK_XPU) && defined(ARK_SYCL_TLA)) || !defined(ARK_XPU)m.def("sdpa", &ark::sdpa);#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cpp b/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cppindex 372d1d994..a9bd2040f 100644--- a/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cpp+++ b/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cpp@@ -3,7 +3,8 @@// The bookkeeping below is deliberately defined here rather than in a header so// that the module holds exactly one instance of it, and so that the buffers// themselves are owned by the extension-wide `DeviceMemoryPool` (keyed by-// device UUID) instead of by a header-local static keyed by raw `sycl::queue*`.+// device UUID + SYCL context + queue identity) instead of by a header-local+// static keyed by raw `sycl::queue*`.// See `sycl_tla_moe_decode_scratch.hpp` for the rationale.#include "sycl_tla_moe_decode_scratch.hpp"@@ -32,8 +33,8 @@ struct RepackTag {struct ScratchState {std::mutex mu;- // Device key (`DeviceMemoryPool::get_device_key`) -> a queue handle on that- // device. The queue is held *by value*: a `sycl::queue` is a reference+ // Device+context+queue key (`DeviceMemoryPool::get_device_key`) -> a queue+ // handle on that stream queue. The queue is held *by value*: a `sycl::queue` is a reference// counted handle, so keeping a copy guarantees the queue outlives the device// memory we allocated against it. This is what makes the pool immune to the// caller destroying its queue, and to a later queue landing on the samediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hppindex 7529eab33..ff5e22bb3 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp@@ -14,10 +14,6 @@#include "utils.hpp"-#define GETCTX() \- auto& eng = *DnnlContext::Instance()->get_eng(q); \- auto& stream = *DnnlContext::Instance()->get_stream(q);-namespace ark {class DnnlWrapper {@@ -27,7 +23,8 @@ class DnnlWrapper {static void gemm(sycl::queue* q, int m, int n, int k, const void* a, dt at, const void* b, dt bt, bool BT, void* c,dt ct, const void* bias) {- GETCTX();+ auto& eng = *DnnlContext::Instance()->get_eng(q);+ auto stream = DnnlContext::Instance()->get_stream(q);dnnl::memory::dims a_dims = {m, k};const auto a_in_md = dnnl::memory::desc(a_dims, at, dnnl::memory::format_tag::ab);auto a_mem = dnnl::memory(a_in_md, eng, const_cast<void*>(a));@@ -70,7 +67,8 @@ class DnnlWrapper {static void dyn_quant_s8(sycl::queue* q, int m, int k, const void* a, dt adt, void* a_abs, int8_t* qa, void* maxa,void* scalea) {- GETCTX();+ auto& eng = *DnnlContext::Instance()->get_eng(q);+ auto stream = DnnlContext::Instance()->get_stream(q);using namespace dnnl;auto src_f32_md = memory::desc({m, k}, adt, memory::format_tag::ab);auto src_f32_mem = memory(src_f32_md, eng, const_cast<void*>(a));@@ -172,7 +170,8 @@ class DnnlWrapper {// scale_a:m scale_b:nstatic void igemm_s8s8(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c, dt ct,void* scale_a, void* scale_b, void* bias) {- GETCTX();+ auto& eng = *DnnlContext::Instance()->get_eng(q);+ auto stream = DnnlContext::Instance()->get_stream(q);dnnl::memory::dims a_dims = {m, k};const auto a_in_md = dnnl::memory::desc(a_dims, dt::s8, tag::ab);diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hppindex d8162a5b3..f717c7735 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hpp@@ -22,22 +22,19 @@ namespace moe_decode_detail {// putting an unbounded device sync inside the lock.//// Both buffers are therefore served from the extension-wide `DeviceMemoryPool`,-// which keys on the device UUID rather than on a queue pointer, so slab-// identity follows the device and is immune to queue lifetime and address-// reuse. The bookkeeping below lives in `sycl_tla_moe_decode_scratch.cpp`, so-// there is exactly one instance in the module; this header only declares it.+// which keys on (device UUID, SYCL context, queue identity) rather than on a+// raw queue pointer, so slab identity follows the concrete execution stream and+// is immune to queue lifetime and address reuse. The bookkeeping below lives in+// `sycl_tla_moe_decode_scratch.cpp`, so there is exactly one instance in the+// module; this header only declares it.//// Slabs are intentionally never freed from a static destructor: the SYCL// context may already be torn down at that point. `release_decode_scratch`// provides explicit teardown for callers that need it (exposed to Python as// `moe_decode_release_scratch`).//-// Keying on the device rather than on the queue is what removes the duplicate-// slabs, but it also means two queues on the same device now share one slab-// instead of getting one each. That matches how every other `DeviceMemoryPool`-// slot already behaves, and the decode path runs on the device's current-// queue, but it does mean these entry points must not be driven concurrently-// from two queues on one device.+// Keying includes queue identity, so concurrent streams in one context use+// independent slabs while still reusing within each stream queue.// ----------------------------------------------------------------------------// `DeviceMemoryPool` slots owned by the decode path. Slots 0-7 belong to thediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hppindex ed53dd6c3..280a9cad4 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hpp@@ -178,14 +178,13 @@ using cute_scalar_t = typename cute_scalar<ScalarT>::type;//// Sharing one counter across calls is safe because every launcher call is// synchronous (`event.wait()` in `MoEGEMMLauncher`), so a dispatch never-// returns with the counter still in use. The pool keys buffers by device-// rather than by queue, so two queues on the same device share one slot; that-// is safe under the same host-side serialization the pool itself already-// assumes (its maps are unguarded, and every ark entry point is reached with-// the GIL held). The buffer lives until process exit, matching the singleton-// lifetime already used by `EventManager`. The INT8 and S4 headers re-export-// this helper rather than defining their own, so all three paths share one-// slot -- also safe, for the same reason.+// returns with the counter still in use. The pool keys buffers by+// (device UUID, SYCL context, queue identity), so concurrent queues in one+// context keep independent slots while each queue still reuses its own slot.+// The buffer lives until process exit,+// matching the singleton lifetime already used by `EventManager`. The INT8 and+// S4 headers re-export this helper rather than defining their own, so all+// three paths share one slot per queue.//// Note that a reused slot enters the kernel holding the previous dispatch's// final count rather than fresh (uninitialised) USM, so correctness restsdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hppindex 574149c1d..efa54a48d 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp@@ -16,10 +16,13 @@#include <cstdlib>#include <cstdint>#include <cstdio>+#include <memory>+#include <mutex>#include <stdexcept>#include <string>#include <type_traits>#include <unordered_map>+#include <vector>#if ARK_DNNL#include <dnnl.hpp>@@ -96,8 +99,32 @@ class DeviceMemoryPool {size_t get_device_key(sycl::queue* q) {#if ARK_XPUif (q != nullptr) {- auto uuid = q->get_device().get_info<sycl::ext::intel::info::device::uuid>();- return UUIDHasher{}(uuid);+ std::lock_guard<std::mutex> lock(key_mutex_);+ auto key = get_device_queue_key_locked(q);+ auto it = device_queue_ids_.find(key);+ if (it != device_queue_ids_.end()) {+ return it->second;+ }+ const size_t new_id = next_device_queue_id_++;+ device_queue_ids_.emplace(key, new_id);+ return new_id;+ }+#endif+ return 0;+ }++ size_t get_device_context_key(sycl::queue* q) {+#if ARK_XPU+ if (q != nullptr) {+ std::lock_guard<std::mutex> lock(key_mutex_);+ auto key = get_device_context_key_locked(q);+ auto it = device_context_ids_.find(key);+ if (it != device_context_ids_.end()) {+ return it->second;+ }+ const size_t new_id = next_device_context_id_++;+ device_context_ids_.emplace(key, new_id);+ return new_id;}#endifreturn 0;@@ -108,18 +135,17 @@ class DeviceMemoryPool {return get_scratch_ptr(size, buf_loc, q, key);}- // Current size of the slab held for ``buf_loc`` on ``q``'s device, or 0 when- // no slab is held. Callers that must synchronize before an existing slab is- // freed (because in-flight kernels may still reference it) use this to detect- // the grow path in `get_scratch_ptr` ahead of time.+ // Current size of the slab held for ``buf_loc`` on ``q``'s device+context+queue, or 0 when+ // no slab is held.size_t get_scratch_size(size_t buf_loc, sycl::queue* q) {if (buf_loc >= MaxLocNum) return 0;auto key = get_device_key(q);+ std::lock_guard<std::mutex> lock(scratch_mutex_);auto it = dev_mem_size_map[buf_loc].find(key);return it == dev_mem_size_map[buf_loc].end() ? 0 : it->second;}- // Detach the slab held for ``buf_loc`` on ``q``'s device and hand ownership to+ // Detach the slab held for ``buf_loc`` on ``q``'s device+context+queue and hand ownership to// the caller, which becomes responsible for synchronizing and freeing it.// Returns nullptr when no slab is held.//@@ -130,6 +156,7 @@ class DeviceMemoryPool {void* detach_scratch_mem(size_t buf_loc, sycl::queue* q) {if (buf_loc >= MaxLocNum) return nullptr;auto key = get_device_key(q);+ std::lock_guard<std::mutex> lock(scratch_mutex_);auto it = dev_mem_ptr_map[buf_loc].find(key);if (it == dev_mem_ptr_map[buf_loc].end()) return nullptr;int8_t* ptr = it->second;@@ -140,6 +167,7 @@ class DeviceMemoryPool {void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {if (size == 0 || buf_loc >= MaxLocNum) return nullptr;+ std::lock_guard<std::mutex> lock(scratch_mutex_);auto it = dev_mem_ptr_map[buf_loc].find(key);if (it == dev_mem_ptr_map[buf_loc].end()) {@@ -151,6 +179,26 @@ class DeviceMemoryPool {auto old_size = dev_mem_size_map[buf_loc][key];if (old_size < size) {+#if ARK_XPU+ if (q == nullptr) {+ throw std::invalid_argument("DeviceMemoryPool: XPU grow requires a non-null SYCL queue");+ }+ // Do NOT block the host here with q->wait(). Two reasons:+ // 1) It is redundant for correctness. The scratch slab is keyed+ // per-queue (see get_device_key), so only work enqueued on this+ // same queue `q` ever references this slab. Both the sycl::free+ // in release() below and the sycl::aligned_alloc_device in+ // allocate() are associated with `q`, and `q` is an in-order+ // queue (ARK device queues are built in_order; torch.xpu+ // streams are in-order by default). USM bookkeeping plus the+ // in-order ordering guarantee the old slab is not handed back+ // while any in-flight kernel still references it -- the same+ // guarantee the rest of the codebase relies on (e.g. the async+ // sycl::free in sycl_tla_moe_prefill_fp8_native).+ // 2) A blocking wait() is illegal while `q` is recording to a+ // torch.xpu command graph, so a wait here would throw whenever a+ // scratch slab grows during graph capture (see issue #2206).+#endifrelease(it->second, q);auto newptr = allocate(size, q);dev_mem_size_map[buf_loc][key] = size;@@ -171,6 +219,91 @@ class DeviceMemoryPool {using SizeMap = std::unordered_map<size_t, size_t>;using PtrMap = std::unordered_map<size_t, int8_t*>;+#if ARK_XPU+ struct DeviceContextKey {+ UUIDArray device_uuid;+ size_t context_id;++ bool operator==(const DeviceContextKey& other) const {+ return context_id == other.context_id && device_uuid == other.device_uuid;+ }+ };++ struct DeviceContextKeyHasher {+ size_t operator()(const DeviceContextKey& key) const {+ size_t h = UUIDHasher{}(key.device_uuid);+ h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);+ return h;+ }+ };++ struct DeviceQueueKey {+ UUIDArray device_uuid;+ size_t context_id;+ size_t queue_id;++ bool operator==(const DeviceQueueKey& other) const {+ return queue_id == other.queue_id && context_id == other.context_id && device_uuid == other.device_uuid;+ }+ };++ struct DeviceQueueKeyHasher {+ size_t operator()(const DeviceQueueKey& key) const {+ size_t h = UUIDHasher{}(key.device_uuid);+ h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);+ h ^= std::hash<size_t>{}(key.queue_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);+ return h;+ }+ };++ UUIDArray get_device_uuid_locked(sycl::queue* q) {+ return q->get_device().get_info<sycl::ext::intel::info::device::uuid>();+ }++ DeviceContextKey get_device_context_key_locked(sycl::queue* q) {+ auto context_id = get_context_id_locked(q->get_context());+ return {get_device_uuid_locked(q), context_id};+ }++ size_t get_context_id_locked(const sycl::context& ctx) {+ for (const auto& entry : context_ids_) {+ if (entry.context == ctx) {+ return entry.context_id;+ }+ }+ const size_t new_id = next_context_id_++;+ context_ids_.push_back({ctx, new_id});+ return new_id;+ }++ struct ContextEntry {+ sycl::context context;+ size_t context_id;+ };++ size_t get_queue_id_locked(const sycl::queue& q) {+ for (const auto& entry : queue_ids_) {+ if (entry.queue == q) {+ return entry.queue_id;+ }+ }+ const size_t new_id = next_queue_id_++;+ queue_ids_.push_back({q, new_id});+ return new_id;+ }++ DeviceQueueKey get_device_queue_key_locked(sycl::queue* q) {+ auto context_id = get_context_id_locked(q->get_context());+ auto queue_id = get_queue_id_locked(*q);+ return {get_device_uuid_locked(q), context_id, queue_id};+ }++ struct QueueEntry {+ sycl::queue queue;+ size_t queue_id;+ };+#endif+int8_t* allocate(size_t size, sycl::queue* q) {#if ARK_XPUif (q == nullptr) {@@ -196,6 +329,18 @@ class DeviceMemoryPool {std::array<SizeMap, MaxLocNum> dev_mem_size_map;std::array<PtrMap, MaxLocNum> dev_mem_ptr_map;+ std::mutex scratch_mutex_;+#if ARK_XPU+ std::mutex key_mutex_;+ std::vector<ContextEntry> context_ids_;+ std::vector<QueueEntry> queue_ids_;+ std::unordered_map<DeviceContextKey, size_t, DeviceContextKeyHasher> device_context_ids_;+ std::unordered_map<DeviceQueueKey, size_t, DeviceQueueKeyHasher> device_queue_ids_;+ size_t next_context_id_ = 1;+ size_t next_queue_id_ = 1;+ size_t next_device_context_id_ = 1;+ size_t next_device_queue_id_ = 1;+#endif};#if ARK_DNNL@@ -245,61 +390,156 @@ class DnnlContext {}dnnl::engine* get_eng(sycl::queue* q) {- auto key = check_dnnl_device(q);- return &dev_engine_map[key];- }-- dnnl::stream* get_stream(sycl::queue* q) {- auto key = check_dnnl_device(q);- return &dev_stream_map[key];- }-- size_t check_dnnl_device(sycl::queue* q) {- size_t key = 0;+ std::lock_guard<std::mutex> lock(mutex_);if (q == nullptr) {- if (dev_engine_map.find(key) == dev_engine_map.end()) {- dev_engine_map[key] = dnnl::engine(dnnl::engine::kind::cpu, 0);- dev_stream_map[key] = dnnl::stream(dev_engine_map[key]);+ if (!cpu_engine_) {+ cpu_engine_ = std::make_unique<dnnl::engine>(dnnl::engine::kind::cpu, 0);}- return key;+ return cpu_engine_.get();}#if ARK_XPU- key = DeviceMemoryPool::Instance()->get_device_key(q);- if (dev_engine_map.find(key) == dev_engine_map.end()) {- sycl::device dev = q->get_device();- sycl::context ctx = q->get_context();- dev_engine_map[key] = dnnl::sycl_interop::make_engine(dev, ctx);- dev_stream_map[key] = dnnl::sycl_interop::make_stream(dev_engine_map[key], *q);+ auto key = get_device_context_key_locked(q);+ auto engine_it = xpu_engines_.find(key);+ if (engine_it != xpu_engines_.end()) {+ return engine_it->second.get();}+ auto dev = q->get_device();+ auto ctx = q->get_context();+ auto insert_result =+ xpu_engines_.emplace(key, std::make_unique<dnnl::engine>(dnnl::sycl_interop::make_engine(dev, ctx)));+ return insert_result.first->second.get();#else- if (dev_engine_map.find(key) == dev_engine_map.end()) {- dev_engine_map[key] = dnnl::engine(dnnl::engine::kind::cpu, 0);- dev_stream_map[key] = dnnl::stream(dev_engine_map[key]);+ if (!cpu_engine_) {+ cpu_engine_ = std::make_unique<dnnl::engine>(dnnl::engine::kind::cpu, 0);}+ return cpu_engine_.get();#endif+ }- return key;+ dnnl::stream get_stream(sycl::queue* q) {+ auto* eng = get_eng(q);+ if (q == nullptr) {+ return dnnl::stream(*eng);+ }+#if ARK_XPU+ return dnnl::sycl_interop::make_stream(*eng, *q);+#else+ return dnnl::stream(*eng);+#endif}dnnl::memory get_scratch_mem(dnnl::memory::desc md, sycl::queue* q) {- auto key = check_dnnl_device(q);+ auto key = get_dnnl_key(q);auto ptr = DeviceMemoryPool::Instance()->get_scratch_ptr(md.get_size(), 0, q, key);- return dnnl::memory(md, dev_engine_map[key], ptr);+ return dnnl::memory(md, *get_eng(q), ptr);}void* get_scratch_mem(size_t size, size_t buf_loc, sycl::queue* q) {- return DeviceMemoryPool::Instance()->get_scratch_mem(size, buf_loc, q);+ auto key = get_dnnl_key(q);+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);}void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {+ if (q == nullptr) {+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);+ }+#if ARK_XPU+ std::lock_guard<std::mutex> lock(mutex_);+ auto queue_key = DeviceMemoryPool::Instance()->get_device_key(q);+ auto scratch_key = QueueScratchKey{queue_key, key};+ auto it = dnnl_queue_scratch_ids_.find(scratch_key);+ if (it != dnnl_queue_scratch_ids_.end()) {+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, it->second);+ }+ const size_t new_id = next_dnnl_queue_scratch_id_++;+ dnnl_queue_scratch_ids_.emplace(scratch_key, new_id);+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, new_id);+#elsereturn DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);+#endif}private:- std::unordered_map<size_t, dnnl::engine> dev_engine_map;- std::unordered_map<size_t, dnnl::stream> dev_stream_map;+ struct DeviceContextKey {+ UUIDArray device_uuid;+ size_t context_id;++ bool operator==(const DeviceContextKey& other) const {+ return context_id == other.context_id && device_uuid == other.device_uuid;+ }+ };++ struct DeviceContextKeyHasher {+ size_t operator()(const DeviceContextKey& key) const {+ size_t h = UUIDHasher{}(key.device_uuid);+ h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);+ return h;+ }+ };++ size_t get_dnnl_key(sycl::queue* q) {+ if (q == nullptr) return 0;+#if ARK_XPU+ return DeviceMemoryPool::Instance()->get_device_key(q);+#else+ return 0;+#endif+ }++#if ARK_XPU+ UUIDArray get_device_uuid_locked(sycl::queue* q) {+ return q->get_device().get_info<sycl::ext::intel::info::device::uuid>();+ }++ DeviceContextKey get_device_context_key_locked(sycl::queue* q) {+ auto context_id = get_context_id_locked(q->get_context());+ return {get_device_uuid_locked(q), context_id};+ }++ size_t get_context_id_locked(const sycl::context& ctx) {+ for (const auto& entry : context_ids_) {+ if (entry.context == ctx) {+ return entry.context_id;+ }+ }+ const size_t new_id = next_context_id_++;+ context_ids_.push_back({ctx, new_id});+ return new_id;+ }++ struct ContextEntry {+ sycl::context context;+ size_t context_id;+ };++ struct QueueScratchKey {+ size_t queue_key;+ size_t user_key;++ bool operator==(const QueueScratchKey& other) const {+ return queue_key == other.queue_key && user_key == other.user_key;+ }+ };++ struct QueueScratchKeyHasher {+ size_t operator()(const QueueScratchKey& key) const {+ size_t h = std::hash<size_t>{}(key.queue_key);+ h ^= std::hash<size_t>{}(key.user_key) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);+ return h;+ }+ };++ std::vector<ContextEntry> context_ids_;+ std::unordered_map<QueueScratchKey, size_t, QueueScratchKeyHasher> dnnl_queue_scratch_ids_;+ std::unordered_map<DeviceContextKey, std::unique_ptr<dnnl::engine>, DeviceContextKeyHasher> xpu_engines_;+ size_t next_dnnl_queue_scratch_id_ = 1;+ size_t next_context_id_ = 1;+#endif++ std::unique_ptr<dnnl::engine> cpu_engine_;+ std::mutex mutex_;};#endif // ARK_DNNLdiff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.pyindex d2fa042ac..10bbb17e2 100644--- a/auto_round_extension/ark/test/test_weightonly.py+++ b/auto_round_extension/ark/test/test_weightonly.py@@ -229,6 +229,140 @@ def test_xpu_s8_large_tile(m, n, k):torch.xpu.empty_cache()+ os.environ.get("ARK_DNNL", "0") not in ["1", "ON", "true", "True"],+ reason="Skipped because ARK_DNNL is not enabled in environment variables",+)+def test_xpu_woqgemm_graph_capture_uses_live_queue():+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ required = ("CUDAGraph", "Stream", "graph", "stream")+ if any(not hasattr(torch.xpu, name) for name in required):+ pytest.skip("torch.xpu graph APIs are not available")+ xpu_lib = getattr(ark, "xpu_lib", None)+ debug_required = (+ "_debug_xpu_device_context_key",+ "_debug_xpu_device_queue_key",+ "_debug_xpu_pool_scratch_ptr",+ "_debug_xpu_dnnl_engine_ptr",+ "_debug_xpu_dnnl_scratch_ptr",+ )+ if xpu_lib is None or any(not hasattr(xpu_lib, name) for name in debug_required):+ pytest.skip("XPU debug probes for context-aware scratch/engine isolation are unavailable")++ with torch.no_grad():+ default_stream = torch.xpu.current_stream()+ capture_stream = torch.xpu.Stream()++ def _on_stream(stream, fn, *args):+ with torch.xpu.stream(stream):+ return fn(torch.xpu.current_stream().sycl_queue, *args)++ default_context_key = int(_on_stream(default_stream, xpu_lib._debug_xpu_device_context_key))+ capture_context_key = int(_on_stream(capture_stream, xpu_lib._debug_xpu_device_context_key))+ if default_context_key != capture_context_key:+ pytest.skip("Unable to create two queues in the same SYCL context for this XPU runtime")++ default_queue_key = int(_on_stream(default_stream, xpu_lib._debug_xpu_device_queue_key))+ capture_queue_key = int(_on_stream(capture_stream, xpu_lib._debug_xpu_device_queue_key))+ if default_queue_key == capture_queue_key:+ pytest.skip("Unable to create two distinct queues in this SYCL context")++ def _assert_pool_slot_isolated(slot, small_bytes, large_bytes):+ ptr_a0 = int(_on_stream(default_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))+ ptr_a1 = int(_on_stream(default_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))+ assert ptr_a0 != 0 and ptr_a0 == ptr_a1+ ptr_b0 = int(_on_stream(capture_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))+ assert ptr_b0 != 0 and ptr_b0 != ptr_a0+ ptr_b1 = int(_on_stream(capture_stream, xpu_lib._debug_xpu_pool_scratch_ptr, large_bytes, slot))+ assert ptr_b1 != 0+ ptr_a2 = int(_on_stream(default_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))+ assert ptr_a2 == ptr_a0++ for slot, small, large in ((8, 4, 64), (9, 4096, 8192), (10, 4096, 8192)):+ _assert_pool_slot_isolated(slot, small, large)++ eng_a0 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_engine_ptr))+ eng_a1 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_engine_ptr))+ eng_b = int(_on_stream(capture_stream, xpu_lib._debug_xpu_dnnl_engine_ptr))+ assert eng_a0 != 0 and eng_a0 == eng_a1+ assert eng_b != 0 and eng_b == eng_a0++ dnnl_scratch_a0 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 4096, 0))+ dnnl_scratch_b0 = int(_on_stream(capture_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 4096, 0))+ assert dnnl_scratch_b0 != 0 and dnnl_scratch_b0 != dnnl_scratch_a0+ _ = _on_stream(capture_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 8192, 0)+ dnnl_scratch_a1 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 4096, 0))+ assert dnnl_scratch_a0 != 0 and dnnl_scratch_a1 == dnnl_scratch_a0++ m, n, k = 8, 256, 256+ blocksize = 32+ compute_type = "fp16"+ weight_type = "int4"+ scale_type = "fp16"+ asym = False++ raw_s8_wei = gen_weis8(weight_type, "xpu", k, n)+ scale = torch.rand(k // blocksize, n, dtype=torch.float16, device="xpu") / 300 + 0.002+ bias = torch.randn(1, n, dtype=torch.float16, device="xpu")+ zp = torch.empty(0, device=raw_s8_wei.device)+ packw = ark.repack_quantized_weight(+ raw_s8_wei, scale, zp, blocksize, compute_type, weight_type, scale_type, asym+ )++ def _woq_call(x):+ return ark.woqgemm(x, packw, bias, n, k, blocksize, compute_type, weight_type, scale_type, asym)++ eager_input = torch.randn(m, k, dtype=torch.float16, device="xpu") - 0.5+ replay_input = eager_input + 0.75++ # Force a smaller scratch allocation on capture_stream first so the+ # overlapping call below takes the growth path on that queue.+ with torch.xpu.stream(capture_stream):+ _ = _woq_call(eager_input[:1])++ # Submit work to two queues in the same SYCL context before any global+ # synchronize so the executions can overlap in-flight.+ with torch.xpu.stream(default_stream):+ out_a = _woq_call(eager_input)+ with torch.xpu.stream(capture_stream):+ out_b = _woq_call(replay_input)+ torch.xpu.synchronize()+ expected_a = _woq_call(eager_input)+ expected_b = _woq_call(replay_input)+ torch.xpu.synchronize()+ assert torch.allclose(out_a, expected_a, rtol=0.1, atol=2.0)+ assert torch.allclose(out_b, expected_b, rtol=0.1, atol=2.0)+ assert not torch.allclose(out_a, out_b, rtol=1e-3, atol=1e-3)++ # Warm up on the default queue to reproduce stale stream cache regressions.+ _ = _woq_call(eager_input)+ torch.xpu.synchronize()+ graph_input = eager_input.clone()+ graph = torch.xpu.CUDAGraph()++ try:+ with torch.xpu.stream(capture_stream):+ with torch.xpu.graph(graph):+ graph_output = _woq_call(graph_input)+ except RuntimeError as err:+ pytest.skip(f"XPU graph capture is unavailable in this environment: {err}")++ outputs = []+ for current in (eager_input, replay_input):+ graph_input.copy_(current)+ graph.replay()+ torch.xpu.synchronize()+ replayed = graph_output.clone()+ expected = _woq_call(current)+ torch.xpu.synchronize()+ assert torch.allclose(replayed, expected, rtol=0.1, atol=2.0)+ outputs.append(replayed)++ assert not torch.allclose(outputs[0], outputs[1], rtol=1e-3, atol=1e-3)+ torch.xpu.empty_cache()++@pytest.mark.parametrize("m", [1, 1024])@pytest.mark.parametrize("n, k", [(1024, 768)])@pytest.mark.parametrize("blocksize", [32, 128])