Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

XPU oneDNN: bind matmul execution to live SYCL queue (graph-capture safe) (#2280)

Copilot79516ff3026f
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 94fe51ada..02273e27a 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -45,7 +45,7 @@ steps:
- task: DownloadPipelineArtifact@2
condition: and(succeeded(), eq(variables['BUILD_ARK_WHEEL'], 'true'))
inputs:
- artifact: auto-round-lib-wheel-${{ replace(parameters.imageTag, 'py', 'cp') }}
+ artifact: auto-round-lib-wheel-cp314
patterns: '*.whl'
path: $(Build.SourcesDirectory)/ark_wheel
displayName: "Download ark wheel"
diff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cpp
index aed72f4ca..affb8b5bb 100755
--- a/auto_round_extension/ark/auto_round_kernel/ark.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp
@@ -27,6 +27,7 @@ typedef uintptr_t torch_ptr;
#include <sycl/sycl.hpp>
#include "xpu_wrapper.hpp"
#include "sycl_s8_wrapper.hpp"
+#include "utils.hpp"
#if ARK_SYCL_TLA
#include "sycl_tla_common.hpp"
#endif
@@ -145,6 +146,33 @@ static size_t packed_weight_size(torch_ptr stream, int n, int k, int blocksize,
#endif
}
+#if ARK_XPU
+static size_t debug_xpu_device_context_key(torch_ptr stream) {
+ return DeviceMemoryPool::Instance()->get_device_context_key(reinterpret_cast<sycl::queue*>(stream));
+}
+
+static size_t debug_xpu_device_queue_key(torch_ptr stream) {
+ return DeviceMemoryPool::Instance()->get_device_key(reinterpret_cast<sycl::queue*>(stream));
+}
+
+static size_t debug_xpu_pool_scratch_ptr(torch_ptr stream, size_t bytes, size_t slot) {
+ auto* ptr = DeviceMemoryPool::Instance()->get_scratch_mem(bytes, slot, reinterpret_cast<sycl::queue*>(stream));
+ return reinterpret_cast<size_t>(ptr);
+}
+
+#if ARK_DNNL
+static size_t debug_xpu_dnnl_engine_ptr(torch_ptr stream) {
+ auto* eng = DnnlContext::Instance()->get_eng(reinterpret_cast<sycl::queue*>(stream));
+ return reinterpret_cast<size_t>(eng);
+}
+
+static size_t debug_xpu_dnnl_scratch_ptr(torch_ptr stream, size_t bytes, size_t slot) {
+ auto* ptr = DnnlContext::Instance()->get_scratch_mem(bytes, slot, reinterpret_cast<sycl::queue*>(stream));
+ return reinterpret_cast<size_t>(ptr);
+}
+#endif
+#endif
+
#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)
static void matmul_sycl_tla(torch_ptr stream, int m, int n, int k, torch_ptr A, int Adt, torch_ptr B,
@@ -1322,6 +1350,15 @@ PYBIND11_MODULE(PY_NAME, m) {
m.def("packed_weight_size", &ark::packed_weight_size);
m.def("repack_quantized_weight", &ark::repack_quantized_weight);
m.def("unpack_weight", &ark::unpack_weight);
+#if defined(ARK_XPU)
+ m.def("_debug_xpu_device_context_key", &ark::debug_xpu_device_context_key);
+ m.def("_debug_xpu_device_queue_key", &ark::debug_xpu_device_queue_key);
+ m.def("_debug_xpu_pool_scratch_ptr", &ark::debug_xpu_pool_scratch_ptr);
+#if ARK_DNNL
+ m.def("_debug_xpu_dnnl_engine_ptr", &ark::debug_xpu_dnnl_engine_ptr);
+ m.def("_debug_xpu_dnnl_scratch_ptr", &ark::debug_xpu_dnnl_scratch_ptr);
+#endif
+#endif
#if (defined(ARK_XPU) && defined(ARK_SYCL_TLA)) || !defined(ARK_XPU)
m.def("sdpa", &ark::sdpa);
#endif
diff --git a/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cpp b/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cpp
index 372d1d994..a9bd2040f 100644
--- a/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cpp
+++ b/auto_round_extension/ark/auto_round_kernel/sycl_tla_moe_decode_scratch.cpp
@@ -3,7 +3,8 @@
// The bookkeeping below is deliberately defined here rather than in a header so
// that the module holds exactly one instance of it, and so that the buffers
// themselves are owned by the extension-wide `DeviceMemoryPool` (keyed by
-// device UUID) instead of by a header-local static keyed by raw `sycl::queue*`.
+// device UUID + SYCL context + queue identity) instead of by a header-local
+// static keyed by raw `sycl::queue*`.
// See `sycl_tla_moe_decode_scratch.hpp` for the rationale.
#include "sycl_tla_moe_decode_scratch.hpp"
@@ -32,8 +33,8 @@ struct RepackTag {
struct ScratchState {
std::mutex mu;
- // Device key (`DeviceMemoryPool::get_device_key`) -> a queue handle on that
- // device. The queue is held *by value*: a `sycl::queue` is a reference
+ // Device+context+queue key (`DeviceMemoryPool::get_device_key`) -> a queue
+ // handle on that stream queue. The queue is held *by value*: a `sycl::queue` is a reference
// counted handle, so keeping a copy guarantees the queue outlives the device
// memory we allocated against it. This is what makes the pool immune to the
// caller destroying its queue, and to a later queue landing on the same
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp
index 7529eab33..ff5e22bb3 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp
@@ -14,10 +14,6 @@
#include "utils.hpp"
-#define GETCTX() \
- auto& eng = *DnnlContext::Instance()->get_eng(q); \
- auto& stream = *DnnlContext::Instance()->get_stream(q);
-
namespace ark {
class DnnlWrapper {
@@ -27,7 +23,8 @@ class DnnlWrapper {
static void gemm(sycl::queue* q, int m, int n, int k, const void* a, dt at, const void* b, dt bt, bool BT, void* c,
dt ct, const void* bias) {
- GETCTX();
+ auto& eng = *DnnlContext::Instance()->get_eng(q);
+ auto stream = DnnlContext::Instance()->get_stream(q);
dnnl::memory::dims a_dims = {m, k};
const auto a_in_md = dnnl::memory::desc(a_dims, at, dnnl::memory::format_tag::ab);
auto a_mem = dnnl::memory(a_in_md, eng, const_cast<void*>(a));
@@ -70,7 +67,8 @@ class DnnlWrapper {
static void dyn_quant_s8(sycl::queue* q, int m, int k, const void* a, dt adt, void* a_abs, int8_t* qa, void* maxa,
void* scalea) {
- GETCTX();
+ auto& eng = *DnnlContext::Instance()->get_eng(q);
+ auto stream = DnnlContext::Instance()->get_stream(q);
using namespace dnnl;
auto src_f32_md = memory::desc({m, k}, adt, memory::format_tag::ab);
auto src_f32_mem = memory(src_f32_md, eng, const_cast<void*>(a));
@@ -172,7 +170,8 @@ class DnnlWrapper {
// scale_a:m scale_b:n
static void igemm_s8s8(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c, dt ct,
void* scale_a, void* scale_b, void* bias) {
- GETCTX();
+ auto& eng = *DnnlContext::Instance()->get_eng(q);
+ auto stream = DnnlContext::Instance()->get_stream(q);
dnnl::memory::dims a_dims = {m, k};
const auto a_in_md = dnnl::memory::desc(a_dims, dt::s8, tag::ab);
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hpp
index d8162a5b3..f717c7735 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_decode_scratch.hpp
@@ -22,22 +22,19 @@ namespace moe_decode_detail {
// putting an unbounded device sync inside the lock.
//
// Both buffers are therefore served from the extension-wide `DeviceMemoryPool`,
-// which keys on the device UUID rather than on a queue pointer, so slab
-// identity follows the device and is immune to queue lifetime and address
-// reuse. The bookkeeping below lives in `sycl_tla_moe_decode_scratch.cpp`, so
-// there is exactly one instance in the module; this header only declares it.
+// which keys on (device UUID, SYCL context, queue identity) rather than on a
+// raw queue pointer, so slab identity follows the concrete execution stream and
+// is immune to queue lifetime and address reuse. The bookkeeping below lives in
+// `sycl_tla_moe_decode_scratch.cpp`, so there is exactly one instance in the
+// module; this header only declares it.
//
// Slabs are intentionally never freed from a static destructor: the SYCL
// context may already be torn down at that point. `release_decode_scratch`
// provides explicit teardown for callers that need it (exposed to Python as
// `moe_decode_release_scratch`).
//
-// Keying on the device rather than on the queue is what removes the duplicate
-// slabs, but it also means two queues on the same device now share one slab
-// instead of getting one each. That matches how every other `DeviceMemoryPool`
-// slot already behaves, and the decode path runs on the device's current
-// queue, but it does mean these entry points must not be driven concurrently
-// from two queues on one device.
+// Keying includes queue identity, so concurrent streams in one context use
+// independent slabs while still reusing within each stream queue.
// ----------------------------------------------------------------------------
// `DeviceMemoryPool` slots owned by the decode path. Slots 0-7 belong to the
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hpp
index ed53dd6c3..280a9cad4 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe_prefill_fp8_dpas.hpp
@@ -178,14 +178,13 @@ using cute_scalar_t = typename cute_scalar<ScalarT>::type;
//
// Sharing one counter across calls is safe because every launcher call is
// synchronous (`event.wait()` in `MoEGEMMLauncher`), so a dispatch never
-// returns with the counter still in use. The pool keys buffers by device
-// rather than by queue, so two queues on the same device share one slot; that
-// is safe under the same host-side serialization the pool itself already
-// assumes (its maps are unguarded, and every ark entry point is reached with
-// the GIL held). The buffer lives until process exit, matching the singleton
-// lifetime already used by `EventManager`. The INT8 and S4 headers re-export
-// this helper rather than defining their own, so all three paths share one
-// slot -- also safe, for the same reason.
+// returns with the counter still in use. The pool keys buffers by
+// (device UUID, SYCL context, queue identity), so concurrent queues in one
+// context keep independent slots while each queue still reuses its own slot.
+// The buffer lives until process exit,
+// matching the singleton lifetime already used by `EventManager`. The INT8 and
+// S4 headers re-export this helper rather than defining their own, so all
+// three paths share one slot per queue.
//
// Note that a reused slot enters the kernel holding the previous dispatch's
// final count rather than fresh (uninitialised) USM, so correctness rests
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
index 574149c1d..efa54a48d 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp
@@ -16,10 +16,13 @@
#include <cstdlib>
#include <cstdint>
#include <cstdio>
+#include <memory>
+#include <mutex>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <unordered_map>
+#include <vector>
#if ARK_DNNL
#include <dnnl.hpp>
@@ -96,8 +99,32 @@ class DeviceMemoryPool {
size_t get_device_key(sycl::queue* q) {
#if ARK_XPU
if (q != nullptr) {
- auto uuid = q->get_device().get_info<sycl::ext::intel::info::device::uuid>();
- return UUIDHasher{}(uuid);
+ std::lock_guard<std::mutex> lock(key_mutex_);
+ auto key = get_device_queue_key_locked(q);
+ auto it = device_queue_ids_.find(key);
+ if (it != device_queue_ids_.end()) {
+ return it->second;
+ }
+ const size_t new_id = next_device_queue_id_++;
+ device_queue_ids_.emplace(key, new_id);
+ return new_id;
+ }
+#endif
+ return 0;
+ }
+
+ size_t get_device_context_key(sycl::queue* q) {
+#if ARK_XPU
+ if (q != nullptr) {
+ std::lock_guard<std::mutex> lock(key_mutex_);
+ auto key = get_device_context_key_locked(q);
+ auto it = device_context_ids_.find(key);
+ if (it != device_context_ids_.end()) {
+ return it->second;
+ }
+ const size_t new_id = next_device_context_id_++;
+ device_context_ids_.emplace(key, new_id);
+ return new_id;
}
#endif
return 0;
@@ -108,18 +135,17 @@ class DeviceMemoryPool {
return get_scratch_ptr(size, buf_loc, q, key);
}
- // Current size of the slab held for ``buf_loc`` on ``q``'s device, or 0 when
- // no slab is held. Callers that must synchronize before an existing slab is
- // freed (because in-flight kernels may still reference it) use this to detect
- // the grow path in `get_scratch_ptr` ahead of time.
+ // Current size of the slab held for ``buf_loc`` on ``q``'s device+context+queue, or 0 when
+ // no slab is held.
size_t get_scratch_size(size_t buf_loc, sycl::queue* q) {
if (buf_loc >= MaxLocNum) return 0;
auto key = get_device_key(q);
+ std::lock_guard<std::mutex> lock(scratch_mutex_);
auto it = dev_mem_size_map[buf_loc].find(key);
return it == dev_mem_size_map[buf_loc].end() ? 0 : it->second;
}
- // Detach the slab held for ``buf_loc`` on ``q``'s device and hand ownership to
+ // Detach the slab held for ``buf_loc`` on ``q``'s device+context+queue and hand ownership to
// the caller, which becomes responsible for synchronizing and freeing it.
// Returns nullptr when no slab is held.
//
@@ -130,6 +156,7 @@ class DeviceMemoryPool {
void* detach_scratch_mem(size_t buf_loc, sycl::queue* q) {
if (buf_loc >= MaxLocNum) return nullptr;
auto key = get_device_key(q);
+ std::lock_guard<std::mutex> lock(scratch_mutex_);
auto it = dev_mem_ptr_map[buf_loc].find(key);
if (it == dev_mem_ptr_map[buf_loc].end()) return nullptr;
int8_t* ptr = it->second;
@@ -140,6 +167,7 @@ class DeviceMemoryPool {
void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {
if (size == 0 || buf_loc >= MaxLocNum) return nullptr;
+ std::lock_guard<std::mutex> lock(scratch_mutex_);
auto it = dev_mem_ptr_map[buf_loc].find(key);
if (it == dev_mem_ptr_map[buf_loc].end()) {
@@ -151,6 +179,26 @@ class DeviceMemoryPool {
auto old_size = dev_mem_size_map[buf_loc][key];
if (old_size < size) {
+#if ARK_XPU
+ if (q == nullptr) {
+ throw std::invalid_argument("DeviceMemoryPool: XPU grow requires a non-null SYCL queue");
+ }
+ // Do NOT block the host here with q->wait(). Two reasons:
+ // 1) It is redundant for correctness. The scratch slab is keyed
+ // per-queue (see get_device_key), so only work enqueued on this
+ // same queue `q` ever references this slab. Both the sycl::free
+ // in release() below and the sycl::aligned_alloc_device in
+ // allocate() are associated with `q`, and `q` is an in-order
+ // queue (ARK device queues are built in_order; torch.xpu
+ // streams are in-order by default). USM bookkeeping plus the
+ // in-order ordering guarantee the old slab is not handed back
+ // while any in-flight kernel still references it -- the same
+ // guarantee the rest of the codebase relies on (e.g. the async
+ // sycl::free in sycl_tla_moe_prefill_fp8_native).
+ // 2) A blocking wait() is illegal while `q` is recording to a
+ // torch.xpu command graph, so a wait here would throw whenever a
+ // scratch slab grows during graph capture (see issue #2206).
+#endif
release(it->second, q);
auto newptr = allocate(size, q);
dev_mem_size_map[buf_loc][key] = size;
@@ -171,6 +219,91 @@ class DeviceMemoryPool {
using SizeMap = std::unordered_map<size_t, size_t>;
using PtrMap = std::unordered_map<size_t, int8_t*>;
+#if ARK_XPU
+ struct DeviceContextKey {
+ UUIDArray device_uuid;
+ size_t context_id;
+
+ bool operator==(const DeviceContextKey& other) const {
+ return context_id == other.context_id && device_uuid == other.device_uuid;
+ }
+ };
+
+ struct DeviceContextKeyHasher {
+ size_t operator()(const DeviceContextKey& key) const {
+ size_t h = UUIDHasher{}(key.device_uuid);
+ h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
+ return h;
+ }
+ };
+
+ struct DeviceQueueKey {
+ UUIDArray device_uuid;
+ size_t context_id;
+ size_t queue_id;
+
+ bool operator==(const DeviceQueueKey& other) const {
+ return queue_id == other.queue_id && context_id == other.context_id && device_uuid == other.device_uuid;
+ }
+ };
+
+ struct DeviceQueueKeyHasher {
+ size_t operator()(const DeviceQueueKey& key) const {
+ size_t h = UUIDHasher{}(key.device_uuid);
+ h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
+ h ^= std::hash<size_t>{}(key.queue_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
+ return h;
+ }
+ };
+
+ UUIDArray get_device_uuid_locked(sycl::queue* q) {
+ return q->get_device().get_info<sycl::ext::intel::info::device::uuid>();
+ }
+
+ DeviceContextKey get_device_context_key_locked(sycl::queue* q) {
+ auto context_id = get_context_id_locked(q->get_context());
+ return {get_device_uuid_locked(q), context_id};
+ }
+
+ size_t get_context_id_locked(const sycl::context& ctx) {
+ for (const auto& entry : context_ids_) {
+ if (entry.context == ctx) {
+ return entry.context_id;
+ }
+ }
+ const size_t new_id = next_context_id_++;
+ context_ids_.push_back({ctx, new_id});
+ return new_id;
+ }
+
+ struct ContextEntry {
+ sycl::context context;
+ size_t context_id;
+ };
+
+ size_t get_queue_id_locked(const sycl::queue& q) {
+ for (const auto& entry : queue_ids_) {
+ if (entry.queue == q) {
+ return entry.queue_id;
+ }
+ }
+ const size_t new_id = next_queue_id_++;
+ queue_ids_.push_back({q, new_id});
+ return new_id;
+ }
+
+ DeviceQueueKey get_device_queue_key_locked(sycl::queue* q) {
+ auto context_id = get_context_id_locked(q->get_context());
+ auto queue_id = get_queue_id_locked(*q);
+ return {get_device_uuid_locked(q), context_id, queue_id};
+ }
+
+ struct QueueEntry {
+ sycl::queue queue;
+ size_t queue_id;
+ };
+#endif
+
int8_t* allocate(size_t size, sycl::queue* q) {
#if ARK_XPU
if (q == nullptr) {
@@ -196,6 +329,18 @@ class DeviceMemoryPool {
std::array<SizeMap, MaxLocNum> dev_mem_size_map;
std::array<PtrMap, MaxLocNum> dev_mem_ptr_map;
+ std::mutex scratch_mutex_;
+#if ARK_XPU
+ std::mutex key_mutex_;
+ std::vector<ContextEntry> context_ids_;
+ std::vector<QueueEntry> queue_ids_;
+ std::unordered_map<DeviceContextKey, size_t, DeviceContextKeyHasher> device_context_ids_;
+ std::unordered_map<DeviceQueueKey, size_t, DeviceQueueKeyHasher> device_queue_ids_;
+ size_t next_context_id_ = 1;
+ size_t next_queue_id_ = 1;
+ size_t next_device_context_id_ = 1;
+ size_t next_device_queue_id_ = 1;
+#endif
};
#if ARK_DNNL
@@ -245,61 +390,156 @@ class DnnlContext {
}
dnnl::engine* get_eng(sycl::queue* q) {
- auto key = check_dnnl_device(q);
- return &dev_engine_map[key];
- }
-
- dnnl::stream* get_stream(sycl::queue* q) {
- auto key = check_dnnl_device(q);
- return &dev_stream_map[key];
- }
-
- size_t check_dnnl_device(sycl::queue* q) {
- size_t key = 0;
+ std::lock_guard<std::mutex> lock(mutex_);
if (q == nullptr) {
- if (dev_engine_map.find(key) == dev_engine_map.end()) {
- dev_engine_map[key] = dnnl::engine(dnnl::engine::kind::cpu, 0);
- dev_stream_map[key] = dnnl::stream(dev_engine_map[key]);
+ if (!cpu_engine_) {
+ cpu_engine_ = std::make_unique<dnnl::engine>(dnnl::engine::kind::cpu, 0);
}
- return key;
+ return cpu_engine_.get();
}
#if ARK_XPU
- key = DeviceMemoryPool::Instance()->get_device_key(q);
- if (dev_engine_map.find(key) == dev_engine_map.end()) {
- sycl::device dev = q->get_device();
- sycl::context ctx = q->get_context();
- dev_engine_map[key] = dnnl::sycl_interop::make_engine(dev, ctx);
- dev_stream_map[key] = dnnl::sycl_interop::make_stream(dev_engine_map[key], *q);
+ auto key = get_device_context_key_locked(q);
+ auto engine_it = xpu_engines_.find(key);
+ if (engine_it != xpu_engines_.end()) {
+ return engine_it->second.get();
}
+ auto dev = q->get_device();
+ auto ctx = q->get_context();
+ auto insert_result =
+ xpu_engines_.emplace(key, std::make_unique<dnnl::engine>(dnnl::sycl_interop::make_engine(dev, ctx)));
+ return insert_result.first->second.get();
#else
- if (dev_engine_map.find(key) == dev_engine_map.end()) {
- dev_engine_map[key] = dnnl::engine(dnnl::engine::kind::cpu, 0);
- dev_stream_map[key] = dnnl::stream(dev_engine_map[key]);
+ if (!cpu_engine_) {
+ cpu_engine_ = std::make_unique<dnnl::engine>(dnnl::engine::kind::cpu, 0);
}
+ return cpu_engine_.get();
#endif
+ }
- return key;
+ dnnl::stream get_stream(sycl::queue* q) {
+ auto* eng = get_eng(q);
+ if (q == nullptr) {
+ return dnnl::stream(*eng);
+ }
+#if ARK_XPU
+ return dnnl::sycl_interop::make_stream(*eng, *q);
+#else
+ return dnnl::stream(*eng);
+#endif
}
dnnl::memory get_scratch_mem(dnnl::memory::desc md, sycl::queue* q) {
- auto key = check_dnnl_device(q);
+ auto key = get_dnnl_key(q);
auto ptr = DeviceMemoryPool::Instance()->get_scratch_ptr(md.get_size(), 0, q, key);
- return dnnl::memory(md, dev_engine_map[key], ptr);
+ return dnnl::memory(md, *get_eng(q), ptr);
}
void* get_scratch_mem(size_t size, size_t buf_loc, sycl::queue* q) {
- return DeviceMemoryPool::Instance()->get_scratch_mem(size, buf_loc, q);
+ auto key = get_dnnl_key(q);
+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);
}
void* get_scratch_ptr(size_t size, size_t buf_loc, sycl::queue* q, size_t key) {
+ if (q == nullptr) {
+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);
+ }
+#if ARK_XPU
+ std::lock_guard<std::mutex> lock(mutex_);
+ auto queue_key = DeviceMemoryPool::Instance()->get_device_key(q);
+ auto scratch_key = QueueScratchKey{queue_key, key};
+ auto it = dnnl_queue_scratch_ids_.find(scratch_key);
+ if (it != dnnl_queue_scratch_ids_.end()) {
+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, it->second);
+ }
+ const size_t new_id = next_dnnl_queue_scratch_id_++;
+ dnnl_queue_scratch_ids_.emplace(scratch_key, new_id);
+ return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, new_id);
+#else
return DeviceMemoryPool::Instance()->get_scratch_ptr(size, buf_loc, q, key);
+#endif
}
private:
- std::unordered_map<size_t, dnnl::engine> dev_engine_map;
- std::unordered_map<size_t, dnnl::stream> dev_stream_map;
+ struct DeviceContextKey {
+ UUIDArray device_uuid;
+ size_t context_id;
+
+ bool operator==(const DeviceContextKey& other) const {
+ return context_id == other.context_id && device_uuid == other.device_uuid;
+ }
+ };
+
+ struct DeviceContextKeyHasher {
+ size_t operator()(const DeviceContextKey& key) const {
+ size_t h = UUIDHasher{}(key.device_uuid);
+ h ^= std::hash<size_t>{}(key.context_id) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
+ return h;
+ }
+ };
+
+ size_t get_dnnl_key(sycl::queue* q) {
+ if (q == nullptr) return 0;
+#if ARK_XPU
+ return DeviceMemoryPool::Instance()->get_device_key(q);
+#else
+ return 0;
+#endif
+ }
+
+#if ARK_XPU
+ UUIDArray get_device_uuid_locked(sycl::queue* q) {
+ return q->get_device().get_info<sycl::ext::intel::info::device::uuid>();
+ }
+
+ DeviceContextKey get_device_context_key_locked(sycl::queue* q) {
+ auto context_id = get_context_id_locked(q->get_context());
+ return {get_device_uuid_locked(q), context_id};
+ }
+
+ size_t get_context_id_locked(const sycl::context& ctx) {
+ for (const auto& entry : context_ids_) {
+ if (entry.context == ctx) {
+ return entry.context_id;
+ }
+ }
+ const size_t new_id = next_context_id_++;
+ context_ids_.push_back({ctx, new_id});
+ return new_id;
+ }
+
+ struct ContextEntry {
+ sycl::context context;
+ size_t context_id;
+ };
+
+ struct QueueScratchKey {
+ size_t queue_key;
+ size_t user_key;
+
+ bool operator==(const QueueScratchKey& other) const {
+ return queue_key == other.queue_key && user_key == other.user_key;
+ }
+ };
+
+ struct QueueScratchKeyHasher {
+ size_t operator()(const QueueScratchKey& key) const {
+ size_t h = std::hash<size_t>{}(key.queue_key);
+ h ^= std::hash<size_t>{}(key.user_key) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
+ return h;
+ }
+ };
+
+ std::vector<ContextEntry> context_ids_;
+ std::unordered_map<QueueScratchKey, size_t, QueueScratchKeyHasher> dnnl_queue_scratch_ids_;
+ std::unordered_map<DeviceContextKey, std::unique_ptr<dnnl::engine>, DeviceContextKeyHasher> xpu_engines_;
+ size_t next_dnnl_queue_scratch_id_ = 1;
+ size_t next_context_id_ = 1;
+#endif
+
+ std::unique_ptr<dnnl::engine> cpu_engine_;
+ std::mutex mutex_;
};
#endif // ARK_DNNL
diff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.py
index d2fa042ac..10bbb17e2 100644
--- a/auto_round_extension/ark/test/test_weightonly.py
+++ b/auto_round_extension/ark/test/test_weightonly.py
@@ -229,6 +229,140 @@ def test_xpu_s8_large_tile(m, n, k):
torch.xpu.empty_cache()
+ os.environ.get("ARK_DNNL", "0") not in ["1", "ON", "true", "True"],
+ reason="Skipped because ARK_DNNL is not enabled in environment variables",
+)
+def test_xpu_woqgemm_graph_capture_uses_live_queue():
+ if not torch.xpu.is_available():
+ pytest.skip("No XPU Device")
+ required = ("CUDAGraph", "Stream", "graph", "stream")
+ if any(not hasattr(torch.xpu, name) for name in required):
+ pytest.skip("torch.xpu graph APIs are not available")
+ xpu_lib = getattr(ark, "xpu_lib", None)
+ debug_required = (
+ "_debug_xpu_device_context_key",
+ "_debug_xpu_device_queue_key",
+ "_debug_xpu_pool_scratch_ptr",
+ "_debug_xpu_dnnl_engine_ptr",
+ "_debug_xpu_dnnl_scratch_ptr",
+ )
+ if xpu_lib is None or any(not hasattr(xpu_lib, name) for name in debug_required):
+ pytest.skip("XPU debug probes for context-aware scratch/engine isolation are unavailable")
+
+ with torch.no_grad():
+ default_stream = torch.xpu.current_stream()
+ capture_stream = torch.xpu.Stream()
+
+ def _on_stream(stream, fn, *args):
+ with torch.xpu.stream(stream):
+ return fn(torch.xpu.current_stream().sycl_queue, *args)
+
+ default_context_key = int(_on_stream(default_stream, xpu_lib._debug_xpu_device_context_key))
+ capture_context_key = int(_on_stream(capture_stream, xpu_lib._debug_xpu_device_context_key))
+ if default_context_key != capture_context_key:
+ pytest.skip("Unable to create two queues in the same SYCL context for this XPU runtime")
+
+ default_queue_key = int(_on_stream(default_stream, xpu_lib._debug_xpu_device_queue_key))
+ capture_queue_key = int(_on_stream(capture_stream, xpu_lib._debug_xpu_device_queue_key))
+ if default_queue_key == capture_queue_key:
+ pytest.skip("Unable to create two distinct queues in this SYCL context")
+
+ def _assert_pool_slot_isolated(slot, small_bytes, large_bytes):
+ ptr_a0 = int(_on_stream(default_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))
+ ptr_a1 = int(_on_stream(default_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))
+ assert ptr_a0 != 0 and ptr_a0 == ptr_a1
+ ptr_b0 = int(_on_stream(capture_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))
+ assert ptr_b0 != 0 and ptr_b0 != ptr_a0
+ ptr_b1 = int(_on_stream(capture_stream, xpu_lib._debug_xpu_pool_scratch_ptr, large_bytes, slot))
+ assert ptr_b1 != 0
+ ptr_a2 = int(_on_stream(default_stream, xpu_lib._debug_xpu_pool_scratch_ptr, small_bytes, slot))
+ assert ptr_a2 == ptr_a0
+
+ for slot, small, large in ((8, 4, 64), (9, 4096, 8192), (10, 4096, 8192)):
+ _assert_pool_slot_isolated(slot, small, large)
+
+ eng_a0 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_engine_ptr))
+ eng_a1 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_engine_ptr))
+ eng_b = int(_on_stream(capture_stream, xpu_lib._debug_xpu_dnnl_engine_ptr))
+ assert eng_a0 != 0 and eng_a0 == eng_a1
+ assert eng_b != 0 and eng_b == eng_a0
+
+ dnnl_scratch_a0 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 4096, 0))
+ dnnl_scratch_b0 = int(_on_stream(capture_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 4096, 0))
+ assert dnnl_scratch_b0 != 0 and dnnl_scratch_b0 != dnnl_scratch_a0
+ _ = _on_stream(capture_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 8192, 0)
+ dnnl_scratch_a1 = int(_on_stream(default_stream, xpu_lib._debug_xpu_dnnl_scratch_ptr, 4096, 0))
+ assert dnnl_scratch_a0 != 0 and dnnl_scratch_a1 == dnnl_scratch_a0
+
+ m, n, k = 8, 256, 256
+ blocksize = 32
+ compute_type = "fp16"
+ weight_type = "int4"
+ scale_type = "fp16"
+ asym = False
+
+ raw_s8_wei = gen_weis8(weight_type, "xpu", k, n)
+ scale = torch.rand(k // blocksize, n, dtype=torch.float16, device="xpu") / 300 + 0.002
+ bias = torch.randn(1, n, dtype=torch.float16, device="xpu")
+ zp = torch.empty(0, device=raw_s8_wei.device)
+ packw = ark.repack_quantized_weight(
+ raw_s8_wei, scale, zp, blocksize, compute_type, weight_type, scale_type, asym
+ )
+
+ def _woq_call(x):
+ return ark.woqgemm(x, packw, bias, n, k, blocksize, compute_type, weight_type, scale_type, asym)
+
+ eager_input = torch.randn(m, k, dtype=torch.float16, device="xpu") - 0.5
+ replay_input = eager_input + 0.75
+
+ # Force a smaller scratch allocation on capture_stream first so the
+ # overlapping call below takes the growth path on that queue.
+ with torch.xpu.stream(capture_stream):
+ _ = _woq_call(eager_input[:1])
+
+ # Submit work to two queues in the same SYCL context before any global
+ # synchronize so the executions can overlap in-flight.
+ with torch.xpu.stream(default_stream):
+ out_a = _woq_call(eager_input)
+ with torch.xpu.stream(capture_stream):
+ out_b = _woq_call(replay_input)
+ torch.xpu.synchronize()
+ expected_a = _woq_call(eager_input)
+ expected_b = _woq_call(replay_input)
+ torch.xpu.synchronize()
+ assert torch.allclose(out_a, expected_a, rtol=0.1, atol=2.0)
+ assert torch.allclose(out_b, expected_b, rtol=0.1, atol=2.0)
+ assert not torch.allclose(out_a, out_b, rtol=1e-3, atol=1e-3)
+
+ # Warm up on the default queue to reproduce stale stream cache regressions.
+ _ = _woq_call(eager_input)
+ torch.xpu.synchronize()
+ graph_input = eager_input.clone()
+ graph = torch.xpu.CUDAGraph()
+
+ try:
+ with torch.xpu.stream(capture_stream):
+ with torch.xpu.graph(graph):
+ graph_output = _woq_call(graph_input)
+ except RuntimeError as err:
+ pytest.skip(f"XPU graph capture is unavailable in this environment: {err}")
+
+ outputs = []
+ for current in (eager_input, replay_input):
+ graph_input.copy_(current)
+ graph.replay()
+ torch.xpu.synchronize()
+ replayed = graph_output.clone()
+ expected = _woq_call(current)
+ torch.xpu.synchronize()
+ assert torch.allclose(replayed, expected, rtol=0.1, atol=2.0)
+ outputs.append(replayed)
+
+ assert not torch.allclose(outputs[0], outputs[1], rtol=1e-3, atol=1e-3)
+ torch.xpu.empty_cache()
+
+
@pytest.mark.parametrize("m", [1, 1024])
@pytest.mark.parametrize("n, k", [(1024, 768)])
@pytest.mark.parametrize("blocksize", [32, 128])