Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

[Engine] SYCL: build the grouped-expert bench, and port it to SYCL

Niko1221da3df90eb871
diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt
index 6e817a76e..17e47c514 100644
--- a/sycl/CMakeLists.txt
+++ b/sycl/CMakeLists.txt
@@ -451,6 +451,13 @@ if(STRATA_SYCL_PARITY)
add_executable(native_expert_parity ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/native_expert_parity.cpp)
target_include_directories(native_expert_parity PRIVATE ${STRATA_ROOT}/third_party/ggml)
target_link_libraries(native_expert_parity PRIVATE strata_kernels strata_kernels_cpu)
+ # SYCL port: the same grouped native experts timed per mode on real GGUF rows at the decode window's shape
+ # (G experts, T tokens), checked bit for bit against the reference mode. The source was in the tree without a
+ # target, so the numbers that decide the decode expert path could not be reproduced.
+ # native_expert_bench <shard1.gguf> <layer> <G> <T> <ref mode> <mode> [iters]
+ add_executable(native_expert_bench ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/native_expert_bench.cpp)
+ target_include_directories(native_expert_bench PRIVATE ${STRATA_ROOT}/third_party/ggml)
+ target_link_libraries(native_expert_bench PRIVATE strata_kernels strata_kernels_cpu)
# Gyro (APR) trellis experts on real GGUF tensors: tq_sycl_parity <pack> <model.gguf> [layer ...]
add_executable(tq_sycl_parity ${CMAKE_CURRENT_SOURCE_DIR}/src/kernels/tq_sycl_parity.cpp)
target_include_directories(tq_sycl_parity PRIVATE ${STRATA_ROOT}/third_party/ggml)
diff --git a/sycl/src/kernels/native_expert_bench.cpp b/sycl/src/kernels/native_expert_bench.cpp
index 48e048762..163d2e625 100644
--- a/sycl/src/kernels/native_expert_bench.cpp
+++ b/sycl/src/kernels/native_expert_bench.cpp
@@ -15,6 +15,7 @@
#include "ggml.h"
+#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
@@ -48,9 +49,6 @@ int main(int argc, char** argv) {
int failures = 0;
dpct::queue_ptr s;
s = dpct::get_current_device().create_queue(true);
- dpct::event_ptr e0, e1;
- e0 = new sycl::event();
- e1 = new sycl::event();
for (int l : layers) {
const strata::TensorInfo* t[3] = {};
const char* roles[3] = {"gate", "up", "down"};
@@ -86,21 +84,18 @@ int main(int argc, char** argv) {
std::vector<unsigned long long> ptr(G);
std::vector<int32_t> start(G + 1), dst(NE), tok(NE);
const auto L = K::native_expert_layout(f.gu_type, f.d_type, H, FF);
- uint8_t* dblob;
- void *dx, *dxq, *dscr;
- float* dout;
- unsigned long long* dptr;
- int32_t *dstart, *dn, *ddst, *dtok;
- cudaMalloc((void**) &dblob, blobs.size());
- cudaMalloc(&dx, x.size() * 4);
- cudaMalloc(&dxq, (size_t) NTOK * H / 32 * 36);
- cudaMalloc(&dscr, K::native_expert_scratch_bytes(NE, FF));
- cudaMalloc((void**) &dout, (size_t) NE * H * 4);
- cudaMalloc((void**) &dptr, (size_t) G * 8);
- cudaMalloc((void**) &dstart, (size_t) (G + 1) * 4);
- cudaMalloc((void**) &dn, 4);
- cudaMalloc((void**) &ddst, (size_t) NE * 4);
- cudaMalloc((void**) &dtok, (size_t) NE * 4);
+ // SYCL port: this bench was never a build target, so its CUDA calls were never migrated. Device buffers are
+ // USM allocations on the bench's own in-order queue, and every copy is ordered on that queue.
+ uint8_t* dblob = (uint8_t*) sycl::malloc_device(blobs.size(), *s);
+ void *dx = sycl::malloc_device((size_t) NTOK * H * 4, *s);
+ void *dxq = sycl::malloc_device((size_t) NTOK * H / 32 * 36, *s);
+ void *dscr = sycl::malloc_device(K::native_expert_scratch_bytes(NE, FF), *s);
+ float* dout = (float*) sycl::malloc_device((size_t) NE * H * 4, *s);
+ unsigned long long* dptr = (unsigned long long*) sycl::malloc_device((size_t) G * 8, *s);
+ int32_t* dstart = (int32_t*) sycl::malloc_device((size_t) (G + 1) * 4, *s);
+ int32_t* dn = (int32_t*) sycl::malloc_device(4, *s);
+ int32_t* ddst = (int32_t*) sycl::malloc_device((size_t) NE * 4, *s);
+ int32_t* dtok = (int32_t*) sycl::malloc_device((size_t) NE * 4, *s);
for (int g = 0; g < G; ++g) {
ptr[g] = (unsigned long long) (dblob + (size_t) g * f.bytes);
start[g] = g * T;
@@ -110,26 +105,26 @@ int main(int argc, char** argv) {
}
}
start[G] = NE;
- cudaMemcpy(dblob, blobs.data(), blobs.size(), cudaMemcpyHostToDevice);
- cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice);
- cudaMemcpy(dptr, ptr.data(), (size_t) G * 8, cudaMemcpyHostToDevice);
- cudaMemcpy(dstart, start.data(), (size_t) (G + 1) * 4, cudaMemcpyHostToDevice);
- cudaMemcpy(dn, &G, 4, cudaMemcpyHostToDevice);
- cudaMemcpy(ddst, dst.data(), (size_t) NE * 4, cudaMemcpyHostToDevice);
- cudaMemcpy(dtok, tok.data(), (size_t) NE * 4, cudaMemcpyHostToDevice);
+ s->memcpy(dblob, blobs.data(), blobs.size()).wait();
+ s->memcpy(dx, x.data(), x.size() * 4).wait();
+ s->memcpy(dptr, ptr.data(), (size_t) G * 8).wait();
+ s->memcpy(dstart, start.data(), (size_t) (G + 1) * 4).wait();
+ s->memcpy(dn, &G, 4).wait();
+ s->memcpy(ddst, dst.data(), (size_t) NE * 4).wait();
+ s->memcpy(dtok, tok.data(), (size_t) NE * 4).wait();
K::quantize_q8_1_rows((const float*) dx, NTOK, H, dxq, s);
auto run = [&](int m, int phase) {
K::native_expert_set_mode(m, phase);
K::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, G, NE, dxq, dscr, dout, s);
};
std::vector<float> ref((size_t) NE * H), got((size_t) NE * H);
- cudaMemsetAsync(dout, 0xff, got.size() * 4, s);
+ s->memset(dout, 0xff, got.size() * 4).wait();
run(mref, 0);
- cudaMemcpyAsync(ref.data(), dout, ref.size() * 4, cudaMemcpyDeviceToHost, s);
- cudaMemsetAsync(dout, 0xff, got.size() * 4, s);
+ s->memcpy(ref.data(), dout, ref.size() * 4).wait();
+ s->memset(dout, 0xff, got.size() * 4).wait();
run(mode, 0);
- cudaMemcpyAsync(got.data(), dout, got.size() * 4, cudaMemcpyDeviceToHost, s);
- if (cudaStreamSynchronize(s) != cudaSuccess) { std::printf("layer %d: CUDA error\n", l); return 1; }
+ s->memcpy(got.data(), dout, got.size() * 4).wait();
+ s->wait();
size_t ndiff = 0;
double worst = 0;
for (size_t i = 0; i < ref.size(); ++i)
@@ -139,13 +134,13 @@ int main(int argc, char** argv) {
}
auto time = [&](int m, int phase) {
for (int i = 0; i < 10; ++i) run(m, phase);
- cudaEventRecord(e0, s);
+ s->wait();
+ const auto t0 = std::chrono::steady_clock::now();
for (int i = 0; i < iters; ++i) run(m, phase);
- cudaEventRecord(e1, s);
- cudaEventSynchronize(e1);
- float ms = 0;
- cudaEventElapsedTime(&ms, e0, e1);
- return 1e3 * ms / iters;
+ s->wait();
+ const double ms =
+ std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+ return ms / iters;
};
const double gu_b = (double) G * 2 * f.up_off, d_b = (double) G * dsz;
double us[2][3] = {{1e30, 1e30, 1e30}, {1e30, 1e30, 1e30}};
@@ -163,8 +158,9 @@ int main(int argc, char** argv) {
++failures;
}
K::native_expert_set_mode(-1, 0);
- cudaFree(dblob); cudaFree(dx); cudaFree(dxq); cudaFree(dscr); cudaFree(dout); cudaFree(dptr);
- cudaFree(dstart); cudaFree(dn); cudaFree(ddst); cudaFree(dtok);
+ for (void* p : {(void*) dblob, (void*) dx, (void*) dxq, (void*) dscr, (void*) dout, (void*) dptr,
+ (void*) dstart, (void*) dn, (void*) ddst, (void*) dtok})
+ sycl::free(p, *s);
}
std::printf("native_expert_bench: %d differing\n", failures);
return failures ? 1 : 0;