Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
sycl prefill: DMA an expert straight from the arena only when it is USM host memory (A750 prompt deadlock)
Niko1221fca3c5958666
diff --git a/sycl/src/prefill/prefill.cpp b/sycl/src/prefill/prefill.cppindex 9b7c125b5..a42215e22 100644--- a/sycl/src/prefill/prefill.cpp+++ b/sycl/src/prefill/prefill.cpp@@ -51,6 +51,7 @@#include <memory>#include <mutex>#include <thread>+#include <unordered_map>#include <vector>#ifndef STRATA_PREFILL_MMQ@@ -97,6 +98,32 @@ namespace strata::prefill {namespace {using Clock = std::chrono::steady_clock;++// 0.1.48, A750 deadlock: SYCL has no cudaHostRegister, so the expert arena is ordinary pageable memory even where the engine+// calls it "pinned" (pinned.dp.cpp registers nothing). A DMA straight from it, `copy->memcpy(stage, arena_ptr, 2.3 MB)`, makes the+// Level Zero driver copy through its own staging buffers on the calling thread, and on some prompts that wait never ends (the+// thread spins in sched_yield inside libze_intel_gpu with the GPU idle; tools: /data/148-intel notes). Only a USM host+// allocation (the --stream-experts pinned mirror, the B70's) is DMA'd directly; every other expert is copied into one of the+// stager's pinned buffers by its threads first, the path unpinned experts already take. STRATA_PF_DMA_PAGEABLE=1 restores the+// direct copy from pageable memory.+inline bool dma_direct(const core::ExpertSource* src, int64_t layer, int64_t expert, const void* blob) {+ if (!src->pinned(layer, expert)) return false;+ static const bool pageable_ok = [] { const char* v = std::getenv("STRATA_PF_DMA_PAGEABLE"); return v && v[0] == '1'; }();+ if (pageable_ok) return true;+ if (blob == nullptr) return false;+ // the answer per 2 MiB bucket (a query costs ~2 us, 10k+ experts a chunk): "host USM" only when both ends of the bucket are, so a bucket+ // that straddles two allocations is never taken for USM+ static thread_local std::unordered_map<uintptr_t, bool> usm_bucket;+ const uintptr_t key = (uintptr_t) blob >> 21;+ const auto it = usm_bucket.find(key);+ if (it != usm_bucket.end()) return it->second;+ const auto ctx = dpct::get_in_order_queue().get_context();+ const bool usm = sycl::get_pointer_type((const void*) (key << 21), ctx) == sycl::usm::alloc::host &&+ sycl::get_pointer_type((const void*) (((key + 1) << 21) - 1), ctx) == sycl::usm::alloc::host &&+ sycl::get_pointer_type(blob, ctx) == sycl::usm::alloc::host;+ usm_bucket.emplace(key, usm);+ return usm;+}constexpr float EPS = 1e-6f;constexpr int64_t N = 2560, HC = 4, D = N * HC, LR = 320, K = 10, NE = 512;constexpr int64_t C = 10240, ZV = 6144, HV = 48;@@ -2550,7 +2577,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,continue;}}- if (!m.src->pinned(l, e)) {+ if (!dma_direct(m.src, l, e, b)) {job = (int) js.size();js.push_back({b, (size_t) lay0.blob_bytes(l)});}@@ -3809,7 +3836,7 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,for (size_t j = 0; j < order.size(); ++j) {const int32_t e = order[j];if (m.host_res && m.cache && m.host_res[(size_t) l * m.g->n_expert + e] >= 0) continue;- if (m.src->pinned(l, e)) continue;+ { const uint8_t* pb = m.src->pinned(l, e) ? m.src->blob(l, e) : nullptr; if (dma_direct(m.src, l, e, pb)) continue; }job_of[j] = (int) js.size();if (gsrc != nullptr) { // SYCL port: --stream-experts - the stager's thread reads it from the GGUFjs.push_back({nullptr, (size_t) lay.blob_bytes(l), nullptr, (int32_t) l, e, gsrc});@@ -3834,7 +3861,8 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,const int sl = stage_next;stage_next = (stage_next + 1) % stage_slots();const auto th = Clock::now();- const bool pinned = m.src->pinned(l, e); // pinned: never transient+ const uint8_t* pb0 = m.src->pinned(l, e) ? m.src->blob(l, e) : nullptr;+ const bool pinned = dma_direct(m.src, l, e, pb0); // pinned: never transientconst uint8_t* b = pinned ? m.src->blob(l, e) : nullptr;if (pinned && !b) { err = "prefill: expert source has no blob"; return false; }if (pinned) {diff --git a/sycl/tools/data/prompt_deadlock_2744.tok b/sycl/tools/data/prompt_deadlock_2744.toknew file mode 100644index 000000000..843e1ee31--- /dev/null+++ b/sycl/tools/data/prompt_deadlock_2744.tok@@ -0,0 +1 @@+248045,846,198,8917,5437,537,279,2614,9417,47801,303,4097,16916,3387,13,271,16,21,42,11,220,19,17,13,16,19,1411,220,17,21,13,18,15,9537,506,220,18,17,42,553,357,4220,9640,25,279,1788,7620,440,1132,411,3289,5324,11,264,7478,4560,791,198,2835,11,220,16,16,13139,303,69905,1906,1238,264,41784,799,11,279,4560,579,1510,39242,20759,7561,1396,9640,1301,303,2400,318,2083,66420,198,29567,553,8618,4560,15339,264,41784,220,19,42,9640,1118,11,1521,279,1118,9640,1238,264,1151,33094,777,318,263,1782,1118,198,23693,12389,279,1788,13139,6348,220,24,20,4,26126,314,498,16,13,15,15,19,11,220,16,13,15,18,17,60,321,498,15,13,24,23,16,11,220,16,13,15,16,20,14545,279,49219,1732,1754,975,2957,198,63,14,15318,5285,76,9586,63,6,82,2081,1395,4871,321,2050,5331,7611,30494,599,14217,1990,364,220,18,274,11,321,264,6505,303,864,2892,9640,1990,198,3923,2081,11567,466,2050,55557,557,11878,321,16384,1495,318,18,314,279,220,21,24,13139,11,660,220,18,17,42,440,279,3289,383,26,8341,303,11,220,18,17,42,198,70,1821,220,16,13,15,17,15,321,220,16,13,15,16,20,553,328,9333,220,20,1,369,279,11292,7101,314,1510,759,27270,42247,46,20668,63,3644,220,20,318,4058,1510,6457,4337,76465,49461,11481,1102,28,17,15,19,23,63,1174,198,1,25902,1,369,279,4560,1973,1070,318,2720,220,19,579,38171,16,16,20,16,16184,4565,553,8261,513,1711,5212,26,279,11013,369,279,50717,2977,314,198,1719,220,16,16,974,593,383,39875,440,264,220,24,20,4,26062,9581,26,328,69,2225,1,14159,279,13139,303,864,279,3289,557,10281,25,271,91,10885,735,6560,735,974,1411,383,735,974,593,383,735,10281,735,198,91,4277,25,91,4277,91,4277,25,91,4277,25,91,4277,25,91,198,91,220,19,11,15,24,21,735,3644,220,20,735,220,18,13,22,15,18,1411,220,18,13,21,19,16,274,735,220,16,13,15,16,23,498,16,13,15,16,21,11,220,16,13,15,17,15,60,735,220,16,16,314,220,16,16,735,198,91,220,19,11,15,24,21,735,16184,735,220,20,13,24,16,19,1411,220,20,13,23,17,24,274,735,220,16,13,15,16,19,498,16,13,15,16,15,11,220,16,13,15,16,22,60,735,220,16,15,314,220,16,16,735,198,91,220,18,17,11,22,21,23,735,3644,220,20,735,220,17,21,13,16,20,22,1411,220,17,20,13,20,23,24,274,735,220,16,13,15,17,16,498,16,13,15,17,15,11,220,16,13,15,17,18,60,735,220,16,16,314,220,16,16,735,198,91,220,18,17,11,22,21,23,735,16184,735,220,19,19,13,15,23,17,1411,220,19,18,13,19,22,24,274,735,220,16,13,15,16,20,498,16,13,15,16,17,11,220,16,13,15,16,22,60,735,220,16,16,314,220,16,16,735,198,91,220,16,17,23,11,15,15,15,735,3644,220,20,735,220,16,15,22,13,16,19,24,1411,220,16,15,19,13,23,21,21,274,735,220,16,13,15,17,16,498,16,13,15,17,15,11,220,16,13,15,17,18,60,735,220,16,16,314,220,16,16,735,198,91,220,16,17,23,11,15,15,15,735,16184,735,220,16,24,18,13,19,18,24,1411,220,16,24,16,13,15,20,20,274,735,220,16,13,15,16,19,498,16,13,15,16,17,11,220,16,13,15,16,21,60,735,220,16,16,314,220,16,16,735,271,4272,883,220,17,4,383,1844,314,3644,220,20,321,883,220,16,13,20,4,383,279,16184,13,2332,220,18,21,81787,9141,5777,6738,13224,279,6000,12041,6805,883,198,15,13,20,21,274,364,264,220,18,17,42,9640,26,220,15,13,20,22,274,318,2720,220,20,8,321,220,15,13,21,15,274,318,25902,8,998,16384,13,271,5170,33686,65427,2272,279,1788,1542,1973,279,3289,11,220,16,17,49219,314,220,16,13,20,42,12,17,19,42,10885,25,75862,20819,48,19,6645,50,220,15,13,15,15,21,22,2977,318,15,13,15,19,19,1866,8,383,279,198,25902,321,220,15,13,15,15,18,24,318,15,13,15,17,22,8,383,1844,314,3644,220,20,26,75862,27325,19,9802,6645,43,220,15,13,15,15,21,18,318,15,13,15,19,18,8,321,220,15,13,15,15,19,15,318,15,13,15,16,19,1174,279,1788,1844,3817,303,660,314,198,47092,318,565,2060,279,220,16,17,49219,279,1118,3817,4829,1132,1330,12103,2663,553,10790,220,20,579,30147,7249,513,220,15,13,15,15,21,23,318,59865,19,6645,50,8,321,198,15,13,15,15,18,16,318,48,19,9802,6645,43,8,2272,279,16184,11,264,2086,2222,421,369,9711,364,5281,321,524,430,264,3798,364,411,799,13,8020,279,198,3765,7658,3817,557,7463,25,264,1248,9191,321,70665,477,998,524,16384,13,271,10743,321,1542,1439,11988,303,279,77528,1870,6025,25,271,71093,898,198,6031,706,1137,5653,1870,12,2082,1137,5480,17603,1824,16884,29542,290,17603,1824,16884,6860,12756,198,65590,1137,1877,43774,1870,12,2082,471,49,359,2082,1824,16884,8102,25148,290,91,2670,12756,20973,1137,2911,10020,2149,9094,198,1891,585,303,220,16,220,17,220,18,26,635,12619,5653,12,2082,14,2082,1824,16884,29542,290,1137,26157,13279,26,12619,5653,12,2082,14,2082,1824,16884,29542,290,1137,26157,26,2725,198,71093,271,4699,27502,1542,6473,279,21966,38372,21853,314,7840,8213,1238,1330,7948,8339,11,279,1788,69777,2763,1528,1396,1542,25,198,63,312,26157,63,411,1752,11,1510,312,26157,13279,63,38171,16,16,20,16,579,1581,61369,27718,6457,4337,2559,30237,41832,28,17,7561,1510,6457,4337,2559,30237,8827,56,28,16,57208,561,3289,369,1301,198,12802,791,1817,11,748,279,1330,513,8387,8213,13,1049,2942,799,6000,579,72632,321,7230,11,524,264,4220,9640,13,271,550,4744,321,13092,6343,271,48887,264,7021,1558,3641,321,1141,380,4100,15135,1608,279,6149,7141,13,198,2381,2441,895,9242,35494,18,62,5955,50,11,1732,498,1719,11135,23967,13996,9251,842,4887,20668,1174,198,559,18583,1503,524,2222,9966,1954,11,1558,20421,11,466,3817,11881,13,198,2143,1510,312,4431,15089,47476,63,310,279,1510,15449,15956,80,31902,6971,63,3108,25,68463,7225,279,3641,579,198,63,4431,15089,28434,7561,864,279,1581,9575,34532,3108,1503,524,16201,13,1061,198,64422,287,4945,12977,3433,13,16722,34100,48,12,7146,46,35494,18,62,5955,50,5533,50319,220,17,364,387,841,26,198,2126,895,5533,50319,220,16,13,13262,1754,1558,579,1765,3641,321,44424,3658,13,271,9947,1510,5653,12,2082,14,485,450,63,430,279,31097,303,279,3421,4566,13,8167,279,23547,198,5920,440,1510,1240,8702,55275,81520,63,14,63,65757,55275,81520,63,413,5689,13,4980,440,198,64,25989,2193,321,829,7320,11540,1331,1518,28043,7875,944,31600,13,198,18844,279,387,841,999,383,449,35160,13,3054,524,9370,264,19152,2193,2984,34577,198,29934,2400,40387,42903,13,271,33119,46120,3357,67283,628,3564,383,94450,76,1442,948,18541,21360,369,198,10028,13,561,6149,1510,312,76,2117,17830,15089,63,1752,53007,70290,279,2400,46120,198,71376,24112,26,424,1990,13402,383,9787,999,35464,51088,321,1189,41278,383,198,15696,15339,13,1049,1503,524,1236,35160,2528,13185,310,21360,13,271,23841,2736,364,279,3889,34100,48,12,7146,46,35494,18,62,5955,50,1558,318,7899,279,12351,1590,271,71093,898,198,5653,12,2082,14,485,450,1137,12882,1088,198,220,1137,4595,29318,15956,80,18,4010,82,1088,198,220,1137,28767,593,2267,31358,43765,16451,18,13,23,12,23129,12,5666,11668,62137,12,7146,46,20819,48,18,62,5955,50,12,15,15,15,15,16,8404,12,15,15,15,15,17,75021,1644,1088,198,220,1137,675,2294,70,1644,593,2267,31358,43765,16451,18,13,23,12,23129,12,5666,11668,62137,12,7146,46,20819,48,18,62,5955,50,12,15,15,15,15,17,8404,12,15,15,15,15,17,75021,1644,1088,198,220,1137,76,2117,17830,15089,1137,71376,23435,795,63250,517,23435,28434,1137,71376,35464,3129,1088,198,220,1137,29554,473,220,20,16,17,1137,9253,220,19,1137,9253,13011,2195,220,15,13,20,1137,76,765,295,765,14,3233,1088,198,220,1137,2721,61485,220,19,15,24,21,1137,41973,514,23,1137,9979,60186,220,16,20,1088,198,220,1137,85037,5478,1164,220,15,1137,3858,627,12,35790,220,15,1137,85,2319,11255,5671,1395,564,220,16,15,17,19,198,71093,271,63,312,12882,63,369,279,4560,579,5138,3817,11176,13,1690,264,6713,466,5097,15015,5165,11,198,611,1439,2736,303,279,1510,2048,63,1287,314,279,498,3895,4566,3010,9251,842,4887,20668,2,2359,25459,681,198,723,1510,44644,63,310,1510,5653,12,2082,14,485,450,7561,321,958,279,12219,3641,579,1510,82637,63,6025,13,198,31010,440,71570,1002,85,8484,22542,471,76,8418,12275,1137,8251,592,450,1137,1617,592,450,12,2082,4182,1137,395,220,23,15,23,15,27653,198,760,11525,1698,3294,557,1429,383,264,220,16,21,22728,13540,26,6420,424,364,678,13540,13,198,760,220,19,42,2193,369,264,15728,16347,5732,1406,11,524,264,1558,3798,13,561,6009,6297,198,51368,4924,9184,321,10583,220,16,14944,33,314,18588,1354,1901,2871,13,271,332,760,3575,1048,19505,264,13928,16985,28133,20,21,15,653,20,16,21,188912,2426,777,18588,1354,1845,1056,279,1581,220,22,15,15,20112,33,11,378,1257,13,198,63,1665,14746,2318,1137,85,2319,11255,5671,1395,564,220,18,15,22,17,27653,2985,279,6297,20621,279,3575,11,279,16985,579,1727,18588,1354,1144,3520,1044,33974,5409,3166,198,48213,4779,310,1785,21360,318,38,14472,681,279,496,1833,23743,1179,10029,710,16472,10928,10217,956,598,466,1510,8678,67,12,4053,67,63,10029,10187,11,466,279,198,848,58897,14107,440,328,9129,310,8712,70537,440,1412,471,16,17,3158,271,760,41632,11222,411,18583,318,3924,328,23912,440,6302,1,3294,553,11441,1542,1472,279,13540,22927,364,1381,27718,312,12640,16809,57208,198,21149,67940,799,23547,3575,11,466,3679,440,1510,312,70,17289,63,318,1719,4560,579,6000,6507,26,1436,36737,7096,19,3559,553,271,332,20711,3309,30147,364,36737,7096,17,49219,321,56378,318,15,13,16,13,19,15,11,974,539,1581,188912,1510,6457,4337,1502,6166,1203,34005,28,16,63,28133,23,17,15,11,77528,8,8213,279,198,39242,579,27044,54802,19913,39141,1472,279,514,23,21001,48,61369,4370,314,401,29456,537,478,17603,9110,1846,25,383,279,27255,220,21,24,15,15,59781,264,220,19,42,198,39242,1301,220,19,18,21,1411,220,22,20,22,9341,2600,318,15069,9037,1414,35140,4050,494,279,1581,1238,264,2342,19781,10885,25,279,2715,23,62,16,14658,198,1014,286,553,1510,6457,4337,1971,3191,21811,63736,38325,34380,28,16,63,28133,23,23,19,8,10672,279,45559,25165,579,70664,440,264,9705,4370,314,279,7778,4680,198,8251,11,264,57035,364,279,38171,16,15,18,15,14255,3753,440,279,21001,48,9640,1752,321,45559,70664,318,354,72384,383,279,18387,579,198,31000,553,5342,77528,25,279,5817,16742,61369,23873,836,3436,28133,21,24,22,681,321,279,5904,17830,517,21736,369,974,383,77528,28133,23,16,21,553,271,332,760,5904,17830,517,4129,21736,27718,6457,4337,6620,22847,63,188912,424,369,974,539,1581,383,77528,11,3525,364,279,38171,16,16,20,16,1898,11,321,198,8761,1581,25710,13,561,20934,27376,539,5171,13,220,17,87,27255,220,21,24,15,15,59781,383,13928,28133,16,17,22,17,1590,12680,424,974,6829,220,20,12,16,18,4,198,1020,16401,318,588,3575,220,19,22,13,20,593,220,19,21,13,24,593,220,19,23,13,18,1411,220,20,15,13,20,593,220,19,23,13,20,593,220,20,16,13,18,9341,2600,440,424,383,553,27255,220,21,23,15,15,383,5342,28133,23,16,21,11,198,63,26157,74533,14,17,15,17,21,12,16,15,12,15,19,86695,3667,87,12,21,23,15,15,93843,63,1590,383,557,220,16,17,12,17,17,4,27467,13,220,19,87,423,24,22,15,15,25,21736,974,557,220,16,18,12,19,20,4,10281,13,198,44780,27255,220,21,15,15,15,3719,628,1380,1510,6457,4337,6620,22847,28,16,74927,279,2468,369,18922,2892,1560,13,8439,279,1788,3575,5601,198,20385,5337,383,2086,45266,11,1510,1665,14746,2318,1137,5248,41670,63,10633,424,383,17771,318,1783,2272,383,11,264,16526,1754,8,321,13514,198,63,6457,4337,6620,22847,28,16,63,1083,279,2124,579,51515,3059,38593,1132,948,424,14387,539,777,1056,220,18,15699,264,2843,36367,421,13325,424,874,198,4670,261,10281,27245,424,1495,13,271,332,760,9665,579,3008,4129,27718,6457,4337,1203,4100,75103,22847,63,188912,77528,1503,524,958,279,380,4100,42634,1011,579,8602,4129,11,321,2394,198,15,13,16,13,19,15,13,17,424,1503,524,1791,424,28133,16,17,19,15,25,449,423,24,22,15,15,5384,9640,4478,440,424,553,38171,16,15,18,87,369,279,4531,321,11121,424,25,383,198,276,27255,220,21,23,15,15,1172,5342,27718,6457,4337,1971,3191,68925,2737,1352,16,21,28,16,7561,35494,18,62,5955,50,11,1330,8213,1754,8,264,220,22,13,24,42,33686,9640,15339,506,220,22,24,23,12,23,15,23,9341,2600,198,4058,279,4129,321,220,20,23,19,12,20,24,21,440,282,248046,198,248045,74455,198,248068,271,248069,271\ No newline at end of filediff --git a/sycl/tools/test_prompt_deadlock.sh b/sycl/tools/test_prompt_deadlock.shnew file mode 100644index 000000000..d70d452e0--- /dev/null+++ b/sycl/tools/test_prompt_deadlock.sh@@ -0,0 +1,26 @@+#!/bin/bash+# Regression test for the Arc A750 prompt deadlock (0.1.48): a 2,744-token prompt (a docs excerpt in chat framing, token ids in+# data/prompt_deadlock_2744.tok) made the prompt path hang in the Level Zero driver (the thread spun in sched_yield inside+# libze_intel_gpu while the GPU was idle) when the expert arena was DMA'd straight from pageable memory. The engine now copies+# every expert that is not USM host memory through the stager's pinned buffers.+#+# STRATA_BIN=build-jit/strata MODEL_ARGS="--pack ... --native ... --ple-gguf ... --mtp ... --kv int8 --max-context 8192 ..." \+# sycl/tools/test_prompt_deadlock.sh+#+# MODEL_ARGS are the engine arguments of the box (everything but --tokens-file/--max-new); the A750 needs its usual environment+# (render group, ONEAPI_DEVICE_SELECTOR, IGC_EnableDPEmulation=1, ...). Passes when the prefill line is printed within+# DEADLOCK_TIMEOUT seconds (default 900; the prompt takes about 60 s on an A750 after the start-up). Exit 0 pass, 1 fail, 2 usage.+set -u+HERE=$(cd "$(dirname "$0")" && pwd)+BIN=${STRATA_BIN:-}+[ -n "$BIN" ] && [ -x "$BIN" ] && [ -n "${MODEL_ARGS:-}" ] || { echo "usage: STRATA_BIN=<strata> MODEL_ARGS=\"...\" $0"; exit 2; }+OUT=$(mktemp)+timeout "${DEADLOCK_TIMEOUT:-900}" "$BIN" $MODEL_ARGS --tokens-file "$HERE/data/prompt_deadlock_2744.tok" --max-new 4 --greedy > "$OUT" 2>&1+rc=$?+if grep -q "^prefill *2743 tokens" "$OUT"; then+ echo "PASS: the 2,743-token prompt was read ($(grep -E '^prefill' "$OUT" | head -1 | tr -s ' '))"+ rm -f "$OUT"; exit 0+fi+echo "FAIL: no prefill line (engine exit code $rc); output kept in $OUT"+tail -5 "$OUT"+exit 1