37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri
The table IQ1S_GRID below is the IQ1 grid of ggml (ggml/src/ggml-common.h,
iq1s_grid), copied as data from src/llm/iq1s_grid.h - each entry's eight
levels of -1, 0 or 1 as two bits (the level plus one), an entry in 16
bits, two a word: it defines the formats. Copyright (c) 2023-2026 The
ggml authors, under the MIT licence (see the ggml repository).
Generated - not formatted by hand. */
#version 450
// IQ1 weights times up to NV Q8_K vectors, one row per invocation, as
// janas_iq1s_dot_ref and janas_iq1m_dot_ref compute them, KIND chosen when
// the pipeline is made: 0 IQ1_S (50 bytes a block: d, 32 grid indices'
// low bytes, a 16-bit word a group of 32 - their top three bits each, the
// scale, the shift's sign), 1 IQ1_M (56: the indices' low bytes, a byte a
// pair of sub-groups - their top three bits and shifts' signs - and four
// 16-bit words of scales of three bits each, whose top nibbles together
// are d). A weight is 8 times its level plus or minus 1, -9 to 9, the
// sub-group's integer dot times its scale 2 s + 1, the block's exact
// integer sum in one fma with (d / 8) d8: the CPU's float operations, so
// the results agree bit for bit. IQ1_S's blocks start on 2-byte
// boundaries: their words are realigned. Activations shared across the
// subgroup as in q4k_matvec.comp.
#extension GL_EXT_integer_dot_product : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_ballot : require
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 64) in;
layout(constant_id = 0) const uint NV = 8;
layout(constant_id = 1) const uint SG = 8;
layout(constant_id = 2) const uint KIND = 0;
const uint NX = (19u + SG - 1u) / SG;
const uint BS = KIND == 0u ? 50u : 56u, NW = (BS + 3u) / 4u + 1u;
layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };
layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;
const uint IQ1S_GRID[1024] = uint[1024](
0x00020000u, 0x00080005u, 0x0011000au, 0x00200015u, 0x00280022u, 0x0045002au,
0x00540051u, 0x00650056u, 0x00820080u, 0x008a0088u, 0x00a00095u, 0x00a800a2u,
0x010400aau, 0x01110105u, 0x01160114u, 0x011a0119u, 0x01410125u, 0x01490146u,
0x01550152u, 0x0161015au, 0x01660164u, 0x01850168u, 0x01940191u, 0x01a50196u,
0x02020200u, 0x020a0208u, 0x02200215u, 0x02280222u, 0x0245022au, 0x02590251u,
0x02690264u, 0x02820280u, 0x028a0288u, 0x02950291u, 0x02a00299u, 0x02a802a2u,
0x041102aau, 0x04160414u, 0x04410425u, 0x04550449u, 0x0464045au, 0x04910465u,
0x04a50499u, 0x05040501u, 0x05060505u, 0x05180515u, 0x0529051au, 0x05450540u,
0x0550054au, 0x05540551u, 0x05560555u, 0x05600559u, 0x05650562u, 0x056a0568u,
0x05910581u, 0x05980595u, 0x05a1059au, 0x05a505a4u, 0x05a905a6u, 0x06190614u,
0x06440641u, 0x06520650u, 0x06580655u, 0x06610660u, 0x06690666u, 0x06910685u,
0x06990694u, 0x08020800u, 0x080a0808u, 0x08200815u, 0x08280822u, 0x0845082au,
0x08560851u, 0x08800865u, 0x08880882u, 0x0895088au, 0x08a208a0u, 0x08aa08a8u,
0x09110905u, 0x09190914u, 0x09250924u, 0x09500941u, 0x09550951u, 0x09640961u,
0x09910969u, 0x09960994u, 0x09a50999u, 0x0a020a00u, 0x0a0a0a08u, 0x0a200a15u,
0x0a280a22u, 0x0a450a2au, 0x0a590a51u, 0x0a650a61u, 0x0a820a80u, 0x0a880a85u,
0x0a950a8au, 0x0aa20aa0u, 0x0aaa0aa8u, 0x10111010u, 0x10191014u, 0x10251024u,
0x10441041u, 0x10551050u, 0x10611058u, 0x10651064u, 0x10911069u, 0x10961094u,
0x10a510a1u, 0x11041101u, 0x11091106u, 0x11121110u, 0x11181115u, 0x11241121u,
0x11451129u, 0x1150114au, 0x11521151u, 0x11551154u, 0x11591156u, 0x11651160u,
0x11921184u, 0x11a11195u, 0x121111a4u, 0x12161214u, 0x12401225u, 0x12491246u,
0x12551252u, 0x125a1258u, 0x12661264u, 0x12911285u, 0x12961294u, 0x140112a5u,
0x14091406u, 0x14151414u, 0x14191418u, 0x14261421u, 0x14451441u, 0x14481446u,
0x1451144au, 0x14551454u, 0x14591456u, 0x14651462u, 0x14841468u, 0x14901489u,
0x14951494u, 0x14991498u, 0x14a1149au, 0x14a514a4u, 0x150214a9u, 0x150a1505u,
0x15141511u, 0x15161515u, 0x15201519u, 0x15251522u, 0x152a1528u, 0x15441541u,
0x15461545u, 0x15521551u, 0x15551554u, 0x15591556u, 0x1561155au, 0x15651564u,
0x15691566u, 0x15821580u, 0x15851584u, 0x158a1588u, 0x15911590u, 0x15951594u,
0x15991596u, 0x15a0159au, 0x15a515a2u, 0x16041601u, 0x16061605u, 0x16161615u,
0x161a1618u, 0x16261621u, 0x16421640u, 0x16451644u, 0x164a1648u, 0x16551651u,
0x16581656u, 0x16611659u, 0x16651664u, 0x16691668u, 0x1686166au, 0x1692168au,
0x16a41695u, 0x181116a9u, 0x18251816u, 0x18441841u, 0x18491846u, 0x18551850u,
0x185a1858u, 0x18611860u, 0x18661864u, 0x18851869u, 0x18941891u, 0x191018a5u,
0x19151912u, 0x1921191au, 0x19421925u, 0x19451944u, 0x19511948u, 0x19551954u,
0x19591956u, 0x1960195au, 0x196a1965u, 0x19911989u, 0x19951992u, 0x19a11998u,
0x19a919a6u, 0x1a161a09u, 0x1a261a24u, 0x1a461a44u, 0x1a501a49u, 0x1a551a52u,
0x1a611a58u, 0x1a691a66u, 0x1a911a85u, 0x1a9a1a96u, 0x20022000u, 0x200a2008u,
0x20202015u, 0x20252022u, 0x202a2028u, 0x20512045u, 0x20612059u, 0x20802065u,
0x20882082u, 0x2095208au, 0x20a220a0u, 0x20a820a5u, 0x210520aau, 0x21142111u,
0x21252119u, 0x21442142u, 0x21552149u, 0x215a2158u, 0x21642161u, 0x21662165u,
0x21902185u, 0x21992196u, 0x220121a5u, 0x220a2208u, 0x22152211u, 0x22222220u,
0x222a2228u, 0x22512245u, 0x22592256u, 0x22812265u, 0x228a2288u, 0x22952291u,
0x22a222a0u, 0x22aa22a8u, 0x24142405u, 0x24192416u, 0x24442425u, 0x24462445u,
0x24522449u, 0x24582455u, 0x2466245au, 0x24912485u, 0x24992494u, 0x24a524a1u,
0x25152509u, 0x25292521u, 0x25452540u, 0x25512548u, 0x25552554u, 0x25622559u,
0x25682565u, 0x25902589u, 0x25952594u, 0x259a2598u, 0x25a425a1u, 0x25a925a6u,
0x26102605u, 0x26192612u, 0x26412625u, 0x26552649u, 0x26612660u, 0x26842669u,
0x26902686u, 0x2800269au, 0x28082802u, 0x2815280au, 0x28222820u, 0x282a2828u,
0x28512845u, 0x28652854u, 0x28822880u, 0x288a2888u, 0x28a228a0u, 0x28aa28a8u,
0x29112909u, 0x29192914u, 0x29462925u, 0x29522949u, 0x29612955u, 0x29662964u,
0x29852969u, 0x29962990u, 0x29a42999u, 0x2a0029a5u, 0x2a082a02u, 0x2a202a0au,
0x2a282a22u, 0x2a452a2au, 0x2a562a51u, 0x2a652a59u, 0x2a822a80u, 0x2a8a2a88u,
0x2aa02a95u, 0x2aa82aa2u, 0x40052aaau, 0x40164011u, 0x40494025u, 0x40554052u,
0x405a4058u, 0x40644061u, 0x40944066u, 0x40a14099u, 0x410040a6u, 0x41044101u,
0x41094106u, 0x41154112u, 0x41184116u, 0x4121411au, 0x41294126u, 0x41484145u,
0x4151414au, 0x41554154u, 0x41594156u, 0x4165415au, 0x416a4168u, 0x41844181u,
0x41904186u, 0x41954192u, 0x41a141a0u, 0x420541a2u, 0x42144211u, 0x42254216u,
0x42524241u, 0x425a4255u, 0x42694264u, 0x42944289u, 0x440142a5u, 0x44194415u,
0x44454429u, 0x444a4448u, 0x44544451u, 0x44564455u, 0x44624461u, 0x44684465u,
0x4481446au, 0x44894486u, 0x44924490u, 0x44a04495u, 0x44a944a1u, 0x45024501u,
0x450a4505u, 0x45144511u, 0x45164515u, 0x45204519u, 0x452a4525u, 0x45444541u,
0x45464545u, 0x45504549u, 0x45544551u, 0x45564555u, 0x45594558u, 0x45644561u,
0x45664565u, 0x45824569u, 0x45854584u, 0x45914588u, 0x45954594u, 0x45994596u,
0x45a5459au, 0x45aa45a8u, 0x46054601u, 0x46144609u, 0x46184615u, 0x4621461au,
0x46294624u, 0x46424640u, 0x46484645u, 0x46514650u, 0x46554652u, 0x46594656u,
0x46654662u, 0x46814668u, 0x468a4685u, 0x46954694u, 0x46a446a1u, 0x480546a6u,
0x48154811u, 0x4825481au, 0x48494842u, 0x48554850u, 0x48614858u, 0x48664864u,
0x48854869u, 0x48944891u, 0x48994896u, 0x490148a5u, 0x49064905u, 0x4910490au,
0x49154914u, 0x49214918u, 0x49264924u, 0x49454940u, 0x4951494au, 0x49544952u,
0x49564955u, 0x49604959u, 0x49654962u, 0x496a4966u, 0x49894986u, 0x49954992u,
0x49984996u, 0x49a449a1u, 0x49a949a6u, 0x4a444a16u, 0x4a494a46u, 0x4a584a55u,
0x4a644a5au, 0x4a944a69u, 0x50014aa5u, 0x50055004u, 0x50095006u, 0x50155012u,
0x5021501au, 0x50295024u, 0x50455040u, 0x50515048u, 0x50555054u, 0x50595056u,
0x50685065u, 0x50895086u, 0x50985095u, 0x50a150a0u, 0x50a950a6u, 0x51085105u,
0x510a5109u, 0x51145111u, 0x51165115u, 0x51195118u, 0x51255120u, 0x51285126u,
0x5141512au, 0x51455144u, 0x51495146u, 0x51515150u, 0x51545152u, 0x51565155u,
0x51595158u, 0x5161515au, 0x51655164u, 0x51695166u, 0x51855182u, 0x51945191u,
0x51965195u, 0x51a05199u, 0x51aa51a5u, 0x52065201u, 0x52155212u, 0x5221521au,
0x52425224u, 0x524a5245u, 0x52545251u, 0x52565255u, 0x52625259u, 0x52855265u,
0x52925290u, 0x52995295u, 0x52a4529au, 0x54055404u, 0x54145411u, 0x54165415u,
0x54195418u, 0x54255421u, 0x542a5428u, 0x54445441u, 0x54465445u, 0x544a5449u,
0x54515450u, 0x54555454u, 0x54585456u, 0x545a5459u, 0x54625461u, 0x54655464u,
0x54695466u, 0x54885480u, 0x5491548au, 0x54955494u, 0x54995496u, 0x54a454a1u,
0x54aa54a5u, 0x55025501u, 0x55055504u, 0x55095506u, 0x55115510u, 0x55145512u,
0x55165515u, 0x551a5519u, 0x55245521u, 0x55265525u, 0x55405529u, 0x55425541u,
0x55455544u, 0x55485546u, 0x55505549u, 0x55525551u, 0x55555554u, 0x55585556u,
0x555a5559u, 0x55615560u, 0x55655564u, 0x55685566u, 0x556a5569u, 0x55845581u,
0x55895585u, 0x5590558au, 0x55945591u, 0x55965595u, 0x55995598u, 0x55a455a1u,
0x55a655a5u, 0x560055a9u, 0x56025601u, 0x56065604u, 0x56095608u, 0x56145611u,
0x56185615u, 0x56205619u, 0x56225621u, 0x56255624u, 0x56285626u, 0x56415629u,
0x56465645u, 0x56495648u, 0x5650564au, 0x56525651u, 0x56555654u, 0x56585656u,
0x565a5659u, 0x56645661u, 0x56695665u, 0x56855682u, 0x56885686u, 0x568a5689u,
0x56955691u, 0x56a2569au, 0x56a656a5u, 0x56a956a8u, 0x58055804u, 0x58095806u,
0x58155810u, 0x58215818u, 0x5845582au, 0x584a5848u, 0x58545851u, 0x58565855u,
0x58595858u, 0x58625860u, 0x58655864u, 0x58895882u, 0x58925890u, 0x58985895u,
0x58a958a1u, 0x59025901u, 0x590a5905u, 0x59145911u, 0x59165915u, 0x59255919u,
0x59445941u, 0x59465945u, 0x59505949u, 0x59525951u, 0x59555954u, 0x59585956u,
0x595a5959u, 0x59645961u, 0x59665965u, 0x59815969u, 0x59895985u, 0x59945991u,
0x59965995u, 0x59995998u, 0x5a0459a5u, 0x5a155a08u, 0x5a205a1au, 0x5a265a25u,
0x5a455a29u, 0x5a495a48u, 0x5a555a51u, 0x5a585a56u, 0x5a625a59u, 0x5a685a65u,
0x5a815a6au, 0x5a925a8au, 0x5a965a95u, 0x5a9a5a98u, 0x60055aa1u, 0x60166014u,
0x60256019u, 0x60506044u, 0x60566055u, 0x605a6058u, 0x60646061u, 0x60696066u,
0x60966081u, 0x610160a5u, 0x61066104u, 0x61126109u, 0x61216115u, 0x61266122u,
0x61456129u, 0x61516149u, 0x61566155u, 0x61656159u, 0x616a6166u, 0x618a6184u,
0x61956192u, 0x61a661a1u, 0x621161a9u, 0x62196216u, 0x62416240u, 0x62556246u,
0x62586256u, 0x62856260u, 0x62966291u, 0x641162a5u, 0x64156412u, 0x641a6416u,
0x64266421u, 0x64406429u, 0x64456442u, 0x644a6448u, 0x64546451u, 0x64566455u,
0x645a6459u, 0x64626460u, 0x64846465u, 0x64896485u, 0x64926490u, 0x64956494u,
0x64986496u, 0x64a1649au, 0x64a964a4u, 0x65086505u, 0x6511650au, 0x65166515u,
0x65446519u, 0x65466545u, 0x65506549u, 0x65546551u, 0x65566555u, 0x65616559u,
0x65656564u, 0x65696566u, 0x65896586u, 0x6591658au, 0x65966595u, 0x659a6599u,
0x65a565a2u, 0x65a865a6u, 0x66096602u, 0x66206615u, 0x66286626u, 0x66406629u,
0x66486645u, 0x6651664au, 0x66556654u, 0x66586656u, 0x6660665au, 0x66686665u,
0x66826680u, 0x668a6685u, 0x66966694u, 0x66996698u, 0x66a466a0u, 0x66aa66a6u,
0x68196816u, 0x68416825u, 0x68556852u, 0x6861685au, 0x68856869u, 0x68986891u,
0x690168a6u, 0x69106904u, 0x69216915u, 0x69266924u, 0x69406929u, 0x69456941u,
0x69486946u, 0x69546951u, 0x69566955u, 0x69606959u, 0x696a6965u, 0x69846982u,
0x6995698au, 0x69a469a1u, 0x69a969a5u, 0x6a166a11u, 0x6a416a18u, 0x6a496a44u,
0x6a556a50u, 0x6a5a6a58u, 0x6a656a64u, 0x6a866a69u, 0x6a986a94u, 0x6aa66a9au,
0x80028000u, 0x800a8008u, 0x80228020u, 0x802a8028u, 0x80508045u, 0x80548051u,
0x80598056u, 0x80808065u, 0x80888082u, 0x8095808au, 0x80a280a0u, 0x80aa80a8u,
0x81118105u, 0x81168114u, 0x81258119u, 0x81448141u, 0x81508149u, 0x81558152u,
0x81588156u, 0x81648159u, 0x81698166u, 0x81898185u, 0x81968194u, 0x81a58199u,
0x82028200u, 0x820a8208u, 0x82208215u, 0x82288222u, 0x8251822au, 0x82598254u,
0x82808265u, 0x82888282u, 0x8295828au, 0x82a282a0u, 0x82aa82a8u, 0x84198414u,
0x84448441u, 0x84558451u, 0x8461845au, 0x84698464u, 0x84998494u, 0x85098501u,
0x85158512u, 0x8526851au, 0x85408529u, 0x85458541u, 0x85518548u, 0x85558554u,
0x85598556u, 0x8565855au, 0x85688566u, 0x8581856au, 0x85868584u, 0x85908589u,
0x85958592u, 0x85a68598u, 0x86168611u, 0x86258619u, 0x86448641u, 0x864a8649u,
0x86558650u, 0x865a8659u, 0x86668661u, 0x8685866au, 0x869a8691u, 0x880086a4u,
0x88088802u, 0x8815880au, 0x88228820u, 0x882a8828u, 0x88458841u, 0x88548851u,
0x88658859u, 0x88808869u, 0x88888882u, 0x8895888au, 0x88a288a0u, 0x88aa88a8u,
0x89068905u, 0x89148911u, 0x89258916u, 0x89448941u, 0x89498946u, 0x89528950u,
0x895a8955u, 0x89648961u, 0x89968985u, 0x89a58999u, 0x8a028a00u, 0x8a0a8a08u,
0x8a208a15u, 0x8a288a22u, 0x8a458a2au, 0x8a548a51u, 0x8a808a56u, 0x8a888a82u,
0x8a958a8au, 0x8aa28aa0u, 0x8aaa8aa8u, 0x90119005u, 0x90189016u, 0x90259019u,
0x90469041u, 0x90559049u, 0x905a9058u, 0x906a9069u, 0x90919085u, 0x90969094u,
0x90a59099u, 0x91049101u, 0x91099106u, 0x91159110u, 0x911a9118u, 0x91249121u,
0x91299126u, 0x91459140u, 0x91519150u, 0x91559154u, 0x91599156u, 0x91659162u,
0x91869184u, 0x91959192u, 0x91a19198u, 0x91a691a4u, 0x920591a9u, 0x92149211u,
0x92259219u, 0x92469244u, 0x92509249u, 0x92559252u, 0x92669258u, 0x92859269u,
0x92969294u, 0x940192a9u, 0x94069404u, 0x94159410u, 0x94269418u, 0x944a9440u,
0x94549451u, 0x94569455u, 0x94599458u, 0x94619460u, 0x94659462u, 0x94869484u,
0x94949492u, 0x94989495u, 0x94a994a1u, 0x95059500u, 0x950a9508u, 0x95119510u,
0x95159514u, 0x95199516u, 0x95259521u, 0x952a9529u, 0x95449541u, 0x95469545u,
0x95509549u, 0x95529551u, 0x95559554u, 0x95589556u, 0x955a9559u, 0x95649561u,
0x95669565u, 0x95819569u, 0x95889585u, 0x95929591u, 0x95959594u, 0x95999596u,
0x95a0959au, 0x95a595a2u, 0x95aa95a8u, 0x96049601u, 0x96159610u, 0x96209619u,
0x96299626u, 0x96489645u, 0x96519649u, 0x96559652u, 0x96599656u, 0x96689665u,
0x96849682u, 0x968a9689u, 0x96949692u, 0x96a49695u, 0x96a996a6u, 0x98169805u,
0x98259819u, 0x98469841u, 0x98529850u, 0x98569855u, 0x9864985au, 0x98859865u,
0x98969891u, 0x98a59899u, 0x99069904u, 0x99109909u, 0x99159912u, 0x991a9918u,
0x99219920u, 0x99269924u, 0x99429940u, 0x99489945u, 0x9951994au, 0x99559954u,
0x99599956u, 0x99659962u, 0x996a9966u, 0x99849981u, 0x99929990u, 0x999a9995u,
0x99a699a1u, 0x9a159a05u, 0x9a449a25u, 0x9a499a46u, 0x9a559a50u, 0x9a619a58u,
0x9a919a85u, 0x9a959a94u, 0xa0009a96u, 0xa008a002u, 0xa015a00au, 0xa022a020u,
0xa02aa028u, 0xa051a045u, 0xa056a054u, 0xa080a059u, 0xa088a082u, 0xa095a08au,
0xa0a2a0a0u, 0xa0aaa0a8u, 0xa109a105u, 0xa114a111u, 0xa119a116u, 0xa146a11au,
0xa151a149u, 0xa158a155u, 0xa161a15au, 0xa185a164u, 0xa192a190u, 0xa199a196u,
0xa208a202u, 0xa210a20au, 0xa222a219u, 0xa22aa228u, 0xa251a245u, 0xa259a256u,
0xa280a265u, 0xa288a282u, 0xa295a28au, 0xa2a2a2a0u, 0xa2aaa2a8u, 0xa425a419u,
0xa444a441u, 0xa454a450u, 0xa458a455u, 0xa461a45au, 0xa466a465u, 0xa469a468u,
0xa506a485u, 0xa510a509u, 0xa515a512u, 0xa526a518u, 0xa542a529u, 0xa551a545u,
0xa555a554u, 0xa559a556u, 0xa56aa565u, 0xa584a581u, 0xa586a585u, 0xa592a589u,
0xa598a595u, 0xa611a605u, 0xa61aa616u, 0xa625a621u, 0xa646a644u, 0xa652a64au,
0xa656a655u, 0xa660a658u, 0xa686a662u, 0xa695a690u, 0xa699a696u, 0xa6a4a6a1u,
0xa800a6a6u, 0xa808a802u, 0xa820a80au, 0xa828a822u, 0xa851a82au, 0xa856a854u,
0xa880a859u, 0xa888a882u, 0xa895a88au, 0xa8a2a8a0u, 0xa8aaa8a8u, 0xa914a905u,
0xa921a919u, 0xa941a925u, 0xa955a950u, 0xa961a95au, 0xa969a966u, 0xa996a990u,
0xaa02aa00u, 0xaa0aaa08u, 0xaa22aa20u, 0xaa2aaa28u, 0xaa54aa51u, 0xaa80aa56u,
0xaa88aa82u, 0xaa95aa8au, 0xaaa2aaa0u, 0xaaaaaaa8u);
int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }
uint byte_at(uint wd[NW], uint i) { return (wd[i >> 2] >> (8u * (i & 3u))) & 0xffu; }
uint half_at(uint wd[NW], uint i) { return byte_at(wd, i) | (byte_at(wd, i + 1u) << 8); }
/* the grid in the workgroup's shared memory (iq4xs_matvec.comp) */
shared uint GS[1024];
/* weights 4h..4h+3 of a sub-group, 8 level + delta, as signed bytes */
uint weights4(uint idx, uint h, int delta)
{
uint e = (GS[idx >> 1] >> (16u * (idx & 1u))) & 0xffffu, r = 0u;
for (uint i = 0; i < 4u; i++) {
int lv = int((e >> (2u * (4u * h + i))) & 3u) - 1;
r |= (uint(8 * lv + delta) & 0xffu) << (8u * i);
}
return r;
}
void main()
{
for (uint i = gl_LocalInvocationID.x; i < 1024u; i += 64u)
GS[i] = IQ1S_GRID[i];
barrier();
U1 W = U1(p.w);
U4 X = U4(p.x);
F Y = F(p.y);
uint lane = gl_SubgroupInvocationID;
uint grow = gl_WorkGroupID.y * 64u + gl_LocalInvocationID.x;
uint row = min(grow, p.rows - 1u);
uint v0 = gl_WorkGroupID.x * NV;
uint xo[NV];
for (uint v = 0; v < NV; v++)
xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;
float acc[NV];
for (uint v = 0; v < NV; v++)
acc[v] = 0.0;
uint off = row * p.nb * BS;
for (uint b = 0; b < p.nb; b++, off += BS) {
uint base = off >> 2;
uint wd[NW];
for (uint k = 0; k < NW; k++)
wd[k] = W.v[base + k];
if ((off & 3u) != 0u) {
for (uint k = 0; k + 1u < NW; k++)
wd[k] = (wd[k] >> 16) | (wd[k + 1u] << 16);
wd[NW - 1u] = wd[NW - 1u] >> 16;
}
float d;
if (KIND == 0u) {
d = unpackHalf2x16(wd[0] & 0xffffu).x;
} else {
uint s0 = half_at(wd, 48u), s1 = half_at(wd, 50u), s2 = half_at(wd, 52u),
s3 = half_at(wd, 54u);
d = unpackHalf2x16((s0 >> 12) | ((s1 >> 8) & 0x00f0u) | ((s2 >> 4) & 0x0f00u) |
(s3 & 0xf000u)).x;
}
/* each sub-group of 8 two signed words, and its scale */
uint q[64];
int sc[32];
for (uint g = 0; g < 8u; g++)
for (uint k = 0; k < 4u; k++) {
uint l = 4u * g + k, idx;
int neg;
if (KIND == 0u) {
uint qh = half_at(wd, 34u + 2u * g);
idx = byte_at(wd, 2u + l) | (((qh >> (3u * k)) & 7u) << 8);
neg = int((qh >> 15) & 1u);
sc[l] = 2 * int((qh >> 12) & 7u) + 1;
} else {
uint qh = byte_at(wd, 32u + 2u * g + k / 2u);
idx = byte_at(wd, l) | ((qh << ((k & 1u) != 0u ? 4u : 8u)) & 0x700u);
neg = int((qh >> ((k & 1u) != 0u ? 7u : 3u)) & 1u);
uint s = half_at(wd, 48u + 2u * (g / 2u));
sc[l] = 2 * int((s >> (6u * (g & 1u) + 3u * (k / 2u))) & 7u) + 1;
}
int delta = neg != 0 ? -1 : 1;
q[2u * l] = weights4(idx, 0u, delta);
q[2u * l + 1u] = weights4(idx, 1u, delta);
}
for (uint v = 0; v < NV; v++) {
uvec4 xr[NX];
for (uint j = 0; j < NX; j++)
xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
int isum = 0;
for (uint g = 0; g < 8u; g++) {
uvec4 a0 = bc(xr, 2u * g), a1 = bc(xr, 2u * g + 1u);
uint l = 4u * g;
isum += sc[l] * (dp(q[2u * l], a0.x) + dp(q[2u * l + 1u], a0.y));
isum += sc[l + 1u] * (dp(q[2u * l + 2u], a0.z) + dp(q[2u * l + 3u], a0.w));
isum += sc[l + 2u] * (dp(q[2u * l + 4u], a1.x) + dp(q[2u * l + 5u], a1.y));
isum += sc[l + 3u] * (dp(q[2u * l + 6u], a1.z) + dp(q[2u * l + 7u], a1.w));
}
float d8 = uintBitsToFloat(bc(xr, 18u).x);
acc[v] = fma(d * 0.125 * d8, float(isum), acc[v]);
xo[v] += 20u;
}
}
if (grow < p.rows)
for (uint v = 0; v < NV; v++)
if (v0 + v < p.nv)
Y.v[(v0 + v) * p.ys + row] = acc[v];
}