Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

143 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * gpu.c - optional integrated GPU for quantized products (see gpu.h).
 *
 * Built with the GPU only when the build compiled the shaders
 * (JANAS_GPU_SPV) and the Vulkan headers are present; the Vulkan loader
 * itself is opened at run time, so the library never depends on it.
 */
#include "gpu.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#if defined(JANAS_GPU_SPV) && __has_include(<vulkan/vulkan.h>)

#include "gpu_impl.h"

#include "q4k_matvec.h"    /* janas_spv_q4k_matvec, generated by build.sh */
#include "q6k_matvec.h"    /* janas_spv_q6k_matvec */
#include "q6kp_matvec.h"   /* janas_spv_q6kp_matvec */
#include "q5k_matvec.h"    /* janas_spv_q5k_matvec */
#include "q8_0_matvec.h"   /* janas_spv_q8_0_matvec */
#include "b32_matvec.h"    /* janas_spv_b32_matvec */
#include "iq4xs_matvec.h"  /* janas_spv_iq4xs_matvec */
#include "q3k_matvec.h"    /* janas_spv_q3k_matvec */
#include "q4k_mmq.h"       /* janas_spv_q4k_mmq */
#include "q6kp_mmq.h"      /* janas_spv_q6kp_mmq */
#include "q6k_mmq.h"       /* janas_spv_q6k_mmq */
#include "q3k_mmq.h"       /* janas_spv_q3k_mmq */
#include "iq4xs_mmq.h"     /* janas_spv_iq4xs_mmq */
#include "iq3s_mmq.h"      /* janas_spv_iq3s_mmq */
#include "iq4nl_mmq.h"     /* janas_spv_iq4nl_mmq */
#include "mxfp4_mmid.h"    /* janas_spv_mxfp4_mmid */
#include "b32_mmid.h"      /* janas_spv_b32_mmid */
#include "bf16_matvec.h"   /* janas_spv_bf16_matvec */
#include "iq1_matvec.h"    /* janas_spv_iq1_matvec */
#include "iq2_matvec.h"    /* janas_spv_iq2_matvec */
#include "iq3xxs_matvec.h" /* janas_spv_iq3xxs_matvec */
#include "iqk_mmid.h"      /* janas_spv_iqk_mmid */
#include "iqk_mmq.h"       /* janas_spv_iqk_mmq */
#include "q2k_matvec.h"    /* janas_spv_q2k_matvec */
#include "q80_mmq.h"       /* janas_spv_q80_mmq */
#include "q5k_mmq.h"       /* janas_spv_q5k_mmq */
#include "dense_mm.h"      /* janas_spv_dense_mm */
#include "dense_mv.h"      /* janas_spv_dense_mv */
#include "q3k_mmid.h"      /* janas_spv_q3k_mmid */
#include "q4k_mmid.h"      /* janas_spv_q4k_mmid */
#include "q6kp_mmid.h"     /* janas_spv_q6kp_mmid */
#include "q6k_mmid.h"      /* janas_spv_q6k_mmid */
#include "iq3s_matvec.h"   /* janas_spv_iq3s_matvec */
#include "expert_act.h"    /* janas_spv_expert_act */
#include "q4k_row.h"       /* janas_spv_q4k_row */
#include "q6k_row.h"       /* janas_spv_q6k_row */
#include "q6kp_row.h"      /* janas_spv_q6kp_row */
#include "q6ka_row.h"      /* janas_spv_q6ka_row */
#include "q4k_mv.h"        /* janas_spv_q4k_mv */
#include "q4k_mvd.h"       /* janas_spv_q4k_mvd */
#include "q6k_mvd.h"       /* janas_spv_q6k_mvd */
#include "q5k_mvd.h"       /* janas_spv_q5k_mvd */
#include "iq4xs_mvd.h"     /* janas_spv_iq4xs_mvd */

static double now(void)
{
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    return (double)ts.tv_sec + 1e-9 * (double)ts.tv_nsec;
}

static int set_err(char *err, size_t len, const char *what, VkResult r)
{
    snprintf(err, len, "%s (%d)", what, (int)r);
    return -1;
}

/* A memory type among bits with the wanted flags, preferring cached. */
int janas_g_mem_type(const struct janas_gpu *g, uint32_t bits,
                     VkMemoryPropertyFlags want)
{
    int best = -1;
    for (uint32_t i = 0; i < g->mp.memoryTypeCount; i++) {
        VkMemoryPropertyFlags f = g->mp.memoryTypes[i].propertyFlags;
        if (!(bits & (1u << i)) || (f & want) != want)
            continue;
        if (best < 0 || (f & VK_MEMORY_PROPERTY_HOST_CACHED_BIT))
            best = (int)i;
        if (f & VK_MEMORY_PROPERTY_HOST_CACHED_BIT)
            break;
    }
    return best;
}

VkDeviceAddress janas_g_buffer_address(struct janas_gpu *g, VkBuffer b)
{
    VkBufferDeviceAddressInfo ai = {
        .sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = b};
    return g->vkGetBufferDeviceAddress(g->dev, &ai);
}

/* A mapped buffer in host-cached memory if there is any (on the Arc iGPU
   the coherent type is uncached: reading 1 MB costs ~0.9 ms and writing
   300 KB ~0.7 ms), flushed and invalidated by hand when not coherent. */
int janas_g_make_stage(struct janas_gpu *g, struct stage *s, size_t bytes,
                       int cached)
{
    VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
                             .size = bytes,
                             /* indirect: a mixture's dispatches whose
                                size the CPU writes (gpu_token.c) */
                             .usage =
                                 VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
                                 VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
                                 VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT |
                                 VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
                                 VK_BUFFER_USAGE_TRANSFER_DST_BIT};
    if (g->vkCreateBuffer(g->dev, &bi, NULL, &s->buf) != VK_SUCCESS)
        return -1;
    VkMemoryRequirements req;
    g->vkGetBufferMemoryRequirements(g->dev, s->buf, &req);
    int t =
        janas_g_mem_type(g, req.memoryTypeBits,
                         VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
                             (cached ? VK_MEMORY_PROPERTY_HOST_CACHED_BIT
                                     : VK_MEMORY_PROPERTY_HOST_COHERENT_BIT));
    if (t < 0)
        t = janas_g_mem_type(g, req.memoryTypeBits,
                             VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
                                 VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
    if (t < 0)
        return -1;
    s->coherent = (g->mp.memoryTypes[t].propertyFlags &
                   VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
    VkMemoryAllocateFlagsInfo fl = {
        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
        .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
    VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
                               .pNext = &fl,
                               .allocationSize = req.size,
                               .memoryTypeIndex = (uint32_t)t};
    if (g->vkAllocateMemory(g->dev, &ai, NULL, &s->mem) != VK_SUCCESS ||
        g->vkBindBufferMemory(g->dev, s->buf, s->mem, 0) != VK_SUCCESS ||
        g->vkMapMemory(g->dev, s->mem, 0, VK_WHOLE_SIZE, 0, (void **)&s->map) !=
            VK_SUCCESS)
        return -1;
    s->addr = janas_g_buffer_address(g, s->buf);
    s->bytes = bytes;
    if (getenv("JANAS_GPU_VERBOSE"))
        fprintf(stderr, "gpu stage %zu MiB: memory type %d, flags %#x\n",
                bytes >> 20, t, (unsigned)g->mp.memoryTypes[t].propertyFlags);
    return 0;
}

static int has_extension(struct janas_gpu *g, const char *name)
{
    uint32_t n = 0;
    g->vkEnumerateDeviceExtensionProperties(g->pd, NULL, &n, NULL);
    VkExtensionProperties *e = malloc(n * sizeof(*e));
    if (!e)
        return 0;
    g->vkEnumerateDeviceExtensionProperties(g->pd, NULL, &n, e);
    int found = 0;
    for (uint32_t i = 0; i < n && !found; i++)
        found = strcmp(e[i].extensionName, name) == 0;
    free(e);
    return found;
}

/* An integrated GPU with everything the shader needs, or -1. */
static int pick_device(struct janas_gpu *g, uint32_t *family, char *err,
                       size_t err_len)
{
    uint32_t n = 0;
    g->vkEnumeratePhysicalDevices(g->inst, &n, NULL);
    VkPhysicalDevice pds[16];
    if (n > 16)
        n = 16;
    g->vkEnumeratePhysicalDevices(g->inst, &n, pds);
    /* a discrete GPU first (memory and power of its own), then an
       integrated one; JANAS_GPU_DEVICE=integrated|discrete restricts */
    const char *want_dev = getenv("JANAS_GPU_DEVICE");
    /* the first device turned down and why, for the message: a tester
       with a GPU the engine does not take has to be able to tell why */
    char why[192] = "";
#define REJECT(what)                                                           \
    {                                                                          \
        if (!why[0])                                                           \
            snprintf(why, sizeof(why), "%.80s: %s", pp.deviceName, what);      \
        continue;                                                              \
    }
    for (int pass = 0; pass < 2; pass++)
        for (uint32_t i = 0; i < n; i++) {
            g->pd = pds[i];
            VkPhysicalDeviceProperties pp;
            g->vkGetPhysicalDeviceProperties(g->pd, &pp);
            VkPhysicalDeviceType type =
                pass == 0 ? VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU
                          : VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
            if (pp.deviceType != type ||
                (want_dev &&
                 strcmp(want_dev, pass == 0 ? "discrete" : "integrated") != 0))
                continue;
            if (pp.apiVersion < VK_API_VERSION_1_3)
                REJECT("its driver offers Vulkan below 1.3")
            VkPhysicalDeviceVulkan12Properties p12 = {
                .sType =
                    VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_PROPERTIES};
            VkPhysicalDeviceVulkan13Properties p13 = {
                .sType =
                    VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_PROPERTIES,
                .pNext = &p12};
            VkPhysicalDeviceVulkan11Properties p11 = {
                .sType =
                    VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_PROPERTIES,
                .pNext = &p13};
            VkPhysicalDeviceProperties2 pp2 = {
                .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
                .pNext = &p11};
            g->vkGetPhysicalDeviceProperties2(g->pd, &pp2);
            VkPhysicalDeviceVulkan13Features f13 = {
                .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES};
            VkPhysicalDeviceVulkan12Features f12 = {
                .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES,
                .pNext = &f13};
            VkPhysicalDeviceFeatures2 f2 = {
                .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
                .pNext = &f12};
            g->vkGetPhysicalDeviceFeatures2(g->pd, &f2);
            if (!f12.bufferDeviceAddress || !f13.shaderIntegerDotProduct ||
                !(p11.subgroupSupportedOperations &
                  VK_SUBGROUP_FEATURE_BALLOT_BIT))
                REJECT("no buffer addresses, integer dot product or ballot")
            /* besides ballot, 13 shaders use subgroup arithmetic, clustered
               reductions or shuffles (expert_act, the DeltaNet steps,
               attention, norms, RoPE, the subgroup-per-row products), all in
               compute shaders: a device without one of them cannot build them
               (vkcheck says the same) */
            VkSubgroupFeatureFlags sg_ops =
                (p11.subgroupSupportedStages & VK_SHADER_STAGE_COMPUTE_BIT)
                    ? p11.subgroupSupportedOperations
                    : 0;
            char sg_miss[48] = "", sg_why[96];
            if (!(sg_ops & VK_SUBGROUP_FEATURE_ARITHMETIC_BIT))
                strcat(sg_miss, ", arithmetic");
            if (!(sg_ops & VK_SUBGROUP_FEATURE_CLUSTERED_BIT))
                strcat(sg_miss, ", clustered");
            if (!(sg_ops & VK_SUBGROUP_FEATURE_SHUFFLE_BIT))
                strcat(sg_miss, ", shuffle");
            snprintf(sg_why, sizeof(sg_why),
                     "no %s subgroup operations in compute shaders",
                     sg_miss + 2);
            if (sg_miss[0])
                REJECT(sg_why)
            /* sharing host memory: needed by an integrated GPU (it reads the
               weights where they are), optional for a discrete one (it gets a
               copy in its own memory) */
            g->host_import =
                has_extension(g, VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME);
            g->discrete = type == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
            g->has_f64 = f2.features.shaderFloat64; /* gpu_fma.c */
            if (!g->discrete && !g->host_import)
                REJECT("integrated, without VK_EXT_external_memory_host")
            /* the subgroup size the shader is built for: 8 where the size can
               be chosen (Intel: fewer registers per lane), else the device's
               own if it never varies (NVIDIA: 32); JANAS_GPU_SG forces one */
            int control =
                f13.subgroupSizeControl && f13.computeFullSubgroups &&
                (p13.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT);
            const char *se = getenv("JANAS_GPU_SG");
            uint32_t want = se ? (uint32_t)atoi(se) : 8, sg;
            if (control && want >= p13.minSubgroupSize &&
                want <= p13.maxSubgroupSize)
                sg = want;
            else if (control)
                sg = p11.subgroupSize;
            else if (p13.minSubgroupSize == p13.maxSubgroupSize)
                sg = p11.subgroupSize;
            else
                REJECT("subgroup size varying and not selectable")
            if (sg < 4 || sg > ROWS_PER_GROUP || (sg & (sg - 1)) != 0)
                REJECT("subgroup size not usable")
            g->sg = sg;
            g->sg_control = control;
            g->sg_min = p13.minSubgroupSize;
            g->sg_max = p13.maxSubgroupSize;
            uint32_t nf = 0;
            g->vkGetPhysicalDeviceQueueFamilyProperties(g->pd, &nf, NULL);
            VkQueueFamilyProperties qf[16];
            if (nf > 16)
                nf = 16;
            g->vkGetPhysicalDeviceQueueFamilyProperties(g->pd, &nf, qf);
            for (uint32_t f = 0; f < nf; f++)
                if (qf[f].queueFlags & VK_QUEUE_COMPUTE_BIT) {
                    *family = f;
                    snprintf(g->name, sizeof(g->name), "%s", pp.deviceName);
                    g->vendor_id = pp.vendorID;
                    g->device_id = pp.deviceID;
                    g->driver_version = pp.driverVersion;
                    /* what a report needs to tell one device and driver
                       from another, and whether its float operations can
                       be the CPU's */
                    snprintf(
                        g->about, sizeof(g->about),
                        "%.96s, %s; driver %.64s %.64s (%#x), Vulkan %u.%u.%u; "
                        "vendor %#x device %#x; subgroups %u-%u%s, "
                        "running %u; float32 round to nearest even %s, "
                        "denormals preserved %s, signed zero/inf/nan "
                        "preserved %s",
                        pp.deviceName, g->discrete ? "discrete" : "integrated",
                        p12.driverName, p12.driverInfo,
                        (unsigned)pp.driverVersion,
                        VK_API_VERSION_MAJOR(pp.apiVersion),
                        VK_API_VERSION_MINOR(pp.apiVersion),
                        VK_API_VERSION_PATCH(pp.apiVersion),
                        (unsigned)pp.vendorID, (unsigned)pp.deviceID,
                        p13.minSubgroupSize, p13.maxSubgroupSize,
                        control ? " (chosen)" : "", sg,
                        p12.shaderRoundingModeRTEFloat32 ? "yes" : "no",
                        p12.shaderDenormPreserveFloat32 ? "yes" : "no",
                        p12.shaderSignedZeroInfNanPreserveFloat32 ? "yes"
                                                                  : "no");
                    return 0;
                }
            REJECT("no compute queue")
        }
#undef REJECT
    if (why[0])
        snprintf(err, err_len, "%s", why);
    else if (n == 0)
        snprintf(err, err_len, "Vulkan sees no GPU");
    else if (want_dev)
        snprintf(err, err_len, "no %.20s GPU, as JANAS_GPU_DEVICE asks",
                 want_dev);
    else
        snprintf(err, err_len, "Vulkan sees no integrated or discrete GPU");
    return -1;
}

static void *keep_alive(void *arg)
{
    struct janas_gpu *g = arg;
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->ka_cb};
    int pending = 0;
    while (!atomic_load(&g->ka_stop)) {
        if (!atomic_load(&g->ka_wanted)) {
            usleep(2000);
            continue;
        }
        if (pending && g->vkGetFenceStatus(g->dev, g->ka_fence) == VK_SUCCESS) {
            g->vkResetFences(g->dev, 1, &g->ka_fence);
            pending = 0;
        }
        if (!pending) {
            pthread_mutex_lock(&g->qlock);
            pending =
                g->vkQueueSubmit(g->queue, 1, &sub, g->ka_fence) == VK_SUCCESS;
            pthread_mutex_unlock(&g->qlock);
        }
        usleep(g->keep_us);
    }
    if (pending)
        while (g->vkGetFenceStatus(g->dev, g->ka_fence) == VK_NOT_READY)
            usleep(50);
    return NULL;
}

/* An empty dispatch (no blocks: nothing read; invocation 0 writes one float
   into a buffer of its own), recorded once, for the keep-alive. */
static int record_keep_alive(struct janas_gpu *g)
{
    VkCommandBufferAllocateInfo cba = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
        .commandPool = g->cp,
        .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
        .commandBufferCount = 1};
    VkFenceCreateInfo fci = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
    if (g->vkAllocateCommandBuffers(g->dev, &cba, &g->ka_cb) != VK_SUCCESS ||
        g->vkCreateFence(g->dev, &fci, NULL, &g->ka_fence) != VK_SUCCESS ||
        g->vkBeginCommandBuffer(g->ka_cb, &bb) != VK_SUCCESS)
        return -1;
    if (janas_g_make_stage(g, &g->kas, 64, 0) != 0)
        return -1;
    VkDeviceAddress a = g->kas.addr;
    struct push pc = {.rows = 1, .nb = 0, .nv = 1, .xs = 0, .ys = 0};
    pc.y[0] = pc.x[0] = pc.w[0] = (uint32_t)a;
    pc.y[1] = pc.x[1] = pc.w[1] = (uint32_t)(a >> 32);
    g->vkCmdBindPipeline(g->ka_cb, VK_PIPELINE_BIND_POINT_COMPUTE, g->pipe);
    g->vkCmdPushConstants(g->ka_cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pc), &pc);
    g->vkCmdDispatch(g->ka_cb, 1, 1, 1);
    return g->vkEndCommandBuffer(g->ka_cb) == VK_SUCCESS ? 0 : -1;
}

/* A compute pipeline for a product shader: NV vectors per workgroup, the
   chosen subgroup size (required when the device lets it be chosen). */
VkResult janas_g_make_pipeline(struct janas_gpu *g, const uint32_t *code,
                               size_t bytes, VkShaderModule *sm,
                               VkPipeline *pipe, uint32_t kind)
{
    return janas_g_make_pipeline_nv(g, code, bytes, sm, pipe, kind, g->nvg);
}

static VkResult make_pipeline(struct janas_gpu *g, const uint32_t *code,
                              size_t bytes, VkShaderModule *sm,
                              VkPipeline *pipe, uint32_t kind, uint32_t nv,
                              uint32_t sg)
{
    /* the _xfma variant where the driver splits fma() (gpu_fma.c); made[]
       keeps code, the name janas_gpu_last_shader looks up */
    size_t mb = bytes;
    const uint32_t *mc = janas_g_spv(g, code, &mb);
    VkShaderModuleCreateInfo smi = {
        .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
        .codeSize = mb,
        .pCode = mc};
    VkResult r = g->vkCreateShaderModule(g->dev, &smi, NULL, sm);
    if (r != VK_SUCCESS)
        return r;
    /* constant 2, the kind of a shader that serves several types (ignored
       by the others) */
    uint32_t spec[3] = {nv, sg, kind};
    VkSpecializationMapEntry me[3] = {
        {.constantID = 0, .offset = 0, .size = 4},
        {.constantID = 1, .offset = 4, .size = 4},
        {.constantID = 2, .offset = 8, .size = 4}};
    VkSpecializationInfo si = {
        .mapEntryCount = 3, .pMapEntries = me, .dataSize = 12, .pData = spec};
    VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rss = {
        .sType =
            VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO,
        .requiredSubgroupSize = sg};
    VkComputePipelineCreateInfo cpi = {
        .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
        .stage =
            {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
             .pNext = g->sg_control ? &rss : NULL,
             .flags =
                 g->sg_control
                     ? VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT
                     : 0,
             .stage = VK_SHADER_STAGE_COMPUTE_BIT,
             .module = *sm,
             .pName = "main",
             .pSpecializationInfo = &si},
        .layout = g->pl};
    r = g->vkCreateComputePipelines(g->dev, VK_NULL_HANDLE, 1, &cpi, NULL,
                                    pipe);
    if (r == VK_SUCCESS && g->n_made < MAX_MADE)
        g->made[g->n_made++] = (struct made){
            .p = *pipe, .code = code, .nv = nv, .sg = sg, .kind = kind};
    return r;
}

VkResult janas_g_make_pipeline_nv(struct janas_gpu *g, const uint32_t *code,
                                  size_t bytes, VkShaderModule *sm,
                                  VkPipeline *pipe, uint32_t kind, uint32_t nv)
{
    return make_pipeline(g, code, bytes, sm, pipe, kind, nv, g->sg);
}

uint32_t janas_g_make_pipeline_sg(struct janas_gpu *g, const uint32_t *code,
                                  size_t bytes, VkShaderModule *sm,
                                  VkPipeline *pipe, uint32_t kind, uint32_t sg)
{
    if (!g->sg_control || sg < g->sg_min || sg > g->sg_max ||
        (sg & (sg - 1)) != 0)
        sg = g->sg;
    return make_pipeline(g, code, bytes, sm, pipe, kind, g->nvg, sg) ==
                   VK_SUCCESS
               ? sg
               : 0;
}

/* JANAS_GPU_COPY=1: an integrated GPU taken as a card of its own (its
   value, not its presence: =0 as without) */
int janas_g_copy_asked(void)
{
    static int on = -1;
    if (on < 0) {
        const char *e = getenv("JANAS_GPU_COPY");
        on = e && atoi(e) > 0;
    }
    return on;
}

/* JANAS_GPU_TILES_EXACT=1: the fast prompt's Q4_K and Q6_K tiles summed
   as the CPU sums, a block at a time (q4k_mmq.comp, q6k_mmq.comp EXACT),
   being measured */
static int tiles_exact(void)
{
    const char *e = getenv("JANAS_GPU_TILES_EXACT");
    return e && atoi(e) > 0;
}

struct janas_gpu *janas_gpu_create(char *err, size_t err_len)
{
    struct janas_gpu *g = calloc(1, sizeof(*g));
    if (g)
        g->pass_n = UINT32_MAX;
    /* vectors a product's workgroup takes (JANAS_GPU_NV, being measured:
       with 8 the compiler spills the products' registers) */
    if (g) {
        const char *nve = getenv("JANAS_GPU_NV");
        g->nvg = nve && atoi(nve) > 0 && atoi(nve) <= 16 ? (uint32_t)atoi(nve)
                                                         : NV_PER_GROUP;
        /* the products' paths (janas_gpu_set_path): JANAS_GPU_NO_ROW
           leaves out the subgroup-per-row shaders, JANAS_GPU_ROW_NV (0 to
           8, 4 by default) the vectors they take at most, JANAS_GPU_MVD=0
           the shaders with the lanes on the vectors, JANAS_GPU_MV=1 takes
           q4k_mv (being measured) */
        const char *e;
        g->row_on = !getenv("JANAS_GPU_NO_ROW");
        g->row_nv = (e = getenv("JANAS_GPU_ROW_NV")) ? atoi(e) : 4;
        g->mvd_on = !(e = getenv("JANAS_GPU_MVD")) || atoi(e) > 0;
        g->mv_on = (e = getenv("JANAS_GPU_MV")) && atoi(e) > 0;
    }
    if (!g) {
        snprintf(err, err_len, "out of memory");
        return NULL;
    }
#ifdef _WIN32
#define VULKAN_LOADER "vulkan-1.dll"
#else
#define VULKAN_LOADER "libvulkan.so.1"
#endif
    g->lib = dlopen(VULKAN_LOADER, RTLD_NOW | RTLD_LOCAL);
    if (!g->lib) {
        snprintf(err, err_len, "no Vulkan loader (" VULKAN_LOADER ")");
        free(g);
        return NULL;
    }
    g->gipa = (PFN_vkGetInstanceProcAddr)dlsym(g->lib, "vkGetInstanceProcAddr");
    PFN_vkCreateInstance create_instance =
        g->gipa ? (PFN_vkCreateInstance)g->gipa(NULL, "vkCreateInstance")
                : NULL;
    VkResult r;
    VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
                             .pApplicationName = "janas",
                             .apiVersion = VK_API_VERSION_1_3};
    VkInstanceCreateInfo ici = {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
                                .pApplicationInfo = &app};
    if (!create_instance ||
        (r = create_instance(&ici, NULL, &g->inst)) != VK_SUCCESS) {
        snprintf(err, err_len, "no Vulkan instance");
        dlclose(g->lib);
        free(g);
        return NULL;
    }
#define LOAD_I(f)                                                              \
    if (!(g->f = (PFN_##f)g->gipa(g->inst, #f)))                               \
        goto missing;
    VK_INSTANCE_FUNCS(LOAD_I)
#undef LOAD_I
    uint32_t family = 0;
    if (pick_device(g, &family, err, err_len) != 0)
        goto fail;
    g->vkGetPhysicalDeviceMemoryProperties(g->pd, &g->mp);
    VkPhysicalDeviceProperties pp;
    g->vkGetPhysicalDeviceProperties(g->pd, &pp);
    g->atom =
        pp.limits.nonCoherentAtomSize ? pp.limits.nonCoherentAtomSize : 64;
    g->ts_period = pp.limits.timestampPeriod;

    float prio = 1.0f;
    VkDeviceQueueCreateInfo qci = {
        .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
        .queueFamilyIndex = family,
        .queueCount = 1,
        .pQueuePriorities = &prio};
    VkPhysicalDeviceVulkan13Features f13 = {
        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES,
        .shaderIntegerDotProduct = VK_TRUE,
        .subgroupSizeControl = g->sg_control ? VK_TRUE : VK_FALSE,
        .computeFullSubgroups = g->sg_control ? VK_TRUE : VK_FALSE};
    VkPhysicalDeviceVulkan12Features f12 = {
        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES,
        .pNext = &f13,
        .bufferDeviceAddress = VK_TRUE};
    const char *ext[] = {VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME};
    VkPhysicalDeviceFeatures2 f2 = {
        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
        .pNext = &f12,
        .features = {.shaderFloat64 = g->has_f64 ? VK_TRUE : VK_FALSE}};
    VkDeviceCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
                              .pNext = &f2,
                              .queueCreateInfoCount = 1,
                              .pQueueCreateInfos = &qci,
                              .enabledExtensionCount = g->host_import ? 1 : 0,
                              .ppEnabledExtensionNames = ext};
    if ((r = g->vkCreateDevice(g->pd, &dci, NULL, &g->dev)) != VK_SUCCESS) {
        set_err(err, err_len, "cannot create the Vulkan device", r);
        goto fail;
    }
#define LOAD_D(f)                                                              \
    if (!(g->f = (PFN_##f)g->vkGetDeviceProcAddr(g->dev, #f)))                 \
        goto missing;
    VK_DEVICE_FUNCS(LOAD_D)
#undef LOAD_D
    g->vkGetDeviceQueue(g->dev, family, 0, &g->queue);
    VkCommandPoolCreateInfo cpci = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
        .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
        .queueFamilyIndex = family};
    VkCommandBufferAllocateInfo cba = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
        .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
        .commandBufferCount = 1};
    VkFenceCreateInfo fci = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
    if (g->vkCreateCommandPool(g->dev, &cpci, NULL, &g->cp) != VK_SUCCESS ||
        (cba.commandPool = g->cp,
         g->vkAllocateCommandBuffers(g->dev, &cba, &g->cb) != VK_SUCCESS) ||
        g->vkCreateFence(g->dev, &fci, NULL, &g->fence) != VK_SUCCESS ||
        janas_g_make_stage(g, &g->xs, X_STAGE_BYTES,
                           !getenv("JANAS_GPU_XCOH")) != 0 ||
        janas_g_make_stage(g, &g->ys, Y_STAGE_BYTES, 1) != 0 ||
        janas_g_make_stage(g, &g->ts, 64 << 10, 1) != 0) {
        snprintf(err, err_len, "cannot create the GPU buffers");
        goto fail;
    }

    VkPushConstantRange pcr = {.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
                               .size = JANAS_G_PUSH_BYTES};
    VkPipelineLayoutCreateInfo pli = {
        .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
        .pushConstantRangeCount = 1,
        .pPushConstantRanges = &pcr};
    if ((r = g->vkCreatePipelineLayout(g->dev, &pli, NULL, &g->pl)) !=
        VK_SUCCESS) {
        set_err(err, err_len, "cannot create the shaders", r);
        goto fail;
    }
    /* before any shader is built: whether they need their _xfma variants */
    janas_g_fma_probe(g);
    if ((r = janas_g_make_pipeline(g, janas_spv_q4k_matvec,
                                   sizeof(janas_spv_q4k_matvec), &g->sm,
                                   &g->pipe, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_q6k_matvec,
                                   sizeof(janas_spv_q6k_matvec), &g->sm6,
                                   &g->pipe6, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_q6kp_matvec,
                                   sizeof(janas_spv_q6kp_matvec), &g->sm6p,
                                   &g->pipe6p, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_q5k_matvec,
                                   sizeof(janas_spv_q5k_matvec), &g->sm5,
                                   &g->pipe5, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_q8_0_matvec,
                                   sizeof(janas_spv_q8_0_matvec), &g->sm80,
                                   &g->pipe80, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_expert_act,
                                   sizeof(janas_spv_expert_act), &g->sma,
                                   &g->pipea, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_expert_act,
                                   sizeof(janas_spv_expert_act), &g->sma32,
                                   &g->pipea32, 1)) != VK_SUCCESS) {
        set_err(err, err_len, "cannot create the shaders", r);
        goto fail;
    }
    if ((r = janas_g_make_pipeline(g, janas_spv_iq4xs_matvec,
                                   sizeof(janas_spv_iq4xs_matvec), &g->smx4,
                                   &g->pipex4, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_q3k_matvec,
                                   sizeof(janas_spv_q3k_matvec), &g->sm3,
                                   &g->pipe3, 0)) != VK_SUCCESS ||
        (r = janas_g_make_pipeline(g, janas_spv_iq3s_matvec,
                                   sizeof(janas_spv_iq3s_matvec), &g->smi3,
                                   &g->pipei3, 0)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q4k_mmq, sizeof(janas_spv_q4k_mmq),
                           &g->sm_mm, &g->pipe_mm, (uint32_t)tiles_exact(),
                           NV_PER_GROUP, g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q6kp_mmq, sizeof(janas_spv_q6kp_mmq),
                           &g->sm_mm6, &g->pipe_mm6, 0, NV_PER_GROUP, g->sg)) !=
            VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q6k_mmq, sizeof(janas_spv_q6k_mmq),
                           &g->sm_mm6s, &g->pipe_mm6s, (uint32_t)tiles_exact(),
                           NV_PER_GROUP, g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q3k_mmq, sizeof(janas_spv_q3k_mmq),
                           &g->sm_mm3, &g->pipe_mm3, 0, NV_PER_GROUP, g->sg)) !=
            VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_iq4xs_mmq, sizeof(janas_spv_iq4xs_mmq),
                           &g->sm_mmx4, &g->pipe_mmx4, 0, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_iq3s_mmq, sizeof(janas_spv_iq3s_mmq),
                           &g->sm_mmi3, &g->pipe_mmi3, 0, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_iq4nl_mmq, sizeof(janas_spv_iq4nl_mmq),
                           &g->sm_mmn4, &g->pipe_mmn4, 0, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_iq4nl_mmq, sizeof(janas_spv_iq4nl_mmq),
                           &g->sm_mmmx, &g->pipe_mmmx, 1, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q80_mmq, sizeof(janas_spv_q80_mmq),
                           &g->sm_mm80, &g->pipe_mm80, 0, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q5k_mmq, sizeof(janas_spv_q5k_mmq),
                           &g->sm_mm5, &g->pipe_mm5, 0, NV_PER_GROUP, g->sg)) !=
            VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_dense_mm, sizeof(janas_spv_dense_mm),
                           &g->sm_dm[0], &g->pipe_dm[0], 0, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_dense_mm, sizeof(janas_spv_dense_mm),
                           &g->sm_dm[1], &g->pipe_dm[1], 1, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_dense_mm, sizeof(janas_spv_dense_mm),
                           &g->sm_dm[2], &g->pipe_dm[2], 2, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q4k_mmid, sizeof(janas_spv_q4k_mmid),
                           &g->sm_id4, &g->pipe_id4, 0, NV_PER_GROUP, g->sg)) !=
            VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q6kp_mmid, sizeof(janas_spv_q6kp_mmid),
                           &g->sm_id6, &g->pipe_id6, 0, NV_PER_GROUP, g->sg)) !=
            VK_SUCCESS ||
        (r = make_pipeline(g, janas_spv_q6k_mmid, sizeof(janas_spv_q6k_mmid),
                           &g->sm_id6s, &g->pipe_id6s, 0, NV_PER_GROUP,
                           g->sg)) != VK_SUCCESS ||
        (r = make_pipeline(
             g, janas_spv_mxfp4_mmid, sizeof(janas_spv_mxfp4_mmid), &g->sm_idmx,
             &g->pipe_idmx, 0, NV_PER_GROUP, g->sg)) != VK_SUCCESS) {
        set_err(err, err_len, "cannot create the shaders", r);
        goto fail;
    }
    /* a mixture's Q3_K experts (q3k_mmid.comp): without it they stay on
       the CPU, as they all did until 21 Oct 2026 */
    if (make_pipeline(g, janas_spv_q3k_mmid, sizeof(janas_spv_q3k_mmid),
                      &g->sm_id3, &g->pipe_id3, 0, NV_PER_GROUP,
                      g->sg) != VK_SUCCESS)
        g->pipe_id3 = VK_NULL_HANDLE;
    /* qwen4exp's types for the fast prompt (Q5_0 and Q2_0 tiles, iq4nl_mmq
       KIND 2 and 3) and its experts' (b32_mmid KIND 0, 2, 3: IQ4_NL, Q5_0,
       Q2_0): without them those products stay with the CPU */
    if (make_pipeline(g, janas_spv_iq4nl_mmq, sizeof(janas_spv_iq4nl_mmq),
                      &g->sm_mm50, &g->pipe_mm50, 2, NV_PER_GROUP,
                      g->sg) != VK_SUCCESS)
        g->pipe_mm50 = VK_NULL_HANDLE;
    if (make_pipeline(g, janas_spv_iq4nl_mmq, sizeof(janas_spv_iq4nl_mmq),
                      &g->sm_mm20, &g->pipe_mm20, 3, NV_PER_GROUP,
                      g->sg) != VK_SUCCESS)
        g->pipe_mm20 = VK_NULL_HANDLE;
    for (int i = 0; i < 3; i++)
        if (make_pipeline(g, janas_spv_b32_mmid, sizeof(janas_spv_b32_mmid),
                          &g->sm_idb[i], &g->pipe_idb[i],
                          i ? (uint32_t)i + 1 : 0, NV_PER_GROUP,
                          g->sg) != VK_SUCCESS)
            g->pipe_idb[i] = VK_NULL_HANDLE;
    /* the 256-block types of Qwen3.8-Flash-Next's other files and of
       unsloth's (iqk_index), bit for bit with the CPU: without one, its
       products stay with the CPU */
    {
        const struct {
            const uint32_t *code;
            size_t bytes;
            uint32_t kind;
        } iqk[N_IQK] = {
            {janas_spv_q2k_matvec, sizeof(janas_spv_q2k_matvec), 0},
            {janas_spv_iq2_matvec, sizeof(janas_spv_iq2_matvec), 0},
            {janas_spv_iq2_matvec, sizeof(janas_spv_iq2_matvec), 1},
            {janas_spv_iq2_matvec, sizeof(janas_spv_iq2_matvec), 2},
            {janas_spv_iq3xxs_matvec, sizeof(janas_spv_iq3xxs_matvec), 0},
            {janas_spv_iq1_matvec, sizeof(janas_spv_iq1_matvec), 0},
            {janas_spv_iq1_matvec, sizeof(janas_spv_iq1_matvec), 1}};
        for (int i = 0; i < N_IQK; i++) {
            if (janas_g_make_pipeline(g, iqk[i].code, iqk[i].bytes,
                                      &g->sm_iqk[i], &g->pipe_iqk[i],
                                      iqk[i].kind) != VK_SUCCESS)
                g->pipe_iqk[i] = VK_NULL_HANDLE;
            if (janas_g_make_pipeline_nv(g, iqk[i].code, iqk[i].bytes,
                                         &g->sm1_iqk[i], &g->pp1_iqk[i],
                                         iqk[i].kind, 1) != VK_SUCCESS)
                g->pp1_iqk[i] = VK_NULL_HANDLE;
        }
        /* dense_mm's products on a few vectors: without, dense_mm's */
        for (int i = 0; i < 3; i++)
            if (janas_g_make_pipeline(g, janas_spv_dense_mv,
                                      sizeof(janas_spv_dense_mv), &g->sm_dv[i],
                                      &g->pipe_dv[i], i) != VK_SUCCESS)
                g->pipe_dv[i] = VK_NULL_HANDLE;
        /* the experts' products on a few vectors (gpu_token_moe.c) */
        /* (MXFP4, Q2_0, Q5_1: b32_matvec's kinds; Gemma 4 26B's down
           matrices are Q5_1) */
        const uint32_t *mc[8] = {janas_spv_q3k_matvec, janas_spv_q4k_matvec,
                                 janas_spv_q6k_matvec, janas_spv_q6kp_matvec,
                                 janas_spv_b32_matvec, janas_spv_b32_matvec,
                                 janas_spv_b32_matvec, janas_spv_q8_0_matvec};
        size_t mb[8] = {
            sizeof(janas_spv_q3k_matvec), sizeof(janas_spv_q4k_matvec),
            sizeof(janas_spv_q6k_matvec), sizeof(janas_spv_q6kp_matvec),
            sizeof(janas_spv_b32_matvec), sizeof(janas_spv_b32_matvec),
            sizeof(janas_spv_b32_matvec), sizeof(janas_spv_q8_0_matvec)};
        /* constant 2: ID, b32_matvec's in bit 8 beside its KIND (MXFP4,
           Q2_0, Q5_1), q8_0_matvec's alone (Gemma 4 26B's down matrices
           are Q5_1, its last layer's Q8_0) */
        uint32_t mk[8] = {1, 1, 1, 1, 256 | 4, 256 | 6, 256 | 3, 256};
        for (int i = 0; i < 8; i++)
            if (janas_g_make_pipeline_nv(g, mc[i], mb[i], &g->sm_mvid[i],
                                         &g->pipe_mvid[i], mk[i],
                                         MVID_NV) != VK_SUCCESS)
                g->pipe_mvid[i] = VK_NULL_HANDLE;
        /* their tiles for the fast prompt */
        for (int i = 0; i < N_IQK; i++)
            if (make_pipeline(g, janas_spv_iqk_mmq, sizeof(janas_spv_iqk_mmq),
                              &g->sm_iqq[i], &g->pipe_iqq[i], (uint32_t)i,
                              NV_PER_GROUP, g->sg) != VK_SUCCESS)
                g->pipe_iqq[i] = VK_NULL_HANDLE;
        /* their experts' tiles, and IQ3_S's and IQ4_XS's (kinds 7, 8) */
        for (int i = 0; i < N_IQK + 2; i++)
            if (make_pipeline(g, janas_spv_iqk_mmid, sizeof(janas_spv_iqk_mmid),
                              &g->sm_iqm[i], &g->pipe_iqm[i], (uint32_t)i,
                              NV_PER_GROUP, g->sg) != VK_SUCCESS)
                g->pipe_iqm[i] = VK_NULL_HANDLE;
    }
    /* bf16 products bit for bit (janas_gpu_bf16_mv): a row's eight lanes
       in one subgroup, so subgroups of a multiple of 8 */
    if (g->sg % 8 != 0 ||
        make_pipeline(g, janas_spv_bf16_matvec, sizeof(janas_spv_bf16_matvec),
                      &g->sm_b16, &g->pipe_b16, 0, B16_NV, g->sg) != VK_SUCCESS)
        g->pipe_b16 = VK_NULL_HANDLE;
    /* each product shader made for a single vector too */
    {
        const struct {
            const uint32_t *code;
            size_t bytes;
            uint32_t kind;
        } one[N_PROD] = {
            {janas_spv_q4k_matvec, sizeof(janas_spv_q4k_matvec), 0},
            {janas_spv_q6k_matvec, sizeof(janas_spv_q6k_matvec), 0},
            {janas_spv_q6kp_matvec, sizeof(janas_spv_q6kp_matvec), 0},
            {janas_spv_q5k_matvec, sizeof(janas_spv_q5k_matvec), 0},
            {janas_spv_q8_0_matvec, sizeof(janas_spv_q8_0_matvec), 0},
            {janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec), 0},
            {janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec), 1},
            {janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec), 2},
            {janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec), 3},
            {janas_spv_iq4xs_matvec, sizeof(janas_spv_iq4xs_matvec), 0},
            {janas_spv_q3k_matvec, sizeof(janas_spv_q3k_matvec), 0},
            {janas_spv_iq3s_matvec, sizeof(janas_spv_iq3s_matvec), 0},
            {janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec), 4},
            {janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec), 5},
            {janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec), 6}};
        for (int i = 0; i < N_PROD; i++)
            if ((r = janas_g_make_pipeline_nv(g, one[i].code, one[i].bytes,
                                              &g->pm1[i], &g->pp1[i],
                                              one[i].kind, 1)) != VK_SUCCESS) {
                set_err(err, err_len, "cannot create the shaders", r);
                goto fail;
            }
        /* the subgroup-per-row shaders and q4k_mv take SG / 8 blocks a
           step: none where a subgroup has fewer than 8 lanes (their step
           would be 0, a loop without end), the products going the other
           ways */
        if (g->sg >= 8) {
            /* the subgroup-per-row shaders written so far */
            if ((r = janas_g_make_pipeline_nv(
                     g, janas_spv_q4k_row, sizeof(janas_spv_q4k_row), &g->rm[0],
                     &g->rp[0], 0, 1)) != VK_SUCCESS) {
                set_err(err, err_len, "cannot create the shaders", r);
                goto fail;
            }
            if ((r = janas_g_make_pipeline_nv(
                     g, janas_spv_q6k_row, sizeof(janas_spv_q6k_row), &g->rm[1],
                     &g->rp[1], 0, 1)) != VK_SUCCESS) {
                set_err(err, err_len, "cannot create the shaders", r);
                goto fail;
            }
            if ((r = janas_g_make_pipeline_nv(
                     g, janas_spv_q6kp_row, sizeof(janas_spv_q6kp_row),
                     &g->rm[2], &g->rp[2], 0, 1)) != VK_SUCCESS) {
                set_err(err, err_len, "cannot create the shaders", r);
                goto fail;
            }
            if ((r = janas_g_make_pipeline_nv(
                     g, janas_spv_q6ka_row, sizeof(janas_spv_q6ka_row),
                     &g->rm_q6ka, &g->rp_q6ka, 0, 1)) != VK_SUCCESS) {
                set_err(err, err_len, "cannot create the shaders", r);
                goto fail;
            }
            /* the tiled copies (constant 2: the rows of a tile) */
            if ((r = janas_g_make_pipeline_nv(g, janas_spv_q4k_row,
                                              sizeof(janas_spv_q4k_row),
                                              &g->rm_tile[0], &g->rp_tile[0],
                                              JANAS_TILE, 1)) != VK_SUCCESS ||
                (r = janas_g_make_pipeline_nv(g, janas_spv_q6ka_row,
                                              sizeof(janas_spv_q6ka_row),
                                              &g->rm_tile[1], &g->rp_tile[1],
                                              JANAS_TILE, 1)) != VK_SUCCESS) {
                set_err(err, err_len, "cannot create the shaders", r);
                goto fail;
            }
            /* and for 2, 4 and 8 vectors at once */
            {
                const uint32_t *code[4] = {janas_spv_q4k_row, janas_spv_q6k_row,
                                           janas_spv_q6kp_row,
                                           janas_spv_q6ka_row};
                size_t bytes[4] = {
                    sizeof(janas_spv_q4k_row), sizeof(janas_spv_q6k_row),
                    sizeof(janas_spv_q6kp_row), sizeof(janas_spv_q6ka_row)};
                for (int k = 0; k < 3; k++)
                    for (int i = 0; i < 4; i++)
                        if ((r = janas_g_make_pipeline_nv(
                                 g, code[i], bytes[i], &g->rmv[k][i],
                                 &g->rpv[k][i], 0, 2u << k)) != VK_SUCCESS) {
                            set_err(err, err_len, "cannot create the shaders",
                                    r);
                            goto fail;
                        }
            }
            /* Q4_K on a few vectors, a row an invocation (JANAS_GPU_MV=1, being
               measured) */
            if ((r = janas_g_make_pipeline_nv(
                     g, janas_spv_q4k_mv, sizeof(janas_spv_q4k_mv), &g->sm_mv,
                     &g->pipe_mv, 0, MV_NV)) != VK_SUCCESS) {
                set_err(err, err_len, "cannot create the shaders", r);
                goto fail;
            }
        }
        /* ... and Q4_K, Q6_K, Q5_K and IQ4_XS on 2 to 8 with the lanes on
           the vectors, where a subgroup is 8 lanes, one a vector */
        if (g->sg == 8 &&
            ((r = janas_g_make_pipeline_nv(
                  g, janas_spv_q4k_mvd, sizeof(janas_spv_q4k_mvd), &g->sm_mvd,
                  &g->pipe_mvd, 0, 8)) != VK_SUCCESS ||
             (r = janas_g_make_pipeline_nv(
                  g, janas_spv_q6k_mvd, sizeof(janas_spv_q6k_mvd), &g->sm_mvd6,
                  &g->pipe_mvd6, 0, 8)) != VK_SUCCESS ||
             (r = janas_g_make_pipeline_nv(
                  g, janas_spv_q5k_mvd, sizeof(janas_spv_q5k_mvd), &g->sm_mvd5,
                  &g->pipe_mvd5, 0, 8)) != VK_SUCCESS ||
             (r = janas_g_make_pipeline_nv(
                  g, janas_spv_iq4xs_mvd, sizeof(janas_spv_iq4xs_mvd),
                  &g->sm_mvdx4, &g->pipe_mvdx4, 0, 8)) != VK_SUCCESS)) {
            set_err(err, err_len, "cannot create the shaders", r);
            goto fail;
        }
    }
    for (uint32_t k = 0; k < N_B32; k++)
        if ((r = janas_g_make_pipeline(
                 g, janas_spv_b32_matvec, sizeof(janas_spv_b32_matvec),
                 &g->sm32[k], &g->pipe32[k], k)) != VK_SUCCESS) {
            set_err(err, err_len, "cannot create the shaders", r);
            goto fail;
        }
    pthread_mutex_init(&g->qlock, NULL);
    const char *ka = getenv("JANAS_GPU_KEEPALIVE_US");
    g->keep_us = ka ? (unsigned)atoi(ka) : 500;
    atomic_init(&g->ka_wanted, ka != NULL); /* as before when forced */
    if (g->keep_us > 0 && record_keep_alive(g) == 0 &&
        pthread_create(&g->ka, NULL, keep_alive, g) == 0)
        g->ka_running = 1;
    /* single tokens too on a card with memory of its own: there the GPU
       adds its bandwidth to the RAM's (measured on an RTX 5080, issue #15:
       replies 11-16% faster with only attention and output in its memory);
       an integrated GPU shares the RAM's, and lost 6% */
    const char *e = getenv("JANAS_GPU_MIN_N");
    g->min_n = e && atoi(e) > 0 ? (size_t)atoi(e) : g->discrete ? 1 : 2;
    e = getenv("JANAS_GPU_FULL_N");
    g->full_n = e && atoi(e) > 0 ? (size_t)atoi(e) : (size_t)-1;
    e = getenv("JANAS_GPU_VERBOSE");
    g->verbose = e && atoi(e) > 0 ? atoi(e) : 0;
    /* the pool's idle workers asleep while the GPU computes, and a
       blocking wait for it: on with the fast prompt (=0 turns either off),
       where the GPU computes nearly everything and the CPU's spinning draws
       the power it would use (the laptop's PL4) - Qwen3.5-2B read a prompt
       at 389 tokens/s instead of 315 (each alone: 346, 317; 10 Oct 2026) */
    const char *ns = getenv("JANAS_GPU_NOSPIN"),
               *bw = getenv("JANAS_GPU_BLOCKWAIT");
    g->nospin = ns ? atoi(ns) != 0 : janas_gpu_fast_prompt();
    g->block_wait = bw ? atoi(bw) != 0 : janas_gpu_fast_prompt();
    janas_g_selfcheck(g);
    return g;

missing:
    snprintf(err, err_len, "incomplete Vulkan driver");
fail:
    janas_gpu_destroy(g);
    return NULL;
}

void janas_gpu_destroy(struct janas_gpu *g)
{
    if (!g)
        return;
    if (g->verbose)
        fprintf(stderr, "gpu begin: convert %.3f s, submit %.3f s\n", g->t_conv,
                g->t_sub);
    if (g->verbose && g->b16_calls)
        fprintf(stderr,
                "gpu bf16: %llu products bit for bit, %llu through the "
                "stages\n",
                (unsigned long long)g->b16_calls,
                (unsigned long long)g->b16_staged);
    /* the key: (group's products << 48) ^ (rows << 24) ^ (columns << 8) ^
       vectors, of its first product. Calls with every row on the GPU (the
       fast prompt's) apart: there the CPU waits having nothing to do, and
       neither "late" nor the share says anything (issue #16 read them as a
       GPU always behind) */
    for (int i = 0; g->verbose && i < SHARE_SLOTS; i++) {
        const struct share *s = &g->share[i];
        if (!s->calls)
            continue;
        double c = 1e6 / s->calls;
        if (s->alone == s->calls)
            fprintf(stderr,
                    "gpu alone: key %016llx, %u calls with every row on the "
                    "GPU; us/call: begin %.0f wait %.0f copy %.0f\n",
                    (unsigned long long)s->key, s->calls, c * s->t_begin,
                    c * s->t_wait, c * s->t_copy);
        else
            fprintf(
                stderr,
                "gpu share: key %016llx, %u calls (%u with every row on "
                "the GPU), GPU late in %u of the others, share %.2f "
                "(mean %.2f); us/call: begin %.0f cpu %.0f wait %.0f "
                "copy %.0f\n",
                (unsigned long long)s->key, s->calls, s->alone, s->waits, s->f,
                (double)s->rows_gpu / (double)(s->rows_all ? s->rows_all : 1),
                c * s->t_begin, c * s->t_cpu, c * s->t_wait, c * s->t_copy);
    }
    if (g->ka_running) {
        atomic_store(&g->ka_stop, 1);
        pthread_join(g->ka, NULL);
    }
    if (g->dev) {
        if (g->ka_fence)
            g->vkDestroyFence(g->dev, g->ka_fence, NULL);
        if (g->busy) {
            int waited;
            janas_gpu_finish(g, &waited);
        }
        struct stage *st[4] = {&g->xs, &g->ys, &g->ts, &g->kas};
        for (int i = 0; i < 4; i++) {
            if (st[i]->buf)
                g->vkDestroyBuffer(g->dev, st[i]->buf, NULL);
            if (st[i]->mem)
                g->vkFreeMemory(g->dev, st[i]->mem, NULL);
        }
        for (int i = 0; i < g->n_reg; i++) {
            g->vkDestroyBuffer(g->dev, g->reg[i].buf, NULL);
            g->vkFreeMemory(g->dev, g->reg[i].mem, NULL);
        }
        if (g->fence)
            g->vkDestroyFence(g->dev, g->fence, NULL);
        if (g->cp)
            g->vkDestroyCommandPool(g->dev, g->cp, NULL);
        if (g->pipe)
            g->vkDestroyPipeline(g->dev, g->pipe, NULL);
        if (g->pipe6)
            g->vkDestroyPipeline(g->dev, g->pipe6, NULL);
        if (g->pipe6p)
            g->vkDestroyPipeline(g->dev, g->pipe6p, NULL);
        if (g->pipe5)
            g->vkDestroyPipeline(g->dev, g->pipe5, NULL);
        if (g->pipe80)
            g->vkDestroyPipeline(g->dev, g->pipe80, NULL);
        if (g->rp_q6ka)
            g->vkDestroyPipeline(g->dev, g->rp_q6ka, NULL);
        if (g->rm_q6ka)
            g->vkDestroyShaderModule(g->dev, g->rm_q6ka, NULL);
        if (g->pipe_mv)
            g->vkDestroyPipeline(g->dev, g->pipe_mv, NULL);
        if (g->sm_mv)
            g->vkDestroyShaderModule(g->dev, g->sm_mv, NULL);
        if (g->pipe_mvd)
            g->vkDestroyPipeline(g->dev, g->pipe_mvd, NULL);
        if (g->sm_mvd)
            g->vkDestroyShaderModule(g->dev, g->sm_mvd, NULL);
        if (g->pipe_mvd6)
            g->vkDestroyPipeline(g->dev, g->pipe_mvd6, NULL);
        if (g->sm_mvd6)
            g->vkDestroyShaderModule(g->dev, g->sm_mvd6, NULL);
        if (g->pipe_mvd5)
            g->vkDestroyPipeline(g->dev, g->pipe_mvd5, NULL);
        if (g->sm_mvd5)
            g->vkDestroyShaderModule(g->dev, g->sm_mvd5, NULL);
        if (g->pipe_mvdx4)
            g->vkDestroyPipeline(g->dev, g->pipe_mvdx4, NULL);
        if (g->sm_mvdx4)
            g->vkDestroyShaderModule(g->dev, g->sm_mvdx4, NULL);
        for (int i = 0; i < 2; i++) {
            if (g->rp_tile[i])
                g->vkDestroyPipeline(g->dev, g->rp_tile[i], NULL);
            if (g->rm_tile[i])
                g->vkDestroyShaderModule(g->dev, g->rm_tile[i], NULL);
        }
        for (int k = 0; k < 3; k++)
            for (int i = 0; i < 4; i++) {
                if (g->rpv[k][i])
                    g->vkDestroyPipeline(g->dev, g->rpv[k][i], NULL);
                if (g->rmv[k][i])
                    g->vkDestroyShaderModule(g->dev, g->rmv[k][i], NULL);
            }
        for (int i = 0; i < N_PROD; i++) {
            if (g->pp1[i])
                g->vkDestroyPipeline(g->dev, g->pp1[i], NULL);
            if (g->pm1[i])
                g->vkDestroyShaderModule(g->dev, g->pm1[i], NULL);
            if (g->rp[i])
                g->vkDestroyPipeline(g->dev, g->rp[i], NULL);
            if (g->rm[i])
                g->vkDestroyShaderModule(g->dev, g->rm[i], NULL);
        }
        if (g->pipex4)
            g->vkDestroyPipeline(g->dev, g->pipex4, NULL);
        if (g->pipe3)
            g->vkDestroyPipeline(g->dev, g->pipe3, NULL);
        if (g->pipe_mm)
            g->vkDestroyPipeline(g->dev, g->pipe_mm, NULL);
        if (g->pipe_mm6)
            g->vkDestroyPipeline(g->dev, g->pipe_mm6, NULL);
        if (g->pipe_mm6s)
            g->vkDestroyPipeline(g->dev, g->pipe_mm6s, NULL);
        if (g->pipe_mm3)
            g->vkDestroyPipeline(g->dev, g->pipe_mm3, NULL);
        if (g->pipe_mmx4)
            g->vkDestroyPipeline(g->dev, g->pipe_mmx4, NULL);
        if (g->pipe_mmi3)
            g->vkDestroyPipeline(g->dev, g->pipe_mmi3, NULL);
        if (g->pipe_mmn4)
            g->vkDestroyPipeline(g->dev, g->pipe_mmn4, NULL);
        if (g->pipe_mmmx)
            g->vkDestroyPipeline(g->dev, g->pipe_mmmx, NULL);
        if (g->pipe_mm80)
            g->vkDestroyPipeline(g->dev, g->pipe_mm80, NULL);
        if (g->pipe_mm50)
            g->vkDestroyPipeline(g->dev, g->pipe_mm50, NULL);
        if (g->sm_mm50)
            g->vkDestroyShaderModule(g->dev, g->sm_mm50, NULL);
        if (g->pipe_mm20)
            g->vkDestroyPipeline(g->dev, g->pipe_mm20, NULL);
        if (g->sm_mm20)
            g->vkDestroyShaderModule(g->dev, g->sm_mm20, NULL);
        for (int i = 0; i < 3; i++) {
            if (g->pipe_idb[i])
                g->vkDestroyPipeline(g->dev, g->pipe_idb[i], NULL);
            if (g->sm_idb[i])
                g->vkDestroyShaderModule(g->dev, g->sm_idb[i], NULL);
        }
        if (g->pipe_b16)
            g->vkDestroyPipeline(g->dev, g->pipe_b16, NULL);
        if (g->sm_b16)
            g->vkDestroyShaderModule(g->dev, g->sm_b16, NULL);
        for (int i = 0; i < N_IQK; i++) {
            if (g->pipe_iqq[i])
                g->vkDestroyPipeline(g->dev, g->pipe_iqq[i], NULL);
            if (g->sm_iqq[i])
                g->vkDestroyShaderModule(g->dev, g->sm_iqq[i], NULL);
        }
        for (int i = 0; i < N_IQK + 2; i++) {
            if (g->pipe_iqm[i])
                g->vkDestroyPipeline(g->dev, g->pipe_iqm[i], NULL);
            if (g->sm_iqm[i])
                g->vkDestroyShaderModule(g->dev, g->sm_iqm[i], NULL);
        }
        for (int i = 0; i < N_IQK; i++) {
            if (g->pipe_iqk[i])
                g->vkDestroyPipeline(g->dev, g->pipe_iqk[i], NULL);
            if (g->sm_iqk[i])
                g->vkDestroyShaderModule(g->dev, g->sm_iqk[i], NULL);
            if (g->pp1_iqk[i])
                g->vkDestroyPipeline(g->dev, g->pp1_iqk[i], NULL);
            if (g->sm1_iqk[i])
                g->vkDestroyShaderModule(g->dev, g->sm1_iqk[i], NULL);
        }
        if (g->pipe_mm5)
            g->vkDestroyPipeline(g->dev, g->pipe_mm5, NULL);
        if (g->sm_mm5)
            g->vkDestroyShaderModule(g->dev, g->sm_mm5, NULL);
        for (int i = 0; i < 8; i++) {
            if (g->pipe_mvid[i])
                g->vkDestroyPipeline(g->dev, g->pipe_mvid[i], NULL);
            if (g->sm_mvid[i])
                g->vkDestroyShaderModule(g->dev, g->sm_mvid[i], NULL);
        }
        for (int i = 0; i < 3; i++) {
            if (g->pipe_dm[i])
                g->vkDestroyPipeline(g->dev, g->pipe_dm[i], NULL);
            if (g->sm_dm[i])
                g->vkDestroyShaderModule(g->dev, g->sm_dm[i], NULL);
            if (g->pipe_dv[i])
                g->vkDestroyPipeline(g->dev, g->pipe_dv[i], NULL);
            if (g->sm_dv[i])
                g->vkDestroyShaderModule(g->dev, g->sm_dv[i], NULL);
        }
        for (int i = 0; i < 5; i++) {
            VkPipeline pp = i == 4   ? g->pipe_id3
                            : i == 3 ? g->pipe_idmx
                            : i == 2 ? g->pipe_id6s
                            : i      ? g->pipe_id6
                                     : g->pipe_id4;
            VkShaderModule ss = i == 4   ? g->sm_id3
                                : i == 3 ? g->sm_idmx
                                : i == 2 ? g->sm_id6s
                                : i      ? g->sm_id6
                                         : g->sm_id4;
            if (pp)
                g->vkDestroyPipeline(g->dev, pp, NULL);
            if (ss)
                g->vkDestroyShaderModule(g->dev, ss, NULL);
        }
        if (g->sm_mm6s)
            g->vkDestroyShaderModule(g->dev, g->sm_mm6s, NULL);
        if (g->sm_mm3)
            g->vkDestroyShaderModule(g->dev, g->sm_mm3, NULL);
        if (g->sm_mmx4)
            g->vkDestroyShaderModule(g->dev, g->sm_mmx4, NULL);
        if (g->sm_mmi3)
            g->vkDestroyShaderModule(g->dev, g->sm_mmi3, NULL);
        if (g->sm_mmn4)
            g->vkDestroyShaderModule(g->dev, g->sm_mmn4, NULL);
        if (g->sm_mmmx)
            g->vkDestroyShaderModule(g->dev, g->sm_mmmx, NULL);
        if (g->sm_mm80)
            g->vkDestroyShaderModule(g->dev, g->sm_mm80, NULL);
        if (g->sm_mm6)
            g->vkDestroyShaderModule(g->dev, g->sm_mm6, NULL);
        if (g->sm_mm)
            g->vkDestroyShaderModule(g->dev, g->sm_mm, NULL);
        if (g->pipei3)
            g->vkDestroyPipeline(g->dev, g->pipei3, NULL);
        if (g->sm3)
            g->vkDestroyShaderModule(g->dev, g->sm3, NULL);
        if (g->smi3)
            g->vkDestroyShaderModule(g->dev, g->smi3, NULL);
        if (g->smx4)
            g->vkDestroyShaderModule(g->dev, g->smx4, NULL);
        for (int k = 0; k < N_B32; k++) {
            if (g->pipe32[k])
                g->vkDestroyPipeline(g->dev, g->pipe32[k], NULL);
            if (g->sm32[k])
                g->vkDestroyShaderModule(g->dev, g->sm32[k], NULL);
        }
        if (g->pipea)
            g->vkDestroyPipeline(g->dev, g->pipea, NULL);
        if (g->sma)
            g->vkDestroyShaderModule(g->dev, g->sma, NULL);
        if (g->pipea32)
            g->vkDestroyPipeline(g->dev, g->pipea32, NULL);
        if (g->sma32)
            g->vkDestroyShaderModule(g->dev, g->sma32, NULL);
        if (g->sm6)
            g->vkDestroyShaderModule(g->dev, g->sm6, NULL);
        if (g->sm6p)
            g->vkDestroyShaderModule(g->dev, g->sm6p, NULL);
        if (g->sm5)
            g->vkDestroyShaderModule(g->dev, g->sm5, NULL);
        if (g->sm80)
            g->vkDestroyShaderModule(g->dev, g->sm80, NULL);
        if (g->pl)
            g->vkDestroyPipelineLayout(g->dev, g->pl, NULL);
        if (g->sm)
            g->vkDestroyShaderModule(g->dev, g->sm, NULL);
        g->vkDestroyDevice(g->dev, NULL);
    }
    if (g->inst && g->vkDestroyInstance)
        g->vkDestroyInstance(g->inst, NULL);
    if (g->lib)
        dlclose(g->lib);
    free(g);
}

const char *janas_gpu_name(const struct janas_gpu *g)
{
    return g->name;
}

unsigned janas_gpu_subgroup(const struct janas_gpu *g)
{
    return g->sg;
}

/* The product shaders' names, for janas_gpu_last_shader. */
static const struct {
    const uint32_t *code;
    const char *name;
} spv_names[] = {
#define N(x) {janas_spv_##x, #x}
    N(q4k_matvec),    N(q6k_matvec),  N(q6kp_matvec),  N(q5k_matvec),
    N(q8_0_matvec),   N(b32_matvec),  N(iq4xs_matvec), N(q3k_matvec),
    N(q4k_mmq),       N(q6kp_mmq),    N(q6k_mmq),      N(q3k_mmq),
    N(iq4xs_mmq),     N(iq3s_mmq),    N(iq4nl_mmq),    N(mxfp4_mmid),
    N(b32_mmid),      N(bf16_matvec), N(iq1_matvec),   N(iq2_matvec),
    N(iq3xxs_matvec), N(iqk_mmid),    N(iqk_mmq),      N(q2k_matvec),
    N(q80_mmq),       N(q5k_mmq),     N(dense_mm),     N(q3k_mmid),
    N(q4k_mmid),      N(q6kp_mmid),   N(q6k_mmid),     N(iq3s_matvec),
    N(expert_act),    N(q4k_row),     N(q6k_row),      N(q6kp_row),
    N(q6ka_row),      N(q4k_mv),      N(q4k_mvd),      N(q6k_mvd),
    N(q5k_mvd),       N(iq4xs_mvd)
#undef N
};

size_t janas_gpu_last_shader(const struct janas_gpu *g, char *buf, size_t len)
{
    buf[0] = 0;
    for (int i = 0; i < g->n_made; i++)
        if (g->made[i].p == g->last_pipe) {
            const char *name = "a shader";
            for (size_t k = 0; k < sizeof(spv_names) / sizeof(*spv_names); k++)
                if (spv_names[k].code == g->made[i].code)
                    name = spv_names[k].name;
            int n = snprintf(buf, len, "%s NV %u SG %u", name, g->made[i].nv,
                             g->made[i].sg);
            if (g->made[i].kind && n > 0 && (size_t)n < len)
                n += snprintf(buf + n, len - (size_t)n, " kind %u",
                              g->made[i].kind);
            return n > 0 ? (size_t)n : 0;
        }
    return 0;
}

int janas_gpu_set_path(struct janas_gpu *g, enum janas_gpu_path path, int value)
{
    int *f = path == JANAS_GPU_PATH_ROW      ? &g->row_on
             : path == JANAS_GPU_PATH_ROW_NV ? &g->row_nv
             : path == JANAS_GPU_PATH_MVD    ? &g->mvd_on
                                             : &g->mv_on;
    int was = *f;
    *f = value;
    return was;
}

void janas_gpu_subgroup_range(const struct janas_gpu *g, unsigned *min,
                              unsigned *max)
{
    *min = g->sg_control ? g->sg_min : g->sg;
    *max = g->sg_control ? g->sg_max : g->sg;
}

const char *janas_gpu_about(const struct janas_gpu *g)
{
    return g->about;
}

size_t janas_gpu_min_n(const struct janas_gpu *g)
{
    return g->min_n;
}

void janas_gpu_set_min_n(struct janas_gpu *g, size_t n)
{
    g->min_n = n ? n : 1;
}

void janas_gpu_keep_awake(struct janas_gpu *g, int on)
{
    if (g)
        atomic_store(&g->ka_wanted, on);
}

/* A memory type local to the device, preferably not visible to the host
   (the fastest memory of a discrete GPU). */
static int local_type(const struct janas_gpu *g, uint32_t bits)
{
    int any = -1;
    for (uint32_t i = 0; i < g->mp.memoryTypeCount; i++) {
        VkMemoryPropertyFlags f = g->mp.memoryTypes[i].propertyFlags;
        if (!(bits & (1u << i)) || !(f & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
            continue;
        if (!(f & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT))
            return (int)i;
        if (any < 0)
            any = (int)i;
    }
    return any;
}

/*
 * A copy of host memory in the device's own memory, for a discrete GPU:
 * uploaded once through a staging buffer, in chunks. The CPU keeps its own
 * copy; the region maps the host range to the device copy.
 */
static int copy_to_device(struct janas_gpu *g, const void *p, size_t bytes)
{
    struct region *rg = &g->reg[g->n_reg];
    VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
                             .size = bytes,
                             .usage =
                                 VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
                                 VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
                                 VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
                                 VK_BUFFER_USAGE_TRANSFER_DST_BIT};
    if (g->vkCreateBuffer(g->dev, &bi, NULL, &rg->buf) != VK_SUCCESS)
        return -1;
    VkMemoryRequirements req;
    g->vkGetBufferMemoryRequirements(g->dev, rg->buf, &req);
    int t = local_type(g, req.memoryTypeBits);
    /* room in its heap: the rest of the engine runs on the CPU otherwise */
    if (t < 0 || g->mp.memoryHeaps[g->mp.memoryTypes[t].heapIndex].size <
                     req.size + ((VkDeviceSize)512 << 20)) {
        g->vkDestroyBuffer(g->dev, rg->buf, NULL);
        return -1;
    }
    VkMemoryAllocateFlagsInfo fl = {
        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
        .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
    VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
                               .pNext = &fl,
                               .allocationSize = req.size,
                               .memoryTypeIndex = (uint32_t)t};
    if (g->vkAllocateMemory(g->dev, &ai, NULL, &rg->mem) != VK_SUCCESS) {
        g->vkDestroyBuffer(g->dev, rg->buf, NULL);
        return -1;
    }
    if (g->vkBindBufferMemory(g->dev, rg->buf, rg->mem, 0) != VK_SUCCESS)
        goto fail;
    g->local_bytes += req.size;
    rg->local = req.size;
    /* staging: 64 MiB of host memory, one chunk at a time */
    struct stage st = {0};
    size_t chunk = (size_t)64 << 20;
    VkBufferCreateInfo sbi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
                              .size = chunk,
                              .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT};
    if (g->vkCreateBuffer(g->dev, &sbi, NULL, &st.buf) != VK_SUCCESS)
        goto fail;
    g->vkGetBufferMemoryRequirements(g->dev, st.buf, &req);
    int ts = janas_g_mem_type(g, req.memoryTypeBits,
                              VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
                                  VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
    VkMemoryAllocateInfo sai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
                                .allocationSize = req.size,
                                .memoryTypeIndex = (uint32_t)ts};
    int ok = ts >= 0 &&
             g->vkAllocateMemory(g->dev, &sai, NULL, &st.mem) == VK_SUCCESS &&
             g->vkBindBufferMemory(g->dev, st.buf, st.mem, 0) == VK_SUCCESS &&
             g->vkMapMemory(g->dev, st.mem, 0, VK_WHOLE_SIZE, 0,
                            (void **)&st.map) == VK_SUCCESS;
    for (size_t off = 0; ok && off < bytes; off += chunk) {
        size_t nb = bytes - off < chunk ? bytes - off : chunk;
        memcpy(st.map, (const uint8_t *)p + off, nb);
        VkCommandBufferBeginInfo bb = {
            .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
            .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
        VkBufferCopy cp = {.srcOffset = 0, .dstOffset = off, .size = nb};
        VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                            .commandBufferCount = 1,
                            .pCommandBuffers = &g->cb};
        ok = g->vkResetCommandBuffer(g->cb, 0) == VK_SUCCESS &&
             g->vkBeginCommandBuffer(g->cb, &bb) == VK_SUCCESS;
        if (ok)
            g->vkCmdCopyBuffer(g->cb, st.buf, rg->buf, 1, &cp);
        ok = ok && g->vkEndCommandBuffer(g->cb) == VK_SUCCESS;
        pthread_mutex_lock(&g->qlock);
        ok = ok && g->vkQueueSubmit(g->queue, 1, &sub, g->fence) == VK_SUCCESS;
        pthread_mutex_unlock(&g->qlock);
        ok = ok && g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE,
                                      UINT64_MAX) == VK_SUCCESS;
        g->vkResetFences(g->dev, 1, &g->fence);
    }
    if (st.buf)
        g->vkDestroyBuffer(g->dev, st.buf, NULL);
    if (st.mem)
        g->vkFreeMemory(g->dev, st.mem, NULL);
    if (!ok)
        goto fail;
    rg->host = p;
    rg->bytes = bytes;
    rg->addr = janas_g_buffer_address(g, rg->buf);
    rg->coherent = 1;
    rg->host_shared = 0;
    g->n_reg++;
    return 0;

fail:
    g->vkDestroyBuffer(g->dev, rg->buf, NULL);
    g->vkFreeMemory(g->dev, rg->mem, NULL);
    g->local_bytes -= rg->local;
    rg->local = 0;
    return -1;
}

void janas_g_free_stage(struct janas_gpu *g, struct stage *s)
{
    if (s->buf)
        g->vkDestroyBuffer(g->dev, s->buf, NULL);
    if (s->mem)
        g->vkFreeMemory(g->dev, s->mem, NULL);
    g->local_bytes -= s->local;
    memset(s, 0, sizeof(*s));
}

int janas_g_make_local(struct janas_gpu *g, struct stage *s, size_t bytes)
{
    memset(s, 0, sizeof(*s));
    VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
                             .size = bytes,
                             .usage =
                                 VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
                                 VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
                                 VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT |
                                 VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
                                 VK_BUFFER_USAGE_TRANSFER_DST_BIT};
    if (g->vkCreateBuffer(g->dev, &bi, NULL, &s->buf) != VK_SUCCESS)
        return -1;
    VkMemoryRequirements req;
    g->vkGetBufferMemoryRequirements(g->dev, s->buf, &req);
    int t = local_type(g, req.memoryTypeBits);
    VkMemoryAllocateFlagsInfo fl = {
        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
        .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
    VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
                               .pNext = &fl,
                               .allocationSize = req.size,
                               .memoryTypeIndex = (uint32_t)t};
    if (t < 0 ||
        g->vkAllocateMemory(g->dev, &ai, NULL, &s->mem) != VK_SUCCESS ||
        g->vkBindBufferMemory(g->dev, s->buf, s->mem, 0) != VK_SUCCESS) {
        g->vkDestroyBuffer(g->dev, s->buf, NULL);
        if (s->mem)
            g->vkFreeMemory(g->dev, s->mem, NULL);
        memset(s, 0, sizeof(*s));
        return -1;
    }
    g->local_bytes += req.size;
    s->local = req.size;
    s->addr = janas_g_buffer_address(g, s->buf);
    s->bytes = bytes;
    return 0;
}

int janas_g_record_copies(struct janas_gpu *g, const struct janas_g_cp *c,
                          uint32_t n, int barrier)
{
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
    int ok = g->vkResetCommandBuffer(g->cb, 0) == VK_SUCCESS &&
             g->vkBeginCommandBuffer(g->cb, &bb) == VK_SUCCESS;
    for (uint32_t i = 0; ok && i < n; i++) {
        VkBufferCopy cp = {
            .srcOffset = c[i].so, .dstOffset = c[i].dof, .size = c[i].bytes};
        g->vkCmdCopyBuffer(g->cb, c[i].src, c[i].dst, 1, &cp);
    }
    if (ok && barrier) {
        /* what the commands submitted after it read: shaders, copies and
           indirect dispatches */
        VkMemoryBarrier mb = {
            .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
            .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
            .dstAccessMask =
                VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
                VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT |
                VK_ACCESS_INDIRECT_COMMAND_READ_BIT};
        g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_TRANSFER_BIT,
                                VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
                                    VK_PIPELINE_STAGE_TRANSFER_BIT |
                                    VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
                                0, 1, &mb, 0, NULL, 0, NULL);
    }
    return ok && g->vkEndCommandBuffer(g->cb) == VK_SUCCESS ? 0 : -1;
}

int janas_g_copies(struct janas_gpu *g, const struct janas_g_cp *c, uint32_t n)
{
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->cb};
    int ok = janas_g_record_copies(g, c, n, 0) == 0;
    pthread_mutex_lock(&g->qlock);
    ok = ok && g->vkQueueSubmit(g->queue, 1, &sub, g->fence) == VK_SUCCESS;
    pthread_mutex_unlock(&g->qlock);
    ok = ok && g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE, UINT64_MAX) ==
                   VK_SUCCESS;
    g->vkResetFences(g->dev, 1, &g->fence);
    return ok ? 0 : -1;
}

int janas_g_copy(struct janas_gpu *g, VkBuffer src, VkDeviceSize so,
                 VkBuffer dst, VkDeviceSize dof, VkDeviceSize bytes)
{
    struct janas_g_cp c = {
        .src = src, .dst = dst, .so = so, .dof = dof, .bytes = bytes};
    return janas_g_copies(g, &c, 1);
}

int janas_gpu_sync(struct janas_gpu *g, void *p, size_t bytes, int to_device)
{
    if (bytes == 0)
        return 0;
    const struct region *rg = janas_g_find_region(g, p, bytes);
    if (!rg || g->broken || g->busy)
        return -1;
    if (rg->host_shared)
        return 0; /* the GPU reads and writes the host's memory itself */
    VkDeviceSize at = (VkDeviceSize)((const uint8_t *)p - rg->host);
    /* through the Y stage, idle between passes, a piece at a time */
    for (size_t done = 0; done < bytes;) {
        size_t n = bytes - done < g->ys.bytes ? bytes - done : g->ys.bytes;
        /* the stage may be cached, not coherent: written bytes flushed
           before the copy, read ones invalidated after it */
        VkMappedMemoryRange mr = {
            .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
            .memory = g->ys.mem,
            .size = (n + g->atom - 1) / g->atom * g->atom < g->ys.bytes
                        ? (n + g->atom - 1) / g->atom * g->atom
                        : VK_WHOLE_SIZE};
        if (to_device) {
            memcpy(g->ys.map, (const uint8_t *)p + done, n);
            if (!g->ys.coherent)
                g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);
        }
        VkCommandBufferBeginInfo bb = {
            .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
            .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
        VkBufferCopy cp = {.srcOffset = to_device ? 0 : at + done,
                           .dstOffset = to_device ? at + done : 0,
                           .size = n};
        VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                            .commandBufferCount = 1,
                            .pCommandBuffers = &g->cb};
        int ok = g->vkResetCommandBuffer(g->cb, 0) == VK_SUCCESS &&
                 g->vkBeginCommandBuffer(g->cb, &bb) == VK_SUCCESS;
        if (ok)
            g->vkCmdCopyBuffer(g->cb, to_device ? g->ys.buf : rg->buf,
                               to_device ? rg->buf : g->ys.buf, 1, &cp);
        ok = ok && g->vkEndCommandBuffer(g->cb) == VK_SUCCESS;
        pthread_mutex_lock(&g->qlock);
        ok = ok && g->vkQueueSubmit(g->queue, 1, &sub, g->fence) == VK_SUCCESS;
        pthread_mutex_unlock(&g->qlock);
        ok = ok && g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE,
                                      UINT64_MAX) == VK_SUCCESS;
        g->vkResetFences(g->dev, 1, &g->fence);
        if (!ok)
            return -1;
        if (!to_device) {
            if (!g->ys.coherent)
                g->vkInvalidateMappedMemoryRanges(g->dev, 1, &mr);
            memcpy((uint8_t *)p + done, g->ys.map, n);
        }
        done += n;
    }
    return 0;
}

/* an import refused, and where, with JANAS_GPU_VERBOSE */
static int import_refused(const struct janas_gpu *g, const void *p,
                          size_t bytes, const char *why, int r)
{
    if (g->verbose)
        fprintf(stderr,
                "gpu: %.2f GB at %p not imported: %s (%d), %d regions\n",
                bytes / 1e9, p, why, r, g->n_reg);
    return -1;
}

int janas_gpu_import(struct janas_gpu *g, const void *p, size_t bytes)
{
    /* below 4 GiB: Mesa's Intel driver passes an imported allocation's size
       on in 32 bits, and asked the kernel for 0 bytes of a region of 4 GiB
       (unsloth's 3-bit Qwen3-30B-A3B's experts, 21 Oct 2026) */
    if (g->n_reg == MAX_REGIONS || ((uintptr_t)p | bytes) % 4096 != 0 ||
        bytes == 0 || bytes >= ((size_t)4 << 30))
        return import_refused(g, p, bytes, "regions, alignment or size", 0);
    /* a discrete GPU (or JANAS_GPU_COPY=1, to test that path) reads a copy
       in its own memory */
    if (g->discrete || !g->host_import || janas_g_copy_asked())
        return copy_to_device(g, p, bytes);
    struct region *rg = &g->reg[g->n_reg];
    VkMemoryHostPointerPropertiesEXT hp = {
        .sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT};
    VkResult r = g->vkGetMemoryHostPointerPropertiesEXT(
        g->dev, VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, p, &hp);
    if (r != VK_SUCCESS)
        return import_refused(g, p, bytes, "host pointer properties", r);
    VkExternalMemoryBufferCreateInfo eb = {
        .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
        .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT};
    VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
                             .pNext = &eb,
                             .size = bytes,
                             .usage =
                                 VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
                                 VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT};
    if ((r = g->vkCreateBuffer(g->dev, &bi, NULL, &rg->buf)) != VK_SUCCESS)
        return import_refused(g, p, bytes, "buffer", r);
    VkMemoryRequirements req;
    g->vkGetBufferMemoryRequirements(g->dev, rg->buf, &req);
    /* coherent only: the host writes these pages and reads what the GPU
       wrote with no flush or invalidation in between (some ARM drivers
       offer cached, non-coherent types: refused, the copy way instead) */
    int t = janas_g_mem_type(g, hp.memoryTypeBits & req.memoryTypeBits,
                             VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
    VkImportMemoryHostPointerInfoEXT imp = {
        .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
        .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
        .pHostPointer = (void *)p};
    VkMemoryAllocateFlagsInfo fl = {
        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
        .pNext = &imp,
        .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
    VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
                               .pNext = &fl,
                               .allocationSize = bytes,
                               .memoryTypeIndex = (uint32_t)t};
    if (t < 0 ||
        (r = g->vkAllocateMemory(g->dev, &ai, NULL, &rg->mem)) != VK_SUCCESS) {
        g->vkDestroyBuffer(g->dev, rg->buf, NULL);
        return import_refused(g, p, bytes, t < 0 ? "memory type" : "memory",
                              t < 0 ? 0 : r);
    }
    if ((r = g->vkBindBufferMemory(g->dev, rg->buf, rg->mem, 0)) !=
        VK_SUCCESS) {
        g->vkDestroyBuffer(g->dev, rg->buf, NULL);
        g->vkFreeMemory(g->dev, rg->mem, NULL);
        return import_refused(g, p, bytes, "binding", r);
    }
    rg->host = p;
    rg->bytes = bytes;
    rg->addr = janas_g_buffer_address(g, rg->buf);
    rg->coherent = 1;
    rg->host_shared = 1;
    g->n_reg++;
    return 0;
}

void janas_g_drop_region(struct janas_gpu *g, const void *host)
{
    if (g->n_reg == 0 || g->reg[g->n_reg - 1].host != host)
        return;
    struct region *rg = &g->reg[--g->n_reg];
    g->vkDestroyBuffer(g->dev, rg->buf, NULL);
    g->vkFreeMemory(g->dev, rg->mem, NULL);
    g->local_bytes -= rg->local;
    memset(rg, 0, sizeof(*rg));
}

void *janas_gpu_alloc(struct janas_gpu *g, size_t bytes)
{
    bytes = (bytes + 4095) / 4096 * 4096;
    if (g->n_reg == MAX_REGIONS || bytes == 0 || bytes > ((size_t)4 << 30))
        return NULL;
    struct region *rg = &g->reg[g->n_reg];
    VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
                             .size = bytes,
                             .usage =
                                 VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
                                 VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT};
    if (g->vkCreateBuffer(g->dev, &bi, NULL, &rg->buf) != VK_SUCCESS)
        return NULL;
    VkMemoryRequirements req;
    g->vkGetBufferMemoryRequirements(g->dev, rg->buf, &req);
    int t = janas_g_mem_type(g, req.memoryTypeBits,
                             VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
                                 VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
    VkMemoryAllocateFlagsInfo fl = {
        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
        .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
    VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
                               .pNext = &fl,
                               .allocationSize = req.size,
                               .memoryTypeIndex = (uint32_t)t};
    void *p = NULL;
    if (t < 0 ||
        g->vkAllocateMemory(g->dev, &ai, NULL, &rg->mem) != VK_SUCCESS) {
        g->vkDestroyBuffer(g->dev, rg->buf, NULL);
        return NULL;
    }
    if (g->vkBindBufferMemory(g->dev, rg->buf, rg->mem, 0) != VK_SUCCESS ||
        g->vkMapMemory(g->dev, rg->mem, 0, VK_WHOLE_SIZE, 0, &p) !=
            VK_SUCCESS) {
        g->vkDestroyBuffer(g->dev, rg->buf, NULL);
        g->vkFreeMemory(g->dev, rg->mem, NULL);
        return NULL;
    }
    rg->host = p;
    rg->bytes = bytes;
    rg->addr = janas_g_buffer_address(g, rg->buf);
    rg->coherent = (g->mp.memoryTypes[t].propertyFlags &
                    VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
    rg->host_shared = 1;
    g->n_reg++;
    return p;
}

void janas_gpu_flush(struct janas_gpu *g, const void *p, size_t bytes)
{
    const uint8_t *b = p;
    for (int i = 0; i < g->n_reg; i++) {
        const struct region *rg = &g->reg[i];
        if (b < rg->host || b + bytes > rg->host + rg->bytes || rg->coherent)
            continue;
        VkDeviceSize off = (VkDeviceSize)(b - rg->host) / g->atom * g->atom;
        VkDeviceSize end = (VkDeviceSize)(b + bytes - rg->host);
        end = (end + g->atom - 1) / g->atom * g->atom;
        VkMappedMemoryRange mr = {
            .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
            .memory = rg->mem,
            .offset = off,
            .size = end >= rg->bytes ? VK_WHOLE_SIZE : end - off};
        g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);
    }
}

/* The region holding [p, p + bytes), or NULL. */
const struct region *janas_g_find_region(const struct janas_gpu *g,
                                         const void *p, size_t bytes)
{
    const uint8_t *b = p;
    for (int i = 0; i < g->n_reg; i++)
        if (b >= g->reg[i].host &&
            b + bytes <= g->reg[i].host + g->reg[i].bytes)
            return &g->reg[i];
    return NULL;
}

static int iqk_index(int type);

/* Bytes the shader may read for rows of a task: Q6_K, the high plane of
   Q6_K_P, Q3_K and IQ3_S read up to 2 bytes past the last block (whole
   words). A row ending inside a group of 256 is as long as it is. */
static size_t read_bytes(const struct janas_matvec_task *t, size_t rows)
{
    size_t nb = t->cols / JANAS_QK;
    if (t->cols % JANAS_QK)
        return rows * janas_qtype_row_bytes(t->type, t->cols);
    if (t->type == JANAS_Q4_K_T || t->type == JANAS_Q6_K_AT)
        return (t->rows + JANAS_TILE - 1) / JANAS_TILE * JANAS_TILE * nb *
               (t->type == JANAS_Q4_K_T ? janas_qtype_block_size(JANAS_Q4_K)
                                        : JANAS_Q6KA_BLOCK);
    size_t bs = t->type == JANAS_Q6_K_A ? JANAS_Q6KA_BLOCK
                                        : janas_qtype_block_size(t->type);
    /* iq2, iq3xxs and iq1_matvec read a word past a block, realigned */
    int ik = iqk_index(t->type);
    if (ik > 0)
        return rows * nb * bs + 8;
    return rows * nb * bs + (t->type == JANAS_Q6_K || t->type == JANAS_Q6_K_P ||
                                     t->type == JANAS_Q3_K ||
                                     t->type == JANAS_IQ3_S
                                 ? 4
                                 : 0);
}

/* The bare planes of a Q6_K_P task: level 3 (both), 2 (plane[0]) or 1. */
static int plane_level(const struct janas_matvec_task *t)
{
    return t->plane[0] && t->plane[1] ? 3 : t->plane[0] ? 2 : 1;
}

/* The region holding rows of a bare plane (4-byte aligned), or NULL. */
static const struct region *plane_region(const struct janas_gpu *g,
                                         const void *p, size_t rows, size_t nb)
{
    return (uintptr_t)p % 4 == 0
               ? janas_g_find_region(g, p, rows * nb * JANAS_Q6KP_PLANE)
               : NULL;
}

static int b32_kind(int type);

/* The index of a 256-block type in pipe_iqk, -1 if none. */
static int iqk_index(int type)
{
    static const int types[N_IQK] = {JANAS_Q2_K,  JANAS_IQ2_XXS, JANAS_IQ2_XS,
                                     JANAS_IQ2_S, JANAS_IQ3_XXS, JANAS_IQ1_S,
                                     JANAS_IQ1_M};
    for (int i = 0; i < N_IQK; i++)
        if (types[i] == type)
            return i;
    return -1;
}

/* The kind of iqk_mmid.comp for a type's experts: iqk_index's, IQ3_S 7,
   IQ4_XS 8; -1 if none. */
static int iqm_kind(int type)
{
    return type == JANAS_IQ3_S    ? N_IQK
           : type == JANAS_IQ4_XS ? N_IQK + 1
                                  : iqk_index(type);
}

/* Whether rows of t that end inside a group of 256 (qwen4exp's widths of
   640) are for the GPU: the types of b32_matvec.comp, rows of whole
   words. */
static int tail_ok(const struct janas_matvec_task *t)
{
    /* 0 bytes: a width the type cannot hold */
    size_t rb = janas_qtype_row_bytes(t->type, t->cols);
    return t->cols % JANAS_QK == 0 ||
           ((b32_kind(t->type) >= 0 || t->type == JANAS_Q8_0) && rb > 0 &&
            rb % 4 == 0);
}

int janas_gpu_can(const struct janas_gpu *g, const struct janas_matvec_task *t)
{
    /* the activations of a row ending inside a group: its last group
       whole, past the row's end zero (model_ffn.c) */
    size_t nb = (t->cols + JANAS_QK - 1) / JANAS_QK,
           nv = t->n_vec ? t->n_vec : 1;
    int typed =
        (t->type == JANAS_Q4_K && (uintptr_t)t->w % 16 == 0) ||
        (t->type == JANAS_Q5_K && (uintptr_t)t->w % 16 == 0) ||
        ((t->type == JANAS_Q8_0 || t->type == JANAS_IQ4_XS ||
          t->type == JANAS_Q3_K || t->type == JANAS_IQ3_S ||
          b32_kind(t->type) >= 0) &&
         (uintptr_t)t->w % 4 == 0) ||
        (t->type == JANAS_Q6_K && (uintptr_t)t->w % 4 == 0) ||
        (t->type == JANAS_Q6_K_A && (uintptr_t)t->w % 16 == 0 &&
         (nv == 1 || g->rpv[2][3])) ||
        ((t->type == JANAS_Q4_K_T || t->type == JANAS_Q6_K_AT) &&
         (uintptr_t)t->w % 16 == 0 && nv == 1) ||
        (t->type == JANAS_Q6_K_P && (uintptr_t)t->w % 4 == 0 &&
         (plane_level(t) < 2 || plane_region(g, t->plane[0], t->rows, nb)) &&
         (plane_level(t) < 3 || plane_region(g, t->plane[1], t->rows, nb)));
    int ik = iqk_index(t->type);
    typed |= ik >= 0 && g->pipe_iqk[ik] && (uintptr_t)t->w % 4 == 0;
    return !g->broken && typed && tail_ok(t) && !janas_g_type_off(g, t->type) &&
           nv * nb * GPU_BLOCK <= X_STAGE_BYTES &&
           t->rows * nv * sizeof(float) <= Y_STAGE_BYTES &&
           janas_g_find_region(g, t->w, read_bytes(t, t->rows));
}

/* Copies nv vectors of nb Q8_K blocks into the shader's layout. */
static void convert_x(uint8_t *dst, const struct janas_block_q8k *x, size_t xs,
                      size_t nv, size_t nb, int k32)
{
    _Alignas(64) uint8_t blk[GPU_BLOCK] = {0};
    for (size_t v = 0; v < nv; v++)
        for (size_t b = 0; b < nb; b++, dst += GPU_BLOCK) {
            const struct janas_block_q8k *s = x + v * xs + b;
            if (k32) {
                /* for the weights in blocks of 32: qs32, bsums32, d32 */
                memcpy(blk, s->qs32, JANAS_QK);
                memcpy(blk + JANAS_QK, s->bsums32, sizeof(s->bsums32));
                memcpy(blk + JANAS_QK + sizeof(s->bsums32), s->d32,
                       sizeof(s->d32));
            } else {
                memcpy(blk, s->qs, JANAS_QK);
                memcpy(blk + JANAS_QK, s->bsums, sizeof(s->bsums));
                memcpy(blk + JANAS_QK + sizeof(s->bsums), &s->d, sizeof(float));
            }
            memcpy(dst, blk, GPU_BLOCK);
        }
}

/* The KIND of b32_matvec for a type, -1 if it is not one of its. */
static int b32_kind(int type)
{
    return type == JANAS_Q4_0     ? 0
           : type == JANAS_IQ4_NL ? 1
           : type == JANAS_Q4_1   ? 2
           : type == JANAS_Q5_1   ? 3
           : type == JANAS_MXFP4  ? 4
           : type == JANAS_Q5_0   ? 5
           : type == JANAS_Q2_0   ? 6
           : type == JANAS_NVFP4  ? 7
                                  : -1;
}

/* Whether a type reads the activations in blocks of 32 (convert_x). */
int janas_g_x_in_32(int type)
{
    return type == JANAS_Q8_0 || b32_kind(type) >= 0;
}

int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,
                    const size_t *gpu_rows, size_t n)
{
    if (g->busy || g->broken)
        return -1;
    /* activations: tasks of a group often share theirs, converted once */
    struct {
        const void *x;
        size_t nv, xs, nb, off;
        int k32;
    } xc[MAX_XCACHE];
    size_t n_xc = 0, x_used = 0, y_used = 0;
    g->n_pend = 0;
    if (g->vkResetCommandBuffer(g->cb, 0) != VK_SUCCESS)
        return -1;
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
    if (g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)
        return -1;
    VkPipeline bound = VK_NULL_HANDLE;
    size_t recorded = 0; /* products to run, written directly or copied */
    for (size_t i = 0; i < n; i++) {
        if (gpu_rows[i] == 0)
            continue;
        const struct janas_matvec_task *t = &tasks[i];
        /* a row ending inside a group: its activations' last group whole */
        size_t nb = (t->cols + JANAS_QK - 1) / JANAS_QK,
               nv = t->n_vec ? t->n_vec : 1;
        size_t xs = t->x_stride ? t->x_stride : nb;
        size_t k = 0;
        int k32 = janas_g_x_in_32(t->type);
        while (k < n_xc &&
               !(xc[k].x == t->x && xc[k].nv == nv && xc[k].xs == xs &&
                 xc[k].nb == nb && xc[k].k32 == k32))
            k++;
        if (k == n_xc) {
            size_t bytes = nv * nb * GPU_BLOCK;
            if (n_xc == MAX_XCACHE || x_used + bytes > X_STAGE_BYTES)
                return -1;
            double tc = g->verbose ? now() : 0;
            convert_x(g->xs.map + x_used, t->x, xs, nv, nb, k32);
            if (g->verbose)
                g->t_conv += now() - tc;
            xc[k].x = t->x;
            xc[k].nv = nv;
            xc[k].xs = xs;
            xc[k].nb = nb;
            xc[k].off = x_used;
            xc[k].k32 = k32;
            x_used += bytes;
            n_xc++;
        }
        /* an output the GPU shares (the engine's buffers, imported at
           load): written there, rows as the CPU lays them out, no copy.
           Not a card's copy of it, which the GPU's writes would never
           leave - the CPU read what was there before (issue #17): through
           the stage then */
        size_t ystr = t->y_stride ? t->y_stride : t->rows;
        const struct region *yr = janas_g_find_region(
            g, t->y, ((nv - 1) * ystr + gpu_rows[i]) * sizeof(float));
        if (yr && yr->host_shared) {
            if (janas_g_record_task(
                    g, &bound, t, gpu_rows[i], g->xs.addr + xc[k].off,
                    yr->addr +
                        (VkDeviceAddress)((const uint8_t *)t->y - yr->host),
                    ystr) != 0)
                return -1;
            recorded++;
            continue;
        }
        size_t ybytes = gpu_rows[i] * nv * sizeof(float);
        if (g->n_pend == MAX_PENDING || y_used + ybytes > Y_STAGE_BYTES)
            return -1;
        if (janas_g_record_task(g, &bound, t, gpu_rows[i],
                                g->xs.addr + xc[k].off, g->ys.addr + y_used,
                                gpu_rows[i]) != 0)
            return -1;
        recorded++;
        g->pend[g->n_pend++] =
            (struct pending){.y = t->y,
                             .rows = gpu_rows[i],
                             .nv = nv,
                             .y_stride = t->y_stride ? t->y_stride : t->rows,
                             .off = y_used};
        y_used += ybytes;
    }
    /* the shader's writes made visible to the host once the fence signals
       (without this they can stay in the GPU's cache) */
    VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
                          .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
                          .dstAccessMask = VK_ACCESS_HOST_READ_BIT};
    g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                            VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,
                            NULL);
    if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)
        return -1;
    g->y_used = y_used;
    if (recorded == 0)
        return 0;
    if (!g->xs.coherent && x_used > 0) {
        VkDeviceSize sz = (x_used + g->atom - 1) / g->atom * g->atom;
        VkMappedMemoryRange mr = {
            .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
            .memory = g->xs.mem,
            .size = sz < g->xs.bytes ? sz : VK_WHOLE_SIZE};
        g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);
    }
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->cb};
    double ts = g->verbose ? now() : 0;
    pthread_mutex_lock(&g->qlock);
    VkResult sr = g->vkQueueSubmit(g->queue, 1, &sub, g->fence);
    pthread_mutex_unlock(&g->qlock);
    if (g->verbose)
        g->t_sub += now() - ts;
    if (sr != VK_SUCCESS) {
        g->broken = 1;
        return -1;
    }
    g->busy = 1;
    return 0;
}

/* Push constants of the activation shader. */
struct push_act {
    uint32_t g[2], u[2], x[2];
    uint32_t blocks, gelu; /* the shader's whole block, gelu 0 here */
};

void janas_g_push_addr(uint32_t out[2], VkDeviceAddress a)
{
    out[0] = (uint32_t)a;
    out[1] = (uint32_t)(a >> 32);
}

/* The index of a type's product shader in the single-vector table, -1 if
   none: Q4_K, Q6_K, Q6_K_P, Q5_K, Q8_0, four of b32 (Q4_0, IQ4_NL, Q4_1,
   Q5_1), IQ4_XS, Q3_K, IQ3_S, and b32's MXFP4, Q5_0 and Q2_0. */
static int prod_index(int type)
{
    static const int types[N_PROD] = {
        JANAS_Q4_K, JANAS_Q6_K,   JANAS_Q6_K_P, JANAS_Q5_K, JANAS_Q8_0,
        JANAS_Q4_0, JANAS_IQ4_NL, JANAS_Q4_1,   JANAS_Q5_1, JANAS_IQ4_XS,
        JANAS_Q3_K, JANAS_IQ3_S,  JANAS_MXFP4,  JANAS_Q5_0, JANAS_Q2_0};
    for (int i = 0; i < N_PROD; i++)
        if (types[i] == type)
            return i;
    return -1;
}

/*
 * The fast prompt (on unless JANAS_GPU_FAST_PROMPT=0, since 10 October
 * 2026): a prompt's blocks go to the tiled shaders (*_mmq.comp,
 * *_mmid.comp, dense_mm.comp) and, where they can, whole to the GPU in one
 * submission (gpu_token.c); they round group by group and so are not bit
 * for bit with the CPU. Qwen3-4B, 256 tokens against Qwen's Q8_0
 * (9 Oct 2026): the most likely token 234 of 256 on code with it and
 * without, 219 and 217 on Italian, mean |diff| 0.7234 against 0.7249 and
 * 0.5620 against 0.5637; the block 10% faster on the development laptop.
 */
static int mm_on(void);

int janas_gpu_fast_prompt(void)
{
    return mm_on();
}

void janas_gpu_set_only_fast(struct janas_gpu *g, int on)
{
    g->only_fast = on;
}

void janas_gpu_set_pass(struct janas_gpu *g, uint32_t n)
{
    g->pass_n = n;
}

static int mm_on(void)
{
    static int on = -1;
    if (on < 0)
        on = !getenv("JANAS_GPU_FAST_PROMPT") ||
             atoi(getenv("JANAS_GPU_FAST_PROMPT")) > 0;
    return on;
}

/* q4k_mv for Q4_K on 2 to JANAS_GPU_FAST_N - 1 vectors (JANAS_GPU_MV=1;
   off by default while it is measured) */
static int mv_on(const struct janas_gpu *g)
{
    return g->mv_on;
}

/* q4k_mvd, q6k_mvd, q5k_mvd and iq4xs_mvd for Q4_K, Q6_K, Q5_K and IQ4_XS
   on 2 to 8 vectors (JANAS_GPU_MVD=0 leaves them out) */
static int mvd_on(const struct janas_gpu *g)
{
    return g->mvd_on;
}

/* The shader of a type with the lanes on the vectors (2 to 8 vectors), or
   none. */
static VkPipeline mvd_pipe(const struct janas_gpu *g, int type)
{
    return type == JANAS_Q4_K     ? g->pipe_mvd
           : type == JANAS_Q6_K   ? g->pipe_mvd6
           : type == JANAS_Q5_K   ? g->pipe_mvd5
           : type == JANAS_IQ4_XS ? g->pipe_mvdx4
                                  : VK_NULL_HANDLE;
}

size_t janas_gpu_few_max(const struct janas_gpu *g)
{
    return g->pipe_mvd && mvd_on(g) ? 8 : 0;
}

/* Whether bytes from a lie within one 4 GB of the address space (the high
   word the same throughout), as q4k_mv's and q4k_mvd's addressing needs. */
static int in_4g(VkDeviceAddress a, size_t bytes)
{
    return bytes == 0 || (a >> 32) == ((a + bytes - 1) >> 32);
}

/* Whether a pipeline is one of the shaders of a row an invocation
   (*_matvec.comp, q4k_mv.comp), which take the groups of vectors on
   workgroup x and the rows on y. */
static int vectors_first(const struct janas_gpu *g, VkPipeline p)
{
    if (p == g->pipe || p == g->pipe6 || p == g->pipe6p || p == g->pipe5 ||
        p == g->pipe80 || p == g->pipex4 || p == g->pipe3 || p == g->pipei3 ||
        p == g->pipe_mv || p == g->pipe_mvd || p == g->pipe_mvd6 ||
        p == g->pipe_mvd5 || p == g->pipe_mvdx4)
        return 1;
    for (int k = 0; k < N_B32; k++)
        if (p == g->pipe32[k])
            return 1;
    for (int i = 0; i < N_PROD; i++)
        if (p == g->pp1[i])
            return 1;
    for (int i = 0; i < N_IQK; i++)
        if (p == g->pipe_iqk[i] || p == g->pp1_iqk[i])
            return 1;
    return 0;
}

/* A product the fast prompt's shaders take. */
static int fast_type(const struct janas_matvec_task *t)
{
    return t->type == JANAS_Q4_K || t->type == JANAS_Q6_K ||
           t->type == JANAS_Q8_0 || t->type == JANAS_Q5_K ||
           t->type == JANAS_Q3_K || t->type == JANAS_IQ4_XS ||
           t->type == JANAS_IQ3_S || t->type == JANAS_IQ4_NL ||
           t->type == JANAS_MXFP4 || t->type == JANAS_Q5_0 ||
           t->type == JANAS_Q2_0 || iqk_index(t->type) >= 0 ||
           (t->type == JANAS_Q6_K_P && plane_level(t) == 3);
}

/* The fast prompt's tiles for the types in blocks of 32 or 64
   (iq4nl_mmq.comp, which takes rows ending inside a group too), or
   none. */
static VkPipeline b32_mm_pipe(const struct janas_gpu *g, int type)
{
    return type == JANAS_IQ4_NL  ? g->pipe_mmn4
           : type == JANAS_MXFP4 ? g->pipe_mmmx
           : type == JANAS_Q5_0  ? g->pipe_mm50
           : type == JANAS_Q2_0  ? g->pipe_mm20
                                 : VK_NULL_HANDLE;
}

int janas_g_record_task(struct janas_gpu *g, VkPipeline *bound,
                        const struct janas_matvec_task *t, size_t rows,
                        VkDeviceAddress xa, VkDeviceAddress ya, size_t ys)
{
    VkPipeline want = t->type == JANAS_Q6_K     ? g->pipe6
                      : t->type == JANAS_Q6_K_P ? g->pipe6p
                      : t->type == JANAS_Q5_K   ? g->pipe5
                      : t->type == JANAS_Q8_0   ? g->pipe80
                      : t->type == JANAS_IQ4_XS ? g->pipex4
                      : t->type == JANAS_Q3_K   ? g->pipe3
                      : t->type == JANAS_IQ3_S  ? g->pipei3
                      : b32_kind(t->type) >= 0  ? g->pipe32[b32_kind(t->type)]
                                                : g->pipe;
    size_t nb = t->cols / JANAS_QK, nv = t->n_vec ? t->n_vec : 1;
    /* blocks of 32 past the whole groups (a row ending inside one): the
       activations' stride counts that group whole */
    uint32_t tail = (uint32_t)(t->cols % JANAS_QK / 32);
    size_t xnb = nb + (tail != 0);
    /* a single vector: a subgroup per row where there is such a shader
       (rows_wg rows a workgroup), else the shader made for one vector */
    uint32_t nvg = g->nvg, rows_wg = ROWS_PER_GROUP;
    int pk = prod_index(t->type);
    /* up to row_nv vectors (JANAS_GPU_ROW_NV, 0 to 8; 4 by default) a row
       shader takes them, its variant for 2, 4 or 8 (kv), where the
       shaders reading a row per invocation are slower; more, those */
    int row_nv = g->row_nv;
    int kv = nv <= 2 ? 0 : nv <= 4 ? 1 : 2;
    const struct region *rg = janas_g_find_region(g, t->w, read_bytes(t, rows));
    if (!rg)
        return -1;
    VkDeviceAddress wa =
        rg->addr + (VkDeviceAddress)((const uint8_t *)t->w - rg->host);
    int ik = iqk_index(t->type);
    if (ik >= 0 && nv >= JANAS_GPU_FAST_N && mm_on() && g->pipe_iqq[ik]) {
        /* the fast prompt's tiles (iqk_mmq.comp) */
        want = g->pipe_iqq[ik];
        nvg = 64;
        rows_wg = 64;
    } else if (ik >= 0) {
        /* a row an invocation, for groups of vectors or for one */
        if (!g->pipe_iqk[ik])
            return -1;
        want = nv == 1 && g->pp1_iqk[ik] ? g->pp1_iqk[ik] : g->pipe_iqk[ik];
        nvg = nv == 1 && g->pp1_iqk[ik] ? 1 : g->nvg;
    } else if (nv >= JANAS_GPU_FAST_N && mm_on() && b32_mm_pipe(g, t->type)) {
        /* the fast prompt's tiles, rows ending inside a group too */
        want = b32_mm_pipe(g, t->type);
        nvg = 64;
        rows_wg = 64;
    } else if (tail) {
        /* else only b32_matvec.comp and q8_0_matvec.comp sum a tail
           (janas_gpu_can) */
        want = nv == 1 && pk >= 0 && g->pp1[pk] ? g->pp1[pk] : want;
        nvg = nv == 1 ? 1 : nvg;
    } else if (t->type == JANAS_Q4_K_T || t->type == JANAS_Q6_K_AT) {
        VkPipeline rp = g->rp_tile[t->type == JANAS_Q6_K_AT];
        if (nv != 1 || !rp)
            return -1;
        want = rp;
        nvg = 1;
        rows_wg = 64 / g->sg;
    } else if (t->type == JANAS_Q6_K_A) {
        /* the aligned copy: its row shaders, eight vectors at a time */
        VkPipeline rp = nv == 1 ? g->rp_q6ka : g->rpv[kv][3];
        if (!rp)
            return -1;
        want = rp;
        nvg = nv == 1 ? 1 : 2u << kv;
        rows_wg = 64 / g->sg;
    } else if (nv > 1 && nv <= 8 && mvd_pipe(g, t->type) && mvd_on(g) &&
               in_4g(wa, read_bytes(t, rows)) &&
               in_4g(xa, nv * nb * GPU_BLOCK)) {
        want = mvd_pipe(g, t->type);
        nvg = 8;
        rows_wg = MVD_ROWS;
    } else if (t->type == JANAS_Q4_K && nv > 1 && nv < JANAS_GPU_FAST_N &&
               g->pipe_mv && mv_on(g) && in_4g(wa, read_bytes(t, rows)) &&
               in_4g(xa, nv * nb * GPU_BLOCK)) {
        want = g->pipe_mv;
        nvg = MV_NV;
    } else if (nv == 1 && pk >= 0 && g->rp[pk] && nb <= 128 && g->row_on) {
        want = g->rp[pk];
        nvg = 1;
        rows_wg = 64 / g->sg; /* a subgroup per row */
    } else if (nv > 1 && (int)nv <= row_nv && pk >= 0 && pk < 3 &&
               g->rpv[kv][pk] && nb <= 128 && g->row_on) {
        want = g->rpv[kv][pk];
        nvg = 2u << kv;
        rows_wg = 64 / g->sg;
    } else if (nv == 1 && pk >= 0 && g->pp1[pk]) {
        want = g->pp1[pk];
        nvg = 1;
    } else if (t->type == JANAS_Q6_K_P && plane_level(t) == 3 &&
               nv >= JANAS_GPU_FAST_N && g->pipe_mm6 && mm_on()) {
        want = g->pipe_mm6;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_Q6_K && nv >= JANAS_GPU_FAST_N &&
               g->pipe_mm6s && mm_on()) {
        want = g->pipe_mm6s;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_Q3_K && nv >= JANAS_GPU_FAST_N && g->pipe_mm3 &&
               mm_on()) {
        want = g->pipe_mm3;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_IQ4_XS && nv >= JANAS_GPU_FAST_N &&
               g->pipe_mmx4 && mm_on()) {
        want = g->pipe_mmx4;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_IQ3_S && nv >= JANAS_GPU_FAST_N &&
               g->pipe_mmi3 && mm_on()) {
        want = g->pipe_mmi3;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_IQ4_NL && nv >= JANAS_GPU_FAST_N &&
               g->pipe_mmn4 && mm_on()) {
        want = g->pipe_mmn4;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_MXFP4 && nv >= JANAS_GPU_FAST_N &&
               g->pipe_mmmx && mm_on()) {
        want = g->pipe_mmmx;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_Q5_K && nv >= JANAS_GPU_FAST_N && g->pipe_mm5 &&
               mm_on()) {
        want = g->pipe_mm5;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_Q8_0 && nv >= JANAS_GPU_FAST_N &&
               g->pipe_mm80 && mm_on()) {
        want = g->pipe_mm80;
        nvg = 64;
        rows_wg = 64;
    } else if (t->type == JANAS_Q4_K && nv >= JANAS_GPU_FAST_N && g->pipe_mm &&
               mm_on()) {
        /* the fast prompt: tiles of 64 rows by 64 vectors */
        want = g->pipe_mm;
        nvg = 64;
        rows_wg = 64;
    }
    if (want != *bound) {
        g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, want);
        *bound = want;
    }
    g->last_pipe = want;
    struct push pc = {.rows = (uint32_t)rows,
                      .nb = (uint32_t)nb,
                      .nv = (uint32_t)nv,
                      .xs = (uint32_t)xnb,
                      .ys = (uint32_t)ys,
                      .tail = tail};
    janas_g_push_addr(pc.w, wa);
    janas_g_push_addr(pc.x, xa);
    janas_g_push_addr(pc.y, ya);
    if (t->type == JANAS_Q6_K_P) {
        pc.level = (uint32_t)plane_level(t);
        for (int q = 0; q < (int)pc.level - 1; q++) {
            const struct region *pr = plane_region(g, t->plane[q], rows, nb);
            if (!pr)
                return -1;
            janas_g_push_addr(
                q == 0 ? pc.p1 : pc.p0,
                pr->addr +
                    (VkDeviceAddress)((const uint8_t *)t->plane[q] - pr->host));
        }
    }
    g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pc), &pc);
    /* the shaders of a row an invocation take the vectors' groups on x and
       the rows on y: the groups of the same rows go out together, and those
       after the first find the weights in the L3 (Q4_K 17408 x 5120, 16
       vectors: 1.24 ms instead of 2.06, 13 Oct 2026); the others, rows on x */
    uint32_t gr = (uint32_t)((rows + rows_wg - 1) / rows_wg),
             gv = (uint32_t)((nv + nvg - 1) / nvg);
    if (vectors_first(g, want))
        g->vkCmdDispatch(g->cb, gv, gr, 1);
    else
        g->vkCmdDispatch(g->cb, gr, gv, 1);
    return 0;
}

/* A product of one weight matrix (rows x cols of type) with one vector. */
static int record_product(struct janas_gpu *g, VkPipeline *bound, int type,
                          const void *w, size_t rows, size_t cols,
                          VkDeviceAddress xa, VkDeviceAddress ya)
{
    struct janas_matvec_task t = {
        .type = type, .w = w, .rows = rows, .cols = cols, .n_vec = 1};
    const struct region *rg = janas_g_find_region(g, w, read_bytes(&t, rows));
    if (!rg || (type != JANAS_Q4_K && type != JANAS_Q6_K))
        return -1;
    VkPipeline want = type == JANAS_Q6_K ? g->pipe6 : g->pipe;
    if (want != *bound) {
        g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, want);
        *bound = want;
    }
    VkDeviceAddress wa =
        rg->addr + (VkDeviceAddress)((const uint8_t *)w - rg->host);
    size_t nb = cols / JANAS_QK;
    struct push pc = {.rows = (uint32_t)rows,
                      .nb = (uint32_t)nb,
                      .nv = 1,
                      .xs = (uint32_t)nb,
                      .ys = (uint32_t)rows};
    janas_g_push_addr(pc.w, wa);
    janas_g_push_addr(pc.x, xa);
    janas_g_push_addr(pc.y, ya);
    g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pc), &pc);
    /* q4k_matvec and q6k_matvec: the rows on workgroup y */
    g->vkCmdDispatch(
        g->cb, 1, (uint32_t)((rows + ROWS_PER_GROUP - 1) / ROWS_PER_GROUP), 1);
    return 0;
}

void janas_g_compute_barrier(struct janas_gpu *g)
{
    VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
                          .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
                          .dstAccessMask = VK_ACCESS_SHADER_READ_BIT};
    g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                            VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &mb, 0,
                            NULL, 0, NULL);
}

int janas_gpu_ffn_begin(struct janas_gpu *g, const struct janas_gpu_expert *e,
                        size_t n, const struct janas_block_q8k *x, size_t d,
                        size_t ff, float *out)
{
    if (g->busy || g->broken || n == 0 || d % JANAS_QK || ff % JANAS_QK)
        return -1;
    size_t nbx = d / JANAS_QK, nbh = ff / JANAS_QK;
    /* X stage: x, then every expert's h; Y stage: gate, up, down */
    size_t hoff = (nbx * GPU_BLOCK + 255) / 256 * 256;
    size_t up_off = n * ff * sizeof(float), down_off = 2 * up_off;
    if (hoff + n * nbh * GPU_BLOCK > X_STAGE_BYTES ||
        down_off + n * d * sizeof(float) > Y_STAGE_BYTES)
        return -1;
    convert_x(g->xs.map, x, nbx, 1, nbx, 0);
    if (g->vkResetCommandBuffer(g->cb, 0) != VK_SUCCESS)
        return -1;
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
    if (g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)
        return -1;
    VkPipeline bound = VK_NULL_HANDLE;
    VkDeviceAddress X = g->xs.addr, Y = g->ys.addr;
    for (size_t i = 0; i < n; i++)
        if (record_product(g, &bound, e[i].gate_type, e[i].gate, ff, d, X,
                           Y + i * ff * sizeof(float)) != 0 ||
            record_product(g, &bound, e[i].up_type, e[i].up, ff, d, X,
                           Y + up_off + i * ff * sizeof(float)) != 0)
            return -1;
    janas_g_compute_barrier(g);
    g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, g->pipea);
    bound = g->pipea;
    struct push_act pa = {.blocks = (uint32_t)(n * nbh)};
    janas_g_push_addr(pa.g, Y);
    janas_g_push_addr(pa.u, Y + up_off);
    janas_g_push_addr(pa.x, X + hoff);
    g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pa), &pa);
    g->vkCmdDispatch(g->cb, (uint32_t)(n * nbh), 1, 1);
    janas_g_compute_barrier(g);
    for (size_t i = 0; i < n; i++)
        if (record_product(g, &bound, e[i].down_type, e[i].down, d, ff,
                           X + hoff + i * nbh * GPU_BLOCK,
                           Y + down_off + i * d * sizeof(float)) != 0)
            return -1;
    VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
                          .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
                          .dstAccessMask = VK_ACCESS_HOST_READ_BIT};
    g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                            VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,
                            NULL);
    if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)
        return -1;
    if (!g->xs.coherent) {
        VkMappedMemoryRange mr = {
            .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
            .memory = g->xs.mem,
            .size = (hoff + g->atom - 1) / g->atom * g->atom};
        g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);
    }
    g->n_pend = 1;
    g->pend[0] = (struct pending){
        .y = out, .rows = n * d, .nv = 1, .y_stride = n * d, .off = down_off};
    g->y_used = down_off + n * d * sizeof(float);
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->cb};
    pthread_mutex_lock(&g->qlock);
    VkResult sr = g->vkQueueSubmit(g->queue, 1, &sub, g->fence);
    pthread_mutex_unlock(&g->qlock);
    if (sr != VK_SUCCESS) {
        g->broken = 1;
        g->n_pend = 0;
        return -1;
    }
    g->busy = 1;
    return 0;
}

int janas_gpu_finish(struct janas_gpu *g, int *waited)
{
    *waited = 0;
    if (!g->busy)
        return g->n_pend ? -1 : 0;
    VkResult r;
    /* the CPU's share is done: wait actively, the reply comes sooner */
    double t0 = g->verbose ? now() : 0;
    if (g->block_wait) {
        /* a blocking wait: the driver raises the GPU's clock for waiters */
        if ((r = g->vkGetFenceStatus(g->dev, g->fence)) == VK_NOT_READY) {
            *waited = 1;
            r = g->vkWaitForFences(g->dev, 1, &g->fence, VK_TRUE, UINT64_MAX);
        }
    } else {
        while ((r = g->vkGetFenceStatus(g->dev, g->fence)) == VK_NOT_READY) {
            *waited = 1;
            gpu_relax();
        }
    }
    g->t_waited = g->verbose ? now() - t0 : 0;
    g->busy = 0;
    if (r != VK_SUCCESS) {
        g->broken = 1; /* device lost: the caller computes these rows */
        return -1;
    }
    g->vkResetFences(g->dev, 1, &g->fence);
    if (!g->ys.coherent) {
        /* only the bytes written: invalidating costs per cache line */
        VkDeviceSize sz = (g->y_used + g->atom - 1) / g->atom * g->atom;
        VkMappedMemoryRange mr = {
            .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
            .memory = g->ys.mem,
            .size = sz < g->ys.bytes ? sz : VK_WHOLE_SIZE};
        g->vkInvalidateMappedMemoryRanges(g->dev, 1, &mr);
    }
    for (size_t i = 0; i < g->n_pend; i++) {
        const struct pending *p = &g->pend[i];
        const float *src = (const float *)(g->ys.map + p->off);
        for (size_t v = 0; v < p->nv; v++)
            memcpy(p->y + v * p->y_stride, src + v * p->rows,
                   p->rows * sizeof(float));
    }
    g->n_pend = 0;
    return 0;
}

static struct share *find_share(struct janas_gpu *g, uint64_t key)
{
    size_t i = (size_t)(key * 0x9e3779b97f4a7c15ull >> 58) % SHARE_SLOTS;
    for (size_t k = 0; k < SHARE_SLOTS; k++, i = (i + 1) % SHARE_SLOTS) {
        if (g->share[i].key == key)
            return &g->share[i];
        if (g->share[i].calls == 0) {
            g->share[i] = (struct share){.key = key, .f = 0.4f};
            return &g->share[i];
        }
    }
    return NULL;
}

/*
 * The fast prompt for a mixture's experts: a group of products of one type
 * and shape (the experts of a block: some 16 tokens each of 256 on
 * average, from 1 to 228 seen on Qwen3-30B-A3B) in one dispatch (q4k_mmid.comp,
 * q6kp_mmid.comp), their activations - one array, the pairs of the block in
 * order - converted once, every output written where the engine reads it. Such
 * a group asks too much of janas_gpu_begin (more than MAX_GROUP products, more
 * than MAX_XCACHE activations): a block's experts never reached the GPU before.
 * The GPU takes a fixed share of the tokens (experts_pick), the CPU the
 * rest meanwhile. 0, or -1 if the group is not of that kind (the caller
 * goes on as before).
 */
/* the fewest tokens an expert needs to be given to the GPU, and the
   share of a group's tokens it is given (experts_pick) */
#define EXPERT_GPU_MIN 1
#define EXPERT_GPU_SHARE 0.6

/* Which experts of a group go to the GPU (on[i] = 1): those with the most
   tokens first, at least JANAS_GPU_EXPERT_MIN, until the GPU has
   JANAS_GPU_EXPERT_SHARE of the group's tokens; the CPU computes the
   others meanwhile. A fixed share, so that the results do not depend on
   timings. On Qwen3-30B-A3B, blocks of 256 tokens, development laptop
   (9 Oct 2026), token/s: 75 with the GPU taking the experts of 32 tokens
   and more, 82 at a share of 0.3, 93 at 0.5, 99 at 0.6, 93 at 0.7, 86 at
   0.8; in blocks of 512 (janas_llm_prefill_block), a prompt of 1024
   tokens: 112.7 at 0.4, 122.7 at 0.5, 119.2 at 0.6, 112.2 at 0.7; with
   the shaders reading each word once (10 Oct): 124.8 at 0.5, 131.3-133.4
   at 0.6, 125.7 at 0.7. The largest token count, 0 if none. */
static size_t experts_pick(const struct janas_matvec_task *t, size_t n,
                           unsigned char *on)
{
    static size_t min_nv;
    static double share;
    if (!min_nv) {
        const char *e = getenv("JANAS_GPU_EXPERT_MIN");
        min_nv = e && atoi(e) > 0 ? (size_t)atoi(e) : EXPERT_GPU_MIN;
        e = getenv("JANAS_GPU_EXPERT_SHARE");
        share = e ? atof(e) : EXPERT_GPU_SHARE;
    }
    size_t total = 0, given = 0, maxnv = 0;
    for (size_t i = 0; i < n; i++) {
        on[i] = 0;
        total += t[i].n_vec ? t[i].n_vec : 1;
    }
    for (;;) {
        size_t best = n, bnv = 0;
        for (size_t i = 0; i < n; i++) {
            size_t nv = t[i].n_vec ? t[i].n_vec : 1;
            if (!on[i] && nv >= min_nv && nv > bnv) {
                best = i;
                bnv = nv;
            }
        }
        if (best == n || (double)(given + bnv) > share * (double)total + 0.5)
            break;
        on[best] = 1;
        given += bnv;
        if (bnv > maxnv)
            maxnv = bnv;
    }
    return maxnv;
}

/* A group experts_id leaves to the CPU, and why, with JANAS_GPU_VERBOSE=2:
   -1. */
static int experts_off(const struct janas_gpu *g, size_t n, int type,
                       const char *why)
{
    if (g->verbose > 1)
        fprintf(stderr,
                "experts_id: a group of %zu (type %d first) not taken: %s\n", n,
                type, why);
    return -1;
}

/* JANAS_GPU_REPEATABLE=1: a mixture's experts never on the fast prompt's
   shaders. A group goes to them only when all its experts are in the memory
   given to the GPU, which changes as the cache fills (and the GPU gets the
   cache after the first pass): the same prompt read twice had some groups
   rounded on the GPU and others exact on the CPU, and Qwen3.8-Flash-Next's
   reply to it changed from one run to the next (22 Oct 2026). With it the
   experts stay exact on the CPU, the dense products (weights that never
   move) on the GPU: a prompt gives the same results every time. */
int janas_gpu_repeatable(void)
{
    static int on = -1;
    if (on < 0) {
        const char *e = getenv("JANAS_GPU_REPEATABLE");
        on = e && atoi(e) > 0;
    }
    return on;
}

static int experts_id(struct janas_gpu *g, struct janas_pool *pool,
                      const struct janas_matvec_task *t, size_t n)
{
    /* not in a chain of drafts to verify (janas_gpu_set_pass): with
       Qwen3.8-Flash-Next's ten experts a token, a chain of 13 made the 128
       tokens below, and its text was not the same as without drafts */
    if (!mm_on() || g->busy || g->broken || n < 2 || !g->pipe_id4 ||
        g->pass_n < JANAS_GPU_FAST_N || janas_gpu_repeatable())
        return -1;
    /* a group may open with the layer's shared expert (model_ffn.c: its gate
       and up, or its down) in a type of its own - Q8_0 beside the routed
       experts' Q4_K or MXFP4 on Qwen3.6-35B-A3B, a plain Q6_K down beside
       their Q6_K_P ones on Qwen3-Next - which left the whole group to the
       CPU: those first s0 products the CPU computes now, as it does the
       experts the GPU is not given, and the routed experts go to the GPU */
    size_t s0 = 0;
    while (s0 < 2 && s0 + 2 <= n && t[s0].type != t[n - 1].type)
        s0++;
    int type = t[s0].type;
    if (n * 48 > g->ts.bytes || n > 2 * 128 + 8)
        return experts_off(g, n, type, "more products than its table");
    /* b32_mmid.comp: IQ4_NL, Q5_0, Q2_0 (pipe_idb 0, 1, 2); iqk_mmid.comp:
       Q2_K, the IQ1, IQ2 and IQ3 kinds, IQ4_XS */
    int ib = type == JANAS_IQ4_NL ? 0
             : type == JANAS_Q5_0 ? 1
             : type == JANAS_Q2_0 ? 2
                                  : -1;
    int im = iqm_kind(type);
    if (!(type == JANAS_Q4_K || type == JANAS_Q6_K ||
          (type == JANAS_MXFP4 && g->pipe_idmx) ||
          (type == JANAS_Q3_K && g->pipe_id3) || (ib >= 0 && g->pipe_idb[ib]) ||
          (im >= 0 && g->pipe_iqm[im]) ||
          (type == JANAS_Q6_K_P && plane_level(&t[s0]) == 3)))
        return experts_off(g, n, type, "a type without its shader");
    const struct janas_matvec_task *t0 = &t[s0];
    /* rows ending inside a group of 256 (qwen4exp's down matrices, 640
       wide): b32_mmid.comp's types only, their activations' last group
       whole */
    if (t0->cols % JANAS_QK && ib < 0)
        return experts_off(g, n, type, "rows ending inside a group of 256");
    size_t nb = (t0->cols + JANAS_QK - 1) / JANAS_QK,
           xs = t0->x_stride ? t0->x_stride : nb;
    const struct janas_block_q8k *x0 = t0->x, *x1 = t0->x;
    for (size_t i = s0; i < n; i++) {
        size_t nv = t[i].n_vec ? t[i].n_vec : 1;
        if (t[i].type != type || t[i].rows != t0->rows ||
            t[i].cols != t0->cols ||
            (t[i].x_stride ? t[i].x_stride : nb) != xs ||
            !janas_gpu_can(g, &t[i])) {
            if (g->verbose > 1)
                fprintf(stderr,
                        "experts_id: product %zu of %zu not taken "
                        "(type %d, nv %zu, can %d)\n",
                        i, n, t[i].type, nv, janas_gpu_can(g, &t[i]));
            return -1;
        }
        if (t[i].x < x0)
            x0 = t[i].x;
        if (t[i].x + nv * xs > x1)
            x1 = t[i].x + nv * xs;
    }
    /* experts: tokens of their own each. A dense feed-forward's gate and up
       (one "expert" in the cache, Qwen3-4B) read the same activations, and
       go whole to the GPU as before - shared, the CPU got half of them and
       the 4B read a prompt at 111 tokens/s instead of 127 (9 Oct 2026) */
    int own = 0;
    for (size_t i = s0 + 1; i < n && !own; i++)
        own = t[i].x != t0->x;
    if (!own)
        return experts_off(g, n, type, "no expert with tokens of its own");
    /* a prompt's block only: generating, each expert has a token or a
       few, and the GPU's share cost the 30B three quarters of its speed */
    size_t tokens = 0;
    for (size_t i = s0; i < n; i++)
        tokens += t[i].n_vec ? t[i].n_vec : 1;
    if (tokens < JANAS_GPU_FAST_N * 2)
        return -1;
    unsigned char on[2 * 128 + 8] = {0};
    size_t maxnv = experts_pick(t0, n - s0, on + s0);
    if (maxnv == 0) /* no expert worth the GPU: the CPU's as before */
        return experts_off(g, n, type, "no expert worth the GPU");
    /* the activations: one array of vectors xs blocks apart */
    size_t span = (size_t)(x1 - x0);
    if (span % xs != 0 || span / xs * nb * GPU_BLOCK > X_STAGE_BYTES)
        return experts_off(g, n, type, "activations beyond the stage");
    for (size_t i = s0; i < n; i++)
        if ((size_t)(t[i].x - x0) % xs != 0)
            return experts_off(g, n, type, "activations off the stride");
    /* the table: weights, activations, outputs (shared with the GPU) */
    uint32_t *tb = (uint32_t *)g->ts.map;
    size_t ng = 0;
    for (size_t i = 0; i < n; i++) {
        size_t nv = t[i].n_vec ? t[i].n_vec : 1;
        if (!on[i])
            continue;
        size_t ys = t[i].y_stride ? t[i].y_stride : t[i].rows;
        const struct region *wr =
            janas_g_find_region(g, t[i].w, read_bytes(&t[i], t[i].rows));
        const struct region *yr = janas_g_find_region(
            g, t[i].y, ((nv - 1) * ys + t[i].rows) * sizeof(float));
        if (!wr || !yr || !yr->host_shared) /* outputs written in place */
            return experts_off(g, n, type, "weights or outputs out of reach");
        uint32_t *e = tb + 12 * ng++;
        janas_g_push_addr(
            e,
            wr->addr + (VkDeviceAddress)((const uint8_t *)t[i].w - wr->host));
        janas_g_push_addr(e + 2,
                          g->xs.addr + (VkDeviceAddress)((size_t)(t[i].x - x0) /
                                                         xs * nb * GPU_BLOCK));
        janas_g_push_addr(
            e + 4,
            yr->addr + (VkDeviceAddress)((const uint8_t *)t[i].y - yr->host));
        e[6] = (uint32_t)nv;
        e[7] = (uint32_t)ys;
        e[8] = e[9] = e[10] = e[11] = 0;
        if (type == JANAS_Q6_K_P)
            for (int q = 0; q < 2; q++) {
                const struct region *pr =
                    plane_region(g, t[i].plane[q], t[i].rows, nb);
                if (!pr)
                    return experts_off(g, n, type, "a plane out of reach");
                janas_g_push_addr(
                    e + (q == 0 ? 8 : 10),
                    pr->addr +
                        (VkDeviceAddress)((const uint8_t *)t[i].plane[q] -
                                          pr->host));
            }
    }
    double tc = g->verbose ? now() : 0;
    /* the workgroups' tiles: (expert, first vector) for each 16 vectors of
       each expert, so that none is launched for nothing - with a grid as
       tall as the busiest expert's tiles, most of a block's experts (some
       16 tokens each) had a dozen empty workgroups each */
    size_t nt = 0;
    for (size_t k = 0; k < ng; k++)
        for (uint32_t v = 0; v < tb[12 * k + 6]; v += 16)
            nt++;
    if ((3 * ng + nt) * 16 > g->ts.bytes)
        return experts_off(g, n, type, "more tiles than its table");
    for (size_t k = 0, at = 3 * ng * 4; k < ng; k++)
        for (uint32_t v = 0; v < tb[12 * k + 6]; v += 16, at += 4) {
            tb[at] = (uint32_t)k;
            tb[at + 1] = v;
            tb[at + 2] = tb[at + 3] = 0;
        }
    convert_x(g->xs.map, x0, xs, span / xs, nb, janas_g_x_in_32(type));
    if (g->verbose)
        g->t_conv += now() - tc;
    struct stage *fl[2] = {&g->xs, &g->ts};
    size_t used[2] = {span / xs * nb * GPU_BLOCK, (3 * ng + nt) * 16};
    for (int k = 0; k < 2; k++)
        if (!fl[k]->coherent) {
            VkDeviceSize sz = (used[k] + g->atom - 1) / g->atom * g->atom;
            VkMappedMemoryRange mr = {
                .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
                .memory = fl[k]->mem,
                .size = sz < fl[k]->bytes ? sz : VK_WHOLE_SIZE};
            g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);
        }
    if (g->vkResetCommandBuffer(g->cb, 0) != VK_SUCCESS)
        return -1;
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
    if (g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)
        return -1;
    g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE,
                         ib >= 0               ? g->pipe_idb[ib]
                         : im >= 0             ? g->pipe_iqm[im]
                         : type == JANAS_Q4_K  ? g->pipe_id4
                         : type == JANAS_Q6_K  ? g->pipe_id6s
                         : type == JANAS_MXFP4 ? g->pipe_idmx
                         : type == JANAS_Q3_K  ? g->pipe_id3
                                               : g->pipe_id6);
    /* b32_mmid.comp and iqk_mmid.comp also take the groups of 32 a row
       (pc[5]) */
    uint32_t pc[6];
    janas_g_push_addr(pc, g->ts.addr);
    pc[2] = (uint32_t)t0->rows;
    pc[3] = (uint32_t)nb;
    pc[4] = (uint32_t)(3 * ng);
    pc[5] = (uint32_t)(t0->cols / 32);
    g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pc), pc);
    g->vkCmdDispatch(g->cb, (uint32_t)((t0->rows + 63) / 64), 1, (uint32_t)nt);
    VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
                          .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
                          .dstAccessMask = VK_ACCESS_HOST_READ_BIT};
    g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                            VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,
                            NULL);
    if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)
        return -1;
    g->n_pend = 0;
    g->y_used = 0;
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->cb};
    pthread_mutex_lock(&g->qlock);
    VkResult sr = g->vkQueueSubmit(g->queue, 1, &sub, g->fence);
    pthread_mutex_unlock(&g->qlock);
    if (sr != VK_SUCCESS) {
        g->broken = 1;
        return -1;
    }
    g->busy = 1;
    /* the experts with few tokens on the CPU meanwhile */
    struct janas_matvec_task cpu[2 * 128 + 8];
    size_t nc = 0;
    for (size_t i = 0; i < n; i++)
        if (!on[i] && nc < sizeof(cpu) / sizeof(*cpu))
            cpu[nc++] = t[i];
    if (nc > 0)
        janas_matvec_q4k_group(pool, cpu, nc);
    int waited;
    if (janas_gpu_finish(g, &waited) != 0) {
        /* the GPU failed: all on the CPU (the outputs are overwritten) */
        janas_matvec_q4k_group(pool, t, n);
    }
    return 0;
}

int janas_gpu_dense_mm(struct janas_gpu *g, int bf16, const void *w,
                       const float *x, float *y, size_t rows, size_t cols,
                       size_t n, float scale)
{
    if (!g || g->busy || g->broken || !mm_on() || n < JANAS_GPU_FAST_N ||
        cols % 16 || !g->pipe_dm[bf16 != 0])
        return -1;
    const struct region *wr =
        janas_g_find_region(g, w, rows * cols * (bf16 ? 2 : 4));
    const struct region *xr = janas_g_find_region(g, x, n * cols * 4);
    const struct region *yr = janas_g_find_region(g, y, n * rows * 4);
    /* x read and y written in place: the host's own memory only */
    if (!wr || !xr || !yr || !xr->host_shared || !yr->host_shared)
        return -1;
    if (g->vkResetCommandBuffer(g->cb, 0) != VK_SUCCESS)
        return -1;
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
    if (g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)
        return -1;
    g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE,
                         g->pipe_dm[bf16 != 0]);
    struct {
        uint32_t w[2], x[2], y[2];
        uint32_t rows, cols, nv, ys;
        float scale;
    } pc = {.rows = (uint32_t)rows,
            .cols = (uint32_t)cols,
            .nv = (uint32_t)n,
            .ys = (uint32_t)rows,
            .scale = scale};
    janas_g_push_addr(
        pc.w, wr->addr + (VkDeviceAddress)((const uint8_t *)w - wr->host));
    janas_g_push_addr(
        pc.x, xr->addr + (VkDeviceAddress)((const uint8_t *)x - xr->host));
    janas_g_push_addr(
        pc.y, yr->addr + (VkDeviceAddress)((const uint8_t *)y - yr->host));
    g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pc), &pc);
    g->vkCmdDispatch(g->cb, (uint32_t)((rows + 63) / 64),
                     (uint32_t)((n + 63) / 64), 1);
    VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
                          .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
                          .dstAccessMask = VK_ACCESS_HOST_READ_BIT};
    g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                            VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,
                            NULL);
    if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)
        return -1;
    g->n_pend = 0;
    g->y_used = 0;
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->cb};
    pthread_mutex_lock(&g->qlock);
    VkResult sr = g->vkQueueSubmit(g->queue, 1, &sub, g->fence);
    pthread_mutex_unlock(&g->qlock);
    if (sr != VK_SUCCESS) {
        g->broken = 1;
        return -1;
    }
    g->busy = 1;
    int waited;
    return janas_gpu_finish(g, &waited) == 0 ? 0 : -1;
}

int janas_gpu_bf16_mv(struct janas_gpu *g, const uint16_t *w, const float *x,
                      float *y, size_t rows, size_t cols, size_t n, float scale)
{
    /* not in a pass the tuner gave the CPU (janas_gpu_set_only_fast) */
    if (!g || g->busy || g->broken || g->only_fast || !g->pipe_b16 || n == 0 ||
        rows == 0 || cols == 0 || n < g->min_n)
        return -1;
    const struct region *wr = janas_g_find_region(g, w, rows * cols * 2);
    if (!wr || (uintptr_t)w % 4 != 0)
        return -1;
    /* x read and y written in place where the GPU shares them with the
       host, else through the stages (a card's own memory) */
    size_t xb = n * cols * sizeof(float), yb = n * rows * sizeof(float);
    const struct region *xr = janas_g_find_region(g, x, xb),
                        *yr = janas_g_find_region(g, y, yb);
    int xin = xr && xr->host_shared, yin = yr && yr->host_shared;
    if ((!xin && xb > X_STAGE_BYTES) || (!yin && yb > Y_STAGE_BYTES))
        return -1;
    if (!xin) {
        memcpy(g->xs.map, x, xb);
        if (!g->xs.coherent) {
            VkDeviceSize sz = (xb + g->atom - 1) / g->atom * g->atom;
            VkMappedMemoryRange mr = {
                .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
                .memory = g->xs.mem,
                .size = sz < g->xs.bytes ? sz : VK_WHOLE_SIZE};
            g->vkFlushMappedMemoryRanges(g->dev, 1, &mr);
        }
    }
    if (g->vkResetCommandBuffer(g->cb, 0) != VK_SUCCESS)
        return -1;
    VkCommandBufferBeginInfo bb = {
        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT};
    if (g->vkBeginCommandBuffer(g->cb, &bb) != VK_SUCCESS)
        return -1;
    g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, g->pipe_b16);
    struct {
        uint32_t w[2], x[2], y[2];
        uint32_t rows, cols, nv, xs, ys;
        float scale;
    } pc = {.rows = (uint32_t)rows,
            .cols = (uint32_t)cols,
            .nv = (uint32_t)n,
            .xs = (uint32_t)cols,
            .ys = (uint32_t)rows,
            .scale = scale};
    janas_g_push_addr(
        pc.w, wr->addr + (VkDeviceAddress)((const uint8_t *)w - wr->host));
    janas_g_push_addr(
        pc.x, xin ? xr->addr + (VkDeviceAddress)((const uint8_t *)x - xr->host)
                  : g->xs.addr);
    janas_g_push_addr(
        pc.y, yin ? yr->addr + (VkDeviceAddress)((const uint8_t *)y - yr->host)
                  : g->ys.addr);
    g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
                          sizeof(pc), &pc);
    /* vectors on x, rows on y: eight rows a workgroup (64 invocations) */
    g->vkCmdDispatch(g->cb, (uint32_t)((n + B16_NV - 1) / B16_NV),
                     (uint32_t)((rows + 7) / 8), 1);
    VkMemoryBarrier mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
                          .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
                          .dstAccessMask = VK_ACCESS_HOST_READ_BIT};
    g->vkCmdPipelineBarrier(g->cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                            VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &mb, 0, NULL, 0,
                            NULL);
    if (g->vkEndCommandBuffer(g->cb) != VK_SUCCESS)
        return -1;
    /* a staged y: copied out by janas_gpu_finish */
    g->n_pend = 0;
    g->y_used = 0;
    if (!yin) {
        g->pend[0] = (struct pending){
            .y = y, .rows = rows, .nv = n, .y_stride = rows, .off = 0};
        g->n_pend = 1;
        g->y_used = yb;
    }
    VkSubmitInfo sub = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                        .commandBufferCount = 1,
                        .pCommandBuffers = &g->cb};
    pthread_mutex_lock(&g->qlock);
    VkResult sr = g->vkQueueSubmit(g->queue, 1, &sub, g->fence);
    pthread_mutex_unlock(&g->qlock);
    if (sr != VK_SUCCESS) {
        g->broken = 1;
        g->n_pend = 0;
        return -1;
    }
    g->busy = 1;
    int waited;
    if (janas_gpu_finish(g, &waited) != 0)
        return -1;
    g->b16_calls++;
    g->b16_staged += !xin || !yin;
    return 0;
}

void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,
                            const struct janas_matvec_task *tasks, size_t n)
{
    if (g && experts_id(g, pool, tasks, n) == 0)
        return;
    size_t nv = tasks[0].n_vec ? tasks[0].n_vec : 1;
    if (!g || g->broken || nv < g->min_n || n > MAX_GROUP) {
        janas_matvec_q4k_group(pool, tasks, n);
        return;
    }
    uint64_t key = (uint64_t)n << 48 ^ (uint64_t)tasks[0].rows << 24 ^
                   (uint64_t)tasks[0].cols << 8 ^ nv;
    struct share *sh = find_share(g, key);
    /* JANAS_GPU_VERBOSE=2: each shape once for every set of weight types
       it comes with (up to four; layers of the same shape can differ, as
       the down matrices of a Q4_K_M model), with whether the GPU takes
       them */
    if (g->verbose > 1 && sh) {
        uint32_t ts = 2166136261u;
        for (size_t i = 0; i < n; i++)
            ts = (ts ^ (uint32_t)tasks[i].type) * 16777619u;
        ts |= 1; /* never 0, which marks a free entry */
        int k = 0;
        while (k < 4 && sh->seen[k] && sh->seen[k] != ts)
            k++;
        if (k < 4 && !sh->seen[k]) {
            sh->seen[k] = ts;
            fprintf(stderr, "gpu shape %016llx:", (unsigned long long)key);
            for (size_t i = 0; i < n; i++)
                fprintf(stderr, " type %d %zux%zu%s", tasks[i].type,
                        tasks[i].rows, tasks[i].cols,
                        janas_gpu_can(g, &tasks[i]) ? "" : " (CPU)");
            fprintf(stderr, "\n");
        }
    }
    /* the fast prompt: its products on the GPU, whole, always */
    int fast = mm_on() && nv >= JANAS_GPU_FAST_N;
    if (sh && sh->off > 0 && !fast) {
        /* the GPU was late even with the smallest share: CPU for a while */
        sh->off--;
        janas_matvec_q4k_group(pool, tasks, n);
        return;
    }
    size_t gpu_rows[MAX_GROUP], any = 0;
    struct janas_matvec_task cpu[MAX_GROUP];
    for (size_t i = 0; i < n; i++) {
        const struct janas_matvec_task *t = &tasks[i];
        size_t r = 0;
        if (g->only_fast && !(fast && fast_type(t))) {
            r = 0; /* the CPU's pass: only the fast prompt's products here */
        } else if ((nv >= g->full_n || (fast && fast_type(t))) &&
                   janas_gpu_can(g, t)) {
            r = t->rows; /* the GPU alone */
        } else if (sh && t->rows >= 2 * ROWS_PER_GROUP && janas_gpu_can(g, t)) {
            r = (size_t)(sh->f * (float)t->rows) / ROWS_PER_GROUP *
                ROWS_PER_GROUP;
            if (r > t->rows)
                r = t->rows;
        }
        gpu_rows[i] = r;
        any += r;
        size_t nb = t->cols / JANAS_QK;
        cpu[i] = *t;
        cpu[i].w =
            (const uint8_t *)t->w + r * janas_qtype_row_bytes(t->type, t->cols);
        for (int q = 0; q < 2; q++) /* Q6_K_P: its bare planes follow */
            if (t->plane[q])
                cpu[i].plane[q] =
                    (const uint8_t *)t->plane[q] + r * nb * JANAS_Q6KP_PLANE;
        cpu[i].y = t->y + r;
        cpu[i].rows = t->rows - r;
        cpu[i].y_stride = t->y_stride ? t->y_stride : t->rows;
    }
    double t0 = g->verbose ? now() : 0;
    /* experiment: the pool's idle workers asleep while submitting */
    int spin = janas_pool_get_spin(pool);
    if (g->nospin)
        janas_pool_set_spin(pool, 0);
    int rb = any == 0 ? -1 : janas_gpu_begin(g, tasks, gpu_rows, n);
    if (g->nospin)
        janas_pool_set_spin(pool, spin);
    if (rb != 0) {
        janas_matvec_q4k_group(pool, tasks, n);
        return;
    }
    /* the pool's share meanwhile (tasks left with no rows skipped) */
    double t1 = g->verbose ? now() : 0;
    size_t m = 0, all = 0;
    for (size_t i = 0; i < n; i++) {
        all += tasks[i].rows;
        if (cpu[i].rows > 0)
            cpu[m++] = cpu[i];
    }
    if (m > 0)
        janas_matvec_q4k_group(pool, cpu, m);
    double t2 = g->verbose ? now() : 0;
    int waited;
    if (janas_gpu_finish(g, &waited) != 0) {
        /* the GPU failed: its rows on the CPU, and no GPU from now on */
        for (size_t i = 0; i < n; i++) {
            cpu[i] = tasks[i];
            cpu[i].rows = gpu_rows[i];
            cpu[i].y_stride =
                tasks[i].y_stride ? tasks[i].y_stride : tasks[i].rows;
        }
        janas_matvec_q4k_group(pool, cpu, n);
        return;
    }
    if (nv >= g->full_n) {
        if (g->verbose && sh) {
            sh->calls++;
            if (any == all)
                sh->alone++;
            else
                sh->waits += (uint32_t)waited;
            sh->t_begin += t1 - t0;
            sh->t_cpu += t2 - t1;
            sh->t_wait += g->t_waited;
            sh->t_copy += now() - t2 - g->t_waited;
            sh->rows_gpu += any;
            sh->rows_all += all;
        }
        return;
    }
    if (!sh)
        return; /* the table of shares full: nothing to learn from */
    if (g->verbose) {
        sh->t_begin += t1 - t0;
        sh->t_cpu += t2 - t1;
        sh->t_wait += g->t_waited;
        sh->t_copy += now() - t2 - g->t_waited;
        sh->rows_gpu += any;
        sh->rows_all += all;
    }
    /* late GPU: less next time; early: more */
    sh->calls++;
    if (any == all)
        sh->alone++; /* reported apart; the share learns as before */
    else
        sh->waits += (uint32_t)waited;
    if (waited && sh->f <= 0.05f) {
        sh->off = 256; /* not worth it for this shape: retry later */
        return;
    }
    sh->f = waited ? sh->f * 0.95f : sh->f * 1.05f;
    if (sh->f > 0.95f)
        sh->f = 0.95f;
    if (sh->f < 0.05f)
        sh->f = 0.05f;
}

#else /* no GPU in this build */

struct janas_gpu {
    int unused;
};

struct janas_gpu *janas_gpu_create(char *err, size_t err_len)
{
#if defined(JANAS_GPU_WHY)
    snprintf(err, err_len, "built without GPU support: it needs %s",
             JANAS_GPU_WHY);
#elif !defined(JANAS_GPU_SPV)
    snprintf(err, err_len,
             "built without GPU support: glslangValidator was "
             "not installed when Janas was built");
#else
    snprintf(err, err_len,
             "built without GPU support: the Vulkan headers "
             "(vulkan/vulkan.h) were not installed when Janas "
             "was built");
#endif
    return NULL;
}

void janas_gpu_destroy(struct janas_gpu *g)
{
    (void)g;
}

/* no fast prompt without a GPU (issue #22: the link failed without them) */
int janas_gpu_fast_prompt(void)
{
    return 0;
}

void janas_gpu_set_only_fast(struct janas_gpu *g, int on)
{
    (void)g;
    (void)on;
}

void janas_gpu_set_pass(struct janas_gpu *g, uint32_t n)
{
    (void)g;
    (void)n;
}

int janas_gpu_repeatable(void)
{
    return 0;
}

int janas_gpu_import(struct janas_gpu *g, const void *p, size_t bytes)
{
    (void)g;
    (void)p;
    (void)bytes;
    return -1;
}

void *janas_gpu_alloc(struct janas_gpu *g, size_t bytes)
{
    (void)g;
    (void)bytes;
    return NULL;
}

void janas_gpu_flush(struct janas_gpu *g, const void *p, size_t bytes)
{
    (void)g;
    (void)p;
    (void)bytes;
}

int janas_gpu_can(const struct janas_gpu *g, const struct janas_matvec_task *t)
{
    (void)g;
    (void)t;
    return 0;
}

int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,
                    const size_t *gpu_rows, size_t n)
{
    (void)g;
    (void)tasks;
    (void)gpu_rows;
    (void)n;
    return -1;
}

int janas_gpu_finish(struct janas_gpu *g, int *waited)
{
    (void)g;
    *waited = 0;
    return -1;
}

const char *janas_gpu_name(const struct janas_gpu *g)
{
    (void)g;
    return "";
}

unsigned janas_gpu_subgroup(const struct janas_gpu *g)
{
    (void)g;
    return 0;
}

size_t janas_gpu_last_shader(const struct janas_gpu *g, char *buf, size_t len)
{
    (void)g;
    if (len)
        buf[0] = 0;
    return 0;
}

int janas_gpu_set_path(struct janas_gpu *g, enum janas_gpu_path path, int value)
{
    (void)g;
    (void)path;
    (void)value;
    return 0;
}

void janas_gpu_subgroup_range(const struct janas_gpu *g, unsigned *min,
                              unsigned *max)
{
    (void)g;
    *min = *max = 0;
}

const char *janas_gpu_about(const struct janas_gpu *g)
{
    (void)g;
    return "";
}

int janas_gpu_sync(struct janas_gpu *g, void *p, size_t bytes, int to_device)
{
    (void)g;
    (void)p;
    (void)bytes;
    (void)to_device;
    return -1;
}

size_t janas_gpu_min_n(const struct janas_gpu *g)
{
    (void)g;
    return (size_t)-1;
}

void janas_gpu_set_min_n(struct janas_gpu *g, size_t n)
{
    (void)g;
    (void)n;
}

size_t janas_gpu_few_max(const struct janas_gpu *g)
{
    (void)g;
    return 0;
}

int janas_gpu_bf16_mv(struct janas_gpu *g, const uint16_t *w, const float *x,
                      float *y, size_t rows, size_t cols, size_t n, float scale)
{
    (void)g;
    (void)w;
    (void)x;
    (void)y;
    (void)rows;
    (void)cols;
    (void)n;
    (void)scale;
    return -1;
}

int janas_gpu_dense_mm(struct janas_gpu *g, int bf16, const void *w,
                       const float *x, float *y, size_t rows, size_t cols,
                       size_t n, float scale)
{
    (void)g;
    (void)bf16;
    (void)w;
    (void)x;
    (void)y;
    (void)rows;
    (void)cols;
    (void)n;
    (void)scale;
    return -1;
}

void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,
                            const struct janas_matvec_task *tasks, size_t n)
{
    (void)g;
    janas_matvec_q4k_group(pool, tasks, n);
}

void janas_gpu_keep_awake(struct janas_gpu *g, int on)
{
    (void)g;
    (void)on;
}

int janas_gpu_ffn_begin(struct janas_gpu *g, const struct janas_gpu_expert *e,
                        size_t n, const struct janas_block_q8k *x, size_t d,
                        size_t ff, float *out)
{
    (void)g;
    (void)e;
    (void)n;
    (void)x;
    (void)d;
    (void)ff;
    (void)out;
    return -1;
}

#endif