Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
4.3 KB
Raw Blame
// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).
// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.
// src/ds4/ds4_vision.cuh - DeepSeek-V4.1's vision tower on one GPU (DeepSeek's inference/vision.py):
//   patches (14 x 14 RGB, scaled to [-1, 1]) -> Linear(588 -> 1024)
//   32 blocks: x += attn(rmsnorm(x)) (16 heads of 64, full bidirectional attention over the image, a 2D rotary on q
//              and k: the first half of each head's frequencies the patch row, the second its column);
//              x += w2(silu(gate) * up) with gate, up = w1(rmsnorm(x))
//   rmsnorm, then the aligner: each 3 x 3 block of patches (zero-padded at the right and bottom) as one 9216 row,
//   Linear(9216 -> 5120), GELU, Linear(5120 -> 5120): one LLM embedding row per block, in reading order.
// The BF16 weights live in pinned host memory and stream to the GPU a block at a time (two slots, the next block's
// copy under this one's work), so the tower costs the model ~350 MB of the GPU's memory, not ~1 GB; the GEMMs are
// cuBLAS (BF16 in, FP32 accumulation), the residual stream FP32.
#pragma once

#include <cstddef>
#include <cstdint>
#include <functional>
#include <string>
#include <vector>

namespace strata::ds4 {

struct VisionGeo {
    int layers = 0, dim = 1024, heads = 16, inter = 2816, patch = 14, merge = 3, out = 5120;
    float theta = 10000.0f, eps = 1e-6f;
    int max_tokens = 1024;   // an image's span in the prompt at most: n_h * (n_w + 1) + 2 tokens
};

class VisionTower {
public:
    /// read(name, shape, dst): the tensor's shape, and (dst not null) its BF16 data into dst; false: not there
    using Reader = std::function<bool(const std::string& name, std::vector<int64_t>& shape, void* dst)>;
    VisionTower() = default;
    VisionTower(const VisionTower&) = delete;
    VisionTower& operator=(const VisionTower&) = delete;
    ~VisionTower();
    bool load(const VisionGeo& g, int device, const Reader& read, std::string& err);
    /// Frees everything load took (a load that failed half way too); loaded() is false after it.
    void release();
    bool loaded() const { return dev_ >= 0; }
    int device() const { return dev_; }
    size_t device_bytes() const { return dev_bytes_; }
    /// The patches an image may have (its aligner grid within max_tokens).
    int max_patches() const { return max_n_; }
    /// rgb: height x width x 3 bytes, both sides multiples of the patch size -> the aligner's rows, BF16
    /// [n_h * n_w][out] in reading order (n_h x n_w: the patch grid over 3, rounded up).
    bool encode(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& rows, int& n_h, int& n_w,
                std::string& err);

private:
    struct Piece {             // a weight the encode streams in: host bits, its size
        const uint16_t* host = nullptr;
        size_t n = 0;
    };
    VisionGeo g_;
    int dev_ = -1, at_ = -1, max_n_ = 0, qb_ = 0, rb_ = 0;   // (at_: the device its memory is on)
    size_t dev_bytes_ = 0, slot_n_ = 0;
    std::vector<void*> dallocs_;
    uint16_t* host_ = nullptr;          // pinned: every block's [wqkv, wo, w1, w2], then aligner w1, w2
    size_t block_n_ = 0;                // a block's weights (BF16 values)
    size_t al1_off_ = 0, al2_off_ = 0;  // the aligner's weights in host_
    // resident (small): the patch embedding, the norms and biases
    uint16_t *pe_w_ = nullptr, *pe_b_ = nullptr, *norm_ = nullptr, *al1_b_ = nullptr, *al2_b_ = nullptr;
    std::vector<uint16_t*> n1_, n2_, bqkv_, bo_;
    // work buffers (max_n_ patches)
    uint8_t* rgb_ = nullptr;
    uint16_t *patch_ = nullptr, *xn_ = nullptr, *qkv_ = nullptr, *o_ = nullptr, *mid_ = nullptr, *p_ = nullptr,
             *un_ = nullptr, *a16_ = nullptr, *outb_ = nullptr;
    float *x_ = nullptr, *y_ = nullptr, *s_ = nullptr, *inv_freq_ = nullptr;
    uint16_t* slot_[2] = {nullptr, nullptr};
    void* blas_ = nullptr;              // cublasHandle_t
    uint8_t* ws_ = nullptr;
    void *st_ = nullptr, *cs_ = nullptr;   // cudaStream_t: the work, the weight copies
    void *copied_[2] = {nullptr, nullptr}, *used_[2] = {nullptr, nullptr};   // cudaEvent_t per slot
    template <class T>
    bool dalloc(T*& p, size_t n, std::string& err);
};

}  // namespace strata::ds4