Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
5.5 KB
Raw Blame
// Strata. The trellis routed experts (ds4_iq.cuh's is_tq types) on the CPU lane of the DeepSeek-V4.1 engine path: Strata's
// own decode-and-dot kernels (include/strata/kernels/cpu/tq_cpu.hpp: AVX-512 / AVX2 / scalar, one result whatever the ISA)
// over the file's rows read in place.  The activations are the device's (ds4_gguf.cu's q8_quant: int8 with an F32 scale a 32
// values), so an expert gives the same sums on either lane up to the order of the blocks' float sums.
#include "ds4_gguf_cpu.hpp"

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <string>
#include <vector>

#if defined(STRATA_DSV41_TQ_CPU)
#include "strata/kernels/cpu/tq_cpu.hpp"
#endif

namespace strata::ds4::gguf {

#if defined(STRATA_DSV41_TQ_CPU)
namespace kc = strata::kernels::cpu;
namespace {

int kbits_of(int type) {
    switch (type) {
        case 145: return 6;
        case 146: return 7;
        case 224: case 226: return 5;
        case 225: case 227: return 4;
        default: return 0;
    }
}
constexpr int kActTq = 100;   // IqCpu::gu_act / d_act of these types: the layout below

// an activation of k values: k / 32 F32 scales, then k int8
size_t act_bytes_of(int k) { return (size_t) k / 32 * 4 + (size_t) k; }

void unpack(const void* act, int k, kc::tq_detail::TqAct& a) {
    const uint8_t* p = (const uint8_t*) act;
    std::memcpy(a.d, p, (size_t) k / 32 * 4);
    const int8_t* q = (const int8_t*) (p + (size_t) k / 32 * 4);
    for (int i = 0; i < k; ++i) a.q[i] = q[i];
}

// dots of rows [r0, r1) of one matrix against nt activations: out[t][r]
void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
    static thread_local std::vector<kc::tq_detail::TqAct> unp(kc::kTqMaxTokens);
    const kc::tq_detail::TqAct* ap[kc::kTqMaxTokens];
    for (int t = 0; t < nt; ++t) {
        unpack(act[t], n, unp[(size_t) t]);
        ap[t] = &unp[(size_t) t];
    }
    const int kb = kc::tq_kbits(type);
    const int16_t* tab = kc::tq_state_table_for(type);
    switch (kc::tq_isa()) {
#if defined(__x86_64__) || defined(_M_X64)   // the AVX units exist on x86-64 builds only (arm64: the scalar rows)
        case kc::TqIsa::avx512: kc::tq_detail::tq_rows_avx512(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
        case kc::TqIsa::avx2: kc::tq_detail::tq_rows_avx2(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
#endif
        default: kc::tq_detail::tq_rows_scalar(kb, tab, w, rb, n, ap, nt, out, r0, r1); break;
    }
}
}  // namespace

bool tq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
    if (kbits_of(tg) == 0 || kbits_of(td) == 0) {
        err = "not trellis types";
        return false;
    }
    if (n_embd % 128 != 0 || n_ff % 128 != 0 || n_embd > kc::kTqMaxN || n_ff > kc::kTqMaxN) {
        err = "the trellis CPU kernels take widths of 128 blocks up to " + std::to_string(kc::kTqMaxN);
        return false;
    }
    f.tg = tg;
    f.td = td;
    f.n_embd = n_embd;
    f.n_ff = n_ff;
    f.gu_act = f.d_act = kActTq;
    f.gu_row = (size_t) (n_embd / 128) * (2 + 4 * kbits_of(tg));
    f.d_row = (size_t) (n_ff / 128) * (2 + 4 * kbits_of(td));
    f.act_bytes = act_bytes_of(n_embd);
    f.h_bytes = act_bytes_of(n_ff);
    return true;
}

// x (k values) -> the layout above (the device's q8_quant: sc = max|x| / 127, q = round-half-even(x / sc))
void tq_cpu_quant(const float* x, int k, void* dst) {
    float* d = (float*) dst;
    int8_t* q = (int8_t*) ((uint8_t*) dst + (size_t) k / 32 * 4);
    for (int b = 0; b < k / 32; ++b) {
        const float* xb = x + b * 32;
        float am = 0.0f;
        for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(xb[j]));
        const float sc = am / 127.0f;
        d[b] = sc;
        for (int j = 0; j < 32; ++j) q[b * 32 + j] = (int8_t) (am > 0.0f ? (int) std::nearbyint(xb[j] / sc) : 0);
    }
}

void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
                    int r0, int r1, float limit) {
    static thread_local std::vector<float> tmp[kc::kTqMaxTokens];
    float* up_out[kc::kTqMaxTokens];
    for (int t = 0; t < nt; ++t) {
        tmp[t].resize((size_t) r1);
        up_out[t] = tmp[t].data();
    }
    dots(f.tg, gate, f.gu_row, f.n_embd, act, nt, ff, r0, r1);
    dots(f.tg, up, f.gu_row, f.n_embd, act, nt, up_out, r0, r1);
    for (int t = 0; t < nt; ++t)
        for (int r = r0; r < r1; ++r) {
            float g = ff[t][r], u = up_out[t][r];
            if (limit > 0.0f) {
                g = std::min(g, limit);
                u = std::min(std::max(u, -limit), limit);
            }
            ff[t][r] = g / (1.0f + std::exp(-g)) * u;
        }
}

void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
    dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);
}
bool tq_cpu_type(int type) { return kbits_of(type) > 0; }
int tq_cpu_act() { return kActTq; }
#else
bool tq_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
    err = "this build has no CPU trellis kernels (STRATA_NATIVE_EXPERTS=OFF)";
    return false;
}
void tq_cpu_quant(const float*, int, void*) {}
void tq_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}
void tq_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
bool tq_cpu_type(int) { return false; }
int tq_cpu_act() { return 100; }
#endif

}  // namespace strata::ds4::gguf