Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
4.3 KB
Raw Blame
// 0.1.43 5.2: do two streams move host->device data faster than one (both copy engines)?  And what does one layer's
// worth of expert-sized copies cost end to end (issue, copy, host function, sync)?
//   nvcc -O2 -o h2d_bench tools/h2d_bench.cu && ./h2d_bench [device]
#include <cuda_runtime.h>
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <vector>

#define CK(x) do { cudaError_t e = (x); if (e != cudaSuccess) { std::printf("%s:%d %s\n", __FILE__, __LINE__, cudaGetErrorString(e)); std::exit(1); } } while (0)
using Clock = std::chrono::steady_clock;
static double ms_since(Clock::time_point t) { return std::chrono::duration<double, std::milli>(Clock::now() - t).count(); }

int main(int argc, char** argv) {
    int dev = argc > 1 ? std::atoi(argv[1]) : 0;
    CK(cudaSetDevice(dev));
    cudaDeviceProp p;
    CK(cudaGetDeviceProperties(&p, dev));
    std::printf("%s asyncEngineCount=%d\n", p.name, p.asyncEngineCount);
    const size_t blob = 1382400;                    // a Q2_0 expert
    const size_t nb = 256;                          // blobs in the arena
    const size_t arena_bytes = nb * blob;
    uint8_t* host = nullptr;
    // like the engine's arena: plain pages registered with cudaHostRegister (PORTABLE)
    host = (uint8_t*) aligned_alloc(2 << 20, (arena_bytes + (2 << 20) - 1) / (2 << 20) * (2 << 20));
    for (size_t i = 0; i < arena_bytes; i += 4096) host[i] = (uint8_t) i;
    CK(cudaHostRegister(host, arena_bytes, cudaHostRegisterPortable));
    uint8_t* dbuf;
    CK(cudaMalloc(&dbuf, 2 * 64 * blob));
    cudaStream_t s[2];
    for (int i = 0; i < 2; ++i) CK(cudaStreamCreateWithFlags(&s[i], cudaStreamNonBlocking));

    // 1. throughput: N blobs, one stream vs two streams (alternate blobs)
    for (int nstreams = 1; nstreams <= 2; ++nstreams) {
        double best = 1e9;
        for (int rep = 0; rep < 7; ++rep) {
            auto t0 = Clock::now();
            for (size_t i = 0; i < nb; ++i) CK(cudaMemcpyAsync(dbuf + (i % 64) * blob, host + i * blob, blob, cudaMemcpyHostToDevice, s[i % nstreams]));
            for (int i = 0; i < nstreams; ++i) CK(cudaStreamSynchronize(s[i]));
            best = std::min(best, ms_since(t0));
        }
        std::printf("throughput %d stream(s): %.3f ms for %zu blobs = %.2f GB/s\n", nstreams, best, nb, arena_bytes / best / 1e6);
    }
    // 1b. the same with each stream copying one big contiguous range
    for (int nstreams = 1; nstreams <= 2; ++nstreams) {
        double best = 1e9;
        for (int rep = 0; rep < 7; ++rep) {
            auto t0 = Clock::now();
            size_t half = 64 * blob / nstreams;
            for (int i = 0; i < nstreams; ++i) CK(cudaMemcpyAsync(dbuf + i * half, host + i * half, half, cudaMemcpyHostToDevice, s[i]));
            for (int i = 0; i < nstreams; ++i) CK(cudaStreamSynchronize(s[i]));
            best = std::min(best, ms_since(t0));
        }
        std::printf("big range %d stream(s): %.3f ms for %zu bytes = %.2f GB/s\n", nstreams, best, 64 * blob, 64 * blob / best / 1e6);
    }
    // 2. per-layer latency: k blobs, then a host function, then wait for it (the engine's flag B)
    for (int k : {1, 2, 4, 8, 16}) {
        for (int nstreams = 1; nstreams <= 2; ++nstreams) {
            std::vector<double> v;
            for (int rep = 0; rep < 200; ++rep) {
                static volatile int flag;
                flag = 0;
                auto t0 = Clock::now();
                for (int i = 0; i < k; ++i) CK(cudaMemcpyAsync(dbuf + i * blob, host + ((rep * 13 + i * 7) % nb) * blob, blob, cudaMemcpyHostToDevice, s[i % nstreams]));
                if (nstreams == 2) {
                    cudaEvent_t ev;
                    CK(cudaEventCreateWithFlags(&ev, cudaEventDisableTiming));
                    CK(cudaEventRecord(ev, s[1]));
                    CK(cudaStreamWaitEvent(s[0], ev, 0));
                    CK(cudaEventDestroy(ev));
                }
                CK(cudaLaunchHostFunc(s[0], [](void*) { flag = 1; }, nullptr));
                while (!flag) {}
                v.push_back(ms_since(t0));
            }
            std::sort(v.begin(), v.end());
            std::printf("layer of %2d blobs, %d stream(s): median %.3f ms  min %.3f  p90 %.3f  (ideal at 11 GB/s %.3f)\n", k, nstreams, v[v.size() / 2], v[0], v[v.size() * 9 / 10], k * blob / 11e6);
        }
    }
    return 0;
}