Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
8.2 KB
Raw Blame
// #1996 (0.1.43.1): host-to-device uploads whose host side is pageable go up through pinned bounce slots.
//
// A cudaMemcpyAsync from pageable memory is staged by the runtime, and the staging is not ordered the way a pinned copy
// is.  On gfx1100 (ROCm 7.2.4) a text prompt after a picture one read stale bytes that way (#1712, fixed for the step
// records and the token ids in 0.1.43); a 4-hour multi-slot load (#1996) still faulted after 31 minutes, with the
// image rows, the draft tokens and bounds and the non-mapped routing tables still going up from caller memory.  The
// analysis of which copies those are, and the idea of a ring of pinned slots with one event each, are
// mantovaniluca91's (patch C on #1712); this is our own form of it.
//
// A PinnedRing owns a few pinned slots, each with an event.  h2d() copies the bytes into the next slot (after the copy
// that last read that slot is done), queues the DMA from the slot and records the slot's event.  So the caller may
// overwrite or free its source the moment the call returns, as a pageable copy allowed.  With no pinned RAM (or
// STRATA_TEST_PAGEABLE=1) the copy is made from the caller's memory and the stream is waited for, so the source is
// free on return all the same.  A ring belongs to ONE device (the one current when it is first used) and is used by one
// thread at a time.
#pragma once

#include <cuda_runtime.h>

#include <cstddef>
#include <cstdio>
#include <cstdlib>
#include <cstring>

namespace strata::core {

class PinnedRing {
public:
    static constexpr int kSlots = 4;

    PinnedRing() = default;
    PinnedRing(const PinnedRing&) = delete;
    PinnedRing& operator=(const PinnedRing&) = delete;
    ~PinnedRing() { release(); }

    // dst (device) <- bytes from src (host, any kind of memory); src is free to change when this returns
    cudaError_t h2d(void* dst, const void* src, size_t bytes, cudaStream_t s) {
        if (bytes == 0) return cudaSuccess;
        void* stage = acquire(bytes);
        if (stage == nullptr) return direct(dst, src, bytes, s);
        std::memcpy(stage, src, bytes);
        return submit(dst, stage, bytes, s);
    }

    // For a caller that gathers many small pieces into one upload: acquire() hands out a pinned slot of at least
    // `bytes` (nullptr: none could be had - use direct()), the caller fills it, then submit() queues the copy and
    // guards the slot.  An acquired slot must be submitted before the next acquire().
    void* acquire(size_t bytes) {
        if (bytes == 0 || pageable_forced()) return nullptr;
        int cur = -1;
        if (cudaGetDevice(&cur) != cudaSuccess) { cudaGetLastError(); return nullptr; }
        if (dev_ < 0) dev_ = cur;
        if (cur != dev_) return nullptr;   // a ring is for one device
        Slot& sl = slots_[next_];
        next_ = (next_ + 1) % kSlots;
        if (sl.pending) {                  // the copy that last read this slot is done before it is refilled
            sl.pending = false;
            if (cudaEventSynchronize(sl.ev) != cudaSuccess) { cudaGetLastError(); return nullptr; }
        }
        if (sl.cap < bytes) {
            if (sl.p) { cudaFreeHost(sl.p); sl.p = nullptr; sl.cap = 0; }   // idle: its last copy was waited for above
            const size_t cap = bytes < kMinCap ? kMinCap : bytes;
            if (cudaHostAlloc(&sl.p, cap, cudaHostAllocPortable) != cudaSuccess) {
                cudaGetLastError();
                sl.p = nullptr;
                return nullptr;
            }
            sl.cap = cap;
        }
        if (sl.ev == nullptr && cudaEventCreateWithFlags(&sl.ev, cudaEventDisableTiming) != cudaSuccess) {
            cudaGetLastError();
            sl.ev = nullptr;
            return nullptr;
        }
        acquired_ = &sl;
        return sl.p;
    }

    cudaError_t submit(void* dst, const void* stage, size_t bytes, cudaStream_t s) {
        Slot* sl = acquired_;
        acquired_ = nullptr;
        cudaError_t e = cudaMemcpyAsync(dst, stage, bytes, cudaMemcpyHostToDevice, s);
        if (e != cudaSuccess) {
            std::fprintf(stderr, "strata: pinned upload of %zu bytes failed: %s\n", bytes, cudaGetErrorString(e));
            return e;
        }
        if (sl != nullptr && cudaEventRecord(sl->ev, s) == cudaSuccess) {
            sl->pending = true;
            return cudaSuccess;
        }
        cudaGetLastError();                // no event: the copy is waited for here instead
        return cudaStreamSynchronize(s);
    }

    // the same copy without a pinned slot: from the caller's memory, the stream waited for
    static cudaError_t direct(void* dst, const void* src, size_t bytes, cudaStream_t s) {
        if (bytes == 0) return cudaSuccess;
        cudaError_t e = cudaMemcpyAsync(dst, src, bytes, cudaMemcpyHostToDevice, s);
        if (e == cudaSuccess) e = cudaStreamSynchronize(s);
        if (e != cudaSuccess) std::fprintf(stderr, "strata: upload of %zu bytes failed: %s\n", bytes, cudaGetErrorString(e));
        return e;
    }

    // dst (pageable host) <- device bytes; the stream is drained and the bytes are in dst on return
    cudaError_t d2h_sync(void* dst, const void* src, size_t bytes, cudaStream_t s) {
        if (bytes == 0) return cudaSuccess;
        void* stage = acquire(bytes);
        cudaError_t e = cudaMemcpyAsync(stage != nullptr ? stage : dst, src, bytes, cudaMemcpyDeviceToHost, s);
        if (e == cudaSuccess) e = cudaStreamSynchronize(s);
        if (e == cudaSuccess && stage != nullptr) std::memcpy(dst, stage, bytes);
        if (stage != nullptr) acquired_ = nullptr;   // no event: the stream was drained
        if (e != cudaSuccess) std::fprintf(stderr, "strata: download of %zu bytes failed: %s\n", bytes, cudaGetErrorString(e));
        return e;
    }

    // Every slot holds at least `bytes` from here on: grown now, by a caller that must not allocate later (a pinned
    // allocation or free takes the driver's lock and can wait for the device: STRATA_PIPED_CONCURRENT grows the rings
    // before a read whose windows run beside it).  On the device that is current, as acquire().  False: a slot could
    // not be had (its uploads then grow it, or go up from the caller's memory, as before).
    bool reserve(size_t bytes) {
        if (bytes == 0 || pageable_forced()) return true;
        int cur = -1;
        if (cudaGetDevice(&cur) != cudaSuccess) { cudaGetLastError(); return false; }
        if (dev_ < 0) dev_ = cur;
        if (cur != dev_) return false;
        for (Slot& sl : slots_) {
            if (sl.cap >= bytes && sl.ev != nullptr) continue;
            if (sl.pending) {
                sl.pending = false;
                if (cudaEventSynchronize(sl.ev) != cudaSuccess) { cudaGetLastError(); return false; }
            }
            if (sl.cap < bytes) {
                if (sl.p) { cudaFreeHost(sl.p); sl.p = nullptr; sl.cap = 0; }
                const size_t cap = bytes < kMinCap ? kMinCap : bytes;
                if (cudaHostAlloc(&sl.p, cap, cudaHostAllocPortable) != cudaSuccess) {
                    cudaGetLastError();
                    sl.p = nullptr;
                    return false;
                }
                sl.cap = cap;
            }
            if (sl.ev == nullptr && cudaEventCreateWithFlags(&sl.ev, cudaEventDisableTiming) != cudaSuccess) {
                cudaGetLastError();
                sl.ev = nullptr;
                return false;
            }
        }
        return true;
    }

    void release() {
        for (Slot& sl : slots_) {
            if (sl.pending && sl.ev) cudaEventSynchronize(sl.ev);
            if (sl.ev) cudaEventDestroy(sl.ev);
            if (sl.p) cudaFreeHost(sl.p);
            sl = Slot{};
        }
        cudaGetLastError();
        acquired_ = nullptr;
        dev_ = -1;
        next_ = 0;
    }

private:
    static constexpr size_t kMinCap = 64 * 1024;
    struct Slot {
        void* p = nullptr;
        size_t cap = 0;
        cudaEvent_t ev = nullptr;
        bool pending = false;
    };
    static bool pageable_forced() {
        static const bool v = [] { const char* e = std::getenv("STRATA_TEST_PAGEABLE"); return e != nullptr && e[0] == '1'; }();
        return v;
    }
    Slot slots_[kSlots];
    Slot* acquired_ = nullptr;
    int next_ = 0;
    int dev_ = -1;
};

}  // namespace strata::core