Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
2.6 KB
Raw Blame
#include "strata/core/auto_cache.hpp"

#include <cstdio>
#include <cstdlib>

using namespace strata::core;
namespace {
int checks = 0;
void check(bool ok, const char* label) {
    ++checks;
    if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
}
constexpr int64_t MiB = 1ll << 20;
}

int main() {
    const int64_t blob = 2560ll * 1024;   // 2.5 MiB
    const int64_t min_slots = 300;
    // the A4000 of #4474: 9951 MiB free after the weights, 700 reserved + 184 draft head
    const int64_t free_b = 9951 * MiB, reserve_b = (700 + 184) * MiB;
    const AutoCacheSlots a = auto_cache_slots(free_b, reserve_b, blob, min_slots);
    const AutoCacheSlots plain = auto_cache_slots(free_b, reserve_b, blob, min_slots, 0);
    check(a.late_priced && !plain.late_priced, "the late allocation is priced by default and off with late_mib 0");
    check(plain.slots == (free_b - reserve_b) / blob, "late_mib 0 is the old sizing");
    check(a.slots == (free_b - reserve_b - kLateAllocMib * MiB) / blob, "the cache leaves reserve + late allocation");
    check(plain.slots - a.slots == kLateAllocMib * MiB / blob || plain.slots - a.slots == kLateAllocMib * MiB / blob + 1,
          "256 MiB fewer bytes: about 100 slots of 2.5 MiB");
    // the card ends with the reserve: free - slots*blob - priced - 244 MiB unpriced >= reserve
    check(free_b - a.slots * blob - (184 + 244) * MiB >= 700 * MiB - blob, "the end free is the reserve, not 241 MiB under it");
    // a small card: not enough for min_slots after the late allocation -> the old sizing (#496 lowers the reserve elsewhere)
    const AutoCacheSlots small = auto_cache_slots(1200 * MiB, 700 * MiB, blob, min_slots);
    check(!small.late_priced && small.slots == (500 * MiB) / blob, "a card that cannot spare it keeps the old number");
    const AutoCacheSlots tiny = auto_cache_slots(900 * MiB, 700 * MiB, blob, min_slots);
    check(!tiny.late_priced && tiny.slots == 80, "less than min_slots stays as it was");
    check(auto_cache_slots(500 * MiB, 700 * MiB, blob, min_slots).slots == 0, "a negative room is no slots");
    check(auto_cache_slots(1000 * MiB, 700 * MiB, 0, min_slots).slots == 0, "no blob size: no slots, no division");
    // exactly at the limit: min_slots after the late allocation
    const int64_t edge_free = reserve_b + (kLateAllocMib * MiB) + min_slots * blob;
    check(auto_cache_slots(edge_free, reserve_b, blob, min_slots).late_priced, "exactly min_slots after the late allocation: priced");
    check(!auto_cache_slots(edge_free - 1, reserve_b, blob, min_slots).late_priced, "one byte less: the old sizing");
    std::printf("auto_cache_test: %d checks passed\n", checks);
    return 0;
}