Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
#include "strata/core/expert_source.hpp"
#include "strata/core/foresight_swap.hpp"
#include "strata/kernels/cpu/expert.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
#if defined(STRATA_NATIVE_EXPERTS)
#include "strata/kernels/cpu/native_expert.hpp"
#include "ggml.h"
#endif
#include <cuda_runtime.h>
#include <cstdlib>
#include <cstring>
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <limits>
#include <stdexcept>
#include <string>
#include <system_error>
#include <thread>
#include <utility>
#include <vector>
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
#define NOMINMAX
#endif
#define PSAPI_VERSION 2
#include <windows.h>
#include <psapi.h>
#endif
namespace fs = std::filesystem;
namespace {
void require(bool ok, const std::string& message) {
if (!ok) throw std::runtime_error(message);
}
struct TempDirectory {
fs::path path;
explicit TempDirectory(const fs::path& base = fs::temp_directory_path()) {
const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count();
path = base / ("strata-file-source-test-" + std::to_string(stamp));
fs::create_directories(path);
}
~TempDirectory() {
std::error_code ignored;
fs::remove_all(path, ignored);
}
};
void create_pack(const fs::path& dir, uint64_t bytes, const std::vector<std::pair<uint64_t, char>>& markers = {}) {
const fs::path file = dir / "experts.bin";
{
std::ofstream out(file, std::ios::binary | std::ios::trunc);
require((bool) out, "could not create synthetic experts.bin");
if (bytes > 0) {
out.seekp((std::streamoff) (bytes - 1));
out.put('\0');
}
require((bool) out, "could not size synthetic experts.bin");
}
for (const auto& [offset, marker] : markers) {
require(offset < bytes, "synthetic marker lies beyond experts.bin");
std::fstream file_out(file, std::ios::binary | std::ios::in | std::ios::out);
require((bool) file_out, "could not open synthetic experts.bin for markers");
file_out.seekp((std::streamoff) offset);
file_out.put(marker);
require((bool) file_out, "could not write synthetic expert marker");
}
}
void check_file_size_rejection(strata::core::FileExpertSource& source, const fs::path& dir, int64_t layers,
int64_t experts, uint64_t expected_bytes) {
std::string err;
source.close();
fs::resize_file(dir / "experts.bin", expected_bytes - 1);
require(!source.open(dir.string(), layers, experts, err), "a truncated expert file was accepted");
require(!source.mapped() && !err.empty(), "truncated-file rejection left the source mapped or unreported");
fs::resize_file(dir / "experts.bin", expected_bytes + 1);
err.clear();
require(!source.open(dir.string(), layers, experts, err), "an oversized expert file was accepted");
require(!source.mapped() && !err.empty(), "oversized-file rejection left the source mapped or unreported");
}
#if defined(_WIN32)
void check_mapped_release(strata::core::FileExpertSource& source, int64_t layer, int64_t expert, uint64_t bytes) {
const uint8_t* p = source.blob(layer, expert);
require(p != nullptr, "release fixture has no mapped blob");
const std::vector<uint8_t> saved(p, p + (size_t) bytes);
SYSTEM_INFO info{};
GetSystemInfo(&info);
const uintptr_t page = info.dwPageSize;
const uintptr_t first = ((uintptr_t) p + page - 1) / page * page;
const uintptr_t end = ((uintptr_t) p + bytes) / page * page;
require(end > first, "release fixture has no full interior page");
auto resident = [&](uintptr_t address) {
PSAPI_WORKING_SET_EX_INFORMATION state{};
state.VirtualAddress = (void*) address;
require(QueryWorkingSetEx(GetCurrentProcess(), &state, sizeof state) != 0,
"QueryWorkingSetEx failed for the release fixture");
return state.VirtualAttributes.Valid != 0;
};
const char* setting = std::getenv("STRATA_FILE_RELEASE");
const bool enabled = setting != nullptr && std::strcmp(setting, "1") == 0;
for (int round = 0; round < 3; ++round) {
require(std::memcmp(p, saved.data(), saved.size()) == 0, "release changed the mapped expert bytes");
require(resident(first), "reading the expert did not bring its interior page into the working set");
const uint64_t released = source.release(layer, expert);
require(released == (enabled ? end - first : 0), "release did not honor full pages or its opt-in switch");
// Off: the page stays. On: Windows may keep a trimmed page on its standby list and map it back at once, so a
// page that is still valid is reported, not failed (the byte count above is the contract).
if (!enabled) require(resident(first), "release trimmed pages with the switch off");
else if (resident(first)) std::fprintf(stderr, "note: the trimmed page is already back in the working set\n");
if ((uintptr_t) p < first)
require(resident((uintptr_t) p), "release trimmed the expert's shared first page");
if (end < (uintptr_t) p + bytes)
require(resident(end), "release trimmed the expert's shared last page");
}
require(std::memcmp(p, saved.data(), saved.size()) == 0, "the expert changed after repeated release/read cycles");
require(source.release(-1, expert) == 0 && source.release(layer, -1) == 0 &&
source.release(std::numeric_limits<int64_t>::max(), expert) == 0 &&
source.release(layer, std::numeric_limits<int64_t>::max()) == 0,
"release accepted an invalid layer or expert");
}
#endif
void test_canonical_layout() {
using namespace strata::core;
using namespace strata::kernels::cpu;
constexpr int64_t layers = 2;
constexpr int64_t experts = 3;
const uint64_t layer_bytes = (uint64_t) experts * BLOB;
const uint64_t total = (uint64_t) layers * layer_bytes;
TempDirectory dir;
std::string err;
const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err);
require(layout_ok, "could not load canonical layout: " + err);
create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) BLOB, 'b'}, {layer_bytes, 'c'},
{layer_bytes + (uint64_t) BLOB, 'd'}});
FileExpertSource source;
bool opened = source.open(dir.path.string(), layers, experts, err);
require(opened, "could not map canonical pack: " + err);
require(source.blobs() == layers * experts, "canonical blob count is wrong");
const uint8_t* first = source.blob(0, 0);
const uint8_t* second = source.blob(0, 1);
const uint8_t* next_layer = source.blob(1, 0);
require(first && second && next_layer, "valid canonical blob lookup failed");
require(second - first == (ptrdiff_t) BLOB && next_layer - first == (ptrdiff_t) layer_bytes,
"canonical expert or layer stride is wrong");
require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c',
"canonical blob lookup returned bytes from the wrong expert");
require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr &&
source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr,
"canonical bounds check accepted an invalid layer or expert");
require(source.reads() == 3, "invalid canonical lookups changed the read count");
#if defined(_WIN32)
check_mapped_release(source, 0, 0, BLOB);
check_mapped_release(source, 0, 1, BLOB);
#endif
check_file_size_rejection(source, dir.path, layers, experts, total);
create_pack(dir.path, total, {{layer_bytes, 'c'}});
err.clear();
opened = source.open(dir.path.string(), layers, experts, err);
require(opened, "source failed to reopen after size errors: " + err);
const uint8_t* reopened = source.blob(1, 0);
require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened canonical source kept stale state");
source.close();
require(source.release(0, 0) == 0, "a closed file source released pages");
}
#if defined(STRATA_NATIVE_EXPERTS)
void test_native_variable_layout() {
using namespace strata::core;
using namespace strata::kernels::cpu;
constexpr int64_t layers = 2;
constexpr int64_t experts = 3;
constexpr int64_t embedding = 2560;
constexpr int64_t feed_forward = 640;
TempDirectory dir;
NativeFmt first_fmt, second_fmt;
std::string err;
const bool first_ok = native_fmt(GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, embedding, feed_forward, first_fmt, err);
require(first_ok, "IQ3_XXS/IQ4_NL synthetic format is unavailable: " + err);
err.clear();
const bool second_ok = native_fmt(GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_NL, embedding, feed_forward, second_fmt, err);
require(second_ok, "IQ2_XS/IQ4_NL synthetic format is unavailable: " + err);
require(first_fmt.bytes != second_fmt.bytes, "chosen native formats do not exercise variable layer sizes");
const uint64_t layer0_bytes = (uint64_t) first_fmt.bytes * experts;
const uint64_t total = layer0_bytes + (uint64_t) second_fmt.bytes * experts;
{
std::ofstream metadata(dir.path / "native_experts.txt");
require((bool) metadata, "could not create synthetic native_experts.txt");
metadata << "0 " << GGML_TYPE_IQ3_XXS << ' ' << GGML_TYPE_IQ4_NL << " 0 " << first_fmt.bytes << '\n';
metadata << "1 " << GGML_TYPE_IQ2_XS << ' ' << GGML_TYPE_IQ4_NL << ' ' << layer0_bytes << ' '
<< second_fmt.bytes << '\n';
require((bool) metadata, "could not write synthetic native_experts.txt");
}
create_pack(dir.path, total, {{0, 'a'}, {(uint64_t) first_fmt.bytes, 'b'},
{layer0_bytes, 'c'}, {layer0_bytes + (uint64_t) second_fmt.bytes, 'd'}});
const bool layout_ok = expert_layout_load(dir.path.string(), layers, experts, err);
require(layout_ok, "could not load synthetic native layout: " + err);
FileExpertSource source;
bool opened = source.open(dir.path.string(), layers, experts, err);
require(opened, "could not map native pack: " + err);
require(source.blobs() == layers * experts, "native blob count is wrong");
const uint8_t* first = source.blob(0, 0);
const uint8_t* second = source.blob(0, 1);
const uint8_t* next_layer = source.blob(1, 0);
const uint8_t* next_layer_second = source.blob(1, 1);
require(first && second && next_layer && next_layer_second, "valid native blob lookup failed");
require(second - first == (ptrdiff_t) first_fmt.bytes && next_layer - first == (ptrdiff_t) layer0_bytes &&
next_layer_second - next_layer == (ptrdiff_t) second_fmt.bytes,
"native per-layer expert stride is wrong");
require(first[0] == 'a' && second[0] == 'b' && next_layer[0] == 'c' && next_layer_second[0] == 'd',
"native blob lookup returned bytes from the wrong expert");
require(source.blob(-1, 0) == nullptr && source.blob(0, -1) == nullptr &&
source.blob(layers, 0) == nullptr && source.blob(0, experts) == nullptr,
"native bounds check accepted an invalid layer or expert");
require(source.reads() == 4, "invalid native lookups changed the read count");
#if defined(_WIN32)
check_mapped_release(source, 0, 1, first_fmt.bytes);
check_mapped_release(source, 1, 1, second_fmt.bytes);
#endif
check_file_size_rejection(source, dir.path, layers, experts, total);
create_pack(dir.path, total, {{layer0_bytes, 'c'}});
err.clear();
opened = source.open(dir.path.string(), layers, experts, err);
require(opened, "source failed to reopen after size errors: " + err);
const uint8_t* reopened = source.blob(1, 0);
require(reopened && source.reads() == 1 && reopened[0] == 'c', "reopened native source kept stale state");
source.close();
}
#endif
void test_complement_plan() {
using namespace strata::core::detail;
std::vector<uint64_t> offsets;
uint64_t bytes = 0;
std::string error;
require(make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {1, 2}}, {}, offsets, bytes, error), error);
require(bytes == 16 && offsets == std::vector<uint64_t>{0, kNoCacheComplement, 3, 6, 11, kNoCacheComplement},
"wrong compact offsets for variable native layer sizes");
const uint8_t resident[16] = {}, mapped[5] = {};
require(cache_complement_blob_or_fallback(1, offsets, resident, mapped) == mapped,
"GPU-resident expert lost mmap fallback");
require(cache_complement_blob_or_fallback(4, offsets, resident, mapped) == resident + 11,
"CPU miss did not use resident complement");
require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {0, 1}}, {}, offsets, bytes, error)
&& offsets.empty() && bytes == 0, "duplicate pair accepted");
require(!make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}}, {{0, 1}}, offsets, bytes, error),
"overlapping tiers accepted");
require(!make_cache_complement_plan(2, 3, {3, 5}, {{2, 0}}, {}, offsets, bytes, error),
"out-of-range pair accepted");
require(!make_cache_complement_plan(2, 3, {0, 5}, {}, {}, offsets, bytes, error), "zero-size layer accepted");
}
void test_resident_lend_region() {
using namespace strata::core::detail;
// four slots (sizes 5, 3, 3, 5); the experts no slot holds take 10 bytes
const std::vector<uint64_t> slots{5, 3, 3, 5};
require(choose_resident_keep_from(slots, 10, 9, 1) == -1, "a base larger than the budget was accepted");
require(choose_resident_keep_from(slots, 10, 10, 1) == 4, "no room: the lend region must stay on the file");
require(choose_resident_keep_from(slots, 10, 15, 1) == 3, "the last slot fits and must be kept first");
require(choose_resident_keep_from(slots, 10, 17, 1) == 3, "a slot that does not fit ends the lend region's walk");
require(choose_resident_keep_from(slots, 10, 18, 1) == 2, "two slots fit");
require(choose_resident_keep_from(slots, 10, 100, 1) == 1, "the walk must stop at the lend region's first slot");
require(choose_resident_keep_from(slots, 10, 100, -1) == 4, "no lend region kept slots in RAM");
require(choose_resident_keep_from(slots, 10, 100, 9) == 4, "an out-of-range lend region kept slots in RAM");
require(choose_resident_keep_from({}, 0, 0, 0) == 0, "an empty cache");
}
void test_split_lend_regions() {
using namespace strata::core::detail;
// two layers, three experts; a layer-0 blob is 5 bytes and a layer-1 blob 3. The plan under
// construction already holds the (0,1) expert (an expert no cache holds); stage 0's region lends
// (0,2) and (0,1) from its highest slots, stage 1's lends (1,2), (1,0), (1,1) in that order.
const std::vector<uint64_t> blobs{5, 3};
const std::vector<std::vector<std::pair<int32_t, int32_t>>> regions = {{{0, 2}, {0, 1}}, {{1, 2}, {1, 0}, {1, 1}}};
std::string err;
std::vector<uint64_t> offsets;
uint64_t bytes = 0;
// room for everything: (0,1) is already in the copy and is passed over, not counted twice
offsets = {kNoCacheComplement, 0, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement};
bytes = 5;
require(append_stage_lend_regions(2, 3, blobs, offsets, bytes, 100, regions, err) == 4,
"both stage lend regions must be kept");
require(bytes == 19 &&
offsets == std::vector<uint64_t>{kNoCacheComplement, 0, 5, 13, 16, 10},
"the stage lend regions got wrong compact offsets");
// the cap is the whole copy's: the stage-0 walk fits (0,2) and stage 1 stops at the first pair that
// does not fit; that stage's rest stays on the file
offsets = {kNoCacheComplement, 0, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement};
bytes = 5;
require(append_stage_lend_regions(2, 3, blobs, offsets, bytes, 10, regions, err) == 1, "cap 10 keeps only (0,2)");
require(bytes == 10 && offsets[2] == 5 && offsets[3] == kNoCacheComplement && offsets[5] == kNoCacheComplement,
"the walk placed experts past the cap");
// (0,2)'s 5 bytes do not fit in the 4 left: stage 0 keeps nothing, and stage 1's own walk still runs -
// its (1,2) fits, its (1,0) stops that stage
offsets = {kNoCacheComplement, 0, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement};
bytes = 5;
require(append_stage_lend_regions(2, 3, blobs, offsets, bytes, 9, regions, err) == 1,
"a stage refused at the cap must not stop the next stage's walk");
require(bytes == 8 && offsets[2] == kNoCacheComplement && offsets[5] == 5 && offsets[3] == kNoCacheComplement,
"the refused stage's experts entered the copy, or the next stage's did not");
// a pair outside the geometry is an error, and the plan is left untouched: validate before appending
offsets = {kNoCacheComplement, 0, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement, kNoCacheComplement};
bytes = 5;
require(append_stage_lend_regions(2, 3, blobs, offsets, bytes, 100, {{{0, 2}, {2, 0}}}, err) == -1 &&
!err.empty() && bytes == 5 && offsets[2] == kNoCacheComplement,
"an out-of-range lend-region pair was accepted, or a rejected walk touched the plan");
}
void test_resident_exchange() {
using namespace strata::core;
using namespace strata::core::detail;
// 2 layers x 3 experts; the GPU holds (0,1) and (1,2): the compact copy holds the other four
std::vector<uint64_t> offsets;
uint64_t bytes = 0;
std::string error;
require(make_cache_complement_plan(2, 3, {3, 5}, {{0, 1}, {1, 2}}, {}, offsets, bytes, error), error);
const std::vector<uint64_t> before = offsets;
// (1,0) moves into the GPU, (1,2) leaves it: (1,2) takes (1,0)'s bytes' place
require(exchange_cache_complement(offsets, 3, 5), "a valid exchange was refused");
require(offsets[5] == before[3] && offsets[3] == kNoCacheComplement, "the exchange did not move the place");
for (size_t i : {0u, 1u, 2u, 4u}) require(offsets[i] == before[i], "an exchange touched another expert");
// the copy's size and its set of places are unchanged
std::vector<uint64_t> a, b;
for (uint64_t o : before) if (o != kNoCacheComplement) a.push_back(o);
for (uint64_t o : offsets) if (o != kNoCacheComplement) b.push_back(o);
std::sort(a.begin(), a.end());
std::sort(b.begin(), b.end());
require(a == b, "an exchange changed the compact copy's places");
const std::vector<uint64_t> after = offsets;
require(!exchange_cache_complement(offsets, 3, 0), "an `in` the copy does not hold was accepted");
require(!exchange_cache_complement(offsets, 0, 2), "an `out` the copy holds already was accepted");
require(!exchange_cache_complement(offsets, 0, 0), "a self-exchange was accepted");
require(!exchange_cache_complement(offsets, 0, 6), "an out-of-range `out` was accepted");
require(offsets == after, "a refused exchange changed the offsets");
// and back: the copy is again what the plan makes for the original placement
require(exchange_cache_complement(offsets, 5, 3), "the reverse exchange was refused");
require(offsets == before, "exchanging back did not restore the plan");
}
// #4051: the wait for the RAM an arena needs (an engine still exiting gives it back a few seconds late)
void test_wait_for_ram() {
using namespace strata::core::detail;
constexpr uint64_t GiB = 1ull << 30;
double waited = -1.0;
uint64_t avail = 0;
int slept = 0, noted = 0;
const auto sleeper = [&](double) { ++slept; };
const auto note = [&](double, uint64_t) { ++noted; };
// enough at once: no wait, no sleep, no note
require(wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [](uint64_t& a) { a = 50 * GiB; return true; }, sleeper, note, waited, avail) &&
waited == 0.0 && slept == 0 && noted == 0 && avail == 50 * GiB, "RAM that was there was waited for");
// 10 GiB free for a 40 GiB arena, the exiting engine gives its 50 GiB back after 3 s (12 steps of 0.25 s)
int reads = 0;
slept = noted = 0;
require(wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [&](uint64_t& a) { a = ++reads > 12 ? 60 * GiB : 10 * GiB; return true; },
sleeper, note, waited, avail) && waited == 3.0 && slept == 12 && noted == 1 && avail == 60 * GiB,
"RAM that came back after 3 s was not waited for");
// it never comes: the wait ends at the limit and says so (the load then goes on and warns)
slept = noted = 0;
require(!wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [](uint64_t& a) { a = 10 * GiB; return true; }, sleeper, note, waited, avail) &&
waited == 20.0 && slept == 80 && noted == 4 && avail == 10 * GiB, "the wait did not end at its limit");
// no wait asked for: one reading
slept = 0;
require(!wait_for_ram(40 * GiB, 0.0, 0.25, 5.0, [](uint64_t& a) { a = 10 * GiB; return true; }, sleeper, note, waited, avail) &&
waited == 0.0 && slept == 0, "a zero wait slept");
// the RAM cannot be read: the wait ends at once
slept = 0;
require(!wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [](uint64_t&) { return false; }, sleeper, note, waited, avail) && slept == 0,
"an unreadable RAM was waited for");
}
// #1250: when the page-locked complement is registered in steps, and what a meminfo reading says
void test_pin_pacing() {
using namespace strata::core::detail;
constexpr uint64_t GiB = 1ull << 30;
// the reporter's reading: 54 GiB available but 16 GiB really free for a 26 GiB complement
require(pin_depends_on_reclaim(16 * GiB, 26 * GiB, 4 * GiB), "a complement bigger than the free pages was not flagged");
require(!pin_depends_on_reclaim(40 * GiB, 26 * GiB, 4 * GiB), "a complement the free pages cover was flagged");
require(pin_depends_on_reclaim(30 * GiB, 26 * GiB, 4 * GiB) == false, "exactly free = bytes + reserve is enough");
require(!pin_depends_on_reclaim(0, 0, 4 * GiB), "no complement, nothing to pin");
require(pin_step_fits(6 * GiB, 1 * GiB, 4 * GiB) && !pin_step_fits(4 * GiB, 1 * GiB, 4 * GiB),
"a step must leave the reserve");
const std::string info =
"MemTotal: 65011484 kB\nMemFree: 1284096 kB\nMemAvailable: 33463296 kB\nCached: 54507520 kB\n";
require(meminfo_bytes(info, "MemFree") == 1284096ull * 1024, "MemFree was misread");
require(meminfo_bytes(info, "MemAvailable") == 33463296ull * 1024, "MemAvailable was misread");
require(meminfo_bytes(info, "Mem") == 0 && meminfo_bytes(info, "Shmem") == 0, "an absent key was not 0");
// 0.1.42 (dual-socket report): node 1 full of another model's cache, node 0 empty of it
require(numa_pin_free(100 * GiB, {}) == 100 * GiB && numa_pin_free(100 * GiB, {100 * GiB}) == 100 * GiB,
"a single-node host's figure changed");
require(numa_pin_free(90 * GiB, {88 * GiB, 2 * GiB}) == 4 * GiB, "an emptied node did not lower the figure");
require(numa_pin_free(40 * GiB, {20 * GiB, 20 * GiB}) == 40 * GiB, "balanced nodes changed the figure");
require(numa_pin_free(10 * GiB, {50 * GiB, 50 * GiB}) == 10 * GiB, "the total must still bound it");
require(pin_depends_on_reclaim(numa_pin_free(90 * GiB, {88 * GiB, 2 * GiB}), 26 * GiB, 4 * GiB),
"an unbalanced two-node host was not paced");
require(page_cache_hint(info).find("drop_caches") != std::string::npos &&
page_cache_hint(info).find("STRATA_DROP_CACHE_ON_EXIT") != std::string::npos &&
page_cache_hint(info).find("52.0 GiB") != std::string::npos, "the cache hint is incomplete");
require(page_cache_hint("MemFree: 100 kB\nCached: 1000 kB\n").empty(), "a small cache was reported");
}
// NUMA split: per-slice byte ranges of one complement slot (page rule: a page holding a slice-0 byte is slice 0's)
void test_numa_slice_ranges() {
using strata::core::detail::numa_slice_ranges;
constexpr uint64_t P = 4096;
const uint64_t roles[3] = {870400, 870400, 921600};
const uint64_t slot_bytes = roles[0] + roles[1] + roles[2];
const uint64_t offs[3] = {0, 2662400ull * 7, P * 3 + 100};
for (uint64_t off : offs) {
const auto r0 = numa_slice_ranges(off, roles, 0), r1 = numa_slice_ranges(off, roles, 1);
for (int r = 0; r < 3; ++r) {
require(r0[r].first % P == 0 && r0[r].second % P == 0 && r1[r].first % P == 0 && r1[r].second % P == 0,
"a slice range boundary was not page aligned");
require(r0[r].first <= r0[r].second && r1[r].first <= r1[r].second, "a slice range was inverted");
}
// every page fully inside the slot is in exactly one of the six ranges (and none is in two)
for (uint64_t pg = (off + P - 1) / P * P; pg + P <= off + slot_bytes; pg += P) {
int hits = 0;
for (int r = 0; r < 3; ++r) {
hits += (pg >= r0[r].first && pg + P <= r0[r].second);
hits += (pg >= r1[r].first && pg + P <= r1[r].second);
}
require(hits == 1, "a page was in " + std::to_string(hits) + " slice ranges, not exactly one");
}
}
// (b) no slice-1 range intersects a slice-0 range, within a slot and across two consecutive slots
const auto meet = [](const std::pair<uint64_t, uint64_t>& x, const std::pair<uint64_t, uint64_t>& y) {
return x.first < y.second && y.first < x.second;
};
for (uint64_t off : offs) {
const auto s0 = numa_slice_ranges(off, roles, 0), s1 = numa_slice_ranges(off, roles, 1);
const auto n0 = numa_slice_ranges(off + slot_bytes, roles, 0), n1 = numa_slice_ranges(off + slot_bytes, roles, 1);
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
require(!meet(s0[i], s1[j]), "a slice-1 range intersects a slice-0 range of the same slot");
require(!meet(s1[i], n0[j]) && !meet(n1[i], s0[j]),
"a slice-1 range intersects a slice-0 range of the neighbouring slot");
}
}
const auto a = numa_slice_ranges(0, roles, 0), b = numa_slice_ranges(0, roles, 1);
require(a[0].first <= 435199 && 435199 < a[0].second, "last slice-0 gate byte not in slice 0's gate range");
require(b[0].first <= 435200 + P && 435200 + P < b[0].second, "a slice-1 gate byte not in slice 1's gate range");
require(a[1].first <= 870400 && 870400 < a[1].second, "the gate/up boundary page was not slice 0's up range");
require(!(b[0].first <= 870400 && 870400 < b[0].second), "the gate/up boundary page was in slice 1's gate range");
}
void test_resident_memory_budget() {
using strata::core::detail::clamp_resident_budget;
constexpr uint64_t GiB = 1ull << 30, margin = 256ull << 20, headroom = 4 * GiB;
constexpr uint64_t unlimited = std::numeric_limits<uint64_t>::max();
// #730: RAM can hold the requested cache, but Windows cannot commit it.
require(clamp_resident_budget(66 * GiB, 69 * GiB, 47 * GiB, headroom) == 43 * GiB - margin,
"a RAM budget exceeded available commit capacity");
require(clamp_resident_budget(66 * GiB, 47 * GiB, 69 * GiB, headroom) == 43 * GiB - margin,
"a RAM budget exceeded available physical memory");
require(clamp_resident_budget(40 * GiB, 69 * GiB, 47 * GiB, headroom) == 40 * GiB,
"a smaller explicit budget was changed");
require(clamp_resident_budget(unlimited, 69 * GiB, 47 * GiB, headroom) == 43 * GiB - margin,
"the what-fits budget ignored commit capacity");
require(clamp_resident_budget(66 * GiB, 47 * GiB, unlimited, headroom) == 43 * GiB - margin,
"a platform without a commit reading lost its RAM limit");
require(clamp_resident_budget(43 * GiB, 69 * GiB, 47 * GiB, headroom) == 43 * GiB,
"a budget that exactly fits received the clamping margin");
require(clamp_resident_budget(66 * GiB, 69 * GiB, 0, headroom) == 0,
"exhausted commit capacity was treated as unlimited");
require(clamp_resident_budget(66 * GiB, 0, 69 * GiB, headroom) == 0,
"exhausted RAM produced a positive budget");
require(clamp_resident_budget(66 * GiB, 69 * GiB, headroom - 1, headroom) == 0 &&
clamp_resident_budget(66 * GiB, 69 * GiB, headroom, headroom) == 0,
"subtracting headroom underflowed");
require(clamp_resident_budget(66 * GiB, 69 * GiB, headroom + margin - 1, headroom) == 0 &&
clamp_resident_budget(66 * GiB, 69 * GiB, headroom + margin, headroom) == 0,
"subtracting the clamping margin underflowed");
}
void test_cgroup_memory_budget() {
using namespace strata::core::detail;
constexpr uint64_t GiB = 1ull << 30;
uint64_t bytes = 0;
// the kernel keeps min(half the clean cache, limit / 64) of it: 56 GiB / 64 = 0.875 GiB here
const uint64_t keep56 = 56 * GiB / 64;
CgroupMemoryStat clean_cache{40 * GiB, 32 * GiB, 0, 0, true};
require(cgroup_available_bytes(56 * GiB, clean_cache, bytes), "valid cgroup memory.stat was rejected");
require(bytes == 48 * GiB - keep56, "clean inactive file cache was not credited against the cgroup cap");
CgroupMemoryStat dirty_cache{40 * GiB, 32 * GiB, 4 * GiB, 2 * GiB, true};
require(cgroup_available_bytes(56 * GiB, dirty_cache, bytes), "valid dirty-cache counters were rejected");
require(bytes == 42 * GiB - keep56, "dirty/writeback pages were incorrectly counted as reclaimable");
CgroupMemoryStat oversized_inactive{10 * GiB, 20 * GiB, 0, 0, true};
require(cgroup_available_bytes(12 * GiB, oversized_inactive, bytes),
"valid oversized inactive-file counter was rejected");
require(bytes == 12 * GiB - 12 * GiB / 64, "inactive-file accounting exceeded charged current usage");
// 149: the ACTIVE page cache is reclaimable too (memory.stat "file" = active + inactive), as MemAvailable counts
// it: a 32 GiB group whose own reads left 26 GiB of cache, 25 GiB of it active, and 4.5 GiB of anonymous memory
CgroupMemoryStat active_cache{30 * GiB + GiB / 2, 1 * GiB, 0, 0, true, 26 * GiB};
require(cgroup_available_bytes(32 * GiB, active_cache, bytes), "a memory.stat with file was rejected");
require(bytes == 32 * GiB - (4 * GiB + GiB / 2) - 32 * GiB / 64, "the active page cache was not credited");
// without "file" (an old kernel's stat): inactive_file alone, as before
CgroupMemoryStat no_file{30 * GiB + GiB / 2, 1 * GiB, 0, 0, true};
require(cgroup_available_bytes(32 * GiB, no_file, bytes) && bytes == 32 * GiB - (29 * GiB + GiB / 2) - GiB / 2,
"a stat without file did not fall back to inactive_file");
// dirty and writeback pages of the active cache stay charged; a small cache keeps half of itself
CgroupMemoryStat dirty_active{20 * GiB, 1 * GiB, 2 * GiB, 1 * GiB, true, 10 * GiB};
require(cgroup_available_bytes(32 * GiB, dirty_active, bytes) && bytes == 32 * GiB - (20 * GiB - 7 * GiB) - GiB / 2,
"dirty / writeback pages of the active cache were counted as reclaimable");
CgroupMemoryStat tiny_cache{8 * GiB, 0, 0, 0, true, 256ull << 20};
require(cgroup_available_bytes(32 * GiB, tiny_cache, bytes) && bytes == 32 * GiB - 8 * GiB + (128ull << 20),
"a small clean cache was not kept at half");
// "file" larger than what is charged (a racy read) is bounded by memory.current
CgroupMemoryStat racy{4 * GiB, 1 * GiB, 0, 0, true, 9 * GiB};
require(cgroup_available_bytes(32 * GiB, racy, bytes) && bytes == 32 * GiB - GiB / 2,
"a file counter above memory.current escaped the bound");
const uint64_t max = std::numeric_limits<uint64_t>::max();
CgroupMemoryStat large_counters{max, max, max - 1, max, true};
require(cgroup_available_bytes(max, large_counters, bytes),
"large memory.stat counters were rejected");
require(bytes == 0, "dirty/writeback subtraction overflowed or escaped the usage cap");
CgroupMemoryStat usage_over_limit{60 * GiB, 0, 0, 0, true};
require(cgroup_available_bytes(56 * GiB, usage_over_limit, bytes),
"valid over-limit memory counters were rejected");
require(bytes == 0, "over-limit cgroup usage produced a positive budget");
CgroupMemoryStat missing_stat{};
bytes = 123;
require(!cgroup_available_bytes(56 * GiB, missing_stat, bytes) && bytes == 0,
"missing memory.stat counters did not fail closed");
}
void set_env(const char* name, const char* value) {
#if defined(_WIN32)
_putenv_s(name, value != nullptr ? value : "");
#else
if (value != nullptr) setenv(name, value, 1);
else unsetenv(name);
#endif
}
// #286: the unbuffered file tier (Windows: FILE_FLAG_NO_BUFFERING; Linux: O_DIRECT + io_submit) hands out the same
// bytes as the mapping - one blob at a time, a batch of adjacent experts (merged into one request), and copy_blob - and
// never falls back to the mapping. The pack lives in the working directory, not the temp directory: a tmpfs /tmp has
// no O_DIRECT, and there the test says so and skips.
void test_unbuffered_reads() {
using namespace strata::core;
using namespace strata::kernels::cpu;
constexpr int64_t layers = 2;
constexpr int64_t experts = 3;
const uint64_t layer_bytes = (uint64_t) experts * BLOB;
const uint64_t total = (uint64_t) layers * layer_bytes;
TempDirectory dir(fs::current_path());
std::string err;
require(expert_layout_load(dir.path.string(), layers, experts, err), "could not load canonical layout: " + err);
std::vector<uint8_t> bytes((size_t) total);
uint64_t x = 0x9E3779B97F4A7C15ull;
for (uint8_t& b : bytes) {
x = x * 6364136223846793005ull + 1442695040888963407ull;
b = (uint8_t) (x >> 56);
}
{
std::ofstream out(dir.path / "experts.bin", std::ios::binary | std::ios::trunc);
out.write((const char*) bytes.data(), (std::streamsize) bytes.size());
require((bool) out, "could not write the synthetic experts.bin");
}
FileExpertSource source;
require(source.open(dir.path.string(), layers, experts, err), "could not map the synthetic pack: " + err);
set_env("STRATA_UNBUFFERED_LOAD", "1");
std::string why;
const bool unbuffered = source.set_unbuffered(0, why);
set_env("STRATA_UNBUFFERED_LOAD", nullptr);
#if defined(_WIN32) || defined(__linux__)
if (!unbuffered) {
std::cout << "file_expert_source_test: unbuffered reads skipped (" << why << ")\n";
source.close();
return;
}
#else
require(!unbuffered, "unbuffered reads on a platform without them");
source.close();
return;
#endif
require(source.unbuffered(), "set_unbuffered succeeded but the source is not unbuffered");
const int64_t batch[experts] = {0, 1, 2};
source.prefetch(1, batch, experts); // layer 1 in one batch: three adjacent blobs
for (int64_t l = 0; l < layers; ++l)
for (int64_t e = 0; e < experts; ++e) {
const uint8_t* b = source.blob(l, e);
require(b != nullptr, "an unbuffered blob lookup failed");
require(std::memcmp(b, bytes.data() + (size_t) (l * (int64_t) layer_bytes + e * (int64_t) BLOB),
(size_t) BLOB) == 0,
"an unbuffered blob differs from the file");
}
std::vector<uint8_t> copy((size_t) BLOB);
require(source.copy_blob(0, 2, copy.data()) &&
std::memcmp(copy.data(), bytes.data() + (size_t) (2 * BLOB), (size_t) BLOB) == 0,
"an unbuffered copy_blob differs from the file");
require(source.direct_fallbacks() == 0, "an unbuffered read fell back to the mapping");
source.close();
}
// #633: the host RAM probe with fake /proc and cgroup trees: v2 (a limit, "max", a missing or malformed limit), v1
// (a limit, unlimited), and no cgroup line at all. Elsewhere than Linux it reads the machine's RAM.
void test_host_memory() {
using namespace strata::core::detail;
constexpr uint64_t GiB = 1ull << 30;
HostMemory m;
#if defined(__linux__)
TempDirectory t;
auto put = [&](const fs::path& rel, const std::string& text) {
fs::create_directories((t.path / rel).parent_path());
std::ofstream(t.path / rel) << text;
};
put("meminfo", "MemTotal: 134217728 kB\nMemAvailable: 104857600 kB\n"); // 100 GiB available
const std::string mi = (t.path / "meminfo").string(), cg = (t.path / "cgroup").string(),
root = (t.path / "fs").string();
auto probe = [&](const std::string& self) {
put("cgroup", self);
m.commit = 0;
const bool ok = host_available_memory(m, mi, cg, root);
require(m.commit == ~0ull, "Linux memory probe retained a commit limit");
return ok;
};
// no cgroup line at all: MemAvailable alone (before #633: "cannot determine")
require(probe("") && m.available == 100 * GiB && m.cgroup_limit == ~0ull, "no cgroup: MemAvailable alone");
// v2, a 48 GiB limit with 16 GiB charged, 4 GiB of it clean cache
put("fs/cgroup.controllers", "memory\n");
put("fs/box/memory.max", std::to_string(48 * GiB) + "\n");
put("fs/box/memory.current", std::to_string(16 * GiB) + "\n");
put("fs/box/memory.stat", "inactive_file " + std::to_string(4 * GiB) + "\nfile_dirty 0\nfile_writeback 0\n");
require(probe("0::/box\n") && m.available == 36 * GiB - 48 * GiB / 64 && m.cgroup_limit == 48 * GiB, "v2 limit");
// 149: with "file" (active + inactive cache) the active part counts too: 12 GiB of cache, 4 GiB anonymous
put("fs/box/memory.stat", "file " + std::to_string(12 * GiB) + "\ninactive_file " + std::to_string(4 * GiB) +
"\nfile_dirty 0\nfile_writeback 0\n");
require(probe("0::/box\n") && m.available == 44 * GiB - 48 * GiB / 64 && m.cgroup_limit == 48 * GiB,
"v2 limit, the active page cache credited");
put("fs/box/memory.stat", "inactive_file " + std::to_string(4 * GiB) + "\nfile_dirty 0\nfile_writeback 0\n");
put("fs/box/memory.max", "max\n");
require(probe("0::/box\n") && m.available == 100 * GiB && m.cgroup_limit == ~0ull, "v2 max");
put("fs/box/memory.max", "48G\n");
require(!probe("0::/box\n"), "v2 malformed limit accepted");
fs::remove(t.path / "fs/box/memory.max");
require(!probe("0::/box\n"), "v2 group without memory.max accepted");
// v1: the memory controller's group (a hybrid line list), a 32 GiB limit with 10 GiB used
put("fs/memory/docker/abc/memory.limit_in_bytes", std::to_string(32 * GiB) + "\n");
put("fs/memory/docker/abc/memory.usage_in_bytes", std::to_string(10 * GiB) + "\n");
put("fs/memory/memory.limit_in_bytes", "9223372036854771712\n"); // the root: unlimited
put("fs/memory/memory.usage_in_bytes", std::to_string(50 * GiB) + "\n");
require(probe("12:cpu,cpuacct:/docker/abc\n4:memory:/docker/abc\n1:name=systemd:/docker/abc\n") &&
m.available == 22 * GiB && m.cgroup_limit == 32 * GiB, "v1 limit");
put("fs/memory/docker/abc/memory.limit_in_bytes", "9223372036854771712\n");
require(probe("4:memory:/docker/abc\n") && m.available == 100 * GiB && m.cgroup_limit == ~0ull, "v1 unlimited");
// a v1 group not visible here (another namespace): MemAvailable alone
require(probe("4:memory:/elsewhere\n") && m.available == 100 * GiB, "v1 group not mounted");
#else
require(host_available_memory(m) && m.available > 0 && m.cgroup_limit == ~0ull, "this PC's RAM");
#if defined(_WIN32)
require(m.commit != ~0ull, "Windows memory probe did not report commit capacity");
#else
require(m.commit == ~0ull, "a platform without commit accounting imposed a limit");
#endif
(void) GiB;
#endif
}
// STRATA_IO_PREFETCH: the experts.bin file tier hands out the same bytes with the Linux I/O path on (whole-blob
// preads, the read-ahead workers, the mapping for cached blobs), however the reads and the predictions interleave.
void test_io_prefetch() {
using namespace strata::core;
using namespace strata::kernels::cpu;
constexpr int64_t layers = 3;
constexpr int64_t experts = 8;
const uint64_t layer_bytes = (uint64_t) experts * BLOB;
const uint64_t total = (uint64_t) layers * layer_bytes;
TempDirectory dir(fs::current_path());
std::string err;
require(expert_layout_load(dir.path.string(), layers, experts, err), "could not load canonical layout: " + err);
std::vector<uint8_t> bytes((size_t) total);
uint64_t x = 0xD1B54A32D192ED03ull;
for (uint8_t& b : bytes) {
x = x * 6364136223846793005ull + 1442695040888963407ull;
b = (uint8_t) (x >> 56);
}
{
std::ofstream out(dir.path / "experts.bin", std::ios::binary | std::ios::trunc);
out.write((const char*) bytes.data(), (std::streamsize) bytes.size());
require((bool) out, "could not write the synthetic experts.bin");
}
FileExpertSource source;
require(source.open(dir.path.string(), layers, experts, err), "could not map the synthetic pack: " + err);
source.set_io_prefetch(true, 3);
#if defined(__linux__)
require(source.io_prefetch() && source.warms(), "io prefetch did not turn on");
// predictions for the next layers, some of them wrong or repeated, while the layers are read
for (int round = 0; round < 20; ++round)
for (int64_t l = 0; l < layers; ++l) {
const int64_t pred[5] = {(round + l) % experts, (round * 3 + 1) % experts, (round + 2 * l + 5) % experts,
(round + l) % experts, 7};
source.warm((l + 1) % layers, pred, 5);
source.begin_layer(l, nullptr, 0);
std::vector<int64_t> want = {(round + l) % experts, (round * 5 + 2) % experts, (l * 3 + round) % experts};
source.prefetch(l, want.data(), (int64_t) want.size());
for (int64_t e : want) {
const uint8_t* b = source.blob(l, e);
require(b != nullptr, "an io-prefetch blob lookup failed");
require(std::memcmp(b, bytes.data() + (size_t) (l * (int64_t) layer_bytes + e * (int64_t) BLOB), (size_t) BLOB) == 0,
"an io-prefetch blob differs from the file");
const uint8_t* s2 = source.blob_stable(l, e);
require(s2 != nullptr && std::memcmp(s2, bytes.data() + (size_t) (l * (int64_t) layer_bytes + e * (int64_t) BLOB), (size_t) BLOB) == 0,
"a blob_stable pointer differs from the file");
}
}
std::vector<uint8_t> copy((size_t) BLOB);
require(source.copy_blob(1, 6, copy.data()) &&
std::memcmp(copy.data(), bytes.data() + (size_t) (layer_bytes + 6 * BLOB), (size_t) BLOB) == 0,
"an io-prefetch copy_blob differs from the file");
const FileExpertSource::IoCounters c = source.io_counters();
require(c.cached_bytes + c.uncached_bytes > 0, "no residency counted");
std::cout << "io prefetch: " << c.pf_read_blobs << " read ahead, " << c.pf_used << " used, " << c.pf_resident_skips
<< " skipped, pread " << c.pread_bytes << " B: bytes PASS\n";
#else
require(!source.io_prefetch(), "io prefetch on a platform without it");
#endif
source.close();
}
void test_rotating_source(bool rotate, bool pin) {
using namespace strata::core;
using namespace strata::kernels::cpu;
#if defined(_WIN32)
_putenv_s("STRATA_EXCHANGE_ROTATE", rotate ? "1" : "0");
#else
setenv("STRATA_EXCHANGE_ROTATE", rotate ? "1" : "0", 1);
#endif
TempDirectory dir;
std::string err;
require(expert_layout_load(dir.path.string(), 1, 5, err), err);
const size_t bytes = (size_t)BLOB;
std::vector<std::vector<uint8_t>> truth(5, std::vector<uint8_t>(bytes));
{
std::ofstream out(dir.path / "experts.bin", std::ios::binary);
for (size_t e = 0; e < truth.size(); ++e) {
for (size_t i = 0; i < bytes; ++i) truth[e][i] = (uint8_t)((i * 131u + e * 37u) ^ (i >> 8));
out.write((const char*)truth[e].data(), (std::streamsize)bytes);
}
require((bool)out, "fixture write failed");
}
FileExpertSource src;
ExpertCache cache;
require(src.open(dir.path.string(), 1, 5, err), err);
require(cache.open(2, 1, 5, BLOB, err), err);
for (int e : {2, 3}) {
const int slot = cache.admit(0, e);
require(slot >= 0 && cache.fill_slot_blocking(slot, truth[e].data(), err), err);
}
const std::vector<std::pair<int32_t, int32_t>> rank{{0,0},{0,1},{0,2},{0,3},{0,4}};
// Keep expert 4 on the file tier to check that the fallback survives rotation.
require(src.pin_cache_complement(cache, err, pin, {}, -1, 0, 2 * bytes, &rank), err);
require(src.reserve_exchanges(2, err), err);
require(src.exchange_rotation() == (rotate && pin), "rotation activation/fallback wrong");
require(src.reserve_exchanges(1, err), "smaller capacity rejected");
if (rotate && pin) require(!src.reserve_exchanges(3, err), "live arena growth accepted");
int incoming[2] = {0, 1}, outgoing[2] = {2, 3};
std::vector<uint8_t> actual(bytes);
double commit_ms = 0;
for (int round = 0; round < 32; ++round) {
const uint8_t* prior[2]{};
uint8_t* eviction[2]{};
for (int q = 0; q < 2; ++q) {
prior[q] = src.blob(0, incoming[q]); eviction[q] = src.exchange_buffer(q);
require(prior[q] && !std::memcmp(prior[q], truth[incoming[q]].data(), bytes), "wrong incoming bytes");
require(cudaMemcpy(eviction[q], cache.device_slot(q), bytes, cudaMemcpyDeviceToHost) == cudaSuccess,
"D2H eviction failed");
require(src.stage_exchange(0, incoming[q], outgoing[q], q), "stage rejected");
require(!std::memcmp(src.blob(0, outgoing[q]), truth[outgoing[q]].data(), bytes), "staged override wrong");
require(cache.fill_slot_blocking(q, prior[q], err), err); // H2D completed before commit
}
require(!src.stage_exchange(0, incoming[0], outgoing[0], 0), "duplicate stage accepted");
require(!src.stage_exchange(0, incoming[0], 5, 0), "invalid expert accepted");
const auto t0 = std::chrono::steady_clock::now();
require(src.commit_exchanges() == 2, "commit count wrong");
commit_ms += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
require(src.commit_exchanges() == 0, "empty commit not empty");
for (int q = 0; q < 2; ++q) {
require(!src.has_resident(0, incoming[q]) && src.has_resident(0, outgoing[q]), "residency wrong");
const uint8_t* held = src.blob(0, outgoing[q]);
require(!std::memcmp(held, truth[outgoing[q]].data(), bytes), "evicted bytes corrupted");
require(held == ((rotate && pin) ? eviction[q] : prior[q]), "wrong storage selected");
if (rotate && pin) require(src.exchange_buffer(q) == prior[q], "old input not recycled");
require(src.copy_blob(0, outgoing[q], actual.data()) && actual == truth[outgoing[q]], "copy_blob mismatch");
require(!src.transient(0, outgoing[q]), "resident became transient");
if (pin) {
require(src.pinned(0, outgoing[q]) && src.device_alias(0, outgoing[q]), "mapping lost");
require(cudaMemcpy(actual.data(), src.device_alias(0, outgoing[q]), bytes, cudaMemcpyDeviceToHost) == cudaSuccess,
"device alias read failed");
require(actual == truth[outgoing[q]], "CUDA alias points at stale expert");
}
require(cache.verify_slot(q, truth[incoming[q]].data(), err), err);
std::swap(incoming[q], outgoing[q]);
}
require(!src.has_resident(0, 4) && src.copy_blob(0, 4, actual.data()) && actual == truth[4], "file fallback changed");
}
require(src.exchanges() == 64, "exchange accounting wrong");
require(src.rotated_exchanges() == ((rotate && pin) ? 64u : 0u), "rotation accounting wrong");
require(src.avoided_exchange_copy_bytes() == ((rotate && pin) ? 64 * bytes : 0u), "avoided bytes wrong");
std::cout << "rotation integration: rotate=" << rotate << " pinned=" << pin
<< " exchanges=64 commit_ms=" << commit_ms << " exact_bytes=PASS\n";
src.close();
require(!src.exchange_rotation() && !src.exchange_buffer(0), "close retained storage");
require(src.open(dir.path.string(), 1, 5, err), err);
require(!src.has_resident(0, 0) && !src.exchange_rotation(), "reopen retained residency");
}
// #2134: a Foresight slot copied from an expert's RAM place before an exchange rewrote that place is never served; the
// experts an exchange moves get no copy between commit_copies and commit_flip; and a dropped slot that a window may
// still read waits `depth` finished windows before it is refilled.
void test_foresight_exchange(bool two_phase) {
using namespace strata::core;
using namespace strata::kernels::cpu;
#if defined(_WIN32)
_putenv_s("STRATA_EXCHANGE_ROTATE", "0");
#else
setenv("STRATA_EXCHANGE_ROTATE", "0", 1);
#endif
TempDirectory dir;
std::string err;
require(expert_layout_load(dir.path.string(), 1, 5, err), err);
const size_t bytes = (size_t) BLOB;
std::vector<std::vector<uint8_t>> truth(5, std::vector<uint8_t>(bytes));
{
std::ofstream out(dir.path / "experts.bin", std::ios::binary);
for (size_t e = 0; e < truth.size(); ++e) {
for (size_t i = 0; i < bytes; ++i) truth[e][i] = (uint8_t) ((i * 131u + e * 37u) ^ (i >> 8));
out.write((const char*) truth[e].data(), (std::streamsize) bytes);
}
require((bool) out, "fixture write failed");
}
FileExpertSource src;
ExpertCache cache;
require(src.open(dir.path.string(), 1, 5, err), err);
require(cache.open(2, 1, 5, BLOB, err), err);
for (int e : {2, 3}) {
const int slot = cache.admit(0, e);
require(slot >= 0 && cache.fill_slot_blocking(slot, truth[e].data(), err), err);
}
const std::vector<std::pair<int32_t, int32_t>> rank{{0,0},{0,1},{0,2},{0,3},{0,4}};
require(src.pin_cache_complement(cache, err, true, {}, -1, 0, 2 * bytes, &rank), err);
require(src.reserve_exchanges(1, err), err);
require(src.pinned(0, 0), "expert 0 is not in the pinned RAM copy");
int dev = 0;
cudaGetDevice(&dev);
ForesightSwap fs;
require(fs.init(1, 1, 5, {0}, {dev}, err), err); // one slot, depth 4
fs.verify = true;
src.set_foresight_swap(&fs);
unsigned long long ptr = 0;
auto landed = [&](int32_t e) {
for (int i = 0; i < 5000; ++i) {
if (fs.take(0, e, ptr)) return true;
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
return false;
};
fs.note_miss(&src, 0, 0); // expert 0 -> the slot, from its RAM place
require(landed(0) && fs.verify_ok == 1, "the first copy did not land");
// the adaptive tier swaps expert 0 into cache slot 0 and evicts expert 2 into the exchange buffer
uint8_t* eviction = src.exchange_buffer(0);
const uint8_t* prior = src.blob(0, 0);
require(cudaMemcpy(eviction, cache.device_slot(0), bytes, cudaMemcpyDeviceToHost) == cudaSuccess, "D2H eviction failed");
require(src.stage_exchange(0, 0, 2, 0), "stage rejected");
require(cache.fill_slot_blocking(0, prior, err), err);
if (two_phase) {
src.commit_copies(); // 0's place holds 2's bytes from here on
const uint64_t c0 = fs.copies;
fs.completed.fetch_add(8);
fs.note_miss(&src, 0, 2);
fs.predict(&src, 0, 0);
require(fs.copies == c0, "an expert was copied while its exchange was half done");
require(!fs.take(0, 0, ptr), "a slot dropped by commit_copies was served");
require(src.commit_flip() == 1, "flip count wrong");
} else {
require(src.commit_exchanges() == 1, "commit count wrong");
}
require(fs.dropped == 1, "the exchange did not drop the slot");
require(!fs.take(0, 0, ptr), "a slot copied before the exchange was served");
if (!two_phase) { // the slot was served at window 0: no refill before window 0 + depth
const uint64_t c0 = fs.copies, b0 = fs.busy;
fs.note_miss(&src, 0, 2);
require(fs.copies == c0 && fs.busy == b0 + 1, "a dropped slot was refilled while a window may still read it");
fs.completed.fetch_add(4);
}
require(src.has_resident(0, 2) && !src.has_resident(0, 0), "residency wrong");
fs.note_miss(&src, 0, 2); // expert 2, now at 0's old place
require(landed(2), "the copy after the exchange did not land");
std::vector<uint8_t> back(bytes);
require(cudaMemcpy(back.data(), (const void*) ptr, bytes, cudaMemcpyDeviceToHost) == cudaSuccess && back == truth[2],
"the slot does not hold expert 2's bytes");
require(fs.verify_bad == 0 && fs.verify_ok == 2, "verify counted a bad slot");
src.set_foresight_swap(nullptr);
std::cout << "foresight exchange (" << (two_phase ? "commit_copies + commit_flip" : "commit_exchanges")
<< "): dropped " << fs.dropped << ", " << fs.report() << ": PASS\n";
src.close();
}
// STRATA_RAM_ADAPT: an expert read from the file takes the RAM place of another of its layer.
void test_replace_resident(bool pin) {
using namespace strata::core;
using namespace strata::kernels::cpu;
#if defined(_WIN32)
_putenv_s("STRATA_EXCHANGE_ROTATE", "0");
#else
setenv("STRATA_EXCHANGE_ROTATE", "0", 1);
#endif
TempDirectory dir;
std::string err;
require(expert_layout_load(dir.path.string(), 1, 5, err), err);
const size_t bytes = (size_t)BLOB;
std::vector<std::vector<uint8_t>> truth(5, std::vector<uint8_t>(bytes));
{
std::ofstream out(dir.path / "experts.bin", std::ios::binary);
for (size_t e = 0; e < truth.size(); ++e) {
for (size_t i = 0; i < bytes; ++i) truth[e][i] = (uint8_t)((i * 131u + e * 37u) ^ (i >> 8));
out.write((const char*)truth[e].data(), (std::streamsize)bytes);
}
require((bool)out, "fixture write failed");
}
FileExpertSource src;
ExpertCache cache;
require(src.open(dir.path.string(), 1, 5, err), err);
require(cache.open(2, 1, 5, BLOB, err), err);
for (int e : {2, 3}) {
const int slot = cache.admit(0, e);
require(slot >= 0 && cache.fill_slot_blocking(slot, truth[e].data(), err), err);
}
const std::vector<std::pair<int32_t, int32_t>> rank{{0,0},{0,1},{0,2},{0,3},{0,4}};
// experts 2 and 3 are in VRAM, the RAM budget holds 0 and 1, expert 4 stays on the file tier
require(src.pin_cache_complement(cache, err, pin, {}, -1, 0, 2 * bytes, &rank), err);
require(src.reserve_exchanges(1, err), err);
require(src.has_resident(0, 0) && src.has_resident(0, 1) && !src.has_resident(0, 4), "fixture residency wrong");
std::vector<uint8_t> actual(bytes);
auto holds = [&](int e) {
const uint8_t* b = src.blob(0, e);
return b != nullptr && !std::memcmp(b, truth[(size_t)e].data(), bytes);
};
require(!src.replace_resident(0, 1, 0), "an expert already in RAM accepted");
require(!src.replace_resident(0, 4, 2), "a place the RAM does not hold accepted");
require(!src.replace_resident(0, 4, 4) && !src.replace_resident(0, 5, 0) && !src.replace_resident(1, 4, 0),
"invalid arguments accepted");
require(src.ram_adapts() == 0 && holds(0) && holds(1) && holds(4), "a refused call changed the copy");
// a staged exchange owns the tables until it commits
require(cudaMemcpy(src.exchange_buffer(0), cache.device_slot(0), bytes, cudaMemcpyDeviceToHost) == cudaSuccess,
"D2H eviction failed");
require(src.stage_exchange(0, 0, 2, 0), "stage rejected");
require(!src.replace_resident(0, 4, 1), "accepted beside a staged exchange");
require(cache.fill_slot_blocking(0, truth[0].data(), err), err);
require(src.commit_exchanges() == 1, "commit count wrong"); // RAM: 2 (in 0's place) and 1; VRAM: 0 and 3
const uint8_t* place = src.blob(0, 1);
require(src.replace_resident(0, 4, 1), "replace rejected");
require(src.ram_adapts() == 1 && src.has_resident(0, 4) && !src.has_resident(0, 1), "residency wrong");
require(src.blob(0, 4) == place && holds(4), "the incoming expert is not in the outgoing one's place");
require(holds(1) && holds(2), "another expert's bytes changed");
require(src.copy_blob(0, 1, actual.data()) && actual == truth[1], "the outgoing expert is not read from the file");
require(!src.transient(0, 4), "resident became transient");
if (pin) {
require(src.pinned(0, 4) && src.device_alias(0, 4), "mapping lost");
require(cudaMemcpy(actual.data(), src.device_alias(0, 4), bytes, cudaMemcpyDeviceToHost) == cudaSuccess,
"device alias read failed");
require(actual == truth[4], "CUDA alias points at the old expert");
}
// and back, then an exchange with the VRAM tier still works on the moved place
require(src.replace_resident(0, 1, 4) && holds(1) && src.blob(0, 1) == place && !src.has_resident(0, 4),
"replace back wrong");
require(cudaMemcpy(src.exchange_buffer(0), cache.device_slot(1), bytes, cudaMemcpyDeviceToHost) == cudaSuccess,
"D2H eviction failed");
require(src.stage_exchange(0, 1, 3, 0), "stage after replace rejected");
require(cache.fill_slot_blocking(1, truth[1].data(), err), err);
require(src.commit_exchanges() == 1 && src.has_resident(0, 3) && holds(3) && !src.has_resident(0, 1),
"exchange after replace wrong");
require(src.ram_adapts() == 2, "accounting wrong");
// a staged incoming blob: RAM {2,3}, VRAM {0,1}, 4 on the file tier
require(!src.replace_resident_staged(0, 4, 3) && src.has_resident(0, 3) && src.ram_adapts() == 2,
"an unstaged expert moved");
std::string why;
set_env("STRATA_UNBUFFERED_LOAD", "1");
const bool unbuffered = src.set_unbuffered(0, why);
set_env("STRATA_UNBUFFERED_LOAD", nullptr);
if (unbuffered && src.stages_blobs()) {
require(!src.replace_resident_staged(0, 4, 3) && holds(3), "moved before any demand read");
require(!src.has_resident(0, 4) && holds(4), "demand read wrong"); // leaves 4 in a stage buffer
require(!src.replace_resident_staged(0, 4, 0) && !src.replace_resident_staged(0, 2, 3),
"invalid staged pair accepted");
require(src.replace_resident_staged(0, 4, 3) && src.has_resident(0, 4) && !src.has_resident(0, 3),
"staged replacement residency wrong");
require(src.resident_blob(0, 4) != nullptr && !std::memcmp(src.resident_blob(0, 4), truth[4].data(), bytes),
"staged replacement bytes wrong");
require(holds(2) && holds(3) && src.ram_adapts() == 3 && src.ram_staged() == 1,
"staged replacement changed another expert");
std::cout << "replace_resident_staged: pinned=" << pin << " exact_bytes=PASS\n";
} else {
std::cout << "replace_resident_staged: pinned=" << pin << " unbuffered unavailable here (" << why << "), positive case SKIPPED\n";
}
std::cout << "replace_resident: pinned=" << pin << " exact_bytes=PASS\n";
}
} // namespace
int main(int argc, char** argv) {
try {
test_complement_plan();
test_resident_lend_region();
test_split_lend_regions();
test_resident_exchange();
test_resident_memory_budget();
test_pin_pacing();
test_wait_for_ram();
test_numa_slice_ranges();
test_cgroup_memory_budget();
test_host_memory();
test_canonical_layout();
test_unbuffered_reads();
test_io_prefetch();
#if defined(STRATA_NATIVE_EXPERTS)
test_native_variable_layout();
#endif
if (argc == 2 && std::string(argv[1]) == "--rotation-gpu") {
test_rotating_source(false, true);
test_rotating_source(true, true);
test_rotating_source(true, false);
test_foresight_exchange(false);
test_foresight_exchange(true);
test_replace_resident(true);
test_replace_resident(false);
}
std::cout << "file_expert_source_test: PASS\n";
return 0;
} catch (const std::exception& error) {
std::cerr << "file_expert_source_test: " << error.what() << '\n';
return 1;
}
}