Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: the conversation cache is sized from the free RAM when --conversation-cache-mib is not given (#3493): 40% of what is left above the parking floor, at most 8192, off under 1024, one log line; an explicit value (0 too) is kept; harness_reuse_probe.py; docs
Niko1221f0010a607f32
diff --git a/docs/DETAILS.md b/docs/DETAILS.mdindex 2b8e96591..92ba6924e 100644--- a/docs/DETAILS.md+++ b/docs/DETAILS.md@@ -1177,7 +1177,7 @@ without it. The engine's own key is `pin=N` on the `GEN` line.in a bounded 8 GiB host-RAM cache. This preserves controller/worker histories whentheir requests alternate; it does not execute requests concurrently. No client sessionID is required: only exact token/image prefixes with matching steering mode are reused.-The default budget is 0 (disabled); `--prompt-cache 0` also disables parking.+Without `--conversation-cache-mib` the budget is sized from the RAM that is free once the model is loaded (0.2.3, #3493): 40% of what is left above `--conversation-cache-min-free-mib`, at most 8192, in steps of 256, and off when that is under 1024; the engine log says which in one line. An agent Harness that alternates a long main conversation with shorter ones (sub-agents, summaries) used to read every prompt again (`Reused` 0); with the cache on, the next turn of each one reuses its prompt. `--conversation-cache-mib N` sets it and `--conversation-cache-mib 0` turns it off; `--prompt-cache 0` also disables parking.With `--layer-split`, a parked conversation holds one image per stage (each later stage'ssession and its share of the conversation checkpoints), every image is validated beforeanything is overwritten, and each is restored on its own GPU. Verified on a 4-GPU split:diff --git a/include/strata/core/conversation_memory.hpp b/include/strata/core/conversation_memory.hppindex 81ea0b077..97fc1befd 100644--- a/include/strata/core/conversation_memory.hpp+++ b/include/strata/core/conversation_memory.hpp@@ -64,6 +64,19 @@ std::optional<HostMemoryReading> sample_host_memory();std::optional<uint64_t> available_host_bytes();+/// #3493: the conversation cache budget (MiB) --serve uses when --conversation-cache-mib was not given, from the RAM that is available+/// once the model is loaded: 40% of what is left above the parking floor, at most 8192 (four 100K-token agent conversations were+/// parked in 8 GiB in the report), in steps of 256, and 0 (off) when less than 1024 would be left to give. Unknown telemetry: 0.+inline int64_t auto_conversation_cache_mib(std::optional<uint64_t> available, int64_t min_free_mib) {+ if (!available) return 0;+ const int64_t usable = (int64_t) (*available >> 20) - min_free_mib;+ if (usable <= 0) return 0;+ int64_t mib = usable * 2 / 5;+ if (mib > 8192) mib = 8192;+ mib -= mib % 256;+ return mib >= 1024 ? mib : 0;+}+std::optional<uint64_t> conversation_mem_available(std::istream& meminfo);inline bool conversation_memory_admit(std::optional<uint64_t> available,diff --git a/setup.py b/setup.pyindex bf800d3c3..b39f83101 100644--- a/setup.py+++ b/setup.py@@ -6918,10 +6918,6 @@ def bench_tips(args, env, ram: float, model_ram_gb: float, vram_gb: float, visiotips.append(f"tip: on a PC with {ram:.0f} GB of RAM, \"env\": {{\"STRATA_RESIDENT_HEADROOM_GIB\": \"6\"}} in the ""config kept decode speed in a community benchmark and left the system 2 GiB more (#834); the ""default is 4")- if "--conversation-cache-mib" not in args and ram - model_ram_gb >= AGENT_CACHE_FREE_GB:- tips.append(f"tip: for several agents or clients at once, --conversation-cache-mib {AGENT_CACHE_MIB} in the "- "config's args keeps each one's conversation; without it they were measured re-reading ~90% of "- "their prompts (#882 #440; docs/DETAILS.md, Multiple conversations)")if vision == "gpu" and 0 < vram_gb <= SMALL_VISION_VRAM_GB:tips.append("tip: the image encoder on the GPU makes a prompt's chunks smaller on a card this size (243 vs 750 ""tok/s measured, #469); run setup again with --vision cpu to read prompts ~3x faster, at about "diff --git a/src/core/conversation_memory_test.cpp b/src/core/conversation_memory_test.cppindex c95e70bfa..35d18df13 100644--- a/src/core/conversation_memory_test.cpp+++ b/src/core/conversation_memory_test.cpp@@ -49,6 +49,16 @@ int main() {std::numeric_limits<uint64_t>::max(), 0), "maximal exact bound");check(!conversation_memory_admit(std::numeric_limits<uint64_t>::max(),std::numeric_limits<uint64_t>::max(), 1), "maximal sum overflow rejected");+ // #3493: the budget used when --conversation-cache-mib is not given+ const auto GiB = [](uint64_t n) { return n << 30; };+ check(auto_conversation_cache_mib({}, 2560) == 0, "unknown telemetry: no cache");+ check(auto_conversation_cache_mib(GiB(2), 2560) == 0, "below the parking floor: off");+ check(auto_conversation_cache_mib(GiB(4), 2560) == 0, "4 GiB free: 40% of 1.5 GiB is under 1 GiB: off");+ check(auto_conversation_cache_mib(GiB(8), 2560) == 2048, "8 GiB free: 40% of 5.5 GiB, in steps of 256");+ check(auto_conversation_cache_mib(GiB(20), 2560) == 7168, "20 GiB free: 40% of 17.5 GiB");+ check(auto_conversation_cache_mib(GiB(40), 2560) == 8192, "capped at 8 GiB");+ check(auto_conversation_cache_mib(GiB(1000), 0) == 8192, "a big server: still the cap");+ check(auto_conversation_cache_mib(GiB(40), 1000000) == 0, "a floor above the RAM: off, no underflow");// Test the real provider without assuming any particular amount of free RAM.const auto available = available_host_bytes();check(!available || conversation_memory_admit(available, 0, 0), "provider returns bytes or unknown");diff --git a/src/program/generate.cpp b/src/program/generate.cppindex bd8a398b6..4cd39ac5a 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -722,7 +722,7 @@ struct Options {/// prompt again, the v0.1.2 behaviour). One is the GDN recurrence of the 36 layers, the QSA indexer tails and/// the PLE history (~118 MB of host RAM); the KV cache itself is positional and stays where it is.int prompt_cache = 6;- int64_t conversation_cache_mib = 0; // opt-in host RAM for independent conversations+ int64_t conversation_cache_mib = -1; // host RAM for independent conversations; -1: not given, --serve sizes it from the free RAM (#3493), 0: offint conversation_cache_slots = 4;int64_t conversation_cache_min_free_mib = 2560;/// --serve: park only conversations of at least this many tokens. Every parked conversation costs one of@@ -881,7 +881,7 @@ void usage() {" --vision --serve takes images too (GENI requests; embeddings from strata-vision)\n"" --prompt-cache N --serve: keep N conversation checkpoints between requests (default 6, ~118 MB\n"" of RAM each; 0 = read every prompt from the start)\n"- " --conversation-cache-mib N --serve: RAM budget for parked conversations (default 0 = off)\n"+ " --conversation-cache-mib N --serve: RAM budget for parked conversations (default: sized from the free RAM, up to 8192; 0 = off)\n"" --conversation-cache-slots N --serve: at most N parked conversations (default 4)\n"" --conversation-cache-min-free-mib N --serve: RAM floor (under MemAvailable, the cgroup limits and\n"" --memory-limit-mib) when parking, saving or restoring a session (default 2560)\n"@@ -9280,6 +9280,24 @@ int main(int argc, char** argv) {bool cvec_cached = true; // the control vector's state the live session and the checkpoints were read with// without --mtp the conversation cache stays on: a parked image carries the draft layer's K/V only when there is one// (the snapshots accept a null draft)+ // #3493: no --conversation-cache-mib given: size the cache for agent use from the RAM that is free now that the model is+ // loaded (an agent Harness alternates a long main conversation with shorter ones; with the cache off every request read+ // its whole prompt again, Reused 0 on 12 of 12). An explicit value, 0 included, is never touched.+ if (o.conversation_cache_mib < 0) {+ if (o.serve && o.prompt_cache > 0 && o.conversation_cache_slots > 0) {+ o.conversation_cache_mib = strata::core::auto_conversation_cache_mib(strata::core::available_host_bytes(),+ o.conversation_cache_min_free_mib);+ if (o.conversation_cache_mib > 0)+ std::fprintf(stderr, "strata serve: conversation cache sized from the free RAM: %lld MiB for up to %d parked "+ "conversations, so an agent's next turn reuses its prompt (--conversation-cache-mib N sets it, 0 turns it off)\n",+ (long long) o.conversation_cache_mib, o.conversation_cache_slots);+ else+ std::fprintf(stderr, "strata serve: conversation cache off: not enough free RAM to park conversations "+ "(--conversation-cache-mib N asks for one anyway)\n");+ } else {+ o.conversation_cache_mib = 0;+ }+ }strata::core::ConversationCache conversations(o.prompt_cache > 0 ? (size_t) o.conversation_cache_mib * 1024 * 1024 : 0,(size_t) o.conversation_cache_slots, o.conversation_cache_min_tokens);diff --git a/tools/harness_reuse_probe.py b/tools/harness_reuse_probe.pynew file mode 100644index 000000000..fc6bd0a48--- /dev/null+++ b/tools/harness_reuse_probe.py@@ -0,0 +1,85 @@+"""#3493: does an agent-Harness-style run reuse its prefix with the server's DEFAULT conversation cache?++ python tools/harness_reuse_probe.py http://127.0.0.1:8080 [--turns 4] [--main-tokens 6000] [--key KEY]++An agent Harness keeps a long MAIN conversation (system prompt, tools, a growing list of tool results) and, between its+turns, sends shorter side conversations (a sub-agent, a summary) with their own system prompts. Every turn of either one+extends the previous request of the same conversation. With the conversation cache off, the engine holds one prompt, so+each switch to the other conversation read the whole prompt again (the report: Reused = 0 on 12 of 12 requests).++The probe sends MAIN1 SUB1 MAIN2 SUB2 ... and reads the server's /metrics before and after each request: the tokens the+request took from the cache ("reused") out of its prompt. Exit code 0 when every follow-up turn (all but the first of+each conversation) reused more than half of its prompt, 1 otherwise. Needs an idle server; sampling is greedy, 8 new tokens.+"""+from __future__ import annotations++import argparse+import json+import sys+import time+import urllib.request+++def post(url, key, body, timeout=600):+ req = urllib.request.Request(url + "/v1/chat/completions", data=json.dumps(body).encode(),+ headers={"Content-Type": "application/json", **({"Authorization": "Bearer " + key} if key else {})})+ with urllib.request.urlopen(req, timeout=timeout) as response:+ return json.load(response)+++def metrics(url, key):+ req = urllib.request.Request(url + "/metrics", headers={"Authorization": "Bearer " + key} if key else {})+ with urllib.request.urlopen(req, timeout=20) as response:+ return json.load(response)+++def filler(tag, n_words):+ """Deterministic text of about n_words words that differs per tag (so two conversations share no prefix)."""+ words = ["%s%d" % (tag, i % 977) for i in range(n_words)]+ return " ".join(words)+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("url")+ ap.add_argument("--turns", type=int, default=4)+ ap.add_argument("--main-tokens", type=int, default=6000, help="about this many tokens in the main conversation's first request")+ ap.add_argument("--sub-tokens", type=int, default=1500)+ ap.add_argument("--step-tokens", type=int, default=500, help="what every turn adds (a tool result)")+ ap.add_argument("--key", default=None)+ a = ap.parse_args()+ url = a.url.rstrip("/")+ model = (json.load(urllib.request.urlopen(url + "/v1/models", timeout=20))["data"][0]["id"]+ if not a.key else json.load(urllib.request.urlopen(urllib.request.Request(url + "/v1/models", headers={"Authorization": "Bearer " + a.key}), timeout=20))["data"][0]["id"])+ conv = {"MAIN": [{"role": "system", "content": "You are the main coding agent. " + filler("m", a.main_tokens // 2)}],+ "SUB": [{"role": "system", "content": "You are a sub-agent that summarizes. " + filler("s", a.sub_tokens // 2)}]}+ rows, ok = [], True+ for turn in range(a.turns):+ for name, step in (("MAIN", a.step_tokens), ("SUB", a.step_tokens // 2)):+ conv[name].append({"role": "user", "content": "tool result %d: %s Reply with one word." % (turn, filler(name[0].lower() + "t%d" % turn, step // 2))})+ before = metrics(a.url.rstrip("/"), a.key).get("conversation_cache") or {}+ t0 = time.time()+ reply = post(url, a.key, {"model": model, "messages": conv[name], "max_tokens": 8, "temperature": 0,+ "chat_template_kwargs": {"enable_thinking": False}})+ took = time.time() - t0+ after = metrics(url, a.key).get("conversation_cache") or {}+ prompt = (reply.get("usage") or {}).get("prompt_tokens", 0)+ reused = int(after.get("reused_tokens", 0)) - int(before.get("reused_tokens", 0))+ conv[name].append({"role": "assistant", "content": (reply["choices"][0]["message"].get("content") or "ok")})+ first = turn == 0+ good = first or reused * 2 > prompt+ ok = ok and good+ rows.append((name, turn + 1, prompt, reused, took, good))+ print("%-4s turn %d: prompt %6d tokens, reused %6d (%3.0f%%), %5.1f s%s" % (+ name, turn + 1, prompt, reused, 100.0 * reused / max(prompt, 1), took, "" if good else " <-- not reused"), flush=True)+ follow = [r for r in rows if r[1] > 1]+ total_prompt = sum(r[2] for r in follow)+ total_reused = sum(r[3] for r in follow)+ print("follow-up turns: %d, reused %d of %d prompt tokens (%.0f%%); conversation cache: %s" % (+ len(follow), total_reused, total_prompt, 100.0 * total_reused / max(total_prompt, 1),+ json.dumps({k: v for k, v in (metrics(url, a.key).get("conversation_cache") or {}).items() if k in ("enabled", "budget_mib", "slots", "parked", "requests_reused")})))+ return 0 if ok else 1+++if __name__ == "__main__":+ sys.exit(main())