llama.cpp on a jailbroken PS5: run any GGUF model natively on the console, starting with Qwen 3.8
llama.cpp 04f7226: decision models on /v1/systemone, MiniCPM-V 4.7
Mert Cobanov32a18d9c591e
diff --git a/ps5/app/model_plan.cpp b/ps5/app/model_plan.cppindex b58aab6..3abef0d 100644--- a/ps5/app/model_plan.cpp+++ b/ps5/app/model_plan.cpp@@ -213,6 +213,7 @@ ModelInfo parse_model_info(Reader r, const std::string & path, uint64_t file_bytm.name = strs["general.name"];}const std::string a = m.arch + ".";+ m.decision = strs[a + "decision.type"];auto num = [&](const std::string & k, uint64_t def = 0) {auto it = ints.find(a + k);return it == ints.end() ? def : it->second;@@ -300,6 +301,9 @@ const std::vector<Preset> & presets() {}const Preset * find_preset(const ModelInfo & m) {+ if (!m.decision.empty()) {+ return nullptr; // no sampling: a decision model reads its answers off one forward pass+ }const std::string hay = lower(m.file_name + " " + m.name + " " + m.arch);for (const auto & p : presets()) {if (hay.find(p.match) != std::string::npos) {@@ -315,6 +319,10 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, constif (preset && preset->max_ctx) {ctx_cap = std::min(ctx_cap, preset->max_ctx);}+ // Decision prompts are short; 4k keeps the cache small for the wide batch below.+ if (!m.decision.empty()) {+ ctx_cap = std::min<uint32_t>(ctx_cap, 4096);+ }if (m.n_ctx_train) {ctx_cap = std::min(ctx_cap, m.n_ctx_train);}@@ -324,7 +332,12 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, const// one prompt checkpoint of the recurrent state. Calibrated on the console:// Qwen3.8-27B UD-Q2_K_XL at 64k with a q4_0 cache ran with 11.38 GiB free.// ponytail: 0.25 GiB for the image encoder's buffers is a guess; measure on the console.- p.reserve_gib = 0.75 + p.state_gib + (m.mmproj.empty() ? 0.0 : 0.25);+ // Decision models (laya, clef, vllm-d3, lfm2-d1-omni) and non-causal image+ // encoders (mtmd caps an image to the ubatch) need a whole prompt or image in+ // one ubatch, so they get 2048, the size #13 ran on the console.+ const bool wide = !m.decision.empty() || !m.mmproj.empty();+ // shortcut: 0.5 GiB more compute buffer for the 2048 ubatch is a guess; measure on the console.+ p.reserve_gib = 0.75 + p.state_gib + (m.mmproj.empty() ? 0.0 : 0.25) + (wide ? 0.5 : 0.0);// The longest context first, and at each length the best cache that fits.static const uint32_t ctxs[] = { 131072, 65536, 32768, 16384, 8192, 4096 };@@ -364,13 +377,14 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, constp.why = why;return p;}- std::snprintf(why, sizeof(why), "%uk context, %s cache%s: %.1f of %.1f GiB", p.ctx / 1024, p.kv_type.c_str(),- m.mmproj.empty() ? "" : ", images", p.total_gib, budget_gib);+ std::snprintf(why, sizeof(why), "%s%uk context, %s cache%s: %.1f of %.1f GiB",+ m.decision.empty() ? "" : ("decision model (" + m.decision + "), ").c_str(), p.ctx / 1024,+ p.kv_type.c_str(), m.mmproj.empty() ? "" : ", images", p.total_gib, budget_gib);p.why = why;p.args = { "-m", m.path, "-ngl", "999", "-fit", "off", "-lm", "none","-c", std::to_string(p.ctx), "-fa", "on", "-ctk", p.kv_type, "-ctv", p.kv_type,- "-b", "256", "-ub", "64", "-np", "1",+ "-b", wide ? "2048" : "256", "-ub", wide ? "2048" : "64", "-np", "1",// One prompt checkpoint and no RAM cache: llama-server's defaults// (32 checkpoints, 8 GiB) are for a PC, and the CPU shares this pool."-ctxcp", "1", "-cram", "0",diff --git a/ps5/app/model_plan.hpp b/ps5/app/model_plan.hppindex befbcbc..f76c942 100644--- a/ps5/app/model_plan.hpp+++ b/ps5/app/model_plan.hpp@@ -21,6 +21,7 @@ struct ModelInfo {std::string file_name;std::string name; // general.name, or the file namestd::string arch; // general.architecture+ std::string decision; // "<arch>.decision.type" (openjev, clef, lfm2-d1, ...): answers /v1/systemone, not chatuint64_t file_bytes = 0;uint32_t n_layer = 0;uint32_t n_head_kv = 0;diff --git a/ps5/app/model_plan_test.cpp b/ps5/app/model_plan_test.cppindex 425f0fc..681ac10 100644--- a/ps5/app/model_plan_test.cpp+++ b/ps5/app/model_plan_test.cpp@@ -1,20 +1,73 @@-// Host check for model_plan.cpp against real GGUF files:+// Host check for model_plan.cpp:// c++ -std=c++17 -O1 ps5/app/model_plan_test.cpp ps5/app/model_plan.cpp -o build/model_plan_test-// build/model_plan_test models+// build/model_plan_test synthetic headers only+// build/model_plan_test models also the real GGUF files in models/// Uses the budget the console reported (11.38 GiB free) and the two// configurations measured there.// SPDX-License-Identifier: GPL-3.0-or-later#include "model_plan.hpp"+#include <algorithm>#include <cassert>+#include <cstdint>#include <cstdio>#include <string>+namespace {++// A GGUF header with string and uint32 metadata, no tensors.+struct Gguf {+ std::string kvs;+ uint64_t n = 0;+ template <class T> static std::string raw(T v) { return std::string((const char *) &v, sizeof(v)); }+ static std::string str(const std::string & v) { return raw<uint64_t>(v.size()) + v; }+ Gguf & kv(const std::string & k, const std::string & v) { kvs += str(k) + raw<uint32_t>(8) + str(v); n++; return *this; }+ Gguf & kv(const std::string & k, uint32_t v) { kvs += str(k) + raw<uint32_t>(4) + raw(v); n++; return *this; }+ // magic, version 3, tensor count, metadata count, metadata+ std::string bytes() const { return "GGUF" + raw<uint32_t>(3) + raw<uint64_t>(0) + raw(n) + kvs; }+};++bool has_args(const ps5lm::Plan & p, const char * flag, const char * value) {+ auto it = std::find(p.args.begin(), p.args.end(), flag);+ return it != p.args.end() && it + 1 != p.args.end() && *(it + 1) == value;+}++} // namespace+int main(int argc, char ** argv) {- const std::string dir = argc > 1 ? argv[1] : "models";const double budget = 11.38;++ // A decision model (Clef is Qwen 3.5 9B with a decision head): found from+ // "<arch>.decision.type", a wide ubatch, a 4k cap and no chat sampling.+ Gguf clef;+ clef.kv("general.architecture", "qwen35").kv("general.name", "Clef Flash Qwen3.5 9B")+ .kv("qwen35.block_count", 32u).kv("qwen35.attention.head_count", 16u)+ .kv("qwen35.attention.head_count_kv", 4u).kv("qwen35.embedding_length", 4096u)+ .kv("qwen35.context_length", 262144u).kv("qwen35.decision.type", "clef");+ const ps5lm::ModelInfo d = ps5lm::read_model_header("clef-flash-Q4_K_M.gguf", clef.bytes(), 5ull << 30);+ assert(d.ok && d.decision == "clef");+ assert(!ps5lm::find_preset(d));+ const ps5lm::Plan dp = ps5lm::plan_model(d, budget);+ assert(dp.fits && dp.ctx == 4096 && has_args(dp, "-ub", "2048") && has_args(dp, "-b", "2048"));+ assert(dp.why.find("decision model (clef)") == 0);++ // The same model without the decision key is a chat model with the narrow batch.+ Gguf chat;+ chat.kv("general.architecture", "qwen35").kv("general.name", "Qwen3.5 9B")+ .kv("qwen35.block_count", 32u).kv("qwen35.attention.head_count", 16u)+ .kv("qwen35.attention.head_count_kv", 4u).kv("qwen35.embedding_length", 4096u)+ .kv("qwen35.context_length", 262144u);+ const ps5lm::ModelInfo c = ps5lm::read_model_header("Qwen3.5-9B-Q4_K_M.gguf", chat.bytes(), 5ull << 30);+ assert(c.ok && c.decision.empty() && ps5lm::find_preset(c));+ const ps5lm::Plan cp = ps5lm::plan_model(c, budget);+ assert(cp.fits && cp.ctx > 4096 && has_args(cp, "-ub", "64"));++ if (argc < 2) {+ std::puts("model_plan_test: ok (synthetic only)");+ return 0;+ }bool saw_q2k = false, saw_iq2xxs = false;- for (const auto & path : ps5lm::find_models({ dir })) {+ for (const auto & path : ps5lm::find_models({ argv[1] })) {const ps5lm::ModelInfo m = ps5lm::read_model_info(path);assert(m.ok);const ps5lm::Plan p = ps5lm::plan_model(m, budget);