Skip to content

armgit / PS5LM

Public Imported from GitHub
0

llama.cpp on a jailbroken PS5: run any GGUF model natively on the console, starting with Qwen 3.8

llama.cpp 04f7226: decision models on /v1/systemone, MiniCPM-V 4.7

Mert Cobanov32a18d9c591e
diff --git a/ps5/app/model_plan.cpp b/ps5/app/model_plan.cpp
index b58aab6..3abef0d 100644
--- a/ps5/app/model_plan.cpp
+++ b/ps5/app/model_plan.cpp
@@ -213,6 +213,7 @@ ModelInfo parse_model_info(Reader r, const std::string & path, uint64_t file_byt
m.name = strs["general.name"];
}
const std::string a = m.arch + ".";
+ m.decision = strs[a + "decision.type"];
auto num = [&](const std::string & k, uint64_t def = 0) {
auto it = ints.find(a + k);
return it == ints.end() ? def : it->second;
@@ -300,6 +301,9 @@ const std::vector<Preset> & presets() {
}
const Preset * find_preset(const ModelInfo & m) {
+ if (!m.decision.empty()) {
+ return nullptr; // no sampling: a decision model reads its answers off one forward pass
+ }
const std::string hay = lower(m.file_name + " " + m.name + " " + m.arch);
for (const auto & p : presets()) {
if (hay.find(p.match) != std::string::npos) {
@@ -315,6 +319,10 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, const
if (preset && preset->max_ctx) {
ctx_cap = std::min(ctx_cap, preset->max_ctx);
}
+ // Decision prompts are short; 4k keeps the cache small for the wide batch below.
+ if (!m.decision.empty()) {
+ ctx_cap = std::min<uint32_t>(ctx_cap, 4096);
+ }
if (m.n_ctx_train) {
ctx_cap = std::min(ctx_cap, m.n_ctx_train);
}
@@ -324,7 +332,12 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, const
// one prompt checkpoint of the recurrent state. Calibrated on the console:
// Qwen3.8-27B UD-Q2_K_XL at 64k with a q4_0 cache ran with 11.38 GiB free.
// ponytail: 0.25 GiB for the image encoder's buffers is a guess; measure on the console.
- p.reserve_gib = 0.75 + p.state_gib + (m.mmproj.empty() ? 0.0 : 0.25);
+ // Decision models (laya, clef, vllm-d3, lfm2-d1-omni) and non-causal image
+ // encoders (mtmd caps an image to the ubatch) need a whole prompt or image in
+ // one ubatch, so they get 2048, the size #13 ran on the console.
+ const bool wide = !m.decision.empty() || !m.mmproj.empty();
+ // shortcut: 0.5 GiB more compute buffer for the 2048 ubatch is a guess; measure on the console.
+ p.reserve_gib = 0.75 + p.state_gib + (m.mmproj.empty() ? 0.0 : 0.25) + (wide ? 0.5 : 0.0);
// The longest context first, and at each length the best cache that fits.
static const uint32_t ctxs[] = { 131072, 65536, 32768, 16384, 8192, 4096 };
@@ -364,13 +377,14 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, const
p.why = why;
return p;
}
- std::snprintf(why, sizeof(why), "%uk context, %s cache%s: %.1f of %.1f GiB", p.ctx / 1024, p.kv_type.c_str(),
- m.mmproj.empty() ? "" : ", images", p.total_gib, budget_gib);
+ std::snprintf(why, sizeof(why), "%s%uk context, %s cache%s: %.1f of %.1f GiB",
+ m.decision.empty() ? "" : ("decision model (" + m.decision + "), ").c_str(), p.ctx / 1024,
+ p.kv_type.c_str(), m.mmproj.empty() ? "" : ", images", p.total_gib, budget_gib);
p.why = why;
p.args = { "-m", m.path, "-ngl", "999", "-fit", "off", "-lm", "none",
"-c", std::to_string(p.ctx), "-fa", "on", "-ctk", p.kv_type, "-ctv", p.kv_type,
- "-b", "256", "-ub", "64", "-np", "1",
+ "-b", wide ? "2048" : "256", "-ub", wide ? "2048" : "64", "-np", "1",
// One prompt checkpoint and no RAM cache: llama-server's defaults
// (32 checkpoints, 8 GiB) are for a PC, and the CPU shares this pool.
"-ctxcp", "1", "-cram", "0",
diff --git a/ps5/app/model_plan.hpp b/ps5/app/model_plan.hpp
index befbcbc..f76c942 100644
--- a/ps5/app/model_plan.hpp
+++ b/ps5/app/model_plan.hpp
@@ -21,6 +21,7 @@ struct ModelInfo {
std::string file_name;
std::string name; // general.name, or the file name
std::string arch; // general.architecture
+ std::string decision; // "<arch>.decision.type" (openjev, clef, lfm2-d1, ...): answers /v1/systemone, not chat
uint64_t file_bytes = 0;
uint32_t n_layer = 0;
uint32_t n_head_kv = 0;
diff --git a/ps5/app/model_plan_test.cpp b/ps5/app/model_plan_test.cpp
index 425f0fc..681ac10 100644
--- a/ps5/app/model_plan_test.cpp
+++ b/ps5/app/model_plan_test.cpp
@@ -1,20 +1,73 @@
-// Host check for model_plan.cpp against real GGUF files:
+// Host check for model_plan.cpp:
// c++ -std=c++17 -O1 ps5/app/model_plan_test.cpp ps5/app/model_plan.cpp -o build/model_plan_test
-// build/model_plan_test models
+// build/model_plan_test synthetic headers only
+// build/model_plan_test models also the real GGUF files in models/
// Uses the budget the console reported (11.38 GiB free) and the two
// configurations measured there.
// SPDX-License-Identifier: GPL-3.0-or-later
#include "model_plan.hpp"
+#include <algorithm>
#include <cassert>
+#include <cstdint>
#include <cstdio>
#include <string>
+namespace {
+
+// A GGUF header with string and uint32 metadata, no tensors.
+struct Gguf {
+ std::string kvs;
+ uint64_t n = 0;
+ template <class T> static std::string raw(T v) { return std::string((const char *) &v, sizeof(v)); }
+ static std::string str(const std::string & v) { return raw<uint64_t>(v.size()) + v; }
+ Gguf & kv(const std::string & k, const std::string & v) { kvs += str(k) + raw<uint32_t>(8) + str(v); n++; return *this; }
+ Gguf & kv(const std::string & k, uint32_t v) { kvs += str(k) + raw<uint32_t>(4) + raw(v); n++; return *this; }
+ // magic, version 3, tensor count, metadata count, metadata
+ std::string bytes() const { return "GGUF" + raw<uint32_t>(3) + raw<uint64_t>(0) + raw(n) + kvs; }
+};
+
+bool has_args(const ps5lm::Plan & p, const char * flag, const char * value) {
+ auto it = std::find(p.args.begin(), p.args.end(), flag);
+ return it != p.args.end() && it + 1 != p.args.end() && *(it + 1) == value;
+}
+
+} // namespace
+
int main(int argc, char ** argv) {
- const std::string dir = argc > 1 ? argv[1] : "models";
const double budget = 11.38;
+
+ // A decision model (Clef is Qwen 3.5 9B with a decision head): found from
+ // "<arch>.decision.type", a wide ubatch, a 4k cap and no chat sampling.
+ Gguf clef;
+ clef.kv("general.architecture", "qwen35").kv("general.name", "Clef Flash Qwen3.5 9B")
+ .kv("qwen35.block_count", 32u).kv("qwen35.attention.head_count", 16u)
+ .kv("qwen35.attention.head_count_kv", 4u).kv("qwen35.embedding_length", 4096u)
+ .kv("qwen35.context_length", 262144u).kv("qwen35.decision.type", "clef");
+ const ps5lm::ModelInfo d = ps5lm::read_model_header("clef-flash-Q4_K_M.gguf", clef.bytes(), 5ull << 30);
+ assert(d.ok && d.decision == "clef");
+ assert(!ps5lm::find_preset(d));
+ const ps5lm::Plan dp = ps5lm::plan_model(d, budget);
+ assert(dp.fits && dp.ctx == 4096 && has_args(dp, "-ub", "2048") && has_args(dp, "-b", "2048"));
+ assert(dp.why.find("decision model (clef)") == 0);
+
+ // The same model without the decision key is a chat model with the narrow batch.
+ Gguf chat;
+ chat.kv("general.architecture", "qwen35").kv("general.name", "Qwen3.5 9B")
+ .kv("qwen35.block_count", 32u).kv("qwen35.attention.head_count", 16u)
+ .kv("qwen35.attention.head_count_kv", 4u).kv("qwen35.embedding_length", 4096u)
+ .kv("qwen35.context_length", 262144u);
+ const ps5lm::ModelInfo c = ps5lm::read_model_header("Qwen3.5-9B-Q4_K_M.gguf", chat.bytes(), 5ull << 30);
+ assert(c.ok && c.decision.empty() && ps5lm::find_preset(c));
+ const ps5lm::Plan cp = ps5lm::plan_model(c, budget);
+ assert(cp.fits && cp.ctx > 4096 && has_args(cp, "-ub", "64"));
+
+ if (argc < 2) {
+ std::puts("model_plan_test: ok (synthetic only)");
+ return 0;
+ }
bool saw_q2k = false, saw_iq2xxs = false;
- for (const auto & path : ps5lm::find_models({ dir })) {
+ for (const auto & path : ps5lm::find_models({ argv[1] })) {
const ps5lm::ModelInfo m = ps5lm::read_model_info(path);
assert(m.ok);
const ps5lm::Plan p = ps5lm::plan_model(m, budget);