llama.cpp on a jailbroken PS5: run any GGUF model natively on the console, starting with Qwen 3.8
Chat tools and images: file tools in a scratch folder, mmproj pairing
Abhiram0df54efacdd0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 69e0730..f21c826 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -18,6 +18,13 @@ v0.1.x are the upstream payload releases by cobanov.- THERMALS tile (SoC and CPU temperature, SoC power), STORAGE tile (free spaceon /data and USB), and the CPU clock.+- Chat tools: the browser chat's model can read, write, edit and search files+ (llama-server's built-in tools), confined to `/data/PS5LM/scratch` by+ `patches/0003-tools-root.patch`. Settings has a switch, a size warning+ (default 2 GiB) and a Clear action.+- Images: a vision model's `mmproj` file beside it is loaded with it+ (`--mmproj`), and the planner counts its memory.+### Changed- One MEMORY tile replaces MEMORY and GPU MEMORY (the GPU and CPU share onepool); CONTEXT takes in the session counts.diff --git a/patches/0003-tools-root.patch b/patches/0003-tools-root.patchnew file mode 100644index 0000000..bee9bd5--- /dev/null+++ b/patches/0003-tools-root.patch@@ -0,0 +1,22 @@+PS5LM: confine llama-server's built-in tools to LLAMA_TOOLS_ROOT (the app's scratch folder).++diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp+index 12e9dbb8c..8ff130fab 100644+--- a/tools/server/server-tools.cpp++++ b/tools/server/server-tools.cpp+@@ -323,6 +323,15 @@ public:+ if (!full.has_filename() && full != full.root_path()) {+ full = full.parent_path();+ }++ // PS5LM: with LLAMA_TOOLS_ROOT set, the tools see that folder as the++ // whole filesystem: a path outside it is taken as relative to it.++ if (const char * root_env = std::getenv("LLAMA_TOOLS_ROOT"); root_env && *root_env) {++ const fs::path root = path_from_utf8(root_env).lexically_normal();++ const fs::path rel = full.lexically_relative(root);++ if (rel.empty() || *rel.begin() == "..") {++ full = (root / full.relative_path()).lexically_normal();++ }++ }+ return path_to_utf8(full);+ }+diff --git a/ps5/app/dashboard.cpp b/ps5/app/dashboard.cppindex 72499a0..8d26eb1 100644--- a/ps5/app/dashboard.cpp+++ b/ps5/app/dashboard.cpp@@ -163,7 +163,7 @@ constexpr const char * kPageNames[Dashboard::kPages] = { "Dashboard", "Settings"// ---- Settings: categories of rows, after the kit's "Control Room" ----------------enum class Setting : int { auto_load, default_model, ctx_cap, kv_type, sounds };+enum class Setting : int { auto_load, default_model, ctx_cap, kv_type, sounds, tools, scratch_limit, clear_scratch };struct SettingRow {Setting id;@@ -174,7 +174,7 @@ struct SettingRow {struct SettingCategory {const char * name;const char * about;- SettingRow rows[2];+ SettingRow rows[4];int count;};@@ -187,11 +187,16 @@ constexpr SettingCategory kCategories[] = {{ { Setting::ctx_cap, "Longest context", "The planner picks the longest context up to this that fits." },{ Setting::kv_type, "KV cache", "Auto picks the best that fits; a fixed type trades context for quality." } },2 },+ { "Chat", "The browser chat's tools, and the folder they work in.",+ { { Setting::tools, "File tools", "The model may read, write and search files in /data/PS5LM/scratch. Applies at the next load." },+ { Setting::scratch_limit, "Warn above", "A notification when the scratch folder grows past this." },+ { Setting::clear_scratch, "Clear the scratch folder", "Deletes everything the chat's tools wrote." } },+ 3 },{ "App", "The app itself.",{ { Setting::sounds, "Sounds", "Interface sounds on the TV." } },1 },};-constexpr int kCategoryCount = 3;+constexpr int kCategoryCount = 4;constexpr uint32_t kCtxSteps[] = { 4096, 8192, 16384, 32768, 65536, 131072 };constexpr const char * kKvSteps[] = { "auto", "f16", "q8_0", "q4_0" };@@ -374,11 +379,20 @@ std::string setting_text(const Settings & s, Setting id, const std::vector<Modelstd::snprintf(t, sizeof(t), "%uk tokens", s.ctx_cap / 1024);return t;case Setting::kv_type: return s.kv_type == "auto" ? "Auto" : s.kv_type;+ case Setting::scratch_limit:+ std::snprintf(t, sizeof(t), "%u GiB", s.scratch_limit_gib);+ return t;default: return "";}}-bool is_toggle(Setting id) { return id == Setting::auto_load || id == Setting::sounds; }+bool is_toggle(Setting id) { return id == Setting::auto_load || id == Setting::sounds || id == Setting::tools; }++bool toggle_value(const Settings & s, Setting id) {+ return id == Setting::auto_load ? s.auto_load : id == Setting::sounds ? s.sounds : s.tools;+}++constexpr uint32_t kScratchSteps[] = { 1, 2, 5, 10, 20 };// Moves a stepper one place; false at either end.bool step_setting(Settings & s, Setting id, int dir, const std::vector<ModelRow> & models) {@@ -430,8 +444,23 @@ bool step_setting(Settings & s, Setting id, int dir, const std::vector<ModelRow>s.kv_type = kKvSteps[at + dir];return true;}+ case Setting::scratch_limit: {+ int at = 1;+ for (int i = 0; i < 5; ++i) {+ if (kScratchSteps[i] == s.scratch_limit_gib) {+ at = i;+ }+ }+ if (at + dir < 0 || at + dir >= 5) {+ return false;+ }+ s.scratch_limit_gib = kScratchSteps[at + dir];+ return true;+ }case Setting::auto_load: s.auto_load = !s.auto_load; return true;case Setting::sounds: s.sounds = !s.sounds; return true;+ case Setting::tools: s.tools = !s.tools; return true;+ case Setting::clear_scratch: return false;}return false;}@@ -467,6 +496,17 @@ void Dashboard::update_settings(const hui::InputFrame & input, ui::Feedback & fe}}const Setting id = c.rows[setting_row_].id;+ if (id == Setting::clear_scratch) {+ if (input.is_pressed(Action::confirm)) {+ request_ = { Request::clear_scratch, "" };+ feedback.play(hui::audio::Cue::saved);+ }+ if (input.is_pressed(Action::back)) {+ in_rows_ = false;+ feedback.play(hui::audio::Cue::back);+ }+ return;+ }int dir = 0;if (input.nav == Direction::left || input.nav == Direction::right) {dir = input.nav == Direction::right ? 1 : -1;@@ -477,7 +517,7 @@ void Dashboard::update_settings(const hui::InputFrame & input, ui::Feedback & feif (dir != 0) {if (is_toggle(id) && (input.nav == Direction::left || input.nav == Direction::right)) {// Left turns a switch off, right on; the other way refuses.- const bool on = id == Setting::auto_load ? settings_.auto_load : settings_.sounds;+ const bool on = toggle_value(settings_, id);if (on == (dir > 0)) {if (!input.nav_repeat) {feedback.play(hui::audio::Cue::error, 1.0f, 0.0f, 0.6f);@@ -1127,7 +1167,7 @@ void Dashboard::draw_settings(gfx::DrawList & list) const {kInk.with_alpha(focused ? 1.0f : 0.84f));const float right = row.x + row.w - 28;if (is_toggle(sr.id)) {- const bool on = sr.id == Setting::auto_load ? settings_.auto_load : settings_.sounds;+ const bool on = toggle_value(settings_, sr.id);const Rect pill{ right - 84, row.cy() - 22, 84, 44 };if (focused) {list.glow(pill, 22, 12, kCyan.with_alpha(on ? 0.45f : 0.18f));@@ -1138,6 +1178,15 @@ void Dashboard::draw_settings(gfx::DrawList & list) const {list.rounded_rect(thumb, 17, on ? kPanelBottom : kInk);ui::text(list, f.regular, on ? "On" : "Off", pill.x - 20, row.cy() + 8, 24,kInk.with_alpha(focused ? 0.95f : 0.66f), gfx::Align::right);+ } else if (sr.id == Setting::clear_scratch) {+ // An action row: what it would delete, and a chevron.+ char t[48];+ std::snprintf(t, sizeof(t), "%.2f GiB in the folder", live_.scratch_gib);+ ui::text(list, f.regular, t, right - 34, row.cy() + 8, 24, kInk.with_alpha(focused ? 0.8f : 0.55f),+ gfx::Align::right);+ const Color ink = focused ? kRose : kInk.with_alpha(0.6f);+ list.line(right - 12, row.cy() - 9, right - 3, row.cy(), 3.0f, ink);+ list.line(right - 12, row.cy() + 9, right - 3, row.cy(), 3.0f, ink);} else {// A stepper: the value between two triangles, dimmed at the ends.const float w = 420, cx = right - w * 0.5f;@@ -1231,13 +1280,14 @@ void Dashboard::draw_hints(DashboardFrame & frame) const {const ui::Hint hints[] = { { ui::Button::triangle, "Library" }, { ui::Button::circle, "Back" } };ui::draw_hints(frame.overlay, fonts_, style, hints, 2, kGridX + kGridW, true);} else if (page_ == 1) {- const bool toggle = in_rows_ && is_toggle(kCategories[setting_category_].rows[setting_row_].id);+ const Setting row_id = kCategories[setting_category_].rows[setting_row_].id;+ const bool toggle = in_rows_ && (is_toggle(row_id) || row_id == Setting::clear_scratch);if (!in_rows_) {const ui::Hint hints[] = { { ui::Button::dpad, "Choose" }, { ui::Button::cross, "Open" } };ui::draw_hints(frame.scene, fonts_, style, hints, 2, kGridX + kGridW, true);} else {const ui::Hint hints[] = { { ui::Button::dpad, "Change" },- { ui::Button::cross, toggle ? "Switch" : "" },+ { ui::Button::cross, row_id == Setting::clear_scratch ? "Clear" : "Switch" },{ ui::Button::circle, "Back" } };const ui::Hint plain[] = { { ui::Button::dpad, "Change" }, { ui::Button::circle, "Back" } };ui::draw_hints(frame.scene, fonts_, style, toggle ? hints : plain, toggle ? 3 : 2, kGridX + kGridW, true);diff --git a/ps5/app/dashboard.hpp b/ps5/app/dashboard.hppindex ceb005e..53fe2a5 100644--- a/ps5/app/dashboard.hpp+++ b/ps5/app/dashboard.hpp@@ -54,7 +54,7 @@ class Dashboard {// What the user asked for, once: load or unload a model, or the settings// changed (read them with settings()).struct Request {- enum Kind { none, load, unload, settings } kind = none;+ enum Kind { none, load, unload, settings, clear_scratch } kind = none;std::string path;};Request take_request() {diff --git a/ps5/app/main.cpp b/ps5/app/main.cppindex c724dcb..dd4af58 100644--- a/ps5/app/main.cpp+++ b/ps5/app/main.cpp@@ -23,6 +23,7 @@#include <cstdlib>#include <cstring>#include <exception>+#include <filesystem>#include <fstream>#include <mutex>#include <span>@@ -97,6 +98,9 @@ constexpr const char * kArgs = "/data/PS5LM/app-args.txt";constexpr const char * kChoice = "/data/PS5LM/model.txt";constexpr const char * kQuit = "/data/PS5LM/quit";constexpr const char * kSettings = "/data/PS5LM/settings.json";+constexpr const char * kScratch = "/data/PS5LM/scratch";+// The file tools only: a title cannot start processes (no shell command).+constexpr const char * kTools = "read_file,write_file,edit_file,file_glob_search,grep_search,get_info";constexpr const char * kLlamaLog = "/data/PS5LM/llama.log";constexpr const char * kModels = "/data/PS5LM/models";constexpr const char * kAssets = "/app0/assets";@@ -125,6 +129,19 @@ std::vector<std::string> tail_lines(const char * path) {return lines;}+// Bytes under a folder, or 0.+uint64_t folder_bytes(const char * path) {+ std::error_code ec;+ uint64_t total = 0;+ for (auto it = std::filesystem::recursive_directory_iterator(path, ec);+ !ec && it != std::filesystem::recursive_directory_iterator(); it.increment(ec)) {+ if (it->is_regular_file(ec)) {+ total += it->file_size(ec);+ }+ }+ return total;+}+// A system notification on the TV.void notify(const char * text) {struct {@@ -342,6 +359,15 @@ struct Models {if (std::find(args.begin(), args.end(), "--metrics") == args.end()) {args.push_back("--metrics");}+ // The browser chat's file tools, confined to the scratch folder by+ // patches/0003 (LLAMA_TOOLS_ROOT) and working there by default.+ if (settings.tools && std::find(args.begin(), args.end(), "--tools") == args.end()) {+ std::error_code ec;+ std::filesystem::create_directories(kScratch, ec);+ setenv("LLAMA_TOOLS_ROOT", kScratch, 1);+ chdir(kScratch);+ args.insert(args.end(), { "--tools", kTools });+ }for (const auto & a : args) {std::printf("ps5lm-app: arg %s\n", a.c_str());}@@ -502,6 +528,7 @@ int main(int, char **) {Dashboard dash(fonts);dash.set_settings(models.settings);bool library_shown = false; // opened once at launch when no model is loaded+ bool scratch_warned = false;DashboardFrame frame;hui::ui::Feedback feedback;hui::PadSample samples[64];@@ -547,6 +574,19 @@ int main(int, char **) {}pad.tick(dt);+ // The scratch folder: its size for the dashboard, and one warning each+ // time it grows past the limit.+ if (n % 600 == 300) {+ const double gib = folder_bytes(kScratch) / 1073741824.0;+ stats.set_scratch(gib);+ const bool over = gib > models.settings.scratch_limit_gib;+ if (over && !scratch_warned) {+ char text[128];+ std::snprintf(text, sizeof(text), "PS5LM: the chat's scratch folder holds %.1f GiB; clear it in Settings", gib);+ notify(text);+ }+ scratch_warned = over;+ }if (dash.page() == Dashboard::kLogsPage && n % 60 == 0) {dash.set_logs(tail_lines(kLog), tail_lines(kLlamaLog));}@@ -560,6 +600,15 @@ int main(int, char **) {models.scan(stats);std::printf("ps5lm-app: settings saved\n");}+ if (req.kind == Dashboard::Request::clear_scratch) {+ std::error_code ec;+ for (const auto & e : std::filesystem::directory_iterator(kScratch, ec)) {+ std::filesystem::remove_all(e.path(), ec);+ }+ stats.set_scratch(folder_bytes(kScratch) / 1073741824.0);+ notify("PS5LM: scratch folder cleared");+ std::printf("ps5lm-app: scratch folder cleared\n");+ }// The same requests from the PC (scripts/ps5lm-app.sh load|unload).if (n % 60 == 30 && req.kind == Dashboard::Request::none) {if (std::string p = take_request_file("/data/PS5LM/load"); !p.empty()) {diff --git a/ps5/app/model_plan.cpp b/ps5/app/model_plan.cppindex 84f28af..0c87dba 100644--- a/ps5/app/model_plan.cpp+++ b/ps5/app/model_plan.cpp@@ -2,6 +2,7 @@#include "model_plan.hpp"#include <dirent.h>+#include <sys/stat.h>#include <algorithm>#include <cctype>@@ -85,6 +86,49 @@ std::string lower(std::string s) {} // namespace+// The vision projector for a model: an "mmproj" .gguf in the same folder.+// With several, the one whose name shares the longest start with the+// model's, once "mmproj" and separators are set aside+// ("mmproj-gemma-4-E4B-f16.gguf" for "gemma-4-E4B_q4_0-it.gguf").+std::string find_mmproj(const std::string & model_path) {+ const size_t slash = model_path.find_last_of('/');+ const std::string dir = model_path.substr(0, slash);+ const auto key = [](std::string n) {+ n = lower(n);+ if (const size_t at = n.find("mmproj"); at != std::string::npos) {+ n.erase(at, 6);+ }+ n.erase(std::remove_if(n.begin(), n.end(), [](char c) { return c == '-' || c == '_' || c == '.'; }), n.end());+ return n;+ };+ const std::string want = key(model_path.substr(slash + 1));+ std::string best;+ size_t best_len = 0;+ DIR * d = opendir(dir.c_str());+ if (!d) {+ return best;+ }+ while (dirent * e = readdir(d)) {+ const std::string n = e->d_name;+ if (lower(n).find("mmproj") == std::string::npos || n.size() < 6 || lower(n.substr(n.size() - 5)) != ".gguf") {+ continue;+ }+ const std::string k = key(n);+ size_t common = 0;+ while (common < k.size() && common < want.size() && k[common] == want[common]) {+ ++common;+ }+ // At least the family's name in common ("gemma4e4b"), so a lone+ // projector of another model is not paired.+ if (common >= 5 && common > best_len) {+ best = dir + "/" + n;+ best_len = common;+ }+ }+ closedir(d);+ return best;+}+ModelInfo read_model_info(const std::string & path) {ModelInfo m;m.path = path;@@ -175,6 +219,11 @@ ModelInfo read_model_info(const std::string & path) {if (!m.ok) {m.error = "no layer count for architecture '" + m.arch + "'";}+ m.mmproj = find_mmproj(path);+ if (!m.mmproj.empty()) {+ struct stat st = {};+ m.mmproj_bytes = stat(m.mmproj.c_str(), &st) == 0 ? (uint64_t) st.st_size : 0;+ }return m;}@@ -230,12 +279,13 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, constif (m.n_ctx_train) {ctx_cap = std::min(ctx_cap, m.n_ctx_train);}- p.weights_gib = m.file_bytes / kGiB;+ p.weights_gib = (m.file_bytes + m.mmproj_bytes) / kGiB;p.state_gib = m.state_bytes / kGiB;// Compute buffers and the CPU side (the title heap shares the pool), plus// one prompt checkpoint of the recurrent state. Calibrated on the console:// Qwen3.8-27B UD-Q2_K_XL at 64k with a q4_0 cache ran with 11.38 GiB free.- p.reserve_gib = 0.75 + p.state_gib;+ // ponytail: 0.25 GiB for the image encoder's buffers is a guess; measure on the console.+ p.reserve_gib = 0.75 + p.state_gib + (m.mmproj.empty() ? 0.0 : 0.25);// The longest context first, and at each length the best cache that fits.static const uint32_t ctxs[] = { 131072, 65536, 32768, 16384, 8192, 4096 };@@ -275,8 +325,8 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, constp.why = why;return p;}- std::snprintf(why, sizeof(why), "%uk context, %s cache: %.1f of %.1f GiB", p.ctx / 1024, p.kv_type.c_str(),- p.total_gib, budget_gib);+ std::snprintf(why, sizeof(why), "%uk context, %s cache%s: %.1f of %.1f GiB", p.ctx / 1024, p.kv_type.c_str(),+ m.mmproj.empty() ? "" : ", images", p.total_gib, budget_gib);p.why = why;p.args = { "-m", m.path, "-ngl", "999", "-fit", "off", "-lm", "none",@@ -286,6 +336,9 @@ Plan plan_model(const ModelInfo & m, double budget_gib, uint32_t ctx_cap, const// (32 checkpoints, 8 GiB) are for a PC, and the CPU shares this pool."-ctxcp", "1", "-cram", "0","--host", "0.0.0.0", "--port", "8081" };+ if (!m.mmproj.empty()) {+ p.args.insert(p.args.end(), { "--mmproj", m.mmproj });+ }if (preset) {p.args.insert(p.args.end(), preset->args.begin(), preset->args.end());}diff --git a/ps5/app/model_plan.hpp b/ps5/app/model_plan.hppindex 0b5a180..15a1b51 100644--- a/ps5/app/model_plan.hpp+++ b/ps5/app/model_plan.hpp@@ -30,6 +30,8 @@ struct ModelInfo {uint32_t n_ctx_train = 0;uint32_t n_expert = 0; // 0: denseuint64_t state_bytes = 0; // recurrent state, all layers (f32)+ std::string mmproj; // a vision projector beside the model, or empty+ uint64_t mmproj_bytes = 0;bool ok = false;std::string error;@@ -37,6 +39,9 @@ struct ModelInfo {};// Reads the metadata of a GGUF file (not its tensors).+// The vision projector ("mmproj") beside a model, or empty.+std::string find_mmproj(const std::string & model_path);+ModelInfo read_model_info(const std::string & path);// Bytes of KV cache per token of context, for a cache type ("f16", "q8_0", "q4_0").diff --git a/ps5/app/preview_main.cpp b/ps5/app/preview_main.cppindex ae290c8..fa66726 100644--- a/ps5/app/preview_main.cpp+++ b/ps5/app/preview_main.cpp@@ -271,6 +271,12 @@ int main(int argc, char ** argv) {step(20, nav(hui::Direction::down), false);step(30, press(hui::Action::confirm), false);shot("08-settings-model");+ step(20, press(hui::Action::back), false);+ step(20, nav(hui::Direction::down), false);+ step(20, press(hui::Action::confirm), false);+ step(20, nav(hui::Direction::down), false);+ step(30, nav(hui::Direction::down), false);+ shot("08-settings-chat");step(40, press(hui::Action::page_next), false);shot("09-logs");step(30, press(hui::Action::north), false);diff --git a/ps5/app/settings.cpp b/ps5/app/settings.cppindex a5ee2bc..8bdfe81 100644--- a/ps5/app/settings.cpp+++ b/ps5/app/settings.cpp@@ -22,13 +22,16 @@ Settings load_settings(const char * path) {s.ctx_cap = j.value("ctx_cap", s.ctx_cap);s.kv_type = j.value("kv_type", s.kv_type);s.sounds = j.value("sounds", s.sounds);+ s.tools = j.value("tools", s.tools);+ s.scratch_limit_gib = j.value("scratch_limit_gib", s.scratch_limit_gib);return s;}bool save_settings(const char * path, const Settings & s) {const nlohmann::json j = {{ "auto_load", s.auto_load }, { "default_model", s.default_model }, { "ctx_cap", s.ctx_cap },- { "kv_type", s.kv_type }, { "sounds", s.sounds },+ { "kv_type", s.kv_type }, { "sounds", s.sounds }, { "tools", s.tools },+ { "scratch_limit_gib", s.scratch_limit_gib },};std::ofstream out(path);out << j.dump(2) << "\n";diff --git a/ps5/app/settings.hpp b/ps5/app/settings.hppindex df63536..f66deb4 100644--- a/ps5/app/settings.hpp+++ b/ps5/app/settings.hpp@@ -16,6 +16,8 @@ struct Settings {uint32_t ctx_cap = 65536; // the planner's longest contextstd::string kv_type = "auto"; // "auto", "f16", "q8_0" or "q4_0"bool sounds = true;+ bool tools = true; // file tools in the browser chat, inside the scratch folder+ uint32_t scratch_limit_gib = 2; // warn when the scratch folder grows past thisbool operator==(const Settings &) const = default;};diff --git a/ps5/app/stats.cpp b/ps5/app/stats.cppindex 5b7f3aa..2bbeea4 100644--- a/ps5/app/stats.cpp+++ b/ps5/app/stats.cpp@@ -183,6 +183,11 @@ void StatsCollector::set_models(const std::vector<ModelRow> & rows) {live_.models = rows;}+void StatsCollector::set_scratch(double gib) {+ std::lock_guard<std::mutex> lock(mutex_);+ live_.scratch_gib = gib;+}+void StatsCollector::set_state(ServerState s) {std::lock_guard<std::mutex> lock(mutex_);live_.state = s;diff --git a/ps5/app/stats.hpp b/ps5/app/stats.hppindex ebcf38c..ce8f92e 100644--- a/ps5/app/stats.hpp+++ b/ps5/app/stats.hpp@@ -70,6 +70,7 @@ struct Live {// Storage, GiB; usb_total 0 when no drive is mounted.double data_free = 0, data_total = 0;double usb_free = 0, usb_total = 0;+ double scratch_gib = 0; // the chat tools' scratch folder// Memory, GiB.double pool_gib = 0; // the title's direct memory@@ -98,6 +99,7 @@ class StatsCollector {void set_model(const Live & model_part); // model_* , ctx, kv_*, presetvoid set_models(const std::vector<ModelRow> & rows);void set_state(ServerState s);+ void set_scratch(double gib);private:void run();