llama.cpp on a jailbroken PS5: run any GGUF model natively on the console, starting with Qwen 3.8
App: the splash briefly, then Home with the load moving; the address on the left
Mert Cobanov04f0a5f1d298
diff --git a/ps5/app/dashboard.cpp b/ps5/app/dashboard.cppindex 86e19ff..a2b5d5e 100644--- a/ps5/app/dashboard.cpp+++ b/ps5/app/dashboard.cpp@@ -581,6 +581,10 @@ void Dashboard::update(const hui::InputFrame & raw, float dt, ui::Feedback & feeclock_ += dt;page_age_ += dt;+ // The load's share moves on every frame, not once a second as it is measured.+ const float load = live_.state == ServerState::loading ? std::max(0.0f, live_.load_progress) : 0.0f;+ load_shown_ = load < load_shown_ ? load : load_shown_ + (load - load_shown_) * std::min(1.0f, dt * 2.5f);+// Two sounds come from events, not buttons: a model ready, a download saved.const bool serving = live_.state == ServerState::ready || live_.state == ServerState::generating;if (serving && !was_serving_) {@@ -1185,16 +1189,23 @@ void Dashboard::draw_home(gfx::DrawList & l) const {put(l, f.regular, "Choose a model", kLeft, 340, 64, kInk);put(l, f.regular, "It runs on the PS5's GPU and answers every device on your network.", kLeft, 400, 24, kInk2);put(l, f.regular, format("%.0f models on this PS5", (double) live_.models.size()), kLeft, 444, 24, kInk3);+ if (!live_.address.empty()) {+ put(l, f.regular, "On your network", kLeft, 540, 22, kInk3);+ put(l, f.regular, "http://" + live_.address, kLeft, 586, 32, kInk2);+ }} else {std::string name, build;split_name(live_.model_label, &name, &build);- put(l, f.regular, decision ? "Decision model" : "Language model", kLeft, 250, 26, kInk2);+ put(l, f.regular, !live_.phase.empty() ? "Starting" : decision ? "Decision model" : "Language model", kLeft, 250, 26,+ kInk2);put(l, f.regular, f.regular.font->fit(name, 64, 860), kLeft, 326, 64, kInk);std::string detail = build;if (live_.ctx > 0) {detail += (detail.empty() ? "" : " ") + format("%.0fk context", live_.ctx / 1024.0);}- detail += (detail.empty() ? "" : " ") + gb(live_.model_gib);+ if (live_.model_gib > 0) {+ detail += (detail.empty() ? "" : " ") + gb(live_.model_gib);+ }put(l, f.regular, detail, kLeft, 372, 24, kInk3);// The state, as a car shows its gear: one word, coloured only when it matters.@@ -1206,13 +1217,17 @@ void Dashboard::draw_home(gfx::DrawList & l) const {: live_.state == ServerState::loading ? kInk2.with_alpha(0.5f + 0.5f * std::sin(clock_ * 4.0f)) : kGreen;l.circle(kLeft + 7, 432, 7, dot);put(l, f.regular, word, kLeft + 26, 441, 26, kInk);+ // Where the network reaches it, on the same line.+ if (!live_.address.empty()) {+ put(l, f.regular, "http://" + live_.address, kLeft + 860, 441, 26, kInk2, gfx::Align::right);+ }// The large number: speed while serving, progress while loading.float big = 0;std::string unit;bool known = true;if (live_.state == ServerState::loading) {- big = std::max(0.0f, live_.load_progress) * 100.0f;+ big = load_shown_ * 100.0f;unit = "% on the GPU";} else if (decision) {// Time per decision (one question answered) over the latest requests:@@ -1250,7 +1265,32 @@ void Dashboard::draw_home(gfx::DrawList & l) const {put(l, f.regular, unit, kLeft + nw + 12, 660, 26, kInk2);if (live_.state == ServerState::loading) {- meter(l, kLeft, 712, 860, live_.load_progress, kInk);+ // The model's memory as cells filling column by column, the newest+ // lit and a slow wave over the rest; before the load starts, a light+ // sweeping the empty ones. It never looks stopped.+ constexpr int cols = 60, rows = 4;+ const float cw = (860.0f - 3.0f * (cols - 1)) / cols, ch = 18;+ const int filled = (int) (load_shown_ * cols * rows);+ const float sweep = (0.5f + 0.5f * std::sin(clock_ * 1.7f)) * (cols - 1);+ for (int c = 0; c < cols; ++c) {+ for (int r = 0; r < rows; ++r) {+ const int i = c * rows + r;+ const Color cell = i < filled ? kInk.with_alpha(0.45f + 0.4f * (0.5f + 0.5f * std::sin(clock_ * 3.0f - (float) c * 0.22f)))+ : i == filled ? kBlue.with_alpha(0.6f + 0.4f * std::sin(clock_ * 8.0f))+ : filled == 0 ? gfx::mix(kHair, kInk3, std::max(0.0f, 1.0f - std::fabs((float) c - sweep) / 4.0f))+ : kHair;+ l.rounded_rect({ kLeft + (cw + 3) * (float) c, 700 + (ch + 3) * (float) r, cw, ch }, 3, cell);+ }+ }+ const double total = live_.model_gib + live_.kv_gib;+ put(l, f.regular,+ !live_.phase.empty() ? live_.phase+ : live_.load_progress < 0.985f ? std::string("Copying the weights into GPU memory") : std::string("Preparing the context"),+ kLeft, 822, 22, kInk2);+ if (total > 0) {+ put(l, f.regular, format("%.1f", load_shown_ * total) + " of " + gb(total), kLeft + 860, 822, 22, kInk3,+ gfx::Align::right);+ }} else if (!decision) {trace(l, live_.gen_history, { kLeft, 702, 860, 100 }, kInk.with_alpha(0.75f));}@@ -1268,20 +1308,19 @@ void Dashboard::draw_home(gfx::DrawList & l) const {meter(l, kLeft + 460, my + 16, 400, live_.ctx > 0 ? (float) live_.ctx_used / (float) live_.ctx : 0, kInk.with_alpha(0.8f));}- // Right: where the model is reached, and what the network asks of it, as- // it happens. A single panel, as a car gives the map.+ // Right: what the network asks of it, as it happens. A single panel, as a+ // car gives the map, with what it has done since it started at the top,+ // like a trip meter.const Rect p{ 1056, 196, kRight - 1056, 728 };l.rounded_rect(p, 24, kSurface);const float x = p.x + 56, w = p.w - 112;- put(l, f.regular, "On your network", x, p.y + 70, 22, kInk2);- const std::string url = live_.address.empty() ? std::string("Not connected") : "http://" + live_.address;- put(l, f.regular, f.regular.font->fit(url, 40, w), x, p.y + 124, 40, live_.address.empty() ? kInk3 : kInk);- put(l, f.regular,- !loaded ? "Load a model and it answers here"- : decision ? "TypeSafe decisions at /v1/systemone"- : "OpenAI, Anthropic and a chat page",- x, p.y + 162, 22, kInk3);- l.line(x, p.y + 196, x + w, p.y + 196, 1.5f, kHair);+ put(l, f.regular, "Requests", x, p.y + 62, 22, kInk2);+ std::string trip = thousands(live_.served) + " answered " + thousands(live_.prompt_tokens) + " tokens in";+ if (!decision) {+ trip += " " + thousands(live_.tokens) + " out";+ }+ put(l, f.regular, trip, x + w, p.y + 62, 20, kInk3, gfx::Align::right);+ l.line(x, p.y + 90, x + w, p.y + 90, 1.5f, kHair);// The conversation: what each device asked, on the right as a message// sent, and the model's answer under it on the left, newest at the bottom@@ -1303,7 +1342,7 @@ void Dashboard::draw_home(gfx::DrawList & l) const {lines.back() = f.regular.font->fit(lines.back() + " \xE2\x80\xA6", 22, width);}};- const float top = p.y + 222, bottom = p.y + p.h - 70;+ const float top = p.y + 112, bottom = p.y + p.h - 30;std::vector<Turn> turns;float total = 0;for (auto it = live_.activity.rbegin(); it != live_.activity.rend(); ++it) {@@ -1373,13 +1412,6 @@ void Dashboard::draw_home(gfx::DrawList & l) const {}y += 28 + 26;}-- // What it has done since it started, like a trip meter.- std::string trip = thousands(live_.served) + " requests " + thousands(live_.prompt_tokens) + " tokens in";- if (!decision) {- trip += " " + thousands(live_.tokens) + " out";- }- put(l, f.regular, trip, x, p.y + p.h - 40, 20, kInk3);}// ---- drawing: Models ------------------------------------------------------------------diff --git a/ps5/app/dashboard.hpp b/ps5/app/dashboard.hppindex 6c7c2e2..0e66b28 100644--- a/ps5/app/dashboard.hpp+++ b/ps5/app/dashboard.hpp@@ -120,6 +120,7 @@ class Dashboard {std::string busy_title_, busy_text_;hui::ui::InputPrompt prompt_; // a message typed on Homeint demo_ = 0; // the next demo to send+ float load_shown_ = 0; // the load's share as drawn, eased between samplesbool was_serving_ = false; // for the sound when a model becomes readystd::vector<std::string> unfinished_; // downloads not saved yet, for the sound when one isdiff --git a/ps5/app/main.cpp b/ps5/app/main.cppindex 081678c..529e237 100644--- a/ps5/app/main.cpp+++ b/ps5/app/main.cpp@@ -27,6 +27,7 @@#include <exception>#include <filesystem>#include <fstream>+#include <functional>#include <mutex>#include <span>#include <string>@@ -538,6 +539,19 @@ struct Models {}};+// A thread with a 16 MiB stack, as llama-server's: the console's default is small.+pthread_t start_thread(std::function<void()> & fn) {+ pthread_attr_t attr;+ pthread_attr_init(&attr);+ pthread_attr_setstacksize(&attr, 16u << 20);+ pthread_t t{};+ pthread_create(&t, &attr, [](void * p) -> void * {+ (*static_cast<std::function<void()> *>(p))();+ return nullptr;+ }, &fn);+ return t;+}+bool load_font(hui::gfx::Renderer & renderer, const char * name, hui::gfx::Font * font, hui::ui::FontRef * ref) {std::string data;const std::string path = std::string(kAssets) + "/fonts/" + name;@@ -617,8 +631,7 @@ int main(int, char **) {renderer.draw(frame.overlay);display.present(renderer);};- // The splash before the slow part (the GPU opening, every model's header- // read), so a restart for a model is never a black screen.+ // The splash at once, so a restart for a model is never a black screen.if (ui_ok) {dash.set_busy("Starting", "Opening the GPU and reading the models");render();@@ -626,78 +639,125 @@ int main(int, char **) {}sceSystemServiceHideSplashScreen();- Models models;- for (size_t i = 0; i < ggml_backend_dev_count(); i++) {- ggml_backend_dev_t dev = ggml_backend_dev_get(i);- size_t free = 0, total = 0;- ggml_backend_dev_memory(dev, &free, &total);- std::printf("ps5lm-app: device %zu %s (%s): %.2f / %.2f GiB free\n", i, ggml_backend_dev_name(dev),- ggml_backend_dev_description(dev), free / 1073741824.0, total / 1073741824.0);- // The PS5 is an APU: ggml calls its GPU integrated.- const auto type = ggml_backend_dev_type(dev);- if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {- models.budget_gib = free / 1073741824.0;- }- }-StatsCollector stats;stats.start(kPort);set_activity_sink(&stats); // llama-server's request hooks (patches/0007) report here+ Models models;models.settings = load_settings(kSettings);- models.scan(stats);-- // Which model: app-args.txt verbatim, else model.txt, which the library- // writes just before restarting the app for a switch and which is used- // once, else the default model when Settings asks for it at launch. A- // plain launch otherwise loads nothing and opens the library.- Live model_part;- std::vector<std::string> args = read_lines(kArgs);- std::string path;- if (!args.empty()) {- const auto m = std::find(args.begin(), args.end(), "-m");- path = m != args.end() && m + 1 != args.end() ? *(m + 1) : "";- model_part.model_label = base_name(path);- const ModelInfo info = read_model_info(path);- model_part.model_arch = info.arch;- model_part.model_gib = info.file_bytes / 1073741824.0;- model_part.model_file = path;- model_part.args = args;- std::printf("ps5lm-app: app-args.txt overrides the planner\n");- } else {- auto chosen = read_lines(kChoice);- unlink(kChoice);- if (chosen.empty() && models.settings.auto_load && !models.settings.default_model.empty()) {- chosen = { models.settings.default_model }; // "none" after an unload keeps it from coming back- }- for (const auto & r : models.rows) {- if (r.fits && !chosen.empty() && r.file == chosen[0]) {- path = r.file;- }+ // The model about to load, by name, for Home while the GPU opens.+ {+ std::vector<std::string> next = read_lines(kArgs);+ const auto m = std::find(next.begin(), next.end(), "-m");+ next = m != next.end() && m + 1 != next.end() ? std::vector<std::string>{ *(m + 1) } : read_lines(kChoice);+ if (next.empty() && models.settings.auto_load) {+ next = { models.settings.default_model };}- if (!path.empty()) {- args = models.args_for(path, model_part);+ if (!next.empty() && !next[0].empty() && next[0] != "none") {+ Live pending;+ pending.model_label = base_name(next[0]);+ stats.set_model(pending);+ stats.set_state(ServerState::loading);}+ stats.set_phase("Opening the GPU and reading the models");}- // A projector this llama.cpp cannot read stops the server (#15); the- // restart after it loads the same model text only, this once.- if (access(kTextOnly, F_OK) == 0) {- unlink(kTextOnly);- const auto mm = std::find(args.begin(), args.end(), "--mmproj");- if (mm != args.end()) {- std::printf("ps5lm-app: loading without %s\n", mm + 1 != args.end() ? (mm + 1)->c_str() : "the projector");- args.erase(mm, std::min(mm + 2, args.end()));++ // The slow part on a thread of its own (the GPU opening, every model's+ // header, the model's start) while the screen keeps moving: the splash+ // for a moment, then Home, where the load shows.+ std::function<void()> boot = [&] {+ for (size_t i = 0; i < ggml_backend_dev_count(); i++) {+ ggml_backend_dev_t dev = ggml_backend_dev_get(i);+ size_t free = 0, total = 0;+ ggml_backend_dev_memory(dev, &free, &total);+ std::printf("ps5lm-app: device %zu %s (%s): %.2f / %.2f GiB free\n", i, ggml_backend_dev_name(dev),+ ggml_backend_dev_description(dev), free / 1073741824.0, total / 1073741824.0);+ // The PS5 is an APU: ggml calls its GPU integrated.+ const auto type = ggml_backend_dev_type(dev);+ if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {+ models.budget_gib = free / 1073741824.0;+ }+ }++ models.scan(stats);++ // Which model: app-args.txt verbatim, else model.txt, which the library+ // writes just before restarting the app for a switch and which is used+ // once, else the default model when Settings asks for it at launch. A+ // plain launch otherwise loads nothing and opens the library.+ Live model_part;+ std::vector<std::string> args = read_lines(kArgs);+ std::string path;+ if (!args.empty()) {+ const auto m = std::find(args.begin(), args.end(), "-m");+ path = m != args.end() && m + 1 != args.end() ? *(m + 1) : "";+ model_part.model_label = base_name(path);+ const ModelInfo info = read_model_info(path);+ model_part.model_arch = info.arch;+ model_part.model_gib = info.file_bytes / 1073741824.0;+ model_part.model_file = path;model_part.args = args;+ std::printf("ps5lm-app: app-args.txt overrides the planner\n");+ } else {+ auto chosen = read_lines(kChoice);+ unlink(kChoice);+ if (chosen.empty() && models.settings.auto_load && !models.settings.default_model.empty()) {+ chosen = { models.settings.default_model }; // "none" after an unload keeps it from coming back+ }+ for (const auto & r : models.rows) {+ if (r.fits && !chosen.empty() && r.file == chosen[0]) {+ path = r.file;+ }+ }+ if (!path.empty()) {+ args = models.args_for(path, model_part);+ }}- }- log_memory("before loading");- if (!args.empty()) {- models.start(path, args, stats, model_part);- } else {- stats.set_state(ServerState::no_model);- if (models.rows.empty()) {- notify("PS5LM: no models; copy a .gguf to /data/PS5LM/models or PS5LM/models on a USB drive");+ // A projector this llama.cpp cannot read stops the server (#15); the+ // restart after it loads the same model text only, this once.+ if (access(kTextOnly, F_OK) == 0) {+ unlink(kTextOnly);+ const auto mm = std::find(args.begin(), args.end(), "--mmproj");+ if (mm != args.end()) {+ std::printf("ps5lm-app: loading without %s\n", mm + 1 != args.end() ? (mm + 1)->c_str() : "the projector");+ args.erase(mm, std::min(mm + 2, args.end()));+ model_part.args = args;+ }+ }+ log_memory("before loading");+ if (!args.empty()) {+ models.start(path, args, stats, model_part);+ } else {+ stats.set_state(ServerState::no_model);+ if (models.rows.empty()) {+ notify("PS5LM: no models; copy a .gguf to /data/PS5LM/models or PS5LM/models on a USB drive");+ }+ }+ stats.set_phase("");+ };+ std::atomic<bool> booted{ false };+ std::function<void()> boot_then = [&] {+ boot();+ booted = true;+ };+ const pthread_t booting = start_thread(boot_then);+ {+ hui::ui::Feedback quiet; // no sounds yet: the mixer starts after+ const int64_t start = hui::sys::monotonic_us();+ int64_t at = start;+ while (ui_ok && !booted) {+ const int64_t now = hui::sys::monotonic_us();+ if (now - start > 1200000) {+ dash.set_busy("");+ }+ dash.set_live(stats.snapshot());+ quiet.clear();+ dash.update(hui::InputFrame{}, std::min(0.05f, (float) (now - at) / 1e6f), quiet);+ at = now;+ render();+ display.swap();}}+ pthread_join(booting, nullptr);// Input and sound, as ps5-homebrew-ui's own app does.hui::ps5::Pad pad;diff --git a/ps5/app/preview_main.cpp b/ps5/app/preview_main.cppindex b4ed94b..412f2f0 100644--- a/ps5/app/preview_main.cpp+++ b/ps5/app/preview_main.cpp@@ -54,6 +54,7 @@ bool load_font(hui::gfx::Renderer & r, const std::string & path, hui::gfx::Fontfloat g_loading = -1; // >= 0: draw the loading state at this progressbool g_decision = false; // d1 Omni 600M instead of Qwen 3.8bool g_empty = false; // nothing loaded+std::string g_phase; // the app still startingps5lm::Live sample(float seconds, bool generating) {ps5lm::Live L;@@ -212,6 +213,7 @@ ps5lm::Live sample(float seconds, bool generating) {if (g_loading >= 0) {L.state = ps5lm::ServerState::loading;L.load_progress = g_loading;+ L.phase = g_phase;L.free_gib = 11.03 - g_loading * (L.model_gib + L.kv_gib);L.gen_tps = 0;L.gpu_busy = 0;@@ -378,6 +380,11 @@ int main(int argc, char ** argv) {using D = hui::Direction;using A = hui::Action;+ g_loading = 0;+ g_phase = "Opening the GPU and reading the models";+ step(50, {}, false);+ shot("00-home-starting");+ g_phase.clear();g_loading = 0.45f;step(90, {}, false);shot("00-home-loading");diff --git a/ps5/app/stats.cpp b/ps5/app/stats.cppindex 56d2cf2..0c62387 100644--- a/ps5/app/stats.cpp+++ b/ps5/app/stats.cpp@@ -257,6 +257,11 @@ void StatsCollector::set_model(const Live & m) {live_.args = m.args;}+void StatsCollector::set_phase(std::string phase) {+ std::lock_guard<std::mutex> lock(mutex_);+ live_.phase = std::move(phase);+}+void StatsCollector::set_models(const std::vector<ModelRow> & rows) {std::lock_guard<std::mutex> lock(mutex_);live_.models = rows;diff --git a/ps5/app/stats.hpp b/ps5/app/stats.hppindex e4537b1..57ceb2d 100644--- a/ps5/app/stats.hpp+++ b/ps5/app/stats.hpp@@ -108,6 +108,7 @@ struct Live {// Loading: how much of the model and its cache is on the GPU, 0..1, or -1.float load_progress = -1;+ std::string phase; // what the app does before the load can start ("Opening the GPU"), or empty// Session.double uptime_s = 0;@@ -131,6 +132,7 @@ class StatsCollector {void set_models(const std::vector<ModelRow> & rows);void set_state(ServerState s);void set_scratch(double gib);+ void set_phase(std::string phase);// From llama-server's HTTP threads, through activity.cpp.void activity_begin(int id, std::string client, std::string path, std::string asked, int questions);void activity_text(int id, const std::string & text);