Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

native: pf-serve on a GPU reads the model once, not twice (#3590)

yetone642d430ecb07
diff --git a/native/privacy-filter/patches/weights-once.patch b/native/privacy-filter/patches/weights-once.patch
new file mode 100644
index 000000000..576e2c8db
--- /dev/null
+++ b/native/privacy-filter/patches/weights-once.patch
@@ -0,0 +1,102 @@
+diff --git a/src/model.cpp b/src/model.cpp
+index 80afe56..0ff1edb 100644
+--- a/src/model.cpp
++++ b/src/model.cpp
+@@ -2,8 +2,10 @@
+
+ #include <ggml.h>
+
++#include <algorithm>
+ #include <cmath>
+ #include <cstring>
++#include <fstream>
+
+ namespace pf {
+
+@@ -57,15 +59,19 @@ void yarn_freq_factors(const hparams & hp, std::vector<float> & ff, float & attn
+ bool model::load(const std::string & gguf_path, const std::string & device, int n_threads) {
+ release();
+
+- if (!file.open(gguf_path, /*with_data =*/ true)) {
+- error = file.error;
+- return false;
+- }
+ if (!be.init(device, n_threads)) {
+ error = be.error;
+ return false;
+ }
+- if (!map_tensors() || !realize_weights()) {
++ // The CPU uses the loader's buffer in place, so it is read whole. A
++ // device gets the metadata only, and realize_weights reads each tensor
++ // from the file into the device's buffer: the weights are never held
++ // twice, once in the device's buffer and once here.
++ if (!file.open(gguf_path, /*with_data =*/ be.is_cpu())) {
++ error = file.error;
++ return false;
++ }
++ if (!map_tensors() || !realize_weights(gguf_path)) {
+ return false;
+ }
+ yarn_freq_factors(file.hp, freq_factors, attn_factor);
+@@ -116,7 +122,7 @@ bool model::map_tensors() {
+ return true;
+ }
+
+-bool model::realize_weights() {
++bool model::realize_weights(const std::string & gguf_path) {
+ if (be.is_cpu()) {
+ // Zero-copy: the GGUF loaded with no_alloc=false, so all tensor data
+ // lives in one contiguous ctx buffer. Wrap it as a CPU backend buffer
+@@ -155,11 +161,36 @@ bool model::realize_weights() {
+ error = "device weight alloc failed (out of device memory?)";
+ return false;
+ }
++ // Each tensor's bytes go from the file to the device through one
++ // 16 MB buffer.
++ std::ifstream in(gguf_path, std::ios::binary);
++ if (!in) {
++ error = "failed to reopen GGUF: " + gguf_path;
++ return false;
++ }
++ const size_t data_off = gguf_get_data_offset(file.guf);
++ std::vector<char> chunk(16u << 20);
+ for (ggml_tensor * s = ggml_get_first_tensor(file.ctx); s; s = ggml_get_next_tensor(file.ctx, s)) {
+- ggml_tensor * d = ggml_get_tensor(device_ctx, ggml_get_name(s));
+- ggml_backend_tensor_set(d, s->data, 0, ggml_nbytes(s));
++ ggml_tensor * d = ggml_get_tensor(device_ctx, ggml_get_name(s));
++ const int64_t id = gguf_find_tensor(file.guf, ggml_get_name(s));
++ if (id < 0) {
++ error = std::string("tensor not in GGUF: ") + ggml_get_name(s);
++ return false;
++ }
++ in.seekg((std::streamoff) (data_off + gguf_get_tensor_offset(file.guf, id)));
++ const size_t n = ggml_nbytes(s);
++ for (size_t off = 0; off < n;) {
++ const size_t k = std::min(chunk.size(), n - off);
++ if (!in.read(chunk.data(), (std::streamsize) k)) {
++ error = std::string("short read of tensor ") + ggml_get_name(s);
++ return false;
++ }
++ ggml_backend_tensor_set(d, chunk.data(), off, k);
++ off += k;
++ }
+ }
+- // Repoint the weight map at the device tensors (file.ctx stays as source).
++ // Repoint the weight map at the device tensors (file.ctx holds their
++ // metadata alone).
+ auto remap = [&](ggml_tensor *& t) { if (t) t = ggml_get_tensor(device_ctx, ggml_get_name(t)); };
+ remap(tok_embd); remap(output_norm); remap(cls_w); remap(cls_b);
+ for (auto & l : layers) {
+diff --git a/src/model.h b/src/model.h
+index 9fd8f06..6dc37d6 100644
+--- a/src/model.h
++++ b/src/model.h
+@@ -69,7 +69,7 @@ struct model {
+ tap_map * taps = nullptr);
+
+ private:
+- bool realize_weights();
++ bool realize_weights(const std::string & gguf_path);
+ bool map_tensors();
+ };
+