Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

native: pf-serve, a sidecar that runs OpenAI's Privacy Filter for masking (#1977)

yetoneffa292c7d388
diff --git a/.github/workflows/privacy-filter.yml b/.github/workflows/privacy-filter.yml
new file mode 100644
index 000000000..2c366e5c6
--- /dev/null
+++ b/.github/workflows/privacy-filter.yml
@@ -0,0 +1,77 @@
+# Builds pf-serve, the program that runs Settings → Privacy's local
+# masking model (native/privacy-filter, internal/pii), for each system.
+# A change to it is built on every push; a pf-v* tag publishes the zips as
+# a prerelease of this repo, never Latest, whose sha256s internal/pii pins.
+name: privacy-filter
+
+on:
+ push:
+ branches: [main, "pf/**"]
+ paths: ["native/privacy-filter/**", ".github/workflows/privacy-filter.yml"]
+ tags: ["pf-v*"]
+ pull_request:
+ paths: ["native/privacy-filter/**", ".github/workflows/privacy-filter.yml"]
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ build:
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - { target: darwin-arm64, os: macos-15 }
+ - { target: darwin-amd64, os: macos-15 }
+ # the oldest glibc a runner has, so the program runs on older systems too
+ - { target: linux-amd64, os: ubuntu-22.04 }
+ - { target: linux-arm64, os: ubuntu-22.04-arm }
+ - { target: windows-amd64, os: ubuntu-24.04 }
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: tools (macOS)
+ if: runner.os == 'macOS'
+ run: brew install ninja
+ - name: tools (Linux)
+ if: startsWith(matrix.target, 'linux')
+ run: sudo apt-get update && sudo apt-get install -y ninja-build libvulkan-dev glslc zip
+ - name: tools (Windows, cross)
+ if: matrix.target == 'windows-amd64'
+ run: sudo apt-get update && sudo apt-get install -y ninja-build mingw-w64 glslc zip
+ - name: build
+ run: native/privacy-filter/build.sh ${{ matrix.target }} pf-serve-${{ matrix.target }}.zip
+ - name: starts
+ # each program the runner can start runs to its usage line
+ if: matrix.target != 'windows-amd64' && matrix.target != 'darwin-amd64'
+ run: |
+ mkdir t && cd t && unzip -q ../pf-serve-${{ matrix.target }}.zip
+ for p in pf-serve pf-serve-vulkan; do
+ [ -f "$p" ] || continue
+ ./"$p" | tee out.txt; grep -q usage out.txt
+ done
+ - uses: actions/upload-artifact@v4
+ with:
+ name: pf-serve-${{ matrix.target }}
+ path: pf-serve-${{ matrix.target }}.zip
+
+ release:
+ if: startsWith(github.ref, 'refs/tags/pf-v')
+ needs: build
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - uses: actions/download-artifact@v4
+ with:
+ merge-multiple: true
+ - name: publish
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ sha256sum pf-serve-*.zip | tee SHA256SUMS
+ gh release create "$GITHUB_REF_NAME" -R "$GITHUB_REPOSITORY" --prerelease --latest=false \
+ --title "pf-serve $GITHUB_REF_NAME" \
+ --notes "pf-serve, which runs the local masking model of Settings → Privacy (native/privacy-filter). Not a magpie release: magpie downloads these itself, checked against the sha256s in internal/pii." \
+ pf-serve-*.zip SHA256SUMS
diff --git a/native/privacy-filter/.gitignore b/native/privacy-filter/.gitignore
new file mode 100644
index 000000000..3c1b3b23b
--- /dev/null
+++ b/native/privacy-filter/.gitignore
@@ -0,0 +1 @@
+.work/
diff --git a/native/privacy-filter/CMakeLists.txt b/native/privacy-filter/CMakeLists.txt
new file mode 100644
index 000000000..f83d3b80c
--- /dev/null
+++ b/native/privacy-filter/CMakeLists.txt
@@ -0,0 +1,16 @@
+# pf-serve: privacy-filter.cpp (PF_SRC, patched by build.sh) and the
+# process magpie talks to.
+cmake_minimum_required(VERSION 3.21)
+project(pf-serve C CXX)
+
+set(PF_SRC "" CACHE PATH "a privacy-filter.cpp checkout")
+if (NOT PF_SRC)
+ message(FATAL_ERROR "set PF_SRC, or run build.sh")
+endif()
+set(PF_BUILD_TOOLS OFF CACHE BOOL "" FORCE)
+set(PF_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+add_subdirectory(${PF_SRC} pf)
+
+add_executable(pf-serve pf-serve.cpp)
+target_link_libraries(pf-serve PRIVATE pf)
+target_compile_features(pf-serve PRIVATE cxx_std_17)
diff --git a/native/privacy-filter/build.sh b/native/privacy-filter/build.sh
new file mode 100755
index 000000000..525860df6
--- /dev/null
+++ b/native/privacy-filter/build.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# build.sh <target> <out.zip>: pf-serve for one of darwin-arm64,
+# darwin-amd64, linux-amd64, linux-arm64, windows-amd64, zipped.
+#
+# privacy-filter.cpp and ggml are taken at the commits pinned here, with
+# patches/ applied. A target with a GPU through Vulkan (linux-amd64,
+# windows-amd64) gets two programs: pf-serve-vulkan, linked to the Vulkan
+# loader the GPU's driver installs, and pf-serve, the CPU's alone, for a
+# computer with no loader, where pf-serve-vulkan can't start at all.
+# darwin-arm64's pf-serve uses Metal.
+set -euo pipefail
+
+PF_REPO=https://github.com/localai-org/privacy-filter.cpp
+PF_COMMIT=735a6c28607ee82afc3a670383f41b55266a3b9a
+VULKAN_SDK_TAG=1.4.309
+SPIRV_TAG=vulkan-sdk-1.4.309.0
+
+target=$1
+out=$(cd "$(dirname "$2")" && pwd)/$(basename "$2")
+here=$(cd "$(dirname "$0")" && pwd)
+work=${PF_WORK:-$here/.work}
+src=$work/privacy-filter.cpp
+mkdir -p "$work"
+
+if [ ! -d "$src/.git" ]; then
+ git init -q "$src"
+ git -C "$src" remote add origin "$PF_REPO"
+fi
+if [ "$(git -C "$src" rev-parse -q --verify HEAD || true)" != "$PF_COMMIT" ]; then
+ git -C "$src" fetch -q --depth 1 origin "$PF_COMMIT"
+ git -C "$src" checkout -q --force FETCH_HEAD
+fi
+git -C "$src" submodule update -q --init --depth 1 ggml
+git -C "$src" checkout -q -- .
+git -C "$src/ggml" checkout -q -- .
+for p in "$here"/patches/*.patch; do
+ git -C "$src" apply "$p"
+done
+
+# Vulkan's headers, SPIR-V's and the loader's exports, for the targets
+# that build pf-serve-vulkan
+vk=$work/vulkan
+fetch_vulkan() {
+ [ -f "$vk/.done-$VULKAN_SDK_TAG" ] && return
+ rm -rf "$vk" && mkdir -p "$vk/include" "$vk/lib"
+ curl -fsSL "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v$VULKAN_SDK_TAG.tar.gz" | tar -xz -C "$vk"
+ curl -fsSL "https://github.com/KhronosGroup/SPIRV-Headers/archive/refs/tags/$SPIRV_TAG.tar.gz" | tar -xz -C "$vk"
+ cp -R "$vk/Vulkan-Headers-$VULKAN_SDK_TAG/include/"* "$vk/include/"
+ cp -R "$vk/SPIRV-Headers-$SPIRV_TAG/include/"* "$vk/include/"
+ curl -fsSL -o "$vk/vulkan-1.def" "https://raw.githubusercontent.com/KhronosGroup/Vulkan-Loader/v$VULKAN_SDK_TAG/loader/vulkan-1.def"
+ touch "$vk/.done-$VULKAN_SDK_TAG"
+}
+
+jobs=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)
+stage=$work/stage-$target
+rm -rf "$stage" && mkdir -p "$stage"
+
+# build <name> <cmake args…>: pf-serve built with those, copied to the
+# stage as name
+build() {
+ local name=$1
+ shift
+ local b=$work/build-$target-$name
+ cmake -S "$here" -B "$b" -G Ninja -DCMAKE_BUILD_TYPE=Release -DPF_SRC="$src" \
+ -DBUILD_SHARED_LIBS=OFF -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF "$@"
+ cmake --build "$b" -j "$jobs" --target pf-serve
+ local exe=pf-serve
+ case $target in windows-*) exe=pf-serve.exe ;; esac
+ cp "$b/$exe" "$stage/$name${exe#pf-serve}"
+}
+
+x86=(-DGGML_AVX=ON -DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON -DGGML_BMI2=ON)
+case $target in
+ darwin-arm64)
+ build pf-serve -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 \
+ -DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_BLAS=OFF
+ ;;
+ darwin-amd64)
+ build pf-serve -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 \
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 -DGGML_METAL=OFF -DGGML_BLAS=OFF "${x86[@]}"
+ ;;
+ linux-amd64)
+ fetch_vulkan
+ static=(-DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++")
+ build pf-serve "${static[@]}" "${x86[@]}"
+ build pf-serve-vulkan "${static[@]}" "${x86[@]}" -DPF_VULKAN=ON -DVulkan_INCLUDE_DIR="$vk/include"
+ ;;
+ linux-arm64)
+ build pf-serve -DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++"
+ ;;
+ windows-amd64)
+ fetch_vulkan
+ x86_64-w64-mingw32-dlltool -d "$vk/vulkan-1.def" -l "$vk/lib/libvulkan-1.a"
+ win=(-DCMAKE_TOOLCHAIN_FILE="$here/mingw.cmake" -DCMAKE_C_FLAGS=-D_WIN32_WINNT=0x0A00 -DCMAKE_CXX_FLAGS=-D_WIN32_WINNT=0x0A00)
+ build pf-serve "${win[@]}" "${x86[@]}"
+ build pf-serve-vulkan "${win[@]}" "${x86[@]}" -DPF_VULKAN=ON -DVulkan_INCLUDE_DIR="$vk/include" \
+ -DVulkan_LIBRARY="$vk/lib/libvulkan-1.a" -DVulkan_GLSLC_EXECUTABLE="$(command -v glslc)"
+ ;;
+ *)
+ echo "unknown target $target" >&2
+ exit 2
+ ;;
+esac
+
+cp "$src/LICENSE" "$stage/LICENSE-privacy-filter.cpp"
+cp "$src/ggml/LICENSE" "$stage/LICENSE-ggml"
+rm -f "$out"
+(cd "$stage" && zip -q -X -r "$out" .)
+ls -l "$out"
diff --git a/native/privacy-filter/mingw.cmake b/native/privacy-filter/mingw.cmake
new file mode 100644
index 000000000..9e7d4a03c
--- /dev/null
+++ b/native/privacy-filter/mingw.cmake
@@ -0,0 +1,11 @@
+# Windows on x86-64 with mingw-w64, from Linux: one static .exe each.
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR x86_64)
+set(CMAKE_C_COMPILER x86_64-w64-mingw32-gcc-posix)
+set(CMAKE_CXX_COMPILER x86_64-w64-mingw32-g++-posix)
+set(CMAKE_RC_COMPILER x86_64-w64-mingw32-windres)
+set(CMAKE_FIND_ROOT_PATH /usr/x86_64-w64-mingw32)
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_EXE_LINKER_FLAGS "-static -static-libgcc -static-libstdc++")
diff --git a/native/privacy-filter/patches/ggml-mingw-throttling.patch b/native/privacy-filter/patches/ggml-mingw-throttling.patch
new file mode 100644
index 000000000..d776021c9
--- /dev/null
+++ b/native/privacy-filter/patches/ggml-mingw-throttling.patch
@@ -0,0 +1,13 @@
+diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
+index eb8341c..30891da 100644
+--- a/ggml/src/ggml-cpu/ggml-cpu.c
++++ b/ggml/src/ggml-cpu/ggml-cpu.c
+@@ -2516,7 +2516,7 @@ static bool ggml_thread_apply_priority(int32_t prio) {
+ // Newer Windows 11 versions aggressively park (offline) CPU cores and often place
+ // all our threads onto the first 4 cores which results in terrible performance with
+ // n_threads > 4
+- #if _WIN32_WINNT >= 0x0602
++ #if _WIN32_WINNT >= 0x0602 && defined(THREAD_POWER_THROTTLING_CURRENT_VERSION)
+ THREAD_POWER_THROTTLING_STATE t;
+ ZeroMemory(&t, sizeof(t));
+ t.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;
diff --git a/native/privacy-filter/patches/ner-cstdint.patch b/native/privacy-filter/patches/ner-cstdint.patch
new file mode 100644
index 000000000..f4b422c0d
--- /dev/null
+++ b/native/privacy-filter/patches/ner-cstdint.patch
@@ -0,0 +1,13 @@
+diff --git a/src/ner.h b/src/ner.h
+index b61b264..1d5b597 100644
+--- a/src/ner.h
++++ b/src/ner.h
+@@ -4,6 +4,8 @@
+ // the behavior LocalAI ships today.
+ #pragma once
+
++#include <cstdint>
++
+ #include <string>
+ #include <vector>
+
diff --git a/native/privacy-filter/pf-serve.cpp b/native/privacy-filter/pf-serve.cpp
new file mode 100644
index 000000000..cd73a7bc2
--- /dev/null
+++ b/native/privacy-filter/pf-serve.cpp
@@ -0,0 +1,194 @@
+// pf-serve runs OpenAI's Privacy Filter for magpie's masking (internal/pii):
+// a process of its own, so the model's 1.6 GB is let go by ending it and a
+// crash in a GPU driver ends no gateway.
+//
+// pf-serve <model.gguf> [threshold]
+//
+// It loads the model on a GPU when there is one (PF_DEVICE=cpu keeps it on
+// the CPU), runs a few texts so the GPU's kernels are built before the
+// first request, and writes one JSON line: {"device":…,"gpu":…,"ms":…}, or
+// {"error":…} and exits 1. Then, until stdin ends, each request is
+//
+// u32 n, then n times: u32 len, len bytes of UTF-8
+//
+// and each reply is u32 0 and, for each text in order, u32 count and count
+// times: i32 start, i32 end (byte offsets), f32 score, u8 len, the label;
+// or u32 1, u32 len and an error's text. Little-endian throughout.
+#include "pf.h"
+#include "ggml-backend.h"
+
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#ifdef _WIN32
+#include <fcntl.h>
+#include <io.h>
+#endif
+
+static bool read_all(void * p, size_t n) {
+ return n == 0 || std::fread(p, 1, n, stdin) == n;
+}
+
+static uint32_t le32(const unsigned char * b) {
+ return (uint32_t) b[0] | (uint32_t) b[1] << 8 | (uint32_t) b[2] << 16 | (uint32_t) b[3] << 24;
+}
+
+static void put32(std::string & out, uint32_t v) {
+ const char b[4] = {(char) (v & 0xff), (char) (v >> 8 & 0xff), (char) (v >> 16 & 0xff), (char) (v >> 24 & 0xff)};
+ out.append(b, 4);
+}
+
+static std::string json_str(const std::string & s) {
+ std::string o = "\"";
+ for (unsigned char c : s) {
+ if (c == '"' || c == '\\') {
+ o += '\\';
+ o += (char) c;
+ } else if (c < 0x20) {
+ char b[8];
+ std::snprintf(b, sizeof b, "\\u%04x", c);
+ o += b;
+ } else {
+ o += (char) c;
+ }
+ }
+ return o + "\"";
+}
+
+static void hello(const std::string & line) {
+ std::fwrite(line.data(), 1, line.size(), stdout);
+ std::fputc('\n', stdout);
+ std::fflush(stdout);
+}
+
+// gpu_name is the GPU "gpu" picks, as pf's backend picks it: the first
+// discrete one, else the first integrated one.
+static std::string gpu_name() {
+ for (int want : {(int) GGML_BACKEND_DEVICE_TYPE_GPU, (int) GGML_BACKEND_DEVICE_TYPE_IGPU}) {
+ for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+ ggml_backend_dev_t d = ggml_backend_dev_get(i);
+ if ((int) ggml_backend_dev_type(d) == want) {
+ std::string reg = ggml_backend_reg_name(ggml_backend_dev_backend_reg(d));
+ return reg + ": " + ggml_backend_dev_description(d);
+ }
+ }
+ }
+ return "gpu";
+}
+
+int main(int argc, char ** argv) {
+#ifdef _WIN32
+ _setmode(_fileno(stdin), _O_BINARY);
+ _setmode(_fileno(stdout), _O_BINARY);
+#endif
+ if (argc < 2) {
+ hello("{\"error\":\"usage: pf-serve <model.gguf> [threshold]\"}");
+ return 2;
+ }
+ const float threshold = argc > 2 ? (float) std::atof(argv[2]) : 0.5f;
+ const auto t0 = std::chrono::steady_clock::now();
+
+ const char * env = std::getenv("PF_DEVICE");
+ const bool cpu_only = env && std::strcmp(env, "cpu") == 0;
+ pf_ctx * ctx = nullptr;
+ std::string device, gpu_error;
+ if (!cpu_only) {
+ ctx = pf_load(argv[1], "gpu", 0);
+ if (ctx && !pf_last_error(ctx)) {
+ device = gpu_name();
+ } else {
+ gpu_error = ctx && pf_last_error(ctx) ? pf_last_error(ctx) : "no GPU";
+ pf_free(ctx);
+ ctx = nullptr;
+ }
+ }
+ if (!ctx) {
+ ctx = pf_load(argv[1], "cpu", 0);
+ if (!ctx || pf_last_error(ctx)) {
+ hello("{\"error\":" + json_str(ctx && pf_last_error(ctx) ? pf_last_error(ctx) : "the model didn't load") + "}");
+ return 1;
+ }
+ device = "cpu";
+ }
+
+ // the GPU's kernels are built on first use, a second or more: here,
+ // not on a request
+ // in a short text and longer ones, as kernels are picked by size
+ std::string warm = "Hi Bob";
+ for (int n : {0, 1, 12, 48}) {
+ while ((int) warm.size() < n * 160) {
+ warm += " Ask Jane Doe at [email protected] or +1 415 555 0100 about 221B Baker Street, account 12345678.";
+ }
+ pf_entity * ents = nullptr;
+ size_t ne = 0;
+ if (pf_classify(ctx, warm.data(), warm.size(), threshold, &ents, &ne) != 0) {
+ hello("{\"error\":" + json_str(std::string("warm-up: ") + (pf_last_error(ctx) ? pf_last_error(ctx) : "failed")) + "}");
+ return 1;
+ }
+ pf_entities_free(ents, ne);
+ }
+ const long long ms = (long long) std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();
+ std::string h = "{\"device\":" + json_str(device) + ",\"gpu\":" + (device == "cpu" ? "false" : "true") + ",\"ms\":" + std::to_string(ms);
+ if (!gpu_error.empty()) {
+ h += ",\"gpuError\":" + json_str(gpu_error);
+ }
+ hello(h + "}");
+
+ std::vector<std::string> texts;
+ std::string out;
+ for (;;) {
+ unsigned char b[4];
+ if (!read_all(b, 4)) {
+ break;
+ }
+ const uint32_t n = le32(b);
+ texts.assign(n, std::string());
+ bool ok = true;
+ for (uint32_t i = 0; i < n && ok; i++) {
+ ok = read_all(b, 4);
+ if (ok) {
+ texts[i].resize(le32(b));
+ ok = read_all(texts[i].data(), texts[i].size());
+ }
+ }
+ if (!ok) {
+ break;
+ }
+ out.clear();
+ put32(out, 0);
+ for (const std::string & t : texts) {
+ pf_entity * ents = nullptr;
+ size_t ne = 0;
+ if (!t.empty() && pf_classify(ctx, t.data(), t.size(), threshold, &ents, &ne) != 0) {
+ const std::string e = pf_last_error(ctx) ? pf_last_error(ctx) : "classify failed";
+ out.clear();
+ put32(out, 1);
+ put32(out, (uint32_t) e.size());
+ out += e;
+ break;
+ }
+ put32(out, (uint32_t) ne);
+ for (size_t i = 0; i < ne; i++) {
+ put32(out, (uint32_t) ents[i].start);
+ put32(out, (uint32_t) ents[i].end);
+ uint32_t f;
+ std::memcpy(&f, &ents[i].score, 4);
+ put32(out, f);
+ const size_t ll = std::strlen(ents[i].label);
+ out += (char) (ll > 255 ? 255 : ll);
+ out.append(ents[i].label, ll > 255 ? 255 : ll);
+ }
+ pf_entities_free(ents, ne);
+ }
+ if (std::fwrite(out.data(), 1, out.size(), stdout) != out.size() || std::fflush(stdout) != 0) {
+ break;
+ }
+ }
+ pf_free(ctx);
+ return 0;
+}