Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
native: pf-serve, a sidecar that runs OpenAI's Privacy Filter for masking (#1977)
yetoneffa292c7d388
diff --git a/.github/workflows/privacy-filter.yml b/.github/workflows/privacy-filter.ymlnew file mode 100644index 000000000..2c366e5c6--- /dev/null+++ b/.github/workflows/privacy-filter.yml@@ -0,0 +1,77 @@+# Builds pf-serve, the program that runs Settings → Privacy's local+# masking model (native/privacy-filter, internal/pii), for each system.+# A change to it is built on every push; a pf-v* tag publishes the zips as+# a prerelease of this repo, never Latest, whose sha256s internal/pii pins.+name: privacy-filter++on:+ push:+ branches: [main, "pf/**"]+ paths: ["native/privacy-filter/**", ".github/workflows/privacy-filter.yml"]+ tags: ["pf-v*"]+ pull_request:+ paths: ["native/privacy-filter/**", ".github/workflows/privacy-filter.yml"]+ workflow_dispatch:++permissions:+ contents: read++jobs:+ build:+ strategy:+ fail-fast: false+ matrix:+ include:+ - { target: darwin-arm64, os: macos-15 }+ - { target: darwin-amd64, os: macos-15 }+ # the oldest glibc a runner has, so the program runs on older systems too+ - { target: linux-amd64, os: ubuntu-22.04 }+ - { target: linux-arm64, os: ubuntu-22.04-arm }+ - { target: windows-amd64, os: ubuntu-24.04 }+ runs-on: ${{ matrix.os }}+ steps:+ - uses: actions/checkout@v4+ - name: tools (macOS)+ if: runner.os == 'macOS'+ run: brew install ninja+ - name: tools (Linux)+ if: startsWith(matrix.target, 'linux')+ run: sudo apt-get update && sudo apt-get install -y ninja-build libvulkan-dev glslc zip+ - name: tools (Windows, cross)+ if: matrix.target == 'windows-amd64'+ run: sudo apt-get update && sudo apt-get install -y ninja-build mingw-w64 glslc zip+ - name: build+ run: native/privacy-filter/build.sh ${{ matrix.target }} pf-serve-${{ matrix.target }}.zip+ - name: starts+ # each program the runner can start runs to its usage line+ if: matrix.target != 'windows-amd64' && matrix.target != 'darwin-amd64'+ run: |+ mkdir t && cd t && unzip -q ../pf-serve-${{ matrix.target }}.zip+ for p in pf-serve pf-serve-vulkan; do+ [ -f "$p" ] || continue+ ./"$p" | tee out.txt; grep -q usage out.txt+ done+ - uses: actions/upload-artifact@v4+ with:+ name: pf-serve-${{ matrix.target }}+ path: pf-serve-${{ matrix.target }}.zip++ release:+ if: startsWith(github.ref, 'refs/tags/pf-v')+ needs: build+ runs-on: ubuntu-latest+ permissions:+ contents: write+ steps:+ - uses: actions/download-artifact@v4+ with:+ merge-multiple: true+ - name: publish+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ sha256sum pf-serve-*.zip | tee SHA256SUMS+ gh release create "$GITHUB_REF_NAME" -R "$GITHUB_REPOSITORY" --prerelease --latest=false \+ --title "pf-serve $GITHUB_REF_NAME" \+ --notes "pf-serve, which runs the local masking model of Settings → Privacy (native/privacy-filter). Not a magpie release: magpie downloads these itself, checked against the sha256s in internal/pii." \+ pf-serve-*.zip SHA256SUMSdiff --git a/native/privacy-filter/.gitignore b/native/privacy-filter/.gitignorenew file mode 100644index 000000000..3c1b3b23b--- /dev/null+++ b/native/privacy-filter/.gitignore@@ -0,0 +1 @@+.work/diff --git a/native/privacy-filter/CMakeLists.txt b/native/privacy-filter/CMakeLists.txtnew file mode 100644index 000000000..f83d3b80c--- /dev/null+++ b/native/privacy-filter/CMakeLists.txt@@ -0,0 +1,16 @@+# pf-serve: privacy-filter.cpp (PF_SRC, patched by build.sh) and the+# process magpie talks to.+cmake_minimum_required(VERSION 3.21)+project(pf-serve C CXX)++set(PF_SRC "" CACHE PATH "a privacy-filter.cpp checkout")+if (NOT PF_SRC)+ message(FATAL_ERROR "set PF_SRC, or run build.sh")+endif()+set(PF_BUILD_TOOLS OFF CACHE BOOL "" FORCE)+set(PF_BUILD_TESTS OFF CACHE BOOL "" FORCE)+add_subdirectory(${PF_SRC} pf)++add_executable(pf-serve pf-serve.cpp)+target_link_libraries(pf-serve PRIVATE pf)+target_compile_features(pf-serve PRIVATE cxx_std_17)diff --git a/native/privacy-filter/build.sh b/native/privacy-filter/build.shnew file mode 100755index 000000000..525860df6--- /dev/null+++ b/native/privacy-filter/build.sh@@ -0,0 +1,109 @@+#!/usr/bin/env bash+# build.sh <target> <out.zip>: pf-serve for one of darwin-arm64,+# darwin-amd64, linux-amd64, linux-arm64, windows-amd64, zipped.+#+# privacy-filter.cpp and ggml are taken at the commits pinned here, with+# patches/ applied. A target with a GPU through Vulkan (linux-amd64,+# windows-amd64) gets two programs: pf-serve-vulkan, linked to the Vulkan+# loader the GPU's driver installs, and pf-serve, the CPU's alone, for a+# computer with no loader, where pf-serve-vulkan can't start at all.+# darwin-arm64's pf-serve uses Metal.+set -euo pipefail++PF_REPO=https://github.com/localai-org/privacy-filter.cpp+PF_COMMIT=735a6c28607ee82afc3a670383f41b55266a3b9a+VULKAN_SDK_TAG=1.4.309+SPIRV_TAG=vulkan-sdk-1.4.309.0++target=$1+out=$(cd "$(dirname "$2")" && pwd)/$(basename "$2")+here=$(cd "$(dirname "$0")" && pwd)+work=${PF_WORK:-$here/.work}+src=$work/privacy-filter.cpp+mkdir -p "$work"++if [ ! -d "$src/.git" ]; then+ git init -q "$src"+ git -C "$src" remote add origin "$PF_REPO"+fi+if [ "$(git -C "$src" rev-parse -q --verify HEAD || true)" != "$PF_COMMIT" ]; then+ git -C "$src" fetch -q --depth 1 origin "$PF_COMMIT"+ git -C "$src" checkout -q --force FETCH_HEAD+fi+git -C "$src" submodule update -q --init --depth 1 ggml+git -C "$src" checkout -q -- .+git -C "$src/ggml" checkout -q -- .+for p in "$here"/patches/*.patch; do+ git -C "$src" apply "$p"+done++# Vulkan's headers, SPIR-V's and the loader's exports, for the targets+# that build pf-serve-vulkan+vk=$work/vulkan+fetch_vulkan() {+ [ -f "$vk/.done-$VULKAN_SDK_TAG" ] && return+ rm -rf "$vk" && mkdir -p "$vk/include" "$vk/lib"+ curl -fsSL "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v$VULKAN_SDK_TAG.tar.gz" | tar -xz -C "$vk"+ curl -fsSL "https://github.com/KhronosGroup/SPIRV-Headers/archive/refs/tags/$SPIRV_TAG.tar.gz" | tar -xz -C "$vk"+ cp -R "$vk/Vulkan-Headers-$VULKAN_SDK_TAG/include/"* "$vk/include/"+ cp -R "$vk/SPIRV-Headers-$SPIRV_TAG/include/"* "$vk/include/"+ curl -fsSL -o "$vk/vulkan-1.def" "https://raw.githubusercontent.com/KhronosGroup/Vulkan-Loader/v$VULKAN_SDK_TAG/loader/vulkan-1.def"+ touch "$vk/.done-$VULKAN_SDK_TAG"+}++jobs=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)+stage=$work/stage-$target+rm -rf "$stage" && mkdir -p "$stage"++# build <name> <cmake args…>: pf-serve built with those, copied to the+# stage as name+build() {+ local name=$1+ shift+ local b=$work/build-$target-$name+ cmake -S "$here" -B "$b" -G Ninja -DCMAKE_BUILD_TYPE=Release -DPF_SRC="$src" \+ -DBUILD_SHARED_LIBS=OFF -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF "$@"+ cmake --build "$b" -j "$jobs" --target pf-serve+ local exe=pf-serve+ case $target in windows-*) exe=pf-serve.exe ;; esac+ cp "$b/$exe" "$stage/$name${exe#pf-serve}"+}++x86=(-DGGML_AVX=ON -DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON -DGGML_BMI2=ON)+case $target in+ darwin-arm64)+ build pf-serve -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 \+ -DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_BLAS=OFF+ ;;+ darwin-amd64)+ build pf-serve -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 \+ -DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 -DGGML_METAL=OFF -DGGML_BLAS=OFF "${x86[@]}"+ ;;+ linux-amd64)+ fetch_vulkan+ static=(-DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++")+ build pf-serve "${static[@]}" "${x86[@]}"+ build pf-serve-vulkan "${static[@]}" "${x86[@]}" -DPF_VULKAN=ON -DVulkan_INCLUDE_DIR="$vk/include"+ ;;+ linux-arm64)+ build pf-serve -DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++"+ ;;+ windows-amd64)+ fetch_vulkan+ x86_64-w64-mingw32-dlltool -d "$vk/vulkan-1.def" -l "$vk/lib/libvulkan-1.a"+ win=(-DCMAKE_TOOLCHAIN_FILE="$here/mingw.cmake" -DCMAKE_C_FLAGS=-D_WIN32_WINNT=0x0A00 -DCMAKE_CXX_FLAGS=-D_WIN32_WINNT=0x0A00)+ build pf-serve "${win[@]}" "${x86[@]}"+ build pf-serve-vulkan "${win[@]}" "${x86[@]}" -DPF_VULKAN=ON -DVulkan_INCLUDE_DIR="$vk/include" \+ -DVulkan_LIBRARY="$vk/lib/libvulkan-1.a" -DVulkan_GLSLC_EXECUTABLE="$(command -v glslc)"+ ;;+ *)+ echo "unknown target $target" >&2+ exit 2+ ;;+esac++cp "$src/LICENSE" "$stage/LICENSE-privacy-filter.cpp"+cp "$src/ggml/LICENSE" "$stage/LICENSE-ggml"+rm -f "$out"+(cd "$stage" && zip -q -X -r "$out" .)+ls -l "$out"diff --git a/native/privacy-filter/mingw.cmake b/native/privacy-filter/mingw.cmakenew file mode 100644index 000000000..9e7d4a03c--- /dev/null+++ b/native/privacy-filter/mingw.cmake@@ -0,0 +1,11 @@+# Windows on x86-64 with mingw-w64, from Linux: one static .exe each.+set(CMAKE_SYSTEM_NAME Windows)+set(CMAKE_SYSTEM_PROCESSOR x86_64)+set(CMAKE_C_COMPILER x86_64-w64-mingw32-gcc-posix)+set(CMAKE_CXX_COMPILER x86_64-w64-mingw32-g++-posix)+set(CMAKE_RC_COMPILER x86_64-w64-mingw32-windres)+set(CMAKE_FIND_ROOT_PATH /usr/x86_64-w64-mingw32)+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)+set(CMAKE_EXE_LINKER_FLAGS "-static -static-libgcc -static-libstdc++")diff --git a/native/privacy-filter/patches/ggml-mingw-throttling.patch b/native/privacy-filter/patches/ggml-mingw-throttling.patchnew file mode 100644index 000000000..d776021c9--- /dev/null+++ b/native/privacy-filter/patches/ggml-mingw-throttling.patch@@ -0,0 +1,13 @@+diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c+index eb8341c..30891da 100644+--- a/ggml/src/ggml-cpu/ggml-cpu.c++++ b/ggml/src/ggml-cpu/ggml-cpu.c+@@ -2516,7 +2516,7 @@ static bool ggml_thread_apply_priority(int32_t prio) {+ // Newer Windows 11 versions aggressively park (offline) CPU cores and often place+ // all our threads onto the first 4 cores which results in terrible performance with+ // n_threads > 4+- #if _WIN32_WINNT >= 0x0602++ #if _WIN32_WINNT >= 0x0602 && defined(THREAD_POWER_THROTTLING_CURRENT_VERSION)+ THREAD_POWER_THROTTLING_STATE t;+ ZeroMemory(&t, sizeof(t));+ t.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;diff --git a/native/privacy-filter/patches/ner-cstdint.patch b/native/privacy-filter/patches/ner-cstdint.patchnew file mode 100644index 000000000..f4b422c0d--- /dev/null+++ b/native/privacy-filter/patches/ner-cstdint.patch@@ -0,0 +1,13 @@+diff --git a/src/ner.h b/src/ner.h+index b61b264..1d5b597 100644+--- a/src/ner.h++++ b/src/ner.h+@@ -4,6 +4,8 @@+ // the behavior LocalAI ships today.+ #pragma once+++#include <cstdint>+++ #include <string>+ #include <vector>+diff --git a/native/privacy-filter/pf-serve.cpp b/native/privacy-filter/pf-serve.cppnew file mode 100644index 000000000..cd73a7bc2--- /dev/null+++ b/native/privacy-filter/pf-serve.cpp@@ -0,0 +1,194 @@+// pf-serve runs OpenAI's Privacy Filter for magpie's masking (internal/pii):+// a process of its own, so the model's 1.6 GB is let go by ending it and a+// crash in a GPU driver ends no gateway.+//+// pf-serve <model.gguf> [threshold]+//+// It loads the model on a GPU when there is one (PF_DEVICE=cpu keeps it on+// the CPU), runs a few texts so the GPU's kernels are built before the+// first request, and writes one JSON line: {"device":…,"gpu":…,"ms":…}, or+// {"error":…} and exits 1. Then, until stdin ends, each request is+//+// u32 n, then n times: u32 len, len bytes of UTF-8+//+// and each reply is u32 0 and, for each text in order, u32 count and count+// times: i32 start, i32 end (byte offsets), f32 score, u8 len, the label;+// or u32 1, u32 len and an error's text. Little-endian throughout.+#include "pf.h"+#include "ggml-backend.h"++#include <chrono>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <string>+#include <vector>++#ifdef _WIN32+#include <fcntl.h>+#include <io.h>+#endif++static bool read_all(void * p, size_t n) {+ return n == 0 || std::fread(p, 1, n, stdin) == n;+}++static uint32_t le32(const unsigned char * b) {+ return (uint32_t) b[0] | (uint32_t) b[1] << 8 | (uint32_t) b[2] << 16 | (uint32_t) b[3] << 24;+}++static void put32(std::string & out, uint32_t v) {+ const char b[4] = {(char) (v & 0xff), (char) (v >> 8 & 0xff), (char) (v >> 16 & 0xff), (char) (v >> 24 & 0xff)};+ out.append(b, 4);+}++static std::string json_str(const std::string & s) {+ std::string o = "\"";+ for (unsigned char c : s) {+ if (c == '"' || c == '\\') {+ o += '\\';+ o += (char) c;+ } else if (c < 0x20) {+ char b[8];+ std::snprintf(b, sizeof b, "\\u%04x", c);+ o += b;+ } else {+ o += (char) c;+ }+ }+ return o + "\"";+}++static void hello(const std::string & line) {+ std::fwrite(line.data(), 1, line.size(), stdout);+ std::fputc('\n', stdout);+ std::fflush(stdout);+}++// gpu_name is the GPU "gpu" picks, as pf's backend picks it: the first+// discrete one, else the first integrated one.+static std::string gpu_name() {+ for (int want : {(int) GGML_BACKEND_DEVICE_TYPE_GPU, (int) GGML_BACKEND_DEVICE_TYPE_IGPU}) {+ for (size_t i = 0; i < ggml_backend_dev_count(); i++) {+ ggml_backend_dev_t d = ggml_backend_dev_get(i);+ if ((int) ggml_backend_dev_type(d) == want) {+ std::string reg = ggml_backend_reg_name(ggml_backend_dev_backend_reg(d));+ return reg + ": " + ggml_backend_dev_description(d);+ }+ }+ }+ return "gpu";+}++int main(int argc, char ** argv) {+#ifdef _WIN32+ _setmode(_fileno(stdin), _O_BINARY);+ _setmode(_fileno(stdout), _O_BINARY);+#endif+ if (argc < 2) {+ hello("{\"error\":\"usage: pf-serve <model.gguf> [threshold]\"}");+ return 2;+ }+ const float threshold = argc > 2 ? (float) std::atof(argv[2]) : 0.5f;+ const auto t0 = std::chrono::steady_clock::now();++ const char * env = std::getenv("PF_DEVICE");+ const bool cpu_only = env && std::strcmp(env, "cpu") == 0;+ pf_ctx * ctx = nullptr;+ std::string device, gpu_error;+ if (!cpu_only) {+ ctx = pf_load(argv[1], "gpu", 0);+ if (ctx && !pf_last_error(ctx)) {+ device = gpu_name();+ } else {+ gpu_error = ctx && pf_last_error(ctx) ? pf_last_error(ctx) : "no GPU";+ pf_free(ctx);+ ctx = nullptr;+ }+ }+ if (!ctx) {+ ctx = pf_load(argv[1], "cpu", 0);+ if (!ctx || pf_last_error(ctx)) {+ hello("{\"error\":" + json_str(ctx && pf_last_error(ctx) ? pf_last_error(ctx) : "the model didn't load") + "}");+ return 1;+ }+ device = "cpu";+ }++ // the GPU's kernels are built on first use, a second or more: here,+ // not on a request+ // in a short text and longer ones, as kernels are picked by size+ std::string warm = "Hi Bob";+ for (int n : {0, 1, 12, 48}) {+ while ((int) warm.size() < n * 160) {+ warm += " Ask Jane Doe at [email protected] or +1 415 555 0100 about 221B Baker Street, account 12345678.";+ }+ pf_entity * ents = nullptr;+ size_t ne = 0;+ if (pf_classify(ctx, warm.data(), warm.size(), threshold, &ents, &ne) != 0) {+ hello("{\"error\":" + json_str(std::string("warm-up: ") + (pf_last_error(ctx) ? pf_last_error(ctx) : "failed")) + "}");+ return 1;+ }+ pf_entities_free(ents, ne);+ }+ const long long ms = (long long) std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0).count();+ std::string h = "{\"device\":" + json_str(device) + ",\"gpu\":" + (device == "cpu" ? "false" : "true") + ",\"ms\":" + std::to_string(ms);+ if (!gpu_error.empty()) {+ h += ",\"gpuError\":" + json_str(gpu_error);+ }+ hello(h + "}");++ std::vector<std::string> texts;+ std::string out;+ for (;;) {+ unsigned char b[4];+ if (!read_all(b, 4)) {+ break;+ }+ const uint32_t n = le32(b);+ texts.assign(n, std::string());+ bool ok = true;+ for (uint32_t i = 0; i < n && ok; i++) {+ ok = read_all(b, 4);+ if (ok) {+ texts[i].resize(le32(b));+ ok = read_all(texts[i].data(), texts[i].size());+ }+ }+ if (!ok) {+ break;+ }+ out.clear();+ put32(out, 0);+ for (const std::string & t : texts) {+ pf_entity * ents = nullptr;+ size_t ne = 0;+ if (!t.empty() && pf_classify(ctx, t.data(), t.size(), threshold, &ents, &ne) != 0) {+ const std::string e = pf_last_error(ctx) ? pf_last_error(ctx) : "classify failed";+ out.clear();+ put32(out, 1);+ put32(out, (uint32_t) e.size());+ out += e;+ break;+ }+ put32(out, (uint32_t) ne);+ for (size_t i = 0; i < ne; i++) {+ put32(out, (uint32_t) ents[i].start);+ put32(out, (uint32_t) ents[i].end);+ uint32_t f;+ std::memcpy(&f, &ents[i].score, 4);+ put32(out, f);+ const size_t ll = std::strlen(ents[i].label);+ out += (char) (ll > 255 ? 255 : ll);+ out.append(ents[i].label, ll > 255 ? 255 : ll);+ }+ pf_entities_free(ents, ne);+ }+ if (std::fwrite(out.data(), 1, out.size(), stdout) != out.size() || std::fflush(stdout) != 0) {+ break;+ }+ }+ pf_free(ctx);+ return 0;+}