π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
desktop parity smoke test runner
laurenspriem935dbd97ee75
diff --git a/desktop/scripts/ml-native.js b/desktop/scripts/ml-native.jsindex f189019c39..1572b3ccfa 100644--- a/desktop/scripts/ml-native.js+++ b/desktop/scripts/ml-native.js@@ -123,4 +123,4 @@ const stageMLAddons = async (appDir, platform, arch) => {await fsp.rm(path.join(stageDir, entry), { recursive: true });};-module.exports = { stageMLAddons };+module.exports = { napiTriple, stageMLAddons };diff --git a/infra/ml/test/README.md b/infra/ml/test/README.mdindex 6755e5962e..c517d1baae 100644--- a/infra/ml/test/README.md+++ b/infra/ml/test/README.md@@ -1,6 +1,6 @@# ML Indexing Parity Test Suite-This directory contains the ML indexing parity framework for Android, iOS, and Python ground truth. (Desktop ships the same shared Rust pipeline that the `ente-photos` crate's `ml_indexing` integration test validates against the Python goldens, so it has no separate parity runner.)+This directory contains the ML indexing parity framework for desktop, Android, iOS, and Python ground truth. (The desktop runner is a smoke test of the desktop-specific addon path β the napi bindings and the pinned ONNX Runtime library β since deep validation of the shared Rust pipeline itself lives in the `ente-photos` crate's `ml_indexing` integration test.)## Layout@@ -13,6 +13,7 @@ This directory contains the ML indexing parity framework for Android, iOS, and P## Prerequisites- Python + `uv`+- Node + the desktop build artifacts (`npm ci && npm run postinstall` in `desktop/`) when running desktop parity- Flutter SDK (Android/iOS parity)- Android emulator/device and iOS simulator/device when running mobile parity@@ -24,7 +25,7 @@ bash infra/ml/test/run_ml_parity_tests.shCommon flags:-- `--platforms all|android|ios`+- `--platforms all|desktop|android|ios`- `--output-dir <path>`- `--verbose` (stream full runner/comparator logs to terminal)- `--render-detection-overlays` (generate annotated detection images under `out/parity/detections/<platform>/`; includes selected platforms plus `python` ground truth)diff --git a/infra/ml/test/ground_truth/README.md b/infra/ml/test/ground_truth/README.mdindex a9bd777be3..cd65298ca4 100644--- a/infra/ml/test/ground_truth/README.md+++ b/infra/ml/test/ground_truth/README.md@@ -48,6 +48,7 @@ uv run --project infra/ml python infra/ml/test/tools/generate_goldens.py \```shuv run --project infra/ml python infra/ml/test/tools/compare_parity_outputs.py \--ground-truth infra/ml/test/out/parity/python/results.json \+ --platform-result desktop=infra/ml/test/out/parity/desktop/results.json \--platform-result android=infra/ml/test/out/parity/android/results.json \--platform-result ios=infra/ml/test/out/parity/ios/results.json```diff --git a/infra/ml/test/run_ml_parity_tests.sh b/infra/ml/test/run_ml_parity_tests.shindex cd7dac07c7..9cdd215187 100755--- a/infra/ml/test/run_ml_parity_tests.sh+++ b/infra/ml/test/run_ml_parity_tests.sh@@ -29,7 +29,7 @@ usage() {Usage: infra/ml/test/run_ml_parity_tests.sh [flags]Flags:- --platforms all|android|ios (default: all)+ --platforms all|desktop|android|ios (default: all)--output-dir <path> (default: infra/ml/test/out/parity)--verbose (default: disabled)--render-detection-overlays (default: disabled; render annotated face detection images to out/parity/detections/<platform>/)@@ -283,9 +283,9 @@ print_kv "mobile_ml_route:" "rust"declare -a selected_platforms=()case "$PLATFORMS" inall)- selected_platforms=(android ios)+ selected_platforms=(desktop android ios);;- android|ios)+ desktop|android|ios)selected_platforms=("$PLATFORMS");;*)@@ -572,6 +572,17 @@ run_preflight_checks() {for platform in "${selected_platforms[@]}"; docase "$platform" in+ desktop)+ if ! command -v node >/dev/null 2>&1; then+ preflight_errors+=("node is required for desktop parity")+ fi+ if ! compgen -G "$ROOT_DIR/desktop/rust-bindings/index.*.node" >/dev/null; then+ preflight_errors+=("desktop ML addon not found under desktop/rust-bindings (run 'npm ci && npm run postinstall' in desktop/, or 'cargo codegen napi' from rust/)")+ fi+ if [[ ! -d "$ROOT_DIR/desktop/build/onnxruntime" ]]; then+ preflight_errors+=("ONNX Runtime library not found under desktop/build/onnxruntime (run 'node scripts/ort.js' in desktop/)")+ fi+ ;;android|ios)if ! command -v flutter >/dev/null 2>&1; thenpreflight_errors+=("flutter is required for $platform parity")@@ -744,8 +755,11 @@ for platform in "${selected_platforms[@]}"; doesacdone+# Every platform runner uses the local model cache; only the mobile runners+# additionally need it served over HTTP (the desktop runner reads the files+# directly).+prepare_local_model_mirror_cache "$LOCAL_MODEL_MIRROR_DIR"if $has_mobile_platform; then- prepare_local_model_mirror_cache "$LOCAL_MODEL_MIRROR_DIR"if ! start_local_mirror_server "$ML_DIR" "$LOCAL_MIRROR_LOG"; thenecho "Proceeding without local parity mirror."fi@@ -950,6 +964,30 @@ run_mobile_runner() {return 0}+run_desktop_runner() {+ local platform_output_dir="$OUTPUT_DIR/desktop"+ local output_path="$platform_output_dir/results.json"++ echo "Running desktop parity runner"+ if ! node "$ML_DIR/tools/desktop_parity_runner.js" \+ --manifest "$MANIFEST_PATH" \+ --ml-dir "$ML_DIR" \+ --models-dir "$LOCAL_MODEL_MIRROR_DIR" \+ --asset-lock "$ASSET_LOCK_PATH" \+ --code-revision "$CODE_REVISION" \+ --output "$output_path"; then+ echo "Desktop parity runner failed; desktop parity output not generated."+ return 1+ fi++ if [[ ! -f "$output_path" ]]; then+ echo "Desktop parity runner finished without output at $output_path."+ return 1+ fi++ return 0+}+run_android_runner() {run_mobile_runner \"android" \@@ -967,6 +1005,9 @@ run_ios_runner() {run_platform_runner() {local platform="$1"case "$platform" in+ desktop)+ run_desktop_runner+ ;;android)run_android_runner;;diff --git a/infra/ml/test/tools/desktop_parity_runner.js b/infra/ml/test/tools/desktop_parity_runner.jsnew file mode 100755index 0000000000..7c93960645--- /dev/null+++ b/infra/ml/test/tools/desktop_parity_runner.js@@ -0,0 +1,266 @@+#!/usr/bin/env node+/**+ * Desktop platform runner for the ML indexing parity suite.+ *+ * Run the parity corpus through the desktop build of the Rust ML pipeline+ * (see [Note: ML with Rust] in the desktop app), writing results in the+ * format consumed by compare_parity_outputs.py.+ *+ * This is invoked by run_ml_parity_tests.sh, which syncs the fixture images+ * and models to local disk beforehand; this script performs no network+ * access. It smoke tests the desktop specific pieces β the napi addon and+ * the pinned ONNX Runtime library it loads β while the deep validation of+ * the shared Rust pipeline itself lives in the ente-photos crate's+ * ml_indexing integration test.+ *+ * The desktop build artifacts it needs are produced by the desktop+ * postinstall: the addon in `desktop/rust-bindings/` (via `cargo codegen+ * napi`) and the ONNX Runtime library in `desktop/build/onnxruntime/` (via+ * `node scripts/ort.js`).+ */++const fsp = require("node:fs/promises");+const path = require("node:path");+const { parseArgs } = require("node:util");++const repoRoot = path.resolve(__dirname, "..", "..", "..", "..");+const desktopDir = path.join(repoRoot, "desktop");++// The desktop build scripts own the addon naming scheme.+const { napiTriple } = require(+ path.join(desktopDir, "scripts", "ml-native.js"),+);++const requiredFlags = [+ "manifest",+ "ml-dir",+ "models-dir",+ "asset-lock",+ "code-revision",+ "output",+];++const parseCLIArgs = () => {+ const { values } = parseArgs({+ options: Object.fromEntries(+ requiredFlags.map((flag) => [flag, { type: "string" }]),+ ),+ });+ for (const flag of requiredFlags)+ if (!values[flag]) throw new Error(`Missing required --${flag}`);+ return values;+};++/** Load the Rust ML addon built by `cargo codegen napi`. */+const loadMLAddon = () => {+ const addonPath = path.join(+ desktopDir,+ "rust-bindings",+ `index.${napiTriple(process.platform, process.arch)}.node`,+ );+ return require(addonPath);+};++/**+ * The ONNX Runtime dynamic library downloaded by desktop's `scripts/ort.js`+ * for the current architecture, located by scanning so that this script does+ * not duplicate the pinned library version.+ */+const onnxRuntimeLibraryPath = async () => {+ const libraryDir = path.join(+ desktopDir,+ "build",+ "onnxruntime",+ process.arch,+ );+ const entries = await fsp.readdir(libraryDir, { withFileTypes: true });+ const library = entries.find(+ (entry) =>+ entry.isFile() &&+ (entry.name.startsWith("libonnxruntime.") ||+ entry.name == "onnxruntime.dll"),+ );+ if (!library)+ throw new Error(+ `No ONNX Runtime library found in ${libraryDir} (run "node scripts/ort.js" in desktop/)`,+ );+ return path.join(libraryDir, library.name);+};++/**+ * Resolve the models needed for indexing from the parity harness' local+ * model mirror cache, which run_ml_parity_tests.sh populates (verifying+ * against the SHA-256s in the asset lock) before invoking us.+ */+const resolveModels = async (assetLockPath, modelsDir) => {+ const assetLock = JSON.parse(await fsp.readFile(assetLockPath, "utf8"));++ const resolveModel = async (key) => {+ const spec = assetLock.models[key];+ if (!spec) throw new Error(`No model entry ${key} in ${assetLockPath}`);+ const modelPath = path.join(modelsDir, spec.file_name);+ try {+ await fsp.access(modelPath);+ } catch {+ throw new Error(+ `Model ${spec.file_name} not found in ${modelsDir}`,+ );+ }+ return {+ path: modelPath,+ fileName: spec.file_name,+ sha256: spec.sha256,+ };+ };++ return {+ faceDetection: await resolveModel("face_detection"),+ faceEmbedding: await resolveModel("face_embedding"),+ clipImage: await resolveModel("clip_image"),+ };+};++const mlModelPaths = (models) => ({+ faceDetection: models.faceDetection.path,+ faceEmbedding: models.faceEmbedding.path,+ clipImage: models.clipImage.path,+ clipText: "",+ petFaceDetection: "",+ petFaceEmbeddingDog: "",+ petFaceEmbeddingCat: "",+ petBodyDetection: "",+ petBodyEmbeddingDog: "",+ petBodyEmbeddingCat: "",+});++const analyzeFixture = async (native, fileID, imagePath, modelPaths) => {+ const result = await native.analyzeImage({+ fileId: fileID,+ imagePath,+ runFaces: true,+ runClip: true,+ runPets: false,+ generateFaceCrops: true,+ modelPaths,+ });++ // Also smoke the desktop-only face crop path: crop generation is best+ // effort, but for the parity corpus every face must get a crop.+ const faceCount = result.faces?.length ?? 0;+ const cropCount = (result.faceCrops ?? []).filter(Boolean).length;+ if (cropCount != faceCount)+ throw new Error(+ `Face crop count ${cropCount} does not match face count ${faceCount}`,+ );++ return result;+};++/** Mirror of _toParityResult in mobile's ml_parity_shared.dart. */+const toParityResult = (fileID, result, models, codeRevision, totalMS) => {+ const clip = result.clip;+ if (!clip) throw new Error(`Missing CLIP result for ${fileID}`);++ const faces = (result.faces ?? []).map((face) => {+ const [xMin, yMin, xMax, yMax] = face.detection.boxXyxy;+ return {+ box: [xMin, yMin, xMax - xMin, yMax - yMin],+ landmarks: face.detection.keypoints,+ score: face.detection.score,+ embedding: Array.from(face.embedding),+ };+ });++ return {+ file_id: fileID,+ clip: { embedding: Array.from(clip.embedding) },+ faces,+ runner_metadata: {+ platform: "desktop",+ runtime: "rust-ml",+ models: Object.fromEntries(+ Object.values(models).map((m) => [m.fileName, m.sha256]),+ ),+ code_revision: codeRevision,+ timing_ms: { total: totalMS },+ },+ };+};++const main = async () => {+ const args = parseCLIArgs();++ const native = loadMLAddon();+ native.initOrt(await onnxRuntimeLibraryPath());+ // Match the app's configuration (see ml-worker.ts); a no-op on macOS.+ native.setMlExecutionConfig(true);++ const models = await resolveModels(args["asset-lock"], args["models-dir"]);+ const modelPaths = mlModelPaths(models);++ const manifest = JSON.parse(await fsp.readFile(args.manifest, "utf8"));+ const items = manifest.items;+ if (!Array.isArray(items) || items.length == 0)+ throw new Error(`No items in manifest ${args.manifest}`);++ native.initMlRuntime(modelPaths);++ const results = [];+ const errors = [];+ try {+ for (const [index, item] of items.entries()) {+ const t = Date.now();+ try {+ const imagePath = path.resolve(args["ml-dir"], item.source);+ const result = await analyzeFixture(+ native,+ index + 1,+ imagePath,+ modelPaths,+ );+ results.push(+ toParityResult(+ item.file_id,+ result,+ models,+ args["code-revision"],+ Date.now() - t,+ ),+ );+ console.log(`Analyzed ${item.file_id}`);+ } catch (e) {+ const message = e instanceof Error ? e.message : String(e);+ errors.push({+ file_id: item.file_id,+ error: message,+ timing_ms: Date.now() - t,+ });+ console.log(`Failed to analyze ${item.file_id}: ${message}`);+ }+ }+ } finally {+ native.releaseMlRuntime();+ for (const { severity, message } of native.takeMlRuntimeEvents())+ console.log(`[ml-rt] ${severity}: ${message}`);+ }++ await fsp.mkdir(path.dirname(args.output), { recursive: true });+ await fsp.writeFile(+ args.output,+ JSON.stringify({+ platform: "desktop",+ results,+ ...(errors.length ? { errors } : {}),+ }),+ );++ console.log(+ `Analyzed ${results.length} of ${items.length} fixtures (${errors.length} failed)`,+ );+ if (results.length == 0) throw new Error("All fixtures failed to analyze");+};++main().catch((e) => {+ console.error(e);+ process.exit(1);+});