Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

desktop parity smoke test runner

laurenspriem935dbd97ee75
diff --git a/desktop/scripts/ml-native.js b/desktop/scripts/ml-native.js
index f189019c39..1572b3ccfa 100644
--- a/desktop/scripts/ml-native.js
+++ b/desktop/scripts/ml-native.js
@@ -123,4 +123,4 @@ const stageMLAddons = async (appDir, platform, arch) => {
await fsp.rm(path.join(stageDir, entry), { recursive: true });
};
-module.exports = { stageMLAddons };
+module.exports = { napiTriple, stageMLAddons };
diff --git a/infra/ml/test/README.md b/infra/ml/test/README.md
index 6755e5962e..c517d1baae 100644
--- a/infra/ml/test/README.md
+++ b/infra/ml/test/README.md
@@ -1,6 +1,6 @@
# ML Indexing Parity Test Suite
-This directory contains the ML indexing parity framework for Android, iOS, and Python ground truth. (Desktop ships the same shared Rust pipeline that the `ente-photos` crate's `ml_indexing` integration test validates against the Python goldens, so it has no separate parity runner.)
+This directory contains the ML indexing parity framework for desktop, Android, iOS, and Python ground truth. (The desktop runner is a smoke test of the desktop-specific addon path β€” the napi bindings and the pinned ONNX Runtime library β€” since deep validation of the shared Rust pipeline itself lives in the `ente-photos` crate's `ml_indexing` integration test.)
## Layout
@@ -13,6 +13,7 @@ This directory contains the ML indexing parity framework for Android, iOS, and P
## Prerequisites
- Python + `uv`
+- Node + the desktop build artifacts (`npm ci && npm run postinstall` in `desktop/`) when running desktop parity
- Flutter SDK (Android/iOS parity)
- Android emulator/device and iOS simulator/device when running mobile parity
@@ -24,7 +25,7 @@ bash infra/ml/test/run_ml_parity_tests.sh
Common flags:
-- `--platforms all|android|ios`
+- `--platforms all|desktop|android|ios`
- `--output-dir <path>`
- `--verbose` (stream full runner/comparator logs to terminal)
- `--render-detection-overlays` (generate annotated detection images under `out/parity/detections/<platform>/`; includes selected platforms plus `python` ground truth)
diff --git a/infra/ml/test/ground_truth/README.md b/infra/ml/test/ground_truth/README.md
index a9bd777be3..cd65298ca4 100644
--- a/infra/ml/test/ground_truth/README.md
+++ b/infra/ml/test/ground_truth/README.md
@@ -48,6 +48,7 @@ uv run --project infra/ml python infra/ml/test/tools/generate_goldens.py \
```sh
uv run --project infra/ml python infra/ml/test/tools/compare_parity_outputs.py \
--ground-truth infra/ml/test/out/parity/python/results.json \
+ --platform-result desktop=infra/ml/test/out/parity/desktop/results.json \
--platform-result android=infra/ml/test/out/parity/android/results.json \
--platform-result ios=infra/ml/test/out/parity/ios/results.json
```
diff --git a/infra/ml/test/run_ml_parity_tests.sh b/infra/ml/test/run_ml_parity_tests.sh
index cd7dac07c7..9cdd215187 100755
--- a/infra/ml/test/run_ml_parity_tests.sh
+++ b/infra/ml/test/run_ml_parity_tests.sh
@@ -29,7 +29,7 @@ usage() {
Usage: infra/ml/test/run_ml_parity_tests.sh [flags]
Flags:
- --platforms all|android|ios (default: all)
+ --platforms all|desktop|android|ios (default: all)
--output-dir <path> (default: infra/ml/test/out/parity)
--verbose (default: disabled)
--render-detection-overlays (default: disabled; render annotated face detection images to out/parity/detections/<platform>/)
@@ -283,9 +283,9 @@ print_kv "mobile_ml_route:" "rust"
declare -a selected_platforms=()
case "$PLATFORMS" in
all)
- selected_platforms=(android ios)
+ selected_platforms=(desktop android ios)
;;
- android|ios)
+ desktop|android|ios)
selected_platforms=("$PLATFORMS")
;;
*)
@@ -572,6 +572,17 @@ run_preflight_checks() {
for platform in "${selected_platforms[@]}"; do
case "$platform" in
+ desktop)
+ if ! command -v node >/dev/null 2>&1; then
+ preflight_errors+=("node is required for desktop parity")
+ fi
+ if ! compgen -G "$ROOT_DIR/desktop/rust-bindings/index.*.node" >/dev/null; then
+ preflight_errors+=("desktop ML addon not found under desktop/rust-bindings (run 'npm ci && npm run postinstall' in desktop/, or 'cargo codegen napi' from rust/)")
+ fi
+ if [[ ! -d "$ROOT_DIR/desktop/build/onnxruntime" ]]; then
+ preflight_errors+=("ONNX Runtime library not found under desktop/build/onnxruntime (run 'node scripts/ort.js' in desktop/)")
+ fi
+ ;;
android|ios)
if ! command -v flutter >/dev/null 2>&1; then
preflight_errors+=("flutter is required for $platform parity")
@@ -744,8 +755,11 @@ for platform in "${selected_platforms[@]}"; do
esac
done
+# Every platform runner uses the local model cache; only the mobile runners
+# additionally need it served over HTTP (the desktop runner reads the files
+# directly).
+prepare_local_model_mirror_cache "$LOCAL_MODEL_MIRROR_DIR"
if $has_mobile_platform; then
- prepare_local_model_mirror_cache "$LOCAL_MODEL_MIRROR_DIR"
if ! start_local_mirror_server "$ML_DIR" "$LOCAL_MIRROR_LOG"; then
echo "Proceeding without local parity mirror."
fi
@@ -950,6 +964,30 @@ run_mobile_runner() {
return 0
}
+run_desktop_runner() {
+ local platform_output_dir="$OUTPUT_DIR/desktop"
+ local output_path="$platform_output_dir/results.json"
+
+ echo "Running desktop parity runner"
+ if ! node "$ML_DIR/tools/desktop_parity_runner.js" \
+ --manifest "$MANIFEST_PATH" \
+ --ml-dir "$ML_DIR" \
+ --models-dir "$LOCAL_MODEL_MIRROR_DIR" \
+ --asset-lock "$ASSET_LOCK_PATH" \
+ --code-revision "$CODE_REVISION" \
+ --output "$output_path"; then
+ echo "Desktop parity runner failed; desktop parity output not generated."
+ return 1
+ fi
+
+ if [[ ! -f "$output_path" ]]; then
+ echo "Desktop parity runner finished without output at $output_path."
+ return 1
+ fi
+
+ return 0
+}
+
run_android_runner() {
run_mobile_runner \
"android" \
@@ -967,6 +1005,9 @@ run_ios_runner() {
run_platform_runner() {
local platform="$1"
case "$platform" in
+ desktop)
+ run_desktop_runner
+ ;;
android)
run_android_runner
;;
diff --git a/infra/ml/test/tools/desktop_parity_runner.js b/infra/ml/test/tools/desktop_parity_runner.js
new file mode 100755
index 0000000000..7c93960645
--- /dev/null
+++ b/infra/ml/test/tools/desktop_parity_runner.js
@@ -0,0 +1,266 @@
+#!/usr/bin/env node
+/**
+ * Desktop platform runner for the ML indexing parity suite.
+ *
+ * Run the parity corpus through the desktop build of the Rust ML pipeline
+ * (see [Note: ML with Rust] in the desktop app), writing results in the
+ * format consumed by compare_parity_outputs.py.
+ *
+ * This is invoked by run_ml_parity_tests.sh, which syncs the fixture images
+ * and models to local disk beforehand; this script performs no network
+ * access. It smoke tests the desktop specific pieces β€” the napi addon and
+ * the pinned ONNX Runtime library it loads β€” while the deep validation of
+ * the shared Rust pipeline itself lives in the ente-photos crate's
+ * ml_indexing integration test.
+ *
+ * The desktop build artifacts it needs are produced by the desktop
+ * postinstall: the addon in `desktop/rust-bindings/` (via `cargo codegen
+ * napi`) and the ONNX Runtime library in `desktop/build/onnxruntime/` (via
+ * `node scripts/ort.js`).
+ */
+
+const fsp = require("node:fs/promises");
+const path = require("node:path");
+const { parseArgs } = require("node:util");
+
+const repoRoot = path.resolve(__dirname, "..", "..", "..", "..");
+const desktopDir = path.join(repoRoot, "desktop");
+
+// The desktop build scripts own the addon naming scheme.
+const { napiTriple } = require(
+ path.join(desktopDir, "scripts", "ml-native.js"),
+);
+
+const requiredFlags = [
+ "manifest",
+ "ml-dir",
+ "models-dir",
+ "asset-lock",
+ "code-revision",
+ "output",
+];
+
+const parseCLIArgs = () => {
+ const { values } = parseArgs({
+ options: Object.fromEntries(
+ requiredFlags.map((flag) => [flag, { type: "string" }]),
+ ),
+ });
+ for (const flag of requiredFlags)
+ if (!values[flag]) throw new Error(`Missing required --${flag}`);
+ return values;
+};
+
+/** Load the Rust ML addon built by `cargo codegen napi`. */
+const loadMLAddon = () => {
+ const addonPath = path.join(
+ desktopDir,
+ "rust-bindings",
+ `index.${napiTriple(process.platform, process.arch)}.node`,
+ );
+ return require(addonPath);
+};
+
+/**
+ * The ONNX Runtime dynamic library downloaded by desktop's `scripts/ort.js`
+ * for the current architecture, located by scanning so that this script does
+ * not duplicate the pinned library version.
+ */
+const onnxRuntimeLibraryPath = async () => {
+ const libraryDir = path.join(
+ desktopDir,
+ "build",
+ "onnxruntime",
+ process.arch,
+ );
+ const entries = await fsp.readdir(libraryDir, { withFileTypes: true });
+ const library = entries.find(
+ (entry) =>
+ entry.isFile() &&
+ (entry.name.startsWith("libonnxruntime.") ||
+ entry.name == "onnxruntime.dll"),
+ );
+ if (!library)
+ throw new Error(
+ `No ONNX Runtime library found in ${libraryDir} (run "node scripts/ort.js" in desktop/)`,
+ );
+ return path.join(libraryDir, library.name);
+};
+
+/**
+ * Resolve the models needed for indexing from the parity harness' local
+ * model mirror cache, which run_ml_parity_tests.sh populates (verifying
+ * against the SHA-256s in the asset lock) before invoking us.
+ */
+const resolveModels = async (assetLockPath, modelsDir) => {
+ const assetLock = JSON.parse(await fsp.readFile(assetLockPath, "utf8"));
+
+ const resolveModel = async (key) => {
+ const spec = assetLock.models[key];
+ if (!spec) throw new Error(`No model entry ${key} in ${assetLockPath}`);
+ const modelPath = path.join(modelsDir, spec.file_name);
+ try {
+ await fsp.access(modelPath);
+ } catch {
+ throw new Error(
+ `Model ${spec.file_name} not found in ${modelsDir}`,
+ );
+ }
+ return {
+ path: modelPath,
+ fileName: spec.file_name,
+ sha256: spec.sha256,
+ };
+ };
+
+ return {
+ faceDetection: await resolveModel("face_detection"),
+ faceEmbedding: await resolveModel("face_embedding"),
+ clipImage: await resolveModel("clip_image"),
+ };
+};
+
+const mlModelPaths = (models) => ({
+ faceDetection: models.faceDetection.path,
+ faceEmbedding: models.faceEmbedding.path,
+ clipImage: models.clipImage.path,
+ clipText: "",
+ petFaceDetection: "",
+ petFaceEmbeddingDog: "",
+ petFaceEmbeddingCat: "",
+ petBodyDetection: "",
+ petBodyEmbeddingDog: "",
+ petBodyEmbeddingCat: "",
+});
+
+const analyzeFixture = async (native, fileID, imagePath, modelPaths) => {
+ const result = await native.analyzeImage({
+ fileId: fileID,
+ imagePath,
+ runFaces: true,
+ runClip: true,
+ runPets: false,
+ generateFaceCrops: true,
+ modelPaths,
+ });
+
+ // Also smoke the desktop-only face crop path: crop generation is best
+ // effort, but for the parity corpus every face must get a crop.
+ const faceCount = result.faces?.length ?? 0;
+ const cropCount = (result.faceCrops ?? []).filter(Boolean).length;
+ if (cropCount != faceCount)
+ throw new Error(
+ `Face crop count ${cropCount} does not match face count ${faceCount}`,
+ );
+
+ return result;
+};
+
+/** Mirror of _toParityResult in mobile's ml_parity_shared.dart. */
+const toParityResult = (fileID, result, models, codeRevision, totalMS) => {
+ const clip = result.clip;
+ if (!clip) throw new Error(`Missing CLIP result for ${fileID}`);
+
+ const faces = (result.faces ?? []).map((face) => {
+ const [xMin, yMin, xMax, yMax] = face.detection.boxXyxy;
+ return {
+ box: [xMin, yMin, xMax - xMin, yMax - yMin],
+ landmarks: face.detection.keypoints,
+ score: face.detection.score,
+ embedding: Array.from(face.embedding),
+ };
+ });
+
+ return {
+ file_id: fileID,
+ clip: { embedding: Array.from(clip.embedding) },
+ faces,
+ runner_metadata: {
+ platform: "desktop",
+ runtime: "rust-ml",
+ models: Object.fromEntries(
+ Object.values(models).map((m) => [m.fileName, m.sha256]),
+ ),
+ code_revision: codeRevision,
+ timing_ms: { total: totalMS },
+ },
+ };
+};
+
+const main = async () => {
+ const args = parseCLIArgs();
+
+ const native = loadMLAddon();
+ native.initOrt(await onnxRuntimeLibraryPath());
+ // Match the app's configuration (see ml-worker.ts); a no-op on macOS.
+ native.setMlExecutionConfig(true);
+
+ const models = await resolveModels(args["asset-lock"], args["models-dir"]);
+ const modelPaths = mlModelPaths(models);
+
+ const manifest = JSON.parse(await fsp.readFile(args.manifest, "utf8"));
+ const items = manifest.items;
+ if (!Array.isArray(items) || items.length == 0)
+ throw new Error(`No items in manifest ${args.manifest}`);
+
+ native.initMlRuntime(modelPaths);
+
+ const results = [];
+ const errors = [];
+ try {
+ for (const [index, item] of items.entries()) {
+ const t = Date.now();
+ try {
+ const imagePath = path.resolve(args["ml-dir"], item.source);
+ const result = await analyzeFixture(
+ native,
+ index + 1,
+ imagePath,
+ modelPaths,
+ );
+ results.push(
+ toParityResult(
+ item.file_id,
+ result,
+ models,
+ args["code-revision"],
+ Date.now() - t,
+ ),
+ );
+ console.log(`Analyzed ${item.file_id}`);
+ } catch (e) {
+ const message = e instanceof Error ? e.message : String(e);
+ errors.push({
+ file_id: item.file_id,
+ error: message,
+ timing_ms: Date.now() - t,
+ });
+ console.log(`Failed to analyze ${item.file_id}: ${message}`);
+ }
+ }
+ } finally {
+ native.releaseMlRuntime();
+ for (const { severity, message } of native.takeMlRuntimeEvents())
+ console.log(`[ml-rt] ${severity}: ${message}`);
+ }
+
+ await fsp.mkdir(path.dirname(args.output), { recursive: true });
+ await fsp.writeFile(
+ args.output,
+ JSON.stringify({
+ platform: "desktop",
+ results,
+ ...(errors.length ? { errors } : {}),
+ }),
+ );
+
+ console.log(
+ `Analyzed ${results.length} of ${items.length} fixtures (${errors.length} failed)`,
+ );
+ if (results.length == 0) throw new Error("All fixtures failed to analyze");
+};
+
+main().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});