Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Use the asset store for Photos desktop models

Manav Rathi44b026d0e11e
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts
index e85cbe6e7e..2b57fc3d65 100644
--- a/desktop/src/main/services/ml-worker.ts
+++ b/desktop/src/main/services/ml-worker.ts
@@ -4,25 +4,20 @@
* The ML pipeline itself β€” image decode, preprocessing, ONNX inference and
* postprocessing β€” is implemented by the Rust crate shared with the mobile
* apps, and reaches us via a Node native addon (see [Note: ML with Rust]).
- * This file downloads the models the pipeline needs, loads the addon, and
- * forwards requests that arrive from the web layer over our MessagePort.
+ * This file loads the addon and forwards requests that arrive from the web
+ * layer over our MessagePort. The addon downloads the models through the
+ * shared Rust asset store.
*/
// See [Note: Using Electron APIs in UtilityProcess] about what we can and
// cannot import.
import { expose } from "comlink";
-import { net } from "electron/main";
-import { existsSync } from "fs";
-import { createHash } from "node:crypto";
-import { createReadStream } from "node:fs";
-import fs from "node:fs/promises";
import path from "node:path";
import { z } from "zod";
import log from "../log-worker";
import { messagePortMainEndpoint } from "../utils/comlink";
import { wait } from "../utils/common";
-import { writeStream } from "../utils/stream";
import { fsStatMtime } from "./fs";
// The addon is loaded at runtime from the path the main process gives us, so
@@ -101,6 +96,7 @@ class CategorizedMLWorkerError extends Error {
}
let _native: MLNative | undefined;
+let _assetStore: MLNativeModule.AssetStore | undefined;
let _nativeLoadError: string | undefined;
/**
@@ -118,6 +114,10 @@ const loadMLNative = (paths: MLNativePaths) => {
// Enables the guarded WebGPU path on Linux and Windows. This is a
// no-op on macOS, where CoreML remains the preferred provider.
native.setMlExecutionConfig(true);
+ _assetStore = new native.AssetStore(
+ path.join(userDataPath(), "assets"),
+ path.join(userDataPath(), "models"),
+ );
_native = native;
} catch (e) {
_nativeLoadError = e instanceof Error ? e.message : String(e);
@@ -137,6 +137,14 @@ const mlNative = () => {
);
};
+const assetStore = () => {
+ if (_assetStore) return _assetStore;
+ throw new CategorizedMLWorkerError(
+ "init",
+ `ML is unavailable: ${_nativeLoadError ?? "asset store not loaded"}`,
+ );
+};
+
/**
* Drain the runtime events (execution provider fallbacks, self-test failures)
* buffered on the Rust side, writing each to our log at its severity.
@@ -151,279 +159,35 @@ const logMLRuntimeEvents = (native: MLNative) => {
};
/**
- * The ML models (and the CLIP tokenizer vocabulary) used by the Rust
- * pipeline, identified by their name in the models.ente.com bucket. These are
- * the same assets that the mobile apps use (see ml_model_assets.dart).
- *
- * Fresh downloads are verified against the pinned SHA-256; copies already on
- * disk are revalidated with only a cheap size check (a corrupt model that
- * nevertheless retains the expected size gets caught by the Rust side, which
- * reports it back to us for eviction and redownload).
- */
-const modelSpecs = {
- faceDetection: {
- name: "yolov5s_face_640_640_static_b1.onnx",
- byteSize: 32355091,
- sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",
- },
- faceEmbedding: {
- name: "mobilefacenet_portable_static_b1.onnx",
- byteSize: 5278803,
- sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",
- },
- clipImage: {
- name: "mobileclip_s2_image_gelu_opset20.onnx",
- byteSize: 143057352,
- sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",
- },
- clipText: {
- name: "mobileclip_s2_text_opset18_quant.onnx",
- byteSize: 67144712,
- sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",
- },
- clipTextVocab: {
- name: "bpe_simple_vocab_16e6.txt",
- byteSize: 3194984,
- sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",
- },
- petFaceDetection: {
- name: "yolov5s_pet_face_fp16_V2.onnx",
- byteSize: 14758020,
- sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",
- },
- petFaceEmbeddingDog: {
- name: "dog_face_embedding128.onnx",
- byteSize: 4141071,
- sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",
- },
- petFaceEmbeddingCat: {
- name: "cat_face_embedding128.onnx",
- byteSize: 4141071,
- sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",
- },
- petBodyDetection: {
- name: "yolov5s_object_fp16.onnx",
- byteSize: 14987107,
- sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",
- },
- petBodyEmbeddingDog: {
- name: "dog_body_embedding192.onnx",
- byteSize: 4569361,
- sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",
- },
- petBodyEmbeddingCat: {
- name: "cat_body_embedding192.onnx",
- byteSize: 4569361,
- sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",
- },
-} as const;
-
-type ModelSpec = (typeof modelSpecs)[keyof typeof modelSpecs];
-
-/** Return the path where the given {@link modelName} is meant to be saved */
-const modelSavePath = (modelName: string) =>
- path.join(userDataPath(), "models", modelName);
-
-/**
- * Cached promises to the on-disk paths of verified models, so that each model
- * is downloaded and verified at most once per process. Multiple parallel
- * calls for the same model await the same promise.
- */
-const _verifiedModelPaths = new Map<string, Promise<string>>();
-
-/** The single cleanup attempt shared by all model loads in this process. */
-let _oldModelsCleanup: Promise<void> | undefined;
-
-const modelPath = (spec: ModelSpec) => {
- let promise = _verifiedModelPaths.get(spec.name);
- if (!promise) {
- promise = modelPathDownloadingIfNeeded(spec).catch((e: unknown) => {
- // Allow a later call to retry the download.
- _verifiedModelPaths.delete(spec.name);
- throw e;
- });
- _verifiedModelPaths.set(spec.name, promise);
- }
- return promise;
-};
-
-/**
- * Forget the cached verification of the model at {@link modelPath} and delete
- * it from disk, so that the next use downloads a fresh copy. Used when the
- * Rust side reports the on-disk model as corrupt.
- */
-const evictModel = async (modelPath: string) => {
- const name = path.basename(modelPath);
- _verifiedModelPaths.delete(name);
- await fs.rm(modelPath, { force: true });
-};
-
-/**
- * Download the model described by {@link spec} if we don't already have it.
- *
- * An existing on-disk copy is revalidated with only a cheap size check so
- * that resolving already downloaded models stays fast; fresh downloads are
- * verified against the pinned SHA-256.
- *
- * @returns the path to the model on the local machine.
- */
-const modelPathDownloadingIfNeeded = async (spec: ModelSpec) => {
- const savePath = modelSavePath(spec.name);
-
- await cleanupOldModelsIfNeeded();
-
- if (existsSync(savePath)) {
- const size = (await fs.stat(savePath)).size;
- if (size == spec.byteSize) return savePath;
- log.error(
- `The size ${size} of model ${spec.name} does not match the expected size, downloading again`,
- );
- } else {
- log.info(`ML model ${spec.name} not found, downloading`);
- }
-
- await downloadModelVerifyingSHA256(savePath, spec);
- return savePath;
-};
-
-/**
- * Download the model described by {@link spec} to {@link savePath}, verifying
- * the SHA-256 of the downloaded contents and retrying once on mismatch.
- */
-const downloadModelVerifyingSHA256 = async (
- savePath: string,
- spec: ModelSpec,
-) => {
- await downloadModel(savePath, spec.name);
- if ((await computeFileSHA256(savePath)) == spec.sha256) return;
-
- log.error(
- `The SHA-256 of the downloaded model ${spec.name} does not match the expected value, downloading again`,
- );
- await downloadModel(savePath, spec.name);
- if ((await computeFileSHA256(savePath)) == spec.sha256) return;
-
- // Don't leave the bad download behind: were it to happen to pass the size
- // check, a later call would use it without noticing.
- await fs.rm(savePath, { force: true });
- throw new Error(
- `The SHA-256 of the downloaded model ${spec.name} does not match the expected value`,
- );
-};
-
-const computeFileSHA256 = (filePath: string) =>
- new Promise<string>((resolve, reject) => {
- const hash = createHash("sha256");
- const stream = createReadStream(filePath);
- stream.on("error", reject);
- stream.on("data", (chunk) => hash.update(chunk));
- stream.on("end", () => resolve(hash.digest("hex")));
- });
-
-/**
- * Cleanup old models.
- *
- * This code runs at most once in each utility process, before its first model
- * is resolved. This is a good time to delete previously existent but now
- * unused models and reclaim their disk space.
- */
-const cleanupOldModelsIfNeeded = () =>
- (_oldModelsCleanup ??= cleanupOldModels().catch((e: unknown) => {
- // Removing obsolete cache files is best effort and must not prevent the
- // current models from loading. Retry on the next process start.
- log.warn("Failed to clean up old ML models", e);
- }));
-
-const cleanupOldModels = async () => {
- const oldModelNames = [
- "clip-image-vit-32-float32.onnx",
- "clip-text-vit-32-uint8.onnx",
- "mobileclip_s2_image.onnx",
- "mobileclip_s2_image_opset18_rgba_sim.onnx",
- "mobileclip_s2_image_opset18_rgba_opt.onnx",
- "mobileclip_s2_text_int32.onnx",
- "yolov5s_face_640_640_dynamic.onnx",
- "yolov5s_face_opset18_rgba_opt.onnx",
- "yolov5s_face_opset18_rgba_opt_nosplits.onnx",
- "mobilefacenet_opset15.onnx",
- ];
-
- for (const modelName of oldModelNames) {
- const modelPath = modelSavePath(modelName);
- if (existsSync(modelPath)) {
- log.info(`Removing unused ML model at ${modelPath}`);
- await fs.rm(modelPath, { force: true });
- }
- }
-};
-
-const downloadModel = async (saveLocation: string, name: string) => {
- // `mkdir -p` the directory where we want to save the model.
- const saveDir = path.dirname(saveLocation);
- await fs.mkdir(saveDir, { recursive: true });
- // Download.
- log.info(`Downloading ML model from ${name}`);
- const url = `https://models.ente.com/${name}`;
- const res = await net.fetch(url);
- if (!res.ok) throw new Error(`Failed to fetch ${url}: HTTP ${res.status}`);
- const body = res.body;
- if (!body) throw new Error(`Received an null response for ${url}`);
- // Save.
- await writeStream(saveLocation, body);
- log.info(`Downloaded ML model ${name}`);
-};
-
-/**
- * The subset of {@link modelSpecs} needed for the indexing related tasks that
- * are enabled in the given request, mapped to verified on-disk paths.
+ * The models needed for the indexing tasks enabled in the given request,
+ * mapped to verified on-disk paths.
*
* Models for tasks that are not requested are left as empty strings, which
* the Rust side treats as "not configured".
*/
-const indexingModelPaths = async (
+const _indexingModelPaths = new Map<
+ number,
+ Promise<MLNativeModule.ModelPaths>
+>();
+
+const indexingModelPaths = (
runFaces: boolean,
runClip: boolean,
runPets: boolean,
-): Promise<MLNativeModule.ModelPaths> => {
- const [faceDetection, faceEmbedding] = runFaces
- ? await Promise.all([
- modelPath(modelSpecs.faceDetection),
- modelPath(modelSpecs.faceEmbedding),
- ])
- : ["", ""];
- const clipImage = runClip ? await modelPath(modelSpecs.clipImage) : "";
- const [
- petFaceDetection,
- petFaceEmbeddingDog,
- petFaceEmbeddingCat,
- petBodyDetection,
- petBodyEmbeddingDog,
- petBodyEmbeddingCat,
- ] = runPets
- ? await Promise.all([
- modelPath(modelSpecs.petFaceDetection),
- modelPath(modelSpecs.petFaceEmbeddingDog),
- modelPath(modelSpecs.petFaceEmbeddingCat),
- modelPath(modelSpecs.petBodyDetection),
- modelPath(modelSpecs.petBodyEmbeddingDog),
- modelPath(modelSpecs.petBodyEmbeddingCat),
- ])
- : ["", "", "", "", "", ""];
-
- return {
- faceDetection,
- faceEmbedding,
- clipImage,
- // The CLIP text session is managed lazily by the Rust side when
- // serving text queries; it is not part of the indexing runtime.
- clipText: "",
- petFaceDetection,
- petFaceEmbeddingDog,
- petFaceEmbeddingCat,
- petBodyDetection,
- petBodyEmbeddingDog,
- petBodyEmbeddingCat,
- };
+) => {
+ const key =
+ Number(runFaces) | (Number(runClip) << 1) | (Number(runPets) << 2);
+ let promise = _indexingModelPaths.get(key);
+ if (!promise) {
+ promise = assetStore()
+ .indexingModelPaths(runFaces, runClip, runPets)
+ .catch((e: unknown) => {
+ _indexingModelPaths.delete(key);
+ throw e;
+ });
+ _indexingModelPaths.set(key, promise);
+ }
+ return promise;
};
/**
@@ -493,7 +257,7 @@ const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {
try {
return await analyzeImageOnce(native, req);
} catch (e) {
- if (!(await evictModelIfReportedCorrupt(e))) throw e;
+ if (!evictModelIfReportedCorrupt(e)) throw e;
_preparedRuntimeKey = undefined;
return await analyzeImageOnce(native, req);
} finally {
@@ -550,28 +314,27 @@ const analyzeImageOnce = async (
/**
* If {@link e} is the addon's tagged "CorruptModel" error, return the path of
- * the corrupt model file it reports (but only if that path is under our
- * models directory, as a safety check before we delete it).
+ * the corrupt model file it reports.
*/
const corruptModelPathFromError = (e: unknown) => {
if (!(e instanceof Error)) return undefined;
if (!e.message.startsWith("CorruptModel: ")) return undefined;
- const corruptPath = e.message.slice("CorruptModel: ".length).trim();
- const modelsDir = path.join(userDataPath(), "models") + path.sep;
- return corruptPath.startsWith(modelsDir) ? corruptPath : undefined;
+ return e.message.slice("CorruptModel: ".length).trim();
};
/**
* If {@link e} is the addon's tagged "CorruptModel" error, evict the model it
* names so that its next use downloads a fresh copy, and return `true`.
*/
-const evictModelIfReportedCorrupt = async (e: unknown) => {
+const evictModelIfReportedCorrupt = (e: unknown) => {
const corruptModelPath = corruptModelPathFromError(e);
if (!corruptModelPath) return false;
+ if (!assetStore().removeModel(corruptModelPath)) return false;
+ _indexingModelPaths.clear();
log.error(
`Rust ML reported a corrupt model at ${corruptModelPath}, evicting it so that it gets redownloaded`,
);
- await evictModel(corruptModelPath);
+ _clipTextModelPaths = undefined;
return true;
};
@@ -579,6 +342,16 @@ const evictModelIfReportedCorrupt = async (e: unknown) => {
const uint8ArrayToBuffer = (bytes: Uint8Array) =>
Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+let _clipTextModelPaths: Promise<MLNativeModule.ClipTextModelPaths> | undefined;
+
+const clipTextModelPaths = () =>
+ (_clipTextModelPaths ??= assetStore()
+ .clipTextModelPaths()
+ .catch((e: unknown) => {
+ _clipTextModelPaths = undefined;
+ throw e;
+ }));
+
/**
* Run a throwaway CLIP text query so that the Rust side builds and caches the
* text encoder session before the user's first search. Mirrors the mobile
@@ -588,17 +361,14 @@ const uint8ArrayToBuffer = (bytes: Uint8Array) =>
*/
const warmUpClipTextEncoder = async (native: MLNative) => {
try {
- const [model, vocab] = await Promise.all([
- modelPath(modelSpecs.clipText),
- modelPath(modelSpecs.clipTextVocab),
- ]);
+ const { modelPath, vocabPath } = await clipTextModelPaths();
await native.runClipText({
text: "warm up text encoder",
- modelPath: model,
- vocabPath: vocab,
+ modelPath,
+ vocabPath,
});
} catch (e) {
- if (!(await evictModelIfReportedCorrupt(e)))
+ if (!evictModelIfReportedCorrupt(e))
log.warn("Failed to warm up the CLIP text encoder", e);
} finally {
logMLRuntimeEvents(native);
@@ -617,10 +387,7 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {
const native = _native;
const pathsOrSkip = await Promise.race([
- Promise.all([
- modelPath(modelSpecs.clipText),
- modelPath(modelSpecs.clipTextVocab),
- ]),
+ clipTextModelPaths(),
// Wait a bit for the models to resolve the first time this code runs
// on each app start (when they're already downloaded, resolving them
// is quick), but don't block on a download.
@@ -634,18 +401,18 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {
return undefined;
}
- const [clipTextModelPath, vocabPath] = pathsOrSkip;
+ const { modelPath, vocabPath } = pathsOrSkip;
try {
const { embedding } = await native.runClipText({
text,
- modelPath: clipTextModelPath,
+ modelPath,
vocabPath,
});
return embedding;
} catch (e) {
// Don't block this query on the redownload; a subsequent query will
// pick up the fresh copy.
- if (!(await evictModelIfReportedCorrupt(e))) throw e;
+ if (!evictModelIfReportedCorrupt(e)) throw e;
return undefined;
} finally {
logMLRuntimeEvents(native);
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 2070bc01ef..462ad0e584 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2608,6 +2608,7 @@ dependencies = [
name = "ente_photos_napi"
version = "0.0.0"
dependencies = [
+ "ente-assets",
"ente-photos",
"napi",
"napi-build",
@@ -5626,6 +5627,7 @@ dependencies = [
"napi-sys",
"nohash-hasher",
"rustc-hash",
+ "tokio",
]
[[package]]
diff --git a/rust/bindings/napi/photos/Cargo.toml b/rust/bindings/napi/photos/Cargo.toml
index c6d70f3091..6ebe052ca2 100644
--- a/rust/bindings/napi/photos/Cargo.toml
+++ b/rust/bindings/napi/photos/Cargo.toml
@@ -7,8 +7,9 @@ crate-type = ["cdylib"]
[dependencies]
# usearch requires a newer GCC than napi-cli provides.
+ente-assets.workspace = true
ente-photos.workspace = true
-napi = { version = "3", default-features = false, features = ["napi8"] }
+napi = { version = "3", default-features = false, features = ["napi8", "tokio_rt"] }
napi-derive = "3"
ort = { workspace = true, features = ["load-dynamic"] }
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rs
index f3f675569a..bedbc25caf 100644
--- a/rust/bindings/napi/photos/src/lib.rs
+++ b/rust/bindings/napi/photos/src/lib.rs
@@ -1,5 +1,5 @@
use ente_photos::ml::{
- error::MlError, indexing as shared_indexing, runtime::ModelPaths as SharedModelPaths,
+ assets, error::MlError, indexing as shared_indexing, runtime::ModelPaths as SharedModelPaths,
types as shared_types,
};
use napi::bindgen_prelude::*;
@@ -46,6 +46,84 @@ pub struct ModelPaths {
pub pet_body_embedding_cat: String,
}
+#[napi(object)]
+pub struct ClipTextModelPaths {
+ pub model_path: String,
+ pub vocab_path: String,
+}
+
+#[napi]
+pub struct AssetStore {
+ inner: ente_assets::AssetStore,
+}
+
+#[napi]
+impl AssetStore {
+ #[napi(constructor)]
+ pub fn new(assets_dir: String, legacy_models_dir: String) -> Self {
+ let inner = ente_assets::AssetStore::new(assets_dir);
+ for warning in
+ assets::migrate_desktop_models(&inner, std::path::Path::new(&legacy_models_dir))
+ {
+ eprintln!("Photos model migration: {warning}");
+ }
+ Self { inner }
+ }
+
+ #[napi]
+ pub async fn indexing_model_paths(
+ &self,
+ run_faces: bool,
+ run_clip: bool,
+ run_pets: bool,
+ ) -> Result<ModelPaths> {
+ let models = assets::indexing_models(run_faces, run_clip, run_pets);
+ let model_assets = models
+ .iter()
+ .copied()
+ .map(assets::model_asset)
+ .collect::<Vec<_>>();
+ self.inner
+ .download(
+ &model_assets,
+ |_| {},
+ ente_assets::download::CancellationToken::default(),
+ )
+ .await
+ .map_err(asset_error_to_napi)?;
+ Ok(to_napi_model_paths(assets::indexing_model_paths(
+ &self.inner,
+ run_faces,
+ run_clip,
+ run_pets,
+ )))
+ }
+
+ #[napi]
+ pub async fn clip_text_model_paths(&self) -> Result<ClipTextModelPaths> {
+ let asset = assets::model_asset(assets::Model::ClipText);
+ self.inner
+ .download(
+ std::slice::from_ref(&asset),
+ |_| {},
+ ente_assets::download::CancellationToken::default(),
+ )
+ .await
+ .map_err(asset_error_to_napi)?;
+ let paths = assets::clip_text_paths(&self.inner);
+ Ok(ClipTextModelPaths {
+ model_path: paths.model.to_string_lossy().into_owned(),
+ vocab_path: paths.vocab.to_string_lossy().into_owned(),
+ })
+ }
+
+ #[napi]
+ pub fn remove_model(&self, model_path: String) -> Result<bool> {
+ assets::remove_model_at_path(&self.inner, std::path::Path::new(&model_path))
+ .map_err(|error| Error::from_reason(format!("Assets: {error}")))
+ }
+}
+
#[napi(object)]
pub struct AnalyzeImageRequest {
pub file_id: i64,
@@ -255,6 +333,10 @@ fn ml_error_to_napi(error: MlError) -> Error {
Error::from_reason(format!("{kind}: {detail}"))
}
+fn asset_error_to_napi(error: ente_assets::download::Error) -> Error {
+ Error::from_reason(format!("Assets: {error}"))
+}
+
fn to_shared_model_paths(paths: &ModelPaths) -> SharedModelPaths {
SharedModelPaths {
face_detection: paths.face_detection.clone(),
@@ -270,6 +352,21 @@ fn to_shared_model_paths(paths: &ModelPaths) -> SharedModelPaths {
}
}
+fn to_napi_model_paths(paths: SharedModelPaths) -> ModelPaths {
+ ModelPaths {
+ face_detection: paths.face_detection,
+ face_embedding: paths.face_embedding,
+ clip_image: paths.clip_image,
+ clip_text: paths.clip_text,
+ pet_face_detection: paths.pet_face_detection,
+ pet_face_embedding_dog: paths.pet_face_embedding_dog,
+ pet_face_embedding_cat: paths.pet_face_embedding_cat,
+ pet_body_detection: paths.pet_body_detection,
+ pet_body_embedding_dog: paths.pet_body_embedding_dog,
+ pet_body_embedding_cat: paths.pet_body_embedding_cat,
+ }
+}
+
fn to_napi_analyze_image_result(result: shared_indexing::AnalyzeImageResult) -> AnalyzeImageResult {
AnalyzeImageResult {
file_id: result.file_id,
diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.toml
index f030ab2b58..6480110972 100644
--- a/rust/crates/photos/Cargo.toml
+++ b/rust/crates/photos/Cargo.toml
@@ -7,6 +7,7 @@ default = []
ml-assets = []
[dependencies]
+ente-assets.workspace = true
ente-image.workspace = true
fast_image_resize = { version = "5.5.0", features = ["image", "rayon"] }
half = "2.1"
@@ -80,7 +81,6 @@ ort = { workspace = true, features = [
[dev-dependencies]
anyhow = "1"
-ente-assets.workspace = true
flate2 = "1.1"
serde = { version = "1", features = ["derive"] }
serde_json.workspace = true
diff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rs
new file mode 100644
index 0000000000..05a841c08d
--- /dev/null
+++ b/rust/crates/photos/src/ml/assets.rs
@@ -0,0 +1,436 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use ente_assets::{Asset, AssetFile, AssetStore, AssetStoreError};
+
+use super::runtime::ModelPaths;
+
+const MODELS: &str = "models";
+
+const RETIRED_MODEL_KEYS: &[&str] = &[];
+
+const RETIRED_LEGACY_MODEL_FILES: &[&str] = &[
+ "clip-image-vit-32-float32.onnx",
+ "clip-text-vit-32-uint8.onnx",
+ "mobileclip_s2_image.onnx",
+ "mobileclip_s2_image_opset18_rgba_sim.onnx",
+ "mobileclip_s2_image_opset18_rgba_opt.onnx",
+ "mobileclip_s2_text_int32.onnx",
+ "yolov5s_face_640_640_dynamic.onnx",
+ "yolov5s_face_opset18_rgba_opt.onnx",
+ "yolov5s_face_opset18_rgba_opt_nosplits.onnx",
+ "mobilefacenet_opset15.onnx",
+];
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Model {
+ FaceDetection,
+ FaceEmbedding,
+ ClipImage,
+ ClipText,
+ PetFaceDetection,
+ PetFaceEmbeddingDog,
+ PetFaceEmbeddingCat,
+ PetBodyDetection,
+ PetBodyEmbeddingDog,
+ PetBodyEmbeddingCat,
+}
+
+const ALL_MODELS: [Model; 10] = [
+ Model::FaceDetection,
+ Model::FaceEmbedding,
+ Model::ClipImage,
+ Model::ClipText,
+ Model::PetFaceDetection,
+ Model::PetFaceEmbeddingDog,
+ Model::PetFaceEmbeddingCat,
+ Model::PetBodyDetection,
+ Model::PetBodyEmbeddingDog,
+ Model::PetBodyEmbeddingCat,
+];
+
+pub struct ClipTextPaths {
+ pub model: PathBuf,
+ pub vocab: PathBuf,
+}
+
+pub fn model_asset(model: Model) -> Asset {
+ let spec = model.spec();
+ Asset::files(
+ model_key(spec.key),
+ spec.files
+ .iter()
+ .map(|file| AssetFile {
+ name: file.name.to_string(),
+ url: format!("https://models.ente.com/{}", file.name),
+ sha256: file.sha256.to_string(),
+ })
+ .collect(),
+ )
+ .expect("valid Photos model catalog")
+}
+
+pub fn indexing_models(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Model> {
+ let mut models = Vec::new();
+ if run_faces {
+ models.extend([Model::FaceDetection, Model::FaceEmbedding]);
+ }
+ if run_clip {
+ models.push(Model::ClipImage);
+ }
+ if run_pets {
+ models.extend([
+ Model::PetFaceDetection,
+ Model::PetFaceEmbeddingDog,
+ Model::PetFaceEmbeddingCat,
+ Model::PetBodyDetection,
+ Model::PetBodyEmbeddingDog,
+ Model::PetBodyEmbeddingCat,
+ ]);
+ }
+ models
+}
+
+pub fn indexing_model_paths(
+ store: &AssetStore,
+ run_faces: bool,
+ run_clip: bool,
+ run_pets: bool,
+) -> ModelPaths {
+ ModelPaths {
+ face_detection: model_path_if(store, Model::FaceDetection, run_faces),
+ face_embedding: model_path_if(store, Model::FaceEmbedding, run_faces),
+ clip_image: model_path_if(store, Model::ClipImage, run_clip),
+ clip_text: String::new(),
+ pet_face_detection: model_path_if(store, Model::PetFaceDetection, run_pets),
+ pet_face_embedding_dog: model_path_if(store, Model::PetFaceEmbeddingDog, run_pets),
+ pet_face_embedding_cat: model_path_if(store, Model::PetFaceEmbeddingCat, run_pets),
+ pet_body_detection: model_path_if(store, Model::PetBodyDetection, run_pets),
+ pet_body_embedding_dog: model_path_if(store, Model::PetBodyEmbeddingDog, run_pets),
+ pet_body_embedding_cat: model_path_if(store, Model::PetBodyEmbeddingCat, run_pets),
+ }
+}
+
+pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {
+ let asset = model_asset(Model::ClipText);
+ let files = Model::ClipText.spec().files;
+ ClipTextPaths {
+ model: store
+ .file_path(&asset, files[0].name)
+ .expect("CLIP text model file"),
+ vocab: store
+ .file_path(&asset, files[1].name)
+ .expect("CLIP text vocabulary file"),
+ }
+}
+
+pub fn remove_model_at_path(store: &AssetStore, path: &Path) -> Result<bool, AssetStoreError> {
+ for model in ALL_MODELS {
+ let asset = model_asset(model);
+ if model
+ .spec()
+ .files
+ .iter()
+ .filter_map(|file| store.file_path(&asset, file.name))
+ .any(|candidate| candidate == path)
+ {
+ store.remove(&asset)?;
+ return Ok(true);
+ }
+ }
+ Ok(false)
+}
+
+pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {
+ let mut warnings = Vec::new();
+ if legacy_dir.exists() {
+ for model in ALL_MODELS {
+ if let Err(error) = migrate_desktop_model(store, legacy_dir, model) {
+ warnings.push(format!("Failed to migrate {}: {error}", model.spec().key));
+ }
+ }
+ for name in RETIRED_LEGACY_MODEL_FILES {
+ if let Err(error) = remove_file_if_exists(&legacy_dir.join(name)) {
+ warnings.push(format!("Failed to remove retired model {name}: {error}"));
+ }
+ }
+ let _ = fs::remove_dir(legacy_dir);
+ }
+
+ let retired_keys = RETIRED_MODEL_KEYS
+ .iter()
+ .map(|key| model_key(key))
+ .collect::<Vec<_>>();
+ if let Err(error) = store.remove_keys(&retired_keys) {
+ warnings.push(format!("Failed to remove retired model assets: {error}"));
+ }
+ warnings
+}
+
+impl Model {
+ fn spec(self) -> ModelSpec {
+ match self {
+ Self::FaceDetection => ModelSpec {
+ key: "yolov5s_face_640_640_static_b1",
+ files: &[ModelFileSpec {
+ name: "yolov5s_face_640_640_static_b1.onnx",
+ size: 32_355_091,
+ sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",
+ }],
+ },
+ Self::FaceEmbedding => ModelSpec {
+ key: "mobilefacenet_portable_static_b1",
+ files: &[ModelFileSpec {
+ name: "mobilefacenet_portable_static_b1.onnx",
+ size: 5_278_803,
+ sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",
+ }],
+ },
+ Self::ClipImage => ModelSpec {
+ key: "mobileclip_s2_image_gelu_opset20",
+ files: &[ModelFileSpec {
+ name: "mobileclip_s2_image_gelu_opset20.onnx",
+ size: 143_057_352,
+ sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",
+ }],
+ },
+ Self::ClipText => ModelSpec {
+ key: "mobileclip_s2_text_opset18_quant",
+ files: &[
+ ModelFileSpec {
+ name: "mobileclip_s2_text_opset18_quant.onnx",
+ size: 67_144_712,
+ sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",
+ },
+ ModelFileSpec {
+ name: "bpe_simple_vocab_16e6.txt",
+ size: 3_194_984,
+ sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",
+ },
+ ],
+ },
+ Self::PetFaceDetection => ModelSpec {
+ key: "yolov5s_pet_face_fp16_V2",
+ files: &[ModelFileSpec {
+ name: "yolov5s_pet_face_fp16_V2.onnx",
+ size: 14_758_020,
+ sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",
+ }],
+ },
+ Self::PetFaceEmbeddingDog => ModelSpec {
+ key: "dog_face_embedding128",
+ files: &[ModelFileSpec {
+ name: "dog_face_embedding128.onnx",
+ size: 4_141_071,
+ sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",
+ }],
+ },
+ Self::PetFaceEmbeddingCat => ModelSpec {
+ key: "cat_face_embedding128",
+ files: &[ModelFileSpec {
+ name: "cat_face_embedding128.onnx",
+ size: 4_141_071,
+ sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",
+ }],
+ },
+ Self::PetBodyDetection => ModelSpec {
+ key: "yolov5s_object_fp16",
+ files: &[ModelFileSpec {
+ name: "yolov5s_object_fp16.onnx",
+ size: 14_987_107,
+ sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",
+ }],
+ },
+ Self::PetBodyEmbeddingDog => ModelSpec {
+ key: "dog_body_embedding192",
+ files: &[ModelFileSpec {
+ name: "dog_body_embedding192.onnx",
+ size: 4_569_361,
+ sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",
+ }],
+ },
+ Self::PetBodyEmbeddingCat => ModelSpec {
+ key: "cat_body_embedding192",
+ files: &[ModelFileSpec {
+ name: "cat_body_embedding192.onnx",
+ size: 4_569_361,
+ sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",
+ }],
+ },
+ }
+ }
+}
+
+struct ModelSpec {
+ key: &'static str,
+ files: &'static [ModelFileSpec],
+}
+
+struct ModelFileSpec {
+ name: &'static str,
+ size: u64,
+ sha256: &'static str,
+}
+
+fn model_path_if(store: &AssetStore, model: Model, enabled: bool) -> String {
+ if !enabled {
+ return String::new();
+ }
+ let asset = model_asset(model);
+ store
+ .file_path(&asset, model.spec().files[0].name)
+ .expect("model file")
+ .to_string_lossy()
+ .into_owned()
+}
+
+fn model_key(key: &str) -> Vec<String> {
+ vec![MODELS.to_string(), key.to_string()]
+}
+
+fn migrate_desktop_model(
+ store: &AssetStore,
+ legacy_dir: &Path,
+ model: Model,
+) -> Result<(), String> {
+ let spec = model.spec();
+ let asset = model_asset(model);
+ let legacy_files = spec
+ .files
+ .iter()
+ .map(|file| (file, legacy_dir.join(file.name)))
+ .collect::<Vec<_>>();
+ let states = legacy_files
+ .iter()
+ .map(|(file, path)| match fs::symlink_metadata(path) {
+ Ok(metadata) => Ok(Some(
+ metadata.file_type().is_file() && metadata.len() == file.size,
+ )),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+ Err(error) => Err(error),
+ })
+ .collect::<Result<Vec<_>, _>>()
+ .map_err(|error| error.to_string())?;
+
+ if states.iter().all(Option::is_none) {
+ return Ok(());
+ }
+ if store.is_downloaded(&asset) || !states.iter().all(|state| *state == Some(true)) {
+ return legacy_files
+ .iter()
+ .try_for_each(|(_, path)| remove_file_if_exists(path))
+ .map_err(|error| error.to_string());
+ }
+
+ store.remove(&asset).map_err(|error| error.to_string())?;
+ let destination = store.asset_dir(&asset);
+ fs::create_dir_all(&destination).map_err(|error| error.to_string())?;
+ legacy_files
+ .iter()
+ .try_for_each(|(file, source)| fs::rename(source, destination.join(file.name)))
+ .map_err(|error| error.to_string())
+}
+
+fn remove_file_if_exists(path: &Path) -> std::io::Result<()> {
+ match fs::remove_file(path) {
+ Ok(()) => Ok(()),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(error) => Err(error),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use tempfile::TempDir;
+
+ use super::*;
+
+ #[test]
+ fn catalog_keys_are_unique_and_prefix_free() {
+ let root = Path::new("assets");
+ let store = AssetStore::new(root);
+ let paths = ALL_MODELS
+ .map(|model| store.asset_dir(&model_asset(model)))
+ .to_vec();
+ for (index, path) in paths.iter().enumerate() {
+ assert!(path.starts_with(root));
+ for other in &paths[index + 1..] {
+ assert_ne!(path, other);
+ assert!(!path.starts_with(other));
+ assert!(!other.starts_with(path));
+ }
+ }
+ }
+
+ #[test]
+ fn retired_keys_are_not_in_the_catalog() {
+ for key in RETIRED_MODEL_KEYS {
+ assert!(ALL_MODELS.iter().all(|model| model.spec().key != *key));
+ }
+ }
+
+ #[test]
+ fn test_model_pins_match_the_catalog() {
+ let path = Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("../../../infra/ml/test/ml_indexing/assets.json");
+ let contents = fs::read_to_string(&path).unwrap();
+ let lock: serde_json::Value = serde_json::from_str(&contents).unwrap();
+ for model in lock["models"].as_object().unwrap().values() {
+ let name = model["file_name"].as_str().unwrap();
+ let sha256 = model["sha256"].as_str().unwrap();
+ let catalog_file = ALL_MODELS
+ .iter()
+ .flat_map(|model| model.spec().files)
+ .find(|file| file.name == name)
+ .unwrap_or_else(|| panic!("{name} is missing from the Photos model catalog"));
+ assert_eq!(catalog_file.sha256, sha256, "{name}");
+ }
+ }
+
+ #[test]
+ fn migrates_complete_legacy_models_and_prunes_retired_files() {
+ let temp = TempDir::new().unwrap();
+ let legacy = temp.path().join("models");
+ fs::create_dir(&legacy).unwrap();
+ create_legacy_file(&legacy, &Model::FaceEmbedding.spec().files[0]);
+ fs::write(legacy.join(RETIRED_LEGACY_MODEL_FILES[0]), b"retired").unwrap();
+ let store = AssetStore::new(temp.path().join("assets"));
+
+ assert!(migrate_desktop_models(&store, &legacy).is_empty());
+
+ let asset = model_asset(Model::FaceEmbedding);
+ assert!(store.is_downloaded(&asset));
+ assert_eq!(
+ fs::metadata(model_path_if(&store, Model::FaceEmbedding, true))
+ .unwrap()
+ .len(),
+ Model::FaceEmbedding.spec().files[0].size
+ );
+ assert!(!legacy.join(RETIRED_LEGACY_MODEL_FILES[0]).exists());
+ }
+
+ #[test]
+ fn rejects_partial_or_wrong_sized_legacy_models() {
+ let temp = TempDir::new().unwrap();
+ let legacy = temp.path().join("models");
+ fs::create_dir(&legacy).unwrap();
+ create_legacy_file(&legacy, &Model::ClipText.spec().files[0]);
+ fs::write(
+ legacy.join(Model::FaceDetection.spec().files[0].name),
+ b"truncated",
+ )
+ .unwrap();
+ let store = AssetStore::new(temp.path().join("assets"));
+
+ assert!(migrate_desktop_models(&store, &legacy).is_empty());
+
+ assert!(!store.is_downloaded(&model_asset(Model::ClipText)));
+ assert!(!store.is_downloaded(&model_asset(Model::FaceDetection)));
+ assert!(!legacy.exists());
+ }
+
+ fn create_legacy_file(directory: &Path, file: &ModelFileSpec) {
+ let handle = fs::File::create(directory.join(file.name)).unwrap();
+ handle.set_len(file.size).unwrap();
+ }
+}
diff --git a/rust/crates/photos/src/ml/mod.rs b/rust/crates/photos/src/ml/mod.rs
index 88b35b1544..c5e72de245 100644
--- a/rust/crates/photos/src/ml/mod.rs
+++ b/rust/crates/photos/src/ml/mod.rs
@@ -1,3 +1,4 @@
+pub mod assets;
mod clip;
pub mod error;
pub mod events;