π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Use the asset store for Photos desktop models
Manav Rathi44b026d0e11e
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.tsindex e85cbe6e7e..2b57fc3d65 100644--- a/desktop/src/main/services/ml-worker.ts+++ b/desktop/src/main/services/ml-worker.ts@@ -4,25 +4,20 @@* The ML pipeline itself β image decode, preprocessing, ONNX inference and* postprocessing β is implemented by the Rust crate shared with the mobile* apps, and reaches us via a Node native addon (see [Note: ML with Rust]).- * This file downloads the models the pipeline needs, loads the addon, and- * forwards requests that arrive from the web layer over our MessagePort.+ * This file loads the addon and forwards requests that arrive from the web+ * layer over our MessagePort. The addon downloads the models through the+ * shared Rust asset store.*/// See [Note: Using Electron APIs in UtilityProcess] about what we can and// cannot import.import { expose } from "comlink";-import { net } from "electron/main";-import { existsSync } from "fs";-import { createHash } from "node:crypto";-import { createReadStream } from "node:fs";-import fs from "node:fs/promises";import path from "node:path";import { z } from "zod";import log from "../log-worker";import { messagePortMainEndpoint } from "../utils/comlink";import { wait } from "../utils/common";-import { writeStream } from "../utils/stream";import { fsStatMtime } from "./fs";// The addon is loaded at runtime from the path the main process gives us, so@@ -101,6 +96,7 @@ class CategorizedMLWorkerError extends Error {}let _native: MLNative | undefined;+let _assetStore: MLNativeModule.AssetStore | undefined;let _nativeLoadError: string | undefined;/**@@ -118,6 +114,10 @@ const loadMLNative = (paths: MLNativePaths) => {// Enables the guarded WebGPU path on Linux and Windows. This is a// no-op on macOS, where CoreML remains the preferred provider.native.setMlExecutionConfig(true);+ _assetStore = new native.AssetStore(+ path.join(userDataPath(), "assets"),+ path.join(userDataPath(), "models"),+ );_native = native;} catch (e) {_nativeLoadError = e instanceof Error ? e.message : String(e);@@ -137,6 +137,14 @@ const mlNative = () => {);};+const assetStore = () => {+ if (_assetStore) return _assetStore;+ throw new CategorizedMLWorkerError(+ "init",+ `ML is unavailable: ${_nativeLoadError ?? "asset store not loaded"}`,+ );+};+/*** Drain the runtime events (execution provider fallbacks, self-test failures)* buffered on the Rust side, writing each to our log at its severity.@@ -151,279 +159,35 @@ const logMLRuntimeEvents = (native: MLNative) => {};/**- * The ML models (and the CLIP tokenizer vocabulary) used by the Rust- * pipeline, identified by their name in the models.ente.com bucket. These are- * the same assets that the mobile apps use (see ml_model_assets.dart).- *- * Fresh downloads are verified against the pinned SHA-256; copies already on- * disk are revalidated with only a cheap size check (a corrupt model that- * nevertheless retains the expected size gets caught by the Rust side, which- * reports it back to us for eviction and redownload).- */-const modelSpecs = {- faceDetection: {- name: "yolov5s_face_640_640_static_b1.onnx",- byteSize: 32355091,- sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",- },- faceEmbedding: {- name: "mobilefacenet_portable_static_b1.onnx",- byteSize: 5278803,- sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",- },- clipImage: {- name: "mobileclip_s2_image_gelu_opset20.onnx",- byteSize: 143057352,- sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",- },- clipText: {- name: "mobileclip_s2_text_opset18_quant.onnx",- byteSize: 67144712,- sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",- },- clipTextVocab: {- name: "bpe_simple_vocab_16e6.txt",- byteSize: 3194984,- sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",- },- petFaceDetection: {- name: "yolov5s_pet_face_fp16_V2.onnx",- byteSize: 14758020,- sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",- },- petFaceEmbeddingDog: {- name: "dog_face_embedding128.onnx",- byteSize: 4141071,- sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",- },- petFaceEmbeddingCat: {- name: "cat_face_embedding128.onnx",- byteSize: 4141071,- sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",- },- petBodyDetection: {- name: "yolov5s_object_fp16.onnx",- byteSize: 14987107,- sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",- },- petBodyEmbeddingDog: {- name: "dog_body_embedding192.onnx",- byteSize: 4569361,- sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",- },- petBodyEmbeddingCat: {- name: "cat_body_embedding192.onnx",- byteSize: 4569361,- sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",- },-} as const;--type ModelSpec = (typeof modelSpecs)[keyof typeof modelSpecs];--/** Return the path where the given {@link modelName} is meant to be saved */-const modelSavePath = (modelName: string) =>- path.join(userDataPath(), "models", modelName);--/**- * Cached promises to the on-disk paths of verified models, so that each model- * is downloaded and verified at most once per process. Multiple parallel- * calls for the same model await the same promise.- */-const _verifiedModelPaths = new Map<string, Promise<string>>();--/** The single cleanup attempt shared by all model loads in this process. */-let _oldModelsCleanup: Promise<void> | undefined;--const modelPath = (spec: ModelSpec) => {- let promise = _verifiedModelPaths.get(spec.name);- if (!promise) {- promise = modelPathDownloadingIfNeeded(spec).catch((e: unknown) => {- // Allow a later call to retry the download.- _verifiedModelPaths.delete(spec.name);- throw e;- });- _verifiedModelPaths.set(spec.name, promise);- }- return promise;-};--/**- * Forget the cached verification of the model at {@link modelPath} and delete- * it from disk, so that the next use downloads a fresh copy. Used when the- * Rust side reports the on-disk model as corrupt.- */-const evictModel = async (modelPath: string) => {- const name = path.basename(modelPath);- _verifiedModelPaths.delete(name);- await fs.rm(modelPath, { force: true });-};--/**- * Download the model described by {@link spec} if we don't already have it.- *- * An existing on-disk copy is revalidated with only a cheap size check so- * that resolving already downloaded models stays fast; fresh downloads are- * verified against the pinned SHA-256.- *- * @returns the path to the model on the local machine.- */-const modelPathDownloadingIfNeeded = async (spec: ModelSpec) => {- const savePath = modelSavePath(spec.name);-- await cleanupOldModelsIfNeeded();-- if (existsSync(savePath)) {- const size = (await fs.stat(savePath)).size;- if (size == spec.byteSize) return savePath;- log.error(- `The size ${size} of model ${spec.name} does not match the expected size, downloading again`,- );- } else {- log.info(`ML model ${spec.name} not found, downloading`);- }-- await downloadModelVerifyingSHA256(savePath, spec);- return savePath;-};--/**- * Download the model described by {@link spec} to {@link savePath}, verifying- * the SHA-256 of the downloaded contents and retrying once on mismatch.- */-const downloadModelVerifyingSHA256 = async (- savePath: string,- spec: ModelSpec,-) => {- await downloadModel(savePath, spec.name);- if ((await computeFileSHA256(savePath)) == spec.sha256) return;-- log.error(- `The SHA-256 of the downloaded model ${spec.name} does not match the expected value, downloading again`,- );- await downloadModel(savePath, spec.name);- if ((await computeFileSHA256(savePath)) == spec.sha256) return;-- // Don't leave the bad download behind: were it to happen to pass the size- // check, a later call would use it without noticing.- await fs.rm(savePath, { force: true });- throw new Error(- `The SHA-256 of the downloaded model ${spec.name} does not match the expected value`,- );-};--const computeFileSHA256 = (filePath: string) =>- new Promise<string>((resolve, reject) => {- const hash = createHash("sha256");- const stream = createReadStream(filePath);- stream.on("error", reject);- stream.on("data", (chunk) => hash.update(chunk));- stream.on("end", () => resolve(hash.digest("hex")));- });--/**- * Cleanup old models.- *- * This code runs at most once in each utility process, before its first model- * is resolved. This is a good time to delete previously existent but now- * unused models and reclaim their disk space.- */-const cleanupOldModelsIfNeeded = () =>- (_oldModelsCleanup ??= cleanupOldModels().catch((e: unknown) => {- // Removing obsolete cache files is best effort and must not prevent the- // current models from loading. Retry on the next process start.- log.warn("Failed to clean up old ML models", e);- }));--const cleanupOldModels = async () => {- const oldModelNames = [- "clip-image-vit-32-float32.onnx",- "clip-text-vit-32-uint8.onnx",- "mobileclip_s2_image.onnx",- "mobileclip_s2_image_opset18_rgba_sim.onnx",- "mobileclip_s2_image_opset18_rgba_opt.onnx",- "mobileclip_s2_text_int32.onnx",- "yolov5s_face_640_640_dynamic.onnx",- "yolov5s_face_opset18_rgba_opt.onnx",- "yolov5s_face_opset18_rgba_opt_nosplits.onnx",- "mobilefacenet_opset15.onnx",- ];-- for (const modelName of oldModelNames) {- const modelPath = modelSavePath(modelName);- if (existsSync(modelPath)) {- log.info(`Removing unused ML model at ${modelPath}`);- await fs.rm(modelPath, { force: true });- }- }-};--const downloadModel = async (saveLocation: string, name: string) => {- // `mkdir -p` the directory where we want to save the model.- const saveDir = path.dirname(saveLocation);- await fs.mkdir(saveDir, { recursive: true });- // Download.- log.info(`Downloading ML model from ${name}`);- const url = `https://models.ente.com/${name}`;- const res = await net.fetch(url);- if (!res.ok) throw new Error(`Failed to fetch ${url}: HTTP ${res.status}`);- const body = res.body;- if (!body) throw new Error(`Received an null response for ${url}`);- // Save.- await writeStream(saveLocation, body);- log.info(`Downloaded ML model ${name}`);-};--/**- * The subset of {@link modelSpecs} needed for the indexing related tasks that- * are enabled in the given request, mapped to verified on-disk paths.+ * The models needed for the indexing tasks enabled in the given request,+ * mapped to verified on-disk paths.** Models for tasks that are not requested are left as empty strings, which* the Rust side treats as "not configured".*/-const indexingModelPaths = async (+const _indexingModelPaths = new Map<+ number,+ Promise<MLNativeModule.ModelPaths>+>();++const indexingModelPaths = (runFaces: boolean,runClip: boolean,runPets: boolean,-): Promise<MLNativeModule.ModelPaths> => {- const [faceDetection, faceEmbedding] = runFaces- ? await Promise.all([- modelPath(modelSpecs.faceDetection),- modelPath(modelSpecs.faceEmbedding),- ])- : ["", ""];- const clipImage = runClip ? await modelPath(modelSpecs.clipImage) : "";- const [- petFaceDetection,- petFaceEmbeddingDog,- petFaceEmbeddingCat,- petBodyDetection,- petBodyEmbeddingDog,- petBodyEmbeddingCat,- ] = runPets- ? await Promise.all([- modelPath(modelSpecs.petFaceDetection),- modelPath(modelSpecs.petFaceEmbeddingDog),- modelPath(modelSpecs.petFaceEmbeddingCat),- modelPath(modelSpecs.petBodyDetection),- modelPath(modelSpecs.petBodyEmbeddingDog),- modelPath(modelSpecs.petBodyEmbeddingCat),- ])- : ["", "", "", "", "", ""];-- return {- faceDetection,- faceEmbedding,- clipImage,- // The CLIP text session is managed lazily by the Rust side when- // serving text queries; it is not part of the indexing runtime.- clipText: "",- petFaceDetection,- petFaceEmbeddingDog,- petFaceEmbeddingCat,- petBodyDetection,- petBodyEmbeddingDog,- petBodyEmbeddingCat,- };+) => {+ const key =+ Number(runFaces) | (Number(runClip) << 1) | (Number(runPets) << 2);+ let promise = _indexingModelPaths.get(key);+ if (!promise) {+ promise = assetStore()+ .indexingModelPaths(runFaces, runClip, runPets)+ .catch((e: unknown) => {+ _indexingModelPaths.delete(key);+ throw e;+ });+ _indexingModelPaths.set(key, promise);+ }+ return promise;};/**@@ -493,7 +257,7 @@ const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {try {return await analyzeImageOnce(native, req);} catch (e) {- if (!(await evictModelIfReportedCorrupt(e))) throw e;+ if (!evictModelIfReportedCorrupt(e)) throw e;_preparedRuntimeKey = undefined;return await analyzeImageOnce(native, req);} finally {@@ -550,28 +314,27 @@ const analyzeImageOnce = async (/*** If {@link e} is the addon's tagged "CorruptModel" error, return the path of- * the corrupt model file it reports (but only if that path is under our- * models directory, as a safety check before we delete it).+ * the corrupt model file it reports.*/const corruptModelPathFromError = (e: unknown) => {if (!(e instanceof Error)) return undefined;if (!e.message.startsWith("CorruptModel: ")) return undefined;- const corruptPath = e.message.slice("CorruptModel: ".length).trim();- const modelsDir = path.join(userDataPath(), "models") + path.sep;- return corruptPath.startsWith(modelsDir) ? corruptPath : undefined;+ return e.message.slice("CorruptModel: ".length).trim();};/*** If {@link e} is the addon's tagged "CorruptModel" error, evict the model it* names so that its next use downloads a fresh copy, and return `true`.*/-const evictModelIfReportedCorrupt = async (e: unknown) => {+const evictModelIfReportedCorrupt = (e: unknown) => {const corruptModelPath = corruptModelPathFromError(e);if (!corruptModelPath) return false;+ if (!assetStore().removeModel(corruptModelPath)) return false;+ _indexingModelPaths.clear();log.error(`Rust ML reported a corrupt model at ${corruptModelPath}, evicting it so that it gets redownloaded`,);- await evictModel(corruptModelPath);+ _clipTextModelPaths = undefined;return true;};@@ -579,6 +342,16 @@ const evictModelIfReportedCorrupt = async (e: unknown) => {const uint8ArrayToBuffer = (bytes: Uint8Array) =>Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);+let _clipTextModelPaths: Promise<MLNativeModule.ClipTextModelPaths> | undefined;++const clipTextModelPaths = () =>+ (_clipTextModelPaths ??= assetStore()+ .clipTextModelPaths()+ .catch((e: unknown) => {+ _clipTextModelPaths = undefined;+ throw e;+ }));+/*** Run a throwaway CLIP text query so that the Rust side builds and caches the* text encoder session before the user's first search. Mirrors the mobile@@ -588,17 +361,14 @@ const uint8ArrayToBuffer = (bytes: Uint8Array) =>*/const warmUpClipTextEncoder = async (native: MLNative) => {try {- const [model, vocab] = await Promise.all([- modelPath(modelSpecs.clipText),- modelPath(modelSpecs.clipTextVocab),- ]);+ const { modelPath, vocabPath } = await clipTextModelPaths();await native.runClipText({text: "warm up text encoder",- modelPath: model,- vocabPath: vocab,+ modelPath,+ vocabPath,});} catch (e) {- if (!(await evictModelIfReportedCorrupt(e)))+ if (!evictModelIfReportedCorrupt(e))log.warn("Failed to warm up the CLIP text encoder", e);} finally {logMLRuntimeEvents(native);@@ -617,10 +387,7 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {const native = _native;const pathsOrSkip = await Promise.race([- Promise.all([- modelPath(modelSpecs.clipText),- modelPath(modelSpecs.clipTextVocab),- ]),+ clipTextModelPaths(),// Wait a bit for the models to resolve the first time this code runs// on each app start (when they're already downloaded, resolving them// is quick), but don't block on a download.@@ -634,18 +401,18 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {return undefined;}- const [clipTextModelPath, vocabPath] = pathsOrSkip;+ const { modelPath, vocabPath } = pathsOrSkip;try {const { embedding } = await native.runClipText({text,- modelPath: clipTextModelPath,+ modelPath,vocabPath,});return embedding;} catch (e) {// Don't block this query on the redownload; a subsequent query will// pick up the fresh copy.- if (!(await evictModelIfReportedCorrupt(e))) throw e;+ if (!evictModelIfReportedCorrupt(e)) throw e;return undefined;} finally {logMLRuntimeEvents(native);diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 2070bc01ef..462ad0e584 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2608,6 +2608,7 @@ dependencies = [name = "ente_photos_napi"version = "0.0.0"dependencies = [+ "ente-assets","ente-photos","napi","napi-build",@@ -5626,6 +5627,7 @@ dependencies = ["napi-sys","nohash-hasher","rustc-hash",+ "tokio",][[package]]diff --git a/rust/bindings/napi/photos/Cargo.toml b/rust/bindings/napi/photos/Cargo.tomlindex c6d70f3091..6ebe052ca2 100644--- a/rust/bindings/napi/photos/Cargo.toml+++ b/rust/bindings/napi/photos/Cargo.toml@@ -7,8 +7,9 @@ crate-type = ["cdylib"][dependencies]# usearch requires a newer GCC than napi-cli provides.+ente-assets.workspace = trueente-photos.workspace = true-napi = { version = "3", default-features = false, features = ["napi8"] }+napi = { version = "3", default-features = false, features = ["napi8", "tokio_rt"] }napi-derive = "3"ort = { workspace = true, features = ["load-dynamic"] }diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rsindex f3f675569a..bedbc25caf 100644--- a/rust/bindings/napi/photos/src/lib.rs+++ b/rust/bindings/napi/photos/src/lib.rs@@ -1,5 +1,5 @@use ente_photos::ml::{- error::MlError, indexing as shared_indexing, runtime::ModelPaths as SharedModelPaths,+ assets, error::MlError, indexing as shared_indexing, runtime::ModelPaths as SharedModelPaths,types as shared_types,};use napi::bindgen_prelude::*;@@ -46,6 +46,84 @@ pub struct ModelPaths {pub pet_body_embedding_cat: String,}+#[napi(object)]+pub struct ClipTextModelPaths {+ pub model_path: String,+ pub vocab_path: String,+}++#[napi]+pub struct AssetStore {+ inner: ente_assets::AssetStore,+}++#[napi]+impl AssetStore {+ #[napi(constructor)]+ pub fn new(assets_dir: String, legacy_models_dir: String) -> Self {+ let inner = ente_assets::AssetStore::new(assets_dir);+ for warning in+ assets::migrate_desktop_models(&inner, std::path::Path::new(&legacy_models_dir))+ {+ eprintln!("Photos model migration: {warning}");+ }+ Self { inner }+ }++ #[napi]+ pub async fn indexing_model_paths(+ &self,+ run_faces: bool,+ run_clip: bool,+ run_pets: bool,+ ) -> Result<ModelPaths> {+ let models = assets::indexing_models(run_faces, run_clip, run_pets);+ let model_assets = models+ .iter()+ .copied()+ .map(assets::model_asset)+ .collect::<Vec<_>>();+ self.inner+ .download(+ &model_assets,+ |_| {},+ ente_assets::download::CancellationToken::default(),+ )+ .await+ .map_err(asset_error_to_napi)?;+ Ok(to_napi_model_paths(assets::indexing_model_paths(+ &self.inner,+ run_faces,+ run_clip,+ run_pets,+ )))+ }++ #[napi]+ pub async fn clip_text_model_paths(&self) -> Result<ClipTextModelPaths> {+ let asset = assets::model_asset(assets::Model::ClipText);+ self.inner+ .download(+ std::slice::from_ref(&asset),+ |_| {},+ ente_assets::download::CancellationToken::default(),+ )+ .await+ .map_err(asset_error_to_napi)?;+ let paths = assets::clip_text_paths(&self.inner);+ Ok(ClipTextModelPaths {+ model_path: paths.model.to_string_lossy().into_owned(),+ vocab_path: paths.vocab.to_string_lossy().into_owned(),+ })+ }++ #[napi]+ pub fn remove_model(&self, model_path: String) -> Result<bool> {+ assets::remove_model_at_path(&self.inner, std::path::Path::new(&model_path))+ .map_err(|error| Error::from_reason(format!("Assets: {error}")))+ }+}+#[napi(object)]pub struct AnalyzeImageRequest {pub file_id: i64,@@ -255,6 +333,10 @@ fn ml_error_to_napi(error: MlError) -> Error {Error::from_reason(format!("{kind}: {detail}"))}+fn asset_error_to_napi(error: ente_assets::download::Error) -> Error {+ Error::from_reason(format!("Assets: {error}"))+}+fn to_shared_model_paths(paths: &ModelPaths) -> SharedModelPaths {SharedModelPaths {face_detection: paths.face_detection.clone(),@@ -270,6 +352,21 @@ fn to_shared_model_paths(paths: &ModelPaths) -> SharedModelPaths {}}+fn to_napi_model_paths(paths: SharedModelPaths) -> ModelPaths {+ ModelPaths {+ face_detection: paths.face_detection,+ face_embedding: paths.face_embedding,+ clip_image: paths.clip_image,+ clip_text: paths.clip_text,+ pet_face_detection: paths.pet_face_detection,+ pet_face_embedding_dog: paths.pet_face_embedding_dog,+ pet_face_embedding_cat: paths.pet_face_embedding_cat,+ pet_body_detection: paths.pet_body_detection,+ pet_body_embedding_dog: paths.pet_body_embedding_dog,+ pet_body_embedding_cat: paths.pet_body_embedding_cat,+ }+}+fn to_napi_analyze_image_result(result: shared_indexing::AnalyzeImageResult) -> AnalyzeImageResult {AnalyzeImageResult {file_id: result.file_id,diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex f030ab2b58..6480110972 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -7,6 +7,7 @@ default = []ml-assets = [][dependencies]+ente-assets.workspace = trueente-image.workspace = truefast_image_resize = { version = "5.5.0", features = ["image", "rayon"] }half = "2.1"@@ -80,7 +81,6 @@ ort = { workspace = true, features = [[dev-dependencies]anyhow = "1"-ente-assets.workspace = trueflate2 = "1.1"serde = { version = "1", features = ["derive"] }serde_json.workspace = truediff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rsnew file mode 100644index 0000000000..05a841c08d--- /dev/null+++ b/rust/crates/photos/src/ml/assets.rs@@ -0,0 +1,436 @@+use std::fs;+use std::path::{Path, PathBuf};++use ente_assets::{Asset, AssetFile, AssetStore, AssetStoreError};++use super::runtime::ModelPaths;++const MODELS: &str = "models";++const RETIRED_MODEL_KEYS: &[&str] = &[];++const RETIRED_LEGACY_MODEL_FILES: &[&str] = &[+ "clip-image-vit-32-float32.onnx",+ "clip-text-vit-32-uint8.onnx",+ "mobileclip_s2_image.onnx",+ "mobileclip_s2_image_opset18_rgba_sim.onnx",+ "mobileclip_s2_image_opset18_rgba_opt.onnx",+ "mobileclip_s2_text_int32.onnx",+ "yolov5s_face_640_640_dynamic.onnx",+ "yolov5s_face_opset18_rgba_opt.onnx",+ "yolov5s_face_opset18_rgba_opt_nosplits.onnx",+ "mobilefacenet_opset15.onnx",+];++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum Model {+ FaceDetection,+ FaceEmbedding,+ ClipImage,+ ClipText,+ PetFaceDetection,+ PetFaceEmbeddingDog,+ PetFaceEmbeddingCat,+ PetBodyDetection,+ PetBodyEmbeddingDog,+ PetBodyEmbeddingCat,+}++const ALL_MODELS: [Model; 10] = [+ Model::FaceDetection,+ Model::FaceEmbedding,+ Model::ClipImage,+ Model::ClipText,+ Model::PetFaceDetection,+ Model::PetFaceEmbeddingDog,+ Model::PetFaceEmbeddingCat,+ Model::PetBodyDetection,+ Model::PetBodyEmbeddingDog,+ Model::PetBodyEmbeddingCat,+];++pub struct ClipTextPaths {+ pub model: PathBuf,+ pub vocab: PathBuf,+}++pub fn model_asset(model: Model) -> Asset {+ let spec = model.spec();+ Asset::files(+ model_key(spec.key),+ spec.files+ .iter()+ .map(|file| AssetFile {+ name: file.name.to_string(),+ url: format!("https://models.ente.com/{}", file.name),+ sha256: file.sha256.to_string(),+ })+ .collect(),+ )+ .expect("valid Photos model catalog")+}++pub fn indexing_models(run_faces: bool, run_clip: bool, run_pets: bool) -> Vec<Model> {+ let mut models = Vec::new();+ if run_faces {+ models.extend([Model::FaceDetection, Model::FaceEmbedding]);+ }+ if run_clip {+ models.push(Model::ClipImage);+ }+ if run_pets {+ models.extend([+ Model::PetFaceDetection,+ Model::PetFaceEmbeddingDog,+ Model::PetFaceEmbeddingCat,+ Model::PetBodyDetection,+ Model::PetBodyEmbeddingDog,+ Model::PetBodyEmbeddingCat,+ ]);+ }+ models+}++pub fn indexing_model_paths(+ store: &AssetStore,+ run_faces: bool,+ run_clip: bool,+ run_pets: bool,+) -> ModelPaths {+ ModelPaths {+ face_detection: model_path_if(store, Model::FaceDetection, run_faces),+ face_embedding: model_path_if(store, Model::FaceEmbedding, run_faces),+ clip_image: model_path_if(store, Model::ClipImage, run_clip),+ clip_text: String::new(),+ pet_face_detection: model_path_if(store, Model::PetFaceDetection, run_pets),+ pet_face_embedding_dog: model_path_if(store, Model::PetFaceEmbeddingDog, run_pets),+ pet_face_embedding_cat: model_path_if(store, Model::PetFaceEmbeddingCat, run_pets),+ pet_body_detection: model_path_if(store, Model::PetBodyDetection, run_pets),+ pet_body_embedding_dog: model_path_if(store, Model::PetBodyEmbeddingDog, run_pets),+ pet_body_embedding_cat: model_path_if(store, Model::PetBodyEmbeddingCat, run_pets),+ }+}++pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {+ let asset = model_asset(Model::ClipText);+ let files = Model::ClipText.spec().files;+ ClipTextPaths {+ model: store+ .file_path(&asset, files[0].name)+ .expect("CLIP text model file"),+ vocab: store+ .file_path(&asset, files[1].name)+ .expect("CLIP text vocabulary file"),+ }+}++pub fn remove_model_at_path(store: &AssetStore, path: &Path) -> Result<bool, AssetStoreError> {+ for model in ALL_MODELS {+ let asset = model_asset(model);+ if model+ .spec()+ .files+ .iter()+ .filter_map(|file| store.file_path(&asset, file.name))+ .any(|candidate| candidate == path)+ {+ store.remove(&asset)?;+ return Ok(true);+ }+ }+ Ok(false)+}++pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {+ let mut warnings = Vec::new();+ if legacy_dir.exists() {+ for model in ALL_MODELS {+ if let Err(error) = migrate_desktop_model(store, legacy_dir, model) {+ warnings.push(format!("Failed to migrate {}: {error}", model.spec().key));+ }+ }+ for name in RETIRED_LEGACY_MODEL_FILES {+ if let Err(error) = remove_file_if_exists(&legacy_dir.join(name)) {+ warnings.push(format!("Failed to remove retired model {name}: {error}"));+ }+ }+ let _ = fs::remove_dir(legacy_dir);+ }++ let retired_keys = RETIRED_MODEL_KEYS+ .iter()+ .map(|key| model_key(key))+ .collect::<Vec<_>>();+ if let Err(error) = store.remove_keys(&retired_keys) {+ warnings.push(format!("Failed to remove retired model assets: {error}"));+ }+ warnings+}++impl Model {+ fn spec(self) -> ModelSpec {+ match self {+ Self::FaceDetection => ModelSpec {+ key: "yolov5s_face_640_640_static_b1",+ files: &[ModelFileSpec {+ name: "yolov5s_face_640_640_static_b1.onnx",+ size: 32_355_091,+ sha256: "e047647409403d52696035ecd445792173e50d7fbdcccac97b958a585db9aa3d",+ }],+ },+ Self::FaceEmbedding => ModelSpec {+ key: "mobilefacenet_portable_static_b1",+ files: &[ModelFileSpec {+ name: "mobilefacenet_portable_static_b1.onnx",+ size: 5_278_803,+ sha256: "0763fc33f54e138476194da95987e133b3e976075a6b1d3e1b2caedb251b1a36",+ }],+ },+ Self::ClipImage => ModelSpec {+ key: "mobileclip_s2_image_gelu_opset20",+ files: &[ModelFileSpec {+ name: "mobileclip_s2_image_gelu_opset20.onnx",+ size: 143_057_352,+ sha256: "205a430af825e501c5138e5bb9abea942482a7a4fd4a680e98e47cf0830dce7e",+ }],+ },+ Self::ClipText => ModelSpec {+ key: "mobileclip_s2_text_opset18_quant",+ files: &[+ ModelFileSpec {+ name: "mobileclip_s2_text_opset18_quant.onnx",+ size: 67_144_712,+ sha256: "d92f33dfcff83077fc2e0d3414250710efbb51795dfd89767bdbefb5fdc47322",+ },+ ModelFileSpec {+ name: "bpe_simple_vocab_16e6.txt",+ size: 3_194_984,+ sha256: "67603cfda2e032ad77b5f8808af37789d590db664b26df8705d2bf8b3c553fc8",+ },+ ],+ },+ Self::PetFaceDetection => ModelSpec {+ key: "yolov5s_pet_face_fp16_V2",+ files: &[ModelFileSpec {+ name: "yolov5s_pet_face_fp16_V2.onnx",+ size: 14_758_020,+ sha256: "7876d97992eeb5f3a9f3b35eff5e0e133012928172a8b005093108d8c3ad2d1c",+ }],+ },+ Self::PetFaceEmbeddingDog => ModelSpec {+ key: "dog_face_embedding128",+ files: &[ModelFileSpec {+ name: "dog_face_embedding128.onnx",+ size: 4_141_071,+ sha256: "fb04d781eb1f7adf6ce3432dc0c5873f16cc051b5c98c14c754afb39e2b92462",+ }],+ },+ Self::PetFaceEmbeddingCat => ModelSpec {+ key: "cat_face_embedding128",+ files: &[ModelFileSpec {+ name: "cat_face_embedding128.onnx",+ size: 4_141_071,+ sha256: "32b10694a27f6404d2beaddbd64f07ad555f72dccb12ee60a7afe5dcf6aad6cd",+ }],+ },+ Self::PetBodyDetection => ModelSpec {+ key: "yolov5s_object_fp16",+ files: &[ModelFileSpec {+ name: "yolov5s_object_fp16.onnx",+ size: 14_987_107,+ sha256: "113f0c18632eb2c4f6deebcd40eb01c676492e9b43923c2d336e1b4012fce9ef",+ }],+ },+ Self::PetBodyEmbeddingDog => ModelSpec {+ key: "dog_body_embedding192",+ files: &[ModelFileSpec {+ name: "dog_body_embedding192.onnx",+ size: 4_569_361,+ sha256: "1d85aa20358137e30f11c2d0baa9a2248b9997928d501fe15365d1fc57522770",+ }],+ },+ Self::PetBodyEmbeddingCat => ModelSpec {+ key: "cat_body_embedding192",+ files: &[ModelFileSpec {+ name: "cat_body_embedding192.onnx",+ size: 4_569_361,+ sha256: "62fb5891e61be69a96510d8ec56e7525a9541b0283e54574d27c86c9b4a26ddf",+ }],+ },+ }+ }+}++struct ModelSpec {+ key: &'static str,+ files: &'static [ModelFileSpec],+}++struct ModelFileSpec {+ name: &'static str,+ size: u64,+ sha256: &'static str,+}++fn model_path_if(store: &AssetStore, model: Model, enabled: bool) -> String {+ if !enabled {+ return String::new();+ }+ let asset = model_asset(model);+ store+ .file_path(&asset, model.spec().files[0].name)+ .expect("model file")+ .to_string_lossy()+ .into_owned()+}++fn model_key(key: &str) -> Vec<String> {+ vec![MODELS.to_string(), key.to_string()]+}++fn migrate_desktop_model(+ store: &AssetStore,+ legacy_dir: &Path,+ model: Model,+) -> Result<(), String> {+ let spec = model.spec();+ let asset = model_asset(model);+ let legacy_files = spec+ .files+ .iter()+ .map(|file| (file, legacy_dir.join(file.name)))+ .collect::<Vec<_>>();+ let states = legacy_files+ .iter()+ .map(|(file, path)| match fs::symlink_metadata(path) {+ Ok(metadata) => Ok(Some(+ metadata.file_type().is_file() && metadata.len() == file.size,+ )),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),+ Err(error) => Err(error),+ })+ .collect::<Result<Vec<_>, _>>()+ .map_err(|error| error.to_string())?;++ if states.iter().all(Option::is_none) {+ return Ok(());+ }+ if store.is_downloaded(&asset) || !states.iter().all(|state| *state == Some(true)) {+ return legacy_files+ .iter()+ .try_for_each(|(_, path)| remove_file_if_exists(path))+ .map_err(|error| error.to_string());+ }++ store.remove(&asset).map_err(|error| error.to_string())?;+ let destination = store.asset_dir(&asset);+ fs::create_dir_all(&destination).map_err(|error| error.to_string())?;+ legacy_files+ .iter()+ .try_for_each(|(file, source)| fs::rename(source, destination.join(file.name)))+ .map_err(|error| error.to_string())+}++fn remove_file_if_exists(path: &Path) -> std::io::Result<()> {+ match fs::remove_file(path) {+ Ok(()) => Ok(()),+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),+ Err(error) => Err(error),+ }+}++#[cfg(test)]+mod tests {+ use tempfile::TempDir;++ use super::*;++ #[test]+ fn catalog_keys_are_unique_and_prefix_free() {+ let root = Path::new("assets");+ let store = AssetStore::new(root);+ let paths = ALL_MODELS+ .map(|model| store.asset_dir(&model_asset(model)))+ .to_vec();+ for (index, path) in paths.iter().enumerate() {+ assert!(path.starts_with(root));+ for other in &paths[index + 1..] {+ assert_ne!(path, other);+ assert!(!path.starts_with(other));+ assert!(!other.starts_with(path));+ }+ }+ }++ #[test]+ fn retired_keys_are_not_in_the_catalog() {+ for key in RETIRED_MODEL_KEYS {+ assert!(ALL_MODELS.iter().all(|model| model.spec().key != *key));+ }+ }++ #[test]+ fn test_model_pins_match_the_catalog() {+ let path = Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("../../../infra/ml/test/ml_indexing/assets.json");+ let contents = fs::read_to_string(&path).unwrap();+ let lock: serde_json::Value = serde_json::from_str(&contents).unwrap();+ for model in lock["models"].as_object().unwrap().values() {+ let name = model["file_name"].as_str().unwrap();+ let sha256 = model["sha256"].as_str().unwrap();+ let catalog_file = ALL_MODELS+ .iter()+ .flat_map(|model| model.spec().files)+ .find(|file| file.name == name)+ .unwrap_or_else(|| panic!("{name} is missing from the Photos model catalog"));+ assert_eq!(catalog_file.sha256, sha256, "{name}");+ }+ }++ #[test]+ fn migrates_complete_legacy_models_and_prunes_retired_files() {+ let temp = TempDir::new().unwrap();+ let legacy = temp.path().join("models");+ fs::create_dir(&legacy).unwrap();+ create_legacy_file(&legacy, &Model::FaceEmbedding.spec().files[0]);+ fs::write(legacy.join(RETIRED_LEGACY_MODEL_FILES[0]), b"retired").unwrap();+ let store = AssetStore::new(temp.path().join("assets"));++ assert!(migrate_desktop_models(&store, &legacy).is_empty());++ let asset = model_asset(Model::FaceEmbedding);+ assert!(store.is_downloaded(&asset));+ assert_eq!(+ fs::metadata(model_path_if(&store, Model::FaceEmbedding, true))+ .unwrap()+ .len(),+ Model::FaceEmbedding.spec().files[0].size+ );+ assert!(!legacy.join(RETIRED_LEGACY_MODEL_FILES[0]).exists());+ }++ #[test]+ fn rejects_partial_or_wrong_sized_legacy_models() {+ let temp = TempDir::new().unwrap();+ let legacy = temp.path().join("models");+ fs::create_dir(&legacy).unwrap();+ create_legacy_file(&legacy, &Model::ClipText.spec().files[0]);+ fs::write(+ legacy.join(Model::FaceDetection.spec().files[0].name),+ b"truncated",+ )+ .unwrap();+ let store = AssetStore::new(temp.path().join("assets"));++ assert!(migrate_desktop_models(&store, &legacy).is_empty());++ assert!(!store.is_downloaded(&model_asset(Model::ClipText)));+ assert!(!store.is_downloaded(&model_asset(Model::FaceDetection)));+ assert!(!legacy.exists());+ }++ fn create_legacy_file(directory: &Path, file: &ModelFileSpec) {+ let handle = fs::File::create(directory.join(file.name)).unwrap();+ handle.set_len(file.size).unwrap();+ }+}diff --git a/rust/crates/photos/src/ml/mod.rs b/rust/crates/photos/src/ml/mod.rsindex 88b35b1544..c5e72de245 100644--- a/rust/crates/photos/src/ml/mod.rs+++ b/rust/crates/photos/src/ml/mod.rs@@ -1,3 +1,4 @@+pub mod assets;mod clip;pub mod error;pub mod events;