π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Suspend ML indexing on corrupt model instead of evicting and redownloading
laurenspriem08c98b583edb
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.tsindex 2b57fc3d65..8289d11263 100644--- a/desktop/src/main/services/ml-worker.ts+++ b/desktop/src/main/services/ml-worker.ts@@ -238,9 +238,6 @@ export interface MLWorkerAnalyzeImageRequest {/*** Analyze a single image with the Rust ML pipeline, downloading any needed* models first.- *- * If the Rust side reports the on-disk model as corrupt, delete and- * redownload it, and retry once.*/export const analyzeImage = async (req: MLWorkerAnalyzeImageRequest,@@ -256,29 +253,29 @@ const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {const native = mlNative();try {return await analyzeImageOnce(native, req);- } catch (e) {- if (!evictModelIfReportedCorrupt(e)) throw e;- _preparedRuntimeKey = undefined;- return await analyzeImageOnce(native, req);} finally {logMLRuntimeEvents(native);}};+// A corrupt on-disk model won't fix itself (models are hash-verified on+// download), so treat it like an init failure: suspend indexing for the rest+// of the process lifetime instead of retrying.const categorizeAnalyzeImageError = (e: unknown): MLWorkerAnalyzeImageError => {if (e instanceof CategorizedMLWorkerError) {return { kind: e.kind, message: e.message };}const message = e instanceof Error ? e.message : String(e);- const kind: MLWorkerAnalyzeImageErrorKind =- message.startsWith("Decode: ") || message.startsWith("Image: ")- ? "image"- : message.startsWith("Ort: ") ||- message.startsWith("CorruptModel: ") ||- message.startsWith("Runtime: ")- ? "ort"- : "misc";+ const kind: MLWorkerAnalyzeImageErrorKind = message.startsWith(+ "CorruptModel: ",+ )+ ? "init"+ : message.startsWith("Decode: ") || message.startsWith("Image: ")+ ? "image"+ : message.startsWith("Ort: ") || message.startsWith("Runtime: ")+ ? "ort"+ : "misc";return { kind, message };};@@ -312,32 +309,6 @@ const analyzeImageOnce = async (});};-/**- * If {@link e} is the addon's tagged "CorruptModel" error, return the path of- * the corrupt model file it reports.- */-const corruptModelPathFromError = (e: unknown) => {- if (!(e instanceof Error)) return undefined;- if (!e.message.startsWith("CorruptModel: ")) return undefined;- return e.message.slice("CorruptModel: ".length).trim();-};--/**- * If {@link e} is the addon's tagged "CorruptModel" error, evict the model it- * names so that its next use downloads a fresh copy, and return `true`.- */-const evictModelIfReportedCorrupt = (e: unknown) => {- const corruptModelPath = corruptModelPathFromError(e);- if (!corruptModelPath) return false;- if (!assetStore().removeModel(corruptModelPath)) return false;- _indexingModelPaths.clear();- log.error(- `Rust ML reported a corrupt model at ${corruptModelPath}, evicting it so that it gets redownloaded`,- );- _clipTextModelPaths = undefined;- return true;-};-// Zero-copy view over the transferred bytes.const uint8ArrayToBuffer = (bytes: Uint8Array) =>Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);@@ -368,8 +339,7 @@ const warmUpClipTextEncoder = async (native: MLNative) => {vocabPath,});} catch (e) {- if (!evictModelIfReportedCorrupt(e))- log.warn("Failed to warm up the CLIP text encoder", e);+ log.warn("Failed to warm up the CLIP text encoder", e);} finally {logMLRuntimeEvents(native);}@@ -409,11 +379,6 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {vocabPath,});return embedding;- } catch (e) {- // Don't block this query on the redownload; a subsequent query will- // pick up the fresh copy.- if (!evictModelIfReportedCorrupt(e)) throw e;- return undefined;} finally {logMLRuntimeEvents(native);}diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dartindex bb10a77959..326c92618b 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart@@ -8,7 +8,6 @@ import "package:photos/core/errors.dart";import "package:photos/models/ml/vector.dart";import "package:photos/services/machine_learning/ml_constants.dart";import "package:photos/services/machine_learning/ml_model_assets.dart";-import "package:photos/services/machine_learning/ml_model_download_service.dart";import "package:photos/services/machine_learning/semantic_search/query_result.dart";import "package:photos/services/machine_learning/webgpu_execution_policy.dart";import "package:photos/services/remote_assets_service.dart";@@ -90,11 +89,7 @@ class MLComputer extends SuperIsolate {"clipTextVocabPath": vocabPath,"enableWebGpu": enableWebGpu,});- if (isolateResult is RustCorruptModelCacheDeletedException) {- _clipTextModelPath = null;- MLModelDownloadService.instance.invalidateModelDownloadCache(- includeNonIndexingModels: true,- );+ if (isolateResult is RustCorruptModelException) {throw isolateResult;}final textEmbedding = isolateResult as List<double>;@@ -106,10 +101,8 @@ class MLComputer extends SuperIsolate {s,);rethrow;- } on RustCorruptModelCacheDeletedException catch (e) {- _logger.warning(- "Deleted corrupt Rust CLIP text model cache at ${e.modelPath}",- );+ } on RustCorruptModelException catch (e) {+ _logger.severe("Rust ML reported a corrupt model at ${e.modelPath}");rethrow;} catch (e, s) {_logger.severe("Could not run clip text in isolate", e, s);diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dartindex 38aae475dc..39b5d2938f 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart@@ -63,9 +63,9 @@ class MLIndexingIsolate extends SuperIsolate {...rustRuntimeArgs,"enableWebGpu": enableWebGpu,});- if (isolateResult is RustCorruptModelCacheDeletedException) {- _logger.warning(- "Deleted corrupt Rust ONNX model cache at ${isolateResult.modelPath}; "+ if (isolateResult is RustCorruptModelException) {+ _logger.severe(+ "Rust ML reported a corrupt model at ${isolateResult.modelPath}; ""stopping ML indexing for fileID ${instruction.fileKey}",);shouldPauseIndexingAndClustering = true;@@ -82,8 +82,7 @@ class MLIndexingIsolate extends SuperIsolate {_runtimeFlagCombinations.add(result.remoteFlags);return result;} catch (e, s) {- if (e is RustCorruptModelCacheDeletedException ||- isExpectedMlSkipError(e)) {+ if (e is RustCorruptModelException || isExpectedMlSkipError(e)) {rethrow;}_logger.severe(diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dartindex 0fbcc27108..95cc9fe364 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart@@ -537,9 +537,9 @@ class MLService {"`indexAllImages()` finished. Analyzed $fileAnalyzedCount images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image)",);_logStatus();- } on RustCorruptModelCacheDeletedException catch (e) {- _logger.warning(- "Stopping image indexing because corrupt Rust ONNX model cache was deleted at ${e.modelPath}",+ } on RustCorruptModelException catch (e) {+ _logger.severe(+ "Stopping image indexing because Rust ML reported a corrupt model at ${e.modelPath}",);} catch (e, s) {_logger.severe("indexAllImages failed", e, s);@@ -917,12 +917,12 @@ class MLService {final String format = instruction.file.displayName.split('.').last;final int? size = instruction.file.fileSize;final fileType = instruction.file.fileType;- if (e is RustCorruptModelCacheDeletedException) {+ if (e is RustCorruptModelException) {pauseIndexingAndClustering();- _logger.warning(+ _logger.severe("Stopping ML indexing for fileID ${instruction.fileKey} "- "(format $format, type $fileType, size $size) because corrupt Rust "- "ONNX model cache was deleted at ${e.modelPath}",+ "(format $format, type $fileType, size $size) because Rust ML "+ "reported a corrupt model at ${e.modelPath}",);rethrow;}diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dartindex efd8d62488..b8dad1741a 100644--- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart+++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart@@ -1,4 +1,3 @@-import 'dart:io' show File;import 'dart:typed_data' show Float32List, Uint8List;import "package:flutter_rust_bridge/flutter_rust_bridge.dart" show Uint64List;@@ -25,13 +24,13 @@ const _rustLibLoadedCacheKey = "rustLibLoaded";const _rustMlModelPathsCacheKey = "rustMlModelPaths";final _rustMlRuntimeLogger = Logger("RustMLRuntime");-class RustCorruptModelCacheDeletedException implements Exception {- const RustCorruptModelCacheDeletedException(this.modelPath);+class RustCorruptModelException implements Exception {+ const RustCorruptModelException(this.modelPath);final String modelPath;@override- String toString() => "RustCorruptModelCacheDeletedException: $modelPath";+ String toString() => "RustCorruptModelException: $modelPath";}enum IsolateOperation {@@ -118,11 +117,7 @@ Future<dynamic> isolateFunction(try {result = await analyzeImageRust(args);} on rust_ml.RustMlError_CorruptModel catch (e) {- final file = File(e.field0);- if (await file.exists()) {- await file.delete();- }- return RustCorruptModelCacheDeletedException(e.field0);+ return RustCorruptModelException(e.field0);}return result.toJsonString();@@ -206,11 +201,7 @@ Future<dynamic> isolateFunction(),);} on rust_ml.RustMlError_CorruptModel catch (e) {- final file = File(e.field0);- if (await file.exists()) {- await file.delete();- }- return RustCorruptModelCacheDeletedException(e.field0);+ return RustCorruptModelException(e.field0);}return List<double>.from(result.embedding, growable: false);} finally {diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rsindex bedbc25caf..8bde511f40 100644--- a/rust/bindings/napi/photos/src/lib.rs+++ b/rust/bindings/napi/photos/src/lib.rs@@ -116,12 +116,6 @@ impl AssetStore {vocab_path: paths.vocab.to_string_lossy().into_owned(),})}-- #[napi]- pub fn remove_model(&self, model_path: String) -> Result<bool> {- assets::remove_model_at_path(&self.inner, std::path::Path::new(&model_path))- .map_err(|error| Error::from_reason(format!("Assets: {error}")))- }}#[napi(object)]diff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rsindex 05a841c08d..3e2171ecc5 100644--- a/rust/crates/photos/src/ml/assets.rs+++ b/rust/crates/photos/src/ml/assets.rs@@ -1,7 +1,7 @@use std::fs;use std::path::{Path, PathBuf};-use ente_assets::{Asset, AssetFile, AssetStore, AssetStoreError};+use ente_assets::{Asset, AssetFile, AssetStore};use super::runtime::ModelPaths;@@ -124,23 +124,6 @@ pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {}}-pub fn remove_model_at_path(store: &AssetStore, path: &Path) -> Result<bool, AssetStoreError> {- for model in ALL_MODELS {- let asset = model_asset(model);- if model- .spec()- .files- .iter()- .filter_map(|file| store.file_path(&asset, file.name))- .any(|candidate| candidate == path)- {- store.remove(&asset)?;- return Ok(true);- }- }- Ok(false)-}-pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {let mut warnings = Vec::new();if legacy_dir.exists() {diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.tsindex e5b3423e01..f707409f2d 100644--- a/web/packages/base/types/ipc.ts+++ b/web/packages/base/types/ipc.ts@@ -813,9 +813,10 @@ export interface MLWorkerAnalyzeImageRequest {* Broad category for an error encountered while analyzing an image.** Only `image` indicates a permanent problem inherent to the image. `init`- * blocks indexing for the lifetime of the utility process, while `ort` and- * `misc` remain retryable. None of those other kinds should permanently mark- * the file as unindexable.+ * blocks indexing for the lifetime of the utility process (this includes+ * corrupt on-disk models, which won't fix themselves and need manual+ * intervention), while `ort` and `misc` remain retryable. None of those other+ * kinds should permanently mark the file as unindexable.*/export type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";