Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Suspend ML indexing on corrupt model instead of evicting and redownloading

laurenspriem08c98b583edb
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts
index 2b57fc3d65..8289d11263 100644
--- a/desktop/src/main/services/ml-worker.ts
+++ b/desktop/src/main/services/ml-worker.ts
@@ -238,9 +238,6 @@ export interface MLWorkerAnalyzeImageRequest {
/**
* Analyze a single image with the Rust ML pipeline, downloading any needed
* models first.
- *
- * If the Rust side reports the on-disk model as corrupt, delete and
- * redownload it, and retry once.
*/
export const analyzeImage = async (
req: MLWorkerAnalyzeImageRequest,
@@ -256,29 +253,29 @@ const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {
const native = mlNative();
try {
return await analyzeImageOnce(native, req);
- } catch (e) {
- if (!evictModelIfReportedCorrupt(e)) throw e;
- _preparedRuntimeKey = undefined;
- return await analyzeImageOnce(native, req);
} finally {
logMLRuntimeEvents(native);
}
};
+// A corrupt on-disk model won't fix itself (models are hash-verified on
+// download), so treat it like an init failure: suspend indexing for the rest
+// of the process lifetime instead of retrying.
const categorizeAnalyzeImageError = (e: unknown): MLWorkerAnalyzeImageError => {
if (e instanceof CategorizedMLWorkerError) {
return { kind: e.kind, message: e.message };
}
const message = e instanceof Error ? e.message : String(e);
- const kind: MLWorkerAnalyzeImageErrorKind =
- message.startsWith("Decode: ") || message.startsWith("Image: ")
- ? "image"
- : message.startsWith("Ort: ") ||
- message.startsWith("CorruptModel: ") ||
- message.startsWith("Runtime: ")
- ? "ort"
- : "misc";
+ const kind: MLWorkerAnalyzeImageErrorKind = message.startsWith(
+ "CorruptModel: ",
+ )
+ ? "init"
+ : message.startsWith("Decode: ") || message.startsWith("Image: ")
+ ? "image"
+ : message.startsWith("Ort: ") || message.startsWith("Runtime: ")
+ ? "ort"
+ : "misc";
return { kind, message };
};
@@ -312,32 +309,6 @@ const analyzeImageOnce = async (
});
};
-/**
- * If {@link e} is the addon's tagged "CorruptModel" error, return the path of
- * the corrupt model file it reports.
- */
-const corruptModelPathFromError = (e: unknown) => {
- if (!(e instanceof Error)) return undefined;
- if (!e.message.startsWith("CorruptModel: ")) return undefined;
- return e.message.slice("CorruptModel: ".length).trim();
-};
-
-/**
- * If {@link e} is the addon's tagged "CorruptModel" error, evict the model it
- * names so that its next use downloads a fresh copy, and return `true`.
- */
-const evictModelIfReportedCorrupt = (e: unknown) => {
- const corruptModelPath = corruptModelPathFromError(e);
- if (!corruptModelPath) return false;
- if (!assetStore().removeModel(corruptModelPath)) return false;
- _indexingModelPaths.clear();
- log.error(
- `Rust ML reported a corrupt model at ${corruptModelPath}, evicting it so that it gets redownloaded`,
- );
- _clipTextModelPaths = undefined;
- return true;
-};
-
// Zero-copy view over the transferred bytes.
const uint8ArrayToBuffer = (bytes: Uint8Array) =>
Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
@@ -368,8 +339,7 @@ const warmUpClipTextEncoder = async (native: MLNative) => {
vocabPath,
});
} catch (e) {
- if (!evictModelIfReportedCorrupt(e))
- log.warn("Failed to warm up the CLIP text encoder", e);
+ log.warn("Failed to warm up the CLIP text encoder", e);
} finally {
logMLRuntimeEvents(native);
}
@@ -409,11 +379,6 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => {
vocabPath,
});
return embedding;
- } catch (e) {
- // Don't block this query on the redownload; a subsequent query will
- // pick up the fresh copy.
- if (!evictModelIfReportedCorrupt(e)) throw e;
- return undefined;
} finally {
logMLRuntimeEvents(native);
}
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
index bb10a77959..326c92618b 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_computer.dart
@@ -8,7 +8,6 @@ import "package:photos/core/errors.dart";
import "package:photos/models/ml/vector.dart";
import "package:photos/services/machine_learning/ml_constants.dart";
import "package:photos/services/machine_learning/ml_model_assets.dart";
-import "package:photos/services/machine_learning/ml_model_download_service.dart";
import "package:photos/services/machine_learning/semantic_search/query_result.dart";
import "package:photos/services/machine_learning/webgpu_execution_policy.dart";
import "package:photos/services/remote_assets_service.dart";
@@ -90,11 +89,7 @@ class MLComputer extends SuperIsolate {
"clipTextVocabPath": vocabPath,
"enableWebGpu": enableWebGpu,
});
- if (isolateResult is RustCorruptModelCacheDeletedException) {
- _clipTextModelPath = null;
- MLModelDownloadService.instance.invalidateModelDownloadCache(
- includeNonIndexingModels: true,
- );
+ if (isolateResult is RustCorruptModelException) {
throw isolateResult;
}
final textEmbedding = isolateResult as List<double>;
@@ -106,10 +101,8 @@ class MLComputer extends SuperIsolate {
s,
);
rethrow;
- } on RustCorruptModelCacheDeletedException catch (e) {
- _logger.warning(
- "Deleted corrupt Rust CLIP text model cache at ${e.modelPath}",
- );
+ } on RustCorruptModelException catch (e) {
+ _logger.severe("Rust ML reported a corrupt model at ${e.modelPath}");
rethrow;
} catch (e, s) {
_logger.severe("Could not run clip text in isolate", e, s);
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
index 38aae475dc..39b5d2938f 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_indexing_isolate.dart
@@ -63,9 +63,9 @@ class MLIndexingIsolate extends SuperIsolate {
...rustRuntimeArgs,
"enableWebGpu": enableWebGpu,
});
- if (isolateResult is RustCorruptModelCacheDeletedException) {
- _logger.warning(
- "Deleted corrupt Rust ONNX model cache at ${isolateResult.modelPath}; "
+ if (isolateResult is RustCorruptModelException) {
+ _logger.severe(
+ "Rust ML reported a corrupt model at ${isolateResult.modelPath}; "
"stopping ML indexing for fileID ${instruction.fileKey}",
);
shouldPauseIndexingAndClustering = true;
@@ -82,8 +82,7 @@ class MLIndexingIsolate extends SuperIsolate {
_runtimeFlagCombinations.add(result.remoteFlags);
return result;
} catch (e, s) {
- if (e is RustCorruptModelCacheDeletedException ||
- isExpectedMlSkipError(e)) {
+ if (e is RustCorruptModelException || isExpectedMlSkipError(e)) {
rethrow;
}
_logger.severe(
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
index 0fbcc27108..95cc9fe364 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_service.dart
@@ -537,9 +537,9 @@ class MLService {
"`indexAllImages()` finished. Analyzed $fileAnalyzedCount images, in ${stopwatch.elapsed.inSeconds} seconds (avg of ${stopwatch.elapsed.inSeconds / fileAnalyzedCount} seconds per image)",
);
_logStatus();
- } on RustCorruptModelCacheDeletedException catch (e) {
- _logger.warning(
- "Stopping image indexing because corrupt Rust ONNX model cache was deleted at ${e.modelPath}",
+ } on RustCorruptModelException catch (e) {
+ _logger.severe(
+ "Stopping image indexing because Rust ML reported a corrupt model at ${e.modelPath}",
);
} catch (e, s) {
_logger.severe("indexAllImages failed", e, s);
@@ -917,12 +917,12 @@ class MLService {
final String format = instruction.file.displayName.split('.').last;
final int? size = instruction.file.fileSize;
final fileType = instruction.file.fileType;
- if (e is RustCorruptModelCacheDeletedException) {
+ if (e is RustCorruptModelException) {
pauseIndexingAndClustering();
- _logger.warning(
+ _logger.severe(
"Stopping ML indexing for fileID ${instruction.fileKey} "
- "(format $format, type $fileType, size $size) because corrupt Rust "
- "ONNX model cache was deleted at ${e.modelPath}",
+ "(format $format, type $fileType, size $size) because Rust ML "
+ "reported a corrupt model at ${e.modelPath}",
);
rethrow;
}
diff --git a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
index efd8d62488..b8dad1741a 100644
--- a/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
+++ b/mobile/apps/photos/lib/utils/isolate/isolate_operations.dart
@@ -1,4 +1,3 @@
-import 'dart:io' show File;
import 'dart:typed_data' show Float32List, Uint8List;
import "package:flutter_rust_bridge/flutter_rust_bridge.dart" show Uint64List;
@@ -25,13 +24,13 @@ const _rustLibLoadedCacheKey = "rustLibLoaded";
const _rustMlModelPathsCacheKey = "rustMlModelPaths";
final _rustMlRuntimeLogger = Logger("RustMLRuntime");
-class RustCorruptModelCacheDeletedException implements Exception {
- const RustCorruptModelCacheDeletedException(this.modelPath);
+class RustCorruptModelException implements Exception {
+ const RustCorruptModelException(this.modelPath);
final String modelPath;
@override
- String toString() => "RustCorruptModelCacheDeletedException: $modelPath";
+ String toString() => "RustCorruptModelException: $modelPath";
}
enum IsolateOperation {
@@ -118,11 +117,7 @@ Future<dynamic> isolateFunction(
try {
result = await analyzeImageRust(args);
} on rust_ml.RustMlError_CorruptModel catch (e) {
- final file = File(e.field0);
- if (await file.exists()) {
- await file.delete();
- }
- return RustCorruptModelCacheDeletedException(e.field0);
+ return RustCorruptModelException(e.field0);
}
return result.toJsonString();
@@ -206,11 +201,7 @@ Future<dynamic> isolateFunction(
),
);
} on rust_ml.RustMlError_CorruptModel catch (e) {
- final file = File(e.field0);
- if (await file.exists()) {
- await file.delete();
- }
- return RustCorruptModelCacheDeletedException(e.field0);
+ return RustCorruptModelException(e.field0);
}
return List<double>.from(result.embedding, growable: false);
} finally {
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rs
index bedbc25caf..8bde511f40 100644
--- a/rust/bindings/napi/photos/src/lib.rs
+++ b/rust/bindings/napi/photos/src/lib.rs
@@ -116,12 +116,6 @@ impl AssetStore {
vocab_path: paths.vocab.to_string_lossy().into_owned(),
})
}
-
- #[napi]
- pub fn remove_model(&self, model_path: String) -> Result<bool> {
- assets::remove_model_at_path(&self.inner, std::path::Path::new(&model_path))
- .map_err(|error| Error::from_reason(format!("Assets: {error}")))
- }
}
#[napi(object)]
diff --git a/rust/crates/photos/src/ml/assets.rs b/rust/crates/photos/src/ml/assets.rs
index 05a841c08d..3e2171ecc5 100644
--- a/rust/crates/photos/src/ml/assets.rs
+++ b/rust/crates/photos/src/ml/assets.rs
@@ -1,7 +1,7 @@
use std::fs;
use std::path::{Path, PathBuf};
-use ente_assets::{Asset, AssetFile, AssetStore, AssetStoreError};
+use ente_assets::{Asset, AssetFile, AssetStore};
use super::runtime::ModelPaths;
@@ -124,23 +124,6 @@ pub fn clip_text_paths(store: &AssetStore) -> ClipTextPaths {
}
}
-pub fn remove_model_at_path(store: &AssetStore, path: &Path) -> Result<bool, AssetStoreError> {
- for model in ALL_MODELS {
- let asset = model_asset(model);
- if model
- .spec()
- .files
- .iter()
- .filter_map(|file| store.file_path(&asset, file.name))
- .any(|candidate| candidate == path)
- {
- store.remove(&asset)?;
- return Ok(true);
- }
- }
- Ok(false)
-}
-
pub fn migrate_desktop_models(store: &AssetStore, legacy_dir: &Path) -> Vec<String> {
let mut warnings = Vec::new();
if legacy_dir.exists() {
diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.ts
index e5b3423e01..f707409f2d 100644
--- a/web/packages/base/types/ipc.ts
+++ b/web/packages/base/types/ipc.ts
@@ -813,9 +813,10 @@ export interface MLWorkerAnalyzeImageRequest {
* Broad category for an error encountered while analyzing an image.
*
* Only `image` indicates a permanent problem inherent to the image. `init`
- * blocks indexing for the lifetime of the utility process, while `ort` and
- * `misc` remain retryable. None of those other kinds should permanently mark
- * the file as unindexable.
+ * blocks indexing for the lifetime of the utility process (this includes
+ * corrupt on-disk models, which won't fix themselves and need manual
+ * intervention), while `ort` and `misc` remain retryable. None of those other
+ * kinds should permanently mark the file as unindexable.
*/
export type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";