π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Move retired mobile model cleanup into Rust migration
laurenspriemb08d895c3073
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dartindex 9457af7a51..ebbfb5dbf4 100644--- a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart+++ b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart@@ -6,7 +6,6 @@ import "package:path/path.dart" as p;import "package:path_provider/path_provider.dart";import "package:photos/service_locator.dart"show flagService, hasGrantedMLConsent, isLocalGalleryMode, localSettings;-import "package:photos/services/remote_assets_service.dart";import "package:photos/src/rust/api/ml_indexing_api.dart" as rust_ml;import "package:photos/utils/network_util.dart";import "package:synchronized/synchronized.dart";@@ -110,7 +109,6 @@ class MLModelDownloadService {: "Downloading all ML models",);try {- await RemoteAssetsService.instance.cleanupOldModelsIfNeeded();await for (final update in rust_ml.preloadMlModels(assetsDir: await getAssetsDirectory(),runFaces: downloadIndexing,diff --git a/mobile/apps/photos/lib/services/remote_assets_service.dart b/mobile/apps/photos/lib/services/remote_assets_service.dartindex 5f84e146ba..11c624b4d7 100644--- a/mobile/apps/photos/lib/services/remote_assets_service.dart+++ b/mobile/apps/photos/lib/services/remote_assets_service.dart@@ -20,7 +20,6 @@ class RemoteAssetsService {final StreamController<(String, int, int)> _progressController =StreamController<(String, int, int)>.broadcast();final Map<String, Lock> _assetLocks = {};- Future<void>? _oldModelsCleanupFuture;Stream<(String, int, int)> get progressStream => _progressController.stream;@@ -68,7 +67,6 @@ class RemoteAssetsService {bool refetch = false,String? expectedSha256,}) async {- await cleanupOldModelsIfNeeded();final file = await getAsset(remotePath,refetch: refetch,@@ -198,51 +196,6 @@ class RemoteAssetsService {);}- Future<void> cleanupOldModelsIfNeeded() =>- _oldModelsCleanupFuture ??= _cleanupOldModels();-- Future<void> _cleanupOldModels() async {- try {- const oldModelNames = [- "https://models.ente.io/clip-image-vit-32-float32.onnx",- "https://models.ente.io/clip-text-vit-32-uint8.onnx",- "https://models.ente.io/mobileclip_s2_image_opset18_rgba_sim.onnx",- "https://models.ente.io/mobileclip_s2_image_opset18_rgba_opt.onnx",- "https://models.ente.io/mobileclip_s2_text_int32.onnx",- "https://models.ente.io/yolov5s_face_opset18_rgba_opt.onnx",- "https://models.ente.io/yolov5s_face_opset18_rgba_opt_nosplits.onnx",- "https://models.ente.io/yolov5s_face_640_640_dynamic.onnx",- "https://models.ente.io/mobilefacenet_opset15.onnx",- "https://models.ente.com/yolov5s_face_640_640_dynamic.onnx",- "https://models.ente.com/mobilefacenet_opset15.onnx",- "https://models.ente.io/mobileclip_s2_image.onnx",- "https://models.ente.com/mobileclip_s2_image.onnx",- ];-- await cleanupSelectedModels(oldModelNames);- _logger.info("Old ML models cleaned up");- } catch (_) {- _oldModelsCleanupFuture = null;- rethrow;- }- }-- Future<void> cleanupSelectedModels(List<String> modelRemotePaths) async {- for (final remotePath in modelRemotePaths) {- final localPath = await _getLocalPath(remotePath);- final hasArtifacts =- await File(localPath).exists() ||- await File(_tempPath(localPath)).exists() ||- await File(_resumeMetadataPath(_tempPath(localPath))).exists();- if (hasArtifacts) {- _logger.info(- 'Removing unused ML model ${remotePath.split('/').last} at $localPath',- );- await _deleteAssetArtifacts(localPath);- }- }- }-Dio get _dio => NetworkClient.instance.downloadDio;bool get _resumableDownloadsEnabled => true;diff --git a/rust/crates/ml/src/assets.rs b/rust/crates/ml/src/assets.rsindex b4e6346009..14fc3e3626 100644--- a/rust/crates/ml/src/assets.rs+++ b/rust/crates/ml/src/assets.rs@@ -187,6 +187,13 @@ fn import_mobile_model(staging: &Path,files: &[ModelAssetFile],) -> Result<(), ente_assets::download::Error> {+ for name in RETIRED_LEGACY_MODEL_FILES {+ for path in legacy_mobile_paths(legacy_dir, name) {+ remove_file_if_exists(&path)?;+ remove_file_if_exists(&path.with_extension("temp"))?;+ remove_file_if_exists(&path.with_extension("temp.resume.json"))?;+ }+ }for file in files {for source in legacy_mobile_paths(legacy_dir, file.name) {remove_file_if_exists(&source.with_extension("temp"))?;@@ -495,7 +502,12 @@ mod tests {}}fs::write(root.path().join("unrelated.temp"), b"unrelated").unwrap();+ let retired = root+ .path()+ .join("models_ente_io_mobilefacenet_opset15_onnx");+ fs::write(&retired, b"retired").unwrap();acquire_mobile_test_pair(&store, root.path()).await.unwrap();+ assert!(!retired.exists());for (index, file) in MOBILE_TEST_FILES.iter().enumerate() {let path = store.file_path(&mobile_test_asset(), file.name).unwrap();assert_eq!(fs::read(&path).unwrap(), bytes[index]);@@ -535,6 +547,43 @@ mod tests {);}+ #[tokio::test]+ async fn mobile_cleanup_handles_retired_artifacts_without_current_legacy_models() {+ let root = TempDir::new().unwrap();+ let store = AssetStore::new(root.path());+ let retired = RETIRED_LEGACY_MODEL_FILES+ .iter()+ .flat_map(|name| legacy_mobile_paths(root.path(), name))+ .flat_map(|path| {+ [+ path.with_extension("temp"),+ path.with_extension("temp.resume.json"),+ path,+ ]+ })+ .collect::<Vec<_>>();+ for path in &retired {+ fs::write(path, b"retired").unwrap();+ }+ let preserved = [+ root.path().join("models_ente_com_memory_music_mp3"),+ legacy_mobile_paths(root.path(), "mobilefacenet_portable_static_b1.onnx")[0].clone(),+ root.path().join("models/det_fixed_v1/det_fixed_v1.onnx"),+ ];+ fs::create_dir_all(preserved[2].parent().unwrap()).unwrap();+ for path in &preserved {+ fs::write(path, b"preserved").unwrap();+ }++ for _ in 0..2 {+ assert!(acquire_mobile_test_pair(&store, root.path()).await.is_err());+ assert!(retired.iter().all(|path| !path.exists()));+ for path in &preserved {+ assert_eq!(fs::read(path).unwrap(), b"preserved");+ }+ }+ }+#[tokio::test]async fn either_lone_mobile_member_survives_failure_and_is_reused() {for first in [0, 1] {@@ -599,6 +648,10 @@ mod tests {async fn mobile_ensure_returns_only_the_requested_published_paths() {let root = TempDir::new().unwrap();let store = AssetStore::new(root.path());+ let retired = root+ .path()+ .join("models_ente_io_mobilefacenet_opset15_onnx");+ fs::write(&retired, b"retired").unwrap();for model in models::selected_indexing_models(true, false, false) {fs::create_dir_all(store.asset_dir(&model_asset(model))).unwrap();fs::write(model_path(&store, model), b"trusted published bytes").unwrap();@@ -617,6 +670,7 @@ mod tests {assert!(paths.clip_image.is_empty());assert!(paths.pet_face_detection.is_empty());assert!(!store.asset_dir(&clip_text_asset()).exists());+ assert_eq!(fs::read(retired).unwrap(), b"retired");}#[test]