Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Move retired mobile model cleanup into Rust migration

laurenspriemb08d895c3073
diff --git a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart
index 9457af7a51..ebbfb5dbf4 100644
--- a/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart
+++ b/mobile/apps/photos/lib/services/machine_learning/ml_model_download_service.dart
@@ -6,7 +6,6 @@ import "package:path/path.dart" as p;
import "package:path_provider/path_provider.dart";
import "package:photos/service_locator.dart"
show flagService, hasGrantedMLConsent, isLocalGalleryMode, localSettings;
-import "package:photos/services/remote_assets_service.dart";
import "package:photos/src/rust/api/ml_indexing_api.dart" as rust_ml;
import "package:photos/utils/network_util.dart";
import "package:synchronized/synchronized.dart";
@@ -110,7 +109,6 @@ class MLModelDownloadService {
: "Downloading all ML models",
);
try {
- await RemoteAssetsService.instance.cleanupOldModelsIfNeeded();
await for (final update in rust_ml.preloadMlModels(
assetsDir: await getAssetsDirectory(),
runFaces: downloadIndexing,
diff --git a/mobile/apps/photos/lib/services/remote_assets_service.dart b/mobile/apps/photos/lib/services/remote_assets_service.dart
index 5f84e146ba..11c624b4d7 100644
--- a/mobile/apps/photos/lib/services/remote_assets_service.dart
+++ b/mobile/apps/photos/lib/services/remote_assets_service.dart
@@ -20,7 +20,6 @@ class RemoteAssetsService {
final StreamController<(String, int, int)> _progressController =
StreamController<(String, int, int)>.broadcast();
final Map<String, Lock> _assetLocks = {};
- Future<void>? _oldModelsCleanupFuture;
Stream<(String, int, int)> get progressStream => _progressController.stream;
@@ -68,7 +67,6 @@ class RemoteAssetsService {
bool refetch = false,
String? expectedSha256,
}) async {
- await cleanupOldModelsIfNeeded();
final file = await getAsset(
remotePath,
refetch: refetch,
@@ -198,51 +196,6 @@ class RemoteAssetsService {
);
}
- Future<void> cleanupOldModelsIfNeeded() =>
- _oldModelsCleanupFuture ??= _cleanupOldModels();
-
- Future<void> _cleanupOldModels() async {
- try {
- const oldModelNames = [
- "https://models.ente.io/clip-image-vit-32-float32.onnx",
- "https://models.ente.io/clip-text-vit-32-uint8.onnx",
- "https://models.ente.io/mobileclip_s2_image_opset18_rgba_sim.onnx",
- "https://models.ente.io/mobileclip_s2_image_opset18_rgba_opt.onnx",
- "https://models.ente.io/mobileclip_s2_text_int32.onnx",
- "https://models.ente.io/yolov5s_face_opset18_rgba_opt.onnx",
- "https://models.ente.io/yolov5s_face_opset18_rgba_opt_nosplits.onnx",
- "https://models.ente.io/yolov5s_face_640_640_dynamic.onnx",
- "https://models.ente.io/mobilefacenet_opset15.onnx",
- "https://models.ente.com/yolov5s_face_640_640_dynamic.onnx",
- "https://models.ente.com/mobilefacenet_opset15.onnx",
- "https://models.ente.io/mobileclip_s2_image.onnx",
- "https://models.ente.com/mobileclip_s2_image.onnx",
- ];
-
- await cleanupSelectedModels(oldModelNames);
- _logger.info("Old ML models cleaned up");
- } catch (_) {
- _oldModelsCleanupFuture = null;
- rethrow;
- }
- }
-
- Future<void> cleanupSelectedModels(List<String> modelRemotePaths) async {
- for (final remotePath in modelRemotePaths) {
- final localPath = await _getLocalPath(remotePath);
- final hasArtifacts =
- await File(localPath).exists() ||
- await File(_tempPath(localPath)).exists() ||
- await File(_resumeMetadataPath(_tempPath(localPath))).exists();
- if (hasArtifacts) {
- _logger.info(
- 'Removing unused ML model ${remotePath.split('/').last} at $localPath',
- );
- await _deleteAssetArtifacts(localPath);
- }
- }
- }
-
Dio get _dio => NetworkClient.instance.downloadDio;
bool get _resumableDownloadsEnabled => true;
diff --git a/rust/crates/ml/src/assets.rs b/rust/crates/ml/src/assets.rs
index b4e6346009..14fc3e3626 100644
--- a/rust/crates/ml/src/assets.rs
+++ b/rust/crates/ml/src/assets.rs
@@ -187,6 +187,13 @@ fn import_mobile_model(
staging: &Path,
files: &[ModelAssetFile],
) -> Result<(), ente_assets::download::Error> {
+ for name in RETIRED_LEGACY_MODEL_FILES {
+ for path in legacy_mobile_paths(legacy_dir, name) {
+ remove_file_if_exists(&path)?;
+ remove_file_if_exists(&path.with_extension("temp"))?;
+ remove_file_if_exists(&path.with_extension("temp.resume.json"))?;
+ }
+ }
for file in files {
for source in legacy_mobile_paths(legacy_dir, file.name) {
remove_file_if_exists(&source.with_extension("temp"))?;
@@ -495,7 +502,12 @@ mod tests {
}
}
fs::write(root.path().join("unrelated.temp"), b"unrelated").unwrap();
+ let retired = root
+ .path()
+ .join("models_ente_io_mobilefacenet_opset15_onnx");
+ fs::write(&retired, b"retired").unwrap();
acquire_mobile_test_pair(&store, root.path()).await.unwrap();
+ assert!(!retired.exists());
for (index, file) in MOBILE_TEST_FILES.iter().enumerate() {
let path = store.file_path(&mobile_test_asset(), file.name).unwrap();
assert_eq!(fs::read(&path).unwrap(), bytes[index]);
@@ -535,6 +547,43 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn mobile_cleanup_handles_retired_artifacts_without_current_legacy_models() {
+ let root = TempDir::new().unwrap();
+ let store = AssetStore::new(root.path());
+ let retired = RETIRED_LEGACY_MODEL_FILES
+ .iter()
+ .flat_map(|name| legacy_mobile_paths(root.path(), name))
+ .flat_map(|path| {
+ [
+ path.with_extension("temp"),
+ path.with_extension("temp.resume.json"),
+ path,
+ ]
+ })
+ .collect::<Vec<_>>();
+ for path in &retired {
+ fs::write(path, b"retired").unwrap();
+ }
+ let preserved = [
+ root.path().join("models_ente_com_memory_music_mp3"),
+ legacy_mobile_paths(root.path(), "mobilefacenet_portable_static_b1.onnx")[0].clone(),
+ root.path().join("models/det_fixed_v1/det_fixed_v1.onnx"),
+ ];
+ fs::create_dir_all(preserved[2].parent().unwrap()).unwrap();
+ for path in &preserved {
+ fs::write(path, b"preserved").unwrap();
+ }
+
+ for _ in 0..2 {
+ assert!(acquire_mobile_test_pair(&store, root.path()).await.is_err());
+ assert!(retired.iter().all(|path| !path.exists()));
+ for path in &preserved {
+ assert_eq!(fs::read(path).unwrap(), b"preserved");
+ }
+ }
+ }
+
#[tokio::test]
async fn either_lone_mobile_member_survives_failure_and_is_reused() {
for first in [0, 1] {
@@ -599,6 +648,10 @@ mod tests {
async fn mobile_ensure_returns_only_the_requested_published_paths() {
let root = TempDir::new().unwrap();
let store = AssetStore::new(root.path());
+ let retired = root
+ .path()
+ .join("models_ente_io_mobilefacenet_opset15_onnx");
+ fs::write(&retired, b"retired").unwrap();
for model in models::selected_indexing_models(true, false, false) {
fs::create_dir_all(store.asset_dir(&model_asset(model))).unwrap();
fs::write(model_path(&store, model), b"trusted published bytes").unwrap();
@@ -617,6 +670,7 @@ mod tests {
assert!(paths.clip_image.is_empty());
assert!(paths.pet_face_detection.is_empty());
assert!(!store.asset_dir(&clip_text_asset()).exists());
+ assert_eq!(fs::read(retired).unwrap(), b"retired");
}
#[test]