π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Fix deleted-share tombstone synchronization
Neeraj Gupta211a6154b6fa
diff --git a/cli/pkg/mapper/photo.go b/cli/pkg/mapper/photo.goindex 76cb8a398b..40a06ab7bb 100644--- a/cli/pkg/mapper/photo.go+++ b/cli/pkg/mapper/photo.go@@ -5,23 +5,24 @@ import ("encoding/json""errors""fmt"+ "log"+"github.com/ente/cli/internal/api"eCrypto "github.com/ente/cli/internal/crypto""github.com/ente/cli/pkg/model""github.com/ente/cli/pkg/model/export""github.com/ente/cli/pkg/secrets""github.com/ente/cli/utils/encoding"- "log")func MapCollectionToAlbum(ctx context.Context, collection api.Collection, holder *secrets.KeyHolder) (*model.RemoteAlbum, error) {- var album model.RemoteAlbumuserID := ctx.Value("user_id").(int64)- album.OwnerID = collection.Owner.ID- album.ID = collection.ID- album.IsShared = collection.Owner.ID != userID- album.LastUpdatedAt = collection.UpdationTime- album.IsDeleted = collection.IsDeleted+ album := model.RemoteAlbum{+ ID: collection.ID,+ OwnerID: collection.Owner.ID,+ IsShared: collection.Owner.ID != userID,+ LastUpdatedAt: collection.UpdationTime,+ }collectionKey, err := holder.GetCollectionKey(ctx, collection)if err != nil {return nil, errdiff --git a/cli/pkg/remote_sync.go b/cli/pkg/remote_sync.goindex 9656ae40af..a6d4ef93c6 100644--- a/cli/pkg/remote_sync.go+++ b/cli/pkg/remote_sync.go@@ -4,12 +4,13 @@ import ("context""encoding/json""fmt"- "github.com/ente/cli/pkg/mapper"- "github.com/ente/cli/pkg/model"- "github.com/ente/cli/utils/encoding""log""strconv""time"++ "github.com/ente/cli/pkg/mapper"+ "github.com/ente/cli/pkg/model"+ "github.com/ente/cli/utils/encoding")func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {@@ -23,16 +24,31 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {}maxUpdated := lastSyncTimefor _, collection := range collections {- if lastSyncTime == 0 && collection.IsDeleted {- continue- }- album, mapErr := mapper.MapCollectionToAlbum(ctx, collection, c.KeyHolder)- if mapErr != nil {- return mapErr+ if collection.UpdationTime > maxUpdated {+ maxUpdated = collection.UpdationTime}- if album.LastUpdatedAt > maxUpdated {- maxUpdated = album.LastUpdatedAt++ var album *model.RemoteAlbum+ if collection.IsDeleted {+ key := []byte(strconv.FormatInt(collection.ID, 10))+ existing, err := c.GetValue(ctx, model.RemoteAlbums, key)+ if err != nil {+ return err+ }+ album, err = markRemoteAlbumDeleted(existing, collection.ID, collection.UpdationTime)+ if err != nil {+ return err+ }+ if album == nil {+ continue+ }+ } else {+ album, err = mapper.MapCollectionToAlbum(ctx, collection, c.KeyHolder)+ if err != nil {+ return err+ }}+albumJson := encoding.MustMarshalJSON(album)putErr := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson)if putErr != nil {@@ -48,6 +64,22 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {return nil}+func markRemoteAlbumDeleted(existing []byte, collectionID, updationTime int64) (*model.RemoteAlbum, error) {+ if len(existing) == 0 {+ return nil, nil+ }+ var album model.RemoteAlbum+ if err := json.Unmarshal(existing, &album); err != nil {+ return nil, err+ }+ if album.ID != collectionID {+ return nil, fmt.Errorf("stored album ID %d does not match tombstone ID %d", album.ID, collectionID)+ }+ album.IsDeleted = true+ album.LastUpdatedAt = updationTime+ return &album, nil+}+func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error {albums, err := c.getRemoteAlbums(ctx)filter := ctx.Value(model.FilterKey).(model.Filter)diff --git a/cli/pkg/remote_sync_test.go b/cli/pkg/remote_sync_test.gonew file mode 100644index 0000000000..fd75fa9744--- /dev/null+++ b/cli/pkg/remote_sync_test.go@@ -0,0 +1,99 @@+package pkg++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strconv"+ "testing"++ "github.com/ente/cli/internal/api"+ "github.com/ente/cli/pkg/model"+)++func TestFetchRemoteCollectionsHandlesDeletedCollectionsWithoutDecrypting(t *testing.T) {+ var sinceTime string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/collections/v2" {+ http.NotFound(w, r)+ return+ }+ sinceTime = r.URL.Query().Get("sinceTime")+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"collections":[{"id":7,"owner":{"id":1},"updationTime":9,"isDeleted":true},{"id":8,"owner":{"id":1},"updationTime":11,"isDeleted":true}]}`))+ }))+ defer server.Close()++ db, err := GetDB(t.TempDir() + "/cli.db")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = db.Close() })++ account := model.Account{UserID: 2, App: api.AppPhotos}+ if err := createDataBuckets(db, account); err != nil {+ t.Fatal(err)+ }+ ctx := context.WithValue(context.Background(), "app", string(account.App))+ ctx = context.WithValue(ctx, "account_key", account.AccountKey())+ ctx = context.WithValue(ctx, "user_id", account.UserID)++ ctrl := ClICtrl{+ Client: api.NewClient(api.Params{Host: server.URL}),+ DB: db,+ // A nil holder proves tombstones never enter key or metadata decryption.+ KeyHolder: nil,+ }+ existing := model.RemoteAlbum{+ ID: 7,+ OwnerID: 1,+ IsShared: true,+ AlbumName: "Vacation",+ AlbumKey: model.EncString{+ CipherText: "local-key",+ Nonce: "local-nonce",+ },+ PrivateMeta: map[string]interface{}{"visibility": float64(1)},+ }+ existingJSON, err := json.Marshal(&existing)+ if err != nil {+ t.Fatal(err)+ }+ if err := ctrl.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(existing.ID, 10)), existingJSON); err != nil {+ t.Fatal(err)+ }+ if err := ctrl.PutConfigValue(ctx, model.CollectionsSyncKey, []byte("5")); err != nil {+ t.Fatal(err)+ }++ if err := ctrl.fetchRemoteCollections(ctx); err != nil {+ t.Fatal(err)+ }+ if sinceTime != "5" {+ t.Fatalf("sinceTime = %q, want 5", sinceTime)+ }++ stored, err := ctrl.GetValue(ctx, model.RemoteAlbums, []byte("7"))+ if err != nil {+ t.Fatal(err)+ }+ var album model.RemoteAlbum+ if err := json.Unmarshal(stored, &album); err != nil {+ t.Fatal(err)+ }+ if !album.IsDeleted || album.LastUpdatedAt != 9 || album.AlbumName != existing.AlbumName || album.AlbumKey != existing.AlbumKey || album.PrivateMeta["visibility"] != float64(1) {+ t.Fatalf("deleted album lost local export state: %+v", album)+ }+ unknown, err := ctrl.GetValue(ctx, model.RemoteAlbums, []byte("8"))+ if err != nil {+ t.Fatal(err)+ }+ if unknown != nil {+ t.Fatalf("unknown tombstone created album: %s", unknown)+ }+ cursor, err := ctrl.GetInt64ConfigValue(ctx, model.CollectionsSyncKey)+ if err != nil || cursor != 11 {+ t.Fatalf("cursor = %d, %v; want 11", cursor, err)+ }+}diff --git a/mobile/apps/locker/lib/services/collections/collections_api_client.dart b/mobile/apps/locker/lib/services/collections/collections_api_client.dartindex abf23f7537..d4c41b861f 100644--- a/mobile/apps/locker/lib/services/collections/collections_api_client.dart+++ b/mobile/apps/locker/lib/services/collections/collections_api_client.dart@@ -13,6 +13,7 @@ import 'package:locker/core/errors.dart';import "package:locker/events/collections_updated_event.dart";import "package:locker/services/collections/collections_service.dart";import 'package:locker/services/collections/models/collection.dart';+import 'package:locker/services/collections/models/collection_change.dart';import 'package:locker/services/collections/models/collection_file_item.dart';import 'package:locker/services/collections/models/collection_magic.dart';import 'package:locker/services/collections/models/diff.dart';@@ -42,21 +43,29 @@ class CollectionApiClient {_db = LockerDB.instance;}- Future<List<Collection>> getCollections(int sinceTime) async {+ Future<List<CollectionChange>> getCollections(int sinceTime) async {try {final response = await _enteDio.get("/collections/v2",queryParameters: {"sinceTime": sinceTime},);- final List<Collection> collections = [];+ final List<CollectionChange> changes = [];final c = response.data["collections"];for (final collectionData in c) {- final Collection collection = await _fromRemoteCollection(- collectionData,- );- collections.add(collection);+ if (isSharedCollectionDeletion(collectionData, _config.getUserID()!)) {+ changes.add(+ CollectionDeletion(+ collectionData['id'] as int,+ collectionData['updationTime'] as int,+ ),+ );+ } else {+ changes.add(+ CollectionUpdate(await _fromRemoteCollection(collectionData)),+ );+ }}- return collections;+ return changes;} catch (e, s) {_logger.warning(e, s);if (e is DioException && e.response?.statusCode == 401) {@@ -224,7 +233,7 @@ class CollectionApiClient {}Future<void> _handleCollectionDeletion(Collection collection) async {- await _db.deleteCollection(collection);+ await _db.deleteCollection(collection.id);final deletedCollection = collection.copyWith(isDeleted: true);await _updateCollectionInDB(deletedCollection);}@@ -305,7 +314,7 @@ class CollectionApiClient {"&collectionID=${collection.id}",);if (skipEventFiring) {- await _db.deleteCollection(collection);+ await _db.deleteCollection(collection.id);final deletedCollection = collection.copyWith(isDeleted: true);await _updateCollectionInDB(deletedCollection);} else {diff --git a/mobile/apps/locker/lib/services/collections/collections_service.dart b/mobile/apps/locker/lib/services/collections/collections_service.dartindex eceee2cd47..24f0287c31 100644--- a/mobile/apps/locker/lib/services/collections/collections_service.dart+++ b/mobile/apps/locker/lib/services/collections/collections_service.dart@@ -17,6 +17,7 @@ import 'package:locker/events/collections_updated_event.dart';import 'package:locker/events/user_details_refresh_event.dart';import "package:locker/services/collections/collections_api_client.dart";import 'package:locker/services/collections/models/collection.dart';+import 'package:locker/services/collections/models/collection_change.dart';import "package:locker/services/collections/models/files_split.dart";import "package:locker/services/collections/models/public_url.dart";import 'package:locker/services/configuration.dart';@@ -79,9 +80,10 @@ class CollectionService {final previousSyncTime = _db.getSyncTime();final shouldCheckFirstSyncCompletion = previousSyncTime == 0;- final updatedCollections = await CollectionApiClient.instance- .getCollections(previousSyncTime);- if (updatedCollections.isEmpty) {+ final collectionChanges = await CollectionApiClient.instance.getCollections(+ previousSyncTime,+ );+ if (collectionChanges.isEmpty) {if (shouldCheckFirstSyncCompletion) {final didMarkFirstSync = await _setFirstSyncCompleted();if (didMarkFirstSync) {@@ -93,17 +95,29 @@ class CollectionService {_logger.info("No collections to sync.");return;}- await _db.updateCollections(updatedCollections);- for (final collection in updatedCollections) {+ final persistedCollections = <Collection>[];+ var latestUpdationTime = previousSyncTime;+ for (final change in collectionChanges) {+ latestUpdationTime = max(latestUpdationTime, change.updationTime);+ if (change is CollectionDeletion) {+ await _db.deleteCollection(change.id);+ _collectionIDToCollections.remove(change.id);+ continue;+ }+ final collection = (change as CollectionUpdate).collection;+ persistedCollections.add(collection);_collectionIDToCollections[collection.id] = collection;}- await _db.setSyncTime(updatedCollections.last.updationTime);+ if (persistedCollections.isNotEmpty) {+ await _db.updateCollections(persistedCollections);+ }+ await _db.setSyncTime(latestUpdationTime);if (shouldCheckFirstSyncCompletion) {await _setFirstSyncCompleted();}final List<Future<bool>> fileFutures = [];- for (final collection in updatedCollections) {+ for (final collection in persistedCollections) {if (collection.isDeleted) {continue;}@@ -143,7 +157,7 @@ class CollectionService {"Skipping offline stale cleanup because one or more collection syncs failed",);}- if (updatedCollections.isNotEmpty) {+ if (collectionChanges.isNotEmpty) {Bus.instance.fire(CollectionsUpdatedEvent('sync'));}}diff --git a/mobile/apps/locker/lib/services/collections/models/collection_change.dart b/mobile/apps/locker/lib/services/collections/models/collection_change.dartnew file mode 100644index 0000000000..0de9908331--- /dev/null+++ b/mobile/apps/locker/lib/services/collections/models/collection_change.dart@@ -0,0 +1,28 @@+import 'package:locker/services/collections/models/collection.dart';++sealed class CollectionChange {+ const CollectionChange(this.id, this.updationTime);++ final int id;+ final int updationTime;+}++final class CollectionUpdate extends CollectionChange {+ CollectionUpdate(this.collection)+ : super(collection.id, collection.updationTime);++ final Collection collection;+}++final class CollectionDeletion extends CollectionChange {+ const CollectionDeletion(super.id, super.updationTime);+}++bool isSharedCollectionDeletion(+ Map<String, dynamic> collectionData,+ int currentUserID,+) {+ final ownerID =+ (collectionData['owner'] as Map<String, dynamic>)['id'] as int;+ return collectionData['isDeleted'] == true && ownerID != currentUserID;+}diff --git a/mobile/apps/locker/lib/services/db/locker_db.dart b/mobile/apps/locker/lib/services/db/locker_db.dartindex 176b22c6a5..b1855202d7 100644--- a/mobile/apps/locker/lib/services/db/locker_db.dart+++ b/mobile/apps/locker/lib/services/db/locker_db.dart@@ -328,25 +328,21 @@ class LockerDB extends EnteBaseDatabase {await batch.commit();}- Future<void> deleteCollection(Collection collection) async {+ Future<void> deleteCollection(int collectionID) async {final batch = _db.batch();- batch.delete(- _collectionsTable,- where: 'id = ?',- whereArgs: [collection.id],- );+ batch.delete(_collectionsTable, where: 'id = ?', whereArgs: [collectionID]);batch.delete(_collectionFilesTable,where: 'collection_id = ?',- whereArgs: [collection.id],+ whereArgs: [collectionID],);final filesInCollection = await _db.query(_collectionFilesTable,where: 'collection_id = ?',- whereArgs: [collection.id],+ whereArgs: [collectionID],);for (final fileMap in filesInCollection) {@@ -354,7 +350,7 @@ class LockerDB extends EnteBaseDatabase {final otherCollections = await _db.query(_collectionFilesTable,where: 'uploaded_file_id = ? AND collection_id != ?',- whereArgs: [uploadedFileId, collection.id],+ whereArgs: [uploadedFileId, collectionID],);if (otherCollections.isEmpty) {@@ -366,7 +362,14 @@ class LockerDB extends EnteBaseDatabase {}}+ batch.delete(+ _syncTimesTable,+ where: 'key = ?',+ whereArgs: ['collection_sync_time_$collectionID'],+ );+await batch.commit();+ _collectionSyncTimesCache.remove(collectionID);}Future<List<Collection>> getCollections() async {diff --git a/mobile/apps/locker/test/services/collections/models/collection_change_test.dart b/mobile/apps/locker/test/services/collections/models/collection_change_test.dartnew file mode 100644index 0000000000..0fed451da1--- /dev/null+++ b/mobile/apps/locker/test/services/collections/models/collection_change_test.dart@@ -0,0 +1,59 @@+import 'package:flutter_test/flutter_test.dart';+import 'package:locker/services/collections/models/collection.dart';+import 'package:locker/services/collections/models/collection_change.dart';++void main() {+ group('isSharedCollectionDeletion', () {+ test('accepts sparse and legacy full foreign deletions', () {+ final sparse = {+ 'id': 7,+ 'owner': {'id': 1},+ 'updationTime': 9,+ 'isDeleted': true,+ };+ final legacyFull = {+ ...sparse,+ 'encryptedKey': 'compatibility-key',+ 'keyDecryptionNonce': 'nonce',+ 'type': 'album',+ 'attributes': <String, dynamic>{},+ 'name': 'Deleted Collection',+ };++ expect(isSharedCollectionDeletion(sparse, 2), isTrue);+ expect(isSharedCollectionDeletion(legacyFull, 2), isTrue);+ });++ test('keeps owned deletions and active collections as updates', () {+ final ownedDeletion = {+ 'id': 7,+ 'owner': {'id': 2, 'email': '[email protected]'},+ 'encryptedKey': 'owned-key',+ 'keyDecryptionNonce': 'owned-nonce',+ 'name': 'Deleted Collection',+ 'encryptedName': 'encrypted-name',+ 'nameDecryptionNonce': 'name-nonce',+ 'type': 'folder',+ 'attributes': <String, dynamic>{},+ 'sharees': <Map<String, dynamic>>[],+ 'publicURLs': <Map<String, dynamic>>[],+ 'updationTime': 9,+ 'isDeleted': true,+ };+ final active = {+ ...ownedDeletion,+ 'id': 8,+ 'owner': {'id': 1},+ 'isDeleted': false,+ };++ expect(isSharedCollectionDeletion(ownedDeletion, 2), isFalse);+ expect(isSharedCollectionDeletion(active, 2), isFalse);++ final collection = Collection.fromMap(ownedDeletion);+ expect(collection.encryptedKey, 'owned-key');+ expect(collection.keyDecryptionNonce, 'owned-nonce');+ expect(collection.isDeleted, isTrue);+ });+ });+}diff --git a/mobile/apps/photos/lib/gateways/collections/collections_gateway.dart b/mobile/apps/photos/lib/gateways/collections/collections_gateway.dartindex 1ea4f837ca..b705cb5a45 100644--- a/mobile/apps/photos/lib/gateways/collections/collections_gateway.dart+++ b/mobile/apps/photos/lib/gateways/collections/collections_gateway.dart@@ -1,4 +1,5 @@import "package:dio/dio.dart";+import "package:photos/gateways/collections/models/collection_change.dart";import "package:photos/gateways/collections/models/create_request.dart";import "package:photos/gateways/collections/models/metadata.dart";@@ -95,15 +96,24 @@ class CollectionsGateway {);}- Future<Map<String, dynamic>> getAll({+ Future<List<RemoteCollectionChange>> getAll({required int sinceTime,required String source,+ required int currentUserID,}) async {final response = await _enteDio.get("/collections/v2",queryParameters: {"sinceTime": sinceTime, "source": source},);- return response.data;+ final collections = response.data['collections'] as List<dynamic>;+ return collections+ .map(+ (data) => RemoteCollectionChange.fromMap(+ Map<String, dynamic>.from(data as Map<dynamic, dynamic>),+ currentUserID,+ ),+ )+ .toList(growable: false);}Future<Map<String, dynamic>> fetchPendingRemovalActions() async {diff --git a/mobile/apps/photos/lib/gateways/collections/models/collection_change.dart b/mobile/apps/photos/lib/gateways/collections/models/collection_change.dartnew file mode 100644index 0000000000..7076bf7910--- /dev/null+++ b/mobile/apps/photos/lib/gateways/collections/models/collection_change.dart@@ -0,0 +1,47 @@+sealed class RemoteCollectionChange {+ const RemoteCollectionChange();++ int get id;+ int get updationTime;++ factory RemoteCollectionChange.fromMap(+ Map<String, dynamic> map,+ int currentUserID,+ ) {+ final owner = map['owner'] as Map<String, dynamic>;+ if (map['isDeleted'] == true && owner['id'] as int != currentUserID) {+ return RemoteCollectionDeletion(+ id: map['id'] as int,+ ownerID: owner['id'] as int,+ updationTime: map['updationTime'] as int,+ );+ }+ return RemoteCollectionUpdate(map);+ }+}++final class RemoteCollectionUpdate extends RemoteCollectionChange {+ const RemoteCollectionUpdate(this.data);++ final Map<String, dynamic> data;++ @override+ int get id => data['id'] as int;++ @override+ int get updationTime => data['updationTime'] as int;+}++final class RemoteCollectionDeletion extends RemoteCollectionChange {+ const RemoteCollectionDeletion({+ required this.id,+ required this.ownerID,+ required this.updationTime,+ });++ @override+ final int id;+ final int ownerID;+ @override+ final int updationTime;+}diff --git a/mobile/apps/photos/lib/services/collections_service.dart b/mobile/apps/photos/lib/services/collections_service.dartindex 0b5aa2b847..79dfed5923 100644--- a/mobile/apps/photos/lib/services/collections_service.dart+++ b/mobile/apps/photos/lib/services/collections_service.dart@@ -26,6 +26,7 @@ import 'package:photos/events/files_updated_event.dart';import 'package:photos/events/force_reload_home_gallery_event.dart';import 'package:photos/events/local_photos_updated_event.dart';import "package:photos/gateways/collections/collection_share_gateway.dart";+import 'package:photos/gateways/collections/models/collection_change.dart';import 'package:photos/gateways/collections/models/collection_file_item.dart';import "package:photos/gateways/collections/models/collection_share.dart";import 'package:photos/gateways/collections/models/create_request.dart';@@ -142,23 +143,41 @@ class CollectionsService {_logger.info("[COLLECTIONS] Starting sync");- final fetchedCollections = await _fetchCollections(+ final collectionChanges = await _fetchCollectionChanges(lastCollectionUpdationTime,);_logger.info(- "[COLLECTIONS] Fetched ${fetchedCollections.length} collections from API",+ "[COLLECTIONS] Fetched ${collectionChanges.length} collection changes from API",);- watch.log("remote fetch collections ${fetchedCollections.length}");+ watch.log("remote fetch collections ${collectionChanges.length}");- if (fetchedCollections.isEmpty) {+ if (collectionChanges.isEmpty) {await _ensureUncategorizedCollectionExists();return;}final updatedCollections = <Collection>[];int maxUpdationTime = lastCollectionUpdationTime;- final ownerID = _config.getUserID();bool shouldFireDeleteEvent = false;- for (final collection in fetchedCollections) {+ for (final change in collectionChanges) {+ maxUpdationTime = max(maxUpdationTime, change.updationTime);+ if (change is RemoteCollectionDeletion) {+ await _filesDB.deleteCollection(change.id);+ await _cleanupSocialData(change.id);+ await setCollectionSyncTime(change.id, null);+ if (_collectionIDToCollections.containsKey(change.id)) {+ shouldFireDeleteEvent = true;+ }+ await _db.deleteCollection(change.id);+ _collectionIDToCollections.remove(change.id);+ _cachedKeys.remove(change.id);+ _localPathToCollectionID.removeWhere((_, id) => id == change.id);+ _clearPublicCollectionState(change.id);+ continue;+ }++ final collection = await _fromRemoteCollection(+ (change as RemoteCollectionUpdate).data,+ );if (collection.isDeleted) {await _filesDB.deleteCollection(collection.id);await _cleanupSocialData(collection.id);@@ -167,16 +186,9 @@ class CollectionsService {shouldFireDeleteEvent = true;}}- if (collection.isDeleted && ownerID != collection.owner.id) {- await _db.deleteCollection(collection.id);- } else {- // keep entry for deletedCollection as collectionKey may be used during- // trash file decryption- updatedCollections.add(collection);- }- maxUpdationTime = collection.updationTime > maxUpdationTime- ? collection.updationTime- : maxUpdationTime;+ // Keep owned deleted collections because their keys may be needed to+ // decrypt trashed files.+ updatedCollections.add(collection);}if (shouldFireDeleteEvent) {@@ -193,14 +205,14 @@ class CollectionsService {watch.logAndReset("till DB insertion ${updatedCollections.length}");_logger.info("[COLLECTIONS] Updated ${updatedCollections.length} in DB");- for (final collection in fetchedCollections) {+ for (final collection in updatedCollections) {_cacheLocalPathAndCollection(collection);}_logger.info("Collections synced");- watch.log("${fetchedCollections.length} collection cached refreshed ");+ watch.log("${updatedCollections.length} collection caches refreshed");- if (fetchedCollections.isNotEmpty) {+ if (collectionChanges.isNotEmpty) {Bus.instance.fire(ContactRelationshipsInvalidatedEvent());Bus.instance.fire(CollectionUpdatedEvent(@@ -1532,21 +1544,15 @@ class CollectionsService {}}- Future<List<Collection>> _fetchCollections(int sinceTime) async {+ Future<List<RemoteCollectionChange>> _fetchCollectionChanges(+ int sinceTime,+ ) async {try {- final response = await collectionsGateway.getAll(+ return await collectionsGateway.getAll(sinceTime: sinceTime,source: AppLifecycleService.instance.isForeground ? "fg" : "bg",+ currentUserID: _config.getUserID()!,);- final List<Collection> collections = [];- final c = response["collections"];- for (final collectionData in c) {- final Collection collection = await _fromRemoteCollection(- collectionData,- );- collections.add(collection);- }- return collections;} catch (e, s) {_logger.warning("Failed to fetch collections", e, s);if (e is DioException && e.response?.statusCode == 401) {diff --git a/mobile/apps/photos/test/gateways/collections/models/collection_change_test.dart b/mobile/apps/photos/test/gateways/collections/models/collection_change_test.dartnew file mode 100644index 0000000000..cd4e518d1f--- /dev/null+++ b/mobile/apps/photos/test/gateways/collections/models/collection_change_test.dart@@ -0,0 +1,84 @@+import 'package:flutter_test/flutter_test.dart';+import 'package:photos/gateways/collections/models/collection_change.dart';++void main() {+ const currentUserID = 2;++ test('classifies a full legacy foreign deletion as a tombstone', () {+ final data = _fullCollection(ownerID: 1, isDeleted: true);++ final change = RemoteCollectionChange.fromMap(data, currentUserID);++ expect(change, isA<RemoteCollectionDeletion>());+ _expectDeletion(change as RemoteCollectionDeletion);+ });++ test('classifies the compatibility sparse deletion as a tombstone', () {+ final change = RemoteCollectionChange.fromMap({+ 'id': 7,+ 'owner': {'id': 1, 'email': ''},+ 'encryptedKey': 'compatibility-key',+ 'type': 'album',+ 'attributes': <String, dynamic>{},+ 'updationTime': 9,+ 'isDeleted': true,+ }, currentUserID);++ expect(change, isA<RemoteCollectionDeletion>());+ _expectDeletion(change as RemoteCollectionDeletion);+ });++ test('classifies a future minimal foreign deletion as a tombstone', () {+ final change = RemoteCollectionChange.fromMap({+ 'id': 7,+ 'owner': {'id': 1},+ 'updationTime': 9,+ 'isDeleted': true,+ }, currentUserID);++ expect(change, isA<RemoteCollectionDeletion>());+ _expectDeletion(change as RemoteCollectionDeletion);+ });++ test('retains an active foreign collection as an update', () {+ final data = _fullCollection(ownerID: 1, isDeleted: false);++ final change = RemoteCollectionChange.fromMap(data, currentUserID);++ expect(change, isA<RemoteCollectionUpdate>());+ expect((change as RemoteCollectionUpdate).data, same(data));+ });++ test('retains a full owned deletion as an update', () {+ final data = _fullCollection(ownerID: currentUserID, isDeleted: true);++ final change = RemoteCollectionChange.fromMap(data, currentUserID);++ expect(change, isA<RemoteCollectionUpdate>());+ expect((change as RemoteCollectionUpdate).data, same(data));+ });+}++Map<String, dynamic> _fullCollection({+ required int ownerID,+ required bool isDeleted,+}) => <String, dynamic>{+ 'id': 7,+ 'owner': {'id': ownerID, 'email': '[email protected]'},+ 'encryptedKey': 'encrypted-key',+ 'name': '',+ 'encryptedName': 'encrypted-name',+ 'nameDecryptionNonce': 'name-nonce',+ 'type': 'album',+ 'attributes': <String, dynamic>{},+ 'sharees': <dynamic>[],+ 'publicURLs': <dynamic>[],+ 'updationTime': 9,+ 'isDeleted': isDeleted,+};++void _expectDeletion(RemoteCollectionDeletion deletion) {+ expect(deletion.id, 7);+ expect(deletion.ownerID, 1);+ expect(deletion.updationTime, 9);+}diff --git a/server/pkg/api/collection.go b/server/pkg/api/collection.goindex 7d7c9f0936..19ae24c0c6 100644--- a/server/pkg/api/collection.go+++ b/server/pkg/api/collection.go@@ -76,9 +76,12 @@ func (h *CollectionHandler) GetV2(c *gin.Context) {handler.Error(c, stacktrace.Propagate(err, "Failed to get shared collections"))return}- c.JSON(http.StatusOK, gin.H{- "collections": append(ownedCollections, sharedCollections...),- })+ collections := make([]any, 0, len(ownedCollections)+len(sharedCollections))+ for _, collection := range ownedCollections {+ collections = append(collections, collection)+ }+ collections = append(collections, sharedCollectionResponses(sharedCollections)...)+ c.JSON(http.StatusOK, gin.H{"collections": collections})}func (h *CollectionHandler) GetWithLimit(c *gin.Context) {@@ -105,10 +108,47 @@ func (h *CollectionHandler) GetWithLimit(c *gin.Context) {}c.JSON(http.StatusOK, gin.H{"owned": ownedCollections,- "shared": sharedCollections,+ "shared": sharedCollectionResponses(sharedCollections),})}+type deletedSharedCollectionOwner struct {+ ID int64 `json:"id"`+ Email string `json:"email"`+}++// The compatibility fields can be removed after released clients accept ID-only tombstones.+type deletedSharedCollection struct {+ ID int64 `json:"id"`+ Owner deletedSharedCollectionOwner `json:"owner"`+ EncryptedKey string `json:"encryptedKey"`+ Type string `json:"type"`+ Attributes struct{} `json:"attributes"`+ UpdationTime int64 `json:"updationTime"`+ IsDeleted bool `json:"isDeleted"`+}++func sharedCollectionResponses(collections []ente.Collection) []any {+ responses := make([]any, 0, len(collections))+ for _, collection := range collections {+ if !collection.IsDeleted {+ responses = append(responses, collection)+ continue+ }+ responses = append(responses, deletedSharedCollection{+ ID: collection.ID,+ Owner: deletedSharedCollectionOwner{+ ID: collection.Owner.ID,+ },+ EncryptedKey: collection.EncryptedKey,+ Type: collection.Type,+ UpdationTime: collection.UpdationTime,+ IsDeleted: true,+ })+ }+ return responses+}+func (h *CollectionHandler) Share(c *gin.Context) {var request ente.AlterShareRequestif err := handler.BindJSON(c, &request); err != nil {diff --git a/server/pkg/api/collection_test.go b/server/pkg/api/collection_test.goindex 9644f40391..4f2c5e12dd 100644--- a/server/pkg/api/collection_test.go+++ b/server/pkg/api/collection_test.go@@ -1,11 +1,13 @@package apiimport (+ "encoding/json""net/http""net/http/httptest""strings""testing"+ "github.com/ente/museum/ente""github.com/ente/museum/pkg/controller/collections""github.com/gin-gonic/gin")@@ -38,3 +40,36 @@ func TestBatchShareHandlerValidatesEachShare(t *testing.T) {})}}++func TestDeletedSharedCollectionResponseShape(t *testing.T) {+ collection := ente.Collection{+ ID: 7,+ Owner: ente.CollectionUser{ID: 8, Email: "[email protected]", Name: "Owner", Role: ente.OWNER},+ EncryptedKey: "share-key",+ KeyDecryptionNonce: "key-nonce",+ Name: "secret name",+ EncryptedName: "encrypted-name",+ NameDecryptionNonce: "name-nonce",+ Type: "album",+ Attributes: ente.CollectionAttributes{EncryptedPath: "path", Version: 1},+ Sharees: []ente.CollectionUser{{ID: 9, Email: "[email protected]"}},+ PublicURLs: []ente.PublicURL{{URL: "https://example.com/secret"}},+ UpdationTime: 10,+ SharedAt: func() *int64 { value := int64(9); return &value }(),+ IsDeleted: true,+ MagicMetadata: &ente.MagicMetadata{Data: "private"},+ PublicMagicMetadata: &ente.MagicMetadata{Data: "public"},+ SharedMagicMetadata: &ente.MagicMetadata{Data: "shared"},+ App: string(ente.Photos),+ }++ responses := sharedCollectionResponses([]ente.Collection{collection})+ encoded, err := json.Marshal(responses[0])+ if err != nil {+ t.Fatal(err)+ }+ want := `{"id":7,"owner":{"id":8,"email":""},"encryptedKey":"share-key","type":"album","attributes":{},"updationTime":10,"isDeleted":true}`+ if string(encoded) != want {+ t.Fatalf("deleted shared collection response = %s, want %s", encoded, want)+ }+}diff --git a/server/pkg/controller/collections/shared_sync_test.go b/server/pkg/controller/collections/shared_sync_test.gonew file mode 100644index 0000000000..8b78443e11--- /dev/null+++ b/server/pkg/controller/collections/shared_sync_test.go@@ -0,0 +1,152 @@+package collections++import (+ "database/sql"+ "errors"+ "testing"++ "github.com/ente/museum/ente"+)++func TestSharedCollectionSyncUsesShareAndCollectionChanges(t *testing.T) {+ db, collectionRepo, ownerID, shareeID := setupCollectionShareTest(t)+ collectionID := createShareTestCollection(t, collectionRepo, ownerID)+ addShareTestShare(t, collectionRepo, collectionID, ownerID, shareeID, ente.VIEWER)++ sharedMetadata := ente.MagicMetadata{+ Version: 1,+ Count: 1,+ Data: "sharee-metadata",+ Header: "sharee-header",+ }+ if _, err := db.Exec(+ `UPDATE collection_shares SET magic_metadata = $1, updation_time = 2+ WHERE collection_id = $2 AND to_user_id = $3`,+ sharedMetadata,+ collectionID,+ shareeID,+ ); err != nil {+ t.Fatal(err)+ }++ limit := int64(10)+ changes, err := collectionRepo.GetCollectionsSharedWithUser(shareeID, 1, ente.Photos, &limit)+ if err != nil {+ t.Fatal(err)+ }+ if len(changes) != 1 {+ t.Fatalf("share-only changes = %d, want 1", len(changes))+ }+ if changes[0].UpdationTime != 2 {+ t.Fatalf("share-only updationTime = %d, want 2", changes[0].UpdationTime)+ }+ if changes[0].SharedMagicMetadata == nil || changes[0].SharedMagicMetadata.Data != sharedMetadata.Data {+ t.Fatalf("share-only metadata = %+v, want %+v", changes[0].SharedMagicMetadata, sharedMetadata)+ }++ if _, err := db.Exec(+ `UPDATE collections SET name = 'renamed collection', updation_time = 3+ WHERE collection_id = $1`,+ collectionID,+ ); err != nil {+ t.Fatal(err)+ }+ changes, err = collectionRepo.GetCollectionsSharedWithUser(shareeID, 2, ente.Photos, &limit)+ if err != nil {+ t.Fatal(err)+ }+ if len(changes) != 1 {+ t.Fatalf("collection-only changes = %d, want 1", len(changes))+ }+ if changes[0].UpdationTime != 3 || changes[0].Name != "renamed collection" {+ t.Fatalf("collection-only change = %+v", changes[0])+ }+}++func TestUnshareEmitsOneRedactedTombstone(t *testing.T) {+ db, collectionRepo, ownerID, shareeID := setupCollectionShareTest(t)+ collectionID := createShareTestCollection(t, collectionRepo, ownerID)+ addShareTestShare(t, collectionRepo, collectionID, ownerID, shareeID, ente.VIEWER)++ metadata := ente.MagicMetadata{Version: 1, Count: 1, Data: "secret", Header: "header"}+ if _, err := db.Exec(+ `UPDATE collections+ SET encrypted_name = 'encrypted-name', name_decryption_nonce = 'name-nonce',+ magic_metadata = $1, pub_magic_metadata = $1+ WHERE collection_id = $2`,+ metadata,+ collectionID,+ ); err != nil {+ t.Fatal(err)+ }+ if _, err := db.Exec(+ `UPDATE collection_shares SET magic_metadata = $1+ WHERE collection_id = $2 AND to_user_id = $3`,+ metadata,+ collectionID,+ shareeID,+ ); err != nil {+ t.Fatal(err)+ }++ status, err := collectionRepo.UnShareContext(t.Context(), collectionID, shareeID)+ if err != nil || status != ente.CollectionUnshared {+ t.Fatalf("UnShareContext() = %q, %v", status, err)+ }+ var deletionTime int64+ if err := db.QueryRow(+ `SELECT updation_time FROM collection_shares+ WHERE collection_id = $1 AND to_user_id = $2`,+ collectionID,+ shareeID,+ ).Scan(&deletionTime); err != nil {+ t.Fatal(err)+ }++ if _, err := db.Exec(`UPDATE users SET encrypted_email = NULL, email_decryption_nonce = NULL WHERE user_id = $1`, ownerID); err != nil {+ t.Fatal(err)+ }++ changes, err := collectionRepo.GetCollectionsSharedWithUser(shareeID, deletionTime-1, ente.Photos, nil)+ if err != nil {+ t.Fatal(err)+ }+ if len(changes) != 1 {+ t.Fatalf("changes after unshare = %d, want 1", len(changes))+ }+ tombstone := changes[0]+ if !tombstone.IsDeleted || tombstone.UpdationTime != deletionTime {+ t.Fatalf("tombstone deletion state = %+v, want time %d", tombstone, deletionTime)+ }+ if tombstone.ID != collectionID || tombstone.Owner.ID != ownerID || tombstone.EncryptedKey != "share-key" || tombstone.Type != "album" {+ t.Fatalf("tombstone compatibility fields = %+v", tombstone)+ }+ if tombstone.Owner.Email != "" || tombstone.Name != "" || tombstone.EncryptedName != "" || tombstone.NameDecryptionNonce != "" || tombstone.KeyDecryptionNonce != "" || tombstone.App != "" || tombstone.SharedAt != nil || tombstone.MagicMetadata != nil || tombstone.PublicMagicMetadata != nil || tombstone.SharedMagicMetadata != nil || len(tombstone.Sharees) != 0 || len(tombstone.PublicURLs) != 0 {+ t.Fatalf("tombstone contains live collection data: %+v", tombstone)+ }++ changes, err = collectionRepo.GetCollectionsSharedWithUser(shareeID, deletionTime, ente.Photos, nil)+ if err != nil {+ t.Fatal(err)+ }+ if len(changes) != 0 {+ t.Fatalf("replayed tombstones = %d, want 0", len(changes))+ }++ if _, err := db.Exec(`UPDATE collections SET name = 'post-unshare secret', updation_time = $1 WHERE collection_id = $2`, deletionTime+1, collectionID); err != nil {+ t.Fatal(err)+ }+ changes, err = collectionRepo.GetCollectionsSharedWithUser(shareeID, deletionTime, ente.Photos, nil)+ if err != nil {+ t.Fatal(err)+ }+ if len(changes) != 0 {+ t.Fatalf("post-unshare collection changes = %d, want 0", len(changes))+ }++ controller := newBatchShareTestController(db, collectionRepo)+ _, _, err = controller.GetDiffV2(newBatchShareTestContext(shareeID), collectionID, shareeID, 0)+ if !errors.Is(err, sql.ErrNoRows) {+ t.Fatalf("GetDiffV2() error = %v, want sql.ErrNoRows", err)+ }+}diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.goindex 26ceaf28c1..43bc05f41e 100644--- a/server/pkg/repo/collection.go+++ b/server/pkg/repo/collection.go@@ -255,15 +255,36 @@ pct.access_token, pct.valid_till, pct.device_limit, pct.created_at, pct.updated_func (repo *CollectionRepository) GetCollectionsSharedWithUser(userID int64, updationTime int64, app ente.App, limit *int64) ([]ente.Collection, error) {query := `- SELECT collections.collection_id, collections.owner_id, users.encrypted_email, users.email_decryption_nonce, collection_shares.encrypted_key, collections.name, collections.encrypted_name, collections.name_decryption_nonce, collections.type, collections.app, collections.pub_magic_metadata, collection_shares.magic_metadata, collections.updation_time, collection_shares.is_deleted, collection_shares.role_type, collection_shares.shared_at+ SELECT collections.collection_id, collections.owner_id,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE users.encrypted_email END,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE users.email_decryption_nonce END,+ collection_shares.encrypted_key,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.name END,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.encrypted_name END,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.name_decryption_nonce END,+ collections.type,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.app::text END,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.pub_magic_metadata END,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collection_shares.magic_metadata END,+ CASE WHEN collection_shares.is_deleted THEN collection_shares.updation_time ELSE GREATEST(collection_shares.updation_time, collections.updation_time) END AS effective_updation_time,+ collection_shares.is_deleted,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collection_shares.role_type END,+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collection_shares.shared_at ENDFROM collectionsINNER JOIN usersON collections.owner_id = users.user_idINNER JOIN collection_shares- ON collections.collection_id = collection_shares.collection_id AND collection_shares.to_user_id = $1 AND (collection_shares.updation_time > $2 OR collections.updation_time > $2) AND users.encrypted_email IS NOT NULL AND app = $3`+ ON collections.collection_id = collection_shares.collection_id+ WHERE collection_shares.to_user_id = $1+ AND (collection_shares.is_deleted = TRUE OR users.encrypted_email IS NOT NULL)+ AND collections.app = $3+ AND (+ (collection_shares.is_deleted = FALSE AND (collection_shares.updation_time > $2 OR collections.updation_time > $2))+ OR (collection_shares.is_deleted = TRUE AND collection_shares.updation_time > $2)+ )`args := []interface{}{userID, updationTime, string(app)}if limit != nil {- query += " ORDER BY collections.updation_time ASC LIMIT $4"+ query += " ORDER BY effective_updation_time ASC LIMIT $4"args = append(args, *limit)}@@ -279,11 +300,12 @@ func (repo *CollectionRepository) GetCollectionsSharedWithUser(userID int64, updvar c ente.Collectionvar collectionName, encryptedName, nameDecryptionNonce sql.NullStringvar encryptedEmail, emailDecryptionNonce []byte- var roleType sql.NullString+ var collectionApp, roleType sql.NullStringvar sharedAt sql.NullInt64- if err := rows.Scan(&c.ID, &c.Owner.ID, &encryptedEmail, &emailDecryptionNonce, &c.EncryptedKey, &collectionName, &encryptedName, &nameDecryptionNonce, &c.Type, &c.App, &c.PublicMagicMetadata, &c.SharedMagicMetadata, &c.UpdationTime, &c.IsDeleted, &roleType, &sharedAt); err != nil {+ if err := rows.Scan(&c.ID, &c.Owner.ID, &encryptedEmail, &emailDecryptionNonce, &c.EncryptedKey, &collectionName, &encryptedName, &nameDecryptionNonce, &c.Type, &collectionApp, &c.PublicMagicMetadata, &c.SharedMagicMetadata, &c.UpdationTime, &c.IsDeleted, &roleType, &sharedAt); err != nil {return collections, stacktrace.Propagate(err, "")}+ c.App = collectionApp.Stringif sharedAt.Valid {sharedAtValue := sharedAt.Int64c.SharedAt = &sharedAtValuediff --git a/web/apps/locker/src/services/locker-db.ts b/web/apps/locker/src/services/locker-db.tsindex 8af459a989..f0dad78811 100644--- a/web/apps/locker/src/services/locker-db.ts+++ b/web/apps/locker/src/services/locker-db.ts@@ -256,6 +256,20 @@ export const saveCollectionRecords = async (await tx.done;};+export const deleteCollectionRecords = async (+ collectionIDs: number[],+ userID = ensureLocalUser().id,+) => {+ if (collectionIDs.length === 0) return;++ const db = await resolveLockerDB(userID);+ const tx = db.transaction("collections", "readwrite");+ await Promise.all(+ collectionIDs.map((collectionID) => tx.store.delete(collectionID)),+ );+ await tx.done;+};+export const saveCollectionsSinceTime = async (sinceTime: number,userID = ensureLocalUser().id,diff --git a/web/apps/locker/src/services/sync/schemas.ts b/web/apps/locker/src/services/sync/schemas.tsindex d6caf300c8..36db132f3c 100644--- a/web/apps/locker/src/services/sync/schemas.ts+++ b/web/apps/locker/src/services/sync/schemas.ts@@ -28,8 +28,26 @@ export const RemoteCollection = z.object({export type RemoteCollection = z.infer<typeof RemoteCollection>;+export const RemoteCollectionDeletionTombstone = z.object({+ id: z.number(),+ owner: z.object({ id: z.number() }),+ updationTime: z.number(),+ isDeleted: z.literal(true),+});++export type RemoteCollectionDeletionTombstone = z.infer<+ typeof RemoteCollectionDeletionTombstone+>;++export const RemoteCollectionChange = z.union([+ RemoteCollection,+ RemoteCollectionDeletionTombstone,+]);++export type RemoteCollectionChange = z.infer<typeof RemoteCollectionChange>;+export const CollectionsResponse = z.object({- collections: z.array(RemoteCollection),+ collections: z.array(RemoteCollectionChange),});const RemoteEncryptedMetadata = z.object({diff --git a/web/apps/locker/src/services/sync/sync.ts b/web/apps/locker/src/services/sync/sync.tsindex c15e487c64..f2cf6b8e98 100644--- a/web/apps/locker/src/services/sync/sync.ts+++ b/web/apps/locker/src/services/sync/sync.ts@@ -1,4 +1,5 @@import type { LockerCollection, LockerItem } from "@/types";+import { ensureLocalUser } from "ente-accounts/services/user";import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";import log from "ente-base/log";import { apiURL } from "ente-base/origins";@@ -11,6 +12,7 @@ import {} from "../locker-cache";import {type StoredTrashFileRecord,+ deleteCollectionRecords,deleteCollectionSinceTime,deleteFileRecords,deleteFileRecordsForCollection,@@ -220,6 +222,7 @@ export const syncLockerState = async (): Promise<LockerHydratedState> => {let latestCollectionsSinceTime = snapshot.collectionsSinceTime;const changedCollections: EncryptedCollectionRecord[] = [];+ const revokedCollectionIDs: number[] = [];const deletedCollectionIDs: number[] = [];for (const change of collectionChanges) {@@ -227,6 +230,15 @@ export const syncLockerState = async (): Promise<LockerHydratedState> => {latestCollectionsSinceTime,change.updationTime,);+ if (change.isDeleted && change.owner.id != ensureLocalUser().id) {+ revokedCollectionIDs.push(change.id);+ deletedCollectionIDs.push(change.id);+ continue;+ }++ if (!("encryptedKey" in change)) {+ throw new Error(`Incomplete owned collection change ${change.id}`);+ }const record = await toEncryptedCollectionRecord(change);changedCollections.push(record);if (record.isDeleted) {@@ -237,6 +249,7 @@ export const syncLockerState = async (): Promise<LockerHydratedState> => {if (changedCollections.length > 0) {await saveCollectionRecords(changedCollections);}+ await deleteCollectionRecords(revokedCollectionIDs);for (const collectionID of deletedCollectionIDs) {await deleteFileRecordsForCollection(collectionID);await deleteCollectionSinceTime(collectionID);diff --git a/web/apps/locker/tests/sync.test.ts b/web/apps/locker/tests/sync.test.tsindex a814881f64..e26e46eec8 100644--- a/web/apps/locker/tests/sync.test.ts+++ b/web/apps/locker/tests/sync.test.ts@@ -13,6 +13,7 @@ import {vi.mock("../src/services/locker-db", () => ({loadLockerSnapshotFromDB: vi.fn(),+ deleteCollectionRecords: vi.fn(),saveCollectionRecords: vi.fn(),saveCollectionsSinceTime: vi.fn(),saveCollectionSinceTime: vi.fn(),@@ -24,6 +25,9 @@ vi.mock("../src/services/locker-db", () => ({deleteTrashFileRecords: vi.fn(),saveTrashSinceTime: vi.fn(),}));+vi.mock("ente-accounts/services/user", () => ({+ ensureLocalUser: () => ({ id: 1 }),+}));const { openCollectionKey, decryptMetadataJSON } = vi.hoisted(() => ({openCollectionKey: vi.fn(),decryptMetadataJSON: vi.fn(),@@ -242,6 +246,94 @@ test("incremental sync paginates changed collections and trash, skipping up-to-d);});+test("collection changes remove foreign deletions and retain owned records", async () => {+ const afterDeletion = {+ ...snapshot(),+ collections: new Map([[2, collection(2)]]),+ files: [file(10, 2)],+ collectionsSinceTime: 9,+ collectionSinceTimeByID: new Map([[2, 20]]),+ };+ vi.mocked(db.loadLockerSnapshotFromDB)+ .mockResolvedValueOnce(snapshot())+ .mockResolvedValueOnce(afterDeletion)+ .mockResolvedValueOnce(afterDeletion);+ fetchMock+ .mockResolvedValueOnce(+ response({+ collections: [+ {+ id: 1,+ owner: { id: 7, email: "" },+ encryptedKey: "legacy-key",+ type: "folder",+ updationTime: 9,+ isDeleted: true,+ },+ {+ id: 3,+ owner: { id: 7 },+ updationTime: 10,+ isDeleted: true,+ futureField: "ignored",+ },+ {+ id: 4,+ owner: { id: 1 },+ encryptedKey: "owned-key",+ type: "folder",+ updationTime: 11,+ isDeleted: true,+ },+ {+ id: 5,+ owner: { id: 1 },+ encryptedKey: "active-key",+ type: "folder",+ updationTime: 12,+ isDeleted: false,+ },+ ],+ }),+ )+ .mockResolvedValueOnce(response({ diff: [], hasMore: false }));++ await syncLockerState();++ expect(db.deleteCollectionRecords).toHaveBeenCalledExactlyOnceWith([1, 3]);+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledTimes(3);+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledWith(1);+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledWith(3);+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledWith(4);+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledTimes(3);+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledWith(1);+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledWith(3);+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledWith(4);+ expect(db.saveCollectionRecords).toHaveBeenCalledExactlyOnceWith([+ expect.objectContaining({+ id: 4,+ encryptedKey: "owned-key",+ isDeleted: true,+ }),+ expect.objectContaining({+ id: 5,+ encryptedKey: "active-key",+ isDeleted: false,+ }),+ ]);+ expect(db.saveCollectionsSinceTime).toHaveBeenCalledExactlyOnceWith(12);+ expect(openCollectionKey).not.toHaveBeenCalledWith(+ expect.anything(),+ expect.anything(),+ "legacy-key",+ expect.anything(),+ );+ expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([+ "/collections/v2?sinceTime=5",+ "/trash/v2/diff?sinceTime=40",+ ]);+});+test("failed file pagination does not persist a partial diff or advance its cursor", async () => {fetchMock.mockResolvedValueOnce(response({ collections: [] }))diff --git a/web/apps/photos/src/services/export-collection-state.ts b/web/apps/photos/src/services/export-collection-state.tsnew file mode 100644index 0000000000..9f94e0c88d--- /dev/null+++ b/web/apps/photos/src/services/export-collection-state.ts@@ -0,0 +1,11 @@+export const deletedExportedCollectionIDs = (+ presentCollectionIDs: Iterable<number>,+ collectionExportNames: Readonly<Record<number, string>> | undefined,+) => {+ if (!collectionExportNames) return [];++ const presentIDs = new Set(presentCollectionIDs);+ return Object.keys(collectionExportNames)+ .map(Number)+ .filter((collectionID) => !presentIDs.has(collectionID));+};diff --git a/web/apps/photos/src/services/export.ts b/web/apps/photos/src/services/export.tsindex 633b54d4dd..acd451b191 100644--- a/web/apps/photos/src/services/export.ts+++ b/web/apps/photos/src/services/export.ts@@ -36,6 +36,7 @@ import { PromiseQueue } from "ente-utils/promise";import { nullToUndefined } from "ente-utils/transform";import i18n from "i18next";import { z } from "zod";+import { deletedExportedCollectionIDs } from "./export-collection-state";const exportStoppedError = () => namedError("export_stopped", "export stopped");@@ -1224,25 +1225,11 @@ const getRenamedExportedCollections = (const getDeletedExportedCollections = (collections: Collection[],exportRecord: ExportRecord,-) => {- if (!exportRecord?.collectionExportNames) {- return [];- }- const presentCollections = new Set(- collections.map((collection) => collection.id),+) =>+ deletedExportedCollectionIDs(+ collections.map(({ id }) => id),+ exportRecord.collectionExportNames,);- const deletedExportedCollections = Object.keys(- exportRecord?.collectionExportNames,- )- .map(Number)- .filter((collectionID) => {- if (!presentCollections.has(collectionID)) {- return true;- }- return false;- });- return deletedExportedCollections;-};const readOnDiskFileExportRecordIDs = async (files: EnteFile[],diff --git a/web/apps/photos/tests/collection-change.test.ts b/web/apps/photos/tests/collection-change.test.tsnew file mode 100644index 0000000000..219fb97b2a--- /dev/null+++ b/web/apps/photos/tests/collection-change.test.ts@@ -0,0 +1,52 @@+import {+ RemoteCollectionChange,+ RemoteCollectionDeletionTombstone,+} from "ente-media/collection";+import { describe, expect, test } from "vitest";+import { deletedExportedCollectionIDs } from "../src/services/export-collection-state";++describe("collection deletion tombstones", () => {+ test("parses without active collection fields", () => {+ const parsed = RemoteCollectionChange.parse({+ id: 7,+ owner: { id: 1, email: "" },+ encryptedKey: "compatibility-key",+ type: "album",+ attributes: {},+ updationTime: 9,+ isDeleted: true,+ name: "must not survive parsing",+ magicMetadata: { data: "must not survive parsing" },+ });++ expect(parsed).toEqual({+ id: 7,+ owner: { id: 1 },+ updationTime: 9,+ isDeleted: true,+ });+ expect(RemoteCollectionDeletionTombstone.parse(parsed)).toStrictEqual(+ parsed,+ );+ });++ test("keeps active collection fields required", () => {+ expect(() =>+ RemoteCollectionChange.parse({+ id: 7,+ owner: { id: 1 },+ updationTime: 9,+ isDeleted: false,+ }),+ ).toThrow();+ });++ test("desktop export finds removed collections using IDs only", () => {+ expect(+ deletedExportedCollectionIDs([1], {+ 1: "Current album",+ 7: "Former shared album",+ }),+ ).toEqual([7]);+ });+});diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.tsindex cb4569e722..334823fd5b 100644--- a/web/packages/media/collection.ts+++ b/web/packages/media/collection.ts@@ -136,6 +136,26 @@ export const RemoteCollection = z.looseObject({export type RemoteCollection = z.infer<typeof RemoteCollection>;+export const RemoteCollectionDeletionTombstone = z.object({+ id: z.number(),+ owner: z.object({ id: z.number() }),+ updationTime: z.number(),+ isDeleted: z.literal(true),+});++export type RemoteCollectionDeletionTombstone = z.infer<+ typeof RemoteCollectionDeletionTombstone+>;++const RemoteActiveCollection = RemoteCollection.extend({+ isDeleted: z.literal(false).nullish().transform(nullToUndefined),+});++export const RemoteCollectionChange = z.union([+ RemoteCollectionDeletionTombstone,+ RemoteActiveCollection,+]);+export const decryptRemoteCollection = async (collection: RemoteCollection,collectionKey: string,diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.tsindex edf36f3e08..ead155c206 100644--- a/web/packages/new/photos/services/collection.ts+++ b/web/packages/new/photos/services/collection.ts@@ -12,6 +12,7 @@ import {findUserUncategorizedCollection,maxAlbumDescriptionLength,RemoteCollection,+ RemoteCollectionChange,RemotePublicURL,type Collection,type CollectionNewParticipantRole,@@ -156,7 +157,7 @@ export const getCollectionByID = async (};const CollectionsResponse = z.object({- collections: z.array(RemoteCollection),+ collections: z.array(RemoteCollectionChange),});export interface CollectionChange {