Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Fix deleted-share tombstone synchronization

Neeraj Gupta211a6154b6fa
diff --git a/cli/pkg/mapper/photo.go b/cli/pkg/mapper/photo.go
index 76cb8a398b..40a06ab7bb 100644
--- a/cli/pkg/mapper/photo.go
+++ b/cli/pkg/mapper/photo.go
@@ -5,23 +5,24 @@ import (
"encoding/json"
"errors"
"fmt"
+ "log"
+
"github.com/ente/cli/internal/api"
eCrypto "github.com/ente/cli/internal/crypto"
"github.com/ente/cli/pkg/model"
"github.com/ente/cli/pkg/model/export"
"github.com/ente/cli/pkg/secrets"
"github.com/ente/cli/utils/encoding"
- "log"
)
func MapCollectionToAlbum(ctx context.Context, collection api.Collection, holder *secrets.KeyHolder) (*model.RemoteAlbum, error) {
- var album model.RemoteAlbum
userID := ctx.Value("user_id").(int64)
- album.OwnerID = collection.Owner.ID
- album.ID = collection.ID
- album.IsShared = collection.Owner.ID != userID
- album.LastUpdatedAt = collection.UpdationTime
- album.IsDeleted = collection.IsDeleted
+ album := model.RemoteAlbum{
+ ID: collection.ID,
+ OwnerID: collection.Owner.ID,
+ IsShared: collection.Owner.ID != userID,
+ LastUpdatedAt: collection.UpdationTime,
+ }
collectionKey, err := holder.GetCollectionKey(ctx, collection)
if err != nil {
return nil, err
diff --git a/cli/pkg/remote_sync.go b/cli/pkg/remote_sync.go
index 9656ae40af..a6d4ef93c6 100644
--- a/cli/pkg/remote_sync.go
+++ b/cli/pkg/remote_sync.go
@@ -4,12 +4,13 @@ import (
"context"
"encoding/json"
"fmt"
- "github.com/ente/cli/pkg/mapper"
- "github.com/ente/cli/pkg/model"
- "github.com/ente/cli/utils/encoding"
"log"
"strconv"
"time"
+
+ "github.com/ente/cli/pkg/mapper"
+ "github.com/ente/cli/pkg/model"
+ "github.com/ente/cli/utils/encoding"
)
func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {
@@ -23,16 +24,31 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {
}
maxUpdated := lastSyncTime
for _, collection := range collections {
- if lastSyncTime == 0 && collection.IsDeleted {
- continue
- }
- album, mapErr := mapper.MapCollectionToAlbum(ctx, collection, c.KeyHolder)
- if mapErr != nil {
- return mapErr
+ if collection.UpdationTime > maxUpdated {
+ maxUpdated = collection.UpdationTime
}
- if album.LastUpdatedAt > maxUpdated {
- maxUpdated = album.LastUpdatedAt
+
+ var album *model.RemoteAlbum
+ if collection.IsDeleted {
+ key := []byte(strconv.FormatInt(collection.ID, 10))
+ existing, err := c.GetValue(ctx, model.RemoteAlbums, key)
+ if err != nil {
+ return err
+ }
+ album, err = markRemoteAlbumDeleted(existing, collection.ID, collection.UpdationTime)
+ if err != nil {
+ return err
+ }
+ if album == nil {
+ continue
+ }
+ } else {
+ album, err = mapper.MapCollectionToAlbum(ctx, collection, c.KeyHolder)
+ if err != nil {
+ return err
+ }
}
+
albumJson := encoding.MustMarshalJSON(album)
putErr := c.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(album.ID, 10)), albumJson)
if putErr != nil {
@@ -48,6 +64,22 @@ func (c *ClICtrl) fetchRemoteCollections(ctx context.Context) error {
return nil
}
+func markRemoteAlbumDeleted(existing []byte, collectionID, updationTime int64) (*model.RemoteAlbum, error) {
+ if len(existing) == 0 {
+ return nil, nil
+ }
+ var album model.RemoteAlbum
+ if err := json.Unmarshal(existing, &album); err != nil {
+ return nil, err
+ }
+ if album.ID != collectionID {
+ return nil, fmt.Errorf("stored album ID %d does not match tombstone ID %d", album.ID, collectionID)
+ }
+ album.IsDeleted = true
+ album.LastUpdatedAt = updationTime
+ return &album, nil
+}
+
func (c *ClICtrl) fetchRemoteFiles(ctx context.Context) error {
albums, err := c.getRemoteAlbums(ctx)
filter := ctx.Value(model.FilterKey).(model.Filter)
diff --git a/cli/pkg/remote_sync_test.go b/cli/pkg/remote_sync_test.go
new file mode 100644
index 0000000000..fd75fa9744
--- /dev/null
+++ b/cli/pkg/remote_sync_test.go
@@ -0,0 +1,99 @@
+package pkg
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "testing"
+
+ "github.com/ente/cli/internal/api"
+ "github.com/ente/cli/pkg/model"
+)
+
+func TestFetchRemoteCollectionsHandlesDeletedCollectionsWithoutDecrypting(t *testing.T) {
+ var sinceTime string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/collections/v2" {
+ http.NotFound(w, r)
+ return
+ }
+ sinceTime = r.URL.Query().Get("sinceTime")
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"collections":[{"id":7,"owner":{"id":1},"updationTime":9,"isDeleted":true},{"id":8,"owner":{"id":1},"updationTime":11,"isDeleted":true}]}`))
+ }))
+ defer server.Close()
+
+ db, err := GetDB(t.TempDir() + "/cli.db")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = db.Close() })
+
+ account := model.Account{UserID: 2, App: api.AppPhotos}
+ if err := createDataBuckets(db, account); err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.WithValue(context.Background(), "app", string(account.App))
+ ctx = context.WithValue(ctx, "account_key", account.AccountKey())
+ ctx = context.WithValue(ctx, "user_id", account.UserID)
+
+ ctrl := ClICtrl{
+ Client: api.NewClient(api.Params{Host: server.URL}),
+ DB: db,
+ // A nil holder proves tombstones never enter key or metadata decryption.
+ KeyHolder: nil,
+ }
+ existing := model.RemoteAlbum{
+ ID: 7,
+ OwnerID: 1,
+ IsShared: true,
+ AlbumName: "Vacation",
+ AlbumKey: model.EncString{
+ CipherText: "local-key",
+ Nonce: "local-nonce",
+ },
+ PrivateMeta: map[string]interface{}{"visibility": float64(1)},
+ }
+ existingJSON, err := json.Marshal(&existing)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := ctrl.PutValue(ctx, model.RemoteAlbums, []byte(strconv.FormatInt(existing.ID, 10)), existingJSON); err != nil {
+ t.Fatal(err)
+ }
+ if err := ctrl.PutConfigValue(ctx, model.CollectionsSyncKey, []byte("5")); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := ctrl.fetchRemoteCollections(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if sinceTime != "5" {
+ t.Fatalf("sinceTime = %q, want 5", sinceTime)
+ }
+
+ stored, err := ctrl.GetValue(ctx, model.RemoteAlbums, []byte("7"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var album model.RemoteAlbum
+ if err := json.Unmarshal(stored, &album); err != nil {
+ t.Fatal(err)
+ }
+ if !album.IsDeleted || album.LastUpdatedAt != 9 || album.AlbumName != existing.AlbumName || album.AlbumKey != existing.AlbumKey || album.PrivateMeta["visibility"] != float64(1) {
+ t.Fatalf("deleted album lost local export state: %+v", album)
+ }
+ unknown, err := ctrl.GetValue(ctx, model.RemoteAlbums, []byte("8"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if unknown != nil {
+ t.Fatalf("unknown tombstone created album: %s", unknown)
+ }
+ cursor, err := ctrl.GetInt64ConfigValue(ctx, model.CollectionsSyncKey)
+ if err != nil || cursor != 11 {
+ t.Fatalf("cursor = %d, %v; want 11", cursor, err)
+ }
+}
diff --git a/mobile/apps/locker/lib/services/collections/collections_api_client.dart b/mobile/apps/locker/lib/services/collections/collections_api_client.dart
index abf23f7537..d4c41b861f 100644
--- a/mobile/apps/locker/lib/services/collections/collections_api_client.dart
+++ b/mobile/apps/locker/lib/services/collections/collections_api_client.dart
@@ -13,6 +13,7 @@ import 'package:locker/core/errors.dart';
import "package:locker/events/collections_updated_event.dart";
import "package:locker/services/collections/collections_service.dart";
import 'package:locker/services/collections/models/collection.dart';
+import 'package:locker/services/collections/models/collection_change.dart';
import 'package:locker/services/collections/models/collection_file_item.dart';
import 'package:locker/services/collections/models/collection_magic.dart';
import 'package:locker/services/collections/models/diff.dart';
@@ -42,21 +43,29 @@ class CollectionApiClient {
_db = LockerDB.instance;
}
- Future<List<Collection>> getCollections(int sinceTime) async {
+ Future<List<CollectionChange>> getCollections(int sinceTime) async {
try {
final response = await _enteDio.get(
"/collections/v2",
queryParameters: {"sinceTime": sinceTime},
);
- final List<Collection> collections = [];
+ final List<CollectionChange> changes = [];
final c = response.data["collections"];
for (final collectionData in c) {
- final Collection collection = await _fromRemoteCollection(
- collectionData,
- );
- collections.add(collection);
+ if (isSharedCollectionDeletion(collectionData, _config.getUserID()!)) {
+ changes.add(
+ CollectionDeletion(
+ collectionData['id'] as int,
+ collectionData['updationTime'] as int,
+ ),
+ );
+ } else {
+ changes.add(
+ CollectionUpdate(await _fromRemoteCollection(collectionData)),
+ );
+ }
}
- return collections;
+ return changes;
} catch (e, s) {
_logger.warning(e, s);
if (e is DioException && e.response?.statusCode == 401) {
@@ -224,7 +233,7 @@ class CollectionApiClient {
}
Future<void> _handleCollectionDeletion(Collection collection) async {
- await _db.deleteCollection(collection);
+ await _db.deleteCollection(collection.id);
final deletedCollection = collection.copyWith(isDeleted: true);
await _updateCollectionInDB(deletedCollection);
}
@@ -305,7 +314,7 @@ class CollectionApiClient {
"&collectionID=${collection.id}",
);
if (skipEventFiring) {
- await _db.deleteCollection(collection);
+ await _db.deleteCollection(collection.id);
final deletedCollection = collection.copyWith(isDeleted: true);
await _updateCollectionInDB(deletedCollection);
} else {
diff --git a/mobile/apps/locker/lib/services/collections/collections_service.dart b/mobile/apps/locker/lib/services/collections/collections_service.dart
index eceee2cd47..24f0287c31 100644
--- a/mobile/apps/locker/lib/services/collections/collections_service.dart
+++ b/mobile/apps/locker/lib/services/collections/collections_service.dart
@@ -17,6 +17,7 @@ import 'package:locker/events/collections_updated_event.dart';
import 'package:locker/events/user_details_refresh_event.dart';
import "package:locker/services/collections/collections_api_client.dart";
import 'package:locker/services/collections/models/collection.dart';
+import 'package:locker/services/collections/models/collection_change.dart';
import "package:locker/services/collections/models/files_split.dart";
import "package:locker/services/collections/models/public_url.dart";
import 'package:locker/services/configuration.dart';
@@ -79,9 +80,10 @@ class CollectionService {
final previousSyncTime = _db.getSyncTime();
final shouldCheckFirstSyncCompletion = previousSyncTime == 0;
- final updatedCollections = await CollectionApiClient.instance
- .getCollections(previousSyncTime);
- if (updatedCollections.isEmpty) {
+ final collectionChanges = await CollectionApiClient.instance.getCollections(
+ previousSyncTime,
+ );
+ if (collectionChanges.isEmpty) {
if (shouldCheckFirstSyncCompletion) {
final didMarkFirstSync = await _setFirstSyncCompleted();
if (didMarkFirstSync) {
@@ -93,17 +95,29 @@ class CollectionService {
_logger.info("No collections to sync.");
return;
}
- await _db.updateCollections(updatedCollections);
- for (final collection in updatedCollections) {
+ final persistedCollections = <Collection>[];
+ var latestUpdationTime = previousSyncTime;
+ for (final change in collectionChanges) {
+ latestUpdationTime = max(latestUpdationTime, change.updationTime);
+ if (change is CollectionDeletion) {
+ await _db.deleteCollection(change.id);
+ _collectionIDToCollections.remove(change.id);
+ continue;
+ }
+ final collection = (change as CollectionUpdate).collection;
+ persistedCollections.add(collection);
_collectionIDToCollections[collection.id] = collection;
}
- await _db.setSyncTime(updatedCollections.last.updationTime);
+ if (persistedCollections.isNotEmpty) {
+ await _db.updateCollections(persistedCollections);
+ }
+ await _db.setSyncTime(latestUpdationTime);
if (shouldCheckFirstSyncCompletion) {
await _setFirstSyncCompleted();
}
final List<Future<bool>> fileFutures = [];
- for (final collection in updatedCollections) {
+ for (final collection in persistedCollections) {
if (collection.isDeleted) {
continue;
}
@@ -143,7 +157,7 @@ class CollectionService {
"Skipping offline stale cleanup because one or more collection syncs failed",
);
}
- if (updatedCollections.isNotEmpty) {
+ if (collectionChanges.isNotEmpty) {
Bus.instance.fire(CollectionsUpdatedEvent('sync'));
}
}
diff --git a/mobile/apps/locker/lib/services/collections/models/collection_change.dart b/mobile/apps/locker/lib/services/collections/models/collection_change.dart
new file mode 100644
index 0000000000..0de9908331
--- /dev/null
+++ b/mobile/apps/locker/lib/services/collections/models/collection_change.dart
@@ -0,0 +1,28 @@
+import 'package:locker/services/collections/models/collection.dart';
+
+sealed class CollectionChange {
+ const CollectionChange(this.id, this.updationTime);
+
+ final int id;
+ final int updationTime;
+}
+
+final class CollectionUpdate extends CollectionChange {
+ CollectionUpdate(this.collection)
+ : super(collection.id, collection.updationTime);
+
+ final Collection collection;
+}
+
+final class CollectionDeletion extends CollectionChange {
+ const CollectionDeletion(super.id, super.updationTime);
+}
+
+bool isSharedCollectionDeletion(
+ Map<String, dynamic> collectionData,
+ int currentUserID,
+) {
+ final ownerID =
+ (collectionData['owner'] as Map<String, dynamic>)['id'] as int;
+ return collectionData['isDeleted'] == true && ownerID != currentUserID;
+}
diff --git a/mobile/apps/locker/lib/services/db/locker_db.dart b/mobile/apps/locker/lib/services/db/locker_db.dart
index 176b22c6a5..b1855202d7 100644
--- a/mobile/apps/locker/lib/services/db/locker_db.dart
+++ b/mobile/apps/locker/lib/services/db/locker_db.dart
@@ -328,25 +328,21 @@ class LockerDB extends EnteBaseDatabase {
await batch.commit();
}
- Future<void> deleteCollection(Collection collection) async {
+ Future<void> deleteCollection(int collectionID) async {
final batch = _db.batch();
- batch.delete(
- _collectionsTable,
- where: 'id = ?',
- whereArgs: [collection.id],
- );
+ batch.delete(_collectionsTable, where: 'id = ?', whereArgs: [collectionID]);
batch.delete(
_collectionFilesTable,
where: 'collection_id = ?',
- whereArgs: [collection.id],
+ whereArgs: [collectionID],
);
final filesInCollection = await _db.query(
_collectionFilesTable,
where: 'collection_id = ?',
- whereArgs: [collection.id],
+ whereArgs: [collectionID],
);
for (final fileMap in filesInCollection) {
@@ -354,7 +350,7 @@ class LockerDB extends EnteBaseDatabase {
final otherCollections = await _db.query(
_collectionFilesTable,
where: 'uploaded_file_id = ? AND collection_id != ?',
- whereArgs: [uploadedFileId, collection.id],
+ whereArgs: [uploadedFileId, collectionID],
);
if (otherCollections.isEmpty) {
@@ -366,7 +362,14 @@ class LockerDB extends EnteBaseDatabase {
}
}
+ batch.delete(
+ _syncTimesTable,
+ where: 'key = ?',
+ whereArgs: ['collection_sync_time_$collectionID'],
+ );
+
await batch.commit();
+ _collectionSyncTimesCache.remove(collectionID);
}
Future<List<Collection>> getCollections() async {
diff --git a/mobile/apps/locker/test/services/collections/models/collection_change_test.dart b/mobile/apps/locker/test/services/collections/models/collection_change_test.dart
new file mode 100644
index 0000000000..0fed451da1
--- /dev/null
+++ b/mobile/apps/locker/test/services/collections/models/collection_change_test.dart
@@ -0,0 +1,59 @@
+import 'package:flutter_test/flutter_test.dart';
+import 'package:locker/services/collections/models/collection.dart';
+import 'package:locker/services/collections/models/collection_change.dart';
+
+void main() {
+ group('isSharedCollectionDeletion', () {
+ test('accepts sparse and legacy full foreign deletions', () {
+ final sparse = {
+ 'id': 7,
+ 'owner': {'id': 1},
+ 'updationTime': 9,
+ 'isDeleted': true,
+ };
+ final legacyFull = {
+ ...sparse,
+ 'encryptedKey': 'compatibility-key',
+ 'keyDecryptionNonce': 'nonce',
+ 'type': 'album',
+ 'attributes': <String, dynamic>{},
+ 'name': 'Deleted Collection',
+ };
+
+ expect(isSharedCollectionDeletion(sparse, 2), isTrue);
+ expect(isSharedCollectionDeletion(legacyFull, 2), isTrue);
+ });
+
+ test('keeps owned deletions and active collections as updates', () {
+ final ownedDeletion = {
+ 'id': 7,
+ 'owner': {'id': 2, 'email': '[email protected]'},
+ 'encryptedKey': 'owned-key',
+ 'keyDecryptionNonce': 'owned-nonce',
+ 'name': 'Deleted Collection',
+ 'encryptedName': 'encrypted-name',
+ 'nameDecryptionNonce': 'name-nonce',
+ 'type': 'folder',
+ 'attributes': <String, dynamic>{},
+ 'sharees': <Map<String, dynamic>>[],
+ 'publicURLs': <Map<String, dynamic>>[],
+ 'updationTime': 9,
+ 'isDeleted': true,
+ };
+ final active = {
+ ...ownedDeletion,
+ 'id': 8,
+ 'owner': {'id': 1},
+ 'isDeleted': false,
+ };
+
+ expect(isSharedCollectionDeletion(ownedDeletion, 2), isFalse);
+ expect(isSharedCollectionDeletion(active, 2), isFalse);
+
+ final collection = Collection.fromMap(ownedDeletion);
+ expect(collection.encryptedKey, 'owned-key');
+ expect(collection.keyDecryptionNonce, 'owned-nonce');
+ expect(collection.isDeleted, isTrue);
+ });
+ });
+}
diff --git a/mobile/apps/photos/lib/gateways/collections/collections_gateway.dart b/mobile/apps/photos/lib/gateways/collections/collections_gateway.dart
index 1ea4f837ca..b705cb5a45 100644
--- a/mobile/apps/photos/lib/gateways/collections/collections_gateway.dart
+++ b/mobile/apps/photos/lib/gateways/collections/collections_gateway.dart
@@ -1,4 +1,5 @@
import "package:dio/dio.dart";
+import "package:photos/gateways/collections/models/collection_change.dart";
import "package:photos/gateways/collections/models/create_request.dart";
import "package:photos/gateways/collections/models/metadata.dart";
@@ -95,15 +96,24 @@ class CollectionsGateway {
);
}
- Future<Map<String, dynamic>> getAll({
+ Future<List<RemoteCollectionChange>> getAll({
required int sinceTime,
required String source,
+ required int currentUserID,
}) async {
final response = await _enteDio.get(
"/collections/v2",
queryParameters: {"sinceTime": sinceTime, "source": source},
);
- return response.data;
+ final collections = response.data['collections'] as List<dynamic>;
+ return collections
+ .map(
+ (data) => RemoteCollectionChange.fromMap(
+ Map<String, dynamic>.from(data as Map<dynamic, dynamic>),
+ currentUserID,
+ ),
+ )
+ .toList(growable: false);
}
Future<Map<String, dynamic>> fetchPendingRemovalActions() async {
diff --git a/mobile/apps/photos/lib/gateways/collections/models/collection_change.dart b/mobile/apps/photos/lib/gateways/collections/models/collection_change.dart
new file mode 100644
index 0000000000..7076bf7910
--- /dev/null
+++ b/mobile/apps/photos/lib/gateways/collections/models/collection_change.dart
@@ -0,0 +1,47 @@
+sealed class RemoteCollectionChange {
+ const RemoteCollectionChange();
+
+ int get id;
+ int get updationTime;
+
+ factory RemoteCollectionChange.fromMap(
+ Map<String, dynamic> map,
+ int currentUserID,
+ ) {
+ final owner = map['owner'] as Map<String, dynamic>;
+ if (map['isDeleted'] == true && owner['id'] as int != currentUserID) {
+ return RemoteCollectionDeletion(
+ id: map['id'] as int,
+ ownerID: owner['id'] as int,
+ updationTime: map['updationTime'] as int,
+ );
+ }
+ return RemoteCollectionUpdate(map);
+ }
+}
+
+final class RemoteCollectionUpdate extends RemoteCollectionChange {
+ const RemoteCollectionUpdate(this.data);
+
+ final Map<String, dynamic> data;
+
+ @override
+ int get id => data['id'] as int;
+
+ @override
+ int get updationTime => data['updationTime'] as int;
+}
+
+final class RemoteCollectionDeletion extends RemoteCollectionChange {
+ const RemoteCollectionDeletion({
+ required this.id,
+ required this.ownerID,
+ required this.updationTime,
+ });
+
+ @override
+ final int id;
+ final int ownerID;
+ @override
+ final int updationTime;
+}
diff --git a/mobile/apps/photos/lib/services/collections_service.dart b/mobile/apps/photos/lib/services/collections_service.dart
index 0b5aa2b847..79dfed5923 100644
--- a/mobile/apps/photos/lib/services/collections_service.dart
+++ b/mobile/apps/photos/lib/services/collections_service.dart
@@ -26,6 +26,7 @@ import 'package:photos/events/files_updated_event.dart';
import 'package:photos/events/force_reload_home_gallery_event.dart';
import 'package:photos/events/local_photos_updated_event.dart';
import "package:photos/gateways/collections/collection_share_gateway.dart";
+import 'package:photos/gateways/collections/models/collection_change.dart';
import 'package:photos/gateways/collections/models/collection_file_item.dart';
import "package:photos/gateways/collections/models/collection_share.dart";
import 'package:photos/gateways/collections/models/create_request.dart';
@@ -142,23 +143,41 @@ class CollectionsService {
_logger.info("[COLLECTIONS] Starting sync");
- final fetchedCollections = await _fetchCollections(
+ final collectionChanges = await _fetchCollectionChanges(
lastCollectionUpdationTime,
);
_logger.info(
- "[COLLECTIONS] Fetched ${fetchedCollections.length} collections from API",
+ "[COLLECTIONS] Fetched ${collectionChanges.length} collection changes from API",
);
- watch.log("remote fetch collections ${fetchedCollections.length}");
+ watch.log("remote fetch collections ${collectionChanges.length}");
- if (fetchedCollections.isEmpty) {
+ if (collectionChanges.isEmpty) {
await _ensureUncategorizedCollectionExists();
return;
}
final updatedCollections = <Collection>[];
int maxUpdationTime = lastCollectionUpdationTime;
- final ownerID = _config.getUserID();
bool shouldFireDeleteEvent = false;
- for (final collection in fetchedCollections) {
+ for (final change in collectionChanges) {
+ maxUpdationTime = max(maxUpdationTime, change.updationTime);
+ if (change is RemoteCollectionDeletion) {
+ await _filesDB.deleteCollection(change.id);
+ await _cleanupSocialData(change.id);
+ await setCollectionSyncTime(change.id, null);
+ if (_collectionIDToCollections.containsKey(change.id)) {
+ shouldFireDeleteEvent = true;
+ }
+ await _db.deleteCollection(change.id);
+ _collectionIDToCollections.remove(change.id);
+ _cachedKeys.remove(change.id);
+ _localPathToCollectionID.removeWhere((_, id) => id == change.id);
+ _clearPublicCollectionState(change.id);
+ continue;
+ }
+
+ final collection = await _fromRemoteCollection(
+ (change as RemoteCollectionUpdate).data,
+ );
if (collection.isDeleted) {
await _filesDB.deleteCollection(collection.id);
await _cleanupSocialData(collection.id);
@@ -167,16 +186,9 @@ class CollectionsService {
shouldFireDeleteEvent = true;
}
}
- if (collection.isDeleted && ownerID != collection.owner.id) {
- await _db.deleteCollection(collection.id);
- } else {
- // keep entry for deletedCollection as collectionKey may be used during
- // trash file decryption
- updatedCollections.add(collection);
- }
- maxUpdationTime = collection.updationTime > maxUpdationTime
- ? collection.updationTime
- : maxUpdationTime;
+ // Keep owned deleted collections because their keys may be needed to
+ // decrypt trashed files.
+ updatedCollections.add(collection);
}
if (shouldFireDeleteEvent) {
@@ -193,14 +205,14 @@ class CollectionsService {
watch.logAndReset("till DB insertion ${updatedCollections.length}");
_logger.info("[COLLECTIONS] Updated ${updatedCollections.length} in DB");
- for (final collection in fetchedCollections) {
+ for (final collection in updatedCollections) {
_cacheLocalPathAndCollection(collection);
}
_logger.info("Collections synced");
- watch.log("${fetchedCollections.length} collection cached refreshed ");
+ watch.log("${updatedCollections.length} collection caches refreshed");
- if (fetchedCollections.isNotEmpty) {
+ if (collectionChanges.isNotEmpty) {
Bus.instance.fire(ContactRelationshipsInvalidatedEvent());
Bus.instance.fire(
CollectionUpdatedEvent(
@@ -1532,21 +1544,15 @@ class CollectionsService {
}
}
- Future<List<Collection>> _fetchCollections(int sinceTime) async {
+ Future<List<RemoteCollectionChange>> _fetchCollectionChanges(
+ int sinceTime,
+ ) async {
try {
- final response = await collectionsGateway.getAll(
+ return await collectionsGateway.getAll(
sinceTime: sinceTime,
source: AppLifecycleService.instance.isForeground ? "fg" : "bg",
+ currentUserID: _config.getUserID()!,
);
- final List<Collection> collections = [];
- final c = response["collections"];
- for (final collectionData in c) {
- final Collection collection = await _fromRemoteCollection(
- collectionData,
- );
- collections.add(collection);
- }
- return collections;
} catch (e, s) {
_logger.warning("Failed to fetch collections", e, s);
if (e is DioException && e.response?.statusCode == 401) {
diff --git a/mobile/apps/photos/test/gateways/collections/models/collection_change_test.dart b/mobile/apps/photos/test/gateways/collections/models/collection_change_test.dart
new file mode 100644
index 0000000000..cd4e518d1f
--- /dev/null
+++ b/mobile/apps/photos/test/gateways/collections/models/collection_change_test.dart
@@ -0,0 +1,84 @@
+import 'package:flutter_test/flutter_test.dart';
+import 'package:photos/gateways/collections/models/collection_change.dart';
+
+void main() {
+ const currentUserID = 2;
+
+ test('classifies a full legacy foreign deletion as a tombstone', () {
+ final data = _fullCollection(ownerID: 1, isDeleted: true);
+
+ final change = RemoteCollectionChange.fromMap(data, currentUserID);
+
+ expect(change, isA<RemoteCollectionDeletion>());
+ _expectDeletion(change as RemoteCollectionDeletion);
+ });
+
+ test('classifies the compatibility sparse deletion as a tombstone', () {
+ final change = RemoteCollectionChange.fromMap({
+ 'id': 7,
+ 'owner': {'id': 1, 'email': ''},
+ 'encryptedKey': 'compatibility-key',
+ 'type': 'album',
+ 'attributes': <String, dynamic>{},
+ 'updationTime': 9,
+ 'isDeleted': true,
+ }, currentUserID);
+
+ expect(change, isA<RemoteCollectionDeletion>());
+ _expectDeletion(change as RemoteCollectionDeletion);
+ });
+
+ test('classifies a future minimal foreign deletion as a tombstone', () {
+ final change = RemoteCollectionChange.fromMap({
+ 'id': 7,
+ 'owner': {'id': 1},
+ 'updationTime': 9,
+ 'isDeleted': true,
+ }, currentUserID);
+
+ expect(change, isA<RemoteCollectionDeletion>());
+ _expectDeletion(change as RemoteCollectionDeletion);
+ });
+
+ test('retains an active foreign collection as an update', () {
+ final data = _fullCollection(ownerID: 1, isDeleted: false);
+
+ final change = RemoteCollectionChange.fromMap(data, currentUserID);
+
+ expect(change, isA<RemoteCollectionUpdate>());
+ expect((change as RemoteCollectionUpdate).data, same(data));
+ });
+
+ test('retains a full owned deletion as an update', () {
+ final data = _fullCollection(ownerID: currentUserID, isDeleted: true);
+
+ final change = RemoteCollectionChange.fromMap(data, currentUserID);
+
+ expect(change, isA<RemoteCollectionUpdate>());
+ expect((change as RemoteCollectionUpdate).data, same(data));
+ });
+}
+
+Map<String, dynamic> _fullCollection({
+ required int ownerID,
+ required bool isDeleted,
+}) => <String, dynamic>{
+ 'id': 7,
+ 'owner': {'id': ownerID, 'email': '[email protected]'},
+ 'encryptedKey': 'encrypted-key',
+ 'name': '',
+ 'encryptedName': 'encrypted-name',
+ 'nameDecryptionNonce': 'name-nonce',
+ 'type': 'album',
+ 'attributes': <String, dynamic>{},
+ 'sharees': <dynamic>[],
+ 'publicURLs': <dynamic>[],
+ 'updationTime': 9,
+ 'isDeleted': isDeleted,
+};
+
+void _expectDeletion(RemoteCollectionDeletion deletion) {
+ expect(deletion.id, 7);
+ expect(deletion.ownerID, 1);
+ expect(deletion.updationTime, 9);
+}
diff --git a/server/pkg/api/collection.go b/server/pkg/api/collection.go
index 7d7c9f0936..19ae24c0c6 100644
--- a/server/pkg/api/collection.go
+++ b/server/pkg/api/collection.go
@@ -76,9 +76,12 @@ func (h *CollectionHandler) GetV2(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(err, "Failed to get shared collections"))
return
}
- c.JSON(http.StatusOK, gin.H{
- "collections": append(ownedCollections, sharedCollections...),
- })
+ collections := make([]any, 0, len(ownedCollections)+len(sharedCollections))
+ for _, collection := range ownedCollections {
+ collections = append(collections, collection)
+ }
+ collections = append(collections, sharedCollectionResponses(sharedCollections)...)
+ c.JSON(http.StatusOK, gin.H{"collections": collections})
}
func (h *CollectionHandler) GetWithLimit(c *gin.Context) {
@@ -105,10 +108,47 @@ func (h *CollectionHandler) GetWithLimit(c *gin.Context) {
}
c.JSON(http.StatusOK, gin.H{
"owned": ownedCollections,
- "shared": sharedCollections,
+ "shared": sharedCollectionResponses(sharedCollections),
})
}
+type deletedSharedCollectionOwner struct {
+ ID int64 `json:"id"`
+ Email string `json:"email"`
+}
+
+// The compatibility fields can be removed after released clients accept ID-only tombstones.
+type deletedSharedCollection struct {
+ ID int64 `json:"id"`
+ Owner deletedSharedCollectionOwner `json:"owner"`
+ EncryptedKey string `json:"encryptedKey"`
+ Type string `json:"type"`
+ Attributes struct{} `json:"attributes"`
+ UpdationTime int64 `json:"updationTime"`
+ IsDeleted bool `json:"isDeleted"`
+}
+
+func sharedCollectionResponses(collections []ente.Collection) []any {
+ responses := make([]any, 0, len(collections))
+ for _, collection := range collections {
+ if !collection.IsDeleted {
+ responses = append(responses, collection)
+ continue
+ }
+ responses = append(responses, deletedSharedCollection{
+ ID: collection.ID,
+ Owner: deletedSharedCollectionOwner{
+ ID: collection.Owner.ID,
+ },
+ EncryptedKey: collection.EncryptedKey,
+ Type: collection.Type,
+ UpdationTime: collection.UpdationTime,
+ IsDeleted: true,
+ })
+ }
+ return responses
+}
+
func (h *CollectionHandler) Share(c *gin.Context) {
var request ente.AlterShareRequest
if err := handler.BindJSON(c, &request); err != nil {
diff --git a/server/pkg/api/collection_test.go b/server/pkg/api/collection_test.go
index 9644f40391..4f2c5e12dd 100644
--- a/server/pkg/api/collection_test.go
+++ b/server/pkg/api/collection_test.go
@@ -1,11 +1,13 @@
package api
import (
+ "encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
+ "github.com/ente/museum/ente"
"github.com/ente/museum/pkg/controller/collections"
"github.com/gin-gonic/gin"
)
@@ -38,3 +40,36 @@ func TestBatchShareHandlerValidatesEachShare(t *testing.T) {
})
}
}
+
+func TestDeletedSharedCollectionResponseShape(t *testing.T) {
+ collection := ente.Collection{
+ ID: 7,
+ Owner: ente.CollectionUser{ID: 8, Email: "[email protected]", Name: "Owner", Role: ente.OWNER},
+ EncryptedKey: "share-key",
+ KeyDecryptionNonce: "key-nonce",
+ Name: "secret name",
+ EncryptedName: "encrypted-name",
+ NameDecryptionNonce: "name-nonce",
+ Type: "album",
+ Attributes: ente.CollectionAttributes{EncryptedPath: "path", Version: 1},
+ Sharees: []ente.CollectionUser{{ID: 9, Email: "[email protected]"}},
+ PublicURLs: []ente.PublicURL{{URL: "https://example.com/secret"}},
+ UpdationTime: 10,
+ SharedAt: func() *int64 { value := int64(9); return &value }(),
+ IsDeleted: true,
+ MagicMetadata: &ente.MagicMetadata{Data: "private"},
+ PublicMagicMetadata: &ente.MagicMetadata{Data: "public"},
+ SharedMagicMetadata: &ente.MagicMetadata{Data: "shared"},
+ App: string(ente.Photos),
+ }
+
+ responses := sharedCollectionResponses([]ente.Collection{collection})
+ encoded, err := json.Marshal(responses[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := `{"id":7,"owner":{"id":8,"email":""},"encryptedKey":"share-key","type":"album","attributes":{},"updationTime":10,"isDeleted":true}`
+ if string(encoded) != want {
+ t.Fatalf("deleted shared collection response = %s, want %s", encoded, want)
+ }
+}
diff --git a/server/pkg/controller/collections/shared_sync_test.go b/server/pkg/controller/collections/shared_sync_test.go
new file mode 100644
index 0000000000..8b78443e11
--- /dev/null
+++ b/server/pkg/controller/collections/shared_sync_test.go
@@ -0,0 +1,152 @@
+package collections
+
+import (
+ "database/sql"
+ "errors"
+ "testing"
+
+ "github.com/ente/museum/ente"
+)
+
+func TestSharedCollectionSyncUsesShareAndCollectionChanges(t *testing.T) {
+ db, collectionRepo, ownerID, shareeID := setupCollectionShareTest(t)
+ collectionID := createShareTestCollection(t, collectionRepo, ownerID)
+ addShareTestShare(t, collectionRepo, collectionID, ownerID, shareeID, ente.VIEWER)
+
+ sharedMetadata := ente.MagicMetadata{
+ Version: 1,
+ Count: 1,
+ Data: "sharee-metadata",
+ Header: "sharee-header",
+ }
+ if _, err := db.Exec(
+ `UPDATE collection_shares SET magic_metadata = $1, updation_time = 2
+ WHERE collection_id = $2 AND to_user_id = $3`,
+ sharedMetadata,
+ collectionID,
+ shareeID,
+ ); err != nil {
+ t.Fatal(err)
+ }
+
+ limit := int64(10)
+ changes, err := collectionRepo.GetCollectionsSharedWithUser(shareeID, 1, ente.Photos, &limit)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(changes) != 1 {
+ t.Fatalf("share-only changes = %d, want 1", len(changes))
+ }
+ if changes[0].UpdationTime != 2 {
+ t.Fatalf("share-only updationTime = %d, want 2", changes[0].UpdationTime)
+ }
+ if changes[0].SharedMagicMetadata == nil || changes[0].SharedMagicMetadata.Data != sharedMetadata.Data {
+ t.Fatalf("share-only metadata = %+v, want %+v", changes[0].SharedMagicMetadata, sharedMetadata)
+ }
+
+ if _, err := db.Exec(
+ `UPDATE collections SET name = 'renamed collection', updation_time = 3
+ WHERE collection_id = $1`,
+ collectionID,
+ ); err != nil {
+ t.Fatal(err)
+ }
+ changes, err = collectionRepo.GetCollectionsSharedWithUser(shareeID, 2, ente.Photos, &limit)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(changes) != 1 {
+ t.Fatalf("collection-only changes = %d, want 1", len(changes))
+ }
+ if changes[0].UpdationTime != 3 || changes[0].Name != "renamed collection" {
+ t.Fatalf("collection-only change = %+v", changes[0])
+ }
+}
+
+func TestUnshareEmitsOneRedactedTombstone(t *testing.T) {
+ db, collectionRepo, ownerID, shareeID := setupCollectionShareTest(t)
+ collectionID := createShareTestCollection(t, collectionRepo, ownerID)
+ addShareTestShare(t, collectionRepo, collectionID, ownerID, shareeID, ente.VIEWER)
+
+ metadata := ente.MagicMetadata{Version: 1, Count: 1, Data: "secret", Header: "header"}
+ if _, err := db.Exec(
+ `UPDATE collections
+ SET encrypted_name = 'encrypted-name', name_decryption_nonce = 'name-nonce',
+ magic_metadata = $1, pub_magic_metadata = $1
+ WHERE collection_id = $2`,
+ metadata,
+ collectionID,
+ ); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(
+ `UPDATE collection_shares SET magic_metadata = $1
+ WHERE collection_id = $2 AND to_user_id = $3`,
+ metadata,
+ collectionID,
+ shareeID,
+ ); err != nil {
+ t.Fatal(err)
+ }
+
+ status, err := collectionRepo.UnShareContext(t.Context(), collectionID, shareeID)
+ if err != nil || status != ente.CollectionUnshared {
+ t.Fatalf("UnShareContext() = %q, %v", status, err)
+ }
+ var deletionTime int64
+ if err := db.QueryRow(
+ `SELECT updation_time FROM collection_shares
+ WHERE collection_id = $1 AND to_user_id = $2`,
+ collectionID,
+ shareeID,
+ ).Scan(&deletionTime); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := db.Exec(`UPDATE users SET encrypted_email = NULL, email_decryption_nonce = NULL WHERE user_id = $1`, ownerID); err != nil {
+ t.Fatal(err)
+ }
+
+ changes, err := collectionRepo.GetCollectionsSharedWithUser(shareeID, deletionTime-1, ente.Photos, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(changes) != 1 {
+ t.Fatalf("changes after unshare = %d, want 1", len(changes))
+ }
+ tombstone := changes[0]
+ if !tombstone.IsDeleted || tombstone.UpdationTime != deletionTime {
+ t.Fatalf("tombstone deletion state = %+v, want time %d", tombstone, deletionTime)
+ }
+ if tombstone.ID != collectionID || tombstone.Owner.ID != ownerID || tombstone.EncryptedKey != "share-key" || tombstone.Type != "album" {
+ t.Fatalf("tombstone compatibility fields = %+v", tombstone)
+ }
+ if tombstone.Owner.Email != "" || tombstone.Name != "" || tombstone.EncryptedName != "" || tombstone.NameDecryptionNonce != "" || tombstone.KeyDecryptionNonce != "" || tombstone.App != "" || tombstone.SharedAt != nil || tombstone.MagicMetadata != nil || tombstone.PublicMagicMetadata != nil || tombstone.SharedMagicMetadata != nil || len(tombstone.Sharees) != 0 || len(tombstone.PublicURLs) != 0 {
+ t.Fatalf("tombstone contains live collection data: %+v", tombstone)
+ }
+
+ changes, err = collectionRepo.GetCollectionsSharedWithUser(shareeID, deletionTime, ente.Photos, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(changes) != 0 {
+ t.Fatalf("replayed tombstones = %d, want 0", len(changes))
+ }
+
+ if _, err := db.Exec(`UPDATE collections SET name = 'post-unshare secret', updation_time = $1 WHERE collection_id = $2`, deletionTime+1, collectionID); err != nil {
+ t.Fatal(err)
+ }
+ changes, err = collectionRepo.GetCollectionsSharedWithUser(shareeID, deletionTime, ente.Photos, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(changes) != 0 {
+ t.Fatalf("post-unshare collection changes = %d, want 0", len(changes))
+ }
+
+ controller := newBatchShareTestController(db, collectionRepo)
+ _, _, err = controller.GetDiffV2(newBatchShareTestContext(shareeID), collectionID, shareeID, 0)
+ if !errors.Is(err, sql.ErrNoRows) {
+ t.Fatalf("GetDiffV2() error = %v, want sql.ErrNoRows", err)
+ }
+}
diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.go
index 26ceaf28c1..43bc05f41e 100644
--- a/server/pkg/repo/collection.go
+++ b/server/pkg/repo/collection.go
@@ -255,15 +255,36 @@ pct.access_token, pct.valid_till, pct.device_limit, pct.created_at, pct.updated_
func (repo *CollectionRepository) GetCollectionsSharedWithUser(userID int64, updationTime int64, app ente.App, limit *int64) ([]ente.Collection, error) {
query := `
- SELECT collections.collection_id, collections.owner_id, users.encrypted_email, users.email_decryption_nonce, collection_shares.encrypted_key, collections.name, collections.encrypted_name, collections.name_decryption_nonce, collections.type, collections.app, collections.pub_magic_metadata, collection_shares.magic_metadata, collections.updation_time, collection_shares.is_deleted, collection_shares.role_type, collection_shares.shared_at
+ SELECT collections.collection_id, collections.owner_id,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE users.encrypted_email END,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE users.email_decryption_nonce END,
+ collection_shares.encrypted_key,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.name END,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.encrypted_name END,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.name_decryption_nonce END,
+ collections.type,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.app::text END,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collections.pub_magic_metadata END,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collection_shares.magic_metadata END,
+ CASE WHEN collection_shares.is_deleted THEN collection_shares.updation_time ELSE GREATEST(collection_shares.updation_time, collections.updation_time) END AS effective_updation_time,
+ collection_shares.is_deleted,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collection_shares.role_type END,
+ CASE WHEN collection_shares.is_deleted THEN NULL ELSE collection_shares.shared_at END
FROM collections
INNER JOIN users
ON collections.owner_id = users.user_id
INNER JOIN collection_shares
- ON collections.collection_id = collection_shares.collection_id AND collection_shares.to_user_id = $1 AND (collection_shares.updation_time > $2 OR collections.updation_time > $2) AND users.encrypted_email IS NOT NULL AND app = $3`
+ ON collections.collection_id = collection_shares.collection_id
+ WHERE collection_shares.to_user_id = $1
+ AND (collection_shares.is_deleted = TRUE OR users.encrypted_email IS NOT NULL)
+ AND collections.app = $3
+ AND (
+ (collection_shares.is_deleted = FALSE AND (collection_shares.updation_time > $2 OR collections.updation_time > $2))
+ OR (collection_shares.is_deleted = TRUE AND collection_shares.updation_time > $2)
+ )`
args := []interface{}{userID, updationTime, string(app)}
if limit != nil {
- query += " ORDER BY collections.updation_time ASC LIMIT $4"
+ query += " ORDER BY effective_updation_time ASC LIMIT $4"
args = append(args, *limit)
}
@@ -279,11 +300,12 @@ func (repo *CollectionRepository) GetCollectionsSharedWithUser(userID int64, upd
var c ente.Collection
var collectionName, encryptedName, nameDecryptionNonce sql.NullString
var encryptedEmail, emailDecryptionNonce []byte
- var roleType sql.NullString
+ var collectionApp, roleType sql.NullString
var sharedAt sql.NullInt64
- if err := rows.Scan(&c.ID, &c.Owner.ID, &encryptedEmail, &emailDecryptionNonce, &c.EncryptedKey, &collectionName, &encryptedName, &nameDecryptionNonce, &c.Type, &c.App, &c.PublicMagicMetadata, &c.SharedMagicMetadata, &c.UpdationTime, &c.IsDeleted, &roleType, &sharedAt); err != nil {
+ if err := rows.Scan(&c.ID, &c.Owner.ID, &encryptedEmail, &emailDecryptionNonce, &c.EncryptedKey, &collectionName, &encryptedName, &nameDecryptionNonce, &c.Type, &collectionApp, &c.PublicMagicMetadata, &c.SharedMagicMetadata, &c.UpdationTime, &c.IsDeleted, &roleType, &sharedAt); err != nil {
return collections, stacktrace.Propagate(err, "")
}
+ c.App = collectionApp.String
if sharedAt.Valid {
sharedAtValue := sharedAt.Int64
c.SharedAt = &sharedAtValue
diff --git a/web/apps/locker/src/services/locker-db.ts b/web/apps/locker/src/services/locker-db.ts
index 8af459a989..f0dad78811 100644
--- a/web/apps/locker/src/services/locker-db.ts
+++ b/web/apps/locker/src/services/locker-db.ts
@@ -256,6 +256,20 @@ export const saveCollectionRecords = async (
await tx.done;
};
+export const deleteCollectionRecords = async (
+ collectionIDs: number[],
+ userID = ensureLocalUser().id,
+) => {
+ if (collectionIDs.length === 0) return;
+
+ const db = await resolveLockerDB(userID);
+ const tx = db.transaction("collections", "readwrite");
+ await Promise.all(
+ collectionIDs.map((collectionID) => tx.store.delete(collectionID)),
+ );
+ await tx.done;
+};
+
export const saveCollectionsSinceTime = async (
sinceTime: number,
userID = ensureLocalUser().id,
diff --git a/web/apps/locker/src/services/sync/schemas.ts b/web/apps/locker/src/services/sync/schemas.ts
index d6caf300c8..36db132f3c 100644
--- a/web/apps/locker/src/services/sync/schemas.ts
+++ b/web/apps/locker/src/services/sync/schemas.ts
@@ -28,8 +28,26 @@ export const RemoteCollection = z.object({
export type RemoteCollection = z.infer<typeof RemoteCollection>;
+export const RemoteCollectionDeletionTombstone = z.object({
+ id: z.number(),
+ owner: z.object({ id: z.number() }),
+ updationTime: z.number(),
+ isDeleted: z.literal(true),
+});
+
+export type RemoteCollectionDeletionTombstone = z.infer<
+ typeof RemoteCollectionDeletionTombstone
+>;
+
+export const RemoteCollectionChange = z.union([
+ RemoteCollection,
+ RemoteCollectionDeletionTombstone,
+]);
+
+export type RemoteCollectionChange = z.infer<typeof RemoteCollectionChange>;
+
export const CollectionsResponse = z.object({
- collections: z.array(RemoteCollection),
+ collections: z.array(RemoteCollectionChange),
});
const RemoteEncryptedMetadata = z.object({
diff --git a/web/apps/locker/src/services/sync/sync.ts b/web/apps/locker/src/services/sync/sync.ts
index c15e487c64..f2cf6b8e98 100644
--- a/web/apps/locker/src/services/sync/sync.ts
+++ b/web/apps/locker/src/services/sync/sync.ts
@@ -1,4 +1,5 @@
import type { LockerCollection, LockerItem } from "@/types";
+import { ensureLocalUser } from "ente-accounts/services/user";
import { authenticatedRequestHeaders, ensureOk } from "ente-base/http";
import log from "ente-base/log";
import { apiURL } from "ente-base/origins";
@@ -11,6 +12,7 @@ import {
} from "../locker-cache";
import {
type StoredTrashFileRecord,
+ deleteCollectionRecords,
deleteCollectionSinceTime,
deleteFileRecords,
deleteFileRecordsForCollection,
@@ -220,6 +222,7 @@ export const syncLockerState = async (): Promise<LockerHydratedState> => {
let latestCollectionsSinceTime = snapshot.collectionsSinceTime;
const changedCollections: EncryptedCollectionRecord[] = [];
+ const revokedCollectionIDs: number[] = [];
const deletedCollectionIDs: number[] = [];
for (const change of collectionChanges) {
@@ -227,6 +230,15 @@ export const syncLockerState = async (): Promise<LockerHydratedState> => {
latestCollectionsSinceTime,
change.updationTime,
);
+ if (change.isDeleted && change.owner.id != ensureLocalUser().id) {
+ revokedCollectionIDs.push(change.id);
+ deletedCollectionIDs.push(change.id);
+ continue;
+ }
+
+ if (!("encryptedKey" in change)) {
+ throw new Error(`Incomplete owned collection change ${change.id}`);
+ }
const record = await toEncryptedCollectionRecord(change);
changedCollections.push(record);
if (record.isDeleted) {
@@ -237,6 +249,7 @@ export const syncLockerState = async (): Promise<LockerHydratedState> => {
if (changedCollections.length > 0) {
await saveCollectionRecords(changedCollections);
}
+ await deleteCollectionRecords(revokedCollectionIDs);
for (const collectionID of deletedCollectionIDs) {
await deleteFileRecordsForCollection(collectionID);
await deleteCollectionSinceTime(collectionID);
diff --git a/web/apps/locker/tests/sync.test.ts b/web/apps/locker/tests/sync.test.ts
index a814881f64..e26e46eec8 100644
--- a/web/apps/locker/tests/sync.test.ts
+++ b/web/apps/locker/tests/sync.test.ts
@@ -13,6 +13,7 @@ import {
vi.mock("../src/services/locker-db", () => ({
loadLockerSnapshotFromDB: vi.fn(),
+ deleteCollectionRecords: vi.fn(),
saveCollectionRecords: vi.fn(),
saveCollectionsSinceTime: vi.fn(),
saveCollectionSinceTime: vi.fn(),
@@ -24,6 +25,9 @@ vi.mock("../src/services/locker-db", () => ({
deleteTrashFileRecords: vi.fn(),
saveTrashSinceTime: vi.fn(),
}));
+vi.mock("ente-accounts/services/user", () => ({
+ ensureLocalUser: () => ({ id: 1 }),
+}));
const { openCollectionKey, decryptMetadataJSON } = vi.hoisted(() => ({
openCollectionKey: vi.fn(),
decryptMetadataJSON: vi.fn(),
@@ -242,6 +246,94 @@ test("incremental sync paginates changed collections and trash, skipping up-to-d
);
});
+test("collection changes remove foreign deletions and retain owned records", async () => {
+ const afterDeletion = {
+ ...snapshot(),
+ collections: new Map([[2, collection(2)]]),
+ files: [file(10, 2)],
+ collectionsSinceTime: 9,
+ collectionSinceTimeByID: new Map([[2, 20]]),
+ };
+ vi.mocked(db.loadLockerSnapshotFromDB)
+ .mockResolvedValueOnce(snapshot())
+ .mockResolvedValueOnce(afterDeletion)
+ .mockResolvedValueOnce(afterDeletion);
+ fetchMock
+ .mockResolvedValueOnce(
+ response({
+ collections: [
+ {
+ id: 1,
+ owner: { id: 7, email: "" },
+ encryptedKey: "legacy-key",
+ type: "folder",
+ updationTime: 9,
+ isDeleted: true,
+ },
+ {
+ id: 3,
+ owner: { id: 7 },
+ updationTime: 10,
+ isDeleted: true,
+ futureField: "ignored",
+ },
+ {
+ id: 4,
+ owner: { id: 1 },
+ encryptedKey: "owned-key",
+ type: "folder",
+ updationTime: 11,
+ isDeleted: true,
+ },
+ {
+ id: 5,
+ owner: { id: 1 },
+ encryptedKey: "active-key",
+ type: "folder",
+ updationTime: 12,
+ isDeleted: false,
+ },
+ ],
+ }),
+ )
+ .mockResolvedValueOnce(response({ diff: [], hasMore: false }));
+
+ await syncLockerState();
+
+ expect(db.deleteCollectionRecords).toHaveBeenCalledExactlyOnceWith([1, 3]);
+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledTimes(3);
+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledWith(1);
+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledWith(3);
+ expect(db.deleteFileRecordsForCollection).toHaveBeenCalledWith(4);
+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledTimes(3);
+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledWith(1);
+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledWith(3);
+ expect(db.deleteCollectionSinceTime).toHaveBeenCalledWith(4);
+ expect(db.saveCollectionRecords).toHaveBeenCalledExactlyOnceWith([
+ expect.objectContaining({
+ id: 4,
+ encryptedKey: "owned-key",
+ isDeleted: true,
+ }),
+ expect.objectContaining({
+ id: 5,
+ encryptedKey: "active-key",
+ isDeleted: false,
+ }),
+ ]);
+ expect(db.saveCollectionsSinceTime).toHaveBeenCalledExactlyOnceWith(12);
+ expect(openCollectionKey).not.toHaveBeenCalledWith(
+ expect.anything(),
+ expect.anything(),
+ "legacy-key",
+ expect.anything(),
+ );
+ expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
+ "/collections/v2?sinceTime=5",
+ "/trash/v2/diff?sinceTime=40",
+ ]);
+});
+
test("failed file pagination does not persist a partial diff or advance its cursor", async () => {
fetchMock
.mockResolvedValueOnce(response({ collections: [] }))
diff --git a/web/apps/photos/src/services/export-collection-state.ts b/web/apps/photos/src/services/export-collection-state.ts
new file mode 100644
index 0000000000..9f94e0c88d
--- /dev/null
+++ b/web/apps/photos/src/services/export-collection-state.ts
@@ -0,0 +1,11 @@
+export const deletedExportedCollectionIDs = (
+ presentCollectionIDs: Iterable<number>,
+ collectionExportNames: Readonly<Record<number, string>> | undefined,
+) => {
+ if (!collectionExportNames) return [];
+
+ const presentIDs = new Set(presentCollectionIDs);
+ return Object.keys(collectionExportNames)
+ .map(Number)
+ .filter((collectionID) => !presentIDs.has(collectionID));
+};
diff --git a/web/apps/photos/src/services/export.ts b/web/apps/photos/src/services/export.ts
index 633b54d4dd..acd451b191 100644
--- a/web/apps/photos/src/services/export.ts
+++ b/web/apps/photos/src/services/export.ts
@@ -36,6 +36,7 @@ import { PromiseQueue } from "ente-utils/promise";
import { nullToUndefined } from "ente-utils/transform";
import i18n from "i18next";
import { z } from "zod";
+import { deletedExportedCollectionIDs } from "./export-collection-state";
const exportStoppedError = () => namedError("export_stopped", "export stopped");
@@ -1224,25 +1225,11 @@ const getRenamedExportedCollections = (
const getDeletedExportedCollections = (
collections: Collection[],
exportRecord: ExportRecord,
-) => {
- if (!exportRecord?.collectionExportNames) {
- return [];
- }
- const presentCollections = new Set(
- collections.map((collection) => collection.id),
+) =>
+ deletedExportedCollectionIDs(
+ collections.map(({ id }) => id),
+ exportRecord.collectionExportNames,
);
- const deletedExportedCollections = Object.keys(
- exportRecord?.collectionExportNames,
- )
- .map(Number)
- .filter((collectionID) => {
- if (!presentCollections.has(collectionID)) {
- return true;
- }
- return false;
- });
- return deletedExportedCollections;
-};
const readOnDiskFileExportRecordIDs = async (
files: EnteFile[],
diff --git a/web/apps/photos/tests/collection-change.test.ts b/web/apps/photos/tests/collection-change.test.ts
new file mode 100644
index 0000000000..219fb97b2a
--- /dev/null
+++ b/web/apps/photos/tests/collection-change.test.ts
@@ -0,0 +1,52 @@
+import {
+ RemoteCollectionChange,
+ RemoteCollectionDeletionTombstone,
+} from "ente-media/collection";
+import { describe, expect, test } from "vitest";
+import { deletedExportedCollectionIDs } from "../src/services/export-collection-state";
+
+describe("collection deletion tombstones", () => {
+ test("parses without active collection fields", () => {
+ const parsed = RemoteCollectionChange.parse({
+ id: 7,
+ owner: { id: 1, email: "" },
+ encryptedKey: "compatibility-key",
+ type: "album",
+ attributes: {},
+ updationTime: 9,
+ isDeleted: true,
+ name: "must not survive parsing",
+ magicMetadata: { data: "must not survive parsing" },
+ });
+
+ expect(parsed).toEqual({
+ id: 7,
+ owner: { id: 1 },
+ updationTime: 9,
+ isDeleted: true,
+ });
+ expect(RemoteCollectionDeletionTombstone.parse(parsed)).toStrictEqual(
+ parsed,
+ );
+ });
+
+ test("keeps active collection fields required", () => {
+ expect(() =>
+ RemoteCollectionChange.parse({
+ id: 7,
+ owner: { id: 1 },
+ updationTime: 9,
+ isDeleted: false,
+ }),
+ ).toThrow();
+ });
+
+ test("desktop export finds removed collections using IDs only", () => {
+ expect(
+ deletedExportedCollectionIDs([1], {
+ 1: "Current album",
+ 7: "Former shared album",
+ }),
+ ).toEqual([7]);
+ });
+});
diff --git a/web/packages/media/collection.ts b/web/packages/media/collection.ts
index cb4569e722..334823fd5b 100644
--- a/web/packages/media/collection.ts
+++ b/web/packages/media/collection.ts
@@ -136,6 +136,26 @@ export const RemoteCollection = z.looseObject({
export type RemoteCollection = z.infer<typeof RemoteCollection>;
+export const RemoteCollectionDeletionTombstone = z.object({
+ id: z.number(),
+ owner: z.object({ id: z.number() }),
+ updationTime: z.number(),
+ isDeleted: z.literal(true),
+});
+
+export type RemoteCollectionDeletionTombstone = z.infer<
+ typeof RemoteCollectionDeletionTombstone
+>;
+
+const RemoteActiveCollection = RemoteCollection.extend({
+ isDeleted: z.literal(false).nullish().transform(nullToUndefined),
+});
+
+export const RemoteCollectionChange = z.union([
+ RemoteCollectionDeletionTombstone,
+ RemoteActiveCollection,
+]);
+
export const decryptRemoteCollection = async (
collection: RemoteCollection,
collectionKey: string,
diff --git a/web/packages/new/photos/services/collection.ts b/web/packages/new/photos/services/collection.ts
index edf36f3e08..ead155c206 100644
--- a/web/packages/new/photos/services/collection.ts
+++ b/web/packages/new/photos/services/collection.ts
@@ -12,6 +12,7 @@ import {
findUserUncategorizedCollection,
maxAlbumDescriptionLength,
RemoteCollection,
+ RemoteCollectionChange,
RemotePublicURL,
type Collection,
type CollectionNewParticipantRole,
@@ -156,7 +157,7 @@ export const getCollectionByID = async (
};
const CollectionsResponse = z.object({
- collections: z.array(RemoteCollection),
+ collections: z.array(RemoteCollectionChange),
});
export interface CollectionChange {