π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Batch Space comment and notification visibility checks
Anand Baburajand8b233360607
diff --git a/server/space/repo/comments.go b/server/space/repo/comments.goindex 48e3b8c223..88654d484e 100644--- a/server/space/repo/comments.go+++ b/server/space/repo/comments.go@@ -52,26 +52,32 @@ const commentVisibleSQL = `NOT EXISTS (WHERE NOT a.has_content OR account.encrypted_email IS NULL)`-const commentRecipientsSQL = `WITH RECURSIVE hidden_comments AS (- SELECT c.comment_id FROM space_comments c+const commentNotHiddenSQL = `NOT EXISTS (+ SELECT 1 FROM hidden_comments hidden WHERE hidden.comment_id = c.comment_id+)`++func hiddenCommentsCTE(postFilter string) string {+ return `hidden_comments AS (+ SELECT c.comment_id, c.post_id FROM space_comments cLEFT JOIN spaces author ON author.space_id = c.author_space_idLEFT JOIN users account ON account.user_id = author.owner_id- WHERE c.post_id = $2 AND (c.cipher IS NULL OR account.encrypted_email IS NULL)+ WHERE ` + postFilter + ` AND (c.cipher IS NULL OR account.encrypted_email IS NULL)UNION- SELECT child.comment_id FROM hidden_comments parent+ SELECT child.comment_id, parent.post_id FROM hidden_comments parentCROSS JOIN LATERAL (SELECT c.comment_id FROM space_comments c- WHERE c.reply_to_id = parent.comment_id AND c.post_id = $2+ WHERE c.reply_to_id = parent.comment_id AND c.post_id = parent.post_idOFFSET 0) child-)+)`+}++var commentRecipientsSQL = `WITH RECURSIVE ` + hiddenCommentsCTE("c.post_id = $2") + `SELECT ARRAY(SELECT participant.space_id FROM (SELECT $1::text AS space_idUNION- SELECT c.author_space_id FROM space_comments c WHERE c.post_id = $2 AND NOT EXISTS (- SELECT 1 FROM hidden_comments hidden WHERE hidden.comment_id = c.comment_id- )+ SELECT c.author_space_id FROM space_comments c WHERE c.post_id = $2 AND ` + commentNotHiddenSQL + `UNIONSELECT l.actor_space_id FROM space_post_likes l WHERE l.post_id = $2) participant@@ -124,8 +130,9 @@ func (r *CommentsRepository) List(ctx context.Context, viewer string, postID intreturn nil, "", err}createdAt, id, _ := parseMessageCursor(cursor)- rows, err := tx.QueryContext(ctx, `SELECT c.comment_id, COALESCE(c.parent_id, ''), COALESCE(c.reply_to_id, ''),- CASE WHEN `+commentVisibleSQL+` THEN c.cipher END, c.created_at,+ rows, err := tx.QueryContext(ctx, `WITH RECURSIVE `+hiddenCommentsCTE("c.post_id = $1")+`+ SELECT c.comment_id, COALESCE(c.parent_id, ''), COALESCE(c.reply_to_id, ''),+ CASE WHEN `+commentNotHiddenSQL+` THEN c.cipher END, c.created_at,(COALESCE(actor.space_id = $2, FALSE) OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = actor.space_id AND f.friend_space_id = $2)),COALESCE(actor.space_id, ''), COALESCE(actor.space_slug, ''), actor.public_key, COALESCE(actor.current_version, 0), actor.encrypted_profile,avatar.object_id, avatar.key_version, avatar.size, COALESCE(actor.updated_at, 0)@@ -190,8 +197,10 @@ func (r *CommentsRepository) List(ctx context.Context, viewer string, postID intreturn nil, "", err}rows.Close()- rows, err = tx.QueryContext(ctx, `SELECT n.comment_id, n.notification_id `+notificationSourcesSQL+`- WHERE n.recipient_space_id = $1 AND n.comment_id = ANY($2) AND n.read_at IS NULL AND `+notificationVisibleSQL, viewer, pq.Array(commentIDs))+ const unreadFilter = `n.recipient_space_id = $1 AND n.comment_id = ANY($2) AND n.read_at IS NULL`+ rows, err = tx.QueryContext(ctx, `WITH RECURSIVE `+notificationHiddenCommentsCTE(unreadFilter)+`+ SELECT n.comment_id, n.notification_id `+notificationSourcesSQL+`+ WHERE `+unreadFilter+` AND `+notificationVisibleSQL(commentNotHiddenSQL), viewer, pq.Array(commentIDs))if err != nil {return nil, "", err}@@ -370,7 +379,7 @@ func (r *CommentsRepository) NotificationRecipients(ctx context.Context, actor srows, err := r.DB.QueryContext(ctx, `SELECT recipient.owner_id, n.kind, post_owner.owner_id, post_owner.space_id, post_owner.space_slug `+notificationSourcesSQL+`JOIN spaces recipient ON recipient.space_id = n.recipient_space_idJOIN spaces post_owner ON post_owner.space_id = p.space_id- WHERE n.actor_space_id = $1 AND p.post_id = $2 AND n.comment_id = $3 AND (n.kind = 'comment_reaction') = $4 AND `+notificationVisibleSQL, actor, postID, commentID, reaction)+ WHERE n.actor_space_id = $1 AND p.post_id = $2 AND n.comment_id = $3 AND (n.kind = 'comment_reaction') = $4 AND `+notificationVisibleSQL(commentVisibleSQL), actor, postID, commentID, reaction)if err != nil {return nil, owner, err}diff --git a/server/space/repo/comments_test.go b/server/space/repo/comments_test.goindex 79931ddc6c..f5cc2315c7 100644--- a/server/space/repo/comments_test.go+++ b/server/space/repo/comments_test.go@@ -4,6 +4,7 @@ import ("context""database/sql""encoding/json"+ "fmt""net/http""testing""time"@@ -174,6 +175,107 @@ func TestPostCommentCountWorkScalesWithReplyChainLength(t *testing.T) {}}+func TestDiscussionBatchReadsWithDeepReplyChain(t *testing.T) {+ module := newSpaceTestModule(t)+ ctx := t.Context()+ owner, friend := notificationTestSpaces(t, module)+ rootAuthorID := insertSpaceUser(t, module, "[email protected]", "root-public")+ rootAuthor, err := testCreateSpace(ctx, module, rootAuthorID, "chain_root", "root", "public", "secret", "nonce", "profile")+ require.NoError(t, err)+ require.NoError(t, testAddFriend(ctx, module, rootAuthorID, rootAuthor.SpaceID, owner.SpaceID, "share", 1, "share", 1))+ postID, err := testCreatePost(ctx, module, owner.OwnerID, owner.SpaceID, "post-key", nil, 1, nil)+ require.NoError(t, err)+ _, err = module.Comments.DB.ExecContext(ctx, `INSERT INTO space_comments+ (comment_id, post_id, author_space_id, parent_id, reply_to_id, cipher, created_at)+ SELECT 'chain-' || i, $1, CASE WHEN i = 1 THEN $2 ELSE $3 END,+ CASE WHEN i > 1 THEN 'chain-1' END,+ CASE WHEN i > 1 THEN 'chain-' || (i - 1) END, '\x01'::bytea, i+ FROM generate_series(1, 1000) i`, postID, rootAuthor.SpaceID, owner.SpaceID)+ require.NoError(t, err)+ _, err = module.Comments.DB.ExecContext(ctx, `INSERT INTO space_comment_reactions (comment_id, actor_space_id, cipher)+ SELECT comment_id, $2, '\x01'::bytea FROM space_comments WHERE post_id = $1 AND created_at > 1`, postID, friend.SpaceID)+ require.NoError(t, err)+ _, err = module.Comments.DB.ExecContext(ctx, `INSERT INTO space_notifications+ (notification_id, recipient_space_id, actor_space_id, kind, comment_id, created_at)+ SELECT 'notification-' || i, $1, $2, 'comment_reaction', 'chain-' || i, i+ FROM generate_series(2, 1000) i`, owner.SpaceID, friend.SpaceID)+ require.NoError(t, err)+ _, err = module.Comments.DB.ExecContext(ctx, `ANALYZE space_comments; ANALYZE space_notifications`)+ require.NoError(t, err)+ notificationIDs := make([]string, 0, 999)+ for i := 2; i <= 1000; i++ {+ notificationIDs = append(notificationIDs, fmt.Sprintf("notification-%d", i))+ }+ for _, state := range []string{"visible", "deleted-comment", "inactive-author", "recovered"} {+ t.Run(state, func(t *testing.T) {+ _, err := module.Comments.DB.ExecContext(ctx, `UPDATE space_comments+ SET cipher = CASE WHEN $1 = 'deleted-comment' THEN NULL ELSE '\x01'::bytea END+ WHERE comment_id = 'chain-1'`, state)+ require.NoError(t, err)+ _, err = module.Comments.DB.ExecContext(ctx, `UPDATE users+ SET encrypted_email = CASE WHEN $2 = 'inactive-author' THEN NULL ELSE '\x00'::bytea END+ WHERE user_id = $1`, rootAuthorID, state)+ require.NoError(t, err)+ ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second)+ defer cancel()+ hidden := state == "deleted-comment" || state == "inactive-author"+ started := time.Now()+ items, cursor, err := module.Comments.List(ctx, owner.SpaceID, postID, "900:chain-900")+ require.NoError(t, err)+ t.Logf("last 100 comments: %s", time.Since(started))+ require.Len(t, items, 100)+ require.Empty(t, cursor)+ for i, item := range items {+ require.Equal(t, fmt.Sprintf("chain-%d", i+901), item.ID)+ require.Equal(t, "chain-1", item.ParentID)+ require.Equal(t, fmt.Sprintf("chain-%d", i+900), item.ReplyToID)+ if hidden {+ require.Empty(t, item.Cipher)+ require.Empty(t, item.Reactions)+ require.Empty(t, item.UnreadNotificationIDs)+ require.False(t, item.CanDelete)+ } else {+ require.Equal(t, []byte{1}, item.Cipher)+ require.Len(t, item.Reactions, 1)+ require.Equal(t, []string{fmt.Sprintf("notification-%d", i+901)}, item.UnreadNotificationIDs)+ }+ }+ items, cursor, err = module.Comments.List(ctx, owner.SpaceID, postID, "")+ require.NoError(t, err)+ require.Len(t, items, 100)+ require.Equal(t, "100:chain-100", cursor)+ started = time.Now()+ notifications, cursor, err := module.Notifications.List(ctx, owner.SpaceID, "", 2)+ require.NoError(t, err)+ t.Logf("notifications: %s", time.Since(started))+ if hidden {+ require.Empty(t, notifications)+ require.Empty(t, cursor)+ } else {+ require.Len(t, notifications, 2)+ require.Equal(t, "chain-1000", notifications[0].CommentID)+ require.Equal(t, "chain-999", notifications[1].CommentID)+ require.Equal(t, "999:notification-999", cursor)+ }+ unread, err := module.Notifications.HasUnread(ctx, owner.SpaceID)+ require.NoError(t, err)+ require.Equal(t, !hidden, unread)+ require.NoError(t, module.Notifications.MarkRead(ctx, owner.SpaceID, notificationIDs))+ unread, err = module.Notifications.HasUnread(ctx, owner.SpaceID)+ require.NoError(t, err)+ require.False(t, unread)+ readCount := countSpaceRows(t, module, `SELECT count(*) FROM space_notifications WHERE read_at IS NOT NULL`)+ if hidden {+ require.Zero(t, readCount)+ } else {+ require.EqualValues(t, 999, readCount)+ }+ _, err = module.Notifications.DB.ExecContext(ctx, `UPDATE space_notifications SET read_at = NULL`)+ require.NoError(t, err)+ })+ }+}+func TestDiscussionParentDeletionRemovesWholeThread(t *testing.T) {for _, deleteAsOwner := range []bool{false, true} {name := "author"diff --git a/server/space/repo/notifications.go b/server/space/repo/notifications.goindex 79246c6b36..978d1995d8 100644--- a/server/space/repo/notifications.go+++ b/server/space/repo/notifications.go@@ -38,11 +38,20 @@ const notificationSourcesSQL = `JOIN users actor_owner ON actor_owner.user_id = actor.owner_id AND actor_owner.encrypted_email IS NOT NULL`-const notificationVisibleSQL = `+func notificationHiddenCommentsCTE(filter string) string {+ return hiddenCommentsCTE(`c.post_id IN (+ SELECT c.post_id FROM space_notifications n+ JOIN space_comments c ON c.comment_id = n.comment_id+ WHERE ` + filter + `+ )`)+}++func notificationVisibleSQL(commentVisibilitySQL string) string {+ return `(n.kind = 'friend_request' OR (n.comment_id IS NOT NULL AND c.comment_id IS NOT NULL AND p.post_id IS NOT NULLAND EXISTS (SELECT 1 FROM spaces s JOIN users u ON u.user_id = s.owner_id AND u.encrypted_email IS NOT NULL WHERE s.space_id = p.space_id)- AND ` + commentVisibleSQL + `+ AND ` + commentVisibilitySQL + `AND (p.space_id = n.recipient_space_id OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = p.space_id AND f.friend_space_id = n.recipient_space_id))AND (p.space_id = n.actor_space_id OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = p.space_id AND f.friend_space_id = n.actor_space_id))AND (n.kind <> 'comment' OR p.space_id = n.recipient_space_id OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = n.recipient_space_id AND f.friend_space_id = n.actor_space_id))@@ -52,13 +61,15 @@ const notificationVisibleSQL = `WHERE share.space_id = n.recipient_space_id AND share.friend_space_id = n.actor_space_id)))`+}func (r *NotificationsRepository) List(ctx context.Context, recipientSpaceID, cursor string, limit int) ([]SpaceNotificationRecord, string, error) {limit = optionalInt(limit, 50)if limit > 100 {limit = 100}- query := `WITH visible AS (+ const filter = `n.recipient_space_id = $1`+ query := `WITH RECURSIVE ` + notificationHiddenCommentsCTE(filter) + `, visible AS (SELECT n.*, p.post_id, p.space_id AS post_space_id, c.reply_to_id, CASEWHEN n.kind IN ('post_like', 'comment') THEN p.post_id::textWHEN n.kind = 'comment_reaction' THEN n.comment_id@@ -66,7 +77,7 @@ func (r *NotificationsRepository) List(ctx context.Context, recipientSpaceID, cuWHEN n.kind IN ('friend_accepted', 'poke') THEN n.actor_space_idELSE n.notification_idEND AS group_id- ` + notificationSourcesSQL + ` WHERE n.recipient_space_id = $1 AND ` + notificationVisibleSQL + `+ ` + notificationSourcesSQL + ` WHERE ` + filter + ` AND ` + notificationVisibleSQL(commentNotHiddenSQL) + `), ranked AS (SELECT notification_id, kind, actor_space_id, post_id, post_space_id, comment_id, reply_to_id, friend_request_id, created_at, group_id,ROW_NUMBER() OVER (PARTITION BY kind, group_id ORDER BY created_at DESC, notification_id DESC) AS position,@@ -147,15 +158,17 @@ func (r *NotificationsRepository) List(ctx context.Context, recipientSpaceID, cufunc (r *NotificationsRepository) HasUnread(ctx context.Context, recipientSpaceID string) (bool, error) {var unread bool- err := r.DB.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 `+notificationSourcesSQL+` WHERE n.recipient_space_id = $1 AND n.read_at IS NULL AND `+notificationVisibleSQL+`)`, recipientSpaceID).Scan(&unread)+ const filter = `n.recipient_space_id = $1 AND n.read_at IS NULL`+ err := r.DB.QueryRowContext(ctx, `WITH RECURSIVE `+notificationHiddenCommentsCTE(filter)+`+ SELECT EXISTS (SELECT 1 `+notificationSourcesSQL+` WHERE `+filter+` AND `+notificationVisibleSQL(commentNotHiddenSQL)+`)`, recipientSpaceID).Scan(&unread)return unread, stacktrace.Propagate(err, "")}func (r *NotificationsRepository) MarkRead(ctx context.Context, recipientSpaceID string, notificationIDs []string) error {- _, err := r.DB.ExecContext(ctx, `- WITH visible AS (+ const filter = `n.recipient_space_id = $1 AND n.notification_id = ANY($2) AND n.read_at IS NULL`+ _, err := r.DB.ExecContext(ctx, `WITH RECURSIVE `+notificationHiddenCommentsCTE(filter)+`, visible AS (SELECT n.notification_id- `+notificationSourcesSQL+` WHERE n.recipient_space_id = $1 AND n.notification_id = ANY($2) AND `+notificationVisibleSQL+`+ `+notificationSourcesSQL+` WHERE `+filter+` AND `+notificationVisibleSQL(commentNotHiddenSQL)+`)UPDATE space_notifications n SET read_at = now_utc_micro_seconds()FROM visible itemdiff --git a/server/space/repo/notifications_test.go b/server/space/repo/notifications_test.goindex 2e576520a1..9db4295b66 100644--- a/server/space/repo/notifications_test.go+++ b/server/space/repo/notifications_test.go@@ -9,6 +9,7 @@ import ("testing""time"+ "github.com/ente/museum/space/models""github.com/lib/pq""github.com/stretchr/testify/require")@@ -202,6 +203,60 @@ func TestNotificationsPaginationAndReadIsolation(t *testing.T) {require.False(t, unread)}+func TestNotificationsHideBranchesAcrossPostsBeforePagination(t *testing.T) {+ module := newSpaceTestModule(t)+ ctx := t.Context()+ owner, friend := notificationTestSpaces(t, module)+ postIDs := make([]int64, 3)+ for i := range postIDs {+ postID, err := testCreatePost(ctx, module, owner.OwnerID, owner.SpaceID, "post-key", nil, 1, nil)+ require.NoError(t, err)+ postIDs[i] = postID+ }+ for _, item := range []struct {+ id, replyTo string+ post int+ createdAt int64+ }{+ {"older", "", 0, 10},+ {"newer", "", 2, 20},+ {"hidden-parent", "", 0, 100},+ {"hidden-child", "hidden-parent", 0, 101},+ {"hidden-post-parent", "", 1, 200},+ {"hidden-post-child", "hidden-post-parent", 1, 201},+ } {+ require.NoError(t, module.Comments.Create(ctx, friend.SpaceID, postIDs[item.post], models.CreateCommentRequest{CommentID: item.id, ReplyToID: item.replyTo}, []byte(item.id)))+ _, err := module.Notifications.DB.ExecContext(ctx, `UPDATE space_notifications SET created_at = $2 WHERE comment_id = $1`, item.id, item.createdAt)+ require.NoError(t, err)+ }+ _, err := module.Comments.DB.ExecContext(ctx, `UPDATE space_comments SET cipher = NULL WHERE comment_id IN ('hidden-parent', 'hidden-post-parent')`)+ require.NoError(t, err)+ first, cursor, err := module.Notifications.List(ctx, owner.SpaceID, "", 1)+ require.NoError(t, err)+ require.Len(t, first, 1)+ require.Equal(t, "newer", first[0].CommentID)+ require.Len(t, first[0].NotificationIDs, 1)+ require.NotEmpty(t, cursor)+ second, next, err := module.Notifications.List(ctx, owner.SpaceID, cursor, 1)+ require.NoError(t, err)+ require.Len(t, second, 1)+ require.Equal(t, "older", second[0].CommentID)+ require.Len(t, second[0].NotificationIDs, 1)+ require.EqualValues(t, 1, second[0].ActorCount)+ require.Empty(t, next)+ require.NoError(t, module.Notifications.MarkRead(ctx, owner.SpaceID, first[0].NotificationIDs))+ unread, err := module.Notifications.HasUnread(ctx, owner.SpaceID)+ require.NoError(t, err)+ require.True(t, unread)+ var allIDs pq.StringArray+ require.NoError(t, module.Notifications.DB.QueryRowContext(ctx, `SELECT array_agg(notification_id) FROM space_notifications WHERE recipient_space_id = $1`, owner.SpaceID).Scan(&allIDs))+ require.NoError(t, module.Notifications.MarkRead(ctx, owner.SpaceID, allIDs))+ unread, err = module.Notifications.HasUnread(ctx, owner.SpaceID)+ require.NoError(t, err)+ require.False(t, unread)+ require.EqualValues(t, 4, countSpaceRows(t, module, `SELECT count(*) FROM space_notifications WHERE read_at IS NULL`))+}+func TestNotificationsPaginateCompletePostGroups(t *testing.T) {module := newSpaceTestModule(t)ctx := t.Context()diff --git a/server/space/repo/posts.go b/server/space/repo/posts.goindex 559a6a6eed..f921817792 100644--- a/server/space/repo/posts.go+++ b/server/space/repo/posts.go@@ -80,23 +80,9 @@ func postRecordSelectSQL(viewerLikedExpr string, viewerSpaceParam string) stringp.encrypted_post_key, p.caption_cipher,p.key_version, p.created_at,` + viewerLikedExpr + ` AS viewer_liked,- (WITH RECURSIVE hidden_comments AS (- SELECT c.comment_id FROM space_comments c- LEFT JOIN spaces author ON author.space_id = c.author_space_id- LEFT JOIN users account ON account.user_id = author.owner_id- WHERE c.post_id = p.post_id AND (c.cipher IS NULL OR account.encrypted_email IS NULL)- UNION- SELECT child.comment_id FROM hidden_comments parent- CROSS JOIN LATERAL (- SELECT c.comment_id FROM space_comments c- WHERE c.reply_to_id = parent.comment_id AND c.post_id = p.post_id- OFFSET 0- ) child- )+ (WITH RECURSIVE ` + hiddenCommentsCTE("c.post_id = p.post_id") + `SELECT COUNT(*) FROM space_comments c- WHERE c.post_id = p.post_id AND NOT EXISTS (- SELECT 1 FROM hidden_comments hidden WHERE hidden.comment_id = c.comment_id- )) AS comment_count,+ WHERE c.post_id = p.post_id AND ` + commentNotHiddenSQL + `) AS comment_count,CASE WHEN p.space_id = ` + viewerSpaceParam + ` THEN (SELECT COUNT(*) FROM space_post_likes lJOIN spaces actor ON actor.space_id = l.actor_space_id