Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Batch Space comment and notification visibility checks

Anand Baburajand8b233360607
diff --git a/server/space/repo/comments.go b/server/space/repo/comments.go
index 48e3b8c223..88654d484e 100644
--- a/server/space/repo/comments.go
+++ b/server/space/repo/comments.go
@@ -52,26 +52,32 @@ const commentVisibleSQL = `NOT EXISTS (
WHERE NOT a.has_content OR account.encrypted_email IS NULL
)`
-const commentRecipientsSQL = `WITH RECURSIVE hidden_comments AS (
- SELECT c.comment_id FROM space_comments c
+const commentNotHiddenSQL = `NOT EXISTS (
+ SELECT 1 FROM hidden_comments hidden WHERE hidden.comment_id = c.comment_id
+)`
+
+func hiddenCommentsCTE(postFilter string) string {
+ return `hidden_comments AS (
+ SELECT c.comment_id, c.post_id FROM space_comments c
LEFT JOIN spaces author ON author.space_id = c.author_space_id
LEFT JOIN users account ON account.user_id = author.owner_id
- WHERE c.post_id = $2 AND (c.cipher IS NULL OR account.encrypted_email IS NULL)
+ WHERE ` + postFilter + ` AND (c.cipher IS NULL OR account.encrypted_email IS NULL)
UNION
- SELECT child.comment_id FROM hidden_comments parent
+ SELECT child.comment_id, parent.post_id FROM hidden_comments parent
CROSS JOIN LATERAL (
SELECT c.comment_id FROM space_comments c
- WHERE c.reply_to_id = parent.comment_id AND c.post_id = $2
+ WHERE c.reply_to_id = parent.comment_id AND c.post_id = parent.post_id
OFFSET 0
) child
-)
+)`
+}
+
+var commentRecipientsSQL = `WITH RECURSIVE ` + hiddenCommentsCTE("c.post_id = $2") + `
SELECT ARRAY(
SELECT participant.space_id FROM (
SELECT $1::text AS space_id
UNION
- SELECT c.author_space_id FROM space_comments c WHERE c.post_id = $2 AND NOT EXISTS (
- SELECT 1 FROM hidden_comments hidden WHERE hidden.comment_id = c.comment_id
- )
+ SELECT c.author_space_id FROM space_comments c WHERE c.post_id = $2 AND ` + commentNotHiddenSQL + `
UNION
SELECT l.actor_space_id FROM space_post_likes l WHERE l.post_id = $2
) participant
@@ -124,8 +130,9 @@ func (r *CommentsRepository) List(ctx context.Context, viewer string, postID int
return nil, "", err
}
createdAt, id, _ := parseMessageCursor(cursor)
- rows, err := tx.QueryContext(ctx, `SELECT c.comment_id, COALESCE(c.parent_id, ''), COALESCE(c.reply_to_id, ''),
- CASE WHEN `+commentVisibleSQL+` THEN c.cipher END, c.created_at,
+ rows, err := tx.QueryContext(ctx, `WITH RECURSIVE `+hiddenCommentsCTE("c.post_id = $1")+`
+ SELECT c.comment_id, COALESCE(c.parent_id, ''), COALESCE(c.reply_to_id, ''),
+ CASE WHEN `+commentNotHiddenSQL+` THEN c.cipher END, c.created_at,
(COALESCE(actor.space_id = $2, FALSE) OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = actor.space_id AND f.friend_space_id = $2)),
COALESCE(actor.space_id, ''), COALESCE(actor.space_slug, ''), actor.public_key, COALESCE(actor.current_version, 0), actor.encrypted_profile,
avatar.object_id, avatar.key_version, avatar.size, COALESCE(actor.updated_at, 0)
@@ -190,8 +197,10 @@ func (r *CommentsRepository) List(ctx context.Context, viewer string, postID int
return nil, "", err
}
rows.Close()
- rows, err = tx.QueryContext(ctx, `SELECT n.comment_id, n.notification_id `+notificationSourcesSQL+`
- WHERE n.recipient_space_id = $1 AND n.comment_id = ANY($2) AND n.read_at IS NULL AND `+notificationVisibleSQL, viewer, pq.Array(commentIDs))
+ const unreadFilter = `n.recipient_space_id = $1 AND n.comment_id = ANY($2) AND n.read_at IS NULL`
+ rows, err = tx.QueryContext(ctx, `WITH RECURSIVE `+notificationHiddenCommentsCTE(unreadFilter)+`
+ SELECT n.comment_id, n.notification_id `+notificationSourcesSQL+`
+ WHERE `+unreadFilter+` AND `+notificationVisibleSQL(commentNotHiddenSQL), viewer, pq.Array(commentIDs))
if err != nil {
return nil, "", err
}
@@ -370,7 +379,7 @@ func (r *CommentsRepository) NotificationRecipients(ctx context.Context, actor s
rows, err := r.DB.QueryContext(ctx, `SELECT recipient.owner_id, n.kind, post_owner.owner_id, post_owner.space_id, post_owner.space_slug `+notificationSourcesSQL+`
JOIN spaces recipient ON recipient.space_id = n.recipient_space_id
JOIN spaces post_owner ON post_owner.space_id = p.space_id
- WHERE n.actor_space_id = $1 AND p.post_id = $2 AND n.comment_id = $3 AND (n.kind = 'comment_reaction') = $4 AND `+notificationVisibleSQL, actor, postID, commentID, reaction)
+ WHERE n.actor_space_id = $1 AND p.post_id = $2 AND n.comment_id = $3 AND (n.kind = 'comment_reaction') = $4 AND `+notificationVisibleSQL(commentVisibleSQL), actor, postID, commentID, reaction)
if err != nil {
return nil, owner, err
}
diff --git a/server/space/repo/comments_test.go b/server/space/repo/comments_test.go
index 79931ddc6c..f5cc2315c7 100644
--- a/server/space/repo/comments_test.go
+++ b/server/space/repo/comments_test.go
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"encoding/json"
+ "fmt"
"net/http"
"testing"
"time"
@@ -174,6 +175,107 @@ func TestPostCommentCountWorkScalesWithReplyChainLength(t *testing.T) {
}
}
+func TestDiscussionBatchReadsWithDeepReplyChain(t *testing.T) {
+ module := newSpaceTestModule(t)
+ ctx := t.Context()
+ owner, friend := notificationTestSpaces(t, module)
+ rootAuthorID := insertSpaceUser(t, module, "[email protected]", "root-public")
+ rootAuthor, err := testCreateSpace(ctx, module, rootAuthorID, "chain_root", "root", "public", "secret", "nonce", "profile")
+ require.NoError(t, err)
+ require.NoError(t, testAddFriend(ctx, module, rootAuthorID, rootAuthor.SpaceID, owner.SpaceID, "share", 1, "share", 1))
+ postID, err := testCreatePost(ctx, module, owner.OwnerID, owner.SpaceID, "post-key", nil, 1, nil)
+ require.NoError(t, err)
+ _, err = module.Comments.DB.ExecContext(ctx, `INSERT INTO space_comments
+ (comment_id, post_id, author_space_id, parent_id, reply_to_id, cipher, created_at)
+ SELECT 'chain-' || i, $1, CASE WHEN i = 1 THEN $2 ELSE $3 END,
+ CASE WHEN i > 1 THEN 'chain-1' END,
+ CASE WHEN i > 1 THEN 'chain-' || (i - 1) END, '\x01'::bytea, i
+ FROM generate_series(1, 1000) i`, postID, rootAuthor.SpaceID, owner.SpaceID)
+ require.NoError(t, err)
+ _, err = module.Comments.DB.ExecContext(ctx, `INSERT INTO space_comment_reactions (comment_id, actor_space_id, cipher)
+ SELECT comment_id, $2, '\x01'::bytea FROM space_comments WHERE post_id = $1 AND created_at > 1`, postID, friend.SpaceID)
+ require.NoError(t, err)
+ _, err = module.Comments.DB.ExecContext(ctx, `INSERT INTO space_notifications
+ (notification_id, recipient_space_id, actor_space_id, kind, comment_id, created_at)
+ SELECT 'notification-' || i, $1, $2, 'comment_reaction', 'chain-' || i, i
+ FROM generate_series(2, 1000) i`, owner.SpaceID, friend.SpaceID)
+ require.NoError(t, err)
+ _, err = module.Comments.DB.ExecContext(ctx, `ANALYZE space_comments; ANALYZE space_notifications`)
+ require.NoError(t, err)
+ notificationIDs := make([]string, 0, 999)
+ for i := 2; i <= 1000; i++ {
+ notificationIDs = append(notificationIDs, fmt.Sprintf("notification-%d", i))
+ }
+ for _, state := range []string{"visible", "deleted-comment", "inactive-author", "recovered"} {
+ t.Run(state, func(t *testing.T) {
+ _, err := module.Comments.DB.ExecContext(ctx, `UPDATE space_comments
+ SET cipher = CASE WHEN $1 = 'deleted-comment' THEN NULL ELSE '\x01'::bytea END
+ WHERE comment_id = 'chain-1'`, state)
+ require.NoError(t, err)
+ _, err = module.Comments.DB.ExecContext(ctx, `UPDATE users
+ SET encrypted_email = CASE WHEN $2 = 'inactive-author' THEN NULL ELSE '\x00'::bytea END
+ WHERE user_id = $1`, rootAuthorID, state)
+ require.NoError(t, err)
+ ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second)
+ defer cancel()
+ hidden := state == "deleted-comment" || state == "inactive-author"
+ started := time.Now()
+ items, cursor, err := module.Comments.List(ctx, owner.SpaceID, postID, "900:chain-900")
+ require.NoError(t, err)
+ t.Logf("last 100 comments: %s", time.Since(started))
+ require.Len(t, items, 100)
+ require.Empty(t, cursor)
+ for i, item := range items {
+ require.Equal(t, fmt.Sprintf("chain-%d", i+901), item.ID)
+ require.Equal(t, "chain-1", item.ParentID)
+ require.Equal(t, fmt.Sprintf("chain-%d", i+900), item.ReplyToID)
+ if hidden {
+ require.Empty(t, item.Cipher)
+ require.Empty(t, item.Reactions)
+ require.Empty(t, item.UnreadNotificationIDs)
+ require.False(t, item.CanDelete)
+ } else {
+ require.Equal(t, []byte{1}, item.Cipher)
+ require.Len(t, item.Reactions, 1)
+ require.Equal(t, []string{fmt.Sprintf("notification-%d", i+901)}, item.UnreadNotificationIDs)
+ }
+ }
+ items, cursor, err = module.Comments.List(ctx, owner.SpaceID, postID, "")
+ require.NoError(t, err)
+ require.Len(t, items, 100)
+ require.Equal(t, "100:chain-100", cursor)
+ started = time.Now()
+ notifications, cursor, err := module.Notifications.List(ctx, owner.SpaceID, "", 2)
+ require.NoError(t, err)
+ t.Logf("notifications: %s", time.Since(started))
+ if hidden {
+ require.Empty(t, notifications)
+ require.Empty(t, cursor)
+ } else {
+ require.Len(t, notifications, 2)
+ require.Equal(t, "chain-1000", notifications[0].CommentID)
+ require.Equal(t, "chain-999", notifications[1].CommentID)
+ require.Equal(t, "999:notification-999", cursor)
+ }
+ unread, err := module.Notifications.HasUnread(ctx, owner.SpaceID)
+ require.NoError(t, err)
+ require.Equal(t, !hidden, unread)
+ require.NoError(t, module.Notifications.MarkRead(ctx, owner.SpaceID, notificationIDs))
+ unread, err = module.Notifications.HasUnread(ctx, owner.SpaceID)
+ require.NoError(t, err)
+ require.False(t, unread)
+ readCount := countSpaceRows(t, module, `SELECT count(*) FROM space_notifications WHERE read_at IS NOT NULL`)
+ if hidden {
+ require.Zero(t, readCount)
+ } else {
+ require.EqualValues(t, 999, readCount)
+ }
+ _, err = module.Notifications.DB.ExecContext(ctx, `UPDATE space_notifications SET read_at = NULL`)
+ require.NoError(t, err)
+ })
+ }
+}
+
func TestDiscussionParentDeletionRemovesWholeThread(t *testing.T) {
for _, deleteAsOwner := range []bool{false, true} {
name := "author"
diff --git a/server/space/repo/notifications.go b/server/space/repo/notifications.go
index 79246c6b36..978d1995d8 100644
--- a/server/space/repo/notifications.go
+++ b/server/space/repo/notifications.go
@@ -38,11 +38,20 @@ const notificationSourcesSQL = `
JOIN users actor_owner ON actor_owner.user_id = actor.owner_id AND actor_owner.encrypted_email IS NOT NULL
`
-const notificationVisibleSQL = `
+func notificationHiddenCommentsCTE(filter string) string {
+ return hiddenCommentsCTE(`c.post_id IN (
+ SELECT c.post_id FROM space_notifications n
+ JOIN space_comments c ON c.comment_id = n.comment_id
+ WHERE ` + filter + `
+ )`)
+}
+
+func notificationVisibleSQL(commentVisibilitySQL string) string {
+ return `
(n.kind = 'friend_request' OR (
n.comment_id IS NOT NULL AND c.comment_id IS NOT NULL AND p.post_id IS NOT NULL
AND EXISTS (SELECT 1 FROM spaces s JOIN users u ON u.user_id = s.owner_id AND u.encrypted_email IS NOT NULL WHERE s.space_id = p.space_id)
- AND ` + commentVisibleSQL + `
+ AND ` + commentVisibilitySQL + `
AND (p.space_id = n.recipient_space_id OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = p.space_id AND f.friend_space_id = n.recipient_space_id))
AND (p.space_id = n.actor_space_id OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = p.space_id AND f.friend_space_id = n.actor_space_id))
AND (n.kind <> 'comment' OR p.space_id = n.recipient_space_id OR EXISTS (SELECT 1 FROM space_friend_shares f WHERE f.space_id = n.recipient_space_id AND f.friend_space_id = n.actor_space_id))
@@ -52,13 +61,15 @@ const notificationVisibleSQL = `
WHERE share.space_id = n.recipient_space_id AND share.friend_space_id = n.actor_space_id)
))
`
+}
func (r *NotificationsRepository) List(ctx context.Context, recipientSpaceID, cursor string, limit int) ([]SpaceNotificationRecord, string, error) {
limit = optionalInt(limit, 50)
if limit > 100 {
limit = 100
}
- query := `WITH visible AS (
+ const filter = `n.recipient_space_id = $1`
+ query := `WITH RECURSIVE ` + notificationHiddenCommentsCTE(filter) + `, visible AS (
SELECT n.*, p.post_id, p.space_id AS post_space_id, c.reply_to_id, CASE
WHEN n.kind IN ('post_like', 'comment') THEN p.post_id::text
WHEN n.kind = 'comment_reaction' THEN n.comment_id
@@ -66,7 +77,7 @@ func (r *NotificationsRepository) List(ctx context.Context, recipientSpaceID, cu
WHEN n.kind IN ('friend_accepted', 'poke') THEN n.actor_space_id
ELSE n.notification_id
END AS group_id
- ` + notificationSourcesSQL + ` WHERE n.recipient_space_id = $1 AND ` + notificationVisibleSQL + `
+ ` + notificationSourcesSQL + ` WHERE ` + filter + ` AND ` + notificationVisibleSQL(commentNotHiddenSQL) + `
), ranked AS (
SELECT notification_id, kind, actor_space_id, post_id, post_space_id, comment_id, reply_to_id, friend_request_id, created_at, group_id,
ROW_NUMBER() OVER (PARTITION BY kind, group_id ORDER BY created_at DESC, notification_id DESC) AS position,
@@ -147,15 +158,17 @@ func (r *NotificationsRepository) List(ctx context.Context, recipientSpaceID, cu
func (r *NotificationsRepository) HasUnread(ctx context.Context, recipientSpaceID string) (bool, error) {
var unread bool
- err := r.DB.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 `+notificationSourcesSQL+` WHERE n.recipient_space_id = $1 AND n.read_at IS NULL AND `+notificationVisibleSQL+`)`, recipientSpaceID).Scan(&unread)
+ const filter = `n.recipient_space_id = $1 AND n.read_at IS NULL`
+ err := r.DB.QueryRowContext(ctx, `WITH RECURSIVE `+notificationHiddenCommentsCTE(filter)+`
+ SELECT EXISTS (SELECT 1 `+notificationSourcesSQL+` WHERE `+filter+` AND `+notificationVisibleSQL(commentNotHiddenSQL)+`)`, recipientSpaceID).Scan(&unread)
return unread, stacktrace.Propagate(err, "")
}
func (r *NotificationsRepository) MarkRead(ctx context.Context, recipientSpaceID string, notificationIDs []string) error {
- _, err := r.DB.ExecContext(ctx, `
- WITH visible AS (
+ const filter = `n.recipient_space_id = $1 AND n.notification_id = ANY($2) AND n.read_at IS NULL`
+ _, err := r.DB.ExecContext(ctx, `WITH RECURSIVE `+notificationHiddenCommentsCTE(filter)+`, visible AS (
SELECT n.notification_id
- `+notificationSourcesSQL+` WHERE n.recipient_space_id = $1 AND n.notification_id = ANY($2) AND `+notificationVisibleSQL+`
+ `+notificationSourcesSQL+` WHERE `+filter+` AND `+notificationVisibleSQL(commentNotHiddenSQL)+`
)
UPDATE space_notifications n SET read_at = now_utc_micro_seconds()
FROM visible item
diff --git a/server/space/repo/notifications_test.go b/server/space/repo/notifications_test.go
index 2e576520a1..9db4295b66 100644
--- a/server/space/repo/notifications_test.go
+++ b/server/space/repo/notifications_test.go
@@ -9,6 +9,7 @@ import (
"testing"
"time"
+ "github.com/ente/museum/space/models"
"github.com/lib/pq"
"github.com/stretchr/testify/require"
)
@@ -202,6 +203,60 @@ func TestNotificationsPaginationAndReadIsolation(t *testing.T) {
require.False(t, unread)
}
+func TestNotificationsHideBranchesAcrossPostsBeforePagination(t *testing.T) {
+ module := newSpaceTestModule(t)
+ ctx := t.Context()
+ owner, friend := notificationTestSpaces(t, module)
+ postIDs := make([]int64, 3)
+ for i := range postIDs {
+ postID, err := testCreatePost(ctx, module, owner.OwnerID, owner.SpaceID, "post-key", nil, 1, nil)
+ require.NoError(t, err)
+ postIDs[i] = postID
+ }
+ for _, item := range []struct {
+ id, replyTo string
+ post int
+ createdAt int64
+ }{
+ {"older", "", 0, 10},
+ {"newer", "", 2, 20},
+ {"hidden-parent", "", 0, 100},
+ {"hidden-child", "hidden-parent", 0, 101},
+ {"hidden-post-parent", "", 1, 200},
+ {"hidden-post-child", "hidden-post-parent", 1, 201},
+ } {
+ require.NoError(t, module.Comments.Create(ctx, friend.SpaceID, postIDs[item.post], models.CreateCommentRequest{CommentID: item.id, ReplyToID: item.replyTo}, []byte(item.id)))
+ _, err := module.Notifications.DB.ExecContext(ctx, `UPDATE space_notifications SET created_at = $2 WHERE comment_id = $1`, item.id, item.createdAt)
+ require.NoError(t, err)
+ }
+ _, err := module.Comments.DB.ExecContext(ctx, `UPDATE space_comments SET cipher = NULL WHERE comment_id IN ('hidden-parent', 'hidden-post-parent')`)
+ require.NoError(t, err)
+ first, cursor, err := module.Notifications.List(ctx, owner.SpaceID, "", 1)
+ require.NoError(t, err)
+ require.Len(t, first, 1)
+ require.Equal(t, "newer", first[0].CommentID)
+ require.Len(t, first[0].NotificationIDs, 1)
+ require.NotEmpty(t, cursor)
+ second, next, err := module.Notifications.List(ctx, owner.SpaceID, cursor, 1)
+ require.NoError(t, err)
+ require.Len(t, second, 1)
+ require.Equal(t, "older", second[0].CommentID)
+ require.Len(t, second[0].NotificationIDs, 1)
+ require.EqualValues(t, 1, second[0].ActorCount)
+ require.Empty(t, next)
+ require.NoError(t, module.Notifications.MarkRead(ctx, owner.SpaceID, first[0].NotificationIDs))
+ unread, err := module.Notifications.HasUnread(ctx, owner.SpaceID)
+ require.NoError(t, err)
+ require.True(t, unread)
+ var allIDs pq.StringArray
+ require.NoError(t, module.Notifications.DB.QueryRowContext(ctx, `SELECT array_agg(notification_id) FROM space_notifications WHERE recipient_space_id = $1`, owner.SpaceID).Scan(&allIDs))
+ require.NoError(t, module.Notifications.MarkRead(ctx, owner.SpaceID, allIDs))
+ unread, err = module.Notifications.HasUnread(ctx, owner.SpaceID)
+ require.NoError(t, err)
+ require.False(t, unread)
+ require.EqualValues(t, 4, countSpaceRows(t, module, `SELECT count(*) FROM space_notifications WHERE read_at IS NULL`))
+}
+
func TestNotificationsPaginateCompletePostGroups(t *testing.T) {
module := newSpaceTestModule(t)
ctx := t.Context()
diff --git a/server/space/repo/posts.go b/server/space/repo/posts.go
index 559a6a6eed..f921817792 100644
--- a/server/space/repo/posts.go
+++ b/server/space/repo/posts.go
@@ -80,23 +80,9 @@ func postRecordSelectSQL(viewerLikedExpr string, viewerSpaceParam string) string
p.encrypted_post_key, p.caption_cipher,
p.key_version, p.created_at,
` + viewerLikedExpr + ` AS viewer_liked,
- (WITH RECURSIVE hidden_comments AS (
- SELECT c.comment_id FROM space_comments c
- LEFT JOIN spaces author ON author.space_id = c.author_space_id
- LEFT JOIN users account ON account.user_id = author.owner_id
- WHERE c.post_id = p.post_id AND (c.cipher IS NULL OR account.encrypted_email IS NULL)
- UNION
- SELECT child.comment_id FROM hidden_comments parent
- CROSS JOIN LATERAL (
- SELECT c.comment_id FROM space_comments c
- WHERE c.reply_to_id = parent.comment_id AND c.post_id = p.post_id
- OFFSET 0
- ) child
- )
+ (WITH RECURSIVE ` + hiddenCommentsCTE("c.post_id = p.post_id") + `
SELECT COUNT(*) FROM space_comments c
- WHERE c.post_id = p.post_id AND NOT EXISTS (
- SELECT 1 FROM hidden_comments hidden WHERE hidden.comment_id = c.comment_id
- )) AS comment_count,
+ WHERE c.post_id = p.post_id AND ` + commentNotHiddenSQL + `) AS comment_count,
CASE WHEN p.space_id = ` + viewerSpaceParam + ` THEN (
SELECT COUNT(*) FROM space_post_likes l
JOIN spaces actor ON actor.space_id = l.actor_space_id