💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Canonicalize custom domain ownership
Neeraj Guptab0cad53b0170
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex 4912f89389..b7fc3d0c4b 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -1093,6 +1093,11 @@ func main() {setKnownAPIs(server.Routes())setupAndStartBackgroundJobs(objectCleanupController, replicationController3, fileDataCtrl, contactController, spaceModule)+ time.AfterFunc(10*time.Minute, func() {+ if err := remoteStoreRepository.BackfillCustomDomainCanonicalValues(context.Background()); err != nil {+ log.WithError(err).Error("Failed to backfill custom domain canonical values")+ }+ })setupAndStartCrons(userAuthRepo, collectionLinkRepo, fileLinkRepo, pasteRepo, twoFactorRepo, passkeysRepo, fileController, taskLockingRepo, emailNotificationCtrl,trashController, pushController, objectController, dataCleanupController, storageBonusCtrl, emergencyCtrl,diff --git a/server/ente/remotestore.go b/server/ente/remotestore.goindex 2302fb6ba2..0d556fed80 100644--- a/server/ente/remotestore.go+++ b/server/ente/remotestore.go@@ -192,6 +192,11 @@ func ResolveCustomDomainValue(value string) (string, error) {return value, nil}+func CanonicalDomain(domain string) (string, error) {+ asciiDomain, err := idna.Lookup.ToASCII(domain)+ return strings.ToLower(asciiDomain), err+}+func isValidDomainWithoutScheme(input string) error {trimmed := strings.TrimSpace(input)if trimmed != input {@@ -204,7 +209,7 @@ func isValidDomainWithoutScheme(input string) error {return NewBadRequestWithMessage("domain should not contain scheme (e.g., http:// or https://)")}- asciiDomain, err := idna.ToASCII(trimmed)+ asciiDomain, err := CanonicalDomain(trimmed)if err != nil {return NewBadRequestWithMessage(fmt.Sprintf("invalid idn domain format: %s", trimmed))}diff --git a/server/migrations/132_custom_domain_canonical.down.sql b/server/migrations/132_custom_domain_canonical.down.sqlnew file mode 100644index 0000000000..70c0642973--- /dev/null+++ b/server/migrations/132_custom_domain_canonical.down.sql@@ -0,0 +1,3 @@+DROP INDEX IF EXISTS remote_store_custom_domain_canonical_unique_idx;++ALTER TABLE remote_store DROP COLUMN canonical_value;diff --git a/server/migrations/132_custom_domain_canonical.up.sql b/server/migrations/132_custom_domain_canonical.up.sqlnew file mode 100644index 0000000000..5effb21330--- /dev/null+++ b/server/migrations/132_custom_domain_canonical.up.sql@@ -0,0 +1,5 @@+ALTER TABLE remote_store ADD COLUMN canonical_value TEXT;++CREATE UNIQUE INDEX remote_store_custom_domain_canonical_unique_idx+ ON remote_store (canonical_value)+ WHERE key_name = 'customDomain' AND canonical_value IS NOT NULL;diff --git a/server/pkg/controller/remotestore/controller.go b/server/pkg/controller/remotestore/controller.goindex 489cb3e93b..8501e1b3cd 100644--- a/server/pkg/controller/remotestore/controller.go+++ b/server/pkg/controller/remotestore/controller.go@@ -154,7 +154,7 @@ func (c *Controller) insertOrUpdateCustomDomain(ctx *gin.Context, userID int64,ownerID, err := c.DomainOwner(ctx, value)if err == nil {if ownerID != nil && *ownerID == userID {- if err := c.Repo.InsertOrUpdate(ctx, userID, string(ente.CustomDomain), value); err != nil {+ if err := c.Repo.InsertOrUpdateCustomDomain(ctx, userID, value); err != nil {return err}return c.updateFamilyCustomDomainsIfAdmin(ctx, userID, value)@@ -179,7 +179,7 @@ func (c *Controller) insertOrUpdateCustomDomain(ctx *gin.Context, userID int64,if !errors.Is(err, &ente.ErrNotFoundError) {return stacktrace.Propagate(err, "")}- if err := c.Repo.InsertOrUpdate(ctx, userID, string(ente.CustomDomain), value); err != nil {+ if err := c.Repo.InsertOrUpdateCustomDomain(ctx, userID, value); err != nil {return err}return c.updateFamilyCustomDomainsIfAdmin(ctx, userID, value)@@ -301,10 +301,10 @@ func (c *Controller) DomainOwner(ctx *gin.Context, domain string) (*int64, error// Retry with ASCII/Unicode variants so IDN domains stored in either form still resolve.var candidates []string- if asciiDomain, convErr := idna.ToASCII(domain); convErr == nil && asciiDomain != domain {+ if asciiDomain, convErr := ente.CanonicalDomain(domain); convErr == nil && !strings.EqualFold(asciiDomain, domain) {candidates = append(candidates, asciiDomain)}- if unicodeDomain, convErr := idna.ToUnicode(domain); convErr == nil && unicodeDomain != domain {+ if unicodeDomain, convErr := idna.Lookup.ToUnicode(domain); convErr == nil && !strings.EqualFold(unicodeDomain, domain) {candidates = append(candidates, unicodeDomain)}diff --git a/server/pkg/repo/remotestore/repository.go b/server/pkg/repo/remotestore/repository.goindex 7ead8dc593..5f06a8d080 100644--- a/server/pkg/repo/remotestore/repository.go+++ b/server/pkg/repo/remotestore/repository.go@@ -16,24 +16,87 @@ type Repository struct {}func (r *Repository) InsertOrUpdate(ctx context.Context, userID int64, key string, value string) error {- _, err := r.DB.ExecContext(ctx, `INSERT INTO remote_store(user_id, key_name, key_value) VALUES ($1,$2,$3)- ON CONFLICT (user_id, key_name) DO UPDATE SET key_value = $3;+ return r.insertOrUpdate(ctx, userID, key, value, nil)+}++func (r *Repository) InsertOrUpdateCustomDomain(ctx context.Context, userID int64, value string) error {+ canonicalDomain, err := ente.CanonicalDomain(value)+ if err != nil {+ return stacktrace.Propagate(err, "failed to canonicalize custom domain")+ }+ return r.insertOrUpdate(ctx, userID, string(ente.CustomDomain), value, &canonicalDomain)+}++func (r *Repository) BackfillCustomDomainCanonicalValues(ctx context.Context) error {+ tx, err := r.DB.BeginTx(ctx, nil)+ if err != nil {+ return stacktrace.Propagate(err, "failed to begin custom domain backfill")+ }+ defer tx.Rollback()++ rows, err := tx.QueryContext(ctx, `SELECT user_id, key_value FROM remote_store+ WHERE key_name = $1 AND left(key_value, 1) <> '_' FOR UPDATE`, ente.CustomDomain)+ if err != nil {+ return stacktrace.Propagate(err, "failed to fetch custom domains")+ }+ defer rows.Close()++ type domainRow struct {+ userID int64+ value string+ }+ var domains []domainRow+ for rows.Next() {+ var domain domainRow+ if err := rows.Scan(&domain.userID, &domain.value); err != nil {+ return stacktrace.Propagate(err, "failed to scan custom domain")+ }+ domains = append(domains, domain)+ }+ if err := rows.Err(); err != nil {+ return stacktrace.Propagate(err, "failed to read custom domains")+ }+ if err := rows.Close(); err != nil {+ return stacktrace.Propagate(err, "failed to close custom domain rows")+ }++ for _, domain := range domains {+ if err := ente.ValidatePublicCustomDomain(domain.value); err != nil {+ return stacktrace.Propagate(err, "invalid stored custom domain")+ }+ canonicalDomain, err := ente.CanonicalDomain(domain.value)+ if err != nil {+ return stacktrace.Propagate(err, "failed to canonicalize stored custom domain")+ }+ if _, err := tx.ExecContext(ctx, `UPDATE remote_store SET canonical_value = $1+ WHERE user_id = $2 AND key_name = $3 AND canonical_value IS DISTINCT FROM $1`,+ canonicalDomain, domain.userID, ente.CustomDomain); err != nil {+ return stacktrace.Propagate(err, "failed to backfill custom domain")+ }+ }+ return stacktrace.Propagate(tx.Commit(), "failed to commit custom domain backfill")+}++func (r *Repository) insertOrUpdate(ctx context.Context, userID int64, key string, value string, canonicalValue *string) error {+ _, err := r.DB.ExecContext(ctx, `INSERT INTO remote_store(user_id, key_name, key_value, canonical_value) VALUES ($1,$2,$3,$4)+ ON CONFLICT (user_id, key_name) DO UPDATE SET key_value = $3, canonical_value = $4;`,userID,key,value,+ canonicalValue,)if err != nil {var pgErr *pq.Errorif errors.As(err, &pgErr) && pgErr.Code == "23505" {- if pgErr.Constraint == "remote_store_custom_domain_unique_idx" {+ if pgErr.Constraint == "remote_store_custom_domain_unique_idx" || pgErr.Constraint == "remote_store_custom_domain_canonical_unique_idx" {return ente.NewConflictError("custom domain already exists for another user")}}return stacktrace.Propagate(err, "failed to insert/update")}- return stacktrace.Propagate(err, "failed to insert/update")+ return nil}func (r *Repository) RemoveKey(ctx context.Context, userID int64, key string) error {@@ -47,7 +110,7 @@ func (r *Repository) RemoveKey(ctx context.Context, userID int64, key string) erfunc (r *Repository) DomainOwner(ctx context.Context, domain string) (*int64, error) {rows := r.DB.QueryRowContext(ctx, `SELECT user_id FROM remote_store- WHERE key_name = $1 AND key_value = $2`,+ WHERE key_name = $1 AND (canonical_value = $2 OR lower(key_value) = lower($2))`,ente.CustomDomain,domain,)diff --git a/server/pkg/repo/remotestore/repository_test.go b/server/pkg/repo/remotestore/repository_test.gonew file mode 100644index 0000000000..edefbe1e96--- /dev/null+++ b/server/pkg/repo/remotestore/repository_test.go@@ -0,0 +1,57 @@+package remotestore++import (+ "testing"++ "github.com/ente/museum/ente"+ "github.com/ente/museum/internal/testutil"+ "github.com/stretchr/testify/require"+)++func TestCustomDomainCanonicalUniqueness(t *testing.T) {+ testutil.WithServerRoot(t)+ db := testutil.RequireTestDB(t)+ testutil.ResetTables(t, db)+ t.Cleanup(func() { testutil.ResetTables(t, db) })++ ownerID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 1, Email: "[email protected]", CreationTime: 1})+ otherID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})+ repository := &Repository{DB: db}++ require.NoError(t, repository.InsertOrUpdateCustomDomain(t.Context(), ownerID, "BÜCHER.example"))+ domain, err := repository.GetDomain(t.Context(), ownerID)+ require.NoError(t, err)+ require.Equal(t, "BÜCHER.example", *domain)+ resolvedOwnerID, err := repository.DomainOwner(t.Context(), "xn--bcher-kva.example")+ require.NoError(t, err)+ require.Equal(t, ownerID, *resolvedOwnerID)++ err = repository.InsertOrUpdateCustomDomain(t.Context(), otherID, "bu\u0308cher.example")+ var apiErr *ente.ApiError+ require.ErrorAs(t, err, &apiErr)+ require.Equal(t, ente.CONFLICT, apiErr.Code)++ require.NoError(t, repository.InsertOrUpdateCustomDomain(t.Context(), ownerID, "xn--bcher-kva.example"))+ domain, err = repository.GetDomain(t.Context(), ownerID)+ require.NoError(t, err)+ require.Equal(t, "xn--bcher-kva.example", *domain)++ pointer := ente.BuildFamilyCustomDomainPointer(ownerID, "family.example")+ require.NoError(t, repository.InsertOrUpdate(t.Context(), ownerID, string(ente.CustomDomain), pointer))+ require.NoError(t, repository.InsertOrUpdateCustomDomain(t.Context(), otherID, "BÜCHER.example"))++ _, err = db.ExecContext(t.Context(), `UPDATE remote_store SET key_value = 'LEGACY.example'+ WHERE user_id = $1 AND key_name = 'customDomain'`, otherID)+ require.NoError(t, err)+ require.NoError(t, repository.BackfillCustomDomainCanonicalValues(t.Context()))++ var value string+ var canonicalValue *string+ require.NoError(t, db.QueryRowContext(t.Context(), `SELECT key_value, canonical_value FROM remote_store+ WHERE user_id = $1 AND key_name = 'customDomain'`, otherID).Scan(&value, &canonicalValue))+ require.Equal(t, "LEGACY.example", value)+ require.Equal(t, "legacy.example", *canonicalValue)+ require.NoError(t, db.QueryRowContext(t.Context(), `SELECT canonical_value FROM remote_store+ WHERE user_id = $1 AND key_name = 'customDomain'`, ownerID).Scan(&canonicalValue))+ require.Nil(t, canonicalValue)+}