Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

26.0 KB
Raw Blame
package store_test

import (
	"context"
	"database/sql"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/bytebase/bytebase/backend/common/testcontainer"

	"github.com/pkg/errors"
	"github.com/stretchr/testify/require"
	"google.golang.org/protobuf/proto"

	storepb "github.com/bytebase/bytebase/backend/generated-go/store"
	"github.com/bytebase/bytebase/backend/store"
)

// newSettingAtomicFixture boots a real PostgreSQL, migrates the schema, seeds
// the workspace and an empty WORKSPACE_PROFILE row, and returns a store with
// the setting cache ENABLED — the production non-HA shape, so the tests also
// pin that the cache reflects committed state.
func newSettingAtomicFixture(t *testing.T) (context.Context, *sql.DB, *store.Store) {
	t.Helper()
	return newSettingAtomicFixtureWithCache(t, true)
}

func newSettingAtomicFixtureWithCache(t *testing.T, enableCache bool) (context.Context, *sql.DB, *store.Store) {
	t.Helper()
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	t.Cleanup(cancel)
	db, s, _ := testcontainer.NewMetadataDBWithCache(t, enableCache)

	_, err := db.ExecContext(ctx, `INSERT INTO workspace (resource_id) VALUES ('default');`)
	require.NoError(t, err)

	_, err = s.UpsertSetting(ctx, &store.SettingMessage{
		Name:      storepb.SettingName_WORKSPACE_PROFILE,
		Workspace: "default",
		Value:     &storepb.WorkspaceProfileSetting{},
	})
	require.NoError(t, err)
	return ctx, db, s
}

func mustProfile(t *testing.T, setting *store.SettingMessage) *storepb.WorkspaceProfileSetting {
	t.Helper()
	profile, ok := setting.Value.(*storepb.WorkspaceProfileSetting)
	require.True(t, ok, "unexpected setting value type %T", setting.Value)
	return profile
}

// TestUpdateSettingAtomicInterleaving pins the lost-update protection: while
// one update holds the row lock inside its apply callback, a concurrent update
// to a DIFFERENT field must wait behind it and then merge onto the committed
// result — both mutations survive in the final row and in the setting cache.
// Against a read-then-UpsertSetting implementation this fails: the second
// update merges onto a stale read and silently reverts the first's field.
func TestUpdateSettingAtomicInterleaving(t *testing.T) {
	t.Parallel()
	ctx, db, s := newSettingAtomicFixture(t)

	entered := make(chan struct{})
	release := make(chan struct{})
	firstResult := make(chan error, 1)
	go func() {
		_, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
			func(current proto.Message) (proto.Message, error) {
				profile, ok := current.(*storepb.WorkspaceProfileSetting)
				if !ok {
					return nil, errors.Errorf("unexpected type %T", current)
				}
				profile.EnableMetricCollection = true
				close(entered)
				<-release
				return profile, nil
			}, nil)
		firstResult <- err
	}()
	// The first update is now inside apply and holds the row lock.
	select {
	case <-entered:
	case <-time.After(15 * time.Second):
		t.Fatal("first update never reached its apply callback")
	}

	secondResult := make(chan error, 1)
	go func() {
		_, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
			func(current proto.Message) (proto.Message, error) {
				profile, ok := current.(*storepb.WorkspaceProfileSetting)
				if !ok {
					return nil, errors.Errorf("unexpected type %T", current)
				}
				profile.Watermark = true
				return profile, nil
			}, nil)
		secondResult <- err
	}()

	// Deterministic barrier: the second update must be observed waiting behind
	// the first's row lock before the first is released.
	require.Eventually(t, func() bool {
		var waiting bool
		err := db.QueryRowContext(ctx, `
			SELECT EXISTS (
				SELECT 1
				FROM pg_stat_activity
				WHERE cardinality(pg_blocking_pids(pid)) > 0
					AND query LIKE '%setting%FOR UPDATE%'
			)
		`).Scan(&waiting)
		return err == nil && waiting
	}, 10*time.Second, 10*time.Millisecond,
		"the second update should wait behind the first's row lock")
	select {
	case err := <-secondResult:
		t.Fatalf("second update completed while the first held the row lock: %v", err)
	default:
	}

	close(release)
	for _, result := range []chan error{firstResult, secondResult} {
		select {
		case err := <-result:
			require.NoError(t, err)
		case <-time.After(15 * time.Second):
			t.Fatal("update did not complete; possible deadlock")
		}
	}

	// Both mutations survive in the database ...
	fresh, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	freshProfile := mustProfile(t, fresh)
	require.True(t, freshProfile.EnableMetricCollection, "first update's field must survive")
	require.True(t, freshProfile.Watermark,
		"second update's field must survive")
	// ... and in the setting cache.
	cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	cachedProfile := mustProfile(t, cached)
	require.True(t, cachedProfile.EnableMetricCollection)
	require.True(t, cachedProfile.Watermark)
}

// TestUpdateSettingAtomicApplyAbort pins that an error from apply aborts the
// transaction with no write, leaves the cache untouched, and surfaces
// unwrapped so callers keep typed errors.
func TestUpdateSettingAtomicApplyAbort(t *testing.T) {
	t.Parallel()
	ctx, _, s := newSettingAtomicFixture(t)

	sentinel := errors.New("validation failed")
	_, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
		func(current proto.Message) (proto.Message, error) {
			profile, ok := current.(*storepb.WorkspaceProfileSetting)
			if !ok {
				return nil, errors.Errorf("unexpected type %T", current)
			}
			// Mutate before failing: the mutation must not leak into the row
			// or the served cache.
			profile.EnableMetricCollection = true
			return nil, sentinel
		}, nil)
	require.ErrorIs(t, err, sentinel)

	fresh, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	require.False(t, mustProfile(t, fresh).EnableMetricCollection)
	cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	require.False(t, mustProfile(t, cached).EnableMetricCollection)
}

// TestUpdateSettingAtomicMissingRow pins that a missing setting row is an
// error, not a silent create — the primitive updates existing state only.
func TestUpdateSettingAtomicMissingRow(t *testing.T) {
	t.Parallel()
	ctx, _, s := newSettingAtomicFixture(t)

	_, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_AI,
		func(current proto.Message) (proto.Message, error) {
			return current, nil
		}, nil)
	require.Error(t, err)
	require.Contains(t, err.Error(), "not found")
}

func TestUpsertSettingAtomicMergesConcurrentWritesOnAMissingRow(t *testing.T) {
	t.Parallel()
	ctx, db, s := newSettingAtomicFixture(t)

	upsertAI := func(wantExisted bool, set func(*storepb.AISetting), before func()) error {
		_, err := s.UpsertSettingAtomic(ctx, "default", storepb.SettingName_AI,
			func(current proto.Message, existed bool) (proto.Message, error) {
				setting, ok := current.(*storepb.AISetting)
				if !ok {
					return nil, errors.Errorf("unexpected type %T", current)
				}
				if existed != wantExisted {
					return nil, errors.Errorf("existed is %t, want %t", existed, wantExisted)
				}
				set(setting)
				before()
				return setting, nil
			}, nil)
		return err
	}

	entered := make(chan struct{})
	release := make(chan struct{})
	firstResult := make(chan error, 1)
	go func() {
		firstResult <- upsertAI(false, func(setting *storepb.AISetting) { setting.SqlReviewEnabled = true }, func() {
			close(entered)
			<-release
		})
	}()
	select {
	case <-entered:
	case <-time.After(15 * time.Second):
		t.Fatal("first upsert never reached its apply callback")
	}

	secondResult := make(chan error, 1)
	go func() {
		secondResult <- upsertAI(true, func(setting *storepb.AISetting) { setting.Model = "second" }, func() {})
	}()
	// The second insert waits on the first's uncommitted row.
	require.Eventually(t, func() bool {
		var waiting bool
		err := db.QueryRowContext(ctx, `
			SELECT EXISTS (
				SELECT 1
				FROM pg_stat_activity
				WHERE datname = current_database()
					AND cardinality(pg_blocking_pids(pid)) > 0
					AND query LIKE '%INSERT INTO setting%ON CONFLICT%'
			)
		`).Scan(&waiting)
		return err == nil && waiting
	}, 10*time.Second, 10*time.Millisecond)

	close(release)
	for _, result := range []chan error{firstResult, secondResult} {
		select {
		case err := <-result:
			require.NoError(t, err)
		case <-time.After(15 * time.Second):
			t.Fatal("upsert did not complete; possible deadlock")
		}
	}

	fresh, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_AI)
	require.NoError(t, err)
	setting, ok := fresh.Value.(*storepb.AISetting)
	require.True(t, ok)
	require.True(t, setting.SqlReviewEnabled)
	require.Equal(t, "second", setting.Model)
	cached, err := s.GetAISetting(ctx, "default")
	require.NoError(t, err)
	require.True(t, cached.SqlReviewEnabled)
	require.Equal(t, "second", cached.Model)
}

// TestUpdateSettingAtomicPublishOrder pins that the setting cache is published
// in commit order: while one update is paused between its commit and its cache
// publish (test hook), a later update must not be able to commit and publish —
// otherwise the paused update's older value would overwrite the newer one in
// the cache, serving reverted state until the next refresh. Red against an
// implementation whose commit releases the row lock before an unordered cache
// write.
func TestUpdateSettingAtomicPublishOrder(t *testing.T) {
	t.Parallel()
	ctx, _, s := newSettingAtomicFixture(t)

	pauseFirst := make(chan struct{})
	resumeFirst := make(chan struct{})
	// Only the first update pauses; later publishes pass through WITHOUT
	// blocking (sync.Once.Do would serialize concurrent callers and mask the
	// very ordering bug this test pins).
	var first atomic.Bool
	s.SetSettingPublishHookForTest(func() {
		if first.CompareAndSwap(false, true) {
			close(pauseFirst)
			<-resumeFirst
		}
	})

	// postCommit callbacks run inside the ordered publish window and receive
	// the freshly re-read value, so derived state converges on committed
	// truth regardless of which updater reaches the mutex first.
	var postCommitMu sync.Mutex
	var postCommitSeen []*storepb.WorkspaceProfileSetting
	recordPostCommit := func() func(current *store.SettingMessage) {
		return func(current *store.SettingMessage) {
			profile, ok := current.Value.(*storepb.WorkspaceProfileSetting)
			if !ok {
				return
			}
			postCommitMu.Lock()
			defer postCommitMu.Unlock()
			postCommitSeen = append(postCommitSeen, profile)
		}
	}

	firstResult := make(chan error, 1)
	go func() {
		_, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
			func(current proto.Message) (proto.Message, error) {
				profile, ok := current.(*storepb.WorkspaceProfileSetting)
				if !ok {
					return nil, errors.Errorf("unexpected type %T", current)
				}
				profile.EnableMetricCollection = true
				return profile, nil
			}, recordPostCommit())
		firstResult <- err
	}()
	// The first update has committed and is paused before publishing.
	select {
	case <-pauseFirst:
	case <-time.After(15 * time.Second):
		t.Fatal("first update never reached the publish window")
	}

	secondResult := make(chan error, 1)
	go func() {
		_, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
			func(current proto.Message) (proto.Message, error) {
				profile, ok := current.(*storepb.WorkspaceProfileSetting)
				if !ok {
					return nil, errors.Errorf("unexpected type %T", current)
				}
				profile.Watermark = true
				return profile, nil
			}, recordPostCommit())
		secondResult <- err
	}()

	// Deterministic barrier: wait until the second update's write is visible
	// in the database. A writer must never retain its transaction (a
	// bounded-pool connection and the row lock) while waiting for the publish
	// mutex — that is the connection-pool deadlock cycle — so the commit must
	// land while the first still holds the publish window.
	require.Eventually(t, func() bool {
		committed, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
		if err != nil {
			return false
		}
		profile, ok := committed.Value.(*storepb.WorkspaceProfileSetting)
		return ok && profile.Watermark
	}, 10*time.Second, 10*time.Millisecond,
		"second update must commit while its publish waits on the mutex")

	// Having committed, it must still not have completed: its publication is
	// blocked behind the first's paused window — completing here would be
	// exactly the out-of-order publish.
	select {
	case err := <-secondResult:
		t.Fatalf("second update published while the first held the publish window: %v", err)
	default:
	}

	close(resumeFirst)
	for _, result := range []chan error{firstResult, secondResult} {
		select {
		case err := <-result:
			require.NoError(t, err)
		case <-time.After(15 * time.Second):
			t.Fatal("update did not complete; possible deadlock")
		}
	}

	// The served cache must hold the final committed state: the second update
	// ran after the first's commit, so its merge carries BOTH fields.
	cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	cachedProfile := mustProfile(t, cached)
	require.True(t, cachedProfile.EnableMetricCollection, "cache must not serve pre-first-update state")
	require.True(t, cachedProfile.Watermark,
		"cache must not be overwritten by the earlier committer's stale publish")
	postCommitMu.Lock()
	defer postCommitMu.Unlock()
	require.Len(t, postCommitSeen, 2, "both postCommit callbacks must run")
	for _, seen := range postCommitSeen {
		require.True(t, seen.EnableMetricCollection,
			"postCommit must observe committed truth, not the caller's own write")
		require.True(t, seen.Watermark,
			"postCommit must observe committed truth, not the caller's own write")
	}
}

// TestSettingCacheNoFillFromUncachedReads pins that uncached reads do not
// populate the setting cache: GetSettingUncached (and ListSettings under it)
// runs outside the publish-ordering mutex, so a fill from that path could
// cache a pre-commit snapshot after a newer value was already published.
// Uncached readers must leave publication to the ordered paths.
func TestSettingCacheNoFillFromUncachedReads(t *testing.T) {
	t.Parallel()
	ctx, db, s := newSettingAtomicFixture(t)
	// Drop the seed's cache entry so the next cached read must fill.
	s.DeleteCache()

	// An uncached read must not leave a cache entry behind ...
	_, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)

	// ... so after an out-of-band change, a cached read serves the new value.
	_, err = db.ExecContext(ctx, `
		UPDATE setting SET value = jsonb_set(value, '{enableMetricCollection}', 'true')
		WHERE workspace = 'default' AND name = 'WORKSPACE_PROFILE';
	`)
	require.NoError(t, err)

	cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	require.True(t, mustProfile(t, cached).EnableMetricCollection,
		"GetSetting must not serve a snapshot cached by an uncached read")
}

// TestGetSettingCacheDisabledNotSerialized pins the HA read path: with the
// cache disabled (store.New(..., !profile.HA)), GetSetting must go straight to
// the database and never touch the publish-ordering mutex — otherwise every
// setting read in an HA process serializes behind a single lock, including
// while a writer sits in its commit-to-publish window.
func TestGetSettingCacheDisabledNotSerialized(t *testing.T) {
	t.Parallel()
	ctx, _, s := newSettingAtomicFixtureWithCache(t, false)

	pause := make(chan struct{})
	resume := make(chan struct{})
	var first atomic.Bool
	s.SetSettingPublishHookForTest(func() {
		if first.CompareAndSwap(false, true) {
			close(pause)
			<-resume
		}
	})

	writerResult := make(chan error, 1)
	go func() {
		_, err := s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
			func(current proto.Message) (proto.Message, error) {
				profile, ok := current.(*storepb.WorkspaceProfileSetting)
				if !ok {
					return nil, errors.Errorf("unexpected type %T", current)
				}
				profile.EnableMetricCollection = true
				return profile, nil
			}, func(*store.SettingMessage) {})
		writerResult <- err
	}()
	select {
	case <-pause:
	case <-time.After(15 * time.Second):
		t.Fatal("writer never reached the publish window")
	}

	// The writer holds the publish mutex; a cache-disabled read must still
	// complete (it reads committed state and consults no cache).
	readResult := make(chan error, 1)
	go func() {
		setting, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
		if err == nil && !mustProfile(t, setting).EnableMetricCollection {
			err = errors.New("read did not observe the committed value")
		}
		readResult <- err
	}()
	select {
	case err := <-readResult:
		require.NoError(t, err)
	case <-time.After(5 * time.Second):
		t.Fatal("cache-disabled GetSetting blocked behind the publish mutex")
	}

	close(resume)
	select {
	case err := <-writerResult:
		require.NoError(t, err)
	case <-time.After(15 * time.Second):
		t.Fatal("writer did not complete")
	}
}

// TestUpdateSettingAtomicPublishSurvivesCancellation pins that publication
// completes even when the request context dies right after commit: the write
// is already durable, so the cache refresh and postCommit must not depend on
// the caller still listening. Red against a publish path that re-reads on the
// request context — the canceled read was swallowed and postCommit silently
// skipped, leaving derived runtime state diverged from the database forever.
func TestUpdateSettingAtomicPublishSurvivesCancellation(t *testing.T) {
	t.Parallel()
	ctx, _, s := newSettingAtomicFixture(t)

	updateCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	var once atomic.Bool
	s.SetSettingPublishHookForTest(func() {
		// Fires after commit, before the publish re-read.
		if once.CompareAndSwap(false, true) {
			cancel()
		}
	})

	var observed *storepb.WorkspaceProfileSetting
	_, err := s.UpdateSettingAtomic(updateCtx, "default", storepb.SettingName_WORKSPACE_PROFILE,
		func(current proto.Message) (proto.Message, error) {
			profile, ok := current.(*storepb.WorkspaceProfileSetting)
			if !ok {
				return nil, errors.Errorf("unexpected type %T", current)
			}
			profile.EnableMetricCollection = true
			return profile, nil
		}, func(current *store.SettingMessage) {
			if profile, ok := current.Value.(*storepb.WorkspaceProfileSetting); ok {
				observed = profile
			}
		})
	require.NoError(t, err)

	require.NotNil(t, observed, "postCommit must run despite request cancellation after commit")
	require.True(t, observed.EnableMetricCollection, "postCommit must observe the committed state")

	cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	require.True(t, mustProfile(t, cached).EnableMetricCollection,
		"the served cache must reflect the committed write despite request cancellation")
}

// TestListSettingsSkipsNamesThisBuildDoesNotKnow pins that a setting row a
// newer replica wrote does not fail the list for the settings this build does
// know. SettingName is a closed enum here, so a name added in a later release
// decodes to nothing — and the unfiltered list is how the settings API serves
// every other setting, so one unreadable row must not take them all down.
func TestListSettingsSkipsNamesThisBuildDoesNotKnow(t *testing.T) {
	t.Parallel()
	ctx, db, s := newSettingAtomicFixture(t)

	// Inserted by hand because the point is a name this build's enum cannot
	// produce; there is no store call that writes one.
	_, err := db.ExecContext(ctx,
		`INSERT INTO setting (name, workspace, value) VALUES ('FUTURE_SETTING', 'default', '{}')`)
	require.NoError(t, err)

	settings, err := s.ListSettings(ctx, &store.FindSettingMessage{Workspace: "default"})
	require.NoError(t, err, "one row this build cannot name must not fail the whole list")

	var names []storepb.SettingName
	for _, setting := range settings {
		names = append(names, setting.Name)
	}
	require.Contains(t, names, storepb.SettingName_WORKSPACE_PROFILE,
		"the settings this build does know still come back")
	require.NotContains(t, names, storepb.SettingName_SETTING_NAME_UNSPECIFIED,
		"the unknown row is skipped, not decoded to the zero name")

	// The filtered read is unaffected either way: it selects by name, so the
	// unknown row never reaches the loop.
	profile, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	require.NotNil(t, profile)
}

// TestListSettingsRefusesAnUnparsedRow pins that a row this build cannot parse
// fails the read rather than reading back absent.
//
// GetSettingUncached reads through this, so a skipped row would reach 24
// callers as "no such setting" — which resolves to the setting's default, and
// enforcement would then run one policy while the console showed another. That
// is BOT-100's defect widened to every setting.
func TestListSettingsRefusesAnUnparsedRow(t *testing.T) {
	t.Parallel()
	// Cache off: the fixture's own write would otherwise serve every read from
	// memory, and the point is what the database hands back.
	ctx, db, s := newSettingAtomicFixtureWithCache(t, false)

	// By hand, because no store call writes a value its own message rejects.
	_, err := db.ExecContext(ctx,
		`UPDATE setting SET value = '{"externalUrl": 5}' WHERE name = 'WORKSPACE_PROFILE' AND workspace = 'default'`)
	require.NoError(t, err)

	_, err = s.ListSettings(ctx, &store.FindSettingMessage{Workspace: "default"})
	require.Error(t, err, "an unparseable row fails the list")

	setting, err := s.GetSettingUncached(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.Error(t, err, "the read must not report a row it cannot parse as absent")
	require.Nil(t, setting)
}

// TestUpdateSettingAtomicRefusesAnUnparsedRow pins that a row this build cannot
// parse never reaches apply.
//
// apply merges onto the message it is handed, and for such a row the only
// message available is the zero value. RotateDirectorySyncToken sets one field
// on current and returns it, so letting it through would write back a
// WorkspaceProfileSetting with the signup settings, password restrictions and
// announcements erased.
func TestUpdateSettingAtomicRefusesAnUnparsedRow(t *testing.T) {
	t.Parallel()
	// Cache off: the point is what the locked row hands the writer.
	ctx, db, s := newSettingAtomicFixtureWithCache(t, false)

	// By hand, because no store call writes a value its own message rejects.
	_, err := db.ExecContext(ctx,
		`UPDATE setting SET value = '{"externalUrl": 5}' WHERE name = 'WORKSPACE_PROFILE' AND workspace = 'default'`)
	require.NoError(t, err)

	applied := false
	_, err = s.UpdateSettingAtomic(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE,
		func(current proto.Message) (proto.Message, error) {
			applied = true
			return current, nil
		}, nil)
	require.Error(t, err, "a row this build cannot parse must not reach a merging apply")
	require.False(t, applied, "apply must not be called at all")

	var stored string
	require.NoError(t, db.QueryRowContext(ctx,
		`SELECT value::text FROM setting WHERE name = 'WORKSPACE_PROFILE' AND workspace = 'default'`).Scan(&stored))
	require.Contains(t, stored, "externalUrl", "the refused write must leave the row as it found it")
}

// The first fill is held inside the lock until every other read has missed
// the cache, so each of them waits for the lock and then looks again.
func TestSimultaneousSettingMissesReadTheDatabaseOnce(t *testing.T) {
	t.Parallel()
	ctx, _, s := newSettingAtomicFixture(t)
	s.DeleteCache()
	var misses, fills atomic.Int32
	firstFillStarted := make(chan struct{})
	releaseFirstFill := make(chan struct{})
	s.SetSettingMissHookForTest(func() { misses.Add(1) })
	s.SetSettingFillHookForTest(func() {
		if fills.Add(1) == 1 {
			close(firstFillStarted)
			<-releaseFirstFill
		}
	})
	results := make(chan error, 5)
	read := func() {
		_, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
		results <- err
	}

	go read()
	<-firstFillStarted
	for range 4 {
		go read()
	}
	require.Eventually(t, func() bool { return misses.Load() == 5 }, 10*time.Second, time.Millisecond)
	close(releaseFirstFill)
	for range 5 {
		require.NoError(t, <-results)
	}

	require.Equal(t, int32(1), fills.Load())
}

// Colliding keys: WORKSPACE_PROFILE in two workspaces, with the second read
// waiting behind the first workspace's fill.
func TestSettingFillWaitingBehindAnotherWorkspaceReadsItsOwnRow(t *testing.T) {
	t.Parallel()
	ctx, db, s := newSettingAtomicFixture(t)
	_, err := db.ExecContext(ctx, `INSERT INTO workspace (resource_id) VALUES ('other');`)
	require.NoError(t, err)
	_, err = s.UpsertSetting(ctx, &store.SettingMessage{
		Name:      storepb.SettingName_WORKSPACE_PROFILE,
		Workspace: "other",
		Value:     &storepb.WorkspaceProfileSetting{DisallowPasswordSignin: true},
	})
	require.NoError(t, err)
	s.DeleteCache()
	var misses, fills atomic.Int32
	firstFillStarted := make(chan struct{})
	releaseFirstFill := make(chan struct{})
	s.SetSettingMissHookForTest(func() { misses.Add(1) })
	s.SetSettingFillHookForTest(func() {
		if fills.Add(1) == 1 {
			close(firstFillStarted)
			<-releaseFirstFill
		}
	})
	type readResult struct {
		setting *store.SettingMessage
		err     error
	}
	read := func(workspace string, results chan<- readResult) {
		setting, err := s.GetSetting(ctx, workspace, storepb.SettingName_WORKSPACE_PROFILE)
		results <- readResult{setting: setting, err: err}
	}

	defaultResult, otherResult := make(chan readResult, 1), make(chan readResult, 1)
	go read("default", defaultResult)
	<-firstFillStarted
	go read("other", otherResult)
	require.Eventually(t, func() bool { return misses.Load() == 2 }, 10*time.Second, time.Millisecond)
	close(releaseFirstFill)

	other := <-otherResult
	require.NoError(t, other.err)
	require.True(t, mustProfile(t, other.setting).DisallowPasswordSignin)
	first := <-defaultResult
	require.NoError(t, first.err)
	require.False(t, mustProfile(t, first.setting).DisallowPasswordSignin)
	require.Equal(t, int32(2), fills.Load())
	cached, err := s.GetSetting(ctx, "default", storepb.SettingName_WORKSPACE_PROFILE)
	require.NoError(t, err)
	require.False(t, mustProfile(t, cached).DisallowPasswordSignin)
}