Skip to content

Git designed for AI agents.

sqlstore: admit SQLite writers first come, first served (PLAN7 WP73b)

Federico Elia054e6ee6c6e8
diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.go
index 7097217..fba4387 100644
--- a/internal/graft/corestore/metastore.go
+++ b/internal/graft/corestore/metastore.go
@@ -145,7 +145,8 @@ type MetaStore interface {
ImportHistory(ctx context.Context, repoID int64, entries []changelog.Entry, reviews []*gitalypb.Review) (importedEntries, importedReviews int, err error)
// InsertReview allocates the next review ID of the repository, calls build
- // with it and stores the returned review (version 1), atomically.
+ // with it and stores the returned review (version 1), atomically. build runs
+ // inside the write transaction and must not call the MetaStore.
InsertReview(ctx context.Context, repoID int64, build func(id string) (*gitalypb.Review, error)) (*gitalypb.Review, error)
// GetReview returns the review and its version, ErrReviewNotFound if unknown.
GetReview(ctx context.Context, repoID int64, id string) (*gitalypb.Review, int64, error)
diff --git a/internal/graft/corestore/sqlstore/export_test.go b/internal/graft/corestore/sqlstore/export_test.go
index ac033c2..e287006 100644
--- a/internal/graft/corestore/sqlstore/export_test.go
+++ b/internal/graft/corestore/sqlstore/export_test.go
@@ -21,6 +21,30 @@ func SetRetryHook(fn func(err error)) (restore func()) {
return func() { retryHook = nil }
}
+// SetQueuedHook installs fn as the hook called each time a writer joins the
+// write queue of a Store, and returns the function that removes it.
+func SetQueuedHook(fn func(waiting int)) (restore func()) {
+ queuedHook = fn
+ return func() { queuedHook = nil }
+}
+
+// HoldWriteAdmission takes the write admission of the Store, as a writer
+// would, and returns the function that releases it. SQLite stores only.
+func (s *Store) HoldWriteAdmission(ctx context.Context) (release func(), _ error) {
+ if s.writers == nil {
+ return nil, errors.New("only a SQLite store has a write queue")
+ }
+ if err := s.writers.acquire(ctx); err != nil {
+ return nil, err
+ }
+ return s.writers.release, nil
+}
+
+// WaitingWriters returns the number of writers waiting for the write admission.
+func (s *Store) WaitingWriters() int {
+ return s.writers.waiting()
+}
+
// IsDeadlock reports whether err is a Postgres deadlock (40P01).
func IsDeadlock(err error) bool {
var pe *pgconn.PgError
diff --git a/internal/graft/corestore/sqlstore/store.go b/internal/graft/corestore/sqlstore/store.go
index 9581487..aee6be1 100644
--- a/internal/graft/corestore/sqlstore/store.go
+++ b/internal/graft/corestore/sqlstore/store.go
@@ -68,6 +68,9 @@ type Store struct {
// of db take. On Postgres it is db.
readDB *sql.DB
dialect Dialect
+ // writers queues the write transactions on SQLite (nil on Postgres, whose
+ // row locks queue their waiters themselves); see writeQueue.
+ writers *writeQueue
}
var _ corestore.MetaStore = (*Store)(nil)
@@ -169,7 +172,11 @@ func open(ctx context.Context, dsn string, migrations int) (*Store, error) {
}
}
- return &Store{db: db, readDB: readDB, dialect: dialect}, nil
+ s := &Store{db: db, readDB: readDB, dialect: dialect}
+ if dialect == SQLite {
+ s.writers = &writeQueue{}
+ }
+ return s, nil
}
// ping checks the connection to the database. On SQLite, the first
@@ -384,6 +391,22 @@ func (s *Store) runTx(ctx context.Context, db *sql.DB, opts *sql.TxOptions, fn f
defer func() { _ = conn.Close() }()
defer restoreBusyTimeout(conn)
+ // A write transaction waits for its turn among the writers of this Store
+ // before it asks SQLite for the write lock (writeQueue). The admission covers
+ // one attempt: it is taken right before BEGIN IMMEDIATE and released when the
+ // attempt has committed or rolled back, so a writer never holds it while
+ // retryTx sleeps before the next attempt, and the writers behind it are never
+ // kept waiting for a lock nobody uses. No transaction function starts another
+ // write transaction, and the admission is always taken before the SQLite
+ // lock, so the two can never wait for each other. Read transactions run on
+ // readDB and need no admission.
+ if db == s.db && s.writers != nil {
+ if err := s.writers.acquire(ctx); err != nil {
+ return fmt.Errorf("wait for the write admission: %w", err)
+ }
+ defer s.writers.release()
+ }
+
tx, err := beginSQLite(ctx, conn, opts)
if err != nil {
return fmt.Errorf("begin transaction: %w", err)
diff --git a/internal/graft/corestore/sqlstore/writequeue.go b/internal/graft/corestore/sqlstore/writequeue.go
new file mode 100644
index 0000000..1b9d343
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/writequeue.go
@@ -0,0 +1,103 @@
+package sqlstore
+
+import (
+ "context"
+ "sync"
+)
+
+// writeQueue admits the SQLite write transactions of one Store one at a time,
+// in the order the writers arrive (doc/graft/PLAN7.md, WP73b).
+//
+// SQLite has a single write lock, and a writer that does not get it waits in
+// SQLite's busy handler, which keeps no queue: the lock goes to whichever
+// waiter happens to retry first after it is released, so an unlucky writer can
+// lose again and again while the lock is never idle. Measured on the dense
+// commit test (32 writers on one reference, two cores): a mean wait of 0.16 s
+// and a longest wait of 3.05 s while a fair queue bounds a wait near 0.3 s;
+// under heavier load the tail crossed the busy timeout of 10 s and the write
+// failed with "database is locked". The Store knows its own writers, so it
+// queues them itself before BEGIN IMMEDIATE: a writer waits behind the writers
+// ahead of it, never behind luck, and the end of its context ends the wait at
+// once. SQLite admits one writer at a time anyway, so nothing is lost.
+//
+// The queue only orders the writers of this process. Several processes on one
+// SQLite file (several nodes on one host) still meet in SQLite's busy handler,
+// which is why multi-node deployments use Postgres (doc/graft/STORAGE.md).
+type writeQueue struct {
+ mu sync.Mutex
+ // busy is true while a writer holds the admission.
+ busy bool
+ // waiters are the writers waiting for the admission, first come first. The
+ // admission is handed to the first one directly (its channel is closed), so
+ // a writer that arrives in between never overtakes it.
+ waiters []chan struct{}
+}
+
+// queuedHook, when set by a test, is called each time a writer joins the queue,
+// with the number of writers then waiting, after its position is fixed. It is
+// nil in production.
+var queuedHook func(waiting int)
+
+// acquire waits for the admission. It returns ctx's error, holding nothing,
+// when ctx ends first.
+func (q *writeQueue) acquire(ctx context.Context) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+
+ q.mu.Lock()
+ if !q.busy {
+ q.busy = true
+ q.mu.Unlock()
+ return nil
+ }
+ turn := make(chan struct{})
+ q.waiters = append(q.waiters, turn)
+ waiting := len(q.waiters)
+ q.mu.Unlock()
+
+ if queuedHook != nil {
+ queuedHook(waiting)
+ }
+
+ select {
+ case <-turn:
+ return nil
+ case <-ctx.Done():
+ }
+
+ // The context ended. Leave the queue, unless the admission was handed over
+ // at the same moment: then pass it on, so that it is never lost.
+ q.mu.Lock()
+ for i, w := range q.waiters {
+ if w == turn {
+ q.waiters = append(q.waiters[:i], q.waiters[i+1:]...)
+ q.mu.Unlock()
+ return ctx.Err()
+ }
+ }
+ q.mu.Unlock()
+ q.release()
+ return ctx.Err()
+}
+
+// release hands the admission to the first waiting writer, or frees it.
+func (q *writeQueue) release() {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+
+ if len(q.waiters) == 0 {
+ q.busy = false
+ return
+ }
+ next := q.waiters[0]
+ q.waiters = q.waiters[1:]
+ close(next)
+}
+
+// waiting returns the number of writers waiting for the admission.
+func (q *writeQueue) waiting() int {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+ return len(q.waiters)
+}
diff --git a/internal/graft/corestore/sqlstore/writequeue_internal_test.go b/internal/graft/corestore/sqlstore/writequeue_internal_test.go
new file mode 100644
index 0000000..5708af3
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/writequeue_internal_test.go
@@ -0,0 +1,52 @@
+package sqlstore
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// TestWriteQueueNeverLosesTheAdmission races the hand-over of the admission
+// against the end of the waiting writer's context: whichever wins, the
+// admission ends up either with that writer or free, never with nobody while
+// marked busy.
+func TestWriteQueueNeverLosesTheAdmission(t *testing.T) {
+ queued := make(chan struct{})
+ queuedHook = func(int) { queued <- struct{}{} }
+ t.Cleanup(func() { queuedHook = nil })
+
+ for range 500 {
+ var q writeQueue
+ require.NoError(t, q.acquire(context.Background()))
+
+ ctx, cancel := context.WithCancel(context.Background())
+ result := make(chan error, 1)
+ go func() { result <- q.acquire(ctx) }()
+
+ // Once the second writer is queued, hand over and cancel at the same
+ // time.
+ <-queued
+ go cancel()
+ q.release()
+
+ if err := <-result; err == nil {
+ // The waiter got the admission: it holds it, and must release it.
+ q.release()
+ }
+ require.Zero(t, q.waiting())
+ require.False(t, q.busy, "the admission is free once every holder released it")
+ }
+}
+
+// TestWriteQueueAcquireWithEndedContext checks that a writer whose context has
+// already ended neither takes nor waits for the admission.
+func TestWriteQueueAcquireWithEndedContext(t *testing.T) {
+ var q writeQueue
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ require.ErrorIs(t, q.acquire(ctx), context.Canceled)
+ require.False(t, q.busy)
+ require.Zero(t, q.waiting())
+}
diff --git a/internal/graft/corestore/sqlstore/writequeue_test.go b/internal/graft/corestore/sqlstore/writequeue_test.go
new file mode 100644
index 0000000..eeb9ee6
--- /dev/null
+++ b/internal/graft/corestore/sqlstore/writequeue_test.go
@@ -0,0 +1,86 @@
+package sqlstore_test
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "golang.org/x/sync/errgroup"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/coretest"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+)
+
+// TestSQLiteWritersAreAdmittedInArrivalOrder checks that the write
+// transactions of a SQLite store take the database's write lock in the order
+// they arrived, not in the order SQLite's busy handler happens to pick
+// (doc/graft/PLAN7.md, WP73b).
+func TestSQLiteWritersAreAdmittedInArrivalOrder(t *testing.T) {
+ ctx := testhelper.Context(t)
+ s := openStore(t, ctx, coretest.DatabaseURL(t, sqlstore.SQLite))
+ repo := register(t, ctx, s, testKey("fifo"))
+
+ queued := make(chan int)
+ defer sqlstore.SetQueuedHook(func(waiting int) { queued <- waiting })()
+
+ release, err := s.HoldWriteAdmission(ctx)
+ require.NoError(t, err)
+
+ const writers = 8
+ var g errgroup.Group
+ for i := range writers {
+ g.Go(func() error {
+ _, err := commit(ctx, s, repo.ID, create(fmt.Sprintf("refs/heads/w%d", i), oid(i+1)))
+ return err
+ })
+ // Writer i is in the queue, behind the i writers started before it,
+ // before the next writer starts.
+ require.Equal(t, i+1, <-queued)
+ }
+
+ release()
+ require.NoError(t, g.Wait())
+
+ changesets, err := s.ListChangesets(ctx, repo.ID, 0, writers)
+ require.NoError(t, err)
+ require.Len(t, changesets, writers)
+ for i, cs := range changesets {
+ require.Equal(t, fmt.Sprintf("refs/heads/w%d", i), cs.Updates[0].Name,
+ "changeset %d is the write of the writer that arrived %d-th", cs.Seq, i)
+ }
+}
+
+// TestSQLiteWriterLeavesTheQueueWhenItsContextEnds checks that a writer whose
+// context ends while it waits leaves the queue at once, writes nothing, and
+// does not hold up the writers that come after it.
+func TestSQLiteWriterLeavesTheQueueWhenItsContextEnds(t *testing.T) {
+ ctx := testhelper.Context(t)
+ s := openStore(t, ctx, coretest.DatabaseURL(t, sqlstore.SQLite))
+ repo := register(t, ctx, s, testKey("leave"))
+
+ queued := make(chan int)
+ defer sqlstore.SetQueuedHook(func(waiting int) { queued <- waiting })()
+
+ release, err := s.HoldWriteAdmission(ctx)
+ require.NoError(t, err)
+
+ writerCtx, cancel := context.WithCancel(ctx)
+ errc := make(chan error, 1)
+ go func() {
+ _, err := commit(writerCtx, s, repo.ID, create("refs/heads/gone", oid(1)))
+ errc <- err
+ }()
+ require.Equal(t, 1, <-queued)
+
+ cancel()
+ require.ErrorIs(t, <-errc, context.Canceled)
+ require.Zero(t, s.WaitingWriters(), "the writer left the queue")
+
+ release()
+ _, err = commit(ctx, s, repo.ID, create("refs/heads/next", oid(2)))
+ require.NoError(t, err, "the admission is free again")
+ requireRefs(t, ctx, s, repo.ID, map[string]corestore.Target{"refs/heads/next": oid(2)})
+}