Git designed for AI agents.
sqlstore: admit SQLite writers first come, first served (PLAN7 WP73b)
Federico Elia054e6ee6c6e8
diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.goindex 7097217..fba4387 100644--- a/internal/graft/corestore/metastore.go+++ b/internal/graft/corestore/metastore.go@@ -145,7 +145,8 @@ type MetaStore interface {ImportHistory(ctx context.Context, repoID int64, entries []changelog.Entry, reviews []*gitalypb.Review) (importedEntries, importedReviews int, err error)// InsertReview allocates the next review ID of the repository, calls build- // with it and stores the returned review (version 1), atomically.+ // with it and stores the returned review (version 1), atomically. build runs+ // inside the write transaction and must not call the MetaStore.InsertReview(ctx context.Context, repoID int64, build func(id string) (*gitalypb.Review, error)) (*gitalypb.Review, error)// GetReview returns the review and its version, ErrReviewNotFound if unknown.GetReview(ctx context.Context, repoID int64, id string) (*gitalypb.Review, int64, error)diff --git a/internal/graft/corestore/sqlstore/export_test.go b/internal/graft/corestore/sqlstore/export_test.goindex ac033c2..e287006 100644--- a/internal/graft/corestore/sqlstore/export_test.go+++ b/internal/graft/corestore/sqlstore/export_test.go@@ -21,6 +21,30 @@ func SetRetryHook(fn func(err error)) (restore func()) {return func() { retryHook = nil }}+// SetQueuedHook installs fn as the hook called each time a writer joins the+// write queue of a Store, and returns the function that removes it.+func SetQueuedHook(fn func(waiting int)) (restore func()) {+ queuedHook = fn+ return func() { queuedHook = nil }+}++// HoldWriteAdmission takes the write admission of the Store, as a writer+// would, and returns the function that releases it. SQLite stores only.+func (s *Store) HoldWriteAdmission(ctx context.Context) (release func(), _ error) {+ if s.writers == nil {+ return nil, errors.New("only a SQLite store has a write queue")+ }+ if err := s.writers.acquire(ctx); err != nil {+ return nil, err+ }+ return s.writers.release, nil+}++// WaitingWriters returns the number of writers waiting for the write admission.+func (s *Store) WaitingWriters() int {+ return s.writers.waiting()+}+// IsDeadlock reports whether err is a Postgres deadlock (40P01).func IsDeadlock(err error) bool {var pe *pgconn.PgErrordiff --git a/internal/graft/corestore/sqlstore/store.go b/internal/graft/corestore/sqlstore/store.goindex 9581487..aee6be1 100644--- a/internal/graft/corestore/sqlstore/store.go+++ b/internal/graft/corestore/sqlstore/store.go@@ -68,6 +68,9 @@ type Store struct {// of db take. On Postgres it is db.readDB *sql.DBdialect Dialect+ // writers queues the write transactions on SQLite (nil on Postgres, whose+ // row locks queue their waiters themselves); see writeQueue.+ writers *writeQueue}var _ corestore.MetaStore = (*Store)(nil)@@ -169,7 +172,11 @@ func open(ctx context.Context, dsn string, migrations int) (*Store, error) {}}- return &Store{db: db, readDB: readDB, dialect: dialect}, nil+ s := &Store{db: db, readDB: readDB, dialect: dialect}+ if dialect == SQLite {+ s.writers = &writeQueue{}+ }+ return s, nil}// ping checks the connection to the database. On SQLite, the first@@ -384,6 +391,22 @@ func (s *Store) runTx(ctx context.Context, db *sql.DB, opts *sql.TxOptions, fn fdefer func() { _ = conn.Close() }()defer restoreBusyTimeout(conn)+ // A write transaction waits for its turn among the writers of this Store+ // before it asks SQLite for the write lock (writeQueue). The admission covers+ // one attempt: it is taken right before BEGIN IMMEDIATE and released when the+ // attempt has committed or rolled back, so a writer never holds it while+ // retryTx sleeps before the next attempt, and the writers behind it are never+ // kept waiting for a lock nobody uses. No transaction function starts another+ // write transaction, and the admission is always taken before the SQLite+ // lock, so the two can never wait for each other. Read transactions run on+ // readDB and need no admission.+ if db == s.db && s.writers != nil {+ if err := s.writers.acquire(ctx); err != nil {+ return fmt.Errorf("wait for the write admission: %w", err)+ }+ defer s.writers.release()+ }+tx, err := beginSQLite(ctx, conn, opts)if err != nil {return fmt.Errorf("begin transaction: %w", err)diff --git a/internal/graft/corestore/sqlstore/writequeue.go b/internal/graft/corestore/sqlstore/writequeue.gonew file mode 100644index 0000000..1b9d343--- /dev/null+++ b/internal/graft/corestore/sqlstore/writequeue.go@@ -0,0 +1,103 @@+package sqlstore++import (+ "context"+ "sync"+)++// writeQueue admits the SQLite write transactions of one Store one at a time,+// in the order the writers arrive (doc/graft/PLAN7.md, WP73b).+//+// SQLite has a single write lock, and a writer that does not get it waits in+// SQLite's busy handler, which keeps no queue: the lock goes to whichever+// waiter happens to retry first after it is released, so an unlucky writer can+// lose again and again while the lock is never idle. Measured on the dense+// commit test (32 writers on one reference, two cores): a mean wait of 0.16 s+// and a longest wait of 3.05 s while a fair queue bounds a wait near 0.3 s;+// under heavier load the tail crossed the busy timeout of 10 s and the write+// failed with "database is locked". The Store knows its own writers, so it+// queues them itself before BEGIN IMMEDIATE: a writer waits behind the writers+// ahead of it, never behind luck, and the end of its context ends the wait at+// once. SQLite admits one writer at a time anyway, so nothing is lost.+//+// The queue only orders the writers of this process. Several processes on one+// SQLite file (several nodes on one host) still meet in SQLite's busy handler,+// which is why multi-node deployments use Postgres (doc/graft/STORAGE.md).+type writeQueue struct {+ mu sync.Mutex+ // busy is true while a writer holds the admission.+ busy bool+ // waiters are the writers waiting for the admission, first come first. The+ // admission is handed to the first one directly (its channel is closed), so+ // a writer that arrives in between never overtakes it.+ waiters []chan struct{}+}++// queuedHook, when set by a test, is called each time a writer joins the queue,+// with the number of writers then waiting, after its position is fixed. It is+// nil in production.+var queuedHook func(waiting int)++// acquire waits for the admission. It returns ctx's error, holding nothing,+// when ctx ends first.+func (q *writeQueue) acquire(ctx context.Context) error {+ if err := ctx.Err(); err != nil {+ return err+ }++ q.mu.Lock()+ if !q.busy {+ q.busy = true+ q.mu.Unlock()+ return nil+ }+ turn := make(chan struct{})+ q.waiters = append(q.waiters, turn)+ waiting := len(q.waiters)+ q.mu.Unlock()++ if queuedHook != nil {+ queuedHook(waiting)+ }++ select {+ case <-turn:+ return nil+ case <-ctx.Done():+ }++ // The context ended. Leave the queue, unless the admission was handed over+ // at the same moment: then pass it on, so that it is never lost.+ q.mu.Lock()+ for i, w := range q.waiters {+ if w == turn {+ q.waiters = append(q.waiters[:i], q.waiters[i+1:]...)+ q.mu.Unlock()+ return ctx.Err()+ }+ }+ q.mu.Unlock()+ q.release()+ return ctx.Err()+}++// release hands the admission to the first waiting writer, or frees it.+func (q *writeQueue) release() {+ q.mu.Lock()+ defer q.mu.Unlock()++ if len(q.waiters) == 0 {+ q.busy = false+ return+ }+ next := q.waiters[0]+ q.waiters = q.waiters[1:]+ close(next)+}++// waiting returns the number of writers waiting for the admission.+func (q *writeQueue) waiting() int {+ q.mu.Lock()+ defer q.mu.Unlock()+ return len(q.waiters)+}diff --git a/internal/graft/corestore/sqlstore/writequeue_internal_test.go b/internal/graft/corestore/sqlstore/writequeue_internal_test.gonew file mode 100644index 0000000..5708af3--- /dev/null+++ b/internal/graft/corestore/sqlstore/writequeue_internal_test.go@@ -0,0 +1,52 @@+package sqlstore++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"+)++// TestWriteQueueNeverLosesTheAdmission races the hand-over of the admission+// against the end of the waiting writer's context: whichever wins, the+// admission ends up either with that writer or free, never with nobody while+// marked busy.+func TestWriteQueueNeverLosesTheAdmission(t *testing.T) {+ queued := make(chan struct{})+ queuedHook = func(int) { queued <- struct{}{} }+ t.Cleanup(func() { queuedHook = nil })++ for range 500 {+ var q writeQueue+ require.NoError(t, q.acquire(context.Background()))++ ctx, cancel := context.WithCancel(context.Background())+ result := make(chan error, 1)+ go func() { result <- q.acquire(ctx) }()++ // Once the second writer is queued, hand over and cancel at the same+ // time.+ <-queued+ go cancel()+ q.release()++ if err := <-result; err == nil {+ // The waiter got the admission: it holds it, and must release it.+ q.release()+ }+ require.Zero(t, q.waiting())+ require.False(t, q.busy, "the admission is free once every holder released it")+ }+}++// TestWriteQueueAcquireWithEndedContext checks that a writer whose context has+// already ended neither takes nor waits for the admission.+func TestWriteQueueAcquireWithEndedContext(t *testing.T) {+ var q writeQueue+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ require.ErrorIs(t, q.acquire(ctx), context.Canceled)+ require.False(t, q.busy)+ require.Zero(t, q.waiting())+}diff --git a/internal/graft/corestore/sqlstore/writequeue_test.go b/internal/graft/corestore/sqlstore/writequeue_test.gonew file mode 100644index 0000000..eeb9ee6--- /dev/null+++ b/internal/graft/corestore/sqlstore/writequeue_test.go@@ -0,0 +1,86 @@+package sqlstore_test++import (+ "context"+ "fmt"+ "testing"++ "github.com/stretchr/testify/require"+ "golang.org/x/sync/errgroup"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/coretest"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/internal/testhelper"+)++// TestSQLiteWritersAreAdmittedInArrivalOrder checks that the write+// transactions of a SQLite store take the database's write lock in the order+// they arrived, not in the order SQLite's busy handler happens to pick+// (doc/graft/PLAN7.md, WP73b).+func TestSQLiteWritersAreAdmittedInArrivalOrder(t *testing.T) {+ ctx := testhelper.Context(t)+ s := openStore(t, ctx, coretest.DatabaseURL(t, sqlstore.SQLite))+ repo := register(t, ctx, s, testKey("fifo"))++ queued := make(chan int)+ defer sqlstore.SetQueuedHook(func(waiting int) { queued <- waiting })()++ release, err := s.HoldWriteAdmission(ctx)+ require.NoError(t, err)++ const writers = 8+ var g errgroup.Group+ for i := range writers {+ g.Go(func() error {+ _, err := commit(ctx, s, repo.ID, create(fmt.Sprintf("refs/heads/w%d", i), oid(i+1)))+ return err+ })+ // Writer i is in the queue, behind the i writers started before it,+ // before the next writer starts.+ require.Equal(t, i+1, <-queued)+ }++ release()+ require.NoError(t, g.Wait())++ changesets, err := s.ListChangesets(ctx, repo.ID, 0, writers)+ require.NoError(t, err)+ require.Len(t, changesets, writers)+ for i, cs := range changesets {+ require.Equal(t, fmt.Sprintf("refs/heads/w%d", i), cs.Updates[0].Name,+ "changeset %d is the write of the writer that arrived %d-th", cs.Seq, i)+ }+}++// TestSQLiteWriterLeavesTheQueueWhenItsContextEnds checks that a writer whose+// context ends while it waits leaves the queue at once, writes nothing, and+// does not hold up the writers that come after it.+func TestSQLiteWriterLeavesTheQueueWhenItsContextEnds(t *testing.T) {+ ctx := testhelper.Context(t)+ s := openStore(t, ctx, coretest.DatabaseURL(t, sqlstore.SQLite))+ repo := register(t, ctx, s, testKey("leave"))++ queued := make(chan int)+ defer sqlstore.SetQueuedHook(func(waiting int) { queued <- waiting })()++ release, err := s.HoldWriteAdmission(ctx)+ require.NoError(t, err)++ writerCtx, cancel := context.WithCancel(ctx)+ errc := make(chan error, 1)+ go func() {+ _, err := commit(writerCtx, s, repo.ID, create("refs/heads/gone", oid(1)))+ errc <- err+ }()+ require.Equal(t, 1, <-queued)++ cancel()+ require.ErrorIs(t, <-errc, context.Canceled)+ require.Zero(t, s.WaitingWriters(), "the writer left the queue")++ release()+ _, err = commit(ctx, s, repo.ID, create("refs/heads/next", oid(2)))+ require.NoError(t, err, "the admission is free again")+ requireRefs(t, ctx, s, repo.ID, map[string]corestore.Target{"refs/heads/next": oid(2)})+}