Git designed for AI agents.
hook: group commit of the changeset log, in ticket order
Federico Eliacc518efa2347
diff --git a/doc/graft/LIMITATIONS.md b/doc/graft/LIMITATIONS.mdindex e1ac8b1..f80c367 100644--- a/doc/graft/LIMITATIONS.md+++ b/doc/graft/LIMITATIONS.md@@ -202,9 +202,20 @@ Git hooks use private.- Only reference updates that pass through Gitaly's transaction path arerecorded. Updates made by other tools that write to the repository directlyare not.-- Recording errors are logged and never fail the update. In that case the log- has a gap in what clients expected. The log is dense only while recording- succeeds.+- With the storage core off or in shadow mode, the file log is written by the+ reference-transaction hook after git committed the update, so a recording+ error cannot fail the update: the committed updates are then missing from the+ log, and its consumers do not see them. Each loss is counted+ (`graft_changelog_records_lost_total`) and logged with the references. The+ sequences stay dense, because an entry that failed to be written was never+ visible and the next entry takes its sequence. In authoritative mode the+ storage core writes the changelog in the database transaction of the update,+ so this cannot happen.+- Entries become visible to `ListChangesets`, `WatchChangesets` and the storage+ core only once they are fsynced, and an RPC returns only after its entries+ are, so a reader never sees an entry that a crash could take back. The entries+ of the transactions that commit together are written with one fsync, in the+ order in which the transactions locked their references.- The log is stored per Gitaly node and is not replicated. Behind Praefect,every replica records the updates it applies itself, so sequence numbers andmetadata differ between replicas. Praefect therefore always routesdiff --git a/internal/gitaly/hook/changeset_order.go b/internal/gitaly/hook/changeset_order.goindex 2ad87d9..39f85a9 100644--- a/internal/gitaly/hook/changeset_order.go+++ b/internal/gitaly/hook/changeset_order.go@@ -1,10 +1,13 @@package hookimport (- "context""crypto/sha256""sync""time"++ "github.com/feder-cr/git-ai/internal/graft/changelog"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto")// changesetTicketTimeout bounds how long recording waits for a reference transaction that was@@ -13,16 +16,68 @@ import (// for it, would otherwise block the changeset log of the repository forever. Tests lower it.var changesetTicketTimeout = 5 * time.Second+var (+ changelogGroupTransactions = promauto.NewHistogram(prometheus.HistogramOpts{+ Name: "graft_changelog_group_entries",+ Help: "Reference transactions recorded in the changeset log by one write and one fsync.",+ Buckets: []float64{1, 2, 4, 8, 16, 32, 64, 128},+ })++ changelogAppendSeconds = promauto.NewHistogram(prometheus.HistogramOpts{+ Name: "graft_changelog_append_seconds",+ Help: "Time of one write of the changeset log, with its fsync, for a group of reference transactions.",+ Buckets: prometheus.DefBuckets,+ })++ changesetTicketWaitSeconds = promauto.NewHistogram(prometheus.HistogramOpts{+ Name: "graft_changeset_ticket_wait_seconds",+ Help: "Time from the committed phase of a reference transaction to its entries being durable in the changeset log.",+ Buckets: prometheus.DefBuckets,+ })++ changesetTicketsExpiredTotal = promauto.NewCounter(prometheus.CounterOpts{+ Name: "graft_changeset_tickets_expired_total",+ Help: "Prepared reference transactions that did not report their outcome in time, so that the transactions after them were recorded without them.",+ })++ changelogRecordsLostTotal = promauto.NewCounter(prometheus.CounterOpts{+ Name: "graft_changelog_records_lost_total",+ Help: "Committed reference updates that the changeset log does not record, because writing them failed.",+ })+)+// changesetTicket is the place of a prepared reference transaction in the order of the changeset// log of its repository.type changesetTicket struct {// key identifies the transaction: the digest of the updates the hook was given. Git gives// the same updates to every phase of a transaction.key [sha256.Size]byte- // done is closed when the transaction was recorded or aborted.+ // done is closed when the entries of the transaction were written (see err), or when the+ // transaction was aborted, expired or records nothing.done chan struct{}// deadline is when later transactions stop waiting for this one.deadline time.Time+ // record holds the updates of the transaction once its committed phase ran. The ticket is+ // ready from then on, and only the group writer removes it.+ record *changelog.Record+ // err is the outcome of the write of the record. It is set before done is closed.+ err error+}++// changesetQueue holds the tickets of one repository in the order of their prepared phases.+type changesetQueue struct {+ tickets []*changesetTicket+ // writing is set while a committed phase writes a group of the queue.+ writing bool+ // changed is closed and replaced whenever the queue changes, which wakes the committed phases+ // that wait.+ changed chan struct{}+}++// signal wakes the committed phases that wait on the queue. The caller must hold changesetTickets.+func (q *changesetQueue) signal() {+ close(q.changed)+ q.changed = make(chan struct{})}// changesetTickets orders the recording of committed reference updates in the changeset log.@@ -32,22 +87,46 @@ type changesetTicket struct {// before the first one is recorded, so the log would describe the updates out of order. The// prepared phase runs while the references are locked, so the order in which transactions reach// it is the order of their updates for every reference they share. Each prepared transaction-// whose updates are recorded takes a ticket in that order, and its committed phase records only-// once every earlier ticket of the repository was recorded or aborted.+// whose updates are recorded takes a ticket in that order, and the log records the transactions of+// a repository in the order of their tickets, whatever references they update: a transaction that+// read a reference which an earlier one updated is recorded after it.+//+// The entries are written in groups (recordInOrder): the committed phase that finds no group being+// written writes every ready ticket at the head of the queue with one write and one fsync. Each+// committed phase returns only once its own entries are durable.//// The tickets live in the process that serves the hook calls, which is the Gitaly process that// spawned Git. A ticket whose transaction never reports its outcome expires after// changesetTicketTimeout, after which the later transactions record without waiting for it.var changesetTickets = struct {sync.Mutex- queues map[string][]*changesetTicket-}{queues: map[string][]*changesetTicket{}}+ queues map[string]*changesetQueue+}{queues: map[string]*changesetQueue{}}// changesetTicketKey returns the key of the transaction with the given hook input.func changesetTicketKey(changes []byte) [sha256.Size]byte {return sha256.Sum256(changes)}+// changesetQueueLocked returns the queue of the repository at repoPath, creating it. The caller+// must hold changesetTickets.+func changesetQueueLocked(repoPath string) *changesetQueue {+ q, ok := changesetTickets.queues[repoPath]+ if !ok {+ q = &changesetQueue{changed: make(chan struct{})}+ changesetTickets.queues[repoPath] = q+ }+ return q+}++// dropIfIdleLocked forgets the queue of repoPath when it holds nothing. The caller must hold+// changesetTickets.+func dropIfIdleLocked(repoPath string, q *changesetQueue) {+ if len(q.tickets) == 0 && !q.writing && changesetTickets.queues[repoPath] == q {+ delete(changesetTickets.queues, repoPath)+ }+}+// takeChangesetTicket appends a ticket for the prepared transaction with the given hook input to// the queue of the repository at repoPath.func takeChangesetTicket(repoPath string, changes []byte) {@@ -55,93 +134,185 @@ func takeChangesetTicket(repoPath string, changes []byte) {defer changesetTickets.Unlock()now := time.Now()- queue := pruneExpiredChangesetTickets(changesetTickets.queues[repoPath], now)- changesetTickets.queues[repoPath] = append(queue, &changesetTicket{+ q := changesetQueueLocked(repoPath)+ pruneExpiredChangesetTickets(q, now)+ q.tickets = append(q.tickets, &changesetTicket{key: changesetTicketKey(changes),done: make(chan struct{}),deadline: now.Add(changesetTicketTimeout),})}-// pruneExpiredChangesetTickets drops the expired tickets from the front of the queue. Expired-// tickets further back are dropped once they reach the front. The caller must hold-// changesetTickets.-func pruneExpiredChangesetTickets(queue []*changesetTicket, now time.Time) []*changesetTicket {- for len(queue) > 0 && !now.Before(queue[0].deadline) {- close(queue[0].done)- queue = queue[1:]+// pruneExpiredChangesetTickets drops the expired tickets from the front of the queue whose+// transactions did not run their committed phase. A ready ticket is never dropped: its committed+// phase waits for its entries to be written. Expired tickets further back are dropped once they+// reach the front. The caller must hold changesetTickets.+func pruneExpiredChangesetTickets(q *changesetQueue, now time.Time) {+ pruned := 0+ for len(q.tickets) > 0 && q.tickets[0].record == nil && !now.Before(q.tickets[0].deadline) {+ close(q.tickets[0].done)+ q.tickets = q.tickets[1:]+ pruned++}- return queue+ if pruned > 0 {+ changesetTicketsExpiredTotal.Add(float64(pruned))+ q.signal()+ }}-// findChangesetTicket returns the oldest ticket of the transaction with the given hook input and-// the tickets that precede it in the queue of the repository at repoPath. It returns a nil ticket-// when the transaction has none, for example because it was prepared before the changeset log was-// enabled or its ticket expired.-func findChangesetTicket(repoPath string, changes []byte) (*changesetTicket, []*changesetTicket) {+// findChangesetTicket returns the oldest ticket of the transaction with the given hook input in+// the queue of the repository at repoPath. It returns nil when the transaction has none, for+// example because it was prepared before the changeset log was enabled or its ticket expired.+func findChangesetTicket(repoPath string, changes []byte) *changesetTicket {changesetTickets.Lock()defer changesetTickets.Unlock()+ q, ok := changesetTickets.queues[repoPath]+ if !ok {+ return nil+ }+key := changesetTicketKey(changes)- queue := changesetTickets.queues[repoPath]- for i, ticket := range queue {- if ticket.key == key {- return ticket, append([]*changesetTicket(nil), queue[:i]...)+ for _, ticket := range q.tickets {+ if ticket.key == key && ticket.record == nil {+ return ticket}}- return nil, nil+ return nil}-// releaseChangesetTicket removes the ticket from the queue of the repository at repoPath and-// wakes the transactions that wait for it.+// releaseChangesetTicket removes the ticket, whose transaction records nothing, from the queue of+// the repository at repoPath and wakes the transactions that wait for it.func releaseChangesetTicket(repoPath string, ticket *changesetTicket) {changesetTickets.Lock()defer changesetTickets.Unlock()- queue := changesetTickets.queues[repoPath]- for i, queued := range queue {+ q, ok := changesetTickets.queues[repoPath]+ if !ok {+ return+ }++ for i, queued := range q.tickets {if queued != ticket {continue}close(ticket.done)- queue = append(queue[:i:i], queue[i+1:]...)+ q.tickets = append(q.tickets[:i:i], q.tickets[i+1:]...)+ q.signal()break}- if len(queue) == 0 {- delete(changesetTickets.queues, repoPath)- return- }- changesetTickets.queues[repoPath] = queue+ dropIfIdleLocked(repoPath, q)}// abortChangesetTicket releases the ticket of the aborted transaction with the given hook input,// if it has one.func abortChangesetTicket(repoPath string, changes []byte) {- if ticket, _ := findChangesetTicket(repoPath, changes); ticket != nil {+ if ticket := findChangesetTicket(repoPath, changes); ticket != nil {releaseChangesetTicket(repoPath, ticket)}}-// waitForChangesetTickets waits until every ticket of earlier was recorded, aborted or expired, or-// until the context is done. It reports whether it waited for all of them.-func waitForChangesetTickets(ctx context.Context, earlier []*changesetTicket) bool {- for _, ticket := range earlier {- timer := time.NewTimer(time.Until(ticket.deadline))+// recordInOrder records the record of the transaction of ticket in the changeset log of the+// repository at repoPath, after the transactions whose tickets are before it, and returns once+// its entries are durable or failed to be written.+//+// The ticket becomes ready. While no group is being written and the ticket at the head of the+// queue is ready, the caller writes every ready ticket at the head with one call of+// changelog.AppendGroup, one write and one fsync, and wakes their committed phases with the+// outcome. Otherwise it waits for the queue to change or for the ticket at the head to expire.+// The wait does not end with the request: the updates are committed in git already, so their+// entries are always written, in order. It ends at the latest once every ticket before this one+// was written, aborted or expired, which changesetTicketTimeout bounds.+func recordInOrder(repoPath string, ticket *changesetTicket, record changelog.Record) error {+ start := time.Now()+ defer func() { changesetTicketWaitSeconds.Observe(time.Since(start).Seconds()) }()++ changesetTickets.Lock()+ q := changesetQueueLocked(repoPath)+ ticket.record = &record+ q.signal()+ for {select {case <-ticket.done:- timer.Stop()- case <-timer.C:- return false- case <-ctx.Done():- timer.Stop()- return false+ changesetTickets.Unlock()+ return ticket.err+ default:+ }++ pruneExpiredChangesetTickets(q, time.Now())++ if !q.writing && len(q.tickets) > 0 && q.tickets[0].record != nil {+ writeChangesetGroupLocked(repoPath, q)+ continue+ }++ changed := q.changed+ var expiry *time.Timer+ var expired <-chan time.Time+ if len(q.tickets) > 0 && q.tickets[0].record == nil {+ expiry = time.NewTimer(time.Until(q.tickets[0].deadline))+ expired = expiry.C}+ changesetTickets.Unlock()++ select {+ case <-ticket.done:+ case <-changed:+ case <-expired:+ }+ if expiry != nil {+ expiry.Stop()+ }++ changesetTickets.Lock()+ }+}++// writeChangesetGroupLocked writes the ready tickets at the head of q, the queue of the repository+// at repoPath, as one group. The caller must hold changesetTickets, which is released while the+// group is written.+func writeChangesetGroupLocked(repoPath string, q *changesetQueue) {+ n := 0+ for n < len(q.tickets) && q.tickets[n].record != nil {+ n+++ }+ group := append([]*changesetTicket(nil), q.tickets[:n]...)+ q.writing = true+ changesetTickets.Unlock()++ records := make([]changelog.Record, 0, len(group))+ for _, ticket := range group {+ records = append(records, *ticket.record)}- return true+ start := time.Now()+ _, err := changelog.AppendGroup(repoPath, records)+ changelogAppendSeconds.Observe(time.Since(start).Seconds())+ changelogGroupTransactions.Observe(float64(len(group)))++ changesetTickets.Lock()+ written := make(map[*changesetTicket]bool, len(group))+ for _, ticket := range group {+ ticket.err = err+ close(ticket.done)+ written[ticket] = true+ }++ // Only the group writer removes ready tickets, and tickets are only added at the back, so the+ // group is still at the head of the queue; the tickets are removed by identity all the same.+ remaining := q.tickets[:0:0]+ for _, ticket := range q.tickets {+ if !written[ticket] {+ remaining = append(remaining, ticket)+ }+ }+ q.tickets = remaining+ q.writing = false+ q.signal()+ dropIfIdleLocked(repoPath, q)}diff --git a/internal/gitaly/hook/changeset_order_test.go b/internal/gitaly/hook/changeset_order_test.goindex 6778ad8..fecc348 100644--- a/internal/gitaly/hook/changeset_order_test.go+++ b/internal/gitaly/hook/changeset_order_test.go@@ -3,6 +3,7 @@ package hookimport ("context""fmt"+ "os""strings""testing""time"@@ -16,16 +17,20 @@ import ("github.com/feder-cr/git-ai/internal/graft/changelog""github.com/feder-cr/git-ai/internal/testhelper""github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/prometheus/client_golang/prometheus/testutil""github.com/stretchr/testify/require")-// changesetOrderSetup is a repository with a branch history base -> a -> b and a hook manager that-// records its reference updates in the changeset log.+// changesetOrderSetup is a repository with a branch history base -> a -> b -> c and a hook manager+// that records its reference updates in the changeset log.type changesetOrderSetup struct {repoPath string- // first moves the branch from base to a, second from a to b.- first, second string- run func(state ReferenceTransactionState, changes string) error+ // first moves the branch from base to a, second from a to b, third from b to c. other creates+ // another branch at base.+ first, second, third, other string+ run func(state ReferenceTransactionState, changes string) error+ // runWith runs the hook with the context ctx.+ runWith func(ctx context.Context, state ReferenceTransactionState, changes string) error}func setupChangesetOrder(t *testing.T, ctx context.Context) changesetOrderSetup {@@ -39,6 +44,7 @@ func setupChangesetOrder(t *testing.T, ctx context.Context) changesetOrderSetupbase := gittest.WriteCommit(t, cfg, repoPath, gittest.WithMessage("base"))a := gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(base), gittest.WithMessage("a"))b := gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(a), gittest.WithMessage("b"))+ c := gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(b), gittest.WithMessage("c"))manager := NewManager(cfg, config.NewLocator(cfg), testhelper.SharedLogger(t), gittest.NewCommandFactory(t, cfg), nil, &transaction.MockManager{}, gitlab.NewMockClient(t, gitlab.MockAllowed, gitlab.MockPreReceive, gitlab.MockPostReceive,@@ -51,13 +57,20 @@ func setupChangesetOrder(t *testing.T, ctx context.Context) changesetOrderSetup}, gitcmd.ReferenceTransactionHook, nil).Env()require.NoError(t, err)+ runWith := func(ctx context.Context, state ReferenceTransactionState, changes string) error {+ return manager.ReferenceTransactionHook(ctx, state, []string{payload}, strings.NewReader(changes))+ }+return changesetOrderSetup{repoPath: repoPath,first: fmt.Sprintf("%s %s refs/heads/main\n", base, a),second: fmt.Sprintf("%s %s refs/heads/main\n", a, b),+ third: fmt.Sprintf("%s %s refs/heads/main\n", b, c),+ other: fmt.Sprintf("%s %s refs/heads/other\n", gittest.DefaultObjectHash.ZeroOID, base),run: func(state ReferenceTransactionState, changes string) error {- return manager.ReferenceTransactionHook(ctx, state, []string{payload}, strings.NewReader(changes))+ return runWith(ctx, state, changes)},+ runWith: runWith,}}@@ -131,7 +144,7 @@ func TestReferenceTransactionHook_changesetOrder(t *testing.T) {require.Equal(t, []string{setup.first}, recordedChanges(t, setup.repoPath))})- t.Run("canceled context stops waiting", func(t *testing.T) {+ t.Run("a committed phase whose caller went away still records in order", func(t *testing.T) {t.Parallel()setup := setupChangesetOrder(t, ctx)@@ -139,21 +152,106 @@ func TestReferenceTransactionHook_changesetOrder(t *testing.T) {require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.first))require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.second))- // The ticket of the first transaction is still pending, but the caller of the second- // committed phase went away. The update is recorded regardless.- repoPath := setup.repoPath- ticket, earlier := findChangesetTicket(repoPath, []byte(setup.second))- require.NotNil(t, ticket)- require.Len(t, earlier, 1)-+ // The caller of the second committed phase went away while the first transaction is+ // still pending. Its update is committed in git, so it is recorded all the same, after+ // the first one.canceledCtx, cancel := context.WithCancel(ctx)cancel()- require.False(t, waitForChangesetTickets(canceledCtx, earlier))+ secondDone := make(chan error, 1)+ go func() { secondDone <- setup.runWith(canceledCtx, ReferenceTransactionCommitted, setup.second) }()- require.NoError(t, setup.run(ReferenceTransactionAborted, setup.first))- require.NoError(t, setup.run(ReferenceTransactionCommitted, setup.second))- require.Equal(t, []string{setup.second}, recordedChanges(t, setup.repoPath))+ select {+ case err := <-secondDone:+ require.FailNow(t, "second transaction recorded before the first", "err: %v", err)+ case <-time.After(200 * time.Millisecond):+ }++ require.NoError(t, setup.run(ReferenceTransactionCommitted, setup.first))+ require.NoError(t, <-secondDone)+ require.Equal(t, []string{setup.first, setup.second}, recordedChanges(t, setup.repoPath))+ })++ t.Run("transactions on other references are recorded in order too", func(t *testing.T) {+ t.Parallel()++ setup := setupChangesetOrder(t, ctx)++ // A later transaction may have read what an earlier one wrote to another reference, so it+ // is recorded after it.+ require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.other))+ require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.first))++ firstDone := make(chan error, 1)+ go func() { firstDone <- setup.run(ReferenceTransactionCommitted, setup.first) }()++ select {+ case err := <-firstDone:+ require.FailNow(t, "transaction recorded before an earlier one on another reference", "err: %v", err)+ case <-time.After(200 * time.Millisecond):+ }++ require.NoError(t, setup.run(ReferenceTransactionCommitted, setup.other))+ require.NoError(t, <-firstDone)+ require.Equal(t, []string{setup.other, setup.first}, recordedChanges(t, setup.repoPath))})++ t.Run("ready transactions are written as one group", func(t *testing.T) {+ t.Parallel()++ setup := setupChangesetOrder(t, ctx)++ require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.first))+ require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.second))+ require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.third))++ // The committed phases of the later transactions run first and wait for the first one.+ laterDone := make(chan error, 2)+ for _, changes := range []string{setup.third, setup.second} {+ go func() { laterDone <- setup.run(ReferenceTransactionCommitted, changes) }()+ }++ // Both later tickets are ready before the first transaction commits.+ require.Eventually(t, func() bool {+ changesetTickets.Lock()+ defer changesetTickets.Unlock()+ q := changesetTickets.queues[setup.repoPath]+ return q != nil && len(q.tickets) == 3 && q.tickets[1].record != nil && q.tickets[2].record != nil+ }, 10*time.Second, 5*time.Millisecond)+ require.Empty(t, recordedChanges(t, setup.repoPath))++ // The first committed phase writes all three with one write: their entries share the+ // time of that write. It returns only once its entry is durable, so it is in the log.+ require.NoError(t, setup.run(ReferenceTransactionCommitted, setup.first))+ require.Equal(t, []string{setup.first, setup.second, setup.third}, recordedChanges(t, setup.repoPath))+ require.NoError(t, <-laterDone)+ require.NoError(t, <-laterDone)++ entries, err := changelog.List(setup.repoPath, 0, 10)+ require.NoError(t, err)+ require.Len(t, entries, 3)+ require.Equal(t, entries[0].RecordedAt, entries[1].RecordedAt)+ require.Equal(t, entries[0].RecordedAt, entries[2].RecordedAt)++ changesetTickets.Lock()+ _, ok := changesetTickets.queues[setup.repoPath]+ changesetTickets.Unlock()+ require.False(t, ok, "the queue is dropped once it is empty")+ })+}++// TestReferenceTransactionHook_changesetRecordsLost is not parallel because it reads the counter+// of lost records, which is global.+func TestReferenceTransactionHook_changesetRecordsLost(t *testing.T) {+ ctx := testhelper.Context(t)+ setup := setupChangesetOrder(t, ctx)++ // The log cannot be written: a directory takes the place of its file.+ require.NoError(t, os.MkdirAll(changelog.Path(setup.repoPath), 0o755))++ before := testutil.ToFloat64(changelogRecordsLostTotal)+ require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.first))+ require.NoError(t, setup.run(ReferenceTransactionCommitted, setup.first), "recording never fails the update")+ require.InDelta(t, 1, testutil.ToFloat64(changelogRecordsLostTotal)-before, 0)}// TestReferenceTransactionHook_changesetOrderExpiredTicket is not parallel because it lowers the@@ -181,7 +279,7 @@ func TestReferenceTransactionHook_changesetOrderExpiredTicket(t *testing.T) {// The expired ticket is dropped when the next transaction is prepared.require.NoError(t, setup.run(ReferenceTransactionPrepared, setup.first))changesetTickets.Lock()- queue := changesetTickets.queues[setup.repoPath]+ queue := changesetTickets.queues[setup.repoPath].ticketschangesetTickets.Unlock()require.Len(t, queue, 1)require.NoError(t, setup.run(ReferenceTransactionAborted, setup.first))diff --git a/internal/gitaly/hook/referencetransaction.go b/internal/gitaly/hook/referencetransaction.goindex 7c40d74..ebd000c 100644--- a/internal/gitaly/hook/referencetransaction.go+++ b/internal/gitaly/hook/referencetransaction.go@@ -240,48 +240,75 @@ func (m *GitLabHookManager) recordChangesets(ctx context.Context, objectHash git}// The updates are recorded in the order in which their transactions were prepared, which- // is the order in which the references were updated. See changesetTickets. The ticket is- // released however recording ends, so that later transactions do not wait for it. A- // packed-refs cleanup records nothing, so it does not wait.- if ticket, earlier := findChangesetTicket(repoPath, changes); ticket != nil {- defer releaseChangesetTicket(repoPath, ticket)-- if !packedCleanup && !waitForChangesetTickets(ctx, earlier) {- m.logger.WarnContext(ctx, "recording changesets without waiting for an earlier reference transaction, the changeset log may list them out of order")+ // is the order in which the references were updated. See changesetTickets. A transaction+ // that records nothing gives up its place at once, so that later transactions do not wait+ // for it: a packed-refs cleanup, a transaction whose metadata cannot be read, and one on+ // the MVCC reference backend, whose updates are recorded after its commit.+ ticket := findChangesetTicket(repoPath, changes)+ release := func() {+ if ticket != nil {+ releaseChangesetTicket(repoPath, ticket)}}if packedCleanup {+ release()return}if metaErr != nil {+ release()m.logger.WithError(metaErr).ErrorContext(ctx, "recording changesets: reading changelog metadata")return}- meta := changelogMetaOf(payload, graftMeta)-- appendChangesets := func() {- if _, err := changelog.Append(repoPath, meta, updates); err != nil {- m.logger.WithError(err).ErrorContext(ctx, "recording changesets failed")- }- }+ record := changelog.Record{Meta: changelogMetaOf(payload, graftMeta), Updates: updates}// On a repository that uses the MVCC reference backend, the updates are staged in the// snapshot of the request that performs them, and are only published when that request// commits its MVCC cache after its handler returned. The commit fails when a concurrent// request committed first, and the updates are then thrown away, so they are only- // recorded once the commit succeeded.+ // recorded once the commit succeeded, outside of the order of the tickets.if graftMeta.MVCCCommit != "" {+ release()cache := mvcc.CacheByCommitHandle(graftMeta.MVCCCommit)- if cache == nil || !cache.AfterCommit(appendChangesets) {+ if cache == nil || !cache.AfterCommit(func() {+ if _, err := changelog.Append(repoPath, record.Meta, record.Updates); err != nil {+ m.reportLostChangesets(ctx, updates, err)+ }+ }) {m.logger.ErrorContext(ctx, "recording changesets: the MVCC cache that commits the updates is gone, not recording them")}return}- appendChangesets()+ // A transaction without a ticket, prepared before the changeset log was enabled or whose+ // ticket expired, is recorded at once.+ var recordErr error+ if ticket == nil {+ _, recordErr = changelog.Append(repoPath, record.Meta, record.Updates)+ } else {+ recordErr = recordInOrder(repoPath, ticket, record)+ }+ if recordErr != nil {+ m.reportLostChangesets(ctx, updates, recordErr)+ }+}++// reportLostChangesets reports committed reference updates that the changeset log does not+// record because writing them failed. Recording never fails the reference update, which git+// committed already, so the loss is made visible instead: it is counted+// (graft_changelog_records_lost_total) and logged with the references.+func (m *GitLabHookManager) reportLostChangesets(ctx context.Context, updates []changelog.Update, err error) {+ changelogRecordsLostTotal.Add(float64(len(updates)))++ references := make([]string, 0, len(updates))+ for _, update := range updates {+ references = append(references, update.Reference)+ }++ m.logger.WithError(err).WithField("references", references).ErrorContext(ctx,+ "recording changesets failed: the changeset log misses these committed reference updates")}// observeCommitted hands the committed reference updates to the eager observer,