Skip to content

Git designed for AI agents.

syncer, sdk: fixes from the review of the flakes range

Federico Eliaa426873a5cb5
diff --git a/doc/graft/CONFIGURATION.md b/doc/graft/CONFIGURATION.md
index 8477cde..8e3e96b 100644
--- a/doc/graft/CONFIGURATION.md
+++ b/doc/graft/CONFIGURATION.md
@@ -210,7 +210,7 @@ are in [STORAGE.md](STORAGE.md).
| `database` | string | required when on | `sqlite:///absolute/path.db`, or `postgres://user@host:port/db?sslmode=...` (`postgresql://` too). All nodes use the same database. Credentials are not written here: Postgres reads the `PG*` environment variables and `.pgpass`. The schema is created at startup. |
| `bucket` | string | required when on | gocloud URL of the object storage: `file:///absolute/dir`, `s3://bucket?region=...`, `gs://bucket` or `azblob://container`. All nodes use the same bucket. Credentials come from the cloud SDK's environment (`AWS_*`, `GOOGLE_APPLICATION_CREDENTIALS`, `AZURE_*`). |
| `node_id` | string | the host name | Name of this node in the database (changesets record it, and `graft storage state` shows it). Must not contain spaces or slashes. Give every node its own. |
-| `sync_timeout` | string (Go duration) | `"10s"` | Budget of one sync round before a request in authoritative mode. A round that cannot reach the head in time keeps what it did: the local repository stays at the changeset it reached, the request fails with `Unavailable`, and the next request's round goes on from there. A round that cannot finish at all (for example because the database is unreachable or a local writer holds a reference lock) fails the request with `Unavailable` too. Concurrent requests share a round; a request that arrives during a round that read the head before it then runs one of its own, so it waits at most two rounds. Clients retry `Unavailable`: the SDK does it for `Land` (`LandOptions.RetryUnavailable`, see [LAND-BATCHING](LAND-BATCHING.md#limits)), and a client without the SDK must do the same. Must be positive. |
+| `sync_timeout` | string (Go duration) | `"10s"` | Budget of one sync round before a request in authoritative mode. A round that cannot reach the head in time keeps what it did: the local repository stays at the changeset it reached, the request fails with `Unavailable`, and the next request's round goes on from there. A round that cannot finish at all (for example because the database is unreachable or a local writer holds a reference lock) fails the request with `Unavailable` too. Concurrent requests share a round; a request that arrives during a round that read the head before it then runs one of its own, so it waits at most two rounds (and stops waiting with `Unavailable` after twice `sync_timeout` plus a second per round, should a round not end within its budget). Clients retry `Unavailable`: the SDK does it for merge and fast-forward lands (`LandOptions.RetryUnavailable`, see [LAND-BATCHING](LAND-BATCHING.md#limits)); rebase lands are not retried by default, because the server cannot yet tell that a rebased change already landed. A client without the SDK must follow the same rules. Must be positive. |
| `shadow_timeout` | string (Go duration) | `"5s"` | Budget of the shadow-mode work of one reference transaction, for its Preparing phase (pack build and upload, before git locks the references) and its Prepared phase together. When it runs out, the work is abandoned, logged and counted, and the git write proceeds. Must be positive. |
| `gc_grace` | string (Go duration) | `"1h"` | How long garbage collection waits before it deletes a pack it observed unreferenced, and how old a checkpoint must be before the packs it covers are deleted. Must be greater than `sync_timeout + gc_margin`. |
| `gc_margin` | string (Go duration) | `"5m"` | Safety margin of the writers' deadline: a write uploads its pack and commits its changeset within `gc_grace - gc_margin`, else it fails. It also covers the difference between a node's clock and the database's. Must be positive. |
diff --git a/internal/gitaly/service/repository/fetch_remote_test.go b/internal/gitaly/service/repository/fetch_remote_test.go
index db3d905..ad25c42 100644
--- a/internal/gitaly/service/repository/fetch_remote_test.go
+++ b/internal/gitaly/service/repository/fetch_remote_test.go
@@ -27,6 +27,7 @@ import (
"github.com/feder-cr/git-ai/proto/go/gitalypb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
)
const (
@@ -1204,10 +1205,10 @@ func testFetchRemote(t *testing.T, ctx context.Context) {
for _, run := range setupData.runs {
response, err := client.FetchRemote(ctx, setupData.request)
if len(run.expectedErrOneOf) > 0 {
+ // The whole status, its details included, must be one of them.
matched := false
for _, expected := range run.expectedErrOneOf {
- matched = matched || (status.Code(err) == status.Code(expected) &&
- status.Convert(err).Message() == status.Convert(expected).Message())
+ matched = matched || proto.Equal(status.Convert(err).Proto(), status.Convert(expected).Proto())
}
require.True(t, matched, "%v is none of %v", err, run.expectedErrOneOf)
} else {
diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.go
index 2eaa2c3..8709bc2 100644
--- a/internal/graft/corestore/syncer/syncer.go
+++ b/internal/graft/corestore/syncer/syncer.go
@@ -238,20 +238,28 @@ func New(core *corestore.Core, logger log.Logger, locator storage.Locator, repoF
// of the caller that started it, so that a caller that goes away does not fail
// the others; every caller still stops waiting when its own context is done. A
// call waits for at most two syncs, the one it joined and one of its own, so at
-// most twice sync_timeout, and a sync that runs out of its budget fails the
-// calls that wait for it with Unavailable.
+// most twice sync_timeout and the report after each, and a sync that runs out of
+// its budget fails the calls that wait for it with Unavailable. A sync that does
+// not end within that bound is a defect; the call then stops waiting with
+// Unavailable, and the error is logged.
func (s *Syncer) EnsureFresh(ctx context.Context, repo storage.Repository) error {
if s == nil || !s.core.Authoritative() {
return nil
}
- // Each sync (flight) runs under its own sync_timeout. The call has no budget of its own: one
- // set at its arrival would also have to cover the end of a flight it joins and then a whole
- // flight of its own, so it could give up while its own flight is still within its budget.
- // It waits at most for both, each bounded by sync_timeout, and stops when its context ends.
+ // Each sync (flight) runs under its own sync_timeout. The call waits for at most two flights,
+ // the one it joined and one of its own, each bounded by sync_timeout plus the report it makes
+ // after it (detached, at most reportInterval): a budget of one sync_timeout from its arrival
+ // would give up while its own flight is still within its budget. That bound holds as long as
+ // every step of a flight respects its context; the backstop below ends the wait of the call
+ // should a flight not end within it (a stalled file system, a driver that ignores its
+ // context), instead of leaving every caller of the repository waiting for it.
timeout := s.syncTimeout()
key := corestore.KeyOf(repo)
flightCtx := context.WithoutCancel(ctx)
+ backstop := 2 * (timeout + reportInterval)
+ waitCtx, cancel := context.WithTimeout(ctx, backstop)
+ defer cancel()
// A call must not be answered by a sync that read the database head before
// the call arrived: a changeset committed (on another node) between that read
@@ -286,8 +294,14 @@ func (s *Syncer) EnsureFresh(ctx context.Context, repo storage.Repository) error
return nil
}
// The joined sync read the head before this call arrived: sync again.
- case <-ctx.Done():
- return structerr.NewUnavailable("storage core: waiting for the sync of %s: %w", key, ctx.Err())
+ case <-waitCtx.Done():
+ if ctx.Err() != nil {
+ return structerr.NewUnavailable("storage core: waiting for the sync of %s: %w", key, ctx.Err())
+ }
+ // A flight outlived its budget: a step of it does not respect its context.
+ s.logger.WithField("repository", key.String()).WithField("backstop", backstop).
+ Error("storage core: a sync did not end within its budget; the call stops waiting for it")
+ return structerr.NewUnavailable("storage core: waiting for the sync of %s: no sync ended within %s", key, backstop)
}
}
}
diff --git a/internal/graft/corestore/syncer/syncer_budget_test.go b/internal/graft/corestore/syncer/syncer_budget_test.go
index 6950668..af27c62 100644
--- a/internal/graft/corestore/syncer/syncer_budget_test.go
+++ b/internal/graft/corestore/syncer/syncer_budget_test.go
@@ -6,6 +6,8 @@ import (
"time"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
"github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
)
@@ -68,3 +70,35 @@ func TestEnsureFreshWaitsForItsOwnRound(t *testing.T) {
require.Greater(t, time.Since(arrived), syncTimeout, "the call waited longer than one sync_timeout since it arrived")
require.Equal(t, int32(2), rounds.Load(), "the second call ran a round of its own")
}
+
+// TestEnsureFreshBackstop: a sync round that never ends (a step that ignores its
+// context, here the "synced" point held) does not hold its callers forever. A
+// call stops waiting after twice sync_timeout and the report after each round,
+// with Unavailable.
+func TestEnsureFreshBackstop(t *testing.T) {
+ a := newEnv(t, envOptions{})
+ src := a.newSource(t)
+ c1 := a.writeCommit(t, src)
+ a.register(t, a.key(localRelPath), src, c1)
+
+ b := a.secondNode(t, sqlstore.SQLite)
+ const syncTimeout = 200 * time.Millisecond
+ b.core.SyncTimeout = syncTimeout
+
+ release := make(chan struct{})
+ b.sy.SetTestHook(func(point string) {
+ if point == "synced" {
+ <-release
+ }
+ })
+ // The held round ends when the test does, so that nothing outlives it.
+ t.Cleanup(func() { close(release) })
+
+ start := time.Now()
+ err := b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath))
+ elapsed := time.Since(start)
+
+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)
+ require.ErrorContains(t, err, "no sync ended within")
+ require.GreaterOrEqual(t, elapsed, 2*syncTimeout, "the call waited for the two rounds it may need")
+}
diff --git a/sdk/land.go b/sdk/land.go
index e265bd9..f14d6dd 100644
--- a/sdk/land.go
+++ b/sdk/land.go
@@ -2,6 +2,7 @@ package sdk
import (
"context"
+ "fmt"
"math/rand/v2"
"strings"
"time"
@@ -98,6 +99,29 @@ type UnavailableRetry struct {
Cap time.Duration
}
+// UnavailableRetriesError is the error of a land that failed after the SDK
+// retried Unavailable answers: the error of the last call, or of the context
+// that ended while Land waited to retry. It carries the status of that error,
+// so Code, LandError and the Is* helpers read it as they read the error itself.
+type UnavailableRetriesError struct {
+ // Retries is the number of Unavailable answers that were retried.
+ Retries int
+ // LastUnavailable is the message of the last of them.
+ LastUnavailable string
+ // Err is the error the land failed with.
+ Err error
+}
+
+func (e *UnavailableRetriesError) Error() string {
+ return fmt.Sprintf("%v (after %d retries of Unavailable, the last: %s)", e.Err, e.Retries, e.LastUnavailable)
+}
+
+// Unwrap returns the error the land failed with.
+func (e *UnavailableRetriesError) Unwrap() error { return e.Err }
+
+// GRPCStatus returns the status of the error the land failed with.
+func (e *UnavailableRetriesError) GRPCStatus() *status.Status { return status.Convert(e.Err) }
+
// Defaults of UnavailableRetry.
const (
defaultUnavailableRetries = 5
@@ -182,8 +206,9 @@ func oldTip(oid string) string {
// concurrent updates, so many writers can land changes on the same branch
// without a merge queue.
//
-// An Unavailable answer is retried as opts.RetryUnavailable says; when the
-// retries run out, the last Unavailable error is returned.
+// An Unavailable answer is retried as opts.RetryUnavailable says. A land that
+// fails after retries returns an *UnavailableRetriesError, with the retries and
+// the status of the last error.
//
// Failures carry structured details: use IsMergeConflict, IsNotAncestor,
// IsReferenceUpdateConflict, IsRejectedByHook or LandError.
@@ -195,6 +220,8 @@ func (r *Repo) Land(ctx context.Context, opts LandOptions) (*LandResult, error)
return nil, status.Error(codes.InvalidArgument, "commit is required")
case opts.MaxAttempts < 0:
return nil, status.Error(codes.InvalidArgument, "max attempts must not be negative")
+ case opts.RetryUnavailable.MaxRetries < 0, opts.RetryUnavailable.Base < 0, opts.RetryUnavailable.Cap < 0:
+ return nil, status.Error(codes.InvalidArgument, "the retries of Unavailable must not be negative")
}
req := &gitalypb.UserLandRequest{
@@ -230,11 +257,14 @@ func (r *Repo) Land(ctx context.Context, opts LandOptions) (*LandResult, error)
select {
case <-ctx.Done():
timer.Stop()
- return nil, status.FromContextError(ctx.Err()).Err()
+ return nil, &UnavailableRetriesError{Retries: retries, LastUnavailable: lastUnavailable, Err: status.FromContextError(ctx.Err()).Err()}
case <-timer.C:
}
}
if err != nil {
+ if retries > 0 {
+ return nil, &UnavailableRetriesError{Retries: retries, LastUnavailable: lastUnavailable, Err: err}
+ }
return nil, err
}
diff --git a/sdk/land_retry_test.go b/sdk/land_retry_test.go
index 857164c..6b2ba5f 100644
--- a/sdk/land_retry_test.go
+++ b/sdk/land_retry_test.go
@@ -2,6 +2,7 @@ package sdk_test
import (
"context"
+ "errors"
"testing"
"time"
@@ -99,10 +100,54 @@ func TestRepo_Land_UnavailableRetriesAreBounded(t *testing.T) {
_, err := repo.Land(t.Context(), opts)
require.Equal(t, codes.Unavailable, sdk.Code(err), "the last Unavailable is returned")
require.Len(t, fakes.operations.landRequests, tc.expectedRequests)
+
+ // The retries are reported with the failure.
+ var retriesErr *sdk.UnavailableRetriesError
+ if tc.expectedRequests == 1 {
+ require.False(t, errors.As(err, &retriesErr), "a land that was not retried returns its error as is")
+ return
+ }
+ require.ErrorAs(t, err, &retriesErr)
+ require.Equal(t, tc.expectedRequests-1, retriesErr.Retries)
+ require.Equal(t, "still syncing", retriesErr.LastUnavailable)
})
}
}
+func TestRepo_Land_FailureAfterRetriesKeepsItsStatus(t *testing.T) {
+ t.Parallel()
+
+ fakes := newFakeServices()
+ fakes.operations.landErrs = []error{status.Error(codes.Unavailable, "the land lane is closed")}
+ fakes.operations.landErr = status.Error(codes.Aborted, "giving up after 20 attempts")
+ repo := dialFake(t, fakes.register).Repo("repo")
+
+ opts := mergeLand()
+ opts.RetryUnavailable = fastRetry
+ _, err := repo.Land(t.Context(), opts)
+ require.Equal(t, codes.Aborted, sdk.Code(err), "the status of the last error")
+ require.Contains(t, status.Convert(err).Message(), "giving up after 20 attempts")
+ var retriesErr *sdk.UnavailableRetriesError
+ require.ErrorAs(t, err, &retriesErr)
+ require.Equal(t, 1, retriesErr.Retries)
+ require.Equal(t, "the land lane is closed", retriesErr.LastUnavailable)
+}
+
+func TestRepo_Land_NegativeRetriesAreInvalid(t *testing.T) {
+ t.Parallel()
+
+ for _, retry := range []sdk.UnavailableRetry{{MaxRetries: -1}, {Base: -time.Second}, {Cap: -time.Second}} {
+ fakes := newFakeServices()
+ repo := dialFake(t, fakes.register).Repo("repo")
+
+ opts := mergeLand()
+ opts.RetryUnavailable = retry
+ _, err := repo.Land(t.Context(), opts)
+ require.Equal(t, codes.InvalidArgument, sdk.Code(err), "%+v", retry)
+ require.Empty(t, fakes.operations.landRequests)
+ }
+}
+
func TestRepo_Land_RetriesOnlyUnavailable(t *testing.T) {
t.Parallel()
@@ -140,6 +185,12 @@ func TestRepo_Land_ContextStopsTheRetries(t *testing.T) {
_, err := repo.Land(ctx, opts)
require.Equal(t, codes.DeadlineExceeded, sdk.Code(err))
require.Less(t, len(fakes.operations.landRequests), 6)
+
+ // The error keeps what the server answered last.
+ var retriesErr *sdk.UnavailableRetriesError
+ require.ErrorAs(t, err, &retriesErr)
+ require.Equal(t, "still syncing", retriesErr.LastUnavailable)
+ require.Contains(t, err.Error(), "still syncing")
}
func TestRepo_Land_RetriedResolveLandReportsItsOutcome(t *testing.T) {