Skip to content

Git designed for AI agents.

syncer, sdk: the Unavailable contract, on both sides

Federico Elia10be6799e83b
diff --git a/cmd/graft-bench/bench.go b/cmd/graft-bench/bench.go
index 617bd53..656c31c 100644
--- a/cmd/graft-bench/bench.go
+++ b/cmd/graft-bench/bench.go
@@ -226,6 +226,9 @@ type LandOutcome struct {
// server does not batch.
BatchSize int
BatchPosition int
+ // UnavailableRetries is the number of Unavailable answers the client
+ // retried before the land succeeded (sdk.UnavailableRetry).
+ UnavailableRetries int
}
// Sample is the measurement of one change.
@@ -289,6 +292,9 @@ type Result struct {
Conflicts int `json:"conflicts"`
Resolutions int `json:"resolutions"`
Attempts int64 `json:"attempts"`
+ // UnavailableRetries counts the Unavailable answers that the client
+ // retried, over the landed changes (a failed land reports none).
+ UnavailableRetries int `json:"unavailable_retries"`
// Contention. ObservedConflictRate is the share of the changes that
// failed on a merge conflict.
@@ -502,6 +508,7 @@ func aggregate(cfg Config, samples []Sample, elapsed time.Duration) Result {
}
res.Attempts += int64(s.Outcome.Attempts)
res.Resolutions += s.Outcome.Resolved
+ res.UnavailableRetries += s.Outcome.UnavailableRetries
if s.Outcome.Attempts > res.MaxAttempts {
res.MaxAttempts = s.Outcome.Attempts
}
diff --git a/cmd/graft-bench/bench_test.go b/cmd/graft-bench/bench_test.go
index 10a90b0..f0326f1 100644
--- a/cmd/graft-bench/bench_test.go
+++ b/cmd/graft-bench/bench_test.go
@@ -274,6 +274,8 @@ func TestLandOutcome(t *testing.T) {
batched := landOutcome(&sdk.LandResult{Batch: sdk.BatchInfo{Size: 7, Position: 3}})
require.Equal(t, 7, batched.BatchSize)
require.Equal(t, 3, batched.BatchPosition)
+
+ require.Equal(t, 2, landOutcome(&sdk.LandResult{UnavailableRetries: 2}).UnavailableRetries)
}
func TestParseOptions(t *testing.T) {
diff --git a/cmd/graft-bench/bench_v2_test.go b/cmd/graft-bench/bench_v2_test.go
index af5d053..0d03c3a 100644
--- a/cmd/graft-bench/bench_v2_test.go
+++ b/cmd/graft-bench/bench_v2_test.go
@@ -268,12 +268,13 @@ func TestAggregateBatchAndAttemptsStats(t *testing.T) {
t.Parallel()
samples := []Sample{
- {Landed: true, Outcome: LandOutcome{Attempts: 1, BatchSize: 2}},
- {Landed: true, Outcome: LandOutcome{Attempts: 3, BatchSize: 4}},
+ {Landed: true, Outcome: LandOutcome{Attempts: 1, BatchSize: 2, UnavailableRetries: 2}},
+ {Landed: true, Outcome: LandOutcome{Attempts: 3, BatchSize: 4, UnavailableRetries: 1}},
{Err: errors.New("x"), Outcome: LandOutcome{Attempts: 5, BatchSize: 9}},
}
res := aggregate(baseConfig(), samples, time.Second)
+ require.Equal(t, 3, res.UnavailableRetries, "the client retries of Unavailable are reported")
require.InDelta(t, 3, res.BatchSizeMean, 1e-9, "a failed change is not part of a landed batch")
require.Equal(t, 4, res.BatchSizeMax)
require.Equal(t, int64(9), res.Attempts)
diff --git a/cmd/graft-bench/compare.go b/cmd/graft-bench/compare.go
index 798ae95..a7817a4 100644
--- a/cmd/graft-bench/compare.go
+++ b/cmd/graft-bench/compare.go
@@ -34,6 +34,7 @@ var comparisonMetrics = []comparisonMetric{
{"max attempts (all changes)", func(r Result) string { return fmt.Sprint(r.MaxAttempts) }},
{"attempts histogram (landed)", func(r Result) string { return formatHistogram(r.AttemptsHistogram) }},
{"attempts histogram (failed)", func(r Result) string { return formatHistogram(r.FailedAttemptsHistogram) }},
+ {"unavailable retries", func(r Result) string { return fmt.Sprint(r.UnavailableRetries) }},
{"batch size mean", func(r Result) string { return fmt.Sprintf("%.2f", r.BatchSizeMean) }},
{"batch size max", func(r Result) string { return fmt.Sprint(r.BatchSizeMax) }},
{"batch size histogram", func(r Result) string { return formatHistogram(r.BatchSizeHistogram) }},
diff --git a/cmd/graft-bench/main.go b/cmd/graft-bench/main.go
index 995e9b0..d86d8e2 100644
--- a/cmd/graft-bench/main.go
+++ b/cmd/graft-bench/main.go
@@ -259,6 +259,7 @@ func writeResult(w io.Writer, format string, res Result) error {
{"attempts", fmt.Sprintf("%d over all changes (%.2f per landed change, max %d)", res.Attempts, res.AttemptsPerLand, res.MaxAttempts)},
{"attempts histogram", formatHistogram(res.AttemptsHistogram) + " (landed changes)"},
{"failed attempts histogram", formatHistogram(res.FailedAttemptsHistogram) + " (failed changes)"},
+ {"unavailable retries", fmt.Sprintf("%d (client retries of Unavailable, landed changes)", res.UnavailableRetries)},
{"batch size", fmt.Sprintf("mean %.2f, max %d", res.BatchSizeMean, res.BatchSizeMax)},
{"batch size histogram", formatHistogram(res.BatchSizeHistogram)},
{"duration", fmt.Sprintf("%.2fs", res.DurationSeconds)},
diff --git a/cmd/graft-bench/sdk_backend.go b/cmd/graft-bench/sdk_backend.go
index bb98879..d4e0293 100644
--- a/cmd/graft-bench/sdk_backend.go
+++ b/cmd/graft-bench/sdk_backend.go
@@ -213,5 +213,7 @@ func landOutcome(res *sdk.LandResult) LandOutcome {
Resolved: len(res.ResolvedPaths),
BatchSize: int(res.Batch.Size),
BatchPosition: int(res.Batch.Position),
+
+ UnavailableRetries: res.UnavailableRetries,
}
}
diff --git a/doc/graft/BENCH.md b/doc/graft/BENCH.md
index 162da75..6b2fd34 100644
--- a/doc/graft/BENCH.md
+++ b/doc/graft/BENCH.md
@@ -245,6 +245,7 @@ Markdown report show the same values.
| `resolutions` | Conflicting paths the server resolved. |
| `attempts` | Compare-and-swap attempts the server used, summed over every change, landed or failed. A land that gave up because the branch kept moving counts the attempts it made (the number the server reports in its "giving up after N attempts" error). Other failed lands count none, because the server does not report their attempts. Above `landed` means the branch moved during lands. |
| `attempts_per_land` | `attempts` divided by `landed`. The attempts wasted on failed changes raise it, so it is the server work per landed change. |
+| `unavailable_retries` | Calls that the server answered with `Unavailable` and the SDK retried (`LandOptions.RetryUnavailable`), summed over the landed changes. Each is a whole land request again, the conflict resolver included. A failed land reports none. |
| `max_attempts` | The largest number of attempts of any change, landed or failed. |
| `attempts_histogram` | Landed changes by their number of attempts (`{"1": 95, "2": 5}`: 95 changes landed at the first attempt). Landed changes that report no attempts are left out. |
| `failed_attempts_histogram` | Failed changes that report attempts, by their number of attempts. A land that gave up after `max_attempts` rounds shows up here. |
diff --git a/doc/graft/CONFIGURATION.md b/doc/graft/CONFIGURATION.md
index 4c7b6d9..8477cde 100644
--- a/doc/graft/CONFIGURATION.md
+++ b/doc/graft/CONFIGURATION.md
@@ -210,7 +210,7 @@ are in [STORAGE.md](STORAGE.md).
| `database` | string | required when on | `sqlite:///absolute/path.db`, or `postgres://user@host:port/db?sslmode=...` (`postgresql://` too). All nodes use the same database. Credentials are not written here: Postgres reads the `PG*` environment variables and `.pgpass`. The schema is created at startup. |
| `bucket` | string | required when on | gocloud URL of the object storage: `file:///absolute/dir`, `s3://bucket?region=...`, `gs://bucket` or `azblob://container`. All nodes use the same bucket. Credentials come from the cloud SDK's environment (`AWS_*`, `GOOGLE_APPLICATION_CREDENTIALS`, `AZURE_*`). |
| `node_id` | string | the host name | Name of this node in the database (changesets record it, and `graft storage state` shows it). Must not contain spaces or slashes. Give every node its own. |
-| `sync_timeout` | string (Go duration) | `"10s"` | Budget of one sync round before a request in authoritative mode. A round that cannot reach the head in time keeps what it did: the local repository stays at the changeset it reached, the request fails with `Unavailable`, and the next request's round goes on from there. A round that cannot finish at all (for example because the database is unreachable or a local writer holds a reference lock) fails the request with `Unavailable` too. Must be positive. |
+| `sync_timeout` | string (Go duration) | `"10s"` | Budget of one sync round before a request in authoritative mode. A round that cannot reach the head in time keeps what it did: the local repository stays at the changeset it reached, the request fails with `Unavailable`, and the next request's round goes on from there. A round that cannot finish at all (for example because the database is unreachable or a local writer holds a reference lock) fails the request with `Unavailable` too. Concurrent requests share a round; a request that arrives during a round that read the head before it then runs one of its own, so it waits at most two rounds. Clients retry `Unavailable`: the SDK does it for `Land` (`LandOptions.RetryUnavailable`, see [LAND-BATCHING](LAND-BATCHING.md#limits)), and a client without the SDK must do the same. Must be positive. |
| `shadow_timeout` | string (Go duration) | `"5s"` | Budget of the shadow-mode work of one reference transaction, for its Preparing phase (pack build and upload, before git locks the references) and its Prepared phase together. When it runs out, the work is abandoned, logged and counted, and the git write proceeds. Must be positive. |
| `gc_grace` | string (Go duration) | `"1h"` | How long garbage collection waits before it deletes a pack it observed unreferenced, and how old a checkpoint must be before the packs it covers are deleted. Must be greater than `sync_timeout + gc_margin`. |
| `gc_margin` | string (Go duration) | `"5m"` | Safety margin of the writers' deadline: a write uploads its pack and commits its changeset within `gc_grace - gc_margin`, else it fails. It also covers the difference between a node's clock and the database's. Must be positive. |
diff --git a/doc/graft/LAND-BATCHING.md b/doc/graft/LAND-BATCHING.md
index f9d99b8..b4f6f27 100644
--- a/doc/graft/LAND-BATCHING.md
+++ b/doc/graft/LAND-BATCHING.md
@@ -478,7 +478,13 @@ member) and removed it; see [PLAN8.md](PLAN8.md).
branch still race on the reference, as before.
- The lane lives in memory. Requests queued when a Gitaly process stops are not
landed. When the lane registry is closed, queued requests fail with
- `Unavailable`, and callers retry.
+ `Unavailable`, and callers retry. The SDK does this for `Land` by default
+ (`LandOptions.RetryUnavailable`: up to 5 retries of `Unavailable` only, with
+ an exponential backoff from 100 ms to 2 s with full jitter, stopped by the
+ context), and reports the retries in `LandResult.UnavailableRetries`. A client
+ that does not use the SDK must do the same: retry `Unavailable`, and only
+ `Unavailable`, with a bounded backoff. A land is safe to retry: a change that
+ already reached the branch is reported as already landed.
- A land that arrives alone waits up to `window` before its batch runs. Set
`window = "0s"` to remove the wait, or `disabled = true` to remove batching.
- Requests that do not qualify (see *When a land is batched*) run unbatched and
diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.go
index 6f3a924..2eaa2c3 100644
--- a/internal/graft/corestore/syncer/syncer.go
+++ b/internal/graft/corestore/syncer/syncer.go
@@ -236,16 +236,20 @@ func New(core *corestore.Core, logger log.Logger, locator storage.Locator, repoF
// Concurrent calls for one repository share one sync (single flight). The
// sync runs under its own sync_timeout budget, detached from the cancellation
// of the caller that started it, so that a caller that goes away does not fail
-// the others; every caller still stops waiting when its own context is done.
+// the others; every caller still stops waiting when its own context is done. A
+// call waits for at most two syncs, the one it joined and one of its own, so at
+// most twice sync_timeout, and a sync that runs out of its budget fails the
+// calls that wait for it with Unavailable.
func (s *Syncer) EnsureFresh(ctx context.Context, repo storage.Repository) error {
if s == nil || !s.core.Authoritative() {
return nil
}
+ // Each sync (flight) runs under its own sync_timeout. The call has no budget of its own: one
+ // set at its arrival would also have to cover the end of a flight it joins and then a whole
+ // flight of its own, so it could give up while its own flight is still within its budget.
+ // It waits at most for both, each bounded by sync_timeout, and stops when its context ends.
timeout := s.syncTimeout()
- ctx, cancel := context.WithTimeout(ctx, timeout)
- defer cancel()
-
key := corestore.KeyOf(repo)
flightCtx := context.WithoutCancel(ctx)
diff --git a/internal/graft/corestore/syncer/syncer_budget_test.go b/internal/graft/corestore/syncer/syncer_budget_test.go
new file mode 100644
index 0000000..6950668
--- /dev/null
+++ b/internal/graft/corestore/syncer/syncer_budget_test.go
@@ -0,0 +1,70 @@
+package syncer_test
+
+import (
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+)
+
+// TestEnsureFreshWaitsForItsOwnRound: a call that arrives during a sync round
+// of its repository joins it and, as that round read the head before the call
+// arrived, then waits for a round of its own. Each round runs under its own
+// sync_timeout. The call must not give up while its own round is within that
+// budget, even when, with the round it joined, it waited longer than one
+// sync_timeout since it arrived.
+//
+// The rounds are made slow at their end (the "synced" point, after the sync
+// itself returned): the joined round ends 0.7 x sync_timeout after the call
+// arrived, and the call's own round 1.2 x sync_timeout after it, which is
+// 0.5 x sync_timeout into that round.
+func TestEnsureFreshWaitsForItsOwnRound(t *testing.T) {
+ a := newEnv(t, envOptions{})
+ src := a.newSource(t)
+ c1 := a.writeCommit(t, src)
+ a.register(t, a.key(localRelPath), src, c1)
+
+ b := a.secondNode(t, sqlstore.SQLite)
+ const syncTimeout = 2 * time.Second
+ b.core.SyncTimeout = syncTimeout
+
+ var (
+ rounds atomic.Int32
+ secondArrived atomic.Int64
+ firstRoundHeld = make(chan struct{})
+ releaseFirst = make(chan struct{})
+ )
+ b.sy.SetTestHook(func(point string) {
+ if point != "synced" {
+ return
+ }
+ switch rounds.Add(1) {
+ case 1:
+ close(firstRoundHeld)
+ <-releaseFirst
+ case 2:
+ time.Sleep(time.Until(time.Unix(0, secondArrived.Load()).Add(syncTimeout * 12 / 10)))
+ }
+ })
+
+ first := make(chan error, 1)
+ go func() { first <- b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)) }()
+ <-firstRoundHeld
+
+ // The second call arrives while the first round is held after it read the head.
+ arrived := time.Now()
+ secondArrived.Store(arrived.UnixNano())
+ second := make(chan error, 1)
+ go func() { second <- b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)) }()
+
+ time.Sleep(time.Until(arrived.Add(syncTimeout * 7 / 10)))
+ close(releaseFirst)
+
+ require.NoError(t, <-first)
+ require.NoError(t, <-second, "the call gave up while its own round was within its budget")
+ require.Greater(t, time.Since(arrived), syncTimeout, "the call waited longer than one sync_timeout since it arrived")
+ require.Equal(t, int32(2), rounds.Load(), "the second call ran a round of its own")
+}
diff --git a/sdk/doc.go b/sdk/doc.go
index c4b54b9..e6029ce 100644
--- a/sdk/doc.go
+++ b/sdk/doc.go
@@ -11,7 +11,8 @@
// - GetObjects fetches raw objects by ID, resuming interrupted transfers.
// - CommitFiles builds a commit on the server from a list of actions.
// - Land lands a commit on a branch, rebasing or merging onto the current
-// tip and retrying concurrent updates on the server.
+// tip and retrying concurrent updates on the server. It retries the
+// calls the server answers with Unavailable (see UnavailableRetry).
// - Blame and PathHistory answer history questions from derived data.
// - Changesets and WatchChangesets read the repository's changeset log.
//
diff --git a/sdk/fakes_test.go b/sdk/fakes_test.go
index 254feac..28231fa 100644
--- a/sdk/fakes_test.go
+++ b/sdk/fakes_test.go
@@ -300,6 +300,9 @@ type fakeOperations struct {
landResponse *gitalypb.UserLandResponse
// landErr, if set, is returned by UserLand instead of landResponse.
landErr error
+ // landErrs are returned by the first calls of UserLand, one per call, before landErr or
+ // landResponse.
+ landErrs []error
}
type recordedAction struct {
@@ -348,8 +351,15 @@ func (f *fakeOperations) UserCommitFiles(stream gitalypb.OperationService_UserCo
func (f *fakeOperations) UserLand(_ context.Context, req *gitalypb.UserLandRequest) (*gitalypb.UserLandResponse, error) {
f.mu.Lock()
f.landRequests = append(f.landRequests, req)
+ var next error
+ if len(f.landErrs) > 0 {
+ next, f.landErrs = f.landErrs[0], f.landErrs[1:]
+ }
f.mu.Unlock()
+ if next != nil {
+ return nil, next
+ }
if f.landErr != nil {
return nil, f.landErr
}
diff --git a/sdk/land.go b/sdk/land.go
index 7361485..bf1ee43 100644
--- a/sdk/land.go
+++ b/sdk/land.go
@@ -2,6 +2,7 @@ package sdk
import (
"context"
+ "math/rand/v2"
"strings"
"time"
@@ -56,6 +57,69 @@ type LandOptions struct {
// resolver is disabled, refuses a file or fails, the landing fails with a
// merge conflict as without this option.
ResolveConflicts bool
+ // RetryUnavailable is how Land retries a call that the server answers
+ // with Unavailable (see UnavailableRetry). It is on by default.
+ RetryUnavailable UnavailableRetry
+}
+
+// UnavailableRetry is the client side of the server's contract for
+// Unavailable: a node that cannot serve a land right now (its storage core
+// sync ran out of sync_timeout, its land lane was closed) answers Unavailable,
+// and the client retries later (doc/graft/LAND-BATCHING.md, Limits). Land
+// retries only codes.Unavailable, never another code, with an exponential
+// backoff with full jitter: before retry n it waits a random time between zero
+// and min(Cap, Base x 2^(n-1)). A context that ends stops the waiting.
+//
+// A land is safe to retry: the server lands a change by compare-and-swap, and
+// a change that already reached the branch, for example because the response
+// of an earlier call was lost, is reported as AlreadyLanded instead of being
+// landed again. A retried land with ResolveConflicts may run the conflict
+// resolver again, a model included, for the same conflict; LandResult reports
+// the retries (UnavailableRetries), so that this cost is visible.
+type UnavailableRetry struct {
+ // Disabled turns the retries off: Land returns Unavailable at once.
+ Disabled bool
+ // MaxRetries bounds the retries after the first call. Zero means 5.
+ MaxRetries int
+ // Base is the backoff of the first retry. Zero means 100ms.
+ Base time.Duration
+ // Cap bounds the backoff of every retry. Zero means 2s.
+ Cap time.Duration
+}
+
+// Defaults of UnavailableRetry.
+const (
+ defaultUnavailableRetries = 5
+ defaultUnavailableBase = 100 * time.Millisecond
+ defaultUnavailableCap = 2 * time.Second
+)
+
+// backoff returns the random wait before retry n (n >= 1).
+func (r UnavailableRetry) backoff(n int) time.Duration {
+ base, maxBackoff := r.Base, r.Cap
+ if base <= 0 {
+ base = defaultUnavailableBase
+ }
+ if maxBackoff <= 0 {
+ maxBackoff = defaultUnavailableCap
+ }
+ limit := base
+ for i := 1; i < n && limit < maxBackoff; i++ {
+ limit *= 2
+ }
+ limit = min(limit, maxBackoff)
+ return time.Duration(rand.Int64N(int64(limit) + 1))
+}
+
+func (r UnavailableRetry) maxRetries() int {
+ switch {
+ case r.Disabled:
+ return 0
+ case r.MaxRetries > 0:
+ return r.MaxRetries
+ default:
+ return defaultUnavailableRetries
+ }
}
// LandResult describes the outcome of a successful Land call.
@@ -84,6 +148,12 @@ type LandResult struct {
// Batch is the land batch the change was part of. A change that landed
// alone has Size 1 and Position 1, see BatchInfo.
Batch BatchInfo
+ // UnavailableRetries is the number of calls the server answered with
+ // Unavailable before this one succeeded (see UnavailableRetry).
+ UnavailableRetries int
+ // LastUnavailable is the message of the last of those answers, empty
+ // without retries.
+ LastUnavailable string
}
// oldTip maps the zero object ID that the server reports for a created branch
@@ -100,6 +170,9 @@ func oldTip(oid string) string {
// concurrent updates, so many writers can land changes on the same branch
// without a merge queue.
//
+// An Unavailable answer is retried as opts.RetryUnavailable says; when the
+// retries run out, the last Unavailable error is returned.
+//
// Failures carry structured details: use IsMergeConflict, IsNotAncestor,
// IsReferenceUpdateConflict, IsRejectedByHook or LandError.
func (r *Repo) Land(ctx context.Context, opts LandOptions) (*LandResult, error) {
@@ -127,7 +200,28 @@ func (r *Repo) Land(ctx context.Context, opts LandOptions) (*LandResult, error)
req.Timestamp = timestamppb.New(opts.Timestamp)
}
- resp, err := r.client.operations.UserLand(ctx, req)
+ var (
+ resp *gitalypb.UserLandResponse
+ err error
+ retries int
+ lastUnavailable string
+ )
+ for {
+ resp, err = r.client.operations.UserLand(ctx, req)
+ if status.Code(err) != codes.Unavailable || retries >= opts.RetryUnavailable.maxRetries() {
+ break
+ }
+ retries++
+ lastUnavailable = status.Convert(err).Message()
+
+ timer := time.NewTimer(opts.RetryUnavailable.backoff(retries))
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return nil, status.FromContextError(ctx.Err()).Err()
+ case <-timer.C:
+ }
+ }
if err != nil {
return nil, err
}
@@ -147,5 +241,8 @@ func (r *Repo) Land(ctx context.Context, opts LandOptions) (*LandResult, error)
ResolvedPaths: resolvedPaths,
Resolver: resp.GetResolver(),
Batch: batchOf(resp),
+
+ UnavailableRetries: retries,
+ LastUnavailable: lastUnavailable,
}, nil
}
diff --git a/sdk/land_retry_test.go b/sdk/land_retry_test.go
new file mode 100644
index 0000000..d164e08
--- /dev/null
+++ b/sdk/land_retry_test.go
@@ -0,0 +1,137 @@
+package sdk_test
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+ "github.com/feder-cr/git-ai/sdk"
+)
+
+// fastRetry is the retry of Unavailable with a backoff short enough for tests.
+var fastRetry = sdk.UnavailableRetry{Base: time.Millisecond, Cap: 2 * time.Millisecond}
+
+func TestRepo_Land_RetriesUnavailable(t *testing.T) {
+ t.Parallel()
+
+ fakes := newFakeServices()
+ fakes.operations.landErrs = []error{
+ status.Error(codes.Unavailable, "storage core: waiting for the sync"),
+ status.Error(codes.Unavailable, "the land lane is closed"),
+ }
+ fakes.operations.landResponse = &gitalypb.UserLandResponse{LandedCommitId: "landed-sha", Attempts: 1}
+ repo := dialFake(t, fakes.register).Repo("repo")
+
+ opts := validLand()
+ opts.RetryUnavailable = fastRetry
+ result, err := repo.Land(t.Context(), opts)
+ require.NoError(t, err)
+ require.Equal(t, "landed-sha", result.LandedCommitID)
+ require.Equal(t, 2, result.UnavailableRetries)
+ require.Equal(t, "the land lane is closed", result.LastUnavailable)
+ require.Len(t, fakes.operations.landRequests, 3)
+}
+
+func TestRepo_Land_UnavailableRetriesAreBounded(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ desc string
+ retry sdk.UnavailableRetry
+ expectedRequests int
+ }{
+ {desc: "default", retry: fastRetry, expectedRequests: 6},
+ {desc: "max retries", retry: sdk.UnavailableRetry{MaxRetries: 2, Base: time.Millisecond}, expectedRequests: 3},
+ {desc: "disabled", retry: sdk.UnavailableRetry{Disabled: true}, expectedRequests: 1},
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ fakes := newFakeServices()
+ fakes.operations.landErr = status.Error(codes.Unavailable, "still syncing")
+ repo := dialFake(t, fakes.register).Repo("repo")
+
+ opts := validLand()
+ opts.RetryUnavailable = tc.retry
+ _, err := repo.Land(t.Context(), opts)
+ require.Equal(t, codes.Unavailable, sdk.Code(err), "the last Unavailable is returned")
+ require.Len(t, fakes.operations.landRequests, tc.expectedRequests)
+ })
+ }
+}
+
+func TestRepo_Land_RetriesOnlyUnavailable(t *testing.T) {
+ t.Parallel()
+
+ for _, code := range []codes.Code{codes.DeadlineExceeded, codes.Internal, codes.Unknown, codes.Aborted, codes.FailedPrecondition} {
+ t.Run(code.String(), func(t *testing.T) {
+ t.Parallel()
+
+ fakes := newFakeServices()
+ fakes.operations.landErr = status.Error(code, "not to be retried")
+ repo := dialFake(t, fakes.register).Repo("repo")
+
+ opts := validLand()
+ opts.RetryUnavailable = fastRetry
+ _, err := repo.Land(t.Context(), opts)
+ require.Equal(t, code, sdk.Code(err))
+ require.Len(t, fakes.operations.landRequests, 1)
+ })
+ }
+}
+
+func TestRepo_Land_ContextStopsTheRetries(t *testing.T) {
+ t.Parallel()
+
+ fakes := newFakeServices()
+ fakes.operations.landErr = status.Error(codes.Unavailable, "still syncing")
+ repo := dialFake(t, fakes.register).Repo("repo")
+
+ // The backoff is longer than the context, which ends during the first wait.
+ ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+ defer cancel()
+ opts := validLand()
+ opts.RetryUnavailable = sdk.UnavailableRetry{Base: time.Hour, Cap: time.Hour}
+ // Each wait is drawn from [0, 1h]: the context ends during one of them unless five draws in
+ // a row fall below 50ms.
+ _, err := repo.Land(ctx, opts)
+ require.Equal(t, codes.DeadlineExceeded, sdk.Code(err))
+ require.Less(t, len(fakes.operations.landRequests), 6)
+}
+
+func TestRepo_Land_RetriedResolveLandReportsItsOutcome(t *testing.T) {
+ t.Parallel()
+
+ // The first call landed the change but its answer was lost to an Unavailable: the retry,
+ // which may run the resolver again, finds the change on the branch.
+ fakes := newFakeServices()
+ fakes.operations.landErrs = []error{status.Error(codes.Unavailable, "connection reset")}
+ fakes.operations.landResponse = &gitalypb.UserLandResponse{
+ LandedCommitId: "tip-sha",
+ AlreadyLanded: true,
+ OldTip: "tip-sha",
+ BatchSize: 3,
+ BatchPosition: 2,
+ }
+ repo := dialFake(t, fakes.register).Repo("repo")
+
+ opts := validLand()
+ opts.ResolveConflicts = true
+ opts.RetryUnavailable = fastRetry
+ result, err := repo.Land(t.Context(), opts)
+ require.NoError(t, err)
+ require.True(t, result.AlreadyLanded)
+ require.Equal(t, "tip-sha", result.LandedCommitID)
+ require.Equal(t, sdk.BatchInfo{Size: 3, Position: 2}, result.Batch)
+ require.Equal(t, 1, result.UnavailableRetries)
+
+ require.Len(t, fakes.operations.landRequests, 2)
+ for _, req := range fakes.operations.landRequests {
+ require.True(t, req.GetResolveConflicts(), "every call asks for the resolution")
+ }
+}