Git designed for AI agents.
operations: a test of concurrent lands while the policy is reloaded
Federico Eliabb5b646391bf
diff --git a/internal/gitaly/service/operations/land_policy_reload_test.go b/internal/gitaly/service/operations/land_policy_reload_test.goindex ed3a2f5..84009a5 100644--- a/internal/gitaly/service/operations/land_policy_reload_test.go+++ b/internal/gitaly/service/operations/land_policy_reload_test.go@@ -2,10 +2,14 @@ package operationsimport ("context"+ "fmt""io""strings"+ "slices"+ "sync""sync/atomic""testing"+ "time""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gittest"@@ -18,6 +22,7 @@ import ("github.com/feder-cr/git-ai/proto/go/gitalypb""github.com/stretchr/testify/require""google.golang.org/grpc/codes"+ "google.golang.org/grpc/metadata")// policyReloader reloads the policy of a running service at a point of a land: once armed, right@@ -51,14 +56,14 @@ func (r *policyReloader) ReferenceTransactionHook(ctx context.Context, state hooreturn r.Manager.ReferenceTransactionHook(ctx, state, env, stdin)}-// setupPolicyReload starts the agents' land service of graft with a policyReloader around its hook-// manager.-func setupPolicyReload(t *testing.T, ctx context.Context, graft config.Graft) (context.Context, config.Cfg, agentLandClient, *policyReloader) {+// setupPolicyReload starts the agents' land service of graft, with more server options, with a+// policyReloader around its hook manager.+func setupPolicyReload(t *testing.T, ctx context.Context, graft config.Graft, options ...testserver.GitalyServerOpt) (context.Context, config.Cfg, agentLandClient, *policyReloader) {reloader := &policyReloader{t: t, store: policystore.New(graft.Policy), running: graft}- ctx, cfg, client := setupAgentLandServiceWith(t, ctx, graft, reloader.store, testserver.WithHookManagerWrapper(func(manager hook.Manager) hook.Manager {+ ctx, cfg, client := setupAgentLandServiceWith(t, ctx, graft, reloader.store, append(options, testserver.WithHookManagerWrapper(func(manager hook.Manager) hook.Manager {reloader.Manager = managerreturn reloader- }))+ }))...)return ctx, cfg, client, reloader}@@ -160,3 +165,158 @@ func testUserLandPolicyReloadedInFlight(t *testing.T, ctx context.Context) {require.ElementsMatch(t, []string{"base", "a", "c"}, mainFiles(t, cfg, repoPath))})}++// TestUserLand_policyReloadsDuringConcurrentLands lands changes of several agents concurrently+// while the policy alternates between two versions, with invalid reloads in between (PLAN12,+// section 8). agent-0 is allowed by one version and denied by the other. Every land either lands+// or is denied; every decision of the hook names one of the two versions, never another one or+// none; and every update of agent-0 that the hook allowed was allowed by the version that allows+// it.+func TestUserLand_policyReloadsDuringConcurrentLands(t *testing.T) {+ t.Parallel()++ runLandBatchTest(t, testUserLandPolicyReloadsDuringConcurrentLands)+}++func testUserLandPolicyReloadsDuringConcurrentLands(t *testing.T, ctx context.Context) {+ t.Parallel()++ const agents, landsPerAgent = 4, 4+ allowAll := []config.GraftPolicyRule{{Actor: "*", Allow: true}}+ denyAgent0 := denyMainFor("agent-0")+ versions := []any{policystore.Hash(allowAll), policystore.Hash(denyAgent0)}++ logger := testhelper.NewLogger(t)+ logs := testhelper.AddLoggerHook(logger)+ ctx, cfg, client, reloader := setupPolicyReload(t, ctx, unbatchedGraft(allowAll), testserver.WithLogger(logger))+ repo, repoPath, _, change := policyReloadRepo(t, ctx, cfg)++ commits := make([][]git.ObjectID, agents)+ for a := range commits {+ for i := 0; i < landsPerAgent; i++ {+ commits[a] = append(commits[a], change(fmt.Sprintf("agent-%d-%d", a, i)))+ }+ }++ // The policy alternates between the two versions every millisecond until the lands are over;+ // between each two, an invalid policy and an unreadable configuration are rejected. The store+ // is read after each step: it always holds one of the two versions.+ landsDone := make(chan struct{})+ reloadsDone := make(chan []string)+ go func() {+ var problems []string+ check := func(step string, outcome policystore.Outcome) {+ current := reloader.store.Current()+ if current == nil || !slices.Contains(versions, any(current.SHA256)) {+ problems = append(problems, fmt.Sprintf("%s: the policy in force is neither version: %+v", step, current))+ }+ if outcome.Result == policystore.Rejected && outcome.Current != outcome.Previous {+ problems = append(problems, step+": a rejected reload replaced the policy")+ }+ }+ reload := func(rules []config.GraftPolicyRule) (policystore.Outcome, error) {+ next := reloader.running+ next.Policy = rules+ return reloader.store.Reload(next, reloader.running)+ }++ ticker := time.NewTicker(time.Millisecond)+ defer ticker.Stop()+ for i := 0; ; i++ {+ select {+ case <-landsDone:+ reloadsDone <- problems+ return+ case <-ticker.C:+ }++ outcome, err := reload([]config.GraftPolicyRule{{Actor: "*", Refs: []string{""}, Allow: true}})+ if err == nil || outcome.Result != policystore.Rejected {+ problems = append(problems, "an invalid policy was not rejected")+ }+ check("invalid policy", outcome)+ check("unreadable configuration", reloader.store.Reject())++ rules := allowAll+ if i%2 == 0 {+ rules = denyAgent0+ }+ outcome, err = reload(rules)+ if err != nil {+ problems = append(problems, fmt.Sprintf("a valid policy was rejected: %v", err))+ }+ check("valid policy", outcome)+ }+ }()++ var wg sync.WaitGroup+ errs := make([][]error, agents)+ // lastStart is the generation in force when each agent's last land started. The hook decides+ // after that, so it decides with that generation or a later one.+ lastStart := make([]uint64, agents)+ for a := 0; a < agents; a++ {+ errs[a] = make([]error, landsPerAgent)+ wg.Add(1)+ go func(a int) {+ defer wg.Done()+ agentCtx := metadata.AppendToOutgoingContext(ctx, testAgentHeader, fmt.Sprintf("agent-%d", a))+ for i, commit := range commits[a] {+ lastStart[a] = reloader.store.Current().Generation+ _, errs[a][i] = client.UserLand(agentCtx, landRequest(repo, git.DefaultBranch, commit))+ }+ }(a)+ }+ wg.Wait()+ close(landsDone)+ require.Empty(t, <-reloadsDone)+ require.Greater(t, reloader.store.Current().Generation, uint64(2), "the policy alternated while the agents landed")++ landed := map[string]int{}+ for a := range errs {+ for i, err := range errs[a] {+ if err != nil {+ testhelper.RequireGrpcCode(t, err, codes.PermissionDenied)+ require.Zero(t, a, "only agent-0 is ever denied, land %d", i)+ continue+ }+ landed[fmt.Sprintf("agent-%d", a)]+++ }+ }+ for a := 1; a < agents; a++ {+ require.Equal(t, landsPerAgent, landed[fmt.Sprintf("agent-%d", a)])+ }++ allowedOfAgent0, decisions := 0, 0+ lastDecided := map[any]uint64{}+ for _, entry := range logs.AllEntries() {+ allowed := entry.Message == "agent reference update allowed by the policy"+ if !allowed && entry.Message != "agent reference update denied" {+ continue+ }+ decisions+++ sha := entry.Data["policy_sha256"]+ require.Contains(t, versions, sha, "a decision names one of the two versions")+ generation, ok := entry.Data["policy_generation"].(uint64)+ require.True(t, ok, "a decision names its generation: %v", entry.Data["policy_generation"])+ lastDecided[entry.Data["actor"]] = generation+ if entry.Data["actor"] != "agent-0" {+ continue+ }+ if allowed {+ require.Equal(t, policystore.Hash(allowAll), sha, "agent-0 is allowed only by the version that allows it")+ allowedOfAgent0+++ } else {+ require.Equal(t, policystore.Hash(denyAgent0), sha, "agent-0 is denied only by the version that denies it")+ }+ }+ require.LessOrEqual(t, landed["agent-0"], allowedOfAgent0, "every land of agent-0 was allowed by the hook")+ require.GreaterOrEqual(t, decisions, landsPerAgent*(agents-1)+landed["agent-0"], "the hook decided every land")+ // The agents that every version allows reach the hook with their last land: the hook decided it+ // with the policy in force then, not an older one.+ for a := 1; a < agents; a++ {+ require.Greater(t, lastStart[a], uint64(1), "agent-%d's last land started after reloads", a)+ require.GreaterOrEqual(t, lastDecided[fmt.Sprintf("agent-%d", a)], lastStart[a], "agent-%d's last land was decided by the policy in force", a)+ }++ require.Len(t, mainFiles(t, cfg, repoPath), 1+landsPerAgent*(agents-1)+landed["agent-0"])+}