Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(review): bound an AI review run by the CI sample and a server-wide limit (#21608)
h3n4l1e16085d3915
diff --git a/backend/api/mcp/gen/openapi.yaml b/backend/api/mcp/gen/openapi.yamlindex dca801b881..c0cbbdcfa2 100644--- a/backend/api/mcp/gen/openapi.yaml+++ b/backend/api/mcp/gen/openapi.yaml@@ -18432,8 +18432,9 @@ components:title: ci_sampling_sizeformat: int32description: |-- The maximum number of database rows to sample during CI data validation.- Without specification, sampling is disabled, resulting in full validation.+ The maximum number of databases to sample during CI data validation and+ AI review. Without specification, sampling is disabled, resulting in full+ validation.parallelTasksPerRollout:type: integertitle: parallel_tasks_per_rolloutdiff --git a/backend/generated-go/store/project.pb.go b/backend/generated-go/store/project.pb.goindex ccef5632b4..b12f830435 100644--- a/backend/generated-go/store/project.pb.go+++ b/backend/generated-go/store/project.pb.go@@ -102,8 +102,9 @@ type Project struct {AllowSelfApproval bool `protobuf:"varint,9,opt,name=allow_self_approval,json=allowSelfApproval,proto3" json:"allow_self_approval,omitempty"`// Configuration for automatic retry on task execution failures.ExecutionRetryPolicy *Project_ExecutionRetryPolicy `protobuf:"bytes,10,opt,name=execution_retry_policy,json=executionRetryPolicy,proto3" json:"execution_retry_policy,omitempty"`- // The maximum number of databases to sample during CI data validation.- // If not specified, sampling is disabled, resulting in a full validation.+ // The maximum number of databases to sample during CI data validation and+ // AI review. If not specified, sampling is disabled, resulting in a full+ // validation.CiSamplingSize int32 `protobuf:"varint,11,opt,name=ci_sampling_size,json=ciSamplingSize,proto3" json:"ci_sampling_size,omitempty"`// The maximum number of parallel tasks to run during the rollout.ParallelTasksPerRollout int32 `protobuf:"varint,12,opt,name=parallel_tasks_per_rollout,json=parallelTasksPerRollout,proto3" json:"parallel_tasks_per_rollout,omitempty"`diff --git a/backend/generated-go/v1/issue_service.pb.go b/backend/generated-go/v1/issue_service.pb.goindex f355f831a9..ad8ee1e5ea 100644--- a/backend/generated-go/v1/issue_service.pb.go+++ b/backend/generated-go/v1/issue_service.pb.go@@ -366,8 +366,11 @@ const (// Created and claimable by any replica.ReviewRun_AVAILABLE ReviewRun_Status = 1ReviewRun_RUNNING ReviewRun_Status = 2- // Every (spec, target) unit was evaluated. DONE does not mean the- // review passed: whether problems remain is judged from open comments.+ // Every (spec, target) unit in the reviewer's scope was evaluated. RULE's+ // scope is every target. AI's scope is the project's CI sample, the first+ // ci_sampling_size targets of each spec, or every target when+ // ci_sampling_size is 0. DONE does not mean the review passed: whether+ // problems remain is judged from open comments.ReviewRun_DONE ReviewRun_Status = 3// At least one unit was not evaluated; the cause is in `error`.ReviewRun_FAILED ReviewRun_Status = 4diff --git a/backend/generated-go/v1/project_service.pb.go b/backend/generated-go/v1/project_service.pb.goindex 88f01c52f2..fbbc4908f7 100644--- a/backend/generated-go/v1/project_service.pb.go+++ b/backend/generated-go/v1/project_service.pb.go@@ -934,8 +934,9 @@ type Project struct {AllowSelfApproval bool `protobuf:"varint,14,opt,name=allow_self_approval,json=allowSelfApproval,proto3" json:"allow_self_approval,omitempty"`// Execution retry policy for task runs.ExecutionRetryPolicy *Project_ExecutionRetryPolicy `protobuf:"bytes,15,opt,name=execution_retry_policy,json=executionRetryPolicy,proto3" json:"execution_retry_policy,omitempty"`- // The maximum number of database rows to sample during CI data validation.- // Without specification, sampling is disabled, resulting in full validation.+ // The maximum number of databases to sample during CI data validation and+ // AI review. Without specification, sampling is disabled, resulting in full+ // validation.CiSamplingSize int32 `protobuf:"varint,16,opt,name=ci_sampling_size,json=ciSamplingSize,proto3" json:"ci_sampling_size,omitempty"`// The maximum number of parallel tasks allowed during rollout execution.ParallelTasksPerRollout int32 `protobuf:"varint,17,opt,name=parallel_tasks_per_rollout,json=parallelTasksPerRollout,proto3" json:"parallel_tasks_per_rollout,omitempty"`diff --git a/backend/runner/plancheck/derive.go b/backend/runner/plancheck/derive.goindex f99969b250..85147cb6ba 100644--- a/backend/runner/plancheck/derive.go+++ b/backend/runner/plancheck/derive.go@@ -13,15 +13,16 @@ import (// DeriveCheckTargets derives check targets from a plan and optional database// group, applying the project's CI sampling limit: plan checks are CI-// validation, and sampling bounds their cost.+// validation, and sampling bounds their cost. The AI review uses the same+// sample to bound its model calls.// This replaces the stored config by computing targets at runtime.func DeriveCheckTargets(ctx context.Context, s *store.Store, project *store.ProjectMessage, plan *store.PlanMessage, databaseGroup *v1pb.DatabaseGroup) ([]*CheckTarget, error) {return deriveTargets(ctx, s, project, plan, databaseGroup, true)}-// DeriveReviewTargets derives the same targets without CI sampling: a review-// run's DONE means every (spec, target) unit was evaluated, so review must-// see the full target set.+// DeriveReviewTargets derives the same targets without CI sampling: a rule+// review run's DONE means every (spec, target) unit was evaluated, so the+// rule review must see the full target set.func DeriveReviewTargets(ctx context.Context, s *store.Store, project *store.ProjectMessage, plan *store.PlanMessage, databaseGroup *v1pb.DatabaseGroup) ([]*CheckTarget, error) {return deriveTargets(ctx, s, project, plan, databaseGroup, false)}diff --git a/backend/runner/plancheck/derive_test.go b/backend/runner/plancheck/derive_test.goindex 3b151a2888..424f3a78da 100644--- a/backend/runner/plancheck/derive_test.go+++ b/backend/runner/plancheck/derive_test.go@@ -11,7 +11,7 @@ import ()// Plan checks are CI validation and honor the project's CI sampling limit;-// a review run's DONE means every (spec, target) unit was evaluated, so+// a rule review run's DONE means every (spec, target) unit was evaluated, so// review derivation must not sample.func TestDeriveReviewTargetsSkipsCISampling(t *testing.T) {ctx := context.Background()@@ -42,7 +42,7 @@ func TestDeriveReviewTargetsSkipsCISampling(t *testing.T) {full, err := DeriveReviewTargets(ctx, nil, project, plan, nil)require.NoError(t, err)- require.Len(t, full, 3, "review must evaluate every target regardless of CI sampling")+ require.Len(t, full, 3, "the rule review evaluates every target regardless of CI sampling")for _, target := range full {require.Equal(t, "spec-1", target.SpecID, "a review result anchors on its spec")}diff --git a/backend/runner/reviewrun/ai_review.go b/backend/runner/reviewrun/ai_review.goindex c4771506c3..d8a0f8ede4 100644--- a/backend/runner/reviewrun/ai_review.go+++ b/backend/runner/reviewrun/ai_review.go@@ -21,10 +21,20 @@ import (const (// maxAIReviewSheetBytes bounds the sheet one review reads whole.maxAIReviewSheetBytes = 256 << 10- // aiReviewConcurrency caps the reviews of one run in flight at the model.- aiReviewConcurrency = 8+ // aiReviewRunConcurrency caps the reviews of one run in flight at the model.+ aiReviewRunConcurrency = 8+ // aiReviewServerConcurrency caps the reviews in flight across every run on+ // the server, so runs side by side cannot multiply the load on the provider.+ aiReviewServerConcurrency = 16)+// reviewSlots is the limit on reviews in flight that every run of an executor+// shares. The server uses a *semaphore.Weighted.+type reviewSlots interface {+ Acquire(ctx context.Context, n int64) error+ Release(n int64)+}+// aiReviewUnit is one AI review: a spec's sheet against one database. It// holds no schema, which the review loads when it starts.type aiReviewUnit struct {@@ -179,15 +189,18 @@ func checkSheetSize(statement string) error {return nil}-// reviewUnits runs review over the units, at most aiReviewConcurrency at a-// time, and returns each unit's result or error by index. A panic in one-// review is that unit's error, so the run fails instead of the process: the-// scheduler's recovery covers only the goroutine that called RunOnce.-func reviewUnits(ctx context.Context, units []*aiReviewUnit, review func(context.Context, *aiReviewUnit) (*aireview.Result, error)) ([]*aiReviewResult, []error) {+// reviewUnits runs review over the units, at most aiReviewRunConcurrency at a+// time, and returns each unit's result or error by index. Each review holds+// one of slots, which every run shares. A unit takes its slot before review+// starts, so the wait does not count against the reviewer's timeout, and+// gives up the wait when ctx is done. A panic in one review is that unit's+// error, so the run fails instead of the process: the scheduler's recovery+// covers only the goroutine that called RunOnce.+func reviewUnits(ctx context.Context, slots reviewSlots, units []*aiReviewUnit, review func(context.Context, *aiReviewUnit) (*aireview.Result, error)) ([]*aiReviewResult, []error) {results := make([]*aiReviewResult, len(units))errs := make([]error, len(units))var group errgroup.Group- group.SetLimit(aiReviewConcurrency)+ group.SetLimit(aiReviewRunConcurrency)for i, unit := range units {group.Go(func() error {defer func() {@@ -196,6 +209,11 @@ func reviewUnits(ctx context.Context, units []*aiReviewUnit, review func(contexterrs[i] = errors.Errorf("%s: review panic: %v", unit.Check.Target, r)}}()+ if err := slots.Acquire(ctx, 1); err != nil {+ errs[i] = errors.Wrapf(err, "%s", unit.Check.Target)+ return nil+ }+ defer slots.Release(1)result, err := review(ctx, unit)if err != nil {errs[i] = errors.Wrapf(err, "%s", unit.Check.Target)diff --git a/backend/runner/reviewrun/ai_review_executor.go b/backend/runner/reviewrun/ai_review_executor.goindex 0c663538a6..3843f9694c 100644--- a/backend/runner/reviewrun/ai_review_executor.go+++ b/backend/runner/reviewrun/ai_review_executor.go@@ -5,29 +5,39 @@ import ("log/slog""github.com/pkg/errors"+ "golang.org/x/sync/semaphore"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/aireview""github.com/bytebase/bytebase/backend/runner/plancheck""github.com/bytebase/bytebase/backend/store")// AIReviewExecutor is the AI review: a model judges each spec's sheet against-// the workspace and project AI review policy, once per database, with that-// database's facts in the prompt and its synced schema behind the tools. The-// reviews of one run share the model and run concurrently under-// aiReviewConcurrency.+// the workspace and project AI review policy, once per database of the+// project's CI sample, with that database's facts in the prompt and its synced+// schema behind the tools. The reviews of one run share the model and run+// concurrently under aiReviewRunConcurrency.type AIReviewExecutor struct {- store *store.Store+ store *store.Store+ newModel func(*storepb.AISetting) aireview.Model+ // slots bounds the reviews in flight across every run of the executor.+ slots reviewSlots}-// NewAIReviewExecutor creates the AI review executor.+// NewAIReviewExecutor creates the AI review executor. The server creates one,+// so aiReviewServerConcurrency bounds the reviews of every run on the server.func NewAIReviewExecutor(s *store.Store) *AIReviewExecutor {- return &AIReviewExecutor{store: s}+ return &AIReviewExecutor{+ store: s,+ newModel: aireview.NewModel,+ slots: semaphore.NewWeighted(aiReviewServerConcurrency),+ }}-// RunOnce implements Executor. Collect-all, no fail-fast: every database is-// attempted, failures aggregate into one message, and the findings are-// returned only when every database was reviewed.+// RunOnce implements Executor. Collect-all, no fail-fast: every database of+// the sample is attempted, failures aggregate into one message, and the+// findings are returned only when every one of them was reviewed.func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueUID int64) ([]*store.IssueCommentMessage, error) {plan, project, err := loadReviewPlan(ctx, e.store, projectID, issueUID)if err != nil {@@ -49,9 +59,10 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueUif err != nil {return nil, errors.Wrapf(err, "failed to get database group for plan")}- // DeriveReviewTargets, not DeriveCheckTargets: every database is reviewed- // with its own facts, so the CI sampling limit does not apply.- checkTargets, err := plancheck.DeriveReviewTargets(ctx, e.store, project, plan, databaseGroup)+ // DeriveCheckTargets, not DeriveReviewTargets: every database costs model+ // calls, so the AI review covers the databases plan checks sample when the+ // project sets ci_sampling_size, and every database otherwise.+ checkTargets, err := plancheck.DeriveCheckTargets(ctx, e.store, project, plan, databaseGroup)if err != nil {return nil, errors.Wrapf(err, "failed to derive review targets")}@@ -72,12 +83,12 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueUunits = append(units, &aiReviewUnit{Check: checkTarget, Statement: statement})}- reviewer := aireview.NewReviewer(aireview.NewModel(aiSetting))- results, reviewErrs := reviewUnits(ctx, units, func(ctx context.Context, unit *aiReviewUnit) (*aireview.Result, error) {- // The schema is loaded inside the bounded pool and the run drops it- // when the review ends, so the run itself holds at most- // aiReviewConcurrency schemas however many databases it reviews. The- // store's schema cache keeps its own bounded copies.+ reviewer := aireview.NewReviewer(e.newModel(aiSetting))+ results, reviewErrs := reviewUnits(ctx, e.slots, units, func(ctx context.Context, unit *aiReviewUnit) (*aireview.Result, error) {+ // The schema is loaded once the unit holds its slot and dropped when+ // the review ends, so the runs together hold at most one schema per+ // slot however many databases they review. The store's schema cache+ // keeps its own bounded copies.request := &aireview.Request{WorkspacePolicy: policy.Workspace,ProjectPolicy: policy.Project,diff --git a/backend/runner/reviewrun/ai_review_executor_test.go b/backend/runner/reviewrun/ai_review_executor_test.gonew file mode 100644index 0000000000..6aeb391471--- /dev/null+++ b/backend/runner/reviewrun/ai_review_executor_test.go@@ -0,0 +1,292 @@+package reviewrun++import (+ "context"+ "fmt"+ "sync/atomic"+ "testing"+ "time"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/bytebase/omni/review"+ "github.com/stretchr/testify/require"+ "golang.org/x/sync/semaphore"+ "google.golang.org/genproto/googleapis/type/expr"+ "google.golang.org/protobuf/encoding/protojson"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/aireview"+ "github.com/bytebase/bytebase/backend/store"+)++// fixtureDatabases is how many databases the fixture's database group+// matches.+const fixtureDatabases = 5++// testWaitLimit fails a test whose reviews stop making progress, instead of+// letting it hang.+const testWaitLimit = 30 * time.Second++func TestCISampleBoundsOnlyTheAIReview(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ stores, issues := setupReviewFixture(ctx, t, 1)+ model := &findingModel{}+ executor := &AIReviewExecutor{+ store: stores,+ newModel: func(*storepb.AISetting) aireview.Model { return model },+ slots: semaphore.NewWeighted(aiReviewServerConcurrency),+ }+ reviewedDatabases := func() []string {+ t.Helper()+ comments, err := executor.RunOnce(ctx, "p", issues[0])+ require.NoError(t, err)+ require.Len(t, comments, 1, "the same finding on every database is one comment")+ return comments[0].Payload.ReviewMetadata.Targets+ }++ setCISamplingSize(ctx, t, stores, 2)+ require.Len(t, reviewedDatabases(), 2)+ require.Equal(t, int32(2), model.calls.Load())++ var ruleTargets int+ ruleExecutor := &RuleExecutor{store: stores, reviewFuncs: map[storepb.Engine]review.ReviewFunc{+ storepb.Engine_POSTGRES: func(_ context.Context, _ string, _ review.Options, targets []review.Target) (*review.Result, error) {+ ruleTargets += len(targets)+ return &review.Result{}, nil+ },+ }}+ _, err := ruleExecutor.RunOnce(ctx, "p", issues[0])+ require.NoError(t, err)+ require.Equal(t, fixtureDatabases, ruleTargets, "the rule review evaluates every database whatever the sample")++ setCISamplingSize(ctx, t, stores, 0)+ require.Len(t, reviewedDatabases(), fixtureDatabases)+}++func TestAIReviewRunsShareTheServerLimit(t *testing.T) {+ t.Parallel()+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ stores, issues := setupReviewFixture(ctx, t, 2)+ const limit = 3+ total := len(issues) * fixtureDatabases+ require.Less(t, limit, fixtureDatabases, "one run alone could exceed the limit")+ model := &blockingModel{entered: make(chan struct{}, total), release: make(chan struct{})}+ slots := &observedSlots{Weighted: semaphore.NewWeighted(limit), waiting: make(chan struct{}, total)}+ executor := &AIReviewExecutor{+ store: stores,+ newModel: func(*storepb.AISetting) aireview.Model { return model },+ slots: slots,+ }++ runErrs := make(chan error, len(issues))+ for _, issueUID := range issues {+ go func() {+ _, err := executor.RunOnce(ctx, "p", issueUID)+ runErrs <- err+ }()+ }+ // Nothing is released yet, so every review either reaches the model or+ // waits for a slot.+ var entered, waiting int+ for entered+waiting < total {+ select {+ case <-model.entered:+ entered+++ case <-slots.waiting:+ waiting+++ case <-time.After(testWaitLimit):+ t.Fatalf("timed out with %d reviews at the model and %d waiting", entered, waiting)+ }+ }+ require.Equal(t, limit, entered, "the reviews of both runs past the limit wait for a slot")+ for range total - limit {+ send(t, model.release)+ receive(t, model.entered)+ }+ for range limit {+ send(t, model.release)+ }+ for range issues {+ require.NoError(t, receive(t, runErrs))+ }+ require.Equal(t, int32(limit), model.maxInFlight.Load())+}++// setupReviewFixture seeds project p with AI enabled, a PostgreSQL instance+// holding fixtureDatabases synced databases, a database group matching all of+// them, and the given number of issues, each with a plan whose one spec+// targets the group. It returns the issue UIDs.+func setupReviewFixture(ctx context.Context, t *testing.T, issueCount int) (*store.Store, []int64) {+ t.Helper()+ db, stores, _ := testcontainer.NewMetadataDB(t)+ _, err := db.ExecContext(ctx, `+ INSERT INTO workspace (resource_id) VALUES ('ws');+ INSERT INTO principal (name, email, password_hash) VALUES ('creator', '[email protected]', 'unused');+ INSERT INTO project (resource_id, workspace, name) VALUES ('p', 'ws', 'P');+ `)+ require.NoError(t, err)+ _, err = stores.UpsertSetting(ctx, &store.SettingMessage{Name: storepb.SettingName_AI, Workspace: "ws", Value: &storepb.AISetting{Enabled: true}})+ require.NoError(t, err)+ rules, err := protojson.Marshal(&storepb.ReviewRulePolicy{Rules: []storepb.ReviewRuleType{storepb.ReviewRuleType_SYNTAX}})+ require.NoError(t, err)+ _, err = stores.CreatePolicy(ctx, &store.PolicyMessage{+ Workspace: "ws",+ ResourceType: storepb.Policy_PROJECT,+ Resource: common.FormatProject("p"),+ Type: storepb.Policy_REVIEW_RULE,+ Payload: string(rules),+ Enforce: true,+ })+ require.NoError(t, err)++ _, err = stores.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: "prod",+ Workspace: "ws",+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)+ for i := range fixtureDatabases {+ name := fmt.Sprintf("db%d", i+1)+ _, err = stores.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "p",+ InstanceID: "prod",+ DatabaseName: name,+ Metadata: &storepb.DatabaseMetadata{Labels: map[string]string{}},+ })+ require.NoError(t, err)+ schema := &metadatapb.DatabaseSchemaMetadata{Name: name, Schemas: []*metadatapb.SchemaMetadata{{Name: "public"}}}+ require.NoError(t, stores.UpsertDBSchema(ctx, "prod", name, schema, &storepb.DatabaseConfig{}, nil))+ }+ _, err = stores.CreateDatabaseGroup(ctx, &store.DatabaseGroupMessage{+ ProjectID: "p",+ ResourceID: "all",+ Title: "all",+ Expression: &expr.Expr{Expression: `resource.instance_id == "prod"`},+ })+ require.NoError(t, err)+ sheets, err := stores.CreateSheets(ctx, "p", &store.SheetMessage{Statement: "ALTER TABLE orders ADD COLUMN note text;"})+ require.NoError(t, err)++ var issues []int64+ for i := range issueCount {+ plan, err := stores.CreatePlan(ctx, &store.PlanMessage{+ ProjectID: "p",+ Name: fmt.Sprintf("plan %d", i),+ Config: &storepb.PlanConfig{+ Specs: []*storepb.PlanConfig_Spec{{+ Id: "spec-1",+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{+ Targets: []string{"projects/p/databaseGroups/all"},+ SheetSha256: sheets[0].Sha256,+ },+ },+ }},+ },+ }, "[email protected]")+ require.NoError(t, err)+ issue, err := stores.CreateIssue(ctx, &store.IssueMessage{+ ProjectID: "p",+ CreatorEmail: "[email protected]",+ Title: fmt.Sprintf("issue %d", i),+ Type: storepb.Issue_DATABASE_CHANGE,+ Payload: &storepb.Issue{},+ PlanUID: &plan.UID,+ })+ require.NoError(t, err)+ issues = append(issues, issue.UID)+ }+ return stores, issues+}++func setCISamplingSize(ctx context.Context, t *testing.T, stores *store.Store, size int32) {+ t.Helper()+ require.NoError(t, stores.UpdateProjects(ctx, &store.UpdateProjectMessage{+ ResourceID: "p",+ Workspace: "ws",+ Setting: &storepb.Project{CiSamplingSize: size},+ }))+}++// findingModel answers every review with the same finding, so a run's one+// comment names every database the run reviewed.+type findingModel struct {+ calls atomic.Int32+}++func (m *findingModel) Chat(context.Context, *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {+ m.calls.Add(1)+ content := `{"findings": [{"title": "Backfill note", "severity": "P2", "line": 1, "rule": "r", "evidence": "e", "fix": "f"}]}`+ return &v1pb.AIChatResponse{Content: &content}, nil+}++// observedSlots tells the test when a review has to wait for a slot.+type observedSlots struct {+ *semaphore.Weighted+ waiting chan struct{}+}++func (s *observedSlots) Acquire(ctx context.Context, n int64) error {+ if s.TryAcquire(n) {+ return nil+ }+ s.waiting <- struct{}{}+ return s.Weighted.Acquire(ctx, n)+}++// blockingModel holds every call until the test releases one, and records+// the most calls in flight at once.+type blockingModel struct {+ entered chan struct{}+ release chan struct{}+ inFlight atomic.Int32+ maxInFlight atomic.Int32+}++func (m *blockingModel) Chat(ctx context.Context, _ *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {+ now := m.inFlight.Add(1)+ defer m.inFlight.Add(-1)+ for {+ seen := m.maxInFlight.Load()+ if now <= seen || m.maxInFlight.CompareAndSwap(seen, now) {+ break+ }+ }+ m.entered <- struct{}{}+ select {+ case <-m.release:+ case <-ctx.Done():+ return nil, ctx.Err()+ }+ content := `{"findings": []}`+ return &v1pb.AIChatResponse{Content: &content}, nil+}++func receive[T any](t *testing.T, ch <-chan T) T {+ t.Helper()+ select {+ case v := <-ch:+ return v+ case <-time.After(testWaitLimit):+ t.Fatal("timed out waiting for the reviews")+ var zero T+ return zero+ }+}++func send(t *testing.T, ch chan<- struct{}) {+ t.Helper()+ select {+ case ch <- struct{}{}:+ case <-time.After(testWaitLimit):+ t.Fatal("timed out waiting for a model call to release")+ }+}diff --git a/backend/runner/reviewrun/ai_review_test.go b/backend/runner/reviewrun/ai_review_test.goindex c48541f1d1..62951bb06f 100644--- a/backend/runner/reviewrun/ai_review_test.go+++ b/backend/runner/reviewrun/ai_review_test.go@@ -10,6 +10,7 @@ import (metadatapb "github.com/bytebase/omni/metadata""github.com/pkg/errors""github.com/stretchr/testify/require"+ "golang.org/x/sync/semaphore"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/aireview"@@ -122,8 +123,9 @@ func TestCheckSheetSize(t *testing.T) {}// TestReviewUnits pins the executor's concurrency guarantees: at most-// aiReviewConcurrency reviews in flight, every unit attempted whatever the-// others do, and a panicking review recorded as that unit's error.+// aiReviewRunConcurrency reviews in flight, every unit attempted whatever the+// others do, a panicking review recorded as that unit's error, and every slot+// returned whatever the review did.func TestReviewUnits(t *testing.T) {t.Parallel()@@ -152,10 +154,11 @@ func TestReviewUnits(t *testing.T) {}}- results, errs := reviewUnits(context.Background(), units, review)+ slots := semaphore.NewWeighted(aiReviewServerConcurrency)+ results, errs := reviewUnits(context.Background(), slots, units, review)require.Len(t, results, 20)require.Len(t, errs, 20)- require.LessOrEqual(t, maxInFlight.Load(), int32(aiReviewConcurrency))+ require.LessOrEqual(t, maxInFlight.Load(), int32(aiReviewRunConcurrency))require.Greater(t, maxInFlight.Load(), int32(1), "the reviews run concurrently")for i, result := range results {switch i {@@ -172,4 +175,27 @@ func TestReviewUnits(t *testing.T) {}}require.Zero(t, inFlight.Load())+ require.True(t, slots.TryAcquire(aiReviewServerConcurrency), "the failed and the panicking reviews return their slots too")+}++func TestReviewUnitsStopWaitingForASlotWhenTheRunEnds(t *testing.T) {+ t.Parallel()++ slots := semaphore.NewWeighted(1)+ require.True(t, slots.TryAcquire(1), "another run holds the only slot")+ units := []*aiReviewUnit{{Check: &plancheck.CheckTarget{SpecID: "spec-1", Target: "instances/i/databases/a"}}}+ var reviews atomic.Int32+ ctx, cancel := context.WithCancel(context.Background())+ done := make(chan []error)+ go func() {+ _, errs := reviewUnits(ctx, slots, units, func(context.Context, *aiReviewUnit) (*aireview.Result, error) {+ reviews.Add(1)+ return &aireview.Result{}, nil+ })+ done <- errs+ }()+ cancel()+ errs := receive(t, done)+ require.ErrorIs(t, errs[0], context.Canceled)+ require.Zero(t, reviews.Load(), "no review starts without a slot")}diff --git a/backend/runner/reviewrun/rule_executor.go b/backend/runner/reviewrun/rule_executor.goindex 7ec1d28c3a..764545d280 100644--- a/backend/runner/reviewrun/rule_executor.go+++ b/backend/runner/reviewrun/rule_executor.go@@ -57,8 +57,8 @@ func (e *RuleExecutor) RunOnce(ctx context.Context, projectID string, issueUID iif err != nil {return nil, errors.Wrapf(err, "failed to get database group for plan")}- // DeriveReviewTargets, not DeriveCheckTargets: review must evaluate every- // (spec, target) unit, so the CI sampling limit does not apply.+ // DeriveReviewTargets, not DeriveCheckTargets: the rule review evaluates+ // every (spec, target) unit, so the CI sampling limit does not apply.checkTargets, err := plancheck.DeriveReviewTargets(ctx, e.store, project, plan, databaseGroup)if err != nil {return nil, errors.Wrapf(err, "failed to derive review targets")diff --git a/backend/runner/reviewrun/scheduler.go b/backend/runner/reviewrun/scheduler.goindex 35dcdcd548..aa1d4d8cd9 100644--- a/backend/runner/reviewrun/scheduler.go+++ b/backend/runner/reviewrun/scheduler.go@@ -31,11 +31,11 @@ const reviewRunSchedulerInterval = 5 * time.Second// Executor executes one review run for one reviewer type.type Executor interface {- // RunOnce evaluates every (spec, target) unit of the issue's plan- // (collect-all, no fail-fast) and returns the results to post as issue- // comments. A nil error marks the run DONE and posts the results; a- // non-nil error marks it FAILED with the aggregated message and posts- // nothing.+ // RunOnce evaluates every (spec, target) unit of the issue's plan in the+ // reviewer's scope (collect-all, no fail-fast) and returns the results to+ // post as issue comments. A nil error marks the run DONE and posts the+ // results; a non-nil error marks it FAILED with the aggregated message+ // and posts nothing.RunOnce(ctx context.Context, projectID string, issueUID int64) ([]*store.IssueCommentMessage, error)}diff --git a/frontend/src/types/proto-es/v1/issue_service_pb.d.ts b/frontend/src/types/proto-es/v1/issue_service_pb.d.tsindex 4385049cc6..2cf09982e1 100644--- a/frontend/src/types/proto-es/v1/issue_service_pb.d.ts+++ b/frontend/src/types/proto-es/v1/issue_service_pb.d.ts@@ -1469,8 +1469,11 @@ export enum ReviewRun_Status {RUNNING = 2,/**- * Every (spec, target) unit was evaluated. DONE does not mean the- * review passed: whether problems remain is judged from open comments.+ * Every (spec, target) unit in the reviewer's scope was evaluated. RULE's+ * scope is every target. AI's scope is the project's CI sample, the first+ * ci_sampling_size targets of each spec, or every target when+ * ci_sampling_size is 0. DONE does not mean the review passed: whether+ * problems remain is judged from open comments.** @generated from enum value: DONE = 3;*/diff --git a/frontend/src/types/proto-es/v1/project_service_pb.d.ts b/frontend/src/types/proto-es/v1/project_service_pb.d.tsindex 6e322ec2af..da8f1f5d92 100644--- a/frontend/src/types/proto-es/v1/project_service_pb.d.ts+++ b/frontend/src/types/proto-es/v1/project_service_pb.d.ts@@ -527,8 +527,9 @@ export declare type Project = Message<"bytebase.v1.Project"> & {executionRetryPolicy?: Project_ExecutionRetryPolicy | undefined;/**- * The maximum number of database rows to sample during CI data validation.- * Without specification, sampling is disabled, resulting in full validation.+ * The maximum number of databases to sample during CI data validation and+ * AI review. Without specification, sampling is disabled, resulting in full+ * validation.** @generated from field: int32 ci_sampling_size = 16;*/diff --git a/proto/gen/grpc-doc/openapi.yaml b/proto/gen/grpc-doc/openapi.yamlindex 2ff6f8755f..5a258d9d56 100644--- a/proto/gen/grpc-doc/openapi.yaml+++ b/proto/gen/grpc-doc/openapi.yaml@@ -25930,8 +25930,9 @@ components:title: ci_sampling_sizeformat: int32description: |-- The maximum number of database rows to sample during CI data validation.- Without specification, sampling is disabled, resulting in full validation.+ The maximum number of databases to sample during CI data validation and+ AI review. Without specification, sampling is disabled, resulting in full+ validation.parallelTasksPerRollout:type: integertitle: parallel_tasks_per_rolloutdiff --git a/proto/gen/grpc-doc/store/README.md b/proto/gen/grpc-doc/store/README.mdindex a96c90d3f9..9bd0747a25 100644--- a/proto/gen/grpc-doc/store/README.md+++ b/proto/gen/grpc-doc/store/README.md@@ -2906,7 +2906,7 @@ Project contains settings and configuration for a Bytebase project.| postgres_database_tenant_mode | [bool](#bool) | | Whether to enable the database tenant mode for PostgreSQL. If enabled, the issue will be created with the prepend "set role <db_owner>" statement. || allow_self_approval | [bool](#bool) | | Whether to allow the issue creator to self-approve the issue. || execution_retry_policy | [Project.ExecutionRetryPolicy](#bytebase-store-Project-ExecutionRetryPolicy) | | Configuration for automatic retry on task execution failures. |-| ci_sampling_size | [int32](#int32) | | The maximum number of databases to sample during CI data validation. If not specified, sampling is disabled, resulting in a full validation. |+| ci_sampling_size | [int32](#int32) | | The maximum number of databases to sample during CI data validation and AI review. If not specified, sampling is disabled, resulting in a full validation. || parallel_tasks_per_rollout | [int32](#int32) | | The maximum number of parallel tasks to run during the rollout. || labels | [Project.LabelsEntry](#bytebase-store-Project-LabelsEntry) | repeated | Labels are key-value pairs that can be attached to the project. For example, { "environment": "production", "team": "backend" } || enforce_sql_review | [bool](#bool) | | Whether to enforce SQL review checks to pass before issue creation. If enabled, issues cannot be created when SQL review finds errors. |diff --git a/proto/gen/grpc-doc/store/index.html b/proto/gen/grpc-doc/store/index.htmlindex f01ce27be8..4d8e0597ac 100644--- a/proto/gen/grpc-doc/store/index.html+++ b/proto/gen/grpc-doc/store/index.html@@ -7823,8 +7823,9 @@ If enabled, the issue will be created with the prepend "set role <db_owne<td>ci_sampling_size</td><td><a href="#int32">int32</a></td><td></td>- <td><p>The maximum number of databases to sample during CI data validation.-If not specified, sampling is disabled, resulting in a full validation. </p></td>+ <td><p>The maximum number of databases to sample during CI data validation and+AI review. If not specified, sampling is disabled, resulting in a full+validation. </p></td></tr><tr>diff --git a/proto/gen/grpc-doc/v1/README.md b/proto/gen/grpc-doc/v1/README.mdindex 6912731c60..585586940a 100644--- a/proto/gen/grpc-doc/v1/README.md+++ b/proto/gen/grpc-doc/v1/README.md@@ -4060,7 +4060,7 @@ completion transaction, so re-running supersedes instead of canceling).| STATUS_UNSPECIFIED | 0 | || AVAILABLE | 1 | Created and claimable by any replica. || RUNNING | 2 | |-| DONE | 3 | Every (spec, target) unit was evaluated. DONE does not mean the review passed: whether problems remain is judged from open comments. |+| DONE | 3 | Every (spec, target) unit in the reviewer's scope was evaluated. RULE's scope is every target. AI's scope is the project's CI sample, the first ci_sampling_size targets of each spec, or every target when ci_sampling_size is 0. DONE does not mean the review passed: whether problems remain is judged from open comments. || FAILED | 4 | At least one unit was not evaluated; the cause is in `error`. |@@ -10405,7 +10405,7 @@ When paginating, all other parameters provided to `ListProjects` must match the| postgres_database_tenant_mode | [bool](#bool) | | Whether to enable database tenant mode for PostgreSQL. If enabled, issues will include "set role <db_owner>" statement. || allow_self_approval | [bool](#bool) | | Whether to allow issue creators to self-approve their own issues. || execution_retry_policy | [Project.ExecutionRetryPolicy](#bytebase-v1-Project-ExecutionRetryPolicy) | | Execution retry policy for task runs. |-| ci_sampling_size | [int32](#int32) | | The maximum number of database rows to sample during CI data validation. Without specification, sampling is disabled, resulting in full validation. |+| ci_sampling_size | [int32](#int32) | | The maximum number of databases to sample during CI data validation and AI review. Without specification, sampling is disabled, resulting in full validation. || parallel_tasks_per_rollout | [int32](#int32) | | The maximum number of parallel tasks allowed during rollout execution. || labels | [Project.LabelsEntry](#bytebase-v1-Project-LabelsEntry) | repeated | Labels are key-value pairs that can be attached to the project. For example, { "environment": "production", "team": "backend" } || enforce_sql_review | [bool](#bool) | | Whether to enforce SQL review checks to pass before issue creation. If enabled, issues cannot be created when SQL review finds errors. |diff --git a/proto/gen/grpc-doc/v1/index.html b/proto/gen/grpc-doc/v1/index.htmlindex 462cedc4d6..0267e336f1 100644--- a/proto/gen/grpc-doc/v1/index.html+++ b/proto/gen/grpc-doc/v1/index.html@@ -12091,8 +12091,11 @@ In this situation, `update_mask` is ignored. </p></td><tr><td>DONE</td><td>3</td>- <td><p>Every (spec, target) unit was evaluated. DONE does not mean the-review passed: whether problems remain is judged from open comments.</p></td>+ <td><p>Every (spec, target) unit in the reviewer's scope was evaluated. RULE's+scope is every target. AI's scope is the project's CI sample, the first+ci_sampling_size targets of each spec, or every target when+ci_sampling_size is 0. DONE does not mean the review passed: whether+problems remain is judged from open comments.</p></td></tr><tr>@@ -29562,8 +29565,9 @@ If enabled, issues will include "set role <db_owner>" statement. <<td>ci_sampling_size</td><td><a href="#int32">int32</a></td><td></td>- <td><p>The maximum number of database rows to sample during CI data validation.-Without specification, sampling is disabled, resulting in full validation. </p></td>+ <td><p>The maximum number of databases to sample during CI data validation and+AI review. Without specification, sampling is disabled, resulting in full+validation. </p></td></tr><tr>diff --git a/proto/store/store/project.proto b/proto/store/store/project.protoindex 891f8ee8e9..9bfb10d7b9 100644--- a/proto/store/store/project.proto+++ b/proto/store/store/project.proto@@ -43,8 +43,9 @@ message Project {int32 maximum_retries = 1;}- // The maximum number of databases to sample during CI data validation.- // If not specified, sampling is disabled, resulting in a full validation.+ // The maximum number of databases to sample during CI data validation and+ // AI review. If not specified, sampling is disabled, resulting in a full+ // validation.int32 ci_sampling_size = 11;// The maximum number of parallel tasks to run during the rollout.diff --git a/proto/v1/v1/issue_service.proto b/proto/v1/v1/issue_service.protoindex ce972523a6..90ce663786 100644--- a/proto/v1/v1/issue_service.proto+++ b/proto/v1/v1/issue_service.proto@@ -822,8 +822,11 @@ message ReviewRun {// Created and claimable by any replica.AVAILABLE = 1;RUNNING = 2;- // Every (spec, target) unit was evaluated. DONE does not mean the- // review passed: whether problems remain is judged from open comments.+ // Every (spec, target) unit in the reviewer's scope was evaluated. RULE's+ // scope is every target. AI's scope is the project's CI sample, the first+ // ci_sampling_size targets of each spec, or every target when+ // ci_sampling_size is 0. DONE does not mean the review passed: whether+ // problems remain is judged from open comments.DONE = 3;// At least one unit was not evaluated; the cause is in `error`.FAILED = 4;diff --git a/proto/v1/v1/project_service.proto b/proto/v1/v1/project_service.protoindex da0b6b3ca4..c29822c0c9 100644--- a/proto/v1/v1/project_service.proto+++ b/proto/v1/v1/project_service.proto@@ -466,8 +466,9 @@ message Project {int32 maximum_retries = 1;}- // The maximum number of database rows to sample during CI data validation.- // Without specification, sampling is disabled, resulting in full validation.+ // The maximum number of databases to sample during CI data validation and+ // AI review. Without specification, sampling is disabled, resulting in full+ // validation.int32 ci_sampling_size = 16;// The maximum number of parallel tasks allowed during rollout execution.