Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(review): bound an AI review run by the CI sample and a server-wide limit (#21608)

h3n4l1e16085d3915
diff --git a/backend/api/mcp/gen/openapi.yaml b/backend/api/mcp/gen/openapi.yaml
index dca801b881..c0cbbdcfa2 100644
--- a/backend/api/mcp/gen/openapi.yaml
+++ b/backend/api/mcp/gen/openapi.yaml
@@ -18432,8 +18432,9 @@ components:
title: ci_sampling_size
format: int32
description: |-
- The maximum number of database rows to sample during CI data validation.
- Without specification, sampling is disabled, resulting in full validation.
+ The maximum number of databases to sample during CI data validation and
+ AI review. Without specification, sampling is disabled, resulting in full
+ validation.
parallelTasksPerRollout:
type: integer
title: parallel_tasks_per_rollout
diff --git a/backend/generated-go/store/project.pb.go b/backend/generated-go/store/project.pb.go
index ccef5632b4..b12f830435 100644
--- a/backend/generated-go/store/project.pb.go
+++ b/backend/generated-go/store/project.pb.go
@@ -102,8 +102,9 @@ type Project struct {
AllowSelfApproval bool `protobuf:"varint,9,opt,name=allow_self_approval,json=allowSelfApproval,proto3" json:"allow_self_approval,omitempty"`
// Configuration for automatic retry on task execution failures.
ExecutionRetryPolicy *Project_ExecutionRetryPolicy `protobuf:"bytes,10,opt,name=execution_retry_policy,json=executionRetryPolicy,proto3" json:"execution_retry_policy,omitempty"`
- // The maximum number of databases to sample during CI data validation.
- // If not specified, sampling is disabled, resulting in a full validation.
+ // The maximum number of databases to sample during CI data validation and
+ // AI review. If not specified, sampling is disabled, resulting in a full
+ // validation.
CiSamplingSize int32 `protobuf:"varint,11,opt,name=ci_sampling_size,json=ciSamplingSize,proto3" json:"ci_sampling_size,omitempty"`
// The maximum number of parallel tasks to run during the rollout.
ParallelTasksPerRollout int32 `protobuf:"varint,12,opt,name=parallel_tasks_per_rollout,json=parallelTasksPerRollout,proto3" json:"parallel_tasks_per_rollout,omitempty"`
diff --git a/backend/generated-go/v1/issue_service.pb.go b/backend/generated-go/v1/issue_service.pb.go
index f355f831a9..ad8ee1e5ea 100644
--- a/backend/generated-go/v1/issue_service.pb.go
+++ b/backend/generated-go/v1/issue_service.pb.go
@@ -366,8 +366,11 @@ const (
// Created and claimable by any replica.
ReviewRun_AVAILABLE ReviewRun_Status = 1
ReviewRun_RUNNING ReviewRun_Status = 2
- // Every (spec, target) unit was evaluated. DONE does not mean the
- // review passed: whether problems remain is judged from open comments.
+ // Every (spec, target) unit in the reviewer's scope was evaluated. RULE's
+ // scope is every target. AI's scope is the project's CI sample, the first
+ // ci_sampling_size targets of each spec, or every target when
+ // ci_sampling_size is 0. DONE does not mean the review passed: whether
+ // problems remain is judged from open comments.
ReviewRun_DONE ReviewRun_Status = 3
// At least one unit was not evaluated; the cause is in `error`.
ReviewRun_FAILED ReviewRun_Status = 4
diff --git a/backend/generated-go/v1/project_service.pb.go b/backend/generated-go/v1/project_service.pb.go
index 88f01c52f2..fbbc4908f7 100644
--- a/backend/generated-go/v1/project_service.pb.go
+++ b/backend/generated-go/v1/project_service.pb.go
@@ -934,8 +934,9 @@ type Project struct {
AllowSelfApproval bool `protobuf:"varint,14,opt,name=allow_self_approval,json=allowSelfApproval,proto3" json:"allow_self_approval,omitempty"`
// Execution retry policy for task runs.
ExecutionRetryPolicy *Project_ExecutionRetryPolicy `protobuf:"bytes,15,opt,name=execution_retry_policy,json=executionRetryPolicy,proto3" json:"execution_retry_policy,omitempty"`
- // The maximum number of database rows to sample during CI data validation.
- // Without specification, sampling is disabled, resulting in full validation.
+ // The maximum number of databases to sample during CI data validation and
+ // AI review. Without specification, sampling is disabled, resulting in full
+ // validation.
CiSamplingSize int32 `protobuf:"varint,16,opt,name=ci_sampling_size,json=ciSamplingSize,proto3" json:"ci_sampling_size,omitempty"`
// The maximum number of parallel tasks allowed during rollout execution.
ParallelTasksPerRollout int32 `protobuf:"varint,17,opt,name=parallel_tasks_per_rollout,json=parallelTasksPerRollout,proto3" json:"parallel_tasks_per_rollout,omitempty"`
diff --git a/backend/runner/plancheck/derive.go b/backend/runner/plancheck/derive.go
index f99969b250..85147cb6ba 100644
--- a/backend/runner/plancheck/derive.go
+++ b/backend/runner/plancheck/derive.go
@@ -13,15 +13,16 @@ import (
// DeriveCheckTargets derives check targets from a plan and optional database
// group, applying the project's CI sampling limit: plan checks are CI
-// validation, and sampling bounds their cost.
+// validation, and sampling bounds their cost. The AI review uses the same
+// sample to bound its model calls.
// This replaces the stored config by computing targets at runtime.
func DeriveCheckTargets(ctx context.Context, s *store.Store, project *store.ProjectMessage, plan *store.PlanMessage, databaseGroup *v1pb.DatabaseGroup) ([]*CheckTarget, error) {
return deriveTargets(ctx, s, project, plan, databaseGroup, true)
}
-// DeriveReviewTargets derives the same targets without CI sampling: a review
-// run's DONE means every (spec, target) unit was evaluated, so review must
-// see the full target set.
+// DeriveReviewTargets derives the same targets without CI sampling: a rule
+// review run's DONE means every (spec, target) unit was evaluated, so the
+// rule review must see the full target set.
func DeriveReviewTargets(ctx context.Context, s *store.Store, project *store.ProjectMessage, plan *store.PlanMessage, databaseGroup *v1pb.DatabaseGroup) ([]*CheckTarget, error) {
return deriveTargets(ctx, s, project, plan, databaseGroup, false)
}
diff --git a/backend/runner/plancheck/derive_test.go b/backend/runner/plancheck/derive_test.go
index 3b151a2888..424f3a78da 100644
--- a/backend/runner/plancheck/derive_test.go
+++ b/backend/runner/plancheck/derive_test.go
@@ -11,7 +11,7 @@ import (
)
// Plan checks are CI validation and honor the project's CI sampling limit;
-// a review run's DONE means every (spec, target) unit was evaluated, so
+// a rule review run's DONE means every (spec, target) unit was evaluated, so
// review derivation must not sample.
func TestDeriveReviewTargetsSkipsCISampling(t *testing.T) {
ctx := context.Background()
@@ -42,7 +42,7 @@ func TestDeriveReviewTargetsSkipsCISampling(t *testing.T) {
full, err := DeriveReviewTargets(ctx, nil, project, plan, nil)
require.NoError(t, err)
- require.Len(t, full, 3, "review must evaluate every target regardless of CI sampling")
+ require.Len(t, full, 3, "the rule review evaluates every target regardless of CI sampling")
for _, target := range full {
require.Equal(t, "spec-1", target.SpecID, "a review result anchors on its spec")
}
diff --git a/backend/runner/reviewrun/ai_review.go b/backend/runner/reviewrun/ai_review.go
index c4771506c3..d8a0f8ede4 100644
--- a/backend/runner/reviewrun/ai_review.go
+++ b/backend/runner/reviewrun/ai_review.go
@@ -21,10 +21,20 @@ import (
const (
// maxAIReviewSheetBytes bounds the sheet one review reads whole.
maxAIReviewSheetBytes = 256 << 10
- // aiReviewConcurrency caps the reviews of one run in flight at the model.
- aiReviewConcurrency = 8
+ // aiReviewRunConcurrency caps the reviews of one run in flight at the model.
+ aiReviewRunConcurrency = 8
+ // aiReviewServerConcurrency caps the reviews in flight across every run on
+ // the server, so runs side by side cannot multiply the load on the provider.
+ aiReviewServerConcurrency = 16
)
+// reviewSlots is the limit on reviews in flight that every run of an executor
+// shares. The server uses a *semaphore.Weighted.
+type reviewSlots interface {
+ Acquire(ctx context.Context, n int64) error
+ Release(n int64)
+}
+
// aiReviewUnit is one AI review: a spec's sheet against one database. It
// holds no schema, which the review loads when it starts.
type aiReviewUnit struct {
@@ -179,15 +189,18 @@ func checkSheetSize(statement string) error {
return nil
}
-// reviewUnits runs review over the units, at most aiReviewConcurrency at a
-// time, and returns each unit's result or error by index. A panic in one
-// review is that unit's error, so the run fails instead of the process: the
-// scheduler's recovery covers only the goroutine that called RunOnce.
-func reviewUnits(ctx context.Context, units []*aiReviewUnit, review func(context.Context, *aiReviewUnit) (*aireview.Result, error)) ([]*aiReviewResult, []error) {
+// reviewUnits runs review over the units, at most aiReviewRunConcurrency at a
+// time, and returns each unit's result or error by index. Each review holds
+// one of slots, which every run shares. A unit takes its slot before review
+// starts, so the wait does not count against the reviewer's timeout, and
+// gives up the wait when ctx is done. A panic in one review is that unit's
+// error, so the run fails instead of the process: the scheduler's recovery
+// covers only the goroutine that called RunOnce.
+func reviewUnits(ctx context.Context, slots reviewSlots, units []*aiReviewUnit, review func(context.Context, *aiReviewUnit) (*aireview.Result, error)) ([]*aiReviewResult, []error) {
results := make([]*aiReviewResult, len(units))
errs := make([]error, len(units))
var group errgroup.Group
- group.SetLimit(aiReviewConcurrency)
+ group.SetLimit(aiReviewRunConcurrency)
for i, unit := range units {
group.Go(func() error {
defer func() {
@@ -196,6 +209,11 @@ func reviewUnits(ctx context.Context, units []*aiReviewUnit, review func(context
errs[i] = errors.Errorf("%s: review panic: %v", unit.Check.Target, r)
}
}()
+ if err := slots.Acquire(ctx, 1); err != nil {
+ errs[i] = errors.Wrapf(err, "%s", unit.Check.Target)
+ return nil
+ }
+ defer slots.Release(1)
result, err := review(ctx, unit)
if err != nil {
errs[i] = errors.Wrapf(err, "%s", unit.Check.Target)
diff --git a/backend/runner/reviewrun/ai_review_executor.go b/backend/runner/reviewrun/ai_review_executor.go
index 0c663538a6..3843f9694c 100644
--- a/backend/runner/reviewrun/ai_review_executor.go
+++ b/backend/runner/reviewrun/ai_review_executor.go
@@ -5,29 +5,39 @@ import (
"log/slog"
"github.com/pkg/errors"
+ "golang.org/x/sync/semaphore"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/aireview"
"github.com/bytebase/bytebase/backend/runner/plancheck"
"github.com/bytebase/bytebase/backend/store"
)
// AIReviewExecutor is the AI review: a model judges each spec's sheet against
-// the workspace and project AI review policy, once per database, with that
-// database's facts in the prompt and its synced schema behind the tools. The
-// reviews of one run share the model and run concurrently under
-// aiReviewConcurrency.
+// the workspace and project AI review policy, once per database of the
+// project's CI sample, with that database's facts in the prompt and its synced
+// schema behind the tools. The reviews of one run share the model and run
+// concurrently under aiReviewRunConcurrency.
type AIReviewExecutor struct {
- store *store.Store
+ store *store.Store
+ newModel func(*storepb.AISetting) aireview.Model
+ // slots bounds the reviews in flight across every run of the executor.
+ slots reviewSlots
}
-// NewAIReviewExecutor creates the AI review executor.
+// NewAIReviewExecutor creates the AI review executor. The server creates one,
+// so aiReviewServerConcurrency bounds the reviews of every run on the server.
func NewAIReviewExecutor(s *store.Store) *AIReviewExecutor {
- return &AIReviewExecutor{store: s}
+ return &AIReviewExecutor{
+ store: s,
+ newModel: aireview.NewModel,
+ slots: semaphore.NewWeighted(aiReviewServerConcurrency),
+ }
}
-// RunOnce implements Executor. Collect-all, no fail-fast: every database is
-// attempted, failures aggregate into one message, and the findings are
-// returned only when every database was reviewed.
+// RunOnce implements Executor. Collect-all, no fail-fast: every database of
+// the sample is attempted, failures aggregate into one message, and the
+// findings are returned only when every one of them was reviewed.
func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueUID int64) ([]*store.IssueCommentMessage, error) {
plan, project, err := loadReviewPlan(ctx, e.store, projectID, issueUID)
if err != nil {
@@ -49,9 +59,10 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueU
if err != nil {
return nil, errors.Wrapf(err, "failed to get database group for plan")
}
- // DeriveReviewTargets, not DeriveCheckTargets: every database is reviewed
- // with its own facts, so the CI sampling limit does not apply.
- checkTargets, err := plancheck.DeriveReviewTargets(ctx, e.store, project, plan, databaseGroup)
+ // DeriveCheckTargets, not DeriveReviewTargets: every database costs model
+ // calls, so the AI review covers the databases plan checks sample when the
+ // project sets ci_sampling_size, and every database otherwise.
+ checkTargets, err := plancheck.DeriveCheckTargets(ctx, e.store, project, plan, databaseGroup)
if err != nil {
return nil, errors.Wrapf(err, "failed to derive review targets")
}
@@ -72,12 +83,12 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueU
units = append(units, &aiReviewUnit{Check: checkTarget, Statement: statement})
}
- reviewer := aireview.NewReviewer(aireview.NewModel(aiSetting))
- results, reviewErrs := reviewUnits(ctx, units, func(ctx context.Context, unit *aiReviewUnit) (*aireview.Result, error) {
- // The schema is loaded inside the bounded pool and the run drops it
- // when the review ends, so the run itself holds at most
- // aiReviewConcurrency schemas however many databases it reviews. The
- // store's schema cache keeps its own bounded copies.
+ reviewer := aireview.NewReviewer(e.newModel(aiSetting))
+ results, reviewErrs := reviewUnits(ctx, e.slots, units, func(ctx context.Context, unit *aiReviewUnit) (*aireview.Result, error) {
+ // The schema is loaded once the unit holds its slot and dropped when
+ // the review ends, so the runs together hold at most one schema per
+ // slot however many databases they review. The store's schema cache
+ // keeps its own bounded copies.
request := &aireview.Request{
WorkspacePolicy: policy.Workspace,
ProjectPolicy: policy.Project,
diff --git a/backend/runner/reviewrun/ai_review_executor_test.go b/backend/runner/reviewrun/ai_review_executor_test.go
new file mode 100644
index 0000000000..6aeb391471
--- /dev/null
+++ b/backend/runner/reviewrun/ai_review_executor_test.go
@@ -0,0 +1,292 @@
+package reviewrun
+
+import (
+ "context"
+ "fmt"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/bytebase/omni/review"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/sync/semaphore"
+ "google.golang.org/genproto/googleapis/type/expr"
+ "google.golang.org/protobuf/encoding/protojson"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/plugin/aireview"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+// fixtureDatabases is how many databases the fixture's database group
+// matches.
+const fixtureDatabases = 5
+
+// testWaitLimit fails a test whose reviews stop making progress, instead of
+// letting it hang.
+const testWaitLimit = 30 * time.Second
+
+func TestCISampleBoundsOnlyTheAIReview(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+ stores, issues := setupReviewFixture(ctx, t, 1)
+ model := &findingModel{}
+ executor := &AIReviewExecutor{
+ store: stores,
+ newModel: func(*storepb.AISetting) aireview.Model { return model },
+ slots: semaphore.NewWeighted(aiReviewServerConcurrency),
+ }
+ reviewedDatabases := func() []string {
+ t.Helper()
+ comments, err := executor.RunOnce(ctx, "p", issues[0])
+ require.NoError(t, err)
+ require.Len(t, comments, 1, "the same finding on every database is one comment")
+ return comments[0].Payload.ReviewMetadata.Targets
+ }
+
+ setCISamplingSize(ctx, t, stores, 2)
+ require.Len(t, reviewedDatabases(), 2)
+ require.Equal(t, int32(2), model.calls.Load())
+
+ var ruleTargets int
+ ruleExecutor := &RuleExecutor{store: stores, reviewFuncs: map[storepb.Engine]review.ReviewFunc{
+ storepb.Engine_POSTGRES: func(_ context.Context, _ string, _ review.Options, targets []review.Target) (*review.Result, error) {
+ ruleTargets += len(targets)
+ return &review.Result{}, nil
+ },
+ }}
+ _, err := ruleExecutor.RunOnce(ctx, "p", issues[0])
+ require.NoError(t, err)
+ require.Equal(t, fixtureDatabases, ruleTargets, "the rule review evaluates every database whatever the sample")
+
+ setCISamplingSize(ctx, t, stores, 0)
+ require.Len(t, reviewedDatabases(), fixtureDatabases)
+}
+
+func TestAIReviewRunsShareTheServerLimit(t *testing.T) {
+ t.Parallel()
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ stores, issues := setupReviewFixture(ctx, t, 2)
+ const limit = 3
+ total := len(issues) * fixtureDatabases
+ require.Less(t, limit, fixtureDatabases, "one run alone could exceed the limit")
+ model := &blockingModel{entered: make(chan struct{}, total), release: make(chan struct{})}
+ slots := &observedSlots{Weighted: semaphore.NewWeighted(limit), waiting: make(chan struct{}, total)}
+ executor := &AIReviewExecutor{
+ store: stores,
+ newModel: func(*storepb.AISetting) aireview.Model { return model },
+ slots: slots,
+ }
+
+ runErrs := make(chan error, len(issues))
+ for _, issueUID := range issues {
+ go func() {
+ _, err := executor.RunOnce(ctx, "p", issueUID)
+ runErrs <- err
+ }()
+ }
+ // Nothing is released yet, so every review either reaches the model or
+ // waits for a slot.
+ var entered, waiting int
+ for entered+waiting < total {
+ select {
+ case <-model.entered:
+ entered++
+ case <-slots.waiting:
+ waiting++
+ case <-time.After(testWaitLimit):
+ t.Fatalf("timed out with %d reviews at the model and %d waiting", entered, waiting)
+ }
+ }
+ require.Equal(t, limit, entered, "the reviews of both runs past the limit wait for a slot")
+ for range total - limit {
+ send(t, model.release)
+ receive(t, model.entered)
+ }
+ for range limit {
+ send(t, model.release)
+ }
+ for range issues {
+ require.NoError(t, receive(t, runErrs))
+ }
+ require.Equal(t, int32(limit), model.maxInFlight.Load())
+}
+
+// setupReviewFixture seeds project p with AI enabled, a PostgreSQL instance
+// holding fixtureDatabases synced databases, a database group matching all of
+// them, and the given number of issues, each with a plan whose one spec
+// targets the group. It returns the issue UIDs.
+func setupReviewFixture(ctx context.Context, t *testing.T, issueCount int) (*store.Store, []int64) {
+ t.Helper()
+ db, stores, _ := testcontainer.NewMetadataDB(t)
+ _, err := db.ExecContext(ctx, `
+ INSERT INTO workspace (resource_id) VALUES ('ws');
+ INSERT INTO principal (name, email, password_hash) VALUES ('creator', '[email protected]', 'unused');
+ INSERT INTO project (resource_id, workspace, name) VALUES ('p', 'ws', 'P');
+ `)
+ require.NoError(t, err)
+ _, err = stores.UpsertSetting(ctx, &store.SettingMessage{Name: storepb.SettingName_AI, Workspace: "ws", Value: &storepb.AISetting{Enabled: true}})
+ require.NoError(t, err)
+ rules, err := protojson.Marshal(&storepb.ReviewRulePolicy{Rules: []storepb.ReviewRuleType{storepb.ReviewRuleType_SYNTAX}})
+ require.NoError(t, err)
+ _, err = stores.CreatePolicy(ctx, &store.PolicyMessage{
+ Workspace: "ws",
+ ResourceType: storepb.Policy_PROJECT,
+ Resource: common.FormatProject("p"),
+ Type: storepb.Policy_REVIEW_RULE,
+ Payload: string(rules),
+ Enforce: true,
+ })
+ require.NoError(t, err)
+
+ _, err = stores.CreateInstance(ctx, &store.InstanceMessage{
+ ResourceID: "prod",
+ Workspace: "ws",
+ Metadata: &storepb.Instance{
+ Engine: storepb.Engine_POSTGRES,
+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},
+ },
+ })
+ require.NoError(t, err)
+ for i := range fixtureDatabases {
+ name := fmt.Sprintf("db%d", i+1)
+ _, err = stores.UpsertDatabase(ctx, &store.DatabaseMessage{
+ ProjectID: "p",
+ InstanceID: "prod",
+ DatabaseName: name,
+ Metadata: &storepb.DatabaseMetadata{Labels: map[string]string{}},
+ })
+ require.NoError(t, err)
+ schema := &metadatapb.DatabaseSchemaMetadata{Name: name, Schemas: []*metadatapb.SchemaMetadata{{Name: "public"}}}
+ require.NoError(t, stores.UpsertDBSchema(ctx, "prod", name, schema, &storepb.DatabaseConfig{}, nil))
+ }
+ _, err = stores.CreateDatabaseGroup(ctx, &store.DatabaseGroupMessage{
+ ProjectID: "p",
+ ResourceID: "all",
+ Title: "all",
+ Expression: &expr.Expr{Expression: `resource.instance_id == "prod"`},
+ })
+ require.NoError(t, err)
+ sheets, err := stores.CreateSheets(ctx, "p", &store.SheetMessage{Statement: "ALTER TABLE orders ADD COLUMN note text;"})
+ require.NoError(t, err)
+
+ var issues []int64
+ for i := range issueCount {
+ plan, err := stores.CreatePlan(ctx, &store.PlanMessage{
+ ProjectID: "p",
+ Name: fmt.Sprintf("plan %d", i),
+ Config: &storepb.PlanConfig{
+ Specs: []*storepb.PlanConfig_Spec{{
+ Id: "spec-1",
+ Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{
+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{
+ Targets: []string{"projects/p/databaseGroups/all"},
+ SheetSha256: sheets[0].Sha256,
+ },
+ },
+ }},
+ },
+ require.NoError(t, err)
+ issue, err := stores.CreateIssue(ctx, &store.IssueMessage{
+ ProjectID: "p",
+ CreatorEmail: "[email protected]",
+ Title: fmt.Sprintf("issue %d", i),
+ Type: storepb.Issue_DATABASE_CHANGE,
+ Payload: &storepb.Issue{},
+ PlanUID: &plan.UID,
+ })
+ require.NoError(t, err)
+ issues = append(issues, issue.UID)
+ }
+ return stores, issues
+}
+
+func setCISamplingSize(ctx context.Context, t *testing.T, stores *store.Store, size int32) {
+ t.Helper()
+ require.NoError(t, stores.UpdateProjects(ctx, &store.UpdateProjectMessage{
+ ResourceID: "p",
+ Workspace: "ws",
+ Setting: &storepb.Project{CiSamplingSize: size},
+ }))
+}
+
+// findingModel answers every review with the same finding, so a run's one
+// comment names every database the run reviewed.
+type findingModel struct {
+ calls atomic.Int32
+}
+
+func (m *findingModel) Chat(context.Context, *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {
+ m.calls.Add(1)
+ content := `{"findings": [{"title": "Backfill note", "severity": "P2", "line": 1, "rule": "r", "evidence": "e", "fix": "f"}]}`
+ return &v1pb.AIChatResponse{Content: &content}, nil
+}
+
+// observedSlots tells the test when a review has to wait for a slot.
+type observedSlots struct {
+ *semaphore.Weighted
+ waiting chan struct{}
+}
+
+func (s *observedSlots) Acquire(ctx context.Context, n int64) error {
+ if s.TryAcquire(n) {
+ return nil
+ }
+ s.waiting <- struct{}{}
+ return s.Weighted.Acquire(ctx, n)
+}
+
+// blockingModel holds every call until the test releases one, and records
+// the most calls in flight at once.
+type blockingModel struct {
+ entered chan struct{}
+ release chan struct{}
+ inFlight atomic.Int32
+ maxInFlight atomic.Int32
+}
+
+func (m *blockingModel) Chat(ctx context.Context, _ *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {
+ now := m.inFlight.Add(1)
+ defer m.inFlight.Add(-1)
+ for {
+ seen := m.maxInFlight.Load()
+ if now <= seen || m.maxInFlight.CompareAndSwap(seen, now) {
+ break
+ }
+ }
+ m.entered <- struct{}{}
+ select {
+ case <-m.release:
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+ content := `{"findings": []}`
+ return &v1pb.AIChatResponse{Content: &content}, nil
+}
+
+func receive[T any](t *testing.T, ch <-chan T) T {
+ t.Helper()
+ select {
+ case v := <-ch:
+ return v
+ case <-time.After(testWaitLimit):
+ t.Fatal("timed out waiting for the reviews")
+ var zero T
+ return zero
+ }
+}
+
+func send(t *testing.T, ch chan<- struct{}) {
+ t.Helper()
+ select {
+ case ch <- struct{}{}:
+ case <-time.After(testWaitLimit):
+ t.Fatal("timed out waiting for a model call to release")
+ }
+}
diff --git a/backend/runner/reviewrun/ai_review_test.go b/backend/runner/reviewrun/ai_review_test.go
index c48541f1d1..62951bb06f 100644
--- a/backend/runner/reviewrun/ai_review_test.go
+++ b/backend/runner/reviewrun/ai_review_test.go
@@ -10,6 +10,7 @@ import (
metadatapb "github.com/bytebase/omni/metadata"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
+ "golang.org/x/sync/semaphore"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/aireview"
@@ -122,8 +123,9 @@ func TestCheckSheetSize(t *testing.T) {
}
// TestReviewUnits pins the executor's concurrency guarantees: at most
-// aiReviewConcurrency reviews in flight, every unit attempted whatever the
-// others do, and a panicking review recorded as that unit's error.
+// aiReviewRunConcurrency reviews in flight, every unit attempted whatever the
+// others do, a panicking review recorded as that unit's error, and every slot
+// returned whatever the review did.
func TestReviewUnits(t *testing.T) {
t.Parallel()
@@ -152,10 +154,11 @@ func TestReviewUnits(t *testing.T) {
}
}
- results, errs := reviewUnits(context.Background(), units, review)
+ slots := semaphore.NewWeighted(aiReviewServerConcurrency)
+ results, errs := reviewUnits(context.Background(), slots, units, review)
require.Len(t, results, 20)
require.Len(t, errs, 20)
- require.LessOrEqual(t, maxInFlight.Load(), int32(aiReviewConcurrency))
+ require.LessOrEqual(t, maxInFlight.Load(), int32(aiReviewRunConcurrency))
require.Greater(t, maxInFlight.Load(), int32(1), "the reviews run concurrently")
for i, result := range results {
switch i {
@@ -172,4 +175,27 @@ func TestReviewUnits(t *testing.T) {
}
}
require.Zero(t, inFlight.Load())
+ require.True(t, slots.TryAcquire(aiReviewServerConcurrency), "the failed and the panicking reviews return their slots too")
+}
+
+func TestReviewUnitsStopWaitingForASlotWhenTheRunEnds(t *testing.T) {
+ t.Parallel()
+
+ slots := semaphore.NewWeighted(1)
+ require.True(t, slots.TryAcquire(1), "another run holds the only slot")
+ units := []*aiReviewUnit{{Check: &plancheck.CheckTarget{SpecID: "spec-1", Target: "instances/i/databases/a"}}}
+ var reviews atomic.Int32
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan []error)
+ go func() {
+ _, errs := reviewUnits(ctx, slots, units, func(context.Context, *aiReviewUnit) (*aireview.Result, error) {
+ reviews.Add(1)
+ return &aireview.Result{}, nil
+ })
+ done <- errs
+ }()
+ cancel()
+ errs := receive(t, done)
+ require.ErrorIs(t, errs[0], context.Canceled)
+ require.Zero(t, reviews.Load(), "no review starts without a slot")
}
diff --git a/backend/runner/reviewrun/rule_executor.go b/backend/runner/reviewrun/rule_executor.go
index 7ec1d28c3a..764545d280 100644
--- a/backend/runner/reviewrun/rule_executor.go
+++ b/backend/runner/reviewrun/rule_executor.go
@@ -57,8 +57,8 @@ func (e *RuleExecutor) RunOnce(ctx context.Context, projectID string, issueUID i
if err != nil {
return nil, errors.Wrapf(err, "failed to get database group for plan")
}
- // DeriveReviewTargets, not DeriveCheckTargets: review must evaluate every
- // (spec, target) unit, so the CI sampling limit does not apply.
+ // DeriveReviewTargets, not DeriveCheckTargets: the rule review evaluates
+ // every (spec, target) unit, so the CI sampling limit does not apply.
checkTargets, err := plancheck.DeriveReviewTargets(ctx, e.store, project, plan, databaseGroup)
if err != nil {
return nil, errors.Wrapf(err, "failed to derive review targets")
diff --git a/backend/runner/reviewrun/scheduler.go b/backend/runner/reviewrun/scheduler.go
index 35dcdcd548..aa1d4d8cd9 100644
--- a/backend/runner/reviewrun/scheduler.go
+++ b/backend/runner/reviewrun/scheduler.go
@@ -31,11 +31,11 @@ const reviewRunSchedulerInterval = 5 * time.Second
// Executor executes one review run for one reviewer type.
type Executor interface {
- // RunOnce evaluates every (spec, target) unit of the issue's plan
- // (collect-all, no fail-fast) and returns the results to post as issue
- // comments. A nil error marks the run DONE and posts the results; a
- // non-nil error marks it FAILED with the aggregated message and posts
- // nothing.
+ // RunOnce evaluates every (spec, target) unit of the issue's plan in the
+ // reviewer's scope (collect-all, no fail-fast) and returns the results to
+ // post as issue comments. A nil error marks the run DONE and posts the
+ // results; a non-nil error marks it FAILED with the aggregated message
+ // and posts nothing.
RunOnce(ctx context.Context, projectID string, issueUID int64) ([]*store.IssueCommentMessage, error)
}
diff --git a/frontend/src/types/proto-es/v1/issue_service_pb.d.ts b/frontend/src/types/proto-es/v1/issue_service_pb.d.ts
index 4385049cc6..2cf09982e1 100644
--- a/frontend/src/types/proto-es/v1/issue_service_pb.d.ts
+++ b/frontend/src/types/proto-es/v1/issue_service_pb.d.ts
@@ -1469,8 +1469,11 @@ export enum ReviewRun_Status {
RUNNING = 2,
/**
- * Every (spec, target) unit was evaluated. DONE does not mean the
- * review passed: whether problems remain is judged from open comments.
+ * Every (spec, target) unit in the reviewer's scope was evaluated. RULE's
+ * scope is every target. AI's scope is the project's CI sample, the first
+ * ci_sampling_size targets of each spec, or every target when
+ * ci_sampling_size is 0. DONE does not mean the review passed: whether
+ * problems remain is judged from open comments.
*
* @generated from enum value: DONE = 3;
*/
diff --git a/frontend/src/types/proto-es/v1/project_service_pb.d.ts b/frontend/src/types/proto-es/v1/project_service_pb.d.ts
index 6e322ec2af..da8f1f5d92 100644
--- a/frontend/src/types/proto-es/v1/project_service_pb.d.ts
+++ b/frontend/src/types/proto-es/v1/project_service_pb.d.ts
@@ -527,8 +527,9 @@ export declare type Project = Message<"bytebase.v1.Project"> & {
executionRetryPolicy?: Project_ExecutionRetryPolicy | undefined;
/**
- * The maximum number of database rows to sample during CI data validation.
- * Without specification, sampling is disabled, resulting in full validation.
+ * The maximum number of databases to sample during CI data validation and
+ * AI review. Without specification, sampling is disabled, resulting in full
+ * validation.
*
* @generated from field: int32 ci_sampling_size = 16;
*/
diff --git a/proto/gen/grpc-doc/openapi.yaml b/proto/gen/grpc-doc/openapi.yaml
index 2ff6f8755f..5a258d9d56 100644
--- a/proto/gen/grpc-doc/openapi.yaml
+++ b/proto/gen/grpc-doc/openapi.yaml
@@ -25930,8 +25930,9 @@ components:
title: ci_sampling_size
format: int32
description: |-
- The maximum number of database rows to sample during CI data validation.
- Without specification, sampling is disabled, resulting in full validation.
+ The maximum number of databases to sample during CI data validation and
+ AI review. Without specification, sampling is disabled, resulting in full
+ validation.
parallelTasksPerRollout:
type: integer
title: parallel_tasks_per_rollout
diff --git a/proto/gen/grpc-doc/store/README.md b/proto/gen/grpc-doc/store/README.md
index a96c90d3f9..9bd0747a25 100644
--- a/proto/gen/grpc-doc/store/README.md
+++ b/proto/gen/grpc-doc/store/README.md
@@ -2906,7 +2906,7 @@ Project contains settings and configuration for a Bytebase project.
| postgres_database_tenant_mode | [bool](#bool) | | Whether to enable the database tenant mode for PostgreSQL. If enabled, the issue will be created with the prepend &#34;set role &lt;db_owner&gt;&#34; statement. |
| allow_self_approval | [bool](#bool) | | Whether to allow the issue creator to self-approve the issue. |
| execution_retry_policy | [Project.ExecutionRetryPolicy](#bytebase-store-Project-ExecutionRetryPolicy) | | Configuration for automatic retry on task execution failures. |
-| ci_sampling_size | [int32](#int32) | | The maximum number of databases to sample during CI data validation. If not specified, sampling is disabled, resulting in a full validation. |
+| ci_sampling_size | [int32](#int32) | | The maximum number of databases to sample during CI data validation and AI review. If not specified, sampling is disabled, resulting in a full validation. |
| parallel_tasks_per_rollout | [int32](#int32) | | The maximum number of parallel tasks to run during the rollout. |
| labels | [Project.LabelsEntry](#bytebase-store-Project-LabelsEntry) | repeated | Labels are key-value pairs that can be attached to the project. For example, { &#34;environment&#34;: &#34;production&#34;, &#34;team&#34;: &#34;backend&#34; } |
| enforce_sql_review | [bool](#bool) | | Whether to enforce SQL review checks to pass before issue creation. If enabled, issues cannot be created when SQL review finds errors. |
diff --git a/proto/gen/grpc-doc/store/index.html b/proto/gen/grpc-doc/store/index.html
index f01ce27be8..4d8e0597ac 100644
--- a/proto/gen/grpc-doc/store/index.html
+++ b/proto/gen/grpc-doc/store/index.html
@@ -7823,8 +7823,9 @@ If enabled, the issue will be created with the prepend &#34;set role &lt;db_owne
<td>ci_sampling_size</td>
<td><a href="#int32">int32</a></td>
<td></td>
- <td><p>The maximum number of databases to sample during CI data validation.
-If not specified, sampling is disabled, resulting in a full validation. </p></td>
+ <td><p>The maximum number of databases to sample during CI data validation and
+AI review. If not specified, sampling is disabled, resulting in a full
+validation. </p></td>
</tr>
<tr>
diff --git a/proto/gen/grpc-doc/v1/README.md b/proto/gen/grpc-doc/v1/README.md
index 6912731c60..585586940a 100644
--- a/proto/gen/grpc-doc/v1/README.md
+++ b/proto/gen/grpc-doc/v1/README.md
@@ -4060,7 +4060,7 @@ completion transaction, so re-running supersedes instead of canceling).
| STATUS_UNSPECIFIED | 0 | |
| AVAILABLE | 1 | Created and claimable by any replica. |
| RUNNING | 2 | |
-| DONE | 3 | Every (spec, target) unit was evaluated. DONE does not mean the review passed: whether problems remain is judged from open comments. |
+| DONE | 3 | Every (spec, target) unit in the reviewer&#39;s scope was evaluated. RULE&#39;s scope is every target. AI&#39;s scope is the project&#39;s CI sample, the first ci_sampling_size targets of each spec, or every target when ci_sampling_size is 0. DONE does not mean the review passed: whether problems remain is judged from open comments. |
| FAILED | 4 | At least one unit was not evaluated; the cause is in `error`. |
@@ -10405,7 +10405,7 @@ When paginating, all other parameters provided to `ListProjects` must match the
| postgres_database_tenant_mode | [bool](#bool) | | Whether to enable database tenant mode for PostgreSQL. If enabled, issues will include &#34;set role &lt;db_owner&gt;&#34; statement. |
| allow_self_approval | [bool](#bool) | | Whether to allow issue creators to self-approve their own issues. |
| execution_retry_policy | [Project.ExecutionRetryPolicy](#bytebase-v1-Project-ExecutionRetryPolicy) | | Execution retry policy for task runs. |
-| ci_sampling_size | [int32](#int32) | | The maximum number of database rows to sample during CI data validation. Without specification, sampling is disabled, resulting in full validation. |
+| ci_sampling_size | [int32](#int32) | | The maximum number of databases to sample during CI data validation and AI review. Without specification, sampling is disabled, resulting in full validation. |
| parallel_tasks_per_rollout | [int32](#int32) | | The maximum number of parallel tasks allowed during rollout execution. |
| labels | [Project.LabelsEntry](#bytebase-v1-Project-LabelsEntry) | repeated | Labels are key-value pairs that can be attached to the project. For example, { &#34;environment&#34;: &#34;production&#34;, &#34;team&#34;: &#34;backend&#34; } |
| enforce_sql_review | [bool](#bool) | | Whether to enforce SQL review checks to pass before issue creation. If enabled, issues cannot be created when SQL review finds errors. |
diff --git a/proto/gen/grpc-doc/v1/index.html b/proto/gen/grpc-doc/v1/index.html
index 462cedc4d6..0267e336f1 100644
--- a/proto/gen/grpc-doc/v1/index.html
+++ b/proto/gen/grpc-doc/v1/index.html
@@ -12091,8 +12091,11 @@ In this situation, `update_mask` is ignored. </p></td>
<tr>
<td>DONE</td>
<td>3</td>
- <td><p>Every (spec, target) unit was evaluated. DONE does not mean the
-review passed: whether problems remain is judged from open comments.</p></td>
+ <td><p>Every (spec, target) unit in the reviewer&#39;s scope was evaluated. RULE&#39;s
+scope is every target. AI&#39;s scope is the project&#39;s CI sample, the first
+ci_sampling_size targets of each spec, or every target when
+ci_sampling_size is 0. DONE does not mean the review passed: whether
+problems remain is judged from open comments.</p></td>
</tr>
<tr>
@@ -29562,8 +29565,9 @@ If enabled, issues will include &#34;set role &lt;db_owner&gt;&#34; statement. <
<td>ci_sampling_size</td>
<td><a href="#int32">int32</a></td>
<td></td>
- <td><p>The maximum number of database rows to sample during CI data validation.
-Without specification, sampling is disabled, resulting in full validation. </p></td>
+ <td><p>The maximum number of databases to sample during CI data validation and
+AI review. Without specification, sampling is disabled, resulting in full
+validation. </p></td>
</tr>
<tr>
diff --git a/proto/store/store/project.proto b/proto/store/store/project.proto
index 891f8ee8e9..9bfb10d7b9 100644
--- a/proto/store/store/project.proto
+++ b/proto/store/store/project.proto
@@ -43,8 +43,9 @@ message Project {
int32 maximum_retries = 1;
}
- // The maximum number of databases to sample during CI data validation.
- // If not specified, sampling is disabled, resulting in a full validation.
+ // The maximum number of databases to sample during CI data validation and
+ // AI review. If not specified, sampling is disabled, resulting in a full
+ // validation.
int32 ci_sampling_size = 11;
// The maximum number of parallel tasks to run during the rollout.
diff --git a/proto/v1/v1/issue_service.proto b/proto/v1/v1/issue_service.proto
index ce972523a6..90ce663786 100644
--- a/proto/v1/v1/issue_service.proto
+++ b/proto/v1/v1/issue_service.proto
@@ -822,8 +822,11 @@ message ReviewRun {
// Created and claimable by any replica.
AVAILABLE = 1;
RUNNING = 2;
- // Every (spec, target) unit was evaluated. DONE does not mean the
- // review passed: whether problems remain is judged from open comments.
+ // Every (spec, target) unit in the reviewer's scope was evaluated. RULE's
+ // scope is every target. AI's scope is the project's CI sample, the first
+ // ci_sampling_size targets of each spec, or every target when
+ // ci_sampling_size is 0. DONE does not mean the review passed: whether
+ // problems remain is judged from open comments.
DONE = 3;
// At least one unit was not evaluated; the cause is in `error`.
FAILED = 4;
diff --git a/proto/v1/v1/project_service.proto b/proto/v1/v1/project_service.proto
index da0b6b3ca4..c29822c0c9 100644
--- a/proto/v1/v1/project_service.proto
+++ b/proto/v1/v1/project_service.proto
@@ -466,8 +466,9 @@ message Project {
int32 maximum_retries = 1;
}
- // The maximum number of database rows to sample during CI data validation.
- // Without specification, sampling is disabled, resulting in full validation.
+ // The maximum number of databases to sample during CI data validation and
+ // AI review. Without specification, sampling is disabled, resulting in full
+ // validation.
int32 ci_sampling_size = 16;
// The maximum number of parallel tasks allowed during rollout execution.