Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(review): keep a review panic inside its run and test the run's bounds
h3n4ldc0138311121
diff --git a/backend/runner/reviewrun/ai_review.go b/backend/runner/reviewrun/ai_review.goindex 76b1f30323..d81008bc74 100644--- a/backend/runner/reviewrun/ai_review.go+++ b/backend/runner/reviewrun/ai_review.go@@ -1,18 +1,30 @@package reviewrunimport (+ "context""fmt"+ "log/slog""slices""strings"metadatapb "github.com/bytebase/omni/metadata"+ "github.com/pkg/errors"+ "golang.org/x/sync/errgroup"+ "github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/aireview""github.com/bytebase/bytebase/backend/runner/plancheck""github.com/bytebase/bytebase/backend/store")+const (+ // maxAIReviewSheetBytes bounds the sheet one review reads whole.+ maxAIReviewSheetBytes = 256 << 10+ // aiReviewConcurrency caps the reviews of one run in flight at the model.+ aiReviewConcurrency = 8+)+// aiReviewUnit is one AI review: a spec's sheet against one database, with// the facts the prompt states about that database.type aiReviewUnit struct {@@ -132,3 +144,63 @@ func aiReviewPriority(severity aireview.Severity) storepb.IssueCommentPayload_Rereturn storepb.IssueCommentPayload_ReviewMetadata_P2}}++// uniqueCheckTargets drops a database a spec names twice, so that each+// database is reviewed once per spec.+func uniqueCheckTargets(targets []*plancheck.CheckTarget) []*plancheck.CheckTarget {+ type key struct {+ specID string+ target string+ }+ seen := make(map[key]bool, len(targets))+ unique := make([]*plancheck.CheckTarget, 0, len(targets))+ for _, target := range targets {+ k := key{specID: target.SpecID, target: target.Target}+ if seen[k] {+ continue+ }+ seen[k] = true+ unique = append(unique, target)+ }+ return unique+}++// checkSheetSize refuses a sheet over maxAIReviewSheetBytes. A larger sheet+// is reported as not reviewed, never truncated: the tail would pass+// unreviewed.+func checkSheetSize(statement string) error {+ if len(statement) > maxAIReviewSheetBytes {+ return errors.Errorf("not reviewed: the sheet is %d bytes, over the %d byte limit", len(statement), maxAIReviewSheetBytes)+ }+ return nil+}++// reviewUnits runs review over the units, at most aiReviewConcurrency at a+// time, and returns each unit's result or error by index. A panic in one+// review is that unit's error, so the run fails instead of the process: the+// scheduler's recovery covers only the goroutine that called RunOnce.+func reviewUnits(ctx context.Context, units []*aiReviewUnit, review func(context.Context, *aiReviewUnit) (*aireview.Result, error)) ([]*aiReviewResult, []error) {+ results := make([]*aiReviewResult, len(units))+ errs := make([]error, len(units))+ var group errgroup.Group+ group.SetLimit(aiReviewConcurrency)+ for i, unit := range units {+ group.Go(func() error {+ defer func() {+ if r := recover(); r != nil {+ slog.Error("AI review PANIC RECOVER", slog.String("target", unit.Check.Target), slog.Any("panic", r), log.BBStack("panic-stack"))+ errs[i] = errors.Errorf("%s: review panic: %v", unit.Check.Target, r)+ }+ }()+ result, err := review(ctx, unit)+ if err != nil {+ errs[i] = errors.Wrapf(err, "%s", unit.Check.Target)+ return nil+ }+ results[i] = &aiReviewResult{Unit: unit, Findings: result.Findings}+ return nil+ })+ }+ _ = group.Wait()+ return results, errs+}diff --git a/backend/runner/reviewrun/ai_review_executor.go b/backend/runner/reviewrun/ai_review_executor.goindex 63ef8e4870..b3099101d0 100644--- a/backend/runner/reviewrun/ai_review_executor.go+++ b/backend/runner/reviewrun/ai_review_executor.go@@ -5,22 +5,12 @@ import ("log/slog""github.com/pkg/errors"- "golang.org/x/sync/errgroup""github.com/bytebase/bytebase/backend/plugin/aireview""github.com/bytebase/bytebase/backend/runner/plancheck""github.com/bytebase/bytebase/backend/store")-const (- // maxAIReviewSheetBytes bounds the sheet one review reads whole. A larger- // sheet is reported as not reviewed, never truncated: the tail would pass- // unreviewed.- maxAIReviewSheetBytes = 256 << 10- // aiReviewConcurrency caps the reviews of one run in flight at the model.- aiReviewConcurrency = 8-)-// AIReviewExecutor is the AI review: a model judges each spec's sheet against// the workspace and project AI review policy, once per database, with that// database's facts in the prompt. The reviews of one run share the model and@@ -64,6 +54,7 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueUif err != nil {return nil, errors.Wrapf(err, "failed to derive review targets")}+ checkTargets = uniqueCheckTargets(checkTargets)var unitErrs []errorvar units []*aiReviewUnit@@ -86,38 +77,29 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueU}reviewer := aireview.NewReviewer(aireview.NewModel(aiSetting))- results := make([]*aiReviewResult, len(units))- reviewErrs := make([]error, len(units))- var group errgroup.Group- group.SetLimit(aiReviewConcurrency)- for i, unit := range units {- group.Go(func() error {- result, err := reviewer.Review(ctx, &aireview.Request{- WorkspacePolicy: policy.Workspace,- ProjectPolicy: policy.Project,- Target: unit.Target,- Statement: unit.Statement,- }, aireview.NoTools{})- if err != nil {- reviewErrs[i] = errors.Wrapf(err, "%s", unit.Check.Target)- return nil- }- results[i] = &aiReviewResult{Unit: unit, Findings: result.Findings}- // The notes name the facts the model could not get. They are- // for the operator, never a finding.- slog.Info("AI review completed",- slog.String("project", projectID),- slog.Int64("issue_id", issueUID),- slog.String("spec", unit.Check.SpecID),- slog.String("target", unit.Check.Target),- slog.Int("model_calls", result.Calls),- slog.Int("tokens", result.TotalTokens),- slog.Int("findings", len(result.Findings)),- slog.Any("notes", result.Notes))- return nil- })- }- _ = group.Wait()+ results, reviewErrs := reviewUnits(ctx, units, func(ctx context.Context, unit *aiReviewUnit) (*aireview.Result, error) {+ result, err := reviewer.Review(ctx, &aireview.Request{+ WorkspacePolicy: policy.Workspace,+ ProjectPolicy: policy.Project,+ Target: unit.Target,+ Statement: unit.Statement,+ }, aireview.NoTools{})+ if err != nil {+ return nil, err+ }+ // The notes name the facts the model could not get. They are for+ // the operator, never a finding.+ slog.Info("AI review completed",+ slog.String("project", projectID),+ slog.Int64("issue_id", issueUID),+ slog.String("spec", unit.Check.SpecID),+ slog.String("target", unit.Check.Target),+ slog.Int("model_calls", result.Calls),+ slog.Int("tokens", result.TotalTokens),+ slog.Int("findings", len(result.Findings)),+ slog.Any("notes", result.Notes))+ return result, nil+ })// A canceled context is shutdown: the scheduler leaves the run to the// reaper instead of failing it.if err := ctx.Err(); err != nil {@@ -147,8 +129,8 @@ func (e *AIReviewExecutor) sheetStatement(ctx context.Context, sheets map[stringif sheet == nil {return "", errors.Errorf("sheet %s not found", sha256)}- if len(sheet.Statement) > maxAIReviewSheetBytes {- return "", errors.Errorf("not reviewed: the sheet is %d bytes, over the %d byte limit", len(sheet.Statement), maxAIReviewSheetBytes)+ if err := checkSheetSize(sheet.Statement); err != nil {+ return "", err}sheets[sha256] = sheet.Statementreturn sheet.Statement, nildiff --git a/backend/runner/reviewrun/ai_review_test.go b/backend/runner/reviewrun/ai_review_test.goindex 9607074c06..61a73a0ae8 100644--- a/backend/runner/reviewrun/ai_review_test.go+++ b/backend/runner/reviewrun/ai_review_test.go@@ -1,9 +1,14 @@package reviewrunimport (+ "context"+ "strings"+ "sync/atomic""testing"+ "time"metadatapb "github.com/bytebase/omni/metadata"+ "github.com/pkg/errors""github.com/stretchr/testify/require"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -89,3 +94,79 @@ func TestAIReviewComments(t *testing.T) {require.Empty(t, aiReviewComments("p", 7, nil))}++func TestUniqueCheckTargets(t *testing.T) {+ t.Parallel()++ targets := []*plancheck.CheckTarget{+ {SpecID: "spec-1", Target: "instances/i/databases/a"},+ {SpecID: "spec-1", Target: "instances/i/databases/b"},+ {SpecID: "spec-1", Target: "instances/i/databases/a"},+ {SpecID: "spec-2", Target: "instances/i/databases/a"},+ }+ unique := uniqueCheckTargets(targets)+ require.Equal(t, []*plancheck.CheckTarget{targets[0], targets[1], targets[3]}, unique, "the same database under another spec is another review")+ require.Empty(t, uniqueCheckTargets(nil))+}++func TestCheckSheetSize(t *testing.T) {+ t.Parallel()++ require.NoError(t, checkSheetSize(strings.Repeat("s", maxAIReviewSheetBytes)))+ err := checkSheetSize(strings.Repeat("s", maxAIReviewSheetBytes+1))+ require.ErrorContains(t, err, "not reviewed")+ require.ErrorContains(t, err, "262145 bytes")+}++// TestReviewUnits pins the executor's concurrency guarantees: at most+// aiReviewConcurrency reviews in flight, every unit attempted whatever the+// others do, and a panicking review recorded as that unit's error.+func TestReviewUnits(t *testing.T) {+ t.Parallel()++ units := make([]*aiReviewUnit, 20)+ for i := range units {+ units[i] = &aiReviewUnit{Check: &plancheck.CheckTarget{SpecID: "spec-1", Target: "instances/i/databases/db" + string(rune('a'+i))}}+ }+ var inFlight, maxInFlight atomic.Int32+ review := func(_ context.Context, unit *aiReviewUnit) (*aireview.Result, error) {+ now := inFlight.Add(1)+ defer inFlight.Add(-1)+ for {+ seen := maxInFlight.Load()+ if now <= seen || maxInFlight.CompareAndSwap(seen, now) {+ break+ }+ }+ time.Sleep(20 * time.Millisecond)+ switch unit.Check.Target {+ case units[3].Check.Target:+ return nil, errors.New("model call 1 failed")+ case units[5].Check.Target:+ panic("boom")+ default:+ return &aireview.Result{Findings: []aireview.Finding{{Title: "finding on " + unit.Check.Target, Line: 1}}}, nil+ }+ }++ results, errs := reviewUnits(context.Background(), units, review)+ require.Len(t, results, 20)+ require.Len(t, errs, 20)+ require.LessOrEqual(t, maxInFlight.Load(), int32(aiReviewConcurrency))+ require.Greater(t, maxInFlight.Load(), int32(1), "the reviews run concurrently")+ for i, result := range results {+ switch i {+ case 3:+ require.Nil(t, result)+ require.ErrorContains(t, errs[i], units[3].Check.Target+": model call 1 failed")+ case 5:+ require.Nil(t, result)+ require.ErrorContains(t, errs[i], units[5].Check.Target+": review panic: boom")+ default:+ require.NoError(t, errs[i])+ require.Same(t, units[i], result.Unit)+ require.Equal(t, "finding on "+units[i].Check.Target, result.Findings[0].Title)+ }+ }+ require.Zero(t, inFlight.Load())+}diff --git a/backend/tests/review_run_lifecycle_test.go b/backend/tests/review_run_lifecycle_test.goindex 6f9d94b09c..613b7232aa 100644--- a/backend/tests/review_run_lifecycle_test.go+++ b/backend/tests/review_run_lifecycle_test.go@@ -117,9 +117,13 @@ func TestCollision_RunReviewLifecycle(t *testing.T) {afterB := listReviewRuns(ctx, t, ctl, projectBID)a.Equal(beforeB, afterB, "project B review_run rows must be untouched by A's re-run")- // With AI enabled but the model unreachable, the AI slot is created,- // claimed, and fails honestly with the model's error.- setAISetting(ctx, t, ctl, "https://ai.invalid")+ // With AI enabled but the model failing, the AI slot is created, claimed,+ // and fails honestly with the model's error.+ brokenModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ http.Error(w, `{"error": {"message": "quota exceeded"}}`, http.StatusServiceUnavailable)+ }))+ defer brokenModel.Close()+ setAISetting(ctx, t, ctl, brokenModel.URL)aiRun, err := runReview(ctx, ctl, issueA.Name, "ai")a.NoError(err)a.Equal(v1pb.ReviewRun_AI, aiRun.Type)