Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(review): keep a review panic inside its run and test the run's bounds

h3n4ldc0138311121
diff --git a/backend/runner/reviewrun/ai_review.go b/backend/runner/reviewrun/ai_review.go
index 76b1f30323..d81008bc74 100644
--- a/backend/runner/reviewrun/ai_review.go
+++ b/backend/runner/reviewrun/ai_review.go
@@ -1,18 +1,30 @@
package reviewrun
import (
+ "context"
"fmt"
+ "log/slog"
"slices"
"strings"
metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/pkg/errors"
+ "golang.org/x/sync/errgroup"
+ "github.com/bytebase/bytebase/backend/common/log"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/aireview"
"github.com/bytebase/bytebase/backend/runner/plancheck"
"github.com/bytebase/bytebase/backend/store"
)
+const (
+ // maxAIReviewSheetBytes bounds the sheet one review reads whole.
+ maxAIReviewSheetBytes = 256 << 10
+ // aiReviewConcurrency caps the reviews of one run in flight at the model.
+ aiReviewConcurrency = 8
+)
+
// aiReviewUnit is one AI review: a spec's sheet against one database, with
// the facts the prompt states about that database.
type aiReviewUnit struct {
@@ -132,3 +144,63 @@ func aiReviewPriority(severity aireview.Severity) storepb.IssueCommentPayload_Re
return storepb.IssueCommentPayload_ReviewMetadata_P2
}
}
+
+// uniqueCheckTargets drops a database a spec names twice, so that each
+// database is reviewed once per spec.
+func uniqueCheckTargets(targets []*plancheck.CheckTarget) []*plancheck.CheckTarget {
+ type key struct {
+ specID string
+ target string
+ }
+ seen := make(map[key]bool, len(targets))
+ unique := make([]*plancheck.CheckTarget, 0, len(targets))
+ for _, target := range targets {
+ k := key{specID: target.SpecID, target: target.Target}
+ if seen[k] {
+ continue
+ }
+ seen[k] = true
+ unique = append(unique, target)
+ }
+ return unique
+}
+
+// checkSheetSize refuses a sheet over maxAIReviewSheetBytes. A larger sheet
+// is reported as not reviewed, never truncated: the tail would pass
+// unreviewed.
+func checkSheetSize(statement string) error {
+ if len(statement) > maxAIReviewSheetBytes {
+ return errors.Errorf("not reviewed: the sheet is %d bytes, over the %d byte limit", len(statement), maxAIReviewSheetBytes)
+ }
+ return nil
+}
+
+// reviewUnits runs review over the units, at most aiReviewConcurrency at a
+// time, and returns each unit's result or error by index. A panic in one
+// review is that unit's error, so the run fails instead of the process: the
+// scheduler's recovery covers only the goroutine that called RunOnce.
+func reviewUnits(ctx context.Context, units []*aiReviewUnit, review func(context.Context, *aiReviewUnit) (*aireview.Result, error)) ([]*aiReviewResult, []error) {
+ results := make([]*aiReviewResult, len(units))
+ errs := make([]error, len(units))
+ var group errgroup.Group
+ group.SetLimit(aiReviewConcurrency)
+ for i, unit := range units {
+ group.Go(func() error {
+ defer func() {
+ if r := recover(); r != nil {
+ slog.Error("AI review PANIC RECOVER", slog.String("target", unit.Check.Target), slog.Any("panic", r), log.BBStack("panic-stack"))
+ errs[i] = errors.Errorf("%s: review panic: %v", unit.Check.Target, r)
+ }
+ }()
+ result, err := review(ctx, unit)
+ if err != nil {
+ errs[i] = errors.Wrapf(err, "%s", unit.Check.Target)
+ return nil
+ }
+ results[i] = &aiReviewResult{Unit: unit, Findings: result.Findings}
+ return nil
+ })
+ }
+ _ = group.Wait()
+ return results, errs
+}
diff --git a/backend/runner/reviewrun/ai_review_executor.go b/backend/runner/reviewrun/ai_review_executor.go
index 63ef8e4870..b3099101d0 100644
--- a/backend/runner/reviewrun/ai_review_executor.go
+++ b/backend/runner/reviewrun/ai_review_executor.go
@@ -5,22 +5,12 @@ import (
"log/slog"
"github.com/pkg/errors"
- "golang.org/x/sync/errgroup"
"github.com/bytebase/bytebase/backend/plugin/aireview"
"github.com/bytebase/bytebase/backend/runner/plancheck"
"github.com/bytebase/bytebase/backend/store"
)
-const (
- // maxAIReviewSheetBytes bounds the sheet one review reads whole. A larger
- // sheet is reported as not reviewed, never truncated: the tail would pass
- // unreviewed.
- maxAIReviewSheetBytes = 256 << 10
- // aiReviewConcurrency caps the reviews of one run in flight at the model.
- aiReviewConcurrency = 8
-)
-
// AIReviewExecutor is the AI review: a model judges each spec's sheet against
// the workspace and project AI review policy, once per database, with that
// database's facts in the prompt. The reviews of one run share the model and
@@ -64,6 +54,7 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueU
if err != nil {
return nil, errors.Wrapf(err, "failed to derive review targets")
}
+ checkTargets = uniqueCheckTargets(checkTargets)
var unitErrs []error
var units []*aiReviewUnit
@@ -86,38 +77,29 @@ func (e *AIReviewExecutor) RunOnce(ctx context.Context, projectID string, issueU
}
reviewer := aireview.NewReviewer(aireview.NewModel(aiSetting))
- results := make([]*aiReviewResult, len(units))
- reviewErrs := make([]error, len(units))
- var group errgroup.Group
- group.SetLimit(aiReviewConcurrency)
- for i, unit := range units {
- group.Go(func() error {
- result, err := reviewer.Review(ctx, &aireview.Request{
- WorkspacePolicy: policy.Workspace,
- ProjectPolicy: policy.Project,
- Target: unit.Target,
- Statement: unit.Statement,
- }, aireview.NoTools{})
- if err != nil {
- reviewErrs[i] = errors.Wrapf(err, "%s", unit.Check.Target)
- return nil
- }
- results[i] = &aiReviewResult{Unit: unit, Findings: result.Findings}
- // The notes name the facts the model could not get. They are
- // for the operator, never a finding.
- slog.Info("AI review completed",
- slog.String("project", projectID),
- slog.Int64("issue_id", issueUID),
- slog.String("spec", unit.Check.SpecID),
- slog.String("target", unit.Check.Target),
- slog.Int("model_calls", result.Calls),
- slog.Int("tokens", result.TotalTokens),
- slog.Int("findings", len(result.Findings)),
- slog.Any("notes", result.Notes))
- return nil
- })
- }
- _ = group.Wait()
+ results, reviewErrs := reviewUnits(ctx, units, func(ctx context.Context, unit *aiReviewUnit) (*aireview.Result, error) {
+ result, err := reviewer.Review(ctx, &aireview.Request{
+ WorkspacePolicy: policy.Workspace,
+ ProjectPolicy: policy.Project,
+ Target: unit.Target,
+ Statement: unit.Statement,
+ }, aireview.NoTools{})
+ if err != nil {
+ return nil, err
+ }
+ // The notes name the facts the model could not get. They are for
+ // the operator, never a finding.
+ slog.Info("AI review completed",
+ slog.String("project", projectID),
+ slog.Int64("issue_id", issueUID),
+ slog.String("spec", unit.Check.SpecID),
+ slog.String("target", unit.Check.Target),
+ slog.Int("model_calls", result.Calls),
+ slog.Int("tokens", result.TotalTokens),
+ slog.Int("findings", len(result.Findings)),
+ slog.Any("notes", result.Notes))
+ return result, nil
+ })
// A canceled context is shutdown: the scheduler leaves the run to the
// reaper instead of failing it.
if err := ctx.Err(); err != nil {
@@ -147,8 +129,8 @@ func (e *AIReviewExecutor) sheetStatement(ctx context.Context, sheets map[string
if sheet == nil {
return "", errors.Errorf("sheet %s not found", sha256)
}
- if len(sheet.Statement) > maxAIReviewSheetBytes {
- return "", errors.Errorf("not reviewed: the sheet is %d bytes, over the %d byte limit", len(sheet.Statement), maxAIReviewSheetBytes)
+ if err := checkSheetSize(sheet.Statement); err != nil {
+ return "", err
}
sheets[sha256] = sheet.Statement
return sheet.Statement, nil
diff --git a/backend/runner/reviewrun/ai_review_test.go b/backend/runner/reviewrun/ai_review_test.go
index 9607074c06..61a73a0ae8 100644
--- a/backend/runner/reviewrun/ai_review_test.go
+++ b/backend/runner/reviewrun/ai_review_test.go
@@ -1,9 +1,14 @@
package reviewrun
import (
+ "context"
+ "strings"
+ "sync/atomic"
"testing"
+ "time"
metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/pkg/errors"
"github.com/stretchr/testify/require"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -89,3 +94,79 @@ func TestAIReviewComments(t *testing.T) {
require.Empty(t, aiReviewComments("p", 7, nil))
}
+
+func TestUniqueCheckTargets(t *testing.T) {
+ t.Parallel()
+
+ targets := []*plancheck.CheckTarget{
+ {SpecID: "spec-1", Target: "instances/i/databases/a"},
+ {SpecID: "spec-1", Target: "instances/i/databases/b"},
+ {SpecID: "spec-1", Target: "instances/i/databases/a"},
+ {SpecID: "spec-2", Target: "instances/i/databases/a"},
+ }
+ unique := uniqueCheckTargets(targets)
+ require.Equal(t, []*plancheck.CheckTarget{targets[0], targets[1], targets[3]}, unique, "the same database under another spec is another review")
+ require.Empty(t, uniqueCheckTargets(nil))
+}
+
+func TestCheckSheetSize(t *testing.T) {
+ t.Parallel()
+
+ require.NoError(t, checkSheetSize(strings.Repeat("s", maxAIReviewSheetBytes)))
+ err := checkSheetSize(strings.Repeat("s", maxAIReviewSheetBytes+1))
+ require.ErrorContains(t, err, "not reviewed")
+ require.ErrorContains(t, err, "262145 bytes")
+}
+
+// TestReviewUnits pins the executor's concurrency guarantees: at most
+// aiReviewConcurrency reviews in flight, every unit attempted whatever the
+// others do, and a panicking review recorded as that unit's error.
+func TestReviewUnits(t *testing.T) {
+ t.Parallel()
+
+ units := make([]*aiReviewUnit, 20)
+ for i := range units {
+ units[i] = &aiReviewUnit{Check: &plancheck.CheckTarget{SpecID: "spec-1", Target: "instances/i/databases/db" + string(rune('a'+i))}}
+ }
+ var inFlight, maxInFlight atomic.Int32
+ review := func(_ context.Context, unit *aiReviewUnit) (*aireview.Result, error) {
+ now := inFlight.Add(1)
+ defer inFlight.Add(-1)
+ for {
+ seen := maxInFlight.Load()
+ if now <= seen || maxInFlight.CompareAndSwap(seen, now) {
+ break
+ }
+ }
+ time.Sleep(20 * time.Millisecond)
+ switch unit.Check.Target {
+ case units[3].Check.Target:
+ return nil, errors.New("model call 1 failed")
+ case units[5].Check.Target:
+ panic("boom")
+ default:
+ return &aireview.Result{Findings: []aireview.Finding{{Title: "finding on " + unit.Check.Target, Line: 1}}}, nil
+ }
+ }
+
+ results, errs := reviewUnits(context.Background(), units, review)
+ require.Len(t, results, 20)
+ require.Len(t, errs, 20)
+ require.LessOrEqual(t, maxInFlight.Load(), int32(aiReviewConcurrency))
+ require.Greater(t, maxInFlight.Load(), int32(1), "the reviews run concurrently")
+ for i, result := range results {
+ switch i {
+ case 3:
+ require.Nil(t, result)
+ require.ErrorContains(t, errs[i], units[3].Check.Target+": model call 1 failed")
+ case 5:
+ require.Nil(t, result)
+ require.ErrorContains(t, errs[i], units[5].Check.Target+": review panic: boom")
+ default:
+ require.NoError(t, errs[i])
+ require.Same(t, units[i], result.Unit)
+ require.Equal(t, "finding on "+units[i].Check.Target, result.Findings[0].Title)
+ }
+ }
+ require.Zero(t, inFlight.Load())
+}
diff --git a/backend/tests/review_run_lifecycle_test.go b/backend/tests/review_run_lifecycle_test.go
index 6f9d94b09c..613b7232aa 100644
--- a/backend/tests/review_run_lifecycle_test.go
+++ b/backend/tests/review_run_lifecycle_test.go
@@ -117,9 +117,13 @@ func TestCollision_RunReviewLifecycle(t *testing.T) {
afterB := listReviewRuns(ctx, t, ctl, projectBID)
a.Equal(beforeB, afterB, "project B review_run rows must be untouched by A's re-run")
- // With AI enabled but the model unreachable, the AI slot is created,
- // claimed, and fails honestly with the model's error.
- setAISetting(ctx, t, ctl, "https://ai.invalid")
+ // With AI enabled but the model failing, the AI slot is created, claimed,
+ // and fails honestly with the model's error.
+ brokenModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ http.Error(w, `{"error": {"message": "quota exceeded"}}`, http.StatusServiceUnavailable)
+ }))
+ defer brokenModel.Close()
+ setAISetting(ctx, t, ctl, brokenModel.URL)
aiRun, err := runReview(ctx, ctl, issueA.Name, "ai")
a.NoError(err)
a.Equal(v1pb.ReviewRun_AI, aiRun.Type)