Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

14.1 KB
Raw Blame
package reviewrun

import (
	"context"
	"fmt"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	metadatapb "github.com/bytebase/omni/metadata"
	"github.com/bytebase/omni/review"
	"github.com/stretchr/testify/require"
	"golang.org/x/sync/semaphore"
	"google.golang.org/genproto/googleapis/type/expr"
	"google.golang.org/protobuf/encoding/protojson"

	"github.com/bytebase/bytebase/backend/common"
	"github.com/bytebase/bytebase/backend/common/testcontainer"
	storepb "github.com/bytebase/bytebase/backend/generated-go/store"
	v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
	"github.com/bytebase/bytebase/backend/plugin/aireview"
	"github.com/bytebase/bytebase/backend/store"
)

// fixtureDatabases is how many databases the fixture's database group
// matches.
const fixtureDatabases = 5

// testWaitLimit fails a test whose reviews stop making progress, instead of
// letting it hang.
const testWaitLimit = 30 * time.Second

func TestCISampleBoundsOnlyTheAIReview(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	stores, issues := setupReviewFixture(ctx, t, 1)
	model := &findingModel{}
	executor := &AIReviewExecutor{
		store:    stores,
		newModel: func(*storepb.AISetting, ...aireview.ModelOption) aireview.Model { return model },
		slots:    semaphore.NewWeighted(aiReviewServerConcurrency),
	}
	reviewedDatabases := func() []string {
		t.Helper()
		comments, err := executor.RunOnce(ctx, "p", issues[0])
		require.NoError(t, err)
		require.Len(t, comments, 1, "the same finding on every database is one comment")
		return comments[0].Payload.ReviewMetadata.Targets
	}

	setCISamplingSize(ctx, t, stores, 2)
	require.Len(t, reviewedDatabases(), 2)
	require.Equal(t, int32(4), model.calls.Load(), "a review per database, one merge, and one summary")
	require.Equal(t, int32(1), model.merges.Load())
	require.Equal(t, int32(1), model.summaries.Load())

	var ruleTargets int
	ruleExecutor := &RuleExecutor{store: stores, reviewFuncs: map[storepb.Engine]review.ReviewFunc{
		storepb.Engine_POSTGRES: func(_ context.Context, _ string, _ review.Options, targets []review.Target) (*review.Result, error) {
			ruleTargets += len(targets)
			return &review.Result{}, nil
		},
	}}
	_, err := ruleExecutor.RunOnce(ctx, "p", issues[0])
	require.NoError(t, err)
	require.Equal(t, fixtureDatabases, ruleTargets, "the rule review evaluates every database whatever the sample")

	setCISamplingSize(ctx, t, stores, 0)
	require.Len(t, reviewedDatabases(), fixtureDatabases)
}

func TestAIReviewRunsShareTheServerLimit(t *testing.T) {
	t.Parallel()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stores, issues := setupReviewFixture(ctx, t, 2)
	const limit = 3
	total := len(issues) * fixtureDatabases
	require.Less(t, limit, fixtureDatabases, "one run alone could exceed the limit")
	model := &blockingModel{entered: make(chan struct{}, total), release: make(chan struct{})}
	slots := &observedSlots{Weighted: semaphore.NewWeighted(limit), waiting: make(chan struct{}, total)}
	executor := &AIReviewExecutor{
		store:    stores,
		newModel: func(*storepb.AISetting, ...aireview.ModelOption) aireview.Model { return model },
		slots:    slots,
	}

	runErrs := make(chan error, len(issues))
	for _, issueUID := range issues {
		go func() {
			_, err := executor.RunOnce(ctx, "p", issueUID)
			runErrs <- err
		}()
	}
	// Nothing is released yet, so every review either reaches the model or
	// waits for a slot.
	var entered, waiting int
	for entered+waiting < total {
		select {
		case <-model.entered:
			entered++
		case <-slots.waiting:
			waiting++
		case <-time.After(testWaitLimit):
			t.Fatalf("timed out with %d reviews at the model and %d waiting", entered, waiting)
		}
	}
	require.Equal(t, limit, entered, "the reviews of both runs past the limit wait for a slot")
	for range total - limit {
		send(t, model.release)
		receive(t, model.entered)
	}
	for range limit {
		send(t, model.release)
	}
	for range issues {
		require.NoError(t, receive(t, runErrs))
	}
	require.Equal(t, int32(limit), model.maxInFlight.Load())
}

// setupReviewFixture seeds project p with AI enabled, a PostgreSQL instance
// holding fixtureDatabases synced databases, a database group matching all of
// them, and the given number of issues, each with a plan whose one spec
// targets the group. It returns the issue UIDs.
func setupReviewFixture(ctx context.Context, t *testing.T, issueCount int) (*store.Store, []int64) {
	t.Helper()
	db, stores, _ := testcontainer.NewMetadataDB(t)
	_, err := db.ExecContext(ctx, `
		INSERT INTO workspace (resource_id) VALUES ('ws');
		INSERT INTO principal (name, email, password_hash) VALUES ('creator', '[email protected]', 'unused');
		INSERT INTO project (resource_id, workspace, name) VALUES ('p', 'ws', 'P');
	`)
	require.NoError(t, err)
	_, err = stores.UpsertSetting(ctx, &store.SettingMessage{Name: storepb.SettingName_AI, Workspace: "ws", Value: &storepb.AISetting{Enabled: true, SqlReviewEnabled: true}})
	require.NoError(t, err)
	rules, err := protojson.Marshal(&storepb.ReviewRulePolicy{Rules: []storepb.ReviewRuleType{storepb.ReviewRuleType_SYNTAX}})
	require.NoError(t, err)
	_, err = stores.CreatePolicy(ctx, &store.PolicyMessage{
		Workspace:    "ws",
		ResourceType: storepb.Policy_PROJECT,
		Resource:     common.FormatProject("p"),
		Type:         storepb.Policy_REVIEW_RULE,
		Payload:      string(rules),
		Enforce:      true,
	})
	require.NoError(t, err)

	_, err = stores.CreateInstance(ctx, &store.InstanceMessage{
		ResourceID: "prod",
		Workspace:  "ws",
		Metadata: &storepb.Instance{
			Engine:      storepb.Engine_POSTGRES,
			DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},
		},
	})
	require.NoError(t, err)
	for i := range fixtureDatabases {
		name := fmt.Sprintf("db%d", i+1)
		_, err = stores.UpsertDatabase(ctx, &store.DatabaseMessage{
			ProjectID:    "p",
			InstanceID:   "prod",
			DatabaseName: name,
			Metadata:     &storepb.DatabaseMetadata{Labels: map[string]string{}},
		})
		require.NoError(t, err)
		schema := &metadatapb.DatabaseSchemaMetadata{Name: name, Schemas: []*metadatapb.SchemaMetadata{{Name: "public"}}}
		_, err = stores.UpsertDBSchema(ctx, "prod", name, schema, nil, time.Now())
		require.NoError(t, err)
	}
	_, err = stores.CreateDatabaseGroup(ctx, &store.DatabaseGroupMessage{
		ProjectID:  "p",
		ResourceID: "all",
		Title:      "all",
		Expression: &expr.Expr{Expression: `resource.instance_id == "prod"`},
	})
	require.NoError(t, err)
	sheets, err := stores.CreateSheets(ctx, "p", &store.SheetMessage{Statement: "ALTER TABLE orders ADD COLUMN note text;"})
	require.NoError(t, err)

	var issues []int64
	for i := range issueCount {
		plan, err := stores.CreatePlan(ctx, &store.PlanMessage{
			ProjectID: "p",
			Name:      fmt.Sprintf("plan %d", i),
			Config: &storepb.PlanConfig{
				Specs: []*storepb.PlanConfig_Spec{{
					Id: "spec-1",
					Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{
						ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{
							Targets:     []string{"projects/p/databaseGroups/all"},
							SheetSha256: sheets[0].Sha256,
						},
					},
				}},
			},
		}, "[email protected]")
		require.NoError(t, err)
		issue, err := stores.CreateIssue(ctx, &store.IssueMessage{
			ProjectID:    "p",
			CreatorEmail: "[email protected]",
			Title:        fmt.Sprintf("issue %d", i),
			Type:         storepb.Issue_DATABASE_CHANGE,
			Payload:      &storepb.Issue{},
			PlanUID:      &plan.UID,
		})
		require.NoError(t, err)
		issues = append(issues, issue.UID)
	}
	return stores, issues
}

func setCISamplingSize(ctx context.Context, t *testing.T, stores *store.Store, size int32) {
	t.Helper()
	require.NoError(t, stores.UpdateProjects(ctx, &store.UpdateProjectMessage{
		ResourceID: "p",
		Workspace:  "ws",
		Setting:    &storepb.Project{CiSamplingSize: size},
	}))
}

// summaryDatabaseID finds the database ids of a summary prompt's findings.
var summaryDatabaseID = regexp.MustCompile(`"database":"([a-z2-7]+)"`)

// findingModel answers every review with the same finding, the merge with one
// group of every finding, and the summary with one group of every database,
// so a run's one comment names every database the run reviewed.
type findingModel struct {
	calls     atomic.Int32
	merges    atomic.Int32
	summaries atomic.Int32
	// onSummary runs when the summary call arrives.
	onSummary func()
}

func (m *findingModel) Chat(_ context.Context, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {
	m.calls.Add(1)
	content := `{"findings": [{"title": "Backfill note", "severity": "P2", "line": 1, "rule": "r", "evidence": "e", "fix": "f"}]}`
	// A review offers the schema tools, and the merge and the summary offer
	// none. A merge numbers the findings, and a summary names the databases
	// by id.
	if len(request.GetToolDefinitions()) == 0 {
		user := request.GetMessages()[1].GetContent()
		var ids []string
		if count := strings.Count(user, `{"id":`); count > 0 {
			m.merges.Add(1)
			for i := range count {
				ids = append(ids, strconv.Itoa(i+1))
			}
			content = fmt.Sprintf(`{"groups": [[%s]]}`, strings.Join(ids, ", "))
		} else {
			m.summaries.Add(1)
			if m.onSummary != nil {
				m.onSummary()
			}
			for _, match := range summaryDatabaseID.FindAllStringSubmatch(user, -1) {
				if id := strconv.Quote(match[1]); !slices.Contains(ids, id) {
					ids = append(ids, id)
				}
			}
			content = fmt.Sprintf(`{"title": "Backfill note", "evidence": "e", "groups": [{"databases": [%s], "facts": ""}], "fix": "f"}`, strings.Join(ids, ", "))
		}
	}
	return &v1pb.AIChatResponse{Content: &content}, nil
}

func TestAIReviewStopsWhenSwitchedOffBeforeItPosts(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	stores, issues := setupReviewFixture(ctx, t, 1)
	var switchOffErr error
	model := &findingModel{onSummary: func() {
		_, switchOffErr = stores.UpsertSetting(ctx, &store.SettingMessage{Name: storepb.SettingName_AI, Workspace: "ws", Value: &storepb.AISetting{Enabled: true}})
	}}
	executor := &AIReviewExecutor{
		store:    stores,
		newModel: func(*storepb.AISetting, ...aireview.ModelOption) aireview.Model { return model },
		slots:    semaphore.NewWeighted(aiReviewServerConcurrency),
	}
	comments, err := executor.RunOnce(ctx, "p", issues[0])
	require.NoError(t, switchOffErr)
	require.Equal(t, int32(1), model.summaries.Load())
	require.ErrorContains(t, err, "AI SQL review is not enabled", "a switch-off during the summary stops the run before it posts")
	require.Empty(t, comments)
}

// observedSlots tells the test when a review has to wait for a slot.
type observedSlots struct {
	*semaphore.Weighted
	waiting chan struct{}
}

func (s *observedSlots) Acquire(ctx context.Context, n int64) error {
	if s.TryAcquire(n) {
		return nil
	}
	s.waiting <- struct{}{}
	return s.Weighted.Acquire(ctx, n)
}

// blockingModel holds every call until the test releases one, and records
// the most calls in flight at once.
type blockingModel struct {
	entered     chan struct{}
	release     chan struct{}
	inFlight    atomic.Int32
	maxInFlight atomic.Int32
}

func (m *blockingModel) Chat(ctx context.Context, _ *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {
	now := m.inFlight.Add(1)
	defer m.inFlight.Add(-1)
	for {
		seen := m.maxInFlight.Load()
		if now <= seen || m.maxInFlight.CompareAndSwap(seen, now) {
			break
		}
	}
	m.entered <- struct{}{}
	select {
	case <-m.release:
	case <-ctx.Done():
		return nil, ctx.Err()
	}
	content := `{"findings": []}`
	return &v1pb.AIChatResponse{Content: &content}, nil
}

func receive[T any](t *testing.T, ch <-chan T) T {
	t.Helper()
	select {
	case v := <-ch:
		return v
	case <-time.After(testWaitLimit):
		t.Fatal("timed out waiting for the reviews")
		var zero T
		return zero
	}
}

func send(t *testing.T, ch chan<- struct{}) {
	t.Helper()
	select {
	case ch <- struct{}{}:
	case <-time.After(testWaitLimit):
		t.Fatal("timed out waiting for a model call to release")
	}
}

func TestAIReviewExecutorChecksTheSettingAgain(t *testing.T) {
	t.Parallel()
	ctx := context.Background()
	db, stores, _ := testcontainer.NewMetadataDBWithCache(t, true)
	_, err := db.ExecContext(ctx, `
		INSERT INTO workspace (resource_id) VALUES ('ws');
		INSERT INTO principal (name, email, password_hash) VALUES ('creator', '[email protected]', 'unused');
		INSERT INTO project (resource_id, workspace, name) VALUES ('p', 'ws', 'P');
	`)
	require.NoError(t, err)

	plan, err := stores.CreatePlan(ctx, &store.PlanMessage{
		ProjectID: "p",
		Name:      "plan",
		Config: &storepb.PlanConfig{
			Specs: []*storepb.PlanConfig_Spec{{
				Id: "spec-1",
				Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{
					ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{
						Targets: []string{"instances/missing/databases/app"},
					},
				},
			}},
		},
	}, "[email protected]")
	require.NoError(t, err)
	issue, err := stores.CreateIssue(ctx, &store.IssueMessage{
		ProjectID:    "p",
		CreatorEmail: "[email protected]",
		Title:        "issue",
		Type:         storepb.Issue_DATABASE_CHANGE,
		Payload:      &storepb.Issue{},
		PlanUID:      &plan.UID,
	})
	require.NoError(t, err)

	setAISetting := func(setting *storepb.AISetting) {
		t.Helper()
		_, err := stores.UpsertSetting(ctx, &store.SettingMessage{
			Name:      storepb.SettingName_AI,
			Workspace: "ws",
			Value:     setting,
		})
		require.NoError(t, err)
	}
	executor := NewAIReviewExecutor(stores)

	setAISetting(&storepb.AISetting{Enabled: true})
	_, err = executor.RunOnce(ctx, "p", issue.UID)
	require.ErrorContains(t, err, "AI SQL review is not enabled")

	// With the switch on the executor goes on to the target, which does not
	// exist in this fixture.
	setAISetting(&storepb.AISetting{Enabled: true, SqlReviewEnabled: true})
	_, err = executor.RunOnce(ctx, "p", issue.UID)
	require.Error(t, err)
	require.NotContains(t, err.Error(), "is not enabled")

	// A switch-off written outside this process leaves the cached copy on.
	_, err = db.ExecContext(ctx, `UPDATE setting SET value = '{"enabled": true}' WHERE workspace = 'ws' AND name = 'AI'`)
	require.NoError(t, err)
	cached, err := stores.GetAISetting(ctx, "ws")
	require.NoError(t, err)
	require.True(t, cached.SqlReviewEnabled)
	_, err = executor.RunOnce(ctx, "p", issue.UID)
	require.ErrorContains(t, err, "AI SQL review is not enabled")
}