Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(review): run the AI review on Claude with room for its answer (#21597)
h3n4l7a7f8519b712
diff --git a/backend/plugin/ai/ai.go b/backend/plugin/ai/ai.goindex 047f46b678..3daac81205 100644--- a/backend/plugin/ai/ai.go+++ b/backend/plugin/ai/ai.go@@ -12,14 +12,14 @@ import ()// Chat sends a chat request to the configured AI provider.-func Chat(ctx context.Context, aiSetting *storepb.AISetting, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {+func Chat(ctx context.Context, aiSetting *storepb.AISetting, request *v1pb.AIChatRequest, options ...ChatOption) (*v1pb.AIChatResponse, error) {switch aiSetting.Provider {case storepb.AISetting_OPEN_AI, storepb.AISetting_AZURE_OPENAI:return chatOpenAI(ctx, aiSetting, request)case storepb.AISetting_GEMINI:return chatGemini(ctx, aiSetting, request)case storepb.AISetting_CLAUDE:- return chatClaude(ctx, aiSetting, request)+ return chatClaude(ctx, aiSetting, request, options...)default:return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupported AI provider %s", aiSetting.Provider))}diff --git a/backend/plugin/ai/chat_option.go b/backend/plugin/ai/chat_option.gonew file mode 100644index 0000000000..d9d7df7f51--- /dev/null+++ b/backend/plugin/ai/chat_option.go@@ -0,0 +1,25 @@+package ai++// ChatOption changes one Chat call.+type ChatOption func(*chatOptions)++type chatOptions struct {+ maxOutputTokens int+}++// WithMaxOutputTokens asks for an output limit of up to tokens. Only the Claude+// adapter sends a limit, lowered to the model's documented maximum; the OpenAI+// and Gemini adapters send none.+func WithMaxOutputTokens(tokens int) ChatOption {+ return func(o *chatOptions) {+ o.maxOutputTokens = tokens+ }+}++func newChatOptions(options []ChatOption) chatOptions {+ var o chatOptions+ for _, option := range options {+ option(&o)+ }+ return o+}diff --git a/backend/plugin/ai/claude.go b/backend/plugin/ai/claude.goindex acf8c25045..79583d5e4b 100644--- a/backend/plugin/ai/claude.go+++ b/backend/plugin/ai/claude.go@@ -5,6 +5,8 @@ import ("context""encoding/json""net/http"+ "regexp"+ "strings""github.com/pkg/errors"@@ -84,10 +86,57 @@ func readClaudeFailure(httpErr *HTTPError) {httpErr.QuotaExhausted = reply.Error.Details.ErrorCode == "enforced_spend_limit_reached"}-func chatClaude(ctx context.Context, aiSetting *storepb.AISetting, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {+// defaultClaudeMaxTokens is the output limit of a call that asks for none.+const defaultClaudeMaxTokens = 4096++// claudeOutputCeilings are the documented output maximums of older models, and+// Anthropic rejects a larger max_tokens with a 400. A model id matches the+// first family it contains, ignoring case, so dated, aliased, Bedrock, and+// Vertex ids all match.+var claudeOutputCeilings = []struct {+ family *regexp.Regexp+ maxTokens int+}{+ {family: claudeFamily("claude-3-5-sonnet"), maxTokens: 8192},+ {family: claudeFamily("claude-3-5-haiku"), maxTokens: 8192},+ {family: claudeFamily("claude-3-opus"), maxTokens: 4096},+ {family: claudeFamily("claude-3-sonnet"), maxTokens: 4096},+ {family: claudeFamily("claude-3-haiku"), maxTokens: 4096},+ {family: claudeFamily("claude-2"), maxTokens: 4096},+ {family: claudeFamily("claude-v2"), maxTokens: 4096},+ {family: claudeFamily("claude-instant"), maxTokens: 4096},+}++// claudeFamily matches the name anywhere in a model id. A name that ends in a+// version number must not run on into another digit, so claude-2 matches+// claude-2.1 but not claude-20.+func claudeFamily(name string) *regexp.Regexp {+ pattern := regexp.QuoteMeta(name)+ if last := name[len(name)-1]; last >= '0' && last <= '9' {+ pattern += `($|[^0-9])`+ }+ return regexp.MustCompile(pattern)+}++// claudeMaxTokens caps the requested output limit at the model's ceiling. A+// model missing from the table keeps the request.+func claudeMaxTokens(model string, requested int) int {+ if requested <= 0 {+ return defaultClaudeMaxTokens+ }+ model = strings.ToLower(model)+ for _, ceiling := range claudeOutputCeilings {+ if ceiling.family.MatchString(model) {+ return min(requested, ceiling.maxTokens)+ }+ }+ return requested+}++func chatClaude(ctx context.Context, aiSetting *storepb.AISetting, request *v1pb.AIChatRequest, options ...ChatOption) (*v1pb.AIChatResponse, error) {payload := chatClaudeRequest{Model: aiSetting.Model,- MaxTokens: 4096,+ MaxTokens: claudeMaxTokens(aiSetting.Model, newChatOptions(options).maxOutputTokens),}for _, m := range request.Messages {@@ -132,16 +181,24 @@ func chatClaude(ctx context.Context, aiSetting *storepb.AISetting, request *v1pbContent: contentBlocks,})case v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_TOOL:- // Claude uses "user" role with tool_result content block.+ result := chatClaudeContentBlock{+ Type: "tool_result",+ ToolUseID: m.GetToolCallId(),+ Content: m.GetContent(),+ }+ // Anthropic asks for the results of one turn's tool calls in a single+ // user message, since separate messages teach Claude to stop calling+ // tools in parallel. A user message holds blocks only for tool results.+ if last := len(payload.Messages) - 1; last >= 0 && payload.Messages[last].Role == "user" {+ if results, ok := payload.Messages[last].Content.([]chatClaudeContentBlock); ok {+ results = append(results, result)+ payload.Messages[last].Content = results+ continue+ }+ }payload.Messages = append(payload.Messages, chatClaudeMsg{- Role: "user",- Content: []chatClaudeContentBlock{- {- Type: "tool_result",- ToolUseID: m.GetToolCallId(),- Content: m.GetContent(),- },- },+ Role: "user",+ Content: []chatClaudeContentBlock{result},})default:continuediff --git a/backend/plugin/ai/claude_test.go b/backend/plugin/ai/claude_test.gonew file mode 100644index 0000000000..97db7ca063--- /dev/null+++ b/backend/plugin/ai/claude_test.go@@ -0,0 +1,155 @@+package ai++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// claudeParallelToolUseReply is a claude-sonnet-5 reply that calls two tools at once.+const claudeParallelToolUseReply = `{"model":"claude-sonnet-5","id":"msg_011Cg1p54tbDVDRbm4fQJkuW","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"EpUECscCCBIYAipAigTKC74ZSFufS5ml4CkcNdWZ4uxv2BwwxbQA8KazXjRTqt55Jjsq6SA0BdiW/FgAY4ofAsZ+pf32L5SLABPh1zIPY2xhdWRlLXNvbm5ldC01OABCCHRoaW5raW5nWiQxYTU5NTdlZS0wNzJhLTRhZjItYjdhYS0wYmU3MDNjNWMxMzCaAREKD2NsYXVkZS1zb25uZXQtNagB3fm81gbCAT0IBRAAGg8BD7kCU+gDTzcFsw4EJq0gAjIEEnHPJToEjA2WMkIE1N6MUEoEETUNSVACWgTZBeFWYAJoAXAB2gFgChZwcmVmaXhfYmVmb3JlX2Jsb2NrX3YxEiByuG/a9x+QeTzXTjzga6+KFIGRoQYicqTphtfqRKND5CIAMiDCwBPIapyS4i+Wbz2GF3aPi3pEyQKtzAl7+h4JivtOH0gDEgzR8qcO/wzHH99ilVYaDMo7edEn68Q/I0lw7CIwxKWtUMYG+mzApWDsNah7EW7wi+CkJnubi2/HCyMD/GzlUQKayLF+MZZrkujuZYjdKntfH/3EMtbCq0e6OLRKaqoDfb0m862T5/1CryN2gfrmrOxZuVI6TdQQzppDuRasveEs0RQNtGePxPCVflP9Yx2ktJLio2asyO6nDnnfMe33c7TsDhy6wDq2plkjvn2aSaDBDQfbf3YFYfkXuscjD8Mh7KSql3CqJI9qOggYAQ=="},{"type":"tool_use","id":"toolu_013EgXcQDNpKPQoYB3SsWFxY","name":"read","input":{"objects":[{"name":"orders","schema":"public"}]},"caller":{"type":"direct"}},{"type":"tool_use","id":"toolu_01Jkq5nRsJX4dh5kHHcBhYXg","name":"search","input":{"text":"legacy_status"},"caller":{"type":"direct"}}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2100,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":143,"output_tokens_details":{"thinking_tokens":32},"service_tier":"standard","inference_geo":"global"},"diagnostics":null}`++// claudeRequest is the part of the Claude wire format the tests assert on.+type claudeRequest struct {+ MaxTokens int `json:"max_tokens"`+ Messages []struct {+ Role string `json:"role"`+ Content json.RawMessage `json:"content"`+ } `json:"messages"`+}++func newClaudeServer(t *testing.T, reply string, requests *[]claudeRequest) *httptest.Server {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var request claudeRequest+ require.NoError(t, json.NewDecoder(r.Body).Decode(&request))+ *requests = append(*requests, request)+ w.Header().Set("Content-Type", "application/json")+ _, err := w.Write([]byte(reply))+ require.NoError(t, err)+ }))+ t.Cleanup(server.Close)+ return server+}++func TestChatClaudeOutputLimit(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ model string+ options []ChatOption+ want int+ }{+ {name: "the SQL editor asks for no limit", model: "claude-sonnet-5", want: 4096},+ {name: "a caller asks for a larger limit", model: "claude-sonnet-5", options: []ChatOption{WithMaxOutputTokens(16000)}, want: 16000},+ {name: "an older model caps the larger limit", model: "claude-3-5-haiku-20241022", options: []ChatOption{WithMaxOutputTokens(16000)}, want: 8192},+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ var requests []claudeRequest+ server := newClaudeServer(t, `{"content": [{"type": "text", "text": "ok"}]}`, &requests)+ setting := &storepb.AISetting{Provider: storepb.AISetting_CLAUDE, Endpoint: server.URL, Model: tc.model, ApiKey: "test-key"}+ userContent := "Explain the query."+ _, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{+ Messages: []*v1pb.AIChatMessage{{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_USER, Content: &userContent}},+ }, tc.options...)+ require.NoError(t, err)+ require.Len(t, requests, 1)+ require.Equal(t, tc.want, requests[0].MaxTokens)+ })+ }+}++func TestClaudeMaxTokens(t *testing.T) {+ t.Parallel()++ tests := []struct {+ model string+ requested int+ want int+ }{+ // Callers that ask for no limit get the default whatever the model.+ {model: "claude-sonnet-5", requested: 0, want: 4096},+ {model: "claude-3-haiku-20240307", requested: 0, want: 4096},++ // Models at or above the request keep it.+ {model: "claude-sonnet-5", requested: 16000, want: 16000},+ {model: "claude-haiku-4-5", requested: 16000, want: 16000},+ {model: "claude-3-7-sonnet-20250219", requested: 16000, want: 16000},+ {model: "claude-3-7-sonnet-latest", requested: 16000, want: 16000},+ {model: "anthropic.claude-sonnet-4-20250514-v1:0", requested: 16000, want: 16000},+ {model: "a-custom-model", requested: 16000, want: 16000},+ {model: "claude-20-sonnet", requested: 16000, want: 16000},+ {model: "anthropic.claude-v20-test", requested: 16000, want: 16000},++ {model: "claude-3-5-sonnet-20241022", requested: 16000, want: 8192},+ {model: "claude-3-5-sonnet-20240620", requested: 16000, want: 8192},+ {model: "claude-3-5-sonnet-latest", requested: 16000, want: 8192},+ {model: "anthropic.claude-3-5-sonnet-20241022-v2:0", requested: 16000, want: 8192},+ {model: "claude-3-5-sonnet-v2@20241022", requested: 16000, want: 8192},+ {model: "claude-3-5-haiku-20241022", requested: 16000, want: 8192},+ {model: "anthropic.claude-3-5-haiku-20241022-v1:0", requested: 16000, want: 8192},+ {model: "us.anthropic.claude-3-5-haiku-20241022-v1:0", requested: 16000, want: 8192},+ {model: "myclaude-3-5-sonnet-20240620", requested: 16000, want: 8192},+ {model: "proxy/claude-3-5-sonnet20240620", requested: 16000, want: 8192},+ {model: "claude-3-opus-20240229", requested: 16000, want: 4096},+ {model: "CLAUDE-3-OPUS-LATEST", requested: 16000, want: 4096},+ {model: "claude-3-sonnet-20240229", requested: 16000, want: 4096},+ {model: "claude-3-haiku-20240307", requested: 16000, want: 4096},+ {model: "claude-3-haiku@20240307", requested: 16000, want: 4096},+ {model: "claude-2", requested: 16000, want: 4096},+ {model: "claude-2.1", requested: 16000, want: 4096},+ {model: "claude-2.0", requested: 16000, want: 4096},+ {model: "anthropic.claude-v2:1", requested: 16000, want: 4096},+ {model: "claude-instant-1.2", requested: 16000, want: 4096},+ {model: "anthropic.claude-instant-v1", requested: 16000, want: 4096},++ // A request below the ceiling is sent as asked.+ {model: "claude-3-5-haiku-20241022", requested: 2000, want: 2000},+ {model: "claude-3-opus-20240229", requested: 1024, want: 1024},+ }+ for _, tc := range tests {+ require.Equal(t, tc.want, claudeMaxTokens(tc.model, tc.requested), "%s asking for %d", tc.model, tc.requested)+ }+}++func TestChatClaudeSendsParallelToolResultsInOneUserTurn(t *testing.T) {+ t.Parallel()++ var requests []claudeRequest+ server := newClaudeServer(t, claudeParallelToolUseReply, &requests)+ setting := &storepb.AISetting{Provider: storepb.AISetting_CLAUDE, Endpoint: server.URL, Model: "claude-sonnet-5", ApiKey: "test-key"}+ userContent := "Review the change."+ messages := []*v1pb.AIChatMessage{{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_USER, Content: &userContent}}++ first, err := chatClaude(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages})+ require.NoError(t, err)+ require.Len(t, first.ToolCalls, 2)++ messages = append(messages, &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_ASSISTANT, ToolCalls: first.ToolCalls})+ for _, call := range first.ToolCalls {+ output := "result of " + call.Name+ messages = append(messages, &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_TOOL, Content: &output, ToolCallId: &call.Id})+ }+ _, err = chatClaude(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages})+ require.NoError(t, err)++ require.Len(t, requests, 2)+ second := requests[1]+ require.Len(t, second.Messages, 3, "both results go in the one user turn after the tool calls")+ require.Equal(t, "assistant", second.Messages[1].Role)+ require.Equal(t, "user", second.Messages[2].Role)+ require.JSONEq(t, `[+ {"type": "tool_result", "tool_use_id": "toolu_013EgXcQDNpKPQoYB3SsWFxY", "content": "result of read"},+ {"type": "tool_result", "tool_use_id": "toolu_01Jkq5nRsJX4dh5kHHcBhYXg", "content": "result of search"}+ ]`, string(second.Messages[2].Content))+}diff --git a/backend/plugin/aireview/model_ai.go b/backend/plugin/aireview/model_ai.goindex 3cf01bfb8c..5fffb24c7b 100644--- a/backend/plugin/aireview/model_ai.go+++ b/backend/plugin/aireview/model_ai.go@@ -19,6 +19,11 @@ const (maxChatRetries = 3// firstRetryBackoff is the middle of the first backoff. Each retry doubles it.firstRetryBackoff = time.Second+ // maxOutputTokens holds Claude's thinking, which counts against the limit,+ // and a long findings list. claude-sonnet-5 allows 128K, but Anthropic+ // advises streaming above about 16K and ai.Chat does not stream. The Claude+ // adapter lowers it for an older model whose maximum is smaller.+ maxOutputTokens = 16000)type settingModel struct {@@ -37,7 +42,7 @@ func NewModel(setting *storepb.AISetting) Model {// review loop sees one model call however many attempts it takes.func (m *settingModel) Chat(ctx context.Context, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {for retry := 0; ; retry++ {- response, err := ai.Chat(ctx, m.setting, request)+ response, err := ai.Chat(ctx, m.setting, request, ai.WithMaxOutputTokens(maxOutputTokens))if err == nil {return response, nil}diff --git a/backend/plugin/aireview/model_ai_claude_test.go b/backend/plugin/aireview/model_ai_claude_test.gonew file mode 100644index 0000000000..1cdd4a627c--- /dev/null+++ b/backend/plugin/aireview/model_ai_claude_test.go@@ -0,0 +1,84 @@+package aireview++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// claudeRequest is the part of the Claude wire format the tests assert on.+type claudeRequest struct {+ System string `json:"system"`+ MaxTokens int `json:"max_tokens"`+ Messages []struct {+ Role string `json:"role"`+ Content json.RawMessage `json:"content"`+ } `json:"messages"`+ Tools []struct {+ Name string `json:"name"`+ } `json:"tools"`+}++// The scripted Model in reviewer_test.go stands in for ai.Chat, so this test+// holds the loop to the real Claude adapter's wire format.+func TestSettingModelRoundTripsThroughClaude(t *testing.T) {+ t.Parallel()++ var requests []claudeRequest+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var request claudeRequest+ require.NoError(t, json.NewDecoder(r.Body).Decode(&request))+ requests = append(requests, request)++ w.Header().Set("Content-Type", "application/json")+ reply := `{"type": "message", "role": "assistant", "content": [+ {"type": "thinking", "thinking": "", "signature": "sig-1"},+ {"type": "tool_use", "id": "toolu_1", "name": "read", "input": {"objects": [{"name": "orders", "schema": "public"}]}, "caller": {"type": "direct"}},+ {"type": "tool_use", "id": "toolu_2", "name": "search", "input": {"text": "orders"}, "caller": {"type": "direct"}}+ ], "stop_reason": "tool_use", "usage": {"input_tokens": 2100, "output_tokens": 143, "output_tokens_details": {"thinking_tokens": 32}}}`+ if len(requests) == 2 {+ reply = `{"type": "message", "role": "assistant", "content": [{"type": "text", "text": "{\"findings\": [], \"notes\": []}"}], "stop_reason": "end_turn", "usage": {"input_tokens": 2436, "output_tokens": 21}}`+ }+ _, err := w.Write([]byte(reply))+ require.NoError(t, err)+ }))+ defer server.Close()++ model := NewModel(&storepb.AISetting{Provider: storepb.AISetting_CLAUDE, Endpoint: server.URL, Model: "claude-sonnet-5", ApiKey: "test-key"})+ tools := &catalogTools{}+ result, err := NewReviewer(model).Review(context.Background(), &Request{Statement: "DROP TABLE orders;"}, tools)+ require.NoError(t, err)+ require.Empty(t, result.Findings)+ require.Equal(t, 2, result.Calls)+ require.Equal(t, 2100+143+2436+21, result.TotalTokens)+ require.Len(t, tools.calls, 2)++ require.Len(t, requests, 2)+ first := requests[0]+ require.Equal(t, 16000, first.MaxTokens, "the SQL editor's 4096 is too little for thinking and a long findings list")+ require.Contains(t, first.System, "# Answer format")+ require.Len(t, first.Messages, 1)+ require.Equal(t, "user", first.Messages[0].Role)+ require.Len(t, first.Tools, 2)++ second := requests[1]+ require.Equal(t, 16000, second.MaxTokens)+ require.Len(t, second.Messages, 3)+ require.Equal(t, "assistant", second.Messages[1].Role)+ require.Contains(t, string(second.Messages[1].Content), `"signature":"sig-1"`, "Claude rejects a tool turn that returns without its thinking")+ require.Equal(t, "user", second.Messages[2].Role)+ var results []struct {+ Type string `json:"type"`+ ToolUseID string `json:"tool_use_id"`+ }+ require.NoError(t, json.Unmarshal(second.Messages[2].Content, &results))+ require.Len(t, results, 2)+ require.Equal(t, "toolu_1", results[0].ToolUseID)+ require.Equal(t, "toolu_2", results[1].ToolUseID)+}diff --git a/backend/plugin/aireview/model_ai_live_claude_test.go b/backend/plugin/aireview/model_ai_live_claude_test.gonew file mode 100644index 0000000000..0623604317--- /dev/null+++ b/backend/plugin/aireview/model_ai_live_claude_test.go@@ -0,0 +1,63 @@+package aireview++import (+ "context"+ "os"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// TestReviewLiveClaude reviews a risky change with a real Claude model. It+// needs a key and network access that CI does not have:+//+// AIREVIEW_LIVE_CLAUDE_API_KEY=... go test ./backend/plugin/aireview/ -run '^TestReviewLiveClaude$' -v -count=1+func TestReviewLiveClaude(t *testing.T) {+ apiKey := os.Getenv("AIREVIEW_LIVE_CLAUDE_API_KEY")+ if apiKey == "" {+ t.Skip("AIREVIEW_LIVE_CLAUDE_API_KEY is not set")+ }+ modelName := os.Getenv("AIREVIEW_LIVE_CLAUDE_MODEL")+ if modelName == "" {+ modelName = "claude-sonnet-5"+ }++ statement := strings.Join([]string{+ "-- Speed up the order history page.",+ "CREATE INDEX idx_orders_created_at ON orders (created_at);",+ "",+ "ALTER TABLE orders DROP COLUMN legacy_status;",+ }, "\n")+ request := &Request{+ WorkspacePolicy: "An index on a table with more than 1 million rows must be created with CONCURRENTLY.",+ Target: Target{+ Engine: "POSTGRES",+ Version: "16.2",+ Environment: "prod",+ Schemas: []SchemaSummary{{Name: "public", ObjectCount: 2}},+ },+ Statement: statement,+ }+ model := NewModel(&storepb.AISetting{+ Provider: storepb.AISetting_CLAUDE,+ Endpoint: "https://api.anthropic.com/v1/messages",+ Model: modelName,+ ApiKey: apiKey,+ })+ tools := &catalogTools{}++ result, err := NewReviewer(model).Review(context.Background(), request, tools)+ require.NoError(t, err)+ t.Logf("model calls: %d, tokens: %d, tool calls: %v", result.Calls, result.TotalTokens, tools.calls)+ for _, finding := range result.Findings {+ t.Logf("%s line %d: %s\n rule: %s\n evidence: %s\n fix: %s", finding.Severity, finding.Line, finding.Title, finding.Rule, finding.Evidence, finding.Fix)+ }+ for _, note := range result.Notes {+ t.Logf("note: %s", note)+ }+ require.NotEmpty(t, tools.calls, "the model should look up the objects before it judges")+ require.NotEmpty(t, result.Findings)+}