Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
test(review): cover the AI review on Azure OpenAI (#21598)
h3n4l08aca2d2beba
diff --git a/backend/plugin/ai/openai_azure_test.go b/backend/plugin/ai/openai_azure_test.gonew file mode 100644index 0000000000..7b212855eb--- /dev/null+++ b/backend/plugin/ai/openai_azure_test.go@@ -0,0 +1,452 @@+package ai++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// The requests and replies in this file follow the Azure OpenAI v1 API pages on+// learn.microsoft.com: api-version-lifecycle, how-to/responses,+// how-to/function-calling, how-to/reasoning, and how-to/quota.++const (+ azureDeployment = "review-deployment"+ azureAPIKey = "azure-test-key"+)++// azureRequest is one request the fake Azure resource received.+type azureRequest struct {+ method string+ path string+ query string+ apiKey string+ auth string+ contentType string+ body map[string]any+}++// newAzureServer records each request and answers it with the next reply.+func newAzureServer(t *testing.T, replies ...string) (*httptest.Server, *[]azureRequest) {+ t.Helper()+ var requests []azureRequest+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ request := azureRequest{+ method: r.Method,+ path: r.URL.Path,+ query: r.URL.RawQuery,+ apiKey: r.Header.Get("api-key"),+ auth: r.Header.Get("Authorization"),+ contentType: r.Header.Get("Content-Type"),+ }+ if err := json.NewDecoder(r.Body).Decode(&request.body); err != nil {+ t.Errorf("failed to decode the request body: %v", err)+ }+ requests = append(requests, request)+ if len(requests) > len(replies) {+ t.Errorf("unexpected request %d", len(requests))+ w.WriteHeader(http.StatusInternalServerError)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(replies[len(requests)-1]))+ }))+ t.Cleanup(server.Close)+ return server, &requests+}++func azureSetting(endpoint string) *storepb.AISetting {+ return &storepb.AISetting{+ Provider: storepb.AISetting_AZURE_OPENAI,+ Endpoint: endpoint,+ Model: azureDeployment,+ ApiKey: azureAPIKey,+ }+}++func azureSearchTool() *v1pb.AIChatToolDefinition {+ return &v1pb.AIChatToolDefinition{+ Name: "search",+ Description: "Find objects by name.",+ ParametersSchema: `{"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]}`,+ }+}++var azureSearchParameters = map[string]any{+ "type": "object",+ "properties": map[string]any{"text": map[string]any{"type": "string"}},+ "required": []any{"text"},+}++func azureReviewMessages() []*v1pb.AIChatMessage {+ system := "You review SQL."+ user := "DROP TABLE orders;"+ return []*v1pb.AIChatMessage{+ {Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_SYSTEM, Content: &system},+ {Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_USER, Content: &user},+ }+}++// requireAzureHeaders asserts what Azure documents for every call: a POST with+// the key in the api-key header, a JSON body, and the deployment name as the model.+func requireAzureHeaders(t *testing.T, request azureRequest) {+ t.Helper()+ require.Equal(t, http.MethodPost, request.method)+ require.Equal(t, azureAPIKey, request.apiKey)+ require.Empty(t, request.auth)+ require.Equal(t, "application/json", request.contentType)+ require.Equal(t, azureDeployment, request.body["model"])+}++const (+ azureResponsesText = `{+ "id": "resp_2",+ "object": "response",+ "created_at": 1741369940.0,+ "status": "completed",+ "error": null,+ "incomplete_details": null,+ "model": "gpt-5.6-terra",+ "output": [{+ "id": "msg_2",+ "type": "message",+ "role": "assistant",+ "status": "completed",+ "content": [{"type": "output_text", "text": "{\"findings\": []}", "annotations": [], "logprobs": null}]+ }],+ "parallel_tool_calls": true,+ "store": false,+ "usage": {"input_tokens": 140, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 10, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 150},+ "content_filters": [{"source_type": "prompt", "blocked": false, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}}}]+ }`+ azureChatCompletionsText = `{+ "id": "chatcmpl-2",+ "object": "chat.completion",+ "created": 1741369940,+ "model": "gpt-4o-2024-11-20",+ "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}}}],+ "choices": [{+ "index": 0,+ "finish_reason": "stop",+ "logprobs": null,+ "message": {"role": "assistant", "content": "{\"findings\": []}", "refusal": null},+ "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}}+ }],+ "usage": {"completion_tokens": 10, "prompt_tokens": 140, "total_tokens": 150}+ }`+)++func TestChatAzureOpenAISendsTheDocumentedRequest(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ endpoint string+ wantPath string+ wantQuery string+ wantResponses bool+ }{+ {+ name: "v1 responses, the default the settings page offers",+ endpoint: "/openai/v1/responses",+ wantPath: "/openai/v1/responses",+ wantResponses: true,+ },+ {+ name: "v1 chat completions",+ endpoint: "/openai/v1/chat/completions",+ wantPath: "/openai/v1/chat/completions",+ },+ {+ name: "v1 responses with the optional api-version",+ endpoint: "/openai/v1/responses?api-version=preview",+ wantPath: "/openai/v1/responses",+ wantQuery: "api-version=preview",+ wantResponses: true,+ },+ {+ name: "dated chat completions with the deployment in the path",+ endpoint: "/openai/deployments/review-deployment/chat/completions?api-version=2024-10-21",+ wantPath: "/openai/deployments/review-deployment/chat/completions",+ wantQuery: "api-version=2024-10-21",+ },+ {+ name: "dated responses",+ endpoint: "/openai/responses?api-version=2025-04-01-preview",+ wantPath: "/openai/responses",+ wantQuery: "api-version=2025-04-01-preview",+ wantResponses: true,+ },+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()++ reply := azureChatCompletionsText+ if test.wantResponses {+ reply = azureResponsesText+ }+ server, requests := newAzureServer(t, reply)+ response, err := Chat(context.Background(), azureSetting(server.URL+test.endpoint), &v1pb.AIChatRequest{+ Messages: azureReviewMessages(),+ ToolDefinitions: []*v1pb.AIChatToolDefinition{azureSearchTool()},+ })+ require.NoError(t, err)+ require.Equal(t, `{"findings": []}`, response.GetContent())+ require.EqualValues(t, 150, response.GetUsage().GetTotalTokens())++ require.Len(t, *requests, 1)+ request := (*requests)[0]+ requireAzureHeaders(t, request)+ require.Equal(t, test.wantPath, request.path)+ require.Equal(t, test.wantQuery, request.query)+ if test.wantResponses {+ require.Contains(t, request.body, "input")+ require.NotContains(t, request.body, "messages")+ require.Equal(t, false, request.body["store"])+ require.Equal(t, []any{"reasoning.encrypted_content"}, request.body["include"])+ } else {+ require.Contains(t, request.body, "messages")+ require.NotContains(t, request.body, "input")+ }+ })+ }+}++func TestChatAzureOpenAIResponsesRoundTripsAToolCall(t *testing.T) {+ t.Parallel()++ reasoning := map[string]any{+ "id": "rs_1",+ "type": "reasoning",+ "summary": []any{},+ "encrypted_content": "gAAAAB-encrypted-reasoning",+ "status": nil,+ }+ functionCall := map[string]any{+ "id": "fc_1",+ "type": "function_call",+ "call_id": "call_1",+ "name": "search",+ "arguments": `{"text":"orders"}`,+ "status": "completed",+ }+ toolCallReply, err := json.Marshal(map[string]any{+ "id": "resp_1",+ "object": "response",+ "created_at": 1741369938.0,+ "status": "completed",+ "error": nil,+ "incomplete_details": nil,+ "model": "gpt-5.6-terra",+ "output": []any{reasoning, functionCall},+ "parallel_tool_calls": true,+ "store": false,+ "usage": map[string]any{+ "input_tokens": 90,+ "input_tokens_details": map[string]any{"cached_tokens": 0},+ "output_tokens": 30,+ "output_tokens_details": map[string]any{"reasoning_tokens": 18},+ "total_tokens": 120,+ },+ "content_filters": []any{map[string]any{"source_type": "prompt", "blocked": false}},+ })+ require.NoError(t, err)+ server, requests := newAzureServer(t, string(toolCallReply), azureResponsesText)+ setting := azureSetting(server.URL + "/openai/v1/responses")+ messages := azureReviewMessages()+ tools := []*v1pb.AIChatToolDefinition{azureSearchTool()}++ first, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})+ require.NoError(t, err)+ require.Nil(t, first.Content)+ require.EqualValues(t, 120, first.GetUsage().GetTotalTokens())+ require.Len(t, first.ToolCalls, 1)+ require.Equal(t, "call_1", first.ToolCalls[0].GetId())+ require.Equal(t, "search", first.ToolCalls[0].GetName())+ require.Equal(t, `{"text":"orders"}`, first.ToolCalls[0].GetArguments())++ toolResult := `[{"kind": "table", "schema": "public", "name": "orders"}]`+ messages = append(messages,+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_ASSISTANT, ToolCalls: first.ToolCalls},+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_TOOL, Content: &toolResult, ToolCallId: &first.ToolCalls[0].Id},+ )+ second, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})+ require.NoError(t, err)+ require.Equal(t, `{"findings": []}`, second.GetContent())+ require.Empty(t, second.ToolCalls)++ require.Len(t, *requests, 2)+ for _, request := range *requests {+ requireAzureHeaders(t, request)+ require.Equal(t, "/openai/v1/responses", request.path)+ require.Empty(t, request.query)+ require.Equal(t, false, request.body["store"])+ require.Equal(t, []any{"reasoning.encrypted_content"}, request.body["include"])+ require.Equal(t, []any{map[string]any{+ "type": "function",+ "name": "search",+ "description": "Find objects by name.",+ "parameters": azureSearchParameters,+ }}, request.body["tools"])+ }+ conversation := []any{+ map[string]any{"role": "developer", "content": "You review SQL."},+ map[string]any{"role": "user", "content": "DROP TABLE orders;"},+ }+ require.Equal(t, conversation, (*requests)[0].body["input"])+ // With store false Azure keeps nothing, so every output item goes back as+ // received, the encrypted reasoning included.+ require.Equal(t, append(conversation,+ reasoning,+ functionCall,+ map[string]any{"type": "function_call_output", "call_id": "call_1", "output": toolResult},+ ), (*requests)[1].body["input"])+}++func TestChatAzureOpenAIChatCompletionsRoundTripsAToolCall(t *testing.T) {+ t.Parallel()++ toolCall := map[string]any{+ "id": "call_1",+ "type": "function",+ "function": map[string]any{"name": "search", "arguments": `{"text":"orders"}`},+ }+ toolCallReply, err := json.Marshal(map[string]any{+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "created": 1741369938,+ "model": "gpt-4o-2024-11-20",+ "prompt_filter_results": []any{map[string]any{+ "prompt_index": 0,+ "content_filter_results": map[string]any{"hate": map[string]any{"filtered": false, "severity": "safe"}},+ }},+ "choices": []any{map[string]any{+ "index": 0,+ "finish_reason": "tool_calls",+ "logprobs": nil,+ "message": map[string]any{+ "role": "assistant",+ "content": nil,+ "refusal": nil,+ "tool_calls": []any{toolCall},+ },+ "content_filter_results": map[string]any{},+ }},+ "usage": map[string]any{"completion_tokens": 20, "prompt_tokens": 100, "total_tokens": 120},+ })+ require.NoError(t, err)+ server, requests := newAzureServer(t, string(toolCallReply), azureChatCompletionsText)+ setting := azureSetting(server.URL + "/openai/v1/chat/completions")+ messages := azureReviewMessages()+ tools := []*v1pb.AIChatToolDefinition{azureSearchTool()}++ first, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})+ require.NoError(t, err)+ require.Nil(t, first.Content)+ require.EqualValues(t, 120, first.GetUsage().GetTotalTokens())+ require.Len(t, first.ToolCalls, 1)+ require.Equal(t, "call_1", first.ToolCalls[0].GetId())+ require.Equal(t, "search", first.ToolCalls[0].GetName())+ require.Equal(t, `{"text":"orders"}`, first.ToolCalls[0].GetArguments())++ toolResult := `[{"kind": "table", "schema": "public", "name": "orders"}]`+ messages = append(messages,+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_ASSISTANT, ToolCalls: first.ToolCalls},+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_TOOL, Content: &toolResult, ToolCallId: &first.ToolCalls[0].Id},+ )+ second, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})+ require.NoError(t, err)+ require.Equal(t, `{"findings": []}`, second.GetContent())+ require.Empty(t, second.ToolCalls)++ require.Len(t, *requests, 2)+ for _, request := range *requests {+ requireAzureHeaders(t, request)+ require.Equal(t, "/openai/v1/chat/completions", request.path)+ require.Empty(t, request.query)+ require.Equal(t, []any{map[string]any{+ "type": "function",+ "function": map[string]any{+ "name": "search",+ "description": "Find objects by name.",+ "parameters": azureSearchParameters,+ },+ }}, request.body["tools"])+ }+ conversation := []any{+ map[string]any{"role": "system", "content": "You review SQL."},+ map[string]any{"role": "user", "content": "DROP TABLE orders;"},+ }+ require.Equal(t, conversation, (*requests)[0].body["messages"])+ require.Equal(t, append(conversation,+ map[string]any{"role": "assistant", "content": nil, "tool_calls": []any{toolCall}},+ map[string]any{"role": "tool", "content": toolResult, "tool_call_id": "call_1"},+ ), (*requests)[1].body["messages"])+}++func TestChatAzureOpenAIReadsTheDocumentedFailures(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ status int+ header map[string]string+ body string+ want time.Duration+ }{+ {+ name: "rate limit with retry-after-ms",+ status: http.StatusTooManyRequests,+ header: map[string]string{"retry-after-ms": "2000", "x-ratelimit-remaining-requests": "0"},+ body: `{"error": {"code": "429", "message": "Requests to the Responses_Create Operation under Azure OpenAI API version v1 have been limited. Please retry after 2 seconds."}}`,+ want: 2 * time.Second,+ },+ {+ name: "capacity throttling with retry-after-ms",+ status: http.StatusTooManyRequests,+ header: map[string]string{"retry-after-ms": "500"},+ body: `{"error": {"code": "429", "message": "The service is temporarily unable to process your request. Please try again later."}}`,+ want: 500 * time.Millisecond,+ },+ {+ name: "prompt blocked by a guardrail",+ status: http.StatusBadRequest,+ body: `{"error": {"code": "content_filter", "message": "The response was filtered due to the prompt triggering content management policy."}}`,+ },+ }+ for _, endpoint := range []struct {+ kind string+ path string+ }{+ {kind: "responses", path: "/openai/v1/responses"},+ {kind: "chat completions", path: "/openai/v1/chat/completions"},+ } {+ for _, test := range tests {+ t.Run(fmt.Sprintf("%s %s", endpoint.kind, test.name), func(t *testing.T) {+ t.Parallel()++ server := failingProvider(t, test.status, test.header, test.body)+ _, err := Chat(context.Background(), azureSetting(server.URL+endpoint.path), &v1pb.AIChatRequest{Messages: azureReviewMessages()})++ var httpErr *HTTPError+ require.ErrorAs(t, err, &httpErr)+ require.Equal(t, HTTPError{+ Provider: "OpenAI",+ StatusCode: test.status,+ RetryAfter: test.want,+ Body: test.body,+ }, *httpErr, "every 429 Azure documents is cleared by a wait, so none is a used-up quota")+ })+ }+ }+}diff --git a/backend/plugin/aireview/model_ai_azure_openai_test.go b/backend/plugin/aireview/model_ai_azure_openai_test.gonew file mode 100644index 0000000000..5c6994e6aa--- /dev/null+++ b/backend/plugin/aireview/model_ai_azure_openai_test.go@@ -0,0 +1,178 @@+package aireview++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// azureOpenAIEndpoints are the endpoint paths the settings page offers for Azure OpenAI.+var azureOpenAIEndpoints = []struct {+ kind string+ path string+}{+ {kind: "responses", path: "/openai/v1/responses"},+ {kind: "chat completions", path: "/openai/v1/chat/completions"},+}++const (+ azureOpenAIDeployment = "review-deployment"++ azureResponsesToolCall = `{"id": "resp_1", "object": "response", "status": "completed", "error": null, "incomplete_details": null, "output": [{"id": "rs_1", "type": "reasoning", "summary": [], "encrypted_content": "gAAAAB-encrypted-reasoning", "status": null}, {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "search", "arguments": "{\"text\":\"orders\"}", "status": "completed"}], "usage": {"total_tokens": 40}, "content_filters": []}`+ azureResponsesPass = `{"id": "resp_2", "object": "response", "status": "completed", "error": null, "incomplete_details": null, "output": [{"id": "msg_2", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "{\"findings\": []}", "annotations": []}]}], "usage": {"total_tokens": 60}, "content_filters": []}`+ azureChatToolCall = `{"id": "chatcmpl-1", "object": "chat.completion", "prompt_filter_results": [], "choices": [{"index": 0, "finish_reason": "tool_calls", "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "search", "arguments": "{\"text\":\"orders\"}"}}]}, "content_filter_results": {}}], "usage": {"total_tokens": 40}}`+ azureChatPass = `{"id": "chatcmpl-2", "object": "chat.completion", "prompt_filter_results": [], "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "{\"findings\": []}"}, "content_filter_results": {}}], "usage": {"total_tokens": 60}}`++ azureRateLimit = `{"error": {"code": "429", "message": "Requests to the Responses_Create Operation under Azure OpenAI API version v1 have been limited. Please retry after 2 seconds."}}`+ azureContentFilter = `{"error": {"code": "content_filter", "message": "The response was filtered due to the prompt triggering content management policy."}}`+)++func azureOpenAISetting(endpoint string) *storepb.AISetting {+ return &storepb.AISetting{+ Provider: storepb.AISetting_AZURE_OPENAI,+ Endpoint: endpoint,+ Model: azureOpenAIDeployment,+ ApiKey: testAPIKey,+ }+}++// azureOpenAIRequest is the part of a request the tests assert on.+type azureOpenAIRequest struct {+ path string+ apiKey string+ body map[string]any+}++// The scripted Model in reviewer_test.go stands in for ai.Chat, so this test+// holds the loop to the real adapter's Azure OpenAI wire format.+func TestSettingModelRoundTripsThroughAzureOpenAI(t *testing.T) {+ t.Parallel()++ replies := map[string][]string{+ "responses": {azureResponsesToolCall, azureResponsesPass},+ "chat completions": {azureChatToolCall, azureChatPass},+ }+ for _, endpoint := range azureOpenAIEndpoints {+ t.Run(endpoint.kind, func(t *testing.T) {+ t.Parallel()++ var requests []azureOpenAIRequest+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ request := azureOpenAIRequest{path: r.URL.Path, apiKey: r.Header.Get("api-key")}+ if err := json.NewDecoder(r.Body).Decode(&request.body); err != nil {+ t.Errorf("failed to decode the request body: %v", err)+ }+ requests = append(requests, request)+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(replies[endpoint.kind][min(len(requests), 2)-1]))+ }))+ defer server.Close()++ model := NewModel(azureOpenAISetting(server.URL + endpoint.path))+ tools := &catalogTools{}+ result, err := NewReviewer(model).Review(context.Background(), &Request{Statement: "DROP TABLE orders;"}, tools)+ require.NoError(t, err)+ require.Empty(t, result.Findings)+ require.Equal(t, 2, result.Calls)+ require.Equal(t, 100, result.TotalTokens)+ require.Equal(t, []string{`search({"text":"orders"})`}, tools.calls)++ searchOutput, err := (&catalogTools{}).Call(context.Background(), "search", `{"text":"orders"}`)+ require.NoError(t, err)+ require.Len(t, requests, 2)+ for _, request := range requests {+ require.Equal(t, endpoint.path, request.path)+ require.Equal(t, testAPIKey, request.apiKey)+ require.Equal(t, azureOpenAIDeployment, request.body["model"])+ }+ if endpoint.kind == "responses" {+ input, ok := requests[1].body["input"].([]any)+ require.True(t, ok)+ require.Contains(t, input, map[string]any{"type": "reasoning", "id": "rs_1", "summary": []any{}, "encrypted_content": "gAAAAB-encrypted-reasoning", "status": nil})+ require.Equal(t, map[string]any{"type": "function_call_output", "call_id": "call_1", "output": searchOutput}, input[len(input)-1])+ } else {+ messages, ok := requests[1].body["messages"].([]any)+ require.True(t, ok)+ require.Equal(t, map[string]any{"role": "tool", "tool_call_id": "call_1", "content": searchOutput}, messages[len(messages)-1])+ }+ })+ }+}++func TestSettingModelRetriesAzureOpenAIRateLimitsButNotGuardrailBlocks(t *testing.T) {+ t.Parallel()++ success := map[string]string{"responses": azureResponsesPass, "chat completions": azureChatPass}+ for _, endpoint := range azureOpenAIEndpoints {+ t.Run(endpoint.kind+" rate limit waits retry-after-ms", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t,+ providerReply{status: http.StatusTooManyRequests, retryAfterMs: "2000", body: azureRateLimit},+ providerReply{status: http.StatusOK, body: success[endpoint.kind]})+ recorder := &sleepRecorder{}+ model := &settingModel{setting: azureOpenAISetting(provider.url + endpoint.path), sleep: recorder.sleep}+ response, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, `{"findings": []}`, response.GetContent())+ require.Equal(t, int32(2), provider.requests.Load())+ require.Equal(t, []time.Duration{2 * time.Second}, recorder.waits)+ })++ t.Run(endpoint.kind+" prompt blocked by a guardrail", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{status: http.StatusBadRequest, body: azureContentFilter})+ recorder := &sleepRecorder{}+ model := &settingModel{setting: azureOpenAISetting(provider.url + endpoint.path), sleep: recorder.sleep}+ _, err := model.Chat(context.Background(), chatRequest())+ require.EqualError(t, err, "OpenAI API returned status 400: "+azureContentFilter)+ require.Equal(t, int32(1), provider.requests.Load())+ require.Empty(t, recorder.waits)+ })+ }+}++func TestReviewFailsOnAnAzureOpenAIReplyThatWasCutShort(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ path string+ reply string+ }{+ {+ name: "responses out of output tokens before any text",+ path: "/openai/v1/responses",+ reply: `{"id": "resp_1", "object": "response", "status": "incomplete", "error": null, "incomplete_details": {"reason": "max_output_tokens"}, "output": [{"id": "rs_1", "type": "reasoning", "summary": [], "encrypted_content": "gAAAAB", "status": null}], "usage": {"total_tokens": 25000}}`,+ },+ {+ name: "responses output filtered after text that would pass",+ path: "/openai/v1/responses",+ reply: `{"id": "resp_1", "object": "response", "status": "incomplete", "error": null, "incomplete_details": {"reason": "content_filter"}, "output": [{"id": "msg_1", "type": "message", "role": "assistant", "status": "incomplete", "content": [{"type": "output_text", "text": "{\"findings\": []}", "annotations": []}]}], "usage": {"total_tokens": 60}}`,+ },+ {+ name: "chat completions output filtered",+ path: "/openai/v1/chat/completions",+ reply: `{"id": "chatcmpl-1", "object": "chat.completion", "choices": [{"index": 0, "finish_reason": "content_filter", "message": {"role": "assistant", "content": null}, "content_filter_results": {"violence": {"filtered": true, "severity": "high"}}}], "usage": {"total_tokens": 60}}`,+ },+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{status: http.StatusOK, body: test.reply})+ model := NewModel(azureOpenAISetting(provider.url + test.path))+ result, err := NewReviewer(model).Review(context.Background(), &Request{Statement: "DROP TABLE orders;"}, &catalogTools{})+ require.Error(t, err)+ require.Nil(t, result)+ })+ }+}diff --git a/backend/plugin/aireview/model_ai_live_azure_openai_test.go b/backend/plugin/aireview/model_ai_live_azure_openai_test.gonew file mode 100644index 0000000000..dfb17b459b--- /dev/null+++ b/backend/plugin/aireview/model_ai_live_azure_openai_test.go@@ -0,0 +1,72 @@+package aireview++import (+ "context"+ "os"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// TestReviewLiveAzureOpenAI reviews a risky change with a real Azure OpenAI+// deployment, once through each endpoint the settings page offers. It needs a+// resource that CI does not have. The endpoint is the resource's, without a path:+//+// AIREVIEW_LIVE_AZURE_OPENAI_ENDPOINT=https://YOUR-RESOURCE-NAME.openai.azure.com \+// AIREVIEW_LIVE_AZURE_OPENAI_API_KEY=... \+// AIREVIEW_LIVE_AZURE_OPENAI_DEPLOYMENT=... \+// go test ./backend/plugin/aireview/ -run '^TestReviewLiveAzureOpenAI$' -v -count=1+//+// Azure rejects tools on chat completions for a gpt-5.6 deployment, so run that+// case with a deployment such as gpt-4o and -run '^TestReviewLiveAzureOpenAI$/^chat_completions$'.+func TestReviewLiveAzureOpenAI(t *testing.T) {+ resource := os.Getenv("AIREVIEW_LIVE_AZURE_OPENAI_ENDPOINT")+ apiKey := os.Getenv("AIREVIEW_LIVE_AZURE_OPENAI_API_KEY")+ deployment := os.Getenv("AIREVIEW_LIVE_AZURE_OPENAI_DEPLOYMENT")+ if resource == "" || apiKey == "" || deployment == "" {+ t.Skip("AIREVIEW_LIVE_AZURE_OPENAI_ENDPOINT, AIREVIEW_LIVE_AZURE_OPENAI_API_KEY, and AIREVIEW_LIVE_AZURE_OPENAI_DEPLOYMENT are not all set")+ }++ statement := strings.Join([]string{+ "-- Speed up the order history page.",+ "CREATE INDEX idx_orders_created_at ON orders (created_at);",+ "",+ "ALTER TABLE orders DROP COLUMN legacy_status;",+ }, "\n")+ request := &Request{+ WorkspacePolicy: "An index on a table with more than 1 million rows must be created with CONCURRENTLY.",+ Target: Target{+ Engine: "POSTGRES",+ Version: "16.2",+ Environment: "prod",+ Schemas: []SchemaSummary{{Name: "public", ObjectCount: 2}},+ },+ Statement: statement,+ }+ for _, endpoint := range azureOpenAIEndpoints {+ t.Run(endpoint.kind, func(t *testing.T) {+ model := NewModel(&storepb.AISetting{+ Provider: storepb.AISetting_AZURE_OPENAI,+ Endpoint: strings.TrimRight(resource, "/") + endpoint.path,+ Model: deployment,+ ApiKey: apiKey,+ })+ tools := &catalogTools{}++ result, err := NewReviewer(model).Review(context.Background(), request, tools)+ require.NoError(t, err)+ t.Logf("model calls: %d, tokens: %d, tool calls: %v", result.Calls, result.TotalTokens, tools.calls)+ for _, finding := range result.Findings {+ t.Logf("%s line %d: %s\n rule: %s\n evidence: %s\n fix: %s", finding.Severity, finding.Line, finding.Title, finding.Rule, finding.Evidence, finding.Fix)+ }+ for _, note := range result.Notes {+ t.Logf("note: %s", note)+ }+ require.NotEmpty(t, tools.calls, "the model should look up the objects before it judges")+ require.NotEmpty(t, result.Findings)+ })+ }+}