Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

test(review): cover the AI review on Azure OpenAI (#21598)

h3n4l08aca2d2beba
diff --git a/backend/plugin/ai/openai_azure_test.go b/backend/plugin/ai/openai_azure_test.go
new file mode 100644
index 0000000000..7b212855eb
--- /dev/null
+++ b/backend/plugin/ai/openai_azure_test.go
@@ -0,0 +1,452 @@
+package ai
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// The requests and replies in this file follow the Azure OpenAI v1 API pages on
+// learn.microsoft.com: api-version-lifecycle, how-to/responses,
+// how-to/function-calling, how-to/reasoning, and how-to/quota.
+
+const (
+ azureDeployment = "review-deployment"
+ azureAPIKey = "azure-test-key"
+)
+
+// azureRequest is one request the fake Azure resource received.
+type azureRequest struct {
+ method string
+ path string
+ query string
+ apiKey string
+ auth string
+ contentType string
+ body map[string]any
+}
+
+// newAzureServer records each request and answers it with the next reply.
+func newAzureServer(t *testing.T, replies ...string) (*httptest.Server, *[]azureRequest) {
+ t.Helper()
+ var requests []azureRequest
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ request := azureRequest{
+ method: r.Method,
+ path: r.URL.Path,
+ query: r.URL.RawQuery,
+ apiKey: r.Header.Get("api-key"),
+ auth: r.Header.Get("Authorization"),
+ contentType: r.Header.Get("Content-Type"),
+ }
+ if err := json.NewDecoder(r.Body).Decode(&request.body); err != nil {
+ t.Errorf("failed to decode the request body: %v", err)
+ }
+ requests = append(requests, request)
+ if len(requests) > len(replies) {
+ t.Errorf("unexpected request %d", len(requests))
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(replies[len(requests)-1]))
+ }))
+ t.Cleanup(server.Close)
+ return server, &requests
+}
+
+func azureSetting(endpoint string) *storepb.AISetting {
+ return &storepb.AISetting{
+ Provider: storepb.AISetting_AZURE_OPENAI,
+ Endpoint: endpoint,
+ Model: azureDeployment,
+ ApiKey: azureAPIKey,
+ }
+}
+
+func azureSearchTool() *v1pb.AIChatToolDefinition {
+ return &v1pb.AIChatToolDefinition{
+ Name: "search",
+ Description: "Find objects by name.",
+ ParametersSchema: `{"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]}`,
+ }
+}
+
+var azureSearchParameters = map[string]any{
+ "type": "object",
+ "properties": map[string]any{"text": map[string]any{"type": "string"}},
+ "required": []any{"text"},
+}
+
+func azureReviewMessages() []*v1pb.AIChatMessage {
+ system := "You review SQL."
+ user := "DROP TABLE orders;"
+ return []*v1pb.AIChatMessage{
+ {Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_SYSTEM, Content: &system},
+ {Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_USER, Content: &user},
+ }
+}
+
+// requireAzureHeaders asserts what Azure documents for every call: a POST with
+// the key in the api-key header, a JSON body, and the deployment name as the model.
+func requireAzureHeaders(t *testing.T, request azureRequest) {
+ t.Helper()
+ require.Equal(t, http.MethodPost, request.method)
+ require.Equal(t, azureAPIKey, request.apiKey)
+ require.Empty(t, request.auth)
+ require.Equal(t, "application/json", request.contentType)
+ require.Equal(t, azureDeployment, request.body["model"])
+}
+
+const (
+ azureResponsesText = `{
+ "id": "resp_2",
+ "object": "response",
+ "created_at": 1741369940.0,
+ "status": "completed",
+ "error": null,
+ "incomplete_details": null,
+ "model": "gpt-5.6-terra",
+ "output": [{
+ "id": "msg_2",
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{"type": "output_text", "text": "{\"findings\": []}", "annotations": [], "logprobs": null}]
+ }],
+ "parallel_tool_calls": true,
+ "store": false,
+ "usage": {"input_tokens": 140, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 10, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 150},
+ "content_filters": [{"source_type": "prompt", "blocked": false, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}}}]
+ }`
+ azureChatCompletionsText = `{
+ "id": "chatcmpl-2",
+ "object": "chat.completion",
+ "created": 1741369940,
+ "model": "gpt-4o-2024-11-20",
+ "prompt_filter_results": [{"prompt_index": 0, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}}}],
+ "choices": [{
+ "index": 0,
+ "finish_reason": "stop",
+ "logprobs": null,
+ "message": {"role": "assistant", "content": "{\"findings\": []}", "refusal": null},
+ "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}}
+ }],
+ "usage": {"completion_tokens": 10, "prompt_tokens": 140, "total_tokens": 150}
+ }`
+)
+
+func TestChatAzureOpenAISendsTheDocumentedRequest(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ endpoint string
+ wantPath string
+ wantQuery string
+ wantResponses bool
+ }{
+ {
+ name: "v1 responses, the default the settings page offers",
+ endpoint: "/openai/v1/responses",
+ wantPath: "/openai/v1/responses",
+ wantResponses: true,
+ },
+ {
+ name: "v1 chat completions",
+ endpoint: "/openai/v1/chat/completions",
+ wantPath: "/openai/v1/chat/completions",
+ },
+ {
+ name: "v1 responses with the optional api-version",
+ endpoint: "/openai/v1/responses?api-version=preview",
+ wantPath: "/openai/v1/responses",
+ wantQuery: "api-version=preview",
+ wantResponses: true,
+ },
+ {
+ name: "dated chat completions with the deployment in the path",
+ endpoint: "/openai/deployments/review-deployment/chat/completions?api-version=2024-10-21",
+ wantPath: "/openai/deployments/review-deployment/chat/completions",
+ wantQuery: "api-version=2024-10-21",
+ },
+ {
+ name: "dated responses",
+ endpoint: "/openai/responses?api-version=2025-04-01-preview",
+ wantPath: "/openai/responses",
+ wantQuery: "api-version=2025-04-01-preview",
+ wantResponses: true,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ t.Parallel()
+
+ reply := azureChatCompletionsText
+ if test.wantResponses {
+ reply = azureResponsesText
+ }
+ server, requests := newAzureServer(t, reply)
+ response, err := Chat(context.Background(), azureSetting(server.URL+test.endpoint), &v1pb.AIChatRequest{
+ Messages: azureReviewMessages(),
+ ToolDefinitions: []*v1pb.AIChatToolDefinition{azureSearchTool()},
+ })
+ require.NoError(t, err)
+ require.Equal(t, `{"findings": []}`, response.GetContent())
+ require.EqualValues(t, 150, response.GetUsage().GetTotalTokens())
+
+ require.Len(t, *requests, 1)
+ request := (*requests)[0]
+ requireAzureHeaders(t, request)
+ require.Equal(t, test.wantPath, request.path)
+ require.Equal(t, test.wantQuery, request.query)
+ if test.wantResponses {
+ require.Contains(t, request.body, "input")
+ require.NotContains(t, request.body, "messages")
+ require.Equal(t, false, request.body["store"])
+ require.Equal(t, []any{"reasoning.encrypted_content"}, request.body["include"])
+ } else {
+ require.Contains(t, request.body, "messages")
+ require.NotContains(t, request.body, "input")
+ }
+ })
+ }
+}
+
+func TestChatAzureOpenAIResponsesRoundTripsAToolCall(t *testing.T) {
+ t.Parallel()
+
+ reasoning := map[string]any{
+ "id": "rs_1",
+ "type": "reasoning",
+ "summary": []any{},
+ "encrypted_content": "gAAAAB-encrypted-reasoning",
+ "status": nil,
+ }
+ functionCall := map[string]any{
+ "id": "fc_1",
+ "type": "function_call",
+ "call_id": "call_1",
+ "name": "search",
+ "arguments": `{"text":"orders"}`,
+ "status": "completed",
+ }
+ toolCallReply, err := json.Marshal(map[string]any{
+ "id": "resp_1",
+ "object": "response",
+ "created_at": 1741369938.0,
+ "status": "completed",
+ "error": nil,
+ "incomplete_details": nil,
+ "model": "gpt-5.6-terra",
+ "output": []any{reasoning, functionCall},
+ "parallel_tool_calls": true,
+ "store": false,
+ "usage": map[string]any{
+ "input_tokens": 90,
+ "input_tokens_details": map[string]any{"cached_tokens": 0},
+ "output_tokens": 30,
+ "output_tokens_details": map[string]any{"reasoning_tokens": 18},
+ "total_tokens": 120,
+ },
+ "content_filters": []any{map[string]any{"source_type": "prompt", "blocked": false}},
+ })
+ require.NoError(t, err)
+ server, requests := newAzureServer(t, string(toolCallReply), azureResponsesText)
+ setting := azureSetting(server.URL + "/openai/v1/responses")
+ messages := azureReviewMessages()
+ tools := []*v1pb.AIChatToolDefinition{azureSearchTool()}
+
+ first, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})
+ require.NoError(t, err)
+ require.Nil(t, first.Content)
+ require.EqualValues(t, 120, first.GetUsage().GetTotalTokens())
+ require.Len(t, first.ToolCalls, 1)
+ require.Equal(t, "call_1", first.ToolCalls[0].GetId())
+ require.Equal(t, "search", first.ToolCalls[0].GetName())
+ require.Equal(t, `{"text":"orders"}`, first.ToolCalls[0].GetArguments())
+
+ toolResult := `[{"kind": "table", "schema": "public", "name": "orders"}]`
+ messages = append(messages,
+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_ASSISTANT, ToolCalls: first.ToolCalls},
+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_TOOL, Content: &toolResult, ToolCallId: &first.ToolCalls[0].Id},
+ )
+ second, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})
+ require.NoError(t, err)
+ require.Equal(t, `{"findings": []}`, second.GetContent())
+ require.Empty(t, second.ToolCalls)
+
+ require.Len(t, *requests, 2)
+ for _, request := range *requests {
+ requireAzureHeaders(t, request)
+ require.Equal(t, "/openai/v1/responses", request.path)
+ require.Empty(t, request.query)
+ require.Equal(t, false, request.body["store"])
+ require.Equal(t, []any{"reasoning.encrypted_content"}, request.body["include"])
+ require.Equal(t, []any{map[string]any{
+ "type": "function",
+ "name": "search",
+ "description": "Find objects by name.",
+ "parameters": azureSearchParameters,
+ }}, request.body["tools"])
+ }
+ conversation := []any{
+ map[string]any{"role": "developer", "content": "You review SQL."},
+ map[string]any{"role": "user", "content": "DROP TABLE orders;"},
+ }
+ require.Equal(t, conversation, (*requests)[0].body["input"])
+ // With store false Azure keeps nothing, so every output item goes back as
+ // received, the encrypted reasoning included.
+ require.Equal(t, append(conversation,
+ reasoning,
+ functionCall,
+ map[string]any{"type": "function_call_output", "call_id": "call_1", "output": toolResult},
+ ), (*requests)[1].body["input"])
+}
+
+func TestChatAzureOpenAIChatCompletionsRoundTripsAToolCall(t *testing.T) {
+ t.Parallel()
+
+ toolCall := map[string]any{
+ "id": "call_1",
+ "type": "function",
+ "function": map[string]any{"name": "search", "arguments": `{"text":"orders"}`},
+ }
+ toolCallReply, err := json.Marshal(map[string]any{
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "created": 1741369938,
+ "model": "gpt-4o-2024-11-20",
+ "prompt_filter_results": []any{map[string]any{
+ "prompt_index": 0,
+ "content_filter_results": map[string]any{"hate": map[string]any{"filtered": false, "severity": "safe"}},
+ }},
+ "choices": []any{map[string]any{
+ "index": 0,
+ "finish_reason": "tool_calls",
+ "logprobs": nil,
+ "message": map[string]any{
+ "role": "assistant",
+ "content": nil,
+ "refusal": nil,
+ "tool_calls": []any{toolCall},
+ },
+ "content_filter_results": map[string]any{},
+ }},
+ "usage": map[string]any{"completion_tokens": 20, "prompt_tokens": 100, "total_tokens": 120},
+ })
+ require.NoError(t, err)
+ server, requests := newAzureServer(t, string(toolCallReply), azureChatCompletionsText)
+ setting := azureSetting(server.URL + "/openai/v1/chat/completions")
+ messages := azureReviewMessages()
+ tools := []*v1pb.AIChatToolDefinition{azureSearchTool()}
+
+ first, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})
+ require.NoError(t, err)
+ require.Nil(t, first.Content)
+ require.EqualValues(t, 120, first.GetUsage().GetTotalTokens())
+ require.Len(t, first.ToolCalls, 1)
+ require.Equal(t, "call_1", first.ToolCalls[0].GetId())
+ require.Equal(t, "search", first.ToolCalls[0].GetName())
+ require.Equal(t, `{"text":"orders"}`, first.ToolCalls[0].GetArguments())
+
+ toolResult := `[{"kind": "table", "schema": "public", "name": "orders"}]`
+ messages = append(messages,
+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_ASSISTANT, ToolCalls: first.ToolCalls},
+ &v1pb.AIChatMessage{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_TOOL, Content: &toolResult, ToolCallId: &first.ToolCalls[0].Id},
+ )
+ second, err := Chat(context.Background(), setting, &v1pb.AIChatRequest{Messages: messages, ToolDefinitions: tools})
+ require.NoError(t, err)
+ require.Equal(t, `{"findings": []}`, second.GetContent())
+ require.Empty(t, second.ToolCalls)
+
+ require.Len(t, *requests, 2)
+ for _, request := range *requests {
+ requireAzureHeaders(t, request)
+ require.Equal(t, "/openai/v1/chat/completions", request.path)
+ require.Empty(t, request.query)
+ require.Equal(t, []any{map[string]any{
+ "type": "function",
+ "function": map[string]any{
+ "name": "search",
+ "description": "Find objects by name.",
+ "parameters": azureSearchParameters,
+ },
+ }}, request.body["tools"])
+ }
+ conversation := []any{
+ map[string]any{"role": "system", "content": "You review SQL."},
+ map[string]any{"role": "user", "content": "DROP TABLE orders;"},
+ }
+ require.Equal(t, conversation, (*requests)[0].body["messages"])
+ require.Equal(t, append(conversation,
+ map[string]any{"role": "assistant", "content": nil, "tool_calls": []any{toolCall}},
+ map[string]any{"role": "tool", "content": toolResult, "tool_call_id": "call_1"},
+ ), (*requests)[1].body["messages"])
+}
+
+func TestChatAzureOpenAIReadsTheDocumentedFailures(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ status int
+ header map[string]string
+ body string
+ want time.Duration
+ }{
+ {
+ name: "rate limit with retry-after-ms",
+ status: http.StatusTooManyRequests,
+ header: map[string]string{"retry-after-ms": "2000", "x-ratelimit-remaining-requests": "0"},
+ body: `{"error": {"code": "429", "message": "Requests to the Responses_Create Operation under Azure OpenAI API version v1 have been limited. Please retry after 2 seconds."}}`,
+ want: 2 * time.Second,
+ },
+ {
+ name: "capacity throttling with retry-after-ms",
+ status: http.StatusTooManyRequests,
+ header: map[string]string{"retry-after-ms": "500"},
+ body: `{"error": {"code": "429", "message": "The service is temporarily unable to process your request. Please try again later."}}`,
+ want: 500 * time.Millisecond,
+ },
+ {
+ name: "prompt blocked by a guardrail",
+ status: http.StatusBadRequest,
+ body: `{"error": {"code": "content_filter", "message": "The response was filtered due to the prompt triggering content management policy."}}`,
+ },
+ }
+ for _, endpoint := range []struct {
+ kind string
+ path string
+ }{
+ {kind: "responses", path: "/openai/v1/responses"},
+ {kind: "chat completions", path: "/openai/v1/chat/completions"},
+ } {
+ for _, test := range tests {
+ t.Run(fmt.Sprintf("%s %s", endpoint.kind, test.name), func(t *testing.T) {
+ t.Parallel()
+
+ server := failingProvider(t, test.status, test.header, test.body)
+ _, err := Chat(context.Background(), azureSetting(server.URL+endpoint.path), &v1pb.AIChatRequest{Messages: azureReviewMessages()})
+
+ var httpErr *HTTPError
+ require.ErrorAs(t, err, &httpErr)
+ require.Equal(t, HTTPError{
+ Provider: "OpenAI",
+ StatusCode: test.status,
+ RetryAfter: test.want,
+ Body: test.body,
+ }, *httpErr, "every 429 Azure documents is cleared by a wait, so none is a used-up quota")
+ })
+ }
+ }
+}
diff --git a/backend/plugin/aireview/model_ai_azure_openai_test.go b/backend/plugin/aireview/model_ai_azure_openai_test.go
new file mode 100644
index 0000000000..5c6994e6aa
--- /dev/null
+++ b/backend/plugin/aireview/model_ai_azure_openai_test.go
@@ -0,0 +1,178 @@
+package aireview
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// azureOpenAIEndpoints are the endpoint paths the settings page offers for Azure OpenAI.
+var azureOpenAIEndpoints = []struct {
+ kind string
+ path string
+}{
+ {kind: "responses", path: "/openai/v1/responses"},
+ {kind: "chat completions", path: "/openai/v1/chat/completions"},
+}
+
+const (
+ azureOpenAIDeployment = "review-deployment"
+
+ azureResponsesToolCall = `{"id": "resp_1", "object": "response", "status": "completed", "error": null, "incomplete_details": null, "output": [{"id": "rs_1", "type": "reasoning", "summary": [], "encrypted_content": "gAAAAB-encrypted-reasoning", "status": null}, {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "search", "arguments": "{\"text\":\"orders\"}", "status": "completed"}], "usage": {"total_tokens": 40}, "content_filters": []}`
+ azureResponsesPass = `{"id": "resp_2", "object": "response", "status": "completed", "error": null, "incomplete_details": null, "output": [{"id": "msg_2", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "{\"findings\": []}", "annotations": []}]}], "usage": {"total_tokens": 60}, "content_filters": []}`
+ azureChatToolCall = `{"id": "chatcmpl-1", "object": "chat.completion", "prompt_filter_results": [], "choices": [{"index": 0, "finish_reason": "tool_calls", "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "search", "arguments": "{\"text\":\"orders\"}"}}]}, "content_filter_results": {}}], "usage": {"total_tokens": 40}}`
+ azureChatPass = `{"id": "chatcmpl-2", "object": "chat.completion", "prompt_filter_results": [], "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "{\"findings\": []}"}, "content_filter_results": {}}], "usage": {"total_tokens": 60}}`
+
+ azureRateLimit = `{"error": {"code": "429", "message": "Requests to the Responses_Create Operation under Azure OpenAI API version v1 have been limited. Please retry after 2 seconds."}}`
+ azureContentFilter = `{"error": {"code": "content_filter", "message": "The response was filtered due to the prompt triggering content management policy."}}`
+)
+
+func azureOpenAISetting(endpoint string) *storepb.AISetting {
+ return &storepb.AISetting{
+ Provider: storepb.AISetting_AZURE_OPENAI,
+ Endpoint: endpoint,
+ Model: azureOpenAIDeployment,
+ ApiKey: testAPIKey,
+ }
+}
+
+// azureOpenAIRequest is the part of a request the tests assert on.
+type azureOpenAIRequest struct {
+ path string
+ apiKey string
+ body map[string]any
+}
+
+// The scripted Model in reviewer_test.go stands in for ai.Chat, so this test
+// holds the loop to the real adapter's Azure OpenAI wire format.
+func TestSettingModelRoundTripsThroughAzureOpenAI(t *testing.T) {
+ t.Parallel()
+
+ replies := map[string][]string{
+ "responses": {azureResponsesToolCall, azureResponsesPass},
+ "chat completions": {azureChatToolCall, azureChatPass},
+ }
+ for _, endpoint := range azureOpenAIEndpoints {
+ t.Run(endpoint.kind, func(t *testing.T) {
+ t.Parallel()
+
+ var requests []azureOpenAIRequest
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ request := azureOpenAIRequest{path: r.URL.Path, apiKey: r.Header.Get("api-key")}
+ if err := json.NewDecoder(r.Body).Decode(&request.body); err != nil {
+ t.Errorf("failed to decode the request body: %v", err)
+ }
+ requests = append(requests, request)
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(replies[endpoint.kind][min(len(requests), 2)-1]))
+ }))
+ defer server.Close()
+
+ model := NewModel(azureOpenAISetting(server.URL + endpoint.path))
+ tools := &catalogTools{}
+ result, err := NewReviewer(model).Review(context.Background(), &Request{Statement: "DROP TABLE orders;"}, tools)
+ require.NoError(t, err)
+ require.Empty(t, result.Findings)
+ require.Equal(t, 2, result.Calls)
+ require.Equal(t, 100, result.TotalTokens)
+ require.Equal(t, []string{`search({"text":"orders"})`}, tools.calls)
+
+ searchOutput, err := (&catalogTools{}).Call(context.Background(), "search", `{"text":"orders"}`)
+ require.NoError(t, err)
+ require.Len(t, requests, 2)
+ for _, request := range requests {
+ require.Equal(t, endpoint.path, request.path)
+ require.Equal(t, testAPIKey, request.apiKey)
+ require.Equal(t, azureOpenAIDeployment, request.body["model"])
+ }
+ if endpoint.kind == "responses" {
+ input, ok := requests[1].body["input"].([]any)
+ require.True(t, ok)
+ require.Contains(t, input, map[string]any{"type": "reasoning", "id": "rs_1", "summary": []any{}, "encrypted_content": "gAAAAB-encrypted-reasoning", "status": nil})
+ require.Equal(t, map[string]any{"type": "function_call_output", "call_id": "call_1", "output": searchOutput}, input[len(input)-1])
+ } else {
+ messages, ok := requests[1].body["messages"].([]any)
+ require.True(t, ok)
+ require.Equal(t, map[string]any{"role": "tool", "tool_call_id": "call_1", "content": searchOutput}, messages[len(messages)-1])
+ }
+ })
+ }
+}
+
+func TestSettingModelRetriesAzureOpenAIRateLimitsButNotGuardrailBlocks(t *testing.T) {
+ t.Parallel()
+
+ success := map[string]string{"responses": azureResponsesPass, "chat completions": azureChatPass}
+ for _, endpoint := range azureOpenAIEndpoints {
+ t.Run(endpoint.kind+" rate limit waits retry-after-ms", func(t *testing.T) {
+ t.Parallel()
+
+ provider := newScriptedProvider(t,
+ providerReply{status: http.StatusTooManyRequests, retryAfterMs: "2000", body: azureRateLimit},
+ providerReply{status: http.StatusOK, body: success[endpoint.kind]})
+ recorder := &sleepRecorder{}
+ model := &settingModel{setting: azureOpenAISetting(provider.url + endpoint.path), sleep: recorder.sleep}
+ response, err := model.Chat(context.Background(), chatRequest())
+ require.NoError(t, err)
+ require.Equal(t, `{"findings": []}`, response.GetContent())
+ require.Equal(t, int32(2), provider.requests.Load())
+ require.Equal(t, []time.Duration{2 * time.Second}, recorder.waits)
+ })
+
+ t.Run(endpoint.kind+" prompt blocked by a guardrail", func(t *testing.T) {
+ t.Parallel()
+
+ provider := newScriptedProvider(t, providerReply{status: http.StatusBadRequest, body: azureContentFilter})
+ recorder := &sleepRecorder{}
+ model := &settingModel{setting: azureOpenAISetting(provider.url + endpoint.path), sleep: recorder.sleep}
+ _, err := model.Chat(context.Background(), chatRequest())
+ require.EqualError(t, err, "OpenAI API returned status 400: "+azureContentFilter)
+ require.Equal(t, int32(1), provider.requests.Load())
+ require.Empty(t, recorder.waits)
+ })
+ }
+}
+
+func TestReviewFailsOnAnAzureOpenAIReplyThatWasCutShort(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ path string
+ reply string
+ }{
+ {
+ name: "responses out of output tokens before any text",
+ path: "/openai/v1/responses",
+ reply: `{"id": "resp_1", "object": "response", "status": "incomplete", "error": null, "incomplete_details": {"reason": "max_output_tokens"}, "output": [{"id": "rs_1", "type": "reasoning", "summary": [], "encrypted_content": "gAAAAB", "status": null}], "usage": {"total_tokens": 25000}}`,
+ },
+ {
+ name: "responses output filtered after text that would pass",
+ path: "/openai/v1/responses",
+ reply: `{"id": "resp_1", "object": "response", "status": "incomplete", "error": null, "incomplete_details": {"reason": "content_filter"}, "output": [{"id": "msg_1", "type": "message", "role": "assistant", "status": "incomplete", "content": [{"type": "output_text", "text": "{\"findings\": []}", "annotations": []}]}], "usage": {"total_tokens": 60}}`,
+ },
+ {
+ name: "chat completions output filtered",
+ path: "/openai/v1/chat/completions",
+ reply: `{"id": "chatcmpl-1", "object": "chat.completion", "choices": [{"index": 0, "finish_reason": "content_filter", "message": {"role": "assistant", "content": null}, "content_filter_results": {"violence": {"filtered": true, "severity": "high"}}}], "usage": {"total_tokens": 60}}`,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ t.Parallel()
+
+ provider := newScriptedProvider(t, providerReply{status: http.StatusOK, body: test.reply})
+ model := NewModel(azureOpenAISetting(provider.url + test.path))
+ result, err := NewReviewer(model).Review(context.Background(), &Request{Statement: "DROP TABLE orders;"}, &catalogTools{})
+ require.Error(t, err)
+ require.Nil(t, result)
+ })
+ }
+}
diff --git a/backend/plugin/aireview/model_ai_live_azure_openai_test.go b/backend/plugin/aireview/model_ai_live_azure_openai_test.go
new file mode 100644
index 0000000000..dfb17b459b
--- /dev/null
+++ b/backend/plugin/aireview/model_ai_live_azure_openai_test.go
@@ -0,0 +1,72 @@
+package aireview
+
+import (
+ "context"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// TestReviewLiveAzureOpenAI reviews a risky change with a real Azure OpenAI
+// deployment, once through each endpoint the settings page offers. It needs a
+// resource that CI does not have. The endpoint is the resource's, without a path:
+//
+// AIREVIEW_LIVE_AZURE_OPENAI_ENDPOINT=https://YOUR-RESOURCE-NAME.openai.azure.com \
+// AIREVIEW_LIVE_AZURE_OPENAI_API_KEY=... \
+// AIREVIEW_LIVE_AZURE_OPENAI_DEPLOYMENT=... \
+// go test ./backend/plugin/aireview/ -run '^TestReviewLiveAzureOpenAI$' -v -count=1
+//
+// Azure rejects tools on chat completions for a gpt-5.6 deployment, so run that
+// case with a deployment such as gpt-4o and -run '^TestReviewLiveAzureOpenAI$/^chat_completions$'.
+func TestReviewLiveAzureOpenAI(t *testing.T) {
+ resource := os.Getenv("AIREVIEW_LIVE_AZURE_OPENAI_ENDPOINT")
+ apiKey := os.Getenv("AIREVIEW_LIVE_AZURE_OPENAI_API_KEY")
+ deployment := os.Getenv("AIREVIEW_LIVE_AZURE_OPENAI_DEPLOYMENT")
+ if resource == "" || apiKey == "" || deployment == "" {
+ t.Skip("AIREVIEW_LIVE_AZURE_OPENAI_ENDPOINT, AIREVIEW_LIVE_AZURE_OPENAI_API_KEY, and AIREVIEW_LIVE_AZURE_OPENAI_DEPLOYMENT are not all set")
+ }
+
+ statement := strings.Join([]string{
+ "-- Speed up the order history page.",
+ "CREATE INDEX idx_orders_created_at ON orders (created_at);",
+ "",
+ "ALTER TABLE orders DROP COLUMN legacy_status;",
+ }, "\n")
+ request := &Request{
+ WorkspacePolicy: "An index on a table with more than 1 million rows must be created with CONCURRENTLY.",
+ Target: Target{
+ Engine: "POSTGRES",
+ Version: "16.2",
+ Environment: "prod",
+ Schemas: []SchemaSummary{{Name: "public", ObjectCount: 2}},
+ },
+ Statement: statement,
+ }
+ for _, endpoint := range azureOpenAIEndpoints {
+ t.Run(endpoint.kind, func(t *testing.T) {
+ model := NewModel(&storepb.AISetting{
+ Provider: storepb.AISetting_AZURE_OPENAI,
+ Endpoint: strings.TrimRight(resource, "/") + endpoint.path,
+ Model: deployment,
+ ApiKey: apiKey,
+ })
+ tools := &catalogTools{}
+
+ result, err := NewReviewer(model).Review(context.Background(), request, tools)
+ require.NoError(t, err)
+ t.Logf("model calls: %d, tokens: %d, tool calls: %v", result.Calls, result.TotalTokens, tools.calls)
+ for _, finding := range result.Findings {
+ t.Logf("%s line %d: %s\n rule: %s\n evidence: %s\n fix: %s", finding.Severity, finding.Line, finding.Title, finding.Rule, finding.Evidence, finding.Fix)
+ }
+ for _, note := range result.Notes {
+ t.Logf("note: %s", note)
+ }
+ require.NotEmpty(t, tools.calls, "the model should look up the objects before it judges")
+ require.NotEmpty(t, result.Findings)
+ })
+ }
+}