Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(assistant): send user feedback to Braintrust traces (#43021)

Matt Rossman20d1ac0f83c3
diff --git a/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx b/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx
index 930fbb33a4b..9916de1310f 100644
--- a/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx
+++ b/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx
@@ -231,6 +231,7 @@ export const AIAssistant = ({ className }: AIAssistantProps) => {
projectRef: project.ref,
orgSlug: selectedOrganization.slug,
reason,
+ spanId: state.messageSpanIds[messageId],
})
sendEvent({
@@ -254,7 +255,7 @@ export const AIAssistant = ({ className }: AIAssistantProps) => {
})
}
},
- [chatMessages, project?.ref, selectedOrganization?.slug, rateMessage, sendEvent]
+ [chatMessages, project?.ref, selectedOrganization?.slug, rateMessage, sendEvent, state]
)
const isContextExceededError =
diff --git a/apps/studio/data/ai/rate-message-mutation.ts b/apps/studio/data/ai/rate-message-mutation.ts
index 5f8b080e46b..556c3a7bbd9 100644
--- a/apps/studio/data/ai/rate-message-mutation.ts
+++ b/apps/studio/data/ai/rate-message-mutation.ts
@@ -13,6 +13,7 @@ export type RateMessageVariables = {
projectRef: string
orgSlug?: string
reason?: string
+ spanId?: string
}
export async function rateMessage({
@@ -22,6 +23,7 @@ export async function rateMessage({
projectRef,
orgSlug,
reason,
+ spanId,
}: RateMessageVariables) {
const url = `${BASE_PATH}/api/ai/feedback/rate`
@@ -29,7 +31,7 @@ export async function rateMessage({
const response = await fetchHandler(url, {
headers,
method: 'POST',
- body: JSON.stringify({ rating, messages, messageId, projectRef, orgSlug, reason }),
+ body: JSON.stringify({ rating, messages, messageId, projectRef, orgSlug, reason, spanId }),
})
let body: any
diff --git a/apps/studio/lib/ai/generate-assistant-response.ts b/apps/studio/lib/ai/generate-assistant-response.ts
index eef76fdddd9..dcfb2d09126 100644
--- a/apps/studio/lib/ai/generate-assistant-response.ts
+++ b/apps/studio/lib/ai/generate-assistant-response.ts
@@ -41,7 +41,9 @@ export async function generateAssistantResponse({
planId,
promptProviderOptions,
providerOptions,
+ requestedModel,
abortSignal,
+ onSpanCreated,
}: {
messages: UIMessage[]
model: LanguageModel
@@ -55,13 +57,19 @@ export async function generateAssistantResponse({
userId?: string
orgId?: number
planId?: string
+ requestedModel?: string
promptProviderOptions?: Record<string, any>
providerOptions?: Record<string, any>
abortSignal?: AbortSignal
+ onSpanCreated?: (spanId: string) => void
}) {
const shouldTrace = IS_TRACING_ENABLED && !isHipaaEnabled
const run = async (span?: Span) => {
+ if (span) {
+ onSpanCreated?.(span.id)
+ }
+
// Only returns last 7 messages
// Filters out tools with invalid states
// Filters out tool outputs based on opt-in level using renderingToolOutputParser
@@ -171,6 +179,7 @@ export async function generateAssistantResponse({
userId,
orgId,
planId,
+ requestedModel,
gitBranch: process.env.VERCEL_GIT_COMMIT_REF,
environment: process.env.NEXT_PUBLIC_ENVIRONMENT,
},
diff --git a/apps/studio/pages/api/ai/feedback/rate.ts b/apps/studio/pages/api/ai/feedback/rate.ts
index 2892cabb6d5..c1cc9e8a1bf 100644
--- a/apps/studio/pages/api/ai/feedback/rate.ts
+++ b/apps/studio/pages/api/ai/feedback/rate.ts
@@ -1,14 +1,15 @@
import { generateObject } from 'ai'
-import { NextApiRequest, NextApiResponse } from 'next'
-import { z } from 'zod'
-
+import { currentLogger } from 'braintrust'
import { IS_PLATFORM } from 'common'
+import { rateMessageResponseSchema } from 'components/ui/AIAssistantPanel/Message.utils'
import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'
+import { IS_TRACING_ENABLED } from 'lib/ai/braintrust-logger'
import { getModel } from 'lib/ai/model'
import { getOrgAIDetails } from 'lib/ai/org-ai-details'
import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'
import apiWrapper from 'lib/api/apiWrapper'
-import { rateMessageResponseSchema } from 'components/ui/AIAssistantPanel/Message.utils'
+import { NextApiRequest, NextApiResponse } from 'next'
+import { z } from 'zod'
export const maxDuration = 30
@@ -31,6 +32,7 @@ const requestBodySchema = z.object({
projectRef: z.string(),
orgSlug: z.string().optional(),
reason: z.string().optional(),
+ spanId: z.string().optional(),
})
export async function handlePost(req: NextApiRequest, res: NextApiResponse) {
@@ -48,9 +50,10 @@ export async function handlePost(req: NextApiRequest, res: NextApiResponse) {
return res.status(400).json({ error: 'Invalid request body', issues: parseError.issues })
}
- const { rating, messages: rawMessages, projectRef, orgSlug, reason } = data
+ const { rating, messages: rawMessages, projectRef, orgSlug, reason, spanId } = data
let aiOptInLevel: AiOptInLevel = 'disabled'
+ let isHipaaEnabled = false
if (!IS_PLATFORM) {
aiOptInLevel = 'schema'
@@ -59,13 +62,15 @@ export async function handlePost(req: NextApiRequest, res: NextApiResponse) {
if (IS_PLATFORM && orgSlug && authorization && projectRef) {
try {
// Get organizations and compute opt in level server-side
- const { aiOptInLevel: orgAIOptInLevel, isLimited: orgAILimited } = await getOrgAIDetails({
- orgSlug,
- authorization,
- projectRef,
- })
+ const { aiOptInLevel: orgAIOptInLevel, isHipaaEnabled: orgIsHipaaEnabled } =
+ await getOrgAIDetails({
+ orgSlug,
+ authorization,
+ projectRef,
+ })
aiOptInLevel = orgAIOptInLevel
+ isHipaaEnabled = orgIsHipaaEnabled
} catch (error) {
return res.status(400).json({
error: 'There was an error fetching your organization details',
@@ -126,6 +131,25 @@ Instructions:
`,
})
+ // Log feedback to Braintrust if tracing is enabled and span ID is available
+ if (IS_TRACING_ENABLED && !isHipaaEnabled && spanId) {
+ try {
+ const logger = currentLogger()
+ logger?.logFeedback({
+ id: spanId,
+ scores: { 'User Rating': rating === 'positive' ? 1 : 0 },
+ comment: reason,
+ source: 'external',
+ })
+ logger?.updateSpan({
+ id: spanId,
+ metadata: { feedbackCategory: object.category },
+ })
+ } catch (error) {
+ console.error('Failed to log feedback to Braintrust:', error)
+ }
+ }
+
return res.json({
category: object.category,
})
diff --git a/apps/studio/pages/api/ai/sql/generate-v4.ts b/apps/studio/pages/api/ai/sql/generate-v4.ts
index 20e5209d364..27b5e08f592 100644
--- a/apps/studio/pages/api/ai/sql/generate-v4.ts
+++ b/apps/studio/pages/api/ai/sql/generate-v4.ts
@@ -1,19 +1,18 @@
import pgMeta from '@supabase/pg-meta'
-import { safeValidateUIMessages } from 'ai'
import type { JwtPayload } from '@supabase/supabase-js'
-import type { NextApiRequest, NextApiResponse } from 'next'
-import z from 'zod'
-
+import { safeValidateUIMessages } from 'ai'
import { IS_PLATFORM } from 'common'
import { executeSql } from 'data/sql/execute-sql-query'
import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'
+import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'
import { getModel } from 'lib/ai/model'
import { getOrgAIDetails } from 'lib/ai/org-ai-details'
-import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'
import { getTools } from 'lib/ai/tools'
-import { getURL } from 'lib/helpers'
import apiWrapper from 'lib/api/apiWrapper'
import { executeQuery } from 'lib/api/self-hosted/query'
+import { getURL } from 'lib/helpers'
+import type { NextApiRequest, NextApiResponse } from 'next'
+import z from 'zod'
export const maxDuration = 120
@@ -196,9 +195,13 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse, claims?: Jw
userId,
orgId,
planId,
+ requestedModel,
promptProviderOptions,
providerOptions,
abortSignal: abortController.signal,
+ onSpanCreated: (spanId) => {
+ res.setHeader('x-braintrust-span-id', spanId)
+ },
})
result.pipeUIMessageStreamToResponse(res, {
diff --git a/apps/studio/state/ai-assistant-state.tsx b/apps/studio/state/ai-assistant-state.tsx
index 1a19f379b4f..44999807e8b 100644
--- a/apps/studio/state/ai-assistant-state.tsx
+++ b/apps/studio/state/ai-assistant-state.tsx
@@ -234,6 +234,14 @@ function createChatInstance(
sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithToolCalls,
transport: new DefaultChatTransport({
api: `${BASE_PATH}/api/ai/sql/generate-v4`,
+ fetch: async (url, init) => {
+ const response = await globalThis.fetch(url as RequestInfo, init)
+ const spanId = response.headers.get('x-braintrust-span-id')
+ if (spanId) {
+ state.pendingSpanIds[options.id] = spanId
+ }
+ return response
+ },
async prepareSendMessagesRequest({ messages, ...opts }) {
const cleanedMessages = prepareMessagesForAPI(messages)
const headerData = await constructHeaders()
@@ -282,6 +290,16 @@ function createChatInstance(
chat.messages = messages as AssistantMessageType[]
chat.updatedAt = new Date()
}
+
+ // Associate pending span ID with the last assistant message
+ const pendingSpanId = state.pendingSpanIds[options.id]
+ if (pendingSpanId) {
+ const lastAssistantMsg = [...messages].reverse().find((m) => m.role === 'assistant')
+ if (lastAssistantMsg) {
+ state.messageSpanIds[lastAssistantMsg.id] = pendingSpanId
+ }
+ delete state.pendingSpanIds[options.id]
+ }
}
},
})
@@ -294,6 +312,8 @@ export const createAiAssistantState = (): AiAssistantState => {
const state: AiAssistantState = proxy({
...initialState, // Spread initial values directly
chatInstances: {},
+ pendingSpanIds: {},
+ messageSpanIds: {},
setContext: (context: Partial<AiAssistantContext>) => {
state.context = { ...state.context, ...context }
@@ -511,6 +531,8 @@ export type AiAssistantState = AiAssistantData & {
resetAiAssistantPanel: () => void
activeChat: ChatSession | undefined
chatInstances: Record<string, Chat<MessageType>>
+ pendingSpanIds: Record<string, string>
+ messageSpanIds: Record<string, string>
setContext: (context: Partial<AiAssistantContext>) => void
setModel: (model: AssistantModel) => void
newChat: (