The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(assistant): send user feedback to Braintrust traces (#43021)
Matt Rossman20d1ac0f83c3
diff --git a/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx b/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsxindex 930fbb33a4b..9916de1310f 100644--- a/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx+++ b/apps/studio/components/ui/AIAssistantPanel/AIAssistant.tsx@@ -231,6 +231,7 @@ export const AIAssistant = ({ className }: AIAssistantProps) => {projectRef: project.ref,orgSlug: selectedOrganization.slug,reason,+ spanId: state.messageSpanIds[messageId],})sendEvent({@@ -254,7 +255,7 @@ export const AIAssistant = ({ className }: AIAssistantProps) => {})}},- [chatMessages, project?.ref, selectedOrganization?.slug, rateMessage, sendEvent]+ [chatMessages, project?.ref, selectedOrganization?.slug, rateMessage, sendEvent, state])const isContextExceededError =diff --git a/apps/studio/data/ai/rate-message-mutation.ts b/apps/studio/data/ai/rate-message-mutation.tsindex 5f8b080e46b..556c3a7bbd9 100644--- a/apps/studio/data/ai/rate-message-mutation.ts+++ b/apps/studio/data/ai/rate-message-mutation.ts@@ -13,6 +13,7 @@ export type RateMessageVariables = {projectRef: stringorgSlug?: stringreason?: string+ spanId?: string}export async function rateMessage({@@ -22,6 +23,7 @@ export async function rateMessage({projectRef,orgSlug,reason,+ spanId,}: RateMessageVariables) {const url = `${BASE_PATH}/api/ai/feedback/rate`@@ -29,7 +31,7 @@ export async function rateMessage({const response = await fetchHandler(url, {headers,method: 'POST',- body: JSON.stringify({ rating, messages, messageId, projectRef, orgSlug, reason }),+ body: JSON.stringify({ rating, messages, messageId, projectRef, orgSlug, reason, spanId }),})let body: anydiff --git a/apps/studio/lib/ai/generate-assistant-response.ts b/apps/studio/lib/ai/generate-assistant-response.tsindex eef76fdddd9..dcfb2d09126 100644--- a/apps/studio/lib/ai/generate-assistant-response.ts+++ b/apps/studio/lib/ai/generate-assistant-response.ts@@ -41,7 +41,9 @@ export async function generateAssistantResponse({planId,promptProviderOptions,providerOptions,+ requestedModel,abortSignal,+ onSpanCreated,}: {messages: UIMessage[]model: LanguageModel@@ -55,13 +57,19 @@ export async function generateAssistantResponse({userId?: stringorgId?: numberplanId?: string+ requestedModel?: stringpromptProviderOptions?: Record<string, any>providerOptions?: Record<string, any>abortSignal?: AbortSignal+ onSpanCreated?: (spanId: string) => void}) {const shouldTrace = IS_TRACING_ENABLED && !isHipaaEnabledconst run = async (span?: Span) => {+ if (span) {+ onSpanCreated?.(span.id)+ }+// Only returns last 7 messages// Filters out tools with invalid states// Filters out tool outputs based on opt-in level using renderingToolOutputParser@@ -171,6 +179,7 @@ export async function generateAssistantResponse({userId,orgId,planId,+ requestedModel,gitBranch: process.env.VERCEL_GIT_COMMIT_REF,environment: process.env.NEXT_PUBLIC_ENVIRONMENT,},diff --git a/apps/studio/pages/api/ai/feedback/rate.ts b/apps/studio/pages/api/ai/feedback/rate.tsindex 2892cabb6d5..c1cc9e8a1bf 100644--- a/apps/studio/pages/api/ai/feedback/rate.ts+++ b/apps/studio/pages/api/ai/feedback/rate.ts@@ -1,14 +1,15 @@import { generateObject } from 'ai'-import { NextApiRequest, NextApiResponse } from 'next'-import { z } from 'zod'-+import { currentLogger } from 'braintrust'import { IS_PLATFORM } from 'common'+import { rateMessageResponseSchema } from 'components/ui/AIAssistantPanel/Message.utils'import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'+import { IS_TRACING_ENABLED } from 'lib/ai/braintrust-logger'import { getModel } from 'lib/ai/model'import { getOrgAIDetails } from 'lib/ai/org-ai-details'import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'import apiWrapper from 'lib/api/apiWrapper'-import { rateMessageResponseSchema } from 'components/ui/AIAssistantPanel/Message.utils'+import { NextApiRequest, NextApiResponse } from 'next'+import { z } from 'zod'export const maxDuration = 30@@ -31,6 +32,7 @@ const requestBodySchema = z.object({projectRef: z.string(),orgSlug: z.string().optional(),reason: z.string().optional(),+ spanId: z.string().optional(),})export async function handlePost(req: NextApiRequest, res: NextApiResponse) {@@ -48,9 +50,10 @@ export async function handlePost(req: NextApiRequest, res: NextApiResponse) {return res.status(400).json({ error: 'Invalid request body', issues: parseError.issues })}- const { rating, messages: rawMessages, projectRef, orgSlug, reason } = data+ const { rating, messages: rawMessages, projectRef, orgSlug, reason, spanId } = datalet aiOptInLevel: AiOptInLevel = 'disabled'+ let isHipaaEnabled = falseif (!IS_PLATFORM) {aiOptInLevel = 'schema'@@ -59,13 +62,15 @@ export async function handlePost(req: NextApiRequest, res: NextApiResponse) {if (IS_PLATFORM && orgSlug && authorization && projectRef) {try {// Get organizations and compute opt in level server-side- const { aiOptInLevel: orgAIOptInLevel, isLimited: orgAILimited } = await getOrgAIDetails({- orgSlug,- authorization,- projectRef,- })+ const { aiOptInLevel: orgAIOptInLevel, isHipaaEnabled: orgIsHipaaEnabled } =+ await getOrgAIDetails({+ orgSlug,+ authorization,+ projectRef,+ })aiOptInLevel = orgAIOptInLevel+ isHipaaEnabled = orgIsHipaaEnabled} catch (error) {return res.status(400).json({error: 'There was an error fetching your organization details',@@ -126,6 +131,25 @@ Instructions:`,})+ // Log feedback to Braintrust if tracing is enabled and span ID is available+ if (IS_TRACING_ENABLED && !isHipaaEnabled && spanId) {+ try {+ const logger = currentLogger()+ logger?.logFeedback({+ id: spanId,+ scores: { 'User Rating': rating === 'positive' ? 1 : 0 },+ comment: reason,+ source: 'external',+ })+ logger?.updateSpan({+ id: spanId,+ metadata: { feedbackCategory: object.category },+ })+ } catch (error) {+ console.error('Failed to log feedback to Braintrust:', error)+ }+ }+return res.json({category: object.category,})diff --git a/apps/studio/pages/api/ai/sql/generate-v4.ts b/apps/studio/pages/api/ai/sql/generate-v4.tsindex 20e5209d364..27b5e08f592 100644--- a/apps/studio/pages/api/ai/sql/generate-v4.ts+++ b/apps/studio/pages/api/ai/sql/generate-v4.ts@@ -1,19 +1,18 @@import pgMeta from '@supabase/pg-meta'-import { safeValidateUIMessages } from 'ai'import type { JwtPayload } from '@supabase/supabase-js'-import type { NextApiRequest, NextApiResponse } from 'next'-import z from 'zod'-+import { safeValidateUIMessages } from 'ai'import { IS_PLATFORM } from 'common'import { executeSql } from 'data/sql/execute-sql-query'import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'+import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'import { getModel } from 'lib/ai/model'import { getOrgAIDetails } from 'lib/ai/org-ai-details'-import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'import { getTools } from 'lib/ai/tools'-import { getURL } from 'lib/helpers'import apiWrapper from 'lib/api/apiWrapper'import { executeQuery } from 'lib/api/self-hosted/query'+import { getURL } from 'lib/helpers'+import type { NextApiRequest, NextApiResponse } from 'next'+import z from 'zod'export const maxDuration = 120@@ -196,9 +195,13 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse, claims?: JwuserId,orgId,planId,+ requestedModel,promptProviderOptions,providerOptions,abortSignal: abortController.signal,+ onSpanCreated: (spanId) => {+ res.setHeader('x-braintrust-span-id', spanId)+ },})result.pipeUIMessageStreamToResponse(res, {diff --git a/apps/studio/state/ai-assistant-state.tsx b/apps/studio/state/ai-assistant-state.tsxindex 1a19f379b4f..44999807e8b 100644--- a/apps/studio/state/ai-assistant-state.tsx+++ b/apps/studio/state/ai-assistant-state.tsx@@ -234,6 +234,14 @@ function createChatInstance(sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithToolCalls,transport: new DefaultChatTransport({api: `${BASE_PATH}/api/ai/sql/generate-v4`,+ fetch: async (url, init) => {+ const response = await globalThis.fetch(url as RequestInfo, init)+ const spanId = response.headers.get('x-braintrust-span-id')+ if (spanId) {+ state.pendingSpanIds[options.id] = spanId+ }+ return response+ },async prepareSendMessagesRequest({ messages, ...opts }) {const cleanedMessages = prepareMessagesForAPI(messages)const headerData = await constructHeaders()@@ -282,6 +290,16 @@ function createChatInstance(chat.messages = messages as AssistantMessageType[]chat.updatedAt = new Date()}++ // Associate pending span ID with the last assistant message+ const pendingSpanId = state.pendingSpanIds[options.id]+ if (pendingSpanId) {+ const lastAssistantMsg = [...messages].reverse().find((m) => m.role === 'assistant')+ if (lastAssistantMsg) {+ state.messageSpanIds[lastAssistantMsg.id] = pendingSpanId+ }+ delete state.pendingSpanIds[options.id]+ }}},})@@ -294,6 +312,8 @@ export const createAiAssistantState = (): AiAssistantState => {const state: AiAssistantState = proxy({...initialState, // Spread initial values directlychatInstances: {},+ pendingSpanIds: {},+ messageSpanIds: {},setContext: (context: Partial<AiAssistantContext>) => {state.context = { ...state.context, ...context }@@ -511,6 +531,8 @@ export type AiAssistantState = AiAssistantData & {resetAiAssistantPanel: () => voidactiveChat: ChatSession | undefinedchatInstances: Record<string, Chat<MessageType>>+ pendingSpanIds: Record<string, string>+ messageSpanIds: Record<string, string>setContext: (context: Partial<AiAssistantContext>) => voidsetModel: (model: AssistantModel) => voidnewChat: (