The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(assistant): enable Braintrust tracing for non-sensitive chats (#42963)
Matt Rossmane8ab92408ffd
diff --git a/apps/studio/lib/ai/braintrust-logger.ts b/apps/studio/lib/ai/braintrust-logger.tsnew file mode 100644index 00000000000..cae983f9b34--- /dev/null+++ b/apps/studio/lib/ai/braintrust-logger.ts@@ -0,0 +1,17 @@+import { initLogger } from 'braintrust'++const BRAINTRUST_API_KEY = process.env.BRAINTRUST_API_KEY+const BRAINTRUST_PROJECT_ID = process.env.BRAINTRUST_PROJECT_ID++const IS_PRODUCTION = process.env.NEXT_PUBLIC_ENVIRONMENT === 'prod'++// NOTE(mattrossman): Temporary killswitch to disable tracing in production+export const IS_TRACING_ENABLED =+ BRAINTRUST_API_KEY !== undefined && BRAINTRUST_PROJECT_ID !== undefined && !IS_PRODUCTION++if (IS_TRACING_ENABLED) {+ initLogger({+ apiKey: BRAINTRUST_API_KEY,+ projectId: BRAINTRUST_PROJECT_ID,+ })+}diff --git a/apps/studio/lib/ai/generate-assistant-response.ts b/apps/studio/lib/ai/generate-assistant-response.tsindex d0bc75ba733..eef76fdddd9 100644--- a/apps/studio/lib/ai/generate-assistant-response.ts+++ b/apps/studio/lib/ai/generate-assistant-response.ts@@ -2,16 +2,16 @@ import * as ai from 'ai'import {convertToModelMessages,isToolUIPart,+ stepCountIs,type LanguageModel,type ModelMessage,- stepCountIs,type ToolSet,type UIMessage,} from 'ai'-import { wrapAISDK } from 'braintrust'+import { traced, wrapAISDK, type Span } from 'braintrust'import { source } from 'common-tags'-import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'+import { IS_TRACING_ENABLED } from 'lib/ai/braintrust-logger'import {CHAT_PROMPT,EDGE_FUNCTION_PROMPT,@@ -24,7 +24,7 @@ import {} from 'lib/ai/prompts'import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'-const { streamText } = wrapAISDK(ai)+const { streamText: tracedStreamText } = wrapAISDK(ai)export async function generateAssistantResponse({messages: rawMessages,@@ -33,7 +33,12 @@ export async function generateAssistantResponse({aiOptInLevel = 'schema',getSchemas,projectRef,+ chatId,chatName,+ isHipaaEnabled,+ userId,+ orgId,+ planId,promptProviderOptions,providerOptions,abortSignal,@@ -44,91 +49,139 @@ export async function generateAssistantResponse({aiOptInLevel?: AiOptInLevelgetSchemas?: () => Promise<string>projectRef?: string+ chatId?: stringchatName?: string- // TODO(mattrossman): use for excluding HIPAA projects from assistant tracingisHipaaEnabled?: boolean+ userId?: string+ orgId?: number+ planId?: stringpromptProviderOptions?: Record<string, any>providerOptions?: Record<string, any>abortSignal?: AbortSignal}) {- // Only returns last 7 messages- // Filters out tools with invalid states- // Filters out tool outputs based on opt-in level using renderingToolOutputParser- const messages = (rawMessages || []).slice(-7).map((msg) => {- if (msg && msg.role === 'assistant' && 'results' in msg) {- const cleanedMsg = { ...msg }- delete cleanedMsg.results- return cleanedMsg- }- if (msg && msg.role === 'assistant' && msg.parts) {- const cleanedParts = msg.parts- .filter((part) => {- if (isToolUIPart(part)) {- const invalidStates = ['input-streaming', 'input-available', 'output-error']- return !invalidStates.includes(part.state)- }- return true- })- .map((part) => {- return sanitizeMessagePart(part, aiOptInLevel)- })- return { ...msg, parts: cleanedParts }- }- return msg- })+ const shouldTrace = IS_TRACING_ENABLED && !isHipaaEnabled++ const run = async (span?: Span) => {+ // Only returns last 7 messages+ // Filters out tools with invalid states+ // Filters out tool outputs based on opt-in level using renderingToolOutputParser+ const messages = (rawMessages || []).slice(-7).map((msg) => {+ if (msg && msg.role === 'assistant' && 'results' in msg) {+ const cleanedMsg = { ...msg }+ delete cleanedMsg.results+ return cleanedMsg+ }+ if (msg && msg.role === 'assistant' && msg.parts) {+ const cleanedParts = msg.parts+ .filter((part) => {+ if (isToolUIPart(part)) {+ const invalidStates = ['input-streaming', 'input-available', 'output-error']+ return !invalidStates.includes(part.state)+ }+ return true+ })+ .map((part) => {+ return sanitizeMessagePart(part, aiOptInLevel)+ })+ return { ...msg, parts: cleanedParts }+ }+ return msg+ })++ const schemasString =+ aiOptInLevel !== 'disabled' && getSchemas+ ? await traced(async () => getSchemas(), { name: 'getSchemas', type: 'function' })+ : "You don't have access to any schemas."- const schemasString =- aiOptInLevel !== 'disabled' && getSchemas- ? await getSchemas()- : "You don't have access to any schemas."+ // Important: do not use dynamic content in the system prompt or Bedrock will not cache it+ const system = source`+ ${GENERAL_PROMPT}+ ${CHAT_PROMPT}+ ${PG_BEST_PRACTICES}+ ${RLS_PROMPT}+ ${EDGE_FUNCTION_PROMPT}+ ${REALTIME_PROMPT}+ ${SECURITY_PROMPT}+ ${LIMITATIONS_PROMPT}+ `++ // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`+ // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88+ const hasProjectContext =+ projectRef || chatName || schemasString !== "You don't have access to any schemas."++ const assistantContent = hasProjectContext+ ? `The user's current project is ${projectRef || 'unknown'}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName || 'unnamed'}.`+ : undefined++ const coreMessages: ModelMessage[] = [+ {+ role: 'system',+ content: system,+ ...(promptProviderOptions && {+ providerOptions: promptProviderOptions,+ }),+ },+ ...(assistantContent+ ? [+ {+ role: 'assistant' as const,+ // Add any dynamic context here+ content: assistantContent,+ },+ ]+ : []),+ ...convertToModelMessages(messages),+ ]++ const streamTextFn = shouldTrace ? tracedStreamText : ai.streamText++ const streamTextArgs = {+ model,+ stopWhen: stepCountIs(5),+ messages: coreMessages,+ ...(providerOptions && { providerOptions }),+ tools,+ ...(abortSignal && { abortSignal }),+ onFinish: ({ steps }) => {+ for (const step of steps) {+ for (const toolCall of step.toolCalls) {+ if (toolCall.toolName === 'rename_chat') {+ const { newName } = toolCall.input as { newName: string }+ span?.log({ metadata: { chatName: newName } })+ }+ }+ }+ },+ } satisfies Parameters<typeof ai.streamText>[0]- // Important: do not use dynamic content in the system prompt or Bedrock will not cache it- const system = source`- ${GENERAL_PROMPT}- ${CHAT_PROMPT}- ${PG_BEST_PRACTICES}- ${RLS_PROMPT}- ${EDGE_FUNCTION_PROMPT}- ${REALTIME_PROMPT}- ${SECURITY_PROMPT}- ${LIMITATIONS_PROMPT}- `+ const lastUserMessage = rawMessages.findLast((m) => m.role === 'user')+ const lastUserText = lastUserMessage?.parts+ ?.filter((p): p is { type: 'text'; text: string } => p.type === 'text')+ .map((p) => p.text)+ .join('\n')- // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`- // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88- const hasProjectContext =- projectRef || chatName || schemasString !== "You don't have access to any schemas."+ span?.log({+ input: lastUserText,+ metadata: {+ projectRef,+ chatId,+ chatName,+ aiOptInLevel,+ userId,+ orgId,+ planId,+ gitBranch: process.env.VERCEL_GIT_COMMIT_REF,+ environment: process.env.NEXT_PUBLIC_ENVIRONMENT,+ },+ })- const assistantContent = hasProjectContext- ? `The user's current project is ${projectRef || 'unknown'}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName || 'unnamed'}.`- : undefined+ return streamTextFn(streamTextArgs)+ }- const coreMessages: ModelMessage[] = [- {- role: 'system',- content: system,- ...(promptProviderOptions && {- providerOptions: promptProviderOptions,- }),- },- ...(assistantContent- ? [- {- role: 'assistant' as const,- // Add any dynamic context here- content: assistantContent,- },- ]- : []),- ...convertToModelMessages(messages),- ]+ if (shouldTrace) {+ return traced(run, { type: 'function', name: 'generateAssistantResponse' })+ }- return streamText({- model,- stopWhen: stepCountIs(5),- messages: coreMessages,- ...(providerOptions && { providerOptions }),- tools,- ...(abortSignal && { abortSignal }),- })+ return run()}diff --git a/apps/studio/lib/ai/org-ai-details.test.ts b/apps/studio/lib/ai/org-ai-details.test.tsindex e0926a2b4d4..6098b80b08f 100644--- a/apps/studio/lib/ai/org-ai-details.test.ts+++ b/apps/studio/lib/ai/org-ai-details.test.ts@@ -121,6 +121,8 @@ describe('ai/org-ai-details', () => {aiOptInLevel: 'schema_only',isLimited: true,isHipaaEnabled: false,+ orgId: 1,+ planId: 'free',})})diff --git a/apps/studio/lib/ai/org-ai-details.ts b/apps/studio/lib/ai/org-ai-details.tsindex 6457d390914..5ce932d45a6 100644--- a/apps/studio/lib/ai/org-ai-details.ts+++ b/apps/studio/lib/ai/org-ai-details.ts@@ -41,5 +41,7 @@ export const getOrgAIDetails = async ({aiOptInLevel,isLimited,isHipaaEnabled,+ orgId: selectedOrg?.id,+ planId: selectedOrg?.plan.id,}}diff --git a/apps/studio/lib/api/apiWrapper.ts b/apps/studio/lib/api/apiWrapper.tsindex 901c5a5501b..e98f7559bae 100644--- a/apps/studio/lib/api/apiWrapper.ts+++ b/apps/studio/lib/api/apiWrapper.ts@@ -1,6 +1,7 @@+import type { JwtPayload } from '@supabase/supabase-js'import type { NextApiRequest, NextApiResponse } from 'next'-import { ResponseError, ResponseFailure } from 'types'+import { IS_PLATFORM } from '../constants'import { apiAuthenticate } from './apiAuthenticate'@@ -26,11 +27,16 @@ export function isResponseOk<T>(response: T | ResponseFailure | undefined): respexport default async function apiWrapper(req: NextApiRequest,res: NextApiResponse,- handler: (req: NextApiRequest, res: NextApiResponse) => Promise<Response | void>,+ handler: (+ req: NextApiRequest,+ res: NextApiResponse,+ claims?: JwtPayload+ ) => Promise<Response | void>,options?: { withAuth: boolean }): Promise<Response | void> {try {const { withAuth } = options || {}+ let claims: JwtPayload | undefinedif (IS_PLATFORM && withAuth) {const response = await apiAuthenticate(req, res)@@ -41,9 +47,10 @@ export default async function apiWrapper(},})}+ claims = response}- return handler(req, res)+ return handler(req, res, claims)} catch (error) {return res.status(500).json({ error })}diff --git a/apps/studio/lib/api/apiWrappers.test.ts b/apps/studio/lib/api/apiWrappers.test.tsindex 9fc8d326220..eeb96e7b401 100644--- a/apps/studio/lib/api/apiWrappers.test.ts+++ b/apps/studio/lib/api/apiWrappers.test.ts@@ -1,4 +1,7 @@+import type { JwtPayload } from '@supabase/supabase-js'+import type { NextApiRequest, NextApiResponse } from 'next'import { describe, it, expect, vi, beforeEach } from 'vitest'+import { ResponseError } from 'types'import apiWrapper from './apiWrapper'import { apiAuthenticate } from './apiAuthenticate'@@ -12,11 +15,11 @@ vi.mock('./apiAuthenticate', () => ({}))describe('apiWrapper', () => {- const mockReq = {} as any+ const mockReq = {} as NextApiRequestconst mockRes = {status: vi.fn().mockReturnThis(),json: vi.fn().mockReturnThis(),- } as any+ } as unknown as NextApiResponseconst mockHandler = vi.fn()beforeEach(() => {@@ -25,7 +28,34 @@ describe('apiWrapper', () => {it('should call handler directly when withAuth is false', async () => {await apiWrapper(mockReq, mockRes, mockHandler, { withAuth: false })- expect(mockHandler).toHaveBeenCalledWith(mockReq, mockRes)+ expect(mockHandler).toHaveBeenCalledWith(mockReq, mockRes, undefined)expect(apiAuthenticate).not.toHaveBeenCalled()})++ it('should pass JWT claims to handler when withAuth is true', async () => {+ const mockClaims: JwtPayload = {+ iss: 'supabase',+ sub: 'user-123',+ aud: 'authenticated',+ exp: 9999999999,+ iat: 1000000000,+ role: 'authenticated',+ aal: 'aal1',+ session_id: 'session-123',+ }+ vi.mocked(apiAuthenticate).mockResolvedValue(mockClaims)++ await apiWrapper(mockReq, mockRes, mockHandler, { withAuth: true })+ expect(apiAuthenticate).toHaveBeenCalledWith(mockReq, mockRes)+ expect(mockHandler).toHaveBeenCalledWith(mockReq, mockRes, mockClaims)+ })++ it('should return 401 when authentication fails', async () => {+ const mockError = { error: new ResponseError('Invalid token') }+ vi.mocked(apiAuthenticate).mockResolvedValue(mockError)++ await apiWrapper(mockReq, mockRes, mockHandler, { withAuth: true })+ expect(mockRes.status).toHaveBeenCalledWith(401)+ expect(mockHandler).not.toHaveBeenCalled()+ })})diff --git a/apps/studio/package.json b/apps/studio/package.jsonindex 90bf696ee9d..c0c422e830d 100644--- a/apps/studio/package.json+++ b/apps/studio/package.json@@ -184,7 +184,7 @@"@vitest/ui": "^3.2.0","api-types": "workspace:*","autoevals": "^0.0.131",- "braintrust": "^1.0.2",+ "braintrust": "^3.0.0","common": "workspace:*","config": "workspace:*","date-fns": "^2.30.0",diff --git a/apps/studio/pages/api/ai/sql/generate-v4.ts b/apps/studio/pages/api/ai/sql/generate-v4.tsindex 354a30f4b93..20e5209d364 100644--- a/apps/studio/pages/api/ai/sql/generate-v4.ts+++ b/apps/studio/pages/api/ai/sql/generate-v4.ts@@ -1,5 +1,6 @@import pgMeta from '@supabase/pg-meta'import { safeValidateUIMessages } from 'ai'+import type { JwtPayload } from '@supabase/supabase-js'import type { NextApiRequest, NextApiResponse } from 'next'import z from 'zod'@@ -24,12 +25,12 @@ export const config = {},}-async function handler(req: NextApiRequest, res: NextApiResponse) {+async function handler(req: NextApiRequest, res: NextApiResponse, claims?: JwtPayload) {const { method } = reqswitch (method) {case 'POST':- return handlePost(req, res)+ return handlePost(req, res, claims)default:res.setHeader('Allow', ['POST'])res.status(405).json({@@ -50,12 +51,13 @@ const requestBodySchema = z.object({connectionString: z.string(),schema: z.string().optional(),table: z.string().optional(),+ chatId: z.string().optional(),chatName: z.string().optional(),orgSlug: z.string().optional(),model: z.enum(['gpt-5', 'gpt-5-mini']).optional(),})-async function handlePost(req: NextApiRequest, res: NextApiResponse) {+async function handlePost(req: NextApiRequest, res: NextApiResponse, claims?: JwtPayload) {const authorization = req.headers.authorizationconst accessToken = authorization?.replace('Bearer ', '')@@ -63,6 +65,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {return res.status(401).json({ error: 'Authorization token is required' })}+ const userId = claims?.sub+const body = typeof req.body === 'string' ? JSON.parse(req.body) : req.bodyconst { data, error: parseError } = requestBodySchema.safeParse(body)@@ -75,6 +79,7 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {projectRef,connectionString,orgSlug,+ chatId,chatName,model: requestedModel,} = data@@ -90,6 +95,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {let aiOptInLevel: AiOptInLevel = 'disabled'let isLimited = falselet isHipaaEnabled = false+ let orgId: number | undefined+ let planId: string | undefinedif (!IS_PLATFORM) {aiOptInLevel = 'schema'@@ -102,6 +109,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {aiOptInLevel: orgAIOptInLevel,isLimited: orgAILimited,isHipaaEnabled: orgIsHipaaEnabled,+ orgId: fetchedOrgId,+ planId: fetchedPlanId,} = await getOrgAIDetails({orgSlug,authorization,@@ -111,6 +120,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {aiOptInLevel = orgAIOptInLevelisLimited = orgAILimitedisHipaaEnabled = orgIsHipaaEnabled+ orgId = fetchedOrgId+ planId = fetchedPlanId} catch (error) {return res.status(400).json({error: 'There was an error fetching your organization details',@@ -179,8 +190,12 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {aiOptInLevel,getSchemas: aiOptInLevel !== 'disabled' ? getSchemas : undefined,projectRef,+ chatId,chatName,isHipaaEnabled,+ userId,+ orgId,+ planId,promptProviderOptions,providerOptions,abortSignal: abortController.signal,diff --git a/apps/studio/state/ai-assistant-state.tsx b/apps/studio/state/ai-assistant-state.tsxindex 8c7c8534551..1a19f379b4f 100644--- a/apps/studio/state/ai-assistant-state.tsx+++ b/apps/studio/state/ai-assistant-state.tsx@@ -248,6 +248,7 @@ function createChatInstance(messages: cleanedMessages,projectRef: state.context.projectRef,connectionString: state.context.connectionString,+ chatId: options.id,chatName: chat?.name,orgSlug: state.context.orgSlug,context: state.context,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 5d230b4396a..748b1f7feec 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1207,8 +1207,8 @@ importers:specifier: ^0.0.131version: 0.0.131([email protected])([email protected])braintrust:- specifier: ^1.0.2+ specifier: ^3.0.0date-fns:specifier: ^2.30.0version: 2.30.0@@ -9779,11 +9779,11 @@ packages:resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}engines: {node: '>=8'}- resolution: {integrity: sha512-Zm8WbD18FLv5HN38TyUKw659vJdZALQ5VVVWTncMxZgPtFXeWoC7H0tInw/1EQ1pGq50v7WZtlaFBOXhUaf8DQ==}+ resolution: {integrity: sha512-mhpflunGgPhZC8wJxP6GPPcrFNIDgO0E77Gy/hkxSlIUaLpszfxLgQAIESccdbjH+x+PHUDiJiMIgDxNpagVHg==}hasBin: truepeerDependencies:- zod: ^3.25.34+ zod: ^3.25.34 || ^4.0resolution: {integrity: sha512-ECFzp6uFOSB+dcZ5BK/IBaGWssbSYBHvuMeMt3MMFyhI0Z8SqGgEkBLARgpRH3hutIgPVsALcMwbDrJqPxQ65A==}@@ -27485,11 +27485,12 @@ snapshots:dependencies:fill-range: 7.1.1dependencies:'@ai-sdk/provider': 1.1.3'@next/env': 14.2.35'@vercel/functions': 1.6.0(@aws-sdk/[email protected])+ ajv: 8.17.1argparse: 2.0.1boxen: 8.0.1chalk: 4.1.2@@ -27506,7 +27507,6 @@ snapshots:mustache: 4.2.0pluralize: 8.0.0simple-git: 3.28.0([email protected])- slugify: 1.6.6source-map: 0.7.6termi-link: 1.1.0uuid: 9.0.1diff --git a/turbo.json b/turbo.jsonindex 39f698d8c20..c8026da1181 100644--- a/turbo.json+++ b/turbo.json@@ -112,6 +112,7 @@],"passThroughEnv": ["CURRENT_CLI_VERSION",+ "VERCEL_GIT_COMMIT_REF","VERCEL_GIT_COMMIT_SHA","SNIPPETS_MANAGEMENT_FOLDER","EDGE_FUNCTIONS_MANAGEMENT_FOLDER"