Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(assistant): enable Braintrust tracing for non-sensitive chats (#42963)

Matt Rossmane8ab92408ffd
diff --git a/apps/studio/lib/ai/braintrust-logger.ts b/apps/studio/lib/ai/braintrust-logger.ts
new file mode 100644
index 00000000000..cae983f9b34
--- /dev/null
+++ b/apps/studio/lib/ai/braintrust-logger.ts
@@ -0,0 +1,17 @@
+import { initLogger } from 'braintrust'
+
+const BRAINTRUST_API_KEY = process.env.BRAINTRUST_API_KEY
+const BRAINTRUST_PROJECT_ID = process.env.BRAINTRUST_PROJECT_ID
+
+const IS_PRODUCTION = process.env.NEXT_PUBLIC_ENVIRONMENT === 'prod'
+
+// NOTE(mattrossman): Temporary killswitch to disable tracing in production
+export const IS_TRACING_ENABLED =
+ BRAINTRUST_API_KEY !== undefined && BRAINTRUST_PROJECT_ID !== undefined && !IS_PRODUCTION
+
+if (IS_TRACING_ENABLED) {
+ initLogger({
+ apiKey: BRAINTRUST_API_KEY,
+ projectId: BRAINTRUST_PROJECT_ID,
+ })
+}
diff --git a/apps/studio/lib/ai/generate-assistant-response.ts b/apps/studio/lib/ai/generate-assistant-response.ts
index d0bc75ba733..eef76fdddd9 100644
--- a/apps/studio/lib/ai/generate-assistant-response.ts
+++ b/apps/studio/lib/ai/generate-assistant-response.ts
@@ -2,16 +2,16 @@ import * as ai from 'ai'
import {
convertToModelMessages,
isToolUIPart,
+ stepCountIs,
type LanguageModel,
type ModelMessage,
- stepCountIs,
type ToolSet,
type UIMessage,
} from 'ai'
-import { wrapAISDK } from 'braintrust'
+import { traced, wrapAISDK, type Span } from 'braintrust'
import { source } from 'common-tags'
-
import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'
+import { IS_TRACING_ENABLED } from 'lib/ai/braintrust-logger'
import {
CHAT_PROMPT,
EDGE_FUNCTION_PROMPT,
@@ -24,7 +24,7 @@ import {
} from 'lib/ai/prompts'
import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'
-const { streamText } = wrapAISDK(ai)
+const { streamText: tracedStreamText } = wrapAISDK(ai)
export async function generateAssistantResponse({
messages: rawMessages,
@@ -33,7 +33,12 @@ export async function generateAssistantResponse({
aiOptInLevel = 'schema',
getSchemas,
projectRef,
+ chatId,
chatName,
+ isHipaaEnabled,
+ userId,
+ orgId,
+ planId,
promptProviderOptions,
providerOptions,
abortSignal,
@@ -44,91 +49,139 @@ export async function generateAssistantResponse({
aiOptInLevel?: AiOptInLevel
getSchemas?: () => Promise<string>
projectRef?: string
+ chatId?: string
chatName?: string
- // TODO(mattrossman): use for excluding HIPAA projects from assistant tracing
isHipaaEnabled?: boolean
+ userId?: string
+ orgId?: number
+ planId?: string
promptProviderOptions?: Record<string, any>
providerOptions?: Record<string, any>
abortSignal?: AbortSignal
}) {
- // Only returns last 7 messages
- // Filters out tools with invalid states
- // Filters out tool outputs based on opt-in level using renderingToolOutputParser
- const messages = (rawMessages || []).slice(-7).map((msg) => {
- if (msg && msg.role === 'assistant' && 'results' in msg) {
- const cleanedMsg = { ...msg }
- delete cleanedMsg.results
- return cleanedMsg
- }
- if (msg && msg.role === 'assistant' && msg.parts) {
- const cleanedParts = msg.parts
- .filter((part) => {
- if (isToolUIPart(part)) {
- const invalidStates = ['input-streaming', 'input-available', 'output-error']
- return !invalidStates.includes(part.state)
- }
- return true
- })
- .map((part) => {
- return sanitizeMessagePart(part, aiOptInLevel)
- })
- return { ...msg, parts: cleanedParts }
- }
- return msg
- })
+ const shouldTrace = IS_TRACING_ENABLED && !isHipaaEnabled
+
+ const run = async (span?: Span) => {
+ // Only returns last 7 messages
+ // Filters out tools with invalid states
+ // Filters out tool outputs based on opt-in level using renderingToolOutputParser
+ const messages = (rawMessages || []).slice(-7).map((msg) => {
+ if (msg && msg.role === 'assistant' && 'results' in msg) {
+ const cleanedMsg = { ...msg }
+ delete cleanedMsg.results
+ return cleanedMsg
+ }
+ if (msg && msg.role === 'assistant' && msg.parts) {
+ const cleanedParts = msg.parts
+ .filter((part) => {
+ if (isToolUIPart(part)) {
+ const invalidStates = ['input-streaming', 'input-available', 'output-error']
+ return !invalidStates.includes(part.state)
+ }
+ return true
+ })
+ .map((part) => {
+ return sanitizeMessagePart(part, aiOptInLevel)
+ })
+ return { ...msg, parts: cleanedParts }
+ }
+ return msg
+ })
+
+ const schemasString =
+ aiOptInLevel !== 'disabled' && getSchemas
+ ? await traced(async () => getSchemas(), { name: 'getSchemas', type: 'function' })
+ : "You don't have access to any schemas."
- const schemasString =
- aiOptInLevel !== 'disabled' && getSchemas
- ? await getSchemas()
- : "You don't have access to any schemas."
+ // Important: do not use dynamic content in the system prompt or Bedrock will not cache it
+ const system = source`
+ ${GENERAL_PROMPT}
+ ${CHAT_PROMPT}
+ ${PG_BEST_PRACTICES}
+ ${RLS_PROMPT}
+ ${EDGE_FUNCTION_PROMPT}
+ ${REALTIME_PROMPT}
+ ${SECURITY_PROMPT}
+ ${LIMITATIONS_PROMPT}
+ `
+
+ // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`
+ // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88
+ const hasProjectContext =
+ projectRef || chatName || schemasString !== "You don't have access to any schemas."
+
+ const assistantContent = hasProjectContext
+ ? `The user's current project is ${projectRef || 'unknown'}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName || 'unnamed'}.`
+ : undefined
+
+ const coreMessages: ModelMessage[] = [
+ {
+ role: 'system',
+ content: system,
+ ...(promptProviderOptions && {
+ providerOptions: promptProviderOptions,
+ }),
+ },
+ ...(assistantContent
+ ? [
+ {
+ role: 'assistant' as const,
+ // Add any dynamic context here
+ content: assistantContent,
+ },
+ ]
+ : []),
+ ...convertToModelMessages(messages),
+ ]
+
+ const streamTextFn = shouldTrace ? tracedStreamText : ai.streamText
+
+ const streamTextArgs = {
+ model,
+ stopWhen: stepCountIs(5),
+ messages: coreMessages,
+ ...(providerOptions && { providerOptions }),
+ tools,
+ ...(abortSignal && { abortSignal }),
+ onFinish: ({ steps }) => {
+ for (const step of steps) {
+ for (const toolCall of step.toolCalls) {
+ if (toolCall.toolName === 'rename_chat') {
+ const { newName } = toolCall.input as { newName: string }
+ span?.log({ metadata: { chatName: newName } })
+ }
+ }
+ }
+ },
+ } satisfies Parameters<typeof ai.streamText>[0]
- // Important: do not use dynamic content in the system prompt or Bedrock will not cache it
- const system = source`
- ${GENERAL_PROMPT}
- ${CHAT_PROMPT}
- ${PG_BEST_PRACTICES}
- ${RLS_PROMPT}
- ${EDGE_FUNCTION_PROMPT}
- ${REALTIME_PROMPT}
- ${SECURITY_PROMPT}
- ${LIMITATIONS_PROMPT}
- `
+ const lastUserMessage = rawMessages.findLast((m) => m.role === 'user')
+ const lastUserText = lastUserMessage?.parts
+ ?.filter((p): p is { type: 'text'; text: string } => p.type === 'text')
+ .map((p) => p.text)
+ .join('\n')
- // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`
- // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88
- const hasProjectContext =
- projectRef || chatName || schemasString !== "You don't have access to any schemas."
+ span?.log({
+ input: lastUserText,
+ metadata: {
+ projectRef,
+ chatId,
+ chatName,
+ aiOptInLevel,
+ userId,
+ orgId,
+ planId,
+ gitBranch: process.env.VERCEL_GIT_COMMIT_REF,
+ environment: process.env.NEXT_PUBLIC_ENVIRONMENT,
+ },
+ })
- const assistantContent = hasProjectContext
- ? `The user's current project is ${projectRef || 'unknown'}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName || 'unnamed'}.`
- : undefined
+ return streamTextFn(streamTextArgs)
+ }
- const coreMessages: ModelMessage[] = [
- {
- role: 'system',
- content: system,
- ...(promptProviderOptions && {
- providerOptions: promptProviderOptions,
- }),
- },
- ...(assistantContent
- ? [
- {
- role: 'assistant' as const,
- // Add any dynamic context here
- content: assistantContent,
- },
- ]
- : []),
- ...convertToModelMessages(messages),
- ]
+ if (shouldTrace) {
+ return traced(run, { type: 'function', name: 'generateAssistantResponse' })
+ }
- return streamText({
- model,
- stopWhen: stepCountIs(5),
- messages: coreMessages,
- ...(providerOptions && { providerOptions }),
- tools,
- ...(abortSignal && { abortSignal }),
- })
+ return run()
}
diff --git a/apps/studio/lib/ai/org-ai-details.test.ts b/apps/studio/lib/ai/org-ai-details.test.ts
index e0926a2b4d4..6098b80b08f 100644
--- a/apps/studio/lib/ai/org-ai-details.test.ts
+++ b/apps/studio/lib/ai/org-ai-details.test.ts
@@ -121,6 +121,8 @@ describe('ai/org-ai-details', () => {
aiOptInLevel: 'schema_only',
isLimited: true,
isHipaaEnabled: false,
+ orgId: 1,
+ planId: 'free',
})
})
diff --git a/apps/studio/lib/ai/org-ai-details.ts b/apps/studio/lib/ai/org-ai-details.ts
index 6457d390914..5ce932d45a6 100644
--- a/apps/studio/lib/ai/org-ai-details.ts
+++ b/apps/studio/lib/ai/org-ai-details.ts
@@ -41,5 +41,7 @@ export const getOrgAIDetails = async ({
aiOptInLevel,
isLimited,
isHipaaEnabled,
+ orgId: selectedOrg?.id,
+ planId: selectedOrg?.plan.id,
}
}
diff --git a/apps/studio/lib/api/apiWrapper.ts b/apps/studio/lib/api/apiWrapper.ts
index 901c5a5501b..e98f7559bae 100644
--- a/apps/studio/lib/api/apiWrapper.ts
+++ b/apps/studio/lib/api/apiWrapper.ts
@@ -1,6 +1,7 @@
+import type { JwtPayload } from '@supabase/supabase-js'
import type { NextApiRequest, NextApiResponse } from 'next'
-
import { ResponseError, ResponseFailure } from 'types'
+
import { IS_PLATFORM } from '../constants'
import { apiAuthenticate } from './apiAuthenticate'
@@ -26,11 +27,16 @@ export function isResponseOk<T>(response: T | ResponseFailure | undefined): resp
export default async function apiWrapper(
req: NextApiRequest,
res: NextApiResponse,
- handler: (req: NextApiRequest, res: NextApiResponse) => Promise<Response | void>,
+ handler: (
+ req: NextApiRequest,
+ res: NextApiResponse,
+ claims?: JwtPayload
+ ) => Promise<Response | void>,
options?: { withAuth: boolean }
): Promise<Response | void> {
try {
const { withAuth } = options || {}
+ let claims: JwtPayload | undefined
if (IS_PLATFORM && withAuth) {
const response = await apiAuthenticate(req, res)
@@ -41,9 +47,10 @@ export default async function apiWrapper(
},
})
}
+ claims = response
}
- return handler(req, res)
+ return handler(req, res, claims)
} catch (error) {
return res.status(500).json({ error })
}
diff --git a/apps/studio/lib/api/apiWrappers.test.ts b/apps/studio/lib/api/apiWrappers.test.ts
index 9fc8d326220..eeb96e7b401 100644
--- a/apps/studio/lib/api/apiWrappers.test.ts
+++ b/apps/studio/lib/api/apiWrappers.test.ts
@@ -1,4 +1,7 @@
+import type { JwtPayload } from '@supabase/supabase-js'
+import type { NextApiRequest, NextApiResponse } from 'next'
import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { ResponseError } from 'types'
import apiWrapper from './apiWrapper'
import { apiAuthenticate } from './apiAuthenticate'
@@ -12,11 +15,11 @@ vi.mock('./apiAuthenticate', () => ({
}))
describe('apiWrapper', () => {
- const mockReq = {} as any
+ const mockReq = {} as NextApiRequest
const mockRes = {
status: vi.fn().mockReturnThis(),
json: vi.fn().mockReturnThis(),
- } as any
+ } as unknown as NextApiResponse
const mockHandler = vi.fn()
beforeEach(() => {
@@ -25,7 +28,34 @@ describe('apiWrapper', () => {
it('should call handler directly when withAuth is false', async () => {
await apiWrapper(mockReq, mockRes, mockHandler, { withAuth: false })
- expect(mockHandler).toHaveBeenCalledWith(mockReq, mockRes)
+ expect(mockHandler).toHaveBeenCalledWith(mockReq, mockRes, undefined)
expect(apiAuthenticate).not.toHaveBeenCalled()
})
+
+ it('should pass JWT claims to handler when withAuth is true', async () => {
+ const mockClaims: JwtPayload = {
+ iss: 'supabase',
+ sub: 'user-123',
+ aud: 'authenticated',
+ exp: 9999999999,
+ iat: 1000000000,
+ role: 'authenticated',
+ aal: 'aal1',
+ session_id: 'session-123',
+ }
+ vi.mocked(apiAuthenticate).mockResolvedValue(mockClaims)
+
+ await apiWrapper(mockReq, mockRes, mockHandler, { withAuth: true })
+ expect(apiAuthenticate).toHaveBeenCalledWith(mockReq, mockRes)
+ expect(mockHandler).toHaveBeenCalledWith(mockReq, mockRes, mockClaims)
+ })
+
+ it('should return 401 when authentication fails', async () => {
+ const mockError = { error: new ResponseError('Invalid token') }
+ vi.mocked(apiAuthenticate).mockResolvedValue(mockError)
+
+ await apiWrapper(mockReq, mockRes, mockHandler, { withAuth: true })
+ expect(mockRes.status).toHaveBeenCalledWith(401)
+ expect(mockHandler).not.toHaveBeenCalled()
+ })
})
diff --git a/apps/studio/package.json b/apps/studio/package.json
index 90bf696ee9d..c0c422e830d 100644
--- a/apps/studio/package.json
+++ b/apps/studio/package.json
@@ -184,7 +184,7 @@
"@vitest/ui": "^3.2.0",
"api-types": "workspace:*",
"autoevals": "^0.0.131",
- "braintrust": "^1.0.2",
+ "braintrust": "^3.0.0",
"common": "workspace:*",
"config": "workspace:*",
"date-fns": "^2.30.0",
diff --git a/apps/studio/pages/api/ai/sql/generate-v4.ts b/apps/studio/pages/api/ai/sql/generate-v4.ts
index 354a30f4b93..20e5209d364 100644
--- a/apps/studio/pages/api/ai/sql/generate-v4.ts
+++ b/apps/studio/pages/api/ai/sql/generate-v4.ts
@@ -1,5 +1,6 @@
import pgMeta from '@supabase/pg-meta'
import { safeValidateUIMessages } from 'ai'
+import type { JwtPayload } from '@supabase/supabase-js'
import type { NextApiRequest, NextApiResponse } from 'next'
import z from 'zod'
@@ -24,12 +25,12 @@ export const config = {
},
}
-async function handler(req: NextApiRequest, res: NextApiResponse) {
+async function handler(req: NextApiRequest, res: NextApiResponse, claims?: JwtPayload) {
const { method } = req
switch (method) {
case 'POST':
- return handlePost(req, res)
+ return handlePost(req, res, claims)
default:
res.setHeader('Allow', ['POST'])
res.status(405).json({
@@ -50,12 +51,13 @@ const requestBodySchema = z.object({
connectionString: z.string(),
schema: z.string().optional(),
table: z.string().optional(),
+ chatId: z.string().optional(),
chatName: z.string().optional(),
orgSlug: z.string().optional(),
model: z.enum(['gpt-5', 'gpt-5-mini']).optional(),
})
-async function handlePost(req: NextApiRequest, res: NextApiResponse) {
+async function handlePost(req: NextApiRequest, res: NextApiResponse, claims?: JwtPayload) {
const authorization = req.headers.authorization
const accessToken = authorization?.replace('Bearer ', '')
@@ -63,6 +65,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
return res.status(401).json({ error: 'Authorization token is required' })
}
+ const userId = claims?.sub
+
const body = typeof req.body === 'string' ? JSON.parse(req.body) : req.body
const { data, error: parseError } = requestBodySchema.safeParse(body)
@@ -75,6 +79,7 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
projectRef,
connectionString,
orgSlug,
+ chatId,
chatName,
model: requestedModel,
} = data
@@ -90,6 +95,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
let aiOptInLevel: AiOptInLevel = 'disabled'
let isLimited = false
let isHipaaEnabled = false
+ let orgId: number | undefined
+ let planId: string | undefined
if (!IS_PLATFORM) {
aiOptInLevel = 'schema'
@@ -102,6 +109,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
aiOptInLevel: orgAIOptInLevel,
isLimited: orgAILimited,
isHipaaEnabled: orgIsHipaaEnabled,
+ orgId: fetchedOrgId,
+ planId: fetchedPlanId,
} = await getOrgAIDetails({
orgSlug,
authorization,
@@ -111,6 +120,8 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
aiOptInLevel = orgAIOptInLevel
isLimited = orgAILimited
isHipaaEnabled = orgIsHipaaEnabled
+ orgId = fetchedOrgId
+ planId = fetchedPlanId
} catch (error) {
return res.status(400).json({
error: 'There was an error fetching your organization details',
@@ -179,8 +190,12 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
aiOptInLevel,
getSchemas: aiOptInLevel !== 'disabled' ? getSchemas : undefined,
projectRef,
+ chatId,
chatName,
isHipaaEnabled,
+ userId,
+ orgId,
+ planId,
promptProviderOptions,
providerOptions,
abortSignal: abortController.signal,
diff --git a/apps/studio/state/ai-assistant-state.tsx b/apps/studio/state/ai-assistant-state.tsx
index 8c7c8534551..1a19f379b4f 100644
--- a/apps/studio/state/ai-assistant-state.tsx
+++ b/apps/studio/state/ai-assistant-state.tsx
@@ -248,6 +248,7 @@ function createChatInstance(
messages: cleanedMessages,
projectRef: state.context.projectRef,
connectionString: state.context.connectionString,
+ chatId: options.id,
chatName: chat?.name,
orgSlug: state.context.orgSlug,
context: state.context,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 5d230b4396a..748b1f7feec 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1207,8 +1207,8 @@ importers:
specifier: ^0.0.131
braintrust:
- specifier: ^1.0.2
+ specifier: ^3.0.0
date-fns:
specifier: ^2.30.0
version: 2.30.0
@@ -9779,11 +9779,11 @@ packages:
resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
engines: {node: '>=8'}
- resolution: {integrity: sha512-Zm8WbD18FLv5HN38TyUKw659vJdZALQ5VVVWTncMxZgPtFXeWoC7H0tInw/1EQ1pGq50v7WZtlaFBOXhUaf8DQ==}
+ resolution: {integrity: sha512-mhpflunGgPhZC8wJxP6GPPcrFNIDgO0E77Gy/hkxSlIUaLpszfxLgQAIESccdbjH+x+PHUDiJiMIgDxNpagVHg==}
hasBin: true
peerDependencies:
- zod: ^3.25.34
+ zod: ^3.25.34 || ^4.0
resolution: {integrity: sha512-ECFzp6uFOSB+dcZ5BK/IBaGWssbSYBHvuMeMt3MMFyhI0Z8SqGgEkBLARgpRH3hutIgPVsALcMwbDrJqPxQ65A==}
@@ -27485,11 +27485,12 @@ snapshots:
dependencies:
fill-range: 7.1.1
dependencies:
'@ai-sdk/provider': 1.1.3
'@next/env': 14.2.35
'@vercel/functions': 1.6.0(@aws-sdk/[email protected])
+ ajv: 8.17.1
argparse: 2.0.1
boxen: 8.0.1
chalk: 4.1.2
@@ -27506,7 +27507,6 @@ snapshots:
mustache: 4.2.0
pluralize: 8.0.0
simple-git: 3.28.0([email protected])
- slugify: 1.6.6
source-map: 0.7.6
termi-link: 1.1.0
uuid: 9.0.1
diff --git a/turbo.json b/turbo.json
index 39f698d8c20..c8026da1181 100644
--- a/turbo.json
+++ b/turbo.json
@@ -112,6 +112,7 @@
],
"passThroughEnv": [
"CURRENT_CLI_VERSION",
+ "VERCEL_GIT_COMMIT_REF",
"VERCEL_GIT_COMMIT_SHA",
"SNIPPETS_MANAGEMENT_FOLDER",
"EDGE_FUNCTIONS_MANAGEMENT_FOLDER"