Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat: assistant evals (#41311)

Matt Rossman072883bcecb8
diff --git a/.github/workflows/braintrust-evals.yml b/.github/workflows/braintrust-evals.yml
new file mode 100644
index 00000000000..915d2e3bb38
--- /dev/null
+++ b/.github/workflows/braintrust-evals.yml
@@ -0,0 +1,60 @@
+name: Run Braintrust evals
+
+on:
+ push:
+ branches: [master]
+ pull_request:
+ types: [opened, synchronize, labeled]
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ pull-requests: write
+ contents: read
+
+jobs:
+ eval:
+ name: Run evals
+ if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-evals'))
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+
+ env:
+ BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ fetch-depth: 0
+ # For PR events, checkout the actual branch so Braintrust can report the correct branch name instead of detached HEAD.
+ # github.head_ref is the PR source branch, github.ref_name is the fallback for push events (e.g., master).
+ ref: ${{ github.head_ref || github.ref_name }}
+
+ - name: Install pnpm
+ uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
+ with:
+ run_install: false
+
+ - name: Use Node.js
+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+ with:
+ node-version-file: ".nvmrc"
+ cache: "pnpm"
+
+ - name: Install Dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Setup Evals
+ run: cd apps/studio && pnpm evals:setup
+
+ - name: Run Evals
+ uses: braintrustdata/eval-action@v1
+ with:
+ api_key: ${{ secrets.BRAINTRUST_API_KEY }}
+ runtime: node
+ package_manager: pnpm
+ root: apps/studio
diff --git a/apps/studio/evals/.gitignore b/apps/studio/evals/.gitignore
new file mode 100644
index 00000000000..8085866a958
--- /dev/null
+++ b/apps/studio/evals/.gitignore
@@ -0,0 +1 @@
+libpg-query.wasm
\ No newline at end of file
diff --git a/apps/studio/evals/assistant.eval.ts b/apps/studio/evals/assistant.eval.ts
new file mode 100644
index 00000000000..d2c27f40303
--- /dev/null
+++ b/apps/studio/evals/assistant.eval.ts
@@ -0,0 +1,123 @@
+import { openai } from '@ai-sdk/openai'
+import { Eval } from 'braintrust'
+import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'
+import { getMockTools } from 'lib/ai/tools/mock-tools'
+import assert from 'node:assert'
+import { dataset } from './dataset'
+import {
+ completenessScorer,
+ concisenessScorer,
+ goalCompletionScorer,
+ sqlSyntaxScorer,
+ toolUsageScorer,
+} from './scorer'
+import { ToolSet, TypedToolCall, TypedToolResult } from 'ai'
+
+assert(process.env.BRAINTRUST_PROJECT_ID, 'BRAINTRUST_PROJECT_ID is not set')
+assert(process.env.OPENAI_API_KEY, 'OPENAI_API_KEY is not set')
+
+Eval('Assistant', {
+ projectId: process.env.BRAINTRUST_PROJECT_ID,
+ data: () => dataset,
+ task: async (input) => {
+ const result = await generateAssistantResponse({
+ model: openai('gpt-5-mini'),
+ messages: [{ id: '1', role: 'user', parts: [{ type: 'text', text: input }] }],
+ tools: await getMockTools(),
+ })
+
+ // `result.toolCalls` only shows the last step, instead aggregate tools across all steps
+ const steps = await result.steps
+
+ const stepsSerialized = steps
+ .map((step) => {
+ const toolCalls = step.toolCalls
+ ?.map((call) => JSON.stringify({ tool: call.toolName, input: call.input }))
+ .join('\n')
+
+ const text = step.text
+ return toolCalls ? `${text}\n${toolCalls}` : text
+ })
+ .join('\n')
+
+ const toolNames: string[] = []
+ const sqlQueries: string[] = []
+ const docs: string[] = []
+
+ for (const step of steps) {
+ for (const [i, toolCall] of step.toolCalls.entries()) {
+ toolNames.push(toolCall.toolName)
+
+ const toolResult = step.toolResults.at(i)
+ if (!toolResult) {
+ continue
+ }
+
+ const parsed = parseToolCall(toolCall, toolResult)
+
+ if (parsed.sqlQuery) {
+ sqlQueries.push(parsed.sqlQuery)
+ }
+ if (parsed.docs) {
+ docs.push(...parsed.docs)
+ }
+ }
+ }
+
+ return {
+ stepsSerialized,
+ toolNames,
+ sqlQueries,
+ docs,
+ }
+ },
+ scores: [
+ toolUsageScorer,
+ sqlSyntaxScorer,
+ goalCompletionScorer,
+ concisenessScorer,
+ completenessScorer,
+ ],
+})
+
+type ParsedToolCall = {
+ /** Query generated by `execute_sql` */
+ sqlQuery?: string
+
+ /** Docs text pulled in from `search_docs` */
+ docs?: string[]
+}
+
+/**
+ * Validate and extract relevant info from a tool call/result
+ */
+function parseToolCall(
+ toolCall: TypedToolCall<ToolSet>,
+ toolResult: TypedToolResult<ToolSet>
+): ParsedToolCall {
+ switch (toolCall.toolName) {
+ case 'execute_sql': {
+ const sqlQuery = toolCall.input.sql
+ if (typeof sqlQuery !== 'string') {
+ return {}
+ }
+
+ return { sqlQuery }
+ }
+ case 'search_docs': {
+ const content = toolResult.output.content
+ if (!content || !Array.isArray(content)) {
+ return {}
+ }
+
+ const docs = content.map((item) => item?.text).filter((text) => typeof text === 'string')
+ if (docs.length === 0) {
+ return {}
+ }
+
+ return { docs }
+ }
+ }
+
+ return {}
+}
diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.ts
new file mode 100644
index 00000000000..58c31926091
--- /dev/null
+++ b/apps/studio/evals/dataset.ts
@@ -0,0 +1,38 @@
+import { AssistantEvalCase } from './scorer'
+
+export const dataset: AssistantEvalCase[] = [
+ {
+ input: 'How do I run WASM in edge functions? Use `search_docs`.',
+ expected: { requiredTools: ['search_docs'] },
+ metadata: { category: ['general_help'] },
+ },
+ {
+ input: 'Check if my project is having issues right now and tell me what to fix first.',
+ expected: {
+ requiredTools: ['get_advisors', 'get_logs'],
+ },
+ metadata: { category: ['debugging', 'rls_policies'] },
+ },
+ {
+ input: 'Create a new table "foods" with columns for "name" and "color"',
+ expected: {
+ requiredTools: ['execute_sql'],
+ },
+ metadata: { category: ['sql_generation', 'schema_design'] },
+ },
+ {
+ input:
+ 'Write a SQL query to select all products from the products table where the price is greater than 100',
+ expected: {
+ requiredTools: ['execute_sql'],
+ },
+ metadata: { category: ['sql_generation'] },
+ },
+ {
+ input: 'Create an index on the products table for the name column',
+ expected: {
+ requiredTools: ['execute_sql'],
+ },
+ metadata: { category: ['sql_generation', 'database_optimization'] },
+ },
+]
diff --git a/apps/studio/evals/scorer.ts b/apps/studio/evals/scorer.ts
new file mode 100644
index 00000000000..62f68f41d7c
--- /dev/null
+++ b/apps/studio/evals/scorer.ts
@@ -0,0 +1,159 @@
+import { EvalCase, EvalScorer } from 'braintrust'
+import { LLMClassifierFromTemplate } from 'autoevals'
+import { stripIndent } from 'common-tags'
+import { parse } from 'libpg-query'
+
+const LLM_AS_A_JUDGE_MODEL = 'gpt-5.2-2025-12-11'
+
+type Input = string
+
+type Output = {
+ stepsSerialized: string
+ toolNames: string[]
+ sqlQueries: string[]
+ docs: string[]
+}
+
+export type Expected = {
+ requiredTools?: string[]
+}
+
+// Based on categories in the AssistantMessageRatingSubmittedEvent
+export type AssistantEvalCaseCategory =
+ | 'sql_generation'
+ | 'schema_design'
+ | 'rls_policies'
+ | 'edge_functions'
+ | 'database_optimization'
+ | 'debugging'
+ | 'general_help'
+ | 'other'
+
+export type AssistantEvalCaseMetadata = {
+ category?: AssistantEvalCaseCategory[]
+}
+
+export type AssistantEvalCase = EvalCase<Input, Expected, AssistantEvalCaseMetadata>
+
+export const toolUsageScorer: EvalScorer<Input, Output, Expected> = async ({
+ output,
+ expected,
+}) => {
+ if (!expected.requiredTools) return null
+
+ const presentCount = expected.requiredTools.filter((tool) =>
+ output.toolNames.includes(tool)
+ ).length
+ const totalCount = expected.requiredTools.length
+ const ratio = totalCount === 0 ? 1 : presentCount / totalCount
+
+ return {
+ name: 'Tool Usage',
+ score: ratio,
+ }
+}
+
+export const sqlSyntaxScorer: EvalScorer<Input, Output, Expected> = async ({ output }) => {
+ if (output.sqlQueries === undefined || output.sqlQueries.length === 0) {
+ return null
+ }
+
+ const errors: string[] = []
+ let validQueries = 0
+
+ for (const sql of output.sqlQueries) {
+ try {
+ await parse(sql)
+ validQueries++
+ } catch (error) {
+ const errorMessage = error instanceof Error ? error.message : String(error)
+ errors.push(`SQL syntax error: ${errorMessage}`)
+ }
+ }
+
+ return {
+ name: 'SQL Validity',
+ score: validQueries / output.sqlQueries.length,
+ metadata: errors.length > 0 ? { errors } : undefined,
+ }
+}
+
+const concisenessEvaluator = LLMClassifierFromTemplate<{ input: string }>({
+ name: 'Conciseness',
+ promptTemplate: stripIndent`
+ Evaluate the conciseness of this response.
+
+ Input: {{input}}
+ Output: {{output}}
+
+ Is the response concise and free of unnecessary words?
+ a) Very concise - no wasted words
+ b) Acceptable verbosity - some verbosity but acceptable
+ c) Too verbose - contains superfluous wording or overly verbose
+ `,
+ choiceScores: { a: 1, b: 0.5, c: 0 },
+ useCoT: true,
+ model: LLM_AS_A_JUDGE_MODEL,
+})
+
+export const concisenessScorer: EvalScorer<Input, Output, Expected> = async ({ input, output }) => {
+ return await concisenessEvaluator({
+ input,
+ output: output.stepsSerialized,
+ })
+}
+
+const completenessEvaluator = LLMClassifierFromTemplate<{ input: string }>({
+ name: 'Completeness',
+ promptTemplate: stripIndent`
+ Evaluate whether this response is complete and finished, or if it appears cut off or incomplete.
+
+ Input: {{input}}
+ Output: {{output}}
+
+ Does the response appear complete and finished?
+ a) Complete - response is complete and finished
+ b) Incomplete - response appears cut off, missing parts, or severely incomplete
+ `,
+ choiceScores: { a: 1, b: 0 },
+ useCoT: true,
+ model: LLM_AS_A_JUDGE_MODEL,
+})
+
+export const completenessScorer: EvalScorer<Input, Output, Expected> = async ({
+ input,
+ output,
+}) => {
+ return await completenessEvaluator({
+ input,
+ output: output.stepsSerialized,
+ })
+}
+
+const goalCompletionEvaluator = LLMClassifierFromTemplate<{ input: string }>({
+ name: 'Goal Completion',
+ promptTemplate: stripIndent`
+ Evaluate whether this response addresses what the user asked.
+
+ Input: {{input}}
+ Output: {{output}}
+
+ Does the response address what the user asked?
+ a) Fully addresses - completely answers the question or fulfills the request
+ b) Partially addresses - addresses some aspects but misses key parts
+ c) Doesn't address - off-topic or fails to address the request
+ `,
+ choiceScores: { a: 1, b: 0.5, c: 0 },
+ useCoT: true,
+ model: LLM_AS_A_JUDGE_MODEL,
+})
+
+export const goalCompletionScorer: EvalScorer<Input, Output, Expected> = async ({
+ input,
+ output,
+}) => {
+ return await goalCompletionEvaluator({
+ input,
+ output: output.stepsSerialized,
+ })
+}
diff --git a/apps/studio/lib/ai/generate-assistant-response.ts b/apps/studio/lib/ai/generate-assistant-response.ts
new file mode 100644
index 00000000000..1f35a8223d1
--- /dev/null
+++ b/apps/studio/lib/ai/generate-assistant-response.ts
@@ -0,0 +1,130 @@
+import * as ai from 'ai'
+import {
+ convertToModelMessages,
+ isToolUIPart,
+ type LanguageModel,
+ type ModelMessage,
+ stepCountIs,
+ type ToolSet,
+ type UIMessage,
+} from 'ai'
+import { wrapAISDK } from 'braintrust'
+import { source } from 'common-tags'
+
+import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'
+import {
+ CHAT_PROMPT,
+ EDGE_FUNCTION_PROMPT,
+ GENERAL_PROMPT,
+ PG_BEST_PRACTICES,
+ RLS_PROMPT,
+ REALTIME_PROMPT,
+ SECURITY_PROMPT,
+ LIMITATIONS_PROMPT,
+} from 'lib/ai/prompts'
+import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'
+
+const { streamText } = wrapAISDK(ai)
+
+export async function generateAssistantResponse({
+ messages: rawMessages,
+ model,
+ tools,
+ aiOptInLevel = 'schema',
+ getSchemas,
+ projectRef,
+ chatName,
+ promptProviderOptions,
+ providerOptions,
+ abortSignal,
+}: {
+ messages: UIMessage[]
+ model: LanguageModel
+ tools: ToolSet
+ aiOptInLevel?: AiOptInLevel
+ getSchemas?: () => Promise<string>
+ projectRef?: string
+ chatName?: string
+ promptProviderOptions?: Record<string, any>
+ providerOptions?: Record<string, any>
+ abortSignal?: AbortSignal
+}) {
+ // Only returns last 7 messages
+ // Filters out tools with invalid states
+ // Filters out tool outputs based on opt-in level using renderingToolOutputParser
+ const messages = (rawMessages || []).slice(-7).map((msg) => {
+ if (msg && msg.role === 'assistant' && 'results' in msg) {
+ const cleanedMsg = { ...msg }
+ delete cleanedMsg.results
+ return cleanedMsg
+ }
+ if (msg && msg.role === 'assistant' && msg.parts) {
+ const cleanedParts = msg.parts
+ .filter((part) => {
+ if (isToolUIPart(part)) {
+ const invalidStates = ['input-streaming', 'input-available', 'output-error']
+ return !invalidStates.includes(part.state)
+ }
+ return true
+ })
+ .map((part) => {
+ return sanitizeMessagePart(part, aiOptInLevel)
+ })
+ return { ...msg, parts: cleanedParts }
+ }
+ return msg
+ })
+
+ const schemasString =
+ aiOptInLevel !== 'disabled' && getSchemas
+ ? await getSchemas()
+ : "You don't have access to any schemas."
+
+ // Important: do not use dynamic content in the system prompt or Bedrock will not cache it
+ const system = source`
+ ${GENERAL_PROMPT}
+ ${CHAT_PROMPT}
+ ${PG_BEST_PRACTICES}
+ ${RLS_PROMPT}
+ ${EDGE_FUNCTION_PROMPT}
+ ${REALTIME_PROMPT}
+ ${SECURITY_PROMPT}
+ ${LIMITATIONS_PROMPT}
+ `
+
+ // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`
+ // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88
+ const assistantContent =
+ projectRef || chatName || schemasString !== "You don't have access to any schemas."
+ ? `The user's current project is ${projectRef || 'unknown'}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName || 'unnamed'}`
+ : undefined
+
+ const coreMessages: ModelMessage[] = [
+ {
+ role: 'system',
+ content: system,
+ ...(promptProviderOptions && {
+ providerOptions: promptProviderOptions,
+ }),
+ },
+ ...(assistantContent
+ ? [
+ {
+ role: 'assistant' as const,
+ // Add any dynamic context here
+ content: assistantContent,
+ },
+ ]
+ : []),
+ ...convertToModelMessages(messages),
+ ]
+
+ return streamText({
+ model,
+ stopWhen: stepCountIs(5),
+ messages: coreMessages,
+ ...(providerOptions && { providerOptions }),
+ tools,
+ ...(abortSignal && { abortSignal }),
+ })
+}
diff --git a/apps/studio/lib/ai/tools/mock-tools.ts b/apps/studio/lib/ai/tools/mock-tools.ts
new file mode 100644
index 00000000000..7802112589f
--- /dev/null
+++ b/apps/studio/lib/ai/tools/mock-tools.ts
@@ -0,0 +1,324 @@
+import { tool, type ToolSet } from 'ai'
+import { getRenderingTools } from '../tools/rendering-tools'
+import { z } from 'zod'
+import { getMcpTools } from 'lib/ai/tools/mcp-tools'
+import assert from 'node:assert'
+
+const listTablesInputSchema = z.object({
+ schemas: z.array(z.string()).describe('The schema names to list.'),
+})
+
+const getAdvisorsInputSchema = z.object({
+ type: z.enum(['security', 'performance']).optional(),
+})
+
+const getLogsInputSchema = z.object({
+ limit: z.number().min(1).max(100).optional(),
+ level: z.enum(['debug', 'info', 'warning', 'error']).optional(),
+ source: z.enum(['postgres', 'auth', 'storage', 'edge_function']).optional(),
+ search: z.string().optional(),
+})
+
+const listPoliciesInputSchema = z.object({
+ schemas: z.array(z.string()).describe('The schema names to get the policies for'),
+})
+
+const MOCK_TABLES_DATA = [
+ {
+ name: 'user_documents',
+ rls_enabled: false,
+ columns: [
+ { name: 'id', data_type: 'bigint' },
+ { name: 'user_id', data_type: 'uuid' },
+ { name: 'title', data_type: 'text' },
+ ],
+ },
+ {
+ name: 'customers',
+ rls_enabled: true,
+ columns: [
+ { name: 'id', data_type: 'uuid' },
+ { name: 'tenant_id', data_type: 'uuid' },
+ { name: 'email', data_type: 'text' },
+ ],
+ },
+ {
+ name: 'projects',
+ rls_enabled: false,
+ columns: [
+ { name: 'id', data_type: 'uuid' },
+ { name: 'organization_id', data_type: 'uuid' },
+ { name: 'name', data_type: 'text' },
+ ],
+ },
+ {
+ name: 'user_organizations',
+ rls_enabled: true,
+ columns: [
+ { name: 'user_id', data_type: 'uuid' },
+ { name: 'organization_id', data_type: 'uuid' },
+ ],
+ },
+]
+
+const MOCK_EXTENSIONS_DATA = [
+ { name: 'pgcrypto', schema: 'extensions', installed_version: '1.3' },
+ { name: 'uuid-ossp', schema: 'extensions', installed_version: '1.1' },
+]
+
+const MOCK_EDGE_FUNCTIONS_DATA = [
+ { name: 'hello-world', last_deployed_at: '2024-06-10T12:30:00Z' },
+ { name: 'daily-metrics-sync', last_deployed_at: '2024-06-18T08:15:00Z' },
+ { name: 'select-from-table-with-auth-rls', last_deployed_at: '2024-06-19T09:20:00Z' },
+]
+
+const MOCK_ADVISORIES_DATA = [
+ {
+ id: '0016_materialized_view_in_api',
+ level: 'warning',
+ category: 'security',
+ message: 'Materialized views in API schema can bypass RLS. Move them to private schema.',
+ remediationUrl:
+ 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0016_materialized_view_in_api',
+ },
+ {
+ id: '0031_functions_no_rls_guard',
+ level: 'notice',
+ category: 'security',
+ message: 'Function api.health_check should verify auth context before querying tables.',
+ remediationUrl:
+ 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0031_functions_no_rls_guard',
+ },
+ {
+ id: '1012_slow_query',
+ level: 'info',
+ category: 'performance',
+ message:
+ 'Query on table edge_function_logs exceeded 3s average execution time over the last hour.',
+ remediationUrl: 'https://supabase.com/docs/guides/platform/performance-advisors#slow-queries',
+ },
+]
+
+const MOCK_LOGS_DATA = [
+ {
+ id: 'log-001',
+ timestamp: '2024-06-20T14:12:00Z',
+ level: 'error',
+ source: 'edge_function' as const,
+ target: 'hello-world',
+ message: "TypeError: fetch failed at await supabase.functions.invoke('analytics')",
+ },
+ {
+ id: 'log-002',
+ timestamp: '2024-06-20T14:05:30Z',
+ level: 'warning',
+ source: 'postgres' as const,
+ target: 'connection_pool',
+ message: 'Query timeout exceeded for statement SELECT * FROM public.audit_log_entries',
+ },
+ {
+ id: 'log-003',
+ timestamp: '2024-06-20T13:59:10Z',
+ level: 'info',
+ source: 'edge_function' as const,
+ target: 'daily-metrics-sync',
+ message: 'Invocation completed in 520ms',
+ },
+ {
+ id: 'log-004',
+ timestamp: '2024-06-20T13:50:00Z',
+ level: 'error',
+ source: 'postgres' as const,
+ target: 'trigger:refresh_materialized_views',
+ message: 'permission denied for relation user_documents',
+ },
+ {
+ id: 'log-005',
+ timestamp: '2024-06-20T13:45:00Z',
+ level: 'info',
+ source: 'auth' as const,
+ target: 'email-confirmation',
+ message: 'Sent verification email to [email protected]',
+ },
+]
+
+function createMockedRenderingTools() {
+ const renderingTools = getRenderingTools()
+
+ return Object.fromEntries(
+ Object.entries(renderingTools).map(([name, baseTool]) => {
+ if (typeof baseTool.execute === 'function') {
+ return [name, baseTool]
+ }
+
+ const statusMessage =
+ name === 'execute_sql'
+ ? 'SQL execution mocked successfully.'
+ : name === 'deploy_edge_function'
+ ? 'Edge Function deployment mocked successfully.'
+ : 'Tool call mocked successfully.'
+
+ return [
+ name,
+ {
+ ...baseTool,
+ execute: async () => ({ status: statusMessage }),
+ },
+ ]
+ })
+ ) as typeof renderingTools
+}
+
+function createMockListTablesTool() {
+ return tool({
+ description: 'Lists tables and columns for the provided schemas.',
+ inputSchema: listTablesInputSchema,
+ execute: async ({ schemas }: { schemas: string[] }) => {
+ const effectiveSchemas = schemas?.length ? schemas : ['public']
+ return effectiveSchemas.map((schema) => ({
+ schema,
+ tables: MOCK_TABLES_DATA,
+ }))
+ },
+ })
+}
+
+function createMockListExtensionsTool() {
+ return tool({
+ description: 'Lists installed database extensions.',
+ inputSchema: z.object({}),
+ execute: async () => {
+ return MOCK_EXTENSIONS_DATA
+ },
+ })
+}
+
+function createMockListEdgeFunctionsTool() {
+ return tool({
+ description: 'Lists available Supabase Edge Functions.',
+ inputSchema: z.object({}),
+ execute: async () => {
+ return MOCK_EDGE_FUNCTIONS_DATA
+ },
+ })
+}
+
+function createMockGetAdvisorsTool() {
+ return tool({
+ description: 'Returns advisory notices for the project (mocked).',
+ inputSchema: getAdvisorsInputSchema,
+ execute: async ({ type }: { type?: 'security' | 'performance' }) => {
+ if (type) {
+ return MOCK_ADVISORIES_DATA.filter((advisory) => advisory.category === type)
+ }
+ return MOCK_ADVISORIES_DATA
+ },
+ })
+}
+
+function createMockGetLogsTool() {
+ return tool({
+ description: 'Fetches recent project logs for debugging or health checks (mocked).',
+ inputSchema: getLogsInputSchema,
+ execute: async ({
+ limit = 10,
+ level,
+ source,
+ search,
+ }: {
+ limit?: number
+ level?: 'debug' | 'info' | 'warning' | 'error'
+ source?: 'postgres' | 'auth' | 'storage' | 'edge_function'
+ search?: string
+ }) => {
+ let filtered = MOCK_LOGS_DATA
+
+ if (level) {
+ filtered = filtered.filter((entry) => entry.level === level)
+ }
+
+ if (source) {
+ filtered = filtered.filter((entry) => entry.source === source)
+ }
+
+ if (search) {
+ const needle = search.toLowerCase()
+ filtered = filtered.filter((entry) =>
+ `${entry.message} ${entry.target}`.toLowerCase().includes(needle)
+ )
+ }
+
+ return filtered.slice(0, limit)
+ },
+ })
+}
+
+function createMockListPoliciesTool() {
+ return tool({
+ description: 'Get existing RLS policies for provided schemas.',
+ inputSchema: listPoliciesInputSchema,
+ execute: async ({ schemas }: { schemas: string[] }) => {
+ const effectiveSchemas = schemas?.length ? schemas : ['public']
+ const results = [] as Array<{
+ schema: string
+ table: string
+ policies: Array<{
+ name: string
+ command: 'select' | 'insert' | 'update' | 'delete'
+ using?: string
+ check?: string
+ }>
+ }>
+
+ for (const schema of effectiveSchemas) {
+ if (schema !== 'public') continue
+ results.push(
+ {
+ schema,
+ table: 'customers',
+ policies: [
+ {
+ name: 'customers_tenant_select',
+ command: 'select',
+ using: "(auth.jwt() ->> 'tenant_id')::uuid = tenant_id",
+ },
+ ],
+ },
+ { schema, table: 'user_documents', policies: [] },
+ { schema, table: 'projects', policies: [] }
+ )
+ }
+ return results
+ },
+ })
+}
+
+/**
+ * Deterministic mock implementations of MCP/platform tools for evals.
+ * These mirror tool names used in prompts so the model can call them,
+ * but return stable, static data for repeatable tests.
+ *
+ * Note: search_docs uses the real implementation
+ */
+export async function getMockTools() {
+ const mockedRenderingTools = createMockedRenderingTools()
+
+ const { search_docs } = await getMcpTools({
+ accessToken: 'mock-access-token',
+ projectRef: 'mock-project-ref',
+ aiOptInLevel: 'schema_and_log_and_data',
+ })
+
+ assert(search_docs, 'search_docs tool not available from MCP server')
+
+ return {
+ ...mockedRenderingTools,
+ search_docs,
+ list_tables: createMockListTablesTool(),
+ list_extensions: createMockListExtensionsTool(),
+ list_edge_functions: createMockListEdgeFunctionsTool(),
+ get_advisors: createMockGetAdvisorsTool(),
+ get_logs: createMockGetLogsTool(),
+ list_policies: createMockListPoliciesTool(),
+ }
+}
diff --git a/apps/studio/lib/api/generate-v4.test.ts b/apps/studio/lib/api/generate-v4.test.ts
index a04be6ef01a..d584b7e5222 100644
--- a/apps/studio/lib/api/generate-v4.test.ts
+++ b/apps/studio/lib/api/generate-v4.test.ts
@@ -2,9 +2,10 @@ import { expect, test, vi } from 'vitest'
// End of third-party imports
import generateV4 from '../../pages/api/ai/sql/generate-v4'
-import { sanitizeMessagePart } from '../ai/tools/tool-sanitizer'
+import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'
+import { UIMessage } from 'ai'
-vi.mock('../ai/tools/tool-sanitizer', () => ({
+vi.mock('lib/ai/tools/tool-sanitizer', () => ({
sanitizeMessagePart: vi.fn((part) => part),
}))
@@ -17,20 +18,24 @@ test('generateV4 calls the tool sanitizer', async () => {
body: {
messages: [
{
+ id: 'test-msg-id',
role: 'assistant',
parts: [
{
type: 'tool-execute_sql',
state: 'output-available',
- output: 'test output',
+ toolCallId: 'test-tool-call-id',
+ input: { sql: 'SELECT * FROM users' },
+ output: [{ id: 1, name: 'test-output' }],
},
],
},
- ],
+ ] satisfies UIMessage[],
projectRef: 'test-project',
connectionString: 'test-connection',
orgSlug: 'test-org',
},
+ on: vi.fn(),
}
const mockRes = {
@@ -63,13 +68,15 @@ test('generateV4 calls the tool sanitizer', async () => {
getTools: vi.fn().mockResolvedValue({}),
}))
- vi.mock('ai', () => ({
- streamText: vi.fn().mockReturnValue({
- pipeUIMessageStreamToResponse: vi.fn(),
- }),
- convertToModelMessages: vi.fn((msgs) => msgs),
- stepCountIs: vi.fn(),
- }))
+ vi.mock('ai', async () => {
+ const actual = await vi.importActual('ai')
+ return {
+ ...actual,
+ streamText: vi.fn().mockReturnValue({
+ pipeUIMessageStreamToResponse: vi.fn(),
+ }),
+ }
+ })
await generateV4(mockReq as any, mockRes as any)
diff --git a/apps/studio/package.json b/apps/studio/package.json
index aafb78a3d81..58684683e56 100644
--- a/apps/studio/package.json
+++ b/apps/studio/package.json
@@ -23,7 +23,10 @@
"prettier:write": "prettier --cache --write .",
"build:deno-types": "tsx scripts/deno-types.ts",
"build:graphql-types": "tsx scripts/download-graphql-schema.mts && pnpm graphql-codegen --config scripts/codegen.ts",
- "build:graphql-types:watch": "pnpm graphql-codegen --config scripts/codegen.ts --watch"
+ "build:graphql-types:watch": "pnpm graphql-codegen --config scripts/codegen.ts --watch",
+ "evals:setup": "cp node_modules/libpg-query/wasm/libpg-query.wasm evals/libpg-query.wasm",
+ "evals:run": "braintrust eval --no-send-logs evals/assistant.eval.ts",
+ "evals:upload": "braintrust eval evals/assistant.eval.ts"
},
"dependencies": {
"@ai-sdk/amazon-bedrock": "^3.0.0",
@@ -183,6 +186,8 @@
"@vitest/coverage-v8": "^3.2.0",
"@vitest/ui": "^3.2.0",
"api-types": "workspace:*",
+ "autoevals": "^0.0.131",
+ "braintrust": "^1.0.2",
"common": "workspace:*",
"config": "workspace:*",
"date-fns": "^2.30.0",
@@ -191,6 +196,7 @@
"graphql-ws": "5.14.1",
"import-in-the-middle": "^2.0.0",
"jsdom-testing-mocks": "^1.13.1",
+ "libpg-query": "17.6.0",
"msw": "^2.3.0",
"next-router-mock": "^0.9.13",
"node-mocks-http": "^1.17.2",
diff --git a/apps/studio/pages/api/ai/sql/generate-v4.ts b/apps/studio/pages/api/ai/sql/generate-v4.ts
index ecd441f5030..93aa261b14c 100644
--- a/apps/studio/pages/api/ai/sql/generate-v4.ts
+++ b/apps/studio/pages/api/ai/sql/generate-v4.ts
@@ -1,6 +1,5 @@
import pgMeta from '@supabase/pg-meta'
-import { convertToModelMessages, type ModelMessage, stepCountIs, streamText } from 'ai'
-import { source } from 'common-tags'
+import { safeValidateUIMessages } from 'ai'
import type { NextApiRequest, NextApiResponse } from 'next'
import z from 'zod'
@@ -9,18 +8,8 @@ import { executeSql } from 'data/sql/execute-sql-query'
import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'
import { getModel } from 'lib/ai/model'
import { getOrgAIDetails } from 'lib/ai/org-ai-details'
-import {
- CHAT_PROMPT,
- EDGE_FUNCTION_PROMPT,
- GENERAL_PROMPT,
- PG_BEST_PRACTICES,
- RLS_PROMPT,
- REALTIME_PROMPT,
- SECURITY_PROMPT,
- LIMITATIONS_PROMPT,
-} from 'lib/ai/prompts'
+import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'
import { getTools } from 'lib/ai/tools'
-import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'
import apiWrapper from 'lib/api/apiWrapper'
import { executeQuery } from 'lib/api/self-hosted/query'
@@ -89,6 +78,14 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
model: requestedModel,
} = data
+ const messagesValidation = await safeValidateUIMessages({ messages: rawMessages })
+ if (!messagesValidation.success) {
+ return res
+ .status(400)
+ .json({ error: 'Invalid request body', message: messagesValidation.error.message })
+ }
+ const messages = messagesValidation.data
+
let aiOptInLevel: AiOptInLevel = 'disabled'
let isLimited = false
@@ -114,32 +111,6 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
}
}
- // Only returns last 7 messages
- // Filters out tools with invalid states
- // Filters out tool outputs based on opt-in level using renderingToolOutputParser
- const messages = (rawMessages || []).slice(-7).map((msg: any) => {
- if (msg && msg.role === 'assistant' && 'results' in msg) {
- const cleanedMsg = { ...msg }
- delete cleanedMsg.results
- return cleanedMsg
- }
- if (msg && msg.role === 'assistant' && msg.parts) {
- const cleanedParts = msg.parts
- .filter((part: any) => {
- if (part.type.startsWith('tool-')) {
- const invalidStates = ['input-streaming', 'input-available', 'output-error']
- return !invalidStates.includes(part.state)
- }
- return true
- })
- .map((part: any) => {
- return sanitizeMessagePart(part, aiOptInLevel)
- })
- return { ...msg, parts: cleanedParts }
- }
- return msg
- })
-
const {
model,
error: modelError,
@@ -157,67 +128,10 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
}
try {
- // Get a list of all schemas to add to context
- const pgMetaSchemasList = pgMeta.schemas.list()
- type Schemas = z.infer<(typeof pgMetaSchemasList)['zod']>
-
- const { result: schemas } =
- aiOptInLevel !== 'disabled'
- ? await executeSql<Schemas>(
- {
- projectRef,
- connectionString,
- sql: pgMetaSchemasList.sql,
- },
- undefined,
- {
- 'Content-Type': 'application/json',
- ...(authorization && { Authorization: authorization }),
- },
- IS_PLATFORM ? undefined : executeQuery
- )
- : { result: [] }
-
- const schemasString =
- schemas?.length > 0
- ? `The available database schema names are: ${JSON.stringify(schemas)}`
- : "You don't have access to any schemas."
-
- // Important: do not use dynamic content in the system prompt or Bedrock will not cache it
- const system = source`
- ${GENERAL_PROMPT}
- ${CHAT_PROMPT}
- ${PG_BEST_PRACTICES}
- ${RLS_PROMPT}
- ${EDGE_FUNCTION_PROMPT}
- ${REALTIME_PROMPT}
- ${SECURITY_PROMPT}
- ${LIMITATIONS_PROMPT}
- `
-
- // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`
- // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88
- const coreMessages: ModelMessage[] = [
- {
- role: 'system',
- content: system,
- ...(promptProviderOptions && {
- providerOptions: promptProviderOptions,
- }),
- },
- {
- role: 'assistant',
- // Add any dynamic context here
- content: `The user's current project is ${projectRef}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName}`,
- },
- ...convertToModelMessages(messages),
- ]
-
const abortController = new AbortController()
req.on('close', () => abortController.abort())
req.on('aborted', () => abortController.abort())
- // Get tools
const tools = await getTools({
projectRef,
connectionString,
@@ -226,12 +140,40 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {
accessToken,
})
- const result = streamText({
+ // Get a list of all schemas to add to context
+ const getSchemas = async (): Promise<string> => {
+ const pgMetaSchemasList = pgMeta.schemas.list()
+ type Schemas = z.infer<(typeof pgMetaSchemasList)['zod']>
+
+ const { result: schemas } = await executeSql<Schemas>(
+ {
+ projectRef,
+ connectionString,
+ sql: pgMetaSchemasList.sql,
+ },
+ undefined,
+ {
+ 'Content-Type': 'application/json',
+ ...(authorization && { Authorization: authorization }),
+ },
+ IS_PLATFORM ? undefined : executeQuery
+ )
+
+ return schemas?.length > 0
+ ? `The available database schema names are: ${JSON.stringify(schemas)}`
+ : "You don't have access to any schemas."
+ }
+
+ const result = await generateAssistantResponse({
+ messages,
model,
- stopWhen: stepCountIs(5),
- messages: coreMessages,
- ...(providerOptions && { providerOptions }),
tools,
+ aiOptInLevel,
+ getSchemas: aiOptInLevel !== 'disabled' ? getSchemas : undefined,
+ projectRef,
+ chatName,
+ promptProviderOptions,
+ providerOptions,
abortSignal: abortController.signal,
})
diff --git a/package.json b/package.json
index ac42d0ed66b..a35f01b65ab 100644
--- a/package.json
+++ b/package.json
@@ -49,7 +49,7 @@
"prettier-plugin-sql-cst": "^0.11.0",
"rimraf": "^6.0.0",
"sass": "^1.72.0",
- "supabase": "^2.58.5",
+ "supabase": "^2.65.6",
"supports-color": "^8.0.0",
"tailwindcss": "catalog:",
"turbo": "2.3.3",
diff --git a/packages/shared-data/package.json b/packages/shared-data/package.json
index 2533f1c5331..9aacccfae9e 100644
--- a/packages/shared-data/package.json
+++ b/packages/shared-data/package.json
@@ -8,6 +8,9 @@
"preinstall": "npx only-allow pnpm",
"clean": "rimraf node_modules"
},
+ "devDependencies": {
+ "tsconfig": "workspace:"
+ },
"author": "",
"license": "MIT"
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1febf67dc73..a046f17af6c 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -6,12 +6,63 @@ settings:
catalogs:
default:
+ '@sentry/nextjs':
+ specifier: ^10.26.0
+ version: 10.27.0
+ '@supabase/auth-js':
+ specifier: 2.87.0
+ version: 2.87.0
+ '@supabase/postgrest-js':
+ specifier: 2.87.0
+ version: 2.87.0
+ '@supabase/realtime-js':
+ specifier: 2.87.0
+ version: 2.87.0
'@supabase/supabase-js':
specifier: 2.87.0
version: 2.87.0
+ '@types/node':
+ specifier: ^22.0.0
+ version: 22.13.14
+ '@types/react':
+ specifier: ^18.3.0
+ version: 18.3.3
+ '@types/react-dom':
+ specifier: ^18.3.0
+ version: 18.3.0
+ next:
+ specifier: ^15.5.9
+ version: 15.5.9
+ react:
+ specifier: ^18.3.0
+ version: 18.3.1
+ react-dom:
+ specifier: ^18.3.0
+ version: 18.3.1
+ recharts:
+ specifier: ^2.15.4
+ version: 2.15.4
+ tailwindcss:
+ specifier: 3.4.1
+ version: 3.4.1
tsx:
specifier: 4.20.3
version: 4.20.3
+ typescript:
+ specifier: ~5.9.0
+ version: 5.9.2
+ valtio:
+ specifier: ^1.12.0
+ version: 1.12.0
+ vite:
+ specifier: ^7.1.11
+ version: 7.1.11
+ vitest:
+ specifier: ^3.2.0
+ version: 3.2.4
+ zod:
+ specifier: ^3.25.76
+ version: 3.25.76
overrides:
'@eslint/eslintrc>js-yaml': ^4.1.1
@@ -52,8 +103,8 @@ importers:
specifier: ^1.72.0
version: 1.72.0
supabase:
- specifier: ^2.58.5
- version: 2.58.5([email protected])
+ specifier: ^2.65.6
+ version: 2.67.1([email protected])
supports-color:
specifier: ^8.0.0
version: 8.1.1
@@ -489,7 +540,7 @@ importers:
nuqs:
specifier: ^1.19.1
openai:
specifier: ^4.75.1
@@ -1170,6 +1221,12 @@ importers:
api-types:
specifier: workspace:*
version: link:../../packages/api-types
+ autoevals:
+ specifier: ^0.0.131
+ braintrust:
+ specifier: ^1.0.2
date-fns:
specifier: ^2.30.0
version: 2.30.0
@@ -1188,6 +1245,9 @@ importers:
jsdom-testing-mocks:
specifier: ^1.13.1
version: 1.13.1
+ libpg-query:
+ specifier: 17.6.0
+ version: 17.6.0
msw:
specifier: ^2.3.0
version: 2.11.3(@types/[email protected])([email protected])
@@ -2173,7 +2233,11 @@ importers:
specifier: 'catalog:'
- packages/shared-data: {}
+ packages/shared-data:
+ devDependencies:
+ tsconfig:
+ specifier: 'workspace:'
+ version: link:../tsconfig
packages/tsconfig: {}
@@ -2671,6 +2735,10 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==}
+ engines: {node: '>=18'}
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==}
engines: {node: '>=18'}
@@ -3244,6 +3312,10 @@ packages:
react: ^16.8.0 || ^17 || ^18 || ^19
react-dom: ^16.8.0 || ^17 || ^18 || ^19
+ '@colors/[email protected]':
+ resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
+ engines: {node: '>=0.1.90'}
+
'@contentlayer2/[email protected]':
resolution: {integrity: sha512-ZJ+Iiu2rVI50x60XoqnrsO/Q8eqFX5AlP1L0U/3ygaAas3tnOqTzQZ1UsxYQMpJzcLok24ddlhKfQKbCMUJPiQ==}
@@ -4916,6 +4988,9 @@ packages:
resolution: {integrity: sha512-6IajJ23QrXW5RTJj2lRHcBM8mxcEl+vgd7XXVODQG/BcJyjgIP1k5OefdRl+P80btPvHeHoV4fIgC1so25pXcg==}
+ resolution: {integrity: sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==}
+
resolution: {integrity: sha512-4GlTZ+EJM7WaW2HEZcyU317tIQDjkQIyENDLxYJfSWlfqguN+dHkZgyQTV/7ykvobU7yEH5gKvreNrH4B6QgIg==}
@@ -6190,6 +6265,9 @@ packages:
resolution: {integrity: sha512-K3gtnbqbSUuUVmPm143qx5Gy2EmKuooshV95yMD48EUQ1256sgZBriEfY61OWJnlzdREdqHTIOxQqpZAb7XdZg==}
+ '@pgsql/[email protected]':
+ resolution: {integrity: sha512-1UtbELdbqNdyOShhrVfSz3a1gDi0s9XXiQemx+6QqtsrXe62a6zOGU+vjb2GRfG5jeEokI1zBBcfD42enRv0Rw==}
+
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
@@ -9749,6 +9827,15 @@ packages:
'@usercentrics/[email protected]':
resolution: {integrity: sha512-/cik01TdeiYUV1+EasK83Ip05TDqmowM5tmWGfRDdaOrneSic5BedBdJHKya4pP2vwiYAtzDYVJe4BwPt2M16g==}
+ '@vercel/[email protected]':
+ resolution: {integrity: sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA==}
+ engines: {node: '>= 16'}
+ peerDependencies:
+ '@aws-sdk/credential-provider-web-identity': '*'
+ peerDependenciesMeta:
+ '@aws-sdk/credential-provider-web-identity':
+ optional: true
+
'@vercel/[email protected]':
resolution: {integrity: sha512-1gSbK9zfrbJxk1JTBVERDhLi01mK3fz+gw4GjOjZwHnqs0zsBhQA70HGVtXQX/Z3BTRMfbpAEMVDfhecRw0lDA==}
engines: {node: '>= 18'}
@@ -10358,6 +10445,9 @@ packages:
resolution: {integrity: sha512-Hdw8qdNiqdJ8LqT0iK0sVzkFbzg6fhnQqqfWhBDxcHZvU75+B+ayzTy8x+k5Ix0Y92XOhOUlx74ps+bA6BeYMQ==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-F+3lraja+Ms7n1M2cpWl65N7AYx4sPocRW454H5HlSGabYMfuFOUxw8IXmEYDkQ38BxtZ0Wd5ZAQj9RF59YJWw==}
+
resolution: {integrity: sha512-zQAFO1Dlsn69eXaO6+7YZc+v84aquQKbwpzCE3L0stj56ERn9hutFxPopViLjo9G+rWwjozRhgS5KJ25Xy19cQ==}
@@ -10458,6 +10548,9 @@ packages:
resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==}
+
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
@@ -10491,6 +10584,10 @@ packages:
resolution: {integrity: sha512-2hCgjEmP8YLWQ130n2FerGv7rYpfBmnmp9Uy2Le1vge6X3gZIfSmEzP5QTDElFxcvVcXlEn8Aq6MU/PZygIOog==}
engines: {node: '>=14.16'}
+ resolution: {integrity: sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}
@@ -10501,6 +10598,12 @@ packages:
resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-Zm8WbD18FLv5HN38TyUKw659vJdZALQ5VVVWTncMxZgPtFXeWoC7H0tInw/1EQ1pGq50v7WZtlaFBOXhUaf8DQ==}
+ hasBin: true
+ peerDependencies:
+ zod: ^3.25.34
+
resolution: {integrity: sha512-ECFzp6uFOSB+dcZ5BK/IBaGWssbSYBHvuMeMt3MMFyhI0Z8SqGgEkBLARgpRH3hutIgPVsALcMwbDrJqPxQ65A==}
engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7}
@@ -10593,6 +10696,10 @@ packages:
resolution: {integrity: sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==}
engines: {node: '>=14.16'}
+ resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==}
+ engines: {node: '>=16'}
+
resolution: {integrity: sha512-dU+Tx2fsypxTgtLoE36npi3UqcjSSMNYfkqgmoEhtZrraP5VWq0K7FkWVTYa8eMPtnU/G2txVsfdCJTn9uzpuQ==}
@@ -10682,6 +10789,9 @@ packages:
resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}
engines: {node: '>= 6'}
+ resolution: {integrity: sha512-FRcpVkox+cRovffgqNdDFQ1eUav+i/Vq/CUd1hcfEl2bevntFlzznL+jE8g4twl6ElB7gZjCko6pYpXyMn+6dA==}
+
resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==}
peerDependencies:
@@ -10754,10 +10864,18 @@ packages:
resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==}
+ engines: {node: '>=4'}
+
resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==}
engines: {node: '>=6'}
+ resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==}
+ engines: {node: 10.* || >= 12.*}
+
resolution: {integrity: sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==}
engines: {node: '>=8'}
@@ -10950,6 +11068,15 @@ packages:
resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==}
engines: {node: '>= 14'}
+ resolution: {integrity: sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==}
+
+ resolution: {integrity: sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==}
+
+ resolution: {integrity: sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==}
+
resolution: {integrity: sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw==}
@@ -12363,6 +12490,10 @@ packages:
resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
engines: {node: '>=0.8.x'}
+ resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==}
+ engines: {node: '>=14.18'}
+
resolution: {integrity: sha512-nVpZkTMM9rF6AQ9gPJpFsNAMt48wIzB5TQgiTLdHiuO8XEDhUgZEhqKlZWXbIzo9VmJ/HvysHqEaVeD5v9TPvA==}
engines: {node: '>=20.0.0'}
@@ -12581,6 +12712,9 @@ packages:
resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}
+ resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==}
+
resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==}
engines: {node: '>=8'}
@@ -13597,6 +13731,9 @@ packages:
resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==}
+ resolution: {integrity: sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==}
+
resolution: {integrity: sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==}
engines: {node: '>= 0.4'}
@@ -14247,6 +14384,9 @@ packages:
resolution: {integrity: sha512-fBi14tsU3/ahtfU0C+/tvmtlt0EJM6kXe0Dl4/cSbiNmJ3IwLIuk3lawwiK5ByXU4sXwyAzbkvSl26jVXzYuaA==}
+ resolution: {integrity: sha512-r4zOTcLTGYS5PlLQAicJ6Yi/tvZFag42YUuNEO8pi8bwt/ZZ4kj514J4QV5bOx0mZzPLF6agbfNXQVxGgmHR8g==}
+
resolution: {integrity: sha512-aORPWntbpH5esaYpGOOmri0OHDOe3wC5M2MQxZ9dvMLZm6DnaAn0kJlcbU9hwsQgLzmZyReKwFwwPkR+nHu5kA==}
@@ -14258,6 +14398,9 @@ packages:
resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==}
engines: {node: '>=14'}
+ resolution: {integrity: sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==}
+
resolution: {integrity: sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ==}
@@ -15119,6 +15262,24 @@ packages:
engines: {node: '>=10'}
hasBin: true
+ resolution: {integrity: sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==}
+
+ resolution: {integrity: sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==}
+
+ resolution: {integrity: sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==}
+
+ resolution: {integrity: sha512-TJ+8eOFdp+INvzR4zAuwBQJznDUfktMtOB6g/hUcGh3rcyjxbz4Te57Pgri8Q9bhSQ7Zys4IYOGhFdnlgeB6Lw==}
+
+ resolution: {integrity: sha512-n9aXXxdf+ogpYeU6FLZgClOteAE10DyUMkjeBGnvew9vmewKLfINVbMZBosW4Q1aqFDMsZjaItiJJ/UCApTIKg==}
+
+ resolution: {integrity: sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==}
+
resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==}
@@ -15199,6 +15360,10 @@ packages:
resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==}
+ resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}
+ hasBin: true
+
resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==}
@@ -18212,8 +18377,8 @@ packages:
resolution: {integrity: sha512-TDQg6Nh7e2DaLmMjqpJt7Wwh2Au5loVPH2gwKMkD89e3pb5F/zMQGWt22asEXme031jEmhjazixFH0lwMgbRJA==}
engines: {node: '>=18.0.0'}
- resolution: {integrity: sha512-mYZSkUIePTdmwlHd26Pff8wpmjfre8gcuWzrc5QqhZgZvCXugVzAQQhcjaQisw5kusbPQWNIjUwcHYEKmejhPw==}
+ resolution: {integrity: sha512-d/trGytTjB/Hi625zHh2RFFttjLOkSdRTsY/N1pjxoAfG0blDiHKqPwu12VJYitg4nzgjfhjnD3pLyfU0ko5vg==}
engines: {npm: '>=8'}
hasBin: true
@@ -18316,6 +18481,10 @@ packages:
resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
+ resolution: {integrity: sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA==}
+ engines: {node: '>=12'}
+
resolution: {integrity: sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==}
engines: {node: '>= 10.13.0'}
@@ -19158,6 +19327,12 @@ packages:
resolution: {integrity: sha512-OljLrQ9SQdOUqTaQxqL5dEfZWrXExyyWsozYlAWFawPVNuD83igl7uJD2RTkNMbniIYgt8l81eCJGIdQF7avLQ==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
+ resolution: {integrity: sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==}
+
+ resolution: {integrity: sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==}
+
resolution: {integrity: sha512-co8NkCHeY0NsL0XsL/cSICt5VhTjwZlYT8mi50dYY5thx3r3w1D15A04Lvs9WL/y/Rf98vUKY5PAAJCTLHvkJw==}
engines: {node: '>=12.20.0'}
@@ -19543,6 +19718,10 @@ packages:
resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==}
@@ -19558,6 +19737,10 @@ packages:
resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
@@ -19797,6 +19980,10 @@ snapshots:
eventsource-parser: 3.0.6
zod: 3.25.76
+ '@ai-sdk/[email protected]':
+ dependencies:
+ json-schema: 0.4.0
+
'@ai-sdk/[email protected]':
dependencies:
json-schema: 0.4.0
@@ -21204,6 +21391,9 @@ snapshots:
react-dom: 18.3.1([email protected])
react-is: 17.0.2
+ '@colors/[email protected]':
+ optional: true
+
dependencies:
@@ -23329,6 +23519,8 @@ snapshots:
- bufferutil
- utf-8-validate
+ '@next/[email protected]': {}
+
'@next/[email protected]': {}
'@next/[email protected]': {}
@@ -24823,6 +25015,8 @@ snapshots:
'@pgsql/[email protected]': {}
+ '@pgsql/[email protected]': {}
+
optional: true
@@ -29322,6 +29516,10 @@ snapshots:
lz-string: 1.5.0
uuid: 9.0.1
+ '@vercel/[email protected](@aws-sdk/[email protected])':
+ optionalDependencies:
+ '@aws-sdk/credential-provider-web-identity': 3.830.0
+
optionalDependencies:
'@aws-sdk/credential-provider-web-identity': 3.830.0
@@ -30098,6 +30296,21 @@ snapshots:
+ dependencies:
+ ajv: 8.17.1
+ compute-cosine-similarity: 1.1.0
+ js-levenshtein: 1.1.6
+ js-yaml: 4.1.1
+ linear-sum-assignment: 1.0.9
+ mustache: 4.2.0
+ zod: 3.25.76
+ zod-to-json-schema: 3.25.0([email protected])
+ transitivePeerDependencies:
+ - encoding
+ - ws
+
dependencies:
gulp-header: 1.8.12
@@ -30205,6 +30418,8 @@ snapshots:
+
dependencies:
file-uri-to-path: 1.0.0
@@ -30267,6 +30482,17 @@ snapshots:
widest-line: 4.0.1
wrap-ansi: 8.1.0
+ dependencies:
+ ansi-align: 3.0.1
+ camelcase: 8.0.0
+ chalk: 5.4.1
+ cli-boxes: 3.0.0
+ string-width: 7.2.0
+ type-fest: 4.30.0
+ widest-line: 5.0.0
+ wrap-ansi: 9.0.2
+
dependencies:
balanced-match: 1.0.2
@@ -30280,6 +30506,37 @@ snapshots:
dependencies:
fill-range: 7.1.1
+ dependencies:
+ '@ai-sdk/provider': 1.1.3
+ '@next/env': 14.2.35
+ '@vercel/functions': 1.6.0(@aws-sdk/[email protected])
+ argparse: 2.0.1
+ boxen: 8.0.1
+ chalk: 4.1.2
+ cli-progress: 3.12.0
+ cli-table3: 0.6.5
+ cors: 2.8.5
+ dotenv: 16.5.0
+ esbuild: 0.25.2
+ eventsource-parser: 1.1.2
+ express: 4.22.1([email protected])
+ graceful-fs: 4.2.11
+ http-errors: 2.0.1
+ minimatch: 9.0.5
+ mustache: 4.2.0
+ pluralize: 8.0.0
+ simple-git: 3.28.0([email protected])
+ slugify: 1.6.6
+ source-map: 0.7.6
+ termi-link: 1.1.0
+ uuid: 9.0.1
+ zod: 3.25.76
+ zod-to-json-schema: 3.25.0([email protected])
+ transitivePeerDependencies:
+ - '@aws-sdk/credential-provider-web-identity'
+ - supports-color
+
dependencies:
baseline-browser-mapping: 2.8.5
@@ -30416,6 +30673,8 @@ snapshots:
+
@@ -30534,6 +30793,8 @@ snapshots:
parse5: 7.1.2
parse5-htmlparser2-tree-adapter: 7.0.0
+
dependencies:
chevrotain: 11.0.3
@@ -30606,8 +30867,18 @@ snapshots:
dependencies:
restore-cursor: 5.1.0
+ dependencies:
+ string-width: 4.2.3
+
+ dependencies:
+ string-width: 4.2.3
+ optionalDependencies:
+ '@colors/colors': 1.5.0
+
dependencies:
slice-ansi: 3.0.0
@@ -30784,6 +31055,23 @@ snapshots:
normalize-path: 3.0.0
readable-stream: 4.6.0
+ dependencies:
+ compute-dot: 1.1.0
+ compute-l2norm: 1.1.0
+ validate.io-array: 1.0.6
+ validate.io-function: 1.0.2
+
+ dependencies:
+ validate.io-array: 1.0.6
+ validate.io-function: 1.0.2
+
+ dependencies:
+ validate.io-array: 1.0.6
+ validate.io-function: 1.0.2
+
@@ -32263,6 +32551,8 @@ snapshots:
+
@@ -32582,6 +32872,8 @@ snapshots:
+
dependencies:
escape-string-regexp: 1.0.5
@@ -33836,6 +34128,8 @@ snapshots:
is-alphabetical: 2.0.1
is-decimal: 2.0.1
+
dependencies:
call-bind: 1.0.8
@@ -34448,6 +34742,10 @@ snapshots:
- encoding
- supports-color
+ dependencies:
+ '@pgsql/types': 17.6.2
+
dependencies:
cookie: 0.7.2
@@ -34458,6 +34756,12 @@ snapshots:
+ dependencies:
+ cheminfo-types: 1.8.1
+ ml-matrix: 6.12.1
+ ml-spectra-processing: 14.18.2
+
dependencies:
base64-js: 0.0.8
@@ -35975,6 +36279,36 @@ snapshots:
+ dependencies:
+ is-any-array: 2.0.1
+
+ dependencies:
+ is-any-array: 2.0.1
+
+ dependencies:
+ is-any-array: 2.0.1
+ ml-array-max: 1.2.4
+ ml-array-min: 1.2.3
+
+ dependencies:
+ is-any-array: 2.0.1
+ ml-array-rescale: 1.3.7
+
+ dependencies:
+ binary-search: 1.3.6
+ cheminfo-types: 1.8.1
+ fft.js: 4.0.4
+ is-any-array: 2.0.1
+ ml-matrix: 6.12.1
+ ml-xsadd: 3.0.1
+
+
dependencies:
acorn: 8.15.0
@@ -36054,6 +36388,8 @@ snapshots:
+
@@ -36520,7 +36856,7 @@ snapshots:
dependencies:
mitt: 3.0.1
@@ -39921,7 +40257,7 @@ snapshots:
dependencies:
openapi-fetch: 0.6.2
dependencies:
bin-links: 6.0.0
https-proxy-agent: 7.0.6([email protected])
@@ -40065,6 +40401,8 @@ snapshots:
dependencies:
bintrees: 1.0.2
+
dependencies:
'@jridgewell/trace-mapping': 0.3.31
@@ -40898,6 +41236,10 @@ snapshots:
+
+
dependencies:
@@ -41471,6 +41813,10 @@ snapshots:
dependencies:
string-width: 5.1.2
+ dependencies:
+ string-width: 7.2.0
+
@@ -41491,6 +41837,12 @@ snapshots:
string-width: 5.1.2
strip-ansi: 7.1.0
+ dependencies:
+ ansi-styles: 6.2.1
+ string-width: 7.2.0
+ strip-ansi: 7.1.0
+
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 423c8e85fb1..52f428ea86d 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -51,6 +51,7 @@ minimumReleaseAgeExclude:
- iceberg-js
- '@vitejs/plugin-rsc'
- stripe-experiment-sync # TODO(matlin) remove, temp just to unblock launch
+ - braintrust
onlyBuiltDependencies:
- supabase
diff --git a/turbo.json b/turbo.json
index b2042f2d1d6..8cb394fc1ba 100644
--- a/turbo.json
+++ b/turbo.json
@@ -104,6 +104,8 @@
"DEFAULT_PROJECT_NAME",
"DEFAULT_ORGANIZATION_NAME",
"OPENAI_API_KEY",
+ "BRAINTRUST_API_KEY",
+ "BRAINTRUST_PROJECT_ID",
"AUTH_JWT_SECRET",
"LOGFLARE_API_KEY",
"LOGFLARE_PUBLIC_ACCESS_TOKEN",