The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat: assistant evals (#41311)
Matt Rossman072883bcecb8
diff --git a/.github/workflows/braintrust-evals.yml b/.github/workflows/braintrust-evals.ymlnew file mode 100644index 00000000000..915d2e3bb38--- /dev/null+++ b/.github/workflows/braintrust-evals.yml@@ -0,0 +1,60 @@+name: Run Braintrust evals++on:+ push:+ branches: [master]+ pull_request:+ types: [opened, synchronize, labeled]++concurrency:+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: true++permissions:+ pull-requests: write+ contents: read++jobs:+ eval:+ name: Run evals+ if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-evals'))+ runs-on: ubuntu-latest+ timeout-minutes: 20++ env:+ BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}++ steps:+ - name: Checkout+ uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0+ with:+ fetch-depth: 0+ # For PR events, checkout the actual branch so Braintrust can report the correct branch name instead of detached HEAD.+ # github.head_ref is the PR source branch, github.ref_name is the fallback for push events (e.g., master).+ ref: ${{ github.head_ref || github.ref_name }}++ - name: Install pnpm+ uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0+ with:+ run_install: false++ - name: Use Node.js+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0+ with:+ node-version-file: ".nvmrc"+ cache: "pnpm"++ - name: Install Dependencies+ run: pnpm install --frozen-lockfile++ - name: Setup Evals+ run: cd apps/studio && pnpm evals:setup++ - name: Run Evals+ uses: braintrustdata/eval-action@v1+ with:+ api_key: ${{ secrets.BRAINTRUST_API_KEY }}+ runtime: node+ package_manager: pnpm+ root: apps/studiodiff --git a/apps/studio/evals/.gitignore b/apps/studio/evals/.gitignorenew file mode 100644index 00000000000..8085866a958--- /dev/null+++ b/apps/studio/evals/.gitignore@@ -0,0 +1 @@+libpg-query.wasm\ No newline at end of filediff --git a/apps/studio/evals/assistant.eval.ts b/apps/studio/evals/assistant.eval.tsnew file mode 100644index 00000000000..d2c27f40303--- /dev/null+++ b/apps/studio/evals/assistant.eval.ts@@ -0,0 +1,123 @@+import { openai } from '@ai-sdk/openai'+import { Eval } from 'braintrust'+import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'+import { getMockTools } from 'lib/ai/tools/mock-tools'+import assert from 'node:assert'+import { dataset } from './dataset'+import {+ completenessScorer,+ concisenessScorer,+ goalCompletionScorer,+ sqlSyntaxScorer,+ toolUsageScorer,+} from './scorer'+import { ToolSet, TypedToolCall, TypedToolResult } from 'ai'++assert(process.env.BRAINTRUST_PROJECT_ID, 'BRAINTRUST_PROJECT_ID is not set')+assert(process.env.OPENAI_API_KEY, 'OPENAI_API_KEY is not set')++Eval('Assistant', {+ projectId: process.env.BRAINTRUST_PROJECT_ID,+ data: () => dataset,+ task: async (input) => {+ const result = await generateAssistantResponse({+ model: openai('gpt-5-mini'),+ messages: [{ id: '1', role: 'user', parts: [{ type: 'text', text: input }] }],+ tools: await getMockTools(),+ })++ // `result.toolCalls` only shows the last step, instead aggregate tools across all steps+ const steps = await result.steps++ const stepsSerialized = steps+ .map((step) => {+ const toolCalls = step.toolCalls+ ?.map((call) => JSON.stringify({ tool: call.toolName, input: call.input }))+ .join('\n')++ const text = step.text+ return toolCalls ? `${text}\n${toolCalls}` : text+ })+ .join('\n')++ const toolNames: string[] = []+ const sqlQueries: string[] = []+ const docs: string[] = []++ for (const step of steps) {+ for (const [i, toolCall] of step.toolCalls.entries()) {+ toolNames.push(toolCall.toolName)++ const toolResult = step.toolResults.at(i)+ if (!toolResult) {+ continue+ }++ const parsed = parseToolCall(toolCall, toolResult)++ if (parsed.sqlQuery) {+ sqlQueries.push(parsed.sqlQuery)+ }+ if (parsed.docs) {+ docs.push(...parsed.docs)+ }+ }+ }++ return {+ stepsSerialized,+ toolNames,+ sqlQueries,+ docs,+ }+ },+ scores: [+ toolUsageScorer,+ sqlSyntaxScorer,+ goalCompletionScorer,+ concisenessScorer,+ completenessScorer,+ ],+})++type ParsedToolCall = {+ /** Query generated by `execute_sql` */+ sqlQuery?: string++ /** Docs text pulled in from `search_docs` */+ docs?: string[]+}++/**+ * Validate and extract relevant info from a tool call/result+ */+function parseToolCall(+ toolCall: TypedToolCall<ToolSet>,+ toolResult: TypedToolResult<ToolSet>+): ParsedToolCall {+ switch (toolCall.toolName) {+ case 'execute_sql': {+ const sqlQuery = toolCall.input.sql+ if (typeof sqlQuery !== 'string') {+ return {}+ }++ return { sqlQuery }+ }+ case 'search_docs': {+ const content = toolResult.output.content+ if (!content || !Array.isArray(content)) {+ return {}+ }++ const docs = content.map((item) => item?.text).filter((text) => typeof text === 'string')+ if (docs.length === 0) {+ return {}+ }++ return { docs }+ }+ }++ return {}+}diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.tsnew file mode 100644index 00000000000..58c31926091--- /dev/null+++ b/apps/studio/evals/dataset.ts@@ -0,0 +1,38 @@+import { AssistantEvalCase } from './scorer'++export const dataset: AssistantEvalCase[] = [+ {+ input: 'How do I run WASM in edge functions? Use `search_docs`.',+ expected: { requiredTools: ['search_docs'] },+ metadata: { category: ['general_help'] },+ },+ {+ input: 'Check if my project is having issues right now and tell me what to fix first.',+ expected: {+ requiredTools: ['get_advisors', 'get_logs'],+ },+ metadata: { category: ['debugging', 'rls_policies'] },+ },+ {+ input: 'Create a new table "foods" with columns for "name" and "color"',+ expected: {+ requiredTools: ['execute_sql'],+ },+ metadata: { category: ['sql_generation', 'schema_design'] },+ },+ {+ input:+ 'Write a SQL query to select all products from the products table where the price is greater than 100',+ expected: {+ requiredTools: ['execute_sql'],+ },+ metadata: { category: ['sql_generation'] },+ },+ {+ input: 'Create an index on the products table for the name column',+ expected: {+ requiredTools: ['execute_sql'],+ },+ metadata: { category: ['sql_generation', 'database_optimization'] },+ },+]diff --git a/apps/studio/evals/scorer.ts b/apps/studio/evals/scorer.tsnew file mode 100644index 00000000000..62f68f41d7c--- /dev/null+++ b/apps/studio/evals/scorer.ts@@ -0,0 +1,159 @@+import { EvalCase, EvalScorer } from 'braintrust'+import { LLMClassifierFromTemplate } from 'autoevals'+import { stripIndent } from 'common-tags'+import { parse } from 'libpg-query'++const LLM_AS_A_JUDGE_MODEL = 'gpt-5.2-2025-12-11'++type Input = string++type Output = {+ stepsSerialized: string+ toolNames: string[]+ sqlQueries: string[]+ docs: string[]+}++export type Expected = {+ requiredTools?: string[]+}++// Based on categories in the AssistantMessageRatingSubmittedEvent+export type AssistantEvalCaseCategory =+ | 'sql_generation'+ | 'schema_design'+ | 'rls_policies'+ | 'edge_functions'+ | 'database_optimization'+ | 'debugging'+ | 'general_help'+ | 'other'++export type AssistantEvalCaseMetadata = {+ category?: AssistantEvalCaseCategory[]+}++export type AssistantEvalCase = EvalCase<Input, Expected, AssistantEvalCaseMetadata>++export const toolUsageScorer: EvalScorer<Input, Output, Expected> = async ({+ output,+ expected,+}) => {+ if (!expected.requiredTools) return null++ const presentCount = expected.requiredTools.filter((tool) =>+ output.toolNames.includes(tool)+ ).length+ const totalCount = expected.requiredTools.length+ const ratio = totalCount === 0 ? 1 : presentCount / totalCount++ return {+ name: 'Tool Usage',+ score: ratio,+ }+}++export const sqlSyntaxScorer: EvalScorer<Input, Output, Expected> = async ({ output }) => {+ if (output.sqlQueries === undefined || output.sqlQueries.length === 0) {+ return null+ }++ const errors: string[] = []+ let validQueries = 0++ for (const sql of output.sqlQueries) {+ try {+ await parse(sql)+ validQueries+++ } catch (error) {+ const errorMessage = error instanceof Error ? error.message : String(error)+ errors.push(`SQL syntax error: ${errorMessage}`)+ }+ }++ return {+ name: 'SQL Validity',+ score: validQueries / output.sqlQueries.length,+ metadata: errors.length > 0 ? { errors } : undefined,+ }+}++const concisenessEvaluator = LLMClassifierFromTemplate<{ input: string }>({+ name: 'Conciseness',+ promptTemplate: stripIndent`+ Evaluate the conciseness of this response.++ Input: {{input}}+ Output: {{output}}++ Is the response concise and free of unnecessary words?+ a) Very concise - no wasted words+ b) Acceptable verbosity - some verbosity but acceptable+ c) Too verbose - contains superfluous wording or overly verbose+ `,+ choiceScores: { a: 1, b: 0.5, c: 0 },+ useCoT: true,+ model: LLM_AS_A_JUDGE_MODEL,+})++export const concisenessScorer: EvalScorer<Input, Output, Expected> = async ({ input, output }) => {+ return await concisenessEvaluator({+ input,+ output: output.stepsSerialized,+ })+}++const completenessEvaluator = LLMClassifierFromTemplate<{ input: string }>({+ name: 'Completeness',+ promptTemplate: stripIndent`+ Evaluate whether this response is complete and finished, or if it appears cut off or incomplete.++ Input: {{input}}+ Output: {{output}}++ Does the response appear complete and finished?+ a) Complete - response is complete and finished+ b) Incomplete - response appears cut off, missing parts, or severely incomplete+ `,+ choiceScores: { a: 1, b: 0 },+ useCoT: true,+ model: LLM_AS_A_JUDGE_MODEL,+})++export const completenessScorer: EvalScorer<Input, Output, Expected> = async ({+ input,+ output,+}) => {+ return await completenessEvaluator({+ input,+ output: output.stepsSerialized,+ })+}++const goalCompletionEvaluator = LLMClassifierFromTemplate<{ input: string }>({+ name: 'Goal Completion',+ promptTemplate: stripIndent`+ Evaluate whether this response addresses what the user asked.++ Input: {{input}}+ Output: {{output}}++ Does the response address what the user asked?+ a) Fully addresses - completely answers the question or fulfills the request+ b) Partially addresses - addresses some aspects but misses key parts+ c) Doesn't address - off-topic or fails to address the request+ `,+ choiceScores: { a: 1, b: 0.5, c: 0 },+ useCoT: true,+ model: LLM_AS_A_JUDGE_MODEL,+})++export const goalCompletionScorer: EvalScorer<Input, Output, Expected> = async ({+ input,+ output,+}) => {+ return await goalCompletionEvaluator({+ input,+ output: output.stepsSerialized,+ })+}diff --git a/apps/studio/lib/ai/generate-assistant-response.ts b/apps/studio/lib/ai/generate-assistant-response.tsnew file mode 100644index 00000000000..1f35a8223d1--- /dev/null+++ b/apps/studio/lib/ai/generate-assistant-response.ts@@ -0,0 +1,130 @@+import * as ai from 'ai'+import {+ convertToModelMessages,+ isToolUIPart,+ type LanguageModel,+ type ModelMessage,+ stepCountIs,+ type ToolSet,+ type UIMessage,+} from 'ai'+import { wrapAISDK } from 'braintrust'+import { source } from 'common-tags'++import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'+import {+ CHAT_PROMPT,+ EDGE_FUNCTION_PROMPT,+ GENERAL_PROMPT,+ PG_BEST_PRACTICES,+ RLS_PROMPT,+ REALTIME_PROMPT,+ SECURITY_PROMPT,+ LIMITATIONS_PROMPT,+} from 'lib/ai/prompts'+import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'++const { streamText } = wrapAISDK(ai)++export async function generateAssistantResponse({+ messages: rawMessages,+ model,+ tools,+ aiOptInLevel = 'schema',+ getSchemas,+ projectRef,+ chatName,+ promptProviderOptions,+ providerOptions,+ abortSignal,+}: {+ messages: UIMessage[]+ model: LanguageModel+ tools: ToolSet+ aiOptInLevel?: AiOptInLevel+ getSchemas?: () => Promise<string>+ projectRef?: string+ chatName?: string+ promptProviderOptions?: Record<string, any>+ providerOptions?: Record<string, any>+ abortSignal?: AbortSignal+}) {+ // Only returns last 7 messages+ // Filters out tools with invalid states+ // Filters out tool outputs based on opt-in level using renderingToolOutputParser+ const messages = (rawMessages || []).slice(-7).map((msg) => {+ if (msg && msg.role === 'assistant' && 'results' in msg) {+ const cleanedMsg = { ...msg }+ delete cleanedMsg.results+ return cleanedMsg+ }+ if (msg && msg.role === 'assistant' && msg.parts) {+ const cleanedParts = msg.parts+ .filter((part) => {+ if (isToolUIPart(part)) {+ const invalidStates = ['input-streaming', 'input-available', 'output-error']+ return !invalidStates.includes(part.state)+ }+ return true+ })+ .map((part) => {+ return sanitizeMessagePart(part, aiOptInLevel)+ })+ return { ...msg, parts: cleanedParts }+ }+ return msg+ })++ const schemasString =+ aiOptInLevel !== 'disabled' && getSchemas+ ? await getSchemas()+ : "You don't have access to any schemas."++ // Important: do not use dynamic content in the system prompt or Bedrock will not cache it+ const system = source`+ ${GENERAL_PROMPT}+ ${CHAT_PROMPT}+ ${PG_BEST_PRACTICES}+ ${RLS_PROMPT}+ ${EDGE_FUNCTION_PROMPT}+ ${REALTIME_PROMPT}+ ${SECURITY_PROMPT}+ ${LIMITATIONS_PROMPT}+ `++ // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`+ // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88+ const assistantContent =+ projectRef || chatName || schemasString !== "You don't have access to any schemas."+ ? `The user's current project is ${projectRef || 'unknown'}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName || 'unnamed'}`+ : undefined++ const coreMessages: ModelMessage[] = [+ {+ role: 'system',+ content: system,+ ...(promptProviderOptions && {+ providerOptions: promptProviderOptions,+ }),+ },+ ...(assistantContent+ ? [+ {+ role: 'assistant' as const,+ // Add any dynamic context here+ content: assistantContent,+ },+ ]+ : []),+ ...convertToModelMessages(messages),+ ]++ return streamText({+ model,+ stopWhen: stepCountIs(5),+ messages: coreMessages,+ ...(providerOptions && { providerOptions }),+ tools,+ ...(abortSignal && { abortSignal }),+ })+}diff --git a/apps/studio/lib/ai/tools/mock-tools.ts b/apps/studio/lib/ai/tools/mock-tools.tsnew file mode 100644index 00000000000..7802112589f--- /dev/null+++ b/apps/studio/lib/ai/tools/mock-tools.ts@@ -0,0 +1,324 @@+import { tool, type ToolSet } from 'ai'+import { getRenderingTools } from '../tools/rendering-tools'+import { z } from 'zod'+import { getMcpTools } from 'lib/ai/tools/mcp-tools'+import assert from 'node:assert'++const listTablesInputSchema = z.object({+ schemas: z.array(z.string()).describe('The schema names to list.'),+})++const getAdvisorsInputSchema = z.object({+ type: z.enum(['security', 'performance']).optional(),+})++const getLogsInputSchema = z.object({+ limit: z.number().min(1).max(100).optional(),+ level: z.enum(['debug', 'info', 'warning', 'error']).optional(),+ source: z.enum(['postgres', 'auth', 'storage', 'edge_function']).optional(),+ search: z.string().optional(),+})++const listPoliciesInputSchema = z.object({+ schemas: z.array(z.string()).describe('The schema names to get the policies for'),+})++const MOCK_TABLES_DATA = [+ {+ name: 'user_documents',+ rls_enabled: false,+ columns: [+ { name: 'id', data_type: 'bigint' },+ { name: 'user_id', data_type: 'uuid' },+ { name: 'title', data_type: 'text' },+ ],+ },+ {+ name: 'customers',+ rls_enabled: true,+ columns: [+ { name: 'id', data_type: 'uuid' },+ { name: 'tenant_id', data_type: 'uuid' },+ { name: 'email', data_type: 'text' },+ ],+ },+ {+ name: 'projects',+ rls_enabled: false,+ columns: [+ { name: 'id', data_type: 'uuid' },+ { name: 'organization_id', data_type: 'uuid' },+ { name: 'name', data_type: 'text' },+ ],+ },+ {+ name: 'user_organizations',+ rls_enabled: true,+ columns: [+ { name: 'user_id', data_type: 'uuid' },+ { name: 'organization_id', data_type: 'uuid' },+ ],+ },+]++const MOCK_EXTENSIONS_DATA = [+ { name: 'pgcrypto', schema: 'extensions', installed_version: '1.3' },+ { name: 'uuid-ossp', schema: 'extensions', installed_version: '1.1' },+]++const MOCK_EDGE_FUNCTIONS_DATA = [+ { name: 'hello-world', last_deployed_at: '2024-06-10T12:30:00Z' },+ { name: 'daily-metrics-sync', last_deployed_at: '2024-06-18T08:15:00Z' },+ { name: 'select-from-table-with-auth-rls', last_deployed_at: '2024-06-19T09:20:00Z' },+]++const MOCK_ADVISORIES_DATA = [+ {+ id: '0016_materialized_view_in_api',+ level: 'warning',+ category: 'security',+ message: 'Materialized views in API schema can bypass RLS. Move them to private schema.',+ remediationUrl:+ 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0016_materialized_view_in_api',+ },+ {+ id: '0031_functions_no_rls_guard',+ level: 'notice',+ category: 'security',+ message: 'Function api.health_check should verify auth context before querying tables.',+ remediationUrl:+ 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0031_functions_no_rls_guard',+ },+ {+ id: '1012_slow_query',+ level: 'info',+ category: 'performance',+ message:+ 'Query on table edge_function_logs exceeded 3s average execution time over the last hour.',+ remediationUrl: 'https://supabase.com/docs/guides/platform/performance-advisors#slow-queries',+ },+]++const MOCK_LOGS_DATA = [+ {+ id: 'log-001',+ timestamp: '2024-06-20T14:12:00Z',+ level: 'error',+ source: 'edge_function' as const,+ target: 'hello-world',+ message: "TypeError: fetch failed at await supabase.functions.invoke('analytics')",+ },+ {+ id: 'log-002',+ timestamp: '2024-06-20T14:05:30Z',+ level: 'warning',+ source: 'postgres' as const,+ target: 'connection_pool',+ message: 'Query timeout exceeded for statement SELECT * FROM public.audit_log_entries',+ },+ {+ id: 'log-003',+ timestamp: '2024-06-20T13:59:10Z',+ level: 'info',+ source: 'edge_function' as const,+ target: 'daily-metrics-sync',+ message: 'Invocation completed in 520ms',+ },+ {+ id: 'log-004',+ timestamp: '2024-06-20T13:50:00Z',+ level: 'error',+ source: 'postgres' as const,+ target: 'trigger:refresh_materialized_views',+ message: 'permission denied for relation user_documents',+ },+ {+ id: 'log-005',+ timestamp: '2024-06-20T13:45:00Z',+ level: 'info',+ source: 'auth' as const,+ target: 'email-confirmation',+ message: 'Sent verification email to [email protected]',+ },+]++function createMockedRenderingTools() {+ const renderingTools = getRenderingTools()++ return Object.fromEntries(+ Object.entries(renderingTools).map(([name, baseTool]) => {+ if (typeof baseTool.execute === 'function') {+ return [name, baseTool]+ }++ const statusMessage =+ name === 'execute_sql'+ ? 'SQL execution mocked successfully.'+ : name === 'deploy_edge_function'+ ? 'Edge Function deployment mocked successfully.'+ : 'Tool call mocked successfully.'++ return [+ name,+ {+ ...baseTool,+ execute: async () => ({ status: statusMessage }),+ },+ ]+ })+ ) as typeof renderingTools+}++function createMockListTablesTool() {+ return tool({+ description: 'Lists tables and columns for the provided schemas.',+ inputSchema: listTablesInputSchema,+ execute: async ({ schemas }: { schemas: string[] }) => {+ const effectiveSchemas = schemas?.length ? schemas : ['public']+ return effectiveSchemas.map((schema) => ({+ schema,+ tables: MOCK_TABLES_DATA,+ }))+ },+ })+}++function createMockListExtensionsTool() {+ return tool({+ description: 'Lists installed database extensions.',+ inputSchema: z.object({}),+ execute: async () => {+ return MOCK_EXTENSIONS_DATA+ },+ })+}++function createMockListEdgeFunctionsTool() {+ return tool({+ description: 'Lists available Supabase Edge Functions.',+ inputSchema: z.object({}),+ execute: async () => {+ return MOCK_EDGE_FUNCTIONS_DATA+ },+ })+}++function createMockGetAdvisorsTool() {+ return tool({+ description: 'Returns advisory notices for the project (mocked).',+ inputSchema: getAdvisorsInputSchema,+ execute: async ({ type }: { type?: 'security' | 'performance' }) => {+ if (type) {+ return MOCK_ADVISORIES_DATA.filter((advisory) => advisory.category === type)+ }+ return MOCK_ADVISORIES_DATA+ },+ })+}++function createMockGetLogsTool() {+ return tool({+ description: 'Fetches recent project logs for debugging or health checks (mocked).',+ inputSchema: getLogsInputSchema,+ execute: async ({+ limit = 10,+ level,+ source,+ search,+ }: {+ limit?: number+ level?: 'debug' | 'info' | 'warning' | 'error'+ source?: 'postgres' | 'auth' | 'storage' | 'edge_function'+ search?: string+ }) => {+ let filtered = MOCK_LOGS_DATA++ if (level) {+ filtered = filtered.filter((entry) => entry.level === level)+ }++ if (source) {+ filtered = filtered.filter((entry) => entry.source === source)+ }++ if (search) {+ const needle = search.toLowerCase()+ filtered = filtered.filter((entry) =>+ `${entry.message} ${entry.target}`.toLowerCase().includes(needle)+ )+ }++ return filtered.slice(0, limit)+ },+ })+}++function createMockListPoliciesTool() {+ return tool({+ description: 'Get existing RLS policies for provided schemas.',+ inputSchema: listPoliciesInputSchema,+ execute: async ({ schemas }: { schemas: string[] }) => {+ const effectiveSchemas = schemas?.length ? schemas : ['public']+ const results = [] as Array<{+ schema: string+ table: string+ policies: Array<{+ name: string+ command: 'select' | 'insert' | 'update' | 'delete'+ using?: string+ check?: string+ }>+ }>++ for (const schema of effectiveSchemas) {+ if (schema !== 'public') continue+ results.push(+ {+ schema,+ table: 'customers',+ policies: [+ {+ name: 'customers_tenant_select',+ command: 'select',+ using: "(auth.jwt() ->> 'tenant_id')::uuid = tenant_id",+ },+ ],+ },+ { schema, table: 'user_documents', policies: [] },+ { schema, table: 'projects', policies: [] }+ )+ }+ return results+ },+ })+}++/**+ * Deterministic mock implementations of MCP/platform tools for evals.+ * These mirror tool names used in prompts so the model can call them,+ * but return stable, static data for repeatable tests.+ *+ * Note: search_docs uses the real implementation+ */+export async function getMockTools() {+ const mockedRenderingTools = createMockedRenderingTools()++ const { search_docs } = await getMcpTools({+ accessToken: 'mock-access-token',+ projectRef: 'mock-project-ref',+ aiOptInLevel: 'schema_and_log_and_data',+ })++ assert(search_docs, 'search_docs tool not available from MCP server')++ return {+ ...mockedRenderingTools,+ search_docs,+ list_tables: createMockListTablesTool(),+ list_extensions: createMockListExtensionsTool(),+ list_edge_functions: createMockListEdgeFunctionsTool(),+ get_advisors: createMockGetAdvisorsTool(),+ get_logs: createMockGetLogsTool(),+ list_policies: createMockListPoliciesTool(),+ }+}diff --git a/apps/studio/lib/api/generate-v4.test.ts b/apps/studio/lib/api/generate-v4.test.tsindex a04be6ef01a..d584b7e5222 100644--- a/apps/studio/lib/api/generate-v4.test.ts+++ b/apps/studio/lib/api/generate-v4.test.ts@@ -2,9 +2,10 @@ import { expect, test, vi } from 'vitest'// End of third-party importsimport generateV4 from '../../pages/api/ai/sql/generate-v4'-import { sanitizeMessagePart } from '../ai/tools/tool-sanitizer'+import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'+import { UIMessage } from 'ai'-vi.mock('../ai/tools/tool-sanitizer', () => ({+vi.mock('lib/ai/tools/tool-sanitizer', () => ({sanitizeMessagePart: vi.fn((part) => part),}))@@ -17,20 +18,24 @@ test('generateV4 calls the tool sanitizer', async () => {body: {messages: [{+ id: 'test-msg-id',role: 'assistant',parts: [{type: 'tool-execute_sql',state: 'output-available',- output: 'test output',+ toolCallId: 'test-tool-call-id',+ input: { sql: 'SELECT * FROM users' },+ output: [{ id: 1, name: 'test-output' }],},],},- ],+ ] satisfies UIMessage[],projectRef: 'test-project',connectionString: 'test-connection',orgSlug: 'test-org',},+ on: vi.fn(),}const mockRes = {@@ -63,13 +68,15 @@ test('generateV4 calls the tool sanitizer', async () => {getTools: vi.fn().mockResolvedValue({}),}))- vi.mock('ai', () => ({- streamText: vi.fn().mockReturnValue({- pipeUIMessageStreamToResponse: vi.fn(),- }),- convertToModelMessages: vi.fn((msgs) => msgs),- stepCountIs: vi.fn(),- }))+ vi.mock('ai', async () => {+ const actual = await vi.importActual('ai')+ return {+ ...actual,+ streamText: vi.fn().mockReturnValue({+ pipeUIMessageStreamToResponse: vi.fn(),+ }),+ }+ })await generateV4(mockReq as any, mockRes as any)diff --git a/apps/studio/package.json b/apps/studio/package.jsonindex aafb78a3d81..58684683e56 100644--- a/apps/studio/package.json+++ b/apps/studio/package.json@@ -23,7 +23,10 @@"prettier:write": "prettier --cache --write .","build:deno-types": "tsx scripts/deno-types.ts","build:graphql-types": "tsx scripts/download-graphql-schema.mts && pnpm graphql-codegen --config scripts/codegen.ts",- "build:graphql-types:watch": "pnpm graphql-codegen --config scripts/codegen.ts --watch"+ "build:graphql-types:watch": "pnpm graphql-codegen --config scripts/codegen.ts --watch",+ "evals:setup": "cp node_modules/libpg-query/wasm/libpg-query.wasm evals/libpg-query.wasm",+ "evals:run": "braintrust eval --no-send-logs evals/assistant.eval.ts",+ "evals:upload": "braintrust eval evals/assistant.eval.ts"},"dependencies": {"@ai-sdk/amazon-bedrock": "^3.0.0",@@ -183,6 +186,8 @@"@vitest/coverage-v8": "^3.2.0","@vitest/ui": "^3.2.0","api-types": "workspace:*",+ "autoevals": "^0.0.131",+ "braintrust": "^1.0.2","common": "workspace:*","config": "workspace:*","date-fns": "^2.30.0",@@ -191,6 +196,7 @@"graphql-ws": "5.14.1","import-in-the-middle": "^2.0.0","jsdom-testing-mocks": "^1.13.1",+ "libpg-query": "17.6.0","msw": "^2.3.0","next-router-mock": "^0.9.13","node-mocks-http": "^1.17.2",diff --git a/apps/studio/pages/api/ai/sql/generate-v4.ts b/apps/studio/pages/api/ai/sql/generate-v4.tsindex ecd441f5030..93aa261b14c 100644--- a/apps/studio/pages/api/ai/sql/generate-v4.ts+++ b/apps/studio/pages/api/ai/sql/generate-v4.ts@@ -1,6 +1,5 @@import pgMeta from '@supabase/pg-meta'-import { convertToModelMessages, type ModelMessage, stepCountIs, streamText } from 'ai'-import { source } from 'common-tags'+import { safeValidateUIMessages } from 'ai'import type { NextApiRequest, NextApiResponse } from 'next'import z from 'zod'@@ -9,18 +8,8 @@ import { executeSql } from 'data/sql/execute-sql-query'import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi'import { getModel } from 'lib/ai/model'import { getOrgAIDetails } from 'lib/ai/org-ai-details'-import {- CHAT_PROMPT,- EDGE_FUNCTION_PROMPT,- GENERAL_PROMPT,- PG_BEST_PRACTICES,- RLS_PROMPT,- REALTIME_PROMPT,- SECURITY_PROMPT,- LIMITATIONS_PROMPT,-} from 'lib/ai/prompts'+import { generateAssistantResponse } from 'lib/ai/generate-assistant-response'import { getTools } from 'lib/ai/tools'-import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer'import apiWrapper from 'lib/api/apiWrapper'import { executeQuery } from 'lib/api/self-hosted/query'@@ -89,6 +78,14 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {model: requestedModel,} = data+ const messagesValidation = await safeValidateUIMessages({ messages: rawMessages })+ if (!messagesValidation.success) {+ return res+ .status(400)+ .json({ error: 'Invalid request body', message: messagesValidation.error.message })+ }+ const messages = messagesValidation.data+let aiOptInLevel: AiOptInLevel = 'disabled'let isLimited = false@@ -114,32 +111,6 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {}}- // Only returns last 7 messages- // Filters out tools with invalid states- // Filters out tool outputs based on opt-in level using renderingToolOutputParser- const messages = (rawMessages || []).slice(-7).map((msg: any) => {- if (msg && msg.role === 'assistant' && 'results' in msg) {- const cleanedMsg = { ...msg }- delete cleanedMsg.results- return cleanedMsg- }- if (msg && msg.role === 'assistant' && msg.parts) {- const cleanedParts = msg.parts- .filter((part: any) => {- if (part.type.startsWith('tool-')) {- const invalidStates = ['input-streaming', 'input-available', 'output-error']- return !invalidStates.includes(part.state)- }- return true- })- .map((part: any) => {- return sanitizeMessagePart(part, aiOptInLevel)- })- return { ...msg, parts: cleanedParts }- }- return msg- })-const {model,error: modelError,@@ -157,67 +128,10 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {}try {- // Get a list of all schemas to add to context- const pgMetaSchemasList = pgMeta.schemas.list()- type Schemas = z.infer<(typeof pgMetaSchemasList)['zod']>-- const { result: schemas } =- aiOptInLevel !== 'disabled'- ? await executeSql<Schemas>(- {- projectRef,- connectionString,- sql: pgMetaSchemasList.sql,- },- undefined,- {- 'Content-Type': 'application/json',- ...(authorization && { Authorization: authorization }),- },- IS_PLATFORM ? undefined : executeQuery- )- : { result: [] }-- const schemasString =- schemas?.length > 0- ? `The available database schema names are: ${JSON.stringify(schemas)}`- : "You don't have access to any schemas."-- // Important: do not use dynamic content in the system prompt or Bedrock will not cache it- const system = source`- ${GENERAL_PROMPT}- ${CHAT_PROMPT}- ${PG_BEST_PRACTICES}- ${RLS_PROMPT}- ${EDGE_FUNCTION_PROMPT}- ${REALTIME_PROMPT}- ${SECURITY_PROMPT}- ${LIMITATIONS_PROMPT}- `-- // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions`- // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88- const coreMessages: ModelMessage[] = [- {- role: 'system',- content: system,- ...(promptProviderOptions && {- providerOptions: promptProviderOptions,- }),- },- {- role: 'assistant',- // Add any dynamic context here- content: `The user's current project is ${projectRef}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName}`,- },- ...convertToModelMessages(messages),- ]-const abortController = new AbortController()req.on('close', () => abortController.abort())req.on('aborted', () => abortController.abort())- // Get toolsconst tools = await getTools({projectRef,connectionString,@@ -226,12 +140,40 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) {accessToken,})- const result = streamText({+ // Get a list of all schemas to add to context+ const getSchemas = async (): Promise<string> => {+ const pgMetaSchemasList = pgMeta.schemas.list()+ type Schemas = z.infer<(typeof pgMetaSchemasList)['zod']>++ const { result: schemas } = await executeSql<Schemas>(+ {+ projectRef,+ connectionString,+ sql: pgMetaSchemasList.sql,+ },+ undefined,+ {+ 'Content-Type': 'application/json',+ ...(authorization && { Authorization: authorization }),+ },+ IS_PLATFORM ? undefined : executeQuery+ )++ return schemas?.length > 0+ ? `The available database schema names are: ${JSON.stringify(schemas)}`+ : "You don't have access to any schemas."+ }++ const result = await generateAssistantResponse({+ messages,model,- stopWhen: stepCountIs(5),- messages: coreMessages,- ...(providerOptions && { providerOptions }),tools,+ aiOptInLevel,+ getSchemas: aiOptInLevel !== 'disabled' ? getSchemas : undefined,+ projectRef,+ chatName,+ promptProviderOptions,+ providerOptions,abortSignal: abortController.signal,})diff --git a/package.json b/package.jsonindex ac42d0ed66b..a35f01b65ab 100644--- a/package.json+++ b/package.json@@ -49,7 +49,7 @@"prettier-plugin-sql-cst": "^0.11.0","rimraf": "^6.0.0","sass": "^1.72.0",- "supabase": "^2.58.5",+ "supabase": "^2.65.6","supports-color": "^8.0.0","tailwindcss": "catalog:","turbo": "2.3.3",diff --git a/packages/shared-data/package.json b/packages/shared-data/package.jsonindex 2533f1c5331..9aacccfae9e 100644--- a/packages/shared-data/package.json+++ b/packages/shared-data/package.json@@ -8,6 +8,9 @@"preinstall": "npx only-allow pnpm","clean": "rimraf node_modules"},+ "devDependencies": {+ "tsconfig": "workspace:"+ },"author": "","license": "MIT"}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 1febf67dc73..a046f17af6c 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -6,12 +6,63 @@ settings:catalogs:default:+ '@sentry/nextjs':+ specifier: ^10.26.0+ version: 10.27.0+ '@supabase/auth-js':+ specifier: 2.87.0+ version: 2.87.0+ '@supabase/postgrest-js':+ specifier: 2.87.0+ version: 2.87.0+ '@supabase/realtime-js':+ specifier: 2.87.0+ version: 2.87.0'@supabase/supabase-js':specifier: 2.87.0version: 2.87.0+ '@types/node':+ specifier: ^22.0.0+ version: 22.13.14+ '@types/react':+ specifier: ^18.3.0+ version: 18.3.3+ '@types/react-dom':+ specifier: ^18.3.0+ version: 18.3.0+ next:+ specifier: ^15.5.9+ version: 15.5.9+ react:+ specifier: ^18.3.0+ version: 18.3.1+ react-dom:+ specifier: ^18.3.0+ version: 18.3.1+ recharts:+ specifier: ^2.15.4+ version: 2.15.4+ tailwindcss:+ specifier: 3.4.1+ version: 3.4.1tsx:specifier: 4.20.3version: 4.20.3+ typescript:+ specifier: ~5.9.0+ version: 5.9.2+ valtio:+ specifier: ^1.12.0+ version: 1.12.0+ vite:+ specifier: ^7.1.11+ version: 7.1.11+ vitest:+ specifier: ^3.2.0+ version: 3.2.4+ zod:+ specifier: ^3.25.76+ version: 3.25.76overrides:'@eslint/eslintrc>js-yaml': ^4.1.1@@ -52,8 +103,8 @@ importers:specifier: ^1.72.0version: 1.72.0supabase:- specifier: ^2.58.5- version: 2.58.5([email protected])+ specifier: ^2.65.6+ version: 2.67.1([email protected])supports-color:specifier: ^8.0.0version: 8.1.1@@ -489,7 +540,7 @@ importers:nuqs:specifier: ^1.19.1- version: 1.19.1([email protected](@opentelemetry/[email protected])(@playwright/[email protected])([email protected]([email protected]))([email protected])([email protected]))+ version: 1.19.1([email protected](@babel/[email protected]([email protected]))(@opentelemetry/[email protected])(@playwright/[email protected])([email protected]([email protected]))([email protected])([email protected]))openai:specifier: ^4.75.1@@ -1170,6 +1221,12 @@ importers:api-types:specifier: workspace:*version: link:../../packages/api-types+ autoevals:+ specifier: ^0.0.131+ version: 0.0.131([email protected])([email protected])+ braintrust:+ specifier: ^1.0.2date-fns:specifier: ^2.30.0version: 2.30.0@@ -1188,6 +1245,9 @@ importers:jsdom-testing-mocks:specifier: ^1.13.1version: 1.13.1+ libpg-query:+ specifier: 17.6.0+ version: 17.6.0msw:specifier: ^2.3.0version: 2.11.3(@types/[email protected])([email protected])@@ -2173,7 +2233,11 @@ importers:specifier: 'catalog:'version: 3.2.4(@types/[email protected])(@vitest/[email protected])([email protected])([email protected]([email protected]))([email protected](@types/[email protected])([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])- packages/shared-data: {}+ packages/shared-data:+ devDependencies:+ tsconfig:+ specifier: 'workspace:'+ version: link:../tsconfigpackages/tsconfig: {}@@ -2671,6 +2735,10 @@ packages:peerDependencies:zod: ^3.25.76 || ^4+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==}+ engines: {node: '>=18'}+'@ai-sdk/[email protected]':resolution: {integrity: sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==}engines: {node: '>=18'}@@ -3244,6 +3312,10 @@ packages:react: ^16.8.0 || ^17 || ^18 || ^19react-dom: ^16.8.0 || ^17 || ^18 || ^19+ '@colors/[email protected]':+ resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}+ engines: {node: '>=0.1.90'}+'@contentlayer2/[email protected]':resolution: {integrity: sha512-ZJ+Iiu2rVI50x60XoqnrsO/Q8eqFX5AlP1L0U/3ygaAas3tnOqTzQZ1UsxYQMpJzcLok24ddlhKfQKbCMUJPiQ==}@@ -4916,6 +4988,9 @@ packages:'@next/[email protected]':resolution: {integrity: sha512-6IajJ23QrXW5RTJj2lRHcBM8mxcEl+vgd7XXVODQG/BcJyjgIP1k5OefdRl+P80btPvHeHoV4fIgC1so25pXcg==}+ '@next/[email protected]':+ resolution: {integrity: sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==}+'@next/[email protected]':resolution: {integrity: sha512-4GlTZ+EJM7WaW2HEZcyU317tIQDjkQIyENDLxYJfSWlfqguN+dHkZgyQTV/7ykvobU7yEH5gKvreNrH4B6QgIg==}@@ -6190,6 +6265,9 @@ packages:'@pgsql/[email protected]':resolution: {integrity: sha512-K3gtnbqbSUuUVmPm143qx5Gy2EmKuooshV95yMD48EUQ1256sgZBriEfY61OWJnlzdREdqHTIOxQqpZAb7XdZg==}+ '@pgsql/[email protected]':+ resolution: {integrity: sha512-1UtbELdbqNdyOShhrVfSz3a1gDi0s9XXiQemx+6QqtsrXe62a6zOGU+vjb2GRfG5jeEokI1zBBcfD42enRv0Rw==}+'@pkgjs/[email protected]':resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}engines: {node: '>=14'}@@ -9749,6 +9827,15 @@ packages:'@usercentrics/[email protected]':resolution: {integrity: sha512-/cik01TdeiYUV1+EasK83Ip05TDqmowM5tmWGfRDdaOrneSic5BedBdJHKya4pP2vwiYAtzDYVJe4BwPt2M16g==}+ '@vercel/[email protected]':+ resolution: {integrity: sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA==}+ engines: {node: '>= 16'}+ peerDependencies:+ '@aws-sdk/credential-provider-web-identity': '*'+ peerDependenciesMeta:+ '@aws-sdk/credential-provider-web-identity':+ optional: true+'@vercel/[email protected]':resolution: {integrity: sha512-1gSbK9zfrbJxk1JTBVERDhLi01mK3fz+gw4GjOjZwHnqs0zsBhQA70HGVtXQX/Z3BTRMfbpAEMVDfhecRw0lDA==}engines: {node: '>= 18'}@@ -10358,6 +10445,9 @@ packages:resolution: {integrity: sha512-Hdw8qdNiqdJ8LqT0iK0sVzkFbzg6fhnQqqfWhBDxcHZvU75+B+ayzTy8x+k5Ix0Y92XOhOUlx74ps+bA6BeYMQ==}engines: {node: '>=8'}+ resolution: {integrity: sha512-F+3lraja+Ms7n1M2cpWl65N7AYx4sPocRW454H5HlSGabYMfuFOUxw8IXmEYDkQ38BxtZ0Wd5ZAQj9RF59YJWw==}+resolution: {integrity: sha512-zQAFO1Dlsn69eXaO6+7YZc+v84aquQKbwpzCE3L0stj56ERn9hutFxPopViLjo9G+rWwjozRhgS5KJ25Xy19cQ==}@@ -10458,6 +10548,9 @@ packages:resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}engines: {node: '>=8'}+ resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==}+resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}@@ -10491,6 +10584,10 @@ packages:resolution: {integrity: sha512-2hCgjEmP8YLWQ130n2FerGv7rYpfBmnmp9Uy2Le1vge6X3gZIfSmEzP5QTDElFxcvVcXlEn8Aq6MU/PZygIOog==}engines: {node: '>=14.16'}+ resolution: {integrity: sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==}@@ -10501,6 +10598,12 @@ packages:resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}engines: {node: '>=8'}+ resolution: {integrity: sha512-Zm8WbD18FLv5HN38TyUKw659vJdZALQ5VVVWTncMxZgPtFXeWoC7H0tInw/1EQ1pGq50v7WZtlaFBOXhUaf8DQ==}+ hasBin: true+ peerDependencies:+ zod: ^3.25.34+resolution: {integrity: sha512-ECFzp6uFOSB+dcZ5BK/IBaGWssbSYBHvuMeMt3MMFyhI0Z8SqGgEkBLARgpRH3hutIgPVsALcMwbDrJqPxQ65A==}engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7}@@ -10593,6 +10696,10 @@ packages:resolution: {integrity: sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==}engines: {node: '>=14.16'}+ resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==}+ engines: {node: '>=16'}+resolution: {integrity: sha512-dU+Tx2fsypxTgtLoE36npi3UqcjSSMNYfkqgmoEhtZrraP5VWq0K7FkWVTYa8eMPtnU/G2txVsfdCJTn9uzpuQ==}@@ -10682,6 +10789,9 @@ packages:resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}engines: {node: '>= 6'}+ resolution: {integrity: sha512-FRcpVkox+cRovffgqNdDFQ1eUav+i/Vq/CUd1hcfEl2bevntFlzznL+jE8g4twl6ElB7gZjCko6pYpXyMn+6dA==}+resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==}peerDependencies:@@ -10754,10 +10864,18 @@ packages:resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==}engines: {node: '>=18'}+ resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==}+ engines: {node: '>=4'}+resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==}engines: {node: '>=6'}+ resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==}+ engines: {node: 10.* || >= 12.*}+resolution: {integrity: sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==}engines: {node: '>=8'}@@ -10950,6 +11068,15 @@ packages:resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==}engines: {node: '>= 14'}+ resolution: {integrity: sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==}++ resolution: {integrity: sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==}++ resolution: {integrity: sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==}+resolution: {integrity: sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw==}@@ -12363,6 +12490,10 @@ packages:resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}engines: {node: '>=0.8.x'}+ resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==}+ engines: {node: '>=14.18'}+resolution: {integrity: sha512-nVpZkTMM9rF6AQ9gPJpFsNAMt48wIzB5TQgiTLdHiuO8XEDhUgZEhqKlZWXbIzo9VmJ/HvysHqEaVeD5v9TPvA==}engines: {node: '>=20.0.0'}@@ -12581,6 +12712,9 @@ packages:resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}+ resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==}+resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==}engines: {node: '>=8'}@@ -13597,6 +13731,9 @@ packages:resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==}+ resolution: {integrity: sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==}+resolution: {integrity: sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==}engines: {node: '>= 0.4'}@@ -14247,6 +14384,9 @@ packages:resolution: {integrity: sha512-fBi14tsU3/ahtfU0C+/tvmtlt0EJM6kXe0Dl4/cSbiNmJ3IwLIuk3lawwiK5ByXU4sXwyAzbkvSl26jVXzYuaA==}+ resolution: {integrity: sha512-r4zOTcLTGYS5PlLQAicJ6Yi/tvZFag42YUuNEO8pi8bwt/ZZ4kj514J4QV5bOx0mZzPLF6agbfNXQVxGgmHR8g==}+resolution: {integrity: sha512-aORPWntbpH5esaYpGOOmri0OHDOe3wC5M2MQxZ9dvMLZm6DnaAn0kJlcbU9hwsQgLzmZyReKwFwwPkR+nHu5kA==}@@ -14258,6 +14398,9 @@ packages:resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==}engines: {node: '>=14'}+ resolution: {integrity: sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==}+resolution: {integrity: sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ==}@@ -15119,6 +15262,24 @@ packages:engines: {node: '>=10'}hasBin: true+ resolution: {integrity: sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==}++ resolution: {integrity: sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==}++ resolution: {integrity: sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==}++ resolution: {integrity: sha512-TJ+8eOFdp+INvzR4zAuwBQJznDUfktMtOB6g/hUcGh3rcyjxbz4Te57Pgri8Q9bhSQ7Zys4IYOGhFdnlgeB6Lw==}++ resolution: {integrity: sha512-n9aXXxdf+ogpYeU6FLZgClOteAE10DyUMkjeBGnvew9vmewKLfINVbMZBosW4Q1aqFDMsZjaItiJJ/UCApTIKg==}++ resolution: {integrity: sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==}+resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==}@@ -15199,6 +15360,10 @@ packages:resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==}+ resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}+ hasBin: true+resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==}@@ -18212,8 +18377,8 @@ packages:resolution: {integrity: sha512-TDQg6Nh7e2DaLmMjqpJt7Wwh2Au5loVPH2gwKMkD89e3pb5F/zMQGWt22asEXme031jEmhjazixFH0lwMgbRJA==}engines: {node: '>=18.0.0'}- resolution: {integrity: sha512-mYZSkUIePTdmwlHd26Pff8wpmjfre8gcuWzrc5QqhZgZvCXugVzAQQhcjaQisw5kusbPQWNIjUwcHYEKmejhPw==}+ resolution: {integrity: sha512-d/trGytTjB/Hi625zHh2RFFttjLOkSdRTsY/N1pjxoAfG0blDiHKqPwu12VJYitg4nzgjfhjnD3pLyfU0ko5vg==}engines: {npm: '>=8'}hasBin: true@@ -18316,6 +18481,10 @@ packages:resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}+ resolution: {integrity: sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA==}+ engines: {node: '>=12'}+resolution: {integrity: sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==}engines: {node: '>= 10.13.0'}@@ -19158,6 +19327,12 @@ packages:resolution: {integrity: sha512-OljLrQ9SQdOUqTaQxqL5dEfZWrXExyyWsozYlAWFawPVNuD83igl7uJD2RTkNMbniIYgt8l81eCJGIdQF7avLQ==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==}++ resolution: {integrity: sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==}+resolution: {integrity: sha512-co8NkCHeY0NsL0XsL/cSICt5VhTjwZlYT8mi50dYY5thx3r3w1D15A04Lvs9WL/y/Rf98vUKY5PAAJCTLHvkJw==}engines: {node: '>=12.20.0'}@@ -19543,6 +19718,10 @@ packages:resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==}engines: {node: '>=12'}+ resolution: {integrity: sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==}@@ -19558,6 +19737,10 @@ packages:resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}engines: {node: '>=12'}+ resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}@@ -19797,6 +19980,10 @@ snapshots:eventsource-parser: 3.0.6zod: 3.25.76+ '@ai-sdk/[email protected]':+ dependencies:+ json-schema: 0.4.0+'@ai-sdk/[email protected]':dependencies:json-schema: 0.4.0@@ -21204,6 +21391,9 @@ snapshots:react-dom: 18.3.1([email protected])react-is: 17.0.2+ '@colors/[email protected]':+ optional: true+dependencies:@@ -23329,6 +23519,8 @@ snapshots:- bufferutil- utf-8-validate+ '@next/[email protected]': {}+'@next/[email protected]': {}'@next/[email protected]': {}@@ -24823,6 +25015,8 @@ snapshots:'@pgsql/[email protected]': {}+ '@pgsql/[email protected]': {}+'@pkgjs/[email protected]':optional: true@@ -29322,6 +29516,10 @@ snapshots:lz-string: 1.5.0uuid: 9.0.1+ '@vercel/[email protected](@aws-sdk/[email protected])':+ optionalDependencies:+ '@aws-sdk/credential-provider-web-identity': 3.830.0+'@vercel/[email protected](@aws-sdk/[email protected])':optionalDependencies:'@aws-sdk/credential-provider-web-identity': 3.830.0@@ -30098,6 +30296,21 @@ snapshots:+ dependencies:+ ajv: 8.17.1+ compute-cosine-similarity: 1.1.0+ js-levenshtein: 1.1.6+ js-yaml: 4.1.1+ linear-sum-assignment: 1.0.9+ mustache: 4.2.0+ zod: 3.25.76+ zod-to-json-schema: 3.25.0([email protected])+ transitivePeerDependencies:+ - encoding+ - ws+dependencies:gulp-header: 1.8.12@@ -30205,6 +30418,8 @@ snapshots:+ [email protected]: {}+dependencies:file-uri-to-path: 1.0.0@@ -30267,6 +30482,17 @@ snapshots:widest-line: 4.0.1wrap-ansi: 8.1.0+ dependencies:+ ansi-align: 3.0.1+ camelcase: 8.0.0+ chalk: 5.4.1+ cli-boxes: 3.0.0+ string-width: 7.2.0+ type-fest: 4.30.0+ widest-line: 5.0.0+ wrap-ansi: 9.0.2+dependencies:balanced-match: 1.0.2@@ -30280,6 +30506,37 @@ snapshots:dependencies:fill-range: 7.1.1+ dependencies:+ '@ai-sdk/provider': 1.1.3+ '@next/env': 14.2.35+ '@vercel/functions': 1.6.0(@aws-sdk/[email protected])+ argparse: 2.0.1+ boxen: 8.0.1+ chalk: 4.1.2+ cli-progress: 3.12.0+ cli-table3: 0.6.5+ cors: 2.8.5+ dotenv: 16.5.0+ esbuild: 0.25.2+ eventsource-parser: 1.1.2+ express: 4.22.1([email protected])+ graceful-fs: 4.2.11+ http-errors: 2.0.1+ minimatch: 9.0.5+ mustache: 4.2.0+ pluralize: 8.0.0+ simple-git: 3.28.0([email protected])+ slugify: 1.6.6+ source-map: 0.7.6+ termi-link: 1.1.0+ uuid: 9.0.1+ zod: 3.25.76+ zod-to-json-schema: 3.25.0([email protected])+ transitivePeerDependencies:+ - '@aws-sdk/credential-provider-web-identity'+ - supports-color+dependencies:baseline-browser-mapping: 2.8.5@@ -30416,6 +30673,8 @@ snapshots:+ [email protected]: {}+@@ -30534,6 +30793,8 @@ snapshots:parse5: 7.1.2parse5-htmlparser2-tree-adapter: 7.0.0+ [email protected]: {}+dependencies:chevrotain: 11.0.3@@ -30606,8 +30867,18 @@ snapshots:dependencies:restore-cursor: 5.1.0+ dependencies:+ string-width: 4.2.3++ dependencies:+ string-width: 4.2.3+ optionalDependencies:+ '@colors/colors': 1.5.0+dependencies:slice-ansi: 3.0.0@@ -30784,6 +31055,23 @@ snapshots:normalize-path: 3.0.0readable-stream: 4.6.0+ dependencies:+ compute-dot: 1.1.0+ compute-l2norm: 1.1.0+ validate.io-array: 1.0.6+ validate.io-function: 1.0.2++ dependencies:+ validate.io-array: 1.0.6+ validate.io-function: 1.0.2++ dependencies:+ validate.io-array: 1.0.6+ validate.io-function: 1.0.2+@@ -32263,6 +32551,8 @@ snapshots:+ [email protected]: {}+@@ -32582,6 +32872,8 @@ snapshots:+ [email protected]: {}+dependencies:escape-string-regexp: 1.0.5@@ -33836,6 +34128,8 @@ snapshots:is-alphabetical: 2.0.1is-decimal: 2.0.1+ [email protected]: {}+dependencies:call-bind: 1.0.8@@ -34448,6 +34742,10 @@ snapshots:- encoding- supports-color+ dependencies:+ '@pgsql/types': 17.6.2+dependencies:cookie: 0.7.2@@ -34458,6 +34756,12 @@ snapshots:+ dependencies:+ cheminfo-types: 1.8.1+ ml-matrix: 6.12.1+ ml-spectra-processing: 14.18.2+dependencies:base64-js: 0.0.8@@ -35975,6 +36279,36 @@ snapshots:+ dependencies:+ is-any-array: 2.0.1++ dependencies:+ is-any-array: 2.0.1++ dependencies:+ is-any-array: 2.0.1+ ml-array-max: 1.2.4+ ml-array-min: 1.2.3++ dependencies:+ is-any-array: 2.0.1+ ml-array-rescale: 1.3.7++ dependencies:+ binary-search: 1.3.6+ cheminfo-types: 1.8.1+ fft.js: 4.0.4+ is-any-array: 2.0.1+ ml-matrix: 6.12.1+ ml-xsadd: 3.0.1++ [email protected]: {}+dependencies:acorn: 8.15.0@@ -36054,6 +36388,8 @@ snapshots:+ [email protected]: {}+@@ -36520,7 +36856,7 @@ snapshots:- [email protected]([email protected](@opentelemetry/[email protected])(@playwright/[email protected])([email protected]([email protected]))([email protected])([email protected])):+ [email protected]([email protected](@babel/[email protected]([email protected]))(@opentelemetry/[email protected])(@playwright/[email protected])([email protected]([email protected]))([email protected])([email protected])):dependencies:mitt: 3.0.1next: 15.5.9(@babel/[email protected]([email protected]))(@opentelemetry/[email protected])(@playwright/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])@@ -39921,7 +40257,7 @@ snapshots:dependencies:openapi-fetch: 0.6.2dependencies:bin-links: 6.0.0https-proxy-agent: 7.0.6([email protected])@@ -40065,6 +40401,8 @@ snapshots:dependencies:bintrees: 1.0.2+ [email protected]: {}+dependencies:'@jridgewell/trace-mapping': 0.3.31@@ -40898,6 +41236,10 @@ snapshots:+ [email protected]: {}++ [email protected]: {}+dependencies:@@ -41471,6 +41813,10 @@ snapshots:dependencies:string-width: 5.1.2+ dependencies:+ string-width: 7.2.0+@@ -41491,6 +41837,12 @@ snapshots:string-width: 5.1.2strip-ansi: 7.1.0+ dependencies:+ ansi-styles: 6.2.1+ string-width: 7.2.0+ strip-ansi: 7.1.0+diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 423c8e85fb1..52f428ea86d 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -51,6 +51,7 @@ minimumReleaseAgeExclude:- iceberg-js- '@vitejs/plugin-rsc'- stripe-experiment-sync # TODO(matlin) remove, temp just to unblock launch+ - braintrustonlyBuiltDependencies:- supabasediff --git a/turbo.json b/turbo.jsonindex b2042f2d1d6..8cb394fc1ba 100644--- a/turbo.json+++ b/turbo.json@@ -104,6 +104,8 @@"DEFAULT_PROJECT_NAME","DEFAULT_ORGANIZATION_NAME","OPENAI_API_KEY",+ "BRAINTRUST_API_KEY",+ "BRAINTRUST_PROJECT_ID","AUTH_JWT_SECRET","LOGFLARE_API_KEY","LOGFLARE_PUBLIC_ACCESS_TOKEN",