The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(assistant): hallucination scorers + corrective measures for storage versioning answers (#41655)
Matt Rossman4553f09bb516
diff --git a/apps/studio/evals/assistant.eval.ts b/apps/studio/evals/assistant.eval.tsindex d2c27f40303..da25f794b35 100644--- a/apps/studio/evals/assistant.eval.ts+++ b/apps/studio/evals/assistant.eval.ts@@ -7,6 +7,8 @@ import { dataset } from './dataset'import {completenessScorer,concisenessScorer,+ correctnessScorer,+ docsFaithfulnessScorer,goalCompletionScorer,sqlSyntaxScorer,toolUsageScorer,@@ -26,19 +28,18 @@ Eval('Assistant', {tools: await getMockTools(),})+ const finishReason = await result.finishReason+// `result.toolCalls` only shows the last step, instead aggregate tools across all stepsconst steps = await result.steps- const stepsSerialized = steps- .map((step) => {- const toolCalls = step.toolCalls- ?.map((call) => JSON.stringify({ tool: call.toolName, input: call.input }))- .join('\n')-- const text = step.text- return toolCalls ? `${text}\n${toolCalls}` : text- })- .join('\n')+ const simplifiedSteps = steps.map((step) => ({+ text: step.text,+ toolCalls: step.toolCalls.map((call) => ({+ toolName: call.toolName,+ input: call.input,+ })),+ }))const toolNames: string[] = []const sqlQueries: string[] = []@@ -65,7 +66,8 @@ Eval('Assistant', {}return {- stepsSerialized,+ finishReason,+ steps: simplifiedSteps,toolNames,sqlQueries,docs,@@ -77,6 +79,8 @@ Eval('Assistant', {goalCompletionScorer,concisenessScorer,completenessScorer,+ docsFaithfulnessScorer,+ correctnessScorer,],})diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.tsindex 58c31926091..16ce4af0437 100644--- a/apps/studio/evals/dataset.ts+++ b/apps/studio/evals/dataset.ts@@ -35,4 +35,29 @@ export const dataset: AssistantEvalCase[] = [},metadata: { category: ['sql_generation', 'database_optimization'] },},+ {+ input: 'How many projects are included in the free tier?',+ expected: {+ requiredTools: ['search_docs'],+ correctAnswer: '2',+ },+ metadata: { category: ['general_help'] },+ },+ {+ input: 'Restore my Supabase Storage files to the state from 3 days ago',+ expected: {+ requiredTools: ['search_docs'],+ correctAnswer:+ 'There is no way to restore these files. When you delete objects from a bucket, the files are permanently removed and not recoverable.',+ },+ metadata: { category: ['general_help'] },+ },+ {+ input: 'How do I enable S3 versioning in Supabase Storage?',+ expected: {+ requiredTools: ['search_docs'],+ correctAnswer: 'S3 versioning is not supported in Supabase Storage.',+ },+ metadata: { category: ['general_help'] },+ },]diff --git a/apps/studio/evals/scorer.ts b/apps/studio/evals/scorer.tsindex 62f68f41d7c..718f712ec60 100644--- a/apps/studio/evals/scorer.ts+++ b/apps/studio/evals/scorer.ts@@ -1,5 +1,6 @@-import { EvalCase, EvalScorer } from 'braintrust'+import { FinishReason } from 'ai'import { LLMClassifierFromTemplate } from 'autoevals'+import { EvalCase, EvalScorer } from 'braintrust'import { stripIndent } from 'common-tags'import { parse } from 'libpg-query'@@ -8,7 +9,8 @@ const LLM_AS_A_JUDGE_MODEL = 'gpt-5.2-2025-12-11'type Input = stringtype Output = {- stepsSerialized: string+ finishReason: FinishReason+ steps: Array<{ text: string; toolCalls: Array<{ toolName: string; input: unknown }> }>toolNames: string[]sqlQueries: string[]docs: string[]@@ -16,6 +18,7 @@ type Output = {export type Expected = {requiredTools?: string[]+ correctAnswer?: string}// Based on categories in the AssistantMessageRatingSubmittedEvent@@ -35,6 +38,30 @@ export type AssistantEvalCaseMetadata = {export type AssistantEvalCase = EvalCase<Input, Expected, AssistantEvalCaseMetadata>+/**+ * Serialize steps into a string representation including text and tool calls+ */+function serializeSteps(steps: Output['steps']): string {+ return steps+ .map((step) => {+ const toolCalls = step.toolCalls+ ?.map((call) => JSON.stringify({ tool: call.toolName, input: call.input }))+ .join('\n')+ return toolCalls ? `${step.text}\n${toolCalls}` : step.text+ })+ .join('\n')+}++/**+ * Extract only the text content from steps, filtering out empty text+ */+function extractTextOnly(steps: Output['steps']): string {+ return steps+ .map((step) => step.text)+ .filter((text) => text && text.trim().length > 0)+ .join('\n')+}+export const toolUsageScorer: EvalScorer<Input, Output, Expected> = async ({output,expected,@@ -99,7 +126,7 @@ const concisenessEvaluator = LLMClassifierFromTemplate<{ input: string }>({export const concisenessScorer: EvalScorer<Input, Output, Expected> = async ({ input, output }) => {return await concisenessEvaluator({input,- output: output.stepsSerialized,+ output: serializeSteps(output.steps),})}@@ -126,7 +153,7 @@ export const completenessScorer: EvalScorer<Input, Output, Expected> = async ({}) => {return await completenessEvaluator({input,- output: output.stepsSerialized,+ output: serializeSteps(output.steps),})}@@ -154,6 +181,85 @@ export const goalCompletionScorer: EvalScorer<Input, Output, Expected> = async (}) => {return await goalCompletionEvaluator({input,- output: output.stepsSerialized,+ output: serializeSteps(output.steps),+ })+}++const docsFaithfulnessEvaluator = LLMClassifierFromTemplate<{ docs: string }>({+ name: 'Docs Faithfulness',+ promptTemplate: stripIndent`+ Evaluate whether the assistant's response accurately reflects the information in the retrieved documentation.++ Retrieved Documentation:+ {{docs}}++ Assistant Response:+ {{output}}++ Does the assistant's response accurately reflect the documentation without contradicting it or adding unsupported claims?+ a) Faithful - response accurately reflects the docs, no contradictions or unsupported claims+ b) Partially faithful - mostly accurate but has minor inaccuracies or unsupported details+ c) Not faithful - contradicts the docs or makes significant unsupported claims+ `,+ choiceScores: { a: 1, b: 0.5, c: 0 },+ useCoT: true,+ model: LLM_AS_A_JUDGE_MODEL,+})++export const docsFaithfulnessScorer: EvalScorer<Input, Output, Expected> = async ({ output }) => {+ // Skip scoring if no docs were retrieved+ if (!output.docs || output.docs.length === 0) {+ return null+ }++ const docsText = output.docs.join('\n\n')++ return await docsFaithfulnessEvaluator({+ docs: docsText,+ output: extractTextOnly(output.steps),+ })+}++const correctnessEvaluator = LLMClassifierFromTemplate<{ input: string; expected: string }>({+ name: 'Correctness',+ promptTemplate: stripIndent`+ Evaluate whether the assistant's answer is correct according to the expected answer.++ Question:+ {{input}}++ Expected Answer:+ {{expected}}++ Assistant Response:+ {{output}}++ Is the assistant's response correct? The response can contain additional information beyond the expected answer, but it must:+ - Include the expected answer (or equivalent information)+ - Not contradict the expected answer++ a) Correct - response includes the expected answer, no contradictions or omissions+ b) Partially correct - includes most of the expected answer but has minor omissions or contradictions+ c) Incorrect - contradicts or fails to provide the expected answer+ `,+ choiceScores: { a: 1, b: 0.5, c: 0 },+ useCoT: true,+ model: LLM_AS_A_JUDGE_MODEL,+})++export const correctnessScorer: EvalScorer<Input, Output, Expected> = async ({+ input,+ output,+ expected,+}) => {+ // Skip scoring if no ground truth is provided+ if (!expected.correctAnswer) {+ return null+ }++ return await correctnessEvaluator({+ input,+ expected: expected.correctAnswer,+ output: extractTextOnly(output.steps),})}diff --git a/apps/studio/lib/ai/prompts.ts b/apps/studio/lib/ai/prompts.tsindex d3c9ece814b..5f52d254789 100644--- a/apps/studio/lib/ai/prompts.ts+++ b/apps/studio/lib/ai/prompts.ts@@ -567,13 +567,25 @@ export const GENERAL_PROMPT = `Act as a Supabase Postgres expert to assist users in efficiently managing their Supabase projects.## InstructionsSupport the user by:-- Gathering context from the database using the \`list_tables\`, \`list_extensions\`, and \`list_edge_functions\` tools+- Gathering context from Supabase official documentation and the user's database- Writing SQL queries- Creating Edge Functions- Debugging issues- Monitoring project status+## Tool Selection Strategy+Before using tools, determine the task type (not exhaustive):++**For questions about Supabase features/capabilities/limitations, or tasks**+- Use \`search_docs\` FIRST before making claims or gathering database context+- Examples: "How do I...", "Can Supabase...", "Is it possible to..."++**For database interactions:**+- Use \`list_tables\`, \`list_extensions\` to understand current schema++**For Edge Function interactions:**+- Use \`list_edge_functions\` to understand current Edge Functions## Tools-- Always use available context gathering tools such as \`list_tables\`, \`list_extensions\`, and \`list_edge_functions\`+- Always call context gathering tools in parallel, not sequentially.- Tools are for assistant use only; do not imply user access to them.- Only use the tools listed above. For read-only or information-gathering operations, call tools automatically; for potentially destructive actions, obtain explicit user confirmation before proceeding.- Tool access may be limited by organizational settings. If required permissions for a task are unavailable, inform the user of this limitation and propose alternatives if possible.@@ -585,7 +597,9 @@ Support the user by:- Never use tables in responses and use emojis minimally.If a tool output should be summarized, integrate the information clearly into the Markdown response. When a tool call returns an error, provide a concise inline explanation or summary of the error. Quote large error messages only if essential to user action. Upon each tool call or code edit, validate the result in 1–2 lines and proceed or self-correct if validation fails.## Documentation Search-- Use \`search_docs\` to query Supabase documentation for questions involving Supabase features or complex database operations.+- When users ask about Supabase features, limitations, or capabilities, use \`search_docs\` BEFORE attempting database operations or making claims+- If \`search_docs\` reveals a limitation, inform the user immediately without gathering database context+- Do not make claims unsupported by documentation`export const CHAT_PROMPT = `@@ -622,6 +636,11 @@ export const CHAT_PROMPT = `- To check organization usage, use the organization's usage page. Link directly to https://supabase.com/dashboard/org/_/usage.- Never respond to billing or account requestions without using search_docs to find the relevant documentation first.- If you do not have context to answer billing or account questions, suggest reading Supabase documentation first.+# Data Recovery+When asked about restoring/recovering deleted data:+1. Search docs for how deletion works for that data type (e.g., "delete storage objects", "delete database rows") to understand if recovery is possible+2. If recovery is possible (or inconclusive), search docs for restore/backup options+DO NOT start searching for recovery docs before checking deletion docs`export const OUTPUT_ONLY_PROMPT = `