The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
fix(ai): contextualize cron schedule as SQL writes, score in "Tool Usage" (#45997)
Matt Rossman212ccf813563
diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.tsindex 57b3671172f..ebd7ed66503 100644--- a/apps/studio/evals/dataset.ts+++ b/apps/studio/evals/dataset.ts@@ -189,6 +189,27 @@ export const dataset: AssistantEvalCase[] = [},metadata: { category: ['general_help'] },},+ {+ input: {+ prompt: 'Create a cron job named assistant-cron that runs every minute with select 1',+ },+ expected: {+ requiredTools: [+ {+ name: 'execute_sql',+ input: {+ sql: { stringIncludes: 'cron.schedule' },+ isWriteQuery: { equals: true },+ },+ },+ ],+ },+ metadata: {+ category: ['sql_generation'],+ description:+ "Ensures execute_sql marks cron.schedule() calls as write queries so jobs aren't created under the supabase_read_only_user role.",+ },+ },{input: {prompt:diff --git a/apps/studio/evals/scorer.ts b/apps/studio/evals/scorer.tsindex dfcf57c66bd..81302aca1e4 100644--- a/apps/studio/evals/scorer.ts+++ b/apps/studio/evals/scorer.ts@@ -26,8 +26,12 @@ export type AssistantEvalOutput = {finishReason: FinishReason}+type ToolInputExactValue = string | number | boolean | null | string[]+type ToolInputFieldExpectation = { equals: ToolInputExactValue } | { stringIncludes: string }+type RequiredTool = string | { name: string; input?: Record<string, ToolInputFieldExpectation> }+export type Expected = {- requiredTools?: string[]+ requiredTools?: RequiredTool[]requiredKnowledge?: string[]correctAnswer?: string/** When true, the safetyScorer evaluates whether the response handles destructive or out-of-scope requests appropriately. */@@ -60,6 +64,25 @@ const mcpTextContentSpanOutputSchema = z.object({// --- Scorers ---+const matchesToolInputField = (actual: unknown, expected: ToolInputFieldExpectation) => {+ if ('stringIncludes' in expected) {+ return typeof actual === 'string' && actual.includes(expected.stringIncludes)+ }++ return JSON.stringify(actual) === JSON.stringify(expected.equals)+}++const matchesExpectedToolInput = (+ actual: unknown,+ expected: Record<string, ToolInputFieldExpectation>+) => {+ if (typeof actual !== 'object' || actual === null || Array.isArray(actual)) return false++ return Object.entries(expected).every(([key, expectedValue]) => {+ return matchesToolInputField(Reflect.get(actual, key), expectedValue)+ })+}+export const toolUsageScorer: EvalScorer<AssistantEvalInput,AssistantEvalOutput,@@ -68,9 +91,19 @@ export const toolUsageScorer: EvalScorer<if (!expected.requiredTools || !trace) return nullconst toolSpans = await getToolSpans(trace)- const toolNames = toolSpans.map((s) => s.span.span_attributes?.name).filter(Boolean)- const presentCount = expected.requiredTools.filter((tool) => toolNames.includes(tool)).length+ const presentCount = expected.requiredTools.filter((requiredTool) => {+ if (typeof requiredTool === 'string') {+ return toolSpans.some((span) => span.span.span_attributes?.name === requiredTool)+ }++ return toolSpans.some((span) => {+ if (span.span.span_attributes?.name !== requiredTool.name) return false+ if (!requiredTool.input) return true+ return matchesExpectedToolInput(span.input, requiredTool.input)+ })+ }).length+const totalCount = expected.requiredTools.lengthconst ratio = totalCount === 0 ? 1 : presentCount / totalCountdiff --git a/apps/studio/lib/ai/tools/mock-tools.ts b/apps/studio/lib/ai/tools/mock-tools.tsindex 4ad7b9d63d4..8465459d3d3 100644--- a/apps/studio/lib/ai/tools/mock-tools.ts+++ b/apps/studio/lib/ai/tools/mock-tools.ts@@ -65,6 +65,7 @@ export const MOCK_TABLES_DATA = [const MOCK_EXTENSIONS_DATA = [{ name: 'pgcrypto', schema: 'extensions', installed_version: '1.3' },{ name: 'uuid-ossp', schema: 'extensions', installed_version: '1.1' },+ { name: 'pg_cron', schema: 'pg_catalog', installed_version: '1.6.4' },]const MOCK_EDGE_FUNCTIONS_DATA = [diff --git a/apps/studio/lib/ai/tools/studio-tools.ts b/apps/studio/lib/ai/tools/studio-tools.tsindex 1825636363a..c7bb8718105 100644--- a/apps/studio/lib/ai/tools/studio-tools.ts+++ b/apps/studio/lib/ai/tools/studio-tools.ts@@ -39,7 +39,7 @@ export const executeSqlInputSchema = z.object({.boolean().default(false).describe(- 'Whether the SQL statement performs a write operation of any kind instead of a read operation'+ 'Whether the SQL statement performs a write operation or has side effects. Set true for INSERT/UPDATE/DELETE/DDL and for SELECT statements that call side-effecting functions, such as select cron.schedule(...), cron.unschedule(...), or functions that create, modify, schedule, enqueue, notify, or trigger work.'),})