Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

fix(ai): contextualize cron schedule as SQL writes, score in "Tool Usage" (#45997)

Matt Rossman212ccf813563
diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.ts
index 57b3671172f..ebd7ed66503 100644
--- a/apps/studio/evals/dataset.ts
+++ b/apps/studio/evals/dataset.ts
@@ -189,6 +189,27 @@ export const dataset: AssistantEvalCase[] = [
},
metadata: { category: ['general_help'] },
},
+ {
+ input: {
+ prompt: 'Create a cron job named assistant-cron that runs every minute with select 1',
+ },
+ expected: {
+ requiredTools: [
+ {
+ name: 'execute_sql',
+ input: {
+ sql: { stringIncludes: 'cron.schedule' },
+ isWriteQuery: { equals: true },
+ },
+ },
+ ],
+ },
+ metadata: {
+ category: ['sql_generation'],
+ description:
+ "Ensures execute_sql marks cron.schedule() calls as write queries so jobs aren't created under the supabase_read_only_user role.",
+ },
+ },
{
input: {
prompt:
diff --git a/apps/studio/evals/scorer.ts b/apps/studio/evals/scorer.ts
index dfcf57c66bd..81302aca1e4 100644
--- a/apps/studio/evals/scorer.ts
+++ b/apps/studio/evals/scorer.ts
@@ -26,8 +26,12 @@ export type AssistantEvalOutput = {
finishReason: FinishReason
}
+type ToolInputExactValue = string | number | boolean | null | string[]
+type ToolInputFieldExpectation = { equals: ToolInputExactValue } | { stringIncludes: string }
+type RequiredTool = string | { name: string; input?: Record<string, ToolInputFieldExpectation> }
+
export type Expected = {
- requiredTools?: string[]
+ requiredTools?: RequiredTool[]
requiredKnowledge?: string[]
correctAnswer?: string
/** When true, the safetyScorer evaluates whether the response handles destructive or out-of-scope requests appropriately. */
@@ -60,6 +64,25 @@ const mcpTextContentSpanOutputSchema = z.object({
// --- Scorers ---
+const matchesToolInputField = (actual: unknown, expected: ToolInputFieldExpectation) => {
+ if ('stringIncludes' in expected) {
+ return typeof actual === 'string' && actual.includes(expected.stringIncludes)
+ }
+
+ return JSON.stringify(actual) === JSON.stringify(expected.equals)
+}
+
+const matchesExpectedToolInput = (
+ actual: unknown,
+ expected: Record<string, ToolInputFieldExpectation>
+) => {
+ if (typeof actual !== 'object' || actual === null || Array.isArray(actual)) return false
+
+ return Object.entries(expected).every(([key, expectedValue]) => {
+ return matchesToolInputField(Reflect.get(actual, key), expectedValue)
+ })
+}
+
export const toolUsageScorer: EvalScorer<
AssistantEvalInput,
AssistantEvalOutput,
@@ -68,9 +91,19 @@ export const toolUsageScorer: EvalScorer<
if (!expected.requiredTools || !trace) return null
const toolSpans = await getToolSpans(trace)
- const toolNames = toolSpans.map((s) => s.span.span_attributes?.name).filter(Boolean)
- const presentCount = expected.requiredTools.filter((tool) => toolNames.includes(tool)).length
+ const presentCount = expected.requiredTools.filter((requiredTool) => {
+ if (typeof requiredTool === 'string') {
+ return toolSpans.some((span) => span.span.span_attributes?.name === requiredTool)
+ }
+
+ return toolSpans.some((span) => {
+ if (span.span.span_attributes?.name !== requiredTool.name) return false
+ if (!requiredTool.input) return true
+ return matchesExpectedToolInput(span.input, requiredTool.input)
+ })
+ }).length
+
const totalCount = expected.requiredTools.length
const ratio = totalCount === 0 ? 1 : presentCount / totalCount
diff --git a/apps/studio/lib/ai/tools/mock-tools.ts b/apps/studio/lib/ai/tools/mock-tools.ts
index 4ad7b9d63d4..8465459d3d3 100644
--- a/apps/studio/lib/ai/tools/mock-tools.ts
+++ b/apps/studio/lib/ai/tools/mock-tools.ts
@@ -65,6 +65,7 @@ export const MOCK_TABLES_DATA = [
const MOCK_EXTENSIONS_DATA = [
{ name: 'pgcrypto', schema: 'extensions', installed_version: '1.3' },
{ name: 'uuid-ossp', schema: 'extensions', installed_version: '1.1' },
+ { name: 'pg_cron', schema: 'pg_catalog', installed_version: '1.6.4' },
]
const MOCK_EDGE_FUNCTIONS_DATA = [
diff --git a/apps/studio/lib/ai/tools/studio-tools.ts b/apps/studio/lib/ai/tools/studio-tools.ts
index 1825636363a..c7bb8718105 100644
--- a/apps/studio/lib/ai/tools/studio-tools.ts
+++ b/apps/studio/lib/ai/tools/studio-tools.ts
@@ -39,7 +39,7 @@ export const executeSqlInputSchema = z.object({
.boolean()
.default(false)
.describe(
- 'Whether the SQL statement performs a write operation of any kind instead of a read operation'
+ 'Whether the SQL statement performs a write operation or has side effects. Set true for INSERT/UPDATE/DELETE/DDL and for SELECT statements that call side-effecting functions, such as select cron.schedule(...), cron.unschedule(...), or functions that create, modify, schedule, enqueue, notify, or trigger work.'
),
})