Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

test(studio): add get_notebook eval cases (FE-4088) (#49067)

Charis794e45378c18
diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.ts
index ab3871c2d90..9101b66f98a 100644
--- a/apps/studio/evals/dataset.ts
+++ b/apps/studio/evals/dataset.ts
@@ -463,4 +463,47 @@ export const dataset: AssistantEvalCase[] = [
'Guards against inventing a notebook instead of calling the tool and reporting the real, negative result',
},
},
+ {
+ input: { prompt: 'What queries does my Auth health check notebook run?' },
+ expected: {
+ requiredTools: ['list_notebooks', 'get_notebook'],
+ correctAnswer:
+ "Reports exactly two queries: a database query for signups per day from auth.users, and a logs query for auth errors against the auth_logs source scoped to the last hour. May (but does not have to) note that the remaining cell is markdown and runs no query. Accurate statements about a cell's configuration — the signups cell's 30-row limit or its line chart, the auth errors cell's relative time range — are acceptable. Does not attribute any query, table, or log source the notebook does not contain, and does not misstate the auth errors cell's time range.",
+ },
+ metadata: {
+ category: ['general_help'],
+ description:
+ 'Resolves a notebook name to an id via list_notebooks, then reads it — guards against paraphrasing cells into queries the notebook does not contain',
+ },
+ },
+ {
+ input: { prompt: "Summarize what's in my Edge function error triage notebook" },
+ expected: {
+ requiredTools: ['list_notebooks', 'get_notebook'],
+ correctAnswer:
+ 'Summarizes the notebook as a markdown intro plus one log cell ("hello-world failures") that queries function_edge_logs for TypeError failures over the last day. Log cells hold SQL against a logs source, so describing that cell\'s SQL, columns, or time range is correct and acceptable. Does not attribute a third cell, or any Postgres database cell, to the notebook.',
+ },
+ metadata: {
+ category: ['general_help'],
+ description: 'Baseline read of a smaller, single-query notebook',
+ },
+ },
+ {
+ input: { prompt: 'Get the notebook with id 00000000-0000-0000-0000-000000000000' },
+ expected: {
+ requiredTools: [
+ {
+ name: 'get_notebook',
+ input: { id: { equals: '00000000-0000-0000-0000-000000000000' } },
+ },
+ ],
+ correctAnswer:
+ 'States that no notebook with that id was found, and does not describe any notebook contents.',
+ },
+ metadata: {
+ category: ['general_help'],
+ description:
+ 'Exercises the not-found error path of get_notebook and guards against hallucinating contents for a notebook that does not exist',
+ },
+ },
]
diff --git a/apps/studio/lib/ai/prompts.ts b/apps/studio/lib/ai/prompts.ts
index cdd1cd93603..b0c637c4d1f 100644
--- a/apps/studio/lib/ai/prompts.ts
+++ b/apps/studio/lib/ai/prompts.ts
@@ -767,6 +767,7 @@ export const NOTEBOOKS_PROMPT = `
- Use \`execute_sql\` for a single ad-hoc question with no need to persist it.
- When the request clearly calls for a notebook, call \`create_notebook\` or \`update_notebook\` directly; both tools handle user approval.
- \`update_notebook\` re-fetches the notebook right before applying edits, so the latest save always wins — it cannot detect edits made by someone else in between.
+- When describing an existing notebook, report each query cell's configuration that changes what it returns — a log cell's time range, a database cell's row limit — and don't count markdown cells as queries.
`
export const OUTPUT_ONLY_PROMPT = `