The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
test(studio): mock notebook tools (#48952)
Charisb9835e419c9b
diff --git a/apps/studio/data/content/notebooks/notebook-operations.ts b/apps/studio/data/content/notebooks/notebook-operations.tsindex 8b9ba411ddc..8ff279ac980 100644--- a/apps/studio/data/content/notebooks/notebook-operations.ts+++ b/apps/studio/data/content/notebooks/notebook-operations.ts@@ -67,6 +67,17 @@ export type ApplyNotebookOperationsResult =| { success: true; notebook: NotebookOperationsResult }| { success: false; error: NotebookOperationError }+export function describeNotebookOperationError(error: NotebookOperationError): string {+ switch (error._tag) {+ case 'unknown_cell_id':+ return `No cell with id "${error.cell_id}" exists in this notebook.`+ case 'conflicting_operations':+ return `More than one operation targets cell "${error.cell_id}".`+ case 'empty_result':+ return 'This update would leave the notebook with no cells.'+ }+}+function targetCellId(operation: NotebookOperation): string | undefined {switch (operation._tag) {case 'insert_cell':diff --git a/apps/studio/lib/ai/tools/mock-tools.test.ts b/apps/studio/lib/ai/tools/mock-tools.test.tsindex 9e86efbb9c4..a9b808b703f 100644--- a/apps/studio/lib/ai/tools/mock-tools.test.ts+++ b/apps/studio/lib/ai/tools/mock-tools.test.ts@@ -1,6 +1,8 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'-import { getMockTools } from './mock-tools'+import { getMockTools, MOCK_NOTEBOOKS_DATA } from './mock-tools'+import { getNotebookTools } from './notebook-tools'+import type { AgentNotebook } from '@/data/content/notebooks/notebook-schema'import { createInProcessSupabaseMCPClient } from '@/lib/ai/supabase-mcp'// The one real tool in the eval harness (search_docs) is sourced from an@@ -61,4 +63,201 @@ describe('ai/tools/mock-tools getMockTools', () => {afterEach(() => {vi.clearAllMocks()})++ describe('notebook tools', () => {+ const AUTH_HEALTH_NOTEBOOK_ID = MOCK_NOTEBOOKS_DATA[0].id+ const EDGE_FUNCTION_NOTEBOOK_ID = MOCK_NOTEBOOKS_DATA[1].id++ it('list_notebooks reflects the two seeded fixtures', async () => {+ const mockTools = await getMockTools(undefined, new AbortController().signal)+ if (!mockTools.list_notebooks.execute) throw new Error('execute is undefined')++ const result = await mockTools.list_notebooks.execute(+ { limit: 20 },+ { toolCallId: 'test', messages: [] }+ )++ expect(result.notebooks.map((notebook) => notebook.name)).toEqual([+ 'Auth health check',+ 'Edge function error triage',+ ])+ expect(result.notebooks.map((notebook) => notebook.cell_count)).toEqual([3, 2])+ expect(result.notebooks[1].description).toBeUndefined()+ expect(result.cursor).toBeUndefined()+ })++ it('get_notebook resolves cells in order and rejects an unknown id', async () => {+ const mockTools = await getMockTools(undefined, new AbortController().signal)+ if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')++ const result = await mockTools.get_notebook.execute(+ { id: AUTH_HEALTH_NOTEBOOK_ID },+ { toolCallId: 'test', messages: [] }+ )++ expect(result.cells.map((cell) => cell._tag)).toEqual([+ 'markdown_cell',+ 'database_cell',+ 'log_cell',+ ])++ const [, databaseCell, logCell] = result.cells+ if (databaseCell._tag !== 'database_cell') throw new Error('expected database_cell')+ if (logCell._tag !== 'log_cell') throw new Error('expected log_cell')++ expect(databaseCell.sql).toContain('signups')+ expect(databaseCell.row_limit).toBe(30)+ expect(logCell.time_range).toEqual({ _tag: 'relative_time_range', unit: 'hour', amount: 1 })++ await expect(+ mockTools.get_notebook.execute(+ { id: 'unknown-notebook-id' },+ { toolCallId: 'test', messages: [] }+ )+ ).rejects.toThrow(/not found/i)+ })++ it('overrides create_notebook needsApproval to false, unlike the real tool', async () => {+ const mockTools = await getMockTools(undefined, new AbortController().signal)++ expect(getNotebookTools().create_notebook.needsApproval).toBe(true)+ expect(mockTools.create_notebook.needsApproval).toBe(false)+ })++ it('create_notebook stores a new notebook visible via get_notebook and list_notebooks', async () => {+ const mockTools = await getMockTools(undefined, new AbortController().signal)+ if (!mockTools.create_notebook.execute) throw new Error('execute is undefined')+ if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')+ if (!mockTools.list_notebooks.execute) throw new Error('execute is undefined')++ const content: AgentNotebook = {+ schema_version: 1,+ cells: [+ { _tag: 'markdown_cell', text: '# New notebook' },+ { _tag: 'database_cell', sql: 'select 1', row_limit: 10 },+ ],+ }++ const created = await mockTools.create_notebook.execute(+ { name: 'New notebook', content },+ { toolCallId: 'test', messages: [] }+ )+ expect(created).toEqual({ id: expect.any(String), name: 'New notebook' })++ const fetched = await mockTools.get_notebook.execute(+ { id: created.id },+ { toolCallId: 'test', messages: [] }+ )+ expect(fetched.cells).toHaveLength(2)+ expect(fetched.cells.every((cell) => typeof cell.id === 'string')).toBe(true)++ const listed = await mockTools.list_notebooks.execute(+ { limit: 20 },+ { toolCallId: 'test', messages: [] }+ )+ expect(listed.notebooks).toHaveLength(3)+ const newEntry = listed.notebooks.find((notebook) => notebook.id === created.id)+ expect(newEntry?.cell_count).toBe(2)+ })++ it('overrides update_notebook needsApproval to false, unlike the real tool', async () => {+ const mockTools = await getMockTools(undefined, new AbortController().signal)++ expect(getNotebookTools().update_notebook.needsApproval).toBe(true)+ expect(mockTools.update_notebook.needsApproval).toBe(false)+ })++ it('update_notebook inserts and deletes cells, and list_notebooks reflects the new cell count', async () => {+ const mockTools = await getMockTools(undefined, new AbortController().signal)+ if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')+ if (!mockTools.update_notebook.execute) throw new Error('execute is undefined')+ if (!mockTools.list_notebooks.execute) throw new Error('execute is undefined')++ const before = await mockTools.get_notebook.execute(+ { id: AUTH_HEALTH_NOTEBOOK_ID },+ { toolCallId: 'test', messages: [] }+ )+ const [markdownCell, , logCell] = before.cells++ const result = await mockTools.update_notebook.execute(+ {+ id: AUTH_HEALTH_NOTEBOOK_ID,+ operations: [+ {+ _tag: 'insert_cell',+ after_cell_id: markdownCell.id,+ cell: { _tag: 'database_cell', sql: 'select 1', row_limit: 10 },+ },+ { _tag: 'delete_cell', cell_id: logCell.id },+ ],+ },+ { toolCallId: 'test', messages: [] }+ )+ expect(result).toEqual({ id: AUTH_HEALTH_NOTEBOOK_ID, name: 'Auth health check' })++ const after = await mockTools.get_notebook.execute(+ { id: AUTH_HEALTH_NOTEBOOK_ID },+ { toolCallId: 'test', messages: [] }+ )+ expect(after.cells.map((cell) => cell._tag)).toEqual([+ 'markdown_cell',+ 'database_cell',+ 'database_cell',+ ])++ const listed = await mockTools.list_notebooks.execute(+ { limit: 20 },+ { toolCallId: 'test', messages: [] }+ )+ const entry = listed.notebooks.find((notebook) => notebook.id === AUTH_HEALTH_NOTEBOOK_ID)+ expect(entry?.cell_count).toBe(3)+ })++ it('update_notebook rejects an unknown cell_id without mutating the notebook', async () => {+ const mockTools = await getMockTools(undefined, new AbortController().signal)+ if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')+ if (!mockTools.update_notebook.execute) throw new Error('execute is undefined')++ await expect(+ mockTools.update_notebook.execute(+ {+ id: EDGE_FUNCTION_NOTEBOOK_ID,+ operations: [{ _tag: 'delete_cell', cell_id: 'does-not-exist' }],+ },+ { toolCallId: 'test', messages: [] }+ )+ ).rejects.toThrow(/does-not-exist/)++ const after = await mockTools.get_notebook.execute(+ { id: EDGE_FUNCTION_NOTEBOOK_ID },+ { toolCallId: 'test', messages: [] }+ )+ expect(after.cells.map((cell) => cell._tag)).toEqual(['markdown_cell', 'log_cell'])+ })++ it('is isolated per call to getMockTools', async () => {+ const firstCall = await getMockTools(undefined, new AbortController().signal)+ if (!firstCall.create_notebook.execute) throw new Error('execute is undefined')++ await firstCall.create_notebook.execute(+ {+ name: 'Ephemeral notebook',+ content: { schema_version: 1, cells: [{ _tag: 'markdown_cell', text: 'hi' }] },+ },+ { toolCallId: 'test', messages: [] }+ )++ const secondCall = await getMockTools(undefined, new AbortController().signal)+ if (!secondCall.list_notebooks.execute) throw new Error('execute is undefined')++ const result = await secondCall.list_notebooks.execute(+ { limit: 20 },+ { toolCallId: 'test', messages: [] }+ )+ expect(result.notebooks.map((notebook) => notebook.name)).toEqual([+ 'Auth health check',+ 'Edge function error triage',+ ])+ })+ })})diff --git a/apps/studio/lib/ai/tools/mock-tools.ts b/apps/studio/lib/ai/tools/mock-tools.tsindex 049b0b5fecb..2ceb1b452b2 100644--- a/apps/studio/lib/ai/tools/mock-tools.ts+++ b/apps/studio/lib/ai/tools/mock-tools.ts@@ -1,8 +1,20 @@import assert from 'node:assert'-import { tool, type ToolSet } from 'ai'+import { tool, type ToolCallOptions, type ToolSet } from 'ai'import { z } from 'zod'import { getStudioTools } from '../tools/studio-tools'+import { getNotebookTools } from './notebook-tools'+import {+ applyNotebookOperations,+ describeNotebookOperationError,+ type NotebookOperation,+ type OperationResultCell,+} from '@/data/content/notebooks/notebook-operations'+import type {+ AgentNotebook,+ CellWire,+ NotebookWire,+} from '@/data/content/notebooks/notebook-schema'import { createInProcessSupabaseMCPClient } from '@/lib/ai/supabase-mcp'const listTablesInputSchema = z.object({@@ -143,6 +155,75 @@ const MOCK_LOGS_DATA = [},]+type MockNotebook = {+ id: string+ name: string+ description?: string+ visibility: 'project'+ updated_at: string+ content: NotebookWire+}++const MOCK_NOTEBOOK_TIMESTAMP = '2024-06-20T14:30:00Z'++export const MOCK_NOTEBOOKS_DATA: MockNotebook[] = [+ {+ id: '6f1d3a54-8c2b-4d19-9f60-2a7b5c8e1d40',+ name: 'Auth health check',+ description: 'Daily signups plus any auth errors from the last hour.',+ visibility: 'project',+ updated_at: MOCK_NOTEBOOK_TIMESTAMP,+ content: {+ schema_version: 1,+ cells: [+ {+ _tag: 'markdown_cell',+ id: 'c1a0b8e2-3f47-4a52-9d18-6b0c4e2f7a91',+ text: '# Auth health\n\nRun this daily: signup volume, then anything the auth service logged as an error.',+ },+ {+ _tag: 'database_cell',+ id: 'd2b1c9f3-4a58-4b63-8e29-7c1d5f3a8b02',+ title: 'Signups per day',+ sql: "select date_trunc('day', created_at) as day, count(*) as signups\nfrom auth.users\ngroup by day\norder by day desc",+ row_limit: 30,+ chart: { x_column: 'day', y_column: 'signups', cumulative: false },+ },+ {+ _tag: 'log_cell',+ id: 'e3c2d0a4-5b69-4c74-9f3a-8d2e6a4b9c13',+ title: 'Auth errors',+ sql: "select timestamp, event_message\nfrom auth_logs\nwhere event_message like '%error%'\norder by timestamp desc",+ time_range: { _tag: 'relative_time_range', unit: 'hour', amount: 1 },+ },+ ],+ },+ },+ {+ id: '9a4e7b21-6d0c-4f38-8b57-3e1f9c6a2d84',+ name: 'Edge function error triage',+ visibility: 'project',+ updated_at: MOCK_NOTEBOOK_TIMESTAMP,+ content: {+ schema_version: 1,+ cells: [+ {+ _tag: 'markdown_cell',+ id: 'f4d3e1b5-7c80-4d85-8a4b-9e3f7b5c0d24',+ text: '# Edge function errors\n\nFailures from the last day, newest first.',+ },+ {+ _tag: 'log_cell',+ id: '0a5e4f2c-8d91-4e96-9b5c-af408c6d1e35',+ title: 'hello-world failures',+ sql: "select timestamp, event_message\nfrom function_edge_logs\nwhere event_message like '%TypeError%'\norder by timestamp desc",+ time_range: { _tag: 'relative_time_range', unit: 'day', amount: 1 },+ },+ ],+ },+ },+]+function createMockedStudioTools() {const studioTools = getStudioTools()@@ -267,6 +348,152 @@ function createMockListPoliciesTool() {})}+function createMockNotebookStore() {+ const notebooks = new Map(MOCK_NOTEBOOKS_DATA.map((notebook) => [notebook.id, notebook]))++ let notebookCount = 0+ let cellCount = 0++ const assignCellIds = (cells: OperationResultCell[]): CellWire[] =>+ cells.map((cell): CellWire => {+ if ('id' in cell) return cell+ const id = `mock-cell-${++cellCount}`+ switch (cell._tag) {+ case 'markdown_cell':+ return { ...cell, id }+ case 'database_cell':+ return { ...cell, id }+ case 'log_cell':+ return { ...cell, id }+ }+ })++ return {+ list: () => [...notebooks.values()],+ get: (id: string) => notebooks.get(id),+ create: ({+ name,+ description,+ content,+ }: {+ name: string+ description?: string+ content: AgentNotebook+ }) => {+ const notebook: MockNotebook = {+ id: `mock-notebook-${++notebookCount}`,+ name,+ description,+ visibility: 'project',+ updated_at: MOCK_NOTEBOOK_TIMESTAMP,+ content: { schema_version: content.schema_version, cells: assignCellIds(content.cells) },+ }+ notebooks.set(notebook.id, notebook)+ return notebook+ },+ replaceCells: (id: string, cells: OperationResultCell[]) => {+ const existing = notebooks.get(id)+ if (!existing) return+ notebooks.set(id, {+ ...existing,+ content: { schema_version: existing.content.schema_version, cells: assignCellIds(cells) },+ })+ },+ }+}++type MockNotebookStore = ReturnType<typeof createMockNotebookStore>++// All four notebook tools are real, locally-defined ai-SDK tools, so wrap them and+// override only execute/needsApproval — evals must validate the model's arguments+// against the exact schemas production uses (agentCellSchema's `.strict()` rejection of+// agent-authored cell ids, update_notebook's real operations schema, etc).+function createMockNotebookTools(store: MockNotebookStore) {+ const { list_notebooks, get_notebook, create_notebook, update_notebook } = getNotebookTools()++ return {+ list_notebooks: {+ ...list_notebooks,+ execute: async (+ { limit = 20 }: { cursor?: string; limit?: number },+ _options: ToolCallOptions+ ) => ({+ notebooks: store+ .list()+ .slice(0, limit)+ .map((notebook) => ({+ id: notebook.id,+ name: notebook.name,+ description: notebook.description,+ visibility: notebook.visibility,+ updated_at: notebook.updated_at,+ cell_count: notebook.content.cells.length,+ })),+ // The in-memory store never paginates: one page holds everything.+ cursor: undefined,+ }),+ },+ get_notebook: {+ ...get_notebook,+ execute: async ({ id }: { id: string }, _options: ToolCallOptions) => {+ const notebook = store.get(id)+ if (!notebook) throw new Error(`Notebook ${id} not found.`)++ return {+ id: notebook.id,+ name: notebook.name,+ description: notebook.description,+ visibility: notebook.visibility,+ cells: notebook.content.cells,+ }+ },+ },+ create_notebook: {+ ...create_notebook,+ // The eval harness can't answer an approval gate (generate-assistant-response+ // drops tool parts in 'approval-requested' state when cleaning messages), so the+ // real needsApproval: true would stall the eval turn — same override as+ // execute_sql/deploy_edge_function above. Because that gate is gone, this mock+ // deliberately skips acceptUntrustedSql/acceptUntrustedLogsSql promotion: nothing+ // here is executed or sent anywhere, cells are stored as plain data in a Map.+ needsApproval: false,+ execute: async (+ {+ name,+ description,+ content,+ }: {+ name: string+ description?: string+ content: AgentNotebook+ },+ _options: ToolCallOptions+ ) => {+ const created = store.create({ name, description, content })+ return { id: created.id, name: created.name }+ },+ },+ update_notebook: {+ ...update_notebook,+ // Same reasoning as create_notebook's override above.+ needsApproval: false,+ execute: async (+ { id, operations }: { id: string; operations: NotebookOperation[] },+ _options: ToolCallOptions+ ) => {+ const notebook = store.get(id)+ if (!notebook) throw new Error(`Notebook ${id} not found.`)++ const result = applyNotebookOperations(notebook.content, operations)+ if (!result.success) throw new Error(describeNotebookOperationError(result.error))++ store.replaceCells(id, result.notebook.cells)+ return { id, name: notebook.name }+ },+ },+ }+}+export type MockToolOverrides = {list_tables?: Record<string, typeof MOCK_TABLES_DATA>}@@ -280,6 +507,7 @@ export type MockToolOverrides = {*/export async function getMockTools(overrides: MockToolOverrides | undefined, signal: AbortSignal) {const mockedStudioTools = createMockedStudioTools()+ const notebookStore = createMockNotebookStore()// Every tool here is a deterministic mock except `search_docs`, which uses the// real implementation. We source it from an in-process MCP server directly@@ -309,5 +537,6 @@ export async function getMockTools(overrides: MockToolOverrides | undefined, sigget_advisors: createMockGetAdvisorsTool(),query_logs: createMockQueryLogsTool(),list_policies: createMockListPoliciesTool(),+ ...createMockNotebookTools(notebookStore),}}diff --git a/apps/studio/lib/ai/tools/notebook-tools.ts b/apps/studio/lib/ai/tools/notebook-tools.tsindex 64ae842fd24..222ec181175 100644--- a/apps/studio/lib/ai/tools/notebook-tools.ts+++ b/apps/studio/lib/ai/tools/notebook-tools.ts@@ -5,8 +5,8 @@ import { z } from 'zod'import { getContent } from '@/data/content/content-infinite-query'import {applyNotebookOperations,+ describeNotebookOperationError,notebookOperationsSchema,- type NotebookOperationError,} from '@/data/content/notebooks/notebook-operations'import { getNotebook } from '@/data/content/notebooks/notebook-query'import {@@ -20,17 +20,6 @@ import { createNotebook, updateNotebook } from '@/data/content/notebooks/notebooimport { acceptUntrustedLogsSql, untrustedLogSql } from '@/data/logs/safe-analytics-sql'import type { Notebooks } from '@/types'-function describeOperationError(error: NotebookOperationError): string {- switch (error._tag) {- case 'unknown_cell_id':- return `No cell with id "${error.cell_id}" exists in this notebook.`- case 'conflicting_operations':- return `More than one operation targets cell "${error.cell_id}".`- case 'empty_result':- return 'This update would leave the notebook with no cells.'- }-}-export type NotebookToolsContext = {projectRef?: stringauthorization?: string@@ -191,7 +180,7 @@ export const getNotebookTools = (ctx: NotebookToolsContext = {}) => {const result = applyNotebookOperations(wireNotebook, operations)if (!result.success) {- throw new Error(describeOperationError(result.error))+ throw new Error(describeNotebookOperationError(result.error))}// Same promotion as create_notebook above, inlined here for the same auditability