The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
import { describe, expect, it, vi } from 'vitest'
import { getMockTools, MOCK_NOTEBOOKS_DATA } from './mock-tools'
import { getNotebookTools } from './notebook-tools'
import type { AgentNotebook } from '@/data/content/notebooks/notebook-schema'
import { createSearchDocsTool } from '@/lib/ai/tools/search-docs-tool'
import type * as SearchDocsToolModule from '@/lib/ai/tools/search-docs-tool'
// search_docs normally fetches the live docs GraphQL schema on construction
// (see search-docs-tool.ts), which would make every test in this file depend
// on the public docs API. Mock it with a deterministic local fixture instead
// — live connectivity (including the schema fetch) is covered separately by
// evals/preflight.ts. The regression-guard test below overrides this default
// to verify the missing-tool guard fires.
vi.mock('@/lib/ai/tools/search-docs-tool', () => ({
createSearchDocsTool: vi.fn().mockResolvedValue({
description: 'Search the Supabase documentation using GraphQL.',
execute: async () => ({ content: [{ type: 'text' as const, text: '{}' }] }),
} as unknown as SearchDocsToolModule.SearchDocsTool),
}))
describe('ai/tools/mock-tools getMockTools', () => {
it('wires the mocked search_docs tool through from the shared module, alongside the deterministic mocks', async () => {
const result = await getMockTools(undefined)
// The mocked tool, wired through from the shared module
expect(result.search_docs).toBeDefined()
expect(result.search_docs.description).toContain('Search the Supabase documentation')
expect(typeof result.search_docs.execute).toBe('function')
// A couple of the deterministic mocks, to confirm the merge
expect(result).toHaveProperty('list_tables')
expect(result).toHaveProperty('query_logs')
})
// This is the regression guard: if a refactor stops sourcing search_docs
// (drops it from getMockTools, or createSearchDocsTool breaks), fail loudly
// in normal CI instead of only surfacing during an opt-in Braintrust eval run.
it('throws a clear error when search_docs is missing from the harness tools', async () => {
vi.mocked(createSearchDocsTool).mockResolvedValueOnce(
undefined as unknown as SearchDocsToolModule.SearchDocsTool
)
await expect(getMockTools(undefined)).rejects.toThrow(
'search_docs tool is missing from the eval harness'
)
})
describe('notebook tools', () => {
const AUTH_HEALTH_NOTEBOOK_ID = MOCK_NOTEBOOKS_DATA[0].id
const EDGE_FUNCTION_NOTEBOOK_ID = MOCK_NOTEBOOKS_DATA[1].id
it('list_notebooks reflects the two seeded fixtures', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.list_notebooks.execute) throw new Error('execute is undefined')
const result = await mockTools.list_notebooks.execute(
{ limit: 20 },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(result.notebooks.map((notebook) => notebook.name)).toEqual([
'Auth health check',
'Edge function error triage',
])
expect(result.notebooks.map((notebook) => notebook.cell_count)).toEqual([3, 2])
expect(result.notebooks[1].description).toBeUndefined()
expect(result.cursor).toBeUndefined()
})
it('get_notebook resolves cells in order and rejects an unknown id', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')
const result = await mockTools.get_notebook.execute(
{ id: AUTH_HEALTH_NOTEBOOK_ID },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(result.cells.map((cell) => cell._tag)).toEqual([
'markdown_cell',
'database_cell',
'log_cell',
])
const [, databaseCell, logCell] = result.cells
if (databaseCell._tag !== 'database_cell') throw new Error('expected database_cell')
if (logCell._tag !== 'log_cell') throw new Error('expected log_cell')
expect(databaseCell.sql).toContain('signups')
expect(databaseCell.row_limit).toBe(30)
expect(logCell.time_range).toEqual({ _tag: 'relative_time_range', unit: 'hour', amount: 1 })
await expect(
mockTools.get_notebook.execute(
{ id: 'unknown-notebook-id' },
{ toolCallId: 'test', messages: [], context: {} }
)
).rejects.toThrow(/not found/i)
})
it('shares deterministic run_notebook output with eval models', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.run_notebook.execute) throw new Error('execute is undefined')
if (!mockTools.run_notebook.toModelOutput) throw new Error('toModelOutput is undefined')
const output = await mockTools.run_notebook.execute(
{ id: AUTH_HEALTH_NOTEBOOK_ID, expected_updated_at: MOCK_NOTEBOOKS_DATA[0].updated_at },
{ toolCallId: 'test', messages: [], context: {} }
)
const modelOutput = mockTools.run_notebook.toModelOutput({
toolCallId: 'test',
input: { id: AUTH_HEALTH_NOTEBOOK_ID, expected_updated_at: output.updated_at },
output,
})
expect(modelOutput).toMatchObject({
type: 'json',
value: {
cells: expect.arrayContaining([expect.objectContaining({ rows: [] })]),
},
})
})
it('overrides needsApproval to false for notebook mutations, unlike the real tools', async () => {
const mockTools = await getMockTools(undefined)
const realTools = getNotebookTools({ aiOptInLevel: 'schema' })
for (const name of ['create_notebook', 'update_notebook', 'delete_notebook'] as const) {
expect(realTools[name].needsApproval).toBe(true)
expect(mockTools[name].needsApproval).toBe(false)
}
})
it('create_notebook stores a new notebook visible via get_notebook and list_notebooks', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.create_notebook.execute) throw new Error('execute is undefined')
if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')
if (!mockTools.list_notebooks.execute) throw new Error('execute is undefined')
const content: AgentNotebook = {
schema_version: 1,
cells: [
{ _tag: 'markdown_cell', text: '# New notebook' },
{ _tag: 'database_cell', sql: 'select 1', row_limit: 10 },
],
}
const created = await mockTools.create_notebook.execute(
{ name: 'New notebook', content },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(created).toEqual({ id: expect.any(String), name: 'New notebook' })
const fetched = await mockTools.get_notebook.execute(
{ id: created.id },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(fetched.cells).toHaveLength(2)
expect(fetched.cells.every((cell) => typeof cell._id === 'string')).toBe(true)
const listed = await mockTools.list_notebooks.execute(
{ limit: 20 },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(listed.notebooks).toHaveLength(3)
const newEntry = listed.notebooks.find((notebook) => notebook.id === created.id)
expect(newEntry?.cell_count).toBe(2)
})
it('update_notebook inserts and deletes cells, and list_notebooks reflects the new cell count', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')
if (!mockTools.update_notebook.execute) throw new Error('execute is undefined')
if (!mockTools.list_notebooks.execute) throw new Error('execute is undefined')
const before = await mockTools.get_notebook.execute(
{ id: AUTH_HEALTH_NOTEBOOK_ID },
{ toolCallId: 'test', messages: [], context: {} }
)
const [markdownCell, , logCell] = before.cells
const result = await mockTools.update_notebook.execute(
{
id: AUTH_HEALTH_NOTEBOOK_ID,
expected_updated_at: before.updated_at,
operations: [
{
_tag: 'insert_cell',
after_cell_id: markdownCell._id,
cell: { _tag: 'database_cell', sql: 'select 1', row_limit: 10 },
},
{ _tag: 'delete_cell', cell_id: logCell._id },
],
},
{ toolCallId: 'test', messages: [], context: {} }
)
expect(result).toEqual({ id: AUTH_HEALTH_NOTEBOOK_ID, name: 'Auth health check' })
const after = await mockTools.get_notebook.execute(
{ id: AUTH_HEALTH_NOTEBOOK_ID },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(after.cells.map((cell) => cell._tag)).toEqual([
'markdown_cell',
'database_cell',
'database_cell',
])
const listed = await mockTools.list_notebooks.execute(
{ limit: 20 },
{ toolCallId: 'test', messages: [], context: {} }
)
const entry = listed.notebooks.find((notebook) => notebook.id === AUTH_HEALTH_NOTEBOOK_ID)
expect(entry?.cell_count).toBe(3)
})
it('update_notebook rejects an unknown cell_id without mutating the notebook', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.get_notebook.execute) throw new Error('execute is undefined')
if (!mockTools.update_notebook.execute) throw new Error('execute is undefined')
const before = await mockTools.get_notebook.execute(
{ id: EDGE_FUNCTION_NOTEBOOK_ID },
{ toolCallId: 'test', messages: [], context: {} }
)
await expect(
mockTools.update_notebook.execute(
{
id: EDGE_FUNCTION_NOTEBOOK_ID,
expected_updated_at: before.updated_at,
operations: [{ _tag: 'delete_cell', cell_id: 'does-not-exist' }],
},
{ toolCallId: 'test', messages: [], context: {} }
)
).rejects.toThrow(/does-not-exist/)
const after = await mockTools.get_notebook.execute(
{ id: EDGE_FUNCTION_NOTEBOOK_ID },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(after.cells.map((cell) => cell._tag)).toEqual(['markdown_cell', 'log_cell'])
})
it('delete_notebook removes the notebook, and list_notebooks no longer returns it', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.delete_notebook.execute) throw new Error('execute is undefined')
if (!mockTools.list_notebooks.execute) throw new Error('execute is undefined')
const result = await mockTools.delete_notebook.execute(
{ id: AUTH_HEALTH_NOTEBOOK_ID },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(result).toEqual({ id: AUTH_HEALTH_NOTEBOOK_ID, name: 'Auth health check' })
const listed = await mockTools.list_notebooks.execute(
{ limit: 20 },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(listed.notebooks.map((notebook) => notebook.id)).toEqual([EDGE_FUNCTION_NOTEBOOK_ID])
})
it('delete_notebook rejects an unknown id', async () => {
const mockTools = await getMockTools(undefined)
if (!mockTools.delete_notebook.execute) throw new Error('execute is undefined')
await expect(
mockTools.delete_notebook.execute(
{ id: 'unknown-notebook-id' },
{ toolCallId: 'test', messages: [], context: {} }
)
).rejects.toThrow(/unknown-notebook-id/)
})
it('is isolated per call to getMockTools', async () => {
const firstCall = await getMockTools(undefined)
if (!firstCall.create_notebook.execute) throw new Error('execute is undefined')
await firstCall.create_notebook.execute(
{
name: 'Ephemeral notebook',
content: { schema_version: 1, cells: [{ _tag: 'markdown_cell', text: 'hi' }] },
},
{ toolCallId: 'test', messages: [], context: {} }
)
const secondCall = await getMockTools(undefined)
if (!secondCall.list_notebooks.execute) throw new Error('execute is undefined')
const result = await secondCall.list_notebooks.execute(
{ limit: 20 },
{ toolCallId: 'test', messages: [], context: {} }
)
expect(result.notebooks.map((notebook) => notebook.name)).toEqual([
'Auth health check',
'Edge function error triage',
])
})
})
})