Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.7 KB
Raw Blame
import { v4 as uuidv4 } from 'uuid'
import { z } from 'zod'

import type { Schemas } from '@/api/generated'
import type { Context, ToolBase } from '@/tools/types'

import {
    NOT_RUN_HINT,
    runCellAndWriteBack,
    wrapRunResultAsInformational,
    type NotRunResult,
    type ShapedRunResult,
} from './cellRuns'
import {
    buildCellTag,
    COMPONENT_TAG_REGEX,
    DATAFRAME_NAME_REGEX,
    findCellTag,
    parseCellTags,
    uniqueDataframeName,
} from './cellTags'
import { applyVisualization, CellVisualizationSchema, VISUALIZATION_PARAM_DESCRIPTION } from './cellVisualization'
import { applyMarkdownEdit, fetchMarkdownNotebook, notebookPathFor } from './markdownDoc'
import { NOTEBOOK_SHORT_ID_DESCRIPTION, notebookIdAliases } from './notebookId'
import { getNotebookWidgetTagNames, getNotebookWidgetViewError } from './widgetCatalog'

/** The cell header renders a title on a single ellipsized line, so anything longer is cut off anyway. */
const CELL_TITLE_MAX_LENGTH = 120

const AddCellInputSchema = z
    .object({
        notebook_id: z.string().describe(NOTEBOOK_SHORT_ID_DESCRIPTION),
        cell_type: z
            .enum(['sql', 'python', 'markdown', 'saved_insight', 'component'])
            .describe(
                "Cell kind: 'sql' (HogQL against PostHog data) and 'python' run immediately unless run is false; 'markdown' inserts prose (headings, tables, code/mermaid fences); 'saved_insight' embeds an existing insight; 'component' inserts any other notebook component tag (charts, media, PostHog entities)."
            ),
        code: z.string().optional().describe('The SQL or Python source. Required for sql/python cells.'),
        markdown: z.string().optional().describe('Prose to insert. Required for markdown cells.'),
        insight_short_id: z
            .string()
            .optional()
            .describe('Short id of the saved insight to embed. Required for saved_insight cells.'),
        tag_name: z
            .string()
            .regex(COMPONENT_TAG_REGEX)
            .optional()
            .describe(
                `Component cells: the notebook component to insert. Object widgets with named views: ${getNotebookWidgetTagNames().join(', ')}. Other components include Query (product analytics charts and event tables via its query prop), Image, Embed, Latex, Person, Recording, and RecordingPlaylist. For generated interactive visualizations, search for notebooks-widget-generate to check availability and learn how to insert a Widget tag. Executable cells are not allowed here — use cell_type sql/python.`
            ),
        props: z
            .record(z.string(), z.unknown())
            .optional()
            .describe(
                'Component cells: the props for the tag, matching what the notebook UI stores for that component. Object widgets take their identity prop plus an optional view, for example {"id": 123, "view": "summary"}. For Query: {"query": {"kind": "InsightVizNode", "source": <TrendsQuery|FunnelsQuery|RetentionQuery|PathsQuery|StickinessQuery|LifecycleQuery>}} for insights, or {"query": {"kind": "DataTableNode", "source": {"kind": "EventsQuery", …}}} for event tables. HogQLQuery sources are not accepted here: use cell_type sql, and set visualization to chart its result.'
            ),
        dataframe_name: z
            .string()
            .regex(DATAFRAME_NAME_REGEX)
            .optional()
            .describe(
                'Name other cells use to reference this cell\'s result dataframe (e.g. in SQL joins or Python code). Auto-assigned ("sql_df", "df", …) when omitted for sql/python cells.'
            ),
        title: z
            .string()
            .max(CELL_TITLE_MAX_LENGTH)
            .optional()
            .describe(
                'Short label shown in the cell header, e.g. "Weekly signups by source". Set it on every cell you add so a reader can skim the notebook without reading the code — say what the cell shows, not that it is SQL or Python. Not accepted for markdown cells; give those a markdown heading instead.'
            ),
        after_node_id: z
            .string()
            .optional()
            .describe(
                'Insert after this block: any node_id notebooks-get returns, a markdown block included. Defaults to the end of the document. A markdown block id comes from the text, so pass one from the read you are acting on.'
            ),
        visualization: CellVisualizationSchema.optional().describe(VISUALIZATION_PARAM_DESCRIPTION),
        run: z
            .boolean()
            .optional()
            .describe(
                'SQL and Python cells only. Defaults to true: the cell runs as soon as it is added. Pass false to add the cell without a run, for example to lay out dependent cells before running them in order, or to leave a heavy query for the user to run. Run it later with notebooks-run-cell, or run the whole notebook with notebooks-run.'
            ),
    })
    .strict()

export const NotebooksAddCellSchema = z.preprocess(notebookIdAliases('notebook_id'), AddCellInputSchema)

export interface AddCellResult {
    node_id?: string
    dataframe_name?: string
    run?: ShapedRunResult | NotRunResult
    visualization_warnings?: string[]
}

/**
 * One blank line separates two blocks of the same card in the notebook editor; a second one starts
 * a new card. Cells added here are nodes in their own right, so they are separated on both sides —
 * otherwise consecutive markdown cells land as one card (`startsGroup` in the editor's types.ts).
 */
const BLOCK_SEPARATOR = '\n\n\n'

/**
 * A block id the document stores, written on its own line above the block it names. Mirrors
 * `serializeNodeAnchor` in frontend/src/lib/components/MarkdownNotebook/markdown.ts.
 *
 * A markdown block otherwise has no identity of its own: the backend derives its id from the
 * block text, so the id dies with the next edit to that text. An anchor gives a block the same
 * durable id a cell tag carries in its `nodeId` prop.
 */
function anchoredMarkdownBlock(nodeId: string, markdown: string): string {
    return `<!--ph:${nodeId}-->\n${markdown}`
}

/** Matches `STORED_NODE_ID_PREFIX` in the editor, which writes an anchor back only for these. */
const STORED_NODE_ID_PREFIX = 'phb-'

/**
 * A prose block resolved through the state endpoint, which owns the block grammar this side
 * cannot reproduce. `source` re-locates the block when an edit elsewhere has moved the offsets.
 */
interface ProseAnchor {
    nodeId: string
    source: string
    start: number
    end: number
    /** Notebook version the span was read at. The span is exact for that version only. */
    version: number | null
}

// Notebook markdown keeps CRLF and lone-CR line endings, and the backend walk splits on all three.
// The lookahead keeps the two halves of a CRLF from counting as two line breaks.
const EOL = '(?:\\r\\n|\\r(?!\\n)|\\n)'

/**
 * Whether the text at `index` stands as a block of its own, rather than sitting inside a longer
 * one. A blank line separates two blocks, so a match with one on each side covers a whole block.
 *
 * Deliberately strict. A layout this does not recognize is refused and the caller reads again,
 * which costs a round trip. Accepting a match inside a longer paragraph would split that
 * paragraph around the inserted cell, which costs the reader their text.
 */
const BLOCK_START_BOUNDARY = new RegExp(`(?:^|${EOL}[ \\t]*${EOL}[ \\t]*)$`)
const BLOCK_END_BOUNDARY = new RegExp(`^(?:[ \\t]*${EOL}[ \\t]*${EOL}|[ \\t]*${EOL}?$)`)

function isWholeBlockAt(markdown: string, index: number, source: string): boolean {
    return (
        BLOCK_START_BOUNDARY.test(markdown.slice(0, index)) &&
        BLOCK_END_BOUNDARY.test(markdown.slice(index + source.length))
    )
}

/**
 * Where the anchor block ends in `markdown`, by the offsets the read reported, or by its text
 * when an edit elsewhere has moved them.
 *
 * Refuses a source that now appears more than once rather than picking one: the insert would
 * land on a block the caller never saw. The same refusal guards a prose edit in updateCell.ts.
 */
function locateProseAnchorEnd(markdown: string, anchor: ProseAnchor, version: number): number {
    // The backend's grammar set this span, so it is trusted only for the version it was read at.
    // In a later version the same offsets can still hold the same text while the block has grown
    // past them, and an insert at the old end would split the block.
    if (version === anchor.version && markdown.slice(anchor.start, anchor.end) === anchor.source) {
        return anchor.end
    }
    // A stored id is written into the document above its block, so it names the block exactly.
    // Matching on it never confuses two blocks that read the same, which the text search below
    // can only refuse.
    if (anchor.nodeId.startsWith(STORED_NODE_ID_PREFIX)) {
        return locateStoredAnchorEnd(markdown, anchor)
    }
    const matches: number[] = []
    for (
        let index = markdown.indexOf(anchor.source);
        index !== -1;
        index = markdown.indexOf(anchor.source, index + 1)
    ) {
        if (isWholeBlockAt(markdown, index, anchor.source)) {
            matches.push(index)
        }
    }
    if (matches.length === 0) {
        throw new Error(
            `Block ${anchor.nodeId} is no longer a block of its own in notebook, so a cell cannot be placed after it. Read the notebook again with notebooks-get and retry with the id it returns.`
        )
    }
    if (matches.length > 1) {
        throw new Error(
            `Block ${anchor.nodeId} now matches more than one block, so it cannot name one of them. Read the notebook again with notebooks-get and retry with the id it returns.`
        )
    }
    return matches[0]! + anchor.source.length
}

const ANCHOR_TERMINATOR = new RegExp(`^${EOL}`)

/**
 * Where the block under `<!--ph:id-->` ends. The anchor finds the block, and the source the read
 * reported measures it, because a fenced block holds blank lines and a search for the next one
 * would end the block inside the fence.
 */
function locateStoredAnchorEnd(markdown: string, anchor: ProseAnchor): number {
    const marker = `<!--ph:${anchor.nodeId}-->`
    const bodyStarts: number[] = []
    for (let at = markdown.indexOf(marker); at !== -1; at = markdown.indexOf(marker, at + 1)) {
        const terminator = ANCHOR_TERMINATOR.exec(markdown.slice(at + marker.length, at + marker.length + 2))
        if (terminator) {
            bodyStarts.push(at + marker.length + terminator[0].length)
        }
    }
    if (bodyStarts.length === 0) {
        throw new Error(
            `Block ${anchor.nodeId} is no longer in notebook, so a cell cannot be placed after it. Read the notebook again with notebooks-get and retry with the id it returns.`
        )
    }
    if (bodyStarts.length > 1) {
        throw new Error(
            `Block ${anchor.nodeId} names more than one block in notebook, so it cannot name one of them. Read the notebook again with notebooks-get.`
        )
    }
    const bodyStart = bodyStarts[0]!
    const bodyEnd = bodyStart + anchor.source.length
    if (!markdown.startsWith(anchor.source, bodyStart) || !BLOCK_END_BOUNDARY.test(markdown.slice(bodyEnd))) {
        throw new Error(
            `Block ${anchor.nodeId} changed since it was read, so a cell cannot be placed after it. Read the notebook again with notebooks-get and retry.`
        )
    }
    return bodyEnd
}

function insertBlock(
    markdown: string,
    block: string,
    afterNodeId: string | undefined,
    proseAnchor: ProseAnchor | undefined,
    version: number
): string {
    const trimmed = markdown.replace(/\s+$/, '')
    if (proseAnchor) {
        const end = locateProseAnchorEnd(markdown, proseAnchor, version)
        const rest = markdown.slice(end).replace(/^[\r\n]+/, '')
        const head = `${markdown.slice(0, end)}${BLOCK_SEPARATOR}${block}`
        return rest ? `${head}${BLOCK_SEPARATOR}${rest}` : `${head}\n`
    }
    if (afterNodeId) {
        const anchor = findCellTag(markdown, afterNodeId)
        if (!anchor) {
            throw new Error(`No cell with node_id ${afterNodeId} found to insert after.`)
        }
        const rest = markdown.slice(anchor.end).replace(/^\n+/, '')
        const head = `${markdown.slice(0, anchor.end)}${BLOCK_SEPARATOR}${block}`
        return rest ? `${head}${BLOCK_SEPARATOR}${rest}` : `${head}\n`
    }
    return trimmed ? `${trimmed}${BLOCK_SEPARATOR}${block}\n` : `${block}\n`
}

/**
 * Resolve `after_node_id` to a prose anchor, or to `undefined` when a cell tag already carries
 * the id and `insertBlock` can find it in the markdown on its own.
 *
 * A prose id is derived from block text, so it resolves only against the read that produced it.
 * Only the state endpoint walks prose, so a miss in the tag scan goes there rather than failing.
 */
async function resolveInsertAnchor(
    context: Context,
    notebookId: string,
    afterNodeId: string | undefined
): Promise<ProseAnchor | undefined> {
    if (!afterNodeId) {
        return undefined
    }
    const { markdown } = await fetchMarkdownNotebook(context, notebookId)
    if (findCellTag(markdown, afterNodeId)) {
        return undefined
    }

    const projectId = await context.stateManager.getProjectId()
    const state = await context.api.request<{ version: number | null; cells: Schemas.NotebookCellState[] }>({
        method: 'GET',
        path: `${notebookPathFor(projectId, notebookId)}sql_v2/state/`,
    })
    const matches = state.cells.filter((cell) => cell.node_id === afterNodeId)
    if (matches.length === 0) {
        throw new Error(
            `No block with node_id ${afterNodeId} in notebook ${notebookId} to insert after. Read the current ids with notebooks-get; an insight short id is not one of them.`
        )
    }
    if (matches.length > 1) {
        throw new Error(
            `Block ${afterNodeId} names ${matches.length} blocks in notebook ${notebookId}, so it cannot name one of them. Read the notebook again with notebooks-get.`
        )
    }
    const block = matches[0]!
    if (block.cell_type !== 'markdown') {
        throw new Error(
            `Cell ${afterNodeId} is a ${block.cell_type} cell but carries no tag identity, so a cell cannot be placed after it. Insert at the end by omitting after_node_id.`
        )
    }
    return { nodeId: afterNodeId, source: block.code, start: block.start, end: block.end, version: state.version }
}

/**
 * A `<Query>` whose source is HogQL is the legacy SQL cell. It renders a result table or chart but
 * does not run through the sandbox, so it names no dataframe other cells can reference and keeps no
 * run history. SQLV2 (cell_type 'sql') supersedes it and charts its result the same way, so SQL
 * authored over MCP must land there. The editor's insert menu makes the same choice behind the
 * revamped-py-notebooks flag that gates this tool.
 */
function hasHogQLQuerySource(props: Record<string, unknown> | undefined): boolean {
    const query = props?.query
    if (!query || typeof query !== 'object') {
        return false
    }
    const { kind, source } = query as { kind?: unknown; source?: unknown }
    if (kind === 'HogQLQuery') {
        return true
    }
    return !!source && typeof source === 'object' && (source as { kind?: unknown }).kind === 'HogQLQuery'
}

export const addCellHandler: ToolBase<typeof NotebooksAddCellSchema, AddCellResult>['handler'] = async (
    context: Context,
    params: z.infer<typeof NotebooksAddCellSchema>
) => {
    if ((params.cell_type === 'sql' || params.cell_type === 'python') && !params.code?.trim()) {
        throw new Error(`A ${params.cell_type} cell requires non-empty code.`)
    }
    if (params.cell_type === 'markdown' && !params.markdown?.trim()) {
        throw new Error('A markdown cell requires non-empty markdown.')
    }
    if (params.visualization && params.cell_type !== 'sql') {
        throw new Error(
            `visualization applies to sql cells only, not ${params.cell_type} cells. A python cell plots with matplotlib.`
        )
    }
    if (params.cell_type === 'saved_insight' && !params.insight_short_id?.trim()) {
        throw new Error('A saved_insight cell requires insight_short_id.')
    }
    if (params.cell_type === 'component') {
        if (!params.tag_name) {
            throw new Error('A component cell requires tag_name.')
        }
        // Executable and deprecated tags must go through their owning cell types so runs,
        // identity, and result write-back stay consistent.
        if (['SQLV2', 'PythonV2', 'Python', 'DuckSQL', 'HogQLSQL'].includes(params.tag_name)) {
            throw new Error(
                `Use cell_type 'sql' or 'python' for executable cells instead of tag_name ${params.tag_name}.`
            )
        }
        if (hasHogQLQuerySource(params.props)) {
            throw new Error(
                "Use cell_type 'sql' for SQL instead of a component with a HogQLQuery source. A sql cell runs the query, names a dataframe other cells can reference, and charts its result when you set visualization."
            )
        }
        const widgetViewError = getNotebookWidgetViewError(params.tag_name, params.props?.view)
        if (widgetViewError) {
            throw new Error(widgetViewError)
        }
    }

    const title = params.title?.trim() || undefined
    // Resolved before any branch, so every cell type places after prose the same way and a bad
    // anchor fails before a cell is created or a query runs.
    const proseAnchor = await resolveInsertAnchor(context, params.notebook_id, params.after_node_id)

    if (params.cell_type === 'markdown') {
        if (title) {
            throw new Error(
                'A markdown cell has no header to title — put the heading in the markdown itself (e.g. "## Weekly signups").'
            )
        }
        const proseNodeId = `${STORED_NODE_ID_PREFIX}${uuidv4()}`
        await applyMarkdownEdit(context, params.notebook_id, (markdown, version) =>
            insertBlock(
                markdown,
                anchoredMarkdownBlock(proseNodeId, params.markdown!.trim()),
                params.after_node_id,
                proseAnchor,
                version
            )
        )
        return { node_id: proseNodeId }
    }

    const nodeId = uuidv4()

    if (params.cell_type === 'saved_insight') {
        const tag = buildCellTag('Query', {
            nodeId,
            title,
            query: { kind: 'SavedInsightNode', shortId: params.insight_short_id },
        })
        await applyMarkdownEdit(context, params.notebook_id, (markdown, version) =>
            insertBlock(markdown, tag, params.after_node_id, proseAnchor, version)
        )
        return { node_id: nodeId }
    }

    if (params.cell_type === 'component') {
        // `title` last only when set, so a component that carries its own title prop keeps it.
        const tag = buildCellTag(params.tag_name!, { ...params.props, nodeId, ...(title ? { title } : {}) })
        await applyMarkdownEdit(context, params.notebook_id, (markdown, version) =>
            insertBlock(markdown, tag, params.after_node_id, proseAnchor, version)
        )
        return { node_id: nodeId }
    }

    const tagName = params.cell_type === 'sql' ? 'SQLV2' : 'PythonV2'
    const initial = await fetchMarkdownNotebook(context, params.notebook_id)
    const dataframeName =
        params.dataframe_name ??
        uniqueDataframeName(
            params.cell_type === 'sql' ? 'sql_df' : 'df',
            parseCellTags(initial.markdown),
            (initial.notebook.variables ?? []).map((variable) => variable.name)
        )
    const plainTag = buildCellTag(tagName, { nodeId, title, code: params.code, returnVariable: dataframeName })
    const tag = params.visualization ? applyVisualization(plainTag, params.visualization) : plainTag

    // The save response carries the notebook as it stood when the save committed, so it holds a
    // variable edit that landed after the read above. The run binds those values to stay in step
    // with what the notebook now declares.
    const { notebook, markdown } = await applyMarkdownEdit(context, params.notebook_id, (current, version) =>
        insertBlock(current, tag, params.after_node_id, proseAnchor, version)
    )
    if (params.run === false) {
        return { node_id: nodeId, dataframe_name: dataframeName, run: { status: 'not_run', hint: NOT_RUN_HINT } }
    }
    const { run, warnings } = await runCellAndWriteBack(
        context,
        params.notebook_id,
        { nodeId, tagName, code: params.code!, returnVariable: dataframeName },
        parseCellTags(markdown),
        notebook.variables
    )
    return wrapRunResultAsInformational({
        node_id: nodeId,
        dataframe_name: dataframeName,
        run,
        ...(warnings.length ? { visualization_warnings: warnings } : {}),
    })
}

const tool = (): ToolBase<typeof NotebooksAddCellSchema, AddCellResult> => ({
    name: 'notebooks-add-cell',
    schema: NotebooksAddCellSchema,
    handler: addCellHandler,
})

export default tool