Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.7 KB
Raw Blame
import { IconDatabase, IconDocument } from '@posthog/icons'

import { TreeDataItem } from 'lib/lemon-ui/LemonTree/LemonTree'

import { NotebookKernelFrame } from '../../Notebook/notebookKernelInfoLogic'
import { NotebookFrameNodeSummary } from '../notebookNodeContent'

export const DATAFRAMES_FOLDER_ID = 'notebook-dataframes'

// These render as tooltips on a row in *another* cell's sidebar, so they can't say "this cell"
// — that already means "the cell you're in" on the run controls, and would point at the wrong
// one for every row but the current cell's own output.
const NEVER_RAN_REASON = 'Run the cell that creates it to make it available'
const NO_FRAME_REASON = "The last run of the cell that creates it didn't produce a dataframe"
const NOT_IN_KERNEL_REASON = 'Not in the kernel right now. Run the cell that creates it to make it available'

type DataframeEntry = {
    name: string
    columns: [string, string][]
    rowCount: number | null
    /** A DuckDB table or view reads differently from a dataframe, so they get different icons. */
    isTable: boolean
    /** Set when the name can't be referenced as things stand; renders greyed out with this reason. */
    disabledReason?: string
}

/**
 * One entry per name. Python cells deliberately don't disambiguate their names — they are the
 * raw kernel variables, where a repeated name means last-run-wins — and they default to `df`,
 * so two un-run Python cells is enough to produce the same name twice. Collapsing to the last
 * matches what the kernel actually holds, and keeps the tree's item ids unique.
 */
const dedupeByName = (entries: DataframeEntry[]): DataframeEntry[] => {
    const byName = new Map<string, DataframeEntry>()
    entries.forEach((entry) => byName.set(entry.name, entry))
    return [...byName.values()]
}

/**
 * The tree renders `row_count` as a plain count, so an estimate can't go through it honestly —
 * DuckDB's `estimated_size` doesn't track deletes and can read 100,000 for a table holding 10.
 * A DDL table therefore lists without a count rather than with a confident wrong one.
 */
const kernelEntry = (frame: NotebookKernelFrame): DataframeEntry => ({
    name: frame.name,
    columns: frame.columns,
    rowCount: frame.row_count_is_estimate ? null : frame.row_count,
    isTable: frame.kind !== 'frame',
})

/**
 * Drop cells that bind nothing worth showing: an empty one nobody has written yet holds no query
 * and no result, and its name is only the default. Listing those means every blank cell you add
 * spawns a greyed-out row (`sql_df_2`, `sql_df_3`, …) for a frame that was never conceived of.
 * A cell with a query but no run is different — it is a real intent to bind that name.
 */
const listableNodes = (nodes: NotebookFrameNodeSummary[]): NotebookFrameNodeSummary[] =>
    nodes.filter((node) => node.hasRun || node.code.trim())

/**
 * Merge what the notebook knows with what the kernel knows. Neither is sufficient alone:
 *
 * - The kernel's catalog is the only thing that can see a table a DuckDB cell created, and the
 *   only honest answer to whether a Python cell's frame still exists.
 * - It cannot see a SQL cell's output, though. That output lives in ClickHouse and only enters
 *   the kernel if some cell materializes it — yet it stays referenceable from SQL the whole
 *   time, re-inlined as a CTE. Listing the kernel's view alone makes a SQL frame appear or
 *   vanish depending on whether an unrelated Python cell happens to use it.
 */
function mergeEntries(nodes: NotebookFrameNodeSummary[], kernelFrames: NotebookKernelFrame[]): DataframeEntry[] {
    const kernelByName = new Map(kernelFrames.map((frame) => [frame.name, frame]))
    const listable = listableNodes(nodes)
    const boundByCell = new Set(listable.map((node) => node.name))

    const fromNodes = listable.map((node): DataframeEntry => {
        const kernelFrame = kernelByName.get(node.name)
        if (kernelFrame) {
            // The kernel holds it, so prefer its shape: that reflects the frame as it is now,
            // while the cell's stored result is only as fresh as that cell's last run.
            return kernelEntry(kernelFrame)
        }
        const base = { name: node.name, columns: node.columns, rowCount: node.rowCount, isTable: false }
        if (!node.hasRun) {
            return { ...base, disabledReason: NEVER_RAN_REASON }
        }
        if (!node.columns.length) {
            // It ran and bound nothing — its code produces no frame, or it was DDL. Telling the
            // user to run it would be a lie; running it again changes nothing.
            return { ...base, disabledReason: NO_FRAME_REASON }
        }
        // A SQL cell that ran is referenceable with no kernel at all; a Python cell's frame only
        // ever exists in the kernel, so its absence from the catalog means it is really gone.
        return node.nodeType === 'sql' ? base : { ...base, disabledReason: NOT_IN_KERNEL_REASON }
    })

    // Anything the kernel has that no cell binds: a table a DuckDB cell created with DDL, which
    // exists only in the kernel and is invisible to the notebook document.
    const kernelOnly = kernelFrames.filter((frame) => !boundByCell.has(frame.name)).map(kernelEntry)

    return dedupeByName([...fromNodes, ...kernelOnly])
}

// Frame and column ids share one namespace, so the separator must be something an identifier
// can't contain — `-` would let a frame named `a-col-b` collide with column `b` of frame `a`.
const frameId = (index: number): string => `nb-frame:${index}`

const entryNode = (entry: DataframeEntry, index: number): TreeDataItem => ({
    id: frameId(index),
    name: entry.name,
    type: 'node',
    icon: entry.isTable ? <IconDatabase /> : <IconDocument />,
    disabledReason: entry.disabledReason,
    record: {
        type: 'notebook-frame',
        // QueryDatabase renders this natively next to the name, the same as a warehouse table's.
        row_count: entry.rowCount ?? undefined,
    },
    children: entry.columns.map(([columnName, columnType], columnIndex) => ({
        id: `${frameId(index)}:col:${columnIndex}`,
        name: columnName,
        type: 'node' as const,
        // The shape QueryDatabase expects for a column: monospaced name plus a muted type
        // suffix, and `columnName` for the click that inserts it at the cursor. A dataframe is
        // referenceable by name from SQL, so its columns insert exactly like a table's.
        record: { type: 'column', columnName, field: { name: columnName, type: columnType } },
    })),
})

/**
 * The "Dataframes" section for the SQLV2 node's schema browser: the names this notebook's cells
 * bind, plus anything extra its kernel holds. A name that can't be referenced as things stand
 * renders greyed out rather than disappearing — a cell you can see in the notebook going missing
 * from the browser is more confusing than one shown as unavailable.
 *
 * Unfiltered: QueryDatabase applies its own search term to these sections, so this stays out of
 * queryDatabaseLogic (reading it would mount its warehouse loaders in every notebook).
 */
export function buildDataframeTreeSection(
    nodes: NotebookFrameNodeSummary[],
    kernelFrames: NotebookKernelFrame[]
): TreeDataItem[] {
    const entries = mergeEntries(nodes, kernelFrames)
    if (!entries.length) {
        return []
    }
    return [
        {
            id: DATAFRAMES_FOLDER_ID,
            name: 'Dataframes',
            // Bold like the built-in section headers, which key off record types we don't share.
            displayName: <span className="font-semibold">Dataframes</span>,
            type: 'node',
            icon: <IconDatabase />,
            record: { type: 'notebook-dataframes' },
            children: entries.map(entryNode),
        },
    ]
}