:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import * as markdownNotebookParser from 'lib/components/MarkdownNotebook/markdown'
import dataframeNameCases from 'products/notebooks/dataframe-names.test.json'
import { buildMarkdownNotebookContent, serializeMarkdownNotebookComponent } from '../Notebook/markdownNotebookV2'
import { NotebookNodeType } from '../types'
import {
buildNotebookDependencyGraph,
collectNotebookFrameNodes,
collectRunnableCellNodeIds,
extractPythonIdentifiers,
} from './notebookNodeContent'
describe('buildNotebookDependencyGraph', () => {
it.each(dataframeNameCases)('$name', ({ markdown, owners }) => {
const content = buildMarkdownNotebookContent(markdown)
expect(
Object.fromEntries(collectNotebookFrameNodes(content).map(({ name, nodeId }) => [name, nodeId]))
).toEqual(owners)
const graph = buildNotebookDependencyGraph(content)
expect(
Object.fromEntries(graph.nodes.flatMap(({ nodeId, exports }) => exports.map((name) => [name, nodeId])))
).toEqual(owners)
})
afterEach(() => {
jest.restoreAllMocks()
})
const sqlV2Node = (nodeId: string, returnVariable: string, code: string): Record<string, unknown> => ({
type: NotebookNodeType.SQLV2,
attrs: { nodeId, returnVariable, code },
})
const pythonV2Node = (nodeId: string, returnVariable: string, code: string): Record<string, unknown> => ({
type: NotebookNodeType.PythonV2,
attrs: { nodeId, returnVariable, code },
})
it('links a SQLV2 node to a downstream SQLV2 node that references it by table name', () => {
// The "Used in" back-links depend on SQLV2 producing an export and its reader listing it as a use.
const content = {
type: 'doc',
content: [sqlV2Node('a', 'df1', 'select id from events'), sqlV2Node('b', 'joined', 'select * from df1')],
}
const graph = buildNotebookDependencyGraph(content)
expect(graph.downstreamUsageByNode['a'].df1.map((usage) => usage.nodeId)).toEqual(['b'])
expect(graph.upstreamSourcesByNode['b'].df1.nodeId).toEqual('a')
})
it('disambiguates two nodes that share a return variable so a reference links to only the first', () => {
// Without the uniquification pass both nodes would export `sql_df` and `from sql_df`
// would ambiguously attribute to both; the second node's export becomes `sql_df_2`.
const content = {
type: 'doc',
content: [
sqlV2Node('a', 'sql_df', 'select id from events'),
sqlV2Node('b', 'sql_df', 'select id from persons'),
sqlV2Node('c', 'joined', 'select * from sql_df'),
],
}
const graph = buildNotebookDependencyGraph(content)
expect(graph.nodesById['b'].exports).toEqual(['sql_df_2'])
expect(graph.downstreamUsageByNode['a'].sql_df.map((usage) => usage.nodeId)).toEqual(['c'])
expect(graph.upstreamSourcesByNode['c'].sql_df.nodeId).toEqual('a')
})
it('an unnamed SQL cell is not browsable as a dataframe', () => {
// The schema browser lists document-derived frames; a blank-name cell binds no
// dataframe, so it must not appear (nor push a real 'sql_df' cell to 'sql_df_2').
const content = {
type: 'doc',
content: [sqlV2Node('blank', '', 'select 1'), sqlV2Node('named', 'sql_df', 'select id from events')],
}
const frames = collectNotebookFrameNodes(content)
expect(frames.map((frame) => frame.name)).toEqual(['sql_df'])
expect(frames[0].nodeId).toEqual('named')
})
it('an unnamed SQL cell exports nothing and reserves no name', () => {
// The dataframe name is optional: a blank-name cell is display-only, and it must not
// squat the 'sql_df' default (which would misroute references to a named sibling).
const content = {
type: 'doc',
content: [
sqlV2Node('blank', '', 'select 1'),
sqlV2Node('named', 'sql_df', 'select id from events'),
sqlV2Node('reader', 'joined', 'select * from sql_df'),
],
}
const graph = buildNotebookDependencyGraph(content)
expect(graph.nodesById['blank'].exports).toEqual([])
expect(graph.nodesById['named'].exports).toEqual(['sql_df'])
expect(graph.upstreamSourcesByNode['reader'].sql_df.nodeId).toEqual('named')
})
it('an invalid SQL frame name binds nothing (not exported, not browsable)', () => {
// A non-empty but invalid name like `people-df` can never be referenced (bare
// `people-df` is not a valid identifier), so it must not enter the dependency
// graph or the schema browser as an export the way a valid name does.
const content = {
type: 'doc',
content: [sqlV2Node('bad', 'people-df', 'select 1'), sqlV2Node('ok', 'sql_df', 'select id from events')],
}
const graph = buildNotebookDependencyGraph(content)
expect(graph.nodesById['bad'].exports).toEqual([])
expect(graph.nodesById['ok'].exports).toEqual(['sql_df'])
expect(collectNotebookFrameNodes(content).map((frame) => frame.name)).toEqual(['sql_df'])
})
it('links PythonV2 cells in both directions (python reads sql, sql joins the python frame)', () => {
// Journey 10 staleness walks these edges: without PythonV2 in the graph, running a
// SQL cell never marks its Python dependents stale, and vice versa.
const content = {
type: 'doc',
content: [
sqlV2Node('a', 'sql_df', 'select id from events'),
pythonV2Node('py', 'new_events', 'new_events = sql_df.head(50)'),
sqlV2Node('c', 'joined', 'select * from new_events'),
],
}
const graph = buildNotebookDependencyGraph(content)
expect(graph.downstreamUsageByNode['a'].sql_df.map((usage) => usage.nodeId)).toEqual(['py'])
expect(graph.upstreamSourcesByNode['py'].sql_df.nodeId).toEqual('a')
expect(graph.downstreamUsageByNode['py'].new_events.map((usage) => usage.nodeId)).toEqual(['c'])
})
it('links PythonV2 cells held inside a markdown notebook', () => {
// Markdown notebooks are the only V2 surface; without expanding PythonV2 tags the
// graph misses every real-world Python cell.
const markdown = [
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'a',
returnVariable: 'sql_df',
code: 'select id from events',
}),
serializeMarkdownNotebookComponent('PythonV2', {
nodeId: 'py',
returnVariable: 'new_events',
code: 'new_events = sql_df.head()',
}),
].join('\n\n')
const graph = buildNotebookDependencyGraph(buildMarkdownNotebookContent(markdown))
expect(graph.downstreamUsageByNode['a'].sql_df.map((usage) => usage.nodeId)).toEqual(['py'])
})
it('extractPythonIdentifiers ignores strings, comments, and attribute tails', () => {
// A frame name inside a string or after a dot is not a read; matching it would
// create a false dependency edge and mark unrelated cells stale.
const identifiers = extractPythonIdentifiers(
'result = sql_df.head(10) # other_df is just a comment\nprint("other_df")\nlabel = frame.other_df'
)
expect(identifiers).toEqual(expect.arrayContaining(['result', 'sql_df', 'print', 'label', 'frame']))
expect(identifiers).not.toContain('other_df')
expect(identifiers).not.toContain('head')
})
it('links a SQLV2 cell that carries its SQL in a query prop', () => {
// Cells authored with the `<Query query={…} />` prop shape have no `code` attr, so
// without recovering the SQL they read as empty and reference no upstream frame.
const markdown = [
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'a',
returnVariable: 'df1',
code: 'select id from events',
}),
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'b',
returnVariable: 'joined',
query: { kind: 'HogQLQuery', query: 'select * from df1' },
}),
].join('\n\n')
const graph = buildNotebookDependencyGraph(buildMarkdownNotebookContent(markdown))
expect(graph.downstreamUsageByNode['a'].df1.map((usage) => usage.nodeId)).toEqual(['b'])
})
it('links SQLV2 cells inside a markdown notebook', () => {
// Markdown notebooks hold cells as tags inside one markdown attribute; without
// expanding them the graph is empty and the "Used in" back-links never render.
const markdown = [
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'a',
returnVariable: 'df1',
code: 'select id from events',
}),
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'b',
returnVariable: 'joined',
code: 'select * from df1',
}),
].join('\n\n')
const graph = buildNotebookDependencyGraph(buildMarkdownNotebookContent(markdown))
expect(graph.downstreamUsageByNode['a'].df1.map((usage) => usage.nodeId)).toEqual(['b'])
expect(graph.upstreamSourcesByNode['b'].df1.nodeId).toEqual('a')
})
it('parses the markdown once for two collectors reading the same content', () => {
// The parse is cached on the content node so collectors share one parse per edit. The
// cell assertions alone would still pass with the cache gone, so count the parses too:
// that is the per-keystroke work this guards. A cache keyed by anything other than the
// node would also hand the second collector the wrong cells.
const markdown = [
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'a',
returnVariable: 'sql_df',
code: 'select id from events',
}),
serializeMarkdownNotebookComponent('PythonV2', {
nodeId: 'py',
returnVariable: 'new_events',
code: 'new_events = sql_df.head()',
}),
].join('\n\n')
const content = buildMarkdownNotebookContent(markdown)
const parseSpy = jest.spyOn(markdownNotebookParser, 'parseMarkdownNotebook')
// Populate the cache from one collector, then read it from another.
expect(collectNotebookFrameNodes(content).map((frame) => frame.name)).toEqual(['sql_df', 'new_events'])
const graph = buildNotebookDependencyGraph(content)
expect(graph.downstreamUsageByNode['a'].sql_df.map((usage) => usage.nodeId)).toEqual(['py'])
expect(parseSpy).toHaveBeenCalledTimes(1)
})
it('does not serve one content its parse for a later content with different markdown', () => {
// Each edit builds a new content node, so the cache must key on it and stay fresh. A
// cache that leaked the first parse would report the first document's cells forever.
const graphA = buildNotebookDependencyGraph(
buildMarkdownNotebookContent(
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'a',
returnVariable: 'df_a',
code: 'select id from events',
})
)
)
const graphB = buildNotebookDependencyGraph(
buildMarkdownNotebookContent(
serializeMarkdownNotebookComponent('SQLV2', {
nodeId: 'b',
returnVariable: 'df_b',
code: 'select id from persons',
})
)
)
expect(Object.keys(graphA.nodesById)).toEqual(['a'])
expect(Object.keys(graphB.nodesById)).toEqual(['b'])
expect(graphB.nodesById['b'].exports).toEqual(['df_b'])
})
})
describe('collectRunnableCellNodeIds', () => {
// Run all used to key off the SQL summaries, so a notebook of Python cells alone never
// got the button — though the backend plan runs exactly these two kinds.
it.each([
['python only', [{ tag: 'PythonV2', nodeId: 'p1', code: 'print(1)' }], ['p1']],
['sql only', [{ tag: 'SQLV2', nodeId: 's1', code: 'select 1' }], ['s1']],
[
'both, in document order',
[
{ tag: 'SQLV2', nodeId: 's1', code: 'select 1' },
{ tag: 'PythonV2', nodeId: 'p1', code: 'print(1)' },
],
['s1', 'p1'],
],
['an empty cell is not runnable', [{ tag: 'PythonV2', nodeId: 'p1', code: ' ' }], []],
])('%s', (_name, cells, expected) => {
const markdown = cells
.map(({ tag, nodeId, code }) => serializeMarkdownNotebookComponent(tag, { nodeId, code }))
.join('\n\n')
expect(collectRunnableCellNodeIds(buildMarkdownNotebookContent(markdown))).toEqual(expected)
})
})