Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): keep SQL editor lineage interactive and filtered (#125477)

Jovan Sakovic790536cc2dda
diff --git a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsx b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsx
index 19583fba931a..f43ca246b76d 100644
--- a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsx
+++ b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsx
@@ -1,5 +1,5 @@
import { useActions, useMountedLogic, useValues } from 'kea'
-import { router } from 'kea-router'
+import { combineUrl, router } from 'kea-router'
import { IconExternal, IconTarget } from '@posthog/icons'
import { LemonBanner, LemonTable, Link, Spinner, lemonToast } from '@posthog/lemon-ui'
@@ -45,8 +45,8 @@ export function QueryInfo({ tabId, view, tabbed = false }: QueryInfoProps): JSX.
// Mounting it loads the lineage for the view being edited.
const infoLogic = infoTabLogic({ tabId, viewId: targetView?.id })
useMountedLogic(infoLogic)
- const { activeSection, lineageVisited } = useValues(infoLogic)
- const { setActiveSection } = useActions(infoLogic)
+ const { activeSection, lineageVisited, lineageNodePositions } = useValues(infoLogic)
+ const { setActiveSection, lineageNodeDragStopped, resetLineageNodePositions } = useActions(infoLogic)
const { saveAsView, setUpstreamViewMode, editView, loadUpstream } = useActions(sqlEditorLogic)
// The loaded lineage is shared across views, so only use it when it was loaded for this one.
const upstream = loadedUpstream && targetView && loadedUpstream.modelId === targetView.id ? loadedUpstream : null
@@ -288,13 +288,22 @@ export function QueryInfo({ tabId, view, tabbed = false }: QueryInfoProps): JSX.
variant="full"
fitViewOptions={tabbed ? { maxZoom: 1 } : undefined}
interactive
+ nodesDraggable
+ nodePositions={lineageNodePositions}
+ onNodeDragStop={(node, position) => lineageNodeDragStopped(node.id, position)}
+ onResetNodePositions={resetLineageNodePositions}
showControls
showMinimap
panels={
<LemonButton
type="secondary"
size="small"
- to={urls.models('lineage')}
+ to={
+ combineUrl(urls.models('lineage'), {
+ search: `+${targetView.name}+`,
+ node_id: currentNodeId,
+ }).url
+ }
targetBlank
tooltip="Open the full graph"
aria-label="Open the full graph"
diff --git a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.ts b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.ts
index abbe5c12a87d..d764c4b72214 100644
--- a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.ts
+++ b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.ts
@@ -1,3 +1,4 @@
+import type { XYPosition } from '@xyflow/react'
import { MakeLogicType, actions, reducers, afterMount, connect, kea, key, path, props } from 'kea'
import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'
@@ -22,6 +23,7 @@ export interface infoTabLogicValues {
posthogTablesMap: Record<string, DatabaseSchemaTable> // databaseTableListLogic
metadata: HogQLMetadataResponse | null // sqlEditorLogic
activeSection: 'lineage' | 'materialization' | 'quality'
+ lineageNodePositions: Record<string, XYPosition>
lineageVisited: boolean
}
@@ -30,6 +32,16 @@ export interface infoTabLogicActions {
loadUpstream: (modelId: string) => {
modelId: string
} // sqlEditorLogic
+ lineageNodeDragStopped: (
+ nodeId: string,
+ position: XYPosition
+ ) => {
+ nodeId: string
+ position: XYPosition
+ }
+ resetLineageNodePositions: () => {
+ value: true
+ }
setActiveSection: (section: 'lineage' | 'materialization' | 'quality') => {
section: 'lineage' | 'materialization' | 'quality'
}
@@ -61,8 +73,19 @@ export const infoTabLogic = kea<infoTabLogicType>([
actions: [sqlEditorLogic({ tabId: props.tabId }), ['loadUpstream']],
logic: props.viewId ? [materializationJobsLogic({ viewId: props.viewId })] : [],
})),
- actions({ setActiveSection: (section: 'materialization' | 'lineage' | 'quality') => ({ section }) }),
+ actions({
+ setActiveSection: (section: 'materialization' | 'lineage' | 'quality') => ({ section }),
+ lineageNodeDragStopped: (nodeId: string, position: XYPosition) => ({ nodeId, position }),
+ resetLineageNodePositions: true,
+ }),
reducers({
+ lineageNodePositions: [
+ {} as Record<string, XYPosition>,
+ {
+ lineageNodeDragStopped: (positions, { nodeId, position }) => ({ ...positions, [nodeId]: position }),
+ resetLineageNodePositions: () => ({}),
+ },
+ ],
lineageVisited: [false, { setActiveSection: (visited, { section }) => visited || section === 'lineage' }],
activeSection: [
'materialization' as 'materialization' | 'lineage' | 'quality',
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e9f50dc6c62c..35da12ef43d8 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -3437,6 +3437,9 @@ importers:
kea-router:
specifier: 'catalog:'
version: 3.4.2([email protected](patch_hash=139b8d1f1304f9d9da452a9a1244c94ea679dbcb85687d8999563146879fb6f5)([email protected]))
+ kea-subscriptions:
+ specifier: 'catalog:'
+ version: 3.0.1([email protected](patch_hash=139b8d1f1304f9d9da452a9a1244c94ea679dbcb85687d8999563146879fb6f5)([email protected]))
posthog-js:
specifier: 'catalog:'
version: 1.439.0(@types/[email protected])([email protected])
diff --git a/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx b/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx
index baee0da47063..1eedad0c5579 100644
--- a/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx
+++ b/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx
@@ -7,6 +7,7 @@ import { mswDecorator } from '~/mocks/browser'
import { DataModelingEdge, DataModelingNode } from '~/types'
import { LineageGraph } from './LineageGraph'
+import { modelsLineageLogic } from './modelsLineageLogic'
import { ModelsLineageTab } from './ModelsLineageTab'
function mockNode(
@@ -471,6 +472,7 @@ export const SelectorFocus: Story = {
play: async ({ canvasElement }) => {
const canvas = within(canvasElement)
await canvas.findByText('revenue_summary')
+ modelsLineageLogic.actions.nodeDragStopped('4', { x: 100000, y: 100000 })
const search = canvas.getByPlaceholderText('Search, or +name for upstream')
fireEvent.change(search, { target: { value: 'revenue_summary+' } })
await canvas.findByText('4 models · downstream')
@@ -486,6 +488,13 @@ export const SelectorFocus: Story = {
if (!graph || graph.querySelectorAll('.react-flow__node').length !== 4) {
throw new Error('A downstream selector must keep only its lineage cone visible')
}
+ if (Object.keys(modelsLineageLogic.values.nodePositions).length > 0) {
+ throw new Error('A lineage filter must clear positions from the previous graph')
+ }
+ const model = graph.querySelector<HTMLElement>('.react-flow__node[data-id="4"]')
+ if (!model || model.style.transform.includes('100000')) {
+ throw new Error('A lineage filter must place distant nodes in the new layout')
+ }
},
{ timeout: VIEWPORT_SETTLE_MS }
)
diff --git a/products/data_modeling/frontend/lineage/lineageSearch.ts b/products/data_modeling/frontend/lineage/lineageSearch.ts
index b139ee350ede..f4b707d6a7e5 100644
--- a/products/data_modeling/frontend/lineage/lineageSearch.ts
+++ b/products/data_modeling/frontend/lineage/lineageSearch.ts
@@ -4,6 +4,7 @@ export type LineageSearchMode = 'search' | 'upstream' | 'downstream' | 'both'
export interface ParsedLineageSearch {
mode: LineageSearchMode
+ anchorNodeId?: string
/** The name to anchor lineage traversal on, or the term to match for plain search */
term: string
}
@@ -140,7 +141,9 @@ export function orderedNodesForLineageSearch(
if (parsed.mode === 'search' || !parsed.term) {
return null
}
- const anchor = matchNodesByName(nodes, parsed.term)[0]
+ const anchor = parsed.anchorNodeId
+ ? nodes.find((node) => node.id === parsed.anchorNodeId)
+ : matchNodesByName(nodes, parsed.term)[0]
if (!anchor) {
return []
}
diff --git a/products/data_modeling/frontend/lineage/modelsLineageLogic.test.ts b/products/data_modeling/frontend/lineage/modelsLineageLogic.test.ts
index 65f3f1736326..b65a5189ab1a 100644
--- a/products/data_modeling/frontend/lineage/modelsLineageLogic.test.ts
+++ b/products/data_modeling/frontend/lineage/modelsLineageLogic.test.ts
@@ -1,6 +1,9 @@
+import { combineUrl, router } from 'kea-router'
import { expectLogic } from 'kea-test-utils'
import posthog from 'posthog-js'
+import { urls } from 'scenes/urls'
+
import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
import { DataModelingEdge, DataModelingNode } from '~/types'
@@ -39,7 +42,49 @@ describe('modelsLineageLogic', () => {
logic.unmount()
})
+ it.each([false, true])('opens the selected lineage from a URL with a stable ID: %s', async (useNodeId) => {
+ const selectedNode = { ...NODES[0], id: 'selected-view' }
+ const baseNodes = [selectedNode, ...NODES.slice(1)]
+ const nodes = useNodeId
+ ? [{ ...NODES[0], id: 'namesake-metric', type: 'metric' as const }, ...baseNodes]
+ : baseNodes
+ const edges = EDGES.map((edge) => ({ ...edge, target_id: selectedNode.id }))
+ lineageDataLogic.actions.loadNodesSuccess(nodes)
+ lineageDataLogic.actions.loadEdgesSuccess(edges)
+ logic.actions.setTypeFilter(['view'])
+
+ await expectLogic(logic, () => {
+ router.actions.push(
+ combineUrl(urls.models('lineage'), {
+ search: '+website_leads+',
+ ...(useNodeId ? { node_id: selectedNode.id } : {}),
+ }).url
+ )
+ }).toMatchValues({
+ searchTerm: '+website_leads+',
+ debouncedSearchTerm: '+website_leads+',
+ visibleNodes: [selectedNode, NODES[3]],
+ visibleEdges: edges,
+ lineageSearchAnchor: selectedNode,
+ })
+
+ await expectLogic(logic, () => {
+ logic.actions.setSearchTerm('+customers+')
+ logic.actions.setDebouncedSearchTerm('+customers+')
+ }).toMatchValues({ searchAnchorNodeId: null, lineageSearchAnchor: NODES[3] })
+
+ await expectLogic(logic, () => {
+ router.actions.push(urls.models('lineage'))
+ }).toMatchValues({
+ searchTerm: '',
+ visibleNodes: nodes,
+ })
+ })
+
it('ranks plain search results without moving the graph', async () => {
+ const position = { x: 120, y: 80 }
+ logic.actions.nodeDragStopped('1', position)
+
await expectLogic(logic, () => {
logic.actions.setSearchTerm('leads')
logic.actions.setDebouncedSearchTerm('leads')
@@ -50,6 +95,7 @@ describe('modelsLineageLogic', () => {
highlightedNodeIds: new Set(['1', '2', '3']),
focusNodeIds: null,
searchFocusRequest: null,
+ nodePositions: { '1': position },
})
})
@@ -189,23 +235,32 @@ describe('modelsLineageLogic', () => {
})
})
- it('keeps manual node positions through filtering until the layout is reset', async () => {
- const position = { x: 120, y: 80 }
+ it.each(['type', 'lineage', 'clear'] as const)(
+ 'clears distant node positions when the %s filter changes the graph',
+ async (filter) => {
+ const position = { x: 100000, y: 100000 }
- logic.actions.nodeDragStopped('1', position)
- logic.actions.setTypeFilter(['view'])
+ if (filter === 'clear') {
+ logic.actions.setSearchTerm('+website_leads+')
+ logic.actions.setDebouncedSearchTerm('+website_leads+')
+ }
- await expectLogic(logic)
- .toFinishAllListeners()
- .toMatchValues({ nodePositions: { '1': position } })
-
- expect(posthog.capture).toHaveBeenCalledWith('models lineage node moved', {
- node_type: 'table',
- is_filtered: false,
- })
+ logic.actions.nodeDragStopped('1', position)
+ expect(posthog.capture).toHaveBeenCalledWith('models lineage node moved', {
+ node_type: 'table',
+ is_filtered: filter === 'clear',
+ })
- await expectLogic(logic, () => {
- logic.actions.resetNodePositions()
- }).toMatchValues({ nodePositions: {} })
- })
+ await expectLogic(logic, () => {
+ if (filter === 'type') {
+ logic.actions.setTypeFilter(['view'])
+ } else if (filter === 'lineage') {
+ logic.actions.setSearchTerm('+website_leads+')
+ logic.actions.setDebouncedSearchTerm('+website_leads+')
+ } else {
+ logic.actions.resetFilters()
+ }
+ }).toMatchValues({ nodePositions: {} })
+ }
+ )
})
diff --git a/products/data_modeling/frontend/lineage/modelsLineageLogic.ts b/products/data_modeling/frontend/lineage/modelsLineageLogic.ts
index 0213b162d2fd..223df8fe36a9 100644
--- a/products/data_modeling/frontend/lineage/modelsLineageLogic.ts
+++ b/products/data_modeling/frontend/lineage/modelsLineageLogic.ts
@@ -1,7 +1,11 @@
import type { XYPosition } from '@xyflow/react'
import { MakeLogicType, actions, connect, kea, listeners, path, reducers, selectors } from 'kea'
+import { urlToAction } from 'kea-router'
+import { subscriptions } from 'kea-subscriptions'
import posthog from 'posthog-js'
+import { urls } from 'scenes/urls'
+
import { DataModelingEdge, DataModelingNode, DataModelingNodeType } from '~/types'
import { NodeTypeEnumApi } from '../generated/api.schemas'
@@ -33,6 +37,7 @@ export interface modelsLineageLogicValues {
edgesLoading: boolean // lineageDataLogic
searchTerm: string
debouncedSearchTerm: string
+ searchAnchorNodeId: string | null
typeFilter: DataModelingNodeType[]
legendCollapsed: boolean
selectedSearchResultId: string | null
@@ -54,6 +59,7 @@ export interface modelsLineageLogicValues {
}
export interface modelsLineageLogicActions {
+ setSearchAnchorNodeId: (nodeId: string | null) => { nodeId: string | null }
setSearchTerm: (searchTerm: string) => { searchTerm: string }
setDebouncedSearchTerm: (searchTerm: string) => { searchTerm: string }
setTypeFilter: (typeFilter: DataModelingNodeType[]) => { typeFilter: DataModelingNodeType[] }
@@ -82,10 +88,22 @@ export type modelsLineageLogicType = MakeLogicType<modelsLineageLogicValues, mod
export const modelsLineageLogic = kea<modelsLineageLogicType>([
path(['scenes', 'models', 'modelsLineageLogic']),
+ urlToAction(({ actions }) => ({
+ [urls.models()]: (_, searchParams) => {
+ if (searchParams.tab === 'lineage') {
+ const searchTerm = typeof searchParams.search === 'string' ? searchParams.search : ''
+ actions.resetFilters()
+ actions.setSearchTerm(searchTerm)
+ actions.setDebouncedSearchTerm(searchTerm)
+ actions.setSearchAnchorNodeId(typeof searchParams.node_id === 'string' ? searchParams.node_id : null)
+ }
+ },
+ })),
connect(() => ({
values: [lineageDataLogic, ['nodes', 'nodesLoading', 'edges', 'edgesLoading']],
})),
actions({
+ setSearchAnchorNodeId: (nodeId: string | null) => ({ nodeId }),
setSearchTerm: (searchTerm: string) => ({ searchTerm }),
setDebouncedSearchTerm: (searchTerm: string) => ({ searchTerm }),
setTypeFilter: (typeFilter: DataModelingNodeType[]) => ({ typeFilter }),
@@ -98,6 +116,14 @@ export const modelsLineageLogic = kea<modelsLineageLogicType>([
resetNodePositions: true,
}),
reducers({
+ searchAnchorNodeId: [
+ null as string | null,
+ {
+ setSearchAnchorNodeId: (_, { nodeId }) => nodeId,
+ setSearchTerm: () => null,
+ resetFilters: () => null,
+ },
+ ],
searchTerm: [
'',
{
@@ -200,14 +226,37 @@ export const modelsLineageLogic = kea<modelsLineageLogicType>([
}
},
})),
+ subscriptions(({ actions }) => ({
+ visibleNodes: (nodes: DataModelingNode[], previousNodes: DataModelingNode[] | undefined) => {
+ if (
+ previousNodes &&
+ (nodes.length !== previousNodes.length ||
+ nodes.some((node, index) => node.id !== previousNodes[index].id))
+ ) {
+ actions.resetNodePositions()
+ }
+ },
+ })),
selectors({
- parsedSearchTerm: [(s) => [s.searchTerm], (searchTerm: string) => parseLineageSearch(searchTerm)],
- parsedSearch: [(s) => [s.debouncedSearchTerm], (searchTerm: string) => parseLineageSearch(searchTerm)],
+ parsedSearchTerm: [
+ (s) => [s.searchTerm, s.searchAnchorNodeId],
+ (searchTerm: string, anchorNodeId: string | null): ParsedLineageSearch =>
+ anchorNodeId ? { ...parseLineageSearch(searchTerm), anchorNodeId } : parseLineageSearch(searchTerm),
+ ],
+ parsedSearch: [
+ (s) => [s.debouncedSearchTerm, s.searchAnchorNodeId],
+ (searchTerm: string, anchorNodeId: string | null): ParsedLineageSearch =>
+ anchorNodeId ? { ...parseLineageSearch(searchTerm), anchorNodeId } : parseLineageSearch(searchTerm),
+ ],
lineageSearchAnchor: [
(s) => [s.nodes, s.parsedSearchTerm],
(nodes: DataModelingNode[], parsedSearchTerm: ParsedLineageSearch): DataModelingNode | null =>
- parsedSearchTerm.mode === 'search' ? null : (matchNodesByName(nodes, parsedSearchTerm.term)[0] ?? null),
+ parsedSearchTerm.mode === 'search'
+ ? null
+ : parsedSearchTerm.anchorNodeId
+ ? (nodes.find((node) => node.id === parsedSearchTerm.anchorNodeId) ?? null)
+ : (matchNodesByName(nodes, parsedSearchTerm.term)[0] ?? null),
],
// Both the result list and the graph need the lineage cone. Walking it once here keeps the
diff --git a/products/data_modeling/package.json b/products/data_modeling/package.json
index 266f9f2a672e..1ad4c3f3d04d 100644
--- a/products/data_modeling/package.json
+++ b/products/data_modeling/package.json
@@ -10,6 +10,7 @@
"kea-forms": "catalog:",
"kea-loaders": "catalog:",
"kea-router": "catalog:",
+ "kea-subscriptions": "catalog:",
"posthog-js": "catalog:"
},
"devDependencies": {