:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): keep SQL editor lineage interactive and filtered (#125477)
Jovan Sakovic790536cc2dda
diff --git a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsx b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsxindex 19583fba931a..f43ca246b76d 100644--- a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsx+++ b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/QueryInfo.tsx@@ -1,5 +1,5 @@import { useActions, useMountedLogic, useValues } from 'kea'-import { router } from 'kea-router'+import { combineUrl, router } from 'kea-router'import { IconExternal, IconTarget } from '@posthog/icons'import { LemonBanner, LemonTable, Link, Spinner, lemonToast } from '@posthog/lemon-ui'@@ -45,8 +45,8 @@ export function QueryInfo({ tabId, view, tabbed = false }: QueryInfoProps): JSX.// Mounting it loads the lineage for the view being edited.const infoLogic = infoTabLogic({ tabId, viewId: targetView?.id })useMountedLogic(infoLogic)- const { activeSection, lineageVisited } = useValues(infoLogic)- const { setActiveSection } = useActions(infoLogic)+ const { activeSection, lineageVisited, lineageNodePositions } = useValues(infoLogic)+ const { setActiveSection, lineageNodeDragStopped, resetLineageNodePositions } = useActions(infoLogic)const { saveAsView, setUpstreamViewMode, editView, loadUpstream } = useActions(sqlEditorLogic)// The loaded lineage is shared across views, so only use it when it was loaded for this one.const upstream = loadedUpstream && targetView && loadedUpstream.modelId === targetView.id ? loadedUpstream : null@@ -288,13 +288,22 @@ export function QueryInfo({ tabId, view, tabbed = false }: QueryInfoProps): JSX.variant="full"fitViewOptions={tabbed ? { maxZoom: 1 } : undefined}interactive+ nodesDraggable+ nodePositions={lineageNodePositions}+ onNodeDragStop={(node, position) => lineageNodeDragStopped(node.id, position)}+ onResetNodePositions={resetLineageNodePositions}showControlsshowMinimappanels={<LemonButtontype="secondary"size="small"- to={urls.models('lineage')}+ to={+ combineUrl(urls.models('lineage'), {+ search: `+${targetView.name}+`,+ node_id: currentNodeId,+ }).url+ }targetBlanktooltip="Open the full graph"aria-label="Open the full graph"diff --git a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.ts b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.tsindex abbe5c12a87d..d764c4b72214 100644--- a/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.ts+++ b/frontend/src/scenes/data-warehouse/editor/output-pane-tabs/infoTabLogic.ts@@ -1,3 +1,4 @@+import type { XYPosition } from '@xyflow/react'import { MakeLogicType, actions, reducers, afterMount, connect, kea, key, path, props } from 'kea'import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'@@ -22,6 +23,7 @@ export interface infoTabLogicValues {posthogTablesMap: Record<string, DatabaseSchemaTable> // databaseTableListLogicmetadata: HogQLMetadataResponse | null // sqlEditorLogicactiveSection: 'lineage' | 'materialization' | 'quality'+ lineageNodePositions: Record<string, XYPosition>lineageVisited: boolean}@@ -30,6 +32,16 @@ export interface infoTabLogicActions {loadUpstream: (modelId: string) => {modelId: string} // sqlEditorLogic+ lineageNodeDragStopped: (+ nodeId: string,+ position: XYPosition+ ) => {+ nodeId: string+ position: XYPosition+ }+ resetLineageNodePositions: () => {+ value: true+ }setActiveSection: (section: 'lineage' | 'materialization' | 'quality') => {section: 'lineage' | 'materialization' | 'quality'}@@ -61,8 +73,19 @@ export const infoTabLogic = kea<infoTabLogicType>([actions: [sqlEditorLogic({ tabId: props.tabId }), ['loadUpstream']],logic: props.viewId ? [materializationJobsLogic({ viewId: props.viewId })] : [],})),- actions({ setActiveSection: (section: 'materialization' | 'lineage' | 'quality') => ({ section }) }),+ actions({+ setActiveSection: (section: 'materialization' | 'lineage' | 'quality') => ({ section }),+ lineageNodeDragStopped: (nodeId: string, position: XYPosition) => ({ nodeId, position }),+ resetLineageNodePositions: true,+ }),reducers({+ lineageNodePositions: [+ {} as Record<string, XYPosition>,+ {+ lineageNodeDragStopped: (positions, { nodeId, position }) => ({ ...positions, [nodeId]: position }),+ resetLineageNodePositions: () => ({}),+ },+ ],lineageVisited: [false, { setActiveSection: (visited, { section }) => visited || section === 'lineage' }],activeSection: ['materialization' as 'materialization' | 'lineage' | 'quality',diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex e9f50dc6c62c..35da12ef43d8 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -3437,6 +3437,9 @@ importers:kea-router:specifier: 'catalog:'version: 3.4.2([email protected](patch_hash=139b8d1f1304f9d9da452a9a1244c94ea679dbcb85687d8999563146879fb6f5)([email protected]))+ kea-subscriptions:+ specifier: 'catalog:'+ version: 3.0.1([email protected](patch_hash=139b8d1f1304f9d9da452a9a1244c94ea679dbcb85687d8999563146879fb6f5)([email protected]))posthog-js:specifier: 'catalog:'version: 1.439.0(@types/[email protected])([email protected])diff --git a/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx b/products/data_modeling/frontend/lineage/LineageGraph.stories.tsxindex baee0da47063..1eedad0c5579 100644--- a/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx+++ b/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx@@ -7,6 +7,7 @@ import { mswDecorator } from '~/mocks/browser'import { DataModelingEdge, DataModelingNode } from '~/types'import { LineageGraph } from './LineageGraph'+import { modelsLineageLogic } from './modelsLineageLogic'import { ModelsLineageTab } from './ModelsLineageTab'function mockNode(@@ -471,6 +472,7 @@ export const SelectorFocus: Story = {play: async ({ canvasElement }) => {const canvas = within(canvasElement)await canvas.findByText('revenue_summary')+ modelsLineageLogic.actions.nodeDragStopped('4', { x: 100000, y: 100000 })const search = canvas.getByPlaceholderText('Search, or +name for upstream')fireEvent.change(search, { target: { value: 'revenue_summary+' } })await canvas.findByText('4 models · downstream')@@ -486,6 +488,13 @@ export const SelectorFocus: Story = {if (!graph || graph.querySelectorAll('.react-flow__node').length !== 4) {throw new Error('A downstream selector must keep only its lineage cone visible')}+ if (Object.keys(modelsLineageLogic.values.nodePositions).length > 0) {+ throw new Error('A lineage filter must clear positions from the previous graph')+ }+ const model = graph.querySelector<HTMLElement>('.react-flow__node[data-id="4"]')+ if (!model || model.style.transform.includes('100000')) {+ throw new Error('A lineage filter must place distant nodes in the new layout')+ }},{ timeout: VIEWPORT_SETTLE_MS })diff --git a/products/data_modeling/frontend/lineage/lineageSearch.ts b/products/data_modeling/frontend/lineage/lineageSearch.tsindex b139ee350ede..f4b707d6a7e5 100644--- a/products/data_modeling/frontend/lineage/lineageSearch.ts+++ b/products/data_modeling/frontend/lineage/lineageSearch.ts@@ -4,6 +4,7 @@ export type LineageSearchMode = 'search' | 'upstream' | 'downstream' | 'both'export interface ParsedLineageSearch {mode: LineageSearchMode+ anchorNodeId?: string/** The name to anchor lineage traversal on, or the term to match for plain search */term: string}@@ -140,7 +141,9 @@ export function orderedNodesForLineageSearch(if (parsed.mode === 'search' || !parsed.term) {return null}- const anchor = matchNodesByName(nodes, parsed.term)[0]+ const anchor = parsed.anchorNodeId+ ? nodes.find((node) => node.id === parsed.anchorNodeId)+ : matchNodesByName(nodes, parsed.term)[0]if (!anchor) {return []}diff --git a/products/data_modeling/frontend/lineage/modelsLineageLogic.test.ts b/products/data_modeling/frontend/lineage/modelsLineageLogic.test.tsindex 65f3f1736326..b65a5189ab1a 100644--- a/products/data_modeling/frontend/lineage/modelsLineageLogic.test.ts+++ b/products/data_modeling/frontend/lineage/modelsLineageLogic.test.ts@@ -1,6 +1,9 @@+import { combineUrl, router } from 'kea-router'import { expectLogic } from 'kea-test-utils'import posthog from 'posthog-js'+import { urls } from 'scenes/urls'+import { useMocks } from '~/mocks/jest'import { initKeaTests } from '~/test/init'import { DataModelingEdge, DataModelingNode } from '~/types'@@ -39,7 +42,49 @@ describe('modelsLineageLogic', () => {logic.unmount()})+ it.each([false, true])('opens the selected lineage from a URL with a stable ID: %s', async (useNodeId) => {+ const selectedNode = { ...NODES[0], id: 'selected-view' }+ const baseNodes = [selectedNode, ...NODES.slice(1)]+ const nodes = useNodeId+ ? [{ ...NODES[0], id: 'namesake-metric', type: 'metric' as const }, ...baseNodes]+ : baseNodes+ const edges = EDGES.map((edge) => ({ ...edge, target_id: selectedNode.id }))+ lineageDataLogic.actions.loadNodesSuccess(nodes)+ lineageDataLogic.actions.loadEdgesSuccess(edges)+ logic.actions.setTypeFilter(['view'])++ await expectLogic(logic, () => {+ router.actions.push(+ combineUrl(urls.models('lineage'), {+ search: '+website_leads+',+ ...(useNodeId ? { node_id: selectedNode.id } : {}),+ }).url+ )+ }).toMatchValues({+ searchTerm: '+website_leads+',+ debouncedSearchTerm: '+website_leads+',+ visibleNodes: [selectedNode, NODES[3]],+ visibleEdges: edges,+ lineageSearchAnchor: selectedNode,+ })++ await expectLogic(logic, () => {+ logic.actions.setSearchTerm('+customers+')+ logic.actions.setDebouncedSearchTerm('+customers+')+ }).toMatchValues({ searchAnchorNodeId: null, lineageSearchAnchor: NODES[3] })++ await expectLogic(logic, () => {+ router.actions.push(urls.models('lineage'))+ }).toMatchValues({+ searchTerm: '',+ visibleNodes: nodes,+ })+ })+it('ranks plain search results without moving the graph', async () => {+ const position = { x: 120, y: 80 }+ logic.actions.nodeDragStopped('1', position)+await expectLogic(logic, () => {logic.actions.setSearchTerm('leads')logic.actions.setDebouncedSearchTerm('leads')@@ -50,6 +95,7 @@ describe('modelsLineageLogic', () => {highlightedNodeIds: new Set(['1', '2', '3']),focusNodeIds: null,searchFocusRequest: null,+ nodePositions: { '1': position },})})@@ -189,23 +235,32 @@ describe('modelsLineageLogic', () => {})})- it('keeps manual node positions through filtering until the layout is reset', async () => {- const position = { x: 120, y: 80 }+ it.each(['type', 'lineage', 'clear'] as const)(+ 'clears distant node positions when the %s filter changes the graph',+ async (filter) => {+ const position = { x: 100000, y: 100000 }- logic.actions.nodeDragStopped('1', position)- logic.actions.setTypeFilter(['view'])+ if (filter === 'clear') {+ logic.actions.setSearchTerm('+website_leads+')+ logic.actions.setDebouncedSearchTerm('+website_leads+')+ }- await expectLogic(logic)- .toFinishAllListeners()- .toMatchValues({ nodePositions: { '1': position } })-- expect(posthog.capture).toHaveBeenCalledWith('models lineage node moved', {- node_type: 'table',- is_filtered: false,- })+ logic.actions.nodeDragStopped('1', position)+ expect(posthog.capture).toHaveBeenCalledWith('models lineage node moved', {+ node_type: 'table',+ is_filtered: filter === 'clear',+ })- await expectLogic(logic, () => {- logic.actions.resetNodePositions()- }).toMatchValues({ nodePositions: {} })- })+ await expectLogic(logic, () => {+ if (filter === 'type') {+ logic.actions.setTypeFilter(['view'])+ } else if (filter === 'lineage') {+ logic.actions.setSearchTerm('+website_leads+')+ logic.actions.setDebouncedSearchTerm('+website_leads+')+ } else {+ logic.actions.resetFilters()+ }+ }).toMatchValues({ nodePositions: {} })+ }+ )})diff --git a/products/data_modeling/frontend/lineage/modelsLineageLogic.ts b/products/data_modeling/frontend/lineage/modelsLineageLogic.tsindex 0213b162d2fd..223df8fe36a9 100644--- a/products/data_modeling/frontend/lineage/modelsLineageLogic.ts+++ b/products/data_modeling/frontend/lineage/modelsLineageLogic.ts@@ -1,7 +1,11 @@import type { XYPosition } from '@xyflow/react'import { MakeLogicType, actions, connect, kea, listeners, path, reducers, selectors } from 'kea'+import { urlToAction } from 'kea-router'+import { subscriptions } from 'kea-subscriptions'import posthog from 'posthog-js'+import { urls } from 'scenes/urls'+import { DataModelingEdge, DataModelingNode, DataModelingNodeType } from '~/types'import { NodeTypeEnumApi } from '../generated/api.schemas'@@ -33,6 +37,7 @@ export interface modelsLineageLogicValues {edgesLoading: boolean // lineageDataLogicsearchTerm: stringdebouncedSearchTerm: string+ searchAnchorNodeId: string | nulltypeFilter: DataModelingNodeType[]legendCollapsed: booleanselectedSearchResultId: string | null@@ -54,6 +59,7 @@ export interface modelsLineageLogicValues {}export interface modelsLineageLogicActions {+ setSearchAnchorNodeId: (nodeId: string | null) => { nodeId: string | null }setSearchTerm: (searchTerm: string) => { searchTerm: string }setDebouncedSearchTerm: (searchTerm: string) => { searchTerm: string }setTypeFilter: (typeFilter: DataModelingNodeType[]) => { typeFilter: DataModelingNodeType[] }@@ -82,10 +88,22 @@ export type modelsLineageLogicType = MakeLogicType<modelsLineageLogicValues, modexport const modelsLineageLogic = kea<modelsLineageLogicType>([path(['scenes', 'models', 'modelsLineageLogic']),+ urlToAction(({ actions }) => ({+ [urls.models()]: (_, searchParams) => {+ if (searchParams.tab === 'lineage') {+ const searchTerm = typeof searchParams.search === 'string' ? searchParams.search : ''+ actions.resetFilters()+ actions.setSearchTerm(searchTerm)+ actions.setDebouncedSearchTerm(searchTerm)+ actions.setSearchAnchorNodeId(typeof searchParams.node_id === 'string' ? searchParams.node_id : null)+ }+ },+ })),connect(() => ({values: [lineageDataLogic, ['nodes', 'nodesLoading', 'edges', 'edgesLoading']],})),actions({+ setSearchAnchorNodeId: (nodeId: string | null) => ({ nodeId }),setSearchTerm: (searchTerm: string) => ({ searchTerm }),setDebouncedSearchTerm: (searchTerm: string) => ({ searchTerm }),setTypeFilter: (typeFilter: DataModelingNodeType[]) => ({ typeFilter }),@@ -98,6 +116,14 @@ export const modelsLineageLogic = kea<modelsLineageLogicType>([resetNodePositions: true,}),reducers({+ searchAnchorNodeId: [+ null as string | null,+ {+ setSearchAnchorNodeId: (_, { nodeId }) => nodeId,+ setSearchTerm: () => null,+ resetFilters: () => null,+ },+ ],searchTerm: ['',{@@ -200,14 +226,37 @@ export const modelsLineageLogic = kea<modelsLineageLogicType>([}},})),+ subscriptions(({ actions }) => ({+ visibleNodes: (nodes: DataModelingNode[], previousNodes: DataModelingNode[] | undefined) => {+ if (+ previousNodes &&+ (nodes.length !== previousNodes.length ||+ nodes.some((node, index) => node.id !== previousNodes[index].id))+ ) {+ actions.resetNodePositions()+ }+ },+ })),selectors({- parsedSearchTerm: [(s) => [s.searchTerm], (searchTerm: string) => parseLineageSearch(searchTerm)],- parsedSearch: [(s) => [s.debouncedSearchTerm], (searchTerm: string) => parseLineageSearch(searchTerm)],+ parsedSearchTerm: [+ (s) => [s.searchTerm, s.searchAnchorNodeId],+ (searchTerm: string, anchorNodeId: string | null): ParsedLineageSearch =>+ anchorNodeId ? { ...parseLineageSearch(searchTerm), anchorNodeId } : parseLineageSearch(searchTerm),+ ],+ parsedSearch: [+ (s) => [s.debouncedSearchTerm, s.searchAnchorNodeId],+ (searchTerm: string, anchorNodeId: string | null): ParsedLineageSearch =>+ anchorNodeId ? { ...parseLineageSearch(searchTerm), anchorNodeId } : parseLineageSearch(searchTerm),+ ],lineageSearchAnchor: [(s) => [s.nodes, s.parsedSearchTerm],(nodes: DataModelingNode[], parsedSearchTerm: ParsedLineageSearch): DataModelingNode | null =>- parsedSearchTerm.mode === 'search' ? null : (matchNodesByName(nodes, parsedSearchTerm.term)[0] ?? null),+ parsedSearchTerm.mode === 'search'+ ? null+ : parsedSearchTerm.anchorNodeId+ ? (nodes.find((node) => node.id === parsedSearchTerm.anchorNodeId) ?? null)+ : (matchNodesByName(nodes, parsedSearchTerm.term)[0] ?? null),],// Both the result list and the graph need the lineage cone. Walking it once here keeps thediff --git a/products/data_modeling/package.json b/products/data_modeling/package.jsonindex 266f9f2a672e..1ad4c3f3d04d 100644--- a/products/data_modeling/package.json+++ b/products/data_modeling/package.json@@ -10,6 +10,7 @@"kea-forms": "catalog:","kea-loaders": "catalog:","kea-router": "catalog:",+ "kea-subscriptions": "catalog:","posthog-js": "catalog:"},"devDependencies": {