Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): keep lineage connected when node types are hidden (#125055)

Thiago Salvatore9682b2b32d6d
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 43e8fd4db2c2..2b4990ed8f97 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -7258,6 +7258,10 @@ snapshots:
hash: v1.k794b7964.108d63e29073bc99117a6ea3d89ccefd287b3c6cc202591b5655c9e223cde734.hiNs3nTXs3ir0zT1DSxG_wIPNH-n9_KUSSlRYUSSns0
products-data-modeling-lineage-graph--full--light:
hash: v1.k794b7964.89f63325605426975447eb5c49f37c2373d2a10992449917cfda6372e06957d3.B-XUikFmmQLn-LgIQMFAiw0dxyMlPsj8bzRTflNqU7s
+ products-data-modeling-lineage-graph--hidden-node-types--dark:
+ hash: v1.k794b7964.cd8b92367c83ea44fa57d6b6c78b9cb7d9a82039cab54d02eec068511fd86495.tjJ-_Dp8mPzPxo24xYjEpWeeL803wR5jsGR_yCufUW8
+ products-data-modeling-lineage-graph--hidden-node-types--light:
+ hash: v1.k794b7964.4c5652c146c8be268a0e363f92ee0db3cc8eb0456b7e167bc329f240677088a1.NRHh0EDDb3ba3ulfpFhJHku36qr3NCwdK3fyhabX0g4
products-data-modeling-lineage-graph--loading--dark:
hash: v1.k794b7964.0e48330c3af8b4788b41e6938d6957d7db3f2fac227302133d4482dde3f4e84f.1_ZYFp5H2cK3OTMZ8UrEjcgHn36LPpbKeEo0Zy1BRPk
products-data-modeling-lineage-graph--loading--light:
@@ -7483,9 +7487,9 @@ snapshots:
products-data-modeling-node-detail-scene--paginated-columns--light:
hash: v1.k794b7964.b31c476c47d1978ff552fddbfcb77ad263c0c8f22dcbcc44b60e70e4653ac060.JNntlPiJsIrZcnvJLXSh7lRKeUdb7PKpKtQR-SLODAg
products-data-modeling-node-detail-scene--post-hog-table--dark:
- hash: v1.k794b7964.7d56e07025e9880943429dc254d50ef80a3de15b68828f6c18195873bffe8b10.--fqtn3db5JGeZaXVPI-Ha331NTs7paA3SGE47kitNo
+ hash: v1.k794b7964.c78e11eb6e7902a7077068efb7aa4a3d43d696ca4f8d0231023f97cbd01eafae.OfzBF6GaAr-VuQ4K-FCzo1_VuEI19pWUbE4OU9pWr0w
products-data-modeling-node-detail-scene--post-hog-table--light:
- hash: v1.k794b7964.f6ada3c96b3c194bd4b4f98f16aadc1073e5e34a2a52876b19a4bd57739b56e4.6K8XdVUSHhMJ57PbPsI0aQzVAROz0dZilxs0nPoXvLw
+ hash: v1.k794b7964.06581169b32fa3f15e09a4bfac108203a346a26bd365c38fdfdaa297e377e7ca.7nrTSQ3FHI_p5_ccauI8tqxrSrUjUJrGZT8hYa2yu3I
products-data-modeling-node-detail-scene--suspended-with-run-history--dark:
hash: v1.k794b7964.8777021fae0861f86ccd00ee197d14fcacf12d0ae1ba8a8fe7b38f20591cbc41.r_GyHK2h2ozRGIYXoZ3N4CAlJMClpccnfeC5TEgiDOo
products-data-modeling-node-detail-scene--suspended-with-run-history--light:
@@ -7495,9 +7499,9 @@ snapshots:
products-data-modeling-node-detail-scene--view--light:
hash: v1.k794b7964.e743a0848352753df7ab1cbbc6538044d9de0a387f319617de876b210d4bce47.n2hUpnRhsVX_Bo6mGpZaqgJ9kw0dbt46jONG85BMNdU
products-data-modeling-node-detail-scene--warehouse-table--dark:
- hash: v1.k794b7964.be58922f69d26ddd9efddb1b1e43a3d8de95472703f74b19a959fe48285a9200.JJ2OsKi210rpYYCKb7ekK_ILlI2s28umWtPmPbhctjI
+ hash: v1.k794b7964.cfe93c72c59b2bad259c3886fa09c4d67cdd176f06711d23573efa2f5fe2725e.A1zJ-83qIxsR3ToDYysYd8jcoCZoMUIKlihswyQSgg4
products-data-modeling-node-detail-scene--warehouse-table--light:
- hash: v1.k794b7964.ab08f887631042e5e621e933edf8ce7293e18faeac2d27d9115eb89362e6386f.sIE7j7MMgZZENRhOIvsoK8ySfu9caxb7CU75zvzu4f4
+ hash: v1.k794b7964.9a86d60eb9ed315ccc93368566e6efce95cb90e19e1e10267131ee30dc1bb477.QpVekP3q5k6TykdTmxWuyDbks8oLv6EASkXggCxz_Ro
products-data-modeling-views-list--actions--dark:
hash: v1.k794b7964.eceb80b180d692b6f616365ea7269d24bb6f084128cbfafc661cf6159dc281ec.FagEpfMfYMDwAJaH7ctNLBVSS-sThFgcTnQ8Ghwk8wI
products-data-modeling-views-list--actions--light:
diff --git a/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx b/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx
index 1eedad0c5579..f4c99919e3af 100644
--- a/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx
+++ b/products/data_modeling/frontend/lineage/LineageGraph.stories.tsx
@@ -7,6 +7,7 @@ import { mswDecorator } from '~/mocks/browser'
import { DataModelingEdge, DataModelingNode } from '~/types'
import { LineageGraph } from './LineageGraph'
+import { bridgeHiddenNodes, nodesWithHubs } from './lineageHiddenNodes'
import { modelsLineageLogic } from './modelsLineageLogic'
import { ModelsLineageTab } from './ModelsLineageTab'
@@ -206,6 +207,47 @@ export const Canvas: Story = {
),
}
+const HIDDEN_TYPES_NODES: DataModelingNode[] = [
+ mockNode({ id: 'orders', name: 'orders', type: 'table' }),
+ mockNode({ id: 'customers', name: 'customers', type: 'table' }),
+ mockNode({ id: 'payments', name: 'payments', type: 'table' }),
+ mockNode({ id: 'enriched_orders', name: 'enriched_orders', type: 'matview' }),
+ mockNode({ id: 'orders_api', name: 'orders_api', type: 'endpoint' }),
+ mockNode({ id: 'revenue_api', name: 'revenue_api', type: 'endpoint' }),
+ mockNode({ id: 'churn_api', name: 'churn_api', type: 'endpoint' }),
+ mockNode({ id: 'weekly_orders', name: 'weekly_orders', type: 'metric', metric_id: 'metric-weekly' }),
+ mockNode({ id: 'revenue_summary', name: 'revenue_summary', type: 'view', user_tag: 'finance' }),
+ mockNode({ id: 'monthly_report', name: 'monthly_report', type: 'matview' }),
+ mockNode({ id: 'forecast_api', name: 'forecast_api', type: 'endpoint' }),
+]
+const HIDDEN_TYPES_EDGES: DataModelingEdge[] = [
+ ...['orders', 'customers', 'payments'].map((source) => mockEdge(`${source}-hub`, source, 'enriched_orders')),
+ ...['orders_api', 'revenue_api', 'churn_api', 'weekly_orders'].map((target) =>
+ mockEdge(`hub-${target}`, 'enriched_orders', target)
+ ),
+ mockEdge('orders-summary', 'orders', 'revenue_summary'),
+ mockEdge('summary-report', 'revenue_summary', 'monthly_report'),
+ mockEdge('report-forecast', 'monthly_report', 'forecast_api'),
+]
+const HIDDEN_TYPES_VISIBLE_IDS = new Set(
+ HIDDEN_TYPES_NODES.filter((node) => ['table', 'view', 'endpoint'].includes(node.type)).map((node) => node.id)
+)
+const HIDDEN_TYPES_BRIDGING = bridgeHiddenNodes(
+ HIDDEN_TYPES_EDGES,
+ new Set(HIDDEN_TYPES_NODES.map((node) => node.id)),
+ HIDDEN_TYPES_VISIBLE_IDS
+)
+
+export const HiddenNodeTypes: Story = {
+ render: () => (
+ <LineageGraph
+ nodes={nodesWithHubs(HIDDEN_TYPES_NODES, HIDDEN_TYPES_VISIBLE_IDS, HIDDEN_TYPES_BRIDGING.hubNodeIds)}
+ edges={HIDDEN_TYPES_BRIDGING.edges}
+ variant="canvas"
+ />
+ ),
+}
+
export const Selectable: Story = {
render: () => <LineageGraph nodes={GRAPH_NODES} edges={GRAPH_EDGES} variant="canvas" selectable interactive />,
play: async ({ canvasElement }) => {
diff --git a/products/data_modeling/frontend/lineage/LineageGraph.tsx b/products/data_modeling/frontend/lineage/LineageGraph.tsx
index 7c8c6a9e13bb..1b24a0a808fb 100644
--- a/products/data_modeling/frontend/lineage/LineageGraph.tsx
+++ b/products/data_modeling/frontend/lineage/LineageGraph.tsx
@@ -31,26 +31,27 @@ import {
import { IconArchive, IconExpand45, IconRefresh } from '@posthog/icons'
import { themeLogic } from '~/layout/navigation-3000/themeLogic'
-import { DataModelingEdge, DataModelingNode } from '~/types'
+import { DataModelingNode } from '~/types'
import { ElkDirection } from './autolayout'
import { LINEAGE_MIN_ZOOM, lineageFitTarget } from './lineageFit'
import { LineageGraphLoading } from './LineageGraphLoading'
import { lineageGraphLogic } from './lineageGraphLogic'
+import { LineageEdge, LineageGraphNode } from './lineageHiddenNodes'
import { LINEAGE_NODE_TYPES, LineageNodeCallbacks, LineageNodeState, LineageVariant } from './LineageNode'
import { lineageCone } from './lineageSelection'
import { useNodesMeasured } from './useNodesMeasured'
export type { LineageVariant, LineageNodeState, LineageNodeCallbacks } from './LineageNode'
-const EMPTY_NODES: DataModelingNode[] = []
-const EMPTY_EDGES: DataModelingEdge[] = []
+const EMPTY_NODES: LineageGraphNode[] = []
+const EMPTY_EDGES: LineageEdge[] = []
const FOCUSED_NODES_MAX_ZOOM = 2
const CURRENT_NODE_MAX_ZOOM = 1
export interface LineageGraphProps {
- nodes: DataModelingNode[]
- edges: DataModelingEdge[]
+ nodes: LineageGraphNode[]
+ edges: LineageEdge[]
/** Highlighted "you are here" node, rendered with a target marker + accent border */
currentNodeId?: string
variant?: LineageVariant
diff --git a/products/data_modeling/frontend/lineage/LineageNode.tsx b/products/data_modeling/frontend/lineage/LineageNode.tsx
index 38186048fcf4..80cacc6bc826 100644
--- a/products/data_modeling/frontend/lineage/LineageNode.tsx
+++ b/products/data_modeling/frontend/lineage/LineageNode.tsx
@@ -35,6 +35,7 @@ import { servingSuspension } from 'products/data_modeling/frontend/suspension'
import { syncIntervalToShorthand } from 'products/data_warehouse/frontend/utils'
import { ElkDirection, NodeHandle } from './autolayout'
+import { LineageGraphNode } from './lineageHiddenNodes'
import { NODE_TYPE_TAG_SETTINGS, statusBackgroundClass } from './nodeStyles'
import { NodeTypeTag } from './NodeTypeTag'
@@ -54,7 +55,8 @@ export type LineageNodeShape = Pick<
| 'user_tag'
| 'suspended'
| 'lineage_issue'
->
+> &
+ Pick<LineageGraphNode, 'collapsed'>
export interface LineageNodeState {
isCurrent?: boolean
@@ -208,6 +210,57 @@ function MetadataBar({ node }: { node: LineageNodeShape }): JSX.Element {
)
}
+function NodeHandles({ handles }: { handles: NodeHandle[] }): JSX.Element {
+ return (
+ <>
+ {handles.map((handle) => (
+ <Handle
+ key={handle.id}
+ id={handle.id}
+ type={handle.type}
+ position={handle.position ?? (handle.type === 'target' ? Position.Left : Position.Right)}
+ className="opacity-0"
+ isConnectable={false}
+ />
+ ))}
+ </>
+ )
+}
+
+function CollapsedLineageNode({ data }: { data: LineageNodeData }): JSX.Element {
+ const { node, state, callbacks } = data
+ const description = `Hidden ${NODE_TYPE_TAG_SETTINGS[node.type].label.toLowerCase()}: ${node.name}`
+ return (
+ <Tooltip title={description}>
+ <div
+ className={clsx(
+ 'relative flex h-full w-full items-center justify-center rounded-full border border-dashed bg-bg-light text-secondary pointer-events-auto',
+ data.draggable && 'cursor-grab active:cursor-grabbing',
+ !data.draggable && callbacks.onClick && 'cursor-pointer',
+ state.isSelected && 'border-link ring-2 ring-link/40',
+ state.isDimmed && 'opacity-30'
+ )}
+ onClick={callbacks.onClick}
+ data-attr="lineage-collapsed-node"
+ >
+ {callbacks.onClick && (
+ <button
+ type="button"
+ className="absolute inset-0 pointer-events-none rounded-full focus-visible:ring-2 focus-visible:ring-link/40"
+ onClick={(event) => {
+ event.stopPropagation()
+ callbacks.onClick?.(event)
+ }}
+ aria-label={description}
+ />
+ )}
+ <NodeHandles handles={data.handles} />
+ <span aria-hidden="true">⋯</span>
+ </div>
+ </Tooltip>
+ )
+}
+
function LineageNodeContextMenu({
data,
children,
@@ -289,16 +342,7 @@ export function LineageNode({ data }: { data: LineageNodeData }): JSX.Element {
state.loading === 'focus' ? `color-mix(in srgb, ${color} 60%, transparent)` : undefined,
}}
>
- {data.handles.map((handle) => (
- <Handle
- key={handle.id}
- id={handle.id}
- type={handle.type}
- position={handle.position ?? (handle.type === 'target' ? Position.Left : Position.Right)}
- className="opacity-0"
- isConnectable={false}
- />
- ))}
+ <NodeHandles handles={data.handles} />
<div className="flex flex-1 flex-col justify-center gap-2 px-3">
{state.loading === 'focus' ? (
<div className="opacity-70">
@@ -327,6 +371,10 @@ export function LineageNode({ data }: { data: LineageNodeData }): JSX.Element {
)
}
+ if (node.collapsed) {
+ return <CollapsedLineageNode data={data} />
+ }
+
const stop = (fn?: () => void) => (e: React.MouseEvent) => {
e.stopPropagation()
fn?.()
@@ -376,16 +424,7 @@ export function LineageNode({ data }: { data: LineageNodeData }): JSX.Element {
aria-label={ariaLabel}
/>
)}
- {data.handles.map((handle) => (
- <Handle
- key={handle.id}
- id={handle.id}
- type={handle.type}
- position={handle.position ?? (handle.type === 'target' ? Position.Left : Position.Right)}
- className="opacity-0"
- isConnectable={false}
- />
- ))}
+ <NodeHandles handles={data.handles} />
{showRunArrows && node.upstream_count > 0 && callbacks.onRunUpstream && (
<RunArrow direction="upstream" layoutDirection={direction} onClick={stop(callbacks.onRunUpstream)} />
diff --git a/products/data_modeling/frontend/lineage/LineageTypeFilter.tsx b/products/data_modeling/frontend/lineage/LineageTypeFilter.tsx
new file mode 100644
index 000000000000..e31deb94990c
--- /dev/null
+++ b/products/data_modeling/frontend/lineage/LineageTypeFilter.tsx
@@ -0,0 +1,58 @@
+import { IconChevronDown, IconDatabase } from '@posthog/icons'
+import { LemonButton, LemonCheckbox, LemonDropdown } from '@posthog/lemon-ui'
+
+import { DataModelingNodeType } from '~/types'
+
+import { LINEAGE_FILTER_TYPES } from './modelsLineageLogic'
+import { NODE_TYPE_TAG_SETTINGS } from './nodeStyles'
+
+export interface LineageTypeFilterProps {
+ value: DataModelingNodeType[]
+ onChange: (value: DataModelingNodeType[]) => void
+ dataAttr: string
+}
+
+function summarize(selected: DataModelingNodeType[]): string {
+ if (selected.length === LINEAGE_FILTER_TYPES.length) {
+ return 'All types'
+ }
+ if (selected.length === 0) {
+ return 'No types'
+ }
+ if (selected.length === 1) {
+ return NODE_TYPE_TAG_SETTINGS[selected[0]].label
+ }
+ return `${selected.length} types`
+}
+
+export function LineageTypeFilter({ value, onChange, dataAttr }: LineageTypeFilterProps): JSX.Element {
+ const toggle = (type: DataModelingNodeType): void =>
+ onChange(value.includes(type) ? value.filter((selected) => selected !== type) : [...value, type])
+
+ return (
+ <LemonDropdown
+ closeOnClickInside={false}
+ overlay={
+ <div className="flex flex-col gap-px p-1">
+ {LINEAGE_FILTER_TYPES.map((type) => (
+ <LemonButton
+ key={type}
+ type="tertiary"
+ size="small"
+ fullWidth
+ icon={<LemonCheckbox checked={value.includes(type)} className="pointer-events-none" />}
+ sideIcon={<IconDatabase style={{ color: NODE_TYPE_TAG_SETTINGS[type].color }} />}
+ onClick={() => toggle(type)}
+ >
+ {NODE_TYPE_TAG_SETTINGS[type].label}
+ </LemonButton>
+ ))}
+ </div>
+ }
+ >
+ <LemonButton type="secondary" size="small" sideIcon={<IconChevronDown />} data-attr={dataAttr}>
+ {summarize(value)}
+ </LemonButton>
+ </LemonDropdown>
+ )
+}
diff --git a/products/data_modeling/frontend/lineage/ModelsLineageTab.tsx b/products/data_modeling/frontend/lineage/ModelsLineageTab.tsx
index e787bf112350..d9014855a4e6 100644
--- a/products/data_modeling/frontend/lineage/ModelsLineageTab.tsx
+++ b/products/data_modeling/frontend/lineage/ModelsLineageTab.tsx
@@ -4,24 +4,16 @@ import { useRef } from 'react'
import { IconInfo } from '@posthog/icons'
import { LemonButton, LemonInput, Tooltip } from '@posthog/lemon-ui'
-import { LemonInputSelect } from 'lib/lemon-ui/LemonInputSelect'
import { pluralize } from 'lib/utils/strings'
-import { DataModelingNodeType } from '~/types'
-
import { LineageGraph } from './LineageGraph'
import { lineageNodeUrl } from './lineageNodeUrl'
import { LineageSearchResults } from './LineageSearchResults'
-import { LINEAGE_FILTER_TYPES, modelsLineageLogic } from './modelsLineageLogic'
-import { NODE_TYPE_TAG_SETTINGS } from './nodeStyles'
+import { LineageTypeFilter } from './LineageTypeFilter'
+import { modelsLineageLogic } from './modelsLineageLogic'
import { NodeTypeLegend } from './NodeTypeLegend'
import { SEARCH_SYNTAX_HELP } from './SearchSyntaxHelp'
-const TYPE_OPTIONS = LINEAGE_FILTER_TYPES.map((type) => ({
- key: type,
- label: NODE_TYPE_TAG_SETTINGS[type].label,
-}))
-
export function ModelsLineageTab(): JSX.Element {
const searchInputRef = useRef<HTMLInputElement>(null)
const {
@@ -41,6 +33,7 @@ export function ModelsLineageTab(): JSX.Element {
searchFocusRequest,
focusNodeIds,
visibleNodes,
+ graphNodes,
visibleEdges,
isFiltered,
nodePositions,
@@ -104,19 +97,7 @@ export function ModelsLineageTab(): JSX.Element {
<Tooltip title={SEARCH_SYNTAX_HELP}>
<IconInfo className="text-base text-secondary" />
</Tooltip>
- <div className="w-44">
- <LemonInputSelect
- mode="multiple"
- size="small"
- placeholder="All types"
- options={TYPE_OPTIONS}
- value={typeFilter}
- onChange={(value) => setTypeFilter(value as DataModelingNodeType[])}
- displayMode="count"
- bulkActions="select-and-clear-all"
- data-attr="models-lineage-type-filter"
- />
- </div>
+ <LineageTypeFilter value={typeFilter} onChange={setTypeFilter} dataAttr="models-lineage-type-filter" />
{isFiltered && (
<>
<span className="text-xs text-secondary">
@@ -153,7 +134,7 @@ export function ModelsLineageTab(): JSX.Element {
/>
)}
<LineageGraph
- nodes={visibleNodes}
+ nodes={graphNodes}
edges={visibleEdges}
focusNodeIds={focusNodeIds}
searchFocusRequest={searchFocusRequest}
diff --git a/products/data_modeling/frontend/lineage/lineageGraphLogic.ts b/products/data_modeling/frontend/lineage/lineageGraphLogic.ts
index 2162c1bae357..08e825e7a7a3 100644
--- a/products/data_modeling/frontend/lineage/lineageGraphLogic.ts
+++ b/products/data_modeling/frontend/lineage/lineageGraphLogic.ts
@@ -1,9 +1,8 @@
import { MarkerType, Position, type Edge as ReactFlowEdge, type Node as ReactFlowNode } from '@xyflow/react'
import { MakeLogicType, actions, afterMount, kea, key, listeners, path, props, propsChanged, reducers } from 'kea'
-import { DataModelingEdge, DataModelingNode } from '~/types'
-
import { ElkDirection, NodeHandle, getFormattedNodes } from './autolayout'
+import { LineageEdge, LineageGraphNode } from './lineageHiddenNodes'
import { LineageNodeData, LineageVariant } from './LineageNode'
const NODE_SIZES: Record<LineageVariant, { width: number; height: number }> = {
@@ -11,16 +10,29 @@ const NODE_SIZES: Record<LineageVariant, { width: number; height: number }> = {
canvas: { width: 180, height: 120 },
}
+const COLLAPSED_NODE_SIZE = { width: 40, height: 24 }
+const ARROW_MARKER = { type: MarkerType.ArrowClosed, width: 20, height: 20 }
+
const NODE_GAP = 60
+const THROUGH_HIDDEN_NODES_EDGE: Partial<ReactFlowEdge> = {
+ label: '⋯',
+ ariaLabel: 'Connected through hidden models',
+ style: { strokeDasharray: '4 4' },
+ labelStyle: { fill: 'var(--color-text-secondary)' },
+ labelBgStyle: { fill: 'var(--color-bg-surface-primary)', fillOpacity: 0.8 },
+ labelBgPadding: [2, 0],
+ labelBgBorderRadius: 4,
+}
+
export interface LineageGraphLayout {
nodes: ReactFlowNode<LineageNodeData>[]
edges: ReactFlowEdge[]
}
export interface LineageGraphLogicProps {
- nodes: DataModelingNode[]
- edges: DataModelingEdge[]
+ nodes: LineageGraphNode[]
+ edges: LineageEdge[]
variant: LineageVariant
direction: ElkDirection
}
@@ -34,8 +46,8 @@ function handlesForDirection(nodeId: string, direction: ElkDirection): NodeHandl
}
export function initialLineageGraphLayout(
- nodes: DataModelingNode[],
- edges: DataModelingEdge[],
+ nodes: LineageGraphNode[],
+ edges: LineageEdge[],
variant: LineageVariant,
direction: ElkDirection
): LineageGraphLayout {
@@ -46,8 +58,7 @@ export function initialLineageGraphLayout(
type: 'lineage',
position:
direction === 'DOWN' ? { x: 0, y: index * (height + NODE_GAP) } : { x: index * (width + NODE_GAP), y: 0 },
- width,
- height,
+ ...(node.collapsed ? COLLAPSED_NODE_SIZE : { width, height }),
data: {
node,
variant,
@@ -58,13 +69,15 @@ export function initialLineageGraphLayout(
},
}))
+ const collapsedNodeIds = new Set(nodes.filter((node) => node.collapsed).map((node) => node.id))
const rfEdges: ReactFlowEdge[] = edges.map((edge) => ({
id: edge.id,
source: edge.source_id,
target: edge.target_id,
sourceHandle: `source_${edge.source_id}`,
targetHandle: `target_${edge.target_id}`,
- markerEnd: { type: MarkerType.ArrowClosed, width: 20, height: 20 },
+ markerEnd: collapsedNodeIds.has(edge.target_id) ? undefined : ARROW_MARKER,
+ ...(edge.throughHiddenNodes && THROUGH_HIDDEN_NODES_EDGE),
}))
return { nodes: rfNodes, edges: rfEdges }
@@ -73,8 +86,8 @@ export function initialLineageGraphLayout(
// Layout depends only on the graph's structure — per-node state/callbacks are decorated onto the
// laid-out nodes at render time, so caller-supplied inline callbacks never retrigger the ELK pass.
async function layoutGraph(
- nodes: DataModelingNode[],
- edges: DataModelingEdge[],
+ nodes: LineageGraphNode[],
+ edges: LineageEdge[],
variant: LineageVariant,
direction: ElkDirection
): Promise<LineageGraphLayout> {
diff --git a/products/data_modeling/frontend/lineage/lineageHiddenNodes.test.ts b/products/data_modeling/frontend/lineage/lineageHiddenNodes.test.ts
new file mode 100644
index 000000000000..695b303a1b44
--- /dev/null
+++ b/products/data_modeling/frontend/lineage/lineageHiddenNodes.test.ts
@@ -0,0 +1,73 @@
+import { DataModelingEdge } from '~/types'
+
+import { bridgeHiddenNodes } from './lineageHiddenNodes'
+
+const edge = (source_id: string, target_id: string): DataModelingEdge => ({
+ id: `${source_id}>${target_id}`,
+ source_id,
+ target_id,
+})
+
+describe('bridgeHiddenNodes', () => {
+ test.each([
+ {
+ name: 'connects across one hidden node',
+ edges: [edge('a', 'h'), edge('h', 'b')],
+ scope: ['a', 'h', 'b'],
+ visible: ['a', 'b'],
+ expectedEdges: ['a>b hidden'],
+ },
+ {
+ name: 'connects across a chain of hidden nodes',
+ edges: [edge('a', 'h1'), edge('h1', 'h2'), edge('h2', 'b')],
+ scope: ['a', 'h1', 'h2', 'b'],
+ visible: ['a', 'b'],
+ expectedEdges: ['a>b hidden'],
+ },
+ {
+ name: 'keeps the real edge when a hidden path runs alongside it',
+ edges: [edge('a', 'b'), edge('a', 'h'), edge('h', 'b')],
+ scope: ['a', 'h', 'b'],
+ visible: ['a', 'b'],
+ expectedEdges: ['a>b'],
+ },
+ {
+ name: 'fans out from a hidden node to every visible child',
+ edges: [edge('a', 'h'), edge('h', 'b'), edge('h', 'c')],
+ scope: ['a', 'h', 'b', 'c'],
+ visible: ['a', 'b', 'c'],
+ expectedEdges: ['a>b hidden', 'a>c hidden'],
+ },
+ {
+ name: 'keeps hidden hubs as nodes instead of multiplying edges through them',
+ edges: [
+ edge('a1', 'h1'),
+ edge('a2', 'h1'),
+ edge('a3', 'h1'),
+ edge('h1', 'h2'),
+ edge('h2', 'b1'),
+ edge('h2', 'b2'),
+ edge('h2', 'b3'),
+ ],
+ scope: ['a1', 'a2', 'a3', 'h1', 'h2', 'b1', 'b2', 'b3'],
+ visible: ['a1', 'a2', 'a3', 'b1', 'b2', 'b3'],
+ expectedEdges: ['a1>h1', 'a2>h1', 'a3>h1', 'h1>h2', 'h2>b1', 'h2>b2', 'h2>b3'],
+ expectedHubs: ['h1', 'h2'],
+ },
+ {
+ name: 'does not walk through a node outside the scope',
+ edges: [edge('a', 'out'), edge('out', 'b')],
+ scope: ['a', 'b'],
+ visible: ['a', 'b'],
+ expectedEdges: [],
+ },
+ ])('$name', ({ edges, scope, visible, expectedEdges, expectedHubs = [] }) => {
+ const bridging = bridgeHiddenNodes(edges, new Set(scope), new Set(visible))
+
+ const renderedEdges = bridging.edges.map(
+ (e) => `${e.source_id}>${e.target_id}${e.throughHiddenNodes ? ' hidden' : ''}`
+ )
+ expect(renderedEdges.sort()).toEqual(expectedEdges)
+ expect([...bridging.hubNodeIds].sort()).toEqual(expectedHubs)
+ })
+})
diff --git a/products/data_modeling/frontend/lineage/lineageHiddenNodes.ts b/products/data_modeling/frontend/lineage/lineageHiddenNodes.ts
new file mode 100644
index 000000000000..1dc9a6f24938
--- /dev/null
+++ b/products/data_modeling/frontend/lineage/lineageHiddenNodes.ts
@@ -0,0 +1,193 @@
+import { DataModelingEdge, DataModelingNode } from '~/types'
+
+import { AdjacencyMaps, buildAdjacencyMaps, edgesWithinNodes } from './lineageSearch'
+
+const PATH_COUNT_CEILING = 1_000_000
+const CACHED_REACH_LIMIT = 2
+const EMPTY_NODE_IDS: ReadonlySet<string> = new Set()
+
+export interface LineageEdge extends DataModelingEdge {
+ throughHiddenNodes?: boolean
+}
+
+export interface LineageGraphNode extends DataModelingNode {
+ collapsed?: boolean
+}
+
+export interface HiddenNodeBridging {
+ edges: LineageEdge[]
+ hubNodeIds: Set<string>
+}
+
+function childrenFirstOrder(
+ nodeIds: Iterable<string>,
+ adjacency: Map<string, string[]>,
+ include: (nodeId: string) => boolean
+): string[] {
+ const order: string[] = []
+ const visited = new Set<string>()
+ for (const startId of nodeIds) {
+ if (!include(startId) || visited.has(startId)) {
+ continue
+ }
+ visited.add(startId)
+ const stack = [{ nodeId: startId, nextChild: 0 }]
+ while (stack.length > 0) {
+ const frame = stack[stack.length - 1]
+ const children = adjacency.get(frame.nodeId) ?? []
+ if (frame.nextChild === children.length) {
+ order.push(frame.nodeId)
+ stack.pop()
+ continue
+ }
+ const childId = children[frame.nextChild++]
+ if (include(childId) && !visited.has(childId)) {
+ visited.add(childId)
+ stack.push({ nodeId: childId, nextChild: 0 })
+ }
+ }
+ }
+ return order
+}
+
+class HiddenSubgraph {
+ readonly terminals: Set<string>
+ readonly hubNodeIds: Set<string>
+ private readonly maps: AdjacencyMaps
+ private readonly downstreamPaths: Map<string, number>
+ private readonly cachedReach = new Map<string, ReadonlySet<string>>()
+
+ constructor(
+ scopedEdges: DataModelingEdge[],
+ private readonly scopeNodeIds: Set<string>,
+ private readonly visibleNodeIds: Set<string>
+ ) {
+ this.maps = buildAdjacencyMaps(scopedEdges)
+ this.downstreamPaths = this.countVisiblePaths(this.maps.downstream)
+ this.hubNodeIds = this.findHubs(this.countVisiblePaths(this.maps.upstream))
+ this.terminals = new Set([...visibleNodeIds, ...this.hubNodeIds])
+ this.walkHiddenNodes()
+ }
+
+ terminalsReachedFrom(sourceId: string): Set<string> {
+ const reached = new Set<string>()
+ const visited = new Set<string>()
+ const stack = this.hiddenChildren(sourceId)
+ while (stack.length > 0) {
+ const nodeId = stack.pop() as string
+ if (visited.has(nodeId)) {
+ continue
+ }
+ visited.add(nodeId)
+ const cached = this.cachedReach.get(nodeId)
+ if (cached) {
+ cached.forEach((targetId) => reached.add(targetId))
+ continue
+ }
+ for (const childId of this.maps.downstream.get(nodeId) ?? []) {
+ if (this.terminals.has(childId)) {
+ reached.add(childId)
+ } else {
+ stack.push(childId)
+ }
+ }
+ }
+ reached.delete(sourceId)
+ return reached
+ }
+
+ private isHidden = (nodeId: string): boolean => !this.visibleNodeIds.has(nodeId)
+
+ private isHiddenNonTerminal = (nodeId: string): boolean => !this.terminals.has(nodeId)
+
+ private hiddenChildren(nodeId: string): string[] {
+ return (this.maps.downstream.get(nodeId) ?? []).filter(this.isHiddenNonTerminal)
+ }
+
+ private findHubs(upstreamPaths: Map<string, number>): Set<string> {
+ return new Set(
+ [...upstreamPaths.keys()].filter((nodeId) => {
+ const upstream = upstreamPaths.get(nodeId) ?? 0
+ const downstream = this.downstreamPaths.get(nodeId) ?? 0
+ return upstream * downstream > upstream + downstream
+ })
+ )
+ }
+
+ private countVisiblePaths(adjacency: Map<string, string[]>): Map<string, number> {
+ const counts = new Map<string, number>()
+ for (const nodeId of childrenFirstOrder(this.scopeNodeIds, adjacency, this.isHidden)) {
+ let total = 0
+ for (const neighborId of adjacency.get(nodeId) ?? []) {
+ total += this.isHidden(neighborId) ? (counts.get(neighborId) ?? 0) : 1
+ }
+ counts.set(nodeId, Math.min(total, PATH_COUNT_CEILING))
+ }
+ return counts
+ }
+
+ private walkHiddenNodes(): void {
+ for (const nodeId of childrenFirstOrder(this.scopeNodeIds, this.maps.downstream, this.isHiddenNonTerminal)) {
+ if ((this.downstreamPaths.get(nodeId) ?? 0) <= CACHED_REACH_LIMIT) {
+ this.cachedReach.set(nodeId, this.reachFromCachedChildren(nodeId))
+ }
+ }
+ }
+
+ private reachFromCachedChildren(nodeId: string): ReadonlySet<string> {
+ const reached = new Set<string>()
+ for (const childId of this.maps.downstream.get(nodeId) ?? []) {
+ if (this.terminals.has(childId)) {
+ reached.add(childId)
+ } else {
+ this.cachedReach.get(childId)?.forEach((targetId) => reached.add(targetId))
+ }
+ }
+ return reached.size === 0 ? EMPTY_NODE_IDS : reached
+ }
+}
+
+export function nodesWithHubs(
+ scopedNodes: DataModelingNode[],
+ visibleNodeIds: Set<string>,
+ hubNodeIds: Set<string>
+): LineageGraphNode[] {
+ return scopedNodes
+ .filter((node) => visibleNodeIds.has(node.id) || hubNodeIds.has(node.id))
+ .map((node) => (hubNodeIds.has(node.id) ? { ...node, collapsed: true } : node))
+}
+
+const edgeKey = (sourceId: string, targetId: string): string => `${sourceId}>${targetId}`
+
+export function bridgeHiddenNodes(
+ edges: DataModelingEdge[],
+ scopeNodeIds: Set<string>,
+ visibleNodeIds: Set<string>
+): HiddenNodeBridging {
+ if (scopeNodeIds.size === visibleNodeIds.size) {
+ return {
+ edges: edgesWithinNodes(edges, visibleNodeIds),
+ hubNodeIds: new Set(),
+ }
+ }
+
+ const subgraph = new HiddenSubgraph(edgesWithinNodes(edges, scopeNodeIds), scopeNodeIds, visibleNodeIds)
+ const realEdges = edgesWithinNodes(edges, subgraph.terminals)
+ const realEdgeKeys = new Set(realEdges.map((edge) => edgeKey(edge.source_id, edge.target_id)))
+ const bridgedEdges: LineageEdge[] = []
+
+ for (const sourceId of subgraph.terminals) {
+ for (const targetId of subgraph.terminalsReachedFrom(sourceId)) {
+ if (!realEdgeKeys.has(edgeKey(sourceId, targetId))) {
+ bridgedEdges.push({
+ id: `hidden:${sourceId}:${targetId}`,
+ source_id: sourceId,
+ target_id: targetId,
+ throughHiddenNodes: true,
+ })
+ }
+ }
+ }
+
+ return { edges: [...realEdges, ...bridgedEdges], hubNodeIds: subgraph.hubNodeIds }
+}
diff --git a/products/data_modeling/frontend/lineage/modelsLineageLogic.ts b/products/data_modeling/frontend/lineage/modelsLineageLogic.ts
index 223df8fe36a9..bdaba9298f77 100644
--- a/products/data_modeling/frontend/lineage/modelsLineageLogic.ts
+++ b/products/data_modeling/frontend/lineage/modelsLineageLogic.ts
@@ -10,9 +10,15 @@ import { DataModelingEdge, DataModelingNode, DataModelingNodeType } from '~/type
import { NodeTypeEnumApi } from '../generated/api.schemas'
import { lineageDataLogic } from './lineageDataLogic'
+import {
+ HiddenNodeBridging,
+ LineageEdge,
+ LineageGraphNode,
+ bridgeHiddenNodes,
+ nodesWithHubs,
+} from './lineageHiddenNodes'
import {
ParsedLineageSearch,
- edgesWithinNodes,
matchNodesByName,
orderedNodesForLineageSearch,
parseLineageSearch,
@@ -24,6 +30,9 @@ type SearchResultFocusTrigger = 'keyboard' | 'click' | 'previous' | 'next'
// Product analytics can depend on this event name, so keep it stable.
const MODELS_LINEAGE_NODE_MOVED_EVENT = 'models lineage node moved'
+export const LINEAGE_TYPE_FILTER_CHANGED_EVENT = 'models lineage type filter changed'
+
+const nodeIds = (nodes: DataModelingNode[]): Set<string> => new Set(nodes.map((node) => node.id))
export interface SearchFocusRequest {
nodeId: string
@@ -52,8 +61,11 @@ export interface modelsLineageLogicValues {
searchResultAnnouncement: string
highlightedNodeIds: Set<string>
focusNodeIds: Set<string> | null
+ scopedNodes: DataModelingNode[]
visibleNodes: DataModelingNode[]
- visibleEdges: DataModelingEdge[]
+ hiddenNodeBridging: HiddenNodeBridging
+ graphNodes: LineageGraphNode[]
+ visibleEdges: LineageEdge[]
isFiltered: boolean
nodePositions: Record<string, XYPosition>
}
@@ -216,6 +228,13 @@ export const modelsLineageLogic = kea<modelsLineageLogicType>([
})
}
},
+ setTypeFilter: ({ typeFilter }) => {
+ posthog.capture(LINEAGE_TYPE_FILTER_CHANGED_EVENT, {
+ surface: 'models',
+ visible_types: typeFilter,
+ hidden_type_count: LINEAGE_FILTER_TYPES.length - typeFilter.length,
+ })
+ },
nodeDragStopped: ({ nodeId }) => {
const node = values.nodes.find((candidate) => candidate.id === nodeId)
if (node) {
@@ -301,7 +320,7 @@ export const modelsLineageLogic = kea<modelsLineageLogicType>([
if (searchTerm !== debouncedSearchTerm) {
return []
}
- const visibleNodeIds = new Set(visibleNodes.map((node) => node.id))
+ const visibleNodeIds = nodeIds(visibleNodes)
return (orderedLineageNodes ?? []).filter((node) => visibleNodeIds.has(node.id))
},
],
@@ -357,42 +376,59 @@ export const modelsLineageLogic = kea<modelsLineageLogicType>([
parsedSearchTerm.mode === 'search' ? new Set(searchResults.map((node) => node.id)) : new Set(),
],
- visibleNodes: [
- (s) => [s.nodes, s.orderedLineageNodes, s.typeFilter],
- (
- nodes: DataModelingNode[],
- orderedLineageNodes: DataModelingNode[] | null,
- typeFilter: DataModelingNodeType[]
- ): DataModelingNode[] => {
- let kept = nodes
-
- if (orderedLineageNodes) {
- const reached = new Set(orderedLineageNodes.map((node) => node.id))
- kept = kept.filter((node) => reached.has(node.id))
+ scopedNodes: [
+ (s) => [s.nodes, s.orderedLineageNodes],
+ (nodes: DataModelingNode[], orderedLineageNodes: DataModelingNode[] | null): DataModelingNode[] => {
+ if (!orderedLineageNodes) {
+ return nodes
}
-
- if (typeFilter.length !== LINEAGE_FILTER_TYPES.length) {
- kept = kept.filter((node) => typeFilter.includes(node.type))
- }
-
- return kept
+ const reached = nodeIds(orderedLineageNodes)
+ return nodes.filter((node) => reached.has(node.id))
},
],
+ visibleNodes: [
+ (s) => [s.scopedNodes, s.typeFilter],
+ (scopedNodes: DataModelingNode[], typeFilter: DataModelingNodeType[]): DataModelingNode[] =>
+ typeFilter.length === LINEAGE_FILTER_TYPES.length
+ ? scopedNodes
+ : scopedNodes.filter((node) => typeFilter.includes(node.type)),
+ ],
+
focusNodeIds: [
(s) => [s.parsedSearch, s.visibleNodes],
(parsedSearch: ParsedLineageSearch, visibleNodes: DataModelingNode[]): Set<string> | null => {
if (parsedSearch.mode === 'search') {
return parsedSearch.term ? null : new Set()
}
- return new Set(visibleNodes.map((node) => node.id))
+ return nodeIds(visibleNodes)
},
],
+ hiddenNodeBridging: [
+ (s) => [s.edges, s.scopedNodes, s.visibleNodes],
+ (
+ edges: DataModelingEdge[],
+ scopedNodes: DataModelingNode[],
+ visibleNodes: DataModelingNode[]
+ ): HiddenNodeBridging => bridgeHiddenNodes(edges, nodeIds(scopedNodes), nodeIds(visibleNodes)),
+ ],
+
+ graphNodes: [
+ (s) => [s.scopedNodes, s.visibleNodes, s.hiddenNodeBridging],
+ (
+ scopedNodes: DataModelingNode[],
+ visibleNodes: DataModelingNode[],
+ hiddenNodeBridging: HiddenNodeBridging
+ ): LineageGraphNode[] =>
+ hiddenNodeBridging.hubNodeIds.size === 0
+ ? visibleNodes
+ : nodesWithHubs(scopedNodes, nodeIds(visibleNodes), hiddenNodeBridging.hubNodeIds),
+ ],
+
visibleEdges: [
- (s) => [s.edges, s.visibleNodes],
- (edges: DataModelingEdge[], visibleNodes: DataModelingNode[]): DataModelingEdge[] =>
- edgesWithinNodes(edges, new Set(visibleNodes.map((node) => node.id))),
+ (s) => [s.hiddenNodeBridging],
+ (hiddenNodeBridging: HiddenNodeBridging): LineageEdge[] => hiddenNodeBridging.edges,
],
isFiltered: [
diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailLineage.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailLineage.tsx
index 6f83698e096c..52f63e67d07f 100644
--- a/products/data_modeling/frontend/nodeDetail/NodeDetailLineage.tsx
+++ b/products/data_modeling/frontend/nodeDetail/NodeDetailLineage.tsx
@@ -10,7 +10,9 @@ import { urls } from 'scenes/urls'
import { DataModelingJobStatus, DataModelingNode } from '~/types'
import { LineageGraph } from '../lineage/LineageGraph'
+import { LineageGraphNode, nodesWithHubs } from '../lineage/lineageHiddenNodes'
import { lineageNodeUrl } from '../lineage/lineageNodeUrl'
+import { LineageTypeFilter } from '../lineage/LineageTypeFilter'
import { nodeDetailSceneLogic } from './nodeDetailSceneLogic'
function nodeLineageUrl(node: DataModelingNode): string {
@@ -29,17 +31,27 @@ export function NodeDetailLineage({ id }: { id: string }): JSX.Element {
effectiveLastRunStatus,
lineageModalOpen,
lineageNodePositions,
+ lineageTypeFilter,
+ lineageVisibleNodeIds,
+ lineageHiddenNodeBridging,
node,
} = useValues(nodeDetailSceneLogic({ id }))
- const { openLineageModal, closeLineageModal, loadLineageGraph, lineageNodeDragStopped, resetLineageNodePositions } =
- useActions(nodeDetailSceneLogic({ id }))
+ const {
+ openLineageModal,
+ closeLineageModal,
+ loadLineageGraph,
+ lineageNodeDragStopped,
+ resetLineageNodePositions,
+ setLineageTypeFilter,
+ } = useActions(nodeDetailSceneLogic({ id }))
// The current node's freshest status/run come from its materialization jobs, not the graph payload
- const nodes = useMemo((): DataModelingNode[] => {
+ const nodes = useMemo((): LineageGraphNode[] => {
if (!lineageGraph) {
return []
}
- return lineageGraph.nodes.map((node) =>
+ const hubNodeIds = lineageHiddenNodeBridging?.hubNodeIds ?? new Set<string>()
+ return nodesWithHubs(lineageGraph.nodes, lineageVisibleNodeIds, hubNodeIds).map((node) =>
node.id === lineageGraph.currentNodeId
? {
...node,
@@ -48,7 +60,8 @@ export function NodeDetailLineage({ id }: { id: string }): JSX.Element {
}
: node
)
- }, [lineageGraph, effectiveLastRunAt, effectiveLastRunStatus])
+ }, [lineageGraph, lineageVisibleNodeIds, lineageHiddenNodeBridging, effectiveLastRunAt, effectiveLastRunStatus])
+ const edges = lineageHiddenNodeBridging?.edges ?? []
if (!lineageGraphLoading && lineageGraphError) {
return (
@@ -58,7 +71,7 @@ export function NodeDetailLineage({ id }: { id: string }): JSX.Element {
)
}
- if (!lineageGraphLoading && nodes.length <= 1 && !nodes[0]?.lineage_issue) {
+ if (!lineageGraphLoading && (lineageGraph?.nodes.length ?? 0) <= 1 && !nodes[0]?.lineage_issue) {
return (
<div className="flex min-h-64 flex-col items-center justify-center gap-3 rounded border bg-bg-light p-6 text-center">
<div className="max-w-120">
@@ -76,11 +89,16 @@ export function NodeDetailLineage({ id }: { id: string }): JSX.Element {
}
return (
- <>
+ <div className="flex flex-col items-start gap-2">
+ <LineageTypeFilter
+ value={lineageTypeFilter}
+ onChange={setLineageTypeFilter}
+ dataAttr="node-detail-lineage-type-filter"
+ />
<div className="h-[calc(100vh-20rem)] min-h-[400px] w-full border rounded bg-bg-light overflow-hidden">
<LineageGraph
nodes={nodes}
- edges={lineageGraph?.edges ?? []}
+ edges={edges}
currentNodeId={lineageGraph?.currentNodeId}
loading={lineageGraphLoading}
loadingCenter={lineageGraphLoading && node ? { name: node.name, type: node.type } : undefined}
@@ -124,7 +142,7 @@ export function NodeDetailLineage({ id }: { id: string }): JSX.Element {
<div className="h-[calc(100vh-12rem)]">
<LineageGraph
nodes={nodes}
- edges={lineageGraph?.edges ?? []}
+ edges={edges}
currentNodeId={lineageGraph?.currentNodeId}
variant="full"
interactive
@@ -137,6 +155,6 @@ export function NodeDetailLineage({ id }: { id: string }): JSX.Element {
/>
</div>
</LemonModal>
- </>
+ </div>
)
}
diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts
index f938536e51a2..fb4146e2539c 100644
--- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts
+++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts
@@ -185,6 +185,22 @@ describe('nodeDetailSceneLogic', () => {
expect(logic.values.currentTab).toEqual('lineage')
})
+ it('keeps the current model in its lineage when its type is hidden', async () => {
+ await mountScene(urls.nodeDetail(NODE_ID, 'lineage'))
+ logic.actions.loadLineageGraphSuccess({
+ currentNodeId: NODE_ID,
+ nodes: [buildNode('table', { id: 'source' }), node, buildNode('metric', { id: 'revenue' })],
+ edges: [
+ { id: 'e1', source_id: 'source', target_id: NODE_ID },
+ { id: 'e2', source_id: NODE_ID, target_id: 'revenue' },
+ ],
+ })
+
+ logic.actions.setLineageTypeFilter(['table'])
+
+ expect(logic.values.lineageVisibleNodeIds).toEqual(new Set(['source', NODE_ID]))
+ })
+
it('replaces a backing-resource URL with the canonical node URL', async () => {
await mountScene(
combineUrl(urls.nodeDetail(SAVED_QUERY_ID, 'lineage'), { lookup: 'saved_query' }).url,
diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts
index 0bb703d7ba2c..f479d01717fe 100644
--- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts
+++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts
@@ -2,6 +2,7 @@ import type { XYPosition } from '@xyflow/react'
import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'
import { loaders } from 'kea-loaders'
import { router, urlToAction } from 'kea-router'
+import posthog from 'posthog-js'
import api from 'lib/api'
import { userHasAccess } from 'lib/utils/accessControlUtils'
@@ -19,6 +20,7 @@ import {
Breadcrumb,
DataModelingEdge,
DataModelingNode,
+ DataModelingNodeType,
DataWarehouseSavedQuery,
DataWarehouseTable,
ExternalDataSchemaWithSource,
@@ -31,6 +33,8 @@ import type { DataQualitySubjectApi } from 'products/data_quality/frontend/gener
import { MATERIALIZING_TYPES } from '../freshness'
import { dataModelingNodesRetrieve } from '../generated/api'
import { DataModelingNodesRetrieveLookup, type NodeTypeEnumApi } from '../generated/api.schemas'
+import { HiddenNodeBridging, bridgeHiddenNodes } from '../lineage/lineageHiddenNodes'
+import { LINEAGE_FILTER_TYPES, LINEAGE_TYPE_FILTER_CHANGED_EVENT } from '../lineage/modelsLineageLogic'
export const NODE_DETAIL_SCENE_TABS = ['query', 'lineage', 'materialization', 'data-quality', 'history'] as const
export type NodeDetailSceneTab = (typeof NODE_DETAIL_SCENE_TABS)[number]
@@ -98,8 +102,11 @@ export interface nodeDetailSceneLogicValues {
lineageGraph: LineageGraphData | null
lineageGraphError: boolean
lineageGraphLoading: boolean
+ lineageHiddenNodeBridging: HiddenNodeBridging | null
lineageModalOpen: boolean
lineageNodePositions: Record<string, XYPosition>
+ lineageTypeFilter: NodeTypeEnumApi[]
+ lineageVisibleNodeIds: Set<string>
node: DataModelingNode | null
nodeLoading: boolean
nodeType: NodeTypeEnumApi | null
@@ -258,6 +265,9 @@ export interface nodeDetailSceneLogicActions {
setCurrentTab: (tab: NodeDetailSceneTab | null) => {
tab: 'data-quality' | 'history' | 'lineage' | 'materialization' | 'query' | null
}
+ setLineageTypeFilter: (typeFilter: DataModelingNodeType[]) => {
+ typeFilter: NodeTypeEnumApi[]
+ }
setQueryEditorState: (state: NodeDetailQueryEditorState | null) => {
state: NodeDetailQueryEditorState | null
}
@@ -312,6 +322,14 @@ export interface nodeDetailSceneLogicMeta {
savedQuery: DataWarehouseSavedQuery | null
) => string | null
canEditQuery: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => boolean
+ lineageVisibleNodeIds: (
+ lineageGraph: LineageGraphData | null,
+ lineageTypeFilter: NodeTypeEnumApi[]
+ ) => Set<string>
+ lineageHiddenNodeBridging: (
+ lineageGraph: LineageGraphData | null,
+ lineageVisibleNodeIds: Set<string>
+ ) => HiddenNodeBridging | null
}
}
@@ -344,6 +362,7 @@ export const nodeDetailSceneLogic = kea<nodeDetailSceneLogicType>([
closeLineageModal: true,
lineageNodeDragStopped: (nodeId: string, position: XYPosition) => ({ nodeId, position }),
resetLineageNodePositions: true,
+ setLineageTypeFilter: (typeFilter: DataModelingNodeType[]) => ({ typeFilter }),
markSavedQueryDeleted: true,
}),
reducers({
@@ -445,6 +464,12 @@ export const nodeDetailSceneLogic = kea<nodeDetailSceneLogicType>([
resetLineageNodePositions: () => ({}),
},
],
+ lineageTypeFilter: [
+ LINEAGE_FILTER_TYPES,
+ {
+ setLineageTypeFilter: (_, { typeFilter }) => typeFilter,
+ },
+ ],
}),
loaders(({ actions, props, values }) => ({
node: {
@@ -688,6 +713,28 @@ export const nodeDetailSceneLogic = kea<nodeDetailSceneLogicType>([
savedQuery.user_access_level
),
],
+ lineageVisibleNodeIds: [
+ (s) => [s.lineageGraph, s.lineageTypeFilter],
+ (lineageGraph: LineageGraphData | null, lineageTypeFilter: DataModelingNodeType[]): Set<string> =>
+ new Set(
+ (lineageGraph?.nodes ?? [])
+ .filter(
+ (node) => node.id === lineageGraph?.currentNodeId || lineageTypeFilter.includes(node.type)
+ )
+ .map((node) => node.id)
+ ),
+ ],
+ lineageHiddenNodeBridging: [
+ (s) => [s.lineageGraph, s.lineageVisibleNodeIds],
+ (lineageGraph: LineageGraphData | null, lineageVisibleNodeIds: Set<string>): HiddenNodeBridging | null =>
+ lineageGraph
+ ? bridgeHiddenNodes(
+ lineageGraph.edges,
+ new Set(lineageGraph.nodes.map((node) => node.id)),
+ lineageVisibleNodeIds
+ )
+ : null,
+ ],
}),
listeners(({ actions, props, values }) => {
const embeddedQueryEditor = (): ReturnType<typeof sqlEditorLogic.findMounted> =>
@@ -698,6 +745,13 @@ export const nodeDetailSceneLogic = kea<nodeDetailSceneLogicType>([
})
: null
return {
+ setLineageTypeFilter: ({ typeFilter }) => {
+ posthog.capture(LINEAGE_TYPE_FILTER_CHANGED_EVENT, {
+ surface: 'node_detail',
+ visible_types: typeFilter,
+ hidden_type_count: LINEAGE_FILTER_TYPES.length - typeFilter.length,
+ })
+ },
loadNodeSuccess: () => {
const node = values.node
if (node && node.id !== props.id) {