Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): sort and star sources in the sources table (#124095)

Chris Shinef15e5013af11
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 38753c326803..63c544190b10 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -11335,13 +11335,13 @@ snapshots:
scenes-app-errortracking-suppress-similar-issues-dialog--similar-messages--light:
hash: v1.k794b7964.8328c04966ba5a6b33fe31dede3262184b0cfa19ebb7f4e5c4bf6d052fd2a829.KKd1GJ0I_20JnUEE7ik1rNIhu_U2pREp86C629xT_fU
scenes-app-etl--healthy--dark:
- hash: v1.k794b7964.342346c35b32e21a02a09fe488480c9d100dbfda52c5b7c6b8dac3d2f83f3cf2.conExopWBDSppCq6BcJkzZ7MFg2JFCvBC0FF5QDc_xs
+ hash: v1.k794b7964.af47a2089a0e96523319000942dcf95855be7e7308011501bbfb31b622ae9e1a.-e6qEOIGiP6o9vjiFSABwNrwon1CD8l1RQSIlICCxX0
scenes-app-etl--healthy--light:
- hash: v1.k794b7964.88245f9f3d4dbc35485f0f2cad3b516dd603e94441cf19b9e19bda8fd4fe2de6.krGZaFW2zaFivovleByCTzMPyLN3PwIxzf4IuBeUPO0
+ hash: v1.k794b7964.0cb87db06e5e43d01959d5f94be7a9bacaad563cb20c92bf1904469024e83eb0.v-5SypA8vnzxOFthNrBFzBN-cpW31vrV73927kTKErc
scenes-app-etl--needs-attention--dark:
- hash: v1.k794b7964.d2e091fbea79cfa1fe3300241367e48b811b2400d49e880a24f4b02785776646.PTrR95S1l8scT-93IqNFp8x7EGVPCC_xlIRPKbLXbCo
+ hash: v1.k794b7964.defd9f632823f206ae1bfc974149f52f75209f628f258718e4528b07b52b6098.lB1GLRp9-JpBeKohvWGP21kt_ptmvAJw9bRf-vu-OJw
scenes-app-etl--needs-attention--light:
- hash: v1.k794b7964.2191373653f651b1c140b616c7cc6d9d97addf1f79dd3f663e7664f47040c538.yY9v-EpvhW9u0XUs2p7VyYxHmN8dA_QJm6D8zQikTkA
+ hash: v1.k794b7964.ac1d6223bebce3e5f589fbb186a144de1b755e8740af807b67de85e89190fe56.9btgAA5QBskCVRhcfc0okuxRcwp8XyEiYt-PvugkLuQ
scenes-app-events--event-explorer--dark:
hash: v1.k794b7964.fcae9c7e7645e82b24fbafc58f3c9f3292ccff3e4016b05be08120eecd4a724a.voNrg5FBYbpwdD1dxVnsQLtgFXc2GJ-wOgaW9uQvWGg
scenes-app-events--event-explorer--light:
diff --git a/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.test.ts b/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.test.ts
index 21eb365ebc7f..3f17f633ef4f 100644
--- a/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.test.ts
+++ b/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.test.ts
@@ -124,6 +124,24 @@ describe('projectTreeDataLogic', () => {
expect(logic.values.shortcutData).toEqual(updatedShortcuts)
})
+ it('loads shortcuts past the first page', async () => {
+ await expectLogic(logic).toFinishAllListeners()
+ const firstPage = [
+ { id: 'star-1', path: 'Notes', type: 'notebook', ref: 'notes' },
+ { id: 'star-2', path: 'Revenue', type: 'insight', ref: 'revenue' },
+ ]
+ const secondPage = [{ id: 'star-3', path: 'Stripe (stripe_)', type: 'data_source', ref: 'source-1' }]
+ jest.mocked(api.fileSystemShortcuts.list).mockImplementation(async (params) =>
+ params?.offset === firstPage.length
+ ? { count: 3, next: null, results: secondPage }
+ : { count: 3, next: 'next-page', results: firstPage }
+ )
+
+ await expectLogic(logic, () => logic.actions.loadShortcuts()).toFinishAllListeners()
+
+ expect(logic.values.shortcutData.map((shortcut) => shortcut.id)).toEqual(['star-1', 'star-2', 'star-3'])
+ })
+
it.each([
{ listedLocally: true, created: false },
{ listedLocally: false, created: false },
diff --git a/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.tsx b/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.tsx
index f7e3c699af12..1258564c6cf6 100644
--- a/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.tsx
+++ b/frontend/src/layout/panel-layout/ProjectTree/projectTreeDataLogic.tsx
@@ -1082,12 +1082,20 @@ export const projectTreeDataLogic = kea<projectTreeDataLogicType>([
// the loader settle: on timeout it rejects, firing loadShortcutsFailure (which
// flips shortcutDataHasLoaded), and kea-loaders' global onFailure handler in
// initKea captures the exception so the previously-silent hang is surfaced.
- const response = await withTimeout(
- (signal) => api.fileSystemShortcuts.list({ signal }),
- SHORTCUTS_LOADER_TIMEOUT_MS,
- 'loadShortcuts timed out'
- )
- return response.results.map(withProductShortcutHref)
+ // Each page gets its own timeout. A star past the first page would otherwise look unstarred.
+ const listPage = (offset: number): ReturnType<typeof api.fileSystemShortcuts.list> =>
+ withTimeout(
+ (signal) => api.fileSystemShortcuts.list(offset ? { offset, signal } : { signal }),
+ SHORTCUTS_LOADER_TIMEOUT_MS,
+ 'loadShortcuts timed out'
+ )
+ let page = await listPage(0)
+ const shortcuts = [...page.results]
+ while (page.next) {
+ page = await listPage(shortcuts.length)
+ shortcuts.push(...page.results)
+ }
+ return shortcuts.map(withProductShortcutHref)
},
addShortcutItem: async ({ item, successMessage }) => {
const shortcutItem = shortcutFromEntry(item)
diff --git a/frontend/src/layout/panel-layout/ProjectTree/utils.test.ts b/frontend/src/layout/panel-layout/ProjectTree/utils.test.ts
index 8db2313d1562..f6d271288c0a 100644
--- a/frontend/src/layout/panel-layout/ProjectTree/utils.test.ts
+++ b/frontend/src/layout/panel-layout/ProjectTree/utils.test.ts
@@ -10,6 +10,7 @@ import { productsItemName } from '../navbar/tabs/productsCatalog'
import { getCustomIcon } from './customIconRegistry'
import { getDefaultTreeData, getDefaultTreeProducts, iconForType } from './defaultTree'
import {
+ DATA_SOURCE_SHORTCUT_TYPE,
convertFileSystemEntryToTreeDataItem,
escapePath,
isFileSystemEntryTypeEnabled,
@@ -144,6 +145,21 @@ describe('project tree utils', () => {
folder: 'Research',
})
})
+
+ it.each([
+ ['shortcuts://My home', 'onto', null],
+ ['project://Research', 'onto', null],
+ [
+ 'shortcuts://My home',
+ 'before',
+ { type: 'reorder', activeId: 'shortcuts/source', overId: 'shortcuts://My home' },
+ ],
+ ] as const)('keeps a source star out of folder moves when dropped on %s (%s)', (overId, position, expected) => {
+ const sourceStar = { id: 'source', type: DATA_SOURCE_SHORTCUT_TYPE, ref: 'source-id', path: 'Stripe' }
+ expect(
+ resolveProjectTreeDrop('shortcuts/source', overId, [folder], [...shortcuts, sourceStar], position)
+ ).toEqual(expected)
+ })
})
describe('escapePath', () => {
diff --git a/frontend/src/layout/panel-layout/ProjectTree/utils.tsx b/frontend/src/layout/panel-layout/ProjectTree/utils.tsx
index 9d6fdba11735..fa191ba155cb 100644
--- a/frontend/src/layout/panel-layout/ProjectTree/utils.tsx
+++ b/frontend/src/layout/panel-layout/ProjectTree/utils.tsx
@@ -64,6 +64,9 @@ export function getItemId(item: FileSystemImport | FileSystemEntry, protocol = '
return item.type === 'folder' ? `${root}://${item.path}` : `${root}/${item.id || item.path}`
}
+// pinned: stored as each source star's type. Its source id `ref` keeps it apart from the ref-less Sources page star.
+export const DATA_SOURCE_SHORTCUT_TYPE = 'data_source'
+
type ProjectTreeDrop =
| { type: 'reorder'; activeId: string; overId: string }
| { type: 'star'; item: FileSystemEntry }
@@ -86,9 +89,11 @@ export function resolveProjectTreeDrop(
if (!item) {
return null
}
+ // A source star's ref is a source id, not a file-system entry, so there is no file to move.
+ const canMove = !activeShortcut || (!!activeShortcut.ref && activeShortcut.type !== DATA_SOURCE_SHORTCUT_TYPE)
if (overShortcut?.type === 'folder' && position === 'onto') {
// A shortcut's path is its label; ref points to the real folder, including unloaded home folders.
- return overShortcut.ref && (!activeShortcut || activeShortcut.ref)
+ return overShortcut.ref && canMove
? { type: activeShortcut ? 'move-shortcut' : 'move', item, folder: overShortcut.ref }
: null
}
@@ -97,9 +102,7 @@ export function resolveProjectTreeDrop(
? ''
: items.find((entry) => entry.type === 'folder' && getItemId(entry) === overId)?.path
if (folder !== undefined) {
- return !activeShortcut || activeShortcut.ref
- ? { type: activeShortcut ? 'move-shortcut' : 'move', item, folder }
- : null
+ return canMove ? { type: activeShortcut ? 'move-shortcut' : 'move', item, folder } : null
}
if (activeShortcut) {
return overShortcut ? { type: 'reorder', activeId, overId } : null
diff --git a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
index 1c0618e555fa..ea41d12bd6b6 100644
--- a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
+++ b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
@@ -1,7 +1,7 @@
import { useActions, useValues } from 'kea'
import { router } from 'kea-router'
-import { IconPlusSmall } from '@posthog/icons'
+import { IconPlusSmall, IconStar, IconStarFilled } from '@posthog/icons'
import { LemonButton, LemonDialog, LemonInput, LemonTable, LemonTag, Spinner, Tooltip } from '@posthog/lemon-ui'
import { AccessControlAction } from 'lib/components/AccessControlAction'
@@ -13,30 +13,25 @@ import { LemonTableLink } from 'lib/lemon-ui/LemonTable/LemonTableLink'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { urls } from 'scenes/urls'
-import { AccessControlLevel, AccessControlResourceType, ExternalDataSchemaStatus } from '~/types'
+import { AccessControlLevel, AccessControlResourceType } from '~/types'
import { StatusTagSetting } from 'products/data_warehouse/frontend/utils'
import { availableSourcesLogic } from '../../scenes/NewSourceScene/availableSourcesLogic'
import { isFileUploadSource, sourceTypeLabel } from '../fileUploadSources'
+import { managedSourcesTableLogic, syncingSchemasByStatus, totalRowsSynced } from '../logics/managedSourcesTableLogic'
import { sourceManagementLogic } from '../logics/sourceManagementLogic'
import { FreeHistoricalSyncsBanner } from './FreeHistoricalSyncsBanner'
import { DATA_WAREHOUSE_APP_SOURCE } from './metrics/DataWarehouseMetrics'
// eslint-disable-next-line import/no-cycle
import { SourceIcon } from './SourceIcon'
-const SCHEMA_STATUS_ORDER: ExternalDataSchemaStatus[] = [
- ExternalDataSchemaStatus.Failed,
- ExternalDataSchemaStatus.Paused,
- ExternalDataSchemaStatus.Cancelled,
- ExternalDataSchemaStatus.Running,
- ExternalDataSchemaStatus.Completed,
-]
-
export function ManagedSourcesTable(): JSX.Element {
- const { filteredManagedSources, dataWarehouseSourcesLoading, sourceReloadingById, managedSearchTerm } =
- useValues(sourceManagementLogic)
+ const { dataWarehouseSourcesLoading, sourceReloadingById, managedSearchTerm } = useValues(sourceManagementLogic)
const { deleteSource, reloadSource, setManagedSearchTerm } = useActions(sourceManagementLogic)
+ const { sortedManagedSources, managedSourcesSorting, starredSourceShortcuts, shortcutDataLoading } =
+ useValues(managedSourcesTableLogic)
+ const { setManagedSourcesSorting, toggleSourceStarred } = useActions(managedSourcesTableLogic)
const { availableSources, availableSourcesLoading } = useValues(availableSourcesLogic)
const { featureFlags } = useValues(featureFlagLogic)
const showMetrics = !!featureFlags[FEATURE_FLAGS.DWH_SOURCE_METRICS]
@@ -49,17 +44,22 @@ export function ManagedSourcesTable(): JSX.Element {
placeholder="Search..."
onChange={setManagedSearchTerm}
value={managedSearchTerm}
+ data-attr="managed-sources-search"
/>
</div>
<LemonTable
id="managed-sources"
// Labels and icons come from `availableSources`, so rows shown before it answers
// would render raw source types and then swap under the reader.
- dataSource={availableSourcesLoading ? [] : filteredManagedSources}
+ dataSource={availableSourcesLoading ? [] : sortedManagedSources}
loading={dataWarehouseSourcesLoading || availableSourcesLoading}
disableTableWhileLoading={false}
pagination={{ pageSize: 10 }}
scrollToTopOnPageChange={false}
+ // The logic sorts, because LemonTable reverses a column sorter and would sink starred sources.
+ sorting={managedSourcesSorting}
+ onSort={setManagedSourcesSorting}
+ useURLForSorting={false}
emptyState={
<div className="flex flex-col items-center gap-2 py-2">
<span>{managedSearchTerm ? 'No sources matching your search' : 'No managed sources'}</span>
@@ -77,11 +77,29 @@ export function ManagedSourcesTable(): JSX.Element {
columns={[
{
width: 0,
- render: (_, source) => <SourceIcon type={source.source_type} engine={source.engine} />,
+ className: '!pl-2',
+ render: (_, source) => {
+ const starred = !!starredSourceShortcuts[source.id]
+ return (
+ <div className="flex items-center gap-2">
+ <LemonButton
+ size="xsmall"
+ noPadding
+ icon={starred ? <IconStarFilled /> : <IconStar />}
+ tooltip={starred ? 'Remove from starred' : 'Add to starred'}
+ onClick={() => toggleSourceStarred(source)}
+ disabledReason={shortcutDataLoading ? 'Updating starred items' : undefined}
+ data-attr="managed-sources-star"
+ />
+ <SourceIcon type={source.source_type} engine={source.engine} />
+ </div>
+ )
+ },
},
{
title: 'Source',
key: 'name',
+ sorter: true,
render: (_, source) => (
<LemonTableLink
to={urls.dataWarehouseSource(`managed-${source.id}`)}
@@ -93,12 +111,14 @@ export function ManagedSourcesTable(): JSX.Element {
{
title: 'Table prefix',
key: 'prefix',
+ sorter: true,
render: (_, source) => source.prefix || '-',
},
{
title: 'Last Successful Run',
key: 'last_run_at',
tooltip: 'Time of the last run that completed a data import',
+ sorter: true,
render: (_, run) => {
return run.last_run_at ? (
<TZLabel time={run.last_run_at} formatDate="MMM DD, YYYY" formatTime="HH:mm" />
@@ -111,10 +131,8 @@ export function ManagedSourcesTable(): JSX.Element {
title: 'Total Rows Synced',
key: 'rows_synced',
tooltip: 'Total number of rows synced across all schemas in this source',
- render: (_, source) =>
- source.schemas
- .reduce((acc, schema) => acc + (schema.table?.row_count ?? 0), 0)
- .toLocaleString(),
+ sorter: true,
+ render: (_, source) => totalRowsSynced(source).toLocaleString(),
},
...(showMetrics
? [
@@ -145,18 +163,12 @@ export function ManagedSourcesTable(): JSX.Element {
{
title: 'Status',
key: 'status',
+ sorter: true,
render: (_, source) => {
if (!source.status) {
return null
}
- // A file upload schema never syncs on a schedule, so its imports still count.
- const syncingSchemas = isFileUploadSource(source)
- ? source.schemas
- : source.schemas.filter((s) => s.should_sync)
- const counts = SCHEMA_STATUS_ORDER.map((status) => ({
- status,
- schemas: syncingSchemas.filter((s) => s.status === status),
- })).filter(({ schemas }) => schemas.length > 0)
+ const counts = syncingSchemasByStatus(source)
if (counts.length === 0) {
// Source has schemas but none are enabled — source.status can be stale
diff --git a/products/data_warehouse/frontend/shared/logics/managedSourcesTableLogic.ts b/products/data_warehouse/frontend/shared/logics/managedSourcesTableLogic.ts
new file mode 100644
index 000000000000..7cf590375e76
--- /dev/null
+++ b/products/data_warehouse/frontend/shared/logics/managedSourcesTableLogic.ts
@@ -0,0 +1,258 @@
+import { MakeLogicType, actions, connect, kea, listeners, path, reducers, selectors } from 'kea'
+import posthog from 'posthog-js'
+
+import { dayjs } from 'lib/dayjs'
+import { Sorting } from 'lib/lemon-ui/LemonTable'
+import { urls } from 'scenes/urls'
+
+import { projectTreeDataLogic } from '~/layout/panel-layout/ProjectTree/projectTreeDataLogic'
+import { DATA_SOURCE_SHORTCUT_TYPE, escapePath } from '~/layout/panel-layout/ProjectTree/utils'
+import { FileSystemEntry } from '~/queries/schema/schema-general'
+import { ExternalDataJobStatus, ExternalDataSchemaStatus, ExternalDataSource, ExternalDataSourceSchema } from '~/types'
+
+import type { SourceConfigResponseApi } from '../../../../warehouse_sources/frontend/generated/api.schemas'
+import { isFileUploadSource, sourceTypeLabel } from '../fileUploadSources'
+import { sourceManagementLogic } from './sourceManagementLogic'
+
+const SCHEMA_STATUS_ORDER: ExternalDataSchemaStatus[] = [
+ ExternalDataSchemaStatus.Failed,
+ ExternalDataSchemaStatus.Paused,
+ ExternalDataSchemaStatus.Cancelled,
+ ExternalDataSchemaStatus.Running,
+ ExternalDataSchemaStatus.Completed,
+]
+
+// The backend names a source failure "Error". Billing limits block syncing, so they rank with failures.
+const SOURCE_STATUS_RANK: Record<string, number> = {
+ Error: 0,
+ [ExternalDataJobStatus.Failed]: 0,
+ [ExternalDataJobStatus.BillingLimits]: 0,
+ [ExternalDataJobStatus.BillingLimitTooLow]: 0,
+ [ExternalDataSchemaStatus.Paused]: SCHEMA_STATUS_ORDER.indexOf(ExternalDataSchemaStatus.Paused),
+ [ExternalDataSchemaStatus.Cancelled]: SCHEMA_STATUS_ORDER.indexOf(ExternalDataSchemaStatus.Cancelled),
+ [ExternalDataJobStatus.Running]: SCHEMA_STATUS_ORDER.indexOf(ExternalDataSchemaStatus.Running),
+ [ExternalDataJobStatus.Completed]: SCHEMA_STATUS_ORDER.indexOf(ExternalDataSchemaStatus.Completed),
+}
+
+export const totalRowsSynced = (source: ExternalDataSource): number =>
+ source.schemas.reduce((acc, schema) => acc + (schema.table?.row_count ?? 0), 0)
+
+export const syncingSchemasByStatus = (
+ source: ExternalDataSource
+): { status: ExternalDataSchemaStatus; schemas: ExternalDataSourceSchema[] }[] => {
+ // A file upload schema never syncs on a schedule, so its imports still count.
+ const syncingSchemas = isFileUploadSource(source)
+ ? source.schemas
+ : source.schemas.filter((schema) => schema.should_sync)
+ return SCHEMA_STATUS_ORDER.map((status) => ({
+ status,
+ schemas: syncingSchemas.filter((schema) => schema.status === status),
+ })).filter(({ schemas }) => schemas.length > 0)
+}
+
+// dayjs(undefined) is the current time, so a source that has never run needs its own value to sort as the oldest.
+const lastRunTime = (source: ExternalDataSource): number =>
+ source.last_run_at ? dayjs(source.last_run_at).valueOf() : 0
+
+const statusSortRank = (source: ExternalDataSource): number => {
+ if (!source.status) {
+ return SCHEMA_STATUS_ORDER.length
+ }
+ const [worst] = syncingSchemasByStatus(source)
+ if (worst) {
+ return SCHEMA_STATUS_ORDER.indexOf(worst.status)
+ }
+ if (source.schemas.length === 0) {
+ return SOURCE_STATUS_RANK[source.status] ?? SCHEMA_STATUS_ORDER.length
+ }
+ return SCHEMA_STATUS_ORDER.length
+}
+
+type SortKey = string | number
+
+const compareSortKeys = (a: SortKey[], b: SortKey[]): number => {
+ for (const [i, x] of a.entries()) {
+ const y = b[i]
+ const result = typeof x === 'number' && typeof y === 'number' ? x - y : String(x).localeCompare(String(y))
+ if (result !== 0) {
+ return result
+ }
+ }
+ return 0
+}
+
+// The selector computes these once per source, because the rows synced and status keys walk every schema.
+const MANAGED_SOURCE_SORT_KEYS: Record<
+ string,
+ (source: ExternalDataSource, availableSources: Record<string, SourceConfigResponseApi> | null) => SortKey[]
+> = {
+ name: (source, availableSources) => [sourceTypeLabel(source.source_type, availableSources), source.prefix ?? ''],
+ prefix: (source) => [source.prefix ?? ''],
+ last_run_at: (source) => [lastRunTime(source)],
+ rows_synced: (source) => [totalRowsSynced(source)],
+ status: (source) => [statusSortRank(source)],
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface managedSourcesTableLogicValues {
+ shortcutData: FileSystemEntry[] // projectTreeDataLogic
+ shortcutDataLoading: boolean // projectTreeDataLogic
+ availableSources: Record<string, SourceConfigResponseApi> | null // sourceManagementLogic
+ filteredManagedSources: ExternalDataSource[] // sourceManagementLogic
+ managedSearchTerm: string // sourceManagementLogic
+ managedSourcesSorting: Sorting | null
+ sortedManagedSources: ExternalDataSource[]
+ starredSourceShortcuts: Record<string, FileSystemEntry>
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface managedSourcesTableLogicActions {
+ addShortcutItem: (
+ item: FileSystemEntry,
+ successMessage?: string | undefined
+ ) => {
+ item: FileSystemEntry
+ successMessage: string | undefined
+ } // projectTreeDataLogic
+ deleteShortcut: (
+ id: string,
+ successMessage?: string | undefined
+ ) => {
+ id: string
+ successMessage: string | undefined
+ } // projectTreeDataLogic
+ sourceDeleted: (source: ExternalDataSource) => {
+ source: ExternalDataSource
+ } // sourceManagementLogic
+ setManagedSourcesSorting: (sorting: Sorting | null) => {
+ sorting: Sorting | null
+ }
+ toggleSourceStarred: (source: ExternalDataSource) => {
+ source: ExternalDataSource
+ }
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface managedSourcesTableLogicMeta {
+ __keaTypeGenInternalSelectorTypes: {
+ starredSourceShortcuts: (shortcutData: FileSystemEntry[]) => Record<string, FileSystemEntry>
+ sortedManagedSources: (
+ filteredManagedSources: ExternalDataSource[],
+ managedSourcesSorting: Sorting | null,
+ starredSourceShortcuts: Record<string, FileSystemEntry>,
+ managedSearchTerm: string,
+ availableSources: Record<string, SourceConfigResponseApi> | null
+ ) => ExternalDataSource[]
+ }
+}
+
+export type managedSourcesTableLogicType = MakeLogicType<
+ managedSourcesTableLogicValues,
+ managedSourcesTableLogicActions,
+ Record<string, any>,
+ managedSourcesTableLogicMeta
+>
+
+// Kept out of sourceManagementLogic, which also mounts on pages that should not load the sidebar's shortcuts.
+export const managedSourcesTableLogic = kea<managedSourcesTableLogicType>([
+ path(['products', 'dataWarehouse', 'managedSourcesTableLogic']),
+ connect(() => ({
+ values: [
+ sourceManagementLogic,
+ ['availableSources', 'filteredManagedSources', 'managedSearchTerm'],
+ projectTreeDataLogic,
+ ['shortcutData', 'shortcutDataLoading'],
+ ],
+ actions: [
+ sourceManagementLogic,
+ ['sourceDeleted'],
+ projectTreeDataLogic,
+ ['addShortcutItem', 'deleteShortcut'],
+ ],
+ })),
+ actions({
+ setManagedSourcesSorting: (sorting: Sorting | null) => ({ sorting }),
+ toggleSourceStarred: (source: ExternalDataSource) => ({ source }),
+ }),
+ reducers({
+ managedSourcesSorting: [
+ null as Sorting | null,
+ {
+ setManagedSourcesSorting: (_, { sorting }) => sorting,
+ },
+ ],
+ }),
+ selectors({
+ starredSourceShortcuts: [
+ (s) => [s.shortcutData],
+ (shortcutData: FileSystemEntry[]): Record<string, FileSystemEntry> =>
+ Object.fromEntries(
+ shortcutData
+ .filter((shortcut) => shortcut.type === DATA_SOURCE_SHORTCUT_TYPE && !!shortcut.ref)
+ .map((shortcut) => [shortcut.ref as string, shortcut])
+ ),
+ ],
+ sortedManagedSources: [
+ (s) => [
+ s.filteredManagedSources,
+ s.managedSourcesSorting,
+ s.starredSourceShortcuts,
+ s.managedSearchTerm,
+ s.availableSources,
+ ],
+ (
+ filteredManagedSources: ExternalDataSource[],
+ managedSourcesSorting: Sorting | null,
+ starredSourceShortcuts: Record<string, FileSystemEntry>,
+ managedSearchTerm: string,
+ availableSources: Record<string, SourceConfigResponseApi> | null
+ ): ExternalDataSource[] => {
+ const sortKey = managedSourcesSorting && MANAGED_SOURCE_SORT_KEYS[managedSourcesSorting.columnKey]
+ const order = managedSourcesSorting?.order ?? 1
+ const sorted = sortKey
+ ? filteredManagedSources
+ .map((source) => ({ source, key: sortKey(source, availableSources) }))
+ .sort((a, b) => order * compareSortKeys(a.key, b.key))
+ .map(({ source }) => source)
+ : filteredManagedSources
+ // While searching, starred sources do not float above better matches.
+ if (managedSearchTerm?.trim()) {
+ return sorted
+ }
+ return [
+ ...sorted.filter((source) => starredSourceShortcuts[source.id]),
+ ...sorted.filter((source) => !starredSourceShortcuts[source.id]),
+ ]
+ },
+ ],
+ }),
+ listeners(({ actions, values }) => ({
+ toggleSourceStarred: ({ source }) => {
+ const shortcut = values.starredSourceShortcuts[source.id]
+ if (shortcut) {
+ actions.deleteShortcut(shortcut.id)
+ return
+ }
+ const label = sourceTypeLabel(source.source_type, values.availableSources)
+ actions.addShortcutItem({
+ id: source.id,
+ path: escapePath(source.prefix ? `${label} (${source.prefix})` : label),
+ type: DATA_SOURCE_SHORTCUT_TYPE,
+ ref: source.id,
+ href: urls.dataWarehouseSource(`managed-${source.id}`),
+ })
+ },
+ // The backend removes stars only for file-system items, so a deleted source's star would stay behind.
+ sourceDeleted: ({ source }) => {
+ const shortcut = values.starredSourceShortcuts[source.id]
+ if (shortcut) {
+ actions.deleteShortcut(shortcut.id)
+ }
+ },
+ setManagedSourcesSorting: ({ sorting }) => {
+ if (sorting) {
+ posthog.capture('managed sources sorted', { columnKey: sorting.columnKey, order: sorting.order })
+ }
+ },
+ })),
+])
diff --git a/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts b/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts
index 39d4e3a760eb..656650a70e71 100644
--- a/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts
+++ b/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts
@@ -33,6 +33,7 @@ import type { DatabaseSchemaQueryResponse } from '../../../../../frontend/src/qu
import type { ExternalDataSourceRevenueAnalyticsConfig } from '../../../../../frontend/src/types'
import type { SourceConfigResponseApi } from '../../../../warehouse_sources/frontend/generated/api.schemas'
import { availableSourcesLogic } from '../../scenes/NewSourceScene/availableSourcesLogic'
+import { sourceTypeLabel } from '../fileUploadSources'
import { joinsDataLogic } from './joinsDataLogic'
import { sourcesDataLogic } from './sourcesDataLogic'
@@ -177,6 +178,9 @@ export interface sourceManagementLogicActions {
setSearchTerm: (searchTerm: string) => {
searchTerm: string
}
+ sourceDeleted: (source: ExternalDataSource) => {
+ source: ExternalDataSource
+ }
sourceLoadingFinished: (source: ExternalDataSource) => {
source: ExternalDataSource
}
@@ -258,6 +262,7 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([
})),
actions({
deleteSource: (source: ExternalDataSource) => ({ source }),
+ sourceDeleted: (source: ExternalDataSource) => ({ source }),
reloadSource: (source: ExternalDataSource) => ({ source }),
sourceLoadingFinished: (source: ExternalDataSource) => ({ source }),
schemaLoadingFinished: (schema: ExternalDataSourceSchema) => ({ schema }),
@@ -405,7 +410,7 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([
keys: [
{
name: 'label',
- getFn: (source) => availableSources?.[source.source_type]?.label ?? source.source_type,
+ getFn: (source) => sourceTypeLabel(source.source_type, availableSources),
},
'source_type',
'prefix',
@@ -486,6 +491,7 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([
// after a delete that failed offline would blank a list of sources that still exist.
actions.loadSources()
posthog.capture('source deleted', { sourceType: source.source_type })
+ actions.sourceDeleted(source)
} catch (e) {
lemonToast.error("We couldn't delete this source. Refresh the page and try again.")
posthog.captureException(e)
diff --git a/products/data_warehouse/frontend/shared/logics/tests/managedSourcesTableLogic.test.ts b/products/data_warehouse/frontend/shared/logics/tests/managedSourcesTableLogic.test.ts
new file mode 100644
index 000000000000..bbc1e3433424
--- /dev/null
+++ b/products/data_warehouse/frontend/shared/logics/tests/managedSourcesTableLogic.test.ts
@@ -0,0 +1,122 @@
+import { expectLogic } from 'kea-test-utils'
+
+import api, { PaginatedResponse } from 'lib/api'
+
+import { projectTreeDataLogic } from '~/layout/panel-layout/ProjectTree/projectTreeDataLogic'
+import { performQuery } from '~/queries/query'
+import { DatabaseSchemaQueryResponse, FileSystemEntry } from '~/queries/schema/schema-general'
+import { initKeaTests } from '~/test/init'
+import { ExternalDataSource } from '~/types'
+
+import { managedSourcesTableLogic } from '../managedSourcesTableLogic'
+import { sourceManagementLogic } from '../sourceManagementLogic'
+
+jest.mock('lib/api')
+jest.mock('~/queries/query')
+
+const SOURCES = [
+ {
+ id: 'recent',
+ source_type: 'Stripe',
+ prefix: 'b',
+ status: 'Completed',
+ last_run_at: '2026-01-02T00:00:00Z',
+ schemas: [{ id: 'schema-1', should_sync: true, status: 'Paused' }],
+ },
+ {
+ id: 'starred',
+ source_type: 'Stripe',
+ prefix: 'c',
+ status: 'Completed',
+ last_run_at: '2026-01-01T00:00:00Z',
+ schemas: [],
+ },
+ { id: 'never-run', source_type: 'Stripe', prefix: 'a', status: 'Error', schemas: [] },
+].map((source) => ({ ...source, access_method: 'warehouse' }) as unknown as ExternalDataSource)
+
+const source = (id: string): ExternalDataSource => SOURCES.find((s) => s.id === id) as ExternalDataSource
+
+describe('managedSourcesTableLogic', () => {
+ let logic: ReturnType<typeof managedSourcesTableLogic.build>
+
+ beforeEach(async () => {
+ initKeaTests()
+ jest.spyOn(api.externalDataSources, 'list').mockResolvedValue({
+ results: [],
+ count: 0,
+ next: null,
+ previous: null,
+ } as PaginatedResponse<ExternalDataSource>)
+ jest.spyOn(api.dataWarehouseViewLinks, 'list').mockResolvedValue({ results: [] } as any)
+ ;(performQuery as jest.Mock).mockResolvedValue({ tables: {}, joins: [] } as DatabaseSchemaQueryResponse)
+
+ logic = managedSourcesTableLogic()
+ logic.mount()
+ await expectLogic(sourceManagementLogic).toDispatchActions(['loadSourcesSuccess'])
+ await expectLogic(projectTreeDataLogic).toFinishAllListeners()
+ sourceManagementLogic.actions.loadSourcesSuccess({
+ results: SOURCES,
+ count: SOURCES.length,
+ next: null,
+ previous: null,
+ })
+ })
+
+ afterEach(() => {
+ logic.unmount()
+ })
+
+ const starSources = (ids: string[]): void => {
+ projectTreeDataLogic.actions.loadShortcutsSuccess(
+ ids.map((id) => ({ id: `shortcut-${id}`, path: id, type: 'data_source', ref: id }))
+ )
+ }
+
+ it.each([
+ ['last run, ascending', 'last_run_at', 1 as const, '', ['starred', 'never-run', 'recent']],
+ ['last run, descending', 'last_run_at', -1 as const, '', ['starred', 'recent', 'never-run']],
+ ['last run, while searching', 'last_run_at', 1 as const, 'stripe', ['never-run', 'starred', 'recent']],
+ ['status, with a backend error status', 'status', 1 as const, '', ['starred', 'never-run', 'recent']],
+ ])('orders managed sources by %s', (_, columnKey, order, searchTerm, expectedIds) => {
+ starSources(['starred'])
+
+ sourceManagementLogic.actions.setManagedSearchTerm(searchTerm)
+ logic.actions.setManagedSourcesSorting({ columnKey, order })
+
+ expect(logic.values.sortedManagedSources.map((s) => s.id)).toEqual(expectedIds)
+ })
+
+ it('stars and unstars a source through the sidebar shortcuts', async () => {
+ const createShortcut = jest.spyOn(api.fileSystemShortcuts, 'create').mockImplementation(async (data) => ({
+ shortcut: { id: 'shortcut-1', ...data } as FileSystemEntry,
+ created: true,
+ }))
+ const deleteShortcut = jest.spyOn(api.fileSystemShortcuts, 'delete').mockResolvedValue()
+ starSources([])
+
+ await expectLogic(logic, () => logic.actions.toggleSourceStarred(source('never-run'))).toDispatchActions(
+ projectTreeDataLogic,
+ ['addShortcutItemSuccess']
+ )
+ expect(createShortcut).toHaveBeenCalledTimes(1)
+ expect(logic.values.sortedManagedSources[0].id).toEqual('never-run')
+
+ await expectLogic(logic, () => logic.actions.toggleSourceStarred(source('never-run'))).toDispatchActions(
+ projectTreeDataLogic,
+ ['deleteShortcutSuccess']
+ )
+ expect(deleteShortcut).toHaveBeenCalledWith('shortcut-1')
+ expect(logic.values.sortedManagedSources[0].id).toEqual('recent')
+ })
+
+ it('removes the star of a deleted source', async () => {
+ jest.spyOn(api.externalDataSources, 'delete').mockResolvedValue()
+ const deleteShortcut = jest.spyOn(api.fileSystemShortcuts, 'delete').mockResolvedValue()
+ starSources(['starred'])
+
+ sourceManagementLogic.actions.deleteSource(source('starred'))
+ await expectLogic(projectTreeDataLogic).toDispatchActions(['deleteShortcutSuccess'])
+
+ expect(deleteShortcut).toHaveBeenCalledWith('shortcut-starred')
+ })
+})
diff --git a/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts b/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts
index 4d1cce45772a..a97f18220b90 100644
--- a/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts
+++ b/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts
@@ -174,7 +174,9 @@ describe('sourceManagementLogic', () => {
await expectLogic(logic).toDispatchActions(['loadSourcesSuccess'])
logic.actions.deleteSource(source)
- await expectLogic(logic).toDispatchActions(['sourceLoadingFinished'])
+ await expectLogic(logic)
+ .toDispatchActions(['sourceLoadingFinished'])
+ .toNotHaveDispatchedActions(['sourceDeleted'])
expect(logic.values.sourceReloadingById['source-1']).toBe(false)
// A transient list failure resolves to an empty page, so reloading after a failed delete