Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): show linked self-managed tables only under their managed source (#125654)

Tom Owers783a1f44a834
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index de722dbafddf..acb0c61466e0 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6290,6 +6290,8 @@ export interface DataWarehouseTable {
credential: DataWarehouseCredential | null
external_data_source?: ExternalDataSource
external_schema?: SimpleExternalDataSourceSchema
+ /** The managed source that will import this self-managed table on its first sync, if one is linked. */
+ pending_source_id?: string | null
options?: { csv_allow_double_quotes?: boolean | null }
user_access_level?: AccessControlLevel
}
@@ -6700,6 +6702,8 @@ export type IncrementalSyncBlockedReason = IncrementalSyncBlockedReasonEnumApi
export interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema {
table?: SimpleDataWarehouseTable
+ /** The schema is linked to a self-managed table that its first sync will import. */
+ pending_self_managed_table?: boolean
incremental: boolean
sync_type: 'incremental' | 'full_refresh' | 'append' | 'webhook' | 'cdc' | 'xmin' | null
sync_time_of_day: string | null
diff --git a/products/data_warehouse/backend/presentation/views/table.py b/products/data_warehouse/backend/presentation/views/table.py
index 4335cea8453c..d49be2ae1f09 100644
--- a/products/data_warehouse/backend/presentation/views/table.py
+++ b/products/data_warehouse/backend/presentation/views/table.py
@@ -5,6 +5,7 @@ from urllib.parse import urlparse
from django.conf import settings
from django.db.models import QuerySet
+from django.db.models.manager import BaseManager
import boto3
import posthoganalytics
@@ -54,6 +55,7 @@ from products.warehouse_sources.backend.facade.api import (
build_file_upload_url_pattern,
denied_table_ids,
hosted_upload_s3_path,
+ pending_source_ids_for_tables,
)
from products.warehouse_sources.backend.facade.contracts import (
FileUploadSourceError,
@@ -279,6 +281,22 @@ class CredentialSerializer(serializers.ModelSerializer):
}
+class TableListSerializer(serializers.ListSerializer):
+ """Resolves the pending source of every table in the page with one query."""
+
+ def to_representation(self, data):
+ tables = list(data.all() if isinstance(data, BaseManager) else data)
+ unsourced = [table for table in tables if table.external_data_source_id is None]
+ if unsourced:
+ team_ids = {table.team_id for table in unsourced}
+ cache = self.context.setdefault("pending_source_ids", {})
+ for team_id in team_ids:
+ ids = [table.id for table in unsourced if table.team_id == team_id]
+ found = pending_source_ids_for_tables(team_id, ids)
+ cache.update({table_id: found.get(table_id) for table_id in ids})
+ return super().to_representation(tables)
+
+
class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSerializer):
created_by = UserBasicSerializer(read_only=True)
credential = CredentialSerializer()
@@ -290,6 +308,10 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSeriali
)
external_data_source = SimpleExternalDataSourceSerializers(read_only=True)
external_schema = serializers.SerializerMethodField(read_only=True)
+ pending_source_id = serializers.SerializerMethodField(
+ read_only=True,
+ help_text="The managed source that will import this self-managed table on its first sync, if one is linked.",
+ )
options = serializers.DictField(
required=False,
default=dict,
@@ -326,9 +348,11 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSeriali
"columns",
"external_data_source",
"external_schema",
+ "pending_source_id",
"options",
"user_access_level",
]
+ list_serializer_class = TableListSerializer
read_only_fields = [
"id",
"created_by",
@@ -338,6 +362,7 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSeriali
"columns",
"external_data_source",
"external_schema",
+ "pending_source_id",
"user_access_level",
]
extra_kwargs = {
@@ -402,6 +427,15 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSeriali
for field in serializes_fields
]
+ @extend_schema_field(serializers.UUIDField(allow_null=True))
+ def get_pending_source_id(self, table: DataWarehouseTable) -> uuid.UUID | None:
+ if table.external_data_source_id is not None:
+ return None
+ cache = self.context.get("pending_source_ids")
+ if cache is not None and table.id in cache:
+ return cache[table.id]
+ return pending_source_ids_for_tables(table.team_id, [table.id]).get(table.id)
+
@extend_schema_field(serializers.DictField(allow_null=True))
def get_external_schema(self, instance: DataWarehouseTable):
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
@@ -618,7 +652,13 @@ class CreateTableFromUploadSerializer(UploadedFileSerializer):
bool,
description="Include table columns. Disable for lightweight table existence or name lookups.",
default=True,
- )
+ ),
+ OpenApiParameter(
+ "pending_source",
+ bool,
+ description="Only return self-managed tables that a managed source will import on its first sync.",
+ default=False,
+ ),
]
)
)
@@ -646,13 +686,18 @@ class TableViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viewsets.M
return context
def safely_get_queryset(self, queryset):
- return (
+ queryset = (
queryset.filter(team_id=self.team_id)
.exclude(deleted=True)
.exclude(external_data_source__access_method=ExternalDataSourceAccessMethod.DIRECT)
.prefetch_related("created_by", "externaldataschema_set")
.order_by(self.ordering)
)
+ if self.action == "list" and self.request.query_params.get("pending_source", "").lower() == "true":
+ queryset = queryset.filter(id__in=list(pending_source_ids_for_tables(self.team_id))).select_related(
+ "credential"
+ )
+ return queryset
def _filter_queryset_by_access_level(self, queryset: QuerySet) -> QuerySet:
queryset = super()._filter_queryset_by_access_level(queryset)
diff --git a/products/data_warehouse/backend/tests/api/test_table.py b/products/data_warehouse/backend/tests/api/test_table.py
index 116bd56617b1..451a8ef02f5f 100644
--- a/products/data_warehouse/backend/tests/api/test_table.py
+++ b/products/data_warehouse/backend/tests/api/test_table.py
@@ -6,7 +6,9 @@ from unittest.mock import ANY, MagicMock, Mock, patch
from django.conf import settings
from django.core.exceptions import ValidationError as DjangoValidationError
+from django.db import connection
from django.test import SimpleTestCase, override_settings
+from django.test.utils import CaptureQueriesContext
import boto3
from clickhouse_driver.errors import ServerException
@@ -14,6 +16,7 @@ from parameterized import parameterized
from rest_framework.exceptions import ValidationError
from rest_framework.test import APIRequestFactory
+from posthog.models import Team
from posthog.models.activity_logging.activity_log import ActivityLog
from products.data_warehouse.backend.direct_postgres import DIRECT_POSTGRES_URL_PATTERN
@@ -26,6 +29,7 @@ from products.data_warehouse.backend.presentation.views.table import (
from products.warehouse_sources.backend.facade.models import (
DataWarehouseCredential,
DataWarehouseTable,
+ ExternalDataSchema,
ExternalDataSource,
)
@@ -641,6 +645,144 @@ class TestTable(APIBaseTest):
assert response.json()["count"] == 1
assert response.json()["results"][0]["id"] == str(table.id)
+ def _pinned_table(self, name: str = "orders") -> tuple[DataWarehouseTable, ExternalDataSchema]:
+ table = DataWarehouseTable.objects.create(
+ name=name,
+ format="Parquet",
+ team=self.team,
+ team_id=self.team.pk,
+ url_pattern=f"https://example.com/{name}/*.parquet",
+ )
+ source = ExternalDataSource.objects.create(
+ team=self.team,
+ team_id=self.team.pk,
+ source_type="AwsS3",
+ access_method=ExternalDataSource.AccessMethod.WAREHOUSE,
+ )
+ schema = ExternalDataSchema.objects.create(
+ name=name,
+ team=self.team,
+ source=source,
+ sync_type_config={
+ "schema_metadata": {
+ "self_managed_table": {
+ "table_id": str(table.id),
+ "url_pattern": table.url_pattern,
+ "format": "Parquet",
+ }
+ }
+ },
+ )
+ return table, schema
+
+ @parameterized.expand(
+ [
+ ("pinned_by_live_schema", lambda t, s, other: None, True),
+ (
+ "schema_deleted",
+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(deleted=True),
+ False,
+ ),
+ (
+ "source_deleted",
+ lambda t, s, other: ExternalDataSource.objects.filter(pk=s.source_id).update(deleted=True),
+ False,
+ ),
+ (
+ "table_already_has_source",
+ lambda t, s, other: DataWarehouseTable.objects.filter(pk=t.pk).update(external_data_source=s.source),
+ False,
+ ),
+ (
+ "pin_abandoned",
+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(
+ sync_type_config={
+ "schema_metadata": {
+ "self_managed_table": {
+ "table_id": str(t.id),
+ "url_pattern": t.url_pattern,
+ "format": "Parquet",
+ "abandon_reason": "gone",
+ }
+ }
+ }
+ ),
+ False,
+ ),
+ (
+ "pin_in_other_team",
+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(team_id=other.pk),
+ False,
+ ),
+ (
+ "table_url_edited_since_migration",
+ lambda t, s, other: DataWarehouseTable.objects.filter(pk=t.pk).update(
+ url_pattern="https://example.com/elsewhere/*.parquet"
+ ),
+ False,
+ ),
+ (
+ "table_format_edited_since_migration",
+ lambda t, s, other: DataWarehouseTable.objects.filter(pk=t.pk).update(format="CSV"),
+ False,
+ ),
+ (
+ "schema_linked_to_replacement_table",
+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(
+ table=DataWarehouseTable.objects.create(
+ name="replacement",
+ format="Parquet",
+ team_id=t.team_id,
+ url_pattern="https://example.com/replacement/*.parquet",
+ external_data_source_id=s.source_id,
+ )
+ ),
+ False,
+ ),
+ ]
+ )
+ def test_pending_source_id(self, _name, mutate, expect_pending):
+ table, schema = self._pinned_table()
+ unrelated = DataWarehouseTable.objects.create(
+ name="unrelated",
+ format="Parquet",
+ team=self.team,
+ team_id=self.team.pk,
+ url_pattern="https://example.com/unrelated/*.parquet",
+ )
+ other_team = Team.objects.create(organization=self.organization, name="Other")
+ mutate(table, schema, other_team)
+
+ results = {
+ row["id"]: row
+ for row in self.client.get(f"/api/projects/{self.team.pk}/warehouse_tables/?include_columns=false").json()[
+ "results"
+ ]
+ }
+ expected = str(schema.source_id) if expect_pending else None
+ assert results[str(table.id)]["pending_source_id"] == expected
+ assert results[str(unrelated.id)]["pending_source_id"] is None
+
+ detail = self.client.get(f"/api/projects/{self.team.pk}/warehouse_tables/{table.id}/").json()
+ assert detail["pending_source_id"] == expected
+
+ filtered = self.client.get(
+ f"/api/projects/{self.team.pk}/warehouse_tables/?include_columns=false&pending_source=true"
+ ).json()["results"]
+ assert [row["id"] for row in filtered] == ([str(table.id)] if expect_pending else [])
+
+ def test_pending_source_id_is_resolved_once_per_list(self):
+ self._pinned_table("one")
+ url = f"/api/projects/{self.team.pk}/warehouse_tables/?include_columns=false"
+ for name in ("two", "three", "four"):
+ self._pinned_table(name)
+ with CaptureQueriesContext(connection) as several:
+ response = self.client.get(url)
+
+ assert [row["pending_source_id"] is not None for row in response.json()["results"]] == [True] * 4
+ pending_lookups = [q["sql"] for q in several.captured_queries if "self_managed_table" in q["sql"]]
+ assert len(pending_lookups) == 1
+
def test_list_tables_exposes_hogql_name_and_honors_include_columns(self):
source = ExternalDataSource.objects.create(
team=self.team,
diff --git a/products/data_warehouse/frontend/generated/api.schemas.ts b/products/data_warehouse/frontend/generated/api.schemas.ts
index 3f6f73ef7ca5..45068ef93678 100644
--- a/products/data_warehouse/frontend/generated/api.schemas.ts
+++ b/products/data_warehouse/frontend/generated/api.schemas.ts
@@ -5334,6 +5334,11 @@ export interface TableApi {
readonly external_data_source: SimpleExternalDataSourceSerializersApi
/** @nullable */
readonly external_schema: TableApiExternalSchema
+ /**
+ * The managed source that will import this self-managed table on its first sync, if one is linked.
+ * @nullable
+ */
+ readonly pending_source_id: string | null
/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */
options?: TableApiOptions
/**
@@ -5413,6 +5418,11 @@ export interface PatchedTableApi {
readonly external_data_source?: SimpleExternalDataSourceSerializersApi
/** @nullable */
readonly external_schema?: PatchedTableApiExternalSchema
+ /**
+ * The managed source that will import this self-managed table on its first sync, if one is linked.
+ * @nullable
+ */
+ readonly pending_source_id?: string | null
/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */
options?: PatchedTableApiOptions
/**
@@ -6052,6 +6062,10 @@ export type WarehouseTablesListParams = {
* The initial index from which to return the results.
*/
offset?: number
+ /**
+ * Only return self-managed tables that a managed source will import on its first sync.
+ */
+ pending_source?: boolean
/**
* A search term.
*/
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
index ff378ea5b529..4423fa0f4d10 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
@@ -85,6 +85,9 @@ const schemaEditDisabledReason = (schema: ExternalDataSourceSchema): string | nu
const dimWhenNotSyncing = (_: unknown, schema: ExternalDataSourceSchema): string =>
schema.should_sync ? '' : 'opacity-60'
+const PENDING_SELF_MANAGED_TABLE_NOTE =
+ 'This table reads your bucket until you turn on syncing. The first sync imports it into PostHog.'
+
type SchemaColumn = LemonTableColumn<ExternalDataSourceSchema, keyof ExternalDataSourceSchema | undefined>
// A file upload schema has no schedule or sync method to choose, and never syncs on its own, so it isn't dimmed.
@@ -549,6 +552,9 @@ function ManagedSchemaTable({
}
description={((): JSX.Element | undefined => {
const tableName = schema.table?.hogql_name ?? schema.table?.name
+ if (!tableName && schema.pending_self_managed_table && !schema.should_sync) {
+ return <span>{PENDING_SELF_MANAGED_TABLE_NOTE}</span>
+ }
return tableName ? (
<CopyToClipboardInline
explicitValue={tableName}
diff --git a/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsx b/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsx
index fb146879ae69..d5ed8aef4e00 100644
--- a/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsx
+++ b/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsx
@@ -9,7 +9,8 @@ import { sourceManagementLogic } from '../logics/sourceManagementLogic'
import { SourceIcon, mapUrlToProvider } from './SourceIcon'
export function SelfManagedSourcesTable(): JSX.Element {
- const { filteredSelfManagedTables, searchTerm, databaseLoading } = useValues(sourceManagementLogic)
+ const { filteredSelfManagedTables, searchTerm, databaseLoading, pendingSelfManagedTableIdsLoading } =
+ useValues(sourceManagementLogic)
const { deleteSelfManagedTable, refreshSelfManagedTableSchema, setSearchTerm } = useActions(sourceManagementLogic)
return (
@@ -19,8 +20,8 @@ export function SelfManagedSourcesTable(): JSX.Element {
</div>
<LemonTable
id="self-managed-sources"
- dataSource={filteredSelfManagedTables}
- loading={databaseLoading}
+ dataSource={pendingSelfManagedTableIdsLoading ? [] : filteredSelfManagedTables}
+ loading={databaseLoading || pendingSelfManagedTableIdsLoading}
pagination={{ pageSize: 10 }}
scrollToTopOnPageChange={false}
columns={[
diff --git a/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts b/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts
index 656650a70e71..776777c9bee9 100644
--- a/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts
+++ b/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts
@@ -14,7 +14,7 @@ import { loaders } from 'kea-loaders'
import { router, urlToAction } from 'kea-router'
import posthog from 'posthog-js'
-import api from 'lib/api'
+import api, { ApiConfig } from 'lib/api'
import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
import { createFuse, Fuse } from 'lib/utils/fuseSearch'
import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'
@@ -28,6 +28,8 @@ import {
ExternalDataSourceSchema,
} from '~/types'
+import { warehouseTablesList } from 'products/data_warehouse/frontend/generated/api'
+
import type { PaginatedResponse } from '../../../../../frontend/src/lib/api'
import type { DatabaseSchemaQueryResponse } from '../../../../../frontend/src/queries/schema/schema-general'
import type { ExternalDataSourceRevenueAnalyticsConfig } from '../../../../../frontend/src/types'
@@ -81,6 +83,8 @@ export interface sourceManagementLogicValues {
managedSearchTerm: string
managedSources: ExternalDataSource[]
managedSourcesFuse: Fuse<ExternalDataSource>
+ pendingSelfManagedTableIds: string[]
+ pendingSelfManagedTableIdsLoading: boolean
schemaReloadingById: Record<string, boolean>
schemas: null
schemasLoading: boolean
@@ -160,6 +164,21 @@ export interface sourceManagementLogicActions {
deleteSource: (source: ExternalDataSource) => {
source: ExternalDataSource
}
+ loadPendingSelfManagedTableIds: () => any
+ loadPendingSelfManagedTableIdsFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadPendingSelfManagedTableIdsSuccess: (
+ pendingSelfManagedTableIds: string[],
+ payload?: any
+ ) => {
+ pendingSelfManagedTableIds: string[]
+ payload?: any
+ }
refreshSelfManagedTableSchema: (tableId: string) => {
tableId: string
}
@@ -205,7 +224,8 @@ export interface sourceManagementLogicActions {
export interface sourceManagementLogicMeta {
__keaTypeGenInternalSelectorTypes: {
selfManagedTables: (
- dataWarehouseTables: DatabaseSchemaDataWarehouseTable[]
+ dataWarehouseTables: DatabaseSchemaDataWarehouseTable[],
+ pendingSelfManagedTableIds: string[]
) => DatabaseSchemaDataWarehouseTable[]
filteredSelfManagedTables: (
selfManagedTables: DatabaseSchemaDataWarehouseTable[],
@@ -273,6 +293,27 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([
setDirectSearchTerm: (directSearchTerm: string) => ({ directSearchTerm }),
}),
loaders(({ actions, values }) => ({
+ pendingSelfManagedTableIds: [
+ [] as string[],
+ {
+ loadPendingSelfManagedTableIds: async () => {
+ const ids: string[] = []
+ const limit = 1000
+ for (let offset = 0; ; offset += limit) {
+ const response = await warehouseTablesList(String(ApiConfig.getCurrentProjectId()), {
+ pending_source: true,
+ include_columns: false,
+ limit,
+ offset,
+ })
+ ids.push(...response.results.map((table) => table.id))
+ if (!response.next || response.results.length === 0) {
+ return ids
+ }
+ }
+ },
+ },
+ ],
schemas: [
null,
{
@@ -348,9 +389,14 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([
})),
selectors({
selfManagedTables: [
- (s) => [s.dataWarehouseTables],
- (dataWarehouseTables: DatabaseSchemaDataWarehouseTable[]): DatabaseSchemaDataWarehouseTable[] => {
- return dataWarehouseTables.filter((table) => !table.source)
+ (s) => [s.dataWarehouseTables, s.pendingSelfManagedTableIds],
+ (
+ dataWarehouseTables: DatabaseSchemaDataWarehouseTable[],
+ pendingSelfManagedTableIds: string[]
+ ): DatabaseSchemaDataWarehouseTable[] => {
+ // A table that a managed source will import on its first sync is listed under that source instead.
+ const pendingIds = new Set(pendingSelfManagedTableIds)
+ return dataWarehouseTables.filter((table) => !table.source && !pendingIds.has(table.id))
},
],
filteredSelfManagedTables: [
@@ -490,6 +536,7 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([
// Only on success: a transient list failure resolves to an empty page, so reloading
// after a delete that failed offline would blank a list of sources that still exist.
actions.loadSources()
+ actions.loadPendingSelfManagedTableIds()
posthog.capture('source deleted', { sourceType: source.source_type })
actions.sourceDeleted(source)
} catch (e) {
@@ -580,6 +627,7 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([
}
}
actions.loadSources()
+ actions.loadPendingSelfManagedTableIds()
}),
beforeUnmount(() => {
// Disposables plugin handles cleanup automatically
diff --git a/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts b/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts
index a97f18220b90..0f97e990f386 100644
--- a/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts
+++ b/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts
@@ -8,10 +8,15 @@ import { DatabaseSchemaQueryResponse } from '~/queries/schema/schema-general'
import { initKeaTests } from '~/test/init'
import { ExternalDataSource } from '~/types'
+import { warehouseTablesList } from 'products/data_warehouse/frontend/generated/api'
+
import { availableSourcesLogic } from '../../../scenes/NewSourceScene/availableSourcesLogic'
import { sourceManagementLogic } from '../sourceManagementLogic'
jest.mock('lib/api')
+jest.mock('products/data_warehouse/frontend/generated/api', () => ({
+ warehouseTablesList: jest.fn(),
+}))
jest.mock('~/queries/query')
describe('sourceManagementLogic', () => {
@@ -33,6 +38,13 @@ describe('sourceManagementLogic', () => {
results: [],
} as any)
+ ;(warehouseTablesList as jest.Mock).mockResolvedValue({
+ results: [],
+ count: 0,
+ next: null,
+ previous: null,
+ })
+
;(performQuery as jest.Mock).mockResolvedValue({
tables: {},
joins: [],
@@ -203,4 +215,40 @@ describe('sourceManagementLogic', () => {
resolveShallowLoad?.({ tables: {}, joins: [] } as DatabaseSchemaQueryResponse)
await shallowLoad
})
+
+ it.each([
+ ['no linked tables', [], ['orders', 'users', 'events']],
+ ['one linked table', ['t-orders'], ['users', 'events']],
+ ['every self-managed table linked', ['t-orders', 't-users'], ['events']],
+ ['id of an unknown table', ['t-missing'], ['orders', 'users', 'events']],
+ ])('hides tables a managed source will import from the self-managed list: %s', async (_, linkedIds, expected) => {
+ ;(warehouseTablesList as jest.Mock).mockResolvedValue({
+ results: linkedIds.map((id) => ({ id, pending_source_id: 's2' })),
+ })
+ databaseLogic.mount()
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadPendingSelfManagedTableIdsSuccess'])
+ databaseTableListLogic.actions.loadDatabaseSuccess({
+ tables: {
+ orders: { type: 'data_warehouse', id: 't-orders', name: 'orders', fields: {} },
+ users: { type: 'data_warehouse', id: 't-users', name: 'users', fields: {} },
+ // Linked tables are only hidden when they have no source of their own.
+ events: { type: 'data_warehouse', id: 't-events', name: 'events', fields: {} },
+ stripe_charges: {
+ type: 'data_warehouse',
+ id: 't-charges',
+ name: 'stripe_charges',
+ fields: {},
+ source: { id: 's1', status: 'Running', source_type: 'Stripe', prefix: '' },
+ },
+ },
+ joins: [],
+ } as unknown as DatabaseSchemaQueryResponse)
+
+ expect(warehouseTablesList).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({ pending_source: true })
+ )
+ expect(logic.values.selfManagedTables.map((table) => table.name)).toEqual(expected)
+ })
})
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 2212b56b8405..91fc3813119d 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10937,6 +10937,10 @@ export namespace Schemas {
latest_error: string | null;
incremental: boolean;
status: string | null;
+ /**
+ * Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket.
+ */
+ pending_self_managed_table: boolean;
/**
* A schema of an external data source: its sync configuration and the warehouse table it syncs into.
*/
diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.py
index 32e72aa4ae61..07e8d403abba 100644
--- a/products/warehouse_sources/backend/facade/api.py
+++ b/products/warehouse_sources/backend/facade/api.py
@@ -47,7 +47,11 @@ from products.warehouse_sources.backend.models.external_data_job import (
ExternalDataJob as _ExternalDataJob,
latest_completed_job_subquery,
)
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema as _ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_schema import (
+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,
+ ExternalDataSchema as _ExternalDataSchema,
+ self_managed_table_ref,
+)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource as _ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable as _DataWarehouseTable
@@ -93,6 +97,7 @@ __all__ = [
"record_table_row_count",
"allowed_table_ids",
"denied_table_ids",
+ "pending_source_ids_for_tables",
"all_queryable_tables",
"all_queryable_table_columns",
"list_tables_for_source",
@@ -530,6 +535,55 @@ def _resolve_allowed_table_ids(
return _allowed_tables(_tables_for_access_check(candidates), user_access_control, required_level)
+def pending_source_ids_for_tables(team_id: int, table_ids: Collection[UUID] | None = None) -> dict[UUID, UUID]:
+ """Map each self-managed table to the managed source that will import it on its first sync.
+
+ A table is pending when a live, still unlinked schema of a live source in the same team pins it,
+ and the table has no source yet and still reads the files the schema was pinned to. That matches
+ the conditions under which the first sync adopts the table rather than creating a new one.
+ `table_ids` of None checks every table in the team.
+ """
+ if table_ids is not None and not table_ids:
+ return {}
+ pinned = (
+ _ExternalDataSchema.objects.filter(team_id=team_id, source__team_id=team_id)
+ .exclude(deleted=True)
+ .exclude(source__deleted=True)
+ .filter(table__isnull=True)
+ .filter(sync_type_config__schema_metadata__has_key=SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY)
+ .order_by("created_at", "id")
+ )
+ if table_ids is not None:
+ pinned = pinned.filter(
+ sync_type_config__schema_metadata__self_managed_table__table_id__in=[str(i) for i in table_ids]
+ )
+ by_table: dict[UUID, list[tuple[UUID, str, str]]] = {}
+ for schema in pinned:
+ ref = self_managed_table_ref(schema)
+ if ref is None or ref.abandon_reason is not None or not UUIDT.is_valid_uuid(ref.table_id):
+ continue
+ by_table.setdefault(UUID(ref.table_id), []).append((schema.source_id, ref.url_pattern, ref.format))
+ if not by_table:
+ return {}
+ unsourced = _DataWarehouseTable.raw_objects.filter(
+ team_id=team_id, id__in=list(by_table), external_data_source__isnull=True
+ ).values_list("id", "url_pattern", "format")
+ found: dict[UUID, UUID] = {}
+ for table_id, url_pattern, table_format in unsourced:
+ # A table edited since the migration reads other files, so its sync creates a new table instead.
+ source_id = next(
+ (
+ sid
+ for sid, ref_url, ref_format in by_table[table_id]
+ if (ref_url, ref_format) == (url_pattern, table_format)
+ ),
+ None,
+ )
+ if source_id is not None:
+ found[table_id] = source_id
+ return found
+
+
def denied_table_ids(team_id: int, user_access_control: "UserAccessControl") -> frozenset[UUID]:
"""Every table this caller may not view.
diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.py
index 8dc1ba4d8d07..18950bd54785 100644
--- a/products/warehouse_sources/backend/facade/models.py
+++ b/products/warehouse_sources/backend/facade/models.py
@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
get_schemas_for_direct_reconciliation,
mark_schema_running_unless_halted,
resolve_sync_type,
+ self_managed_table_ref,
sync_frequency_interval_to_sync_frequency,
sync_frequency_to_sync_frequency_interval,
sync_old_schemas_with_new_schemas,
@@ -143,6 +144,7 @@ __all__ = [
"update_sync_type_config_keys",
"UnsupportedSyncTypeError",
"resolve_sync_type",
+ "self_managed_table_ref",
"CDC_SNAPSHOT_LANE_KEY",
"validate_source_prefix",
"validate_warehouse_table_url_pattern",
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 882f0bd4744b..7b1c56eabbea 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -57,6 +57,7 @@ from products.warehouse_sources.backend.facade.models import (
mark_schema_running_unless_halted,
resolve_destinations,
resolve_sync_type,
+ self_managed_table_ref,
sync_frequency_interval_to_sync_frequency,
sync_frequency_to_sync_frequency_interval,
update_sync_type_config_keys,
@@ -551,6 +552,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"""A schema of an external data source: its sync configuration and the warehouse table it syncs into."""
table = serializers.SerializerMethodField(read_only=True)
+ pending_self_managed_table = serializers.SerializerMethodField(
+ read_only=True,
+ help_text="Whether this schema is linked to an existing self-managed table that its first sync will import. "
+ "Until then the table keeps reading the original bucket.",
+ )
incremental = serializers.SerializerMethodField(read_only=True)
status = serializers.SerializerMethodField(read_only=True)
sync_type = serializers.ChoiceField(
@@ -737,6 +743,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"latest_error",
"incremental",
"status",
+ "pending_self_managed_table",
"sync_type",
"incremental_field",
"incremental_field_type",
@@ -770,6 +777,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"last_synced_at",
"latest_error",
"status",
+ "pending_self_managed_table",
"incremental_sync_blocked",
"next_full_refresh_at",
"description",
@@ -948,6 +956,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
def get_status(self, schema: ExternalDataSchema) -> str | None:
return schema_display_status(schema)
+ @extend_schema_field(serializers.BooleanField())
+ def get_pending_self_managed_table(self, schema: ExternalDataSchema) -> bool:
+ ref = self_managed_table_ref(schema)
+ return schema.table_id is None and ref is not None and ref.abandon_reason is None
+
def get_incremental(self, schema: ExternalDataSchema) -> bool:
return schema.is_incremental
@@ -2080,6 +2093,11 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):
read_only=True, help_text="The synced warehouse table (id, name, row_count), or null if not yet synced."
)
status = serializers.SerializerMethodField(read_only=True, help_text="Current sync status for this schema.")
+ pending_self_managed_table = serializers.SerializerMethodField(
+ read_only=True,
+ help_text="Whether this schema is linked to an existing self-managed table that its first sync will import. "
+ "Until then the table keeps reading the original bucket.",
+ )
sync_frequency = serializers.SerializerMethodField(
read_only=True, help_text="How often this table is scheduled to sync, or null if no interval is set."
@@ -2098,6 +2116,7 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):
"sync_frequency",
"latest_error",
"table",
+ "pending_self_managed_table",
]
read_only_fields = fields
@@ -2122,6 +2141,11 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):
def get_status(self, schema: ExternalDataSchema) -> str | None:
return schema_display_status(schema)
+ @extend_schema_field(serializers.BooleanField())
+ def get_pending_self_managed_table(self, schema: ExternalDataSchema) -> bool:
+ ref = self_managed_table_ref(schema)
+ return schema.table_id is None and ref is not None and ref.abandon_reason is None
+
@extend_schema_field(serializers.CharField(allow_null=True))
def get_sync_frequency(self, schema: ExternalDataSchema) -> str | None:
try:
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index 67bccb2a624c..e8811c418a53 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -5070,6 +5070,38 @@ class TestExternalDataSchemaRetrieveSource(APIBaseTest):
assert summary["supports_row_filters"] is expected_row_filters
assert "user_access_level" in summary
+ @parameterized.expand(
+ [
+ ("pinned_and_unsynced", True, False, True),
+ ("pinned_but_synced_table_set", True, True, False),
+ ("no_pin", False, False, False),
+ ]
+ )
+ def test_pending_self_managed_table(self, _name: str, pinned: bool, has_table: bool, expected: bool):
+ _source, schema = self._create()
+ if pinned:
+ schema.sync_type_config = {
+ "schema_metadata": {
+ "self_managed_table": {
+ "table_id": str(uuid.uuid4()),
+ "url_pattern": "https://example.com/a/*.parquet",
+ "format": "Parquet",
+ }
+ }
+ }
+ schema.save()
+ if has_table:
+ schema.table = DataWarehouseTable.objects.create(
+ name="orders", format="Parquet", team=self.team, url_pattern="https://example.com/a/*.parquet"
+ )
+ schema.save()
+
+ detail = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()
+ listed = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/").json()["results"]
+
+ assert detail["pending_self_managed_table"] is expected
+ assert [row["pending_self_managed_table"] for row in listed] == [expected]
+
def test_list_omits_source_summary(self):
self._create()
response = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/")
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index a8d2add0c3cd..20f7c4af410b 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -279,6 +279,7 @@ class TestExternalDataSource(APIBaseTest):
"sync_frequency",
"latest_error",
"table",
+ "pending_self_managed_table",
},
)
self.assertEqual(listed_schema["sync_frequency"], "6hour")
@@ -3392,6 +3393,7 @@ class TestExternalDataSource(APIBaseTest):
"latest_error": schema.latest_error,
"should_sync": schema.should_sync,
"status": schema.status,
+ "pending_self_managed_table": False,
"sync_type": schema.sync_type,
"table": schema.table,
"sync_frequency": sync_frequency_interval_to_sync_frequency(schema.sync_frequency_interval),
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index 7c64bc002ba6..3212b06594ee 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -430,6 +430,8 @@ export interface ExternalDataSchemaApi {
readonly incremental: boolean
/** @nullable */
readonly status: string | null
+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */
+ readonly pending_self_managed_table: boolean
/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.
*
* * `full_refresh` - full_refresh
@@ -625,6 +627,8 @@ export interface PatchedExternalDataSchemaApi {
readonly incremental?: boolean
/** @nullable */
readonly status?: string | null
+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */
+ readonly pending_self_managed_table?: boolean
/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.
*
* * `full_refresh` - full_refresh
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 3eb46bebd515..508a1bcacbba 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -51765,6 +51765,8 @@ export namespace Schemas {
readonly incremental: boolean;
/** @nullable */
readonly status: string | null;
+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */
+ readonly pending_self_managed_table: boolean;
/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.
*
* * `full_refresh` - full_refresh
@@ -82323,6 +82325,11 @@ export namespace Schemas {
readonly external_data_source: SimpleExternalDataSourceSerializers;
/** @nullable */
readonly external_schema: TableExternalSchema;
+ /**
+ * The managed source that will import this self-managed table on its first sync, if one is linked.
+ * @nullable
+ */
+ readonly pending_source_id: string | null;
/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */
options?: TableOptions;
/**
@@ -88340,6 +88347,8 @@ export namespace Schemas {
readonly incremental?: boolean;
/** @nullable */
readonly status?: string | null;
+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */
+ readonly pending_self_managed_table?: boolean;
/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.
*
* * `full_refresh` - full_refresh
@@ -94298,6 +94307,11 @@ export namespace Schemas {
readonly external_data_source?: SimpleExternalDataSourceSerializers;
/** @nullable */
readonly external_schema?: PatchedTableExternalSchema;
+ /**
+ * The managed source that will import this self-managed table on its first sync, if one is linked.
+ * @nullable
+ */
+ readonly pending_source_id?: string | null;
/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */
options?: PatchedTableOptions;
/**
@@ -144519,6 +144533,10 @@ export namespace Schemas {
* The initial index from which to return the results.
*/
offset?: number;
+ /**
+ * Only return self-managed tables that a managed source will import on its first sync.
+ */
+ pending_source?: boolean;
/**
* A search term.
*/