:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): show linked self-managed tables only under their managed source (#125654)
Tom Owers783a1f44a834
diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex de722dbafddf..acb0c61466e0 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -6290,6 +6290,8 @@ export interface DataWarehouseTable {credential: DataWarehouseCredential | nullexternal_data_source?: ExternalDataSourceexternal_schema?: SimpleExternalDataSourceSchema+ /** The managed source that will import this self-managed table on its first sync, if one is linked. */+ pending_source_id?: string | nulloptions?: { csv_allow_double_quotes?: boolean | null }user_access_level?: AccessControlLevel}@@ -6700,6 +6702,8 @@ export type IncrementalSyncBlockedReason = IncrementalSyncBlockedReasonEnumApiexport interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema {table?: SimpleDataWarehouseTable+ /** The schema is linked to a self-managed table that its first sync will import. */+ pending_self_managed_table?: booleanincremental: booleansync_type: 'incremental' | 'full_refresh' | 'append' | 'webhook' | 'cdc' | 'xmin' | nullsync_time_of_day: string | nulldiff --git a/products/data_warehouse/backend/presentation/views/table.py b/products/data_warehouse/backend/presentation/views/table.pyindex 4335cea8453c..d49be2ae1f09 100644--- a/products/data_warehouse/backend/presentation/views/table.py+++ b/products/data_warehouse/backend/presentation/views/table.py@@ -5,6 +5,7 @@ from urllib.parse import urlparsefrom django.conf import settingsfrom django.db.models import QuerySet+from django.db.models.manager import BaseManagerimport boto3import posthoganalytics@@ -54,6 +55,7 @@ from products.warehouse_sources.backend.facade.api import (build_file_upload_url_pattern,denied_table_ids,hosted_upload_s3_path,+ pending_source_ids_for_tables,)from products.warehouse_sources.backend.facade.contracts import (FileUploadSourceError,@@ -279,6 +281,22 @@ class CredentialSerializer(serializers.ModelSerializer):}+class TableListSerializer(serializers.ListSerializer):+ """Resolves the pending source of every table in the page with one query."""++ def to_representation(self, data):+ tables = list(data.all() if isinstance(data, BaseManager) else data)+ unsourced = [table for table in tables if table.external_data_source_id is None]+ if unsourced:+ team_ids = {table.team_id for table in unsourced}+ cache = self.context.setdefault("pending_source_ids", {})+ for team_id in team_ids:+ ids = [table.id for table in unsourced if table.team_id == team_id]+ found = pending_source_ids_for_tables(team_id, ids)+ cache.update({table_id: found.get(table_id) for table_id in ids})+ return super().to_representation(tables)++class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSerializer):created_by = UserBasicSerializer(read_only=True)credential = CredentialSerializer()@@ -290,6 +308,10 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSeriali)external_data_source = SimpleExternalDataSourceSerializers(read_only=True)external_schema = serializers.SerializerMethodField(read_only=True)+ pending_source_id = serializers.SerializerMethodField(+ read_only=True,+ help_text="The managed source that will import this self-managed table on its first sync, if one is linked.",+ )options = serializers.DictField(required=False,default=dict,@@ -326,9 +348,11 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSeriali"columns","external_data_source","external_schema",+ "pending_source_id","options","user_access_level",]+ list_serializer_class = TableListSerializerread_only_fields = ["id","created_by",@@ -338,6 +362,7 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSeriali"columns","external_data_source","external_schema",+ "pending_source_id","user_access_level",]extra_kwargs = {@@ -402,6 +427,15 @@ class TableSerializer(UserAccessControlSerializerMixin, serializers.ModelSerialifor field in serializes_fields]+ @extend_schema_field(serializers.UUIDField(allow_null=True))+ def get_pending_source_id(self, table: DataWarehouseTable) -> uuid.UUID | None:+ if table.external_data_source_id is not None:+ return None+ cache = self.context.get("pending_source_ids")+ if cache is not None and table.id in cache:+ return cache[table.id]+ return pending_source_ids_for_tables(table.team_id, [table.id]).get(table.id)+@extend_schema_field(serializers.DictField(allow_null=True))def get_external_schema(self, instance: DataWarehouseTable):from products.warehouse_sources.backend.presentation.views.external_data_schema import (@@ -618,7 +652,13 @@ class CreateTableFromUploadSerializer(UploadedFileSerializer):bool,description="Include table columns. Disable for lightweight table existence or name lookups.",default=True,- )+ ),+ OpenApiParameter(+ "pending_source",+ bool,+ description="Only return self-managed tables that a managed source will import on its first sync.",+ default=False,+ ),]))@@ -646,13 +686,18 @@ class TableViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viewsets.Mreturn contextdef safely_get_queryset(self, queryset):- return (+ queryset = (queryset.filter(team_id=self.team_id).exclude(deleted=True).exclude(external_data_source__access_method=ExternalDataSourceAccessMethod.DIRECT).prefetch_related("created_by", "externaldataschema_set").order_by(self.ordering))+ if self.action == "list" and self.request.query_params.get("pending_source", "").lower() == "true":+ queryset = queryset.filter(id__in=list(pending_source_ids_for_tables(self.team_id))).select_related(+ "credential"+ )+ return querysetdef _filter_queryset_by_access_level(self, queryset: QuerySet) -> QuerySet:queryset = super()._filter_queryset_by_access_level(queryset)diff --git a/products/data_warehouse/backend/tests/api/test_table.py b/products/data_warehouse/backend/tests/api/test_table.pyindex 116bd56617b1..451a8ef02f5f 100644--- a/products/data_warehouse/backend/tests/api/test_table.py+++ b/products/data_warehouse/backend/tests/api/test_table.py@@ -6,7 +6,9 @@ from unittest.mock import ANY, MagicMock, Mock, patchfrom django.conf import settingsfrom django.core.exceptions import ValidationError as DjangoValidationError+from django.db import connectionfrom django.test import SimpleTestCase, override_settings+from django.test.utils import CaptureQueriesContextimport boto3from clickhouse_driver.errors import ServerException@@ -14,6 +16,7 @@ from parameterized import parameterizedfrom rest_framework.exceptions import ValidationErrorfrom rest_framework.test import APIRequestFactory+from posthog.models import Teamfrom posthog.models.activity_logging.activity_log import ActivityLogfrom products.data_warehouse.backend.direct_postgres import DIRECT_POSTGRES_URL_PATTERN@@ -26,6 +29,7 @@ from products.data_warehouse.backend.presentation.views.table import (from products.warehouse_sources.backend.facade.models import (DataWarehouseCredential,DataWarehouseTable,+ ExternalDataSchema,ExternalDataSource,)@@ -641,6 +645,144 @@ class TestTable(APIBaseTest):assert response.json()["count"] == 1assert response.json()["results"][0]["id"] == str(table.id)+ def _pinned_table(self, name: str = "orders") -> tuple[DataWarehouseTable, ExternalDataSchema]:+ table = DataWarehouseTable.objects.create(+ name=name,+ format="Parquet",+ team=self.team,+ team_id=self.team.pk,+ url_pattern=f"https://example.com/{name}/*.parquet",+ )+ source = ExternalDataSource.objects.create(+ team=self.team,+ team_id=self.team.pk,+ source_type="AwsS3",+ access_method=ExternalDataSource.AccessMethod.WAREHOUSE,+ )+ schema = ExternalDataSchema.objects.create(+ name=name,+ team=self.team,+ source=source,+ sync_type_config={+ "schema_metadata": {+ "self_managed_table": {+ "table_id": str(table.id),+ "url_pattern": table.url_pattern,+ "format": "Parquet",+ }+ }+ },+ )+ return table, schema++ @parameterized.expand(+ [+ ("pinned_by_live_schema", lambda t, s, other: None, True),+ (+ "schema_deleted",+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(deleted=True),+ False,+ ),+ (+ "source_deleted",+ lambda t, s, other: ExternalDataSource.objects.filter(pk=s.source_id).update(deleted=True),+ False,+ ),+ (+ "table_already_has_source",+ lambda t, s, other: DataWarehouseTable.objects.filter(pk=t.pk).update(external_data_source=s.source),+ False,+ ),+ (+ "pin_abandoned",+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(+ sync_type_config={+ "schema_metadata": {+ "self_managed_table": {+ "table_id": str(t.id),+ "url_pattern": t.url_pattern,+ "format": "Parquet",+ "abandon_reason": "gone",+ }+ }+ }+ ),+ False,+ ),+ (+ "pin_in_other_team",+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(team_id=other.pk),+ False,+ ),+ (+ "table_url_edited_since_migration",+ lambda t, s, other: DataWarehouseTable.objects.filter(pk=t.pk).update(+ url_pattern="https://example.com/elsewhere/*.parquet"+ ),+ False,+ ),+ (+ "table_format_edited_since_migration",+ lambda t, s, other: DataWarehouseTable.objects.filter(pk=t.pk).update(format="CSV"),+ False,+ ),+ (+ "schema_linked_to_replacement_table",+ lambda t, s, other: ExternalDataSchema.objects.filter(pk=s.pk).update(+ table=DataWarehouseTable.objects.create(+ name="replacement",+ format="Parquet",+ team_id=t.team_id,+ url_pattern="https://example.com/replacement/*.parquet",+ external_data_source_id=s.source_id,+ )+ ),+ False,+ ),+ ]+ )+ def test_pending_source_id(self, _name, mutate, expect_pending):+ table, schema = self._pinned_table()+ unrelated = DataWarehouseTable.objects.create(+ name="unrelated",+ format="Parquet",+ team=self.team,+ team_id=self.team.pk,+ url_pattern="https://example.com/unrelated/*.parquet",+ )+ other_team = Team.objects.create(organization=self.organization, name="Other")+ mutate(table, schema, other_team)++ results = {+ row["id"]: row+ for row in self.client.get(f"/api/projects/{self.team.pk}/warehouse_tables/?include_columns=false").json()[+ "results"+ ]+ }+ expected = str(schema.source_id) if expect_pending else None+ assert results[str(table.id)]["pending_source_id"] == expected+ assert results[str(unrelated.id)]["pending_source_id"] is None++ detail = self.client.get(f"/api/projects/{self.team.pk}/warehouse_tables/{table.id}/").json()+ assert detail["pending_source_id"] == expected++ filtered = self.client.get(+ f"/api/projects/{self.team.pk}/warehouse_tables/?include_columns=false&pending_source=true"+ ).json()["results"]+ assert [row["id"] for row in filtered] == ([str(table.id)] if expect_pending else [])++ def test_pending_source_id_is_resolved_once_per_list(self):+ self._pinned_table("one")+ url = f"/api/projects/{self.team.pk}/warehouse_tables/?include_columns=false"+ for name in ("two", "three", "four"):+ self._pinned_table(name)+ with CaptureQueriesContext(connection) as several:+ response = self.client.get(url)++ assert [row["pending_source_id"] is not None for row in response.json()["results"]] == [True] * 4+ pending_lookups = [q["sql"] for q in several.captured_queries if "self_managed_table" in q["sql"]]+ assert len(pending_lookups) == 1+def test_list_tables_exposes_hogql_name_and_honors_include_columns(self):source = ExternalDataSource.objects.create(team=self.team,diff --git a/products/data_warehouse/frontend/generated/api.schemas.ts b/products/data_warehouse/frontend/generated/api.schemas.tsindex 3f6f73ef7ca5..45068ef93678 100644--- a/products/data_warehouse/frontend/generated/api.schemas.ts+++ b/products/data_warehouse/frontend/generated/api.schemas.ts@@ -5334,6 +5334,11 @@ export interface TableApi {readonly external_data_source: SimpleExternalDataSourceSerializersApi/** @nullable */readonly external_schema: TableApiExternalSchema+ /**+ * The managed source that will import this self-managed table on its first sync, if one is linked.+ * @nullable+ */+ readonly pending_source_id: string | null/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */options?: TableApiOptions/**@@ -5413,6 +5418,11 @@ export interface PatchedTableApi {readonly external_data_source?: SimpleExternalDataSourceSerializersApi/** @nullable */readonly external_schema?: PatchedTableApiExternalSchema+ /**+ * The managed source that will import this self-managed table on its first sync, if one is linked.+ * @nullable+ */+ readonly pending_source_id?: string | null/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */options?: PatchedTableApiOptions/**@@ -6052,6 +6062,10 @@ export type WarehouseTablesListParams = {* The initial index from which to return the results.*/offset?: number+ /**+ * Only return self-managed tables that a managed source will import on its first sync.+ */+ pending_source?: boolean/*** A search term.*/diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsxindex ff378ea5b529..4423fa0f4d10 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx@@ -85,6 +85,9 @@ const schemaEditDisabledReason = (schema: ExternalDataSourceSchema): string | nuconst dimWhenNotSyncing = (_: unknown, schema: ExternalDataSourceSchema): string =>schema.should_sync ? '' : 'opacity-60'+const PENDING_SELF_MANAGED_TABLE_NOTE =+ 'This table reads your bucket until you turn on syncing. The first sync imports it into PostHog.'+type SchemaColumn = LemonTableColumn<ExternalDataSourceSchema, keyof ExternalDataSourceSchema | undefined>// A file upload schema has no schedule or sync method to choose, and never syncs on its own, so it isn't dimmed.@@ -549,6 +552,9 @@ function ManagedSchemaTable({}description={((): JSX.Element | undefined => {const tableName = schema.table?.hogql_name ?? schema.table?.name+ if (!tableName && schema.pending_self_managed_table && !schema.should_sync) {+ return <span>{PENDING_SELF_MANAGED_TABLE_NOTE}</span>+ }return tableName ? (<CopyToClipboardInlineexplicitValue={tableName}diff --git a/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsx b/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsxindex fb146879ae69..d5ed8aef4e00 100644--- a/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsx+++ b/products/data_warehouse/frontend/shared/components/SelfManagedSourcesTable.tsx@@ -9,7 +9,8 @@ import { sourceManagementLogic } from '../logics/sourceManagementLogic'import { SourceIcon, mapUrlToProvider } from './SourceIcon'export function SelfManagedSourcesTable(): JSX.Element {- const { filteredSelfManagedTables, searchTerm, databaseLoading } = useValues(sourceManagementLogic)+ const { filteredSelfManagedTables, searchTerm, databaseLoading, pendingSelfManagedTableIdsLoading } =+ useValues(sourceManagementLogic)const { deleteSelfManagedTable, refreshSelfManagedTableSchema, setSearchTerm } = useActions(sourceManagementLogic)return (@@ -19,8 +20,8 @@ export function SelfManagedSourcesTable(): JSX.Element {</div><LemonTableid="self-managed-sources"- dataSource={filteredSelfManagedTables}- loading={databaseLoading}+ dataSource={pendingSelfManagedTableIdsLoading ? [] : filteredSelfManagedTables}+ loading={databaseLoading || pendingSelfManagedTableIdsLoading}pagination={{ pageSize: 10 }}scrollToTopOnPageChange={false}columns={[diff --git a/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts b/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.tsindex 656650a70e71..776777c9bee9 100644--- a/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts+++ b/products/data_warehouse/frontend/shared/logics/sourceManagementLogic.ts@@ -14,7 +14,7 @@ import { loaders } from 'kea-loaders'import { router, urlToAction } from 'kea-router'import posthog from 'posthog-js'-import api from 'lib/api'+import api, { ApiConfig } from 'lib/api'import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'import { createFuse, Fuse } from 'lib/utils/fuseSearch'import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'@@ -28,6 +28,8 @@ import {ExternalDataSourceSchema,} from '~/types'+import { warehouseTablesList } from 'products/data_warehouse/frontend/generated/api'+import type { PaginatedResponse } from '../../../../../frontend/src/lib/api'import type { DatabaseSchemaQueryResponse } from '../../../../../frontend/src/queries/schema/schema-general'import type { ExternalDataSourceRevenueAnalyticsConfig } from '../../../../../frontend/src/types'@@ -81,6 +83,8 @@ export interface sourceManagementLogicValues {managedSearchTerm: stringmanagedSources: ExternalDataSource[]managedSourcesFuse: Fuse<ExternalDataSource>+ pendingSelfManagedTableIds: string[]+ pendingSelfManagedTableIdsLoading: booleanschemaReloadingById: Record<string, boolean>schemas: nullschemasLoading: boolean@@ -160,6 +164,21 @@ export interface sourceManagementLogicActions {deleteSource: (source: ExternalDataSource) => {source: ExternalDataSource}+ loadPendingSelfManagedTableIds: () => any+ loadPendingSelfManagedTableIdsFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadPendingSelfManagedTableIdsSuccess: (+ pendingSelfManagedTableIds: string[],+ payload?: any+ ) => {+ pendingSelfManagedTableIds: string[]+ payload?: any+ }refreshSelfManagedTableSchema: (tableId: string) => {tableId: string}@@ -205,7 +224,8 @@ export interface sourceManagementLogicActions {export interface sourceManagementLogicMeta {__keaTypeGenInternalSelectorTypes: {selfManagedTables: (- dataWarehouseTables: DatabaseSchemaDataWarehouseTable[]+ dataWarehouseTables: DatabaseSchemaDataWarehouseTable[],+ pendingSelfManagedTableIds: string[]) => DatabaseSchemaDataWarehouseTable[]filteredSelfManagedTables: (selfManagedTables: DatabaseSchemaDataWarehouseTable[],@@ -273,6 +293,27 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([setDirectSearchTerm: (directSearchTerm: string) => ({ directSearchTerm }),}),loaders(({ actions, values }) => ({+ pendingSelfManagedTableIds: [+ [] as string[],+ {+ loadPendingSelfManagedTableIds: async () => {+ const ids: string[] = []+ const limit = 1000+ for (let offset = 0; ; offset += limit) {+ const response = await warehouseTablesList(String(ApiConfig.getCurrentProjectId()), {+ pending_source: true,+ include_columns: false,+ limit,+ offset,+ })+ ids.push(...response.results.map((table) => table.id))+ if (!response.next || response.results.length === 0) {+ return ids+ }+ }+ },+ },+ ],schemas: [null,{@@ -348,9 +389,14 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([})),selectors({selfManagedTables: [- (s) => [s.dataWarehouseTables],- (dataWarehouseTables: DatabaseSchemaDataWarehouseTable[]): DatabaseSchemaDataWarehouseTable[] => {- return dataWarehouseTables.filter((table) => !table.source)+ (s) => [s.dataWarehouseTables, s.pendingSelfManagedTableIds],+ (+ dataWarehouseTables: DatabaseSchemaDataWarehouseTable[],+ pendingSelfManagedTableIds: string[]+ ): DatabaseSchemaDataWarehouseTable[] => {+ // A table that a managed source will import on its first sync is listed under that source instead.+ const pendingIds = new Set(pendingSelfManagedTableIds)+ return dataWarehouseTables.filter((table) => !table.source && !pendingIds.has(table.id))},],filteredSelfManagedTables: [@@ -490,6 +536,7 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([// Only on success: a transient list failure resolves to an empty page, so reloading// after a delete that failed offline would blank a list of sources that still exist.actions.loadSources()+ actions.loadPendingSelfManagedTableIds()posthog.capture('source deleted', { sourceType: source.source_type })actions.sourceDeleted(source)} catch (e) {@@ -580,6 +627,7 @@ export const sourceManagementLogic = kea<sourceManagementLogicType>([}}actions.loadSources()+ actions.loadPendingSelfManagedTableIds()}),beforeUnmount(() => {// Disposables plugin handles cleanup automaticallydiff --git a/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts b/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.tsindex a97f18220b90..0f97e990f386 100644--- a/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts+++ b/products/data_warehouse/frontend/shared/logics/tests/sourceManagementLogic.test.ts@@ -8,10 +8,15 @@ import { DatabaseSchemaQueryResponse } from '~/queries/schema/schema-general'import { initKeaTests } from '~/test/init'import { ExternalDataSource } from '~/types'+import { warehouseTablesList } from 'products/data_warehouse/frontend/generated/api'+import { availableSourcesLogic } from '../../../scenes/NewSourceScene/availableSourcesLogic'import { sourceManagementLogic } from '../sourceManagementLogic'jest.mock('lib/api')+jest.mock('products/data_warehouse/frontend/generated/api', () => ({+ warehouseTablesList: jest.fn(),+}))jest.mock('~/queries/query')describe('sourceManagementLogic', () => {@@ -33,6 +38,13 @@ describe('sourceManagementLogic', () => {results: [],} as any)+ ;(warehouseTablesList as jest.Mock).mockResolvedValue({+ results: [],+ count: 0,+ next: null,+ previous: null,+ })+;(performQuery as jest.Mock).mockResolvedValue({tables: {},joins: [],@@ -203,4 +215,40 @@ describe('sourceManagementLogic', () => {resolveShallowLoad?.({ tables: {}, joins: [] } as DatabaseSchemaQueryResponse)await shallowLoad})++ it.each([+ ['no linked tables', [], ['orders', 'users', 'events']],+ ['one linked table', ['t-orders'], ['users', 'events']],+ ['every self-managed table linked', ['t-orders', 't-users'], ['events']],+ ['id of an unknown table', ['t-missing'], ['orders', 'users', 'events']],+ ])('hides tables a managed source will import from the self-managed list: %s', async (_, linkedIds, expected) => {+ ;(warehouseTablesList as jest.Mock).mockResolvedValue({+ results: linkedIds.map((id) => ({ id, pending_source_id: 's2' })),+ })+ databaseLogic.mount()+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadPendingSelfManagedTableIdsSuccess'])+ databaseTableListLogic.actions.loadDatabaseSuccess({+ tables: {+ orders: { type: 'data_warehouse', id: 't-orders', name: 'orders', fields: {} },+ users: { type: 'data_warehouse', id: 't-users', name: 'users', fields: {} },+ // Linked tables are only hidden when they have no source of their own.+ events: { type: 'data_warehouse', id: 't-events', name: 'events', fields: {} },+ stripe_charges: {+ type: 'data_warehouse',+ id: 't-charges',+ name: 'stripe_charges',+ fields: {},+ source: { id: 's1', status: 'Running', source_type: 'Stripe', prefix: '' },+ },+ },+ joins: [],+ } as unknown as DatabaseSchemaQueryResponse)++ expect(warehouseTablesList).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({ pending_source: true })+ )+ expect(logic.values.selfManagedTables.map((table) => table.name)).toEqual(expected)+ })})diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 2212b56b8405..91fc3813119d 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -10937,6 +10937,10 @@ export namespace Schemas {latest_error: string | null;incremental: boolean;status: string | null;+ /**+ * Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket.+ */+ pending_self_managed_table: boolean;/*** A schema of an external data source: its sync configuration and the warehouse table it syncs into.*/diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.pyindex 32e72aa4ae61..07e8d403abba 100644--- a/products/warehouse_sources/backend/facade/api.py+++ b/products/warehouse_sources/backend/facade/api.py@@ -47,7 +47,11 @@ from products.warehouse_sources.backend.models.external_data_job import (ExternalDataJob as _ExternalDataJob,latest_completed_job_subquery,)-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema as _ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_schema import (+ SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY,+ ExternalDataSchema as _ExternalDataSchema,+ self_managed_table_ref,+)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource as _ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTable as _DataWarehouseTable@@ -93,6 +97,7 @@ __all__ = ["record_table_row_count","allowed_table_ids","denied_table_ids",+ "pending_source_ids_for_tables","all_queryable_tables","all_queryable_table_columns","list_tables_for_source",@@ -530,6 +535,55 @@ def _resolve_allowed_table_ids(return _allowed_tables(_tables_for_access_check(candidates), user_access_control, required_level)+def pending_source_ids_for_tables(team_id: int, table_ids: Collection[UUID] | None = None) -> dict[UUID, UUID]:+ """Map each self-managed table to the managed source that will import it on its first sync.++ A table is pending when a live, still unlinked schema of a live source in the same team pins it,+ and the table has no source yet and still reads the files the schema was pinned to. That matches+ the conditions under which the first sync adopts the table rather than creating a new one.+ `table_ids` of None checks every table in the team.+ """+ if table_ids is not None and not table_ids:+ return {}+ pinned = (+ _ExternalDataSchema.objects.filter(team_id=team_id, source__team_id=team_id)+ .exclude(deleted=True)+ .exclude(source__deleted=True)+ .filter(table__isnull=True)+ .filter(sync_type_config__schema_metadata__has_key=SCHEMA_SELF_MANAGED_TABLE_METADATA_KEY)+ .order_by("created_at", "id")+ )+ if table_ids is not None:+ pinned = pinned.filter(+ sync_type_config__schema_metadata__self_managed_table__table_id__in=[str(i) for i in table_ids]+ )+ by_table: dict[UUID, list[tuple[UUID, str, str]]] = {}+ for schema in pinned:+ ref = self_managed_table_ref(schema)+ if ref is None or ref.abandon_reason is not None or not UUIDT.is_valid_uuid(ref.table_id):+ continue+ by_table.setdefault(UUID(ref.table_id), []).append((schema.source_id, ref.url_pattern, ref.format))+ if not by_table:+ return {}+ unsourced = _DataWarehouseTable.raw_objects.filter(+ team_id=team_id, id__in=list(by_table), external_data_source__isnull=True+ ).values_list("id", "url_pattern", "format")+ found: dict[UUID, UUID] = {}+ for table_id, url_pattern, table_format in unsourced:+ # A table edited since the migration reads other files, so its sync creates a new table instead.+ source_id = next(+ (+ sid+ for sid, ref_url, ref_format in by_table[table_id]+ if (ref_url, ref_format) == (url_pattern, table_format)+ ),+ None,+ )+ if source_id is not None:+ found[table_id] = source_id+ return found++def denied_table_ids(team_id: int, user_access_control: "UserAccessControl") -> frozenset[UUID]:"""Every table this caller may not view.diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.pyindex 8dc1ba4d8d07..18950bd54785 100644--- a/products/warehouse_sources/backend/facade/models.py+++ b/products/warehouse_sources/backend/facade/models.py@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (get_schemas_for_direct_reconciliation,mark_schema_running_unless_halted,resolve_sync_type,+ self_managed_table_ref,sync_frequency_interval_to_sync_frequency,sync_frequency_to_sync_frequency_interval,sync_old_schemas_with_new_schemas,@@ -143,6 +144,7 @@ __all__ = ["update_sync_type_config_keys","UnsupportedSyncTypeError","resolve_sync_type",+ "self_managed_table_ref","CDC_SNAPSHOT_LANE_KEY","validate_source_prefix","validate_warehouse_table_url_pattern",diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex 882f0bd4744b..7b1c56eabbea 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -57,6 +57,7 @@ from products.warehouse_sources.backend.facade.models import (mark_schema_running_unless_halted,resolve_destinations,resolve_sync_type,+ self_managed_table_ref,sync_frequency_interval_to_sync_frequency,sync_frequency_to_sync_frequency_interval,update_sync_type_config_keys,@@ -551,6 +552,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"""A schema of an external data source: its sync configuration and the warehouse table it syncs into."""table = serializers.SerializerMethodField(read_only=True)+ pending_self_managed_table = serializers.SerializerMethodField(+ read_only=True,+ help_text="Whether this schema is linked to an existing self-managed table that its first sync will import. "+ "Until then the table keeps reading the original bucket.",+ )incremental = serializers.SerializerMethodField(read_only=True)status = serializers.SerializerMethodField(read_only=True)sync_type = serializers.ChoiceField(@@ -737,6 +743,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"latest_error","incremental","status",+ "pending_self_managed_table","sync_type","incremental_field","incremental_field_type",@@ -770,6 +777,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"last_synced_at","latest_error","status",+ "pending_self_managed_table","incremental_sync_blocked","next_full_refresh_at","description",@@ -948,6 +956,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersdef get_status(self, schema: ExternalDataSchema) -> str | None:return schema_display_status(schema)+ @extend_schema_field(serializers.BooleanField())+ def get_pending_self_managed_table(self, schema: ExternalDataSchema) -> bool:+ ref = self_managed_table_ref(schema)+ return schema.table_id is None and ref is not None and ref.abandon_reason is None+def get_incremental(self, schema: ExternalDataSchema) -> bool:return schema.is_incremental@@ -2080,6 +2093,11 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):read_only=True, help_text="The synced warehouse table (id, name, row_count), or null if not yet synced.")status = serializers.SerializerMethodField(read_only=True, help_text="Current sync status for this schema.")+ pending_self_managed_table = serializers.SerializerMethodField(+ read_only=True,+ help_text="Whether this schema is linked to an existing self-managed table that its first sync will import. "+ "Until then the table keeps reading the original bucket.",+ )sync_frequency = serializers.SerializerMethodField(read_only=True, help_text="How often this table is scheduled to sync, or null if no interval is set."@@ -2098,6 +2116,7 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):"sync_frequency","latest_error","table",+ "pending_self_managed_table",]read_only_fields = fields@@ -2122,6 +2141,11 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):def get_status(self, schema: ExternalDataSchema) -> str | None:return schema_display_status(schema)+ @extend_schema_field(serializers.BooleanField())+ def get_pending_self_managed_table(self, schema: ExternalDataSchema) -> bool:+ ref = self_managed_table_ref(schema)+ return schema.table_id is None and ref is not None and ref.abandon_reason is None+@extend_schema_field(serializers.CharField(allow_null=True))def get_sync_frequency(self, schema: ExternalDataSchema) -> str | None:try:diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex 67bccb2a624c..e8811c418a53 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -5070,6 +5070,38 @@ class TestExternalDataSchemaRetrieveSource(APIBaseTest):assert summary["supports_row_filters"] is expected_row_filtersassert "user_access_level" in summary+ @parameterized.expand(+ [+ ("pinned_and_unsynced", True, False, True),+ ("pinned_but_synced_table_set", True, True, False),+ ("no_pin", False, False, False),+ ]+ )+ def test_pending_self_managed_table(self, _name: str, pinned: bool, has_table: bool, expected: bool):+ _source, schema = self._create()+ if pinned:+ schema.sync_type_config = {+ "schema_metadata": {+ "self_managed_table": {+ "table_id": str(uuid.uuid4()),+ "url_pattern": "https://example.com/a/*.parquet",+ "format": "Parquet",+ }+ }+ }+ schema.save()+ if has_table:+ schema.table = DataWarehouseTable.objects.create(+ name="orders", format="Parquet", team=self.team, url_pattern="https://example.com/a/*.parquet"+ )+ schema.save()++ detail = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()+ listed = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/").json()["results"]++ assert detail["pending_self_managed_table"] is expected+ assert [row["pending_self_managed_table"] for row in listed] == [expected]+def test_list_omits_source_summary(self):self._create()response = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/")diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex a8d2add0c3cd..20f7c4af410b 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -279,6 +279,7 @@ class TestExternalDataSource(APIBaseTest):"sync_frequency","latest_error","table",+ "pending_self_managed_table",},)self.assertEqual(listed_schema["sync_frequency"], "6hour")@@ -3392,6 +3393,7 @@ class TestExternalDataSource(APIBaseTest):"latest_error": schema.latest_error,"should_sync": schema.should_sync,"status": schema.status,+ "pending_self_managed_table": False,"sync_type": schema.sync_type,"table": schema.table,"sync_frequency": sync_frequency_interval_to_sync_frequency(schema.sync_frequency_interval),diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.tsindex 7c64bc002ba6..3212b06594ee 100644--- a/products/warehouse_sources/frontend/generated/api.schemas.ts+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts@@ -430,6 +430,8 @@ export interface ExternalDataSchemaApi {readonly incremental: boolean/** @nullable */readonly status: string | null+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */+ readonly pending_self_managed_table: boolean/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.** * `full_refresh` - full_refresh@@ -625,6 +627,8 @@ export interface PatchedExternalDataSchemaApi {readonly incremental?: boolean/** @nullable */readonly status?: string | null+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */+ readonly pending_self_managed_table?: boolean/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.** * `full_refresh` - full_refreshdiff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 3eb46bebd515..508a1bcacbba 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -51765,6 +51765,8 @@ export namespace Schemas {readonly incremental: boolean;/** @nullable */readonly status: string | null;+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */+ readonly pending_self_managed_table: boolean;/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.** * `full_refresh` - full_refresh@@ -82323,6 +82325,11 @@ export namespace Schemas {readonly external_data_source: SimpleExternalDataSourceSerializers;/** @nullable */readonly external_schema: TableExternalSchema;+ /**+ * The managed source that will import this self-managed table on its first sync, if one is linked.+ * @nullable+ */+ readonly pending_source_id: string | null;/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */options?: TableOptions;/**@@ -88340,6 +88347,8 @@ export namespace Schemas {readonly incremental?: boolean;/** @nullable */readonly status?: string | null;+ /** Whether this schema is linked to an existing self-managed table that its first sync will import. Until then the table keeps reading the original bucket. */+ readonly pending_self_managed_table?: boolean;/** Sync strategy: incremental, full_refresh, append, cdc, or xmin.** * `full_refresh` - full_refresh@@ -94298,6 +94307,11 @@ export namespace Schemas {readonly external_data_source?: SimpleExternalDataSourceSerializers;/** @nullable */readonly external_schema?: PatchedTableExternalSchema;+ /**+ * The managed source that will import this self-managed table on its first sync, if one is linked.+ * @nullable+ */+ readonly pending_source_id?: string | null;/** Per-format read options. The only one read today is `csv_allow_double_quotes` (boolean), for CSV files that quote fields with doubled quotes. */options?: PatchedTableOptions;/**@@ -144519,6 +144533,10 @@ export namespace Schemas {* The initial index from which to return the results.*/offset?: number;+ /**+ * Only return self-managed tables that a managed source will import on its first sync.+ */+ pending_source?: boolean;/*** A search term.*/