:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): link upload schemas to their tables and stop re-syncing uploaded files (#125561)
Tom Owers023dc5766c18
diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex df12ffad4f58..b57d56afd6d6 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -6520,6 +6520,8 @@ export interface ExternalDataSource {latest_error: string | nulllast_run_at?: Dayjsschemas: ExternalDataSourceSchema[]+ /** The name of the file a file upload source imports. */+ file_name?: string | nullsync_frequency: DataWarehouseSyncIntervaljob_inputs: Record<string, any>revenue_analytics_config: ExternalDataSourceRevenueAnalyticsConfig@@ -6704,6 +6706,8 @@ export type IncrementalSyncBlockedReason = IncrementalSyncBlockedReasonEnumApiexport interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema {table?: SimpleDataWarehouseTable+ /** The live table the schema syncs into, also before its first sync when the schema already names one. */+ table_id?: string | null/** The schema is linked to a self-managed table that its first sync will import. */pending_self_managed_table?: booleanincremental: boolean@@ -6759,6 +6763,8 @@ export interface ExternalDataSchemaSourceSummary {supports_column_selection?: booleansupports_row_filters?: booleanrequires_exact_column_metadata?: boolean+ /** The name of the file a file upload source imports. */+ file_name?: string | nulluser_access_level: AccessControlLevel | null/** The source's effective vendor API version — what schemas without an override sync on */api_version?: string | nulldiff --git a/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.py b/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.pyindex 4bfcd1c76168..0369772afa81 100644--- a/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.py+++ b/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.py@@ -33,6 +33,7 @@ def _stub_source_security_gate(source_mock) -> None:source_mock.job_inputs_add_connection_host.return_value = Falsesource_mock.has_preserved_row_backed_credentials.return_value = Falsesource_mock.serialize_config.side_effect = lambda config: config.to_dict()+ source_mock.uploaded_file_name.return_value = Noneclass TestPostgresWarehouseMigration(APIBaseTest):diff --git a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.pyindex 68a86a8be06e..24d8d31c42fd 100644--- a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py+++ b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py@@ -11,6 +11,8 @@ from django.test import override_settingsfrom parameterized import parameterizedfrom rest_framework import status+from posthog.models.team import Team+from products.warehouse_sources.backend.facade.api import FILE_FORMAT_READ_HINTSfrom products.warehouse_sources.backend.facade.testing import (find_replaced_self_managed_table,@@ -881,6 +883,58 @@ class TestFileUploadSources(APIBaseTest):assert not ExternalDataJob.objects.filter(team=self.team).exists()self.trigger.assert_not_called()+ @parameterized.expand(["reload", "resync"])+ def test_a_table_that_imported_its_file_points_to_replace_file(self, action: str) -> None:+ _, schema = self._table_with_source()+ ExternalDataJob.objects.create(+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.COMPLETED+ )+ self.trigger.reset_mock()++ response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")++ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert "Replace file" in response.json()["message"]+ self.trigger.assert_not_called()++ @parameterized.expand(+ [+ ("linked", None),+ ("pinned", "managed/team_{team}/orders.csv"),+ ("pinned_in_upload_folder", "file_uploads/team_{team}/{upload}/orders.csv"),+ ]+ )+ def test_the_schema_and_source_name_the_table_and_the_file(self, _name: str, migrated_key: str | None) -> None:+ if migrated_key is None:+ table, schema = self._table_with_source()+ else:+ table, schema = self._migrated_table(migrated_key)+ assert schema.table_id is None++ schema_body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()+ source_body = self.client.get(+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/"+ ).json()++ assert schema_body["table_id"] == str(table.id)+ assert source_body["schemas"][0]["table_id"] == str(table.id)+ assert source_body["file_name"] == "orders.csv"+ assert schema_body["source"]["file_name"] == "orders.csv"+ assert source_body["job_inputs"] == {}++ @parameterized.expand(["other_team", "deleted"])+ def test_a_pinned_table_that_is_not_live_in_the_team_is_not_returned(self, case: str) -> None:+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")+ if case == "deleted":+ DataWarehouseTable.objects.filter(pk=table.pk).update(deleted=True)+ else:+ other_team = Team.objects.create(organization=self.organization, name="Other")+ DataWarehouseTable.objects.filter(pk=table.pk).update(team=other_team)++ body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()++ assert body["table_id"] is None+@patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")@patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")@patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsxindex b94091d25e7f..b4dd92e6af47 100644--- a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx+++ b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx@@ -41,6 +41,7 @@ import {SchemaEditorAction,useSchemaEditorAccess,} from 'products/data_warehouse/frontend/shared/components/SourceEditorAction'+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'import {IncrementalSyncBlockedMessageMap,StatusTagSetting,@@ -56,6 +57,7 @@ import type { ExternalDataSchemaApi } from 'products/warehouse_sources/frontend/import { ApiVersionDeprecationBanner } from '../SourceScene/SourceScene'import { ColumnSelectionPicker } from '../SourceScene/tabs/ColumnSelectionModal'+import { replaceFileDisabledReason, replaceFileLogic, replaceFileTarget } from '../SourceScene/tabs/replaceFileLogic'import { RowFilterEditor } from '../SourceScene/tabs/RowFilterEditor'import { validateRowFilters } from '../SourceScene/tabs/rowFilterUtils'import { columnAnnotationsLogic } from './columnAnnotationsLogic'@@ -104,6 +106,14 @@ export function ConfigurationTab({const { isProjectTime, refreshingSchemas, resyncingSchema, supportsRowFilters } = useValues(logic)const { setIsProjectTime, updateSchema, reloadSchema, resyncSchema, cancelSchema, deleteTable, refreshSchemas } =useActions(logic)+ const { openReplaceFile } = useActions(replaceFileLogic({ sourceId }))+ const fileUpload: FileUploadDetails | undefined = isFileUploadSource(source)+ ? {+ fileName: source?.file_name ?? null,+ onReplaceFile: () => openReplaceFile(replaceFileTarget(schema)),+ replaceDisabledReason: replaceFileDisabledReason(schema),+ }+ : undefinedswitch (section) {case 'details':@@ -115,6 +125,7 @@ export function ConfigurationTab({updateSchema={updateSchema}onConfigureSyncMethod={onConfigureSyncMethod}syncHistoryUrl={syncHistoryUrl}+ fileUpload={fileUpload}/>)case 'sync-method':@@ -171,6 +182,12 @@ function SectionHeader({ title, description }: { title: string; description?: st)}+interface FileUploadDetails {+ fileName: string | null+ onReplaceFile: () => void+ replaceDisabledReason?: string+}+function DetailsSection({schema,reloadSchema,@@ -178,6 +195,7 @@ function DetailsSection({updateSchema,onConfigureSyncMethod,syncHistoryUrl,+ fileUpload,}: {schema: ExternalDataSourceSchemareloadSchema: (schema: ExternalDataSourceSchema) => void@@ -185,6 +203,8 @@ function DetailsSection({updateSchema: (schema: ExternalDataSourceSchema) => voidonConfigureSyncMethod: () => voidsyncHistoryUrl?: string+ /** Set for a file upload schema, which refreshes by replacing its file and not by syncing. */+ fileUpload?: FileUploadDetails}): JSX.Element {const syncedTableName = schema.table?.hogql_name ?? schema.table?.name@@ -192,7 +212,11 @@ function DetailsSection({<div><SectionHeadertitle="Details"- description="Enable or disable syncing for this schema, see its current state, and trigger a sync on demand."+ description={+ fileUpload+ ? 'See the current state of this table and replace the file it imports.'+ : 'Enable or disable syncing for this schema, see its current state, and trigger a sync on demand.'+ }/><div className="border rounded p-4 bg-surface-primary flex flex-col gap-3">{schema.incremental_sync_blocked && (@@ -203,70 +227,78 @@ function DetailsSection({{IncrementalSyncBlockedMessageMap[schema.incremental_sync_blocked]}</LemonBanner>)}- <div className="flex items-start justify-between gap-4">- <div className="flex flex-col">- <span>Enabled</span>- <span className="text-xs text-muted max-w-md">- When enabled, this schema runs on the configured schedule and data is imported into PostHog.- Disabling pauses all syncs — existing data stays in place but is not updated until you- re-enable.- </span>+ {!fileUpload && (+ <div className="flex items-start justify-between gap-4">+ <div className="flex flex-col">+ <span>Enabled</span>+ <span className="text-xs text-muted max-w-md">+ When enabled, this schema runs on the configured schedule and data is imported into+ PostHog. Disabling pauses all syncs — existing data stays in place but is not updated+ until you re-enable.+ </span>+ </div>+ <SchemaEditorAction schema={schema}>+ <LemonSwitch+ checked={schema.should_sync}+ label={schema.should_sync ? 'Syncing' : 'Disabled'}+ onChange={(active) => {+ // A blocked table is not routed away here on purpose. An operator who fixed+ // the duplicates or added the key at the source has to be able to turn the+ // table back on themselves; the banner above says what the last run found.+ if (active && !schema.sync_type) {+ // No sync method saved yet — open the sync method section to set one up.+ onConfigureSyncMethod()+ return+ }+ if (!active && schema.sync_type === 'cdc') {+ LemonDialog.open({+ title: 'Disable CDC table?',+ content: (+ <div className="text-sm text-secondary space-y-2">+ <p>+ Disabling <strong>{schema.table?.name ?? schema.name}</strong>{' '}+ will remove it from the replication publication. Changes made+ while disabled will be permanently lost.+ </p>+ <p>+ Re-enabling this table will require a{' '}+ <strong>full resync</strong> to ensure data consistency.+ </p>+ </div>+ ),+ primaryButton: {+ children: 'Disable',+ status: 'danger',+ onClick: () => updateSchema({ ...schema, should_sync: false }),+ },+ secondaryButton: { children: 'Cancel', type: 'tertiary' },+ })+ } else if (!active && schema.sync_type === 'webhook') {+ LemonDialog.open({+ title: 'Disable webhook sync?',+ description:+ 'Turning off this table will stop the webhook from consuming any more data. When you re-enable it, a full refresh sync will need to be completed to ensure no data is missing.',+ primaryButton: {+ children: 'Disable',+ status: 'danger',+ onClick: () => updateSchema({ ...schema, should_sync: false }),+ },+ secondaryButton: { children: 'Cancel' },+ })+ } else {+ updateSchema({ ...schema, should_sync: active })+ }+ }}+ />+ </SchemaEditorAction></div>- <SchemaEditorAction schema={schema}>- <LemonSwitch- checked={schema.should_sync}- label={schema.should_sync ? 'Syncing' : 'Disabled'}- onChange={(active) => {- // A blocked table is not routed away here on purpose. An operator who fixed- // the duplicates or added the key at the source has to be able to turn the- // table back on themselves; the banner above says what the last run found.- if (active && !schema.sync_type) {- // No sync method saved yet — open the sync method section to set one up.- onConfigureSyncMethod()- return- }- if (!active && schema.sync_type === 'cdc') {- LemonDialog.open({- title: 'Disable CDC table?',- content: (- <div className="text-sm text-secondary space-y-2">- <p>- Disabling <strong>{schema.table?.name ?? schema.name}</strong> will- remove it from the replication publication. Changes made while- disabled will be permanently lost.- </p>- <p>- Re-enabling this table will require a <strong>full resync</strong>{' '}- to ensure data consistency.- </p>- </div>- ),- primaryButton: {- children: 'Disable',- status: 'danger',- onClick: () => updateSchema({ ...schema, should_sync: false }),- },- secondaryButton: { children: 'Cancel', type: 'tertiary' },- })- } else if (!active && schema.sync_type === 'webhook') {- LemonDialog.open({- title: 'Disable webhook sync?',- description:- 'Turning off this table will stop the webhook from consuming any more data. When you re-enable it, a full refresh sync will need to be completed to ensure no data is missing.',- primaryButton: {- children: 'Disable',- status: 'danger',- onClick: () => updateSchema({ ...schema, should_sync: false }),- },- secondaryButton: { children: 'Cancel' },- })- } else {- updateSchema({ ...schema, should_sync: active })- }- }}- />- </SchemaEditorAction>- </div>+ )}+ {fileUpload && (+ <div className="flex items-center justify-between">+ <span className="text-muted">File</span>+ <span>{fileUpload.fileName ?? 'Unknown'}</span>+ </div>+ )}<div className="flex items-center justify-between"><span className="text-muted">Current status</span>{schema.status ? (@@ -313,34 +345,49 @@ function DetailsSection({</div></div><div className="mt-4 flex gap-2 flex-wrap">- <SchemaEditorAction schema={schema}>- {({ disabledReason }) => (- <Tooltip- title={- schema.sync_type === 'cdc'- ? 'Trigger a CDC extraction run now.'- : schema.incremental- ? 'Sync incremental data since the last run.'- : 'Sync all data.'- }- >+ {fileUpload ? (+ <SchemaEditorAction schema={schema}>+ {({ disabledReason }) => (<LemonButtontype="primary"- onClick={() => reloadSchema(schema)}- disabledReason={- disabledReason ??- (!schema.sync_type- ? 'Set up the sync method first'- : schema.status === 'Running'- ? 'A sync is already running'- : undefined)- }+ onClick={fileUpload.onReplaceFile}+ disabledReason={disabledReason ?? fileUpload.replaceDisabledReason}+ data-attr="schema-replace-file">- {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}+ Replace file</LemonButton>- </Tooltip>- )}- </SchemaEditorAction>+ )}+ </SchemaEditorAction>+ ) : (+ <SchemaEditorAction schema={schema}>+ {({ disabledReason }) => (+ <Tooltip+ title={+ schema.sync_type === 'cdc'+ ? 'Trigger a CDC extraction run now.'+ : schema.incremental+ ? 'Sync incremental data since the last run.'+ : 'Sync all data.'+ }+ >+ <LemonButton+ type="primary"+ onClick={() => reloadSchema(schema)}+ disabledReason={+ disabledReason ??+ (!schema.sync_type+ ? 'Set up the sync method first'+ : schema.status === 'Running'+ ? 'A sync is already running'+ : undefined)+ }+ >+ {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}+ </LemonButton>+ </Tooltip>+ )}+ </SchemaEditorAction>+ )}{schema.status === 'Running' && (<SchemaEditorAction schema={schema}>{({ disabledReason }) => (diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsxindex c4f0251f757f..cbad843653c8 100644--- a/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsx+++ b/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsx@@ -17,9 +17,11 @@ import { ProductKey } from '~/queries/schema/schema-general'import { ActivityScope } from '~/types'import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel'+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'import { cleanSourceId } from 'products/data_warehouse/frontend/utils'import { shouldShowManagedSourceMetricsTab, shouldShowManagedSourceSyncsTab } from '../SourceScene/SourceScene'+import { ReplaceFileModal } from '../SourceScene/tabs/ReplaceFileModal'import { SyncsTab } from '../SourceScene/tabs/SyncsTab'import { ConfigurationTab } from './ConfigurationTab'import { MetricsTab } from './MetricsTab'@@ -50,6 +52,8 @@ const SECTION_LABELS: Record<SchemaConfigurationSection, string> = {'danger-zone': 'Danger zone',}+const FILE_UPLOAD_HIDDEN_SECTIONS: readonly SchemaConfigurationSection[] = ['sync-method', 'schedule', 'destinations']+export function SchemaScene({ sourceId, schemaId }: SchemaSceneProps): JSX.Element {if (!sourceId || !schemaId) {return <NotFound object="Data warehouse schema" />@@ -74,8 +78,13 @@ function SchemaSceneContent({ sourceId, schemaId }: SchemaSceneProps): JSX.Eleme// Same gate as the source's Destinations tab. Without it a table offers a destination set that// the sync ignores, because a run only snapshots destinations when the flag is on.const showDestinations = !!featureFlags[FEATURE_FLAGS.WAREHOUSE_MULTI_DESTINATION]+ // A file upload table has one schema that PostHog imports itself: no sync method, schedule or destinations.+ const isFileUpload = isFileUploadSource(source)const visibleSections = SCHEMA_CONFIGURATION_SECTIONS.filter(- (key) => (key !== 'columns' || showColumnsSection) && (key !== 'destinations' || showDestinations)+ (key) =>+ (key !== 'columns' || showColumnsSection) &&+ (key !== 'destinations' || showDestinations) &&+ !(isFileUpload && FILE_UPLOAD_HIDDEN_SECTIONS.includes(key)))useEffect(() => {@@ -106,6 +115,12 @@ function SchemaSceneContent({ sourceId, schemaId }: SchemaSceneProps): JSX.Eleme}}, [showColumnsSection, currentSection, setCurrentSection])+ useEffect(() => {+ if (isFileUpload && FILE_UPLOAD_HIDDEN_SECTIONS.includes(currentSection)) {+ setCurrentSection('details')+ }+ }, [isFileUpload, currentSection, setCurrentSection])+if (schemaDataLoading && !schema) {return (<SceneContent>@@ -205,6 +220,7 @@ function SchemaSceneContent({ sourceId, schemaId }: SchemaSceneProps): JSX.ElemeresourceType={{ type: 'data_pipeline' }}/><LemonTabs activeKey={activeTab} tabs={tabs} onChange={setCurrentTab} sceneInset />+ {isFileUpload && <ReplaceFileModal sourceId={cleanedSourceId} />}</SceneContent>)}diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsxindex 4423fa0f4d10..fcc3594a1db4 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx@@ -66,7 +66,7 @@ import {import type { ExternalDataDestinationApi } from 'products/warehouse_sources/frontend/generated/api.schemas'import { DirectQuerySchemasTab } from './DirectQuerySchemasTab'-import { replaceFileLogic, replaceFileTarget } from './replaceFileLogic'+import { replaceFileDisabledReason, replaceFileLogic, replaceFileTarget } from './replaceFileLogic'import { ReplaceFileModal } from './ReplaceFileModal'import { BulkSyncMethod, bulkSyncMethodDisabledReason, sourceSettingsLogic } from './sourceSettingsLogic'@@ -102,15 +102,6 @@ function fileUploadColumns(isFileUpload: boolean, columns: SchemaColumn[]): Sche.map((column) => (column.className === dimWhenNotSyncing ? { ...column, className: undefined } : column))}-function replaceFileDisabledReason(schema: ExternalDataSourceSchema | undefined): string | undefined {- if (!schema) {- return 'This source has no table'- }- return schema.status === ExternalDataSchemaStatus.Running- ? 'An import is running. Replace the file once it finishes.'- : undefined-}-interface SchemaStatusDisplay {type: LemonTagTypelabel: string@@ -246,7 +237,10 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {<div className="flex items-start justify-between gap-2 mb-2">{isFileUpload ? (<div className="text-secondary text-sm">- This table comes from a file you uploaded. To update its data, replace the file.+ {source?.file_name+ ? `This table comes from the file ${source.file_name}. `+ : 'This table comes from a file you uploaded. '}+ To update its data, replace the file.</div>) : (<div className="flex flex-wrap items-center gap-3 min-w-0">diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.tsindex 061336f1e306..b30ccd678f98 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts@@ -4,18 +4,16 @@ import api from 'lib/api'import { initKeaTests } from '~/test/init'-import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'-import type { PaginatedTableListApi, TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'+import { warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'+import type { TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'import { MAX_FILE_UPLOAD_SIZE_BYTES } from '../../NewSourceScene/fileUploadSourceLogic'-import { detectFileFormat, replaceFileLogic } from './replaceFileLogic'+import { detectFileFormat, replaceFileLogic, replaceFileTarget } from './replaceFileLogic'jest.mock('products/data_warehouse/frontend/generated/api', () => ({- warehouseTablesList: jest.fn(),warehouseTablesReplaceFileCreate: jest.fn(),}))-const mockList = warehouseTablesList as jest.MockedFunction<typeof warehouseTablesList>const mockReplace = warehouseTablesReplaceFileCreate as jest.MockedFunction<typeof warehouseTablesReplaceFileCreate>const UPLOAD = { upload_id: 'upload-1', filename: 'orders.csv', file_format: 'csv', size_bytes: 3 }@@ -43,7 +41,7 @@ describe('replaceFileLogic', () => {})it('uploads the file, replaces the table file and closes the modal', async () => {- logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders', schemaId: 'schema-1' })logic.actions.selectFile(csvFile())await expectLogic(logic, () => {@@ -55,51 +53,24 @@ describe('replaceFileLogic', () => {filename: 'orders.csv',file_format: 'csv',})- expect(mockList).not.toHaveBeenCalled()expect(logic.values.isReplaceFileModalOpen).toBe(false)expect(logic.values.isReplacing).toBe(false)})- it('finds a table the source does not link to yet by its exact name', async () => {- mockList.mockResolvedValue({- count: 2,- results: [- { id: 'other', name: 'orders_backup' },- { id: 'table-9', name: 'orders' },- ],- } as PaginatedTableListApi)- logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })- logic.actions.selectFile(csvFile())-- await expectLogic(logic, () => {- logic.actions.submitReplaceFile()- }).toDispatchActions(['replaceFileSuccess'])-- expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())- })-- it('follows the result pages when the exact name is not on the first page', async () => {- mockList- .mockResolvedValueOnce({- count: 2,- next: 'next-page',- results: [{ id: 'other', name: 'orders_backup' }],- } as PaginatedTableListApi)- .mockResolvedValueOnce({- count: 2,- next: null,- results: [{ id: 'table-9', name: 'orders' }],- } as PaginatedTableListApi)- logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })- logic.actions.selectFile(csvFile())-- await expectLogic(logic, () => {- logic.actions.submitReplaceFile()- }).toDispatchActions(['replaceFileSuccess'])-- expect(mockList).toHaveBeenCalledTimes(2)- expect(mockList).toHaveBeenLastCalledWith(expect.any(String), expect.objectContaining({ offset: 1 }))- expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())+ it.each([+ {+ name: 'a linked table',+ schema: { table: { id: 'table-1', name: 'orders' }, table_id: 'table-1' },+ id: 'table-1',+ },+ { name: 'a table the schema only pins', schema: { table: undefined, table_id: 'table-9' }, id: 'table-9' },+ { name: 'a table without a table_id', schema: { table: { id: 'table-2', name: 'orders' } }, id: 'table-2' },+ { name: 'no table at all', schema: { table: undefined, table_id: null }, id: null },+ ])('targets the table id of $name', ({ schema, id }) => {+ expect(+ replaceFileTarget({ id: 'schema-1', name: 'orders', ...schema } as Parameters<typeof replaceFileTarget>[0])+ .tableId+ ).toBe(id)})it.each([@@ -122,9 +93,7 @@ describe('replaceFileLogic', () => {},{stage: 'replace without a table',- fail: (): void => {- mockList.mockResolvedValue({ count: 0, results: [] } as PaginatedTableListApi)- },+ fail: (): void => {},message: 'Could not find the table orders. Refresh the page and try again.',},])('shows the API error and keeps the modal open when the $stage step fails', async ({ stage, fail, message }) => {@@ -132,6 +101,7 @@ describe('replaceFileLogic', () => {logic.actions.openReplaceFile({tableId: stage === 'replace without a table' ? null : 'table-1',tableName: 'orders',+ schemaId: 'schema-1',})logic.actions.selectFile(csvFile())@@ -147,7 +117,7 @@ describe('replaceFileLogic', () => {it('ignores a second submit while the first one is in flight', async () => {let releaseUpload: (value: typeof UPLOAD) => void = () => {}uploadSpy.mockReturnValue(new Promise<typeof UPLOAD>((resolve) => (releaseUpload = resolve)))- logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders', schemaId: 'schema-1' })logic.actions.selectFile(csvFile())logic.actions.submitReplaceFile()@@ -165,7 +135,7 @@ describe('replaceFileLogic', () => {it('does not submit a file the backend would refuse', () => {const oversized = csvFile()Object.defineProperty(oversized, 'size', { value: MAX_FILE_UPLOAD_SIZE_BYTES + 1 })- logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders', schemaId: 'schema-1' })logic.actions.selectFile(oversized)logic.actions.submitReplaceFile()diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.tsindex 412a45d3cf5f..c7f0c498b626 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts@@ -6,13 +6,14 @@ import { lemonToast } from '@posthog/lemon-ui'import api from 'lib/api'import { teamLogic } from 'scenes/teamLogic'-import { ExternalDataSourceSchema } from '~/types'+import { ExternalDataSchemaStatus, ExternalDataSourceSchema } from '~/types'-import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'+import { warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'import type { TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'import { FILE_UPLOAD_FORMATS, FileUploadFormat } from '../../NewSourceScene/fileUploadSource'import { MAX_FILE_UPLOAD_SIZE_BYTES, fileFormatMismatchError } from '../../NewSourceScene/fileUploadSourceLogic'+import { schemaSceneLogic } from '../../SchemaScene/schemaSceneLogic'import { sourceSettingsLogic } from './sourceSettingsLogic'export const REPLACE_FILE_SUCCESS_MESSAGE = 'Replacing the file. The table updates when the import finishes.'@@ -21,17 +22,36 @@ export const REPLACE_FILE_SUCCESS_MESSAGE = 'Replacing the file. The table updatexport type ReplaceFileStage = 'upload' | 'replace'export interface ReplaceFileTarget {- /** Null for a table that the source names but does not link to yet, such as a migrated upload table. */+ /** Null when the schema has no live table to replace the file of. */tableId: string | nulltableName: string+ /** The schema the file belongs to, so the schema page can reload after a replacement. */+ schemaId: string}export interface ReplaceFileLogicProps {sourceId: string}-export function replaceFileTarget(schema: Pick<ExternalDataSourceSchema, 'name' | 'table'>): ReplaceFileTarget {- return { tableId: schema.table?.id ?? null, tableName: schema.table?.name ?? schema.name }+export function replaceFileTarget(+ schema: Pick<ExternalDataSourceSchema, 'id' | 'name' | 'table' | 'table_id'>+): ReplaceFileTarget {+ return {+ tableId: schema.table_id ?? schema.table?.id ?? null,+ tableName: schema.table?.name ?? schema.name,+ schemaId: schema.id,+ }+}++export function replaceFileDisabledReason(+ schema: Pick<ExternalDataSourceSchema, 'status'> | undefined+): string | undefined {+ if (!schema) {+ return 'This source has no table'+ }+ return schema.status === ExternalDataSchemaStatus.Running+ ? 'An import is running. Replace the file once it finishes.'+ : undefined}/** The one supported format the file's extension names, or null when the extension names none or several. */@@ -56,23 +76,6 @@ function apiErrorMessage(error: any, fallback: string): string {return error?.data?.message ?? error?.detail ?? error?.message ?? fallback}-// A table the migration moved to a file upload source is not linked to it until its first import,-// so the source only knows the table by name. Table names are unique in a project. The search matches-// substrings, so follow the pages until the exact name turns up.-async function findTableIdByName(projectId: string, tableName: string): Promise<string | null> {- for (let offset = 0; ; ) {- const response = await warehouseTablesList(projectId, { search: tableName, include_columns: false, offset })- const match = response.results.find((table) => table.name === tableName)- if (match) {- return match.id- }- if (!response.next || response.results.length === 0) {- return null- }- offset += response.results.length- }-}-// Generated by kea-typegen. Update if you're an agent, ignore if you're human.export interface replaceFileLogicValues {currentTeamId: number | null // teamLogic@@ -241,6 +244,13 @@ export const replaceFileLogic = kea<replaceFileLogicType>([return}const projectId = String(currentTeamId)+ if (!target.tableId) {+ actions.replaceFileFailure(+ `Could not find the table ${target.tableName}. Refresh the page and try again.`+ )+ return+ }+ const { tableId } = targetactions.setReplaceStage('upload')const formData = new FormData()@@ -262,10 +272,6 @@ export const replaceFileLogic = kea<replaceFileLogicType>([actions.setReplaceStage('replace')try {- const tableId = target.tableId ?? (await findTableIdByName(projectId, target.tableName))- if (!tableId) {- throw new Error(`Could not find the table ${target.tableName}. Refresh the page and try again.`)- }const table = await warehouseTablesReplaceFileCreate(projectId, tableId, {upload_id: upload.upload_id,filename: upload.filename,@@ -281,8 +287,12 @@ export const replaceFileLogic = kea<replaceFileLogicType>([},replaceFileSuccess: () => {lemonToast.success(REPLACE_FILE_SUCCESS_MESSAGE)+ const schemaId = values.target?.schemaIdactions.closeReplaceFile()sourceSettingsLogic.findMounted({ id: props.sourceId })?.actions.loadSource()+ if (schemaId) {+ schemaSceneLogic.findMounted({ sourceId: props.sourceId, schemaId })?.actions.loadSchema()+ }},})),])diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 7d78e5e00b8e..b2d9b5fba14b 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -10953,6 +10953,10 @@ export namespace Schemas {name: string;label: string | null;table: Record<string, unknown> | null;+ /**+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.+ */+ table_id: string | null;/*** A schema of an external data source: its sync configuration and the warehouse table it syncs into.*/@@ -11077,6 +11081,7 @@ export namespace Schemas {user_access_level: string | null;api_version: string | null;supported_api_versions: Array<string>;+ file_name: string | null;}> | null;/*** A schema of an external data source: its sync configuration and the warehouse table it syncs into.@@ -15495,6 +15500,10 @@ export namespace Schemas {* Whether this source supports per-column sync selection via `enabled_columns`.*/supports_column_selection: boolean;+ /**+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.+ */+ file_name: string | null;/*** Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.*/@@ -19627,6 +19636,7 @@ export namespace Schemas {user_access_level: string | null;supports_webhooks: boolean;supports_column_selection: boolean;+ file_name: string | null;api_version: string | null;api_version_deprecation: ExternalDataSourceApiVersionDeprecation | null;connection_warning: string | null;diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.pyindex 18950bd54785..59cb9ffce821 100644--- a/products/warehouse_sources/backend/facade/models.py+++ b/products/warehouse_sources/backend/facade/models.py@@ -113,6 +113,7 @@ __all__ = ["asave_datawarehousetable","auto_enable_new_schemas","discovered_schema_finder",+ "self_managed_table_ref","create_datawarehousetable","get_all_schemas_for_source_id","get_schemas_for_direct_reconciliation",diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex 7b1c56eabbea..0d7b7ae86115 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -1,3 +1,4 @@+import uuidimport datetime as dtfrom collections.abc import Callable, Iterablefrom typing import Any, Optional, cast@@ -48,6 +49,7 @@ from products.warehouse_sources.backend.facade.models import (MAX_FULL_REFRESH_INTERVAL_DAYS,SCHEDULED_FULL_REFRESH_SYNC_TYPES,SCHEMA_RESOURCE_ID_METADATA_KEY,+ DataWarehouseTable,ExternalDataJob,ExternalDataSchema,ExternalDataSchemaDestination,@@ -148,6 +150,43 @@ def source_syncs_on_demand(source_type: str) -> bool:return source.syncs_on_demand is True+PINNED_TABLE_IDS_CONTEXT_KEY = "pinned_table_ids"+++def pinned_table_ids(schemas: Iterable[ExternalDataSchema], team_id: int) -> dict[uuid.UUID, uuid.UUID]:+ """The live table each unlinked schema pins in its metadata, such as a migrated table before its first sync.++ One team-scoped query for all schemas. A pin that names another team's table or a deleted one is left out.+ """+ pins: dict[uuid.UUID, uuid.UUID] = {}+ for schema in schemas:+ ref = self_managed_table_ref(schema) if schema.table_id is None else None+ if ref is None:+ continue+ try:+ pins[schema.pk] = uuid.UUID(ref.table_id)+ except ValueError:+ continue+ if not pins:+ return {}+ live = set(+ DataWarehouseTable.objects.filter(team_id=team_id, pk__in=set(pins.values()), deleted=False).values_list(+ "pk", flat=True+ )+ )+ return {schema_id: table_id for schema_id, table_id in pins.items() if table_id in live}+++def linked_table_id(schema: ExternalDataSchema, context: dict[str, Any]) -> uuid.UUID | None:+ """The schema's table id: its linked table, else the live table its metadata pins."""+ if schema.table_id is not None:+ return None if schema.table is None or schema.table.deleted else schema.table_id+ pinned = context.get(PINNED_TABLE_IDS_CONTEXT_KEY)+ if pinned is None:+ pinned = pinned_table_ids([schema], schema.team_id)+ return pinned.get(schema.pk)++def manual_sync_refusal(source_model: ExternalDataSource) -> str | None:try:source = SourceRegistry.get_source(ExternalDataSourceType(source_model.source_type))@@ -552,6 +591,13 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"""A schema of an external data source: its sync configuration and the warehouse table it syncs into."""table = serializers.SerializerMethodField(read_only=True)+ table_id = serializers.SerializerMethodField(+ read_only=True,+ help_text=(+ "The id of the warehouse table this schema syncs into. Set before the first sync for a table "+ "the schema already names, such as a migrated one. Null when there is none."+ ),+ )pending_self_managed_table = serializers.SerializerMethodField(read_only=True,help_text="Whether this schema is linked to an existing self-managed table that its first sync will import. "@@ -738,6 +784,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"name","label","table",+ "table_id","should_sync","last_synced_at","latest_error",@@ -774,6 +821,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"name","label","table",+ "table_id","last_synced_at","latest_error","status",@@ -849,6 +897,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"user_access_level": {"type": "string", "nullable": True},"api_version": {"type": "string", "nullable": True},"supported_api_versions": {"type": "array", "items": {"type": "string"}},+ "file_name": {"type": "string", "nullable": True},},})@@ -868,9 +917,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers# The version the schema falls back to without an override, and the picker's options.source_api_version: str | None = source_impl.resolve_api_version(source.api_version)supported_api_versions = list(source_impl.supported_versions)+ file_name = source_impl.uploaded_file_name(source)except ValueError:source_api_version = Nonesupported_api_versions = []+ file_name = Nonereturn {"id": str(source.id),"source_type": source.source_type,@@ -882,6 +933,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"user_access_level": user_access_level,"api_version": source_api_version,"supported_api_versions": supported_api_versions,+ "file_name": file_name,}def get_user_access_level(self, schema: ExternalDataSchema) -> str | None: # type: ignore[override] # narrows the mixin's Model — DRF always dispatches with this serializer's instance@@ -964,6 +1016,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersdef get_incremental(self, schema: ExternalDataSchema) -> bool:return schema.is_incremental+ @extend_schema_field(serializers.UUIDField(allow_null=True))+ def get_table_id(self, schema: ExternalDataSchema) -> uuid.UUID | None:+ return linked_table_id(schema, self.context)+def get_table(self, schema: ExternalDataSchema) -> Optional[dict]:from products.data_warehouse.backend.presentation.views.table import SimpleTableSerializer@@ -2092,6 +2148,13 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):table = serializers.SerializerMethodField(read_only=True, help_text="The synced warehouse table (id, name, row_count), or null if not yet synced.")+ table_id = serializers.SerializerMethodField(+ read_only=True,+ help_text=(+ "The id of the warehouse table this schema syncs into. Set before the first sync for a table "+ "the schema already names, such as a migrated one. Null when there is none."+ ),+ )status = serializers.SerializerMethodField(read_only=True, help_text="Current sync status for this schema.")pending_self_managed_table = serializers.SerializerMethodField(read_only=True,@@ -2116,6 +2179,7 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):"sync_frequency","latest_error","table",+ "table_id","pending_self_managed_table",]read_only_fields = fields@@ -2137,6 +2201,10 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):return Nonereturn {"id": str(table.id), "name": table.name, "row_count": table.row_count}+ @extend_schema_field(serializers.UUIDField(allow_null=True))+ def get_table_id(self, schema: ExternalDataSchema) -> uuid.UUID | None:+ return linked_table_id(schema, self.context)+@extend_schema_field(serializers.CharField(allow_null=True))def get_status(self, schema: ExternalDataSchema) -> str | None:return schema_display_status(schema)@@ -2233,8 +2301,19 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):# otherwise route POST here, whose serializer create path skips the api_version guards.raise MethodNotAllowed("POST")+ def get_serializer(self, *args: Any, **kwargs: Any) -> serializers.BaseSerializer:+ if "context" not in kwargs:+ kwargs["context"] = self.get_serializer_context()+ if kwargs.get("many") and args:+ context = kwargs["context"]+ context[PINNED_TABLE_IDS_CONTEXT_KEY] = pinned_table_ids(args[0], self.team_id)+ serializer_class = self.get_serializer_class()+ return serializer_class(*args, **kwargs)+def get_serializer_context(self) -> dict[str, Any]:context = super().get_serializer_context()+ if getattr(self, "swagger_fake_view", False):+ return contextcontext["database"] = Database.create_for(team_id=self.team_id, user=cast(User, self.request.user))# Only the single-schema retrieve embeds the parent-source summary (see ExternalDataSchemaSerializer.get_source).context["include_source"] = self.action == "retrieve"diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.pyindex 426b96e8da73..b1e123ec53b4 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py@@ -89,8 +89,10 @@ from products.warehouse_sources.backend.presentation.views.destination_links impset_source_destinations,)from products.warehouse_sources.backend.presentation.views.external_data_schema import (+ PINNED_TABLE_IDS_CONTEXT_KEY,ExternalDataSchemaListSerializer,ExternalDataSchemaSerializer,+ pinned_table_ids,source_supports_column_selection,unsupported_row_filter_reason,)@@ -148,6 +150,10 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializerread_only=True,help_text="Whether this source supports per-column sync selection via `enabled_columns`.",)+ file_name = serializers.SerializerMethodField(+ read_only=True,+ help_text="The name of the file a file upload source imports. Null for other sources or when it is unknown.",+ )# Optional on both create and update. On create, missing values default to `api`# in the viewset to preserve backward compatibility with direct API callers that# predate this field; the in-app UI and MCP tool always send it explicitly.@@ -247,6 +253,7 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer"user_access_level","supports_webhooks","supports_column_selection",+ "file_name","api_version","api_version_deprecation","connection_warning",@@ -266,6 +273,7 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer"access_method","supports_webhooks","supports_column_selection",+ "file_name","api_version","api_version_deprecation","connection_warning",@@ -334,6 +342,14 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializercapture_exception(e)return False+ @extend_schema_field(serializers.CharField(allow_null=True))+ def get_file_name(self, instance: ExternalDataSource) -> str | None:+ try:+ return SourceRegistry.get_source(ExternalDataSourceType(instance.source_type)).uploaded_file_name(instance)+ except Exception as e:+ capture_exception(e)+ return None+def get_supports_column_selection(self, instance: ExternalDataSource) -> bool:return source_supports_column_selection(instance.source_type)@@ -393,7 +409,11 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer# The source list embeds every schema of every source; large projects have tens of thousands.# The list UI only reads a handful of per-schema fields, so serialize the trimmed shape there# and reserve the full serializer for single-source reads.- context = {**self.context, "error_redaction_values": self._error_redaction_values(instance)}+ context = {+ **self.context,+ "error_redaction_values": self._error_redaction_values(instance),+ PINNED_TABLE_IDS_CONTEXT_KEY: pinned_table_ids(schemas, instance.team_id),+ }if self.context.get("schemas_list_only"):return ExternalDataSchemaListSerializer(schemas, many=True, read_only=True, context=context).datareturn ExternalDataSchemaSerializer(schemas, many=True, read_only=True, context=context).datadiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.pyindex 7af300b187c6..f56da6fea593 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py@@ -632,6 +632,10 @@ class _BaseSource(ABC, Generic[ConfigType]):when they can. For a source whose inputs can't run a sync yet, such as one that waits for a file."""return None+ def uploaded_file_name(self, source_model: "ExternalDataSource") -> str | None:+ """The name of the file the source imports, for a source that imports an uploaded file. None by default."""+ return None+class SourceExtractionNotImplementedError(NotImplementedError):"""A source class carries no `source_for_pipeline` of its own, so only the base stub is left.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.pyindex 46e9c8eb23dc..d159856ffd90 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py@@ -9,14 +9,16 @@ from structlog.types import FilteringBoundLoggerfrom products.data_warehouse.backend.facade.api import get_s3_clientfrom products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE+from products.warehouse_sources.backend.facade.contracts import UnreadableSourceInputsErrorfrom products.warehouse_sources.backend.facade.source_config import (DataWarehouseSourceCategory,ReleaseStatus,SourceConfig,)-from products.warehouse_sources.backend.file_uploads import InvalidUploadError, UploadLocation+from products.warehouse_sources.backend.file_uploads import InvalidUploadError, UploadLocation, parse_hosted_uploadfrom products.warehouse_sources.backend.logic.source_inputs import SourceInputsStorefrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+from products.warehouse_sources.backend.models.external_data_schema import self_managed_table_reffrom products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common import configfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import SimpleSource@@ -238,6 +240,34 @@ class FileUploadSource(SimpleSource[FileUploadInputs]):upload_from_job_inputs(source_model.team_id, SourceInputsStore.load_settings(source_model))except InvalidUploadError:return "This table syncs when you upload a new file for it. Upload the file to import it."+ # A sync of the same file would import and bill the same rows again.+ completed_jobs = ExternalDataJob.objects.filter(+ team_id=source_model.team_id, pipeline_id=source_model.pk, status=ExternalDataJob.Status.COMPLETED+ )+ pending_uploads = superseded_uploads(source_model)+ if pending_uploads:+ completed_jobs = completed_jobs.filter(+ created_at__gte=max(entry.superseded_at for entry in pending_uploads)+ )+ if completed_jobs.exists() and source_model.schemas.filter(deleted=False, table__deleted=False).exists():+ return "This table already imported its file. To refresh the data, use Replace file to upload a new one."+ return None++ def uploaded_file_name(self, source_model: ExternalDataSource) -> str | None:+ try:+ return upload_from_job_inputs(source_model.team_id, SourceInputsStore.load_settings(source_model)).filename+ except (InvalidUploadError, UnreadableSourceInputsError):+ pass+ # A migrated source has no upload id yet. Its table still reads the file it was made from.+ schemas = sorted(+ (schema for schema in source_model.schemas.all() if not schema.deleted),+ key=lambda schema: schema.created_at,+ )+ for schema in schemas:+ ref = self_managed_table_ref(schema)+ upload = parse_hosted_upload(source_model.team_id, ref.url_pattern) if ref is not None else None+ if upload is not None:+ return upload.filenamereturn Nonedef has_cleanup_after_sync(self, source_model: ExternalDataSource) -> bool:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.pyindex c8cfe064c5de..075bfbd14c04 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.py@@ -9,8 +9,11 @@ from django.test import override_settingsfrom parameterized import parameterizedfrom products.warehouse_sources.backend.file_uploads import LegacyUpload, UploadLocation+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.source import FileUploadSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.storage import (SupersededUpload,delete_source_uploads,@@ -57,6 +60,66 @@ class TestFileUploadStorage(BaseTest):def _removed(self) -> list[str]:return [call.args[0] for call in self.s3.rm.call_args_list]+ @parameterized.expand(+ [+ ("original", False, -1, "live", True),+ ("replacement_pending", True, -1, "live", False),+ ("replacement_imported", True, 1, "live", True),+ ("table_unlinked", False, -1, "unlinked", False),+ ("table_deleted", False, -1, "deleted", False),+ ]+ )+ def test_manual_sync_refuses_only_an_import_of_the_current_file(+ self, _name: str, replaced: bool, completed_minutes: int, table_state: str, refused: bool+ ) -> None:+ entries = [SupersededUpload(upload=self._upload(OLD_ID), superseded_at=SYNC_STARTED_AT)] if replaced else []+ source = self._source(self._inputs(CURRENT_ID), *entries)+ table = DataWarehouseTable.objects.create(+ team=self.team,+ name="orders",+ format="Parquet",+ url_pattern="https://example.com/orders.parquet",+ deleted=table_state == "deleted",+ )+ ExternalDataSchema.objects.create(+ team=self.team, source=source, name="orders", table=table if table_state != "unlinked" else None+ )+ job = ExternalDataJob.objects.create(team=self.team, pipeline=source, status=ExternalDataJob.Status.COMPLETED)+ ExternalDataJob.objects.filter(pk=job.pk).update(+ created_at=SYNC_STARTED_AT + timedelta(minutes=completed_minutes)+ )+ ExternalDataJob.objects.create(team=self.team, pipeline=source, status=ExternalDataJob.Status.FAILED)++ assert (FileUploadSource().manual_sync_refusal(source) is not None) is refused++ def test_migrated_filename_uses_prefetched_live_schemas_in_creation_order(self) -> None:+ source = self._source({})+ for index, (filename, deleted) in enumerate(+ [("deleted.csv", True), ("first.csv", False), ("second.csv", False)]+ ):+ schema = ExternalDataSchema.objects.create(+ team=self.team,+ source=source,+ name=filename,+ deleted=deleted,+ sync_type_config={+ "schema_metadata": {+ "self_managed_table": {+ "table_id": str(source.pk),+ "url_pattern": self._upload(OLD_ID, filename).url_pattern,+ "format": "CSVWithNames",+ }+ }+ },+ )+ ExternalDataSchema.objects.filter(pk=schema.pk).update(+ created_at=SYNC_STARTED_AT + timedelta(minutes=index)+ )+ source = ExternalDataSource.objects.prefetch_related("schemas").get(pk=source.pk)++ with self.assertNumQueries(0):+ assert FileUploadSource().uploaded_file_name(source) == "first.csv"+def test_a_completed_sync_deletes_only_the_uploads_replaced_before_it_started(self) -> None:before = SupersededUpload(upload=self._upload(OLD_ID), superseded_at=SYNC_STARTED_AT - timedelta(minutes=1))after = SupersededUpload(upload=self._upload(NEWER_ID), superseded_at=SYNC_STARTED_AT + timedelta(minutes=1))diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex e8811c418a53..9ab473e87342 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -18,6 +18,8 @@ from asgiref.sync import sync_to_asyncfrom parameterized import parameterizedfrom rest_framework import statusfrom rest_framework.exceptions import ValidationError+from rest_framework.request import Request+from rest_framework.test import APIRequestFactoryfrom temporalio.service import RPCError, RPCStatusCodefrom posthog.api.test.test_organization import create_organization@@ -45,6 +47,8 @@ from products.warehouse_sources.backend.facade.models import (from products.warehouse_sources.backend.facade.types import ExternalDataSourceTypefrom products.warehouse_sources.backend.presentation.views.external_data_schema import (UNREADABLE_SOURCE_CONFIG_ERROR,+ ExternalDataSchemaSerializer,+ ExternalDataSchemaViewset,schema_display_status,)from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (@@ -5102,6 +5106,38 @@ class TestExternalDataSchemaRetrieveSource(APIBaseTest):assert detail["pending_self_managed_table"] is expectedassert [row["pending_self_managed_table"] for row in listed] == [expected]+ def test_list_serializer_batches_pinned_table_lookups(self) -> None:+ source, _ = self._create()+ schemas = []+ expected = {}+ for index in range(3):+ table = DataWarehouseTable.objects.create(+ team=self.team, name=f"orders_{index}", format="Parquet", url_pattern="https://example.com/a/*.parquet"+ )+ schema = ExternalDataSchema.objects.create(+ team=self.team,+ source=source,+ name=table.name,+ sync_type_config={+ "schema_metadata": {+ "self_managed_table": {+ "table_id": str(table.pk),+ "url_pattern": table.url_pattern,+ "format": table.format,+ }+ }+ },+ )+ schemas.append(schema)+ expected[str(schema.pk)] = table.pk+ view = ExternalDataSchemaViewset()+ view.team_id = self.team.pk++ with self.assertNumQueries(1):+ rows = view.get_serializer(schemas, many=True, context={"include_columns": False}).data++ assert {row["id"]: row["table_id"] for row in rows} == expected+def test_list_omits_source_summary(self):self._create()response = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/")@@ -5561,3 +5597,16 @@ class TestSchemaDisplayStatus(SimpleTestCase):)def test_maps_billing_statuses_to_labels(self, raw_status, expected):assert schema_display_status(ExternalDataSchema(status=raw_status)) == expected+++class TestExternalDataSchemaIntrospection(SimpleTestCase):+ def test_schema_generation_needs_no_team_context(self) -> None:+ view = ExternalDataSchemaViewset()+ view.request = Request(APIRequestFactory().get("/"))+ view.format_kwarg = None+ view.kwargs = {}++ with mock.patch.object(view, "swagger_fake_view", True, create=True):+ serializer = view.get_serializer()+ assert isinstance(serializer, ExternalDataSchemaSerializer)+ assert "table_id" in serializer.fieldsdiff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex 20f7c4af410b..2788dbdf05a6 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -174,6 +174,7 @@ def _configure_source_mock_versioning(mock_get_source) -> None:mock_get_source.return_value.job_inputs_add_connection_host.return_value = Falsemock_get_source.return_value.has_preserved_row_backed_credentials.return_value = Falsemock_get_source.return_value.serialize_config.side_effect = lambda config: config.to_dict()+ mock_get_source.return_value.uploaded_file_name.return_value = Noneclass TestExternalDataSource(APIBaseTest):@@ -279,6 +280,7 @@ class TestExternalDataSource(APIBaseTest):"sync_frequency","latest_error","table",+ "table_id","pending_self_managed_table",},)@@ -3372,6 +3374,7 @@ class TestExternalDataSource(APIBaseTest):"user_access_level","supports_webhooks","supports_column_selection",+ "file_name","api_version","api_version_deprecation","connection_warning",@@ -3396,6 +3399,7 @@ class TestExternalDataSource(APIBaseTest):"pending_self_managed_table": False,"sync_type": schema.sync_type,"table": schema.table,+ "table_id": None,"sync_frequency": sync_frequency_interval_to_sync_frequency(schema.sync_frequency_interval),"sync_time_of_day": schema.sync_time_of_day,"full_refresh_interval_days": None,diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.tsindex 3212b06594ee..216022bdb041 100644--- a/products/warehouse_sources/frontend/generated/api.schemas.ts+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts@@ -407,6 +407,8 @@ export type ExternalDataSchemaApiSource = {/** @nullable */readonly api_version?: string | nullreadonly supported_api_versions?: string[]+ /** @nullable */+ readonly file_name?: string | null} | null/**@@ -419,6 +421,11 @@ export interface ExternalDataSchemaApi {readonly label: string | null/** @nullable */readonly table: ExternalDataSchemaApiTable+ /**+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.+ * @nullable+ */+ readonly table_id: string | nullshould_sync?: boolean/** @nullable */readonly last_synced_at: string | null@@ -604,6 +611,8 @@ export type PatchedExternalDataSchemaApiSource = {/** @nullable */readonly api_version?: string | nullreadonly supported_api_versions?: string[]+ /** @nullable */+ readonly file_name?: string | null} | null/**@@ -616,6 +625,11 @@ export interface PatchedExternalDataSchemaApi {readonly label?: string | null/** @nullable */readonly table?: PatchedExternalDataSchemaApiTable+ /**+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.+ * @nullable+ */+ readonly table_id?: string | nullshould_sync?: boolean/** @nullable */readonly last_synced_at?: string | null@@ -3734,6 +3748,11 @@ export interface ExternalDataSourceSerializersApi {readonly supports_webhooks: boolean/** Whether this source supports per-column sync selection via `enabled_columns`. */readonly supports_column_selection: boolean+ /**+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.+ * @nullable+ */+ readonly file_name: string | null/*** Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.* @nullable@@ -5300,6 +5319,11 @@ export interface PatchedExternalDataSourceSerializersApi {readonly supports_webhooks?: boolean/** Whether this source supports per-column sync selection via `enabled_columns`. */readonly supports_column_selection?: boolean+ /**+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.+ * @nullable+ */+ readonly file_name?: string | null/*** Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.* @nullablediff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 2764df183e02..0bf18993acf8 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -51696,6 +51696,8 @@ export namespace Schemas {/** @nullable */readonly api_version?: string | null;readonly supported_api_versions?: string[];+ /** @nullable */+ readonly file_name?: string | null;} | null;/**@@ -51791,6 +51793,11 @@ export namespace Schemas {readonly label: string | null;/** @nullable */readonly table: ExternalDataSchemaTable;+ /**+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.+ * @nullable+ */+ readonly table_id: string | null;should_sync?: boolean;/** @nullable */readonly last_synced_at: string | null;@@ -55070,6 +55077,11 @@ export namespace Schemas {readonly supports_webhooks: boolean;/** Whether this source supports per-column sync selection via `enabled_columns`. */readonly supports_column_selection: boolean;+ /**+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.+ * @nullable+ */+ readonly file_name: string | null;/*** Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.* @nullable@@ -88508,6 +88520,8 @@ export namespace Schemas {/** @nullable */readonly api_version?: string | null;readonly supported_api_versions?: string[];+ /** @nullable */+ readonly file_name?: string | null;} | null;/**@@ -88520,6 +88534,11 @@ export namespace Schemas {readonly label?: string | null;/** @nullable */readonly table?: PatchedExternalDataSchemaTable;+ /**+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.+ * @nullable+ */+ readonly table_id?: string | null;should_sync?: boolean;/** @nullable */readonly last_synced_at?: string | null;@@ -88731,6 +88750,11 @@ export namespace Schemas {readonly supports_webhooks?: boolean;/** Whether this source supports per-column sync selection via `enabled_columns`. */readonly supports_column_selection?: boolean;+ /**+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.+ * @nullable+ */+ readonly file_name?: string | null;/*** Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.* @nullable