Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): link upload schemas to their tables and stop re-syncing uploaded files (#125561)

Tom Owers023dc5766c18
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index df12ffad4f58..b57d56afd6d6 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6520,6 +6520,8 @@ export interface ExternalDataSource {
latest_error: string | null
last_run_at?: Dayjs
schemas: ExternalDataSourceSchema[]
+ /** The name of the file a file upload source imports. */
+ file_name?: string | null
sync_frequency: DataWarehouseSyncInterval
job_inputs: Record<string, any>
revenue_analytics_config: ExternalDataSourceRevenueAnalyticsConfig
@@ -6704,6 +6706,8 @@ export type IncrementalSyncBlockedReason = IncrementalSyncBlockedReasonEnumApi
export interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema {
table?: SimpleDataWarehouseTable
+ /** The live table the schema syncs into, also before its first sync when the schema already names one. */
+ table_id?: string | null
/** The schema is linked to a self-managed table that its first sync will import. */
pending_self_managed_table?: boolean
incremental: boolean
@@ -6759,6 +6763,8 @@ export interface ExternalDataSchemaSourceSummary {
supports_column_selection?: boolean
supports_row_filters?: boolean
requires_exact_column_metadata?: boolean
+ /** The name of the file a file upload source imports. */
+ file_name?: string | null
user_access_level: AccessControlLevel | null
/** The source's effective vendor API version — what schemas without an override sync on */
api_version?: string | null
diff --git a/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.py b/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.py
index 4bfcd1c76168..0369772afa81 100644
--- a/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.py
+++ b/products/data_warehouse/backend/tests/api/test_postgres_warehouse_migration.py
@@ -33,6 +33,7 @@ def _stub_source_security_gate(source_mock) -> None:
source_mock.job_inputs_add_connection_host.return_value = False
source_mock.has_preserved_row_backed_credentials.return_value = False
source_mock.serialize_config.side_effect = lambda config: config.to_dict()
+ source_mock.uploaded_file_name.return_value = None
class TestPostgresWarehouseMigration(APIBaseTest):
diff --git a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py
index 68a86a8be06e..24d8d31c42fd 100644
--- a/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py
+++ b/products/data_warehouse/backend/tests/api/test_warehouse_table_file_upload.py
@@ -11,6 +11,8 @@ from django.test import override_settings
from parameterized import parameterized
from rest_framework import status
+from posthog.models.team import Team
+
from products.warehouse_sources.backend.facade.api import FILE_FORMAT_READ_HINTS
from products.warehouse_sources.backend.facade.testing import (
find_replaced_self_managed_table,
@@ -881,6 +883,58 @@ class TestFileUploadSources(APIBaseTest):
assert not ExternalDataJob.objects.filter(team=self.team).exists()
self.trigger.assert_not_called()
+ @parameterized.expand(["reload", "resync"])
+ def test_a_table_that_imported_its_file_points_to_replace_file(self, action: str) -> None:
+ _, schema = self._table_with_source()
+ ExternalDataJob.objects.create(
+ team=self.team, pipeline=schema.source, schema=schema, status=ExternalDataJob.Status.COMPLETED
+ )
+ self.trigger.reset_mock()
+
+ response = self.client.post(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/{action}/")
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert "Replace file" in response.json()["message"]
+ self.trigger.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("linked", None),
+ ("pinned", "managed/team_{team}/orders.csv"),
+ ("pinned_in_upload_folder", "file_uploads/team_{team}/{upload}/orders.csv"),
+ ]
+ )
+ def test_the_schema_and_source_name_the_table_and_the_file(self, _name: str, migrated_key: str | None) -> None:
+ if migrated_key is None:
+ table, schema = self._table_with_source()
+ else:
+ table, schema = self._migrated_table(migrated_key)
+ assert schema.table_id is None
+
+ schema_body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()
+ source_body = self.client.get(
+ f"/api/environments/{self.team.pk}/external_data_sources/{schema.source_id}/"
+ ).json()
+
+ assert schema_body["table_id"] == str(table.id)
+ assert source_body["schemas"][0]["table_id"] == str(table.id)
+ assert source_body["file_name"] == "orders.csv"
+ assert schema_body["source"]["file_name"] == "orders.csv"
+ assert source_body["job_inputs"] == {}
+
+ @parameterized.expand(["other_team", "deleted"])
+ def test_a_pinned_table_that_is_not_live_in_the_team_is_not_returned(self, case: str) -> None:
+ table, schema = self._migrated_table("managed/team_{team}/orders.csv")
+ if case == "deleted":
+ DataWarehouseTable.objects.filter(pk=table.pk).update(deleted=True)
+ else:
+ other_team = Team.objects.create(organization=self.organization, name="Other")
+ DataWarehouseTable.objects.filter(pk=table.pk).update(team=other_team)
+
+ body = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}/").json()
+
+ assert body["table_id"] is None
+
@patch(f"{SOURCE_VIEWS}.delete_discover_schemas_schedule")
@patch(f"{SOURCE_VIEWS}.delete_external_data_schedule")
@patch(f"{SOURCE_VIEWS}.bulk_delete_external_data_schedules", return_value=[])
diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
index b94091d25e7f..b4dd92e6af47 100644
--- a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
@@ -41,6 +41,7 @@ import {
SchemaEditorAction,
useSchemaEditorAccess,
} from 'products/data_warehouse/frontend/shared/components/SourceEditorAction'
+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import {
IncrementalSyncBlockedMessageMap,
StatusTagSetting,
@@ -56,6 +57,7 @@ import type { ExternalDataSchemaApi } from 'products/warehouse_sources/frontend/
import { ApiVersionDeprecationBanner } from '../SourceScene/SourceScene'
import { ColumnSelectionPicker } from '../SourceScene/tabs/ColumnSelectionModal'
+import { replaceFileDisabledReason, replaceFileLogic, replaceFileTarget } from '../SourceScene/tabs/replaceFileLogic'
import { RowFilterEditor } from '../SourceScene/tabs/RowFilterEditor'
import { validateRowFilters } from '../SourceScene/tabs/rowFilterUtils'
import { columnAnnotationsLogic } from './columnAnnotationsLogic'
@@ -104,6 +106,14 @@ export function ConfigurationTab({
const { isProjectTime, refreshingSchemas, resyncingSchema, supportsRowFilters } = useValues(logic)
const { setIsProjectTime, updateSchema, reloadSchema, resyncSchema, cancelSchema, deleteTable, refreshSchemas } =
useActions(logic)
+ const { openReplaceFile } = useActions(replaceFileLogic({ sourceId }))
+ const fileUpload: FileUploadDetails | undefined = isFileUploadSource(source)
+ ? {
+ fileName: source?.file_name ?? null,
+ onReplaceFile: () => openReplaceFile(replaceFileTarget(schema)),
+ replaceDisabledReason: replaceFileDisabledReason(schema),
+ }
+ : undefined
switch (section) {
case 'details':
@@ -115,6 +125,7 @@ export function ConfigurationTab({
updateSchema={updateSchema}
onConfigureSyncMethod={onConfigureSyncMethod}
syncHistoryUrl={syncHistoryUrl}
+ fileUpload={fileUpload}
/>
)
case 'sync-method':
@@ -171,6 +182,12 @@ function SectionHeader({ title, description }: { title: string; description?: st
)
}
+interface FileUploadDetails {
+ fileName: string | null
+ onReplaceFile: () => void
+ replaceDisabledReason?: string
+}
+
function DetailsSection({
schema,
reloadSchema,
@@ -178,6 +195,7 @@ function DetailsSection({
updateSchema,
onConfigureSyncMethod,
syncHistoryUrl,
+ fileUpload,
}: {
schema: ExternalDataSourceSchema
reloadSchema: (schema: ExternalDataSourceSchema) => void
@@ -185,6 +203,8 @@ function DetailsSection({
updateSchema: (schema: ExternalDataSourceSchema) => void
onConfigureSyncMethod: () => void
syncHistoryUrl?: string
+ /** Set for a file upload schema, which refreshes by replacing its file and not by syncing. */
+ fileUpload?: FileUploadDetails
}): JSX.Element {
const syncedTableName = schema.table?.hogql_name ?? schema.table?.name
@@ -192,7 +212,11 @@ function DetailsSection({
<div>
<SectionHeader
title="Details"
- description="Enable or disable syncing for this schema, see its current state, and trigger a sync on demand."
+ description={
+ fileUpload
+ ? 'See the current state of this table and replace the file it imports.'
+ : 'Enable or disable syncing for this schema, see its current state, and trigger a sync on demand.'
+ }
/>
<div className="border rounded p-4 bg-surface-primary flex flex-col gap-3">
{schema.incremental_sync_blocked && (
@@ -203,70 +227,78 @@ function DetailsSection({
{IncrementalSyncBlockedMessageMap[schema.incremental_sync_blocked]}
</LemonBanner>
)}
- <div className="flex items-start justify-between gap-4">
- <div className="flex flex-col">
- <span>Enabled</span>
- <span className="text-xs text-muted max-w-md">
- When enabled, this schema runs on the configured schedule and data is imported into PostHog.
- Disabling pauses all syncs — existing data stays in place but is not updated until you
- re-enable.
- </span>
+ {!fileUpload && (
+ <div className="flex items-start justify-between gap-4">
+ <div className="flex flex-col">
+ <span>Enabled</span>
+ <span className="text-xs text-muted max-w-md">
+ When enabled, this schema runs on the configured schedule and data is imported into
+ PostHog. Disabling pauses all syncs — existing data stays in place but is not updated
+ until you re-enable.
+ </span>
+ </div>
+ <SchemaEditorAction schema={schema}>
+ <LemonSwitch
+ checked={schema.should_sync}
+ label={schema.should_sync ? 'Syncing' : 'Disabled'}
+ onChange={(active) => {
+ // A blocked table is not routed away here on purpose. An operator who fixed
+ // the duplicates or added the key at the source has to be able to turn the
+ // table back on themselves; the banner above says what the last run found.
+ if (active && !schema.sync_type) {
+ // No sync method saved yet — open the sync method section to set one up.
+ onConfigureSyncMethod()
+ return
+ }
+ if (!active && schema.sync_type === 'cdc') {
+ LemonDialog.open({
+ title: 'Disable CDC table?',
+ content: (
+ <div className="text-sm text-secondary space-y-2">
+ <p>
+ Disabling <strong>{schema.table?.name ?? schema.name}</strong>{' '}
+ will remove it from the replication publication. Changes made
+ while disabled will be permanently lost.
+ </p>
+ <p>
+ Re-enabling this table will require a{' '}
+ <strong>full resync</strong> to ensure data consistency.
+ </p>
+ </div>
+ ),
+ primaryButton: {
+ children: 'Disable',
+ status: 'danger',
+ onClick: () => updateSchema({ ...schema, should_sync: false }),
+ },
+ secondaryButton: { children: 'Cancel', type: 'tertiary' },
+ })
+ } else if (!active && schema.sync_type === 'webhook') {
+ LemonDialog.open({
+ title: 'Disable webhook sync?',
+ description:
+ 'Turning off this table will stop the webhook from consuming any more data. When you re-enable it, a full refresh sync will need to be completed to ensure no data is missing.',
+ primaryButton: {
+ children: 'Disable',
+ status: 'danger',
+ onClick: () => updateSchema({ ...schema, should_sync: false }),
+ },
+ secondaryButton: { children: 'Cancel' },
+ })
+ } else {
+ updateSchema({ ...schema, should_sync: active })
+ }
+ }}
+ />
+ </SchemaEditorAction>
</div>
- <SchemaEditorAction schema={schema}>
- <LemonSwitch
- checked={schema.should_sync}
- label={schema.should_sync ? 'Syncing' : 'Disabled'}
- onChange={(active) => {
- // A blocked table is not routed away here on purpose. An operator who fixed
- // the duplicates or added the key at the source has to be able to turn the
- // table back on themselves; the banner above says what the last run found.
- if (active && !schema.sync_type) {
- // No sync method saved yet — open the sync method section to set one up.
- onConfigureSyncMethod()
- return
- }
- if (!active && schema.sync_type === 'cdc') {
- LemonDialog.open({
- title: 'Disable CDC table?',
- content: (
- <div className="text-sm text-secondary space-y-2">
- <p>
- Disabling <strong>{schema.table?.name ?? schema.name}</strong> will
- remove it from the replication publication. Changes made while
- disabled will be permanently lost.
- </p>
- <p>
- Re-enabling this table will require a <strong>full resync</strong>{' '}
- to ensure data consistency.
- </p>
- </div>
- ),
- primaryButton: {
- children: 'Disable',
- status: 'danger',
- onClick: () => updateSchema({ ...schema, should_sync: false }),
- },
- secondaryButton: { children: 'Cancel', type: 'tertiary' },
- })
- } else if (!active && schema.sync_type === 'webhook') {
- LemonDialog.open({
- title: 'Disable webhook sync?',
- description:
- 'Turning off this table will stop the webhook from consuming any more data. When you re-enable it, a full refresh sync will need to be completed to ensure no data is missing.',
- primaryButton: {
- children: 'Disable',
- status: 'danger',
- onClick: () => updateSchema({ ...schema, should_sync: false }),
- },
- secondaryButton: { children: 'Cancel' },
- })
- } else {
- updateSchema({ ...schema, should_sync: active })
- }
- }}
- />
- </SchemaEditorAction>
- </div>
+ )}
+ {fileUpload && (
+ <div className="flex items-center justify-between">
+ <span className="text-muted">File</span>
+ <span>{fileUpload.fileName ?? 'Unknown'}</span>
+ </div>
+ )}
<div className="flex items-center justify-between">
<span className="text-muted">Current status</span>
{schema.status ? (
@@ -313,34 +345,49 @@ function DetailsSection({
</div>
</div>
<div className="mt-4 flex gap-2 flex-wrap">
- <SchemaEditorAction schema={schema}>
- {({ disabledReason }) => (
- <Tooltip
- title={
- schema.sync_type === 'cdc'
- ? 'Trigger a CDC extraction run now.'
- : schema.incremental
- ? 'Sync incremental data since the last run.'
- : 'Sync all data.'
- }
- >
+ {fileUpload ? (
+ <SchemaEditorAction schema={schema}>
+ {({ disabledReason }) => (
<LemonButton
type="primary"
- onClick={() => reloadSchema(schema)}
- disabledReason={
- disabledReason ??
- (!schema.sync_type
- ? 'Set up the sync method first'
- : schema.status === 'Running'
- ? 'A sync is already running'
- : undefined)
- }
+ onClick={fileUpload.onReplaceFile}
+ disabledReason={disabledReason ?? fileUpload.replaceDisabledReason}
+ data-attr="schema-replace-file"
>
- {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}
+ Replace file
</LemonButton>
- </Tooltip>
- )}
- </SchemaEditorAction>
+ )}
+ </SchemaEditorAction>
+ ) : (
+ <SchemaEditorAction schema={schema}>
+ {({ disabledReason }) => (
+ <Tooltip
+ title={
+ schema.sync_type === 'cdc'
+ ? 'Trigger a CDC extraction run now.'
+ : schema.incremental
+ ? 'Sync incremental data since the last run.'
+ : 'Sync all data.'
+ }
+ >
+ <LemonButton
+ type="primary"
+ onClick={() => reloadSchema(schema)}
+ disabledReason={
+ disabledReason ??
+ (!schema.sync_type
+ ? 'Set up the sync method first'
+ : schema.status === 'Running'
+ ? 'A sync is already running'
+ : undefined)
+ }
+ >
+ {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}
+ </LemonButton>
+ </Tooltip>
+ )}
+ </SchemaEditorAction>
+ )}
{schema.status === 'Running' && (
<SchemaEditorAction schema={schema}>
{({ disabledReason }) => (
diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsx
index c4f0251f757f..cbad843653c8 100644
--- a/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsx
+++ b/products/data_warehouse/frontend/scenes/SchemaScene/SchemaScene.tsx
@@ -17,9 +17,11 @@ import { ProductKey } from '~/queries/schema/schema-general'
import { ActivityScope } from '~/types'
import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel'
+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import { cleanSourceId } from 'products/data_warehouse/frontend/utils'
import { shouldShowManagedSourceMetricsTab, shouldShowManagedSourceSyncsTab } from '../SourceScene/SourceScene'
+import { ReplaceFileModal } from '../SourceScene/tabs/ReplaceFileModal'
import { SyncsTab } from '../SourceScene/tabs/SyncsTab'
import { ConfigurationTab } from './ConfigurationTab'
import { MetricsTab } from './MetricsTab'
@@ -50,6 +52,8 @@ const SECTION_LABELS: Record<SchemaConfigurationSection, string> = {
'danger-zone': 'Danger zone',
}
+const FILE_UPLOAD_HIDDEN_SECTIONS: readonly SchemaConfigurationSection[] = ['sync-method', 'schedule', 'destinations']
+
export function SchemaScene({ sourceId, schemaId }: SchemaSceneProps): JSX.Element {
if (!sourceId || !schemaId) {
return <NotFound object="Data warehouse schema" />
@@ -74,8 +78,13 @@ function SchemaSceneContent({ sourceId, schemaId }: SchemaSceneProps): JSX.Eleme
// Same gate as the source's Destinations tab. Without it a table offers a destination set that
// the sync ignores, because a run only snapshots destinations when the flag is on.
const showDestinations = !!featureFlags[FEATURE_FLAGS.WAREHOUSE_MULTI_DESTINATION]
+ // A file upload table has one schema that PostHog imports itself: no sync method, schedule or destinations.
+ const isFileUpload = isFileUploadSource(source)
const visibleSections = SCHEMA_CONFIGURATION_SECTIONS.filter(
- (key) => (key !== 'columns' || showColumnsSection) && (key !== 'destinations' || showDestinations)
+ (key) =>
+ (key !== 'columns' || showColumnsSection) &&
+ (key !== 'destinations' || showDestinations) &&
+ !(isFileUpload && FILE_UPLOAD_HIDDEN_SECTIONS.includes(key))
)
useEffect(() => {
@@ -106,6 +115,12 @@ function SchemaSceneContent({ sourceId, schemaId }: SchemaSceneProps): JSX.Eleme
}
}, [showColumnsSection, currentSection, setCurrentSection])
+ useEffect(() => {
+ if (isFileUpload && FILE_UPLOAD_HIDDEN_SECTIONS.includes(currentSection)) {
+ setCurrentSection('details')
+ }
+ }, [isFileUpload, currentSection, setCurrentSection])
+
if (schemaDataLoading && !schema) {
return (
<SceneContent>
@@ -205,6 +220,7 @@ function SchemaSceneContent({ sourceId, schemaId }: SchemaSceneProps): JSX.Eleme
resourceType={{ type: 'data_pipeline' }}
/>
<LemonTabs activeKey={activeTab} tabs={tabs} onChange={setCurrentTab} sceneInset />
+ {isFileUpload && <ReplaceFileModal sourceId={cleanedSourceId} />}
</SceneContent>
)
}
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
index 4423fa0f4d10..fcc3594a1db4 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
@@ -66,7 +66,7 @@ import {
import type { ExternalDataDestinationApi } from 'products/warehouse_sources/frontend/generated/api.schemas'
import { DirectQuerySchemasTab } from './DirectQuerySchemasTab'
-import { replaceFileLogic, replaceFileTarget } from './replaceFileLogic'
+import { replaceFileDisabledReason, replaceFileLogic, replaceFileTarget } from './replaceFileLogic'
import { ReplaceFileModal } from './ReplaceFileModal'
import { BulkSyncMethod, bulkSyncMethodDisabledReason, sourceSettingsLogic } from './sourceSettingsLogic'
@@ -102,15 +102,6 @@ function fileUploadColumns(isFileUpload: boolean, columns: SchemaColumn[]): Sche
.map((column) => (column.className === dimWhenNotSyncing ? { ...column, className: undefined } : column))
}
-function replaceFileDisabledReason(schema: ExternalDataSourceSchema | undefined): string | undefined {
- if (!schema) {
- return 'This source has no table'
- }
- return schema.status === ExternalDataSchemaStatus.Running
- ? 'An import is running. Replace the file once it finishes.'
- : undefined
-}
-
interface SchemaStatusDisplay {
type: LemonTagType
label: string
@@ -246,7 +237,10 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {
<div className="flex items-start justify-between gap-2 mb-2">
{isFileUpload ? (
<div className="text-secondary text-sm">
- This table comes from a file you uploaded. To update its data, replace the file.
+ {source?.file_name
+ ? `This table comes from the file ${source.file_name}. `
+ : 'This table comes from a file you uploaded. '}
+ To update its data, replace the file.
</div>
) : (
<div className="flex flex-wrap items-center gap-3 min-w-0">
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts
index 061336f1e306..b30ccd678f98 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts
@@ -4,18 +4,16 @@ import api from 'lib/api'
import { initKeaTests } from '~/test/init'
-import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'
-import type { PaginatedTableListApi, TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'
+import { warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'
+import type { TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'
import { MAX_FILE_UPLOAD_SIZE_BYTES } from '../../NewSourceScene/fileUploadSourceLogic'
-import { detectFileFormat, replaceFileLogic } from './replaceFileLogic'
+import { detectFileFormat, replaceFileLogic, replaceFileTarget } from './replaceFileLogic'
jest.mock('products/data_warehouse/frontend/generated/api', () => ({
- warehouseTablesList: jest.fn(),
warehouseTablesReplaceFileCreate: jest.fn(),
}))
-const mockList = warehouseTablesList as jest.MockedFunction<typeof warehouseTablesList>
const mockReplace = warehouseTablesReplaceFileCreate as jest.MockedFunction<typeof warehouseTablesReplaceFileCreate>
const UPLOAD = { upload_id: 'upload-1', filename: 'orders.csv', file_format: 'csv', size_bytes: 3 }
@@ -43,7 +41,7 @@ describe('replaceFileLogic', () => {
})
it('uploads the file, replaces the table file and closes the modal', async () => {
- logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })
+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders', schemaId: 'schema-1' })
logic.actions.selectFile(csvFile())
await expectLogic(logic, () => {
@@ -55,51 +53,24 @@ describe('replaceFileLogic', () => {
filename: 'orders.csv',
file_format: 'csv',
})
- expect(mockList).not.toHaveBeenCalled()
expect(logic.values.isReplaceFileModalOpen).toBe(false)
expect(logic.values.isReplacing).toBe(false)
})
- it('finds a table the source does not link to yet by its exact name', async () => {
- mockList.mockResolvedValue({
- count: 2,
- results: [
- { id: 'other', name: 'orders_backup' },
- { id: 'table-9', name: 'orders' },
- ],
- } as PaginatedTableListApi)
- logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })
- logic.actions.selectFile(csvFile())
-
- await expectLogic(logic, () => {
- logic.actions.submitReplaceFile()
- }).toDispatchActions(['replaceFileSuccess'])
-
- expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())
- })
-
- it('follows the result pages when the exact name is not on the first page', async () => {
- mockList
- .mockResolvedValueOnce({
- count: 2,
- next: 'next-page',
- results: [{ id: 'other', name: 'orders_backup' }],
- } as PaginatedTableListApi)
- .mockResolvedValueOnce({
- count: 2,
- next: null,
- results: [{ id: 'table-9', name: 'orders' }],
- } as PaginatedTableListApi)
- logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })
- logic.actions.selectFile(csvFile())
-
- await expectLogic(logic, () => {
- logic.actions.submitReplaceFile()
- }).toDispatchActions(['replaceFileSuccess'])
-
- expect(mockList).toHaveBeenCalledTimes(2)
- expect(mockList).toHaveBeenLastCalledWith(expect.any(String), expect.objectContaining({ offset: 1 }))
- expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())
+ it.each([
+ {
+ name: 'a linked table',
+ schema: { table: { id: 'table-1', name: 'orders' }, table_id: 'table-1' },
+ id: 'table-1',
+ },
+ { name: 'a table the schema only pins', schema: { table: undefined, table_id: 'table-9' }, id: 'table-9' },
+ { name: 'a table without a table_id', schema: { table: { id: 'table-2', name: 'orders' } }, id: 'table-2' },
+ { name: 'no table at all', schema: { table: undefined, table_id: null }, id: null },
+ ])('targets the table id of $name', ({ schema, id }) => {
+ expect(
+ replaceFileTarget({ id: 'schema-1', name: 'orders', ...schema } as Parameters<typeof replaceFileTarget>[0])
+ .tableId
+ ).toBe(id)
})
it.each([
@@ -122,9 +93,7 @@ describe('replaceFileLogic', () => {
},
{
stage: 'replace without a table',
- fail: (): void => {
- mockList.mockResolvedValue({ count: 0, results: [] } as PaginatedTableListApi)
- },
+ fail: (): void => {},
message: 'Could not find the table orders. Refresh the page and try again.',
},
])('shows the API error and keeps the modal open when the $stage step fails', async ({ stage, fail, message }) => {
@@ -132,6 +101,7 @@ describe('replaceFileLogic', () => {
logic.actions.openReplaceFile({
tableId: stage === 'replace without a table' ? null : 'table-1',
tableName: 'orders',
+ schemaId: 'schema-1',
})
logic.actions.selectFile(csvFile())
@@ -147,7 +117,7 @@ describe('replaceFileLogic', () => {
it('ignores a second submit while the first one is in flight', async () => {
let releaseUpload: (value: typeof UPLOAD) => void = () => {}
uploadSpy.mockReturnValue(new Promise<typeof UPLOAD>((resolve) => (releaseUpload = resolve)))
- logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })
+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders', schemaId: 'schema-1' })
logic.actions.selectFile(csvFile())
logic.actions.submitReplaceFile()
@@ -165,7 +135,7 @@ describe('replaceFileLogic', () => {
it('does not submit a file the backend would refuse', () => {
const oversized = csvFile()
Object.defineProperty(oversized, 'size', { value: MAX_FILE_UPLOAD_SIZE_BYTES + 1 })
- logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })
+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders', schemaId: 'schema-1' })
logic.actions.selectFile(oversized)
logic.actions.submitReplaceFile()
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts
index 412a45d3cf5f..c7f0c498b626 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts
@@ -6,13 +6,14 @@ import { lemonToast } from '@posthog/lemon-ui'
import api from 'lib/api'
import { teamLogic } from 'scenes/teamLogic'
-import { ExternalDataSourceSchema } from '~/types'
+import { ExternalDataSchemaStatus, ExternalDataSourceSchema } from '~/types'
-import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'
+import { warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'
import type { TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'
import { FILE_UPLOAD_FORMATS, FileUploadFormat } from '../../NewSourceScene/fileUploadSource'
import { MAX_FILE_UPLOAD_SIZE_BYTES, fileFormatMismatchError } from '../../NewSourceScene/fileUploadSourceLogic'
+import { schemaSceneLogic } from '../../SchemaScene/schemaSceneLogic'
import { sourceSettingsLogic } from './sourceSettingsLogic'
export const REPLACE_FILE_SUCCESS_MESSAGE = 'Replacing the file. The table updates when the import finishes.'
@@ -21,17 +22,36 @@ export const REPLACE_FILE_SUCCESS_MESSAGE = 'Replacing the file. The table updat
export type ReplaceFileStage = 'upload' | 'replace'
export interface ReplaceFileTarget {
- /** Null for a table that the source names but does not link to yet, such as a migrated upload table. */
+ /** Null when the schema has no live table to replace the file of. */
tableId: string | null
tableName: string
+ /** The schema the file belongs to, so the schema page can reload after a replacement. */
+ schemaId: string
}
export interface ReplaceFileLogicProps {
sourceId: string
}
-export function replaceFileTarget(schema: Pick<ExternalDataSourceSchema, 'name' | 'table'>): ReplaceFileTarget {
- return { tableId: schema.table?.id ?? null, tableName: schema.table?.name ?? schema.name }
+export function replaceFileTarget(
+ schema: Pick<ExternalDataSourceSchema, 'id' | 'name' | 'table' | 'table_id'>
+): ReplaceFileTarget {
+ return {
+ tableId: schema.table_id ?? schema.table?.id ?? null,
+ tableName: schema.table?.name ?? schema.name,
+ schemaId: schema.id,
+ }
+}
+
+export function replaceFileDisabledReason(
+ schema: Pick<ExternalDataSourceSchema, 'status'> | undefined
+): string | undefined {
+ if (!schema) {
+ return 'This source has no table'
+ }
+ return schema.status === ExternalDataSchemaStatus.Running
+ ? 'An import is running. Replace the file once it finishes.'
+ : undefined
}
/** The one supported format the file's extension names, or null when the extension names none or several. */
@@ -56,23 +76,6 @@ function apiErrorMessage(error: any, fallback: string): string {
return error?.data?.message ?? error?.detail ?? error?.message ?? fallback
}
-// A table the migration moved to a file upload source is not linked to it until its first import,
-// so the source only knows the table by name. Table names are unique in a project. The search matches
-// substrings, so follow the pages until the exact name turns up.
-async function findTableIdByName(projectId: string, tableName: string): Promise<string | null> {
- for (let offset = 0; ; ) {
- const response = await warehouseTablesList(projectId, { search: tableName, include_columns: false, offset })
- const match = response.results.find((table) => table.name === tableName)
- if (match) {
- return match.id
- }
- if (!response.next || response.results.length === 0) {
- return null
- }
- offset += response.results.length
- }
-}
-
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface replaceFileLogicValues {
currentTeamId: number | null // teamLogic
@@ -241,6 +244,13 @@ export const replaceFileLogic = kea<replaceFileLogicType>([
return
}
const projectId = String(currentTeamId)
+ if (!target.tableId) {
+ actions.replaceFileFailure(
+ `Could not find the table ${target.tableName}. Refresh the page and try again.`
+ )
+ return
+ }
+ const { tableId } = target
actions.setReplaceStage('upload')
const formData = new FormData()
@@ -262,10 +272,6 @@ export const replaceFileLogic = kea<replaceFileLogicType>([
actions.setReplaceStage('replace')
try {
- const tableId = target.tableId ?? (await findTableIdByName(projectId, target.tableName))
- if (!tableId) {
- throw new Error(`Could not find the table ${target.tableName}. Refresh the page and try again.`)
- }
const table = await warehouseTablesReplaceFileCreate(projectId, tableId, {
upload_id: upload.upload_id,
filename: upload.filename,
@@ -281,8 +287,12 @@ export const replaceFileLogic = kea<replaceFileLogicType>([
},
replaceFileSuccess: () => {
lemonToast.success(REPLACE_FILE_SUCCESS_MESSAGE)
+ const schemaId = values.target?.schemaId
actions.closeReplaceFile()
sourceSettingsLogic.findMounted({ id: props.sourceId })?.actions.loadSource()
+ if (schemaId) {
+ schemaSceneLogic.findMounted({ sourceId: props.sourceId, schemaId })?.actions.loadSchema()
+ }
},
})),
])
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 7d78e5e00b8e..b2d9b5fba14b 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10953,6 +10953,10 @@ export namespace Schemas {
name: string;
label: string | null;
table: Record<string, unknown> | null;
+ /**
+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.
+ */
+ table_id: string | null;
/**
* A schema of an external data source: its sync configuration and the warehouse table it syncs into.
*/
@@ -11077,6 +11081,7 @@ export namespace Schemas {
user_access_level: string | null;
api_version: string | null;
supported_api_versions: Array<string>;
+ file_name: string | null;
}> | null;
/**
* A schema of an external data source: its sync configuration and the warehouse table it syncs into.
@@ -15495,6 +15500,10 @@ export namespace Schemas {
* Whether this source supports per-column sync selection via `enabled_columns`.
*/
supports_column_selection: boolean;
+ /**
+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.
+ */
+ file_name: string | null;
/**
* Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.
*/
@@ -19627,6 +19636,7 @@ export namespace Schemas {
user_access_level: string | null;
supports_webhooks: boolean;
supports_column_selection: boolean;
+ file_name: string | null;
api_version: string | null;
api_version_deprecation: ExternalDataSourceApiVersionDeprecation | null;
connection_warning: string | null;
diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.py
index 18950bd54785..59cb9ffce821 100644
--- a/products/warehouse_sources/backend/facade/models.py
+++ b/products/warehouse_sources/backend/facade/models.py
@@ -113,6 +113,7 @@ __all__ = [
"asave_datawarehousetable",
"auto_enable_new_schemas",
"discovered_schema_finder",
+ "self_managed_table_ref",
"create_datawarehousetable",
"get_all_schemas_for_source_id",
"get_schemas_for_direct_reconciliation",
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 7b1c56eabbea..0d7b7ae86115 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -1,3 +1,4 @@
+import uuid
import datetime as dt
from collections.abc import Callable, Iterable
from typing import Any, Optional, cast
@@ -48,6 +49,7 @@ from products.warehouse_sources.backend.facade.models import (
MAX_FULL_REFRESH_INTERVAL_DAYS,
SCHEDULED_FULL_REFRESH_SYNC_TYPES,
SCHEMA_RESOURCE_ID_METADATA_KEY,
+ DataWarehouseTable,
ExternalDataJob,
ExternalDataSchema,
ExternalDataSchemaDestination,
@@ -148,6 +150,43 @@ def source_syncs_on_demand(source_type: str) -> bool:
return source.syncs_on_demand is True
+PINNED_TABLE_IDS_CONTEXT_KEY = "pinned_table_ids"
+
+
+def pinned_table_ids(schemas: Iterable[ExternalDataSchema], team_id: int) -> dict[uuid.UUID, uuid.UUID]:
+ """The live table each unlinked schema pins in its metadata, such as a migrated table before its first sync.
+
+ One team-scoped query for all schemas. A pin that names another team's table or a deleted one is left out.
+ """
+ pins: dict[uuid.UUID, uuid.UUID] = {}
+ for schema in schemas:
+ ref = self_managed_table_ref(schema) if schema.table_id is None else None
+ if ref is None:
+ continue
+ try:
+ pins[schema.pk] = uuid.UUID(ref.table_id)
+ except ValueError:
+ continue
+ if not pins:
+ return {}
+ live = set(
+ DataWarehouseTable.objects.filter(team_id=team_id, pk__in=set(pins.values()), deleted=False).values_list(
+ "pk", flat=True
+ )
+ )
+ return {schema_id: table_id for schema_id, table_id in pins.items() if table_id in live}
+
+
+def linked_table_id(schema: ExternalDataSchema, context: dict[str, Any]) -> uuid.UUID | None:
+ """The schema's table id: its linked table, else the live table its metadata pins."""
+ if schema.table_id is not None:
+ return None if schema.table is None or schema.table.deleted else schema.table_id
+ pinned = context.get(PINNED_TABLE_IDS_CONTEXT_KEY)
+ if pinned is None:
+ pinned = pinned_table_ids([schema], schema.team_id)
+ return pinned.get(schema.pk)
+
+
def manual_sync_refusal(source_model: ExternalDataSource) -> str | None:
try:
source = SourceRegistry.get_source(ExternalDataSourceType(source_model.source_type))
@@ -552,6 +591,13 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"""A schema of an external data source: its sync configuration and the warehouse table it syncs into."""
table = serializers.SerializerMethodField(read_only=True)
+ table_id = serializers.SerializerMethodField(
+ read_only=True,
+ help_text=(
+ "The id of the warehouse table this schema syncs into. Set before the first sync for a table "
+ "the schema already names, such as a migrated one. Null when there is none."
+ ),
+ )
pending_self_managed_table = serializers.SerializerMethodField(
read_only=True,
help_text="Whether this schema is linked to an existing self-managed table that its first sync will import. "
@@ -738,6 +784,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"name",
"label",
"table",
+ "table_id",
"should_sync",
"last_synced_at",
"latest_error",
@@ -774,6 +821,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"name",
"label",
"table",
+ "table_id",
"last_synced_at",
"latest_error",
"status",
@@ -849,6 +897,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"user_access_level": {"type": "string", "nullable": True},
"api_version": {"type": "string", "nullable": True},
"supported_api_versions": {"type": "array", "items": {"type": "string"}},
+ "file_name": {"type": "string", "nullable": True},
},
}
)
@@ -868,9 +917,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
# The version the schema falls back to without an override, and the picker's options.
source_api_version: str | None = source_impl.resolve_api_version(source.api_version)
supported_api_versions = list(source_impl.supported_versions)
+ file_name = source_impl.uploaded_file_name(source)
except ValueError:
source_api_version = None
supported_api_versions = []
+ file_name = None
return {
"id": str(source.id),
"source_type": source.source_type,
@@ -882,6 +933,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"user_access_level": user_access_level,
"api_version": source_api_version,
"supported_api_versions": supported_api_versions,
+ "file_name": file_name,
}
def get_user_access_level(self, schema: ExternalDataSchema) -> str | None: # type: ignore[override] # narrows the mixin's Model — DRF always dispatches with this serializer's instance
@@ -964,6 +1016,10 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
def get_incremental(self, schema: ExternalDataSchema) -> bool:
return schema.is_incremental
+ @extend_schema_field(serializers.UUIDField(allow_null=True))
+ def get_table_id(self, schema: ExternalDataSchema) -> uuid.UUID | None:
+ return linked_table_id(schema, self.context)
+
def get_table(self, schema: ExternalDataSchema) -> Optional[dict]:
from products.data_warehouse.backend.presentation.views.table import SimpleTableSerializer
@@ -2092,6 +2148,13 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):
table = serializers.SerializerMethodField(
read_only=True, help_text="The synced warehouse table (id, name, row_count), or null if not yet synced."
)
+ table_id = serializers.SerializerMethodField(
+ read_only=True,
+ help_text=(
+ "The id of the warehouse table this schema syncs into. Set before the first sync for a table "
+ "the schema already names, such as a migrated one. Null when there is none."
+ ),
+ )
status = serializers.SerializerMethodField(read_only=True, help_text="Current sync status for this schema.")
pending_self_managed_table = serializers.SerializerMethodField(
read_only=True,
@@ -2116,6 +2179,7 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):
"sync_frequency",
"latest_error",
"table",
+ "table_id",
"pending_self_managed_table",
]
read_only_fields = fields
@@ -2137,6 +2201,10 @@ class ExternalDataSchemaListSerializer(serializers.ModelSerializer):
return None
return {"id": str(table.id), "name": table.name, "row_count": table.row_count}
+ @extend_schema_field(serializers.UUIDField(allow_null=True))
+ def get_table_id(self, schema: ExternalDataSchema) -> uuid.UUID | None:
+ return linked_table_id(schema, self.context)
+
@extend_schema_field(serializers.CharField(allow_null=True))
def get_status(self, schema: ExternalDataSchema) -> str | None:
return schema_display_status(schema)
@@ -2233,8 +2301,19 @@ class ExternalDataSchemaViewset(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
# otherwise route POST here, whose serializer create path skips the api_version guards.
raise MethodNotAllowed("POST")
+ def get_serializer(self, *args: Any, **kwargs: Any) -> serializers.BaseSerializer:
+ if "context" not in kwargs:
+ kwargs["context"] = self.get_serializer_context()
+ if kwargs.get("many") and args:
+ context = kwargs["context"]
+ context[PINNED_TABLE_IDS_CONTEXT_KEY] = pinned_table_ids(args[0], self.team_id)
+ serializer_class = self.get_serializer_class()
+ return serializer_class(*args, **kwargs)
+
def get_serializer_context(self) -> dict[str, Any]:
context = super().get_serializer_context()
+ if getattr(self, "swagger_fake_view", False):
+ return context
context["database"] = Database.create_for(team_id=self.team_id, user=cast(User, self.request.user))
# Only the single-schema retrieve embeds the parent-source summary (see ExternalDataSchemaSerializer.get_source).
context["include_source"] = self.action == "retrieve"
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index 426b96e8da73..b1e123ec53b4 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -89,8 +89,10 @@ from products.warehouse_sources.backend.presentation.views.destination_links imp
set_source_destinations,
)
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
+ PINNED_TABLE_IDS_CONTEXT_KEY,
ExternalDataSchemaListSerializer,
ExternalDataSchemaSerializer,
+ pinned_table_ids,
source_supports_column_selection,
unsupported_row_filter_reason,
)
@@ -148,6 +150,10 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
read_only=True,
help_text="Whether this source supports per-column sync selection via `enabled_columns`.",
)
+ file_name = serializers.SerializerMethodField(
+ read_only=True,
+ help_text="The name of the file a file upload source imports. Null for other sources or when it is unknown.",
+ )
# Optional on both create and update. On create, missing values default to `api`
# in the viewset to preserve backward compatibility with direct API callers that
# predate this field; the in-app UI and MCP tool always send it explicitly.
@@ -247,6 +253,7 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
"user_access_level",
"supports_webhooks",
"supports_column_selection",
+ "file_name",
"api_version",
"api_version_deprecation",
"connection_warning",
@@ -266,6 +273,7 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
"access_method",
"supports_webhooks",
"supports_column_selection",
+ "file_name",
"api_version",
"api_version_deprecation",
"connection_warning",
@@ -334,6 +342,14 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
capture_exception(e)
return False
+ @extend_schema_field(serializers.CharField(allow_null=True))
+ def get_file_name(self, instance: ExternalDataSource) -> str | None:
+ try:
+ return SourceRegistry.get_source(ExternalDataSourceType(instance.source_type)).uploaded_file_name(instance)
+ except Exception as e:
+ capture_exception(e)
+ return None
+
def get_supports_column_selection(self, instance: ExternalDataSource) -> bool:
return source_supports_column_selection(instance.source_type)
@@ -393,7 +409,11 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
# The source list embeds every schema of every source; large projects have tens of thousands.
# The list UI only reads a handful of per-schema fields, so serialize the trimmed shape there
# and reserve the full serializer for single-source reads.
- context = {**self.context, "error_redaction_values": self._error_redaction_values(instance)}
+ context = {
+ **self.context,
+ "error_redaction_values": self._error_redaction_values(instance),
+ PINNED_TABLE_IDS_CONTEXT_KEY: pinned_table_ids(schemas, instance.team_id),
+ }
if self.context.get("schemas_list_only"):
return ExternalDataSchemaListSerializer(schemas, many=True, read_only=True, context=context).data
return ExternalDataSchemaSerializer(schemas, many=True, read_only=True, context=context).data
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
index 7af300b187c6..f56da6fea593 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py
@@ -632,6 +632,10 @@ class _BaseSource(ABC, Generic[ConfigType]):
when they can. For a source whose inputs can't run a sync yet, such as one that waits for a file."""
return None
+ def uploaded_file_name(self, source_model: "ExternalDataSource") -> str | None:
+ """The name of the file the source imports, for a source that imports an uploaded file. None by default."""
+ return None
+
class SourceExtractionNotImplementedError(NotImplementedError):
"""A source class carries no `source_for_pipeline` of its own, so only the base stub is left.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py
index 46e9c8eb23dc..d159856ffd90 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/source.py
@@ -9,14 +9,16 @@ from structlog.types import FilteringBoundLogger
from products.data_warehouse.backend.facade.api import get_s3_client
from products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE
+from products.warehouse_sources.backend.facade.contracts import UnreadableSourceInputsError
from products.warehouse_sources.backend.facade.source_config import (
DataWarehouseSourceCategory,
ReleaseStatus,
SourceConfig,
)
-from products.warehouse_sources.backend.file_uploads import InvalidUploadError, UploadLocation
+from products.warehouse_sources.backend.file_uploads import InvalidUploadError, UploadLocation, parse_hosted_upload
from products.warehouse_sources.backend.logic.source_inputs import SourceInputsStore
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.models.external_data_schema import self_managed_table_ref
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import SimpleSource
@@ -238,6 +240,34 @@ class FileUploadSource(SimpleSource[FileUploadInputs]):
upload_from_job_inputs(source_model.team_id, SourceInputsStore.load_settings(source_model))
except InvalidUploadError:
return "This table syncs when you upload a new file for it. Upload the file to import it."
+ # A sync of the same file would import and bill the same rows again.
+ completed_jobs = ExternalDataJob.objects.filter(
+ team_id=source_model.team_id, pipeline_id=source_model.pk, status=ExternalDataJob.Status.COMPLETED
+ )
+ pending_uploads = superseded_uploads(source_model)
+ if pending_uploads:
+ completed_jobs = completed_jobs.filter(
+ created_at__gte=max(entry.superseded_at for entry in pending_uploads)
+ )
+ if completed_jobs.exists() and source_model.schemas.filter(deleted=False, table__deleted=False).exists():
+ return "This table already imported its file. To refresh the data, use Replace file to upload a new one."
+ return None
+
+ def uploaded_file_name(self, source_model: ExternalDataSource) -> str | None:
+ try:
+ return upload_from_job_inputs(source_model.team_id, SourceInputsStore.load_settings(source_model)).filename
+ except (InvalidUploadError, UnreadableSourceInputsError):
+ pass
+ # A migrated source has no upload id yet. Its table still reads the file it was made from.
+ schemas = sorted(
+ (schema for schema in source_model.schemas.all() if not schema.deleted),
+ key=lambda schema: schema.created_at,
+ )
+ for schema in schemas:
+ ref = self_managed_table_ref(schema)
+ upload = parse_hosted_upload(source_model.team_id, ref.url_pattern) if ref is not None else None
+ if upload is not None:
+ return upload.filename
return None
def has_cleanup_after_sync(self, source_model: ExternalDataSource) -> bool:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.py
index c8cfe064c5de..075bfbd14c04 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/file_upload/tests/test_file_upload_storage.py
@@ -9,8 +9,11 @@ from django.test import override_settings
from parameterized import parameterized
from products.warehouse_sources.backend.file_uploads import LegacyUpload, UploadLocation
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.source import FileUploadSource
from products.warehouse_sources.backend.temporal.data_imports.sources.file_upload.storage import (
SupersededUpload,
delete_source_uploads,
@@ -57,6 +60,66 @@ class TestFileUploadStorage(BaseTest):
def _removed(self) -> list[str]:
return [call.args[0] for call in self.s3.rm.call_args_list]
+ @parameterized.expand(
+ [
+ ("original", False, -1, "live", True),
+ ("replacement_pending", True, -1, "live", False),
+ ("replacement_imported", True, 1, "live", True),
+ ("table_unlinked", False, -1, "unlinked", False),
+ ("table_deleted", False, -1, "deleted", False),
+ ]
+ )
+ def test_manual_sync_refuses_only_an_import_of_the_current_file(
+ self, _name: str, replaced: bool, completed_minutes: int, table_state: str, refused: bool
+ ) -> None:
+ entries = [SupersededUpload(upload=self._upload(OLD_ID), superseded_at=SYNC_STARTED_AT)] if replaced else []
+ source = self._source(self._inputs(CURRENT_ID), *entries)
+ table = DataWarehouseTable.objects.create(
+ team=self.team,
+ name="orders",
+ format="Parquet",
+ url_pattern="https://example.com/orders.parquet",
+ deleted=table_state == "deleted",
+ )
+ ExternalDataSchema.objects.create(
+ team=self.team, source=source, name="orders", table=table if table_state != "unlinked" else None
+ )
+ job = ExternalDataJob.objects.create(team=self.team, pipeline=source, status=ExternalDataJob.Status.COMPLETED)
+ ExternalDataJob.objects.filter(pk=job.pk).update(
+ created_at=SYNC_STARTED_AT + timedelta(minutes=completed_minutes)
+ )
+ ExternalDataJob.objects.create(team=self.team, pipeline=source, status=ExternalDataJob.Status.FAILED)
+
+ assert (FileUploadSource().manual_sync_refusal(source) is not None) is refused
+
+ def test_migrated_filename_uses_prefetched_live_schemas_in_creation_order(self) -> None:
+ source = self._source({})
+ for index, (filename, deleted) in enumerate(
+ [("deleted.csv", True), ("first.csv", False), ("second.csv", False)]
+ ):
+ schema = ExternalDataSchema.objects.create(
+ team=self.team,
+ source=source,
+ name=filename,
+ deleted=deleted,
+ sync_type_config={
+ "schema_metadata": {
+ "self_managed_table": {
+ "table_id": str(source.pk),
+ "url_pattern": self._upload(OLD_ID, filename).url_pattern,
+ "format": "CSVWithNames",
+ }
+ }
+ },
+ )
+ ExternalDataSchema.objects.filter(pk=schema.pk).update(
+ created_at=SYNC_STARTED_AT + timedelta(minutes=index)
+ )
+ source = ExternalDataSource.objects.prefetch_related("schemas").get(pk=source.pk)
+
+ with self.assertNumQueries(0):
+ assert FileUploadSource().uploaded_file_name(source) == "first.csv"
+
def test_a_completed_sync_deletes_only_the_uploads_replaced_before_it_started(self) -> None:
before = SupersededUpload(upload=self._upload(OLD_ID), superseded_at=SYNC_STARTED_AT - timedelta(minutes=1))
after = SupersededUpload(upload=self._upload(NEWER_ID), superseded_at=SYNC_STARTED_AT + timedelta(minutes=1))
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index e8811c418a53..9ab473e87342 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -18,6 +18,8 @@ from asgiref.sync import sync_to_async
from parameterized import parameterized
from rest_framework import status
from rest_framework.exceptions import ValidationError
+from rest_framework.request import Request
+from rest_framework.test import APIRequestFactory
from temporalio.service import RPCError, RPCStatusCode
from posthog.api.test.test_organization import create_organization
@@ -45,6 +47,8 @@ from products.warehouse_sources.backend.facade.models import (
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
UNREADABLE_SOURCE_CONFIG_ERROR,
+ ExternalDataSchemaSerializer,
+ ExternalDataSchemaViewset,
schema_display_status,
)
from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (
@@ -5102,6 +5106,38 @@ class TestExternalDataSchemaRetrieveSource(APIBaseTest):
assert detail["pending_self_managed_table"] is expected
assert [row["pending_self_managed_table"] for row in listed] == [expected]
+ def test_list_serializer_batches_pinned_table_lookups(self) -> None:
+ source, _ = self._create()
+ schemas = []
+ expected = {}
+ for index in range(3):
+ table = DataWarehouseTable.objects.create(
+ team=self.team, name=f"orders_{index}", format="Parquet", url_pattern="https://example.com/a/*.parquet"
+ )
+ schema = ExternalDataSchema.objects.create(
+ team=self.team,
+ source=source,
+ name=table.name,
+ sync_type_config={
+ "schema_metadata": {
+ "self_managed_table": {
+ "table_id": str(table.pk),
+ "url_pattern": table.url_pattern,
+ "format": table.format,
+ }
+ }
+ },
+ )
+ schemas.append(schema)
+ expected[str(schema.pk)] = table.pk
+ view = ExternalDataSchemaViewset()
+ view.team_id = self.team.pk
+
+ with self.assertNumQueries(1):
+ rows = view.get_serializer(schemas, many=True, context={"include_columns": False}).data
+
+ assert {row["id"]: row["table_id"] for row in rows} == expected
+
def test_list_omits_source_summary(self):
self._create()
response = self.client.get(f"/api/environments/{self.team.pk}/external_data_schemas/")
@@ -5561,3 +5597,16 @@ class TestSchemaDisplayStatus(SimpleTestCase):
)
def test_maps_billing_statuses_to_labels(self, raw_status, expected):
assert schema_display_status(ExternalDataSchema(status=raw_status)) == expected
+
+
+class TestExternalDataSchemaIntrospection(SimpleTestCase):
+ def test_schema_generation_needs_no_team_context(self) -> None:
+ view = ExternalDataSchemaViewset()
+ view.request = Request(APIRequestFactory().get("/"))
+ view.format_kwarg = None
+ view.kwargs = {}
+
+ with mock.patch.object(view, "swagger_fake_view", True, create=True):
+ serializer = view.get_serializer()
+ assert isinstance(serializer, ExternalDataSchemaSerializer)
+ assert "table_id" in serializer.fields
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 20f7c4af410b..2788dbdf05a6 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -174,6 +174,7 @@ def _configure_source_mock_versioning(mock_get_source) -> None:
mock_get_source.return_value.job_inputs_add_connection_host.return_value = False
mock_get_source.return_value.has_preserved_row_backed_credentials.return_value = False
mock_get_source.return_value.serialize_config.side_effect = lambda config: config.to_dict()
+ mock_get_source.return_value.uploaded_file_name.return_value = None
class TestExternalDataSource(APIBaseTest):
@@ -279,6 +280,7 @@ class TestExternalDataSource(APIBaseTest):
"sync_frequency",
"latest_error",
"table",
+ "table_id",
"pending_self_managed_table",
},
)
@@ -3372,6 +3374,7 @@ class TestExternalDataSource(APIBaseTest):
"user_access_level",
"supports_webhooks",
"supports_column_selection",
+ "file_name",
"api_version",
"api_version_deprecation",
"connection_warning",
@@ -3396,6 +3399,7 @@ class TestExternalDataSource(APIBaseTest):
"pending_self_managed_table": False,
"sync_type": schema.sync_type,
"table": schema.table,
+ "table_id": None,
"sync_frequency": sync_frequency_interval_to_sync_frequency(schema.sync_frequency_interval),
"sync_time_of_day": schema.sync_time_of_day,
"full_refresh_interval_days": None,
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index 3212b06594ee..216022bdb041 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -407,6 +407,8 @@ export type ExternalDataSchemaApiSource = {
/** @nullable */
readonly api_version?: string | null
readonly supported_api_versions?: string[]
+ /** @nullable */
+ readonly file_name?: string | null
} | null
/**
@@ -419,6 +421,11 @@ export interface ExternalDataSchemaApi {
readonly label: string | null
/** @nullable */
readonly table: ExternalDataSchemaApiTable
+ /**
+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.
+ * @nullable
+ */
+ readonly table_id: string | null
should_sync?: boolean
/** @nullable */
readonly last_synced_at: string | null
@@ -604,6 +611,8 @@ export type PatchedExternalDataSchemaApiSource = {
/** @nullable */
readonly api_version?: string | null
readonly supported_api_versions?: string[]
+ /** @nullable */
+ readonly file_name?: string | null
} | null
/**
@@ -616,6 +625,11 @@ export interface PatchedExternalDataSchemaApi {
readonly label?: string | null
/** @nullable */
readonly table?: PatchedExternalDataSchemaApiTable
+ /**
+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.
+ * @nullable
+ */
+ readonly table_id?: string | null
should_sync?: boolean
/** @nullable */
readonly last_synced_at?: string | null
@@ -3734,6 +3748,11 @@ export interface ExternalDataSourceSerializersApi {
readonly supports_webhooks: boolean
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection: boolean
+ /**
+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.
+ * @nullable
+ */
+ readonly file_name: string | null
/**
* Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.
* @nullable
@@ -5300,6 +5319,11 @@ export interface PatchedExternalDataSourceSerializersApi {
readonly supports_webhooks?: boolean
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection?: boolean
+ /**
+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.
+ * @nullable
+ */
+ readonly file_name?: string | null
/**
* Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.
* @nullable
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 2764df183e02..0bf18993acf8 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -51696,6 +51696,8 @@ export namespace Schemas {
/** @nullable */
readonly api_version?: string | null;
readonly supported_api_versions?: string[];
+ /** @nullable */
+ readonly file_name?: string | null;
} | null;
/**
@@ -51791,6 +51793,11 @@ export namespace Schemas {
readonly label: string | null;
/** @nullable */
readonly table: ExternalDataSchemaTable;
+ /**
+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.
+ * @nullable
+ */
+ readonly table_id: string | null;
should_sync?: boolean;
/** @nullable */
readonly last_synced_at: string | null;
@@ -55070,6 +55077,11 @@ export namespace Schemas {
readonly supports_webhooks: boolean;
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection: boolean;
+ /**
+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.
+ * @nullable
+ */
+ readonly file_name: string | null;
/**
* Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.
* @nullable
@@ -88508,6 +88520,8 @@ export namespace Schemas {
/** @nullable */
readonly api_version?: string | null;
readonly supported_api_versions?: string[];
+ /** @nullable */
+ readonly file_name?: string | null;
} | null;
/**
@@ -88520,6 +88534,11 @@ export namespace Schemas {
readonly label?: string | null;
/** @nullable */
readonly table?: PatchedExternalDataSchemaTable;
+ /**
+ * The id of the warehouse table this schema syncs into. Set before the first sync for a table the schema already names, such as a migrated one. Null when there is none.
+ * @nullable
+ */
+ readonly table_id?: string | null;
should_sync?: boolean;
/** @nullable */
readonly last_synced_at?: string | null;
@@ -88731,6 +88750,11 @@ export namespace Schemas {
readonly supports_webhooks?: boolean;
/** Whether this source supports per-column sync selection via `enabled_columns`. */
readonly supports_column_selection?: boolean;
+ /**
+ * The name of the file a file upload source imports. Null for other sources or when it is unknown.
+ * @nullable
+ */
+ readonly file_name?: string | null;
/**
* Vendor API version this source is pinned to (an opaque vendor label, e.g. a Stripe date version). Null resolves to the source type's default version at sync time.
* @nullable