:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): replace files and manage file upload sources in the app (#125530)
Tom Owers34ae120d3d08
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsxindex ac954fdea98f..c9c0cb226975 100644--- a/frontend/src/lib/constants.tsx+++ b/frontend/src/lib/constants.tsx@@ -305,6 +305,7 @@ export const FEATURE_FLAGS = {DATA_MODELING_USAGE: 'data-modeling-usage', // owner: #team-data-modeling, gates the Models usage tabDATA_WAREHOUSE_CUSTOM_SOURCE_AI_BUILDER: 'dwh-custom-source-ai-builder', // owner: #team-warehouse-sourcesDATA_WAREHOUSE_CUSTOM_SOURCE_OAUTH2: 'dwh-custom-source-oauth2', // owner: #team-warehouse-sources+ DATA_WAREHOUSE_OBJECT_STORAGE_SOURCES: 'dwh-object-storage-sources', // owner: #team-warehouse-sourcesDATA_WAREHOUSE_RENAME_SAVED_VIEW: 'data-warehouse-rename-saved-view', // owner: #team-data-modelingDATA_WAREHOUSE_SCENE: 'data-warehouse-scene', // owner: #team-data-modelingDISCUSSIONS_SLACK_SYNC: 'discussions-slack-sync', // owner: #team-platform-featuresdiff --git a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.ts b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.tsindex 232ee2af1e37..1278837eb630 100644--- a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.ts+++ b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.ts@@ -1,6 +1,7 @@import {getDefaultDataWarehouseSourceSceneTab,isManagedSourceSceneId,+ shouldShowManagedSourceConfigurationTab,shouldShowManagedSourceMetricsTab,shouldShowManagedSourceSyncsTab,} from 'products/data_warehouse/frontend/scenes/SourceScene/SourceScene'@@ -35,4 +36,12 @@ describe('DataWarehouseSourceScene', () => {])('hides the metrics tab for direct query sources (source %p, flag %p)', (source, flagEnabled, expected) => {expect(shouldShowManagedSourceMetricsTab(source, flagEnabled)).toEqual(expected)})++ it.each([+ [null, false],+ [{ source_type: 'FileUpload' as const }, false],+ [{ source_type: 'Stripe' as const }, true],+ ])('hides the configuration tab for file upload sources (source %p)', (source, expected) => {+ expect(shouldShowManagedSourceConfigurationTab(source)).toEqual(expected)+ })})diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsx b/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsxindex 916543f54094..824618ecca2e 100644--- a/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsx+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsx@@ -3,6 +3,7 @@ import { Form } from 'kea-forms'import { LemonButton, LemonInput, LemonSelect, Spinner } from '@posthog/lemon-ui'+import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'import { LemonField } from 'lib/lemon-ui/LemonField'import { LemonFileInput } from 'lib/lemon-ui/LemonFileInput/LemonFileInput'import { LemonProgress } from 'lib/lemon-ui/LemonProgress'@@ -13,6 +14,7 @@ import { FILE_UPLOAD_ACCEPT, fileUploadSourceLogic } from '../fileUploadSourceLoexport function FileUploadSourceForm(): JSX.Element {const { fileUpload, isFileUploadSubmitting, uploadStage, uploadPercent } = useValues(fileUploadSourceLogic)const { selectFiles, setTableName, setFileFormat } = useActions(fileUploadSourceLogic)+ const objectStorageSourcesEnabled = useFeatureFlag('DATA_WAREHOUSE_OBJECT_STORAGE_SOURCES')const busyReason = !isFileUploadSubmitting? undefined@@ -56,7 +58,14 @@ export function FileUploadSourceForm(): JSX.Element {/>)}</LemonField>- <div className="mb-4 text-xs text-secondary">Files can be up to 50MB.</div>+ <div className="mb-4 flex flex-col gap-1 text-xs text-secondary">+ <span>Files can be up to 50MB.</span>+ {objectStorageSourcesEnabled && (+ <span data-attr="file-upload-billing-note">+ Rows imported from this file count toward your data warehouse rows synced.+ </span>+ )}+ </div><LemonField name="table_name" label="Table name">{({ value = '' }) => (diff --git a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsxindex 57d31897e257..856182caca85 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx+++ b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx@@ -42,6 +42,7 @@ import {SidePanelTab,} from '~/types'+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'import { cleanSourceId, isSelfManagedSourceId } from 'products/data_warehouse/frontend/utils'import { FailingDestinationsBanner } from './FailingDestinationsBanner'@@ -91,6 +92,13 @@ export function shouldShowManagedSourceMetricsTab(return metricsFeatureFlagEnabled && shouldShowManagedSourceSyncsTab(source)}+// PostHog manages every input of a file upload source, so its configuration has nothing to edit.+export function shouldShowManagedSourceConfigurationTab(+ source: Pick<ExternalDataSource, 'source_type'> | null | undefined+): boolean {+ return !!source && !isFileUploadSource(source)+}+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.export interface sourceSceneLogicValues {breadcrumbName: string@@ -292,6 +300,7 @@ function ManagedSourceTabs({const showWebhookTab = !!source?.supports_webhooksconst showDestinationsTab = !!featureFlags[FEATURE_FLAGS.WAREHOUSE_MULTI_DESTINATION]const showMetricsTab = shouldShowManagedSourceMetricsTab(source, !!featureFlags[FEATURE_FLAGS.DWH_SOURCE_METRICS])+ const showConfigurationTab = shouldShowManagedSourceConfigurationTab(source)useEffect(() => {// Wait until the source has loaded before deciding a tab is unavailable.@@ -310,7 +319,10 @@ function ManagedSourceTabs({if (!showMetricsTab && currentTab === 'metrics') {setCurrentTab('schemas')}- }, [source, showSyncsTab, showWebhookTab, showMetricsTab, currentTab, setCurrentTab])+ if (!showConfigurationTab && currentTab === 'configuration') {+ setCurrentTab('schemas')+ }+ }, [source, showSyncsTab, showWebhookTab, showMetricsTab, showConfigurationTab, currentTab, setCurrentTab])if (sourceLoading && !source) {return <LemonSkeleton className="w-full h-12" />@@ -335,7 +347,9 @@ function ManagedSourceTabs({tabs.push({ label: 'Destinations', key: 'destinations', content: <DestinationsTab id={sourceId} /> })}- tabs.push({ label: 'Configuration', key: 'configuration', content: <ConfigurationTab id={sourceId} /> })+ if (showConfigurationTab) {+ tabs.push({ label: 'Configuration', key: 'configuration', content: <ConfigurationTab id={sourceId} /> })+ }if (showWebhookTab) {tabs.push({diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ReplaceFileModal.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ReplaceFileModal.tsxnew file mode 100644index 000000000000..9033b417fb4a--- /dev/null+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ReplaceFileModal.tsx@@ -0,0 +1,109 @@+import { useActions, useValues } from 'kea'++import { LemonBanner, LemonButton, LemonModal, LemonSelect, Spinner } from '@posthog/lemon-ui'++import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'+import { LemonField } from 'lib/lemon-ui/LemonField'+import { LemonFileInput } from 'lib/lemon-ui/LemonFileInput/LemonFileInput'+import { LemonProgress } from 'lib/lemon-ui/LemonProgress'+import { humanizeBytes } from 'lib/utils/numbers'++import { FILE_UPLOAD_ACCEPT } from '../../NewSourceScene/fileUploadSourceLogic'+import { replaceFileLogic } from './replaceFileLogic'++export function ReplaceFileModal({ sourceId }: { sourceId: string }): JSX.Element {+ const logic = replaceFileLogic({ sourceId })+ const {+ target,+ isReplaceFileModalOpen,+ file,+ fileFormat,+ fileError,+ replaceError,+ replaceStage,+ isReplacing,+ uploadPercent,+ } = useValues(logic)+ const { closeReplaceFile, selectFile, setFileFormat, submitReplaceFile } = useActions(logic)+ const objectStorageSourcesEnabled = useFeatureFlag('DATA_WAREHOUSE_OBJECT_STORAGE_SOURCES')++ const busyReason = isReplacing ? 'The file is being replaced' : undefined++ return (+ <LemonModal+ isOpen={isReplaceFileModalOpen}+ onClose={isReplacing ? undefined : closeReplaceFile}+ closable={!isReplacing}+ title={target ? `Replace the file of ${target.tableName}` : 'Replace file'}+ description="Upload a new file to replace all rows in this table. The table keeps its name, so your queries keep working."+ footer={+ <>+ <LemonButton+ type="secondary"+ onClick={closeReplaceFile}+ disabledReason={busyReason}+ data-attr="replace-file-cancel"+ >+ Cancel+ </LemonButton>+ <LemonButton+ type="primary"+ onClick={submitReplaceFile}+ loading={isReplacing}+ disabledReason={busyReason ?? (!file ? 'Choose a file first' : (fileError ?? undefined))}+ data-attr="replace-file-submit"+ >+ Replace file+ </LemonButton>+ </>+ }+ >+ <div className="flex flex-col gap-3">+ <LemonField.Pure label="File format" className="w-max">+ <LemonSelect+ data-attr="replace-file-format"+ options={[+ { label: 'CSV', value: 'csv' },+ { label: 'JSON', value: 'json' },+ { label: 'Parquet', value: 'parquet' },+ ]}+ value={fileFormat}+ onChange={setFileFormat}+ disabledReason={busyReason}+ />+ </LemonField.Pure>+ <LemonField.Pure label="File" error={fileError ?? undefined} help="Files can be up to 50MB.">+ <LemonFileInput+ multiple={false}+ accept={FILE_UPLOAD_ACCEPT[fileFormat]}+ value={file ? [file] : []}+ onChange={(files) => selectFile(files[0] ?? null)}+ disabledReason={busyReason}+ />+ </LemonField.Pure>+ {objectStorageSourcesEnabled && (+ <div className="text-xs text-secondary">+ Rows imported from this file count toward your data warehouse rows synced.+ </div>+ )}+ {replaceStage === 'upload' ? (+ <div className="flex flex-col gap-1" data-attr="replace-file-progress">+ <LemonProgress percent={uploadPercent} />+ <div className="text-xs text-secondary">+ <span>Uploading file: </span>+ <span translate="no">+ {uploadPercent}% of {humanizeBytes(file?.size ?? null)}+ </span>+ </div>+ </div>+ ) : replaceStage === 'replace' ? (+ <div className="flex items-center gap-2 text-xs text-secondary" data-attr="replace-file-progress">+ <Spinner />+ <span>File uploaded. Starting the import.</span>+ </div>+ ) : null}+ {replaceError && <LemonBanner type="error">{replaceError}</LemonBanner>}+ </div>+ </LemonModal>+ )+}diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsxindex 16c15cd9954f..644dd1257aae 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx@@ -25,6 +25,7 @@ import { FEATURE_FLAGS } from 'lib/constants'import { dayjs } from 'lib/dayjs'import { More } from 'lib/lemon-ui/LemonButton/More'import { LemonMenu } from 'lib/lemon-ui/LemonMenu'+import { LemonTableColumn } from 'lib/lemon-ui/LemonTable'import { LemonTableLink } from 'lib/lemon-ui/LemonTable/LemonTableLink'import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils'@@ -51,6 +52,7 @@ import {SchemaEditorAction,SourceEditorAction,} from 'products/data_warehouse/frontend/shared/components/SourceEditorAction'+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'import { destinationLookupLogic } from 'products/data_warehouse/frontend/shared/logics/destinationLookupLogic'import { sourceManagementLogic } from 'products/data_warehouse/frontend/shared/logics/sourceManagementLogic'import {@@ -64,6 +66,8 @@ import {import type { ExternalDataDestinationApi } from 'products/warehouse_sources/frontend/generated/api.schemas'import { DirectQuerySchemasTab } from './DirectQuerySchemasTab'+import { replaceFileLogic, replaceFileTarget } from './replaceFileLogic'+import { ReplaceFileModal } from './ReplaceFileModal'import { BulkSyncMethod, bulkSyncMethodDisabledReason, sourceSettingsLogic } from './sourceSettingsLogic'const frequencyRank = (frequency: DataWarehouseSyncInterval | null | undefined): number =>@@ -81,6 +85,29 @@ const schemaEditDisabledReason = (schema: ExternalDataSourceSchema): string | nuconst dimWhenNotSyncing = (_: unknown, schema: ExternalDataSourceSchema): string =>schema.should_sync ? '' : 'opacity-60'+type SchemaColumn = LemonTableColumn<ExternalDataSourceSchema, keyof ExternalDataSourceSchema | undefined>++// A file upload schema has no schedule or sync method to choose, and never syncs on its own, so it isn't dimmed.+const FILE_UPLOAD_HIDDEN_COLUMN_KEYS = ['sync_type', 'sync_frequency', 'should_sync']++function fileUploadColumns(isFileUpload: boolean, columns: SchemaColumn[]): SchemaColumn[] {+ if (!isFileUpload) {+ return columns+ }+ return columns+ .filter((column) => !FILE_UPLOAD_HIDDEN_COLUMN_KEYS.includes(column.key ?? ''))+ .map((column) => (column.className === dimWhenNotSyncing ? { ...column, className: undefined } : column))+}++function replaceFileDisabledReason(schema: ExternalDataSourceSchema | undefined): string | undefined {+ if (!schema) {+ return 'This source has no table'+ }+ return schema.status === ExternalDataSchemaStatus.Running+ ? 'An import is running. Replace the file once it finishes.'+ : undefined+}+interface SchemaStatusDisplay {type: LemonTagTypelabel: string@@ -174,7 +201,17 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {const { featureFlags } = useValues(featureFlagLogic)const { loadDestinations } = useActions(destinationLookupLogic)const { destinationsById } = useValues(destinationLookupLogic)+ const { openReplaceFile } = useActions(replaceFileLogic({ sourceId: id }))const showDestinations = !!featureFlags[FEATURE_FLAGS.WAREHOUSE_MULTI_DESTINATION]+ const isFileUpload = isFileUploadSource(source)+ const fileUploadSchema = isFileUpload ? source?.schemas[0] : undefined+ const onReplaceFile = isFileUpload+ ? (schema: ExternalDataSourceSchema): void => openReplaceFile(replaceFileTarget(schema))+ : undefined+ // The schema filters are hidden for file uploads, so a saved filter (e.g. "Show enabled only",+ // which an upload's schema never matches) must not hide its table.+ const visibleSchemas = isFileUpload ? (source?.schemas ?? []) : filteredSchemas+ const visibleSchemaGroups = isFileUpload ? [] : groupedFilteredSchemasuseEffect(() => {if (showDestinations) {@@ -204,58 +241,78 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {return (<><div className="flex items-start justify-between gap-2 mb-2">- <div className="flex flex-wrap items-center gap-3 min-w-0">- <LemonSwitch- checked={showEnabledSchemasOnly}- onChange={setShowEnabledSchemasOnly}- label="Show enabled only"- />- <LemonInput- type="search"- placeholder="Filter schemas"- size="small"- value={schemaNameFilter}- onChange={setSchemaNameFilter}- />- <LemonSelect- size="small"- placeholder="All statuses"- value={statusFilter}- onChange={setStatusFilter}- options={[- { value: null, label: 'All statuses' },- ...schemaFilterOptions.statuses.map((status) => ({ value: status, label: status })),- ]}- />- <LemonSelect- size="small"- placeholder="All sync methods"- value={syncMethodFilter}- onChange={setSyncMethodFilter}- options={[- { value: null, label: 'All sync methods' },- ...schemaFilterOptions.syncMethods.map((method) => ({- value: method === 'none' ? 'none' : method,- label: method === 'none' ? 'Not set up' : SyncTypeLabelMap[method],- })),- ]}- />- <LemonSelect- size="small"- placeholder="All frequencies"- value={frequencyFilter}- onChange={setFrequencyFilter}- options={[- { value: null, label: 'All frequencies' },- ...schemaFilterOptions.frequencies.map((frequency) => ({- value: frequency,- label: SyncFrequencyLabelMap[frequency],- })),- ]}- />- <span className="text-muted text-sm">{pluralize(filteredSchemas.length, 'schema', 'schemas')}</span>- </div>+ {isFileUpload ? (+ <div className="text-secondary text-sm">+ This table comes from a file you uploaded. To update its data, replace the file.+ </div>+ ) : (+ <div className="flex flex-wrap items-center gap-3 min-w-0">+ <LemonSwitch+ checked={showEnabledSchemasOnly}+ onChange={setShowEnabledSchemasOnly}+ label="Show enabled only"+ />+ <LemonInput+ type="search"+ placeholder="Filter schemas"+ size="small"+ value={schemaNameFilter}+ onChange={setSchemaNameFilter}+ />+ <LemonSelect+ size="small"+ placeholder="All statuses"+ value={statusFilter}+ onChange={setStatusFilter}+ options={[+ { value: null, label: 'All statuses' },+ ...schemaFilterOptions.statuses.map((status) => ({ value: status, label: status })),+ ]}+ />+ <LemonSelect+ size="small"+ placeholder="All sync methods"+ value={syncMethodFilter}+ onChange={setSyncMethodFilter}+ options={[+ { value: null, label: 'All sync methods' },+ ...schemaFilterOptions.syncMethods.map((method) => ({+ value: method === 'none' ? 'none' : method,+ label: method === 'none' ? 'Not set up' : SyncTypeLabelMap[method],+ })),+ ]}+ />+ <LemonSelect+ size="small"+ placeholder="All frequencies"+ value={frequencyFilter}+ onChange={setFrequencyFilter}+ options={[+ { value: null, label: 'All frequencies' },+ ...schemaFilterOptions.frequencies.map((frequency) => ({+ value: frequency,+ label: SyncFrequencyLabelMap[frequency],+ })),+ ]}+ />+ <span className="text-muted text-sm">+ {pluralize(filteredSchemas.length, 'schema', 'schemas')}+ </span>+ </div>+ )}<div className="flex items-center gap-2 shrink-0">+ {isFileUpload && (+ <SchemaEditorAction schema={fileUploadSchema ?? null}>+ <LemonButton+ type="primary"+ onClick={() => fileUploadSchema && onReplaceFile?.(fileUploadSchema)}+ disabledReason={replaceFileDisabledReason(fileUploadSchema)}+ data-attr="schemas-tab-replace-file"+ >+ Replace file+ </LemonButton>+ </SchemaEditorAction>+ )}{source?.direct_query_enabled && supportsDirectQuery(source.source_type) && (<LemonButtontype="secondary"@@ -266,59 +323,67 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {Query directly</LemonButton>)}- <SourceEditorAction source={source}>- <LemonButton- type="secondary"- loading={syncingNow}- onClick={() => {- LemonDialog.open({- title: 'Sync all enabled schemas?',- content: (- <div className="text-sm text-secondary">- This will trigger a sync for all schemas you have enabled. New sync jobs- will appear in the Syncs tab.- </div>- ),- primaryButton: {- children: 'Sync now',- type: 'primary',- onClick: () => syncNow(),- },- secondaryButton: { children: 'Cancel', type: 'tertiary' },- })- }}- disabledReason={- sourceLoading- ? 'Source is loading'- : refreshingSchemas- ? 'Schema refresh in progress'- : undefined- }- >- Sync all enabled schemas- </LemonButton>- </SourceEditorAction>- <SourceEditorAction source={source}>- <LemonButton- type="secondary"- loading={refreshingSchemas}- onClick={() => refreshSchemas()}- disabledReason={- sourceLoading ? 'Source is loading' : syncingNow ? 'Sync in progress' : undefined- }- >- Pull new schemas- </LemonButton>- </SourceEditorAction>+ {!isFileUpload && (+ <>+ <SourceEditorAction source={source}>+ <LemonButton+ type="secondary"+ loading={syncingNow}+ onClick={() => {+ LemonDialog.open({+ title: 'Sync all enabled schemas?',+ content: (+ <div className="text-sm text-secondary">+ This will trigger a sync for all schemas you have enabled. New sync+ jobs will appear in the Syncs tab.+ </div>+ ),+ primaryButton: {+ children: 'Sync now',+ type: 'primary',+ onClick: () => syncNow(),+ },+ secondaryButton: { children: 'Cancel', type: 'tertiary' },+ })+ }}+ disabledReason={+ sourceLoading+ ? 'Source is loading'+ : refreshingSchemas+ ? 'Schema refresh in progress'+ : undefined+ }+ >+ Sync all enabled schemas+ </LemonButton>+ </SourceEditorAction>+ <SourceEditorAction source={source}>+ <LemonButton+ type="secondary"+ loading={refreshingSchemas}+ onClick={() => refreshSchemas()}+ disabledReason={+ sourceLoading+ ? 'Source is loading'+ : syncingNow+ ? 'Sync in progress'+ : undefined+ }+ >+ Pull new schemas+ </LemonButton>+ </SourceEditorAction>+ </>+ )}</div></div>- {groupedFilteredSchemas.length > 1 ? (+ {visibleSchemaGroups.length > 1 ? (<div className="border rounded bg-bg-light"><LemonCollapsemultipleembedded- defaultActiveKeys={groupedFilteredSchemas.map((group) => group.schemaName)}- panels={groupedFilteredSchemas.map(({ schemaName, tables }) => ({+ defaultActiveKeys={visibleSchemaGroups.map((group) => group.schemaName)}+ panels={visibleSchemaGroups.map(({ schemaName, tables }) => ({key: schemaName,header: (<div className="flex items-center justify-between gap-3 w-full">@@ -343,6 +408,7 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {cancelSchema={cancelSchema}deleteTable={deleteTable}openAccessControl={openAccessControl}+ onReplaceFile={onReplaceFile}showMetrics={showMetrics}destinationsById={showDestinations ? destinationsById : {}}/>@@ -352,7 +418,7 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {</div>) : (<ManagedSchemaTable- schemas={filteredSchemas}+ schemas={visibleSchemas}isLoading={sourceLoading}source={source}sourceId={id}@@ -363,10 +429,12 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {cancelSchema={cancelSchema}deleteTable={deleteTable}openAccessControl={openAccessControl}+ onReplaceFile={onReplaceFile}showMetrics={showMetrics}destinationsById={showDestinations ? destinationsById : {}}/>)}+ {isFileUpload && <ReplaceFileModal sourceId={id} />}{accessControlSchema?.table && (<AccessControlObjectModalisOpen={!!accessControlSchema}@@ -394,6 +462,8 @@ interface ManagedSchemaTableProps {deleteTable: (schema: ExternalDataSourceSchema) => void/** Undefined when the warehouse access control feature is off. */openAccessControl?: (schema: ExternalDataSourceSchema) => void+ /** Set only for a file upload source, whose tables are refreshed by replacing the file. */+ onReplaceFile?: (schema: ExternalDataSourceSchema) => voidshowMetrics: booleandestinationsById: Record<string, ExternalDataDestinationApi>/** Rendered inside a namespace group — the group header already shows the namespace, so strip it from row names. */@@ -412,6 +482,7 @@ function ManagedSchemaTable({cancelSchema,deleteTable,openAccessControl,+ onReplaceFile,showMetrics,destinationsById,inSchemaGroup = false,@@ -427,25 +498,31 @@ function ManagedSchemaTable({}}, [isLoading, initialLoad])+ const isFileUpload = !!onReplaceFile+return (<LemonTabledataSource={schemas}loading={initialLoad}disableTableWhileLoading={false}pagination={{ pageSize: 100, hideOnSinglePage: true }}- bulkSelection={{- getKey: (schema) => schema.id,- isRowSelectable: (schema) => {- const reason = schemaEditDisabledReason(schema)- return reason ? { disabledReason: reason } : true- },- noun: ['schema', 'schemas'],- barClassName: 'mb-2',- renderActions: (ctx) => (- <SchemaBulkActions schemas={ctx.selectedRecords} clearSelection={ctx.clearSelection} />- ),- }}- columns={[+ bulkSelection={+ isFileUpload+ ? undefined+ : {+ getKey: (schema) => schema.id,+ isRowSelectable: (schema) => {+ const reason = schemaEditDisabledReason(schema)+ return reason ? { disabledReason: reason } : true+ },+ noun: ['schema', 'schemas'],+ barClassName: 'mb-2',+ renderActions: (ctx) => (+ <SchemaBulkActions schemas={ctx.selectedRecords} clearSelection={ctx.clearSelection} />+ ),+ }+ }+ columns={fileUploadColumns(isFileUpload, [{title: 'Schema',key: 'name',@@ -728,14 +805,20 @@ function ManagedSchemaTable({}return (<div className="flex justify-end items-center gap-1">- <LemonButton- type="secondary"- size="xsmall"- to={urls.dataWarehouseSourceSchema(prefixedSourceId, schema.id, 'configuration')}- disabledReason={schemaEditDisabledReason(schema)}- >- Configure- </LemonButton>+ {!isFileUpload && (+ <LemonButton+ type="secondary"+ size="xsmall"+ to={urls.dataWarehouseSourceSchema(+ prefixedSourceId,+ schema.id,+ 'configuration'+ )}+ disabledReason={schemaEditDisabledReason(schema)}+ >+ Configure+ </LemonButton>+ )}<SchemaRowMoresource={source}schema={schema}@@ -744,12 +827,13 @@ function ManagedSchemaTable({cancelSchema={cancelSchema}deleteTable={deleteTable}onOpenAccessControl={openAccessControl}+ onReplaceFile={onReplaceFile}/></div>)},},- ]}+ ])}/>)}@@ -947,6 +1031,7 @@ function SchemaRowMore({cancelSchema,deleteTable,onOpenAccessControl,+ onReplaceFile,}: {source: ExternalDataSource | nullschema: ExternalDataSourceSchema@@ -955,8 +1040,11 @@ function SchemaRowMore({cancelSchema: (schema: ExternalDataSourceSchema) => voiddeleteTable: (schema: ExternalDataSourceSchema) => voidonOpenAccessControl?: (schema: ExternalDataSourceSchema) => void+ onReplaceFile?: (schema: ExternalDataSourceSchema) => void}): JSX.Element {const { pausePolling, resumePolling } = useActions(sourceSettingsLogic)+ // An uploaded file has no outside system whose data stays untouched.+ const externalSourceType = isFileUploadSource(source) ? null : source?.source_typereturn (<SchemaEditorAction schema={schema}>@@ -980,31 +1068,44 @@ function SchemaRowMore({Access control</LemonButton>)}- <Tooltip- title={- schema.sync_type === 'cdc'- ? 'Trigger a CDC extraction run now.'- : schema.incremental- ? 'Sync incremental data since the last run.'- : 'Sync all data.'- }- >+ {onReplaceFile ? (<LemonButtontype="tertiary"size="xsmall"fullWidth- onClick={() => reloadSchema(schema)}- disabledReason={- !schema.sync_type- ? 'Set up the sync method first'- : schema.status === 'Running'- ? 'A sync is already running'- : undefined- }+ onClick={() => onReplaceFile(schema)}+ disabledReason={replaceFileDisabledReason(schema)}+ data-attr="schema-row-replace-file">- {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}+ Replace file</LemonButton>- </Tooltip>+ ) : (+ <Tooltip+ title={+ schema.sync_type === 'cdc'+ ? 'Trigger a CDC extraction run now.'+ : schema.incremental+ ? 'Sync incremental data since the last run.'+ : 'Sync all data.'+ }+ >+ <LemonButton+ type="tertiary"+ size="xsmall"+ fullWidth+ onClick={() => reloadSchema(schema)}+ disabledReason={+ !schema.sync_type+ ? 'Set up the sync method first'+ : schema.status === 'Running'+ ? 'A sync is already running'+ : undefined+ }+ >+ {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}+ </LemonButton>+ </Tooltip>+ )}{schema.status === 'Running' && (<LemonButtontype="tertiary"@@ -1078,8 +1179,8 @@ function SchemaRowMore({{schema.table && (<Tooltiptitle={`Delete this table from PostHog. ${- source?.source_type- ? `This will not delete the data in ${source.source_type}`+ externalSourceType+ ? `This will not delete the data in ${externalSourceType}`: ''}`}>@@ -1092,8 +1193,8 @@ function SchemaRowMore({LemonDialog.open({title: `Delete ${schema.table?.name ?? schema.name} from PostHog?`,description: `This also turns off syncing for the table.${- source?.source_type- ? ` The data in ${source.source_type} will not be touched.`+ externalSourceType+ ? ` The data in ${externalSourceType} will not be touched.`: ''}`,primaryButton: {diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.tsnew file mode 100644index 000000000000..061336f1e306--- /dev/null+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts@@ -0,0 +1,193 @@+import { expectLogic } from 'kea-test-utils'++import api from 'lib/api'++import { initKeaTests } from '~/test/init'++import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'+import type { PaginatedTableListApi, TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'++import { MAX_FILE_UPLOAD_SIZE_BYTES } from '../../NewSourceScene/fileUploadSourceLogic'+import { detectFileFormat, replaceFileLogic } from './replaceFileLogic'++jest.mock('products/data_warehouse/frontend/generated/api', () => ({+ warehouseTablesList: jest.fn(),+ warehouseTablesReplaceFileCreate: jest.fn(),+}))++const mockList = warehouseTablesList as jest.MockedFunction<typeof warehouseTablesList>+const mockReplace = warehouseTablesReplaceFileCreate as jest.MockedFunction<typeof warehouseTablesReplaceFileCreate>++const UPLOAD = { upload_id: 'upload-1', filename: 'orders.csv', file_format: 'csv', size_bytes: 3 }++function csvFile(name = 'orders.csv'): File {+ return new File(['a,b'], name)+}++describe('replaceFileLogic', () => {+ let logic: ReturnType<typeof replaceFileLogic.build>+ let uploadSpy: jest.SpyInstance++ beforeEach(() => {+ initKeaTests()+ uploadSpy = jest.spyOn(api.dataWarehouseTables, 'uploadFile').mockResolvedValue(UPLOAD)+ mockReplace.mockResolvedValue({ id: 'table-1', name: 'orders' } as TableApi)+ logic = replaceFileLogic({ sourceId: 'source-1' })+ logic.mount()+ })++ afterEach(() => {+ logic.unmount()+ jest.clearAllMocks()+ uploadSpy.mockRestore()+ })++ it('uploads the file, replaces the table file and closes the modal', async () => {+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })+ logic.actions.selectFile(csvFile())++ await expectLogic(logic, () => {+ logic.actions.submitReplaceFile()+ }).toDispatchActions(['replaceFileSuccess', 'closeReplaceFile'])++ expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-1', {+ upload_id: 'upload-1',+ filename: 'orders.csv',+ file_format: 'csv',+ })+ expect(mockList).not.toHaveBeenCalled()+ expect(logic.values.isReplaceFileModalOpen).toBe(false)+ expect(logic.values.isReplacing).toBe(false)+ })++ it('finds a table the source does not link to yet by its exact name', async () => {+ mockList.mockResolvedValue({+ count: 2,+ results: [+ { id: 'other', name: 'orders_backup' },+ { id: 'table-9', name: 'orders' },+ ],+ } as PaginatedTableListApi)+ logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })+ logic.actions.selectFile(csvFile())++ await expectLogic(logic, () => {+ logic.actions.submitReplaceFile()+ }).toDispatchActions(['replaceFileSuccess'])++ expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())+ })++ it('follows the result pages when the exact name is not on the first page', async () => {+ mockList+ .mockResolvedValueOnce({+ count: 2,+ next: 'next-page',+ results: [{ id: 'other', name: 'orders_backup' }],+ } as PaginatedTableListApi)+ .mockResolvedValueOnce({+ count: 2,+ next: null,+ results: [{ id: 'table-9', name: 'orders' }],+ } as PaginatedTableListApi)+ logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })+ logic.actions.selectFile(csvFile())++ await expectLogic(logic, () => {+ logic.actions.submitReplaceFile()+ }).toDispatchActions(['replaceFileSuccess'])++ expect(mockList).toHaveBeenCalledTimes(2)+ expect(mockList).toHaveBeenLastCalledWith(expect.any(String), expect.objectContaining({ offset: 1 }))+ expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())+ })++ it.each([+ {+ stage: 'upload',+ fail: (): void => {+ uploadSpy.mockRejectedValue({ status: 400, data: { message: 'Invalid filename' } })+ },+ message: 'Invalid filename',+ },+ {+ stage: 'replace',+ fail: (): void => {+ mockReplace.mockRejectedValue({+ status: 400,+ data: { message: 'A sync of this table is running. Replace the file once it finishes.' },+ })+ },+ message: 'A sync of this table is running. Replace the file once it finishes.',+ },+ {+ stage: 'replace without a table',+ fail: (): void => {+ mockList.mockResolvedValue({ count: 0, results: [] } as PaginatedTableListApi)+ },+ message: 'Could not find the table orders. Refresh the page and try again.',+ },+ ])('shows the API error and keeps the modal open when the $stage step fails', async ({ stage, fail, message }) => {+ fail()+ logic.actions.openReplaceFile({+ tableId: stage === 'replace without a table' ? null : 'table-1',+ tableName: 'orders',+ })+ logic.actions.selectFile(csvFile())++ await expectLogic(logic, () => {+ logic.actions.submitReplaceFile()+ }).toDispatchActions(['replaceFileFailure'])++ expect(logic.values.replaceError).toEqual(message)+ expect(logic.values.isReplaceFileModalOpen).toBe(true)+ expect(logic.values.isReplacing).toBe(false)+ })++ it('ignores a second submit while the first one is in flight', async () => {+ let releaseUpload: (value: typeof UPLOAD) => void = () => {}+ uploadSpy.mockReturnValue(new Promise<typeof UPLOAD>((resolve) => (releaseUpload = resolve)))+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })+ logic.actions.selectFile(csvFile())++ logic.actions.submitReplaceFile()+ logic.actions.submitReplaceFile()+ expect(logic.values.isReplacing).toBe(true)++ await expectLogic(logic, () => {+ releaseUpload(UPLOAD)+ }).toDispatchActions(['replaceFileSuccess'])++ expect(uploadSpy).toHaveBeenCalledTimes(1)+ expect(mockReplace).toHaveBeenCalledTimes(1)+ })++ it('does not submit a file the backend would refuse', () => {+ const oversized = csvFile()+ Object.defineProperty(oversized, 'size', { value: MAX_FILE_UPLOAD_SIZE_BYTES + 1 })+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })+ logic.actions.selectFile(oversized)++ logic.actions.submitReplaceFile()++ expect(logic.values.fileError).toContain('50MB')+ expect(uploadSpy).not.toHaveBeenCalled()+ })++ it.each([+ ['orders.csv', 'csv'],+ ['events.ndjson', 'json'],+ ['events.pqt', 'parquet'],+ ['export.txt', null],+ ['sheet.xlsx', null],+ ])('detects the format of %s as %s', (filename, expected) => {+ expect(detectFileFormat(filename)).toEqual(expected)+ })++ it('switches the format to match the chosen file', () => {+ logic.actions.selectFile(csvFile('events.parquet'))++ expect(logic.values.fileFormat).toEqual('parquet')+ expect(logic.values.fileError).toBeNull()+ })+})diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.tsnew file mode 100644index 000000000000..412a45d3cf5f--- /dev/null+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts@@ -0,0 +1,288 @@+import { MakeLogicType, actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'+import posthog from 'posthog-js'++import { lemonToast } from '@posthog/lemon-ui'++import api from 'lib/api'+import { teamLogic } from 'scenes/teamLogic'++import { ExternalDataSourceSchema } from '~/types'++import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'+import type { TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'++import { FILE_UPLOAD_FORMATS, FileUploadFormat } from '../../NewSourceScene/fileUploadSource'+import { MAX_FILE_UPLOAD_SIZE_BYTES, fileFormatMismatchError } from '../../NewSourceScene/fileUploadSourceLogic'+import { sourceSettingsLogic } from './sourceSettingsLogic'++export const REPLACE_FILE_SUCCESS_MESSAGE = 'Replacing the file. The table updates when the import finishes.'++/** Which of the two requests a replacement is on. */+export type ReplaceFileStage = 'upload' | 'replace'++export interface ReplaceFileTarget {+ /** Null for a table that the source names but does not link to yet, such as a migrated upload table. */+ tableId: string | null+ tableName: string+}++export interface ReplaceFileLogicProps {+ sourceId: string+}++export function replaceFileTarget(schema: Pick<ExternalDataSourceSchema, 'name' | 'table'>): ReplaceFileTarget {+ return { tableId: schema.table?.id ?? null, tableName: schema.table?.name ?? schema.name }+}++/** The one supported format the file's extension names, or null when the extension names none or several. */+export function detectFileFormat(filename: string): FileUploadFormat | null {+ const matches = FILE_UPLOAD_FORMATS.map(({ format }) => format).filter(+ (format) => !fileFormatMismatchError(filename, format)+ )+ return matches.length === 1 ? matches[0] : null+}++export function replaceFileError(file: File | null, fileFormat: FileUploadFormat): string | null {+ if (!file) {+ return null+ }+ if (file.size > MAX_FILE_UPLOAD_SIZE_BYTES) {+ return 'This file is larger than the 50MB upload limit.'+ }+ return fileFormatMismatchError(file.name, fileFormat) ?? null+}++function apiErrorMessage(error: any, fallback: string): string {+ return error?.data?.message ?? error?.detail ?? error?.message ?? fallback+}++// A table the migration moved to a file upload source is not linked to it until its first import,+// so the source only knows the table by name. Table names are unique in a project. The search matches+// substrings, so follow the pages until the exact name turns up.+async function findTableIdByName(projectId: string, tableName: string): Promise<string | null> {+ for (let offset = 0; ; ) {+ const response = await warehouseTablesList(projectId, { search: tableName, include_columns: false, offset })+ const match = response.results.find((table) => table.name === tableName)+ if (match) {+ return match.id+ }+ if (!response.next || response.results.length === 0) {+ return null+ }+ offset += response.results.length+ }+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface replaceFileLogicValues {+ currentTeamId: number | null // teamLogic+ file: File | null+ fileError: string | null+ fileFormat: FileUploadFormat+ isReplaceFileModalOpen: boolean+ isReplacing: boolean+ replaceError: string | null+ replaceStage: ReplaceFileStage | null+ target: ReplaceFileTarget | null+ uploadPercent: number+ uploadedBytes: number+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface replaceFileLogicActions {+ closeReplaceFile: () => {+ value: true+ }+ openReplaceFile: (target: ReplaceFileTarget) => {+ target: ReplaceFileTarget+ }+ replaceFileFailure: (error: string) => {+ error: string+ }+ replaceFileSuccess: (table: TableApi) => {+ table: TableApi+ }+ selectFile: (file: File | null) => {+ file: File | null+ }+ setFileFormat: (fileFormat: FileUploadFormat) => {+ fileFormat: FileUploadFormat+ }+ setReplaceStage: (stage: ReplaceFileStage) => {+ stage: ReplaceFileStage+ }+ setUploadedBytes: (uploadedBytes: number) => {+ uploadedBytes: number+ }+ submitReplaceFile: () => {+ value: true+ }+}++// Generated by kea-typegen. Update if you're an agent, ignore if you're human.+export interface replaceFileLogicMeta {+ key: string+ __keaTypeGenInternalSelectorTypes: {+ isReplaceFileModalOpen: (target: ReplaceFileTarget | null) => boolean+ isReplacing: (replaceStage: ReplaceFileStage | null) => boolean+ fileError: (file: File | null, fileFormat: FileUploadFormat) => string | null+ uploadPercent: (uploadedBytes: number, file: File | null) => number+ }+}++export type replaceFileLogicType = MakeLogicType<+ replaceFileLogicValues,+ replaceFileLogicActions,+ ReplaceFileLogicProps,+ replaceFileLogicMeta+>++export const replaceFileLogic = kea<replaceFileLogicType>([+ props({} as ReplaceFileLogicProps),+ key(({ sourceId }) => sourceId),+ path((key) => ['products', 'dataWarehouse', 'replaceFileLogic', key]),+ connect(() => ({+ values: [teamLogic, ['currentTeamId']],+ })),+ actions({+ openReplaceFile: (target: ReplaceFileTarget) => ({ target }),+ closeReplaceFile: true,+ selectFile: (file: File | null) => ({ file }),+ setFileFormat: (fileFormat: FileUploadFormat) => ({ fileFormat }),+ submitReplaceFile: true,+ setReplaceStage: (stage: ReplaceFileStage) => ({ stage }),+ setUploadedBytes: (uploadedBytes: number) => ({ uploadedBytes }),+ replaceFileSuccess: (table: TableApi) => ({ table }),+ replaceFileFailure: (error: string) => ({ error }),+ }),+ reducers({+ target: [+ null as ReplaceFileTarget | null,+ {+ openReplaceFile: (_, { target }) => target,+ closeReplaceFile: () => null,+ },+ ],+ file: [+ null as File | null,+ {+ selectFile: (_, { file }) => file,+ openReplaceFile: () => null,+ },+ ],+ fileFormat: [+ 'csv' as FileUploadFormat,+ {+ setFileFormat: (_, { fileFormat }) => fileFormat,+ },+ ],+ replaceStage: [+ null as ReplaceFileStage | null,+ {+ setReplaceStage: (_, { stage }) => stage,+ replaceFileSuccess: () => null,+ replaceFileFailure: () => null,+ },+ ],+ uploadedBytes: [+ 0,+ {+ setUploadedBytes: (_, { uploadedBytes }) => uploadedBytes,+ openReplaceFile: () => 0,+ selectFile: () => 0,+ },+ ],+ replaceError: [+ null as string | null,+ {+ replaceFileFailure: (_, { error }) => error,+ openReplaceFile: () => null,+ selectFile: () => null,+ setFileFormat: () => null,+ setReplaceStage: () => null,+ },+ ],+ }),+ selectors({+ isReplaceFileModalOpen: [(s) => [s.target], (target: ReplaceFileTarget | null): boolean => target !== null],+ isReplacing: [+ (s) => [s.replaceStage],+ (replaceStage: ReplaceFileStage | null): boolean => replaceStage !== null,+ ],+ fileError: [+ (s) => [s.file, s.fileFormat],+ (file: File | null, fileFormat: FileUploadFormat): string | null => replaceFileError(file, fileFormat),+ ],+ uploadPercent: [+ (s) => [s.uploadedBytes, s.file],+ (uploadedBytes: number, file: File | null): number => {+ if (!file?.size) {+ return 0+ }+ // Multipart framing makes the sent byte count overshoot the file size at the end.+ return Math.min(100, Math.round((uploadedBytes / file.size) * 100))+ },+ ],+ }),+ listeners(({ actions, values, props }) => ({+ selectFile: ({ file }) => {+ const detected = file ? detectFileFormat(file.name) : null+ if (detected) {+ actions.setFileFormat(detected)+ }+ },+ submitReplaceFile: async () => {+ // The stage is set before the first await, so a second submit while one is in flight stops here.+ if (values.replaceStage !== null) {+ return+ }+ const { file, fileFormat, target, fileError, currentTeamId } = values+ if (!file || !target || fileError || !currentTeamId) {+ return+ }+ const projectId = String(currentTeamId)+ actions.setReplaceStage('upload')++ const formData = new FormData()+ formData.append('file', file)+ formData.append('file_format', fileFormat)++ let upload+ try {+ // The generated client sends uploads with fetch, which cannot report upload progress.+ // nosemgrep: prefer-codegen-api-namespaced-data_warehouse+ upload = await api.dataWarehouseTables.uploadFile(formData, {+ onUploadProgress: ({ loaded }) => actions.setUploadedBytes(loaded),+ })+ } catch (e: any) {+ posthog.capture('warehouse file replace failed', { stage: 'upload', file_format: fileFormat })+ actions.replaceFileFailure(apiErrorMessage(e, 'Could not upload the file.'))+ return+ }++ actions.setReplaceStage('replace')+ try {+ const tableId = target.tableId ?? (await findTableIdByName(projectId, target.tableName))+ if (!tableId) {+ throw new Error(`Could not find the table ${target.tableName}. Refresh the page and try again.`)+ }+ const table = await warehouseTablesReplaceFileCreate(projectId, tableId, {+ upload_id: upload.upload_id,+ filename: upload.filename,+ file_format: fileFormat,+ })+ // pinned: analytics event names, renaming breaks dashboards+ posthog.capture('warehouse file replaced', { file_format: fileFormat })+ actions.replaceFileSuccess(table)+ } catch (e: any) {+ posthog.capture('warehouse file replace failed', { stage: 'replace', file_format: fileFormat })+ actions.replaceFileFailure(apiErrorMessage(e, 'Could not replace the file.'))+ }+ },+ replaceFileSuccess: () => {+ lemonToast.success(REPLACE_FILE_SUCCESS_MESSAGE)+ actions.closeReplaceFile()+ sourceSettingsLogic.findMounted({ id: props.sourceId })?.actions.loadSource()+ },+ })),+])diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.tsindex e85c0920b6d1..e0fabc53fb5d 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts@@ -36,6 +36,7 @@ import {} from '~/types'import { groupTablesBySchema } from 'products/data_warehouse/frontend/shared/components/forms/schemaGroupingUtils'+import { FILE_UPLOAD_SOURCE_LABEL, isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'import {clonePayloadPreservingFiles,findUploadedFiles,@@ -1361,7 +1362,9 @@ export const sourceSettingsLogic = kea<sourceSettingsLogicType>([const breadcrumbName =values.source?.access_method === 'direct'? values.source?.prefix || values.source?.source_type || 'Source'- : values.source?.source_type || 'Source'+ : isFileUploadSource(values.source)+ ? FILE_UPLOAD_SOURCE_LABEL+ : values.source?.source_type || 'Source'if (values.pollPauseCount === 0) {cache.disposables.add(() => {diff --git a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsxindex 645e61383b6a..1c0618e555fa 100644--- a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx+++ b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx@@ -18,6 +18,7 @@ import { AccessControlLevel, AccessControlResourceType, ExternalDataSchemaStatusimport { StatusTagSetting } from 'products/data_warehouse/frontend/utils'import { availableSourcesLogic } from '../../scenes/NewSourceScene/availableSourcesLogic'+import { isFileUploadSource, sourceTypeLabel } from '../fileUploadSources'import { sourceManagementLogic } from '../logics/sourceManagementLogic'import { FreeHistoricalSyncsBanner } from './FreeHistoricalSyncsBanner'import { DATA_WAREHOUSE_APP_SOURCE } from './metrics/DataWarehouseMetrics'@@ -84,7 +85,7 @@ export function ManagedSourcesTable(): JSX.Element {render: (_, source) => (<LemonTableLinkto={urls.dataWarehouseSource(`managed-${source.id}`)}- title={availableSources?.[source.source_type]?.label ?? source.source_type}+ title={sourceTypeLabel(source.source_type, availableSources)}description={source.description}/>),@@ -148,7 +149,10 @@ export function ManagedSourcesTable(): JSX.Element {if (!source.status) {return null}- const syncingSchemas = source.schemas.filter((s) => s.should_sync)+ // A file upload schema never syncs on a schedule, so its imports still count.+ const syncingSchemas = isFileUploadSource(source)+ ? source.schemas+ : source.schemas.filter((s) => s.should_sync)const counts = SCHEMA_STATUS_ORDER.map((status) => ({status,schemas: syncingSchemas.filter((s) => s.status === status),@@ -216,27 +220,29 @@ export function ManagedSourcesTable(): JSX.Element {<Moreoverlay={<>- <AccessControlAction- resourceType={AccessControlResourceType.ExternalDataSource}- minAccessLevel={AccessControlLevel.Editor}- userAccessLevel={source.user_access_level}- >- {({ disabledReason }) => (- <Tooltip title="Start the data import for this schema again">- <LemonButton- type="tertiary"- data-attr={`reload-data-warehouse-${source.source_type}`}- key={`reload-data-warehouse-${source.source_type}`}- onClick={() => {- reloadSource(source)- }}- disabledReason={disabledReason}- >- Reload- </LemonButton>- </Tooltip>- )}- </AccessControlAction>+ {!isFileUploadSource(source) && (+ <AccessControlAction+ resourceType={AccessControlResourceType.ExternalDataSource}+ minAccessLevel={AccessControlLevel.Editor}+ userAccessLevel={source.user_access_level}+ >+ {({ disabledReason }) => (+ <Tooltip title="Start the data import for this schema again">+ <LemonButton+ type="tertiary"+ data-attr={`reload-data-warehouse-${source.source_type}`}+ key={`reload-data-warehouse-${source.source_type}`}+ onClick={() => {+ reloadSource(source)+ }}+ disabledReason={disabledReason}+ >+ Reload+ </LemonButton>+ </Tooltip>+ )}+ </AccessControlAction>+ )}<AccessControlActionresourceType={AccessControlResourceType.ExternalDataSource}diff --git a/products/data_warehouse/frontend/shared/fileUploadSources.ts b/products/data_warehouse/frontend/shared/fileUploadSources.tsnew file mode 100644index 000000000000..9e8681d0b1e9--- /dev/null+++ b/products/data_warehouse/frontend/shared/fileUploadSources.ts@@ -0,0 +1,27 @@+import { ExternalDataSource } from '~/types'++import {+ ExternalDataSourceTypeEnumApi,+ SourceConfigResponseApi,+} from 'products/warehouse_sources/frontend/generated/api.schemas'++export const FILE_UPLOAD_SOURCE_LABEL = 'File upload'++/**+ * A file upload source imports one file a user uploaded. It has no schedule and no incremental+ * sync: PostHog starts each import itself, and the user refreshes the data by replacing the file.+ */+export function isFileUploadSource(source: Pick<ExternalDataSource, 'source_type'> | null | undefined): boolean {+ return source?.source_type === ExternalDataSourceTypeEnumApi.FileUpload+}++// The source catalog leaves out file upload sources, so their label can't come from it.+export function sourceTypeLabel(+ sourceType: string,+ availableSources: Record<string, SourceConfigResponseApi> | null | undefined+): string {+ if (sourceType === ExternalDataSourceTypeEnumApi.FileUpload) {+ return FILE_UPLOAD_SOURCE_LABEL+ }+ return availableSources?.[sourceType]?.label ?? sourceType+}