Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): replace files and manage file upload sources in the app (#125530)

Tom Owers34ae120d3d08
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index ac954fdea98f..c9c0cb226975 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -305,6 +305,7 @@ export const FEATURE_FLAGS = {
DATA_MODELING_USAGE: 'data-modeling-usage', // owner: #team-data-modeling, gates the Models usage tab
DATA_WAREHOUSE_CUSTOM_SOURCE_AI_BUILDER: 'dwh-custom-source-ai-builder', // owner: #team-warehouse-sources
DATA_WAREHOUSE_CUSTOM_SOURCE_OAUTH2: 'dwh-custom-source-oauth2', // owner: #team-warehouse-sources
+ DATA_WAREHOUSE_OBJECT_STORAGE_SOURCES: 'dwh-object-storage-sources', // owner: #team-warehouse-sources
DATA_WAREHOUSE_RENAME_SAVED_VIEW: 'data-warehouse-rename-saved-view', // owner: #team-data-modeling
DATA_WAREHOUSE_SCENE: 'data-warehouse-scene', // owner: #team-data-modeling
DISCUSSIONS_SLACK_SYNC: 'discussions-slack-sync', // owner: #team-platform-features
diff --git a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.ts b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.ts
index 232ee2af1e37..1278837eb630 100644
--- a/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.ts
+++ b/frontend/src/scenes/data-warehouse/settings/DataWarehouseSourceScene.test.ts
@@ -1,6 +1,7 @@
import {
getDefaultDataWarehouseSourceSceneTab,
isManagedSourceSceneId,
+ shouldShowManagedSourceConfigurationTab,
shouldShowManagedSourceMetricsTab,
shouldShowManagedSourceSyncsTab,
} from 'products/data_warehouse/frontend/scenes/SourceScene/SourceScene'
@@ -35,4 +36,12 @@ describe('DataWarehouseSourceScene', () => {
])('hides the metrics tab for direct query sources (source %p, flag %p)', (source, flagEnabled, expected) => {
expect(shouldShowManagedSourceMetricsTab(source, flagEnabled)).toEqual(expected)
})
+
+ it.each([
+ [null, false],
+ [{ source_type: 'FileUpload' as const }, false],
+ [{ source_type: 'Stripe' as const }, true],
+ ])('hides the configuration tab for file upload sources (source %p)', (source, expected) => {
+ expect(shouldShowManagedSourceConfigurationTab(source)).toEqual(expected)
+ })
})
diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsx b/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsx
index 916543f54094..824618ecca2e 100644
--- a/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsx
+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/components/FileUploadSourceForm.tsx
@@ -3,6 +3,7 @@ import { Form } from 'kea-forms'
import { LemonButton, LemonInput, LemonSelect, Spinner } from '@posthog/lemon-ui'
+import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'
import { LemonField } from 'lib/lemon-ui/LemonField'
import { LemonFileInput } from 'lib/lemon-ui/LemonFileInput/LemonFileInput'
import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
@@ -13,6 +14,7 @@ import { FILE_UPLOAD_ACCEPT, fileUploadSourceLogic } from '../fileUploadSourceLo
export function FileUploadSourceForm(): JSX.Element {
const { fileUpload, isFileUploadSubmitting, uploadStage, uploadPercent } = useValues(fileUploadSourceLogic)
const { selectFiles, setTableName, setFileFormat } = useActions(fileUploadSourceLogic)
+ const objectStorageSourcesEnabled = useFeatureFlag('DATA_WAREHOUSE_OBJECT_STORAGE_SOURCES')
const busyReason = !isFileUploadSubmitting
? undefined
@@ -56,7 +58,14 @@ export function FileUploadSourceForm(): JSX.Element {
/>
)}
</LemonField>
- <div className="mb-4 text-xs text-secondary">Files can be up to 50MB.</div>
+ <div className="mb-4 flex flex-col gap-1 text-xs text-secondary">
+ <span>Files can be up to 50MB.</span>
+ {objectStorageSourcesEnabled && (
+ <span data-attr="file-upload-billing-note">
+ Rows imported from this file count toward your data warehouse rows synced.
+ </span>
+ )}
+ </div>
<LemonField name="table_name" label="Table name">
{({ value = '' }) => (
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
index 57d31897e257..856182caca85 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/SourceScene.tsx
@@ -42,6 +42,7 @@ import {
SidePanelTab,
} from '~/types'
+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import { cleanSourceId, isSelfManagedSourceId } from 'products/data_warehouse/frontend/utils'
import { FailingDestinationsBanner } from './FailingDestinationsBanner'
@@ -91,6 +92,13 @@ export function shouldShowManagedSourceMetricsTab(
return metricsFeatureFlagEnabled && shouldShowManagedSourceSyncsTab(source)
}
+// PostHog manages every input of a file upload source, so its configuration has nothing to edit.
+export function shouldShowManagedSourceConfigurationTab(
+ source: Pick<ExternalDataSource, 'source_type'> | null | undefined
+): boolean {
+ return !!source && !isFileUploadSource(source)
+}
+
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface sourceSceneLogicValues {
breadcrumbName: string
@@ -292,6 +300,7 @@ function ManagedSourceTabs({
const showWebhookTab = !!source?.supports_webhooks
const showDestinationsTab = !!featureFlags[FEATURE_FLAGS.WAREHOUSE_MULTI_DESTINATION]
const showMetricsTab = shouldShowManagedSourceMetricsTab(source, !!featureFlags[FEATURE_FLAGS.DWH_SOURCE_METRICS])
+ const showConfigurationTab = shouldShowManagedSourceConfigurationTab(source)
useEffect(() => {
// Wait until the source has loaded before deciding a tab is unavailable.
@@ -310,7 +319,10 @@ function ManagedSourceTabs({
if (!showMetricsTab && currentTab === 'metrics') {
setCurrentTab('schemas')
}
- }, [source, showSyncsTab, showWebhookTab, showMetricsTab, currentTab, setCurrentTab])
+ if (!showConfigurationTab && currentTab === 'configuration') {
+ setCurrentTab('schemas')
+ }
+ }, [source, showSyncsTab, showWebhookTab, showMetricsTab, showConfigurationTab, currentTab, setCurrentTab])
if (sourceLoading && !source) {
return <LemonSkeleton className="w-full h-12" />
@@ -335,7 +347,9 @@ function ManagedSourceTabs({
tabs.push({ label: 'Destinations', key: 'destinations', content: <DestinationsTab id={sourceId} /> })
}
- tabs.push({ label: 'Configuration', key: 'configuration', content: <ConfigurationTab id={sourceId} /> })
+ if (showConfigurationTab) {
+ tabs.push({ label: 'Configuration', key: 'configuration', content: <ConfigurationTab id={sourceId} /> })
+ }
if (showWebhookTab) {
tabs.push({
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ReplaceFileModal.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ReplaceFileModal.tsx
new file mode 100644
index 000000000000..9033b417fb4a
--- /dev/null
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ReplaceFileModal.tsx
@@ -0,0 +1,109 @@
+import { useActions, useValues } from 'kea'
+
+import { LemonBanner, LemonButton, LemonModal, LemonSelect, Spinner } from '@posthog/lemon-ui'
+
+import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'
+import { LemonField } from 'lib/lemon-ui/LemonField'
+import { LemonFileInput } from 'lib/lemon-ui/LemonFileInput/LemonFileInput'
+import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
+import { humanizeBytes } from 'lib/utils/numbers'
+
+import { FILE_UPLOAD_ACCEPT } from '../../NewSourceScene/fileUploadSourceLogic'
+import { replaceFileLogic } from './replaceFileLogic'
+
+export function ReplaceFileModal({ sourceId }: { sourceId: string }): JSX.Element {
+ const logic = replaceFileLogic({ sourceId })
+ const {
+ target,
+ isReplaceFileModalOpen,
+ file,
+ fileFormat,
+ fileError,
+ replaceError,
+ replaceStage,
+ isReplacing,
+ uploadPercent,
+ } = useValues(logic)
+ const { closeReplaceFile, selectFile, setFileFormat, submitReplaceFile } = useActions(logic)
+ const objectStorageSourcesEnabled = useFeatureFlag('DATA_WAREHOUSE_OBJECT_STORAGE_SOURCES')
+
+ const busyReason = isReplacing ? 'The file is being replaced' : undefined
+
+ return (
+ <LemonModal
+ isOpen={isReplaceFileModalOpen}
+ onClose={isReplacing ? undefined : closeReplaceFile}
+ closable={!isReplacing}
+ title={target ? `Replace the file of ${target.tableName}` : 'Replace file'}
+ description="Upload a new file to replace all rows in this table. The table keeps its name, so your queries keep working."
+ footer={
+ <>
+ <LemonButton
+ type="secondary"
+ onClick={closeReplaceFile}
+ disabledReason={busyReason}
+ data-attr="replace-file-cancel"
+ >
+ Cancel
+ </LemonButton>
+ <LemonButton
+ type="primary"
+ onClick={submitReplaceFile}
+ loading={isReplacing}
+ disabledReason={busyReason ?? (!file ? 'Choose a file first' : (fileError ?? undefined))}
+ data-attr="replace-file-submit"
+ >
+ Replace file
+ </LemonButton>
+ </>
+ }
+ >
+ <div className="flex flex-col gap-3">
+ <LemonField.Pure label="File format" className="w-max">
+ <LemonSelect
+ data-attr="replace-file-format"
+ options={[
+ { label: 'CSV', value: 'csv' },
+ { label: 'JSON', value: 'json' },
+ { label: 'Parquet', value: 'parquet' },
+ ]}
+ value={fileFormat}
+ onChange={setFileFormat}
+ disabledReason={busyReason}
+ />
+ </LemonField.Pure>
+ <LemonField.Pure label="File" error={fileError ?? undefined} help="Files can be up to 50MB.">
+ <LemonFileInput
+ multiple={false}
+ accept={FILE_UPLOAD_ACCEPT[fileFormat]}
+ value={file ? [file] : []}
+ onChange={(files) => selectFile(files[0] ?? null)}
+ disabledReason={busyReason}
+ />
+ </LemonField.Pure>
+ {objectStorageSourcesEnabled && (
+ <div className="text-xs text-secondary">
+ Rows imported from this file count toward your data warehouse rows synced.
+ </div>
+ )}
+ {replaceStage === 'upload' ? (
+ <div className="flex flex-col gap-1" data-attr="replace-file-progress">
+ <LemonProgress percent={uploadPercent} />
+ <div className="text-xs text-secondary">
+ <span>Uploading file:&nbsp;</span>
+ <span translate="no">
+ {uploadPercent}% of {humanizeBytes(file?.size ?? null)}
+ </span>
+ </div>
+ </div>
+ ) : replaceStage === 'replace' ? (
+ <div className="flex items-center gap-2 text-xs text-secondary" data-attr="replace-file-progress">
+ <Spinner />
+ <span>File uploaded. Starting the import.</span>
+ </div>
+ ) : null}
+ {replaceError && <LemonBanner type="error">{replaceError}</LemonBanner>}
+ </div>
+ </LemonModal>
+ )
+}
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
index 16c15cd9954f..644dd1257aae 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
@@ -25,6 +25,7 @@ import { FEATURE_FLAGS } from 'lib/constants'
import { dayjs } from 'lib/dayjs'
import { More } from 'lib/lemon-ui/LemonButton/More'
import { LemonMenu } from 'lib/lemon-ui/LemonMenu'
+import { LemonTableColumn } from 'lib/lemon-ui/LemonTable'
import { LemonTableLink } from 'lib/lemon-ui/LemonTable/LemonTableLink'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils'
@@ -51,6 +52,7 @@ import {
SchemaEditorAction,
SourceEditorAction,
} from 'products/data_warehouse/frontend/shared/components/SourceEditorAction'
+import { isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import { destinationLookupLogic } from 'products/data_warehouse/frontend/shared/logics/destinationLookupLogic'
import { sourceManagementLogic } from 'products/data_warehouse/frontend/shared/logics/sourceManagementLogic'
import {
@@ -64,6 +66,8 @@ import {
import type { ExternalDataDestinationApi } from 'products/warehouse_sources/frontend/generated/api.schemas'
import { DirectQuerySchemasTab } from './DirectQuerySchemasTab'
+import { replaceFileLogic, replaceFileTarget } from './replaceFileLogic'
+import { ReplaceFileModal } from './ReplaceFileModal'
import { BulkSyncMethod, bulkSyncMethodDisabledReason, sourceSettingsLogic } from './sourceSettingsLogic'
const frequencyRank = (frequency: DataWarehouseSyncInterval | null | undefined): number =>
@@ -81,6 +85,29 @@ const schemaEditDisabledReason = (schema: ExternalDataSourceSchema): string | nu
const dimWhenNotSyncing = (_: unknown, schema: ExternalDataSourceSchema): string =>
schema.should_sync ? '' : 'opacity-60'
+type SchemaColumn = LemonTableColumn<ExternalDataSourceSchema, keyof ExternalDataSourceSchema | undefined>
+
+// A file upload schema has no schedule or sync method to choose, and never syncs on its own, so it isn't dimmed.
+const FILE_UPLOAD_HIDDEN_COLUMN_KEYS = ['sync_type', 'sync_frequency', 'should_sync']
+
+function fileUploadColumns(isFileUpload: boolean, columns: SchemaColumn[]): SchemaColumn[] {
+ if (!isFileUpload) {
+ return columns
+ }
+ return columns
+ .filter((column) => !FILE_UPLOAD_HIDDEN_COLUMN_KEYS.includes(column.key ?? ''))
+ .map((column) => (column.className === dimWhenNotSyncing ? { ...column, className: undefined } : column))
+}
+
+function replaceFileDisabledReason(schema: ExternalDataSourceSchema | undefined): string | undefined {
+ if (!schema) {
+ return 'This source has no table'
+ }
+ return schema.status === ExternalDataSchemaStatus.Running
+ ? 'An import is running. Replace the file once it finishes.'
+ : undefined
+}
+
interface SchemaStatusDisplay {
type: LemonTagType
label: string
@@ -174,7 +201,17 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {
const { featureFlags } = useValues(featureFlagLogic)
const { loadDestinations } = useActions(destinationLookupLogic)
const { destinationsById } = useValues(destinationLookupLogic)
+ const { openReplaceFile } = useActions(replaceFileLogic({ sourceId: id }))
const showDestinations = !!featureFlags[FEATURE_FLAGS.WAREHOUSE_MULTI_DESTINATION]
+ const isFileUpload = isFileUploadSource(source)
+ const fileUploadSchema = isFileUpload ? source?.schemas[0] : undefined
+ const onReplaceFile = isFileUpload
+ ? (schema: ExternalDataSourceSchema): void => openReplaceFile(replaceFileTarget(schema))
+ : undefined
+ // The schema filters are hidden for file uploads, so a saved filter (e.g. "Show enabled only",
+ // which an upload's schema never matches) must not hide its table.
+ const visibleSchemas = isFileUpload ? (source?.schemas ?? []) : filteredSchemas
+ const visibleSchemaGroups = isFileUpload ? [] : groupedFilteredSchemas
useEffect(() => {
if (showDestinations) {
@@ -204,58 +241,78 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {
return (
<>
<div className="flex items-start justify-between gap-2 mb-2">
- <div className="flex flex-wrap items-center gap-3 min-w-0">
- <LemonSwitch
- checked={showEnabledSchemasOnly}
- onChange={setShowEnabledSchemasOnly}
- label="Show enabled only"
- />
- <LemonInput
- type="search"
- placeholder="Filter schemas"
- size="small"
- value={schemaNameFilter}
- onChange={setSchemaNameFilter}
- />
- <LemonSelect
- size="small"
- placeholder="All statuses"
- value={statusFilter}
- onChange={setStatusFilter}
- options={[
- { value: null, label: 'All statuses' },
- ...schemaFilterOptions.statuses.map((status) => ({ value: status, label: status })),
- ]}
- />
- <LemonSelect
- size="small"
- placeholder="All sync methods"
- value={syncMethodFilter}
- onChange={setSyncMethodFilter}
- options={[
- { value: null, label: 'All sync methods' },
- ...schemaFilterOptions.syncMethods.map((method) => ({
- value: method === 'none' ? 'none' : method,
- label: method === 'none' ? 'Not set up' : SyncTypeLabelMap[method],
- })),
- ]}
- />
- <LemonSelect
- size="small"
- placeholder="All frequencies"
- value={frequencyFilter}
- onChange={setFrequencyFilter}
- options={[
- { value: null, label: 'All frequencies' },
- ...schemaFilterOptions.frequencies.map((frequency) => ({
- value: frequency,
- label: SyncFrequencyLabelMap[frequency],
- })),
- ]}
- />
- <span className="text-muted text-sm">{pluralize(filteredSchemas.length, 'schema', 'schemas')}</span>
- </div>
+ {isFileUpload ? (
+ <div className="text-secondary text-sm">
+ This table comes from a file you uploaded. To update its data, replace the file.
+ </div>
+ ) : (
+ <div className="flex flex-wrap items-center gap-3 min-w-0">
+ <LemonSwitch
+ checked={showEnabledSchemasOnly}
+ onChange={setShowEnabledSchemasOnly}
+ label="Show enabled only"
+ />
+ <LemonInput
+ type="search"
+ placeholder="Filter schemas"
+ size="small"
+ value={schemaNameFilter}
+ onChange={setSchemaNameFilter}
+ />
+ <LemonSelect
+ size="small"
+ placeholder="All statuses"
+ value={statusFilter}
+ onChange={setStatusFilter}
+ options={[
+ { value: null, label: 'All statuses' },
+ ...schemaFilterOptions.statuses.map((status) => ({ value: status, label: status })),
+ ]}
+ />
+ <LemonSelect
+ size="small"
+ placeholder="All sync methods"
+ value={syncMethodFilter}
+ onChange={setSyncMethodFilter}
+ options={[
+ { value: null, label: 'All sync methods' },
+ ...schemaFilterOptions.syncMethods.map((method) => ({
+ value: method === 'none' ? 'none' : method,
+ label: method === 'none' ? 'Not set up' : SyncTypeLabelMap[method],
+ })),
+ ]}
+ />
+ <LemonSelect
+ size="small"
+ placeholder="All frequencies"
+ value={frequencyFilter}
+ onChange={setFrequencyFilter}
+ options={[
+ { value: null, label: 'All frequencies' },
+ ...schemaFilterOptions.frequencies.map((frequency) => ({
+ value: frequency,
+ label: SyncFrequencyLabelMap[frequency],
+ })),
+ ]}
+ />
+ <span className="text-muted text-sm">
+ {pluralize(filteredSchemas.length, 'schema', 'schemas')}
+ </span>
+ </div>
+ )}
<div className="flex items-center gap-2 shrink-0">
+ {isFileUpload && (
+ <SchemaEditorAction schema={fileUploadSchema ?? null}>
+ <LemonButton
+ type="primary"
+ onClick={() => fileUploadSchema && onReplaceFile?.(fileUploadSchema)}
+ disabledReason={replaceFileDisabledReason(fileUploadSchema)}
+ data-attr="schemas-tab-replace-file"
+ >
+ Replace file
+ </LemonButton>
+ </SchemaEditorAction>
+ )}
{source?.direct_query_enabled && supportsDirectQuery(source.source_type) && (
<LemonButton
type="secondary"
@@ -266,59 +323,67 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {
Query directly
</LemonButton>
)}
- <SourceEditorAction source={source}>
- <LemonButton
- type="secondary"
- loading={syncingNow}
- onClick={() => {
- LemonDialog.open({
- title: 'Sync all enabled schemas?',
- content: (
- <div className="text-sm text-secondary">
- This will trigger a sync for all schemas you have enabled. New sync jobs
- will appear in the Syncs tab.
- </div>
- ),
- primaryButton: {
- children: 'Sync now',
- type: 'primary',
- onClick: () => syncNow(),
- },
- secondaryButton: { children: 'Cancel', type: 'tertiary' },
- })
- }}
- disabledReason={
- sourceLoading
- ? 'Source is loading'
- : refreshingSchemas
- ? 'Schema refresh in progress'
- : undefined
- }
- >
- Sync all enabled schemas
- </LemonButton>
- </SourceEditorAction>
- <SourceEditorAction source={source}>
- <LemonButton
- type="secondary"
- loading={refreshingSchemas}
- onClick={() => refreshSchemas()}
- disabledReason={
- sourceLoading ? 'Source is loading' : syncingNow ? 'Sync in progress' : undefined
- }
- >
- Pull new schemas
- </LemonButton>
- </SourceEditorAction>
+ {!isFileUpload && (
+ <>
+ <SourceEditorAction source={source}>
+ <LemonButton
+ type="secondary"
+ loading={syncingNow}
+ onClick={() => {
+ LemonDialog.open({
+ title: 'Sync all enabled schemas?',
+ content: (
+ <div className="text-sm text-secondary">
+ This will trigger a sync for all schemas you have enabled. New sync
+ jobs will appear in the Syncs tab.
+ </div>
+ ),
+ primaryButton: {
+ children: 'Sync now',
+ type: 'primary',
+ onClick: () => syncNow(),
+ },
+ secondaryButton: { children: 'Cancel', type: 'tertiary' },
+ })
+ }}
+ disabledReason={
+ sourceLoading
+ ? 'Source is loading'
+ : refreshingSchemas
+ ? 'Schema refresh in progress'
+ : undefined
+ }
+ >
+ Sync all enabled schemas
+ </LemonButton>
+ </SourceEditorAction>
+ <SourceEditorAction source={source}>
+ <LemonButton
+ type="secondary"
+ loading={refreshingSchemas}
+ onClick={() => refreshSchemas()}
+ disabledReason={
+ sourceLoading
+ ? 'Source is loading'
+ : syncingNow
+ ? 'Sync in progress'
+ : undefined
+ }
+ >
+ Pull new schemas
+ </LemonButton>
+ </SourceEditorAction>
+ </>
+ )}
</div>
</div>
- {groupedFilteredSchemas.length > 1 ? (
+ {visibleSchemaGroups.length > 1 ? (
<div className="border rounded bg-bg-light">
<LemonCollapse
multiple
embedded
- defaultActiveKeys={groupedFilteredSchemas.map((group) => group.schemaName)}
- panels={groupedFilteredSchemas.map(({ schemaName, tables }) => ({
+ defaultActiveKeys={visibleSchemaGroups.map((group) => group.schemaName)}
+ panels={visibleSchemaGroups.map(({ schemaName, tables }) => ({
key: schemaName,
header: (
<div className="flex items-center justify-between gap-3 w-full">
@@ -343,6 +408,7 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {
cancelSchema={cancelSchema}
deleteTable={deleteTable}
openAccessControl={openAccessControl}
+ onReplaceFile={onReplaceFile}
showMetrics={showMetrics}
destinationsById={showDestinations ? destinationsById : {}}
/>
@@ -352,7 +418,7 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {
</div>
) : (
<ManagedSchemaTable
- schemas={filteredSchemas}
+ schemas={visibleSchemas}
isLoading={sourceLoading}
source={source}
sourceId={id}
@@ -363,10 +429,12 @@ function ManagedSchemasTab({ id }: { id: string }): JSX.Element {
cancelSchema={cancelSchema}
deleteTable={deleteTable}
openAccessControl={openAccessControl}
+ onReplaceFile={onReplaceFile}
showMetrics={showMetrics}
destinationsById={showDestinations ? destinationsById : {}}
/>
)}
+ {isFileUpload && <ReplaceFileModal sourceId={id} />}
{accessControlSchema?.table && (
<AccessControlObjectModal
isOpen={!!accessControlSchema}
@@ -394,6 +462,8 @@ interface ManagedSchemaTableProps {
deleteTable: (schema: ExternalDataSourceSchema) => void
/** Undefined when the warehouse access control feature is off. */
openAccessControl?: (schema: ExternalDataSourceSchema) => void
+ /** Set only for a file upload source, whose tables are refreshed by replacing the file. */
+ onReplaceFile?: (schema: ExternalDataSourceSchema) => void
showMetrics: boolean
destinationsById: Record<string, ExternalDataDestinationApi>
/** Rendered inside a namespace group — the group header already shows the namespace, so strip it from row names. */
@@ -412,6 +482,7 @@ function ManagedSchemaTable({
cancelSchema,
deleteTable,
openAccessControl,
+ onReplaceFile,
showMetrics,
destinationsById,
inSchemaGroup = false,
@@ -427,25 +498,31 @@ function ManagedSchemaTable({
}
}, [isLoading, initialLoad])
+ const isFileUpload = !!onReplaceFile
+
return (
<LemonTable
dataSource={schemas}
loading={initialLoad}
disableTableWhileLoading={false}
pagination={{ pageSize: 100, hideOnSinglePage: true }}
- bulkSelection={{
- getKey: (schema) => schema.id,
- isRowSelectable: (schema) => {
- const reason = schemaEditDisabledReason(schema)
- return reason ? { disabledReason: reason } : true
- },
- noun: ['schema', 'schemas'],
- barClassName: 'mb-2',
- renderActions: (ctx) => (
- <SchemaBulkActions schemas={ctx.selectedRecords} clearSelection={ctx.clearSelection} />
- ),
- }}
- columns={[
+ bulkSelection={
+ isFileUpload
+ ? undefined
+ : {
+ getKey: (schema) => schema.id,
+ isRowSelectable: (schema) => {
+ const reason = schemaEditDisabledReason(schema)
+ return reason ? { disabledReason: reason } : true
+ },
+ noun: ['schema', 'schemas'],
+ barClassName: 'mb-2',
+ renderActions: (ctx) => (
+ <SchemaBulkActions schemas={ctx.selectedRecords} clearSelection={ctx.clearSelection} />
+ ),
+ }
+ }
+ columns={fileUploadColumns(isFileUpload, [
{
title: 'Schema',
key: 'name',
@@ -728,14 +805,20 @@ function ManagedSchemaTable({
}
return (
<div className="flex justify-end items-center gap-1">
- <LemonButton
- type="secondary"
- size="xsmall"
- to={urls.dataWarehouseSourceSchema(prefixedSourceId, schema.id, 'configuration')}
- disabledReason={schemaEditDisabledReason(schema)}
- >
- Configure
- </LemonButton>
+ {!isFileUpload && (
+ <LemonButton
+ type="secondary"
+ size="xsmall"
+ to={urls.dataWarehouseSourceSchema(
+ prefixedSourceId,
+ schema.id,
+ 'configuration'
+ )}
+ disabledReason={schemaEditDisabledReason(schema)}
+ >
+ Configure
+ </LemonButton>
+ )}
<SchemaRowMore
source={source}
schema={schema}
@@ -744,12 +827,13 @@ function ManagedSchemaTable({
cancelSchema={cancelSchema}
deleteTable={deleteTable}
onOpenAccessControl={openAccessControl}
+ onReplaceFile={onReplaceFile}
/>
</div>
)
},
},
- ]}
+ ])}
/>
)
}
@@ -947,6 +1031,7 @@ function SchemaRowMore({
cancelSchema,
deleteTable,
onOpenAccessControl,
+ onReplaceFile,
}: {
source: ExternalDataSource | null
schema: ExternalDataSourceSchema
@@ -955,8 +1040,11 @@ function SchemaRowMore({
cancelSchema: (schema: ExternalDataSourceSchema) => void
deleteTable: (schema: ExternalDataSourceSchema) => void
onOpenAccessControl?: (schema: ExternalDataSourceSchema) => void
+ onReplaceFile?: (schema: ExternalDataSourceSchema) => void
}): JSX.Element {
const { pausePolling, resumePolling } = useActions(sourceSettingsLogic)
+ // An uploaded file has no outside system whose data stays untouched.
+ const externalSourceType = isFileUploadSource(source) ? null : source?.source_type
return (
<SchemaEditorAction schema={schema}>
@@ -980,31 +1068,44 @@ function SchemaRowMore({
Access control
</LemonButton>
)}
- <Tooltip
- title={
- schema.sync_type === 'cdc'
- ? 'Trigger a CDC extraction run now.'
- : schema.incremental
- ? 'Sync incremental data since the last run.'
- : 'Sync all data.'
- }
- >
+ {onReplaceFile ? (
<LemonButton
type="tertiary"
size="xsmall"
fullWidth
- onClick={() => reloadSchema(schema)}
- disabledReason={
- !schema.sync_type
- ? 'Set up the sync method first'
- : schema.status === 'Running'
- ? 'A sync is already running'
- : undefined
- }
+ onClick={() => onReplaceFile(schema)}
+ disabledReason={replaceFileDisabledReason(schema)}
+ data-attr="schema-row-replace-file"
>
- {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}
+ Replace file
</LemonButton>
- </Tooltip>
+ ) : (
+ <Tooltip
+ title={
+ schema.sync_type === 'cdc'
+ ? 'Trigger a CDC extraction run now.'
+ : schema.incremental
+ ? 'Sync incremental data since the last run.'
+ : 'Sync all data.'
+ }
+ >
+ <LemonButton
+ type="tertiary"
+ size="xsmall"
+ fullWidth
+ onClick={() => reloadSchema(schema)}
+ disabledReason={
+ !schema.sync_type
+ ? 'Set up the sync method first'
+ : schema.status === 'Running'
+ ? 'A sync is already running'
+ : undefined
+ }
+ >
+ {schema.sync_type === 'cdc' ? 'Sync CDC now' : 'Sync now'}
+ </LemonButton>
+ </Tooltip>
+ )}
{schema.status === 'Running' && (
<LemonButton
type="tertiary"
@@ -1078,8 +1179,8 @@ function SchemaRowMore({
{schema.table && (
<Tooltip
title={`Delete this table from PostHog. ${
- source?.source_type
- ? `This will not delete the data in ${source.source_type}`
+ externalSourceType
+ ? `This will not delete the data in ${externalSourceType}`
: ''
}`}
>
@@ -1092,8 +1193,8 @@ function SchemaRowMore({
LemonDialog.open({
title: `Delete ${schema.table?.name ?? schema.name} from PostHog?`,
description: `This also turns off syncing for the table.${
- source?.source_type
- ? ` The data in ${source.source_type} will not be touched.`
+ externalSourceType
+ ? ` The data in ${externalSourceType} will not be touched.`
: ''
}`,
primaryButton: {
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts
new file mode 100644
index 000000000000..061336f1e306
--- /dev/null
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.test.ts
@@ -0,0 +1,193 @@
+import { expectLogic } from 'kea-test-utils'
+
+import api from 'lib/api'
+
+import { initKeaTests } from '~/test/init'
+
+import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'
+import type { PaginatedTableListApi, TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'
+
+import { MAX_FILE_UPLOAD_SIZE_BYTES } from '../../NewSourceScene/fileUploadSourceLogic'
+import { detectFileFormat, replaceFileLogic } from './replaceFileLogic'
+
+jest.mock('products/data_warehouse/frontend/generated/api', () => ({
+ warehouseTablesList: jest.fn(),
+ warehouseTablesReplaceFileCreate: jest.fn(),
+}))
+
+const mockList = warehouseTablesList as jest.MockedFunction<typeof warehouseTablesList>
+const mockReplace = warehouseTablesReplaceFileCreate as jest.MockedFunction<typeof warehouseTablesReplaceFileCreate>
+
+const UPLOAD = { upload_id: 'upload-1', filename: 'orders.csv', file_format: 'csv', size_bytes: 3 }
+
+function csvFile(name = 'orders.csv'): File {
+ return new File(['a,b'], name)
+}
+
+describe('replaceFileLogic', () => {
+ let logic: ReturnType<typeof replaceFileLogic.build>
+ let uploadSpy: jest.SpyInstance
+
+ beforeEach(() => {
+ initKeaTests()
+ uploadSpy = jest.spyOn(api.dataWarehouseTables, 'uploadFile').mockResolvedValue(UPLOAD)
+ mockReplace.mockResolvedValue({ id: 'table-1', name: 'orders' } as TableApi)
+ logic = replaceFileLogic({ sourceId: 'source-1' })
+ logic.mount()
+ })
+
+ afterEach(() => {
+ logic.unmount()
+ jest.clearAllMocks()
+ uploadSpy.mockRestore()
+ })
+
+ it('uploads the file, replaces the table file and closes the modal', async () => {
+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })
+ logic.actions.selectFile(csvFile())
+
+ await expectLogic(logic, () => {
+ logic.actions.submitReplaceFile()
+ }).toDispatchActions(['replaceFileSuccess', 'closeReplaceFile'])
+
+ expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-1', {
+ upload_id: 'upload-1',
+ filename: 'orders.csv',
+ file_format: 'csv',
+ })
+ expect(mockList).not.toHaveBeenCalled()
+ expect(logic.values.isReplaceFileModalOpen).toBe(false)
+ expect(logic.values.isReplacing).toBe(false)
+ })
+
+ it('finds a table the source does not link to yet by its exact name', async () => {
+ mockList.mockResolvedValue({
+ count: 2,
+ results: [
+ { id: 'other', name: 'orders_backup' },
+ { id: 'table-9', name: 'orders' },
+ ],
+ } as PaginatedTableListApi)
+ logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })
+ logic.actions.selectFile(csvFile())
+
+ await expectLogic(logic, () => {
+ logic.actions.submitReplaceFile()
+ }).toDispatchActions(['replaceFileSuccess'])
+
+ expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())
+ })
+
+ it('follows the result pages when the exact name is not on the first page', async () => {
+ mockList
+ .mockResolvedValueOnce({
+ count: 2,
+ next: 'next-page',
+ results: [{ id: 'other', name: 'orders_backup' }],
+ } as PaginatedTableListApi)
+ .mockResolvedValueOnce({
+ count: 2,
+ next: null,
+ results: [{ id: 'table-9', name: 'orders' }],
+ } as PaginatedTableListApi)
+ logic.actions.openReplaceFile({ tableId: null, tableName: 'orders' })
+ logic.actions.selectFile(csvFile())
+
+ await expectLogic(logic, () => {
+ logic.actions.submitReplaceFile()
+ }).toDispatchActions(['replaceFileSuccess'])
+
+ expect(mockList).toHaveBeenCalledTimes(2)
+ expect(mockList).toHaveBeenLastCalledWith(expect.any(String), expect.objectContaining({ offset: 1 }))
+ expect(mockReplace).toHaveBeenCalledWith(expect.any(String), 'table-9', expect.anything())
+ })
+
+ it.each([
+ {
+ stage: 'upload',
+ fail: (): void => {
+ uploadSpy.mockRejectedValue({ status: 400, data: { message: 'Invalid filename' } })
+ },
+ message: 'Invalid filename',
+ },
+ {
+ stage: 'replace',
+ fail: (): void => {
+ mockReplace.mockRejectedValue({
+ status: 400,
+ data: { message: 'A sync of this table is running. Replace the file once it finishes.' },
+ })
+ },
+ message: 'A sync of this table is running. Replace the file once it finishes.',
+ },
+ {
+ stage: 'replace without a table',
+ fail: (): void => {
+ mockList.mockResolvedValue({ count: 0, results: [] } as PaginatedTableListApi)
+ },
+ message: 'Could not find the table orders. Refresh the page and try again.',
+ },
+ ])('shows the API error and keeps the modal open when the $stage step fails', async ({ stage, fail, message }) => {
+ fail()
+ logic.actions.openReplaceFile({
+ tableId: stage === 'replace without a table' ? null : 'table-1',
+ tableName: 'orders',
+ })
+ logic.actions.selectFile(csvFile())
+
+ await expectLogic(logic, () => {
+ logic.actions.submitReplaceFile()
+ }).toDispatchActions(['replaceFileFailure'])
+
+ expect(logic.values.replaceError).toEqual(message)
+ expect(logic.values.isReplaceFileModalOpen).toBe(true)
+ expect(logic.values.isReplacing).toBe(false)
+ })
+
+ it('ignores a second submit while the first one is in flight', async () => {
+ let releaseUpload: (value: typeof UPLOAD) => void = () => {}
+ uploadSpy.mockReturnValue(new Promise<typeof UPLOAD>((resolve) => (releaseUpload = resolve)))
+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })
+ logic.actions.selectFile(csvFile())
+
+ logic.actions.submitReplaceFile()
+ logic.actions.submitReplaceFile()
+ expect(logic.values.isReplacing).toBe(true)
+
+ await expectLogic(logic, () => {
+ releaseUpload(UPLOAD)
+ }).toDispatchActions(['replaceFileSuccess'])
+
+ expect(uploadSpy).toHaveBeenCalledTimes(1)
+ expect(mockReplace).toHaveBeenCalledTimes(1)
+ })
+
+ it('does not submit a file the backend would refuse', () => {
+ const oversized = csvFile()
+ Object.defineProperty(oversized, 'size', { value: MAX_FILE_UPLOAD_SIZE_BYTES + 1 })
+ logic.actions.openReplaceFile({ tableId: 'table-1', tableName: 'orders' })
+ logic.actions.selectFile(oversized)
+
+ logic.actions.submitReplaceFile()
+
+ expect(logic.values.fileError).toContain('50MB')
+ expect(uploadSpy).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ ['orders.csv', 'csv'],
+ ['events.ndjson', 'json'],
+ ['events.pqt', 'parquet'],
+ ['export.txt', null],
+ ['sheet.xlsx', null],
+ ])('detects the format of %s as %s', (filename, expected) => {
+ expect(detectFileFormat(filename)).toEqual(expected)
+ })
+
+ it('switches the format to match the chosen file', () => {
+ logic.actions.selectFile(csvFile('events.parquet'))
+
+ expect(logic.values.fileFormat).toEqual('parquet')
+ expect(logic.values.fileError).toBeNull()
+ })
+})
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts
new file mode 100644
index 000000000000..412a45d3cf5f
--- /dev/null
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/replaceFileLogic.ts
@@ -0,0 +1,288 @@
+import { MakeLogicType, actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'
+import posthog from 'posthog-js'
+
+import { lemonToast } from '@posthog/lemon-ui'
+
+import api from 'lib/api'
+import { teamLogic } from 'scenes/teamLogic'
+
+import { ExternalDataSourceSchema } from '~/types'
+
+import { warehouseTablesList, warehouseTablesReplaceFileCreate } from 'products/data_warehouse/frontend/generated/api'
+import type { TableApi } from 'products/data_warehouse/frontend/generated/api.schemas'
+
+import { FILE_UPLOAD_FORMATS, FileUploadFormat } from '../../NewSourceScene/fileUploadSource'
+import { MAX_FILE_UPLOAD_SIZE_BYTES, fileFormatMismatchError } from '../../NewSourceScene/fileUploadSourceLogic'
+import { sourceSettingsLogic } from './sourceSettingsLogic'
+
+export const REPLACE_FILE_SUCCESS_MESSAGE = 'Replacing the file. The table updates when the import finishes.'
+
+/** Which of the two requests a replacement is on. */
+export type ReplaceFileStage = 'upload' | 'replace'
+
+export interface ReplaceFileTarget {
+ /** Null for a table that the source names but does not link to yet, such as a migrated upload table. */
+ tableId: string | null
+ tableName: string
+}
+
+export interface ReplaceFileLogicProps {
+ sourceId: string
+}
+
+export function replaceFileTarget(schema: Pick<ExternalDataSourceSchema, 'name' | 'table'>): ReplaceFileTarget {
+ return { tableId: schema.table?.id ?? null, tableName: schema.table?.name ?? schema.name }
+}
+
+/** The one supported format the file's extension names, or null when the extension names none or several. */
+export function detectFileFormat(filename: string): FileUploadFormat | null {
+ const matches = FILE_UPLOAD_FORMATS.map(({ format }) => format).filter(
+ (format) => !fileFormatMismatchError(filename, format)
+ )
+ return matches.length === 1 ? matches[0] : null
+}
+
+export function replaceFileError(file: File | null, fileFormat: FileUploadFormat): string | null {
+ if (!file) {
+ return null
+ }
+ if (file.size > MAX_FILE_UPLOAD_SIZE_BYTES) {
+ return 'This file is larger than the 50MB upload limit.'
+ }
+ return fileFormatMismatchError(file.name, fileFormat) ?? null
+}
+
+function apiErrorMessage(error: any, fallback: string): string {
+ return error?.data?.message ?? error?.detail ?? error?.message ?? fallback
+}
+
+// A table the migration moved to a file upload source is not linked to it until its first import,
+// so the source only knows the table by name. Table names are unique in a project. The search matches
+// substrings, so follow the pages until the exact name turns up.
+async function findTableIdByName(projectId: string, tableName: string): Promise<string | null> {
+ for (let offset = 0; ; ) {
+ const response = await warehouseTablesList(projectId, { search: tableName, include_columns: false, offset })
+ const match = response.results.find((table) => table.name === tableName)
+ if (match) {
+ return match.id
+ }
+ if (!response.next || response.results.length === 0) {
+ return null
+ }
+ offset += response.results.length
+ }
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface replaceFileLogicValues {
+ currentTeamId: number | null // teamLogic
+ file: File | null
+ fileError: string | null
+ fileFormat: FileUploadFormat
+ isReplaceFileModalOpen: boolean
+ isReplacing: boolean
+ replaceError: string | null
+ replaceStage: ReplaceFileStage | null
+ target: ReplaceFileTarget | null
+ uploadPercent: number
+ uploadedBytes: number
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface replaceFileLogicActions {
+ closeReplaceFile: () => {
+ value: true
+ }
+ openReplaceFile: (target: ReplaceFileTarget) => {
+ target: ReplaceFileTarget
+ }
+ replaceFileFailure: (error: string) => {
+ error: string
+ }
+ replaceFileSuccess: (table: TableApi) => {
+ table: TableApi
+ }
+ selectFile: (file: File | null) => {
+ file: File | null
+ }
+ setFileFormat: (fileFormat: FileUploadFormat) => {
+ fileFormat: FileUploadFormat
+ }
+ setReplaceStage: (stage: ReplaceFileStage) => {
+ stage: ReplaceFileStage
+ }
+ setUploadedBytes: (uploadedBytes: number) => {
+ uploadedBytes: number
+ }
+ submitReplaceFile: () => {
+ value: true
+ }
+}
+
+// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
+export interface replaceFileLogicMeta {
+ key: string
+ __keaTypeGenInternalSelectorTypes: {
+ isReplaceFileModalOpen: (target: ReplaceFileTarget | null) => boolean
+ isReplacing: (replaceStage: ReplaceFileStage | null) => boolean
+ fileError: (file: File | null, fileFormat: FileUploadFormat) => string | null
+ uploadPercent: (uploadedBytes: number, file: File | null) => number
+ }
+}
+
+export type replaceFileLogicType = MakeLogicType<
+ replaceFileLogicValues,
+ replaceFileLogicActions,
+ ReplaceFileLogicProps,
+ replaceFileLogicMeta
+>
+
+export const replaceFileLogic = kea<replaceFileLogicType>([
+ props({} as ReplaceFileLogicProps),
+ key(({ sourceId }) => sourceId),
+ path((key) => ['products', 'dataWarehouse', 'replaceFileLogic', key]),
+ connect(() => ({
+ values: [teamLogic, ['currentTeamId']],
+ })),
+ actions({
+ openReplaceFile: (target: ReplaceFileTarget) => ({ target }),
+ closeReplaceFile: true,
+ selectFile: (file: File | null) => ({ file }),
+ setFileFormat: (fileFormat: FileUploadFormat) => ({ fileFormat }),
+ submitReplaceFile: true,
+ setReplaceStage: (stage: ReplaceFileStage) => ({ stage }),
+ setUploadedBytes: (uploadedBytes: number) => ({ uploadedBytes }),
+ replaceFileSuccess: (table: TableApi) => ({ table }),
+ replaceFileFailure: (error: string) => ({ error }),
+ }),
+ reducers({
+ target: [
+ null as ReplaceFileTarget | null,
+ {
+ openReplaceFile: (_, { target }) => target,
+ closeReplaceFile: () => null,
+ },
+ ],
+ file: [
+ null as File | null,
+ {
+ selectFile: (_, { file }) => file,
+ openReplaceFile: () => null,
+ },
+ ],
+ fileFormat: [
+ 'csv' as FileUploadFormat,
+ {
+ setFileFormat: (_, { fileFormat }) => fileFormat,
+ },
+ ],
+ replaceStage: [
+ null as ReplaceFileStage | null,
+ {
+ setReplaceStage: (_, { stage }) => stage,
+ replaceFileSuccess: () => null,
+ replaceFileFailure: () => null,
+ },
+ ],
+ uploadedBytes: [
+ 0,
+ {
+ setUploadedBytes: (_, { uploadedBytes }) => uploadedBytes,
+ openReplaceFile: () => 0,
+ selectFile: () => 0,
+ },
+ ],
+ replaceError: [
+ null as string | null,
+ {
+ replaceFileFailure: (_, { error }) => error,
+ openReplaceFile: () => null,
+ selectFile: () => null,
+ setFileFormat: () => null,
+ setReplaceStage: () => null,
+ },
+ ],
+ }),
+ selectors({
+ isReplaceFileModalOpen: [(s) => [s.target], (target: ReplaceFileTarget | null): boolean => target !== null],
+ isReplacing: [
+ (s) => [s.replaceStage],
+ (replaceStage: ReplaceFileStage | null): boolean => replaceStage !== null,
+ ],
+ fileError: [
+ (s) => [s.file, s.fileFormat],
+ (file: File | null, fileFormat: FileUploadFormat): string | null => replaceFileError(file, fileFormat),
+ ],
+ uploadPercent: [
+ (s) => [s.uploadedBytes, s.file],
+ (uploadedBytes: number, file: File | null): number => {
+ if (!file?.size) {
+ return 0
+ }
+ // Multipart framing makes the sent byte count overshoot the file size at the end.
+ return Math.min(100, Math.round((uploadedBytes / file.size) * 100))
+ },
+ ],
+ }),
+ listeners(({ actions, values, props }) => ({
+ selectFile: ({ file }) => {
+ const detected = file ? detectFileFormat(file.name) : null
+ if (detected) {
+ actions.setFileFormat(detected)
+ }
+ },
+ submitReplaceFile: async () => {
+ // The stage is set before the first await, so a second submit while one is in flight stops here.
+ if (values.replaceStage !== null) {
+ return
+ }
+ const { file, fileFormat, target, fileError, currentTeamId } = values
+ if (!file || !target || fileError || !currentTeamId) {
+ return
+ }
+ const projectId = String(currentTeamId)
+ actions.setReplaceStage('upload')
+
+ const formData = new FormData()
+ formData.append('file', file)
+ formData.append('file_format', fileFormat)
+
+ let upload
+ try {
+ // The generated client sends uploads with fetch, which cannot report upload progress.
+ // nosemgrep: prefer-codegen-api-namespaced-data_warehouse
+ upload = await api.dataWarehouseTables.uploadFile(formData, {
+ onUploadProgress: ({ loaded }) => actions.setUploadedBytes(loaded),
+ })
+ } catch (e: any) {
+ posthog.capture('warehouse file replace failed', { stage: 'upload', file_format: fileFormat })
+ actions.replaceFileFailure(apiErrorMessage(e, 'Could not upload the file.'))
+ return
+ }
+
+ actions.setReplaceStage('replace')
+ try {
+ const tableId = target.tableId ?? (await findTableIdByName(projectId, target.tableName))
+ if (!tableId) {
+ throw new Error(`Could not find the table ${target.tableName}. Refresh the page and try again.`)
+ }
+ const table = await warehouseTablesReplaceFileCreate(projectId, tableId, {
+ upload_id: upload.upload_id,
+ filename: upload.filename,
+ file_format: fileFormat,
+ })
+ // pinned: analytics event names, renaming breaks dashboards
+ posthog.capture('warehouse file replaced', { file_format: fileFormat })
+ actions.replaceFileSuccess(table)
+ } catch (e: any) {
+ posthog.capture('warehouse file replace failed', { stage: 'replace', file_format: fileFormat })
+ actions.replaceFileFailure(apiErrorMessage(e, 'Could not replace the file.'))
+ }
+ },
+ replaceFileSuccess: () => {
+ lemonToast.success(REPLACE_FILE_SUCCESS_MESSAGE)
+ actions.closeReplaceFile()
+ sourceSettingsLogic.findMounted({ id: props.sourceId })?.actions.loadSource()
+ },
+ })),
+])
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts b/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts
index e85c0920b6d1..e0fabc53fb5d 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/sourceSettingsLogic.ts
@@ -36,6 +36,7 @@ import {
} from '~/types'
import { groupTablesBySchema } from 'products/data_warehouse/frontend/shared/components/forms/schemaGroupingUtils'
+import { FILE_UPLOAD_SOURCE_LABEL, isFileUploadSource } from 'products/data_warehouse/frontend/shared/fileUploadSources'
import {
clonePayloadPreservingFiles,
findUploadedFiles,
@@ -1361,7 +1362,9 @@ export const sourceSettingsLogic = kea<sourceSettingsLogicType>([
const breadcrumbName =
values.source?.access_method === 'direct'
? values.source?.prefix || values.source?.source_type || 'Source'
- : values.source?.source_type || 'Source'
+ : isFileUploadSource(values.source)
+ ? FILE_UPLOAD_SOURCE_LABEL
+ : values.source?.source_type || 'Source'
if (values.pollPauseCount === 0) {
cache.disposables.add(() => {
diff --git a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
index 645e61383b6a..1c0618e555fa 100644
--- a/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
+++ b/products/data_warehouse/frontend/shared/components/ManagedSourcesTable.tsx
@@ -18,6 +18,7 @@ import { AccessControlLevel, AccessControlResourceType, ExternalDataSchemaStatus
import { StatusTagSetting } from 'products/data_warehouse/frontend/utils'
import { availableSourcesLogic } from '../../scenes/NewSourceScene/availableSourcesLogic'
+import { isFileUploadSource, sourceTypeLabel } from '../fileUploadSources'
import { sourceManagementLogic } from '../logics/sourceManagementLogic'
import { FreeHistoricalSyncsBanner } from './FreeHistoricalSyncsBanner'
import { DATA_WAREHOUSE_APP_SOURCE } from './metrics/DataWarehouseMetrics'
@@ -84,7 +85,7 @@ export function ManagedSourcesTable(): JSX.Element {
render: (_, source) => (
<LemonTableLink
to={urls.dataWarehouseSource(`managed-${source.id}`)}
- title={availableSources?.[source.source_type]?.label ?? source.source_type}
+ title={sourceTypeLabel(source.source_type, availableSources)}
description={source.description}
/>
),
@@ -148,7 +149,10 @@ export function ManagedSourcesTable(): JSX.Element {
if (!source.status) {
return null
}
- const syncingSchemas = source.schemas.filter((s) => s.should_sync)
+ // A file upload schema never syncs on a schedule, so its imports still count.
+ const syncingSchemas = isFileUploadSource(source)
+ ? source.schemas
+ : source.schemas.filter((s) => s.should_sync)
const counts = SCHEMA_STATUS_ORDER.map((status) => ({
status,
schemas: syncingSchemas.filter((s) => s.status === status),
@@ -216,27 +220,29 @@ export function ManagedSourcesTable(): JSX.Element {
<More
overlay={
<>
- <AccessControlAction
- resourceType={AccessControlResourceType.ExternalDataSource}
- minAccessLevel={AccessControlLevel.Editor}
- userAccessLevel={source.user_access_level}
- >
- {({ disabledReason }) => (
- <Tooltip title="Start the data import for this schema again">
- <LemonButton
- type="tertiary"
- data-attr={`reload-data-warehouse-${source.source_type}`}
- key={`reload-data-warehouse-${source.source_type}`}
- onClick={() => {
- reloadSource(source)
- }}
- disabledReason={disabledReason}
- >
- Reload
- </LemonButton>
- </Tooltip>
- )}
- </AccessControlAction>
+ {!isFileUploadSource(source) && (
+ <AccessControlAction
+ resourceType={AccessControlResourceType.ExternalDataSource}
+ minAccessLevel={AccessControlLevel.Editor}
+ userAccessLevel={source.user_access_level}
+ >
+ {({ disabledReason }) => (
+ <Tooltip title="Start the data import for this schema again">
+ <LemonButton
+ type="tertiary"
+ data-attr={`reload-data-warehouse-${source.source_type}`}
+ key={`reload-data-warehouse-${source.source_type}`}
+ onClick={() => {
+ reloadSource(source)
+ }}
+ disabledReason={disabledReason}
+ >
+ Reload
+ </LemonButton>
+ </Tooltip>
+ )}
+ </AccessControlAction>
+ )}
<AccessControlAction
resourceType={AccessControlResourceType.ExternalDataSource}
diff --git a/products/data_warehouse/frontend/shared/fileUploadSources.ts b/products/data_warehouse/frontend/shared/fileUploadSources.ts
new file mode 100644
index 000000000000..9e8681d0b1e9
--- /dev/null
+++ b/products/data_warehouse/frontend/shared/fileUploadSources.ts
@@ -0,0 +1,27 @@
+import { ExternalDataSource } from '~/types'
+
+import {
+ ExternalDataSourceTypeEnumApi,
+ SourceConfigResponseApi,
+} from 'products/warehouse_sources/frontend/generated/api.schemas'
+
+export const FILE_UPLOAD_SOURCE_LABEL = 'File upload'
+
+/**
+ * A file upload source imports one file a user uploaded. It has no schedule and no incremental
+ * sync: PostHog starts each import itself, and the user refreshes the data by replacing the file.
+ */
+export function isFileUploadSource(source: Pick<ExternalDataSource, 'source_type'> | null | undefined): boolean {
+ return source?.source_type === ExternalDataSourceTypeEnumApi.FileUpload
+}
+
+// The source catalog leaves out file upload sources, so their label can't come from it.
+export function sourceTypeLabel(
+ sourceType: string,
+ availableSources: Record<string, SourceConfigResponseApi> | null | undefined
+): string {
+ if (sourceType === ExternalDataSourceTypeEnumApi.FileUpload) {
+ return FILE_UPLOAD_SOURCE_LABEL
+ }
+ return availableSources?.[sourceType]?.label ?? sourceType
+}