Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(marketing-analytics): use managed bucket sources for spend data (#130143)

Tom Owersfc34eb443a9c
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 99b4f8e39c13..08085f45a96f 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -6187,13 +6187,13 @@ snapshots:
marketing-analytics-add-source--new-sources--light:
hash: v1.k794b7964.5fa1307763c5b2c1d15d9da94126801e790cdb76e0d0de4e7afcd9cd27bc4c50.vlfBEVR6C5WxdqBMZe2QoAmr_I-A-oivfeAPBPrwLzc
marketing-analytics-add-source--open-menu--dark:
- hash: v1.k794b7964.d51ec98a614b79accdb49ca8d3cc27c0e5bf41ba6642340ecbf18a9389b0989a.l1VDDtUPoK7-IKhjk-mkyRGrqedbINjNK4n8VP64lBY
+ hash: v1.k794b7964.537899a871e20dfcd96d69281d25269969be5afc40c393f93d3eba46b9b53966.hzegqao9WPhKZFnDNvCBXT1ovPsa_TvnbSFxqFeuw_s
marketing-analytics-add-source--open-menu--light:
- hash: v1.k794b7964.d2e506d81189ed9a23402466d0a847ca0b1815ce8267e86da415a79aba8e2596.Nzx08xQi8RRBnXSJckpdS2LOmhyZ_6zpWMRp94bKQis
+ hash: v1.k794b7964.b5d4f1f2fa6dcafae9374aee7d0b7d9b715c06ccd68d39ef937037bda0959a7c.Bhdfzw817Z_qpcJtQ2l66YA0TXOvEc7rM_oJkTvspWA
marketing-analytics-add-source--sources-disabled--dark:
- hash: v1.k794b7964.f87bf5e8d4be0ba27ba4c75a32609858a380ed888ba6f152c93952e4eb0a890d.Af9g_9Sz2_nccEGsHh3C7U8xRV7qS6pdbVDNBTdfxnE
+ hash: v1.k794b7964.b66f3816ccb3efb854c5dd8313de78e2c6edbb53959644b368aafbfdda0bfbd1.-KjBBG6vtoItLGfeWvctcxm1OAJNWbo7mKyso6YQfVw
marketing-analytics-add-source--sources-disabled--light:
- hash: v1.k794b7964.a205467d269caecad3762b2a5e3ede203bff5181bc78311941b36f6ed7497bca.fwSMqwSNKC2ZfzTH0lJQBzRS4GgL8n2xjqfr0gudBXA
+ hash: v1.k794b7964.5075e5d859558adc6b9cf92469e385da31a35fd5afc495480adf217e7bef8029.w-rkxxX-Jx0KydKQEocVmDUCryDHiMFH477Ax63jfv8
marketing-analytics-conversion-recordings--empty--dark:
hash: v1.k794b7964.8982a2852c9527ff952b660792885e04c2fd81a3302a2c5c820c30e040182a06.h2Rie6hdM7FmCIHijYzFOkelhYaqzQboMKzRrwcxu8w
marketing-analytics-conversion-recordings--empty--light:
@@ -14115,9 +14115,9 @@ snapshots:
scenes-app-marketing-analytics-source-onboarding--manual-selection-with-search-console--light:
hash: v1.k794b7964.8c92db15f2a02f8a857e5415c8ede1df694b03e125117a87ff812481917fb2a8.B-oCrelZD1d7dV0sGePsUpciJbe19tYS8BrDPOcRM4U
scenes-app-marketing-analytics-source-onboarding--skip-during-scan--dark:
- hash: v1.k794b7964.47edb4acaf6be15f5f0f8bd40f30c834c5bc2ff7e47f357a72ec35fa332b159c.EZdlP9uCfGNLBeZX3R8gQtu6FtkHW8sHd0WY0b6fslM
+ hash: v1.k794b7964.3a974269e6af961392c0caae3576aa92c9de15d4f06240d8627d971c3e4664b9.cUqvawncbCGZHSN-llaIaXDipqxeSJS5IF6VgFDybaY
scenes-app-marketing-analytics-source-onboarding--skip-during-scan--light:
- hash: v1.k794b7964.a2d2c7e59c2ced8ef00221ae207eea27dabc4a81645cf14d1b6cb226baa24d2b.29rsM3fbhIfYk4EAExPUWRlURHw4YAiPt5U6WuoFh98
+ hash: v1.k794b7964.788651fbd8e66035ce9c7e416df23cb38e30a90cebc04233fe5b6a669200e735.cM83-F6eR_mRfLYaYzgq66iCukTdjXNxi7bSBudnuVY
scenes-app-marketing-analytics-source-setup-panel--detected-platforms--dark:
hash: v1.k794b7964.ae7dcb8da63611c945d804117b2ea522790cb21b19fc2302422e67decfad7885.2DfPP_Ou6x7e7cgYgTVY63SN-DSltVT-edZMhPPAGHI
scenes-app-marketing-analytics-source-setup-panel--detected-platforms--light:
diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsx b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsx
index 67bc23e97bb4..408a406adc07 100644
--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsx
+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsx
@@ -15,7 +15,7 @@ import {
NativeMarketingSource,
VALID_NATIVE_MARKETING_SOURCES,
} from '~/queries/schema/schema-general'
-import { ExternalDataSchemaStatus, ExternalDataSource, ManualLinkSourceType } from '~/types'
+import { ExternalDataSchemaStatus, ExternalDataSource } from '~/types'
import { SourceIcon } from 'products/data_warehouse/frontend/shared/components/SourceIcon'
import { AddIntegrationButton } from 'products/marketing_analytics/frontend/components/AddIntegrationButton'
@@ -29,12 +29,12 @@ import {
} from '../../logic/marketingAnalyticsLogic'
import { marketingAnalyticsSettingsLogic } from '../../logic/marketingAnalyticsSettingsLogic'
import {
+ ExternalTableTypeLabel,
MAX_ITEMS_TO_SHOW,
NEEDED_FIELDS_FOR_NATIVE_MARKETING_ANALYTICS,
- NonNativeMarketingSource,
- VALID_NON_NATIVE_MARKETING_SOURCES,
- VALID_SELF_MANAGED_MARKETING_SOURCES,
+ externalTableTypeLabel,
findSchemaByFieldName,
+ marketingSourceDisplayLabel,
nativeSourceDisplayLabel,
} from '../../logic/utils'
import { ColumnMappingModal } from './ColumnMappingModal'
@@ -46,7 +46,7 @@ type UnifiedSource = {
id: string
name: string
sourceType: string
- sourceTypeLabel: 'Native' | 'Data warehouse' | 'Self-managed'
+ sourceTypeLabel: 'Native' | ExternalTableTypeLabel
status: SourceStatus
statusMessage: string
sourceUrl?: string
@@ -55,7 +55,7 @@ type UnifiedSource = {
syncingTables?: string[]
tablesToSync?: string[]
nativeSource?: ExternalDataSource & { status?: string; statusMessage?: string }
- // For non-native/self-managed
+ // For non-native and self-managed tables
isTable?: boolean
table?: ExternalTable
}
@@ -115,38 +115,21 @@ export function ExternalDataSourceConfiguration(): JSX.Element {
}
}
- // Check if this is a BigQuery/warehouse source
- if (VALID_NON_NATIVE_MARKETING_SOURCES.includes(item.source_type as NonNativeMarketingSource)) {
- return {
- id: `table-${item.id}`,
- name: item.name,
- sourceType: `${item.source_type} ${item.source_prefix}`,
- sourceTypeLabel: 'Data warehouse' as const,
- status: item.status || MarketingSourceStatus.Error,
- statusMessage: item.statusMessage || 'Unknown status',
- sourceUrl: item.sourceUrl,
- isTable: true,
- table: item,
- }
+ const tableTypeLabel = externalTableTypeLabel(item.source_type)
+ if (!tableTypeLabel) {
+ return null
}
-
- // Check if this is a self-managed source
- if (VALID_SELF_MANAGED_MARKETING_SOURCES.includes(item.source_type as ManualLinkSourceType)) {
- return {
- id: `table-${item.id}`,
- name: item.name,
- sourceType: `${item.source_type} ${item.source_prefix}`,
- sourceTypeLabel: 'Self-managed' as const,
- status: item.status || MarketingSourceStatus.Error,
- statusMessage: item.statusMessage || 'Unknown status',
- sourceUrl: item.sourceUrl,
- isTable: true,
- table: item,
- }
+ return {
+ id: `table-${item.id}`,
+ name: item.name,
+ sourceType: `${marketingSourceDisplayLabel(item.source_type)} ${item.source_prefix}`,
+ sourceTypeLabel: tableTypeLabel,
+ status: item.status || MarketingSourceStatus.Error,
+ statusMessage: item.statusMessage || 'Unknown status',
+ sourceUrl: item.sourceUrl,
+ isTable: true,
+ table: item,
}
-
- // Fallback for unknown types
- return null
})
.filter(Boolean) as UnifiedSource[]
@@ -184,7 +167,7 @@ export function ExternalDataSourceConfiguration(): JSX.Element {
return (
<SceneSection
title="Data source configuration"
- description="Connect and configure data sources to enable marketing analytics. Native sources sync automatically, while warehouse and self-managed sources need column mapping."
+ description="Connect and configure data sources to enable marketing analytics. Native sources sync automatically, while warehouse, file, and self-managed sources need column mapping."
>
{hasNoConfiguredSources &&
(unconfiguredNativeSources.length > 0 ? (
@@ -246,9 +229,9 @@ export function ExternalDataSourceConfiguration(): JSX.Element {
type={
item.sourceTypeLabel === 'Native'
? 'success'
- : item.sourceTypeLabel === 'Data warehouse'
- ? 'default'
- : 'primary'
+ : item.sourceTypeLabel === 'Self-managed'
+ ? 'primary'
+ : 'default'
}
>
{item.sourceTypeLabel}
diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.ts
index ad4dc401ceee..865f3bf4a69b 100644
--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.ts
+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.ts
@@ -57,6 +57,7 @@ import {
marketingAnalyticsTilesLogic,
} from './marketingAnalyticsTilesLogic'
import { activeColumnView, columnsForView, marketingTableDimensionColumns } from './marketingTableColumnViews'
+import { externalTableTypeLabel } from './utils'
jest.mock('posthog-js')
@@ -1133,6 +1134,84 @@ describe('marketingAnalyticsLogic', () => {
}
})
+ describe.each([
+ ['AmazonS3', 'aws', 'https://example-bucket.s3.amazonaws.com/spend/*.csv'],
+ ['GoogleCloudStorage', 'google-cloud', 'https://storage.googleapis.com/example-bucket/spend/*.csv'],
+ ['AzureBlob', 'azure', 'https://example.blob.core.windows.net/spend/*.csv'],
+ ['CloudflareR2', 'cloudflare-r2', 'https://example.r2.cloudflarestorage.com/example-bucket/spend/*.csv'],
+ ])('a %s spend table', (sourceType, provider, customerUrl) => {
+ // A table that was never migrated and one that its source has not synced yet look the same here.
+ it.each([
+ ['still self-managed', 'example-table', provider, 'Self-managed', 'self-managed'],
+ ['adopted by its source', 'example-table', sourceType, 'Files', 'managed'],
+ ['created by its source', 'example-schema', sourceType, 'Files', 'managed'],
+ ])('keeps its mapping when %s', async (state, mappingKey, expectedSourceType, typeLabel, dwSourceType) => {
+ logic = marketingAnalyticsLogic()
+ logic.mount()
+ await expectLogic(logic).toFinishAllListeners()
+ const managed = state !== 'still self-managed'
+ const table: DatabaseSchemaDataWarehouseTable = {
+ id: 'example-table',
+ name: managed ? `${sourceType.toLowerCase()}.example_spend` : 'example_spend',
+ type: 'data_warehouse',
+ url_pattern: managed
+ ? 'https://example-warehouse.s3.amazonaws.com/team_1/example_spend/*.parquet'
+ : customerUrl,
+ ...(managed
+ ? {
+ schema: {
+ id: 'example-schema',
+ name: 'example_spend',
+ should_sync: false,
+ incremental: false,
+ status: ExternalDataSchemaStatus.Completed,
+ },
+ source: { id: 'example-source', source_type: sourceType, status: 'Completed', prefix: '' },
+ }
+ : {}),
+ fields: Object.fromEntries(
+ ['id', 'campaign', 'source', 'spend', 'day'].map((name) => [
+ name,
+ { name, hogql_value: name, type: 'string', schema_valid: true },
+ ])
+ ),
+ }
+ databaseTableListLogic.actions.loadDatabaseSuccess({ tables: { [table.name]: table }, joins: [] })
+ await expectLogic(logic, () =>
+ teamLogic.actions.loadCurrentTeamSuccess({
+ ...teamLogic.values.currentTeam!,
+ marketing_analytics_config: {
+ sources_map: {
+ [mappingKey]: {
+ id: 'id',
+ campaign: 'campaign',
+ source: 'source',
+ cost: 'spend',
+ date: 'day',
+ } as SourceMap,
+ },
+ },
+ })
+ ).toFinishAllListeners()
+
+ expect(logic.values.validExternalTables).toEqual([
+ expect.objectContaining({
+ source_map_id: mappingKey,
+ source_type: expectedSourceType,
+ source_map: expect.objectContaining({ cost: 'spend' }),
+ }),
+ ])
+ expect(logic.values.allExternalTablesWithStatus.filter((row) => row.id === table.id)).toEqual([
+ expect.objectContaining({ source_map_id: mappingKey, source_type: expectedSourceType }),
+ ])
+ expect(externalTableTypeLabel(expectedSourceType)).toBe(typeLabel)
+ expect(logic.values.marketingChartSourceIds).toEqual([mappingKey])
+ expect(logic.values.createMarketingDataWarehouseNodes).toEqual([
+ expect.objectContaining({ id: 'example-table', dw_source_type: dwSourceType }),
+ ])
+ })
+ })
+
it('keeps the selection and drops an unknown key from a filter saved by an older build', async () => {
localStorage.setItem(STORAGE_KEY, JSON.stringify({ integrationSourceIds: ['source-1'], removedOption: true }))
diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.ts
index a9f50973bc75..3293c11740fc 100644
--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.ts
+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.ts
@@ -64,6 +64,7 @@ import {
findSchemaByFieldName,
generateUniqueName,
getEnabledNativeMarketingSources,
+ getSourceMapId,
sanitizeIntegrationFilter,
validColumnsForTiles,
} from './utils'
@@ -1124,14 +1125,15 @@ export const marketingAnalyticsLogic = kea<marketingAnalyticsLogicType>([
const tableType = isDataWarehouse
? DataWarehouseSettingsTab.Managed
: DataWarehouseSettingsTab.SelfManaged
- const sourceMap = sources_map?.[table.schema?.id || ''] ?? sources_map?.[table.id] ?? null
+ const sourceMapId = getSourceMapId(table, sources_map)
externalTables.push({
...table,
name: table.name,
url_pattern: table.url_pattern || '',
+ // An adopted table reads PostHog's storage, so only its source names the provider.
source_type: table.source?.source_type || mapUrlToProvider(table.url_pattern),
- source_map_id: table.schema?.id || table.source?.id || table.id,
+ source_map_id: sourceMapId,
source_prefix: table.source?.prefix || '',
columns: Object.keys(table.fields).map((field) => ({
name: table.fields[field].hogql_value,
@@ -1141,7 +1143,7 @@ export const marketingAnalyticsLogic = kea<marketingAnalyticsLogicType>([
`${tableType}-${dataWarehouseSource?.id || table.source?.id || table.id}`
),
external_type: tableType,
- source_map: sourceMap,
+ source_map: sources_map?.[sourceMapId] ?? null,
schema_name: table.schema?.name || table.name,
dw_source_type: tableType,
schema_status: table.schema?.status,
diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.ts
index 8faa1e5c7c3c..b29e161e44c4 100644
--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.ts
+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.ts
@@ -24,6 +24,7 @@ import { NativeSource } from './marketingAnalyticsLogic'
import {
conversionGoalToSeries,
createMarketingTile,
+ externalTableTypeLabel,
findSchemaByFieldName,
getEnabledNativeMarketingSources,
getOrderBy,
@@ -757,6 +758,15 @@ describe('marketing analytics utils', () => {
}
)
+ it.each([
+ ['BigQuery', 'Data warehouse'],
+ ['FileUpload', 'Files'],
+ ['google-cloud', 'Self-managed'],
+ ['Postgres', null],
+ ])('labels a %s table as %s in the source settings', (sourceType, label) => {
+ expect(externalTableTypeLabel(sourceType)).toBe(label)
+ })
+
describe('sanitizeIntegrationFilter', () => {
it('drops a key the query schema no longer accepts', () => {
const stored = { integrationSourceIds: ['abc'], removedOption: true }
diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.ts
index 48f37f1f4923..182a7b9c22d9 100644
--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.ts
+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.ts
@@ -28,15 +28,49 @@ import { ExternalDataSource, HogQLMathType, ManualLinkSourceType, PropertyMathTy
import { NativeSource } from './marketingAnalyticsLogic'
-export type NonNativeMarketingSource = 'BigQuery'
-
-export const VALID_NON_NATIVE_MARKETING_SOURCES: NonNativeMarketingSource[] = ['BigQuery']
-export const VALID_SELF_MANAGED_MARKETING_SOURCES: ManualLinkSourceType[] = [
- 'aws',
- 'google-cloud',
- 'cloudflare-r2',
- 'azure',
+type WarehouseMarketingSource = 'BigQuery'
+// Managed sources that import spend data from files: a storage bucket or an uploaded file.
+type FileMarketingSource = 'AmazonS3' | 'GoogleCloudStorage' | 'AzureBlob' | 'CloudflareR2' | 'FileUpload'
+
+// Together these mirror the backend VALID_NON_NATIVE_MARKETING_SOURCES (constants.py).
+export const VALID_WAREHOUSE_MARKETING_SOURCES: WarehouseMarketingSource[] = ['BigQuery']
+export const VALID_FILE_MARKETING_SOURCES: FileMarketingSource[] = [
+ 'AmazonS3',
+ 'GoogleCloudStorage',
+ 'AzureBlob',
+ 'CloudflareR2',
+ 'FileUpload',
]
+const VALID_SELF_MANAGED_MARKETING_SOURCES: ManualLinkSourceType[] = ['aws', 'google-cloud', 'cloudflare-r2', 'azure']
+
+export type ExternalTableTypeLabel = 'Data warehouse' | 'Files' | 'Self-managed'
+
+/** The type a mapped table shows in the settings list, or null when marketing analytics can't use the table. */
+export function externalTableTypeLabel(sourceType: string): ExternalTableTypeLabel | null {
+ if (VALID_WAREHOUSE_MARKETING_SOURCES.includes(sourceType as WarehouseMarketingSource)) {
+ return 'Data warehouse'
+ }
+ if (VALID_FILE_MARKETING_SOURCES.includes(sourceType as FileMarketingSource)) {
+ return 'Files'
+ }
+ if (VALID_SELF_MANAGED_MARKETING_SOURCES.includes(sourceType as ManualLinkSourceType)) {
+ return 'Self-managed'
+ }
+ return null
+}
+
+/**
+ * The sources_map key of a warehouse table. A self-managed table keeps its id when a managed source
+ * adopts it, so its mapping stays under the table id. Use the key that holds the mapping, in the order
+ * the backend reads them. With no mapping, use the key for a new mapping.
+ */
+export function getSourceMapId(
+ table: { id: string; schema?: { id: string } | null; source?: { id: string } | null },
+ sourcesMap: Record<string, unknown> | null | undefined
+): string {
+ const savedKey = [table.schema?.id, table.id].find((key) => !!key && sourcesMap?.[key] != null)
+ return savedKey || table.schema?.id || table.source?.id || table.id
+}
export const NATIVE_SOURCE_FEATURE_FLAGS: Partial<Record<NativeMarketingSource, FeatureFlagKey>> = {
AmazonAds: FEATURE_FLAGS.MARKETING_ANALYTICS_AMAZON_ADS,
@@ -117,6 +151,18 @@ export function nativeSourceDisplayLabel(sourceType: string): string {
return NATIVE_SOURCE_DISPLAY_LABELS[sourceType as NativeMarketingSource] ?? sourceType
}
+const FILE_SOURCE_DISPLAY_LABELS: Record<FileMarketingSource, string> = {
+ AmazonS3: 'Amazon S3',
+ GoogleCloudStorage: 'Google Cloud Storage',
+ AzureBlob: 'Azure Blob Storage',
+ CloudflareR2: 'Cloudflare R2',
+ FileUpload: 'File upload',
+}
+
+export function marketingSourceDisplayLabel(sourceType: string): string {
+ return FILE_SOURCE_DISPLAY_LABELS[sourceType as FileMarketingSource] ?? nativeSourceDisplayLabel(sourceType)
+}
+
// The id suffix tells apart two accounts of one platform that have no prefix.
export function marketingSourceName(source: Pick<ExternalDataSource, 'id' | 'prefix' | 'source_type'>): string {
return source.prefix || `${nativeSourceDisplayLabel(source.source_type)} (${source.id.slice(0, 8)})`
diff --git a/products/marketing_analytics/backend/hogql_queries/adapters/factory.py b/products/marketing_analytics/backend/hogql_queries/adapters/factory.py
index 53b0b016b37b..1b66d28d5ed0 100644
--- a/products/marketing_analytics/backend/hogql_queries/adapters/factory.py
+++ b/products/marketing_analytics/backend/hogql_queries/adapters/factory.py
@@ -48,6 +48,7 @@ from .base import (
TikTokAdsConfig,
)
from .bigquery import BigQueryAdapter
+from .file_source import FileSourceAdapter
from .google_ads import GoogleAdsAdapter
from .openai_ads import OpenAIAdsAdapter
from .rokt_ads import RoktAdsAdapter
@@ -97,6 +98,11 @@ class MarketingSourceFactory:
"PinterestAds": PinterestAdsAdapter,
# Non-native adapters
"BigQuery": BigQueryAdapter,
+ "AmazonS3": FileSourceAdapter,
+ "GoogleCloudStorage": FileSourceAdapter,
+ "AzureBlob": FileSourceAdapter,
+ "CloudflareR2": FileSourceAdapter,
+ "FileUpload": FileSourceAdapter,
# Self-managed adapters
"aws": AWSAdapter,
"google-cloud": GoogleCloudAdapter,
@@ -339,14 +345,17 @@ class MarketingSourceFactory:
continue
for table in tables:
- source_map = self._get_source_map_for_table(table, str(source.id))
- if not source_map:
+ source_map_key = self._get_source_map_key_for_table(table, str(source.id))
+ if source_map_key is None:
continue
+ source_map = self._sources_map[source_map_key]
# For non-native: use schema ID to match frontend (table.schema?.id || table.source?.id || table.id).
# Iterate the prefetched set — `.first()` issues a fresh query and bypasses the prefetch cache.
schema = next(iter(table.externaldataschema_set.all()), None)
- source_id = str(schema.id) if schema else str(table.id)
+ # A self-managed table that a source adopts keeps its id, so its mapping stays under the
+ # table id. The frontend then uses the table id too, so the ids must agree.
+ source_id = str(schema.id) if schema and source_map_key != str(table.id) else str(table.id)
config = ExternalConfig(
table=table,
@@ -400,23 +409,19 @@ class MarketingSourceFactory:
def _get_source_map_for_table(self, table: DataWarehouseTable, source_id: str | None = None) -> Optional[SourceMap]:
"""Get source map for a table"""
- if source_id and table.external_data_source and table.external_data_source.source_type:
- # Managed table
- table_id = str(table.id)
- schema_id = self._get_schema_id_for_table(table)
+ key = self._get_source_map_key_for_table(table, source_id)
+ return self._sources_map[key] if key is not None else None
- # Try schema_id, table_id, then source_id
- for key in [schema_id, table_id, source_id]:
- if key and key in self._sources_map:
- return self._sources_map[key]
+ def _get_source_map_key_for_table(self, table: DataWarehouseTable, source_id: str | None = None) -> str | None:
+ """Get the sources_map key that holds the mapping of a table"""
+ if source_id and table.external_data_source and table.external_data_source.source_type:
+ # Managed table: try schema_id, table_id, then source_id
+ keys = [self._get_schema_id_for_table(table), str(table.id), source_id]
else:
- if str(table.id) in self._sources_map:
- # Self-managed table
- return self._sources_map[str(table.id)]
- else:
- return None
+ # Self-managed table
+ keys = [str(table.id)]
- return None
+ return next((key for key in keys if key and key in self._sources_map), None)
def get_valid_adapters(self, adapters: list[MarketingSourceAdapter]) -> list[MarketingSourceAdapter]:
"""Filter adapters to only return valid ones"""
diff --git a/products/marketing_analytics/backend/hogql_queries/adapters/file_source.py b/products/marketing_analytics/backend/hogql_queries/adapters/file_source.py
new file mode 100644
index 000000000000..46385332508d
--- /dev/null
+++ b/products/marketing_analytics/backend/hogql_queries/adapters/file_source.py
@@ -0,0 +1,16 @@
+# File Marketing Source Adapter
+
+from .self_managed import SelfManagedAdapter
+
+
+class FileSourceAdapter(SelfManagedAdapter):
+ """
+ Adapter for managed sources that import spend data from files: Amazon S3, Google Cloud Storage,
+ Azure Blob Storage, Cloudflare R2 and file uploads.
+
+ These sources replace self-managed tables. They need the same column mapping, so the adapter
+ shares all field resolution and query-building logic with SelfManagedAdapter.
+ """
+
+ def get_source_type(self) -> str:
+ return self.config.source_type
diff --git a/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.py b/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.py
index 8c6067b788b9..d92b9140784e 100644
--- a/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.py
+++ b/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.py
@@ -27,9 +27,22 @@ from products.marketing_analytics.backend.hogql_queries.adapters.base import (
TikTokAdsConfig,
)
from products.marketing_analytics.backend.hogql_queries.adapters.factory import MarketingSourceFactory
+from products.marketing_analytics.backend.hogql_queries.adapters.file_source import FileSourceAdapter
from products.marketing_analytics.backend.hogql_queries.adapters.meta_ads import MetaAdsAdapter
+from products.marketing_analytics.backend.hogql_queries.adapters.self_managed import (
+ AWSAdapter,
+ AzureAdapter,
+ CloudflareR2Adapter,
+ GoogleCloudAdapter,
+ SelfManagedAdapter,
+)
from products.marketing_analytics.backend.hogql_queries.constants import TABLE_PATTERNS
-from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSource
+from products.warehouse_sources.backend.facade.models import (
+ DataWarehouseCredential,
+ DataWarehouseTable,
+ ExternalDataSchema,
+ ExternalDataSource,
+)
class FactoryTestMixin:
@@ -598,3 +611,81 @@ class TestNativeSourceKillSwitch(SimpleTestCase):
assert set(errors) == ({adapter.config.source_id for adapter in adapters[1:]} if enabled else set())
assert all(messages for messages in errors.values())
factory.logger.exception.assert_not_called()
+
+
+_BUCKET_PROVIDERS: list[tuple[str, str, type[SelfManagedAdapter]]] = [
+ ("AmazonS3", "https://example-bucket.s3.amazonaws.com/spend/*.csv", AWSAdapter),
+ ("GoogleCloudStorage", "https://storage.googleapis.com/example-bucket/spend/*.csv", GoogleCloudAdapter),
+ ("AzureBlob", "https://example.blob.core.windows.net/spend/*.csv", AzureAdapter),
+ ("CloudflareR2", "https://example.r2.cloudflarestorage.com/example-bucket/spend/*.csv", CloudflareR2Adapter),
+]
+_POSTHOG_STORAGE_URL = "https://example-warehouse.s3.amazonaws.com/team_1/example_spend/*.parquet"
+_TABLE_STATES = ["never_migrated", "migrated_not_adopted", "adopted", "new_managed_table"]
+
+
+class TestFileSourceTableDiscovery(BaseTest):
+ def _create_table_state(self, state: str, source_type: str, customer_url: str) -> tuple[DataWarehouseTable, str]:
+ managed = state in ("adopted", "new_managed_table")
+ source = None
+ if state != "never_migrated":
+ source = ExternalDataSource.objects.create(
+ team=self.team, source_id="example", connection_id="example", source_type=source_type
+ )
+ table = DataWarehouseTable.raw_objects.create(
+ team=self.team,
+ name="example_spend",
+ format="CSVWithNames",
+ url_pattern=_POSTHOG_STORAGE_URL if managed else customer_url,
+ credential=None
+ if managed
+ else DataWarehouseCredential.objects.create(team=self.team, access_key="key", access_secret="secret"),
+ external_data_source=source if managed else None,
+ )
+ schema = None
+ if source is not None:
+ schema = ExternalDataSchema.objects.create(
+ team=self.team, source=source, name="example_spend", table=table if managed else None
+ )
+ mapping_key = str(schema.id) if state == "new_managed_table" and schema else str(table.id)
+ self.team.marketing_analytics_config.sources_map = {
+ mapping_key: {"campaign": "campaign", "source": "source", "cost": "spend", "date": "day"}
+ }
+ self.team.marketing_analytics_config.save()
+ return table, mapping_key
+
+ def _make_factory(self, table: DataWarehouseTable) -> MarketingSourceFactory:
+ database = Mock(get_warehouse_table_names=Mock(return_value=[table.name]))
+ context = QueryContext(date_range=None, team=self.team, base_currency=DEFAULT_CURRENCY, database=database)
+ return MarketingSourceFactory(context=context)
+
+ @parameterized.expand(
+ [
+ (f"{state}_{source_type}", state, source_type, customer_url, self_managed_adapter)
+ for source_type, customer_url, self_managed_adapter in _BUCKET_PROVIDERS
+ for state in _TABLE_STATES
+ ]
+ + [(f"{state}_FileUpload", state, "FileUpload", "", SelfManagedAdapter) for state in _TABLE_STATES[2:]]
+ )
+ def test_table_gets_one_adapter_with_its_mapping(
+ self,
+ _name: str,
+ state: str,
+ source_type: str,
+ customer_url: str,
+ self_managed_adapter: type[SelfManagedAdapter],
+ ) -> None:
+ table, mapping_key = self._create_table_state(state, source_type, customer_url)
+ factory = self._make_factory(table)
+
+ adapters = factory.create_adapters(raise_on_error=True)
+
+ assert len(adapters) == 1
+ adapter = adapters[0]
+ if state in ("adopted", "new_managed_table"):
+ assert type(adapter) is FileSourceAdapter
+ assert adapter.get_source_type() == source_type
+ else:
+ assert type(adapter) is self_managed_adapter
+ assert adapter.config.source_id == mapping_key
+ assert adapter.config.source_map.cost == "spend"
+ assert factory.get_valid_adapters(adapters) == adapters
diff --git a/products/marketing_analytics/backend/hogql_queries/constants.py b/products/marketing_analytics/backend/hogql_queries/constants.py
index b6c0d95c81b5..1f447796e598 100644
--- a/products/marketing_analytics/backend/hogql_queries/constants.py
+++ b/products/marketing_analytics/backend/hogql_queries/constants.py
@@ -502,8 +502,11 @@ MARKETING_ANALYTICS_SCHEMA = {
# Valid native marketing sources - derived from generated enum
VALID_NATIVE_MARKETING_SOURCES = [source.value for source in NativeMarketingSource]
-# Valid non-native marketing sources (managed external sources like BigQuery)
-VALID_NON_NATIVE_MARKETING_SOURCES = ["BigQuery"]
+# Managed sources that import spend data from files: a storage bucket or an uploaded file.
+VALID_FILE_MARKETING_SOURCES = ["AmazonS3", "GoogleCloudStorage", "AzureBlob", "CloudflareR2", "FileUpload"]
+
+# Valid non-native marketing sources (managed external sources that need a column mapping)
+VALID_NON_NATIVE_MARKETING_SOURCES = ["BigQuery", *VALID_FILE_MARKETING_SOURCES]
# Valid self-managed marketing sources (mirrors frontend types)
VALID_SELF_MANAGED_MARKETING_SOURCES = ["aws", "google-cloud", "cloudflare-r2", "azure"]
diff --git a/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsx b/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsx
index 99af0f0f76a1..956b30b06182 100644
--- a/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsx
+++ b/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsx
@@ -61,6 +61,10 @@ const meta: Meta<typeof AddIntegrationButton> = {
AmazonAds: 'amazon_ads.png',
TwitterAds: 'twitter_ads.png',
BigQuery: 'bigquery.png',
+ AmazonS3: 'aws-s3.png',
+ GoogleCloudStorage: 'google-cloud-storage.png',
+ AzureBlob: 'azure_blob.png',
+ CloudflareR2: 'cloudflare.png',
}).map(([name, icon]) => [
name,
{ name, caption: name, iconPath: `/static/services/${icon}`, fields: [] },
diff --git a/products/marketing_analytics/frontend/components/AddIntegrationButton.tsx b/products/marketing_analytics/frontend/components/AddIntegrationButton.tsx
index fc0f2f5bd261..050702b1c210 100644
--- a/products/marketing_analytics/frontend/components/AddIntegrationButton.tsx
+++ b/products/marketing_analytics/frontend/components/AddIntegrationButton.tsx
@@ -9,8 +9,10 @@ import { FEATURE_FLAGS, TeamMembershipLevel } from 'lib/constants'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { urls } from 'scenes/urls'
import {
- VALID_NON_NATIVE_MARKETING_SOURCES,
+ VALID_FILE_MARKETING_SOURCES,
+ VALID_WAREHOUSE_MARKETING_SOURCES,
getEnabledNativeMarketingSources,
+ marketingSourceDisplayLabel,
nativeSourceDisplayLabel,
} from 'scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils'
@@ -53,7 +55,8 @@ export function AddIntegrationButton({ onIntegrationSelect }: AddIntegrationButt
...(featureFlags[FEATURE_FLAGS.MARKETING_ANALYTICS_ORGANIC_KEYWORDS]
? [{ title: 'Organic search', sources: ['GoogleSearchConsole'] }]
: []),
- { title: 'External sources', sources: VALID_NON_NATIVE_MARKETING_SOURCES },
+ { title: 'External sources', sources: VALID_WAREHOUSE_MARKETING_SOURCES },
+ { title: 'Files', sources: VALID_FILE_MARKETING_SOURCES },
]
const items: LemonMenuItems = [
...(showNewSourcesInMenu
@@ -81,7 +84,7 @@ export function AddIntegrationButton({ onIntegrationSelect }: AddIntegrationButt
label:
integrationId === 'GoogleSearchConsole'
? 'Google Search Console'
- : nativeSourceDisplayLabel(integrationId),
+ : marketingSourceDisplayLabel(integrationId),
icon: <SourceIcon type={integrationId} size="xsmall" disableTooltip />,
tag:
showNewSourcesInMenu && newSources.some((source) => source === integrationId)
diff --git a/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsx b/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsx
index 9a1063e2cc72..47dfa6f70b1d 100644
--- a/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsx
+++ b/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsx
@@ -13,9 +13,10 @@ import { teamLogic } from 'scenes/teamLogic'
import { urls } from 'scenes/urls'
import { marketingAnalyticsLogic } from 'scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic'
import {
- VALID_NON_NATIVE_MARKETING_SOURCES,
+ VALID_FILE_MARKETING_SOURCES,
+ VALID_WAREHOUSE_MARKETING_SOURCES,
getEnabledNativeMarketingSourcesInDisplayOrder,
- nativeSourceDisplayLabel,
+ marketingSourceDisplayLabel,
} from 'scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils'
import { ProductIntentContext, ProductKey } from '~/queries/schema/schema-general'
@@ -27,7 +28,7 @@ const HedgehogMoney = pngHoggie(moneyPng)
interface MarketingSource {
id: string
isConnected: boolean
- category: 'native' | 'external'
+ category: 'native' | 'external' | 'file'
}
interface SourceCatalogProps {
@@ -52,11 +53,16 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogP
isConnected: validNativeSources.some((source) => source.source.source_type === sourceType),
category: 'native' as const,
})),
- ...VALID_NON_NATIVE_MARKETING_SOURCES.map((sourceType) => ({
+ ...VALID_WAREHOUSE_MARKETING_SOURCES.map((sourceType) => ({
id: sourceType,
isConnected: validExternalTables.some((table) => table.source_type === sourceType),
category: 'external' as const,
})),
+ ...VALID_FILE_MARKETING_SOURCES.map((sourceType) => ({
+ id: sourceType,
+ isConnected: validExternalTables.some((table) => table.source_type === sourceType),
+ category: 'file' as const,
+ })),
]
const handleSourceSelect = (sourceId: string): void => {
@@ -75,16 +81,12 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogP
)
}
- const nativeSources = allSources.filter(
- (s) =>
- nativeSourceDisplayLabel(s.id).toLowerCase().includes(manualSourceSearch.toLowerCase()) &&
- s.category === 'native'
- )
- const externalSources = allSources.filter(
- (s) =>
- nativeSourceDisplayLabel(s.id).toLowerCase().includes(manualSourceSearch.toLowerCase()) &&
- s.category === 'external'
+ const matchingSources = allSources.filter((s) =>
+ marketingSourceDisplayLabel(s.id).toLowerCase().includes(manualSourceSearch.toLowerCase())
)
+ const nativeSources = matchingSources.filter((s) => s.category === 'native')
+ const externalSources = matchingSources.filter((s) => s.category === 'external')
+ const fileSources = matchingSources.filter((s) => s.category === 'file')
const totalConnected = validNativeSources.length + validExternalTables.length
@@ -111,7 +113,7 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogP
value={manualSourceSearch}
onChange={setManualSourceSearch}
/>
- {!nativeSources.length && !externalSources.length && (
+ {!nativeSources.length && !externalSources.length && !fileSources.length && (
<p className="text-secondary">No integrations match your search.</p>
)}
{/* Native Sources */}
@@ -145,6 +147,26 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogP
</div>
</div>
)}
+
+ {/* File Sources */}
+ {fileSources.length > 0 && (
+ <div>
+ <div className="text-xs font-medium text-muted mb-1.5 flex items-center gap-1">
+ Files
+ <Tooltip
+ title="Import spend data from CSV, JSON, or Parquet files in a storage bucket, or upload a file."
+ delayMs={0}
+ >
+ <IconInfo className="w-3 h-3 cursor-help" />
+ </Tooltip>
+ </div>
+ <div className="flex flex-wrap gap-2">
+ {fileSources.map((source) => (
+ <SourceChip key={source.id} source={source} onSelect={handleSourceSelect} />
+ ))}
+ </div>
+ </div>
+ )}
</div>
<div className="border-t p-4 flex flex-wrap items-center justify-between gap-3 bg-bg-light">
{onBack && (
@@ -184,7 +206,7 @@ function SourceChip({ source, onSelect }: { source: MarketingSource; onSelect: (
onClick={() => onSelect(source.id)}
data-attr="marketing-manual-connect-source"
>
- {nativeSourceDisplayLabel(source.id)}
+ {marketingSourceDisplayLabel(source.id)}
{source.isConnected && <IconCheckCircle className="ml-2 text-success" />}
</LemonButton>
)
diff --git a/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsx b/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsx
index f36fc32c2f44..fd85257a708e 100644
--- a/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsx
+++ b/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsx
@@ -71,6 +71,10 @@ const meta: Meta = {
SnapchatAds: { iconPath: '/static/services/snapchat.png' },
PinterestAds: { iconPath: '/static/services/pinterest_ads.png' },
BigQuery: { iconPath: '/static/services/bigquery.png' },
+ AmazonS3: { iconPath: '/static/services/aws-s3.png' },
+ GoogleCloudStorage: { iconPath: '/static/services/google-cloud-storage.png' },
+ AzureBlob: { iconPath: '/static/services/azure_blob.png' },
+ CloudflareR2: { iconPath: '/static/services/cloudflare.png' },
},
],
},