:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(marketing-analytics): use managed bucket sources for spend data (#130143)
Tom Owersfc34eb443a9c
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 99b4f8e39c13..08085f45a96f 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -6187,13 +6187,13 @@ snapshots:marketing-analytics-add-source--new-sources--light:hash: v1.k794b7964.5fa1307763c5b2c1d15d9da94126801e790cdb76e0d0de4e7afcd9cd27bc4c50.vlfBEVR6C5WxdqBMZe2QoAmr_I-A-oivfeAPBPrwLzcmarketing-analytics-add-source--open-menu--dark:- hash: v1.k794b7964.d51ec98a614b79accdb49ca8d3cc27c0e5bf41ba6642340ecbf18a9389b0989a.l1VDDtUPoK7-IKhjk-mkyRGrqedbINjNK4n8VP64lBY+ hash: v1.k794b7964.537899a871e20dfcd96d69281d25269969be5afc40c393f93d3eba46b9b53966.hzegqao9WPhKZFnDNvCBXT1ovPsa_TvnbSFxqFeuw_smarketing-analytics-add-source--open-menu--light:- hash: v1.k794b7964.d2e506d81189ed9a23402466d0a847ca0b1815ce8267e86da415a79aba8e2596.Nzx08xQi8RRBnXSJckpdS2LOmhyZ_6zpWMRp94bKQis+ hash: v1.k794b7964.b5d4f1f2fa6dcafae9374aee7d0b7d9b715c06ccd68d39ef937037bda0959a7c.Bhdfzw817Z_qpcJtQ2l66YA0TXOvEc7rM_oJkTvspWAmarketing-analytics-add-source--sources-disabled--dark:- hash: v1.k794b7964.f87bf5e8d4be0ba27ba4c75a32609858a380ed888ba6f152c93952e4eb0a890d.Af9g_9Sz2_nccEGsHh3C7U8xRV7qS6pdbVDNBTdfxnE+ hash: v1.k794b7964.b66f3816ccb3efb854c5dd8313de78e2c6edbb53959644b368aafbfdda0bfbd1.-KjBBG6vtoItLGfeWvctcxm1OAJNWbo7mKyso6YQfVwmarketing-analytics-add-source--sources-disabled--light:- hash: v1.k794b7964.a205467d269caecad3762b2a5e3ede203bff5181bc78311941b36f6ed7497bca.fwSMqwSNKC2ZfzTH0lJQBzRS4GgL8n2xjqfr0gudBXA+ hash: v1.k794b7964.5075e5d859558adc6b9cf92469e385da31a35fd5afc495480adf217e7bef8029.w-rkxxX-Jx0KydKQEocVmDUCryDHiMFH477Ax63jfv8marketing-analytics-conversion-recordings--empty--dark:hash: v1.k794b7964.8982a2852c9527ff952b660792885e04c2fd81a3302a2c5c820c30e040182a06.h2Rie6hdM7FmCIHijYzFOkelhYaqzQboMKzRrwcxu8wmarketing-analytics-conversion-recordings--empty--light:@@ -14115,9 +14115,9 @@ snapshots:scenes-app-marketing-analytics-source-onboarding--manual-selection-with-search-console--light:hash: v1.k794b7964.8c92db15f2a02f8a857e5415c8ede1df694b03e125117a87ff812481917fb2a8.B-oCrelZD1d7dV0sGePsUpciJbe19tYS8BrDPOcRM4Uscenes-app-marketing-analytics-source-onboarding--skip-during-scan--dark:- hash: v1.k794b7964.47edb4acaf6be15f5f0f8bd40f30c834c5bc2ff7e47f357a72ec35fa332b159c.EZdlP9uCfGNLBeZX3R8gQtu6FtkHW8sHd0WY0b6fslM+ hash: v1.k794b7964.3a974269e6af961392c0caae3576aa92c9de15d4f06240d8627d971c3e4664b9.cUqvawncbCGZHSN-llaIaXDipqxeSJS5IF6VgFDybaYscenes-app-marketing-analytics-source-onboarding--skip-during-scan--light:- hash: v1.k794b7964.a2d2c7e59c2ced8ef00221ae207eea27dabc4a81645cf14d1b6cb226baa24d2b.29rsM3fbhIfYk4EAExPUWRlURHw4YAiPt5U6WuoFh98+ hash: v1.k794b7964.788651fbd8e66035ce9c7e416df23cb38e30a90cebc04233fe5b6a669200e735.cM83-F6eR_mRfLYaYzgq66iCukTdjXNxi7bSBudnuVYscenes-app-marketing-analytics-source-setup-panel--detected-platforms--dark:hash: v1.k794b7964.ae7dcb8da63611c945d804117b2ea522790cb21b19fc2302422e67decfad7885.2DfPP_Ou6x7e7cgYgTVY63SN-DSltVT-edZMhPPAGHIscenes-app-marketing-analytics-source-setup-panel--detected-platforms--light:diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsx b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsxindex 67bc23e97bb4..408a406adc07 100644--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsx+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/components/settings/ExternalDataSourceConfiguration.tsx@@ -15,7 +15,7 @@ import {NativeMarketingSource,VALID_NATIVE_MARKETING_SOURCES,} from '~/queries/schema/schema-general'-import { ExternalDataSchemaStatus, ExternalDataSource, ManualLinkSourceType } from '~/types'+import { ExternalDataSchemaStatus, ExternalDataSource } from '~/types'import { SourceIcon } from 'products/data_warehouse/frontend/shared/components/SourceIcon'import { AddIntegrationButton } from 'products/marketing_analytics/frontend/components/AddIntegrationButton'@@ -29,12 +29,12 @@ import {} from '../../logic/marketingAnalyticsLogic'import { marketingAnalyticsSettingsLogic } from '../../logic/marketingAnalyticsSettingsLogic'import {+ ExternalTableTypeLabel,MAX_ITEMS_TO_SHOW,NEEDED_FIELDS_FOR_NATIVE_MARKETING_ANALYTICS,- NonNativeMarketingSource,- VALID_NON_NATIVE_MARKETING_SOURCES,- VALID_SELF_MANAGED_MARKETING_SOURCES,+ externalTableTypeLabel,findSchemaByFieldName,+ marketingSourceDisplayLabel,nativeSourceDisplayLabel,} from '../../logic/utils'import { ColumnMappingModal } from './ColumnMappingModal'@@ -46,7 +46,7 @@ type UnifiedSource = {id: stringname: stringsourceType: string- sourceTypeLabel: 'Native' | 'Data warehouse' | 'Self-managed'+ sourceTypeLabel: 'Native' | ExternalTableTypeLabelstatus: SourceStatusstatusMessage: stringsourceUrl?: string@@ -55,7 +55,7 @@ type UnifiedSource = {syncingTables?: string[]tablesToSync?: string[]nativeSource?: ExternalDataSource & { status?: string; statusMessage?: string }- // For non-native/self-managed+ // For non-native and self-managed tablesisTable?: booleantable?: ExternalTable}@@ -115,38 +115,21 @@ export function ExternalDataSourceConfiguration(): JSX.Element {}}- // Check if this is a BigQuery/warehouse source- if (VALID_NON_NATIVE_MARKETING_SOURCES.includes(item.source_type as NonNativeMarketingSource)) {- return {- id: `table-${item.id}`,- name: item.name,- sourceType: `${item.source_type} ${item.source_prefix}`,- sourceTypeLabel: 'Data warehouse' as const,- status: item.status || MarketingSourceStatus.Error,- statusMessage: item.statusMessage || 'Unknown status',- sourceUrl: item.sourceUrl,- isTable: true,- table: item,- }+ const tableTypeLabel = externalTableTypeLabel(item.source_type)+ if (!tableTypeLabel) {+ return null}-- // Check if this is a self-managed source- if (VALID_SELF_MANAGED_MARKETING_SOURCES.includes(item.source_type as ManualLinkSourceType)) {- return {- id: `table-${item.id}`,- name: item.name,- sourceType: `${item.source_type} ${item.source_prefix}`,- sourceTypeLabel: 'Self-managed' as const,- status: item.status || MarketingSourceStatus.Error,- statusMessage: item.statusMessage || 'Unknown status',- sourceUrl: item.sourceUrl,- isTable: true,- table: item,- }+ return {+ id: `table-${item.id}`,+ name: item.name,+ sourceType: `${marketingSourceDisplayLabel(item.source_type)} ${item.source_prefix}`,+ sourceTypeLabel: tableTypeLabel,+ status: item.status || MarketingSourceStatus.Error,+ statusMessage: item.statusMessage || 'Unknown status',+ sourceUrl: item.sourceUrl,+ isTable: true,+ table: item,}-- // Fallback for unknown types- return null}).filter(Boolean) as UnifiedSource[]@@ -184,7 +167,7 @@ export function ExternalDataSourceConfiguration(): JSX.Element {return (<SceneSectiontitle="Data source configuration"- description="Connect and configure data sources to enable marketing analytics. Native sources sync automatically, while warehouse and self-managed sources need column mapping."+ description="Connect and configure data sources to enable marketing analytics. Native sources sync automatically, while warehouse, file, and self-managed sources need column mapping.">{hasNoConfiguredSources &&(unconfiguredNativeSources.length > 0 ? (@@ -246,9 +229,9 @@ export function ExternalDataSourceConfiguration(): JSX.Element {type={item.sourceTypeLabel === 'Native'? 'success'- : item.sourceTypeLabel === 'Data warehouse'- ? 'default'- : 'primary'+ : item.sourceTypeLabel === 'Self-managed'+ ? 'primary'+ : 'default'}>{item.sourceTypeLabel}diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.tsindex ad4dc401ceee..865f3bf4a69b 100644--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.ts+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.test.ts@@ -57,6 +57,7 @@ import {marketingAnalyticsTilesLogic,} from './marketingAnalyticsTilesLogic'import { activeColumnView, columnsForView, marketingTableDimensionColumns } from './marketingTableColumnViews'+import { externalTableTypeLabel } from './utils'jest.mock('posthog-js')@@ -1133,6 +1134,84 @@ describe('marketingAnalyticsLogic', () => {}})+ describe.each([+ ['AmazonS3', 'aws', 'https://example-bucket.s3.amazonaws.com/spend/*.csv'],+ ['GoogleCloudStorage', 'google-cloud', 'https://storage.googleapis.com/example-bucket/spend/*.csv'],+ ['AzureBlob', 'azure', 'https://example.blob.core.windows.net/spend/*.csv'],+ ['CloudflareR2', 'cloudflare-r2', 'https://example.r2.cloudflarestorage.com/example-bucket/spend/*.csv'],+ ])('a %s spend table', (sourceType, provider, customerUrl) => {+ // A table that was never migrated and one that its source has not synced yet look the same here.+ it.each([+ ['still self-managed', 'example-table', provider, 'Self-managed', 'self-managed'],+ ['adopted by its source', 'example-table', sourceType, 'Files', 'managed'],+ ['created by its source', 'example-schema', sourceType, 'Files', 'managed'],+ ])('keeps its mapping when %s', async (state, mappingKey, expectedSourceType, typeLabel, dwSourceType) => {+ logic = marketingAnalyticsLogic()+ logic.mount()+ await expectLogic(logic).toFinishAllListeners()+ const managed = state !== 'still self-managed'+ const table: DatabaseSchemaDataWarehouseTable = {+ id: 'example-table',+ name: managed ? `${sourceType.toLowerCase()}.example_spend` : 'example_spend',+ type: 'data_warehouse',+ url_pattern: managed+ ? 'https://example-warehouse.s3.amazonaws.com/team_1/example_spend/*.parquet'+ : customerUrl,+ ...(managed+ ? {+ schema: {+ id: 'example-schema',+ name: 'example_spend',+ should_sync: false,+ incremental: false,+ status: ExternalDataSchemaStatus.Completed,+ },+ source: { id: 'example-source', source_type: sourceType, status: 'Completed', prefix: '' },+ }+ : {}),+ fields: Object.fromEntries(+ ['id', 'campaign', 'source', 'spend', 'day'].map((name) => [+ name,+ { name, hogql_value: name, type: 'string', schema_valid: true },+ ])+ ),+ }+ databaseTableListLogic.actions.loadDatabaseSuccess({ tables: { [table.name]: table }, joins: [] })+ await expectLogic(logic, () =>+ teamLogic.actions.loadCurrentTeamSuccess({+ ...teamLogic.values.currentTeam!,+ marketing_analytics_config: {+ sources_map: {+ [mappingKey]: {+ id: 'id',+ campaign: 'campaign',+ source: 'source',+ cost: 'spend',+ date: 'day',+ } as SourceMap,+ },+ },+ })+ ).toFinishAllListeners()++ expect(logic.values.validExternalTables).toEqual([+ expect.objectContaining({+ source_map_id: mappingKey,+ source_type: expectedSourceType,+ source_map: expect.objectContaining({ cost: 'spend' }),+ }),+ ])+ expect(logic.values.allExternalTablesWithStatus.filter((row) => row.id === table.id)).toEqual([+ expect.objectContaining({ source_map_id: mappingKey, source_type: expectedSourceType }),+ ])+ expect(externalTableTypeLabel(expectedSourceType)).toBe(typeLabel)+ expect(logic.values.marketingChartSourceIds).toEqual([mappingKey])+ expect(logic.values.createMarketingDataWarehouseNodes).toEqual([+ expect.objectContaining({ id: 'example-table', dw_source_type: dwSourceType }),+ ])+ })+ })+it('keeps the selection and drops an unknown key from a filter saved by an older build', async () => {localStorage.setItem(STORAGE_KEY, JSON.stringify({ integrationSourceIds: ['source-1'], removedOption: true }))diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.tsindex a9f50973bc75..3293c11740fc 100644--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.ts+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic.ts@@ -64,6 +64,7 @@ import {findSchemaByFieldName,generateUniqueName,getEnabledNativeMarketingSources,+ getSourceMapId,sanitizeIntegrationFilter,validColumnsForTiles,} from './utils'@@ -1124,14 +1125,15 @@ export const marketingAnalyticsLogic = kea<marketingAnalyticsLogicType>([const tableType = isDataWarehouse? DataWarehouseSettingsTab.Managed: DataWarehouseSettingsTab.SelfManaged- const sourceMap = sources_map?.[table.schema?.id || ''] ?? sources_map?.[table.id] ?? null+ const sourceMapId = getSourceMapId(table, sources_map)externalTables.push({...table,name: table.name,url_pattern: table.url_pattern || '',+ // An adopted table reads PostHog's storage, so only its source names the provider.source_type: table.source?.source_type || mapUrlToProvider(table.url_pattern),- source_map_id: table.schema?.id || table.source?.id || table.id,+ source_map_id: sourceMapId,source_prefix: table.source?.prefix || '',columns: Object.keys(table.fields).map((field) => ({name: table.fields[field].hogql_value,@@ -1141,7 +1143,7 @@ export const marketingAnalyticsLogic = kea<marketingAnalyticsLogicType>([`${tableType}-${dataWarehouseSource?.id || table.source?.id || table.id}`),external_type: tableType,- source_map: sourceMap,+ source_map: sources_map?.[sourceMapId] ?? null,schema_name: table.schema?.name || table.name,dw_source_type: tableType,schema_status: table.schema?.status,diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.tsindex 8faa1e5c7c3c..b29e161e44c4 100644--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.ts+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.test.ts@@ -24,6 +24,7 @@ import { NativeSource } from './marketingAnalyticsLogic'import {conversionGoalToSeries,createMarketingTile,+ externalTableTypeLabel,findSchemaByFieldName,getEnabledNativeMarketingSources,getOrderBy,@@ -757,6 +758,15 @@ describe('marketing analytics utils', () => {})+ it.each([+ ['BigQuery', 'Data warehouse'],+ ['FileUpload', 'Files'],+ ['google-cloud', 'Self-managed'],+ ['Postgres', null],+ ])('labels a %s table as %s in the source settings', (sourceType, label) => {+ expect(externalTableTypeLabel(sourceType)).toBe(label)+ })+describe('sanitizeIntegrationFilter', () => {it('drops a key the query schema no longer accepts', () => {const stored = { integrationSourceIds: ['abc'], removedOption: true }diff --git a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.ts b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.tsindex 48f37f1f4923..182a7b9c22d9 100644--- a/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.ts+++ b/frontend/src/scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils.ts@@ -28,15 +28,49 @@ import { ExternalDataSource, HogQLMathType, ManualLinkSourceType, PropertyMathTyimport { NativeSource } from './marketingAnalyticsLogic'-export type NonNativeMarketingSource = 'BigQuery'--export const VALID_NON_NATIVE_MARKETING_SOURCES: NonNativeMarketingSource[] = ['BigQuery']-export const VALID_SELF_MANAGED_MARKETING_SOURCES: ManualLinkSourceType[] = [- 'aws',- 'google-cloud',- 'cloudflare-r2',- 'azure',+type WarehouseMarketingSource = 'BigQuery'+// Managed sources that import spend data from files: a storage bucket or an uploaded file.+type FileMarketingSource = 'AmazonS3' | 'GoogleCloudStorage' | 'AzureBlob' | 'CloudflareR2' | 'FileUpload'++// Together these mirror the backend VALID_NON_NATIVE_MARKETING_SOURCES (constants.py).+export const VALID_WAREHOUSE_MARKETING_SOURCES: WarehouseMarketingSource[] = ['BigQuery']+export const VALID_FILE_MARKETING_SOURCES: FileMarketingSource[] = [+ 'AmazonS3',+ 'GoogleCloudStorage',+ 'AzureBlob',+ 'CloudflareR2',+ 'FileUpload',]+const VALID_SELF_MANAGED_MARKETING_SOURCES: ManualLinkSourceType[] = ['aws', 'google-cloud', 'cloudflare-r2', 'azure']++export type ExternalTableTypeLabel = 'Data warehouse' | 'Files' | 'Self-managed'++/** The type a mapped table shows in the settings list, or null when marketing analytics can't use the table. */+export function externalTableTypeLabel(sourceType: string): ExternalTableTypeLabel | null {+ if (VALID_WAREHOUSE_MARKETING_SOURCES.includes(sourceType as WarehouseMarketingSource)) {+ return 'Data warehouse'+ }+ if (VALID_FILE_MARKETING_SOURCES.includes(sourceType as FileMarketingSource)) {+ return 'Files'+ }+ if (VALID_SELF_MANAGED_MARKETING_SOURCES.includes(sourceType as ManualLinkSourceType)) {+ return 'Self-managed'+ }+ return null+}++/**+ * The sources_map key of a warehouse table. A self-managed table keeps its id when a managed source+ * adopts it, so its mapping stays under the table id. Use the key that holds the mapping, in the order+ * the backend reads them. With no mapping, use the key for a new mapping.+ */+export function getSourceMapId(+ table: { id: string; schema?: { id: string } | null; source?: { id: string } | null },+ sourcesMap: Record<string, unknown> | null | undefined+): string {+ const savedKey = [table.schema?.id, table.id].find((key) => !!key && sourcesMap?.[key] != null)+ return savedKey || table.schema?.id || table.source?.id || table.id+}export const NATIVE_SOURCE_FEATURE_FLAGS: Partial<Record<NativeMarketingSource, FeatureFlagKey>> = {AmazonAds: FEATURE_FLAGS.MARKETING_ANALYTICS_AMAZON_ADS,@@ -117,6 +151,18 @@ export function nativeSourceDisplayLabel(sourceType: string): string {return NATIVE_SOURCE_DISPLAY_LABELS[sourceType as NativeMarketingSource] ?? sourceType}+const FILE_SOURCE_DISPLAY_LABELS: Record<FileMarketingSource, string> = {+ AmazonS3: 'Amazon S3',+ GoogleCloudStorage: 'Google Cloud Storage',+ AzureBlob: 'Azure Blob Storage',+ CloudflareR2: 'Cloudflare R2',+ FileUpload: 'File upload',+}++export function marketingSourceDisplayLabel(sourceType: string): string {+ return FILE_SOURCE_DISPLAY_LABELS[sourceType as FileMarketingSource] ?? nativeSourceDisplayLabel(sourceType)+}+// The id suffix tells apart two accounts of one platform that have no prefix.export function marketingSourceName(source: Pick<ExternalDataSource, 'id' | 'prefix' | 'source_type'>): string {return source.prefix || `${nativeSourceDisplayLabel(source.source_type)} (${source.id.slice(0, 8)})`diff --git a/products/marketing_analytics/backend/hogql_queries/adapters/factory.py b/products/marketing_analytics/backend/hogql_queries/adapters/factory.pyindex 53b0b016b37b..1b66d28d5ed0 100644--- a/products/marketing_analytics/backend/hogql_queries/adapters/factory.py+++ b/products/marketing_analytics/backend/hogql_queries/adapters/factory.py@@ -48,6 +48,7 @@ from .base import (TikTokAdsConfig,)from .bigquery import BigQueryAdapter+from .file_source import FileSourceAdapterfrom .google_ads import GoogleAdsAdapterfrom .openai_ads import OpenAIAdsAdapterfrom .rokt_ads import RoktAdsAdapter@@ -97,6 +98,11 @@ class MarketingSourceFactory:"PinterestAds": PinterestAdsAdapter,# Non-native adapters"BigQuery": BigQueryAdapter,+ "AmazonS3": FileSourceAdapter,+ "GoogleCloudStorage": FileSourceAdapter,+ "AzureBlob": FileSourceAdapter,+ "CloudflareR2": FileSourceAdapter,+ "FileUpload": FileSourceAdapter,# Self-managed adapters"aws": AWSAdapter,"google-cloud": GoogleCloudAdapter,@@ -339,14 +345,17 @@ class MarketingSourceFactory:continuefor table in tables:- source_map = self._get_source_map_for_table(table, str(source.id))- if not source_map:+ source_map_key = self._get_source_map_key_for_table(table, str(source.id))+ if source_map_key is None:continue+ source_map = self._sources_map[source_map_key]# For non-native: use schema ID to match frontend (table.schema?.id || table.source?.id || table.id).# Iterate the prefetched set — `.first()` issues a fresh query and bypasses the prefetch cache.schema = next(iter(table.externaldataschema_set.all()), None)- source_id = str(schema.id) if schema else str(table.id)+ # A self-managed table that a source adopts keeps its id, so its mapping stays under the+ # table id. The frontend then uses the table id too, so the ids must agree.+ source_id = str(schema.id) if schema and source_map_key != str(table.id) else str(table.id)config = ExternalConfig(table=table,@@ -400,23 +409,19 @@ class MarketingSourceFactory:def _get_source_map_for_table(self, table: DataWarehouseTable, source_id: str | None = None) -> Optional[SourceMap]:"""Get source map for a table"""- if source_id and table.external_data_source and table.external_data_source.source_type:- # Managed table- table_id = str(table.id)- schema_id = self._get_schema_id_for_table(table)+ key = self._get_source_map_key_for_table(table, source_id)+ return self._sources_map[key] if key is not None else None- # Try schema_id, table_id, then source_id- for key in [schema_id, table_id, source_id]:- if key and key in self._sources_map:- return self._sources_map[key]+ def _get_source_map_key_for_table(self, table: DataWarehouseTable, source_id: str | None = None) -> str | None:+ """Get the sources_map key that holds the mapping of a table"""+ if source_id and table.external_data_source and table.external_data_source.source_type:+ # Managed table: try schema_id, table_id, then source_id+ keys = [self._get_schema_id_for_table(table), str(table.id), source_id]else:- if str(table.id) in self._sources_map:- # Self-managed table- return self._sources_map[str(table.id)]- else:- return None+ # Self-managed table+ keys = [str(table.id)]- return None+ return next((key for key in keys if key and key in self._sources_map), None)def get_valid_adapters(self, adapters: list[MarketingSourceAdapter]) -> list[MarketingSourceAdapter]:"""Filter adapters to only return valid ones"""diff --git a/products/marketing_analytics/backend/hogql_queries/adapters/file_source.py b/products/marketing_analytics/backend/hogql_queries/adapters/file_source.pynew file mode 100644index 000000000000..46385332508d--- /dev/null+++ b/products/marketing_analytics/backend/hogql_queries/adapters/file_source.py@@ -0,0 +1,16 @@+# File Marketing Source Adapter++from .self_managed import SelfManagedAdapter+++class FileSourceAdapter(SelfManagedAdapter):+ """+ Adapter for managed sources that import spend data from files: Amazon S3, Google Cloud Storage,+ Azure Blob Storage, Cloudflare R2 and file uploads.++ These sources replace self-managed tables. They need the same column mapping, so the adapter+ shares all field resolution and query-building logic with SelfManagedAdapter.+ """++ def get_source_type(self) -> str:+ return self.config.source_typediff --git a/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.py b/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.pyindex 8c6067b788b9..d92b9140784e 100644--- a/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.py+++ b/products/marketing_analytics/backend/hogql_queries/adapters/test_factory.py@@ -27,9 +27,22 @@ from products.marketing_analytics.backend.hogql_queries.adapters.base import (TikTokAdsConfig,)from products.marketing_analytics.backend.hogql_queries.adapters.factory import MarketingSourceFactory+from products.marketing_analytics.backend.hogql_queries.adapters.file_source import FileSourceAdapterfrom products.marketing_analytics.backend.hogql_queries.adapters.meta_ads import MetaAdsAdapter+from products.marketing_analytics.backend.hogql_queries.adapters.self_managed import (+ AWSAdapter,+ AzureAdapter,+ CloudflareR2Adapter,+ GoogleCloudAdapter,+ SelfManagedAdapter,+)from products.marketing_analytics.backend.hogql_queries.constants import TABLE_PATTERNS-from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSource+from products.warehouse_sources.backend.facade.models import (+ DataWarehouseCredential,+ DataWarehouseTable,+ ExternalDataSchema,+ ExternalDataSource,+)class FactoryTestMixin:@@ -598,3 +611,81 @@ class TestNativeSourceKillSwitch(SimpleTestCase):assert set(errors) == ({adapter.config.source_id for adapter in adapters[1:]} if enabled else set())assert all(messages for messages in errors.values())factory.logger.exception.assert_not_called()+++_BUCKET_PROVIDERS: list[tuple[str, str, type[SelfManagedAdapter]]] = [+ ("AmazonS3", "https://example-bucket.s3.amazonaws.com/spend/*.csv", AWSAdapter),+ ("GoogleCloudStorage", "https://storage.googleapis.com/example-bucket/spend/*.csv", GoogleCloudAdapter),+ ("AzureBlob", "https://example.blob.core.windows.net/spend/*.csv", AzureAdapter),+ ("CloudflareR2", "https://example.r2.cloudflarestorage.com/example-bucket/spend/*.csv", CloudflareR2Adapter),+]+_POSTHOG_STORAGE_URL = "https://example-warehouse.s3.amazonaws.com/team_1/example_spend/*.parquet"+_TABLE_STATES = ["never_migrated", "migrated_not_adopted", "adopted", "new_managed_table"]+++class TestFileSourceTableDiscovery(BaseTest):+ def _create_table_state(self, state: str, source_type: str, customer_url: str) -> tuple[DataWarehouseTable, str]:+ managed = state in ("adopted", "new_managed_table")+ source = None+ if state != "never_migrated":+ source = ExternalDataSource.objects.create(+ team=self.team, source_id="example", connection_id="example", source_type=source_type+ )+ table = DataWarehouseTable.raw_objects.create(+ team=self.team,+ name="example_spend",+ format="CSVWithNames",+ url_pattern=_POSTHOG_STORAGE_URL if managed else customer_url,+ credential=None+ if managed+ else DataWarehouseCredential.objects.create(team=self.team, access_key="key", access_secret="secret"),+ external_data_source=source if managed else None,+ )+ schema = None+ if source is not None:+ schema = ExternalDataSchema.objects.create(+ team=self.team, source=source, name="example_spend", table=table if managed else None+ )+ mapping_key = str(schema.id) if state == "new_managed_table" and schema else str(table.id)+ self.team.marketing_analytics_config.sources_map = {+ mapping_key: {"campaign": "campaign", "source": "source", "cost": "spend", "date": "day"}+ }+ self.team.marketing_analytics_config.save()+ return table, mapping_key++ def _make_factory(self, table: DataWarehouseTable) -> MarketingSourceFactory:+ database = Mock(get_warehouse_table_names=Mock(return_value=[table.name]))+ context = QueryContext(date_range=None, team=self.team, base_currency=DEFAULT_CURRENCY, database=database)+ return MarketingSourceFactory(context=context)++ @parameterized.expand(+ [+ (f"{state}_{source_type}", state, source_type, customer_url, self_managed_adapter)+ for source_type, customer_url, self_managed_adapter in _BUCKET_PROVIDERS+ for state in _TABLE_STATES+ ]+ + [(f"{state}_FileUpload", state, "FileUpload", "", SelfManagedAdapter) for state in _TABLE_STATES[2:]]+ )+ def test_table_gets_one_adapter_with_its_mapping(+ self,+ _name: str,+ state: str,+ source_type: str,+ customer_url: str,+ self_managed_adapter: type[SelfManagedAdapter],+ ) -> None:+ table, mapping_key = self._create_table_state(state, source_type, customer_url)+ factory = self._make_factory(table)++ adapters = factory.create_adapters(raise_on_error=True)++ assert len(adapters) == 1+ adapter = adapters[0]+ if state in ("adopted", "new_managed_table"):+ assert type(adapter) is FileSourceAdapter+ assert adapter.get_source_type() == source_type+ else:+ assert type(adapter) is self_managed_adapter+ assert adapter.config.source_id == mapping_key+ assert adapter.config.source_map.cost == "spend"+ assert factory.get_valid_adapters(adapters) == adaptersdiff --git a/products/marketing_analytics/backend/hogql_queries/constants.py b/products/marketing_analytics/backend/hogql_queries/constants.pyindex b6c0d95c81b5..1f447796e598 100644--- a/products/marketing_analytics/backend/hogql_queries/constants.py+++ b/products/marketing_analytics/backend/hogql_queries/constants.py@@ -502,8 +502,11 @@ MARKETING_ANALYTICS_SCHEMA = {# Valid native marketing sources - derived from generated enumVALID_NATIVE_MARKETING_SOURCES = [source.value for source in NativeMarketingSource]-# Valid non-native marketing sources (managed external sources like BigQuery)-VALID_NON_NATIVE_MARKETING_SOURCES = ["BigQuery"]+# Managed sources that import spend data from files: a storage bucket or an uploaded file.+VALID_FILE_MARKETING_SOURCES = ["AmazonS3", "GoogleCloudStorage", "AzureBlob", "CloudflareR2", "FileUpload"]++# Valid non-native marketing sources (managed external sources that need a column mapping)+VALID_NON_NATIVE_MARKETING_SOURCES = ["BigQuery", *VALID_FILE_MARKETING_SOURCES]# Valid self-managed marketing sources (mirrors frontend types)VALID_SELF_MANAGED_MARKETING_SOURCES = ["aws", "google-cloud", "cloudflare-r2", "azure"]diff --git a/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsx b/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsxindex 99af0f0f76a1..956b30b06182 100644--- a/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsx+++ b/products/marketing_analytics/frontend/components/AddIntegrationButton.stories.tsx@@ -61,6 +61,10 @@ const meta: Meta<typeof AddIntegrationButton> = {AmazonAds: 'amazon_ads.png',TwitterAds: 'twitter_ads.png',BigQuery: 'bigquery.png',+ AmazonS3: 'aws-s3.png',+ GoogleCloudStorage: 'google-cloud-storage.png',+ AzureBlob: 'azure_blob.png',+ CloudflareR2: 'cloudflare.png',}).map(([name, icon]) => [name,{ name, caption: name, iconPath: `/static/services/${icon}`, fields: [] },diff --git a/products/marketing_analytics/frontend/components/AddIntegrationButton.tsx b/products/marketing_analytics/frontend/components/AddIntegrationButton.tsxindex fc0f2f5bd261..050702b1c210 100644--- a/products/marketing_analytics/frontend/components/AddIntegrationButton.tsx+++ b/products/marketing_analytics/frontend/components/AddIntegrationButton.tsx@@ -9,8 +9,10 @@ import { FEATURE_FLAGS, TeamMembershipLevel } from 'lib/constants'import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { urls } from 'scenes/urls'import {- VALID_NON_NATIVE_MARKETING_SOURCES,+ VALID_FILE_MARKETING_SOURCES,+ VALID_WAREHOUSE_MARKETING_SOURCES,getEnabledNativeMarketingSources,+ marketingSourceDisplayLabel,nativeSourceDisplayLabel,} from 'scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils'@@ -53,7 +55,8 @@ export function AddIntegrationButton({ onIntegrationSelect }: AddIntegrationButt...(featureFlags[FEATURE_FLAGS.MARKETING_ANALYTICS_ORGANIC_KEYWORDS]? [{ title: 'Organic search', sources: ['GoogleSearchConsole'] }]: []),- { title: 'External sources', sources: VALID_NON_NATIVE_MARKETING_SOURCES },+ { title: 'External sources', sources: VALID_WAREHOUSE_MARKETING_SOURCES },+ { title: 'Files', sources: VALID_FILE_MARKETING_SOURCES },]const items: LemonMenuItems = [...(showNewSourcesInMenu@@ -81,7 +84,7 @@ export function AddIntegrationButton({ onIntegrationSelect }: AddIntegrationButtlabel:integrationId === 'GoogleSearchConsole'? 'Google Search Console'- : nativeSourceDisplayLabel(integrationId),+ : marketingSourceDisplayLabel(integrationId),icon: <SourceIcon type={integrationId} size="xsmall" disableTooltip />,tag:showNewSourcesInMenu && newSources.some((source) => source === integrationId)diff --git a/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsx b/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsxindex 9a1063e2cc72..47dfa6f70b1d 100644--- a/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsx+++ b/products/marketing_analytics/frontend/dashboard/SourceCatalog.tsx@@ -13,9 +13,10 @@ import { teamLogic } from 'scenes/teamLogic'import { urls } from 'scenes/urls'import { marketingAnalyticsLogic } from 'scenes/web-analytics/tabs/marketing-analytics/frontend/logic/marketingAnalyticsLogic'import {- VALID_NON_NATIVE_MARKETING_SOURCES,+ VALID_FILE_MARKETING_SOURCES,+ VALID_WAREHOUSE_MARKETING_SOURCES,getEnabledNativeMarketingSourcesInDisplayOrder,- nativeSourceDisplayLabel,+ marketingSourceDisplayLabel,} from 'scenes/web-analytics/tabs/marketing-analytics/frontend/logic/utils'import { ProductIntentContext, ProductKey } from '~/queries/schema/schema-general'@@ -27,7 +28,7 @@ const HedgehogMoney = pngHoggie(moneyPng)interface MarketingSource {id: stringisConnected: boolean- category: 'native' | 'external'+ category: 'native' | 'external' | 'file'}interface SourceCatalogProps {@@ -52,11 +53,16 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogPisConnected: validNativeSources.some((source) => source.source.source_type === sourceType),category: 'native' as const,})),- ...VALID_NON_NATIVE_MARKETING_SOURCES.map((sourceType) => ({+ ...VALID_WAREHOUSE_MARKETING_SOURCES.map((sourceType) => ({id: sourceType,isConnected: validExternalTables.some((table) => table.source_type === sourceType),category: 'external' as const,})),+ ...VALID_FILE_MARKETING_SOURCES.map((sourceType) => ({+ id: sourceType,+ isConnected: validExternalTables.some((table) => table.source_type === sourceType),+ category: 'file' as const,+ })),]const handleSourceSelect = (sourceId: string): void => {@@ -75,16 +81,12 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogP)}- const nativeSources = allSources.filter(- (s) =>- nativeSourceDisplayLabel(s.id).toLowerCase().includes(manualSourceSearch.toLowerCase()) &&- s.category === 'native'- )- const externalSources = allSources.filter(- (s) =>- nativeSourceDisplayLabel(s.id).toLowerCase().includes(manualSourceSearch.toLowerCase()) &&- s.category === 'external'+ const matchingSources = allSources.filter((s) =>+ marketingSourceDisplayLabel(s.id).toLowerCase().includes(manualSourceSearch.toLowerCase()))+ const nativeSources = matchingSources.filter((s) => s.category === 'native')+ const externalSources = matchingSources.filter((s) => s.category === 'external')+ const fileSources = matchingSources.filter((s) => s.category === 'file')const totalConnected = validNativeSources.length + validExternalTables.length@@ -111,7 +113,7 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogPvalue={manualSourceSearch}onChange={setManualSourceSearch}/>- {!nativeSources.length && !externalSources.length && (+ {!nativeSources.length && !externalSources.length && !fileSources.length && (<p className="text-secondary">No integrations match your search.</p>)}{/* Native Sources */}@@ -145,6 +147,26 @@ export function SourceCatalog({ onContinue, hasSources, onBack }: SourceCatalogP</div></div>)}++ {/* File Sources */}+ {fileSources.length > 0 && (+ <div>+ <div className="text-xs font-medium text-muted mb-1.5 flex items-center gap-1">+ Files+ <Tooltip+ title="Import spend data from CSV, JSON, or Parquet files in a storage bucket, or upload a file."+ delayMs={0}+ >+ <IconInfo className="w-3 h-3 cursor-help" />+ </Tooltip>+ </div>+ <div className="flex flex-wrap gap-2">+ {fileSources.map((source) => (+ <SourceChip key={source.id} source={source} onSelect={handleSourceSelect} />+ ))}+ </div>+ </div>+ )}</div><div className="border-t p-4 flex flex-wrap items-center justify-between gap-3 bg-bg-light">{onBack && (@@ -184,7 +206,7 @@ function SourceChip({ source, onSelect }: { source: MarketingSource; onSelect: (onClick={() => onSelect(source.id)}data-attr="marketing-manual-connect-source">- {nativeSourceDisplayLabel(source.id)}+ {marketingSourceDisplayLabel(source.id)}{source.isConnected && <IconCheckCircle className="ml-2 text-success" />}</LemonButton>)diff --git a/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsx b/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsxindex f36fc32c2f44..fd85257a708e 100644--- a/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsx+++ b/products/marketing_analytics/frontend/dashboard/SourceOnboarding.stories.tsx@@ -71,6 +71,10 @@ const meta: Meta = {SnapchatAds: { iconPath: '/static/services/snapchat.png' },PinterestAds: { iconPath: '/static/services/pinterest_ads.png' },BigQuery: { iconPath: '/static/services/bigquery.png' },+ AmazonS3: { iconPath: '/static/services/aws-s3.png' },+ GoogleCloudStorage: { iconPath: '/static/services/google-cloud-storage.png' },+ AzureBlob: { iconPath: '/static/services/azure_blob.png' },+ CloudflareR2: { iconPath: '/static/services/cloudflare.png' },},],},