Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): track history enabled, disabled and queried (#116588)

Daniel RCe48e17a97b03
diff --git a/frontend/src/queries/query.test.ts b/frontend/src/queries/query.test.ts
index e742e96ea427..15e9e3ccaeaf 100644
--- a/frontend/src/queries/query.test.ts
+++ b/frontend/src/queries/query.test.ts
@@ -131,6 +131,44 @@ describe('query', () => {
})
})
+ it.each([
+ [true, 1],
+ [undefined, 0],
+ ])('when uses_history_table is %s, sends %i history queried events', async (usesHistoryTable, expectedEvents) => {
+ useMocks({
+ post: {
+ '/api/environments/:team_id/query/:kind': () => [
+ 200,
+ {
+ results: [],
+ is_cached: false,
+ used_data_warehouse_sources: [
+ {
+ id: 'src-1',
+ source_type: 'Postgres',
+ table_name: 'orders_cdc',
+ uses_history_table: usesHistoryTable,
+ },
+ ],
+ },
+ ],
+ },
+ })
+ const captureSpy = jest.spyOn(posthog, 'capture')
+ const q: HogQLQuery = setLatestVersionsOnQuery({ kind: NodeKind.HogQLQuery, query: 'select * from orders_cdc' })
+ captureSpy.mockClear()
+ await performQuery(q)
+ const historyCalls = captureSpy.mock.calls.filter((call) => call[0] === 'data warehouse history table queried')
+ expect(historyCalls).toHaveLength(expectedEvents)
+ if (expectedEvents) {
+ expect(historyCalls[0][1]).toMatchObject({
+ query_kind: NodeKind.HogQLQuery,
+ data_warehouse_source_ids: ['src-1'],
+ data_warehouse_source_types: ['Postgres'],
+ })
+ }
+ })
+
it('captures precompute strategy and staleness from web analytics responses', async () => {
const captureSpy = jest.spyOn(posthog, 'capture')
const q = setLatestVersionsOnQuery({
@@ -379,6 +417,7 @@ describe('query', () => {
controller.abort()
}
const promise = pollForResults('test-query-id', { signal: controller.signal })
+ // oxlint-disable-next-line jest/valid-expect -- awaited after the abort below; awaiting here hangs the test
const rejected = expect(promise).rejects.toMatchObject({ name: 'AbortError' })
await jest.advanceTimersByTimeAsync(0)
controller.abort()
diff --git a/frontend/src/queries/query.ts b/frontend/src/queries/query.ts
index fe4f70e6300c..abea2c35fc29 100644
--- a/frontend/src/queries/query.ts
+++ b/frontend/src/queries/query.ts
@@ -472,6 +472,16 @@ export async function performQuery<N extends DataNode>(
...logParams,
...submitTelemetry,
})
+ const historySources = warehouseSources.filter((s) => s.uses_history_table)
+ if (historySources.length > 0) {
+ posthog.capture('data warehouse history table queried', {
+ query_kind: queryNode.kind,
+ queryId,
+ is_cached: response?.is_cached,
+ data_warehouse_source_ids: historySources.map((s) => s.id),
+ data_warehouse_source_types: historySources.map((s) => s.source_type).filter(Boolean),
+ })
+ }
return response
} catch (e) {
// A superseded query or navigating away mid-request aborts, not fails — skip so the
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 30141a61c8ce..044649ebebb6 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -21310,6 +21310,10 @@
"table_name": {
"description": "Warehouse table name that was referenced",
"type": "string"
+ },
+ "uses_history_table": {
+ "description": "Whether the query read a history table of this source",
+ "type": "boolean"
}
},
"required": ["id", "table_name"],
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index 19b4f87fa498..997abcaaa7b0 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -2854,6 +2854,8 @@ export interface DataWarehouseSourceUsage {
source_type?: string
/** Warehouse table name that was referenced */
table_name: string
+ /** Whether the query read a history table of this source */
+ uses_history_table?: boolean
}
export interface AnalyticsQueryResponseBase {
diff --git a/posthog/hogql/database/s3_table.py b/posthog/hogql/database/s3_table.py
index 68b4aef2e0a9..d6dfa87f5153 100644
--- a/posthog/hogql/database/s3_table.py
+++ b/posthog/hogql/database/s3_table.py
@@ -334,6 +334,8 @@ class S3Table(FunctionCallTable):
source_type: Optional[str] = None
# Set when this table backs a materialized saved query, so a query that reads the table is attributed to the view.
saved_query_id: Optional[str] = None
+ # Set when the table keeps every version of a row, so a query that reads it counts as history usage.
+ is_history_table: bool = False
def to_printed_hogql(self):
return escape_hogql_identifier(self.name)
diff --git a/posthog/hogql/database/test/test_warehouse_usage.py b/posthog/hogql/database/test/test_warehouse_usage.py
index 51e27df73a47..7295628dd569 100644
--- a/posthog/hogql/database/test/test_warehouse_usage.py
+++ b/posthog/hogql/database/test/test_warehouse_usage.py
@@ -52,6 +52,20 @@ class TestWarehouseUsage(BaseTest):
)
return source
+ def _create_history_table(self, source: ExternalDataSource, name: str) -> None:
+ string_column = {"hogql": "StringDatabaseField", "clickhouse": "Nullable(String)", "schema_valid": True}
+ table = DataWarehouseTable(
+ name=name,
+ format="Parquet",
+ team=self.team,
+ external_data_source=source,
+ credential=DataWarehouseCredential.objects.create(access_key="blah", access_secret="blah", team=self.team),
+ url_pattern="https://bucket.s3/history/*",
+ columns={"id": string_column, "valid_from": string_column, "valid_to": string_column},
+ )
+ table.mark_as_history_table()
+ table.save()
+
def _create_self_managed_table(self, name: str = "self_managed_1") -> None:
credentials = DataWarehouseCredential.objects.create(access_key="blah", access_secret="blah", team=self.team)
DataWarehouseTable.objects.create(
@@ -118,6 +132,41 @@ class TestWarehouseUsage(BaseTest):
assert [s.id for s in response.used_data_warehouse_sources or []] == [str(source.id)]
+ @parameterized.expand(
+ [
+ ("current_table_only", "SELECT id FROM stripe_table_1", False),
+ ("history_table_only", "SELECT id FROM stripe_table_1_cdc", True),
+ (
+ "history_table_joined_after_current_table",
+ "SELECT a.id FROM stripe_table_1 a JOIN stripe_table_1_cdc b ON a.id = b.id",
+ True,
+ ),
+ ]
+ )
+ def test_flags_a_source_whose_history_table_is_read(self, _name, query, expected):
+ source = self._create_stripe_table()
+ self._create_history_table(source, "stripe_table_1_cdc")
+
+ sources = self._sources_for(query)
+
+ assert [(s.id, s.uses_history_table) for s in sources] == [(str(source.id), expected)]
+
+ def test_history_read_survives_a_jev_read_of_the_same_source(self):
+ source = self._create_stripe_table()
+ self._create_history_table(source, "stripe_table_1_cdc")
+ executor = HogQLQueryExecutor(query="SELECT id FROM stripe_table_1_cdc", team=self.team, user=self.user)
+ # A jev source execution of the same connector read only the current table.
+ executor._prompt_jev_warehouse_sources = [
+ WarehouseSourceUsage(id=str(source.id), source_type="Stripe", table_name="stripe_table_1")
+ ]
+
+ with patch("posthog.hogql.query.sync_execute", return_value=([], [("id", "Nullable(String)")])):
+ response = executor.execute()
+
+ assert [(s.id, s.uses_history_table) for s in response.used_data_warehouse_sources or []] == [
+ (str(source.id), True)
+ ]
+
def test_deduplicates_by_source(self):
source = self._create_stripe_table()
diff --git a/posthog/hogql/database/warehouse_usage.py b/posthog/hogql/database/warehouse_usage.py
index b38a3f3d5346..5aef9758b43f 100644
--- a/posthog/hogql/database/warehouse_usage.py
+++ b/posthog/hogql/database/warehouse_usage.py
@@ -1,4 +1,5 @@
-from dataclasses import dataclass
+from collections.abc import Iterable
+from dataclasses import dataclass, replace
from typing import Optional
from posthog.hogql import ast
@@ -14,6 +15,7 @@ class WarehouseSourceUsage:
id: str
source_type: Optional[str]
table_name: str
+ uses_history_table: bool = False
def extract_warehouse_sources(
@@ -29,17 +31,32 @@ def extract_warehouse_sources(
if select_type is None:
return []
- usages: dict[str, WarehouseSourceUsage] = {}
+ usages: list[WarehouseSourceUsage] = []
for table_type in extract_base_table_types(select_type):
table = table_type.table
if not isinstance(table, (S3Table, DirectPostgresTable)):
continue
source_id = getattr(table, "external_data_source_id", None)
- if not source_id or source_id in usages:
+ if not source_id:
continue
- usages[source_id] = WarehouseSourceUsage(
- id=source_id,
- source_type=getattr(table, "source_type", None),
- table_name=getattr(table, "name", "") or "",
+ usages.append(
+ WarehouseSourceUsage(
+ id=source_id,
+ source_type=getattr(table, "source_type", None),
+ table_name=getattr(table, "name", "") or "",
+ uses_history_table=getattr(table, "is_history_table", False),
+ )
)
- return list(usages.values())
+ return merge_warehouse_sources(usages)
+
+
+def merge_warehouse_sources(usages: Iterable[WarehouseSourceUsage]) -> list[WarehouseSourceUsage]:
+ """One entry per source, first-seen order. The source counts as history usage if any read was."""
+ merged: dict[str, WarehouseSourceUsage] = {}
+ for usage in usages:
+ existing = merged.get(usage.id)
+ if existing is None:
+ merged[usage.id] = usage
+ elif usage.uses_history_table and not existing.uses_history_table:
+ merged[usage.id] = replace(existing, uses_history_table=True)
+ return list(merged.values())
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py
index 7f23aaf1fdd8..51d7abdea69d 100644
--- a/posthog/hogql/query.py
+++ b/posthog/hogql/query.py
@@ -33,7 +33,11 @@ from posthog.hogql.database.schema.duckdb_table_functions import (
)
from posthog.hogql.database.schema.information_schema import InformationSchemaTable
from posthog.hogql.database.schema.logs import get_hogql_max_bytes_to_read_for_logs_user_queries
-from posthog.hogql.database.warehouse_usage import WarehouseSourceUsage, extract_warehouse_sources
+from posthog.hogql.database.warehouse_usage import (
+ WarehouseSourceUsage,
+ extract_warehouse_sources,
+ merge_warehouse_sources,
+)
from posthog.hogql.direct_connection import (
INVALID_CONNECTION_ID_ERROR,
RAW_QUERY_TABLE_DENIED_ERROR,
@@ -636,7 +640,7 @@ class HogQLQueryExecutor:
except Exception:
sources = []
# jev source queries read their tables in separate executions.
- sources = list({source.id: source for source in [*sources, *self._prompt_jev_warehouse_sources]}.values())
+ sources = merge_warehouse_sources([*sources, *self._prompt_jev_warehouse_sources])
self.used_data_warehouse_sources = sources
return sources
@@ -1136,7 +1140,12 @@ class HogQLQueryExecutor:
if self.user is None or not self.user.is_authenticated:
return None
return [
- DataWarehouseSourceUsage(id=s.id, source_type=s.source_type, table_name=s.table_name)
+ DataWarehouseSourceUsage(
+ id=s.id,
+ source_type=s.source_type,
+ table_name=s.table_name,
+ uses_history_table=s.uses_history_table or None,
+ )
for s in self.used_data_warehouse_sources
]
diff --git a/posthog/schema.py b/posthog/schema.py
index 33e7f9de4986..e155fba4204a 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -1260,6 +1260,10 @@ class DataWarehouseSourceUsage(BaseModel):
description="Connector type of the source (e.g. Stripe, Postgres), if known",
)
table_name: str = Field(..., description="Warehouse table name that was referenced")
+ uses_history_table: bool | None = Field(
+ default=None,
+ description="Whether the query read a history table of this source",
+ )
class DataWarehouseSyncWarning(BaseModel):
diff --git a/products/customer_analytics/frontend/generated/api.schemas.ts b/products/customer_analytics/frontend/generated/api.schemas.ts
index 4692d40103f3..024624af5a9f 100644
--- a/products/customer_analytics/frontend/generated/api.schemas.ts
+++ b/products/customer_analytics/frontend/generated/api.schemas.ts
@@ -1967,6 +1967,8 @@ export interface DataWarehouseSourceUsageApi {
source_type?: string | null
/** Warehouse table name that was referenced */
table_name: string
+ /** Whether the query read a history table of this source */
+ uses_history_table?: boolean | null
}
export interface DataWarehouseSyncWarningApi {
diff --git a/products/dashboards/frontend/generated/api.schemas.ts b/products/dashboards/frontend/generated/api.schemas.ts
index 845bdf3a0681..318ed813cc73 100644
--- a/products/dashboards/frontend/generated/api.schemas.ts
+++ b/products/dashboards/frontend/generated/api.schemas.ts
@@ -2634,6 +2634,8 @@ export interface DataWarehouseSourceUsageApi {
source_type?: string | null
/** Warehouse table name that was referenced */
table_name: string
+ /** Whether the query read a history table of this source */
+ uses_history_table?: boolean | null
}
export interface DataWarehouseSyncWarningApi {
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 0cc898790fdd..bffda9d998ba 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -346,6 +346,7 @@ export namespace Schemas {
* Warehouse table name that was referenced
*/
table_name: string;
+ uses_history_table?: (boolean | null) | undefined;
};
export type DataWarehouseSyncWarning = {
data_is_current?: (boolean | null) | undefined;
diff --git a/products/marketing_analytics/frontend/generated/api.schemas.ts b/products/marketing_analytics/frontend/generated/api.schemas.ts
index 051e83f51d90..7f77fd8f6cf0 100644
--- a/products/marketing_analytics/frontend/generated/api.schemas.ts
+++ b/products/marketing_analytics/frontend/generated/api.schemas.ts
@@ -1911,6 +1911,8 @@ export interface DataWarehouseSourceUsageApi {
source_type?: string | null
/** Warehouse table name that was referenced */
table_name: string
+ /** Whether the query read a history table of this source */
+ uses_history_table?: boolean | null
}
export interface DataWarehouseSyncWarningApi {
diff --git a/products/persons/frontend/generated/api.schemas.ts b/products/persons/frontend/generated/api.schemas.ts
index acb75cca0bb6..aff144798b88 100644
--- a/products/persons/frontend/generated/api.schemas.ts
+++ b/products/persons/frontend/generated/api.schemas.ts
@@ -697,6 +697,8 @@ export interface DataWarehouseSourceUsageApi {
source_type?: string | null
/** Warehouse table name that was referenced */
table_name: string
+ /** Whether the query read a history table of this source */
+ uses_history_table?: boolean | null
}
export interface DataWarehouseSyncWarningApi {
diff --git a/products/product_analytics/frontend/generated/api.schemas.ts b/products/product_analytics/frontend/generated/api.schemas.ts
index 0c350297bb78..2495033fd9e5 100644
--- a/products/product_analytics/frontend/generated/api.schemas.ts
+++ b/products/product_analytics/frontend/generated/api.schemas.ts
@@ -1460,6 +1460,8 @@ export interface DataWarehouseSourceUsageApi {
source_type?: string | null
/** Warehouse table name that was referenced */
table_name: string
+ /** Whether the query read a history table of this source */
+ uses_history_table?: boolean | null
}
export interface DataWarehouseSyncWarningApi {
diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.py
index e8cbe8be3ca6..a41c1427fb22 100644
--- a/products/warehouse_sources/backend/models/external_data_schema.py
+++ b/products/warehouse_sources/backend/models/external_data_schema.py
@@ -587,6 +587,10 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djan
return self.sync_type_config.get("cdc_table_mode", "consolidated")
return "consolidated"
+ @property
+ def keeps_history(self) -> bool:
+ return self.is_cdc and self.cdc_table_mode in ("cdc_only", "both")
+
@property
def should_use_incremental_field(self):
return self.is_incremental or self.is_append or self.is_webhook
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index e065a60e013b..1551c1f7083f 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -1343,6 +1343,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
table_id=str(self.id),
external_data_source_id=str(self.external_data_source_id) if self.external_data_source_id else None,
source_type=self.external_data_source.source_type if self.external_data_source else None,
+ is_history_table=self.is_history_table,
)
# Must resolve an unset option the same way _describe_settings does, by sending no setting.
diff --git a/products/warehouse_sources/backend/presentation/history_tracking.py b/products/warehouse_sources/backend/presentation/history_tracking.py
new file mode 100644
index 000000000000..3cd2f8636d7f
--- /dev/null
+++ b/products/warehouse_sources/backend/presentation/history_tracking.py
@@ -0,0 +1,54 @@
+from typing import TYPE_CHECKING, Optional
+
+from posthog.event_usage import report_user_action
+
+from products.warehouse_sources.backend.facade.models import ExternalDataSchema
+
+if TYPE_CHECKING:
+ from django.contrib.auth.models import AnonymousUser
+
+ from rest_framework.request import Request
+
+ from posthog.models.user import User
+
+HISTORY_ENABLED_EVENT = "data warehouse history enabled"
+HISTORY_DISABLED_EVENT = "data warehouse history disabled"
+
+
+def report_history_toggled(
+ user: "User | AnonymousUser",
+ schema: ExternalDataSchema,
+ *,
+ enabled: bool,
+ table_mode: str,
+ request: Optional["Request"] = None,
+) -> None:
+ report_user_action(
+ user,
+ HISTORY_ENABLED_EVENT if enabled else HISTORY_DISABLED_EVENT,
+ {
+ "source_type": schema.source.source_type,
+ "source_id": str(schema.source_id),
+ "schema_id": str(schema.id),
+ "table_mode": table_mode,
+ },
+ team=schema.team,
+ request=request,
+ )
+
+
+def report_history_change(
+ user: "User | AnonymousUser",
+ schema: ExternalDataSchema,
+ *,
+ history_mode_before: str | None,
+ request: Optional["Request"] = None,
+) -> None:
+ """`history_mode_before` is the table mode the schema kept history in, or None if it kept none."""
+ if schema.keeps_history == (history_mode_before is not None):
+ return
+ if schema.keeps_history:
+ report_history_toggled(user, schema, enabled=True, table_mode=schema.cdc_table_mode, request=request)
+ else:
+ assert history_mode_before is not None
+ report_history_toggled(user, schema, enabled=False, table_mode=history_mode_before, request=request)
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 55f1dc54fa8f..8fc90f1ecb9b 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -88,6 +88,7 @@ from products.warehouse_sources.backend.facade.types import (
IncrementalFieldType,
IncrementalSyncBlockedReason,
)
+from products.warehouse_sources.backend.presentation.history_tracking import report_history_change
from products.warehouse_sources.backend.presentation.views.destination_links import (
DestinationLinkSerializer,
SchemaDestinationsSerializer,
@@ -1018,6 +1019,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
# Capture the previous cdc_table_mode before any mutation so the post-save hook below can decide
# whether the change adds a new physical write target (and therefore needs a re-snapshot).
previous_cdc_table_mode = instance.cdc_table_mode
+ history_mode_before = previous_cdc_table_mode if instance.keeps_history else None
api_version_changed = "api_version" in validated_data and validated_data["api_version"] != instance.api_version
@@ -1693,6 +1695,12 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
)
self._run_temporal_side_effect(lambda: _reset_cdc_for_full_resnapshot(updated_instance))
+ request = self.context.get("request")
+ if request is not None:
+ report_history_change(
+ request.user, updated_instance, history_mode_before=history_mode_before, request=request
+ )
+
return updated_instance
def _effective_probe_api_version(self, schema: ExternalDataSchema, source_impl: AnySource) -> str | None:
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
index 0cc40dcf71db..39c9478a7363 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
@@ -39,6 +39,7 @@ from products.warehouse_sources.backend.facade.source_management import (
source_type_supports_cdc,
)
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
+from products.warehouse_sources.backend.presentation.history_tracking import report_history_toggled
from . import base
@@ -461,12 +462,13 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
sync_type=ExternalDataSchema.SyncType.CDC,
)
.exclude(deleted=True)
- .select_related("table")
+ .select_related("table", "source", "team")
)
# Disabling cancels jobs, drops the slot, purges buffered change data, and resets
# every CDC schema — editor on the source isn't enough when a table is locked below it.
self._assert_can_write_schemas(cdc_schemas)
cdc_schema_ids = [schema.id for schema in cdc_schemas]
+ history_schemas = [schema for schema in cdc_schemas if schema.keeps_history]
# The bulk update below bypasses update_should_sync, which is what pauses a table's schedule, so
# without this every former CDC table keeps syncing, and billing, on its old schedule. It runs
# first: a failed pause then fails the request while CDC still reads as enabled, so a retry
@@ -564,6 +566,11 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
instance.job_inputs = job_inputs
instance.save(update_fields=["job_inputs", "updated_at"])
+ for schema in history_schemas:
+ report_history_toggled(
+ request.user, schema, enabled=False, table_mode=schema.cdc_table_mode, request=request
+ )
+
return Response(status=status.HTTP_200_OK, data={"success": True})
@extend_schema(
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index 9313df03f593..c02deba7422a 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -77,6 +77,7 @@ from products.warehouse_sources.backend.facade.source_management import (
validate_and_coerce_row_filters,
)
from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind, ExternalDataSourceType
+from products.warehouse_sources.backend.presentation.history_tracking import report_history_toggled
from products.warehouse_sources.backend.presentation.views.destination_links import (
EMPTY_SET_MESSAGE,
set_source_destinations,
@@ -1368,6 +1369,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
)
active_schemas: list[ExternalDataSchema] = []
+ history_schemas: list[ExternalDataSchema] = []
# Pre-fetch PK column names for CDC tables
pk_columns_by_table: dict[str, list[str]] = {}
@@ -1779,6 +1781,8 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
if should_sync and new_source_model.supports_scheduled_sync:
active_schemas.append(schema_model)
+ if schema_model.keeps_history:
+ history_schemas.append(schema_model)
# Attach destinations before any schedule starts. Extraction snapshots the set onto the
# run, so a source whose destinations arrive after its first sync began writes that run
@@ -1867,6 +1871,15 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
request=request,
)
+ for schema_model in history_schemas:
+ report_history_toggled(
+ cast(User, request.user),
+ schema_model,
+ enabled=True,
+ table_mode=schema_model.cdc_table_mode,
+ request=request,
+ )
+
return Response(
status=status.HTTP_201_CREATED,
data=ExternalDataSourceCreateResponseSerializer({"id": new_source_model.pk}).data,
diff --git a/products/warehouse_sources/backend/tests/api/test_cdc_table_mode_patch.py b/products/warehouse_sources/backend/tests/api/test_cdc_table_mode_patch.py
index c22209394cf3..9699d498bb38 100644
--- a/products/warehouse_sources/backend/tests/api/test_cdc_table_mode_patch.py
+++ b/products/warehouse_sources/backend/tests/api/test_cdc_table_mode_patch.py
@@ -622,3 +622,53 @@ def test_patch_cdc_table_mode_drop_target_allowed_when_team_over_billing_limit(t
assert response.status_code == 200, response.content
schema.refresh_from_db()
assert schema.cdc_table_mode == "consolidated"
+
+
+ ("old_mode", "patch", "expected_event", "expected_table_mode"),
+ [
+ ("consolidated", {"cdc_table_mode": "both"}, "data warehouse history enabled", "both"),
+ ("both", {"cdc_table_mode": "consolidated"}, "data warehouse history disabled", "both"),
+ ("cdc_only", {"sync_type": "full_refresh"}, "data warehouse history disabled", "cdc_only"),
+ ("cdc_only", {"cdc_table_mode": "both"}, None, None),
+ ("consolidated", {"cdc_table_mode": "consolidated"}, None, None),
+ ],
+)
+def test_patch_reports_history_enabled_and_disabled(
+ team, user, client: HttpClient, old_mode, patch, expected_event, expected_table_mode
+):
+ source, schema = _make_cdc_source_and_schema(team, cdc_table_mode=old_mode)
+ client.force_login(user)
+
+ with (
+ mock.patch(_PATCH_TARGETS["is_cdc_enabled_for_team"], return_value=True),
+ mock.patch(_PATCH_TARGETS["alter_cdc_publication"]),
+ mock.patch(_PATCH_TARGETS["external_data_workflow_exists"], return_value=True),
+ mock.patch(_PATCH_TARGETS["sync_external_data_job_workflow"]),
+ mock.patch(_PATCH_TARGETS["sync_cdc_extraction_schedule"]),
+ mock.patch(_PATCH_TARGETS["cancel_external_data_workflow"]),
+ mock.patch(_PATCH_TARGETS["facade_sync_cdc_extraction_schedule"]),
+ mock.patch(_PATCH_TARGETS["pause_external_data_schedule"]),
+ mock.patch(_PATCH_TARGETS["trigger_cdc_extraction_schedule"]),
+ mock.patch(_PATCH_TARGETS["has_queued_batches"], return_value=False),
+ mock.patch(_PATCH_TARGETS["trigger_external_data_workflow"]),
+ mock.patch(_PATCH_TARGETS["is_any_external_data_schema_paused"], return_value=False),
+ mock.patch("posthog.event_usage.posthoganalytics.capture") as mock_capture,
+ ):
+ response = client.patch(
+ f"/api/environments/{team.pk}/external_data_schemas/{schema.id}",
+ data=patch,
+ content_type="application/json",
+ )
+
+ assert response.status_code == 200, response.content
+ history_events = [
+ call.kwargs
+ for call in mock_capture.call_args_list
+ if str(call.kwargs.get("event", "")).startswith("data warehouse history")
+ ]
+ assert [event["event"] for event in history_events] == ([expected_event] if expected_event else [])
+ if expected_event:
+ assert history_events[0]["properties"]["schema_id"] == str(schema.id)
+ assert history_events[0]["properties"]["source_id"] == str(source.id)
+ assert history_events[0]["properties"]["table_mode"] == expected_table_mode
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 57fd7be8869e..bedf2ff5c9e2 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -4851,6 +4851,13 @@ class TestExternalDataSource(APIBaseTest):
assert not ExternalDataSource.objects.filter(team_id=self.team.pk).exists()
assert not ExternalDataSchema.objects.filter(team_id=self.team.pk, name="analytics.events").exists()
+ @parameterized.expand(
+ [
+ ("default_mode_keeps_no_history", None, []),
+ ("history_mode_reports_enabled", "both", ["both"]),
+ ]
+ )
+ @patch("posthog.event_usage.posthoganalytics.capture")
@patch(
"products.warehouse_sources.backend.presentation.views.external_data_source.base.is_cdc_enabled_for_team",
return_value=True,
@@ -4866,12 +4873,16 @@ class TestExternalDataSource(APIBaseTest):
@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.SourceRegistry.get_source")
def test_create_postgres_cdc_leaves_unenabled_schemas_without_sync_type(
self,
+ _name,
+ events_table_mode,
+ expected_history_modes,
mock_get_source,
mock_cdc_pg_connection,
mock_get_primary_key_columns,
mock_setup_cdc_resources,
mock_add_table,
_mock_is_cdc_enabled_for_team,
+ mock_capture,
):
_configure_source_mock_versioning(mock_get_source)
# A CDC source discovers every table, but the user only enables a few. Tables the user
@@ -4946,7 +4957,12 @@ class TestExternalDataSource(APIBaseTest):
"schema": "analytics",
"cdc_enabled": True,
"schemas": [
- {"name": "analytics.events", "should_sync": True, "sync_type": "cdc"},
+ {
+ "name": "analytics.events",
+ "should_sync": True,
+ "sync_type": "cdc",
+ **({"cdc_table_mode": events_table_mode} if events_table_mode else {}),
+ },
{"name": "analytics.sessions", "should_sync": False, "sync_type": "cdc"},
],
},
@@ -4958,7 +4974,15 @@ class TestExternalDataSource(APIBaseTest):
enabled = ExternalDataSchema.objects.get(team_id=self.team.pk, name="analytics.events")
assert enabled.sync_type == ExternalDataSchema.SyncType.CDC
assert enabled.sync_type_config["cdc_mode"] == "snapshot"
- assert enabled.sync_type_config["cdc_table_mode"] == "consolidated"
+ assert enabled.sync_type_config["cdc_table_mode"] == (events_table_mode or "consolidated")
+ history_events = [
+ c.kwargs["properties"]
+ for c in mock_capture.call_args_list
+ if c.kwargs.get("event") == "data warehouse history enabled"
+ ]
+ assert [(e["schema_id"], e["table_mode"]) for e in history_events] == [
+ (str(enabled.id), mode) for mode in expected_history_modes
+ ]
unenabled = ExternalDataSchema.objects.get(team_id=self.team.pk, name="analytics.sessions")
assert unenabled.sync_type is None
@@ -11792,6 +11816,14 @@ class TestDisableCDC(APIBaseTest):
sync_type=ExternalDataSchema.SyncType.CDC,
should_sync=True,
)
+ history_schema = ExternalDataSchema.objects.create(
+ name="cdc_history_table",
+ team_id=self.team.pk,
+ source_id=source.pk,
+ sync_type=ExternalDataSchema.SyncType.CDC,
+ sync_type_config={"cdc_table_mode": "both"},
+ should_sync=True,
+ )
non_cdc_schema = ExternalDataSchema.objects.create(
name="incremental_table",
team_id=self.team.pk,
@@ -11800,11 +11832,19 @@ class TestDisableCDC(APIBaseTest):
should_sync=True,
)
- response = self.client.post(
- f"/api/environments/{self.team.pk}/external_data_sources/{source.pk}/disable_cdc/",
- )
+ with patch("posthog.event_usage.posthoganalytics.capture") as mock_capture:
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/external_data_sources/{source.pk}/disable_cdc/",
+ )
assert response.status_code == 200, response.content
+ history_events = [
+ c.kwargs for c in mock_capture.call_args_list if c.kwargs.get("event") == "data warehouse history disabled"
+ ]
+ assert [(e["properties"]["schema_id"], e["properties"]["table_mode"]) for e in history_events] == [
+ (str(history_schema.id), "both")
+ ]
+
source.refresh_from_db()
ji = source.job_inputs or {}
# Every cdc_* key must be gone — leaving a stale consistent_point behind
@@ -11822,7 +11862,7 @@ class TestDisableCDC(APIBaseTest):
non_cdc_schema.refresh_from_db()
assert non_cdc_schema.sync_type == ExternalDataSchema.SyncType.INCREMENTAL
assert non_cdc_schema.should_sync is True
- mock_pause_schedule.assert_called_once_with(str(cdc_schema.id))
+ assert {c.args[0] for c in mock_pause_schedule.call_args_list} == {str(cdc_schema.id), str(history_schema.id)}
@patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.cleanup_resources",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 1748bde0d6bc..83c588c14018 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -2941,6 +2941,8 @@ export namespace Schemas {
source_type?: string | null;
/** Warehouse table name that was referenced */
table_name: string;
+ /** Whether the query read a history table of this source */
+ uses_history_table?: boolean | null;
}
export interface DataWarehouseSyncWarning {