:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add deterministic column masking for sources (#116589)
Daniel RCef9d39e29025
diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 61ea1c9dbe7a..912b0ee02007 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -6637,6 +6637,8 @@ export interface ExternalDataSourceSyncSchema {* PK columns and the active incremental field are always retained server-side.*/enabled_columns?: string[] | null+ /** Source columns whose values are replaced with a one-way hash at sync time. */+ masked_columns?: string[] | null/*** Predicates ANDed onto the source query so only matching rows sync.* `null`/undefined/empty = sync all rows.@@ -6680,6 +6682,8 @@ export interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema* Primary-key + active incremental columns are always retained even if not listed.*/enabled_columns?: string[] | null+ /** Source columns whose values are replaced with a one-way hash at sync time. */+ masked_columns?: string[] | nullavailable_columns?: { name: string; data_type?: string; is_nullable?: boolean }[]/** Exact source identifiers are available for sources that project columns upstream. */source_column_metadata_available?: booleandiff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.pyindex e35d7c040b05..2686bb74b58f 100644--- a/posthog/models/activity_logging/activity_log.py+++ b/posthog/models/activity_logging/activity_log.py@@ -510,6 +510,7 @@ field_name_overrides: dict[AuditableScope, dict[str, str]] = {},"ExternalDataSchema": {"should_sync": "enabled",+ "masked_columns": "masked columns","full_refresh_interval_days": "full refresh interval (days)","full_refresh_time_of_day": "full refresh time (UTC)",},diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex 4f03724d71f9..eb6a37068e46 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -33,6 +33,10 @@ USE_LOCAL_SETUP = get_from_env("USE_LOCAL_SETUP", USE_LOCAL_SETUP_DEFAULT, type_PYARROW_DEBUG_LOGGING = get_from_env("PYARROW_DEBUG_LOGGING", False, type_cast=str_to_bool)+# Keys the one-way hash of masked warehouse columns. Changing it changes every later hash, so every+# masked table needs a full resync afterwards. Without it, column masking is off outside development.+WAREHOUSE_COLUMN_MASKING_KEY = os.getenv("WAREHOUSE_COLUMN_MASKING_KEY", "")+# Load the full warehouse source catalog (every vendor SDK) at web-worker startup, before# the worker starts serving, so its first warehouse query doesn't pay the multi-second# catalog import at request time. WSGI workers load while importing posthog.wsgi, ASGIdiff --git a/products/data_warehouse/backend/direct_virtual_tables.py b/products/data_warehouse/backend/direct_virtual_tables.pyindex 253160116fc1..8b30da1a51af 100644--- a/products/data_warehouse/backend/direct_virtual_tables.py+++ b/products/data_warehouse/backend/direct_virtual_tables.py@@ -65,7 +65,8 @@ def eligible_direct_query_schemas(team_id: int, source_id: str) -> list[Externalcatalog, so the two selections can't drift. Excludes rows without usable ``schema_metadata``(e.g. discovered before metadata persistence shipped) and rows with ``row_filters`` — async-time row restriction the live direct query can't reproduce, so exposing the upstream- table would let a user read rows the schema was configured not to sync.+ table would let a user read rows the schema was configured not to sync. Rows with+ ``masked_columns`` are excluded for the same reason: a live query reads the plaintext values."""return [schema_row@@ -78,7 +79,7 @@ def eligible_direct_query_schemas(team_id: int, source_id: str) -> list[External# `table` is the synced S3 row backing the warehouse access-control check on the build path..select_related("table").order_by("name")- if schema_row.schema_metadata and not schema_row.row_filters+ if schema_row.schema_metadata and not schema_row.row_filters and not schema_row.masked_columns]@@ -91,9 +92,9 @@ def build_direct_table_for_schema(schema: "ExternalDataSchema", source: "Externametadata = schema.schema_metadataengine = source.direct_engine- # Defense in depth for any direct caller: a row-filtered schema is already excluded from the- # catalog by eligible_direct_query_schemas; never build a live table that would bypass it.- if schema.row_filters:+ # Defense in depth for any direct caller: a row-filtered or masked schema is already excluded+ # from the catalog by eligible_direct_query_schemas; never build a live table that would bypass it.+ if schema.row_filters or schema.masked_columns:return Noneif engine == "postgres":diff --git a/products/data_warehouse/backend/logic/data_load/service.py b/products/data_warehouse/backend/logic/data_load/service.pyindex 5b58927e1886..fd31b1dcbc41 100644--- a/products/data_warehouse/backend/logic/data_load/service.py+++ b/products/data_warehouse/backend/logic/data_load/service.py@@ -41,6 +41,7 @@ from posthog.temporal.common.schedule import (pause_schedule,schedule_exists,trigger_schedule,+ trigger_schedule_buffer_one,unpause_schedule,update_schedule,)@@ -214,9 +215,13 @@ def trigger_external_data_source_workflow(external_data_source: ExternalDataSourtrigger_schedule(temporal, schedule_id=str(external_data_source.id))-def trigger_external_data_workflow(external_data_schema: ExternalDataSchema):+def trigger_external_data_workflow(external_data_schema: ExternalDataSchema, *, buffer_one: bool = False):temporal = sync_connect()- trigger_schedule(temporal, schedule_id=str(external_data_schema.id))+ if buffer_one:+ # The schedule skips a trigger while a run is open, and a cancelled run stays open until it closes.+ trigger_schedule_buffer_one(temporal, str(external_data_schema.id))+ else:+ trigger_schedule(temporal, schedule_id=str(external_data_schema.id))async def a_trigger_external_data_workflow(external_data_schema: ExternalDataSchema):diff --git a/products/data_warehouse/backend/test/test_direct_virtual_tables.py b/products/data_warehouse/backend/test/test_direct_virtual_tables.pyindex fd1998598af3..944df2d4fba6 100644--- a/products/data_warehouse/backend/test/test_direct_virtual_tables.py+++ b/products/data_warehouse/backend/test/test_direct_virtual_tables.py@@ -2,7 +2,10 @@ from posthog.test.base import APIBaseTestfrom posthog.hogql.database.direct_motherduck_table import DirectMotherDuckTable-from products.data_warehouse.backend.direct_virtual_tables import build_direct_table_for_schema+from products.data_warehouse.backend.direct_virtual_tables import (+ build_direct_table_for_schema,+ eligible_direct_query_schemas,+)from products.warehouse_sources.backend.facade.models import ExternalDataSchema, ExternalDataSourcefrom products.warehouse_sources.backend.facade.types import ExternalDataSourceType@@ -16,7 +19,9 @@ SCHEMA_METADATA = {class TestBuildDirectTableForSchema(APIBaseTest):- def _create(self, *, enabled_columns: list[str] | None) -> tuple[ExternalDataSchema, ExternalDataSource]:+ def _create(+ self, *, enabled_columns: list[str] | None, masked_columns: list[str] | None = None+ ) -> tuple[ExternalDataSchema, ExternalDataSource]:source = ExternalDataSource.objects.create(team=self.team,source_type=ExternalDataSourceType.POSTGRES,@@ -29,6 +34,7 @@ class TestBuildDirectTableForSchema(APIBaseTest):team=self.team,source=source,enabled_columns=enabled_columns,+ masked_columns=masked_columns,sync_type_config={"schema_metadata": SCHEMA_METADATA, "primary_key_columns": ["id"]},)return schema, source@@ -45,6 +51,13 @@ class TestBuildDirectTableForSchema(APIBaseTest):assert "email" in table.fieldsassert "id" in table.fields # primary key always retained+ def test_masked_schema_is_not_exposed_to_live_queries(self):+ # A live query reads the source directly, so it would return the masked values in plaintext.+ schema, source = self._create(enabled_columns=None, masked_columns=["secret"])++ assert build_direct_table_for_schema(schema, source) is None+ assert eligible_direct_query_schemas(self.team.pk, str(source.pk)) == []+def test_none_enabled_columns_syncs_everything(self):schema, source = self._create(enabled_columns=None)diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx b/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsxindex ec7b3401e174..2619e50e17cb 100644--- a/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx@@ -277,6 +277,11 @@ export function buildCreateSourcePayload(}}+function withoutColumns(columns: string[] | null | undefined, removed: (string | null)[]): string[] | null | undefined {+ const names = new Set(removed.filter((name): name is string => !!name).map((name) => name.toLowerCase()))+ return columns?.filter((column) => !names.has(column.toLowerCase())) ?? columns+}+function webhookResultHasNoPendingInputs(webhookResult: WebhookCreateResult | null | undefined): boolean {return !!webhookResult?.success && (webhookResult.pending_inputs?.length ?? 0) === 0}@@ -735,6 +740,13 @@ export interface sourceWizardLogicActions {returnLabel: stringreturnUrl: string}+ setSchemaMaskedColumns: (+ schema: ExternalDataSourceSyncSchema,+ maskedColumns: string[]+ ) => {+ maskedColumns: string[]+ schema: ExternalDataSourceSyncSchema+ }setSchemaNameFilter: (schemaNameFilter: string) => {schemaNameFilter: string}@@ -1095,6 +1107,10 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([schema,rowFilters,}),+ setSchemaMaskedColumns: (schema: ExternalDataSourceSyncSchema, maskedColumns: string[]) => ({+ schema,+ maskedColumns,+ }),updateSchemaSyncType: (schema: ExternalDataSourceSyncSchema,syncType: ExternalDataSourceSyncSchema['sync_type'],@@ -1267,6 +1283,12 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([row_filters: s.table === schema.table ? rowFilters : s.row_filters,}))},+ setSchemaMaskedColumns: (state, { schema, maskedColumns }) => {+ return state.map((s) => ({+ ...s,+ masked_columns: s.table === schema.table ? maskedColumns : s.masked_columns,+ }))+ },updateSchemaSyncType: (state,{@@ -1290,6 +1312,11 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([? (incrementalFieldLookbackSeconds ?? null): s.incremental_field_lookback_seconds,primary_key_columns: s.table === schema.table ? primaryKeyColumns : s.primary_key_columns,+ // The server refuses to mask a cursor or key column, which would fail the whole source creation.+ masked_columns:+ s.table === schema.table+ ? withoutColumns(s.masked_columns, [incrementalField, ...(primaryKeyColumns ?? [])])+ : s.masked_columns,...(s.table === schema.table && syncType === 'cdc' && cdcTableMode? { cdc_table_mode: cdcTableMode }: {}),@@ -2200,6 +2227,7 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([sync_time_of_day: schema.sync_time_of_day,primary_key_columns: schema.primary_key_columns,enabled_columns: schema.enabled_columns ?? null,+ masked_columns: schema.masked_columns ?? null,row_filters: schema.row_filters ?? null,...(schema.sync_type === 'cdc' && schema.cdc_table_mode? { cdc_table_mode: schema.cdc_table_mode }diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts b/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.tsindex 1cb811d05fb4..c2b8212d0180 100644--- a/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts@@ -1128,6 +1128,36 @@ describe('sourceWizardLogic', () => {}})+ it.each([+ { name: 'cursor', incrementalField: 'Updated_At', primaryKeyColumns: null, kept: ['email'] },+ { name: 'primary key', incrementalField: null, primaryKeyColumns: ['email'], kept: ['updated_at'] },+ ])(+ 'choosing a masked column as the $name drops its mask, since the server refuses it',+ ({ incrementalField, primaryKeyColumns, kept }) => {+ const users = buildSchema({ table: 'users' })+ const orders = buildSchema({ table: 'orders' })+ const { logic, unmount } = mountWithSchemas([users, orders])++ try {+ logic.actions.setSchemaMaskedColumns(users, ['email', 'updated_at'])+ logic.actions.setSchemaMaskedColumns(orders, ['email'])+ logic.actions.updateSchemaSyncType(+ users,+ 'incremental',+ incrementalField,+ 'datetime',+ primaryKeyColumns+ )++ const byTable = Object.fromEntries(logic.values.databaseSchema.map((s) => [s.table, s]))+ expect(byTable['users'].masked_columns).toEqual(kept)+ expect(byTable['orders'].masked_columns).toEqual(['email'])+ } finally {+ unmount()+ }+ }+ )+it('toggleSchemaGroup skips permission_error rows in a group', () => {const { logic, unmount } = mountWithSchemas([buildSchema({ table: 'public.customers' }),diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsxindex 3f7c82688435..6ea57f53ce71 100644--- a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx+++ b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx@@ -760,14 +760,22 @@ function ColumnsAndRowFiltersSection({// Both editors run in `hideActions` mode and report edits up here, so one Save commits both.const [draftColumns, setDraftColumns] = useState<string[] | null>(schema.enabled_columns ?? null)const [draftRowFilters, setDraftRowFilters] = useState<RowFilter[] | null>(schema.row_filters ?? null)+ const [draftMasked, setDraftMasked] = useState<string[]>(schema.masked_columns ?? [])+ const [saving, setSaving] = useState(false)// Reset drafts on schema switch or when server values change (e.g. reload after save).// Keyed on content so unrelated re-renders don't wipe edits.useEffect(() => {setDraftColumns(schema.enabled_columns ?? null)setDraftRowFilters(schema.row_filters ?? null)+ setDraftMasked(schema.masked_columns ?? [])// eslint-disable-next-line react-hooks/exhaustive-deps- }, [schema.id, JSON.stringify(schema.enabled_columns ?? null), JSON.stringify(schema.row_filters ?? null)])+ }, [+ schema.id,+ JSON.stringify(schema.enabled_columns ?? null),+ JSON.stringify(schema.row_filters ?? null),+ JSON.stringify(schema.masked_columns ?? []),+ ])const alwaysRetained = new Set<string>([...(schema.primary_key_columns ?? []),@@ -788,25 +796,57 @@ function ColumnsAndRowFiltersSection({const rowFilterErrors = validateRowFilters(draftRowFilters ?? [], { availableColumns: rowFilterColumns })const hasRowFilterErrors = Object.keys(rowFilterErrors).length > 0+ const savedMasked = schema.masked_columns ?? []+ const newlyMasked = draftMasked.filter((column) => !savedMasked.includes(column))+ const unmasked = savedMasked.filter((column) => !draftMasked.includes(column))+ const masksChanged = newlyMasked.length > 0 || unmasked.length > 0+const isDirty =!sameColumns(draftColumns, schema.enabled_columns ?? null, available) ||- JSON.stringify(draftRowFilters ?? null) !== JSON.stringify(schema.row_filters ?? null)+ JSON.stringify(draftRowFilters ?? null) !== JSON.stringify(schema.row_filters ?? null) ||+ masksChangedconst commit = (resyncAfter: boolean): void => {- const next = { ...schema, enabled_columns: draftColumns, row_filters: draftRowFilters }- if (resyncAfter) {+ // A second save would validate against the old masks and request another rebuild.+ if (saving) {+ return+ }+ const next = {+ ...schema,+ enabled_columns: draftColumns,+ row_filters: draftRowFilters,+ masked_columns: draftMasked,+ }+ if (resyncAfter || masksChanged) {// Bypass the bulk-update debounce so resync reads the new config from the DB, not the// stale one a still-queued PATCH hasn't written yet.+ // The backend starts the resync itself when it saves a mask change, and it can refuse one.+ setSaving(true)void api.externalDataSchemas- .update(schema.id, { enabled_columns: draftColumns, row_filters: draftRowFilters })+ .update(schema.id, {+ enabled_columns: draftColumns,+ row_filters: draftRowFilters,+ // Unchanged masks stay out, so a column that left the source can't block the save.+ ...(masksChanged ? { masked_columns: draftMasked } : {}),+ }).then(() => {updateSchema(next)- resyncSchema(next)- lemonToast.success('Saved — full resync queued')+ if (!masksChanged) {+ resyncSchema(next)+ }+ // The backend only stages the rebuild of a table that does not sync.+ lemonToast.success(+ !resyncAfter+ ? 'Saved'+ : masksChanged && !schema.should_sync+ ? 'Saved. The full resync starts when you turn syncing on.'+ : 'Saved — full resync queued'+ )}).catch((e: any) => {lemonToast.error(e?.message || "Can't save at this time")})+ .finally(() => setSaving(false))return}updateSchema(next)@@ -815,6 +855,43 @@ function ColumnsAndRowFiltersSection({const handleSave = (): void => {const syncType = schema.sync_type++ // A mask change also cancels and restarts a running first sync, which has no last_synced_at yet.+ const isRunning = schema.status === 'Running'+ if (masksChanged && (!!schema.last_synced_at || isRunning)) {+ LemonDialog.open({+ title: 'Changing masked columns starts a full resync',+ description: (+ <div className="flex flex-col gap-2">+ <span>+ The table is rebuilt from the source, because rows that are already synced can't be masked+ or unmasked in place. A masked column no longer matches the same values in tables where it+ isn't masked.+ </span>+ {isRunning && syncType !== 'cdc' && (+ <span>The sync currently running will be canceled when you save.</span>+ )}+ {newlyMasked.length > 0 && newlyMasked.length <= 6 && (+ <span className="text-xs text-muted">+ Masking: {newlyMasked.map((c) => `"${c}"`).join(', ')}+ </span>+ )}+ {unmasked.length > 0 && unmasked.length <= 6 && (+ <span className="text-xs text-muted">+ Unmasking: {unmasked.map((c) => `"${c}"`).join(', ')}+ </span>+ )}+ </div>+ ),+ primaryButton: {+ children: 'Save and resync',+ onClick: () => commit(true),+ },+ secondaryButton: { children: 'Cancel' },+ })+ return+ }+const added = getAddedColumns(schema.enabled_columns ?? null, draftColumns, available)const requiresPrompt =!!schema.last_synced_at &&@@ -850,7 +927,7 @@ function ColumnsAndRowFiltersSection({<div><SectionHeadertitle="Columns"- description="Choose which columns from this table get synced. Primary keys and the active incremental field are always synced."+ description="Choose which columns from this table get synced, and mask sensitive ones to replace their values with a one-way hash. Primary keys and the active incremental field are always synced and can't be masked."/><div className="border rounded p-4 bg-surface-primary flex flex-col gap-3">{!hasAvailableColumns || columnSelectionNeedsRefresh ? (@@ -875,9 +952,18 @@ function ColumnsAndRowFiltersSection({</div>) : (<>- <span className="text-sm text-secondary">{columnsSummary}</span>+ <span className="text-sm text-secondary">+ {columnsSummary}+ {draftMasked.length > 0 && `, ${draftMasked.length} masked`}+ </span><fieldset disabled={!!editorDisabledReason}>- <ColumnSelectionPicker hideActions schema={schema} onChange={setDraftColumns} />+ <ColumnSelectionPicker+ hideActions+ enableMasking={source?.access_method !== 'direct'}+ schema={schema}+ onChange={setDraftColumns}+ onMaskedChange={setDraftMasked}+ /></fieldset></>)}@@ -920,6 +1006,7 @@ function ColumnsAndRowFiltersSection({<LemonButtontype="primary"onClick={handleSave}+ loading={saving}disabledReason={editorDisabledReason ??(hasRowFilterErrorsdiff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.test.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.test.tsxnew file mode 100644index 000000000000..264d55e75b44--- /dev/null+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.test.tsx@@ -0,0 +1,58 @@+import '@testing-library/jest-dom'++import { cleanup, fireEvent, render, screen, within } from '@testing-library/react'++import { initKeaTests } from '~/test/init'++import { ColumnSelectionPicker, ColumnSelectionTarget } from './ColumnSelectionModal'++const COLUMNS = [{ name: 'id' }, { name: 'uuid' }, { name: 'email' }]++const rowFor = (name: string): HTMLElement =>+ screen.getByText(name, { selector: 'code' }).closest('.justify-between') as HTMLElement++const renderPicker = (schema: Partial<ColumnSelectionTarget>, onMaskedChange = jest.fn()): jest.Mock => {+ render(+ <ColumnSelectionPicker+ hideActions+ enableMasking+ schema={{ id: 'users', available_columns: COLUMNS, ...schema }}+ onMaskedChange={onMaskedChange}+ />+ )+ return onMaskedChange+}++describe('ColumnSelectionPicker', () => {+ beforeEach(() => {+ initKeaTests()+ })++ afterEach(() => {+ cleanup()+ })++ test.each([+ ['the stored key', { primary_key_columns: ['email'], detected_primary_keys: ['uuid'] }, 'email'],+ ['the detected key when none is stored', { detected_primary_keys: ['uuid'] }, 'uuid'],+ ['the id column when no key is known', {}, 'id'],+ ])('only %s cannot be masked', (_name, keys, key) => {+ renderPicker(keys)++ for (const { name } of COLUMNS) {+ const mask = within(rowFor(name)).getByText('Mask').closest('button')+ expect(mask).toHaveAttribute('aria-disabled', String(name === key))+ }+ })++ test.each([+ ['a masked column that became the key', ['uuid'], 'uuid'],+ ['a masked column that left the source', ['gone'], 'gone'],+ ])('%s can still be unmasked', (_name, masked, column) => {+ const onMaskedChange = renderPicker({ detected_primary_keys: ['uuid'], masked_columns: masked })++ fireEvent.click(within(rowFor(column)).getByText('Masked'))++ expect(onMaskedChange).toHaveBeenLastCalledWith([])+ })+})diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsxindex e3ddce948711..11b6a38abf13 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsx+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsx@@ -1,5 +1,6 @@import { useEffect, useMemo, useState } from 'react'+import { IconLock } from '@posthog/icons'import { LemonButton, LemonCheckbox, LemonInput, LemonModal, Tooltip } from '@posthog/lemon-ui'import { ExternalDataSourceSchema } from '~/types'@@ -10,7 +11,10 @@ export interface ColumnSelectionTarget {id: stringname?: stringenabled_columns?: string[] | null+ masked_columns?: string[] | nullprimary_key_columns?: string[] | null+ /** The key discovery found, which a sync merges on when no key is stored. */+ detected_primary_keys?: string[] | nullincremental_field?: string | nullavailable_columns?: { name: string; data_type?: string; is_nullable?: boolean }[]}@@ -22,6 +26,9 @@ interface ColumnSelectionPickerProps {hideActions?: boolean/** Fires on every edit, so a parent can drive saving with its own button. */onChange?: (enabledColumns: string[] | null) => void+ /** Show a mask toggle on each column. */+ enableMasking?: boolean+ onMaskedChange?: (maskedColumns: string[]) => void}interface ColumnSelectionModalProps {@@ -39,6 +46,11 @@ interface UseColumnSelectionResult {toggleColumn: (name: string, checked: boolean) => voidisAlwaysRetained: (name: string) => booleanisChecked: (name: string) => boolean+ isMasked: (name: string) => boolean+ isMaskable: (name: string) => boolean+ isSyncKey: (name: string) => boolean+ toggleMask: (name: string) => void+ missingMasked: string[]persistedSelection: () => string[] | nullavailable: { name: string; data_type?: string; is_nullable?: boolean }[]primaryKeys: Set<string>@@ -48,23 +60,26 @@ interface UseColumnSelectionResult {function useColumnSelection(schema: ColumnSelectionTarget | null,- onChange?: (enabledColumns: string[] | null) => void+ onChange?: (enabledColumns: string[] | null) => void,+ onMaskedChange?: (maskedColumns: string[]) => void): UseColumnSelectionResult {const available = schema?.available_columns ?? []const primaryKeys = useMemo(() => new Set(schema?.primary_key_columns ?? []), [schema?.primary_key_columns])const incrementalField = schema?.incremental_fieldconst [selected, setSelected] = useState<Set<string> | null>(null)+ const [masked, setMasked] = useState<Set<string>>(new Set())const [filter, setFilter] = useState('')useEffect(() => {const current = schema?.enabled_columnssetSelected(Array.isArray(current) ? new Set(current) : null)+ setMasked(new Set(schema?.masked_columns ?? []))setFilter('')// Stable serialized key: the array ref changes on every poll even with identical contents,// which would otherwise wipe out in-progress edits.// eslint-disable-next-line react-hooks/exhaustive-deps- }, [schema?.id, schema?.enabled_columns?.join('\0') ?? null])+ }, [schema?.id, schema?.enabled_columns?.join('\0') ?? null, schema?.masked_columns?.join('\0') ?? null])const isAlwaysRetained = (name: string): boolean => primaryKeys.has(name) || name === incrementalField@@ -111,6 +126,48 @@ function useColumnSelection(commit(next)}+ const commitMasked = (name: string, shouldMask: boolean): void => {+ const next = new Set(masked)+ if (shouldMask) {+ next.add(name)+ } else {+ next.delete(name)+ }+ setMasked(next)+ onMaskedChange?.(Array.from(next))+ }++ const isMasked = (name: string): boolean => masked.has(name)++ // Mirrors the server: without a stored key, a sync merges on the detected key, else on `id`.+ const syncKeys = useMemo(() => {+ if (schema?.primary_key_columns?.length) {+ return new Set(schema.primary_key_columns)+ }+ if (schema?.detected_primary_keys?.length) {+ return new Set(schema.detected_primary_keys)+ }+ const idColumn = (schema?.available_columns ?? []).find((column) => column.name.toLowerCase() === 'id')+ return new Set(idColumn ? [idColumn.name] : [])+ }, [schema?.primary_key_columns, schema?.detected_primary_keys, schema?.available_columns])++ const isSyncKey = (name: string): boolean => syncKeys.has(name)++ // A hashed primary key breaks merges and a hashed incremental field cannot order rows.+ const isMaskable = (name: string): boolean => isChecked(name) && !isAlwaysRetained(name) && !isSyncKey(name)++ // A masked column can disappear from the source, and the mask must stay removable.+ const missingMasked = Array.from(masked).filter((name) => !available.some((column) => column.name === name))++ // A column masked before it became a key can still be unmasked, so the rejected save can be fixed.+ const toggleMask = (name: string): void => {+ if (masked.has(name)) {+ commitMasked(name, false)+ } else if (isMaskable(name)) {+ commitMasked(name, true)+ }+ }+const setSyncAll = (): void => commit(null)const filteredColumns = useMemo(() => {@@ -131,6 +188,11 @@ function useColumnSelection(toggleColumn,isAlwaysRetained,isChecked,+ isMasked,+ isMaskable,+ isSyncKey,+ toggleMask,+ missingMasked,persistedSelection,available,primaryKeys,@@ -145,6 +207,8 @@ export function ColumnSelectionPicker({onCancel,hideActions,onChange,+ enableMasking,+ onMaskedChange,}: ColumnSelectionPickerProps): JSX.Element {const {filter,@@ -153,11 +217,16 @@ export function ColumnSelectionPicker({toggleColumn,isAlwaysRetained,isChecked,+ isMasked,+ isMaskable,+ isSyncKey,+ toggleMask,+ missingMasked,persistedSelection,available,primaryKeys,filteredColumns,- } = useColumnSelection(schema, onChange)+ } = useColumnSelection(schema, onChange, onMaskedChange)return (<div className="flex flex-col gap-2">@@ -191,7 +260,7 @@ export function ColumnSelectionPicker({/>)return (- <div key={column.name} className="px-3 py-1">+ <div key={column.name} className="flex items-center justify-between gap-2 px-3 py-1">{retained ? (<Tooltip title="Always synced — required for merges or incremental tracking."><div>{checkbox}</div>@@ -199,12 +268,61 @@ export function ColumnSelectionPicker({) : (checkbox)}+ {enableMasking &&+ (isMasked(column.name) ? (+ <LemonButton+ size="xsmall"+ type="tertiary"+ icon={<IconLock />}+ onClick={() => toggleMask(column.name)}+ tooltip="This column syncs as a one-way hash. Click to unmask it and sync its real values again."+ data-attr="schema-column-unmask"+ >+ Masked+ </LemonButton>+ ) : (+ <LemonButton+ size="xsmall"+ type="secondary"+ onClick={() => toggleMask(column.name)}+ tooltip="Replace this column's values with a one-way hash when syncing. Use it for passwords and personal data. A masked column no longer matches the same values in tables where it isn't masked."+ disabledReason={+ isMaskable(column.name)+ ? undefined+ : retained || isSyncKey(column.name)+ ? "Primary key and incremental columns can't be masked"+ : 'Only synced columns can be masked'+ }+ data-attr="schema-column-mask"+ >+ Mask+ </LemonButton>+ ))}</div>)})}{filteredColumns.length === 0 && available.length > 0 && (<div className="text-center text-muted-alt py-6 text-sm">No columns match.</div>)}+ {enableMasking &&+ missingMasked.map((name) => (+ <div key={name} className="flex items-center justify-between gap-2 px-3 py-1">+ <div className="flex items-center gap-2">+ <code>{name}</code>+ <span className="text-xs text-muted-alt">no longer in the source</span>+ </div>+ <LemonButton+ size="xsmall"+ type="tertiary"+ icon={<IconLock />}+ onClick={() => toggleMask(name)}+ tooltip="This column is no longer in the source. Click to remove its mask."+ data-attr="schema-column-unmask-missing"+ >+ Masked+ </LemonButton>+ </div>+ ))}</div>{!hideActions && (<div className="flex items-center justify-between gap-2 mt-2">diff --git a/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsx b/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsxindex 05ce61b5df0f..552d8d5c0578 100644--- a/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsx+++ b/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsx@@ -1,7 +1,7 @@import { useActions, useValues } from 'kea'import { useEffect, useState } from 'react'-import { IconInfo, IconWarning } from '@posthog/icons'+import { IconInfo, IconLock, IconWarning } from '@posthog/icons'import {LemonBanner,LemonButton,@@ -89,6 +89,7 @@ export default function SchemaForm(): JSX.Element {setSchemaNameFilter,setSchemaSyncedColumns,setSchemaRowFilters,+ setSchemaMaskedColumns,} = useActions(sourceWizardLogic)const [columnSelectionSchema, setColumnSelectionSchema] = useState<ExternalDataSourceSyncSchema | null>(null)const {@@ -383,12 +384,19 @@ export default function SchemaForm(): JSX.Element {const summary = !synced? `All ${schema.available_columns.length}`: `${syncedCount} of ${schema.available_columns.length}`+ const maskedCount = schema.masked_columns?.length ?? 0return (<div className="justify-end flex"><LemonButtonclassName="my-1"size="small"type="secondary"+ icon={maskedCount > 0 ? <IconLock /> : undefined}+ tooltip={+ maskedCount > 0+ ? `${maskedCount} masked ${maskedCount === 1 ? 'column' : 'columns'}`+ : undefined+ }onClick={() => setColumnSelectionSchema(schema)}disabledReason={schema.permission_error ?? undefined}>@@ -655,7 +663,7 @@ export default function SchemaForm(): JSX.Element {</>) : null}- description="Choose which columns to sync and add row filters to sync only matching rows. Primary-key and incremental columns are always synced."+ description="Choose which columns to sync, mask sensitive columns, and add row filters to sync only matching rows. Primary-key and incremental columns are always synced and can't be masked."isOpen={columnSelectionSchema !== null}onClose={() => setColumnSelectionSchema(null)}footer={@@ -669,13 +677,16 @@ export default function SchemaForm(): JSX.Element {<h4 className="font-semibold mb-0">Columns to sync</h4><ColumnSelectionPickerhideActions+ enableMasking={!isDirectQueryMode}schema={columnSelectionSchema? {id: columnSelectionSchema.table,name: columnSelectionSchema.table,enabled_columns: columnSelectionSchema.enabled_columns,+ masked_columns: columnSelectionSchema.masked_columns,primary_key_columns: columnSelectionSchema.primary_key_columns,+ detected_primary_keys: columnSelectionSchema.detected_primary_keys,incremental_field: columnSelectionSchema.incremental_field,available_columns: columnSelectionSchema.available_columns.map((c) => ({name: c.field,@@ -690,6 +701,11 @@ export default function SchemaForm(): JSX.Element {setSchemaSyncedColumns(columnSelectionSchema, enabledColumns)}}}+ onMaskedChange={(maskedColumns) => {+ if (columnSelectionSchema) {+ setSchemaMaskedColumns(columnSelectionSchema, maskedColumns)+ }+ }}/></div>{columnSelectionSchema?.sync_type !== 'cdc' && (diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 2cf04f938642..69abf148d872 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -11153,6 +11153,10 @@ export namespace Schemas {* A schema of an external data source: its sync configuration and the warehouse table it syncs into.*/enabled_columns?: (Array<string> | null) | undefined;+ /**+ * A schema of an external data source: its sync configuration and the warehouse table it syncs into.+ */+ masked_columns?: (Array<string> | null) | undefined;/*** A schema of an external data source: its sync configuration and the warehouse table it syncs into.*/@@ -11238,6 +11242,7 @@ export namespace Schemas {primary_key_columns?: (Array<string> | null) | undefined;cdc_table_mode?: (CdcTableModeEnum | NullEnum) | undefined;enabled_columns?: (Array<string> | null) | undefined;+ masked_columns?: (Array<string> | null) | undefined;row_filters?:| (Array<{column: string;diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.pyindex 263fe85bd193..da62a989e267 100644--- a/products/warehouse_sources/backend/facade/models.py+++ b/products/warehouse_sources/backend/facade/models.py@@ -14,7 +14,10 @@ Consumers that only read fields should use ``facade.api`` (contracts) instead.from products.warehouse_sources.backend.facade.contracts import UnsupportedSyncTypeErrorfrom products.warehouse_sources.backend.models.column_annotation import WarehouseColumnAnnotation-from products.warehouse_sources.backend.models.column_statistics import WarehouseColumnStatistics+from products.warehouse_sources.backend.models.column_statistics import (+ WarehouseColumnStatistics,+ clear_masked_column_bounds,+)from products.warehouse_sources.backend.models.credential import (DataWarehouseCredential,get_or_create_datawarehouse_credential,@@ -125,6 +128,7 @@ __all__ = ["latest_completed_job_prefetch","get_or_create_datawarehouse_credential","get_or_create_warehouse_destination",+ "clear_masked_column_bounds","clickhouse_column_to_dwh_column","clickhouse_columns_to_dwh_columns","motherduck_column_to_dwh_column",diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.pyindex 1db9c1e7b049..2b374eec0098 100644--- a/products/warehouse_sources/backend/facade/source_management.py+++ b/products/warehouse_sources/backend/facade/source_management.py@@ -43,7 +43,9 @@ _LAZY = {"reserved_column_names": "cdc.batcher","BUFFER_LANE": "cdc.snapshot_lane","CDC_RESET_PENDING_KEY": "cdc.snapshot_lane",+ "hand_reset_to_capture": "cdc.snapshot_lane","hand_reset_to_capture_if_sync_running": "cdc.snapshot_lane",+ "stage_handed_over_reset": "cdc.snapshot_lane","resnapshot_stays_in_buffer": "cdc.snapshot_lane","bigquery_client": "sources.bigquery.bigquery","is_bigquery_auth_error": "sources.bigquery.bigquery",@@ -73,6 +75,10 @@ _LAZY = {"discover_trino_catalog": "sources.trino.trino","build_default_schemas": "sources.common.schema","build_default_sync_settings": "sources.common.schema",+ "ambiguous_masked_columns": "pipelines.core.masking",+ "collision_alias_masked_columns": "pipelines.core.masking",+ "fold_column_name": "naming_convention",+ "masking_key_unavailable": "pipelines.core.masking","RowFilterValidationError": "sources.common.sql","filter_dwh_columns_by_enabled_columns": "sources.common.sql","sql_schema_metadata": "sources.common.sql",diff --git a/products/warehouse_sources/backend/migrations/0196_externaldataschema_masked_columns.py b/products/warehouse_sources/backend/migrations/0196_externaldataschema_masked_columns.pynew file mode 100644index 000000000000..aa9db41dd520--- /dev/null+++ b/products/warehouse_sources/backend/migrations/0196_externaldataschema_masked_columns.py@@ -0,0 +1,13 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [("warehouse_sources", "0195_repin_rippling_api_version_v2")]++ operations = [+ migrations.AddField(+ model_name="externaldataschema",+ name="masked_columns",+ field=models.JSONField(blank=True, default=None, null=True),+ ),+ ]diff --git a/products/warehouse_sources/backend/migrations/max_migration.txt b/products/warehouse_sources/backend/migrations/max_migration.txtindex c1111aeb707e..b81047bedf07 100644--- a/products/warehouse_sources/backend/migrations/max_migration.txt+++ b/products/warehouse_sources/backend/migrations/max_migration.txt@@ -1 +1 @@-0195_repin_rippling_api_version_v2+0196_externaldataschema_masked_columnsdiff --git a/products/warehouse_sources/backend/models/column_statistics.py b/products/warehouse_sources/backend/models/column_statistics.pyindex 7aa7e8db8009..cb10fb62f98f 100644--- a/products/warehouse_sources/backend/models/column_statistics.py+++ b/products/warehouse_sources/backend/models/column_statistics.py@@ -1,8 +1,12 @@+import uuid+from django.db import modelsfrom posthog.models.scoping.root_mixin import TeamScopedRootMixinfrom posthog.models.utils import CreatedMetaFields, UpdatedMetaFields, UUIDTModel, sane_repr+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name+class WarehouseColumnStatistics(TeamScopedRootMixin, CreatedMetaFields, UpdatedMetaFields, UUIDTModel):"""Per-column data profile for a synced warehouse table, surfaced to the AI agent.@@ -60,3 +64,16 @@ class WarehouseColumnStatistics(TeamScopedRootMixin, CreatedMetaFields, UpdatedMconstraints = [models.UniqueConstraint(fields=["table", "column_name"], name="unique_table_column_statistics"),]+++def clear_masked_column_bounds(team_id: int, table_id: uuid.UUID | None, masked_columns: list[str] | None) -> None:+ """Drop the stored min and max of masked columns. They hold plaintext from before the mask."""+ masked = {fold_column_name(column) for column in masked_columns or []}+ if not masked or table_id is None:+ return+ statistics = WarehouseColumnStatistics.objects.for_team(team_id).filter(table_id=table_id, has_min_max=True)+ stale = [+ stat_id for stat_id, name in statistics.values_list("id", "column_name") if fold_column_name(name) in masked+ ]+ if stale:+ statistics.filter(id__in=stale).update(min_value=None, max_value=None, has_min_max=False)diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.pyindex 4bee659b1784..6b87d3d1b765 100644--- a/products/warehouse_sources/backend/models/external_data_schema.py+++ b/products/warehouse_sources/backend/models/external_data_schema.py@@ -375,6 +375,8 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djanenabled_columns = models.JSONField(null=True, blank=True, default=None)# null (default) = sync all rows. List of {column, operator, value} predicates ANDed onto the WHERE clause.row_filters = models.JSONField(null=True, blank=True, default=None)+ # null (default) = no masking. Source column names whose values the sync replaces with a one-way digest.+ masked_columns = models.JSONField(null=True, blank=True, default=None)objects = ExternalDataSchemaQuerySet.as_manager()diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex ad2ce8551e59..d533bfd80fa1 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -55,6 +55,7 @@ from products.warehouse_sources.backend.facade.models import (ExternalDataSchemaDestination,ExternalDataSource,UnsupportedSyncTypeError,+ clear_masked_column_bounds,discovered_schema_finder,is_pinned_schema,mark_schema_running_unless_halted,@@ -77,14 +78,19 @@ from products.warehouse_sources.backend.facade.source_management import (SourceSchema,WebhookSource,add_table_failure_message,+ ambiguous_masked_columns,+ collision_alias_masked_columns,filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,+ fold_column_name,get_cdc_adapter,+ hand_reset_to_capture,hand_reset_to_capture_if_sync_running,history_naming_unless_taken,history_table_name,is_another_schemas_history_table,is_history_table_mode,known_source_key,+ masking_key_unavailable,new_history_naming,pin_history_naming,pinned_history_naming,@@ -94,6 +100,7 @@ from products.warehouse_sources.backend.facade.source_management import (reserved_column_names,resnapshot_stays_in_buffer,source_type_supports_cdc,+ stage_handed_over_reset,tables_wait_for_repair,validate_and_coerce_row_filters,)@@ -538,6 +545,176 @@ def _source_column_names(instance: ExternalDataSchema) -> set[str]:return {column["name"] for column in columns if isinstance(column, dict) and isinstance(column.get("name"), str)}+MASKING_KEY_UNAVAILABLE_ERROR = (+ "Column masking isn't set up on this PostHog instance. "+ "Set WAREHOUSE_COLUMN_MASKING_KEY to a secret of at least 32 characters, then try again."+)+++def masked_key_conflict_error(+ masked_columns: list[str],+ primary_key_columns: Iterable[str] | None,+ incremental_fields: Iterable[str | None],+ partition_keys: Iterable[str] = (),+) -> str | None:+ # A partition column is read as a date or a number to pick the partition, which a digest is not.+ protected = {+ fold_column_name(column)+ for column in [*(primary_key_columns or []), *partition_keys, *incremental_fields]+ if column+ }+ conflicting = [column for column in masked_columns if fold_column_name(column) in protected]+ if conflicting:+ return f"Primary-key, incremental-field and partition columns can't be masked: {sorted(conflicting)}."+ return None+++def masked_columns_error(+ masked_columns: list[str],+ *,+ source_type: str,+ is_direct_query: bool,+ source_columns: Iterable[object],+ primary_key_columns: Iterable[str] | None,+ incremental_fields: Iterable[str | None],+ partition_keys: Iterable[str] = (),+) -> str | None:+ """Why this mask cannot be saved, or None. `source_columns` is empty when no columns are known yet."""+ # A direct query source never syncs, so a saved mask would protect nothing.+ if is_direct_query:+ return "Column masking is not supported for direct query sources."+ # A managed-schema source declares fixed column types, which a string digest breaks.+ if not source_supports_column_selection(source_type):+ return "Column masking is not supported for this source type."+ if masking_key_unavailable():+ return MASKING_KEY_UNAVAILABLE_ERROR+ known = [name for name in source_columns if isinstance(name, str)]+ # A mask that names no real column hashes nothing, so the intended column would sync in plaintext.+ if not known:+ return "Can't mask columns before the table's columns are known. Sync the table once, then try again."+ known_folded = {fold_column_name(name) for name in known}+ unknown = [column for column in masked_columns if fold_column_name(column) not in known_folded]+ if unknown:+ return (+ f"Unknown columns in masked_columns: {sorted(unknown)}. "+ "Run `Pull new schemas` to refresh available columns."+ )+ ambiguous = ambiguous_masked_columns(masked_columns, known)+ if ambiguous:+ return (+ f"Can't mask columns whose names collide after normalization: {sorted(ambiguous)}. "+ "Rename the source columns so they differ by more than case."+ )+ return masked_key_conflict_error(masked_columns, primary_key_columns, incremental_fields, partition_keys)+++def canonical_masked_columns(masked_columns: list[str], source_columns: Iterable[str]) -> list[str]:+ """Each mask under its column's own name, once. The UI compares names exactly."""+ by_folded: dict[str, str] = {}+ for name in source_columns:+ by_folded.setdefault(fold_column_name(name), name)+ return list(dict.fromkeys(by_folded.get(fold_column_name(column), column) for column in masked_columns))+++def _stage_mask_rebuild(config: dict[str, Any]) -> None:+ config["reset_pipeline"] = True+++def _stage_disabled_cdc_mask_snapshot(config: dict[str, Any]) -> None:+ config["cdc_mode"] = "snapshot"+ config["reset_pipeline"] = True+ config.pop(CDC_SNAPSHOT_LANE_KEY, None)+ config.pop(CDC_RESET_PENDING_KEY, None)+++def _validate_masked_columns(+ instance: ExternalDataSchema, validated_data: dict[str, Any], data: dict[str, Any]+) -> bool:+ """Reject a mask the sync cannot honor. Returns whether the request changes the masked set."""+ mask_in_request = "masked_columns" in validated_data+ masked_columns = (validated_data["masked_columns"] if mask_in_request else instance.masked_columns) or []++ # Also checked when only the key or the cursor moves, so neither can land on a masked column.+ if masked_columns and (mask_in_request or "primary_key_columns" in data or "incremental_field" in data):+ metadata_columns = (instance.schema_metadata or {}).get("columns")+ source_columns: list[str] = [+ name+ for column in (metadata_columns if isinstance(metadata_columns, list) else [])+ if isinstance(column, dict) and isinstance(name := column.get("name"), str)+ ]+ synced_columns = (+ [column["name"] for column in instance.table.get_user_facing_columns()]+ if instance.table is not None+ else []+ )+ # Non-SQL sources record no column metadata, so their synced columns are the known ones.+ if not source_columns:+ source_columns = synced_columns+ # Some sources record their columns after normalization, so the metadata can list an alias too.+ aliases = collision_alias_masked_columns(masked_columns, [*source_columns, *synced_columns])+ if aliases:+ raise ValidationError(+ f"Can't mask {sorted(aliases)}: the column was renamed because two source columns share its "+ "name. Rename one of the source columns, then mask it."+ )+ primary_key_columns = [*(data.get("primary_key_columns") or []), *(instance.primary_key_columns or [])]+ # Without a stored key, a sync merges and partitions on `id` when the table has one.+ if not primary_key_columns:+ primary_key_columns = [name for name in source_columns if name.lower() == "id"][:1]+ # The stored key and cursor count too: an update can leave them in place, for example when a+ # CDC table ignores the incremental field in the request. A full-refresh table keeps the cursor+ # of an earlier sync method in its settings, but reads no cursor.+ sync_type = data.get("sync_type") or instance.sync_type+ stored_cursor = instance.incremental_field if sync_type != ExternalDataSchema.SyncType.FULL_REFRESH else None+ incremental_fields = [data.get("incremental_field"), stored_cursor]+ partition_keys = [*(instance.partitioning_keys or []), *(instance.partitioning_keys_override or [])]+ # A stored mask whose column left the source must not block unrelated edits, so only new masks+ # are checked against the known columns. Every mask is still checked against the keys.+ stored = {fold_column_name(column) for column in instance.masked_columns or []}+ new_masks = [column for column in masked_columns if fold_column_name(column) not in stored]+ error = (+ masked_columns_error(+ new_masks,+ source_type=instance.source.source_type,+ is_direct_query=instance.source.is_direct_query,+ source_columns=source_columns,+ primary_key_columns=primary_key_columns,+ incremental_fields=incremental_fields,+ partition_keys=partition_keys,+ )+ if new_masks+ else None+ ) or masked_key_conflict_error(masked_columns, primary_key_columns, incremental_fields, partition_keys)+ if error:+ raise ValidationError(error)+ if mask_in_request:+ masked_columns = canonical_masked_columns(masked_columns, source_columns)+ validated_data["masked_columns"] = masked_columns++ # An omitted field is unchanged. Without this, any other edit to a masked table would read as+ # "remove every mask" and start a full resync.+ if not mask_in_request:+ return False+ changed = {fold_column_name(column) for column in (instance.masked_columns or [])} != {+ fold_column_name(column) for column in masked_columns+ }+ if not changed:+ return False+ # A mask change rebuilds the table from the source. A webhook table keeps its rows through a+ # reset, because the source cannot send them again, so the old values would stay as they are.+ if instance.is_webhook and instance.table is not None:+ raise ValidationError(+ "Can't change masked columns on a synced webhook table, because its data can't be fetched "+ "again from the source. Delete and recreate the table to change masking."+ )+ if is_any_external_data_schema_paused(instance.team_id):+ raise ValidationError(+ "Monthly sync limit reached. Please increase your billing limit before changing "+ "masked columns, because the change needs a full resync."+ )+ return True++def _refuse_reserved_columns(instance: ExternalDataSchema,*,@@ -750,6 +927,19 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"even if not listed here."),)+ masked_columns = serializers.ListField(+ child=serializers.CharField(),+ required=False,+ allow_null=True,+ help_text=(+ "Names of source columns whose values are replaced with a deterministic one-way digest at "+ "sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. "+ "Primary-key, partition and incremental columns can't be masked, and direct query "+ "sources don't support masking. Any change to this list starts a full resync of the table "+ "(CDC tables take a new snapshot), or stages it until syncing is turned on. A synced "+ "webhook table can't change its masked columns, because its data can't be fetched again."+ ),+ )row_filters = RowFiltersField(required=False,allow_null=True,@@ -840,6 +1030,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"cdc_table_mode","incremental_sync_blocked","enabled_columns",+ "masked_columns","row_filters","row_filter_columns","available_columns",@@ -1114,6 +1305,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersinstance: ExternalDataSchema,validated_data: dict[str, Any],original_sync_type_config: dict[str, Any],+ mutate_locked_config: Callable[[dict[str, Any]], None] | None = None,) -> ExternalDataSchema:"""Persist the update onto a freshly-locked row, writing only the fields this request changed.@@ -1138,6 +1330,8 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersmerged.update(changed)for key in removed:merged.pop(key, None)+ if mutate_locked_config is not None:+ mutate_locked_config(merged)validated_data["sync_type_config"] = merged# Apply to the locked row rather than the request's copy. The copy still holds whatever# the other writer replaced, and activity logging diffs the stored row against the@@ -1227,6 +1421,8 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"Column metadata is unavailable. Run `Pull new schemas` before selecting source columns.")+ masked_columns_changed = _validate_masked_columns(instance, validated_data, data)+# Validate against the schema's columns; raw filters are persisted as-is and re-coerced at sync time.if "row_filters" in validated_data and validated_data["row_filters"] is not None:# Only sources that push filters into their query (SQL WHERE) can honor them — a@@ -1679,6 +1875,28 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersvalidated_data["sync_type_config"]["reset_pipeline"] = Truetrigger_refresh = True+ # Rows already synced hold the old values, and the column changes type between the source+ # type and text, so a mask change rebuilds the table. CDC takes a new snapshot further down.+ # The rebuild is staged on the locked row: a run can clear an earlier reset, or repair can stage+ # one, between this request's read and its save.+ stage_locked_config: Callable[[dict[str, Any]], None] | None = None+ if masked_columns_changed:+ will_sync = should_sync if should_sync is not None else instance.should_sync+ if not is_cdc:+ validated_data.setdefault("sync_type_config", instance.sync_type_config)+ stage_locked_config = _stage_mask_rebuild+ if will_sync:+ trigger_refresh = True+ elif will_sync:+ # Staged in this save, so capture still finishes the reset if the hand-over after it fails.+ stage_locked_config = stage_handed_over_reset+ else:+ # Capture does not read a table that does not sync, so the new snapshot is staged in this+ # save. A self-managed publication takes no snapshot of its own when the table is turned on.+ stage_locked_config = _stage_disabled_cdc_mask_snapshot+ instance.initial_sync_complete = False+ validated_data["initial_sync_complete"] = False+if source.is_direct_query:direct_engine_adapter = get_direct_query_engine(source.direct_engine)# Direct-mode lifecycle hooks that need a fresh DataWarehouseTable projection:@@ -1732,14 +1950,25 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers# Persist under a row lock, replaying only the sync_type_config keys this request changed# so a concurrent CDC extract activity's writes aren't reverted by the full-instance save.- updated_instance = self._save_merging_sync_type_config(instance, validated_data, original_sync_type_config)+ updated_instance = self._save_merging_sync_type_config(+ instance, validated_data, original_sync_type_config, stage_locked_config+ )if publication_removal is not None:self._run_temporal_side_effect(publication_removal)+ if masked_columns_changed:+ # Profiles computed before the mask hold plaintext bounds, and the profiler recomputes them only daily.+ clear_masked_column_bounds(+ updated_instance.team_id, updated_instance.table_id, updated_instance.masked_columns+ )+# A version repin invalidates any in-flight import: retried/resumed activities re-resolve# the version from the DB, so letting the run finish would mix two vendor API versions in# one table. Cancel it; the user decides when to sync again.- if api_version_changed:+ # A mask change cancels it too, as the resync button does, so the masking rebuild starts now+ # instead of waiting for the next schedule tick.+ import_cancelled = False+ if api_version_changed or (masked_columns_changed and not is_cdc):running_job = (ExternalDataJob.objects.filter(schema_id=instance.pk, team_id=instance.team_id).order_by("-created_at")@@ -1752,12 +1981,15 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializerscancel_external_data_workflow(running_job.workflow_id)except temporalio.service.RPCError as e:logger.exception(- "Could not cancel running workflow after api_version change",+ "Could not cancel running workflow after a schema change",schema_id=str(instance.id),+ workflow_id=running_job.workflow_id,+ reason="api_version_change" if api_version_changed else "mask_change",exc_info=e,)self._run_temporal_side_effect(cancel_running_import)+ import_cancelled = Trueif source.supports_scheduled_sync and (should_sync is not None@@ -1821,7 +2053,9 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersself._run_temporal_side_effect(update_schedule)if trigger_refresh:- self._run_temporal_side_effect(lambda: trigger_external_data_workflow(updated_instance))+ self._run_temporal_side_effect(+ lambda: trigger_external_data_workflow(updated_instance, buffer_one=import_cancelled)+ )if sync_type == ExternalDataSchema.SyncType.WEBHOOK:self._maybe_create_webhook(updated_instance)@@ -1837,10 +2071,17 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersself._run_temporal_side_effect(sync_cdc_schedule)+ # Capture finishes this reset before it reads the change stream again, and it empties the change+ # buffer first: files captured before the change hold the old values and the old column type.+ cdc_mask_reset = is_cdc and masked_columns_changed and updated_instance.should_sync+ if cdc_mask_reset:+ # The save above staged the reset, so the hand-over only pauses the schedule and starts capture.+ self._run_temporal_side_effect(lambda: hand_reset_to_capture(updated_instance, logger, stage=False))+# If the cdc_table_mode change added a new physical write target, kick a full re-snapshot so the# new table is seeded from the current source state. `_seed_cdc_companion_from_snapshot` runs# automatically once the snapshot completes via `run_post_load_operations`.- if is_cdc and "cdc_table_mode" in data:+ if is_cdc and "cdc_table_mode" in data and not cdc_mask_reset:new_cdc_table_mode = data.get("cdc_table_mode")if _cdc_table_mode_change_needs_resnapshot(previous_cdc_table_mode, new_cdc_table_mode):logger.info(diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.pyindex 003cd0da0235..4920df29ec0b 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py@@ -125,6 +125,16 @@ class ExternalDataSourceBulkUpdateSchemaSerializer(serializers.Serializer):allow_empty=True,help_text="Columns to sync. Null means sync all columns.",)+ masked_columns = serializers.ListField(+ child=serializers.CharField(),+ required=False,+ allow_null=True,+ allow_empty=True,+ help_text=(+ "Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. "+ "Any change starts a full resync of the table."+ ),+ )row_filters = RowFiltersField(required=False,allow_null=True,diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.pyindex df5401d30236..8180f1e74788 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py@@ -100,6 +100,8 @@ from products.warehouse_sources.backend.presentation.views.external_data_schemaPINNED_TABLE_IDS_CONTEXT_KEY,ExternalDataSchemaListSerializer,ExternalDataSchemaSerializer,+ canonical_masked_columns,+ masked_columns_error,pinned_table_ids,source_supports_column_selection,unsupported_row_filter_reason,@@ -1184,6 +1186,14 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):request=self.request,)+ @staticmethod+ def _delete_new_source(source: ExternalDataSource, cdc_adapter: CDCSourceAdapter | None) -> None:+ # A hard delete leaves no source row for the orphan-slot sweeper to find, so drop the slot+ # and publication created for this source first.+ if cdc_adapter is not None:+ cdc_adapter.cleanup_resources(source)+ SourceInputsStore.discard_created(source)+def _create_external_data_source(self,request: Request,@@ -1633,15 +1643,30 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):payload_row_filters if isinstance(payload_row_filters, list) and payload_row_filters else None)+ payload_masked_columns = schema.get("masked_columns")+ # Dropping a malformed mask would sync the column in plaintext, so reject it.+ if payload_masked_columns is not None and not (+ isinstance(payload_masked_columns, list)+ and all(isinstance(column, str) for column in payload_masked_columns)+ ):+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={+ "message": f"masked_columns for schema '{schema.get('name')}' must be a list of column names or null."+ },+ )+ masked_columns: list[str] | None = payload_masked_columns or None+if should_sync and requires_incremental_fields and incremental_field is None:- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)return Response(status=status.HTTP_400_BAD_REQUEST,data={"message": "Incremental schemas given do not have an incremental field set"},)if should_sync and requires_incremental_fields and incremental_field_type is None:- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)return Response(status=status.HTTP_400_BAD_REQUEST,data={"message": "Incremental schemas given do not have an incremental field type set"},@@ -1667,7 +1692,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):and not source_schema.detected_primary_keys # type: ignore[union-attr]and not has_id_column):- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)return Response(data={"message": f"Table '{schema_name}' has no primary key to sync incrementally on. "@@ -1719,7 +1744,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):# Only sources that push filters into their query (SQL WHERE) can honor them — a# saved-but-ignored filter would silently sync unfiltered rows.if not source.supports_row_filters:- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)return Response(status=status.HTTP_400_BAD_REQUEST,data={@@ -1730,7 +1755,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):if reason := unsupported_row_filter_reason(is_direct_query=new_source_model.is_direct_query, is_cdc=sync_type == "cdc"):- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)return Response(status=status.HTTP_400_BAD_REQUEST,data={"message": f"Row filter not allowed for schema '{schema_name}': {reason}"},@@ -1740,7 +1765,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):row_filters, schema_metadata, source.row_filter_columns_for_schema(schema_name))except RowFilterValidationError as e:- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)return Response(status=status.HTTP_400_BAD_REQUEST,data={"message": f"Invalid row filter for schema '{schema_name}': {e}"},@@ -1777,7 +1802,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):# bool is an int subclass — exclude it so true/false aren't treated as 1/0.is_valid_int = isinstance(lookback_seconds, int) and not isinstance(lookback_seconds, bool)if not is_valid_int or not (0 <= lookback_seconds <= 5_184_000):- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)return Response(status=status.HTTP_400_BAD_REQUEST,data={@@ -1819,6 +1844,28 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):else:sync_type_config = {"schema_metadata": schema_metadata}+ if masked_columns:+ masking_error = masked_columns_error(+ masked_columns,+ source_type=source_type,+ is_direct_query=is_direct_query,+ source_columns=[column[0] for column in introspected_columns],+ # A sync merges and partitions on `id` when no key is stored or detected.+ primary_key_columns=sync_type_config.get("primary_key_columns")+ or (source_schema.detected_primary_keys if source_schema else None)+ or [column[0] for column in introspected_columns if column[0].lower() == "id"][:1],+ incremental_fields=[incremental_field],+ )+ if masking_error:+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={"message": f"Column masking not allowed for schema '{schema_name}': {masking_error}"},+ )+ masked_columns = canonical_masked_columns(+ masked_columns, [column[0] for column in introspected_columns]+ )+# CDC schemas benefit from a tighter poll cadence — the extraction workflow is cheap# and the value prop is near-real-time. Other sync types use the 6h default.schema_sync_frequency_interval = (@@ -1838,6 +1885,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):label=schema_label_by_name.get(schema_name),sync_frequency_interval=schema_sync_frequency_interval,enabled_columns=enabled_columns,+ masked_columns=masked_columns,row_filters=row_filters,)@@ -1859,10 +1907,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):cast(str, metadata_source_schema),cast(str, metadata_source_table_name),)- # A hard delete leaves no source row for the orphan-slot sweeper to find, so- # drop the slot and publication created above before deleting the source.- cdc_adapter.cleanup_resources(new_source_model)- SourceInputsStore.discard_created(new_source_model)+ self._delete_new_source(new_source_model, cdc_adapter)return Response(status=status.HTTP_400_BAD_REQUEST, data={"message": message})if direct_engine_adapter is not None and is_direct_query and should_sync:diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.pyindex 7949de648778..a450a7e5d671 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py@@ -73,6 +73,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.errors importMAX_FRIENDLY_MESSAGE_LENGTH,CDCErrorCategory,CDCErrorInfo,+ CDCMaskedKeyError,+ CDCMaskingKeyError,CDCReservedColumnError,CDCSlotNotConfiguredError,classify_cdc_error,@@ -96,6 +98,11 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.source_managersnapshot_can_start_in_buffer,)from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import (+ protected_masked_columns,+ value_masker,+)from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableExceptionfrom products.warehouse_sources.backend.temporal.data_imports.workflow_activities.create_job_model import (_build_schema_snapshot,@@ -206,6 +213,11 @@ class CDCExtractActivity:self.split_key_columns: dict[str, list[str]] = {}# Missing entry = sync all columns; otherwise the set is the projection (always includes PKs).self.enabled_columns_by_table: dict[str, set[str]] = {}+ # Folded names of the columns to mask. Missing entry = mask nothing.+ self.masked_columns_by_table: dict[str, set[str]] = {}+ # Whether each raw column name is masked, per table. Folding a name costs too much per event.+ self._masked_names_by_table: dict[str, dict[str, bool]] = {}+ self._masker: Callable[[object], str | None] | None = Noneself.history_naming_by_table: dict[str, HistoryNaming] = {}self.adapter: typing.Any = Noneself.reader: typing.Any = None@@ -644,6 +656,32 @@ class CDCExtractActivity:retained.add(inc)self.enabled_columns_by_table[schema.name] = retained+ self._load_masked_columns(schema)++ def _read_masked_columns(self, schema: ExternalDataSchema) -> list[str] | None:+ return (+ ExternalDataSchema.objects.filter(id=schema.id, team_id=schema.team_id)+ .values_list("masked_columns", flat=True)+ .first()+ )++ def _load_masked_columns(self, schema: ExternalDataSchema) -> None:+ self._masked_names_by_table.pop(schema.name, None)+ if not (isinstance(schema.masked_columns, list) and schema.masked_columns):+ self.masked_columns_by_table.pop(schema.name, None)+ return+ # The API checks a mask against the stored key, but a table saved without one gets+ # its key from the catalog above. Fail before any change of the table is read.+ masked_keys = protected_masked_columns(schema.masked_columns, self.pk_columns_by_table.get(schema.name, []))+ if masked_keys:+ raise CDCMaskedKeyError(schema.name, masked_keys)+ if self._masker is None:+ try:+ self._masker = value_masker(self.inputs.team_id)+ except NonRetryableException as e:+ raise CDCMaskingKeyError(str(e)) from e+ self.masked_columns_by_table[schema.name] = {fold_column_name(column) for column in schema.masked_columns}+def _query_pk_columns(self, table_names: list[str]) -> dict[str, list[str]]:"""Look up primary keys in the source catalog, keyed back on `ExternalDataSchema.name`.@@ -702,6 +740,30 @@ class CDCExtractActivity:omitted_columns=filtered_omitted,)+ def _mask_event_columns(self, event: ChangeEvent) -> ChangeEvent:+ masked = self.masked_columns_by_table.get(event.table_name)+ if not masked:+ return event+ assert self._masker is not None+ mask = self._masker+ decisions = self._masked_names_by_table.setdefault(event.table_name, {})++ def is_masked(name: str) -> bool:+ decision = decisions.get(name)+ if decision is None:+ decision = decisions[name] = fold_column_name(name) in masked+ return decision++ columns = {name: mask(value) if is_masked(name) else value for name, value in event.columns.items()}+ # A digest is text. Left on the source type, a flush where the column is all null would+ # write that type and no longer merge with the flushes that hold digests.+ column_types = event.column_types+ if column_types is not None:+ column_types = {+ name: pa.string() if is_masked(name) else column_type for name, column_type in column_types.items()+ }+ return dataclasses.replace(event, columns=columns, column_types=column_types)+def _split_key_change(self, event: ChangeEvent) -> tuple[ChangeEvent, ...]:"""Turn an update that changes the merge key into a delete of the old key and an insert of the new one.@@ -808,7 +870,7 @@ class CDCExtractActivity:event = dataclasses.replace(event, table_name=canonical_name)for change in self._split_key_change(event):- self.batcher.add(self._project_event_columns(change))+ self.batcher.add(self._mask_event_columns(self._project_event_columns(change)))# A change in position_serialized proves the previous transaction fully# yielded — all of its events are now buffered or flushed. Record its end LSN@@ -1027,6 +1089,10 @@ class CDCExtractActivity:mutate=_merge_pending,extra_model_fields={"initial_sync_complete": False},)+ # Read after the locked write above. A mask change saves its reset together with the masks,+ # so every reset that write merged and this run finishes comes with masks at least this new.+ schema.masked_columns = self._read_masked_columns(schema)+ self._load_masked_columns(schema)self._tables_awaiting_reset.discard(schema.name)return Truediff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.pyindex 45e140eb5a18..66f65255c979 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py@@ -39,6 +39,8 @@ class CDCErrorCategory(enum.StrEnum):WAL_DECODE_ERROR = "wal_decode_error"TRANSACTION_TOO_LARGE = "transaction_too_large"RESERVED_COLUMN_CONFLICT = "reserved_column_conflict"+ MASKED_KEY_CONFLICT = "masked_key_conflict"+ MASKING_KEY_UNAVAILABLE = "masking_key_unavailable"UNKNOWN = "unknown"@@ -84,6 +86,26 @@ class CDCReservedColumnError(Exception):"""+class CDCMaskingKeyError(Exception):+ """A table masks columns, but the instance has no secret key to hash them with.++ Non-retryable: the key is deployment configuration, so replaying re-fails until it is set.+ """+++class CDCMaskedKeyError(Exception):+ """A table masks a column that capture resolved as its primary key.++ Non-retryable: the conflict is in the table's configuration, so replaying re-fails. Capture+ cannot hash a merge key, and it must not write the masked column to the buffer in plaintext.+ """++ def __init__(self, table_name: str, columns: list[str]) -> None:+ super().__init__(f"Table {table_name} masks its primary-key columns {columns}")+ self.table_name = table_name+ self.columns = columns++# (friendly_message, retryable) per category. Messages are STATIC templates — they never# interpolate host/user/password or raw exception text, so stored copy can't leak credentials._CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {@@ -179,6 +201,16 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {"method for that table.",False,),+ CDCErrorCategory.MASKED_KEY_CONFLICT: (+ "A table masks a column that is its primary key, and a primary key can't be masked. "+ "Remove the column from the table's masked columns, then re-enable change data capture.",+ False,+ ),+ CDCErrorCategory.MASKING_KEY_UNAVAILABLE: (+ "A table masks columns, but this PostHog instance has no secret key to hash them with. "+ "Set WAREHOUSE_COLUMN_MASKING_KEY to a secret value, then re-enable change data capture.",+ False,+ ),CDCErrorCategory.UNKNOWN: ("Change data capture hit an unexpected error. PostHog will retry automatically — if it ""persists, check the source's sync logs or contact support.",@@ -217,6 +249,18 @@ def classify_cdc_error(exc: BaseException, adapter: CDCSourceAdapter | None) ->return cdc_error_info(CDCErrorCategory.TRANSACTION_TOO_LARGE)if isinstance(err, CDCReservedColumnError):return cdc_error_info(CDCErrorCategory.RESERVED_COLUMN_CONFLICT)+ if isinstance(err, CDCMaskingKeyError):+ return cdc_error_info(CDCErrorCategory.MASKING_KEY_UNAVAILABLE)+ if isinstance(err, CDCMaskedKeyError):+ # Every table on the source shows this error, so it names the one to fix. Names are not secrets.+ return dataclasses.replace(+ cdc_error_info(CDCErrorCategory.MASKED_KEY_CONFLICT),+ friendly_message=(+ f"Table {err.table_name} masks its primary-key columns {', '.join(err.columns)}, and a primary "+ "key can't be masked. Remove them from the table's masked columns, then re-enable change "+ "data capture."+ ),+ )if adapter is not None:info = adapter.classify_error(err)if info is not None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.pyindex e5d8c64f64fb..2f70945927b7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.py@@ -121,6 +121,7 @@ def hand_reset_to_capture(*,start_capture: bool = True,awaiting_slot: bool = False,+ stage: bool = True,) -> None:"""Pause a CDC table's schedule and mark its reset pending, so capture finishes it once no sync can hand over.@@ -130,6 +131,9 @@ def hand_reset_to_capture(`awaiting_slot` holds the staged reset back until a capture run has read the slot, as Repair CDCneeds while it recreates the slot.++ `stage=False` is for a caller that staged the reset in its own save. A second stage would start+ another generation, and capture would take a second snapshot."""# Deferred: data_load.service participates in the CDC schedule<->workflow import cycle.from products.data_warehouse.backend.facade.api import ( # noqa: PLC0415@@ -152,14 +156,15 @@ def hand_reset_to_capture(# Capture pauses the schedule again before it resets the table.logger.warning("cdc_reset_schedule_pause_failed", schema_id=str(schema.id), exc_info=True)- persisted = update_sync_type_config_keys(- schema.id, schema.team_id, mutate=partial(stage_handed_over_reset, awaiting_slot=awaiting_slot)- )- # Only this key in memory, so a caller that saves the schema afterwards keeps its own edits.- schema.sync_type_config = {- **(schema.sync_type_config or {}),- CDC_RESET_PENDING_KEY: persisted[CDC_RESET_PENDING_KEY],- }+ if stage:+ persisted = update_sync_type_config_keys(+ schema.id, schema.team_id, mutate=partial(stage_handed_over_reset, awaiting_slot=awaiting_slot)+ )+ # Only this key in memory, so a caller that saves the schema afterwards keeps its own edits.+ schema.sync_type_config = {+ **(schema.sync_type_config or {}),+ CDC_RESET_PENDING_KEY: persisted[CDC_RESET_PENDING_KEY],+ }# Capture finishes the reset, so start a run now instead of waiting for its schedule: the last# run may have looked for resets just before this write, and the schedule may be gone. A halted# source is left alone, because Repair CDC or a resumed capture restarts capture itself.diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.pyindex f711d760c24f..6c79dd2ee455 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py@@ -517,6 +517,9 @@ class ConformanceKit:if update_fields is None or "job_inputs" in update_fields:kit._source_job_inputs = copy.deepcopy(source.job_inputs)+ def read_masked_columns(_extraction: CDCExtractActivity, schema: ExternalDataSchema) -> list[str] | None:+ return kit._schema_row(schema.id).get("masked_columns")+def cdc_schemas(extraction: CDCExtractActivity) -> list[ExternalDataSchema]:assert extraction.source is not Nonereturn [kit._load_schema(name, extraction.source) for name in kit._schema_rows]@@ -547,6 +550,7 @@ class ConformanceKit:),patch(f"{_CDC}.activities.ExternalDataSource", source_model),patch.object(CDCExtractActivity, "_get_cdc_schemas", cdc_schemas),+ patch.object(CDCExtractActivity, "_read_masked_columns", read_masked_columns),patch(f"{_CDC}.activities.update_sync_type_config_keys", merge_config),patch.object(ExternalDataSchema, "save", save_schema),patch.object(ExternalDataSource, "save", save_source),diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.pyindex 2c9175f2ce63..abaefa881acd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py@@ -5,6 +5,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.errors importMAX_FRIENDLY_MESSAGE_LENGTH,CDCErrorCategory,CDCErrorInfo,+ CDCMaskedKeyError,+ CDCMaskingKeyError,CDCSlotNotConfiguredError,CDCTransactionTooLargeError,cdc_error_info,@@ -42,6 +44,7 @@ class TestCDCErrorInfo:(CDCErrorCategory.SLOT_IN_USE, True),(CDCErrorCategory.WAL_DECODE_ERROR, False),(CDCErrorCategory.TRANSACTION_TOO_LARGE, False),+ (CDCErrorCategory.MASKED_KEY_CONFLICT, False),(CDCErrorCategory.UNKNOWN, True),])@@ -82,11 +85,32 @@ class TestClassifyCDCError:info = classify_cdc_error(wrapper, PostgresCDCAdapter())assert info.category is CDCErrorCategory.AUTH_FAILED- def test_transaction_too_large_is_classified_without_adapter(self):- info = classify_cdc_error(CDCTransactionTooLargeError("500001 events"), None)- assert info.category is CDCErrorCategory.TRANSACTION_TOO_LARGE+ @parameterized.expand(+ [+ (+ "transaction_too_large",+ CDCTransactionTooLargeError("500001 events"),+ CDCErrorCategory.TRANSACTION_TOO_LARGE,+ ),+ ("masked_key", CDCMaskedKeyError("public.orders", ["id"]), CDCErrorCategory.MASKED_KEY_CONFLICT),+ (+ "masking_key_unavailable",+ CDCMaskingKeyError("WAREHOUSE_COLUMN_MASKING_KEY must be set"),+ CDCErrorCategory.MASKING_KEY_UNAVAILABLE,+ ),+ ]+ )+ def test_config_errors_are_classified_without_adapter(self, _name, exc, category):+ info = classify_cdc_error(exc, None)+ assert info.category is categoryassert info.retryable is False+ def test_masked_key_error_names_the_table_and_columns(self):+ # Capture fails every table on the source, so the message must say which one to fix.+ info = classify_cdc_error(CDCMaskedKeyError("public.orders", ["id", "tenant_id"]), None)+ assert "public.orders" in info.friendly_message+ assert "id, tenant_id" in info.friendly_message+def test_slot_not_configured_is_classified_without_adapter(self):# A CDC-enabled source with no slot name is a config dead end, not a psycopg failure, so it# must classify to a non-retryable category regardless of engine adapter.diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.pyindex ae0b7e550cd4..c09cde9d79c6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py@@ -1,9 +1,10 @@import uuid+import dataclassesfrom collections.abc import Iterable, Iterator, Mappingfrom contextlib import contextmanagerfrom datetime import UTC, datetime, timedeltafrom types import SimpleNamespace-from typing import Literal+from typing import Any, Literal, castimport pytestfrom unittest.mock import ANY, DEFAULT, MagicMock, patch@@ -41,6 +42,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import)from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import (CDCErrorCategory,+ CDCMaskedKeyError,+ CDCMaskingKeyError,CDCReservedColumnError,cdc_error_info,)@@ -50,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane)from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import has_queued_batchesfrom products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCLag, ChangeEvent+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import mask_valuefrom products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapterfrom products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.position import PgLSNfrom products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException@@ -151,6 +155,7 @@ def _stub_app_db_writes():"_update_schema_sync_type_config",side_effect=_fake_update_schema_sync_type_config,),+ patch.object(CDCExtractActivity, "_read_masked_columns", side_effect=lambda schema: schema.masked_columns),patch(f"{_ACTIVITIES}.cancel_running_sync", return_value=None),):yield@@ -319,6 +324,39 @@ def _make_extract_activity(source, log=None) -> CDCExtractActivity:return activity_obj+class TestMaskEventColumns:+ def test_masks_configured_columns_and_types_them_as_text(self):+ activity_obj = _make_extract_activity(_make_source())+ activity_obj._load_masked_columns(cast(Any, SimpleNamespace(name="users", masked_columns=["email", "Visits"])))+ event = dataclasses.replace(+ _make_event(columns={"id": 1, "Email": "[email protected]", "visits": None}),+ column_types={"id": pa.int64(), "Email": pa.string(), "visits": pa.int64()},+ )++ out = activity_obj._mask_event_columns(event)++ assert out.columns == {"id": 1, "Email": mask_value(1, "[email protected]"), "visits": None}+ assert out.column_types == {"id": pa.int64(), "Email": pa.string(), "visits": pa.string()}++ def test_a_table_without_masks_is_untouched(self):+ activity_obj = _make_extract_activity(_make_source())+ event = _make_event(columns={"id": 1, "email": "[email protected]"})++ assert activity_obj._mask_event_columns(event) is event++ def test_no_secret_key_fails_capture_before_any_change_is_read(self):+ activity_obj = _make_extract_activity(_make_source())++ with (+ patch(+ f"{_ACTIVITIES}.value_masker",+ side_effect=NonRetryableException("WAREHOUSE_COLUMN_MASKING_KEY must be set"),+ ),+ pytest.raises(CDCMaskingKeyError),+ ):+ activity_obj._load_masked_columns(cast(Any, SimpleNamespace(name="users", masked_columns=["email"])))++class TestSetupSelfCleansUnrunnableSchedules:@parameterized.expand([("no_adapter_for_the_type", "MySQL"), ("not_a_source_type_at_all", "UnsupportedDB")])@patch("products.warehouse_sources.backend.temporal.data_imports.cdc.activities.get_cdc_adapter")@@ -1429,6 +1467,26 @@ class TestPKColumnLoading:assert queried_namespaces == {"cdc_test", "public"}assert act.pk_columns_by_table == {"cdc_test.orders": ["id"], "public.orders": ["uuid"]}+ def test_a_mask_on_a_discovered_key_fails_capture(self):+ schema = _make_schema("cdc_test.orders")+ schema.masked_columns = ["ID"]+ schema.incremental_field = None+ act = self._activity([schema], queried_pks={"cdc_test": {"orders": ["id"]}})++ with pytest.raises(CDCMaskedKeyError):+ act._load_pk_columns()++ def test_a_mask_on_a_stored_cursor_column_still_masks_it(self):+ # CDC reads no cursor, so a stale incremental_field must not keep its column in plaintext.+ schema = _make_schema("cdc_test.orders")+ schema.masked_columns = ["Updated_At"]+ schema.incremental_field = "updated_at"+ act = self._activity([schema], queried_pks={"cdc_test": {"orders": ["id"]}})++ act._load_pk_columns()++ assert act.masked_columns_by_table == {"cdc_test.orders": {"updated_at"}}+def test_stored_keys_are_not_requeried(self):schema = _make_schema("cdc_test.orders")schema.sync_type_config["primary_key_columns"] = ["id"]@@ -2216,6 +2274,26 @@ class TestBufferedIngressCapture:assert schema.sync_type_config["cdc_reset_pending"] == staged+ def test_a_finished_reset_masks_with_the_masks_saved_with_it(self):+ # The run loads its tables before it finishes resets. A mask change saved after that load+ # stages a reset this run finishes, so the buffer the run then writes must carry the new mask.+ source = _make_source()+ schema = _make_schema("users", cdc_mode="streaming", source=source)+ schema.masked_columns = None+ schema.incremental_field = None+ schema.sync_type_config["cdc_reset_pending"] = {"trigger": True}+ events = [_make_event(op="I", position="0/100", columns={"id": 1, "email": "[email protected]"})]++ with (+ _capture_harness(source, [schema], events) as capture,+ patch.object(CDCExtractActivity, "_read_masked_columns", return_value=["email"]),+ patch("products.data_warehouse.backend.facade.api.trigger_external_data_workflow"),+ ):+ capture.extract()++ written = capture.buffer.write_batch.call_args.kwargs["table"]+ assert written.column("email").to_pylist() == [mask_value(1, "[email protected]")]+def test_a_reset_waiting_on_a_slot_holds_the_table_out_until_the_slot_reads(self):# Recovery leaves this marker when it could not recreate the slot. Finishing the reset here# would unpause the schedule, and the snapshot would start with no slot to resume from. Adiff --git a/products/warehouse_sources/backend/temporal/data_imports/naming_convention.py b/products/warehouse_sources/backend/temporal/data_imports/naming_convention.pyindex bda5f1bd6af8..0640d9df5459 100644--- a/products/warehouse_sources/backend/temporal/data_imports/naming_convention.py+++ b/products/warehouse_sources/backend/temporal/data_imports/naming_convention.py@@ -120,3 +120,11 @@ def _trim_and_tag(identifier: str, tag: str, max_length: int) -> str:)assert len(trimmed) == max_lengthreturn trimmed+++def fold_column_name(name: str) -> str:+ """Fold a source column name the way the pipeline normalizes it, so `ID` matches `id`."""+ try:+ return NamingConvention.normalize_identifier(name)+ except ValueError:+ return namediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.pyindex 432b966098f7..a0440236cb47 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py@@ -24,7 +24,10 @@ from structlog.types import FilteringBoundLoggerfrom posthog.temporal.common.errors import NonReportableError-from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import (+ NamingConvention,+ fold_column_name,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEYfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -317,7 +320,7 @@ def evolve_pyarrow_schema(# Second pass: align with existing Delta table schema.delta_arrow_schema = pyarrow_schema_from_arrow_exportable(delta_schema)- merge_keys = {_fold_column_name_for_match(name) for name in merge_key_columns or []}+ merge_keys = {fold_column_name(name) for name in merge_key_columns or []}for delta_field in delta_arrow_schema:if delta_field.name not in incoming_table.schema.names:new_column_data = (@@ -390,7 +393,7 @@ def evolve_pyarrow_schema(incoming_table.schema.get_field_index(delta_field.name), delta_field.name, parsed_timestamps)elif (- _fold_column_name_for_match(delta_field.name) in merge_keys+ fold_column_name(delta_field.name) in merge_keysand (pa.types.is_binary(delta_field.type) or pa.types.is_large_binary(delta_field.type))and (pa.types.is_string(incoming_column.type) or pa.types.is_large_string(incoming_column.type))):@@ -510,13 +513,6 @@ def drop_internal_columns(table: pa.Table) -> pa.Table:return table.drop_columns(internal) if internal else table-def _fold_column_name_for_match(name: str) -> str:- try:- return NamingConvention.normalize_identifier(name)- except ValueError:- return name--def apply_enabled_columns_projection(table: pa.Table,enabled_columns: list[str] | None,@@ -541,18 +537,16 @@ def apply_enabled_columns_projection(if enabled_columns is None:return table, []- retained = {_fold_column_name_for_match(name) for name in enabled_columns}+ retained = {fold_column_name(name) for name in enabled_columns}for primary_key in primary_keys or []:- retained.add(_fold_column_name_for_match(primary_key))+ retained.add(fold_column_name(primary_key))if incremental_field:- retained.add(_fold_column_name_for_match(incremental_field))+ retained.add(fold_column_name(incremental_field))for partition_key in partition_keys or []:- retained.add(_fold_column_name_for_match(partition_key))+ retained.add(fold_column_name(partition_key))kept_names = [- name- for name in table.column_names- if name in INTERNAL_COLUMN_NAMES or _fold_column_name_for_match(name) in retained+ name for name in table.column_names if name in INTERNAL_COLUMN_NAMES or fold_column_name(name) in retained]dropped_names = [name for name in table.column_names if name not in set(kept_names)]if not dropped_names:@@ -833,12 +827,15 @@ def table_from_iterator(*,primary_keys: Optional[Sequence[str]] = None,binary_reporter: Optional[BinaryColumnReporter] = None,+ masked_columns: Optional[Sequence[str]] = None,) -> pa.Table:batch = list(data_iterator)if not batch:return pa.Table.from_pylist([])- processed_batch = _process_batch(batch, schema, primary_keys=primary_keys, binary_reporter=binary_reporter)+ processed_batch = _process_batch(+ batch, schema, primary_keys=primary_keys, binary_reporter=binary_reporter, masked_columns=masked_columns+ )return processed_batch@@ -849,13 +846,18 @@ def table_from_py_list(*,primary_keys: Optional[Sequence[str]] = None,binary_reporter: Optional[BinaryColumnReporter] = None,+ masked_columns: Optional[Sequence[str]] = None,) -> pa.Table:"""Convert a list of Python dictionaries to a PyArrow Table.This is a wrapper around table_from_iterator for backward compatibility."""return table_from_iterator(- iter(table_data), schema=schema, primary_keys=primary_keys, binary_reporter=binary_reporter+ iter(table_data),+ schema=schema,+ primary_keys=primary_keys,+ binary_reporter=binary_reporter,+ masked_columns=masked_columns,)@@ -1449,6 +1451,7 @@ def _process_batch(*,primary_keys: Optional[Sequence[str]] = None,binary_reporter: Optional[BinaryColumnReporter] = None,+ masked_columns: Optional[Sequence[str]] = None,) -> pa.Table:# Every step below reads a row as a mapping, so without this check a keyless row fails deep in# the conversion with a bare `'list' object has no attribute 'items'`, which names neither the@@ -1458,6 +1461,7 @@ def _process_batch(raise NonMappingRowError(type(row).__name__)table_data, serialized_dict_columns = _serialize_dict_columns(table_data)+ masked = {fold_column_name(column) for column in masked_columns or []}# Support both given schemas and inferred schemasif schema is None or len(schema.names) == 0:@@ -1673,11 +1677,15 @@ def _process_batch(# not str" phrase so source non-retryable matching still fires, but name the# column and show examples so the offending cells can be found in the source.offenders = [x for x in all_values if isinstance(x, str)]- sample = [(x if x != "" else "<blank>") for x in offenders[:5]]+ # A masked column's cells must not reach the sync error, which project members can read.+ if fold_column_name(str(field_name)) in masked:+ examples = ""+ else:+ examples = f" such as {[(x if x != '' else '<blank>') for x in offenders[:5]]}"raise TypeError(f"must be real number, not str: column '{field_name}' is imported as a "- f"number but contains {len(offenders)} non-numeric value(s) such as "- f"{sample}. Ensure every cell in this column is a plain number (or empty)."+ f"number but contains {len(offenders)} non-numeric value(s){examples}. "+ "Ensure every cell in this column is a plain number (or empty).") from eif arrow_schema and pa.types.is_decimal(arrow_schema.field(field_index).type):diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.pyindex 6291c33eb414..377bd88c27b8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.py@@ -109,9 +109,11 @@ class Batcher:coalesce_tables: bool = False,primary_keys: Optional[list[str]] = None,max_buffer_cells: Optional[int] = None,+ masked_columns: Optional[list[str]] = None,) -> None:self._logger = loggerself._primary_keys = primary_keys+ self._masked_columns = masked_columnsself._binary_reporter = BinaryColumnReporter(logger)self._chunk_size = chunk_size or DEFAULT_CHUNK_SIZE@@ -142,7 +144,12 @@ class Batcher:self._ready_bytes = 0def _rows_to_table(self, rows: list[Any]) -> pa.Table:- return table_from_py_list(rows, primary_keys=self._primary_keys, binary_reporter=self._binary_reporter)+ return table_from_py_list(+ rows,+ primary_keys=self._primary_keys,+ binary_reporter=self._binary_reporter,+ masked_columns=self._masked_columns,+ )def _set_ready(self, table: pa.Table) -> None:"""Split `table` so no yielded chunk overflows a 32-bit offset column or exceedsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/masking.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/masking.pynew file mode 100644index 000000000000..4adbb5c8bab7--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/masking.py@@ -0,0 +1,195 @@+"""Deterministic one-way masking of sensitive warehouse-source columns.++A masked value is replaced with an HMAC-SHA256 digest keyed by the server secret+(`WAREHOUSE_COLUMN_MASKING_KEY`) over `team_id` and the value. The secret stops anyone who can query the+column from brute-forcing low-entropy values such as card numbers. `team_id` makes equal values+mask alike within a team, so they still join, and differently across teams.++The value is hashed as `str(value)`. A value that two read paths render differently digests+differently. On a CDC table the snapshot reads Python values while the change stream keeps the+Postgres text for numeric, date and time, JSON, bytea and array columns. A masked column of those+types can therefore hold two digests for one value after a row changes. Text, integer, boolean,+float8 and UUID columns render alike on both paths.++The key is its own secret, so rotating the encryption keys of other fields never changes a digest.+Changing it changes every later digest, while rows already synced keep the old one. Every table with+masked columns therefore needs a full resync after a change.++Masking does not reach data derived before the mask was set. A materialized view built earlier+keeps its plaintext copy until it is materialized again.++A webhook delivery lands in S3 as a raw parquet file before any sync reads it, so a masked column+stays there in plaintext until the next sync hashes the batch and deletes the file.+"""++from __future__ import annotations++import hmac+import asyncio+import hashlib+from collections.abc import Callable, Iterable+from typing import TYPE_CHECKING++from django.conf import settings++import pyarrow as pa++from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name+from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException++if TYPE_CHECKING:+ from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse+++# Anyone can read this key, so a digest keyed with it can be brute-forced. Development and tests only.+_DEVELOPMENT_KEY = "development-only-warehouse-column-masking-key"+_MIN_KEY_LENGTH = 32++MASKING_KEY_UNAVAILABLE_MESSAGE = (+ f"WAREHOUSE_COLUMN_MASKING_KEY must be set to a secret of at least {_MIN_KEY_LENGTH} characters to mask columns."+)+++def masking_key_unavailable() -> bool:+ """Whether this instance has no usable masking key. Development and tests fall back to a fixed one."""+ if settings.DEBUG or settings.TEST:+ return False+ return len(settings.WAREHOUSE_COLUMN_MASKING_KEY) < _MIN_KEY_LENGTH+++def get_masking_key() -> bytes:+ if masking_key_unavailable():+ raise NonRetryableException(MASKING_KEY_UNAVAILABLE_MESSAGE)+ return (settings.WAREHOUSE_COLUMN_MASKING_KEY or _DEVELOPMENT_KEY).encode()+++def _mask_base(team_id: int, key: bytes) -> hmac.HMAC:+ # Callers copy this per value, which skips the key schedule and the shared prefix for each cell.+ return hmac.new(key, f"{team_id}:".encode(), hashlib.sha256)+++def value_masker(team_id: int) -> Callable[[object], str | None]:+ """`mask_value` for one team, with the key resolved once. Raises when no secret key is set."""+ base = _mask_base(team_id, get_masking_key())++ def mask(value: object) -> str | None:+ if value is None:+ return None+ digest = base.copy()+ digest.update(str(value).encode())+ return digest.hexdigest()++ return mask+++def mask_value(team_id: int, value: object, *, key: bytes | None = None) -> str | None:+ """Digest of one value. Null stays null, so a masked column keeps its nullability.++ Pass `key` to reuse one `get_masking_key()` across many values.+ """+ if value is None:+ return None+ digest = _mask_base(team_id, key if key is not None else get_masking_key()).copy()+ digest.update(str(value).encode())+ return digest.hexdigest()+++def ambiguous_masked_columns(masked_columns: list[str], source_columns: Iterable[str]) -> list[str]:+ """Masked entries whose folded name is shared by two or more source columns.++ The pipeline renames the later of two colliding columns (`email` and `Email`), but a mask is+ matched by folded name and cannot say which one it means. Callers reject these, because the+ mask would otherwise land on the wrong column and leave the intended one in plaintext.+ """+ counts: dict[str, int] = {}+ for name in source_columns:+ folded = fold_column_name(name)+ counts[folded] = counts.get(folded, 0) + 1+ return [column for column in masked_columns if counts.get(fold_column_name(column), 0) > 1]+++def collision_alias_masked_columns(masked_columns: list[str], synced_columns: Iterable[str]) -> list[str]:+ """Masked entries that name a synced column the pipeline renamed to avoid a name collision.++ The pipeline prefixes `_` to the later of two colliding columns (`Email` and `email` load as `email`+ and `_email`). Which source column gets the alias depends on the batch, and a batch with only one+ of them loads it under the plain name, so a mask on the alias would leave that value in plaintext.+ """+ folded = {fold_column_name(name) for name in synced_columns}+ return [+ column+ for column in masked_columns+ if column.startswith("_") and fold_column_name(column[1:]) in folded and fold_column_name(column) in folded+ ]+++def protected_masked_columns(masked_columns: list[str] | None, protected_columns: Iterable[str | None]) -> list[str]:+ """Folded masked names that are also a key, partition or cursor column. Callers fail on any.++ A hashed merge key breaks merges and a hashed cursor is not orderable. Skipping the mask instead+ would sync the column in plaintext.+ """+ protected = {fold_column_name(column) for column in protected_columns if column}+ return sorted({fold_column_name(column) for column in masked_columns or []} & protected)+++def mask_table_columns(table: pa.Table, masked_columns: list[str] | None, *, team_id: int) -> pa.Table:+ """Replace each masked column with string digests. Names match by their folded form, normalized or not."""+ masked = {fold_column_name(column) for column in masked_columns or []}+ if not masked:+ return table+ mask = value_masker(team_id)+ for index, name in enumerate(table.column_names):+ if fold_column_name(name) not in masked:+ continue+ # One chunk at a time, so the Python objects in memory stay bounded by the chunk size.+ masked_chunks: list[pa.Array] = []+ for chunk in table.column(index).chunks:+ masked_chunks.append(pa.array([mask(value) for value in chunk.to_pylist()], type=pa.string()))+ table = table.set_column(index, name, pa.chunked_array(masked_chunks, type=pa.string()))+ return table+++def raise_if_masked_key_columns(schema: ExternalDataSchema, resource: SourceResponse) -> None:+ """Fail the sync when a masked column is a key, partition or incremental column.++ The API rejects a mask on a key it knows, but some sources resolve their merge, partition or+ row-replacement key only at sync time. The loader reads those columns as source values, and+ skipping the mask syncs plaintext, so fail the sync. A retry reads the same configuration.+ """+ if not schema.masked_columns:+ return+ key_columns = [+ *(resource.primary_keys or []),+ *(resource.partition_keys or []),+ *(schema.partitioning_keys or []),+ *(schema.partitioning_keys_override or []),+ *([resource.replace_scope.column] if resource.replace_scope else []),+ ]+ # A full-refresh table keeps the cursor of an earlier sync method in its settings, but reads no cursor.+ cursor = schema.incremental_field if schema.should_use_incremental_field or schema.is_cdc else None+ protected = protected_masked_columns(schema.masked_columns, [*key_columns, cursor])+ if protected:+ raise NonRetryableException(+ f"Cannot mask key, partition or incremental columns {protected}. "+ "Remove them from masked_columns to sync this table."+ )+++async def mask_table_if_configured(+ table: pa.Table, schema: ExternalDataSchema, resource: SourceResponse, source_column_names: Iterable[str]+) -> pa.Table:+ """Mask a batch. `source_column_names` are the batch's names before any normalization."""+ if not schema.masked_columns:+ return table+ # A source column added after the mask was saved can fold to a masked name. Normalization then+ # renames one of the two, and the mask could hash the wrong one and leave the other in plaintext.+ ambiguous = ambiguous_masked_columns(schema.masked_columns, source_column_names)+ if ambiguous:+ raise NonRetryableException(+ f"Cannot mask {sorted(ambiguous)}: two source columns have this name after normalization. "+ "Rename one of the source columns to sync this table."+ )+ raise_if_masked_key_columns(schema, resource)+ return await asyncio.to_thread(mask_table_columns, table, schema.masked_columns, team_id=schema.team_id)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_masking.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_masking.pynew file mode 100644index 000000000000..3249afe65c41--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_masking.py@@ -0,0 +1,161 @@+from types import SimpleNamespace+from typing import Any, cast++import pytest++from django.test import override_settings++import pyarrow as pa+from parameterized import parameterized++from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import (+ ambiguous_masked_columns,+ mask_table_columns,+ mask_table_if_configured,+ mask_value,+ protected_masked_columns,+)+from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException+++class TestMaskValue:+ @parameterized.expand(+ [+ ("same_team_same_value", (1, "[email protected]"), (1, "[email protected]"), True),+ ("other_team", (1, "[email protected]"), (2, "[email protected]"), False),+ ("other_value", (1, "[email protected]"), (1, "[email protected]"), False),+ ]+ )+ def test_digest_equality(self, _name: str, left: tuple[int, str], right: tuple[int, str], equal: bool) -> None:+ assert (mask_value(*left) == mask_value(*right)) is equal++ def test_null_stays_null(self) -> None:+ assert mask_value(1, None) is None++ def test_digest_depends_on_the_server_secret(self) -> None:+ with override_settings(WAREHOUSE_COLUMN_MASKING_KEY="a" * 32):+ first = mask_value(1, "secret")+ with override_settings(WAREHOUSE_COLUMN_MASKING_KEY="b" * 32):+ second = mask_value(1, "secret")+ assert first != second++ @parameterized.expand(+ [+ ("no_key", "", False, False),+ ("short_key", "k" * 31, False, False),+ ("secret_key", "k" * 32, False, True),+ ("no_key_in_development", "", True, True),+ ]+ )+ def test_key_checks(self, _name: str, key: str, debug: bool, masks: bool) -> None:+ with override_settings(WAREHOUSE_COLUMN_MASKING_KEY=key, DEBUG=debug, TEST=False):+ if masks:+ assert mask_value(1, "secret") is not None+ else:+ with pytest.raises(NonRetryableException):+ mask_value(1, "secret")+++class TestProtectedMaskedColumns:+ @parameterized.expand(+ [+ ("key_and_cursor", ["a", "b", "c"], ["a", "b", None], ["a", "b"]),+ ("folds_case_before_matching", ["ID", "Email"], ["id"], ["id"]),+ ("unset", None, ["id"], []),+ ]+ )+ def test_protected(+ self, _name: str, masked: list[str] | None, protected: list[str | None], expected: list[str]+ ) -> None:+ assert protected_masked_columns(masked, protected) == expected+++class TestAmbiguousMaskedColumns:+ @parameterized.expand(+ [+ ("collision", ["Email"], ["email", "Email", "id"], ["Email"]),+ ("no_collision", ["email"], ["email", "id", "name"], []),+ ]+ )+ def test_ambiguous(self, _name: str, masked: list[str], source_columns: list[str], expected: list[str]) -> None:+ assert ambiguous_masked_columns(masked, source_columns) == expected+++class TestMaskTableColumns:+ def _table(self) -> pa.Table:+ return pa.table({"id": [1, 2], "email": ["[email protected]", None], "visits": [7, 8]})++ def test_masks_listed_columns_as_strings_and_leaves_the_rest(self) -> None:+ # "Visits" is the source casing of a column the pipeline already normalized.+ table = mask_table_columns(self._table(), ["email", "Visits"], team_id=3)++ assert table.column("email").to_pylist() == [mask_value(3, "[email protected]"), None]+ assert table.column("visits").to_pylist() == [mask_value(3, 7), mask_value(3, 8)]+ assert table.schema.field("visits").type == pa.string()+ assert table.column("id").to_pylist() == [1, 2]+++def _schema(masked: list[str], **fields: Any) -> Any:+ return SimpleNamespace(+ masked_columns=masked,+ team_id=3,+ **{+ "incremental_field": None,+ "should_use_incremental_field": False,+ "is_cdc": False,+ "partitioning_keys": None,+ "partitioning_keys_override": None,+ **fields,+ },+ )+++_RESOURCE = cast(+ Any,+ SimpleNamespace(+ primary_keys=["id"],+ partition_keys=["created_at"],+ replace_scope=SimpleNamespace(column="_file_name"),+ ),+)+++class TestMaskTableIfConfigured:+ @pytest.mark.parametrize(+ "masked,source_column_names,schema_fields",+ [+ pytest.param(["id"], ["id", "email"], {}, id="mask_on_a_key_resolved_at_sync_time"),+ pytest.param(["created_at"], ["id", "created_at"], {}, id="mask_on_a_source_partition_key"),+ pytest.param(["_file_name"], ["id", "_file_name"], {}, id="mask_on_the_row_replacement_column"),+ pytest.param(["Email"], ["id", "Email", "email"], {}, id="source_column_added_with_a_colliding_name"),+ pytest.param(+ ["updated_at"],+ ["id", "updated_at"],+ {"incremental_field": "updated_at", "should_use_incremental_field": True},+ id="mask_on_the_cursor",+ ),+ pytest.param(["region"], ["id", "region"], {"partitioning_keys": ["region"]}, id="mask_on_a_partition_key"),+ pytest.param(+ ["region"],+ ["id", "region"],+ {"partitioning_keys_override": ["region"]},+ id="mask_on_a_partition_key_override",+ ),+ ],+ )+ async def test_fails_the_sync(+ self, masked: list[str], source_column_names: list[str], schema_fields: dict[str, Any]+ ) -> None:+ with pytest.raises(NonRetryableException):+ await mask_table_if_configured(+ pa.table({"id": [1]}), _schema(masked, **schema_fields), _RESOURCE, source_column_names+ )++ async def test_a_full_refresh_table_masks_the_cursor_of_its_earlier_sync_method(self) -> None:+ schema = _schema(["updated_at"], incremental_field="updated_at")++ masked = await mask_table_if_configured(+ pa.table({"id": [1], "updated_at": ["2026-01-01"]}), schema, _RESOURCE, ["id", "updated_at"]+ )++ assert masked.column("updated_at").to_pylist() == [mask_value(3, "2026-01-01")]diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.pyindex 71ff70109440..6eba343da461 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.py@@ -116,17 +116,26 @@ def test_table_from_py_list_keeps_the_key_order_of_the_rows():assert table.column_names == ["zeta", "alpha", "mid", "beta", "omega", "gamma", "delta", "late"]-def test_table_from_py_list_numeric_column_with_non_numeric_value_raises_named_error():+ "masked_columns,shows_cells",+ [+ pytest.param(None, True, id="unmasked"),+ pytest.param(["Revenue"], False, id="masked_column_hides_its_cells"),+ ],+)+def test_table_from_py_list_numeric_column_with_non_numeric_value_raises_named_error(+ masked_columns: list[str] | None, shows_cells: bool+):with pytest.raises(TypeError) as exc_info:- table_from_py_list([{"revenue": 1.5}, {"revenue": "N/A"}, {"revenue": ""}])+ table_from_py_list([{"revenue": 1.5}, {"revenue": "N/A"}, {"revenue": ""}], masked_columns=masked_columns)message = str(exc_info.value)# Preserves the original phrase so source non-retryable matching still firesassert "must be real number, not str" in message- # Names the column and shows the offending text and blank cells+ # Names the column, and shows the offending text and blank cells unless the column is maskedassert "revenue" in message- assert "N/A" in message- assert "<blank>" in message+ assert ("N/A" in message) is shows_cells+ assert ("<blank>" in message) is shows_cells# The message must stay matched by an Any_Source_Errors entry so the schema is paused with# guidance instead of retrying the same non-numeric cells forever, on every source.assert [key for key in Any_Source_Errors if key in message]diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex 107e55e286a4..292b6d9ca1bf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -94,6 +94,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenancefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableReffrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchema+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import (+ mask_table_if_configured,+ raise_if_masked_key_columns,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (append_cursor_partition_keys,)@@ -417,6 +421,7 @@ class PipelineV3(Generic[ResumableData]):resume_manager=self._resumable_source_manager, is_webhook=self._schema.is_webhook),primary_keys=self._resource.primary_keys,+ masked_columns=self._schema.masked_columns,)self._internal_schema = HogQLSchema()self._sinks = build_pipeline_sinks(@@ -886,6 +891,10 @@ class PipelineV3(Generic[ResumableData]):self._resource,is_first_sync=self._table is None or self._reset_pipeline,)+ # Before the reset below deletes the table: a full-refresh table stores no key, so the API+ # can accept a mask on the key that only this run resolves.+ if self._resource.cdc_write_mode is None:+ raise_if_masked_key_columns(self._schema, self._resource)await persist_primary_keys(self._schema, self._resource, self._is_incremental, self._logger)@@ -1174,6 +1183,7 @@ class PipelineV3(Generic[ResumableData]):async def _process_batch(self, pa_table: pa.Table, batch_index: int, row_count: int) -> None:pa_table = _append_debug_column_to_pyarrows_table(drop_internal_columns(pa_table), self._load_id)+ source_column_names = pa_table.column_namesif not self._resource.preserve_column_names:pa_table = normalize_table_column_names(pa_table)@@ -1199,6 +1209,10 @@ class PipelineV3(Generic[ResumableData]):pa_table = evolve_pyarrow_schema(pa_table, None)pa_table = _handle_null_columns_with_definitions(pa_table, self._resource)+ # After the null-column backfill, so a masked column the batch lacked is typed as text too.+ # Capture masks a change before it writes the change buffer, so a buffered batch is masked already.+ if self._resource.cdc_write_mode is None:+ pa_table = await mask_table_if_configured(pa_table, self._schema, self._resource, source_column_names)# Converge this batch onto the column types earlier batches in this run already used,# and backfill columns they had that this one doesn't. The cursor column is named bothdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex 3cbda075a725..bc051bba1ec8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -23,6 +23,7 @@ from posthog.temporal.common.shutdown import ShutdownMonitor, WorkerShuttingDownfrom products.warehouse_sources.backend.models import external_data_schema as schema_modelsfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import CONSOLIDATED_WRITE_MODEfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import (finalize_desc_sort_incremental_value,handle_reset_or_full_refresh,@@ -45,6 +46,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arr)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcherfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import mask_valuefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (append_partition_key_to_table,)@@ -79,6 +81,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typSourceInputs,SourceResponse,)+from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableExceptionfrom products.warehouse_sources.backend.temporal.data_imports.workflow_activities.import_data_sync import (ImportJobModels,)@@ -120,6 +123,7 @@ def _make_pipeline() -> PipelineV3:is_webhook=False,is_append=False,table=None,+ masked_columns=None,)pipeline._table = Nonepipeline._logger = _make_logger()@@ -262,6 +266,32 @@ class TestAttemptScopedRunUuid:mock_reset.assert_not_called()+ @pytest.mark.asyncio+ async def test_a_mask_on_a_key_resolved_at_sync_time_fails_before_the_reset(self) -> None:+ pipeline = _make_pipeline()+ pipeline._reset_pipeline = True+ cast(MagicMock, pipeline._schema).configure_mock(+ masked_columns=["order_uuid"],+ incremental_field=None,+ partitioning_keys=None,+ partitioning_keys_override=None,+ )+ pipeline._resource = MagicMock(+ primary_keys=["order_uuid"], partition_keys=None, replace_scope=None, cdc_write_mode=None+ )++ with (+ patch(f"{_PIPELINE}.reset_rows_synced_if_needed", new_callable=AsyncMock),+ patch(f"{_PIPELINE}.validate_incremental_sync"),+ patch(f"{_PIPELINE}.handle_reset_or_full_refresh", new_callable=AsyncMock) as mock_reset,+ patch(f"{_PIPELINE}.activity") as mock_activity,+ ):+ mock_activity.in_activity.return_value = False+ with pytest.raises(NonRetryableException, match="order_uuid"):+ await pipeline.run()++ mock_reset.assert_not_called()+class TestTableRebuildAcrossAttempts:@pytest.mark.parametrize(@@ -1369,6 +1399,53 @@ class TestSingleTableRunIsUntouched:pipeline._s3_batch_writer.write_batch.assert_called_once()pipeline._pg_producer.hold_batch.assert_called_once()+ @pytest.mark.parametrize(+ "cdc_write_mode,expected",+ [+ pytest.param(None, [mask_value(1, "[email protected]")], id="source_rows"),+ # Capture masked the change before it wrote the buffer, and a second hash would differ from the snapshot's.+ pytest.param(CONSOLIDATED_WRITE_MODE, ["already-masked"], id="buffered_change_rows"),+ ],+ )+ async def test_masked_columns_are_hashed_once_before_anything_is_staged(+ self, cdc_write_mode: str | None, expected: list[str]+ ) -> None:+ pipeline = _make_pipeline()+ cast(MagicMock, pipeline._schema).configure_mock(+ incremental_field=None,+ enabled_columns=None,+ masked_columns=["email"],+ team_id=1,+ partitioning_keys=None,+ partitioning_keys_override=None,+ )+ pipeline._resource = MagicMock(+ primary_keys=["id"],+ column_hints=None,+ cdc_write_mode=cdc_write_mode,+ partition_keys=None,+ replace_scope=None,+ preserve_column_names=False,+ )+ pipeline._last_incremental_field_value = None+ pipeline._earliest_incremental_field_value = None+ module = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline"+ with (+ patch.object(pipeline, "_stage_batch", AsyncMock(return_value=1)) as stage_batch,+ patch.object(pipeline, "_advance_handoff_checkpoint", AsyncMock()),+ patch(+ f"{module}.update_incremental_field_values",+ AsyncMock(return_value=MagicMock(last_value=None, earliest_value=None)),+ ),+ patch(f"{module}.update_row_tracking_after_batch", AsyncMock()),+ ):+ email = "[email protected]" if cdc_write_mode is None else "already-masked"+ await pipeline._process_batch(pa.table({"id": [1], "email": [email]}), 0, 1)++ assert stage_batch.call_args.args[0].column("email").to_pylist() == expected+ sink_table = cast(MagicMock, pipeline._sinks.stage_chunk).call_args.args[1]+ assert sink_table.column("email").to_pylist() == expected+def test_the_activity_runs_the_base_class_for_a_source_without_lanes(self) -> None:# What keeps every other source off the subclass. `lanes=None` is what a source that# never heard of lanes carries, and `[]` what a lane build that found nothing returns.@@ -1779,6 +1856,7 @@ def _runnable_pipeline(manager: ResumableSourceManager[_Cursor], items) -> Pipelis_append=False,should_use_incremental_field=False,table=None,+ masked_columns=None,)pipeline._process_batch = AsyncMock() # type: ignore[method-assign]return pipeline@@ -2360,6 +2438,7 @@ class TestFinalMarkerIsTheLastDataRow:is_append=False,should_use_incremental_field=False,table=None,+ masked_columns=None,)pipeline._s3_batch_writer = MagicMock(write_batch=MagicMock(diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.pyindex fe1050499f88..4ec8cb4eb3f5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py@@ -43,11 +43,15 @@ from posthog.temporal.common.db_errors import is_transient_db_errorfrom posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeater-from products.warehouse_sources.backend.models.column_statistics import WarehouseColumnStatistics+from products.warehouse_sources.backend.models.column_statistics import (+ WarehouseColumnStatistics,+ clear_masked_column_bounds,+)from products.warehouse_sources.backend.models.external_data_job import ExternalDataJobfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.models.util import clean_type+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_namefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry import (retry_on_operational_error,)@@ -657,8 +661,7 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[stbasis = "full"try:- for column_name, stat in stats_by_column.items():- _upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)+ _write_statistics(team, table, schema.id, row_count, stats_by_column, delta_version, basis)except (OperationalError, InterfaceError):# The Delta-log read above can run long enough for the Postgres connection opened by the# earlier metadata queries (team/schema/existing-stats) to go stale — server restart, proxy@@ -668,8 +671,7 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[st# Delta-log read just to redo a handful of upserts.if not settings.TEST:close_old_connections()- for column_name, stat in stats_by_column.items():- _upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)+ _write_statistics(team, table, schema.id, row_count, stats_by_column, delta_version, basis)log.info("warehouse_statistics.done",@@ -774,6 +776,35 @@ def _fold_since_stored_version(return None+def _write_statistics(+ team: Team,+ table: DataWarehouseTable,+ schema_id: uuid.UUID,+ row_count: int,+ stats_by_column: dict[str, _ColumnStat],+ delta_version: int,+ basis: str,+) -> None:+ """Persist every column's stats, with no bounds for a column masked at write time."""++ def masked_columns() -> list[str] | None:+ return (+ ExternalDataSchema.objects.filter(team_id=team.id, id=schema_id)+ .values_list("masked_columns", flat=True)+ .first()+ )++ masked = {fold_column_name(column) for column in masked_columns() or []}+ for column_name, stat in stats_by_column.items():+ if fold_column_name(column_name) in masked:+ # A masked column's bounds are plaintext until its rebuild lands, and digests after it.+ stat = dataclasses.replace(stat, min_value=None, max_value=None, has_min_max=False)+ _upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)+ # A mask saved during the writes clears the bounds after its commit. This second read covers a mask+ # saved before that point, so no lock on the schema row is needed.+ clear_masked_column_bounds(team.id, table.id, masked_columns())++def _upsert_statistics(team: Team,table: DataWarehouseTable,diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.pyindex 12e701e74d10..89509a3ac378 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py@@ -773,6 +773,49 @@ class TestComputeTableStatisticsSync:assert stat.computed_for_delta_version == 12assert stat.column_type == "Int64"+ @pytest.mark.parametrize(+ "saved_during",+ [+ pytest.param("_aggregate_add_action_stats", id="mask_saved_during_the_read"),+ pytest.param("_upsert_statistics", id="mask_saved_during_the_writes"),+ ],+ )+ def test_a_masked_column_keeps_its_counts_but_stores_no_bounds(self, saved_during: str) -> None:+ team = self._team()+ schema, table, _ = self._schema_table_job(+ team, columns={"amount": {"clickhouse": "Nullable(Int64)"}, "email": {"clickhouse": "Nullable(String)"}}+ )+ add_actions = pa.table(+ {+ "num_records": [10],+ "null_count.amount": [0],+ "min.amount": [2],+ "max.amount": [9],+ "null_count.email": [1],+ "min.email": ["[email protected]"],+ "max.email": ["[email protected]"],+ }+ )+ original = getattr(comp, saved_during)++ def save_mask_then_continue(*args: Any) -> Any:+ # The profiler loaded the schema before this save, so it must read the masks again.+ ExternalDataSchema.objects.filter(id=schema.id).update(masked_columns=["Email"])+ return original(*args)++ with (+ patch(DELTA_HELPER_PATH, return_value=self._mock_delta(add_actions)),+ patch.object(comp, saved_during, save_mask_then_continue),+ ):+ compute_table_statistics_sync(team.id, schema.id)++ stats = {+ s.column_name: s for s in WarehouseColumnStatistics.objects.for_team(team.id).filter(table_id=table.id)+ }+ assert (stats["email"].null_count, stats["email"].min_value, stats["email"].max_value) == (1, None, None)+ assert stats["email"].has_min_max is False+ assert (stats["amount"].min_value, stats["amount"].max_value) == ("2", "9")+def test_recovers_from_stale_connection_during_write(self) -> None:# The Delta-log read can run long enough for the pooled connection opened by the earlier# metadata queries to go stale before the write loop runs, raising OperationalError on thediff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex c998f87ea99b..0157afbcfb82 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -41,8 +41,10 @@ from products.data_warehouse.backend.facade.api import (from products.data_warehouse.backend.facade.contracts import WebhookHogFunctionCreateResultfrom products.warehouse_sources.backend.facade.models import (DataWarehouseTable,+ ExternalDataJob,ExternalDataSchema,ExternalDataSource,+ WarehouseColumnStatistics,update_sync_type_config_keys,)from products.warehouse_sources.backend.facade.types import ExternalDataSourceType@@ -51,6 +53,7 @@ from products.warehouse_sources.backend.presentation.views.external_data_schemaUNREADABLE_SOURCE_CONFIG_ERROR,ExternalDataSchemaSerializer,ExternalDataSchemaViewset,+ masked_columns_error,schema_display_status,)from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (@@ -5213,23 +5216,28 @@ class TestExternalDataSchemaRowFilters(APIBaseTest):@parameterized.expand([- ("enabled_columns", ["id", "name"]),- ("row_filters", [{"column": "id", "operator": ">", "value": 10}]),+ ("enabled_columns", ["id", "name"], "enabled_columns"),+ ("row_filters", [{"column": "id", "operator": ">", "value": 10}], "row_filters"),+ # Column names only: the log must never carry a masked value or digest.+ ("masked_columns", ["name"], "masked columns"),])- def test_selection_change_is_written_to_the_activity_log(self, field: str, value: Any):+ def test_selection_change_is_written_to_the_activity_log(self, field: str, value: Any, logged_field: str):schema = self._create()- response = self.client.patch(- f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}", data={field: value}- )+ with mock.patch(+ "products.warehouse_sources.backend.presentation.views.external_data_schema.trigger_external_data_workflow"+ ):+ response = self.client.patch(+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}", data={field: value}+ )assert response.status_code == 200, response.json()log = ActivityLog.objects.get(scope="ExternalDataSchema", item_id=str(schema.id), activity="updated")assert log.user == self.userassert log.detail is not None- assert [c for c in log.detail["changes"] if c["field"] == field] == [- {"type": "ExternalDataSchema", "field": field, "action": "created", "before": None, "after": value}+ assert [c for c in log.detail["changes"] if c["field"] == logged_field] == [+ {"type": "ExternalDataSchema", "field": logged_field, "action": "created", "before": None, "after": value}]def test_null_clears_row_filters(self):@@ -5366,6 +5374,401 @@ class TestExternalDataSchemaRowFilters(APIBaseTest):assert "not supported for CDC" in str(response.json())+class TestMaskedColumnsError(SimpleTestCase):+ @parameterized.expand(+ [+ ("valid", ["password"], None),+ ("source_casing_of_a_known_column", ["Password"], None),+ ("unknown_column", ["does_not_exist"], "Unknown columns in masked_columns"),+ ("primary_key", ["id"], "can't be masked"),+ ("primary_key_in_another_case", ["ID"], "can't be masked"),+ ("incremental_field", ["created_at"], "can't be masked"),+ ("partition_column", ["region"], "can't be masked"),+ ("two_source_columns_fold_to_one_name", ["Email"], "collide after normalization"),+ ("direct_query_source", ["password"], "direct query", ExternalDataSourceType.POSTGRES, True),+ (+ "managed_schema_source",+ ["password"],+ "not supported for this source type",+ ExternalDataSourceType.STRIPE,+ ),+ ]+ )+ def test_masked_columns_error(+ self,+ _name: str,+ masked_columns: list[str],+ expected: str | None,+ source_type: str = ExternalDataSourceType.POSTGRES,+ is_direct_query: bool = False,+ ) -> None:+ error = masked_columns_error(+ masked_columns,+ source_type=source_type,+ is_direct_query=is_direct_query,+ source_columns=["id", "created_at", "password", "email", "Email", "region"],+ primary_key_columns=["id"],+ incremental_fields=["created_at"],+ partition_keys=["region"],+ )+ if expected is None:+ assert error is None+ else:+ assert error is not None and expected in error++ def test_a_mask_is_refused_while_no_columns_are_known(self) -> None:+ # A mask that matches no column hashes nothing, so the column it meant would sync in plaintext.+ error = masked_columns_error(+ ["anything"],+ source_type=ExternalDataSourceType.POSTGRES,+ is_direct_query=False,+ source_columns=[],+ primary_key_columns=None,+ incremental_fields=[],+ )+ assert error is not None and "columns are known" in error+++class TestExternalDataSchemaMaskedColumns(APIBaseTest):+ VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_schema"+ SCHEMA_METADATA = {+ "columns": [+ {"name": "id", "data_type": "integer", "is_nullable": False},+ {"name": "created_at", "data_type": "timestamp", "is_nullable": True},+ {"name": "password", "data_type": "varchar(255)", "is_nullable": True},+ {"name": "email", "data_type": "varchar(255)", "is_nullable": True},+ ]+ }++ def _create(self, access_method: str | None = None, **schema_kwargs: Any) -> ExternalDataSchema:+ source = ExternalDataSource.objects.create(+ team=self.team,+ source_type=ExternalDataSourceType.POSTGRES,+ job_inputs={"host": "h", "port": "5432", "database": "d", "user": "u", "password": "p", "schema": "public"},+ **({"access_method": access_method} if access_method else {}),+ )+ return ExternalDataSchema.objects.create(+ name="Customers",+ team=self.team,+ source=source,+ sync_type_config={+ "schema_metadata": self.SCHEMA_METADATA,+ "primary_key_columns": ["id"],+ "incremental_field": "created_at",+ },+ **schema_kwargs,+ )++ def _patch(self, schema: ExternalDataSchema, data: dict[str, Any]):+ return self.client.patch(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}", data=data)++ @parameterized.expand(+ [+ ("mask_added", None, ["password"], True, True, True, [True], True),+ ("mask_removed", ["password"], [], True, True, True, [True], True),+ ("mask_swapped", ["password"], ["email"], True, True, True, [True], True),+ ("same_mask_in_another_case", ["password"], ["Password"], True, True, False, [], False),+ ("mask_added_with_no_sync_running", None, ["password"], False, True, True, [False], False),+ ("mask_added_on_a_disabled_table", None, ["password"], True, False, True, [], True),+ ]+ )+ def test_a_mask_change_resyncs_the_table(+ self,+ _name: str,+ stored: list[str] | None,+ requested: list[str],+ sync_running: bool,+ should_sync: bool,+ resets: bool,+ triggers_with_buffer_one: list[bool],+ cancels: bool,+ ) -> None:+ schema = self._create(masked_columns=stored, should_sync=should_sync)+ if sync_running:+ # The schedule skips a trigger while a sync runs, so the rebuild would wait a whole interval.+ ExternalDataJob.objects.create(+ team=self.team,+ pipeline=schema.source,+ schema=schema,+ status=ExternalDataJob.Status.RUNNING,+ workflow_id="running-workflow-id",+ )+ with (+ mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger,+ mock.patch(f"{self.VIEWS}.cancel_external_data_workflow") as mock_cancel,+ ):+ response = self._patch(schema, {"masked_columns": requested})++ assert response.status_code == 200, response.json()+ schema.refresh_from_db()+ # Stored under the source column's own name, which the UI compares exactly.+ assert schema.masked_columns == [column.lower() for column in requested]+ assert schema.sync_type_config.get("reset_pipeline") is (True if resets else None)+ assert mock_trigger.call_args_list == [+ mock.call(mock.ANY, buffer_one=buffer_one) for buffer_one in triggers_with_buffer_one+ ]+ assert mock_cancel.call_args_list == ([mock.call("running-workflow-id")] if cancels else [])++ # Some sources record their columns after normalization, so the metadata holds the alias too.+ @parameterized.expand([("synced_table",), ("normalized_metadata",)])+ def test_a_source_checks_masks_against_its_synced_columns(self, columns_from: str) -> None:+ source = ExternalDataSource.objects.create(team=self.team, source_type=ExternalDataSourceType.HUBSPOT)+ names = ["id", "email", "_email"]+ table = DataWarehouseTable.objects.create(+ team=self.team, name="hubspot_contacts", columns={name: {"clickhouse": "String"} for name in names}+ )+ config = (+ {"schema_metadata": {"columns": [{"name": name, "data_type": "text"} for name in names]}}+ if columns_from == "normalized_metadata"+ else {}+ )+ schema = ExternalDataSchema.objects.create(+ name="contacts", team=self.team, source=source, table=table, sync_type_config=config+ )+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):+ unknown = self._patch(schema, {"masked_columns": ["phone"]})+ # `_email` is where the pipeline put the later of two colliding source columns.+ alias = self._patch(schema, {"masked_columns": ["_email"]})+ accepted = self._patch(schema, {"masked_columns": ["EMAIL"]})++ assert unknown.status_code == 400+ assert "Unknown columns" in str(unknown.json())+ assert alias.status_code == 400+ assert "renamed" in str(alias.json())+ assert accepted.status_code == 200, accepted.json()+ schema.refresh_from_db()+ assert schema.masked_columns == ["email"]++ def test_a_mask_change_clears_the_cached_bounds_of_masked_columns(self) -> None:+ table = DataWarehouseTable.objects.create(team=self.team)+ schema = self._create(table=table)+ for column in ("password", "email"):+ WarehouseColumnStatistics.objects.for_team(self.team.id).create(+ team=self.team, table=table, column_name=column, min_value="a", max_value="z", has_min_max=True+ )+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):+ response = self._patch(schema, {"masked_columns": ["Password"]})++ assert response.status_code == 200, response.json()+ bounds = {+ name: (low, high)+ for name, low, high in WarehouseColumnStatistics.objects.for_team(self.team.id).values_list(+ "column_name", "min_value", "max_value"+ )+ }+ assert bounds == {"password": (None, None), "email": ("a", "z")}++ @parameterized.expand(+ [+ ("unrelated_edit", {"incremental_field": "created_at", "incremental_field_type": "datetime"}),+ ("new_mask_next_to_it", {"masked_columns": ["gone", "password"]}),+ ]+ )+ def test_a_stored_mask_on_a_column_that_left_the_source_does_not_block_edits(+ self, _name: str, data: dict[str, Any]+ ) -> None:+ schema = self._create(masked_columns=["gone"], sync_type=ExternalDataSchema.SyncType.INCREMENTAL)+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):+ response = self._patch(schema, data)++ assert response.status_code == 200, response.json()++ def test_a_patch_without_the_field_keeps_the_masks(self) -> None:+ schema = self._create(masked_columns=["password"])+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger:+ response = self._patch(schema, {"enabled_columns": ["id", "created_at", "password"]})++ assert response.status_code == 200, response.json()+ schema.refresh_from_db()+ assert schema.masked_columns == ["password"]+ assert schema.sync_type_config.get("reset_pipeline") is None+ mock_trigger.assert_not_called()++ def test_a_mask_change_on_a_disabled_table_waits_for_the_next_sync(self) -> None:+ schema = self._create(should_sync=False)+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger:+ response = self._patch(schema, {"masked_columns": ["password"]})++ assert response.status_code == 200, response.json()+ schema.refresh_from_db()+ assert schema.sync_type_config.get("reset_pipeline") is True+ mock_trigger.assert_not_called()++ @parameterized.expand(+ [+ ("syncing", True, 1),+ ("disabled_stages_the_snapshot_in_the_save", False, 0),+ ("syncing_with_a_table_mode_change", True, 1, {"cdc_table_mode": "both"}),+ ]+ )+ def test_a_cdc_table_resets_through_a_new_snapshot(+ self, _name: str, should_sync: bool, resets: int, other_changes: dict[str, Any] | None = None+ ) -> None:+ schema = self._create(sync_type=ExternalDataSchema.SyncType.CDC, should_sync=should_sync)+ facade = "products.data_warehouse.backend.facade.api"+ with (+ mock.patch(f"{self.VIEWS}.source_type_supports_cdc", return_value=False),+ mock.patch(f"{self.VIEWS}.sync_cdc_extraction_schedule"),+ mock.patch(f"{facade}.pause_external_data_schedule") as mock_pause,+ mock.patch(f"{facade}.trigger_cdc_extraction_schedule") as mock_start_capture,+ mock.patch(f"{self.VIEWS}._reset_cdc_for_full_resnapshot") as mock_resnapshot,+ mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger,+ ):+ response = self._patch(schema, {"masked_columns": ["password"], **(other_changes or {})})++ assert response.status_code == 200, response.json()+ assert mock_pause.call_count == resets+ assert mock_start_capture.call_count == resets+ mock_resnapshot.assert_not_called()+ mock_trigger.assert_not_called()+ schema.refresh_from_db()+ # Saved with the mask, so capture still resets the table if the hand-over fails.+ assert ("cdc_reset_pending" in schema.sync_type_config) is should_sync+ if should_sync:+ # One staged generation: a second one would make capture take a second snapshot.+ assert schema.sync_type_config["cdc_reset_pending"]["generation"] == 1+ if not should_sync:+ assert schema.sync_type_config["cdc_mode"] == "snapshot"+ assert schema.sync_type_config["reset_pipeline"] is True+ assert schema.initial_sync_complete is False++ @parameterized.expand(+ [+ (+ "repair_staged_a_reset_that_waits_for_a_slot",+ ExternalDataSchema.SyncType.CDC,+ {"cdc_reset_pending": {"trigger": True, "awaiting_slot": True, "generation": 3}},+ [],+ {"cdc_reset_pending": {"trigger": True, "awaiting_slot": True, "generation": 4}},+ ),+ (+ "a_run_cleared_an_earlier_reset",+ ExternalDataSchema.SyncType.FULL_REFRESH,+ {},+ ["reset_pipeline"],+ {"reset_pipeline": True},+ ),+ ]+ )+ def test_a_mask_rebuild_survives_a_config_write_before_the_save(+ self,+ _name: str,+ sync_type: str,+ concurrent_updates: dict[str, Any],+ concurrent_removes: list[str],+ expected: dict[str, Any],+ ) -> None:+ schema = self._create(sync_type=sync_type, should_sync=True)+ update_sync_type_config_keys(schema.id, schema.team_id, updates={"reset_pipeline": True})+ original_save = ExternalDataSchemaSerializer._save_merging_sync_type_config++ def concurrent_write_first(serializer: ExternalDataSchemaSerializer, *args: Any) -> ExternalDataSchema:+ update_sync_type_config_keys(+ schema.id, schema.team_id, updates=concurrent_updates, removes=concurrent_removes+ )+ return original_save(serializer, *args)++ with (+ mock.patch.object(+ ExternalDataSchemaSerializer,+ "_save_merging_sync_type_config",+ autospec=True,+ side_effect=concurrent_write_first,+ ),+ mock.patch(f"{self.VIEWS}.source_type_supports_cdc", return_value=False),+ mock.patch(f"{self.VIEWS}.sync_cdc_extraction_schedule"),+ mock.patch(f"{self.VIEWS}.hand_reset_to_capture"),+ mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"),+ ):+ response = self._patch(schema, {"masked_columns": ["password"]})++ assert response.status_code == 200, response.json()+ schema.refresh_from_db()+ assert {key: schema.sync_type_config.get(key) for key in expected} == expected++ def test_rejected_when_no_masking_key_is_set(self) -> None:+ schema = self._create()+ with mock.patch(f"{self.VIEWS}.masking_key_unavailable", return_value=True):+ response = self._patch(schema, {"masked_columns": ["password"]})++ assert response.status_code == 400+ assert "WAREHOUSE_COLUMN_MASKING_KEY" in str(response.json())++ def test_rejected_for_a_direct_query_source(self) -> None:+ schema = self._create(access_method=ExternalDataSource.AccessMethod.DIRECT)+ response = self._patch(schema, {"masked_columns": ["password"]})++ assert response.status_code == 400+ assert "direct query" in str(response.json())++ def test_rejected_on_a_synced_webhook_table(self) -> None:+ table = DataWarehouseTable.objects.create(team=self.team)+ schema = self._create(masked_columns=["password"], sync_type=ExternalDataSchema.SyncType.WEBHOOK, table=table)+ response = self._patch(schema, {"masked_columns": []})++ assert response.status_code == 400+ assert "webhook" in str(response.json())+ schema.refresh_from_db()+ assert schema.masked_columns == ["password"]++ def test_rejected_while_syncs_are_paused_for_billing(self) -> None:+ schema = self._create()+ with mock.patch(f"{self.VIEWS}.is_any_external_data_schema_paused", return_value=True):+ response = self._patch(schema, {"masked_columns": ["password"]})++ assert response.status_code == 400+ schema.refresh_from_db()+ assert schema.masked_columns is None++ @parameterized.expand(+ [+ ("mask_on_the_primary_key", {"masked_columns": ["id"]}, ExternalDataSchema.SyncType.INCREMENTAL),+ (+ "incremental_field_moved_onto_a_masked_column",+ {"incremental_field": "password"},+ ExternalDataSchema.SyncType.INCREMENTAL,+ ),+ (+ "primary_key_moved_onto_a_masked_column",+ {"primary_key_columns": ["password"]},+ ExternalDataSchema.SyncType.INCREMENTAL,+ ),+ # A CDC table keeps its stored cursor whatever the request sends for it.+ (+ "stored_cursor_with_a_null_cursor_in_the_request",+ {"masked_columns": ["created_at"], "incremental_field": None},+ ExternalDataSchema.SyncType.CDC,+ ),+ # With no stored key, a sync merges and partitions on `id`.+ (+ "mask_on_id_with_no_stored_key",+ {"masked_columns": ["id"]},+ ExternalDataSchema.SyncType.FULL_REFRESH,+ False,+ ),+ ]+ )+ def test_a_mask_never_shares_a_column_with_the_key_or_cursor(+ self, _name: str, data: dict[str, Any], sync_type: str, key_stored: bool = True+ ) -> None:+ schema = self._create(masked_columns=["password"], sync_type=sync_type)+ if not key_stored:+ update_sync_type_config_keys(schema.id, schema.team_id, removes=["primary_key_columns"])+ response = self._patch(schema, {"incremental_field_type": "integer", **data})++ assert response.status_code == 400+ assert "can't be masked" in str(response.json())++ def test_a_full_refresh_table_can_mask_the_cursor_of_its_earlier_sync_method(self) -> None:+ schema = self._create(sync_type=ExternalDataSchema.SyncType.FULL_REFRESH)+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):+ response = self._patch(schema, {"masked_columns": ["created_at"]})++ assert response.status_code == 200, response.json()+ schema.refresh_from_db()+ assert schema.masked_columns == ["created_at"]++class TestExternalDataSchemaApiVersionOverride(APIBaseTest):def _create_schema(self, sync_type=ExternalDataSchema.SyncType.FULL_REFRESH, api_version=None):source = ExternalDataSource.objects.create(diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex f778eafc037a..7605873955b2 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -3666,6 +3666,7 @@ class TestExternalDataSource(APIBaseTest):"cdc_table_mode": "consolidated","incremental_sync_blocked": None,"enabled_columns": None,+ "masked_columns": None,"row_filters": None,"row_filter_columns": None,"available_columns": [],@@ -5004,7 +5005,12 @@ class TestExternalDataSource(APIBaseTest):self.assertEqual(analytics_schema.sync_type_config["schema_metadata"]["source_schema"], "analytics")def _post_postgres_cdc_source_with_one_table(- self, mock_get_source, mock_cdc_pg_connection, mock_get_primary_key_columns, mock_setup_cdc_resources+ self,+ mock_get_source,+ mock_cdc_pg_connection,+ mock_get_primary_key_columns,+ mock_setup_cdc_resources,+ schema_extra: dict[str, t.Any] | None = None,):_configure_source_mock_versioning(mock_get_source)source_mock = mock_get_source.return_value@@ -5065,7 +5071,7 @@ class TestExternalDataSource(APIBaseTest):"schema": "","cdc_enabled": True,"schemas": [- {"name": "analytics.events", "should_sync": True, "sync_type": "cdc"},+ {"name": "analytics.events", "should_sync": True, "sync_type": "cdc", **(schema_extra or {})},],},},@@ -5110,6 +5116,50 @@ class TestExternalDataSource(APIBaseTest):mock_add_table.assert_called_once()assert mock_add_table.call_args.args[1:] == ("analytics", "events")+ @parameterized.expand(+ [+ ("masked_primary_key", {"masked_columns": ["id"]}),+ ("malformed_mask", {"masked_columns": "id"}),+ ("incremental_table_without_a_cursor", {"sync_type": "incremental", "incremental_field": None}),+ ]+ )+ @patch(+ "products.warehouse_sources.backend.presentation.views.external_data_source.base.is_cdc_enabled_for_team",+ return_value=True,+ )+ @patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.cleanup_resources"+ )+ @patch(+ "products.warehouse_sources.backend.presentation.views.external_data_source.viewset.ExternalDataSourceViewSet._setup_cdc_resources"+ )+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.get_primary_key_columns")+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.cdc_pg_connection")+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.SourceRegistry.get_source")+ def test_create_postgres_cdc_refused_table_drops_the_replication_slot(+ self,+ _name,+ schema_extra,+ mock_get_source,+ mock_cdc_pg_connection,+ mock_get_primary_key_columns,+ mock_setup_cdc_resources,+ mock_cleanup_resources,+ _mock_is_cdc_enabled_for_team,+ ):+ mock_get_source.return_value.has_managed_hogql_schema = False+ response = self._post_postgres_cdc_source_with_one_table(+ mock_get_source,+ mock_cdc_pg_connection,+ mock_get_primary_key_columns,+ mock_setup_cdc_resources,+ schema_extra=schema_extra,+ )++ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content+ mock_cleanup_resources.assert_called_once()+ assert not ExternalDataSource.objects.filter(team_id=self.team.pk).exists()+@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.is_cdc_enabled_for_team",return_value=True,@@ -5785,6 +5835,73 @@ class TestExternalDataSource(APIBaseTest):else:assert schema.sync_type_config["primary_key_columns"] == expected_persisted+ @parameterized.expand(+ [+ ("mask_is_saved", ["email"], status.HTTP_201_CREATED, ["email"]),+ ("empty_list_masks_nothing", [], status.HTTP_201_CREATED, None),+ ("mask_on_the_detected_primary_key", ["id"], status.HTTP_400_BAD_REQUEST, "can't be masked"),+ ("mask_on_an_unknown_column", ["nope"], status.HTTP_400_BAD_REQUEST, "Unknown columns"),+ ("mask_that_is_not_a_list", "email", status.HTTP_400_BAD_REQUEST, "must be a list"),+ ]+ )+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.SourceRegistry.get_source")+ def test_create_postgres_validates_and_persists_masked_columns(+ self,+ _name: str,+ masked_columns: list[str] | str,+ expected_status: int,+ expected: list[str] | str | None,+ mock_get_source,+ ):+ _configure_source_mock_versioning(mock_get_source)+ source_mock = mock_get_source.return_value+ source_mock.validate_config.return_value = (True, [])+ source_mock.has_managed_hogql_schema = False+ parsed_config = Mock()+ parsed_config.schema = "public"+ payload = {+ "host": "localhost",+ "port": 5432,+ "database": "app",+ "user": "user",+ "password": "pass",+ "schema": "public",+ }+ parsed_config.to_dict.return_value = payload+ source_mock.parse_config.return_value = parsed_config+ source_mock.validate_credentials.return_value = (True, None)+ source_mock.get_schemas.return_value = [+ SourceSchema(+ name="events",+ supports_incremental=False,+ supports_append=False,+ columns=[("id", "integer", False), ("email", "text", True)],+ foreign_keys=[],+ detected_primary_keys=["id"],+ ),+ ]++ response = self.client.post(+ f"/api/environments/{self.team.pk}/external_data_sources/",+ data={+ "source_type": "Postgres",+ "payload": {+ **payload,+ "schemas": [+ {"name": "events", "should_sync": True, "sync_type": None, "masked_columns": masked_columns}+ ],+ },+ },+ )++ assert response.status_code == expected_status, response.content+ schemas = ExternalDataSchema.objects.filter(team_id=self.team.pk, name="events")+ if expected_status == status.HTTP_201_CREATED:+ assert schemas.get().masked_columns == expected+ else:+ assert isinstance(expected, str) and expected in response.json()["message"]+ assert not schemas.exists()+@parameterized.expand([# Field omitted -> None: sync everything (default).diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.tsindex 382a98f3959d..ce010d5a587d 100644--- a/products/warehouse_sources/frontend/generated/api.schemas.ts+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts@@ -583,6 +583,11 @@ export interface ExternalDataSchemaApi {* @nullable*/enabled_columns?: string[] | null+ /**+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.+ * @nullable+ */+ masked_columns?: string[] | null/*** Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.* @nullable@@ -787,6 +792,11 @@ export interface PatchedExternalDataSchemaApi {* @nullable*/enabled_columns?: string[] | null+ /**+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.+ * @nullable+ */+ masked_columns?: string[] | null/*** Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.* @nullable@@ -1214,6 +1224,11 @@ export interface ExternalDataSourceBulkUpdateSchemaApi {* @nullable*/enabled_columns?: string[] | null+ /**+ * Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.+ * @nullable+ */+ masked_columns?: string[] | null/*** Row-filter predicates ANDed onto the source query. Null/empty means sync all rows.* @nullablediff --git a/products/warehouse_sources/frontend/generated/api.zod.ts b/products/warehouse_sources/frontend/generated/api.zod.tsindex f1ce05ccc461..3ff87cec10fe 100644--- a/products/warehouse_sources/frontend/generated/api.zod.ts+++ b/products/warehouse_sources/frontend/generated/api.zod.ts@@ -243,6 +243,12 @@ export const ExternalDataSchemasUpdateBody = /* @__PURE__ */ zod.describe('Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.'),+ masked_columns: zod+ .array(zod.string())+ .nullish()+ .describe(+ "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."+ ),row_filters: zod.array(zod.object({@@ -360,6 +366,12 @@ export const ExternalDataSchemasPartialUpdateBody = /* @__PURE__ */ zod.describe('Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.'),+ masked_columns: zod+ .array(zod.string())+ .nullish()+ .describe(+ "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."+ ),row_filters: zod.array(zod.object({@@ -638,6 +650,12 @@ export const ExternalDataSourcesBulkUpdateSchemasPartialUpdateBody = /* @__PURE_.array(zod.string()).nullish().describe('Columns to sync. Null means sync all columns.'),+ masked_columns: zod+ .array(zod.string())+ .nullish()+ .describe(+ 'Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.'+ ),row_filters: zod.array(zod.object({diff --git a/products/warehouse_sources/mcp/tools.yaml b/products/warehouse_sources/mcp/tools.yamlindex 81426f63440a..9f95795567db 100644--- a/products/warehouse_sources/mcp/tools.yaml+++ b/products/warehouse_sources/mcp/tools.yaml@@ -184,8 +184,9 @@ tools:(incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled fullrefresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incrementalfield (plus type and lookback window), primary key columns, CDC table mode, column selection- (enabled_columns), row filters, and vendor API version. Changes take effect on the next sync, not- retroactively. Before changing incremental_field or sync_type, call+ (enabled_columns), column masking (masked_columns), row filters, and vendor API version. Changes take effect+ on the next sync, not retroactively, except a masked_columns change, which starts a full resync of the+ table. Before changing incremental_field or sync_type, call'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actuallysupports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. Torun a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a tablediff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 8ab0cfbbc73c..000cc1c69da8 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -6503,7 +6503,7 @@}},"external-data-schemas-partial-update": {- "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.",+ "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), column masking (masked_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively, except a masked_columns change, which starts a full resync of the table. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.","category": "Warehouse sources","feature": "warehouse_sources","summary": "Update data warehouse table schema sync config",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex f648462b5f6d..116ce6663278 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -6636,7 +6636,7 @@}},"external-data-schemas-partial-update": {- "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.",+ "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), column masking (masked_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively, except a masked_columns change, which starts a full resync of the table. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.","category": "Warehouse sources","feature": "warehouse_sources","summary": "Update data warehouse table schema sync config",diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 7fb573aabe9f..6cb6044e1d68 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -47242,6 +47242,11 @@ export namespace Schemas {* @nullable*/enabled_columns?: string[] | null;+ /**+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.+ * @nullable+ */+ masked_columns?: string[] | null;/*** Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.* @nullable@@ -47363,6 +47368,11 @@ export namespace Schemas {* @nullable*/enabled_columns?: string[] | null;+ /**+ * Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.+ * @nullable+ */+ masked_columns?: string[] | null;/*** Row-filter predicates ANDed onto the source query. Null/empty means sync all rows.* @nullable@@ -81702,6 +81712,11 @@ export namespace Schemas {* @nullable*/enabled_columns?: string[] | null;+ /**+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.+ * @nullable+ */+ masked_columns?: string[] | null;/*** Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.* @nullablediff --git a/services/mcp/src/generated/warehouse_sources/api.ts b/services/mcp/src/generated/warehouse_sources/api.tsindex 8ffb3e3b0bbf..f23e23480e29 100644--- a/services/mcp/src/generated/warehouse_sources/api.ts+++ b/services/mcp/src/generated/warehouse_sources/api.ts@@ -131,6 +131,12 @@ export const ExternalDataSchemasPartialUpdateBody = () => zod.describe('Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.'),+ masked_columns: zod+ .array(zod.string())+ .nullish()+ .describe(+ "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."+ ),row_filters: zod.array(zod.object({@@ -461,6 +467,12 @@ export const ExternalDataSourcesBulkUpdateSchemasPartialUpdateBody = () => zod.o.array(zod.string()).nullish().describe('Columns to sync. Null means sync all columns.'),+ masked_columns: zod+ .array(zod.string())+ .nullish()+ .describe(+ 'Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.'+ ),row_filters: zod.array(zod.object({diff --git a/services/mcp/src/tools/generated/warehouse_sources.ts b/services/mcp/src/tools/generated/warehouse_sources.tsindex 9ef6d4ea0bc6..18f50ad3c288 100644--- a/services/mcp/src/tools/generated/warehouse_sources.ts+++ b/services/mcp/src/tools/generated/warehouse_sources.ts@@ -269,6 +269,9 @@ const externalDataSchemasPartialUpdate = (): ToolBase<if (params.enabled_columns !== undefined) {body['enabled_columns'] = params.enabled_columns}+ if (params.masked_columns !== undefined) {+ body['masked_columns'] = params.masked_columns+ }if (params.row_filters !== undefined) {body['row_filters'] = params.row_filters}diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.jsonindex d58ad1c2e9a5..ac766d8ee440 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.json@@ -106,6 +106,20 @@],"description": "Data type of the incremental field.\n\n* `integer` - integer\n* `numeric` - numeric\n* `datetime` - datetime\n* `date` - date\n* `timestamp` - timestamp\n* `objectid` - objectid\n* `xid` - xid"},+ "masked_columns": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "description": "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."+ },"primary_key_columns": {"anyOf": [{diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.jsonindex 6840db34bdfc..c5528c21fbdd 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.json@@ -91,6 +91,20 @@],"description": "Type of the incremental cursor field."},+ "masked_columns": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "description": "Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table."+ },"primary_key_columns": {"anyOf": [{