Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add deterministic column masking for sources (#116589)

Daniel RCef9d39e29025
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 61ea1c9dbe7a..912b0ee02007 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6637,6 +6637,8 @@ export interface ExternalDataSourceSyncSchema {
* PK columns and the active incremental field are always retained server-side.
*/
enabled_columns?: string[] | null
+ /** Source columns whose values are replaced with a one-way hash at sync time. */
+ masked_columns?: string[] | null
/**
* Predicates ANDed onto the source query so only matching rows sync.
* `null`/undefined/empty = sync all rows.
@@ -6680,6 +6682,8 @@ export interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema
* Primary-key + active incremental columns are always retained even if not listed.
*/
enabled_columns?: string[] | null
+ /** Source columns whose values are replaced with a one-way hash at sync time. */
+ masked_columns?: string[] | null
available_columns?: { name: string; data_type?: string; is_nullable?: boolean }[]
/** Exact source identifiers are available for sources that project columns upstream. */
source_column_metadata_available?: boolean
diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.py
index e35d7c040b05..2686bb74b58f 100644
--- a/posthog/models/activity_logging/activity_log.py
+++ b/posthog/models/activity_logging/activity_log.py
@@ -510,6 +510,7 @@ field_name_overrides: dict[AuditableScope, dict[str, str]] = {
},
"ExternalDataSchema": {
"should_sync": "enabled",
+ "masked_columns": "masked columns",
"full_refresh_interval_days": "full refresh interval (days)",
"full_refresh_time_of_day": "full refresh time (UTC)",
},
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.py
index 4f03724d71f9..eb6a37068e46 100644
--- a/posthog/settings/data_warehouse.py
+++ b/posthog/settings/data_warehouse.py
@@ -33,6 +33,10 @@ USE_LOCAL_SETUP = get_from_env("USE_LOCAL_SETUP", USE_LOCAL_SETUP_DEFAULT, type_
PYARROW_DEBUG_LOGGING = get_from_env("PYARROW_DEBUG_LOGGING", False, type_cast=str_to_bool)
+# Keys the one-way hash of masked warehouse columns. Changing it changes every later hash, so every
+# masked table needs a full resync afterwards. Without it, column masking is off outside development.
+WAREHOUSE_COLUMN_MASKING_KEY = os.getenv("WAREHOUSE_COLUMN_MASKING_KEY", "")
+
# Load the full warehouse source catalog (every vendor SDK) at web-worker startup, before
# the worker starts serving, so its first warehouse query doesn't pay the multi-second
# catalog import at request time. WSGI workers load while importing posthog.wsgi, ASGI
diff --git a/products/data_warehouse/backend/direct_virtual_tables.py b/products/data_warehouse/backend/direct_virtual_tables.py
index 253160116fc1..8b30da1a51af 100644
--- a/products/data_warehouse/backend/direct_virtual_tables.py
+++ b/products/data_warehouse/backend/direct_virtual_tables.py
@@ -65,7 +65,8 @@ def eligible_direct_query_schemas(team_id: int, source_id: str) -> list[External
catalog, so the two selections can't drift. Excludes rows without usable ``schema_metadata``
(e.g. discovered before metadata persistence shipped) and rows with ``row_filters`` — a
sync-time row restriction the live direct query can't reproduce, so exposing the upstream
- table would let a user read rows the schema was configured not to sync.
+ table would let a user read rows the schema was configured not to sync. Rows with
+ ``masked_columns`` are excluded for the same reason: a live query reads the plaintext values.
"""
return [
schema_row
@@ -78,7 +79,7 @@ def eligible_direct_query_schemas(team_id: int, source_id: str) -> list[External
# `table` is the synced S3 row backing the warehouse access-control check on the build path.
.select_related("table")
.order_by("name")
- if schema_row.schema_metadata and not schema_row.row_filters
+ if schema_row.schema_metadata and not schema_row.row_filters and not schema_row.masked_columns
]
@@ -91,9 +92,9 @@ def build_direct_table_for_schema(schema: "ExternalDataSchema", source: "Externa
metadata = schema.schema_metadata
engine = source.direct_engine
- # Defense in depth for any direct caller: a row-filtered schema is already excluded from the
- # catalog by eligible_direct_query_schemas; never build a live table that would bypass it.
- if schema.row_filters:
+ # Defense in depth for any direct caller: a row-filtered or masked schema is already excluded
+ # from the catalog by eligible_direct_query_schemas; never build a live table that would bypass it.
+ if schema.row_filters or schema.masked_columns:
return None
if engine == "postgres":
diff --git a/products/data_warehouse/backend/logic/data_load/service.py b/products/data_warehouse/backend/logic/data_load/service.py
index 5b58927e1886..fd31b1dcbc41 100644
--- a/products/data_warehouse/backend/logic/data_load/service.py
+++ b/products/data_warehouse/backend/logic/data_load/service.py
@@ -41,6 +41,7 @@ from posthog.temporal.common.schedule import (
pause_schedule,
schedule_exists,
trigger_schedule,
+ trigger_schedule_buffer_one,
unpause_schedule,
update_schedule,
)
@@ -214,9 +215,13 @@ def trigger_external_data_source_workflow(external_data_source: ExternalDataSour
trigger_schedule(temporal, schedule_id=str(external_data_source.id))
-def trigger_external_data_workflow(external_data_schema: ExternalDataSchema):
+def trigger_external_data_workflow(external_data_schema: ExternalDataSchema, *, buffer_one: bool = False):
temporal = sync_connect()
- trigger_schedule(temporal, schedule_id=str(external_data_schema.id))
+ if buffer_one:
+ # The schedule skips a trigger while a run is open, and a cancelled run stays open until it closes.
+ trigger_schedule_buffer_one(temporal, str(external_data_schema.id))
+ else:
+ trigger_schedule(temporal, schedule_id=str(external_data_schema.id))
async def a_trigger_external_data_workflow(external_data_schema: ExternalDataSchema):
diff --git a/products/data_warehouse/backend/test/test_direct_virtual_tables.py b/products/data_warehouse/backend/test/test_direct_virtual_tables.py
index fd1998598af3..944df2d4fba6 100644
--- a/products/data_warehouse/backend/test/test_direct_virtual_tables.py
+++ b/products/data_warehouse/backend/test/test_direct_virtual_tables.py
@@ -2,7 +2,10 @@ from posthog.test.base import APIBaseTest
from posthog.hogql.database.direct_motherduck_table import DirectMotherDuckTable
-from products.data_warehouse.backend.direct_virtual_tables import build_direct_table_for_schema
+from products.data_warehouse.backend.direct_virtual_tables import (
+ build_direct_table_for_schema,
+ eligible_direct_query_schemas,
+)
from products.warehouse_sources.backend.facade.models import ExternalDataSchema, ExternalDataSource
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
@@ -16,7 +19,9 @@ SCHEMA_METADATA = {
class TestBuildDirectTableForSchema(APIBaseTest):
- def _create(self, *, enabled_columns: list[str] | None) -> tuple[ExternalDataSchema, ExternalDataSource]:
+ def _create(
+ self, *, enabled_columns: list[str] | None, masked_columns: list[str] | None = None
+ ) -> tuple[ExternalDataSchema, ExternalDataSource]:
source = ExternalDataSource.objects.create(
team=self.team,
source_type=ExternalDataSourceType.POSTGRES,
@@ -29,6 +34,7 @@ class TestBuildDirectTableForSchema(APIBaseTest):
team=self.team,
source=source,
enabled_columns=enabled_columns,
+ masked_columns=masked_columns,
sync_type_config={"schema_metadata": SCHEMA_METADATA, "primary_key_columns": ["id"]},
)
return schema, source
@@ -45,6 +51,13 @@ class TestBuildDirectTableForSchema(APIBaseTest):
assert "email" in table.fields
assert "id" in table.fields # primary key always retained
+ def test_masked_schema_is_not_exposed_to_live_queries(self):
+ # A live query reads the source directly, so it would return the masked values in plaintext.
+ schema, source = self._create(enabled_columns=None, masked_columns=["secret"])
+
+ assert build_direct_table_for_schema(schema, source) is None
+ assert eligible_direct_query_schemas(self.team.pk, str(source.pk)) == []
+
def test_none_enabled_columns_syncs_everything(self):
schema, source = self._create(enabled_columns=None)
diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx b/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx
index ec7b3401e174..2619e50e17cb 100644
--- a/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx
+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/sourceWizardLogic.tsx
@@ -277,6 +277,11 @@ export function buildCreateSourcePayload(
}
}
+function withoutColumns(columns: string[] | null | undefined, removed: (string | null)[]): string[] | null | undefined {
+ const names = new Set(removed.filter((name): name is string => !!name).map((name) => name.toLowerCase()))
+ return columns?.filter((column) => !names.has(column.toLowerCase())) ?? columns
+}
+
function webhookResultHasNoPendingInputs(webhookResult: WebhookCreateResult | null | undefined): boolean {
return !!webhookResult?.success && (webhookResult.pending_inputs?.length ?? 0) === 0
}
@@ -735,6 +740,13 @@ export interface sourceWizardLogicActions {
returnLabel: string
returnUrl: string
}
+ setSchemaMaskedColumns: (
+ schema: ExternalDataSourceSyncSchema,
+ maskedColumns: string[]
+ ) => {
+ maskedColumns: string[]
+ schema: ExternalDataSourceSyncSchema
+ }
setSchemaNameFilter: (schemaNameFilter: string) => {
schemaNameFilter: string
}
@@ -1095,6 +1107,10 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
schema,
rowFilters,
}),
+ setSchemaMaskedColumns: (schema: ExternalDataSourceSyncSchema, maskedColumns: string[]) => ({
+ schema,
+ maskedColumns,
+ }),
updateSchemaSyncType: (
schema: ExternalDataSourceSyncSchema,
syncType: ExternalDataSourceSyncSchema['sync_type'],
@@ -1267,6 +1283,12 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
row_filters: s.table === schema.table ? rowFilters : s.row_filters,
}))
},
+ setSchemaMaskedColumns: (state, { schema, maskedColumns }) => {
+ return state.map((s) => ({
+ ...s,
+ masked_columns: s.table === schema.table ? maskedColumns : s.masked_columns,
+ }))
+ },
updateSchemaSyncType: (
state,
{
@@ -1290,6 +1312,11 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
? (incrementalFieldLookbackSeconds ?? null)
: s.incremental_field_lookback_seconds,
primary_key_columns: s.table === schema.table ? primaryKeyColumns : s.primary_key_columns,
+ // The server refuses to mask a cursor or key column, which would fail the whole source creation.
+ masked_columns:
+ s.table === schema.table
+ ? withoutColumns(s.masked_columns, [incrementalField, ...(primaryKeyColumns ?? [])])
+ : s.masked_columns,
...(s.table === schema.table && syncType === 'cdc' && cdcTableMode
? { cdc_table_mode: cdcTableMode }
: {}),
@@ -2200,6 +2227,7 @@ export const sourceWizardLogic = kea<sourceWizardLogicType>([
sync_time_of_day: schema.sync_time_of_day,
primary_key_columns: schema.primary_key_columns,
enabled_columns: schema.enabled_columns ?? null,
+ masked_columns: schema.masked_columns ?? null,
row_filters: schema.row_filters ?? null,
...(schema.sync_type === 'cdc' && schema.cdc_table_mode
? { cdc_table_mode: schema.cdc_table_mode }
diff --git a/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts b/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts
index 1cb811d05fb4..c2b8212d0180 100644
--- a/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts
+++ b/products/data_warehouse/frontend/scenes/NewSourceScene/tests/sourceWizardLogic.test.ts
@@ -1128,6 +1128,36 @@ describe('sourceWizardLogic', () => {
}
})
+ it.each([
+ { name: 'cursor', incrementalField: 'Updated_At', primaryKeyColumns: null, kept: ['email'] },
+ { name: 'primary key', incrementalField: null, primaryKeyColumns: ['email'], kept: ['updated_at'] },
+ ])(
+ 'choosing a masked column as the $name drops its mask, since the server refuses it',
+ ({ incrementalField, primaryKeyColumns, kept }) => {
+ const users = buildSchema({ table: 'users' })
+ const orders = buildSchema({ table: 'orders' })
+ const { logic, unmount } = mountWithSchemas([users, orders])
+
+ try {
+ logic.actions.setSchemaMaskedColumns(users, ['email', 'updated_at'])
+ logic.actions.setSchemaMaskedColumns(orders, ['email'])
+ logic.actions.updateSchemaSyncType(
+ users,
+ 'incremental',
+ incrementalField,
+ 'datetime',
+ primaryKeyColumns
+ )
+
+ const byTable = Object.fromEntries(logic.values.databaseSchema.map((s) => [s.table, s]))
+ expect(byTable['users'].masked_columns).toEqual(kept)
+ expect(byTable['orders'].masked_columns).toEqual(['email'])
+ } finally {
+ unmount()
+ }
+ }
+ )
+
it('toggleSchemaGroup skips permission_error rows in a group', () => {
const { logic, unmount } = mountWithSchemas([
buildSchema({ table: 'public.customers' }),
diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
index 3f7c82688435..6ea57f53ce71 100644
--- a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
@@ -760,14 +760,22 @@ function ColumnsAndRowFiltersSection({
// Both editors run in `hideActions` mode and report edits up here, so one Save commits both.
const [draftColumns, setDraftColumns] = useState<string[] | null>(schema.enabled_columns ?? null)
const [draftRowFilters, setDraftRowFilters] = useState<RowFilter[] | null>(schema.row_filters ?? null)
+ const [draftMasked, setDraftMasked] = useState<string[]>(schema.masked_columns ?? [])
+ const [saving, setSaving] = useState(false)
// Reset drafts on schema switch or when server values change (e.g. reload after save).
// Keyed on content so unrelated re-renders don't wipe edits.
useEffect(() => {
setDraftColumns(schema.enabled_columns ?? null)
setDraftRowFilters(schema.row_filters ?? null)
+ setDraftMasked(schema.masked_columns ?? [])
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [schema.id, JSON.stringify(schema.enabled_columns ?? null), JSON.stringify(schema.row_filters ?? null)])
+ }, [
+ schema.id,
+ JSON.stringify(schema.enabled_columns ?? null),
+ JSON.stringify(schema.row_filters ?? null),
+ JSON.stringify(schema.masked_columns ?? []),
+ ])
const alwaysRetained = new Set<string>([
...(schema.primary_key_columns ?? []),
@@ -788,25 +796,57 @@ function ColumnsAndRowFiltersSection({
const rowFilterErrors = validateRowFilters(draftRowFilters ?? [], { availableColumns: rowFilterColumns })
const hasRowFilterErrors = Object.keys(rowFilterErrors).length > 0
+ const savedMasked = schema.masked_columns ?? []
+ const newlyMasked = draftMasked.filter((column) => !savedMasked.includes(column))
+ const unmasked = savedMasked.filter((column) => !draftMasked.includes(column))
+ const masksChanged = newlyMasked.length > 0 || unmasked.length > 0
+
const isDirty =
!sameColumns(draftColumns, schema.enabled_columns ?? null, available) ||
- JSON.stringify(draftRowFilters ?? null) !== JSON.stringify(schema.row_filters ?? null)
+ JSON.stringify(draftRowFilters ?? null) !== JSON.stringify(schema.row_filters ?? null) ||
+ masksChanged
const commit = (resyncAfter: boolean): void => {
- const next = { ...schema, enabled_columns: draftColumns, row_filters: draftRowFilters }
- if (resyncAfter) {
+ // A second save would validate against the old masks and request another rebuild.
+ if (saving) {
+ return
+ }
+ const next = {
+ ...schema,
+ enabled_columns: draftColumns,
+ row_filters: draftRowFilters,
+ masked_columns: draftMasked,
+ }
+ if (resyncAfter || masksChanged) {
// Bypass the bulk-update debounce so resync reads the new config from the DB, not the
// stale one a still-queued PATCH hasn't written yet.
+ // The backend starts the resync itself when it saves a mask change, and it can refuse one.
+ setSaving(true)
void api.externalDataSchemas
- .update(schema.id, { enabled_columns: draftColumns, row_filters: draftRowFilters })
+ .update(schema.id, {
+ enabled_columns: draftColumns,
+ row_filters: draftRowFilters,
+ // Unchanged masks stay out, so a column that left the source can't block the save.
+ ...(masksChanged ? { masked_columns: draftMasked } : {}),
+ })
.then(() => {
updateSchema(next)
- resyncSchema(next)
- lemonToast.success('Saved — full resync queued')
+ if (!masksChanged) {
+ resyncSchema(next)
+ }
+ // The backend only stages the rebuild of a table that does not sync.
+ lemonToast.success(
+ !resyncAfter
+ ? 'Saved'
+ : masksChanged && !schema.should_sync
+ ? 'Saved. The full resync starts when you turn syncing on.'
+ : 'Saved — full resync queued'
+ )
})
.catch((e: any) => {
lemonToast.error(e?.message || "Can't save at this time")
})
+ .finally(() => setSaving(false))
return
}
updateSchema(next)
@@ -815,6 +855,43 @@ function ColumnsAndRowFiltersSection({
const handleSave = (): void => {
const syncType = schema.sync_type
+
+ // A mask change also cancels and restarts a running first sync, which has no last_synced_at yet.
+ const isRunning = schema.status === 'Running'
+ if (masksChanged && (!!schema.last_synced_at || isRunning)) {
+ LemonDialog.open({
+ title: 'Changing masked columns starts a full resync',
+ description: (
+ <div className="flex flex-col gap-2">
+ <span>
+ The table is rebuilt from the source, because rows that are already synced can't be masked
+ or unmasked in place. A masked column no longer matches the same values in tables where it
+ isn't masked.
+ </span>
+ {isRunning && syncType !== 'cdc' && (
+ <span>The sync currently running will be canceled when you save.</span>
+ )}
+ {newlyMasked.length > 0 && newlyMasked.length <= 6 && (
+ <span className="text-xs text-muted">
+ Masking: {newlyMasked.map((c) => `"${c}"`).join(', ')}
+ </span>
+ )}
+ {unmasked.length > 0 && unmasked.length <= 6 && (
+ <span className="text-xs text-muted">
+ Unmasking: {unmasked.map((c) => `"${c}"`).join(', ')}
+ </span>
+ )}
+ </div>
+ ),
+ primaryButton: {
+ children: 'Save and resync',
+ onClick: () => commit(true),
+ },
+ secondaryButton: { children: 'Cancel' },
+ })
+ return
+ }
+
const added = getAddedColumns(schema.enabled_columns ?? null, draftColumns, available)
const requiresPrompt =
!!schema.last_synced_at &&
@@ -850,7 +927,7 @@ function ColumnsAndRowFiltersSection({
<div>
<SectionHeader
title="Columns"
- description="Choose which columns from this table get synced. Primary keys and the active incremental field are always synced."
+ description="Choose which columns from this table get synced, and mask sensitive ones to replace their values with a one-way hash. Primary keys and the active incremental field are always synced and can't be masked."
/>
<div className="border rounded p-4 bg-surface-primary flex flex-col gap-3">
{!hasAvailableColumns || columnSelectionNeedsRefresh ? (
@@ -875,9 +952,18 @@ function ColumnsAndRowFiltersSection({
</div>
) : (
<>
- <span className="text-sm text-secondary">{columnsSummary}</span>
+ <span className="text-sm text-secondary">
+ {columnsSummary}
+ {draftMasked.length > 0 && `, ${draftMasked.length} masked`}
+ </span>
<fieldset disabled={!!editorDisabledReason}>
- <ColumnSelectionPicker hideActions schema={schema} onChange={setDraftColumns} />
+ <ColumnSelectionPicker
+ hideActions
+ enableMasking={source?.access_method !== 'direct'}
+ schema={schema}
+ onChange={setDraftColumns}
+ onMaskedChange={setDraftMasked}
+ />
</fieldset>
</>
)}
@@ -920,6 +1006,7 @@ function ColumnsAndRowFiltersSection({
<LemonButton
type="primary"
onClick={handleSave}
+ loading={saving}
disabledReason={
editorDisabledReason ??
(hasRowFilterErrors
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.test.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.test.tsx
new file mode 100644
index 000000000000..264d55e75b44
--- /dev/null
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.test.tsx
@@ -0,0 +1,58 @@
+import '@testing-library/jest-dom'
+
+import { cleanup, fireEvent, render, screen, within } from '@testing-library/react'
+
+import { initKeaTests } from '~/test/init'
+
+import { ColumnSelectionPicker, ColumnSelectionTarget } from './ColumnSelectionModal'
+
+const COLUMNS = [{ name: 'id' }, { name: 'uuid' }, { name: 'email' }]
+
+const rowFor = (name: string): HTMLElement =>
+ screen.getByText(name, { selector: 'code' }).closest('.justify-between') as HTMLElement
+
+const renderPicker = (schema: Partial<ColumnSelectionTarget>, onMaskedChange = jest.fn()): jest.Mock => {
+ render(
+ <ColumnSelectionPicker
+ hideActions
+ enableMasking
+ schema={{ id: 'users', available_columns: COLUMNS, ...schema }}
+ onMaskedChange={onMaskedChange}
+ />
+ )
+ return onMaskedChange
+}
+
+describe('ColumnSelectionPicker', () => {
+ beforeEach(() => {
+ initKeaTests()
+ })
+
+ afterEach(() => {
+ cleanup()
+ })
+
+ test.each([
+ ['the stored key', { primary_key_columns: ['email'], detected_primary_keys: ['uuid'] }, 'email'],
+ ['the detected key when none is stored', { detected_primary_keys: ['uuid'] }, 'uuid'],
+ ['the id column when no key is known', {}, 'id'],
+ ])('only %s cannot be masked', (_name, keys, key) => {
+ renderPicker(keys)
+
+ for (const { name } of COLUMNS) {
+ const mask = within(rowFor(name)).getByText('Mask').closest('button')
+ expect(mask).toHaveAttribute('aria-disabled', String(name === key))
+ }
+ })
+
+ test.each([
+ ['a masked column that became the key', ['uuid'], 'uuid'],
+ ['a masked column that left the source', ['gone'], 'gone'],
+ ])('%s can still be unmasked', (_name, masked, column) => {
+ const onMaskedChange = renderPicker({ detected_primary_keys: ['uuid'], masked_columns: masked })
+
+ fireEvent.click(within(rowFor(column)).getByText('Masked'))
+
+ expect(onMaskedChange).toHaveBeenLastCalledWith([])
+ })
+})
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsx
index e3ddce948711..11b6a38abf13 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/ColumnSelectionModal.tsx
@@ -1,5 +1,6 @@
import { useEffect, useMemo, useState } from 'react'
+import { IconLock } from '@posthog/icons'
import { LemonButton, LemonCheckbox, LemonInput, LemonModal, Tooltip } from '@posthog/lemon-ui'
import { ExternalDataSourceSchema } from '~/types'
@@ -10,7 +11,10 @@ export interface ColumnSelectionTarget {
id: string
name?: string
enabled_columns?: string[] | null
+ masked_columns?: string[] | null
primary_key_columns?: string[] | null
+ /** The key discovery found, which a sync merges on when no key is stored. */
+ detected_primary_keys?: string[] | null
incremental_field?: string | null
available_columns?: { name: string; data_type?: string; is_nullable?: boolean }[]
}
@@ -22,6 +26,9 @@ interface ColumnSelectionPickerProps {
hideActions?: boolean
/** Fires on every edit, so a parent can drive saving with its own button. */
onChange?: (enabledColumns: string[] | null) => void
+ /** Show a mask toggle on each column. */
+ enableMasking?: boolean
+ onMaskedChange?: (maskedColumns: string[]) => void
}
interface ColumnSelectionModalProps {
@@ -39,6 +46,11 @@ interface UseColumnSelectionResult {
toggleColumn: (name: string, checked: boolean) => void
isAlwaysRetained: (name: string) => boolean
isChecked: (name: string) => boolean
+ isMasked: (name: string) => boolean
+ isMaskable: (name: string) => boolean
+ isSyncKey: (name: string) => boolean
+ toggleMask: (name: string) => void
+ missingMasked: string[]
persistedSelection: () => string[] | null
available: { name: string; data_type?: string; is_nullable?: boolean }[]
primaryKeys: Set<string>
@@ -48,23 +60,26 @@ interface UseColumnSelectionResult {
function useColumnSelection(
schema: ColumnSelectionTarget | null,
- onChange?: (enabledColumns: string[] | null) => void
+ onChange?: (enabledColumns: string[] | null) => void,
+ onMaskedChange?: (maskedColumns: string[]) => void
): UseColumnSelectionResult {
const available = schema?.available_columns ?? []
const primaryKeys = useMemo(() => new Set(schema?.primary_key_columns ?? []), [schema?.primary_key_columns])
const incrementalField = schema?.incremental_field
const [selected, setSelected] = useState<Set<string> | null>(null)
+ const [masked, setMasked] = useState<Set<string>>(new Set())
const [filter, setFilter] = useState('')
useEffect(() => {
const current = schema?.enabled_columns
setSelected(Array.isArray(current) ? new Set(current) : null)
+ setMasked(new Set(schema?.masked_columns ?? []))
setFilter('')
// Stable serialized key: the array ref changes on every poll even with identical contents,
// which would otherwise wipe out in-progress edits.
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [schema?.id, schema?.enabled_columns?.join('\0') ?? null])
+ }, [schema?.id, schema?.enabled_columns?.join('\0') ?? null, schema?.masked_columns?.join('\0') ?? null])
const isAlwaysRetained = (name: string): boolean => primaryKeys.has(name) || name === incrementalField
@@ -111,6 +126,48 @@ function useColumnSelection(
commit(next)
}
+ const commitMasked = (name: string, shouldMask: boolean): void => {
+ const next = new Set(masked)
+ if (shouldMask) {
+ next.add(name)
+ } else {
+ next.delete(name)
+ }
+ setMasked(next)
+ onMaskedChange?.(Array.from(next))
+ }
+
+ const isMasked = (name: string): boolean => masked.has(name)
+
+ // Mirrors the server: without a stored key, a sync merges on the detected key, else on `id`.
+ const syncKeys = useMemo(() => {
+ if (schema?.primary_key_columns?.length) {
+ return new Set(schema.primary_key_columns)
+ }
+ if (schema?.detected_primary_keys?.length) {
+ return new Set(schema.detected_primary_keys)
+ }
+ const idColumn = (schema?.available_columns ?? []).find((column) => column.name.toLowerCase() === 'id')
+ return new Set(idColumn ? [idColumn.name] : [])
+ }, [schema?.primary_key_columns, schema?.detected_primary_keys, schema?.available_columns])
+
+ const isSyncKey = (name: string): boolean => syncKeys.has(name)
+
+ // A hashed primary key breaks merges and a hashed incremental field cannot order rows.
+ const isMaskable = (name: string): boolean => isChecked(name) && !isAlwaysRetained(name) && !isSyncKey(name)
+
+ // A masked column can disappear from the source, and the mask must stay removable.
+ const missingMasked = Array.from(masked).filter((name) => !available.some((column) => column.name === name))
+
+ // A column masked before it became a key can still be unmasked, so the rejected save can be fixed.
+ const toggleMask = (name: string): void => {
+ if (masked.has(name)) {
+ commitMasked(name, false)
+ } else if (isMaskable(name)) {
+ commitMasked(name, true)
+ }
+ }
+
const setSyncAll = (): void => commit(null)
const filteredColumns = useMemo(() => {
@@ -131,6 +188,11 @@ function useColumnSelection(
toggleColumn,
isAlwaysRetained,
isChecked,
+ isMasked,
+ isMaskable,
+ isSyncKey,
+ toggleMask,
+ missingMasked,
persistedSelection,
available,
primaryKeys,
@@ -145,6 +207,8 @@ export function ColumnSelectionPicker({
onCancel,
hideActions,
onChange,
+ enableMasking,
+ onMaskedChange,
}: ColumnSelectionPickerProps): JSX.Element {
const {
filter,
@@ -153,11 +217,16 @@ export function ColumnSelectionPicker({
toggleColumn,
isAlwaysRetained,
isChecked,
+ isMasked,
+ isMaskable,
+ isSyncKey,
+ toggleMask,
+ missingMasked,
persistedSelection,
available,
primaryKeys,
filteredColumns,
- } = useColumnSelection(schema, onChange)
+ } = useColumnSelection(schema, onChange, onMaskedChange)
return (
<div className="flex flex-col gap-2">
@@ -191,7 +260,7 @@ export function ColumnSelectionPicker({
/>
)
return (
- <div key={column.name} className="px-3 py-1">
+ <div key={column.name} className="flex items-center justify-between gap-2 px-3 py-1">
{retained ? (
<Tooltip title="Always synced — required for merges or incremental tracking.">
<div>{checkbox}</div>
@@ -199,12 +268,61 @@ export function ColumnSelectionPicker({
) : (
checkbox
)}
+ {enableMasking &&
+ (isMasked(column.name) ? (
+ <LemonButton
+ size="xsmall"
+ type="tertiary"
+ icon={<IconLock />}
+ onClick={() => toggleMask(column.name)}
+ tooltip="This column syncs as a one-way hash. Click to unmask it and sync its real values again."
+ data-attr="schema-column-unmask"
+ >
+ Masked
+ </LemonButton>
+ ) : (
+ <LemonButton
+ size="xsmall"
+ type="secondary"
+ onClick={() => toggleMask(column.name)}
+ tooltip="Replace this column's values with a one-way hash when syncing. Use it for passwords and personal data. A masked column no longer matches the same values in tables where it isn't masked."
+ disabledReason={
+ isMaskable(column.name)
+ ? undefined
+ : retained || isSyncKey(column.name)
+ ? "Primary key and incremental columns can't be masked"
+ : 'Only synced columns can be masked'
+ }
+ data-attr="schema-column-mask"
+ >
+ Mask
+ </LemonButton>
+ ))}
</div>
)
})}
{filteredColumns.length === 0 && available.length > 0 && (
<div className="text-center text-muted-alt py-6 text-sm">No columns match.</div>
)}
+ {enableMasking &&
+ missingMasked.map((name) => (
+ <div key={name} className="flex items-center justify-between gap-2 px-3 py-1">
+ <div className="flex items-center gap-2">
+ <code>{name}</code>
+ <span className="text-xs text-muted-alt">no longer in the source</span>
+ </div>
+ <LemonButton
+ size="xsmall"
+ type="tertiary"
+ icon={<IconLock />}
+ onClick={() => toggleMask(name)}
+ tooltip="This column is no longer in the source. Click to remove its mask."
+ data-attr="schema-column-unmask-missing"
+ >
+ Masked
+ </LemonButton>
+ </div>
+ ))}
</div>
{!hideActions && (
<div className="flex items-center justify-between gap-2 mt-2">
diff --git a/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsx b/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsx
index 05ce61b5df0f..552d8d5c0578 100644
--- a/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsx
+++ b/products/data_warehouse/frontend/shared/components/forms/SchemaForm.tsx
@@ -1,7 +1,7 @@
import { useActions, useValues } from 'kea'
import { useEffect, useState } from 'react'
-import { IconInfo, IconWarning } from '@posthog/icons'
+import { IconInfo, IconLock, IconWarning } from '@posthog/icons'
import {
LemonBanner,
LemonButton,
@@ -89,6 +89,7 @@ export default function SchemaForm(): JSX.Element {
setSchemaNameFilter,
setSchemaSyncedColumns,
setSchemaRowFilters,
+ setSchemaMaskedColumns,
} = useActions(sourceWizardLogic)
const [columnSelectionSchema, setColumnSelectionSchema] = useState<ExternalDataSourceSyncSchema | null>(null)
const {
@@ -383,12 +384,19 @@ export default function SchemaForm(): JSX.Element {
const summary = !synced
? `All ${schema.available_columns.length}`
: `${syncedCount} of ${schema.available_columns.length}`
+ const maskedCount = schema.masked_columns?.length ?? 0
return (
<div className="justify-end flex">
<LemonButton
className="my-1"
size="small"
type="secondary"
+ icon={maskedCount > 0 ? <IconLock /> : undefined}
+ tooltip={
+ maskedCount > 0
+ ? `${maskedCount} masked ${maskedCount === 1 ? 'column' : 'columns'}`
+ : undefined
+ }
onClick={() => setColumnSelectionSchema(schema)}
disabledReason={schema.permission_error ?? undefined}
>
@@ -655,7 +663,7 @@ export default function SchemaForm(): JSX.Element {
</>
) : null
}
- description="Choose which columns to sync and add row filters to sync only matching rows. Primary-key and incremental columns are always synced."
+ description="Choose which columns to sync, mask sensitive columns, and add row filters to sync only matching rows. Primary-key and incremental columns are always synced and can't be masked."
isOpen={columnSelectionSchema !== null}
onClose={() => setColumnSelectionSchema(null)}
footer={
@@ -669,13 +677,16 @@ export default function SchemaForm(): JSX.Element {
<h4 className="font-semibold mb-0">Columns to sync</h4>
<ColumnSelectionPicker
hideActions
+ enableMasking={!isDirectQueryMode}
schema={
columnSelectionSchema
? {
id: columnSelectionSchema.table,
name: columnSelectionSchema.table,
enabled_columns: columnSelectionSchema.enabled_columns,
+ masked_columns: columnSelectionSchema.masked_columns,
primary_key_columns: columnSelectionSchema.primary_key_columns,
+ detected_primary_keys: columnSelectionSchema.detected_primary_keys,
incremental_field: columnSelectionSchema.incremental_field,
available_columns: columnSelectionSchema.available_columns.map((c) => ({
name: c.field,
@@ -690,6 +701,11 @@ export default function SchemaForm(): JSX.Element {
setSchemaSyncedColumns(columnSelectionSchema, enabledColumns)
}
}}
+ onMaskedChange={(maskedColumns) => {
+ if (columnSelectionSchema) {
+ setSchemaMaskedColumns(columnSelectionSchema, maskedColumns)
+ }
+ }}
/>
</div>
{columnSelectionSchema?.sync_type !== 'cdc' && (
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 2cf04f938642..69abf148d872 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -11153,6 +11153,10 @@ export namespace Schemas {
* A schema of an external data source: its sync configuration and the warehouse table it syncs into.
*/
enabled_columns?: (Array<string> | null) | undefined;
+ /**
+ * A schema of an external data source: its sync configuration and the warehouse table it syncs into.
+ */
+ masked_columns?: (Array<string> | null) | undefined;
/**
* A schema of an external data source: its sync configuration and the warehouse table it syncs into.
*/
@@ -11238,6 +11242,7 @@ export namespace Schemas {
primary_key_columns?: (Array<string> | null) | undefined;
cdc_table_mode?: (CdcTableModeEnum | NullEnum) | undefined;
enabled_columns?: (Array<string> | null) | undefined;
+ masked_columns?: (Array<string> | null) | undefined;
row_filters?:
| (Array<{
column: string;
diff --git a/products/warehouse_sources/backend/facade/models.py b/products/warehouse_sources/backend/facade/models.py
index 263fe85bd193..da62a989e267 100644
--- a/products/warehouse_sources/backend/facade/models.py
+++ b/products/warehouse_sources/backend/facade/models.py
@@ -14,7 +14,10 @@ Consumers that only read fields should use ``facade.api`` (contracts) instead.
from products.warehouse_sources.backend.facade.contracts import UnsupportedSyncTypeError
from products.warehouse_sources.backend.models.column_annotation import WarehouseColumnAnnotation
-from products.warehouse_sources.backend.models.column_statistics import WarehouseColumnStatistics
+from products.warehouse_sources.backend.models.column_statistics import (
+ WarehouseColumnStatistics,
+ clear_masked_column_bounds,
+)
from products.warehouse_sources.backend.models.credential import (
DataWarehouseCredential,
get_or_create_datawarehouse_credential,
@@ -125,6 +128,7 @@ __all__ = [
"latest_completed_job_prefetch",
"get_or_create_datawarehouse_credential",
"get_or_create_warehouse_destination",
+ "clear_masked_column_bounds",
"clickhouse_column_to_dwh_column",
"clickhouse_columns_to_dwh_columns",
"motherduck_column_to_dwh_column",
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.py
index 1db9c1e7b049..2b374eec0098 100644
--- a/products/warehouse_sources/backend/facade/source_management.py
+++ b/products/warehouse_sources/backend/facade/source_management.py
@@ -43,7 +43,9 @@ _LAZY = {
"reserved_column_names": "cdc.batcher",
"BUFFER_LANE": "cdc.snapshot_lane",
"CDC_RESET_PENDING_KEY": "cdc.snapshot_lane",
+ "hand_reset_to_capture": "cdc.snapshot_lane",
"hand_reset_to_capture_if_sync_running": "cdc.snapshot_lane",
+ "stage_handed_over_reset": "cdc.snapshot_lane",
"resnapshot_stays_in_buffer": "cdc.snapshot_lane",
"bigquery_client": "sources.bigquery.bigquery",
"is_bigquery_auth_error": "sources.bigquery.bigquery",
@@ -73,6 +75,10 @@ _LAZY = {
"discover_trino_catalog": "sources.trino.trino",
"build_default_schemas": "sources.common.schema",
"build_default_sync_settings": "sources.common.schema",
+ "ambiguous_masked_columns": "pipelines.core.masking",
+ "collision_alias_masked_columns": "pipelines.core.masking",
+ "fold_column_name": "naming_convention",
+ "masking_key_unavailable": "pipelines.core.masking",
"RowFilterValidationError": "sources.common.sql",
"filter_dwh_columns_by_enabled_columns": "sources.common.sql",
"sql_schema_metadata": "sources.common.sql",
diff --git a/products/warehouse_sources/backend/migrations/0196_externaldataschema_masked_columns.py b/products/warehouse_sources/backend/migrations/0196_externaldataschema_masked_columns.py
new file mode 100644
index 000000000000..aa9db41dd520
--- /dev/null
+++ b/products/warehouse_sources/backend/migrations/0196_externaldataschema_masked_columns.py
@@ -0,0 +1,13 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [("warehouse_sources", "0195_repin_rippling_api_version_v2")]
+
+ operations = [
+ migrations.AddField(
+ model_name="externaldataschema",
+ name="masked_columns",
+ field=models.JSONField(blank=True, default=None, null=True),
+ ),
+ ]
diff --git a/products/warehouse_sources/backend/migrations/max_migration.txt b/products/warehouse_sources/backend/migrations/max_migration.txt
index c1111aeb707e..b81047bedf07 100644
--- a/products/warehouse_sources/backend/migrations/max_migration.txt
+++ b/products/warehouse_sources/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0195_repin_rippling_api_version_v2
+0196_externaldataschema_masked_columns
diff --git a/products/warehouse_sources/backend/models/column_statistics.py b/products/warehouse_sources/backend/models/column_statistics.py
index 7aa7e8db8009..cb10fb62f98f 100644
--- a/products/warehouse_sources/backend/models/column_statistics.py
+++ b/products/warehouse_sources/backend/models/column_statistics.py
@@ -1,8 +1,12 @@
+import uuid
+
from django.db import models
from posthog.models.scoping.root_mixin import TeamScopedRootMixin
from posthog.models.utils import CreatedMetaFields, UpdatedMetaFields, UUIDTModel, sane_repr
+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name
+
class WarehouseColumnStatistics(TeamScopedRootMixin, CreatedMetaFields, UpdatedMetaFields, UUIDTModel):
"""Per-column data profile for a synced warehouse table, surfaced to the AI agent.
@@ -60,3 +64,16 @@ class WarehouseColumnStatistics(TeamScopedRootMixin, CreatedMetaFields, UpdatedM
constraints = [
models.UniqueConstraint(fields=["table", "column_name"], name="unique_table_column_statistics"),
]
+
+
+def clear_masked_column_bounds(team_id: int, table_id: uuid.UUID | None, masked_columns: list[str] | None) -> None:
+ """Drop the stored min and max of masked columns. They hold plaintext from before the mask."""
+ masked = {fold_column_name(column) for column in masked_columns or []}
+ if not masked or table_id is None:
+ return
+ statistics = WarehouseColumnStatistics.objects.for_team(team_id).filter(table_id=table_id, has_min_max=True)
+ stale = [
+ stat_id for stat_id, name in statistics.values_list("id", "column_name") if fold_column_name(name) in masked
+ ]
+ if stale:
+ statistics.filter(id__in=stale).update(min_value=None, max_value=None, has_min_max=False)
diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.py
index 4bee659b1784..6b87d3d1b765 100644
--- a/products/warehouse_sources/backend/models/external_data_schema.py
+++ b/products/warehouse_sources/backend/models/external_data_schema.py
@@ -375,6 +375,8 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djan
enabled_columns = models.JSONField(null=True, blank=True, default=None)
# null (default) = sync all rows. List of {column, operator, value} predicates ANDed onto the WHERE clause.
row_filters = models.JSONField(null=True, blank=True, default=None)
+ # null (default) = no masking. Source column names whose values the sync replaces with a one-way digest.
+ masked_columns = models.JSONField(null=True, blank=True, default=None)
objects = ExternalDataSchemaQuerySet.as_manager()
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index ad2ce8551e59..d533bfd80fa1 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -55,6 +55,7 @@ from products.warehouse_sources.backend.facade.models import (
ExternalDataSchemaDestination,
ExternalDataSource,
UnsupportedSyncTypeError,
+ clear_masked_column_bounds,
discovered_schema_finder,
is_pinned_schema,
mark_schema_running_unless_halted,
@@ -77,14 +78,19 @@ from products.warehouse_sources.backend.facade.source_management import (
SourceSchema,
WebhookSource,
add_table_failure_message,
+ ambiguous_masked_columns,
+ collision_alias_masked_columns,
filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,
+ fold_column_name,
get_cdc_adapter,
+ hand_reset_to_capture,
hand_reset_to_capture_if_sync_running,
history_naming_unless_taken,
history_table_name,
is_another_schemas_history_table,
is_history_table_mode,
known_source_key,
+ masking_key_unavailable,
new_history_naming,
pin_history_naming,
pinned_history_naming,
@@ -94,6 +100,7 @@ from products.warehouse_sources.backend.facade.source_management import (
reserved_column_names,
resnapshot_stays_in_buffer,
source_type_supports_cdc,
+ stage_handed_over_reset,
tables_wait_for_repair,
validate_and_coerce_row_filters,
)
@@ -538,6 +545,176 @@ def _source_column_names(instance: ExternalDataSchema) -> set[str]:
return {column["name"] for column in columns if isinstance(column, dict) and isinstance(column.get("name"), str)}
+MASKING_KEY_UNAVAILABLE_ERROR = (
+ "Column masking isn't set up on this PostHog instance. "
+ "Set WAREHOUSE_COLUMN_MASKING_KEY to a secret of at least 32 characters, then try again."
+)
+
+
+def masked_key_conflict_error(
+ masked_columns: list[str],
+ primary_key_columns: Iterable[str] | None,
+ incremental_fields: Iterable[str | None],
+ partition_keys: Iterable[str] = (),
+) -> str | None:
+ # A partition column is read as a date or a number to pick the partition, which a digest is not.
+ protected = {
+ fold_column_name(column)
+ for column in [*(primary_key_columns or []), *partition_keys, *incremental_fields]
+ if column
+ }
+ conflicting = [column for column in masked_columns if fold_column_name(column) in protected]
+ if conflicting:
+ return f"Primary-key, incremental-field and partition columns can't be masked: {sorted(conflicting)}."
+ return None
+
+
+def masked_columns_error(
+ masked_columns: list[str],
+ *,
+ source_type: str,
+ is_direct_query: bool,
+ source_columns: Iterable[object],
+ primary_key_columns: Iterable[str] | None,
+ incremental_fields: Iterable[str | None],
+ partition_keys: Iterable[str] = (),
+) -> str | None:
+ """Why this mask cannot be saved, or None. `source_columns` is empty when no columns are known yet."""
+ # A direct query source never syncs, so a saved mask would protect nothing.
+ if is_direct_query:
+ return "Column masking is not supported for direct query sources."
+ # A managed-schema source declares fixed column types, which a string digest breaks.
+ if not source_supports_column_selection(source_type):
+ return "Column masking is not supported for this source type."
+ if masking_key_unavailable():
+ return MASKING_KEY_UNAVAILABLE_ERROR
+ known = [name for name in source_columns if isinstance(name, str)]
+ # A mask that names no real column hashes nothing, so the intended column would sync in plaintext.
+ if not known:
+ return "Can't mask columns before the table's columns are known. Sync the table once, then try again."
+ known_folded = {fold_column_name(name) for name in known}
+ unknown = [column for column in masked_columns if fold_column_name(column) not in known_folded]
+ if unknown:
+ return (
+ f"Unknown columns in masked_columns: {sorted(unknown)}. "
+ "Run `Pull new schemas` to refresh available columns."
+ )
+ ambiguous = ambiguous_masked_columns(masked_columns, known)
+ if ambiguous:
+ return (
+ f"Can't mask columns whose names collide after normalization: {sorted(ambiguous)}. "
+ "Rename the source columns so they differ by more than case."
+ )
+ return masked_key_conflict_error(masked_columns, primary_key_columns, incremental_fields, partition_keys)
+
+
+def canonical_masked_columns(masked_columns: list[str], source_columns: Iterable[str]) -> list[str]:
+ """Each mask under its column's own name, once. The UI compares names exactly."""
+ by_folded: dict[str, str] = {}
+ for name in source_columns:
+ by_folded.setdefault(fold_column_name(name), name)
+ return list(dict.fromkeys(by_folded.get(fold_column_name(column), column) for column in masked_columns))
+
+
+def _stage_mask_rebuild(config: dict[str, Any]) -> None:
+ config["reset_pipeline"] = True
+
+
+def _stage_disabled_cdc_mask_snapshot(config: dict[str, Any]) -> None:
+ config["cdc_mode"] = "snapshot"
+ config["reset_pipeline"] = True
+ config.pop(CDC_SNAPSHOT_LANE_KEY, None)
+ config.pop(CDC_RESET_PENDING_KEY, None)
+
+
+def _validate_masked_columns(
+ instance: ExternalDataSchema, validated_data: dict[str, Any], data: dict[str, Any]
+) -> bool:
+ """Reject a mask the sync cannot honor. Returns whether the request changes the masked set."""
+ mask_in_request = "masked_columns" in validated_data
+ masked_columns = (validated_data["masked_columns"] if mask_in_request else instance.masked_columns) or []
+
+ # Also checked when only the key or the cursor moves, so neither can land on a masked column.
+ if masked_columns and (mask_in_request or "primary_key_columns" in data or "incremental_field" in data):
+ metadata_columns = (instance.schema_metadata or {}).get("columns")
+ source_columns: list[str] = [
+ name
+ for column in (metadata_columns if isinstance(metadata_columns, list) else [])
+ if isinstance(column, dict) and isinstance(name := column.get("name"), str)
+ ]
+ synced_columns = (
+ [column["name"] for column in instance.table.get_user_facing_columns()]
+ if instance.table is not None
+ else []
+ )
+ # Non-SQL sources record no column metadata, so their synced columns are the known ones.
+ if not source_columns:
+ source_columns = synced_columns
+ # Some sources record their columns after normalization, so the metadata can list an alias too.
+ aliases = collision_alias_masked_columns(masked_columns, [*source_columns, *synced_columns])
+ if aliases:
+ raise ValidationError(
+ f"Can't mask {sorted(aliases)}: the column was renamed because two source columns share its "
+ "name. Rename one of the source columns, then mask it."
+ )
+ primary_key_columns = [*(data.get("primary_key_columns") or []), *(instance.primary_key_columns or [])]
+ # Without a stored key, a sync merges and partitions on `id` when the table has one.
+ if not primary_key_columns:
+ primary_key_columns = [name for name in source_columns if name.lower() == "id"][:1]
+ # The stored key and cursor count too: an update can leave them in place, for example when a
+ # CDC table ignores the incremental field in the request. A full-refresh table keeps the cursor
+ # of an earlier sync method in its settings, but reads no cursor.
+ sync_type = data.get("sync_type") or instance.sync_type
+ stored_cursor = instance.incremental_field if sync_type != ExternalDataSchema.SyncType.FULL_REFRESH else None
+ incremental_fields = [data.get("incremental_field"), stored_cursor]
+ partition_keys = [*(instance.partitioning_keys or []), *(instance.partitioning_keys_override or [])]
+ # A stored mask whose column left the source must not block unrelated edits, so only new masks
+ # are checked against the known columns. Every mask is still checked against the keys.
+ stored = {fold_column_name(column) for column in instance.masked_columns or []}
+ new_masks = [column for column in masked_columns if fold_column_name(column) not in stored]
+ error = (
+ masked_columns_error(
+ new_masks,
+ source_type=instance.source.source_type,
+ is_direct_query=instance.source.is_direct_query,
+ source_columns=source_columns,
+ primary_key_columns=primary_key_columns,
+ incremental_fields=incremental_fields,
+ partition_keys=partition_keys,
+ )
+ if new_masks
+ else None
+ ) or masked_key_conflict_error(masked_columns, primary_key_columns, incremental_fields, partition_keys)
+ if error:
+ raise ValidationError(error)
+ if mask_in_request:
+ masked_columns = canonical_masked_columns(masked_columns, source_columns)
+ validated_data["masked_columns"] = masked_columns
+
+ # An omitted field is unchanged. Without this, any other edit to a masked table would read as
+ # "remove every mask" and start a full resync.
+ if not mask_in_request:
+ return False
+ changed = {fold_column_name(column) for column in (instance.masked_columns or [])} != {
+ fold_column_name(column) for column in masked_columns
+ }
+ if not changed:
+ return False
+ # A mask change rebuilds the table from the source. A webhook table keeps its rows through a
+ # reset, because the source cannot send them again, so the old values would stay as they are.
+ if instance.is_webhook and instance.table is not None:
+ raise ValidationError(
+ "Can't change masked columns on a synced webhook table, because its data can't be fetched "
+ "again from the source. Delete and recreate the table to change masking."
+ )
+ if is_any_external_data_schema_paused(instance.team_id):
+ raise ValidationError(
+ "Monthly sync limit reached. Please increase your billing limit before changing "
+ "masked columns, because the change needs a full resync."
+ )
+ return True
+
+
def _refuse_reserved_columns(
instance: ExternalDataSchema,
*,
@@ -750,6 +927,19 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"even if not listed here."
),
)
+ masked_columns = serializers.ListField(
+ child=serializers.CharField(),
+ required=False,
+ allow_null=True,
+ help_text=(
+ "Names of source columns whose values are replaced with a deterministic one-way digest at "
+ "sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. "
+ "Primary-key, partition and incremental columns can't be masked, and direct query "
+ "sources don't support masking. Any change to this list starts a full resync of the table "
+ "(CDC tables take a new snapshot), or stages it until syncing is turned on. A synced "
+ "webhook table can't change its masked columns, because its data can't be fetched again."
+ ),
+ )
row_filters = RowFiltersField(
required=False,
allow_null=True,
@@ -840,6 +1030,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"cdc_table_mode",
"incremental_sync_blocked",
"enabled_columns",
+ "masked_columns",
"row_filters",
"row_filter_columns",
"available_columns",
@@ -1114,6 +1305,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
instance: ExternalDataSchema,
validated_data: dict[str, Any],
original_sync_type_config: dict[str, Any],
+ mutate_locked_config: Callable[[dict[str, Any]], None] | None = None,
) -> ExternalDataSchema:
"""Persist the update onto a freshly-locked row, writing only the fields this request changed.
@@ -1138,6 +1330,8 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
merged.update(changed)
for key in removed:
merged.pop(key, None)
+ if mutate_locked_config is not None:
+ mutate_locked_config(merged)
validated_data["sync_type_config"] = merged
# Apply to the locked row rather than the request's copy. The copy still holds whatever
# the other writer replaced, and activity logging diffs the stored row against the
@@ -1227,6 +1421,8 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"Column metadata is unavailable. Run `Pull new schemas` before selecting source columns."
)
+ masked_columns_changed = _validate_masked_columns(instance, validated_data, data)
+
# Validate against the schema's columns; raw filters are persisted as-is and re-coerced at sync time.
if "row_filters" in validated_data and validated_data["row_filters"] is not None:
# Only sources that push filters into their query (SQL WHERE) can honor them — a
@@ -1679,6 +1875,28 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
validated_data["sync_type_config"]["reset_pipeline"] = True
trigger_refresh = True
+ # Rows already synced hold the old values, and the column changes type between the source
+ # type and text, so a mask change rebuilds the table. CDC takes a new snapshot further down.
+ # The rebuild is staged on the locked row: a run can clear an earlier reset, or repair can stage
+ # one, between this request's read and its save.
+ stage_locked_config: Callable[[dict[str, Any]], None] | None = None
+ if masked_columns_changed:
+ will_sync = should_sync if should_sync is not None else instance.should_sync
+ if not is_cdc:
+ validated_data.setdefault("sync_type_config", instance.sync_type_config)
+ stage_locked_config = _stage_mask_rebuild
+ if will_sync:
+ trigger_refresh = True
+ elif will_sync:
+ # Staged in this save, so capture still finishes the reset if the hand-over after it fails.
+ stage_locked_config = stage_handed_over_reset
+ else:
+ # Capture does not read a table that does not sync, so the new snapshot is staged in this
+ # save. A self-managed publication takes no snapshot of its own when the table is turned on.
+ stage_locked_config = _stage_disabled_cdc_mask_snapshot
+ instance.initial_sync_complete = False
+ validated_data["initial_sync_complete"] = False
+
if source.is_direct_query:
direct_engine_adapter = get_direct_query_engine(source.direct_engine)
# Direct-mode lifecycle hooks that need a fresh DataWarehouseTable projection:
@@ -1732,14 +1950,25 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
# Persist under a row lock, replaying only the sync_type_config keys this request changed
# so a concurrent CDC extract activity's writes aren't reverted by the full-instance save.
- updated_instance = self._save_merging_sync_type_config(instance, validated_data, original_sync_type_config)
+ updated_instance = self._save_merging_sync_type_config(
+ instance, validated_data, original_sync_type_config, stage_locked_config
+ )
if publication_removal is not None:
self._run_temporal_side_effect(publication_removal)
+ if masked_columns_changed:
+ # Profiles computed before the mask hold plaintext bounds, and the profiler recomputes them only daily.
+ clear_masked_column_bounds(
+ updated_instance.team_id, updated_instance.table_id, updated_instance.masked_columns
+ )
+
# A version repin invalidates any in-flight import: retried/resumed activities re-resolve
# the version from the DB, so letting the run finish would mix two vendor API versions in
# one table. Cancel it; the user decides when to sync again.
- if api_version_changed:
+ # A mask change cancels it too, as the resync button does, so the masking rebuild starts now
+ # instead of waiting for the next schedule tick.
+ import_cancelled = False
+ if api_version_changed or (masked_columns_changed and not is_cdc):
running_job = (
ExternalDataJob.objects.filter(schema_id=instance.pk, team_id=instance.team_id)
.order_by("-created_at")
@@ -1752,12 +1981,15 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
cancel_external_data_workflow(running_job.workflow_id)
except temporalio.service.RPCError as e:
logger.exception(
- "Could not cancel running workflow after api_version change",
+ "Could not cancel running workflow after a schema change",
schema_id=str(instance.id),
+ workflow_id=running_job.workflow_id,
+ reason="api_version_change" if api_version_changed else "mask_change",
exc_info=e,
)
self._run_temporal_side_effect(cancel_running_import)
+ import_cancelled = True
if source.supports_scheduled_sync and (
should_sync is not None
@@ -1821,7 +2053,9 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
self._run_temporal_side_effect(update_schedule)
if trigger_refresh:
- self._run_temporal_side_effect(lambda: trigger_external_data_workflow(updated_instance))
+ self._run_temporal_side_effect(
+ lambda: trigger_external_data_workflow(updated_instance, buffer_one=import_cancelled)
+ )
if sync_type == ExternalDataSchema.SyncType.WEBHOOK:
self._maybe_create_webhook(updated_instance)
@@ -1837,10 +2071,17 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
self._run_temporal_side_effect(sync_cdc_schedule)
+ # Capture finishes this reset before it reads the change stream again, and it empties the change
+ # buffer first: files captured before the change hold the old values and the old column type.
+ cdc_mask_reset = is_cdc and masked_columns_changed and updated_instance.should_sync
+ if cdc_mask_reset:
+ # The save above staged the reset, so the hand-over only pauses the schedule and starts capture.
+ self._run_temporal_side_effect(lambda: hand_reset_to_capture(updated_instance, logger, stage=False))
+
# If the cdc_table_mode change added a new physical write target, kick a full re-snapshot so the
# new table is seeded from the current source state. `_seed_cdc_companion_from_snapshot` runs
# automatically once the snapshot completes via `run_post_load_operations`.
- if is_cdc and "cdc_table_mode" in data:
+ if is_cdc and "cdc_table_mode" in data and not cdc_mask_reset:
new_cdc_table_mode = data.get("cdc_table_mode")
if _cdc_table_mode_change_needs_resnapshot(previous_cdc_table_mode, new_cdc_table_mode):
logger.info(
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py
index 003cd0da0235..4920df29ec0b 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/schema_operations.py
@@ -125,6 +125,16 @@ class ExternalDataSourceBulkUpdateSchemaSerializer(serializers.Serializer):
allow_empty=True,
help_text="Columns to sync. Null means sync all columns.",
)
+ masked_columns = serializers.ListField(
+ child=serializers.CharField(),
+ required=False,
+ allow_null=True,
+ allow_empty=True,
+ help_text=(
+ "Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. "
+ "Any change starts a full resync of the table."
+ ),
+ )
row_filters = RowFiltersField(
required=False,
allow_null=True,
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index df5401d30236..8180f1e74788 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -100,6 +100,8 @@ from products.warehouse_sources.backend.presentation.views.external_data_schema
PINNED_TABLE_IDS_CONTEXT_KEY,
ExternalDataSchemaListSerializer,
ExternalDataSchemaSerializer,
+ canonical_masked_columns,
+ masked_columns_error,
pinned_table_ids,
source_supports_column_selection,
unsupported_row_filter_reason,
@@ -1184,6 +1186,14 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
request=self.request,
)
+ @staticmethod
+ def _delete_new_source(source: ExternalDataSource, cdc_adapter: CDCSourceAdapter | None) -> None:
+ # A hard delete leaves no source row for the orphan-slot sweeper to find, so drop the slot
+ # and publication created for this source first.
+ if cdc_adapter is not None:
+ cdc_adapter.cleanup_resources(source)
+ SourceInputsStore.discard_created(source)
+
def _create_external_data_source(
self,
request: Request,
@@ -1633,15 +1643,30 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
payload_row_filters if isinstance(payload_row_filters, list) and payload_row_filters else None
)
+ payload_masked_columns = schema.get("masked_columns")
+ # Dropping a malformed mask would sync the column in plaintext, so reject it.
+ if payload_masked_columns is not None and not (
+ isinstance(payload_masked_columns, list)
+ and all(isinstance(column, str) for column in payload_masked_columns)
+ ):
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={
+ "message": f"masked_columns for schema '{schema.get('name')}' must be a list of column names or null."
+ },
+ )
+ masked_columns: list[str] | None = payload_masked_columns or None
+
if should_sync and requires_incremental_fields and incremental_field is None:
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"message": "Incremental schemas given do not have an incremental field set"},
)
if should_sync and requires_incremental_fields and incremental_field_type is None:
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"message": "Incremental schemas given do not have an incremental field type set"},
@@ -1667,7 +1692,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
and not source_schema.detected_primary_keys # type: ignore[union-attr]
and not has_id_column
):
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
return Response(
data={
"message": f"Table '{schema_name}' has no primary key to sync incrementally on. "
@@ -1719,7 +1744,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
# Only sources that push filters into their query (SQL WHERE) can honor them — a
# saved-but-ignored filter would silently sync unfiltered rows.
if not source.supports_row_filters:
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={
@@ -1730,7 +1755,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
if reason := unsupported_row_filter_reason(
is_direct_query=new_source_model.is_direct_query, is_cdc=sync_type == "cdc"
):
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"message": f"Row filter not allowed for schema '{schema_name}': {reason}"},
@@ -1740,7 +1765,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
row_filters, schema_metadata, source.row_filter_columns_for_schema(schema_name)
)
except RowFilterValidationError as e:
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"message": f"Invalid row filter for schema '{schema_name}': {e}"},
@@ -1777,7 +1802,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
# bool is an int subclass — exclude it so true/false aren't treated as 1/0.
is_valid_int = isinstance(lookback_seconds, int) and not isinstance(lookback_seconds, bool)
if not is_valid_int or not (0 <= lookback_seconds <= 5_184_000):
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={
@@ -1819,6 +1844,28 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
else:
sync_type_config = {"schema_metadata": schema_metadata}
+ if masked_columns:
+ masking_error = masked_columns_error(
+ masked_columns,
+ source_type=source_type,
+ is_direct_query=is_direct_query,
+ source_columns=[column[0] for column in introspected_columns],
+ # A sync merges and partitions on `id` when no key is stored or detected.
+ primary_key_columns=sync_type_config.get("primary_key_columns")
+ or (source_schema.detected_primary_keys if source_schema else None)
+ or [column[0] for column in introspected_columns if column[0].lower() == "id"][:1],
+ incremental_fields=[incremental_field],
+ )
+ if masking_error:
+ self._delete_new_source(new_source_model, cdc_adapter if cdc_enabled else None)
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={"message": f"Column masking not allowed for schema '{schema_name}': {masking_error}"},
+ )
+ masked_columns = canonical_masked_columns(
+ masked_columns, [column[0] for column in introspected_columns]
+ )
+
# CDC schemas benefit from a tighter poll cadence — the extraction workflow is cheap
# and the value prop is near-real-time. Other sync types use the 6h default.
schema_sync_frequency_interval = (
@@ -1838,6 +1885,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
label=schema_label_by_name.get(schema_name),
sync_frequency_interval=schema_sync_frequency_interval,
enabled_columns=enabled_columns,
+ masked_columns=masked_columns,
row_filters=row_filters,
)
@@ -1859,10 +1907,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
cast(str, metadata_source_schema),
cast(str, metadata_source_table_name),
)
- # A hard delete leaves no source row for the orphan-slot sweeper to find, so
- # drop the slot and publication created above before deleting the source.
- cdc_adapter.cleanup_resources(new_source_model)
- SourceInputsStore.discard_created(new_source_model)
+ self._delete_new_source(new_source_model, cdc_adapter)
return Response(status=status.HTTP_400_BAD_REQUEST, data={"message": message})
if direct_engine_adapter is not None and is_direct_query and should_sync:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
index 7949de648778..a450a7e5d671 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
@@ -73,6 +73,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import
MAX_FRIENDLY_MESSAGE_LENGTH,
CDCErrorCategory,
CDCErrorInfo,
+ CDCMaskedKeyError,
+ CDCMaskingKeyError,
CDCReservedColumnError,
CDCSlotNotConfiguredError,
classify_cdc_error,
@@ -96,6 +98,11 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager
snapshot_can_start_in_buffer,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import (
+ protected_masked_columns,
+ value_masker,
+)
from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException
from products.warehouse_sources.backend.temporal.data_imports.workflow_activities.create_job_model import (
_build_schema_snapshot,
@@ -206,6 +213,11 @@ class CDCExtractActivity:
self.split_key_columns: dict[str, list[str]] = {}
# Missing entry = sync all columns; otherwise the set is the projection (always includes PKs).
self.enabled_columns_by_table: dict[str, set[str]] = {}
+ # Folded names of the columns to mask. Missing entry = mask nothing.
+ self.masked_columns_by_table: dict[str, set[str]] = {}
+ # Whether each raw column name is masked, per table. Folding a name costs too much per event.
+ self._masked_names_by_table: dict[str, dict[str, bool]] = {}
+ self._masker: Callable[[object], str | None] | None = None
self.history_naming_by_table: dict[str, HistoryNaming] = {}
self.adapter: typing.Any = None
self.reader: typing.Any = None
@@ -644,6 +656,32 @@ class CDCExtractActivity:
retained.add(inc)
self.enabled_columns_by_table[schema.name] = retained
+ self._load_masked_columns(schema)
+
+ def _read_masked_columns(self, schema: ExternalDataSchema) -> list[str] | None:
+ return (
+ ExternalDataSchema.objects.filter(id=schema.id, team_id=schema.team_id)
+ .values_list("masked_columns", flat=True)
+ .first()
+ )
+
+ def _load_masked_columns(self, schema: ExternalDataSchema) -> None:
+ self._masked_names_by_table.pop(schema.name, None)
+ if not (isinstance(schema.masked_columns, list) and schema.masked_columns):
+ self.masked_columns_by_table.pop(schema.name, None)
+ return
+ # The API checks a mask against the stored key, but a table saved without one gets
+ # its key from the catalog above. Fail before any change of the table is read.
+ masked_keys = protected_masked_columns(schema.masked_columns, self.pk_columns_by_table.get(schema.name, []))
+ if masked_keys:
+ raise CDCMaskedKeyError(schema.name, masked_keys)
+ if self._masker is None:
+ try:
+ self._masker = value_masker(self.inputs.team_id)
+ except NonRetryableException as e:
+ raise CDCMaskingKeyError(str(e)) from e
+ self.masked_columns_by_table[schema.name] = {fold_column_name(column) for column in schema.masked_columns}
+
def _query_pk_columns(self, table_names: list[str]) -> dict[str, list[str]]:
"""Look up primary keys in the source catalog, keyed back on `ExternalDataSchema.name`.
@@ -702,6 +740,30 @@ class CDCExtractActivity:
omitted_columns=filtered_omitted,
)
+ def _mask_event_columns(self, event: ChangeEvent) -> ChangeEvent:
+ masked = self.masked_columns_by_table.get(event.table_name)
+ if not masked:
+ return event
+ assert self._masker is not None
+ mask = self._masker
+ decisions = self._masked_names_by_table.setdefault(event.table_name, {})
+
+ def is_masked(name: str) -> bool:
+ decision = decisions.get(name)
+ if decision is None:
+ decision = decisions[name] = fold_column_name(name) in masked
+ return decision
+
+ columns = {name: mask(value) if is_masked(name) else value for name, value in event.columns.items()}
+ # A digest is text. Left on the source type, a flush where the column is all null would
+ # write that type and no longer merge with the flushes that hold digests.
+ column_types = event.column_types
+ if column_types is not None:
+ column_types = {
+ name: pa.string() if is_masked(name) else column_type for name, column_type in column_types.items()
+ }
+ return dataclasses.replace(event, columns=columns, column_types=column_types)
+
def _split_key_change(self, event: ChangeEvent) -> tuple[ChangeEvent, ...]:
"""Turn an update that changes the merge key into a delete of the old key and an insert of the new one.
@@ -808,7 +870,7 @@ class CDCExtractActivity:
event = dataclasses.replace(event, table_name=canonical_name)
for change in self._split_key_change(event):
- self.batcher.add(self._project_event_columns(change))
+ self.batcher.add(self._mask_event_columns(self._project_event_columns(change)))
# A change in position_serialized proves the previous transaction fully
# yielded — all of its events are now buffered or flushed. Record its end LSN
@@ -1027,6 +1089,10 @@ class CDCExtractActivity:
mutate=_merge_pending,
extra_model_fields={"initial_sync_complete": False},
)
+ # Read after the locked write above. A mask change saves its reset together with the masks,
+ # so every reset that write merged and this run finishes comes with masks at least this new.
+ schema.masked_columns = self._read_masked_columns(schema)
+ self._load_masked_columns(schema)
self._tables_awaiting_reset.discard(schema.name)
return True
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
index 45e140eb5a18..66f65255c979 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
@@ -39,6 +39,8 @@ class CDCErrorCategory(enum.StrEnum):
WAL_DECODE_ERROR = "wal_decode_error"
TRANSACTION_TOO_LARGE = "transaction_too_large"
RESERVED_COLUMN_CONFLICT = "reserved_column_conflict"
+ MASKED_KEY_CONFLICT = "masked_key_conflict"
+ MASKING_KEY_UNAVAILABLE = "masking_key_unavailable"
UNKNOWN = "unknown"
@@ -84,6 +86,26 @@ class CDCReservedColumnError(Exception):
"""
+class CDCMaskingKeyError(Exception):
+ """A table masks columns, but the instance has no secret key to hash them with.
+
+ Non-retryable: the key is deployment configuration, so replaying re-fails until it is set.
+ """
+
+
+class CDCMaskedKeyError(Exception):
+ """A table masks a column that capture resolved as its primary key.
+
+ Non-retryable: the conflict is in the table's configuration, so replaying re-fails. Capture
+ cannot hash a merge key, and it must not write the masked column to the buffer in plaintext.
+ """
+
+ def __init__(self, table_name: str, columns: list[str]) -> None:
+ super().__init__(f"Table {table_name} masks its primary-key columns {columns}")
+ self.table_name = table_name
+ self.columns = columns
+
+
# (friendly_message, retryable) per category. Messages are STATIC templates — they never
# interpolate host/user/password or raw exception text, so stored copy can't leak credentials.
_CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {
@@ -179,6 +201,16 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {
"method for that table.",
False,
),
+ CDCErrorCategory.MASKED_KEY_CONFLICT: (
+ "A table masks a column that is its primary key, and a primary key can't be masked. "
+ "Remove the column from the table's masked columns, then re-enable change data capture.",
+ False,
+ ),
+ CDCErrorCategory.MASKING_KEY_UNAVAILABLE: (
+ "A table masks columns, but this PostHog instance has no secret key to hash them with. "
+ "Set WAREHOUSE_COLUMN_MASKING_KEY to a secret value, then re-enable change data capture.",
+ False,
+ ),
CDCErrorCategory.UNKNOWN: (
"Change data capture hit an unexpected error. PostHog will retry automatically — if it "
"persists, check the source's sync logs or contact support.",
@@ -217,6 +249,18 @@ def classify_cdc_error(exc: BaseException, adapter: CDCSourceAdapter | None) ->
return cdc_error_info(CDCErrorCategory.TRANSACTION_TOO_LARGE)
if isinstance(err, CDCReservedColumnError):
return cdc_error_info(CDCErrorCategory.RESERVED_COLUMN_CONFLICT)
+ if isinstance(err, CDCMaskingKeyError):
+ return cdc_error_info(CDCErrorCategory.MASKING_KEY_UNAVAILABLE)
+ if isinstance(err, CDCMaskedKeyError):
+ # Every table on the source shows this error, so it names the one to fix. Names are not secrets.
+ return dataclasses.replace(
+ cdc_error_info(CDCErrorCategory.MASKED_KEY_CONFLICT),
+ friendly_message=(
+ f"Table {err.table_name} masks its primary-key columns {', '.join(err.columns)}, and a primary "
+ "key can't be masked. Remove them from the table's masked columns, then re-enable change "
+ "data capture."
+ ),
+ )
if adapter is not None:
info = adapter.classify_error(err)
if info is not None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.py
index e5d8c64f64fb..2f70945927b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/snapshot_lane.py
@@ -121,6 +121,7 @@ def hand_reset_to_capture(
*,
start_capture: bool = True,
awaiting_slot: bool = False,
+ stage: bool = True,
) -> None:
"""Pause a CDC table's schedule and mark its reset pending, so capture finishes it once no sync can hand over.
@@ -130,6 +131,9 @@ def hand_reset_to_capture(
`awaiting_slot` holds the staged reset back until a capture run has read the slot, as Repair CDC
needs while it recreates the slot.
+
+ `stage=False` is for a caller that staged the reset in its own save. A second stage would start
+ another generation, and capture would take a second snapshot.
"""
# Deferred: data_load.service participates in the CDC schedule<->workflow import cycle.
from products.data_warehouse.backend.facade.api import ( # noqa: PLC0415
@@ -152,14 +156,15 @@ def hand_reset_to_capture(
# Capture pauses the schedule again before it resets the table.
logger.warning("cdc_reset_schedule_pause_failed", schema_id=str(schema.id), exc_info=True)
- persisted = update_sync_type_config_keys(
- schema.id, schema.team_id, mutate=partial(stage_handed_over_reset, awaiting_slot=awaiting_slot)
- )
- # Only this key in memory, so a caller that saves the schema afterwards keeps its own edits.
- schema.sync_type_config = {
- **(schema.sync_type_config or {}),
- CDC_RESET_PENDING_KEY: persisted[CDC_RESET_PENDING_KEY],
- }
+ if stage:
+ persisted = update_sync_type_config_keys(
+ schema.id, schema.team_id, mutate=partial(stage_handed_over_reset, awaiting_slot=awaiting_slot)
+ )
+ # Only this key in memory, so a caller that saves the schema afterwards keeps its own edits.
+ schema.sync_type_config = {
+ **(schema.sync_type_config or {}),
+ CDC_RESET_PENDING_KEY: persisted[CDC_RESET_PENDING_KEY],
+ }
# Capture finishes the reset, so start a run now instead of waiting for its schedule: the last
# run may have looked for resets just before this write, and the schedule may be gone. A halted
# source is left alone, because Repair CDC or a resumed capture restarts capture itself.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
index f711d760c24f..6c79dd2ee455 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
@@ -517,6 +517,9 @@ class ConformanceKit:
if update_fields is None or "job_inputs" in update_fields:
kit._source_job_inputs = copy.deepcopy(source.job_inputs)
+ def read_masked_columns(_extraction: CDCExtractActivity, schema: ExternalDataSchema) -> list[str] | None:
+ return kit._schema_row(schema.id).get("masked_columns")
+
def cdc_schemas(extraction: CDCExtractActivity) -> list[ExternalDataSchema]:
assert extraction.source is not None
return [kit._load_schema(name, extraction.source) for name in kit._schema_rows]
@@ -547,6 +550,7 @@ class ConformanceKit:
),
patch(f"{_CDC}.activities.ExternalDataSource", source_model),
patch.object(CDCExtractActivity, "_get_cdc_schemas", cdc_schemas),
+ patch.object(CDCExtractActivity, "_read_masked_columns", read_masked_columns),
patch(f"{_CDC}.activities.update_sync_type_config_keys", merge_config),
patch.object(ExternalDataSchema, "save", save_schema),
patch.object(ExternalDataSource, "save", save_source),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
index 2c9175f2ce63..abaefa881acd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
@@ -5,6 +5,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import
MAX_FRIENDLY_MESSAGE_LENGTH,
CDCErrorCategory,
CDCErrorInfo,
+ CDCMaskedKeyError,
+ CDCMaskingKeyError,
CDCSlotNotConfiguredError,
CDCTransactionTooLargeError,
cdc_error_info,
@@ -42,6 +44,7 @@ class TestCDCErrorInfo:
(CDCErrorCategory.SLOT_IN_USE, True),
(CDCErrorCategory.WAL_DECODE_ERROR, False),
(CDCErrorCategory.TRANSACTION_TOO_LARGE, False),
+ (CDCErrorCategory.MASKED_KEY_CONFLICT, False),
(CDCErrorCategory.UNKNOWN, True),
]
)
@@ -82,11 +85,32 @@ class TestClassifyCDCError:
info = classify_cdc_error(wrapper, PostgresCDCAdapter())
assert info.category is CDCErrorCategory.AUTH_FAILED
- def test_transaction_too_large_is_classified_without_adapter(self):
- info = classify_cdc_error(CDCTransactionTooLargeError("500001 events"), None)
- assert info.category is CDCErrorCategory.TRANSACTION_TOO_LARGE
+ @parameterized.expand(
+ [
+ (
+ "transaction_too_large",
+ CDCTransactionTooLargeError("500001 events"),
+ CDCErrorCategory.TRANSACTION_TOO_LARGE,
+ ),
+ ("masked_key", CDCMaskedKeyError("public.orders", ["id"]), CDCErrorCategory.MASKED_KEY_CONFLICT),
+ (
+ "masking_key_unavailable",
+ CDCMaskingKeyError("WAREHOUSE_COLUMN_MASKING_KEY must be set"),
+ CDCErrorCategory.MASKING_KEY_UNAVAILABLE,
+ ),
+ ]
+ )
+ def test_config_errors_are_classified_without_adapter(self, _name, exc, category):
+ info = classify_cdc_error(exc, None)
+ assert info.category is category
assert info.retryable is False
+ def test_masked_key_error_names_the_table_and_columns(self):
+ # Capture fails every table on the source, so the message must say which one to fix.
+ info = classify_cdc_error(CDCMaskedKeyError("public.orders", ["id", "tenant_id"]), None)
+ assert "public.orders" in info.friendly_message
+ assert "id, tenant_id" in info.friendly_message
+
def test_slot_not_configured_is_classified_without_adapter(self):
# A CDC-enabled source with no slot name is a config dead end, not a psycopg failure, so it
# must classify to a non-retryable category regardless of engine adapter.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
index ae0b7e550cd4..c09cde9d79c6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
@@ -1,9 +1,10 @@
import uuid
+import dataclasses
from collections.abc import Iterable, Iterator, Mapping
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
-from typing import Literal
+from typing import Any, Literal, cast
import pytest
from unittest.mock import ANY, DEFAULT, MagicMock, patch
@@ -41,6 +42,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import (
CDCErrorCategory,
+ CDCMaskedKeyError,
+ CDCMaskingKeyError,
CDCReservedColumnError,
cdc_error_info,
)
@@ -50,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import has_queued_batches
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCLag, ChangeEvent
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import mask_value
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapter
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.position import PgLSN
from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException
@@ -151,6 +155,7 @@ def _stub_app_db_writes():
"_update_schema_sync_type_config",
side_effect=_fake_update_schema_sync_type_config,
),
+ patch.object(CDCExtractActivity, "_read_masked_columns", side_effect=lambda schema: schema.masked_columns),
patch(f"{_ACTIVITIES}.cancel_running_sync", return_value=None),
):
yield
@@ -319,6 +324,39 @@ def _make_extract_activity(source, log=None) -> CDCExtractActivity:
return activity_obj
+class TestMaskEventColumns:
+ def test_masks_configured_columns_and_types_them_as_text(self):
+ activity_obj = _make_extract_activity(_make_source())
+ activity_obj._load_masked_columns(cast(Any, SimpleNamespace(name="users", masked_columns=["email", "Visits"])))
+ event = dataclasses.replace(
+ _make_event(columns={"id": 1, "Email": "[email protected]", "visits": None}),
+ column_types={"id": pa.int64(), "Email": pa.string(), "visits": pa.int64()},
+ )
+
+ out = activity_obj._mask_event_columns(event)
+
+ assert out.columns == {"id": 1, "Email": mask_value(1, "[email protected]"), "visits": None}
+ assert out.column_types == {"id": pa.int64(), "Email": pa.string(), "visits": pa.string()}
+
+ def test_a_table_without_masks_is_untouched(self):
+ activity_obj = _make_extract_activity(_make_source())
+ event = _make_event(columns={"id": 1, "email": "[email protected]"})
+
+ assert activity_obj._mask_event_columns(event) is event
+
+ def test_no_secret_key_fails_capture_before_any_change_is_read(self):
+ activity_obj = _make_extract_activity(_make_source())
+
+ with (
+ patch(
+ f"{_ACTIVITIES}.value_masker",
+ side_effect=NonRetryableException("WAREHOUSE_COLUMN_MASKING_KEY must be set"),
+ ),
+ pytest.raises(CDCMaskingKeyError),
+ ):
+ activity_obj._load_masked_columns(cast(Any, SimpleNamespace(name="users", masked_columns=["email"])))
+
+
class TestSetupSelfCleansUnrunnableSchedules:
@parameterized.expand([("no_adapter_for_the_type", "MySQL"), ("not_a_source_type_at_all", "UnsupportedDB")])
@patch("products.warehouse_sources.backend.temporal.data_imports.cdc.activities.get_cdc_adapter")
@@ -1429,6 +1467,26 @@ class TestPKColumnLoading:
assert queried_namespaces == {"cdc_test", "public"}
assert act.pk_columns_by_table == {"cdc_test.orders": ["id"], "public.orders": ["uuid"]}
+ def test_a_mask_on_a_discovered_key_fails_capture(self):
+ schema = _make_schema("cdc_test.orders")
+ schema.masked_columns = ["ID"]
+ schema.incremental_field = None
+ act = self._activity([schema], queried_pks={"cdc_test": {"orders": ["id"]}})
+
+ with pytest.raises(CDCMaskedKeyError):
+ act._load_pk_columns()
+
+ def test_a_mask_on_a_stored_cursor_column_still_masks_it(self):
+ # CDC reads no cursor, so a stale incremental_field must not keep its column in plaintext.
+ schema = _make_schema("cdc_test.orders")
+ schema.masked_columns = ["Updated_At"]
+ schema.incremental_field = "updated_at"
+ act = self._activity([schema], queried_pks={"cdc_test": {"orders": ["id"]}})
+
+ act._load_pk_columns()
+
+ assert act.masked_columns_by_table == {"cdc_test.orders": {"updated_at"}}
+
def test_stored_keys_are_not_requeried(self):
schema = _make_schema("cdc_test.orders")
schema.sync_type_config["primary_key_columns"] = ["id"]
@@ -2216,6 +2274,26 @@ class TestBufferedIngressCapture:
assert schema.sync_type_config["cdc_reset_pending"] == staged
+ def test_a_finished_reset_masks_with_the_masks_saved_with_it(self):
+ # The run loads its tables before it finishes resets. A mask change saved after that load
+ # stages a reset this run finishes, so the buffer the run then writes must carry the new mask.
+ source = _make_source()
+ schema = _make_schema("users", cdc_mode="streaming", source=source)
+ schema.masked_columns = None
+ schema.incremental_field = None
+ schema.sync_type_config["cdc_reset_pending"] = {"trigger": True}
+ events = [_make_event(op="I", position="0/100", columns={"id": 1, "email": "[email protected]"})]
+
+ with (
+ _capture_harness(source, [schema], events) as capture,
+ patch.object(CDCExtractActivity, "_read_masked_columns", return_value=["email"]),
+ patch("products.data_warehouse.backend.facade.api.trigger_external_data_workflow"),
+ ):
+ capture.extract()
+
+ written = capture.buffer.write_batch.call_args.kwargs["table"]
+ assert written.column("email").to_pylist() == [mask_value(1, "[email protected]")]
+
def test_a_reset_waiting_on_a_slot_holds_the_table_out_until_the_slot_reads(self):
# Recovery leaves this marker when it could not recreate the slot. Finishing the reset here
# would unpause the schedule, and the snapshot would start with no slot to resume from. A
diff --git a/products/warehouse_sources/backend/temporal/data_imports/naming_convention.py b/products/warehouse_sources/backend/temporal/data_imports/naming_convention.py
index bda5f1bd6af8..0640d9df5459 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/naming_convention.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/naming_convention.py
@@ -120,3 +120,11 @@ def _trim_and_tag(identifier: str, tag: str, max_length: int) -> str:
)
assert len(trimmed) == max_length
return trimmed
+
+
+def fold_column_name(name: str) -> str:
+ """Fold a source column name the way the pipeline normalizes it, so `ID` matches `id`."""
+ try:
+ return NamingConvention.normalize_identifier(name)
+ except ValueError:
+ return name
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
index 432b966098f7..a0440236cb47 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
@@ -24,7 +24,10 @@ from structlog.types import FilteringBoundLogger
from posthog.temporal.common.errors import NonReportableError
-from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import (
+ NamingConvention,
+ fold_column_name,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -317,7 +320,7 @@ def evolve_pyarrow_schema(
# Second pass: align with existing Delta table schema.
delta_arrow_schema = pyarrow_schema_from_arrow_exportable(delta_schema)
- merge_keys = {_fold_column_name_for_match(name) for name in merge_key_columns or []}
+ merge_keys = {fold_column_name(name) for name in merge_key_columns or []}
for delta_field in delta_arrow_schema:
if delta_field.name not in incoming_table.schema.names:
new_column_data = (
@@ -390,7 +393,7 @@ def evolve_pyarrow_schema(
incoming_table.schema.get_field_index(delta_field.name), delta_field.name, parsed_timestamps
)
elif (
- _fold_column_name_for_match(delta_field.name) in merge_keys
+ fold_column_name(delta_field.name) in merge_keys
and (pa.types.is_binary(delta_field.type) or pa.types.is_large_binary(delta_field.type))
and (pa.types.is_string(incoming_column.type) or pa.types.is_large_string(incoming_column.type))
):
@@ -510,13 +513,6 @@ def drop_internal_columns(table: pa.Table) -> pa.Table:
return table.drop_columns(internal) if internal else table
-def _fold_column_name_for_match(name: str) -> str:
- try:
- return NamingConvention.normalize_identifier(name)
- except ValueError:
- return name
-
-
def apply_enabled_columns_projection(
table: pa.Table,
enabled_columns: list[str] | None,
@@ -541,18 +537,16 @@ def apply_enabled_columns_projection(
if enabled_columns is None:
return table, []
- retained = {_fold_column_name_for_match(name) for name in enabled_columns}
+ retained = {fold_column_name(name) for name in enabled_columns}
for primary_key in primary_keys or []:
- retained.add(_fold_column_name_for_match(primary_key))
+ retained.add(fold_column_name(primary_key))
if incremental_field:
- retained.add(_fold_column_name_for_match(incremental_field))
+ retained.add(fold_column_name(incremental_field))
for partition_key in partition_keys or []:
- retained.add(_fold_column_name_for_match(partition_key))
+ retained.add(fold_column_name(partition_key))
kept_names = [
- name
- for name in table.column_names
- if name in INTERNAL_COLUMN_NAMES or _fold_column_name_for_match(name) in retained
+ name for name in table.column_names if name in INTERNAL_COLUMN_NAMES or fold_column_name(name) in retained
]
dropped_names = [name for name in table.column_names if name not in set(kept_names)]
if not dropped_names:
@@ -833,12 +827,15 @@ def table_from_iterator(
*,
primary_keys: Optional[Sequence[str]] = None,
binary_reporter: Optional[BinaryColumnReporter] = None,
+ masked_columns: Optional[Sequence[str]] = None,
) -> pa.Table:
batch = list(data_iterator)
if not batch:
return pa.Table.from_pylist([])
- processed_batch = _process_batch(batch, schema, primary_keys=primary_keys, binary_reporter=binary_reporter)
+ processed_batch = _process_batch(
+ batch, schema, primary_keys=primary_keys, binary_reporter=binary_reporter, masked_columns=masked_columns
+ )
return processed_batch
@@ -849,13 +846,18 @@ def table_from_py_list(
*,
primary_keys: Optional[Sequence[str]] = None,
binary_reporter: Optional[BinaryColumnReporter] = None,
+ masked_columns: Optional[Sequence[str]] = None,
) -> pa.Table:
"""
Convert a list of Python dictionaries to a PyArrow Table.
This is a wrapper around table_from_iterator for backward compatibility.
"""
return table_from_iterator(
- iter(table_data), schema=schema, primary_keys=primary_keys, binary_reporter=binary_reporter
+ iter(table_data),
+ schema=schema,
+ primary_keys=primary_keys,
+ binary_reporter=binary_reporter,
+ masked_columns=masked_columns,
)
@@ -1449,6 +1451,7 @@ def _process_batch(
*,
primary_keys: Optional[Sequence[str]] = None,
binary_reporter: Optional[BinaryColumnReporter] = None,
+ masked_columns: Optional[Sequence[str]] = None,
) -> pa.Table:
# Every step below reads a row as a mapping, so without this check a keyless row fails deep in
# the conversion with a bare `'list' object has no attribute 'items'`, which names neither the
@@ -1458,6 +1461,7 @@ def _process_batch(
raise NonMappingRowError(type(row).__name__)
table_data, serialized_dict_columns = _serialize_dict_columns(table_data)
+ masked = {fold_column_name(column) for column in masked_columns or []}
# Support both given schemas and inferred schemas
if schema is None or len(schema.names) == 0:
@@ -1673,11 +1677,15 @@ def _process_batch(
# not str" phrase so source non-retryable matching still fires, but name the
# column and show examples so the offending cells can be found in the source.
offenders = [x for x in all_values if isinstance(x, str)]
- sample = [(x if x != "" else "<blank>") for x in offenders[:5]]
+ # A masked column's cells must not reach the sync error, which project members can read.
+ if fold_column_name(str(field_name)) in masked:
+ examples = ""
+ else:
+ examples = f" such as {[(x if x != '' else '<blank>') for x in offenders[:5]]}"
raise TypeError(
f"must be real number, not str: column '{field_name}' is imported as a "
- f"number but contains {len(offenders)} non-numeric value(s) such as "
- f"{sample}. Ensure every cell in this column is a plain number (or empty)."
+ f"number but contains {len(offenders)} non-numeric value(s){examples}. "
+ "Ensure every cell in this column is a plain number (or empty)."
) from e
if arrow_schema and pa.types.is_decimal(arrow_schema.field(field_index).type):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.py
index 6291c33eb414..377bd88c27b8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/batcher.py
@@ -109,9 +109,11 @@ class Batcher:
coalesce_tables: bool = False,
primary_keys: Optional[list[str]] = None,
max_buffer_cells: Optional[int] = None,
+ masked_columns: Optional[list[str]] = None,
) -> None:
self._logger = logger
self._primary_keys = primary_keys
+ self._masked_columns = masked_columns
self._binary_reporter = BinaryColumnReporter(logger)
self._chunk_size = chunk_size or DEFAULT_CHUNK_SIZE
@@ -142,7 +144,12 @@ class Batcher:
self._ready_bytes = 0
def _rows_to_table(self, rows: list[Any]) -> pa.Table:
- return table_from_py_list(rows, primary_keys=self._primary_keys, binary_reporter=self._binary_reporter)
+ return table_from_py_list(
+ rows,
+ primary_keys=self._primary_keys,
+ binary_reporter=self._binary_reporter,
+ masked_columns=self._masked_columns,
+ )
def _set_ready(self, table: pa.Table) -> None:
"""Split `table` so no yielded chunk overflows a 32-bit offset column or exceeds
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/masking.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/masking.py
new file mode 100644
index 000000000000..4adbb5c8bab7
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/masking.py
@@ -0,0 +1,195 @@
+"""Deterministic one-way masking of sensitive warehouse-source columns.
+
+A masked value is replaced with an HMAC-SHA256 digest keyed by the server secret
+(`WAREHOUSE_COLUMN_MASKING_KEY`) over `team_id` and the value. The secret stops anyone who can query the
+column from brute-forcing low-entropy values such as card numbers. `team_id` makes equal values
+mask alike within a team, so they still join, and differently across teams.
+
+The value is hashed as `str(value)`. A value that two read paths render differently digests
+differently. On a CDC table the snapshot reads Python values while the change stream keeps the
+Postgres text for numeric, date and time, JSON, bytea and array columns. A masked column of those
+types can therefore hold two digests for one value after a row changes. Text, integer, boolean,
+float8 and UUID columns render alike on both paths.
+
+The key is its own secret, so rotating the encryption keys of other fields never changes a digest.
+Changing it changes every later digest, while rows already synced keep the old one. Every table with
+masked columns therefore needs a full resync after a change.
+
+Masking does not reach data derived before the mask was set. A materialized view built earlier
+keeps its plaintext copy until it is materialized again.
+
+A webhook delivery lands in S3 as a raw parquet file before any sync reads it, so a masked column
+stays there in plaintext until the next sync hashes the batch and deletes the file.
+"""
+
+from __future__ import annotations
+
+import hmac
+import asyncio
+import hashlib
+from collections.abc import Callable, Iterable
+from typing import TYPE_CHECKING
+
+from django.conf import settings
+
+import pyarrow as pa
+
+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name
+from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException
+
+if TYPE_CHECKING:
+ from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+
+
+# Anyone can read this key, so a digest keyed with it can be brute-forced. Development and tests only.
+_DEVELOPMENT_KEY = "development-only-warehouse-column-masking-key"
+_MIN_KEY_LENGTH = 32
+
+MASKING_KEY_UNAVAILABLE_MESSAGE = (
+ f"WAREHOUSE_COLUMN_MASKING_KEY must be set to a secret of at least {_MIN_KEY_LENGTH} characters to mask columns."
+)
+
+
+def masking_key_unavailable() -> bool:
+ """Whether this instance has no usable masking key. Development and tests fall back to a fixed one."""
+ if settings.DEBUG or settings.TEST:
+ return False
+ return len(settings.WAREHOUSE_COLUMN_MASKING_KEY) < _MIN_KEY_LENGTH
+
+
+def get_masking_key() -> bytes:
+ if masking_key_unavailable():
+ raise NonRetryableException(MASKING_KEY_UNAVAILABLE_MESSAGE)
+ return (settings.WAREHOUSE_COLUMN_MASKING_KEY or _DEVELOPMENT_KEY).encode()
+
+
+def _mask_base(team_id: int, key: bytes) -> hmac.HMAC:
+ # Callers copy this per value, which skips the key schedule and the shared prefix for each cell.
+ return hmac.new(key, f"{team_id}:".encode(), hashlib.sha256)
+
+
+def value_masker(team_id: int) -> Callable[[object], str | None]:
+ """`mask_value` for one team, with the key resolved once. Raises when no secret key is set."""
+ base = _mask_base(team_id, get_masking_key())
+
+ def mask(value: object) -> str | None:
+ if value is None:
+ return None
+ digest = base.copy()
+ digest.update(str(value).encode())
+ return digest.hexdigest()
+
+ return mask
+
+
+def mask_value(team_id: int, value: object, *, key: bytes | None = None) -> str | None:
+ """Digest of one value. Null stays null, so a masked column keeps its nullability.
+
+ Pass `key` to reuse one `get_masking_key()` across many values.
+ """
+ if value is None:
+ return None
+ digest = _mask_base(team_id, key if key is not None else get_masking_key()).copy()
+ digest.update(str(value).encode())
+ return digest.hexdigest()
+
+
+def ambiguous_masked_columns(masked_columns: list[str], source_columns: Iterable[str]) -> list[str]:
+ """Masked entries whose folded name is shared by two or more source columns.
+
+ The pipeline renames the later of two colliding columns (`email` and `Email`), but a mask is
+ matched by folded name and cannot say which one it means. Callers reject these, because the
+ mask would otherwise land on the wrong column and leave the intended one in plaintext.
+ """
+ counts: dict[str, int] = {}
+ for name in source_columns:
+ folded = fold_column_name(name)
+ counts[folded] = counts.get(folded, 0) + 1
+ return [column for column in masked_columns if counts.get(fold_column_name(column), 0) > 1]
+
+
+def collision_alias_masked_columns(masked_columns: list[str], synced_columns: Iterable[str]) -> list[str]:
+ """Masked entries that name a synced column the pipeline renamed to avoid a name collision.
+
+ The pipeline prefixes `_` to the later of two colliding columns (`Email` and `email` load as `email`
+ and `_email`). Which source column gets the alias depends on the batch, and a batch with only one
+ of them loads it under the plain name, so a mask on the alias would leave that value in plaintext.
+ """
+ folded = {fold_column_name(name) for name in synced_columns}
+ return [
+ column
+ for column in masked_columns
+ if column.startswith("_") and fold_column_name(column[1:]) in folded and fold_column_name(column) in folded
+ ]
+
+
+def protected_masked_columns(masked_columns: list[str] | None, protected_columns: Iterable[str | None]) -> list[str]:
+ """Folded masked names that are also a key, partition or cursor column. Callers fail on any.
+
+ A hashed merge key breaks merges and a hashed cursor is not orderable. Skipping the mask instead
+ would sync the column in plaintext.
+ """
+ protected = {fold_column_name(column) for column in protected_columns if column}
+ return sorted({fold_column_name(column) for column in masked_columns or []} & protected)
+
+
+def mask_table_columns(table: pa.Table, masked_columns: list[str] | None, *, team_id: int) -> pa.Table:
+ """Replace each masked column with string digests. Names match by their folded form, normalized or not."""
+ masked = {fold_column_name(column) for column in masked_columns or []}
+ if not masked:
+ return table
+ mask = value_masker(team_id)
+ for index, name in enumerate(table.column_names):
+ if fold_column_name(name) not in masked:
+ continue
+ # One chunk at a time, so the Python objects in memory stay bounded by the chunk size.
+ masked_chunks: list[pa.Array] = []
+ for chunk in table.column(index).chunks:
+ masked_chunks.append(pa.array([mask(value) for value in chunk.to_pylist()], type=pa.string()))
+ table = table.set_column(index, name, pa.chunked_array(masked_chunks, type=pa.string()))
+ return table
+
+
+def raise_if_masked_key_columns(schema: ExternalDataSchema, resource: SourceResponse) -> None:
+ """Fail the sync when a masked column is a key, partition or incremental column.
+
+ The API rejects a mask on a key it knows, but some sources resolve their merge, partition or
+ row-replacement key only at sync time. The loader reads those columns as source values, and
+ skipping the mask syncs plaintext, so fail the sync. A retry reads the same configuration.
+ """
+ if not schema.masked_columns:
+ return
+ key_columns = [
+ *(resource.primary_keys or []),
+ *(resource.partition_keys or []),
+ *(schema.partitioning_keys or []),
+ *(schema.partitioning_keys_override or []),
+ *([resource.replace_scope.column] if resource.replace_scope else []),
+ ]
+ # A full-refresh table keeps the cursor of an earlier sync method in its settings, but reads no cursor.
+ cursor = schema.incremental_field if schema.should_use_incremental_field or schema.is_cdc else None
+ protected = protected_masked_columns(schema.masked_columns, [*key_columns, cursor])
+ if protected:
+ raise NonRetryableException(
+ f"Cannot mask key, partition or incremental columns {protected}. "
+ "Remove them from masked_columns to sync this table."
+ )
+
+
+async def mask_table_if_configured(
+ table: pa.Table, schema: ExternalDataSchema, resource: SourceResponse, source_column_names: Iterable[str]
+) -> pa.Table:
+ """Mask a batch. `source_column_names` are the batch's names before any normalization."""
+ if not schema.masked_columns:
+ return table
+ # A source column added after the mask was saved can fold to a masked name. Normalization then
+ # renames one of the two, and the mask could hash the wrong one and leave the other in plaintext.
+ ambiguous = ambiguous_masked_columns(schema.masked_columns, source_column_names)
+ if ambiguous:
+ raise NonRetryableException(
+ f"Cannot mask {sorted(ambiguous)}: two source columns have this name after normalization. "
+ "Rename one of the source columns to sync this table."
+ )
+ raise_if_masked_key_columns(schema, resource)
+ return await asyncio.to_thread(mask_table_columns, table, schema.masked_columns, team_id=schema.team_id)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_masking.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_masking.py
new file mode 100644
index 000000000000..3249afe65c41
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_masking.py
@@ -0,0 +1,161 @@
+from types import SimpleNamespace
+from typing import Any, cast
+
+import pytest
+
+from django.test import override_settings
+
+import pyarrow as pa
+from parameterized import parameterized
+
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import (
+ ambiguous_masked_columns,
+ mask_table_columns,
+ mask_table_if_configured,
+ mask_value,
+ protected_masked_columns,
+)
+from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException
+
+
+class TestMaskValue:
+ @parameterized.expand(
+ [
+ ("same_team_same_value", (1, "[email protected]"), (1, "[email protected]"), True),
+ ("other_team", (1, "[email protected]"), (2, "[email protected]"), False),
+ ("other_value", (1, "[email protected]"), (1, "[email protected]"), False),
+ ]
+ )
+ def test_digest_equality(self, _name: str, left: tuple[int, str], right: tuple[int, str], equal: bool) -> None:
+ assert (mask_value(*left) == mask_value(*right)) is equal
+
+ def test_null_stays_null(self) -> None:
+ assert mask_value(1, None) is None
+
+ def test_digest_depends_on_the_server_secret(self) -> None:
+ with override_settings(WAREHOUSE_COLUMN_MASKING_KEY="a" * 32):
+ first = mask_value(1, "secret")
+ with override_settings(WAREHOUSE_COLUMN_MASKING_KEY="b" * 32):
+ second = mask_value(1, "secret")
+ assert first != second
+
+ @parameterized.expand(
+ [
+ ("no_key", "", False, False),
+ ("short_key", "k" * 31, False, False),
+ ("secret_key", "k" * 32, False, True),
+ ("no_key_in_development", "", True, True),
+ ]
+ )
+ def test_key_checks(self, _name: str, key: str, debug: bool, masks: bool) -> None:
+ with override_settings(WAREHOUSE_COLUMN_MASKING_KEY=key, DEBUG=debug, TEST=False):
+ if masks:
+ assert mask_value(1, "secret") is not None
+ else:
+ with pytest.raises(NonRetryableException):
+ mask_value(1, "secret")
+
+
+class TestProtectedMaskedColumns:
+ @parameterized.expand(
+ [
+ ("key_and_cursor", ["a", "b", "c"], ["a", "b", None], ["a", "b"]),
+ ("folds_case_before_matching", ["ID", "Email"], ["id"], ["id"]),
+ ("unset", None, ["id"], []),
+ ]
+ )
+ def test_protected(
+ self, _name: str, masked: list[str] | None, protected: list[str | None], expected: list[str]
+ ) -> None:
+ assert protected_masked_columns(masked, protected) == expected
+
+
+class TestAmbiguousMaskedColumns:
+ @parameterized.expand(
+ [
+ ("collision", ["Email"], ["email", "Email", "id"], ["Email"]),
+ ("no_collision", ["email"], ["email", "id", "name"], []),
+ ]
+ )
+ def test_ambiguous(self, _name: str, masked: list[str], source_columns: list[str], expected: list[str]) -> None:
+ assert ambiguous_masked_columns(masked, source_columns) == expected
+
+
+class TestMaskTableColumns:
+ def _table(self) -> pa.Table:
+ return pa.table({"id": [1, 2], "email": ["[email protected]", None], "visits": [7, 8]})
+
+ def test_masks_listed_columns_as_strings_and_leaves_the_rest(self) -> None:
+ # "Visits" is the source casing of a column the pipeline already normalized.
+ table = mask_table_columns(self._table(), ["email", "Visits"], team_id=3)
+
+ assert table.column("email").to_pylist() == [mask_value(3, "[email protected]"), None]
+ assert table.column("visits").to_pylist() == [mask_value(3, 7), mask_value(3, 8)]
+ assert table.schema.field("visits").type == pa.string()
+ assert table.column("id").to_pylist() == [1, 2]
+
+
+def _schema(masked: list[str], **fields: Any) -> Any:
+ return SimpleNamespace(
+ masked_columns=masked,
+ team_id=3,
+ **{
+ "incremental_field": None,
+ "should_use_incremental_field": False,
+ "is_cdc": False,
+ "partitioning_keys": None,
+ "partitioning_keys_override": None,
+ **fields,
+ },
+ )
+
+
+_RESOURCE = cast(
+ Any,
+ SimpleNamespace(
+ primary_keys=["id"],
+ partition_keys=["created_at"],
+ replace_scope=SimpleNamespace(column="_file_name"),
+ ),
+)
+
+
+class TestMaskTableIfConfigured:
+ @pytest.mark.parametrize(
+ "masked,source_column_names,schema_fields",
+ [
+ pytest.param(["id"], ["id", "email"], {}, id="mask_on_a_key_resolved_at_sync_time"),
+ pytest.param(["created_at"], ["id", "created_at"], {}, id="mask_on_a_source_partition_key"),
+ pytest.param(["_file_name"], ["id", "_file_name"], {}, id="mask_on_the_row_replacement_column"),
+ pytest.param(["Email"], ["id", "Email", "email"], {}, id="source_column_added_with_a_colliding_name"),
+ pytest.param(
+ ["updated_at"],
+ ["id", "updated_at"],
+ {"incremental_field": "updated_at", "should_use_incremental_field": True},
+ id="mask_on_the_cursor",
+ ),
+ pytest.param(["region"], ["id", "region"], {"partitioning_keys": ["region"]}, id="mask_on_a_partition_key"),
+ pytest.param(
+ ["region"],
+ ["id", "region"],
+ {"partitioning_keys_override": ["region"]},
+ id="mask_on_a_partition_key_override",
+ ),
+ ],
+ )
+ async def test_fails_the_sync(
+ self, masked: list[str], source_column_names: list[str], schema_fields: dict[str, Any]
+ ) -> None:
+ with pytest.raises(NonRetryableException):
+ await mask_table_if_configured(
+ pa.table({"id": [1]}), _schema(masked, **schema_fields), _RESOURCE, source_column_names
+ )
+
+ async def test_a_full_refresh_table_masks_the_cursor_of_its_earlier_sync_method(self) -> None:
+ schema = _schema(["updated_at"], incremental_field="updated_at")
+
+ masked = await mask_table_if_configured(
+ pa.table({"id": [1], "updated_at": ["2026-01-01"]}), schema, _RESOURCE, ["id", "updated_at"]
+ )
+
+ assert masked.column("updated_at").to_pylist() == [mask_value(3, "2026-01-01")]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.py
index 71ff70109440..6eba343da461 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_utils.py
@@ -116,17 +116,26 @@ def test_table_from_py_list_keeps_the_key_order_of_the_rows():
assert table.column_names == ["zeta", "alpha", "mid", "beta", "omega", "gamma", "delta", "late"]
-def test_table_from_py_list_numeric_column_with_non_numeric_value_raises_named_error():
+ "masked_columns,shows_cells",
+ [
+ pytest.param(None, True, id="unmasked"),
+ pytest.param(["Revenue"], False, id="masked_column_hides_its_cells"),
+ ],
+)
+def test_table_from_py_list_numeric_column_with_non_numeric_value_raises_named_error(
+ masked_columns: list[str] | None, shows_cells: bool
+):
with pytest.raises(TypeError) as exc_info:
- table_from_py_list([{"revenue": 1.5}, {"revenue": "N/A"}, {"revenue": ""}])
+ table_from_py_list([{"revenue": 1.5}, {"revenue": "N/A"}, {"revenue": ""}], masked_columns=masked_columns)
message = str(exc_info.value)
# Preserves the original phrase so source non-retryable matching still fires
assert "must be real number, not str" in message
- # Names the column and shows the offending text and blank cells
+ # Names the column, and shows the offending text and blank cells unless the column is masked
assert "revenue" in message
- assert "N/A" in message
- assert "<blank>" in message
+ assert ("N/A" in message) is shows_cells
+ assert ("<blank>" in message) is shows_cells
# The message must stay matched by an Any_Source_Errors entry so the schema is paused with
# guidance instead of retrying the same non-numeric cells forever, on every source.
assert [key for key in Any_Source_Errors if key in message]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index 107e55e286a4..292b6d9ca1bf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -94,6 +94,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.bat
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenance
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchema
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import (
+ mask_table_if_configured,
+ raise_if_masked_key_columns,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (
append_cursor_partition_keys,
)
@@ -417,6 +421,7 @@ class PipelineV3(Generic[ResumableData]):
resume_manager=self._resumable_source_manager, is_webhook=self._schema.is_webhook
),
primary_keys=self._resource.primary_keys,
+ masked_columns=self._schema.masked_columns,
)
self._internal_schema = HogQLSchema()
self._sinks = build_pipeline_sinks(
@@ -886,6 +891,10 @@ class PipelineV3(Generic[ResumableData]):
self._resource,
is_first_sync=self._table is None or self._reset_pipeline,
)
+ # Before the reset below deletes the table: a full-refresh table stores no key, so the API
+ # can accept a mask on the key that only this run resolves.
+ if self._resource.cdc_write_mode is None:
+ raise_if_masked_key_columns(self._schema, self._resource)
await persist_primary_keys(self._schema, self._resource, self._is_incremental, self._logger)
@@ -1174,6 +1183,7 @@ class PipelineV3(Generic[ResumableData]):
async def _process_batch(self, pa_table: pa.Table, batch_index: int, row_count: int) -> None:
pa_table = _append_debug_column_to_pyarrows_table(drop_internal_columns(pa_table), self._load_id)
+ source_column_names = pa_table.column_names
if not self._resource.preserve_column_names:
pa_table = normalize_table_column_names(pa_table)
@@ -1199,6 +1209,10 @@ class PipelineV3(Generic[ResumableData]):
pa_table = evolve_pyarrow_schema(pa_table, None)
pa_table = _handle_null_columns_with_definitions(pa_table, self._resource)
+ # After the null-column backfill, so a masked column the batch lacked is typed as text too.
+ # Capture masks a change before it writes the change buffer, so a buffered batch is masked already.
+ if self._resource.cdc_write_mode is None:
+ pa_table = await mask_table_if_configured(pa_table, self._schema, self._resource, source_column_names)
# Converge this batch onto the column types earlier batches in this run already used,
# and backfill columns they had that this one doesn't. The cursor column is named both
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index 3cbda075a725..bc051bba1ec8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -23,6 +23,7 @@ from posthog.temporal.common.shutdown import ShutdownMonitor, WorkerShuttingDown
from products.warehouse_sources.backend.models import external_data_schema as schema_models
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import CONSOLIDATED_WRITE_MODE
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.extract import (
finalize_desc_sort_incremental_value,
handle_reset_or_full_refresh,
@@ -45,6 +46,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arr
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.masking import mask_value
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (
append_partition_key_to_table,
)
@@ -79,6 +81,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
SourceInputs,
SourceResponse,
)
+from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException
from products.warehouse_sources.backend.temporal.data_imports.workflow_activities.import_data_sync import (
ImportJobModels,
)
@@ -120,6 +123,7 @@ def _make_pipeline() -> PipelineV3:
is_webhook=False,
is_append=False,
table=None,
+ masked_columns=None,
)
pipeline._table = None
pipeline._logger = _make_logger()
@@ -262,6 +266,32 @@ class TestAttemptScopedRunUuid:
mock_reset.assert_not_called()
+ @pytest.mark.asyncio
+ async def test_a_mask_on_a_key_resolved_at_sync_time_fails_before_the_reset(self) -> None:
+ pipeline = _make_pipeline()
+ pipeline._reset_pipeline = True
+ cast(MagicMock, pipeline._schema).configure_mock(
+ masked_columns=["order_uuid"],
+ incremental_field=None,
+ partitioning_keys=None,
+ partitioning_keys_override=None,
+ )
+ pipeline._resource = MagicMock(
+ primary_keys=["order_uuid"], partition_keys=None, replace_scope=None, cdc_write_mode=None
+ )
+
+ with (
+ patch(f"{_PIPELINE}.reset_rows_synced_if_needed", new_callable=AsyncMock),
+ patch(f"{_PIPELINE}.validate_incremental_sync"),
+ patch(f"{_PIPELINE}.handle_reset_or_full_refresh", new_callable=AsyncMock) as mock_reset,
+ patch(f"{_PIPELINE}.activity") as mock_activity,
+ ):
+ mock_activity.in_activity.return_value = False
+ with pytest.raises(NonRetryableException, match="order_uuid"):
+ await pipeline.run()
+
+ mock_reset.assert_not_called()
+
class TestTableRebuildAcrossAttempts:
@pytest.mark.parametrize(
@@ -1369,6 +1399,53 @@ class TestSingleTableRunIsUntouched:
pipeline._s3_batch_writer.write_batch.assert_called_once()
pipeline._pg_producer.hold_batch.assert_called_once()
+ @pytest.mark.parametrize(
+ "cdc_write_mode,expected",
+ [
+ pytest.param(None, [mask_value(1, "[email protected]")], id="source_rows"),
+ # Capture masked the change before it wrote the buffer, and a second hash would differ from the snapshot's.
+ pytest.param(CONSOLIDATED_WRITE_MODE, ["already-masked"], id="buffered_change_rows"),
+ ],
+ )
+ async def test_masked_columns_are_hashed_once_before_anything_is_staged(
+ self, cdc_write_mode: str | None, expected: list[str]
+ ) -> None:
+ pipeline = _make_pipeline()
+ cast(MagicMock, pipeline._schema).configure_mock(
+ incremental_field=None,
+ enabled_columns=None,
+ masked_columns=["email"],
+ team_id=1,
+ partitioning_keys=None,
+ partitioning_keys_override=None,
+ )
+ pipeline._resource = MagicMock(
+ primary_keys=["id"],
+ column_hints=None,
+ cdc_write_mode=cdc_write_mode,
+ partition_keys=None,
+ replace_scope=None,
+ preserve_column_names=False,
+ )
+ pipeline._last_incremental_field_value = None
+ pipeline._earliest_incremental_field_value = None
+ module = "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.pipeline"
+ with (
+ patch.object(pipeline, "_stage_batch", AsyncMock(return_value=1)) as stage_batch,
+ patch.object(pipeline, "_advance_handoff_checkpoint", AsyncMock()),
+ patch(
+ f"{module}.update_incremental_field_values",
+ AsyncMock(return_value=MagicMock(last_value=None, earliest_value=None)),
+ ),
+ patch(f"{module}.update_row_tracking_after_batch", AsyncMock()),
+ ):
+ email = "[email protected]" if cdc_write_mode is None else "already-masked"
+ await pipeline._process_batch(pa.table({"id": [1], "email": [email]}), 0, 1)
+
+ assert stage_batch.call_args.args[0].column("email").to_pylist() == expected
+ sink_table = cast(MagicMock, pipeline._sinks.stage_chunk).call_args.args[1]
+ assert sink_table.column("email").to_pylist() == expected
+
def test_the_activity_runs_the_base_class_for_a_source_without_lanes(self) -> None:
# What keeps every other source off the subclass. `lanes=None` is what a source that
# never heard of lanes carries, and `[]` what a lane build that found nothing returns.
@@ -1779,6 +1856,7 @@ def _runnable_pipeline(manager: ResumableSourceManager[_Cursor], items) -> Pipel
is_append=False,
should_use_incremental_field=False,
table=None,
+ masked_columns=None,
)
pipeline._process_batch = AsyncMock() # type: ignore[method-assign]
return pipeline
@@ -2360,6 +2438,7 @@ class TestFinalMarkerIsTheLastDataRow:
is_append=False,
should_use_incremental_field=False,
table=None,
+ masked_columns=None,
)
pipeline._s3_batch_writer = MagicMock(
write_batch=MagicMock(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
index fe1050499f88..4ec8cb4eb3f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/compute_table_statistics.py
@@ -43,11 +43,15 @@ from posthog.temporal.common.db_errors import is_transient_db_error
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeater
-from products.warehouse_sources.backend.models.column_statistics import WarehouseColumnStatistics
+from products.warehouse_sources.backend.models.column_statistics import (
+ WarehouseColumnStatistics,
+ clear_masked_column_bounds,
+)
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.models.util import clean_type
+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import fold_column_name
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry import (
retry_on_operational_error,
)
@@ -657,8 +661,7 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[st
basis = "full"
try:
- for column_name, stat in stats_by_column.items():
- _upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)
+ _write_statistics(team, table, schema.id, row_count, stats_by_column, delta_version, basis)
except (OperationalError, InterfaceError):
# The Delta-log read above can run long enough for the Postgres connection opened by the
# earlier metadata queries (team/schema/existing-stats) to go stale — server restart, proxy
@@ -668,8 +671,7 @@ def compute_table_statistics_sync(team_id: int, schema_id: uuid.UUID) -> dict[st
# Delta-log read just to redo a handful of upserts.
if not settings.TEST:
close_old_connections()
- for column_name, stat in stats_by_column.items():
- _upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)
+ _write_statistics(team, table, schema.id, row_count, stats_by_column, delta_version, basis)
log.info(
"warehouse_statistics.done",
@@ -774,6 +776,35 @@ def _fold_since_stored_version(
return None
+def _write_statistics(
+ team: Team,
+ table: DataWarehouseTable,
+ schema_id: uuid.UUID,
+ row_count: int,
+ stats_by_column: dict[str, _ColumnStat],
+ delta_version: int,
+ basis: str,
+) -> None:
+ """Persist every column's stats, with no bounds for a column masked at write time."""
+
+ def masked_columns() -> list[str] | None:
+ return (
+ ExternalDataSchema.objects.filter(team_id=team.id, id=schema_id)
+ .values_list("masked_columns", flat=True)
+ .first()
+ )
+
+ masked = {fold_column_name(column) for column in masked_columns() or []}
+ for column_name, stat in stats_by_column.items():
+ if fold_column_name(column_name) in masked:
+ # A masked column's bounds are plaintext until its rebuild lands, and digests after it.
+ stat = dataclasses.replace(stat, min_value=None, max_value=None, has_min_max=False)
+ _upsert_statistics(team, table, column_name, row_count, stat, delta_version, basis)
+ # A mask saved during the writes clears the bounds after its commit. This second read covers a mask
+ # saved before that point, so no lock on the schema row is needed.
+ clear_masked_column_bounds(team.id, table.id, masked_columns())
+
+
def _upsert_statistics(
team: Team,
table: DataWarehouseTable,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
index 12e701e74d10..89509a3ac378 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_compute_table_statistics.py
@@ -773,6 +773,49 @@ class TestComputeTableStatisticsSync:
assert stat.computed_for_delta_version == 12
assert stat.column_type == "Int64"
+ @pytest.mark.parametrize(
+ "saved_during",
+ [
+ pytest.param("_aggregate_add_action_stats", id="mask_saved_during_the_read"),
+ pytest.param("_upsert_statistics", id="mask_saved_during_the_writes"),
+ ],
+ )
+ def test_a_masked_column_keeps_its_counts_but_stores_no_bounds(self, saved_during: str) -> None:
+ team = self._team()
+ schema, table, _ = self._schema_table_job(
+ team, columns={"amount": {"clickhouse": "Nullable(Int64)"}, "email": {"clickhouse": "Nullable(String)"}}
+ )
+ add_actions = pa.table(
+ {
+ "num_records": [10],
+ "null_count.amount": [0],
+ "min.amount": [2],
+ "max.amount": [9],
+ "null_count.email": [1],
+ "min.email": ["[email protected]"],
+ "max.email": ["[email protected]"],
+ }
+ )
+ original = getattr(comp, saved_during)
+
+ def save_mask_then_continue(*args: Any) -> Any:
+ # The profiler loaded the schema before this save, so it must read the masks again.
+ ExternalDataSchema.objects.filter(id=schema.id).update(masked_columns=["Email"])
+ return original(*args)
+
+ with (
+ patch(DELTA_HELPER_PATH, return_value=self._mock_delta(add_actions)),
+ patch.object(comp, saved_during, save_mask_then_continue),
+ ):
+ compute_table_statistics_sync(team.id, schema.id)
+
+ stats = {
+ s.column_name: s for s in WarehouseColumnStatistics.objects.for_team(team.id).filter(table_id=table.id)
+ }
+ assert (stats["email"].null_count, stats["email"].min_value, stats["email"].max_value) == (1, None, None)
+ assert stats["email"].has_min_max is False
+ assert (stats["amount"].min_value, stats["amount"].max_value) == ("2", "9")
+
def test_recovers_from_stale_connection_during_write(self) -> None:
# The Delta-log read can run long enough for the pooled connection opened by the earlier
# metadata queries to go stale before the write loop runs, raising OperationalError on the
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index c998f87ea99b..0157afbcfb82 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -41,8 +41,10 @@ from products.data_warehouse.backend.facade.api import (
from products.data_warehouse.backend.facade.contracts import WebhookHogFunctionCreateResult
from products.warehouse_sources.backend.facade.models import (
DataWarehouseTable,
+ ExternalDataJob,
ExternalDataSchema,
ExternalDataSource,
+ WarehouseColumnStatistics,
update_sync_type_config_keys,
)
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
@@ -51,6 +53,7 @@ from products.warehouse_sources.backend.presentation.views.external_data_schema
UNREADABLE_SOURCE_CONFIG_ERROR,
ExternalDataSchemaSerializer,
ExternalDataSchemaViewset,
+ masked_columns_error,
schema_display_status,
)
from products.warehouse_sources.backend.presentation.views.external_data_source.helpers import (
@@ -5213,23 +5216,28 @@ class TestExternalDataSchemaRowFilters(APIBaseTest):
@parameterized.expand(
[
- ("enabled_columns", ["id", "name"]),
- ("row_filters", [{"column": "id", "operator": ">", "value": 10}]),
+ ("enabled_columns", ["id", "name"], "enabled_columns"),
+ ("row_filters", [{"column": "id", "operator": ">", "value": 10}], "row_filters"),
+ # Column names only: the log must never carry a masked value or digest.
+ ("masked_columns", ["name"], "masked columns"),
]
)
- def test_selection_change_is_written_to_the_activity_log(self, field: str, value: Any):
+ def test_selection_change_is_written_to_the_activity_log(self, field: str, value: Any, logged_field: str):
schema = self._create()
- response = self.client.patch(
- f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}", data={field: value}
- )
+ with mock.patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_schema.trigger_external_data_workflow"
+ ):
+ response = self.client.patch(
+ f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}", data={field: value}
+ )
assert response.status_code == 200, response.json()
log = ActivityLog.objects.get(scope="ExternalDataSchema", item_id=str(schema.id), activity="updated")
assert log.user == self.user
assert log.detail is not None
- assert [c for c in log.detail["changes"] if c["field"] == field] == [
- {"type": "ExternalDataSchema", "field": field, "action": "created", "before": None, "after": value}
+ assert [c for c in log.detail["changes"] if c["field"] == logged_field] == [
+ {"type": "ExternalDataSchema", "field": logged_field, "action": "created", "before": None, "after": value}
]
def test_null_clears_row_filters(self):
@@ -5366,6 +5374,401 @@ class TestExternalDataSchemaRowFilters(APIBaseTest):
assert "not supported for CDC" in str(response.json())
+class TestMaskedColumnsError(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("valid", ["password"], None),
+ ("source_casing_of_a_known_column", ["Password"], None),
+ ("unknown_column", ["does_not_exist"], "Unknown columns in masked_columns"),
+ ("primary_key", ["id"], "can't be masked"),
+ ("primary_key_in_another_case", ["ID"], "can't be masked"),
+ ("incremental_field", ["created_at"], "can't be masked"),
+ ("partition_column", ["region"], "can't be masked"),
+ ("two_source_columns_fold_to_one_name", ["Email"], "collide after normalization"),
+ ("direct_query_source", ["password"], "direct query", ExternalDataSourceType.POSTGRES, True),
+ (
+ "managed_schema_source",
+ ["password"],
+ "not supported for this source type",
+ ExternalDataSourceType.STRIPE,
+ ),
+ ]
+ )
+ def test_masked_columns_error(
+ self,
+ _name: str,
+ masked_columns: list[str],
+ expected: str | None,
+ source_type: str = ExternalDataSourceType.POSTGRES,
+ is_direct_query: bool = False,
+ ) -> None:
+ error = masked_columns_error(
+ masked_columns,
+ source_type=source_type,
+ is_direct_query=is_direct_query,
+ source_columns=["id", "created_at", "password", "email", "Email", "region"],
+ primary_key_columns=["id"],
+ incremental_fields=["created_at"],
+ partition_keys=["region"],
+ )
+ if expected is None:
+ assert error is None
+ else:
+ assert error is not None and expected in error
+
+ def test_a_mask_is_refused_while_no_columns_are_known(self) -> None:
+ # A mask that matches no column hashes nothing, so the column it meant would sync in plaintext.
+ error = masked_columns_error(
+ ["anything"],
+ source_type=ExternalDataSourceType.POSTGRES,
+ is_direct_query=False,
+ source_columns=[],
+ primary_key_columns=None,
+ incremental_fields=[],
+ )
+ assert error is not None and "columns are known" in error
+
+
+class TestExternalDataSchemaMaskedColumns(APIBaseTest):
+ VIEWS = "products.warehouse_sources.backend.presentation.views.external_data_schema"
+ SCHEMA_METADATA = {
+ "columns": [
+ {"name": "id", "data_type": "integer", "is_nullable": False},
+ {"name": "created_at", "data_type": "timestamp", "is_nullable": True},
+ {"name": "password", "data_type": "varchar(255)", "is_nullable": True},
+ {"name": "email", "data_type": "varchar(255)", "is_nullable": True},
+ ]
+ }
+
+ def _create(self, access_method: str | None = None, **schema_kwargs: Any) -> ExternalDataSchema:
+ source = ExternalDataSource.objects.create(
+ team=self.team,
+ source_type=ExternalDataSourceType.POSTGRES,
+ job_inputs={"host": "h", "port": "5432", "database": "d", "user": "u", "password": "p", "schema": "public"},
+ **({"access_method": access_method} if access_method else {}),
+ )
+ return ExternalDataSchema.objects.create(
+ name="Customers",
+ team=self.team,
+ source=source,
+ sync_type_config={
+ "schema_metadata": self.SCHEMA_METADATA,
+ "primary_key_columns": ["id"],
+ "incremental_field": "created_at",
+ },
+ **schema_kwargs,
+ )
+
+ def _patch(self, schema: ExternalDataSchema, data: dict[str, Any]):
+ return self.client.patch(f"/api/environments/{self.team.pk}/external_data_schemas/{schema.id}", data=data)
+
+ @parameterized.expand(
+ [
+ ("mask_added", None, ["password"], True, True, True, [True], True),
+ ("mask_removed", ["password"], [], True, True, True, [True], True),
+ ("mask_swapped", ["password"], ["email"], True, True, True, [True], True),
+ ("same_mask_in_another_case", ["password"], ["Password"], True, True, False, [], False),
+ ("mask_added_with_no_sync_running", None, ["password"], False, True, True, [False], False),
+ ("mask_added_on_a_disabled_table", None, ["password"], True, False, True, [], True),
+ ]
+ )
+ def test_a_mask_change_resyncs_the_table(
+ self,
+ _name: str,
+ stored: list[str] | None,
+ requested: list[str],
+ sync_running: bool,
+ should_sync: bool,
+ resets: bool,
+ triggers_with_buffer_one: list[bool],
+ cancels: bool,
+ ) -> None:
+ schema = self._create(masked_columns=stored, should_sync=should_sync)
+ if sync_running:
+ # The schedule skips a trigger while a sync runs, so the rebuild would wait a whole interval.
+ ExternalDataJob.objects.create(
+ team=self.team,
+ pipeline=schema.source,
+ schema=schema,
+ status=ExternalDataJob.Status.RUNNING,
+ workflow_id="running-workflow-id",
+ )
+ with (
+ mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger,
+ mock.patch(f"{self.VIEWS}.cancel_external_data_workflow") as mock_cancel,
+ ):
+ response = self._patch(schema, {"masked_columns": requested})
+
+ assert response.status_code == 200, response.json()
+ schema.refresh_from_db()
+ # Stored under the source column's own name, which the UI compares exactly.
+ assert schema.masked_columns == [column.lower() for column in requested]
+ assert schema.sync_type_config.get("reset_pipeline") is (True if resets else None)
+ assert mock_trigger.call_args_list == [
+ mock.call(mock.ANY, buffer_one=buffer_one) for buffer_one in triggers_with_buffer_one
+ ]
+ assert mock_cancel.call_args_list == ([mock.call("running-workflow-id")] if cancels else [])
+
+ # Some sources record their columns after normalization, so the metadata holds the alias too.
+ @parameterized.expand([("synced_table",), ("normalized_metadata",)])
+ def test_a_source_checks_masks_against_its_synced_columns(self, columns_from: str) -> None:
+ source = ExternalDataSource.objects.create(team=self.team, source_type=ExternalDataSourceType.HUBSPOT)
+ names = ["id", "email", "_email"]
+ table = DataWarehouseTable.objects.create(
+ team=self.team, name="hubspot_contacts", columns={name: {"clickhouse": "String"} for name in names}
+ )
+ config = (
+ {"schema_metadata": {"columns": [{"name": name, "data_type": "text"} for name in names]}}
+ if columns_from == "normalized_metadata"
+ else {}
+ )
+ schema = ExternalDataSchema.objects.create(
+ name="contacts", team=self.team, source=source, table=table, sync_type_config=config
+ )
+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):
+ unknown = self._patch(schema, {"masked_columns": ["phone"]})
+ # `_email` is where the pipeline put the later of two colliding source columns.
+ alias = self._patch(schema, {"masked_columns": ["_email"]})
+ accepted = self._patch(schema, {"masked_columns": ["EMAIL"]})
+
+ assert unknown.status_code == 400
+ assert "Unknown columns" in str(unknown.json())
+ assert alias.status_code == 400
+ assert "renamed" in str(alias.json())
+ assert accepted.status_code == 200, accepted.json()
+ schema.refresh_from_db()
+ assert schema.masked_columns == ["email"]
+
+ def test_a_mask_change_clears_the_cached_bounds_of_masked_columns(self) -> None:
+ table = DataWarehouseTable.objects.create(team=self.team)
+ schema = self._create(table=table)
+ for column in ("password", "email"):
+ WarehouseColumnStatistics.objects.for_team(self.team.id).create(
+ team=self.team, table=table, column_name=column, min_value="a", max_value="z", has_min_max=True
+ )
+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):
+ response = self._patch(schema, {"masked_columns": ["Password"]})
+
+ assert response.status_code == 200, response.json()
+ bounds = {
+ name: (low, high)
+ for name, low, high in WarehouseColumnStatistics.objects.for_team(self.team.id).values_list(
+ "column_name", "min_value", "max_value"
+ )
+ }
+ assert bounds == {"password": (None, None), "email": ("a", "z")}
+
+ @parameterized.expand(
+ [
+ ("unrelated_edit", {"incremental_field": "created_at", "incremental_field_type": "datetime"}),
+ ("new_mask_next_to_it", {"masked_columns": ["gone", "password"]}),
+ ]
+ )
+ def test_a_stored_mask_on_a_column_that_left_the_source_does_not_block_edits(
+ self, _name: str, data: dict[str, Any]
+ ) -> None:
+ schema = self._create(masked_columns=["gone"], sync_type=ExternalDataSchema.SyncType.INCREMENTAL)
+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):
+ response = self._patch(schema, data)
+
+ assert response.status_code == 200, response.json()
+
+ def test_a_patch_without_the_field_keeps_the_masks(self) -> None:
+ schema = self._create(masked_columns=["password"])
+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger:
+ response = self._patch(schema, {"enabled_columns": ["id", "created_at", "password"]})
+
+ assert response.status_code == 200, response.json()
+ schema.refresh_from_db()
+ assert schema.masked_columns == ["password"]
+ assert schema.sync_type_config.get("reset_pipeline") is None
+ mock_trigger.assert_not_called()
+
+ def test_a_mask_change_on_a_disabled_table_waits_for_the_next_sync(self) -> None:
+ schema = self._create(should_sync=False)
+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger:
+ response = self._patch(schema, {"masked_columns": ["password"]})
+
+ assert response.status_code == 200, response.json()
+ schema.refresh_from_db()
+ assert schema.sync_type_config.get("reset_pipeline") is True
+ mock_trigger.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("syncing", True, 1),
+ ("disabled_stages_the_snapshot_in_the_save", False, 0),
+ ("syncing_with_a_table_mode_change", True, 1, {"cdc_table_mode": "both"}),
+ ]
+ )
+ def test_a_cdc_table_resets_through_a_new_snapshot(
+ self, _name: str, should_sync: bool, resets: int, other_changes: dict[str, Any] | None = None
+ ) -> None:
+ schema = self._create(sync_type=ExternalDataSchema.SyncType.CDC, should_sync=should_sync)
+ facade = "products.data_warehouse.backend.facade.api"
+ with (
+ mock.patch(f"{self.VIEWS}.source_type_supports_cdc", return_value=False),
+ mock.patch(f"{self.VIEWS}.sync_cdc_extraction_schedule"),
+ mock.patch(f"{facade}.pause_external_data_schedule") as mock_pause,
+ mock.patch(f"{facade}.trigger_cdc_extraction_schedule") as mock_start_capture,
+ mock.patch(f"{self.VIEWS}._reset_cdc_for_full_resnapshot") as mock_resnapshot,
+ mock.patch(f"{self.VIEWS}.trigger_external_data_workflow") as mock_trigger,
+ ):
+ response = self._patch(schema, {"masked_columns": ["password"], **(other_changes or {})})
+
+ assert response.status_code == 200, response.json()
+ assert mock_pause.call_count == resets
+ assert mock_start_capture.call_count == resets
+ mock_resnapshot.assert_not_called()
+ mock_trigger.assert_not_called()
+ schema.refresh_from_db()
+ # Saved with the mask, so capture still resets the table if the hand-over fails.
+ assert ("cdc_reset_pending" in schema.sync_type_config) is should_sync
+ if should_sync:
+ # One staged generation: a second one would make capture take a second snapshot.
+ assert schema.sync_type_config["cdc_reset_pending"]["generation"] == 1
+ if not should_sync:
+ assert schema.sync_type_config["cdc_mode"] == "snapshot"
+ assert schema.sync_type_config["reset_pipeline"] is True
+ assert schema.initial_sync_complete is False
+
+ @parameterized.expand(
+ [
+ (
+ "repair_staged_a_reset_that_waits_for_a_slot",
+ ExternalDataSchema.SyncType.CDC,
+ {"cdc_reset_pending": {"trigger": True, "awaiting_slot": True, "generation": 3}},
+ [],
+ {"cdc_reset_pending": {"trigger": True, "awaiting_slot": True, "generation": 4}},
+ ),
+ (
+ "a_run_cleared_an_earlier_reset",
+ ExternalDataSchema.SyncType.FULL_REFRESH,
+ {},
+ ["reset_pipeline"],
+ {"reset_pipeline": True},
+ ),
+ ]
+ )
+ def test_a_mask_rebuild_survives_a_config_write_before_the_save(
+ self,
+ _name: str,
+ sync_type: str,
+ concurrent_updates: dict[str, Any],
+ concurrent_removes: list[str],
+ expected: dict[str, Any],
+ ) -> None:
+ schema = self._create(sync_type=sync_type, should_sync=True)
+ update_sync_type_config_keys(schema.id, schema.team_id, updates={"reset_pipeline": True})
+ original_save = ExternalDataSchemaSerializer._save_merging_sync_type_config
+
+ def concurrent_write_first(serializer: ExternalDataSchemaSerializer, *args: Any) -> ExternalDataSchema:
+ update_sync_type_config_keys(
+ schema.id, schema.team_id, updates=concurrent_updates, removes=concurrent_removes
+ )
+ return original_save(serializer, *args)
+
+ with (
+ mock.patch.object(
+ ExternalDataSchemaSerializer,
+ "_save_merging_sync_type_config",
+ autospec=True,
+ side_effect=concurrent_write_first,
+ ),
+ mock.patch(f"{self.VIEWS}.source_type_supports_cdc", return_value=False),
+ mock.patch(f"{self.VIEWS}.sync_cdc_extraction_schedule"),
+ mock.patch(f"{self.VIEWS}.hand_reset_to_capture"),
+ mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"),
+ ):
+ response = self._patch(schema, {"masked_columns": ["password"]})
+
+ assert response.status_code == 200, response.json()
+ schema.refresh_from_db()
+ assert {key: schema.sync_type_config.get(key) for key in expected} == expected
+
+ def test_rejected_when_no_masking_key_is_set(self) -> None:
+ schema = self._create()
+ with mock.patch(f"{self.VIEWS}.masking_key_unavailable", return_value=True):
+ response = self._patch(schema, {"masked_columns": ["password"]})
+
+ assert response.status_code == 400
+ assert "WAREHOUSE_COLUMN_MASKING_KEY" in str(response.json())
+
+ def test_rejected_for_a_direct_query_source(self) -> None:
+ schema = self._create(access_method=ExternalDataSource.AccessMethod.DIRECT)
+ response = self._patch(schema, {"masked_columns": ["password"]})
+
+ assert response.status_code == 400
+ assert "direct query" in str(response.json())
+
+ def test_rejected_on_a_synced_webhook_table(self) -> None:
+ table = DataWarehouseTable.objects.create(team=self.team)
+ schema = self._create(masked_columns=["password"], sync_type=ExternalDataSchema.SyncType.WEBHOOK, table=table)
+ response = self._patch(schema, {"masked_columns": []})
+
+ assert response.status_code == 400
+ assert "webhook" in str(response.json())
+ schema.refresh_from_db()
+ assert schema.masked_columns == ["password"]
+
+ def test_rejected_while_syncs_are_paused_for_billing(self) -> None:
+ schema = self._create()
+ with mock.patch(f"{self.VIEWS}.is_any_external_data_schema_paused", return_value=True):
+ response = self._patch(schema, {"masked_columns": ["password"]})
+
+ assert response.status_code == 400
+ schema.refresh_from_db()
+ assert schema.masked_columns is None
+
+ @parameterized.expand(
+ [
+ ("mask_on_the_primary_key", {"masked_columns": ["id"]}, ExternalDataSchema.SyncType.INCREMENTAL),
+ (
+ "incremental_field_moved_onto_a_masked_column",
+ {"incremental_field": "password"},
+ ExternalDataSchema.SyncType.INCREMENTAL,
+ ),
+ (
+ "primary_key_moved_onto_a_masked_column",
+ {"primary_key_columns": ["password"]},
+ ExternalDataSchema.SyncType.INCREMENTAL,
+ ),
+ # A CDC table keeps its stored cursor whatever the request sends for it.
+ (
+ "stored_cursor_with_a_null_cursor_in_the_request",
+ {"masked_columns": ["created_at"], "incremental_field": None},
+ ExternalDataSchema.SyncType.CDC,
+ ),
+ # With no stored key, a sync merges and partitions on `id`.
+ (
+ "mask_on_id_with_no_stored_key",
+ {"masked_columns": ["id"]},
+ ExternalDataSchema.SyncType.FULL_REFRESH,
+ False,
+ ),
+ ]
+ )
+ def test_a_mask_never_shares_a_column_with_the_key_or_cursor(
+ self, _name: str, data: dict[str, Any], sync_type: str, key_stored: bool = True
+ ) -> None:
+ schema = self._create(masked_columns=["password"], sync_type=sync_type)
+ if not key_stored:
+ update_sync_type_config_keys(schema.id, schema.team_id, removes=["primary_key_columns"])
+ response = self._patch(schema, {"incremental_field_type": "integer", **data})
+
+ assert response.status_code == 400
+ assert "can't be masked" in str(response.json())
+
+ def test_a_full_refresh_table_can_mask_the_cursor_of_its_earlier_sync_method(self) -> None:
+ schema = self._create(sync_type=ExternalDataSchema.SyncType.FULL_REFRESH)
+ with mock.patch(f"{self.VIEWS}.trigger_external_data_workflow"):
+ response = self._patch(schema, {"masked_columns": ["created_at"]})
+
+ assert response.status_code == 200, response.json()
+ schema.refresh_from_db()
+ assert schema.masked_columns == ["created_at"]
+
+
class TestExternalDataSchemaApiVersionOverride(APIBaseTest):
def _create_schema(self, sync_type=ExternalDataSchema.SyncType.FULL_REFRESH, api_version=None):
source = ExternalDataSource.objects.create(
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index f778eafc037a..7605873955b2 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -3666,6 +3666,7 @@ class TestExternalDataSource(APIBaseTest):
"cdc_table_mode": "consolidated",
"incremental_sync_blocked": None,
"enabled_columns": None,
+ "masked_columns": None,
"row_filters": None,
"row_filter_columns": None,
"available_columns": [],
@@ -5004,7 +5005,12 @@ class TestExternalDataSource(APIBaseTest):
self.assertEqual(analytics_schema.sync_type_config["schema_metadata"]["source_schema"], "analytics")
def _post_postgres_cdc_source_with_one_table(
- self, mock_get_source, mock_cdc_pg_connection, mock_get_primary_key_columns, mock_setup_cdc_resources
+ self,
+ mock_get_source,
+ mock_cdc_pg_connection,
+ mock_get_primary_key_columns,
+ mock_setup_cdc_resources,
+ schema_extra: dict[str, t.Any] | None = None,
):
_configure_source_mock_versioning(mock_get_source)
source_mock = mock_get_source.return_value
@@ -5065,7 +5071,7 @@ class TestExternalDataSource(APIBaseTest):
"schema": "",
"cdc_enabled": True,
"schemas": [
- {"name": "analytics.events", "should_sync": True, "sync_type": "cdc"},
+ {"name": "analytics.events", "should_sync": True, "sync_type": "cdc", **(schema_extra or {})},
],
},
},
@@ -5110,6 +5116,50 @@ class TestExternalDataSource(APIBaseTest):
mock_add_table.assert_called_once()
assert mock_add_table.call_args.args[1:] == ("analytics", "events")
+ @parameterized.expand(
+ [
+ ("masked_primary_key", {"masked_columns": ["id"]}),
+ ("malformed_mask", {"masked_columns": "id"}),
+ ("incremental_table_without_a_cursor", {"sync_type": "incremental", "incremental_field": None}),
+ ]
+ )
+ @patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_source.base.is_cdc_enabled_for_team",
+ return_value=True,
+ )
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.cleanup_resources"
+ )
+ @patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_source.viewset.ExternalDataSourceViewSet._setup_cdc_resources"
+ )
+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.get_primary_key_columns")
+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.cdc_pg_connection")
+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.SourceRegistry.get_source")
+ def test_create_postgres_cdc_refused_table_drops_the_replication_slot(
+ self,
+ _name,
+ schema_extra,
+ mock_get_source,
+ mock_cdc_pg_connection,
+ mock_get_primary_key_columns,
+ mock_setup_cdc_resources,
+ mock_cleanup_resources,
+ _mock_is_cdc_enabled_for_team,
+ ):
+ mock_get_source.return_value.has_managed_hogql_schema = False
+ response = self._post_postgres_cdc_source_with_one_table(
+ mock_get_source,
+ mock_cdc_pg_connection,
+ mock_get_primary_key_columns,
+ mock_setup_cdc_resources,
+ schema_extra=schema_extra,
+ )
+
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content
+ mock_cleanup_resources.assert_called_once()
+ assert not ExternalDataSource.objects.filter(team_id=self.team.pk).exists()
+
@patch(
"products.warehouse_sources.backend.presentation.views.external_data_source.base.is_cdc_enabled_for_team",
return_value=True,
@@ -5785,6 +5835,73 @@ class TestExternalDataSource(APIBaseTest):
else:
assert schema.sync_type_config["primary_key_columns"] == expected_persisted
+ @parameterized.expand(
+ [
+ ("mask_is_saved", ["email"], status.HTTP_201_CREATED, ["email"]),
+ ("empty_list_masks_nothing", [], status.HTTP_201_CREATED, None),
+ ("mask_on_the_detected_primary_key", ["id"], status.HTTP_400_BAD_REQUEST, "can't be masked"),
+ ("mask_on_an_unknown_column", ["nope"], status.HTTP_400_BAD_REQUEST, "Unknown columns"),
+ ("mask_that_is_not_a_list", "email", status.HTTP_400_BAD_REQUEST, "must be a list"),
+ ]
+ )
+ @patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.SourceRegistry.get_source")
+ def test_create_postgres_validates_and_persists_masked_columns(
+ self,
+ _name: str,
+ masked_columns: list[str] | str,
+ expected_status: int,
+ expected: list[str] | str | None,
+ mock_get_source,
+ ):
+ _configure_source_mock_versioning(mock_get_source)
+ source_mock = mock_get_source.return_value
+ source_mock.validate_config.return_value = (True, [])
+ source_mock.has_managed_hogql_schema = False
+ parsed_config = Mock()
+ parsed_config.schema = "public"
+ payload = {
+ "host": "localhost",
+ "port": 5432,
+ "database": "app",
+ "user": "user",
+ "password": "pass",
+ "schema": "public",
+ }
+ parsed_config.to_dict.return_value = payload
+ source_mock.parse_config.return_value = parsed_config
+ source_mock.validate_credentials.return_value = (True, None)
+ source_mock.get_schemas.return_value = [
+ SourceSchema(
+ name="events",
+ supports_incremental=False,
+ supports_append=False,
+ columns=[("id", "integer", False), ("email", "text", True)],
+ foreign_keys=[],
+ detected_primary_keys=["id"],
+ ),
+ ]
+
+ response = self.client.post(
+ f"/api/environments/{self.team.pk}/external_data_sources/",
+ data={
+ "source_type": "Postgres",
+ "payload": {
+ **payload,
+ "schemas": [
+ {"name": "events", "should_sync": True, "sync_type": None, "masked_columns": masked_columns}
+ ],
+ },
+ },
+ )
+
+ assert response.status_code == expected_status, response.content
+ schemas = ExternalDataSchema.objects.filter(team_id=self.team.pk, name="events")
+ if expected_status == status.HTTP_201_CREATED:
+ assert schemas.get().masked_columns == expected
+ else:
+ assert isinstance(expected, str) and expected in response.json()["message"]
+ assert not schemas.exists()
+
@parameterized.expand(
[
# Field omitted -> None: sync everything (default).
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index 382a98f3959d..ce010d5a587d 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -583,6 +583,11 @@ export interface ExternalDataSchemaApi {
* @nullable
*/
enabled_columns?: string[] | null
+ /**
+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.
+ * @nullable
+ */
+ masked_columns?: string[] | null
/**
* Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.
* @nullable
@@ -787,6 +792,11 @@ export interface PatchedExternalDataSchemaApi {
* @nullable
*/
enabled_columns?: string[] | null
+ /**
+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.
+ * @nullable
+ */
+ masked_columns?: string[] | null
/**
* Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.
* @nullable
@@ -1214,6 +1224,11 @@ export interface ExternalDataSourceBulkUpdateSchemaApi {
* @nullable
*/
enabled_columns?: string[] | null
+ /**
+ * Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.
+ * @nullable
+ */
+ masked_columns?: string[] | null
/**
* Row-filter predicates ANDed onto the source query. Null/empty means sync all rows.
* @nullable
diff --git a/products/warehouse_sources/frontend/generated/api.zod.ts b/products/warehouse_sources/frontend/generated/api.zod.ts
index f1ce05ccc461..3ff87cec10fe 100644
--- a/products/warehouse_sources/frontend/generated/api.zod.ts
+++ b/products/warehouse_sources/frontend/generated/api.zod.ts
@@ -243,6 +243,12 @@ export const ExternalDataSchemasUpdateBody = /* @__PURE__ */ zod
.describe(
'Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.'
),
+ masked_columns: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."
+ ),
row_filters: zod
.array(
zod.object({
@@ -360,6 +366,12 @@ export const ExternalDataSchemasPartialUpdateBody = /* @__PURE__ */ zod
.describe(
'Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.'
),
+ masked_columns: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."
+ ),
row_filters: zod
.array(
zod.object({
@@ -638,6 +650,12 @@ export const ExternalDataSourcesBulkUpdateSchemasPartialUpdateBody = /* @__PURE_
.array(zod.string())
.nullish()
.describe('Columns to sync. Null means sync all columns.'),
+ masked_columns: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.'
+ ),
row_filters: zod
.array(
zod.object({
diff --git a/products/warehouse_sources/mcp/tools.yaml b/products/warehouse_sources/mcp/tools.yaml
index 81426f63440a..9f95795567db 100644
--- a/products/warehouse_sources/mcp/tools.yaml
+++ b/products/warehouse_sources/mcp/tools.yaml
@@ -184,8 +184,9 @@ tools:
(incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full
refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental
field (plus type and lookback window), primary key columns, CDC table mode, column selection
- (enabled_columns), row filters, and vendor API version. Changes take effect on the next sync, not
- retroactively. Before changing incremental_field or sync_type, call
+ (enabled_columns), column masking (masked_columns), row filters, and vendor API version. Changes take effect
+ on the next sync, not retroactively, except a masked_columns change, which starts a full resync of the
+ table. Before changing incremental_field or sync_type, call
'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually
supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To
run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 8ab0cfbbc73c..000cc1c69da8 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -6503,7 +6503,7 @@
}
},
"external-data-schemas-partial-update": {
- "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.",
+ "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), column masking (masked_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively, except a masked_columns change, which starts a full resync of the table. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.",
"category": "Warehouse sources",
"feature": "warehouse_sources",
"summary": "Update data warehouse table schema sync config",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index f648462b5f6d..116ce6663278 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -6636,7 +6636,7 @@
}
},
"external-data-schemas-partial-update": {
- "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.",
+ "description": "Update one table schema's sync configuration: enable/disable syncing (should_sync), sync_type (incremental, full_refresh, append, webhook, cdc, xmin), sync frequency and UTC time of day, scheduled full refresh interval and UTC time of day (full_refresh_interval_days, full_refresh_time_of_day), incremental field (plus type and lookback window), primary key columns, CDC table mode, column selection (enabled_columns), column masking (masked_columns), row filters, and vendor API version. Changes take effect on the next sync, not retroactively, except a masked_columns change, which starts a full resync of the table. Before changing incremental_field or sync_type, call 'external-data-schemas-incremental-fields-create' to see which fields and sync methods the source actually supports; before switching to cdc, validate with 'external-data-sources-check-cdc-prerequisites-create'. To run a sync immediately after reconfiguring, use 'external-data-schemas-reload'. This is also how a table reported by `incremental_sync_blocked` is resolved: send primary_key_columns, or a different sync_type, or should_sync=true to retry a table fixed at the source. For `duplicate_primary_key`, a different key is refused once data has synced, because rows already merged under the old key would repeat; delete the synced data first with 'external-data-schemas-delete-data', or fix the duplicates at the source and retry. That field reports the last run's failure, so it clears once a run succeeds rather than when the update lands.",
"category": "Warehouse sources",
"feature": "warehouse_sources",
"summary": "Update data warehouse table schema sync config",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 7fb573aabe9f..6cb6044e1d68 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -47242,6 +47242,11 @@ export namespace Schemas {
* @nullable
*/
enabled_columns?: string[] | null;
+ /**
+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.
+ * @nullable
+ */
+ masked_columns?: string[] | null;
/**
* Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.
* @nullable
@@ -47363,6 +47368,11 @@ export namespace Schemas {
* @nullable
*/
enabled_columns?: string[] | null;
+ /**
+ * Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.
+ * @nullable
+ */
+ masked_columns?: string[] | null;
/**
* Row-filter predicates ANDed onto the source query. Null/empty means sync all rows.
* @nullable
@@ -81702,6 +81712,11 @@ export namespace Schemas {
* @nullable
*/
enabled_columns?: string[] | null;
+ /**
+ * Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again.
+ * @nullable
+ */
+ masked_columns?: string[] | null;
/**
* Predicates ANDed onto the source query so only matching rows sync. Each is `{column, operator, value}`; `null`/empty (default) syncs all rows. The operator must be one of `> >= < <= = != IN "NOT IN"` and the value must match the column's type (for `IN`/`NOT IN`, a comma-separated list like `1, 2, 3` or `'a','b'`). Applied on the next sync — not retroactive to already-synced rows.
* @nullable
diff --git a/services/mcp/src/generated/warehouse_sources/api.ts b/services/mcp/src/generated/warehouse_sources/api.ts
index 8ffb3e3b0bbf..f23e23480e29 100644
--- a/services/mcp/src/generated/warehouse_sources/api.ts
+++ b/services/mcp/src/generated/warehouse_sources/api.ts
@@ -131,6 +131,12 @@ export const ExternalDataSchemasPartialUpdateBody = () => zod
.describe(
'Names of source columns to sync. `null` (default) syncs all columns. Primary-key columns and the active incremental field are always retained, even if not listed here.'
),
+ masked_columns: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."
+ ),
row_filters: zod
.array(
zod.object({
@@ -461,6 +467,12 @@ export const ExternalDataSourcesBulkUpdateSchemasPartialUpdateBody = () => zod.o
.array(zod.string())
.nullish()
.describe('Columns to sync. Null means sync all columns.'),
+ masked_columns: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table.'
+ ),
row_filters: zod
.array(
zod.object({
diff --git a/services/mcp/src/tools/generated/warehouse_sources.ts b/services/mcp/src/tools/generated/warehouse_sources.ts
index 9ef6d4ea0bc6..18f50ad3c288 100644
--- a/services/mcp/src/tools/generated/warehouse_sources.ts
+++ b/services/mcp/src/tools/generated/warehouse_sources.ts
@@ -269,6 +269,9 @@ const externalDataSchemasPartialUpdate = (): ToolBase<
if (params.enabled_columns !== undefined) {
body['enabled_columns'] = params.enabled_columns
}
+ if (params.masked_columns !== undefined) {
+ body['masked_columns'] = params.masked_columns
+ }
if (params.row_filters !== undefined) {
body['row_filters'] = params.row_filters
}
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.json
index d58ad1c2e9a5..ac766d8ee440 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-schemas-partial-update.json
@@ -106,6 +106,20 @@
],
"description": "Data type of the incremental field.\n\n* `integer` - integer\n* `numeric` - numeric\n* `datetime` - datetime\n* `date` - date\n* `timestamp` - timestamp\n* `objectid` - objectid\n* `xid` - xid"
},
+ "masked_columns": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Names of source columns whose values are replaced with a deterministic one-way digest at sync time, for sensitive data such as passwords or PII. `null` (default) masks nothing. Primary-key, partition and incremental columns can't be masked, and direct query sources don't support masking. Any change to this list starts a full resync of the table (CDC tables take a new snapshot), or stages it until syncing is turned on. A synced webhook table can't change its masked columns, because its data can't be fetched again."
+ },
"primary_key_columns": {
"anyOf": [
{
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.json
index 6840db34bdfc..c5528c21fbdd 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/external-data-sources-bulk-update-schemas.json
@@ -91,6 +91,20 @@
],
"description": "Type of the incremental cursor field."
},
+ "masked_columns": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Columns whose values are replaced with a one-way digest at sync time. Null means mask nothing. Any change starts a full resync of the table."
+ },
"primary_key_columns": {
"anyOf": [
{