:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): lock individual webhook header rows as secret (#117386)
Daniel Marchuk08b3249ad64f
diff --git a/frontend/src/lib/components/CyclotronJob/CyclotronJobInputs.tsx b/frontend/src/lib/components/CyclotronJob/CyclotronJobInputs.tsxindex 9cfcbc1ac905..78fe93d56933 100644--- a/frontend/src/lib/components/CyclotronJob/CyclotronJobInputs.tsx+++ b/frontend/src/lib/components/CyclotronJob/CyclotronJobInputs.tsx@@ -47,6 +47,15 @@ import { CUSTOM_INPUT_RENDERERS } from './customInputRenderers'import { cyclotronJobInputLogic, formatJsonValue } from './cyclotronJobInputLogic'import { CyclotronJobTemplateSuggestionsButton, useTemplateEditorCursor } from './CyclotronJobTemplateSuggestions'import { insertTemplateReference, templateReferenceForOption } from './cyclotronJobTemplateSuggestionsLogic'+import {+ SecretDictionaryRow,+ canAddSecretRow,+ findSecretRowsCompanion,+ isSecretRowsCompanion,+ secretEntries,+ secretRowsFromInput,+ secretRowsToValue,+} from './dictionarySecretRows'import { CyclotronJobInputIntegration } from './integrations/CyclotronJobInputIntegration'import { CyclotronJobInputIntegrationField } from './integrations/CyclotronJobInputIntegrationField'import { CyclotronJobInputIntegrationMulti } from './integrations/CyclotronJobInputIntegrationMulti'@@ -119,6 +128,8 @@ function isValidNonFailureStatusCode(entry: string): boolean {export type CyclotronJobInputsProps = {onInputChange?: (key: string, input: CyclotronJobInputType) => void+ // Applies changes to several inputs in one update, for a host that builds each update from a snapshot+ onInputsChange?: (inputs: Record<string, CyclotronJobInputType>) => voidconfiguration: CyclotronJobInputConfigurationerrors?: Record<string, string>warnings?: Record<string, string>@@ -143,6 +154,7 @@ export function CyclotronJobInputs({parentConfiguration,onInputSchemaChange,onInputChange,+ onInputsChange,errors,warnings,emailFieldErrors,@@ -185,6 +197,7 @@ export function CyclotronJobInputs({parentConfiguration={parentConfiguration}onInputSchemaChange={onInputSchemaChange}onInputChange={onInputChange}+ onInputsChange={onInputsChange}showSource={showSource}sampleGlobalsWithInputs={sampleGlobalsWithInputs}errors={errors}@@ -374,14 +387,21 @@ function CyclotronJobTemplateInput(props: {)}+const isFilledEntry = ([key, val]: [string, any]): boolean =>+ key.trim() !== '' || typeof val !== 'string' || val.trim() !== ''+function DictionaryField({input,onChange,+ onMakeSecret,+ secretLabel,templating,sampleGlobalsWithInputs,}: {input: CyclotronJobInputTypeonChange?: (value: CyclotronJobInputType) => void+ onMakeSecret?: (key: string, value: any, remaining: Record<string, any>) => void+ secretLabel?: stringtemplating: booleansampleGlobalsWithInputs: CyclotronJobInvocationGlobalsWithInputs | null}): JSX.Element {@@ -391,9 +411,7 @@ function DictionaryField({useEffect(() => {// NOTE: Filter out all empty entries as fetch will throw if passed in- const filteredEntries = entries.filter(- ([key, val]) => key.trim() !== '' || typeof val !== 'string' || val.trim() !== ''- )+ const filteredEntries = entries.filter(isFilledEntry)// Compare with previous filtered entries to avoid unnecessary updatesif (objectsEqual(filteredEntries, prevFilteredEntriesRef.current)) {@@ -407,10 +425,29 @@ function DictionaryField({onChange?.({ ...input, value: val }) // oxlint-disable-line react-hooks/exhaustive-deps}, [entries, onChange])+ // Clear changes or a save response replaces the value. Rebuild the rows from it, but not from an echo of our own update.+ useEffect(() => {+ const incoming = Object.entries(input.value ?? {})+ // Duplicate keys collapse in the emitted value, so compare against that rather than the raw rows.+ if (!objectsEqual(incoming, Object.entries(Object.fromEntries(prevFilteredEntriesRef.current)))) {+ prevFilteredEntriesRef.current = incoming+ setEntries(incoming)+ }+ }, [input.value])+const handleEnableIncludeObject = (): void => {setEntries((prev) => [[EXTEND_OBJECT_KEY, '{event.properties}'], ...prev])}+ const makeSecret = (index: number): void => {+ const [key, val] = entries[index]+ const remaining = entries.filter((_, i) => i !== index)+ // onMakeSecret writes both inputs in one change, so the effect above must not write the plain one again.+ prevFilteredEntriesRef.current = remaining.filter(isFilledEntry)+ setEntries(remaining)+ onMakeSecret?.(key, val, Object.fromEntries(prevFilteredEntriesRef.current))+ }+return (<div className="deprecated-space-y-2">{templating && !entries.some(([key]) => key === EXTEND_OBJECT_KEY) ? (@@ -424,7 +461,7 @@ function DictionaryField({<LemonInputvalue={key === EXTEND_OBJECT_KEY ? 'INCLUDE ENTIRE OBJECT' : key}disabled={key === EXTEND_OBJECT_KEY}- className="flex-1 min-w-60"+ className="flex-1 min-w-0"onChange={(key) => {setEntries((prev) => {const newEntries = [...prev]@@ -455,6 +492,17 @@ function DictionaryField({sampleGlobalsWithInputs={sampleGlobalsWithInputs}/>+ {onMakeSecret && key !== EXTEND_OBJECT_KEY ? (+ <LemonButton+ icon={<IconLock />}+ size="small"+ data-attr="dictionary-row-make-secret"+ tooltip={`Move to ${secretLabel}. It is stored encrypted and hidden after saving.`}+ disabledReason={key.trim() === '' ? 'Enter a key first' : undefined}+ onClick={() => makeSecret(index)}+ />+ ) : null}+<LemonButtonicon={<IconX />}size="small"@@ -482,6 +530,87 @@ function DictionaryField({)}+function SecretDictionaryField({+ input,+ onChange,+}: {+ input: CyclotronJobInputType+ onChange?: (value: CyclotronJobInputType) => void+}): JSX.Element {+ const [rows, setRows] = useState<SecretDictionaryRow[]>(() => secretRowsFromInput(input))+ const lastValue = useRef<unknown>(secretRowsToValue(rows))++ // A save response or a row made secret in the plain field replaces the stored entries.+ useEffect(() => {+ const incoming = secretEntries(input) ?? {}+ if (!objectsEqual(incoming, lastValue.current)) {+ lastValue.current = incoming+ setRows(secretRowsFromInput(input))+ }+ }, [input.value]) // oxlint-disable-line react-hooks/exhaustive-deps++ const updateRows = (next: SecretDictionaryRow[]): void => {+ setRows(next)+ const value = secretRowsToValue(next)+ if (!objectsEqual(value, lastValue.current)) {+ lastValue.current = value+ // Save sends a `secret` input as "keep the stored value", so an edited one must drop the flag.+ onChange?.({ ...input, secret: false, value })+ }+ }+ const updateRow = (index: number, patch: Partial<SecretDictionaryRow>): void =>+ updateRows(rows.map((row, i) => (i === index ? { ...row, ...patch } : row)))++ return (+ <div className="deprecated-space-y-2">+ {rows.map((row, index) => (+ <div className="flex gap-2 items-center" key={index}>+ <LemonInput+ value={row.key}+ disabled={row.stored}+ className="flex-1 min-w-0"+ onChange={(key) => updateRow(index, { key })}+ placeholder="Key"+ />+ {row.masked ? (+ <div className="flex flex-2 gap-2 items-center px-2 min-w-0 h-10 rounded border border-dashed overflow-hidden">+ <span className="flex-1 min-w-0 italic truncate text-secondary">Value is hidden</span>+ <LemonButton+ size="xsmall"+ type="secondary"+ onClick={() => updateRow(index, { masked: false })}+ >+ Replace+ </LemonButton>+ </div>+ ) : (+ <LemonInput+ type="password"+ className="flex-2 min-w-0 ph-no-capture"+ value={row.value}+ onChange={(value) => updateRow(index, { value })}+ placeholder="Secret value"+ />+ )}+ <LemonButton+ icon={<IconX />}+ size="small"+ onClick={() => updateRows(rows.filter((_, i) => i !== index))}+ />+ </div>+ ))}+ <LemonButton+ icon={<IconPlus />}+ size="small"+ type="secondary"+ onClick={() => updateRows([...rows, { key: '', value: '', masked: false, stored: false }])}+ >+ Add entry+ </LemonButton>+ </div>+ )+}+function BooleanField({input,onChange,@@ -580,6 +709,7 @@ type CyclotronJobInputProps = {input: CyclotronJobInputTypeonChange?: (value: CyclotronJobInputType) => voidonInputChange?: (key: string, input: CyclotronJobInputType) => void+ onInputsChange?: (inputs: Record<string, CyclotronJobInputType>) => voiddisabled?: booleanconfiguration: CyclotronJobInputConfigurationparentConfiguration?: CyclotronJobInputConfiguration@@ -632,6 +762,7 @@ function NonFailureStatusCodesField({function CyclotronJobInputRenderer({onChange,onInputChange,+ onInputsChange,schema,disabled,input,@@ -689,15 +820,45 @@ function CyclotronJobInputRenderer({disabled={disabled}/>)- case 'dictionary':+ case 'dictionary': {+ if (isSecretRowsCompanion(schema, configuration.inputs_schema)) {+ return <SecretDictionaryField input={input} onChange={onChange} />+ }+ // A caller with only `onChange` (the schema default editor) cannot write the companion input.+ const secretSchema =+ onInputsChange || onInputChange+ ? findSecretRowsCompanion(schema, configuration.inputs_schema)+ : undefined+ const secretInput = secretSchema ? configuration.inputs?.[secretSchema.key] : undefinedreturn (<DictionaryFieldinput={input}onChange={onChange}+ onMakeSecret={+ secretSchema && canAddSecretRow(secretInput)+ ? (key, value, remaining) => {+ const changes = {+ [schema.key]: { ...input, value: remaining },+ [secretSchema.key]: {+ ...secretInput,+ secret: false,+ value: { ...secretEntries(secretInput), [key]: value },+ },+ }+ if (onInputsChange) {+ onInputsChange(changes)+ } else {+ Object.entries(changes).forEach(([k, change]) => onInputChange?.(k, change))+ }+ }+ : undefined+ }+ secretLabel={secretSchema?.label || secretSchema?.key}templating={templating}sampleGlobalsWithInputs={sampleGlobalsWithInputs}/>)+ }case 'boolean':return (<BooleanField@@ -937,6 +1098,7 @@ function CyclotronJobInputWithSchema({parentConfiguration,onInputSchemaChange,onInputChange,+ onInputsChange,showSource,sampleGlobalsWithInputs,errors,@@ -1079,7 +1241,8 @@ function CyclotronJobInputWithSchema({/>)}</div>- {value?.secret ? (+ {value?.secret &&+ !(isSecretRowsCompanion(schema, configuration.inputs_schema) && secretEntries(value)) ? (<div className="flex gap-2 items-center p-1 rounded border border-dashed"><span className="flex-1 p-1 italic text-secondary">This value is secret and is not displayed here.@@ -1100,6 +1263,7 @@ function CyclotronJobInputWithSchema({input={value ?? { value: '' }}onChange={onChange}onInputChange={onInputChange}+ onInputsChange={onInputsChange}configuration={configuration}parentConfiguration={parentConfiguration}sampleGlobalsWithInputs={sampleGlobalsWithInputs}diff --git a/frontend/src/lib/components/CyclotronJob/dictionarySecretRows.test.ts b/frontend/src/lib/components/CyclotronJob/dictionarySecretRows.test.tsnew file mode 100644index 000000000000..400ed950fb47--- /dev/null+++ b/frontend/src/lib/components/CyclotronJob/dictionarySecretRows.test.ts@@ -0,0 +1,66 @@+import { CyclotronJobInputSchemaType, CyclotronJobInputType } from '~/types'++import {+ MASKED_SECRET_VALUE,+ SecretDictionaryRow,+ canAddSecretRow,+ findSecretRowsCompanion,+ secretRowsFromInput,+ secretRowsToValue,+} from './dictionarySecretRows'++const STORED: CyclotronJobInputType = {+ secret: true,+ value: { Authorization: MASKED_SECRET_VALUE, 'X-Api-Key': MASKED_SECRET_VALUE },+}++describe('dictionarySecretRows', () => {+ it.each<[string, (rows: SecretDictionaryRow[]) => SecretDictionaryRow[], Record<string, string>]>([+ ['sends the mask for every untouched entry', (rows) => rows, STORED.value],+ [+ 'sends the mask for entries left hidden next to a replaced one',+ (rows) => rows.map((r) => (r.key === 'Authorization' ? { ...r, value: 'Bearer new', masked: false } : r)),+ { Authorization: 'Bearer new', 'X-Api-Key': MASKED_SECRET_VALUE },+ ],+ [+ 'drops a deleted entry',+ (rows) => rows.filter((r) => r.key !== 'Authorization'),+ { 'X-Api-Key': MASKED_SECRET_VALUE },+ ],+ [+ 'keeps the stored value for an entry opened with Replace but left empty',+ (rows) => rows.map((r) => ({ ...r, masked: false })),+ STORED.value,+ ],+ [+ 'ignores empty rows',+ (rows) => [...rows, { key: ' ', value: '', masked: false, stored: false }],+ STORED.value,+ ],+ ])('%s', (_name, edit, expected) => {+ expect(secretRowsToValue(edit(secretRowsFromInput(STORED)))).toEqual(expected)+ })++ it.each<[string, CyclotronJobInputType | undefined, boolean]>([+ ['a new function', undefined, true],+ ['a stored secret with its keys', STORED, true],+ ['a stored secret without its keys', { secret: true, value: undefined }, false],+ ])('allows adding a secret row for %s', (_name, stored, allowed) => {+ expect(canAddSecretRow(stored)).toBe(allowed)+ })++ const HEADERS = { key: 'headers', type: 'dictionary', label: 'Headers' } as CyclotronJobInputSchemaType+ const SECRET_HEADERS = {+ key: 'secret_headers',+ type: 'dictionary',+ label: 'Secret headers',+ secret: true,+ } as CyclotronJobInputSchemaType++ it.each<[string, CyclotronJobInputSchemaType[], boolean]>([+ ['a secret companion input', [HEADERS, SECRET_HEADERS], true],+ ['no secret companion input', [HEADERS], false],+ ])('pairs the secret rows input for %s', (_name, inputsSchema, paired) => {+ expect(findSecretRowsCompanion(HEADERS, inputsSchema)).toBe(paired ? SECRET_HEADERS : undefined)+ })+})diff --git a/frontend/src/lib/components/CyclotronJob/dictionarySecretRows.ts b/frontend/src/lib/components/CyclotronJob/dictionarySecretRows.tsnew file mode 100644index 000000000000..5abc122315f7--- /dev/null+++ b/frontend/src/lib/components/CyclotronJob/dictionarySecretRows.ts@@ -0,0 +1,52 @@+import { CyclotronJobInputSchemaType, CyclotronJobInputType } from '~/types'++// The API returns this in place of each stored secret entry, and restores the stored value when it comes back.+export const MASKED_SECRET_VALUE = '********'++// `stored` rows already have a saved value; `masked` ones still show it as hidden.+export type SecretDictionaryRow = { key: string; value: string; masked: boolean; stored: boolean }++// A secret dictionary input named `secret_<key>` holds the rows of the plain `<key>` dictionary that the user made secret.+export function findSecretRowsCompanion(+ schema: CyclotronJobInputSchemaType,+ inputsSchema: CyclotronJobInputSchemaType[] | undefined+): CyclotronJobInputSchemaType | undefined {+ if (schema.type !== 'dictionary' || schema.secret) {+ return undefined+ }+ return inputsSchema?.find((s) => s.key === `secret_${schema.key}` && s.type === 'dictionary' && s.secret)+}++export function isSecretRowsCompanion(+ schema: CyclotronJobInputSchemaType,+ inputsSchema: CyclotronJobInputSchemaType[] | undefined+): boolean {+ return !!inputsSchema?.some((s) => findSecretRowsCompanion(s, inputsSchema) === schema)+}++export function secretEntries(input: CyclotronJobInputType | undefined): Record<string, unknown> | undefined {+ const value = input?.value+ return value && typeof value === 'object' && !Array.isArray(value) ? value : undefined+}++// A stored secret that comes back without its keys (a draft) cannot take a new row without losing the stored ones.+export function canAddSecretRow(secretInput: CyclotronJobInputType | undefined): boolean {+ return !secretInput?.secret || !!secretEntries(secretInput)+}++export function secretRowsFromInput(input: CyclotronJobInputType): SecretDictionaryRow[] {+ return Object.entries(secretEntries(input) ?? {}).map(([key, value]) =>+ value === MASKED_SECRET_VALUE+ ? { key, value: '', masked: true, stored: true }+ : { key, value: String(value ?? ''), masked: false, stored: false }+ )+}++export function secretRowsToValue(rows: SecretDictionaryRow[]): Record<string, string> {+ return Object.fromEntries(+ rows+ .filter((r) => r.masked || r.key.trim() !== '' || r.value.trim() !== '')+ // A stored row the user opened with Replace but left empty keeps its stored value.+ .map((r) => [r.key, r.masked || (r.stored && r.value === '') ? MASKED_SECRET_VALUE : r.value])+ )+}diff --git a/frontend/src/scenes/hog-functions/mapping/HogFunctionMappings.tsx b/frontend/src/scenes/hog-functions/mapping/HogFunctionMappings.tsxindex 6c182c1dad00..6de8453d1951 100644--- a/frontend/src/scenes/hog-functions/mapping/HogFunctionMappings.tsx+++ b/frontend/src/scenes/hog-functions/mapping/HogFunctionMappings.tsx@@ -149,6 +149,9 @@ export function HogFunctionMapping({onInputChange={(key, value) => {onChange({ ...mapping, inputs: { ...mapping.inputs, [key]: value } })}}+ onInputsChange={(changes) => {+ onChange({ ...mapping, inputs: { ...mapping.inputs, ...changes } })+ }}showSource={showSource}sampleGlobalsWithInputs={sampleGlobalsWithInputs}/>diff --git a/products/cdp/backend/api/hog_function.py b/products/cdp/backend/api/hog_function.pyindex 54b9c4e87769..0fb4bc524bd6 100644--- a/products/cdp/backend/api/hog_function.py+++ b/products/cdp/backend/api/hog_function.py@@ -78,7 +78,9 @@ from products.cdp.backend.validation import (MappingsSerializer,added_duplicate_input_keys,compile_hog,+ exposes_secret_dictionary_keys,generate_template_bytecode,+ masked_dictionary_secret,masked_secret_input_keys,require_viewer_access_to_added_actions,reserved_functions_used,@@ -172,13 +174,14 @@ def mask_config_secrets(content: dict) -> dict:continueschemas = config.get("inputs_schema") or []secret_keys = {schema["key"] for schema in schemas if schema.get("secret") and "key" in schema}+ dictionary_keys = {schema.get("key") for schema in schemas if exposes_secret_dictionary_keys(schema, schemas)}for schema in schemas:if schema.get("key") in secret_keys:schema.pop("default", None)inputs = config.get("inputs") or {}for key in secret_keys:if key in inputs:- inputs[key] = {"secret": True}+ inputs[key] = masked_dictionary_secret(inputs[key]) if key in dictionary_keys else {"secret": True}if secret_keys and "transpiled" in content:content["transpiled"] = Nonereturn content@@ -867,7 +870,11 @@ class HogFunctionSerializer(HogFunctionMinimalSerializer):has_value = encrypted_inputs.get(schema["key"]) or inputs.get(schema["key"])if has_value:# Marker to indicate to the user that a secret is set- inputs[schema["key"]] = {"secret": True}+ inputs[schema["key"]] = (+ masked_dictionary_secret(has_value)+ if exposes_secret_dictionary_keys(schema, inputs_schema)+ else {"secret": True}+ )data["inputs"] = inputsdata["draft"] = self._mask_draft_secrets(data.get("draft"), draft_encrypted_inputs, encrypted_inputs)diff --git a/products/cdp/backend/api/test/test_hog_function.py b/products/cdp/backend/api/test/test_hog_function.pyindex 7f565c802d08..9f507fb30e72 100644--- a/products/cdp/backend/api/test/test_hog_function.py+++ b/products/cdp/backend/api/test/test_hog_function.py@@ -1179,9 +1179,11 @@ class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):"order": 0,},"gAAAAABlkgC8AAAAAAAAAAAAAAAAAAAAAKvzDjuLG689YjjVhmmbXAtZSRoucXuT8VtokVrCotIx3ttPcVufoVt76dyr2phbuotMldKMVv_Y6uzMDZFjX1VQVJqL13wH-WALMn9obfpLYD_WWOUdMA6VurFg1TxdopwQKcL10Y5Yg8s8Gswibi1pCMfjwSnKwod91SMtLKgNfAU4EPZ6GxA77xCHIjaTLueR3qx-hy2Pu3W0r5Rh1hWy0bq01uIdulQ_LhxkQgpj",+ {"secret": True},+ [],),(- "dictionary_without_templating",+ "dictionary_paired_with_plain_dictionary",{"key": "secret_headers","type": "dictionary",@@ -1193,20 +1195,47 @@ class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):{"Authorization": "Bearer I AM SECRET"},{"value": {"Authorization": "Bearer I AM SECRET"}, "order": 0},None,+ {"secret": True, "value": {"Authorization": "********"}},+ [{"key": "headers", "type": "dictionary", "label": "Headers", "required": False}],+ ),+ (+ "dictionary_without_plain_pair",+ {+ "key": "account_lookup",+ "type": "dictionary",+ "label": "Account lookup",+ "secret": True,+ "required": False,+ "templating": False,+ },+ {"I AM SECRET": "account-1"},+ {"value": {"I AM SECRET": "account-1"}, "order": 0},+ None,+ {"secret": True},+ [],),])def test_secret_inputs_not_returned(- self, _name, schema, secret_value, expected_encrypted_input, expected_ciphertext, *args+ self,+ _name,+ schema,+ secret_value,+ expected_encrypted_input,+ expected_ciphertext,+ expected_masked,+ other_schemas,+ *args,):payload = {"name": "Fetch URL","hog": "fetch(inputs.url);",- "inputs_schema": [schema],+ "inputs_schema": [schema, *other_schemas],"inputs": {schema["key"]: {"value": secret_value}},"type": "destination",}- expectation = {schema["key"]: {"secret": True}}+ unset_inputs = {other["key"]: None for other in other_schemas}+ expectation = {schema["key"]: expected_masked, **unset_inputs}# Fernet encryption is deterministic, but has a temporal component and utilizes os.urandom() for the IVwith time_machine.travel("2024-01-01T00:01:00Z", tick=False):with patch("os.urandom", return_value=b"\x00" * 16):@@ -1219,7 +1248,7 @@ class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):# Finally check the DB has the real valueobj = HogFunction.objects.get(id=res.json()["id"])- assert obj.inputs == {}+ assert obj.inputs == unset_inputsassert obj.encrypted_inputs == {schema["key"]: expected_encrypted_input}raw_encrypted_inputs = get_db_field_value("encrypted_inputs", obj.id)diff --git a/products/cdp/backend/test/test_validation.py b/products/cdp/backend/test/test_validation.pyindex 880bf8c37601..fb45b2764a76 100644--- a/products/cdp/backend/test/test_validation.py+++ b/products/cdp/backend/test/test_validation.py@@ -934,15 +934,21 @@ class TestHogFunctionValidation(ClickhouseTestMixin, APIBaseTest, QueryMatchingT@parameterized.expand([# Read-back mask flagged as secret, nothing stored to restore.- ({"value": "********", "secret": True},),+ ("string", {"value": "********", "secret": True}, {}),# Mask that lost its secret flag - the persistence guard must still refuse it.- ({"value": "********"},),+ ("string", {"value": "********"}, {}),+ # A hidden header row whose stored entry is gone.+ (+ "dictionary",+ {"value": {"Authorization": "********"}},+ {"secret_field": {"value": {"X-Api-Key": "STORED"}}},+ ),])- def test_masked_secret_without_stored_value_is_rejected(self, input_value):+ def test_masked_secret_without_stored_value_is_rejected(self, input_type, input_value, existing_secret_inputs):# The mask must never be encrypted as the real credential when there is nothing to restore.inputs_schema = [- {"key": "secret_field", "type": "string", "required": True, "secret": True},+ {"key": "secret_field", "type": input_type, "required": True, "secret": True},]serializer = FunctionInputsSerializer(@@ -950,11 +956,52 @@ class TestHogFunctionValidation(ClickhouseTestMixin, APIBaseTest, QueryMatchingT"inputs_schema": inputs_schema,"inputs": {"secret_field": input_value},},- context={"function_type": "destination", "encrypted_inputs": {}},+ context={"function_type": "destination", "encrypted_inputs": existing_secret_inputs},)with self.assertRaises(ValidationError):serializer.is_valid(raise_exception=True)+ @parameterized.expand(+ [+ (+ "untouched",+ {"Authorization": "********", "X-Api-Key": "********"},+ {"Authorization": "Bearer STORED", "X-Api-Key": "STORED KEY"},+ ),+ (+ "one_replaced",+ {"Authorization": "Bearer NEW", "X-Api-Key": "********"},+ {"Authorization": "Bearer NEW", "X-Api-Key": "STORED KEY"},+ ),+ (+ "one_removed_one_added",+ {"X-Api-Key": "********", "X-Other": "OTHER"},+ {"X-Api-Key": "STORED KEY", "X-Other": "OTHER"},+ ),+ ]+ )+ def test_secret_dictionary_restores_masked_entries(self, _name, sent, expected):+ serializer = FunctionInputsSerializer(+ data={+ "inputs_schema": [+ {"key": "secret_headers", "type": "dictionary", "required": False, "secret": True},+ ],+ "inputs": {"secret_headers": {"value": sent}},+ },+ context={+ "function_type": "destination",+ "encrypted_inputs": {+ "secret_headers": {+ "value": {"Authorization": "Bearer STORED", "X-Api-Key": "STORED KEY"},+ "templating": "liquid",+ }+ },+ },+ )+ serializer.is_valid(raise_exception=True)+ assert serializer.validated_data["inputs"]["secret_headers"]["value"] == expected+ assert serializer.validated_data["inputs"]["secret_headers"]["templating"] == "liquid"+def test_validate_filters_builds_bytecode(self):filters = {"properties": [{"key": "email", "value": ["[email protected]"], "operator": "exact", "type": "person"}],diff --git a/products/cdp/backend/validation.py b/products/cdp/backend/validation.pyindex aff88702f5fc..c0b8ad0de0ad 100644--- a/products/cdp/backend/validation.py+++ b/products/cdp/backend/validation.py@@ -983,6 +983,39 @@ class InputsItemSerializer(serializers.Serializer):return attrs+def exposes_secret_dictionary_keys(schema: dict, schemas: list) -> bool:+ """Only a `secret_<key>` dictionary paired with a plain `<key>` dictionary holds locked rows the editor lists by key."""+ key = schema.get("key")+ if schema.get("type") != "dictionary" or not isinstance(key, str) or not key.startswith("secret_"):+ return False+ plain_key = key.removeprefix("secret_")+ return any(+ isinstance(s, dict) and s.get("key") == plain_key and s.get("type") == "dictionary" and not s.get("secret")+ for s in schemas+ )+++def masked_dictionary_secret(stored: Any) -> dict:+ """Mask a secret dictionary input entry by entry, so the editor can still list its keys."""+ entries = stored.get("value") if isinstance(stored, dict) else None+ if isinstance(entries, dict):+ return {"secret": True, "value": dict.fromkeys(entries, MASKED_SECRET_VALUE)}+ return {"secret": True}+++def restore_masked_dictionary_entries(value: Any, stored: Any) -> tuple[Any, list[str]]:+ """Put each stored secret back where the editor sent the mask for an entry it did not retype."""+ entries = value.get("value") if isinstance(value, dict) else None+ if not isinstance(entries, dict) or MASKED_SECRET_VALUE not in entries.values():+ return value, []+ stored_entries = stored.get("value") if isinstance(stored, dict) else None+ stored_entries = stored_entries if isinstance(stored_entries, dict) else {}+ missing = [k for k, v in entries.items() if v == MASKED_SECRET_VALUE and k not in stored_entries]+ restored = {k: stored_entries.get(k) if v == MASKED_SECRET_VALUE else v for k, v in entries.items()}+ stored_metadata = {k: v for k, v in stored.items() if k != "value"} if isinstance(stored, dict) else {}+ return {**stored_metadata, **value, "value": restored}, missing++class InputsSerializer(serializers.DictField):"""Provides the same typing as the DictField but with custom validation to only include the inputs that are in the schema@@ -1020,6 +1053,12 @@ class InputsSerializer(serializers.DictField):and bool(value.get("secret"))and ("value" not in value or value.get("value") == MASKED_SECRET_VALUE))+ if not is_masked and schema.get("type") == "dictionary":+ value, missing = restore_masked_dictionary_entries(value, (existing_secret_inputs or {}).get(key))+ if missing:+ errors[key] = f"No value is saved for {', '.join(missing)}. Enter the value again."+ continue+if is_masked or value == {}:existing_value = (existing_secret_inputs or {}).get(key)if existing_value:diff --git a/products/workflows/frontend/Workflows/hogflows/steps/components/HogFlowFunctionConfiguration.tsx b/products/workflows/frontend/Workflows/hogflows/steps/components/HogFlowFunctionConfiguration.tsxindex a03269288c0a..7cb2788dbe9c 100644--- a/products/workflows/frontend/Workflows/hogflows/steps/components/HogFlowFunctionConfiguration.tsx+++ b/products/workflows/frontend/Workflows/hogflows/steps/components/HogFlowFunctionConfiguration.tsx@@ -273,6 +273,7 @@ export function HogFlowFunctionConfiguration({showSource={false}sampleGlobalsWithInputs={sampleGlobals}onInputChange={(key, value) => setInputs({ ...inputs, [key]: value })}+ onInputsChange={(changes) => setInputs({ ...inputs, ...changes })}/>)