:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add clerk webhook sync (#117917)
posthog[bot]55318df0da04
diff --git a/common/hogvm/CHANGELOG.md b/common/hogvm/CHANGELOG.mdindex ec5eae6b9e99..94f7210f6a2d 100644--- a/common/hogvm/CHANGELOG.md+++ b/common/hogvm/CHANGELOG.md@@ -1,5 +1,18 @@# HogQL bytecode changelog+## 2026-10-15 - 1.0.72++Added `sha256HmacBase64Key` to the standard library. No bytecode operations changed.++```bash+sha256HmacBase64Key(key, data, encoding) # HMAC-SHA256 of data, key is base64, encoding defaults to 'hex'+```++The function decodes the key from base64 and uses the raw bytes as the HMAC key. Svix-style webhook+secrets (`whsec_...`) are random bytes, so `sha256HmacChain` cannot verify them: it uses the UTF-8+bytes of a string key. The key is accepted with or without base64 padding. `encoding` is one of+`hex`, `base64`, `base64url`, or `binary`.+## 2026-09-28 - 1.0.71`range` checks the requested length against the memory limit before it builds the array, as thediff --git a/common/hogvm/__tests__/__snapshots__/crypto.hoge b/common/hogvm/__tests__/__snapshots__/crypto.hogeindex 32560867cf55..079e3bcd973b 100644--- a/common/hogvm/__tests__/__snapshots__/crypto.hoge+++ b/common/hogvm/__tests__/__snapshots__/crypto.hoge@@ -33,4 +33,8 @@32, "Hi There", 43, 2, 2, "sha1HmacChainHex", 1, 2, "print", 2, 35, 32, "rfc2202 case 2:", 32, "Jefe", 32,"what do ya want for nothing?", 43, 2, 2, "sha1HmacChainHex", 1, 2, "print", 2, 35, 32, "rfc2202 case 5:", 32,"\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f", 32, "Test With Truncation", 43, 2, 2, "sha1HmacChainHex", 1, 2, "print", 2,-35, 35, 35]+35, 32, "MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw", 32, "msg_p5jXN8AQM9LWM0D4loKWxJek.1614265330.{\"test\": 2432232314}", 32,+"sha256HmacBase64Key(svix, 'base64'):", 36, 2, 36, 3, 32, "base64", 2, "sha256HmacBase64Key", 3, 2, "print", 2, 35, 32,+"sha256HmacBase64Key(svix):", 36, 2, 36, 3, 2, "sha256HmacBase64Key", 2, 2, "print", 2, 35, 32,+"sha256HmacBase64Key(unpadded):", 32, "YWI=", 32, "data", 2, "sha256HmacBase64Key", 2, 32, "YWI", 32, "data", 2,+"sha256HmacBase64Key", 2, 11, 2, "print", 2, 35, 35, 35, 35, 35]diff --git a/common/hogvm/__tests__/__snapshots__/crypto.stdout b/common/hogvm/__tests__/__snapshots__/crypto.stdoutindex 195842a42d1b..2ace24d80d68 100644Binary files a/common/hogvm/__tests__/__snapshots__/crypto.stdout and b/common/hogvm/__tests__/__snapshots__/crypto.stdout differdiff --git a/common/hogvm/__tests__/crypto.hog b/common/hogvm/__tests__/crypto.hogindex be3324ea3909..d71fea9b4102 100644--- a/common/hogvm/__tests__/crypto.hog+++ b/common/hogvm/__tests__/crypto.hog@@ -47,3 +47,11 @@ print('sha1Hex(\'\'):', sha1Hex(''))print('rfc2202 case 1:', sha1HmacChainHex(['\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v', 'Hi There']))print('rfc2202 case 2:', sha1HmacChainHex(['Jefe', 'what do ya want for nothing?']))print('rfc2202 case 5:', sha1HmacChainHex(['\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f', 'Test With Truncation']))++// Svix signature known answer, from the Svix webhook verification docs. The key decodes to bytes+// that are not valid UTF-8, so a string key would give a different digest.+let svixKey := 'MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw'+let svixPayload := 'msg_p5jXN8AQM9LWM0D4loKWxJek.1614265330.{"test": 2432232314}'+print('sha256HmacBase64Key(svix, \'base64\'):', sha256HmacBase64Key(svixKey, svixPayload, 'base64'))+print('sha256HmacBase64Key(svix):', sha256HmacBase64Key(svixKey, svixPayload))+print('sha256HmacBase64Key(unpadded):', sha256HmacBase64Key('YWI', 'data') = sha256HmacBase64Key('YWI=', 'data'))diff --git a/common/hogvm/python/stl/__init__.py b/common/hogvm/python/stl/__init__.pyindex dffde88be173..3fecc0fee728 100644--- a/common/hogvm/python/stl/__init__.py+++ b/common/hogvm/python/stl/__init__.py@@ -22,7 +22,7 @@ from ..utils import (like,regex_extract,)-from .crypto import md5, sha1, sha1HmacChain, sha256, sha256HmacChain+from .crypto import md5, sha1, sha1HmacChain, sha256, sha256HmacBase64Key, sha256HmacChainfrom .date import (formatDateTime,fromUnixTimestamp,@@ -1116,6 +1116,13 @@ STL: dict[str, STLFunction] = {minArgs=1,maxArgs=2,),+ "sha256HmacBase64Key": STLFunction(+ fn=lambda args, team, stdout, timeout: sha256HmacBase64Key(+ args[0], args[1], args[2] if len(args) > 2 and args[2] is not None else "hex"+ ),+ minArgs=2,+ maxArgs=3,+ ),"isIPAddressInRange": STLFunction(fn=lambda args, team, stdout, timeout: isIPAddressInRange(args[0], args[1]), minArgs=2, maxArgs=2),diff --git a/common/hogvm/python/stl/crypto.py b/common/hogvm/python/stl/crypto.pyindex 2eb04d48d5b5..3dcb1a60cb10 100644--- a/common/hogvm/python/stl/crypto.py+++ b/common/hogvm/python/stl/crypto.py@@ -61,3 +61,12 @@ def sha256HmacChain(data: list, encoding: Literal["hex", "base64", "base64url",def sha1HmacChain(data: list, encoding: Literal["hex", "base64", "base64url", "binary"] = "hex") -> str:return _hmac_chain(data, "sha1", encoding)+++def sha256HmacBase64Key(key: str, data: str, encoding: Literal["hex", "base64", "base64url", "binary"] = "hex") -> str:+ # Svix-style webhook secrets are random bytes, which a string key would corrupt. The extra+ # padding makes the decode accept a key with or without padding, as Node does.+ key_bytes = base64.b64decode(key.strip() + "==")+ digest = hmac.new(key_bytes, data.encode(), hashlib.sha256).digest()++ return encode_digest(encoding, digest)diff --git a/common/hogvm/typescript/package.json b/common/hogvm/typescript/package.jsonindex ff69bb6dc18a..3fcc3d3062a2 100644--- a/common/hogvm/typescript/package.json+++ b/common/hogvm/typescript/package.json@@ -1,6 +1,6 @@{"name": "@posthog/hogvm",- "version": "1.0.71",+ "version": "1.0.72","description": "PostHog Hog Virtual Machine","license": "MIT","author": "PostHog <[email protected]>",diff --git a/common/hogvm/typescript/src/stl/crypto.ts b/common/hogvm/typescript/src/stl/crypto.tsindex 150d0e11e48f..960ab35ce604 100644--- a/common/hogvm/typescript/src/stl/crypto.ts+++ b/common/hogvm/typescript/src/stl/crypto.ts@@ -85,3 +85,18 @@ export function sha1HmacChain(): string {return hmacChain('sha1', 'sha1HmacChainHex', data, encoding, options)}++// HMAC-SHA256 with a base64 encoded binary key. Svix-style webhook secrets are random bytes, which a+// string key would corrupt.+export function sha256HmacBase64Key(+ key: string,+ data: string,+ encoding: 'hex' | 'base64' | 'base64url' | 'binary' = 'hex',+ options?: ExecOptions+): string {+ const crypto = options?.external?.crypto+ if (!crypto) {+ throw new HogVMException('The crypto module is required for "sha256HmacBase64Key" to work.', 'contract')+ }+ return crypto.createHmac('sha256', Buffer.from(key, 'base64')).update(data).digest(encoding)+}diff --git a/common/hogvm/typescript/src/stl/stl.ts b/common/hogvm/typescript/src/stl/stl.tsindex 9bb7066bfa46..86d78e497722 100644--- a/common/hogvm/typescript/src/stl/stl.ts+++ b/common/hogvm/typescript/src/stl/stl.ts@@ -4,7 +4,7 @@ import { DEFAULT_MAX_MEMORY } from '../constants'import { isHogAST, isHogCallable, isHogClosure, isHogDate, isHogDateTime, isHogError, newHogError } from '../objects'import { AsyncSTLFunction, ExecOptions, HogDate, HogDateTime, HogInterval, STLFunction } from '../types'import { COST_PER_UNIT, HogVMException, calculateCost, getNestedValue, like } from '../utils'-import { md5, sha1, sha1HmacChain, sha256, sha256HmacChain } from './crypto'+import { md5, sha1, sha1HmacChain, sha256, sha256HmacBase64Key, sha256HmacChain } from './crypto'import {formatDateTime,fromUnixTimestamp,@@ -1231,6 +1231,13 @@ export const STL: Record<string, STLFunction> = {minArgs: 1,maxArgs: 2,},+ sha256HmacBase64Key: {+ fn: ([key, data, encoding], _, options) => sha256HmacBase64Key(key, data, encoding, options),+ description: 'Computes HMAC-SHA256 of a string with a base64 encoded binary key',+ example: 'sha256HmacBase64Key($1, $2, $3)',+ minArgs: 2,+ maxArgs: 3,+ },isIPAddressInRange: {fn: ([address, prefix]) => isIPAddressInRange(address, prefix),description: 'Checks if IP address is in CIDR range',diff --git a/posthog/hogql/compiler/javascript_stl.py b/posthog/hogql/compiler/javascript_stl.pyindex 6017b5c22ebe..d138c0031554 100644--- a/posthog/hogql/compiler/javascript_stl.py+++ b/posthog/hogql/compiler/javascript_stl.py@@ -375,6 +375,10 @@ STL_FUNCTIONS: dict[str, list[str | list[str]]] = {"function sha1HmacChain (data, encoding, options) { return 'sha1HmacChain not implemented' }",[],],+ "sha256HmacBase64Key": [+ "function sha256HmacBase64Key (key, data, encoding, options) { return 'sha256HmacBase64Key not implemented' }",+ [],+ ],"isIPAddressInRange": ["""function isIPAddressInRange(address, prefix) {diff --git a/products/cdp/backend/api/test/__snapshots__/test_hooks.ambr b/products/cdp/backend/api/test/__snapshots__/test_hooks.ambrindex 8af8520ff329..b3d78ac6a923 100644--- a/products/cdp/backend/api/test/__snapshots__/test_hooks.ambr+++ b/products/cdp/backend/api/test/__snapshots__/test_hooks.ambr@@ -148,7 +148,7 @@]),}),}),- 'bytecode_contract': 'bfd6ece60e41383c',+ 'bytecode_contract': '2c9c13700371fbf9','order': 1,'value': dict({'data': dict({@@ -178,7 +178,7 @@32,'hooks/standard/1234/abcd',]),- 'bytecode_contract': 'bfd6ece60e41383c',+ 'bytecode_contract': '2c9c13700371fbf9','order': 0,'value': 'hooks/standard/1234/abcd',}),diff --git a/products/cdp/backend/api/test/test_hog_function.py b/products/cdp/backend/api/test/test_hog_function.pyindex 8077939ecce4..a9713f91b720 100644--- a/products/cdp/backend/api/test/test_hog_function.py+++ b/products/cdp/backend/api/test/test_hog_function.py@@ -1179,7 +1179,7 @@ class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):"value": "I AM SECRET","order": 0,},- "gAAAAABlkgC8AAAAAAAAAAAAAAAAAAAAAKvzDjuLG689YjjVhmmbXAtZSRoucXuT8VtokVrCotIx3ttPcVufoVt76dyr2phbuotMldKMVv_Y6uzMDZFjX1VQVJqL13wH-WALMn9obfpLYD_WWOUdMA6VurFg1TxdopwQKcL10Y5Yg8s8Gswibi1pCMfjwSnKwod91SMtLKgNfAU4EPZ6GxA77xCHIjaTLueR3qx-hy2Pu3W0r5Rh1hWy0bq01uIdulQ_LhxkQgpj",+ "gAAAAABlkgC8AAAAAAAAAAAAAAAAAAAAAKvzDjuLG689YjjVhmmbXAtZSRoucXuT8VtokVrCotIx3ttPcVufoVt76dyr2phbuotMldKMVv_Y6uzMDZFjX1VQVJqL13wH-WALMn9obfpLYD_WWOUdMA6VurFg1TxdopDxFtIztV_BWe5ApI2TtI57CYnHc8Meh0Qo-Ii9TRkDbxtU0E48PqNx6eqhGROrzCtN2rORP2bj-BM983QSq6ufd6CpPUqUUvSHhvhP0klo",{"secret": True},[],),diff --git a/products/cdp/backend/filter_globals.json b/products/cdp/backend/filter_globals.jsonindex a3b949873b27..818f8743782b 100644--- a/products/cdp/backend/filter_globals.json+++ b/products/cdp/backend/filter_globals.json@@ -1,6 +1,6 @@{"$comment": "Generated. Do not edit: run `pnpm --filter=@posthog/nodejs run build:filter-globals`. roots are the data globals the CDP filter runtime builds for a hog function. callables are the standard-library names the VM hands back as values and can then invoke, so they are the ones a filter can pass as a callback. functions are every standard-library name a filter can call directly, with the argument count the VM enforces as [min, max]. template_roots are the names an input template can read. Django reads this to refuse a filter or an input the runtime could not evaluate. contract is a hash of the rest, stamped on compiled bytecode.",- "contract": "bfd6ece60e41383c",+ "contract": "2c9c13700371fbf9","roots": ["$group_0","$group_1",@@ -117,6 +117,7 @@"sha1HmacChainHex","sha256","sha256Hex",+ "sha256HmacBase64Key","sha256HmacChain","sha256HmacChainHex","splitByString",@@ -514,6 +515,10 @@1,1],+ "sha256HmacBase64Key": [+ 2,+ 3+ ],"sha256HmacChain": [1,2diff --git a/products/error_tracking/backend/hogvm_stl.py b/products/error_tracking/backend/hogvm_stl.pyindex e4b82c2d403b..aca51820ca17 100644--- a/products/error_tracking/backend/hogvm_stl.py+++ b/products/error_tracking/backend/hogvm_stl.py@@ -96,6 +96,7 @@ RUST_HOGVM_STL = ["sha256HmacChain","sha1HmacChainHex","sha1HmacChain",+ "sha256HmacBase64Key","base64Encode","base64Decode","tryBase64Decode",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.mdindex dfa326797d1b..06103a9f887e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md@@ -246,7 +246,7 @@ the row lists both.| cloudflare_r2 | HTTP (vendor SDK) | boto3 (listing), s3fs (reads), deltalake (Delta tables) | ⚠️ || clari | HTTP | requests | ✅ || clay | HTTP | requests + `rest_source.RESTClient` | ✅ |-| clerk | HTTP | requests + `rest_source.RESTClient` | ✅ |+| clerk | HTTP + Webhook | requests + `rest_source.RESTClient` + `WebhookSourceManager` | ✅ (pull) / ➖ (webhook) || clevertap | HTTP | requests + `rest_source.RESTClient` | ✅ || clickhouse | DB protocol (HTTP-based) | clickhouse-connect / clickhouse-driver | ➖ || clickhouse_cloud | HTTP | requests | ✅ |diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.pyindex 782ecb7c2509..55af97783645 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py@@ -1,17 +1,22 @@import functoolsimport dataclasses-from collections.abc import Iterator+from collections.abc import AsyncIterable, Iterable, Iteratorfrom typing import Any, Optional+import orjson+import pyarrow as pa+from asgiref.sync import async_to_syncfrom requests import Request, Responsefrom requests.exceptions import HTTPError, RequestExceptionfrom structlog.types import FilteringBoundLoggerfrom posthog.exceptions_capture import capture_exception+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_listfrom products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (CLERK_ENDPOINTS,RETIRED_ENDPOINTS,+ WEBHOOK_SCHEMA_NAMES,ClerkEndpointConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session@@ -26,6 +31,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse+from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager@dataclasses.dataclass@@ -339,6 +345,33 @@ def _skip_when_feature_disabled()+def _webhook_table_transformer(table: pa.Table) -> pa.Table:+ """Extract the Clerk object from each webhook event and keep the latest event per id."""+ rows_by_id: dict[str, tuple[int, dict[str, Any]]] = {}+ for event in table.to_pylist():+ event_type = event.get("type")+ resource = event.get("data")+ if isinstance(resource, str):+ try:+ resource = orjson.loads(resource)+ except orjson.JSONDecodeError:+ continue+ if not isinstance(event_type, str) or not isinstance(resource, dict) or not resource.get("id"):+ continue+ # The Hog template already drops deletions. This keeps a hand-configured webhook from+ # writing an id-only payload over a stored row. See WEBHOOK_EVENTS_BY_SCHEMA.+ if event_type.endswith(".deleted") or resource.get("deleted") is True:+ continue++ raw_timestamp = event.get("timestamp")+ timestamp = raw_timestamp if isinstance(raw_timestamp, int) and not isinstance(raw_timestamp, bool) else 0+ current = rows_by_id.get(str(resource["id"]))+ if current is None or timestamp >= current[0]:+ rows_by_id[str(resource["id"])] = (timestamp, _convert_timestamps(resource))++ return table_from_py_list([row for _, row in rows_by_id.values()])++def clerk_source(secret_key: str,endpoint: str,@@ -346,6 +379,7 @@ def clerk_source(job_id: str,resumable_source_manager: ResumableSourceManager[ClerkResumeConfig],logger: FilteringBoundLogger,+ webhook_source_manager: WebhookSourceManager,) -> SourceResponse:retired_reason = RETIRED_ENDPOINTS.get(endpoint)if retired_reason is not None:@@ -407,7 +441,14 @@ def clerk_source(gated_feature = endpoint_config.gated_feature- def items() -> Iterator[Any]:+ webhook_enabled = endpoint in WEBHOOK_SCHEMA_NAMES and async_to_sync(webhook_source_manager.webhook_enabled)()++ def items() -> Iterable[Any] | AsyncIterable[Any]:+ if webhook_enabled:+ return webhook_source_manager.get_items(table_transformer=_webhook_table_transformer)+ return polled_items()++ def polled_items() -> Iterator[Any]:if gated_feature is None:yield from resourcereturndiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.pyindex c3be3a1814cc..9a27309829cc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py@@ -192,3 +192,17 @@ RETIRED_ENDPOINTS: dict[str, str] = {}ENDPOINTS = tuple(CLERK_ENDPOINTS.keys())++# Clerk delivers webhooks through Svix. A table qualifies for webhook sync only when its events+# carry the full object on create and update. Delete events are not stored: `user.deleted` and+# `organization.deleted` carry only the object id, and the merge also matches on the `created_at`+# partition key, so a tombstone row could not replace the stored row. A deleted record stays in the+# table until the next full resync.+WEBHOOK_EVENTS_BY_SCHEMA: dict[str, tuple[str, ...]] = {+ "users": ("user.created", "user.updated"),+ "organizations": ("organization.created", "organization.updated"),+ "organization_memberships": ("organizationMembership.created", "organizationMembership.updated"),+}++WEBHOOK_EVENT_TO_SCHEMA = {event: schema for schema, events in WEBHOOK_EVENTS_BY_SCHEMA.items() for event in events}+WEBHOOK_SCHEMA_NAMES = frozenset(WEBHOOK_EVENTS_BY_SCHEMA)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.pyindex 16c2e390014c..3c9485ffa9d3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py@@ -1,4 +1,7 @@-from typing import Optional, cast+from typing import TYPE_CHECKING, Optional, cast++if TYPE_CHECKING:+ from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDCfrom products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.facade.source_config import (@@ -15,8 +18,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.clerclerk_source,validate_credentials as validate_clerk_credentials,)-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import ENDPOINTS, RETIRED_ENDPOINTS-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource+from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (+ ENDPOINTS,+ RETIRED_ENDPOINTS,+ WEBHOOK_SCHEMA_NAMES,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (+ FieldType,+ ResumableSource,+ WebhookCreationResult,+ WebhookDeletionResult,+ WebhookSource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (CanonicalDescriptions,)@@ -24,12 +37,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.regfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchemafrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse+from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.clerk import ClerkSourceConfigfrom products.warehouse_sources.backend.types import ExternalDataSourceType@SourceRegistry.register-class ClerkSource(ResumableSource[ClerkSourceConfig, ClerkResumeConfig]):+class ClerkSource(+ ResumableSource[ClerkSourceConfig, ClerkResumeConfig],+ WebhookSource[ClerkSourceConfig],+):codebase_detection = detect(env=("^CLERK_SECRET_KEY$",@@ -150,6 +167,16 @@ class ClerkSource(ResumableSource[ClerkSourceConfig, ClerkResumeConfig]):def source_type(self) -> ExternalDataSourceType:return ExternalDataSourceType.CLERK+ @property+ def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:+ from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.webhook_template import template++ return template++ @property+ def webhook_resource_map(self) -> dict[str, str]:+ return {name: name for name in WEBHOOK_SCHEMA_NAMES}+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(@@ -177,6 +204,28 @@ The secret key starts with `sk_live_`.),],),+ webhookSetupCaption="""Webhook sync keeps the users, organizations, and organization memberships tables up to date. Set it up in Clerk:++1. In the [Clerk Dashboard](https://dashboard.clerk.com/), go to **Configure** > **Webhooks** and select **Add Endpoint**.+2. Paste the webhook URL shown below into **Endpoint URL**.+3. Subscribe to the `user`, `organization`, and `organizationMembership` created and updated events.+4. Open the endpoint, copy its **Signing Secret**, and paste it into the field below.++Clerk delete events only carry the record ID, so webhook sync doesn't remove deleted records. Resync a table to remove them.+""",+ webhookFields=cast(+ list[FieldType],+ [+ SourceFieldInputConfig(+ name="signing_secret",+ label="Signing secret",+ type=SourceFieldInputConfigType.PASSWORD,+ required=True,+ placeholder="whsec_...",+ secret=True,+ )+ ],+ ),)def get_canonical_descriptions(self) -> CanonicalDescriptions:@@ -195,12 +244,14 @@ The secret key starts with `sk_live_`.force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Clerk only supports full refresh - the API doesn't support filtering by updated_at+ # The Clerk API can't filter by updated_at, so a polled sync is always a full refresh. The+ # tables in WEBHOOK_SCHEMA_NAMES can switch to webhook sync after the initial backfill.schemas = [SourceSchema(name=endpoint,supports_incremental=False,supports_append=False,+ supports_webhooks=endpoint in WEBHOOK_SCHEMA_NAMES,incremental_fields=[],)for endpoint in list(ENDPOINTS)@@ -273,6 +324,23 @@ The secret key starts with `sk_live_`.def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[ClerkResumeConfig]:return ResumableSourceManager[ClerkResumeConfig](inputs, ClerkResumeConfig)+ def get_webhook_source_manager(self, inputs: SourceInputs) -> WebhookSourceManager:+ return WebhookSourceManager(inputs, inputs.logger)++ def create_webhook(+ self, config: ClerkSourceConfig, webhook_url: str, team_id: int, api_version: str | None = None+ ) -> WebhookCreationResult:+ return WebhookCreationResult(+ success=False,+ error="Clerk does not support automatic webhook creation. Please follow the manual setup instructions.",+ )++ def delete_webhook(+ self, config: ClerkSourceConfig, webhook_url: str, team_id: int, api_version: str | None = None+ ) -> WebhookDeletionResult:+ # PostHog never creates the Clerk endpoint, so there is nothing to remove on Clerk's side.+ return WebhookDeletionResult(success=True)+def source_for_pipeline(self,config: ClerkSourceConfig,@@ -286,4 +354,5 @@ The secret key starts with `sk_live_`.job_id=inputs.job_id,resumable_source_manager=resumable_source_manager,logger=inputs.logger,+ webhook_source_manager=self.get_webhook_source_manager(inputs),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.pyindex c5d7200155b5..302f1e148fe2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py@@ -5,6 +5,7 @@ from typing import Anyimport pytestfrom unittest.mock import MagicMock+import pyarrow as pafrom requests import Responsefrom requests.exceptions import HTTPError, RequestExceptionfrom structlog.testing import capture_logs@@ -14,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.clerClerkResumeConfig,_convert_timestamps,_strip_sensitive_fields,+ _webhook_table_transformer,validate_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (@@ -413,3 +415,26 @@ class TestClerkValidateCredentials:assert "Couldn't reach Clerk" in (message or "")assert "connection reset by peer" not in (message or "")assert network.requests_log[0].path == "/v1/users"+++class TestClerkWebhookTransformer:+ def test_keeps_latest_event_and_skips_deletes(self) -> None:+ table = pa.Table.from_pylist(+ [+ {+ "type": "user.updated",+ "timestamp": 1767312000000,+ "data": '{"id":"user_1","email":"[email protected]","created_at":1767225600000}',+ },+ {+ "type": "user.created",+ "timestamp": 1767225600000,+ "data": '{"id":"user_1","email":"[email protected]","created_at":1767225600000}',+ },+ {"type": "user.deleted", "timestamp": 1767398400000, "data": '{"id":"user_1","deleted":true}'},+ ]+ )++ assert _webhook_table_transformer(table).to_pylist() == [+ {"id": "user_1", "email": "[email protected]", "created_at": 1767225600}+ ]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.pynew file mode 100644index 000000000000..6bf51cbc06ff--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py@@ -0,0 +1,102 @@+import hmac+import json+import time+import base64+import hashlib++from parameterized import parameterized++from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest+from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import WEBHOOK_EVENT_TO_SCHEMA+from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.webhook_template import template++# Svix secrets are random bytes. These are not valid UTF-8, so a template that uses the secret as a+# string key fails to verify.+KEY_BYTES = bytes(range(0xE0, 0xF8))+SECRET = "whsec_" + base64.b64encode(KEY_BYTES).decode()+++class TestClerkWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):+ template = template++ def createHogGlobals(self, globals=None) -> dict:+ data = {"request": {"method": "POST", "headers": {}, "body": {}, "query": {}, "stringBody": ""}}+ if globals and globals.get("request"):+ data["request"].update(globals["request"])+ return data++ def _request(+ self, body: dict, key: bytes = KEY_BYTES, timestamp: int | None = None, extra_signature: str = ""+ ) -> dict:+ payload = json.dumps(body)+ message_id = "msg_test"+ timestamp_string = str(timestamp if timestamp is not None else int(time.time()))+ digest = hmac.new(key, f"{message_id}.{timestamp_string}.{payload}".encode(), hashlib.sha256).digest()+ signature = f"{extra_signature}v1,{base64.b64encode(digest).decode()}"+ return {+ "request": {+ "method": "POST",+ "headers": {"svix-id": message_id, "svix-timestamp": timestamp_string, "svix-signature": signature},+ "body": body,+ "query": {},+ "stringBody": payload,+ }+ }++ def _run(self, globals: dict, schema_mapping: dict | None = None):+ return self.run_function(+ {"signing_secret": SECRET, "bypass_signature_check": False, "schema_mapping": schema_mapping or {}},+ globals=globals,+ )++ @parameterized.expand(sorted(WEBHOOK_EVENT_TO_SCHEMA.items()))+ def test_routes_supported_events(self, event_type: str, resource: str) -> None:+ body = {"type": event_type, "object": "event", "data": {"id": "resource_1"}}+ self._run(self._request(body), {resource: "schema_1"})+ self.mock_produce_to_warehouse_webhooks.assert_called_once_with(body, "schema_1")++ def test_accepts_any_matching_signature_in_the_header(self) -> None:+ # Svix lists one signature per active secret while a secret rotates.+ body = {"type": "user.created", "data": {"id": "user_1"}}+ self._run(self._request(body, extra_signature="v1,c3RhbGU= "), {"users": "schema_1"})+ self.mock_produce_to_warehouse_webhooks.assert_called_once_with(body, "schema_1")++ @parameterized.expand(+ [+ ("delete_event", "user.deleted", {"users": "schema_1"}, "Unhandled event type"),+ ("unhandled_event", "email.created", {"users": "schema_1"}, "Unhandled event type"),+ ("unmapped_resource", "user.created", {}, "No schema mapping"),+ ]+ )+ def test_acknowledges_events_it_does_not_store(+ self, _label: str, event_type: str, schema_mapping: dict, expected_body: str+ ) -> None:+ # A non-200 makes Svix retry, then disable the endpoint, for events PostHog never stores.+ result = self._run(self._request({"type": event_type, "data": {"id": "user_1"}}), schema_mapping)++ assert result.result["httpResponse"]["status"] == 200+ assert expected_body in result.result["httpResponse"]["body"]+ self.mock_produce_to_warehouse_webhooks.assert_not_called()++ @parameterized.expand(+ [+ ("wrong_key", {"key": bytes(range(24))}, "Bad signature"),+ ("stale", {"timestamp": int(time.time()) - 301}, "Timestamp outside tolerance"),+ ]+ )+ def test_rejects_unverified_delivery(self, _label: str, request_kwargs: dict, expected_body: str) -> None:+ body = {"type": "user.created", "data": {"id": "user_1"}}+ result = self._run(self._request(body, **request_kwargs), {"users": "schema_1"})++ assert result.result == {"httpResponse": {"status": 400, "body": expected_body}}+ self.mock_produce_to_warehouse_webhooks.assert_not_called()++ def test_rejects_delivery_without_svix_headers(self) -> None:+ body = {"type": "user.created", "data": {"id": "user_1"}}+ result = self._run(+ {"request": {"method": "POST", "headers": {}, "body": body, "stringBody": json.dumps(body)}},+ {"users": "schema_1"},+ )++ assert result.result == {"httpResponse": {"status": 400, "body": "Missing signature"}}+ self.mock_produce_to_warehouse_webhooks.assert_not_called()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.pynew file mode 100644index 000000000000..931ab307e90b--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py@@ -0,0 +1,112 @@+from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC++# Clerk sends webhooks through Svix. Svix signs "{svix-id}.{svix-timestamp}.{raw body}" with+# HMAC-SHA256. The key is the base64 part of the whsec_ signing secret, decoded to bytes. The+# svix-signature header holds one or more space-separated "v1,<base64 signature>" entries.+template: HogFunctionTemplateDC = HogFunctionTemplateDC(+ status="alpha",+ free=False,+ type="warehouse_source_webhook",+ id="template-warehouse-source-clerk",+ name="Clerk warehouse source webhook",+ description="Receive Clerk webhook events for data warehouse ingestion",+ icon_url="/static/services/clerk.png",+ category=["Data warehouse"],+ code_language="hog",+ code="""\+if (request.method != 'POST') {+ return {'httpResponse': {'status': 405, 'body': 'Method not allowed'}}+}++if (not inputs.bypass_signature_check) {+ if (empty(inputs.signing_secret)) {+ return {+ 'httpResponse': {'status': 200, 'body': 'Signing secret not configured, delivery dropped'},+ 'appMetric': 'missing_credential'+ }+ }++ let messageId := request.headers['svix-id']+ let timestamp := request.headers['svix-timestamp']+ let signatureHeader := request.headers['svix-signature']+ if (empty(messageId) or empty(timestamp) or empty(signatureHeader)) {+ return {'httpResponse': {'status': 400, 'body': 'Missing signature'}}+ }++ // The standard base64 alphabet has no underscore, so 'whsec_' can only match the prefix.+ let key := replaceOne(trim(inputs.signing_secret), 'whsec_', '')+ let expected := sha256HmacBase64Key(key, concat(messageId, '.', timestamp, '.', request.stringBody), 'base64')+ let matched := false+ for (let _, entry in splitByString(' ', signatureHeader)) {+ let pair := splitByString(',', trim(entry), 2)+ if (length(pair) = 2 and pair[1] = 'v1' and pair[2] = expected) {+ matched := true+ }+ }+ if (not matched) {+ return {'httpResponse': {'status': 400, 'body': 'Bad signature'}}+ }++ let age := toInt(toUnixTimestamp(now())) - toInt(timestamp)+ if (age > 300 or age < -300) {+ return {'httpResponse': {'status': 400, 'body': 'Timestamp outside tolerance'}}+ }+}++let eventType := request.body.type+// Several event types feed one warehouse table, so the event is collapsed to a resource key+// before the schema lookup. Keep in sync with WEBHOOK_EVENTS_BY_SCHEMA in settings.py.+let resourceByEvent := {+ 'user.created': 'users', 'user.updated': 'users',+ 'organization.created': 'organizations', 'organization.updated': 'organizations',+ 'organizationMembership.created': 'organization_memberships',+ 'organizationMembership.updated': 'organization_memberships',+}+let resource := resourceByEvent?.[eventType]+if (empty(resource)) {+ return {'httpResponse': {'status': 200, 'body': f'Unhandled event type: {eventType}, skipping'}}+}+let schemaId := inputs.schema_mapping?.[resource]+if (empty(schemaId)) {+ return {'httpResponse': {'status': 200, 'body': f'No schema mapping for event type: {eventType}, skipping'}}+}+produceToWarehouseWebhooks(request.body, schemaId)""",+ inputs_schema=[+ {+ "type": "string",+ "key": "signing_secret",+ "label": "Signing secret",+ "required": False,+ "secret": True,+ "hidden": False,+ "description": "Used to verify Clerk webhook deliveries",+ },+ {+ "type": "boolean",+ "key": "bypass_signature_check",+ "label": "Bypass signature check",+ "description": "If set, the svix-signature header will not be checked. This is not recommended.",+ "default": False,+ "required": False,+ "secret": False,+ },+ {+ "type": "json",+ "key": "schema_mapping",+ "label": "Schema mapping",+ "description": "Maps Clerk resources to ExternalDataSchema IDs",+ "required": True,+ "secret": False,+ "hidden": True,+ },+ {+ "type": "string",+ "key": "source_id",+ "label": "Source ID",+ "description": "The ExternalDataSource ID this webhook is associated with",+ "required": True,+ "secret": False,+ "hidden": True,+ },+ ],+)diff --git a/rust/common/hogvm/src/stl.rs b/rust/common/hogvm/src/stl.rsindex d134e60a8a49..412aaf425e69 100644--- a/rust/common/hogvm/src/stl.rs+++ b/rust/common/hogvm/src/stl.rs@@ -1204,6 +1204,24 @@ pub fn stl() -> Vec<(String, NativeFunction)> {Ok(HogLiteral::from(encode_digest(&digest, &encoding)?).into())}),),+ (+ "sha256HmacBase64Key",+ native_func(|vm, args| {+ if args.len() < 2 || args.len() > 3 {+ return Err(VmError::NativeCallFailed(+ "sha256HmacBase64Key takes 2 or 3 arguments".to_string(),+ ));+ }+ let key: &str = args[0].deref(&vm.heap)?.try_as()?;+ let data: &str = args[1].deref(&vm.heap)?.try_as()?;+ let mut mac = <HmacSha256 as Mac>::new_from_slice(&base64_decode_bytes(key)?)+ .map_err(|e| VmError::NativeCallFailed(e.to_string()))?;+ mac.update(data.as_bytes());+ let digest = mac.finalize().into_bytes().to_vec();+ let encoding = encoding_arg(vm, &args, 2)?;+ Ok(HogLiteral::from(encode_digest(&digest, &encoding)?).into())+ }),+ ),("base64Encode",native_func(|vm, args| {@@ -2723,17 +2741,20 @@ fn to_hex(bytes: &[u8]) -> String {}// Node's `Buffer.from(s, 'base64').toString()` — lenient about padding, UTF-8 decode lossily.-fn base64_decode_to_string(s: &str) -> Result<String, VmError> {+fn base64_decode_bytes(s: &str) -> Result<Vec<u8>, VmError> {let engine = base64::engine::GeneralPurpose::new(&base64::alphabet::STANDARD,base64::engine::GeneralPurposeConfig::new().with_decode_padding_mode(base64::engine::DecodePaddingMode::Indifferent).with_decode_allow_trailing_bits(true),);- let bytes = engine+ engine.decode(s.trim())- .map_err(|e| VmError::NativeCallFailed(format!("base64 decode failed: {e}")))?;- Ok(String::from_utf8_lossy(&bytes).into_owned())+ .map_err(|e| VmError::NativeCallFailed(format!("base64 decode failed: {e}")))+}++fn base64_decode_to_string(s: &str) -> Result<String, VmError> {+ Ok(String::from_utf8_lossy(&base64_decode_bytes(s)?).into_owned())}// JS `encodeURIComponent`: keep the unreserved set, percent-encode every other UTF-8 byte.diff --git a/rust/common/hogvm/tests/stl.rs b/rust/common/hogvm/tests/stl.rsindex 779771dd0ba2..648916c423c9 100644--- a/rust/common/hogvm/tests/stl.rs+++ b/rust/common/hogvm/tests/stl.rs@@ -194,6 +194,31 @@ fn hmac_sha1_chain_rekeys_and_encodes() {);}+#[test]+fn hmac_sha256_base64_key_matches_svix() {+ // Known answer from the Svix webhook verification docs. The key decodes to bytes that are not+ // valid UTF-8, so this fails if the key is read as a string.+ let key = || str_lit("MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw");+ let payload = || str_lit("msg_p5jXN8AQM9LWM0D4loKWxJek.1614265330.{\"test\": 2432232314}");+ assert_eq!(+ run(call(+ "sha256HmacBase64Key",+ &[key(), payload(), str_lit("base64")]+ )),+ json!("g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=")+ );+ assert_eq!(+ run(call(+ "sha256HmacBase64Key",+ &[str_lit("YWI"), str_lit("data")]+ )),+ run(call(+ "sha256HmacBase64Key",+ &[str_lit("YWI="), str_lit("data")]+ ))+ );+}+#[test]fn base64_round_trips() {let encoded = run(call("base64Encode", &[str_lit("Hello, World! 🦔")]));