Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add clerk webhook sync (#117917)

posthog[bot]55318df0da04
diff --git a/common/hogvm/CHANGELOG.md b/common/hogvm/CHANGELOG.md
index ec5eae6b9e99..94f7210f6a2d 100644
--- a/common/hogvm/CHANGELOG.md
+++ b/common/hogvm/CHANGELOG.md
@@ -1,5 +1,18 @@
# HogQL bytecode changelog
+## 2026-10-15 - 1.0.72
+
+Added `sha256HmacBase64Key` to the standard library. No bytecode operations changed.
+
+```bash
+sha256HmacBase64Key(key, data, encoding) # HMAC-SHA256 of data, key is base64, encoding defaults to 'hex'
+```
+
+The function decodes the key from base64 and uses the raw bytes as the HMAC key. Svix-style webhook
+secrets (`whsec_...`) are random bytes, so `sha256HmacChain` cannot verify them: it uses the UTF-8
+bytes of a string key. The key is accepted with or without base64 padding. `encoding` is one of
+`hex`, `base64`, `base64url`, or `binary`.
+
## 2026-09-28 - 1.0.71
`range` checks the requested length against the memory limit before it builds the array, as the
diff --git a/common/hogvm/__tests__/__snapshots__/crypto.hoge b/common/hogvm/__tests__/__snapshots__/crypto.hoge
index 32560867cf55..079e3bcd973b 100644
--- a/common/hogvm/__tests__/__snapshots__/crypto.hoge
+++ b/common/hogvm/__tests__/__snapshots__/crypto.hoge
@@ -33,4 +33,8 @@
32, "Hi There", 43, 2, 2, "sha1HmacChainHex", 1, 2, "print", 2, 35, 32, "rfc2202 case 2:", 32, "Jefe", 32,
"what do ya want for nothing?", 43, 2, 2, "sha1HmacChainHex", 1, 2, "print", 2, 35, 32, "rfc2202 case 5:", 32,
"\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f", 32, "Test With Truncation", 43, 2, 2, "sha1HmacChainHex", 1, 2, "print", 2,
-35, 35, 35]
+35, 32, "MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw", 32, "msg_p5jXN8AQM9LWM0D4loKWxJek.1614265330.{\"test\": 2432232314}", 32,
+"sha256HmacBase64Key(svix, 'base64'):", 36, 2, 36, 3, 32, "base64", 2, "sha256HmacBase64Key", 3, 2, "print", 2, 35, 32,
+"sha256HmacBase64Key(svix):", 36, 2, 36, 3, 2, "sha256HmacBase64Key", 2, 2, "print", 2, 35, 32,
+"sha256HmacBase64Key(unpadded):", 32, "YWI=", 32, "data", 2, "sha256HmacBase64Key", 2, 32, "YWI", 32, "data", 2,
+"sha256HmacBase64Key", 2, 11, 2, "print", 2, 35, 35, 35, 35, 35]
diff --git a/common/hogvm/__tests__/__snapshots__/crypto.stdout b/common/hogvm/__tests__/__snapshots__/crypto.stdout
index 195842a42d1b..2ace24d80d68 100644
Binary files a/common/hogvm/__tests__/__snapshots__/crypto.stdout and b/common/hogvm/__tests__/__snapshots__/crypto.stdout differ
diff --git a/common/hogvm/__tests__/crypto.hog b/common/hogvm/__tests__/crypto.hog
index be3324ea3909..d71fea9b4102 100644
--- a/common/hogvm/__tests__/crypto.hog
+++ b/common/hogvm/__tests__/crypto.hog
@@ -47,3 +47,11 @@ print('sha1Hex(\'\'):', sha1Hex(''))
print('rfc2202 case 1:', sha1HmacChainHex(['\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v\v', 'Hi There']))
print('rfc2202 case 2:', sha1HmacChainHex(['Jefe', 'what do ya want for nothing?']))
print('rfc2202 case 5:', sha1HmacChainHex(['\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f\f', 'Test With Truncation']))
+
+// Svix signature known answer, from the Svix webhook verification docs. The key decodes to bytes
+// that are not valid UTF-8, so a string key would give a different digest.
+let svixKey := 'MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw'
+let svixPayload := 'msg_p5jXN8AQM9LWM0D4loKWxJek.1614265330.{"test": 2432232314}'
+print('sha256HmacBase64Key(svix, \'base64\'):', sha256HmacBase64Key(svixKey, svixPayload, 'base64'))
+print('sha256HmacBase64Key(svix):', sha256HmacBase64Key(svixKey, svixPayload))
+print('sha256HmacBase64Key(unpadded):', sha256HmacBase64Key('YWI', 'data') = sha256HmacBase64Key('YWI=', 'data'))
diff --git a/common/hogvm/python/stl/__init__.py b/common/hogvm/python/stl/__init__.py
index dffde88be173..3fecc0fee728 100644
--- a/common/hogvm/python/stl/__init__.py
+++ b/common/hogvm/python/stl/__init__.py
@@ -22,7 +22,7 @@ from ..utils import (
like,
regex_extract,
)
-from .crypto import md5, sha1, sha1HmacChain, sha256, sha256HmacChain
+from .crypto import md5, sha1, sha1HmacChain, sha256, sha256HmacBase64Key, sha256HmacChain
from .date import (
formatDateTime,
fromUnixTimestamp,
@@ -1116,6 +1116,13 @@ STL: dict[str, STLFunction] = {
minArgs=1,
maxArgs=2,
),
+ "sha256HmacBase64Key": STLFunction(
+ fn=lambda args, team, stdout, timeout: sha256HmacBase64Key(
+ args[0], args[1], args[2] if len(args) > 2 and args[2] is not None else "hex"
+ ),
+ minArgs=2,
+ maxArgs=3,
+ ),
"isIPAddressInRange": STLFunction(
fn=lambda args, team, stdout, timeout: isIPAddressInRange(args[0], args[1]), minArgs=2, maxArgs=2
),
diff --git a/common/hogvm/python/stl/crypto.py b/common/hogvm/python/stl/crypto.py
index 2eb04d48d5b5..3dcb1a60cb10 100644
--- a/common/hogvm/python/stl/crypto.py
+++ b/common/hogvm/python/stl/crypto.py
@@ -61,3 +61,12 @@ def sha256HmacChain(data: list, encoding: Literal["hex", "base64", "base64url",
def sha1HmacChain(data: list, encoding: Literal["hex", "base64", "base64url", "binary"] = "hex") -> str:
return _hmac_chain(data, "sha1", encoding)
+
+
+def sha256HmacBase64Key(key: str, data: str, encoding: Literal["hex", "base64", "base64url", "binary"] = "hex") -> str:
+ # Svix-style webhook secrets are random bytes, which a string key would corrupt. The extra
+ # padding makes the decode accept a key with or without padding, as Node does.
+ key_bytes = base64.b64decode(key.strip() + "==")
+ digest = hmac.new(key_bytes, data.encode(), hashlib.sha256).digest()
+
+ return encode_digest(encoding, digest)
diff --git a/common/hogvm/typescript/package.json b/common/hogvm/typescript/package.json
index ff69bb6dc18a..3fcc3d3062a2 100644
--- a/common/hogvm/typescript/package.json
+++ b/common/hogvm/typescript/package.json
@@ -1,6 +1,6 @@
{
"name": "@posthog/hogvm",
- "version": "1.0.71",
+ "version": "1.0.72",
"description": "PostHog Hog Virtual Machine",
"license": "MIT",
"author": "PostHog <[email protected]>",
diff --git a/common/hogvm/typescript/src/stl/crypto.ts b/common/hogvm/typescript/src/stl/crypto.ts
index 150d0e11e48f..960ab35ce604 100644
--- a/common/hogvm/typescript/src/stl/crypto.ts
+++ b/common/hogvm/typescript/src/stl/crypto.ts
@@ -85,3 +85,18 @@ export function sha1HmacChain(
): string {
return hmacChain('sha1', 'sha1HmacChainHex', data, encoding, options)
}
+
+// HMAC-SHA256 with a base64 encoded binary key. Svix-style webhook secrets are random bytes, which a
+// string key would corrupt.
+export function sha256HmacBase64Key(
+ key: string,
+ data: string,
+ encoding: 'hex' | 'base64' | 'base64url' | 'binary' = 'hex',
+ options?: ExecOptions
+): string {
+ const crypto = options?.external?.crypto
+ if (!crypto) {
+ throw new HogVMException('The crypto module is required for "sha256HmacBase64Key" to work.', 'contract')
+ }
+ return crypto.createHmac('sha256', Buffer.from(key, 'base64')).update(data).digest(encoding)
+}
diff --git a/common/hogvm/typescript/src/stl/stl.ts b/common/hogvm/typescript/src/stl/stl.ts
index 9bb7066bfa46..86d78e497722 100644
--- a/common/hogvm/typescript/src/stl/stl.ts
+++ b/common/hogvm/typescript/src/stl/stl.ts
@@ -4,7 +4,7 @@ import { DEFAULT_MAX_MEMORY } from '../constants'
import { isHogAST, isHogCallable, isHogClosure, isHogDate, isHogDateTime, isHogError, newHogError } from '../objects'
import { AsyncSTLFunction, ExecOptions, HogDate, HogDateTime, HogInterval, STLFunction } from '../types'
import { COST_PER_UNIT, HogVMException, calculateCost, getNestedValue, like } from '../utils'
-import { md5, sha1, sha1HmacChain, sha256, sha256HmacChain } from './crypto'
+import { md5, sha1, sha1HmacChain, sha256, sha256HmacBase64Key, sha256HmacChain } from './crypto'
import {
formatDateTime,
fromUnixTimestamp,
@@ -1231,6 +1231,13 @@ export const STL: Record<string, STLFunction> = {
minArgs: 1,
maxArgs: 2,
},
+ sha256HmacBase64Key: {
+ fn: ([key, data, encoding], _, options) => sha256HmacBase64Key(key, data, encoding, options),
+ description: 'Computes HMAC-SHA256 of a string with a base64 encoded binary key',
+ example: 'sha256HmacBase64Key($1, $2, $3)',
+ minArgs: 2,
+ maxArgs: 3,
+ },
isIPAddressInRange: {
fn: ([address, prefix]) => isIPAddressInRange(address, prefix),
description: 'Checks if IP address is in CIDR range',
diff --git a/posthog/hogql/compiler/javascript_stl.py b/posthog/hogql/compiler/javascript_stl.py
index 6017b5c22ebe..d138c0031554 100644
--- a/posthog/hogql/compiler/javascript_stl.py
+++ b/posthog/hogql/compiler/javascript_stl.py
@@ -375,6 +375,10 @@ STL_FUNCTIONS: dict[str, list[str | list[str]]] = {
"function sha1HmacChain (data, encoding, options) { return 'sha1HmacChain not implemented' }",
[],
],
+ "sha256HmacBase64Key": [
+ "function sha256HmacBase64Key (key, data, encoding, options) { return 'sha256HmacBase64Key not implemented' }",
+ [],
+ ],
"isIPAddressInRange": [
"""
function isIPAddressInRange(address, prefix) {
diff --git a/products/cdp/backend/api/test/__snapshots__/test_hooks.ambr b/products/cdp/backend/api/test/__snapshots__/test_hooks.ambr
index 8af8520ff329..b3d78ac6a923 100644
--- a/products/cdp/backend/api/test/__snapshots__/test_hooks.ambr
+++ b/products/cdp/backend/api/test/__snapshots__/test_hooks.ambr
@@ -148,7 +148,7 @@
]),
}),
}),
- 'bytecode_contract': 'bfd6ece60e41383c',
+ 'bytecode_contract': '2c9c13700371fbf9',
'order': 1,
'value': dict({
'data': dict({
@@ -178,7 +178,7 @@
32,
'hooks/standard/1234/abcd',
]),
- 'bytecode_contract': 'bfd6ece60e41383c',
+ 'bytecode_contract': '2c9c13700371fbf9',
'order': 0,
'value': 'hooks/standard/1234/abcd',
}),
diff --git a/products/cdp/backend/api/test/test_hog_function.py b/products/cdp/backend/api/test/test_hog_function.py
index 8077939ecce4..a9713f91b720 100644
--- a/products/cdp/backend/api/test/test_hog_function.py
+++ b/products/cdp/backend/api/test/test_hog_function.py
@@ -1179,7 +1179,7 @@ class TestHogFunctionAPI(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
"value": "I AM SECRET",
"order": 0,
},
- "gAAAAABlkgC8AAAAAAAAAAAAAAAAAAAAAKvzDjuLG689YjjVhmmbXAtZSRoucXuT8VtokVrCotIx3ttPcVufoVt76dyr2phbuotMldKMVv_Y6uzMDZFjX1VQVJqL13wH-WALMn9obfpLYD_WWOUdMA6VurFg1TxdopwQKcL10Y5Yg8s8Gswibi1pCMfjwSnKwod91SMtLKgNfAU4EPZ6GxA77xCHIjaTLueR3qx-hy2Pu3W0r5Rh1hWy0bq01uIdulQ_LhxkQgpj",
+ "gAAAAABlkgC8AAAAAAAAAAAAAAAAAAAAAKvzDjuLG689YjjVhmmbXAtZSRoucXuT8VtokVrCotIx3ttPcVufoVt76dyr2phbuotMldKMVv_Y6uzMDZFjX1VQVJqL13wH-WALMn9obfpLYD_WWOUdMA6VurFg1TxdopDxFtIztV_BWe5ApI2TtI57CYnHc8Meh0Qo-Ii9TRkDbxtU0E48PqNx6eqhGROrzCtN2rORP2bj-BM983QSq6ufd6CpPUqUUvSHhvhP0klo",
{"secret": True},
[],
),
diff --git a/products/cdp/backend/filter_globals.json b/products/cdp/backend/filter_globals.json
index a3b949873b27..818f8743782b 100644
--- a/products/cdp/backend/filter_globals.json
+++ b/products/cdp/backend/filter_globals.json
@@ -1,6 +1,6 @@
{
"$comment": "Generated. Do not edit: run `pnpm --filter=@posthog/nodejs run build:filter-globals`. roots are the data globals the CDP filter runtime builds for a hog function. callables are the standard-library names the VM hands back as values and can then invoke, so they are the ones a filter can pass as a callback. functions are every standard-library name a filter can call directly, with the argument count the VM enforces as [min, max]. template_roots are the names an input template can read. Django reads this to refuse a filter or an input the runtime could not evaluate. contract is a hash of the rest, stamped on compiled bytecode.",
- "contract": "bfd6ece60e41383c",
+ "contract": "2c9c13700371fbf9",
"roots": [
"$group_0",
"$group_1",
@@ -117,6 +117,7 @@
"sha1HmacChainHex",
"sha256",
"sha256Hex",
+ "sha256HmacBase64Key",
"sha256HmacChain",
"sha256HmacChainHex",
"splitByString",
@@ -514,6 +515,10 @@
1,
1
],
+ "sha256HmacBase64Key": [
+ 2,
+ 3
+ ],
"sha256HmacChain": [
1,
2
diff --git a/products/error_tracking/backend/hogvm_stl.py b/products/error_tracking/backend/hogvm_stl.py
index e4b82c2d403b..aca51820ca17 100644
--- a/products/error_tracking/backend/hogvm_stl.py
+++ b/products/error_tracking/backend/hogvm_stl.py
@@ -96,6 +96,7 @@ RUST_HOGVM_STL = [
"sha256HmacChain",
"sha1HmacChainHex",
"sha1HmacChain",
+ "sha256HmacBase64Key",
"base64Encode",
"base64Decode",
"tryBase64Decode",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
index dfa326797d1b..06103a9f887e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
@@ -246,7 +246,7 @@ the row lists both.
| cloudflare_r2 | HTTP (vendor SDK) | boto3 (listing), s3fs (reads), deltalake (Delta tables) | ⚠️ |
| clari | HTTP | requests | ✅ |
| clay | HTTP | requests + `rest_source.RESTClient` | ✅ |
-| clerk | HTTP | requests + `rest_source.RESTClient` | ✅ |
+| clerk | HTTP + Webhook | requests + `rest_source.RESTClient` + `WebhookSourceManager` | ✅ (pull) / ➖ (webhook) |
| clevertap | HTTP | requests + `rest_source.RESTClient` | ✅ |
| clickhouse | DB protocol (HTTP-based) | clickhouse-connect / clickhouse-driver | ➖ |
| clickhouse_cloud | HTTP | requests | ✅ |
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
index 782ecb7c2509..55af97783645 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
@@ -1,17 +1,22 @@
import functools
import dataclasses
-from collections.abc import Iterator
+from collections.abc import AsyncIterable, Iterable, Iterator
from typing import Any, Optional
+import orjson
+import pyarrow as pa
+from asgiref.sync import async_to_sync
from requests import Request, Response
from requests.exceptions import HTTPError, RequestException
from structlog.types import FilteringBoundLogger
from posthog.exceptions_capture import capture_exception
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (
CLERK_ENDPOINTS,
RETIRED_ENDPOINTS,
+ WEBHOOK_SCHEMA_NAMES,
ClerkEndpointConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -26,6 +31,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
@dataclasses.dataclass
@@ -339,6 +345,33 @@ def _skip_when_feature_disabled(
)
+def _webhook_table_transformer(table: pa.Table) -> pa.Table:
+ """Extract the Clerk object from each webhook event and keep the latest event per id."""
+ rows_by_id: dict[str, tuple[int, dict[str, Any]]] = {}
+ for event in table.to_pylist():
+ event_type = event.get("type")
+ resource = event.get("data")
+ if isinstance(resource, str):
+ try:
+ resource = orjson.loads(resource)
+ except orjson.JSONDecodeError:
+ continue
+ if not isinstance(event_type, str) or not isinstance(resource, dict) or not resource.get("id"):
+ continue
+ # The Hog template already drops deletions. This keeps a hand-configured webhook from
+ # writing an id-only payload over a stored row. See WEBHOOK_EVENTS_BY_SCHEMA.
+ if event_type.endswith(".deleted") or resource.get("deleted") is True:
+ continue
+
+ raw_timestamp = event.get("timestamp")
+ timestamp = raw_timestamp if isinstance(raw_timestamp, int) and not isinstance(raw_timestamp, bool) else 0
+ current = rows_by_id.get(str(resource["id"]))
+ if current is None or timestamp >= current[0]:
+ rows_by_id[str(resource["id"])] = (timestamp, _convert_timestamps(resource))
+
+ return table_from_py_list([row for _, row in rows_by_id.values()])
+
+
def clerk_source(
secret_key: str,
endpoint: str,
@@ -346,6 +379,7 @@ def clerk_source(
job_id: str,
resumable_source_manager: ResumableSourceManager[ClerkResumeConfig],
logger: FilteringBoundLogger,
+ webhook_source_manager: WebhookSourceManager,
) -> SourceResponse:
retired_reason = RETIRED_ENDPOINTS.get(endpoint)
if retired_reason is not None:
@@ -407,7 +441,14 @@ def clerk_source(
gated_feature = endpoint_config.gated_feature
- def items() -> Iterator[Any]:
+ webhook_enabled = endpoint in WEBHOOK_SCHEMA_NAMES and async_to_sync(webhook_source_manager.webhook_enabled)()
+
+ def items() -> Iterable[Any] | AsyncIterable[Any]:
+ if webhook_enabled:
+ return webhook_source_manager.get_items(table_transformer=_webhook_table_transformer)
+ return polled_items()
+
+ def polled_items() -> Iterator[Any]:
if gated_feature is None:
yield from resource
return
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py
index c3be3a1814cc..9a27309829cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py
@@ -192,3 +192,17 @@ RETIRED_ENDPOINTS: dict[str, str] = {
}
ENDPOINTS = tuple(CLERK_ENDPOINTS.keys())
+
+# Clerk delivers webhooks through Svix. A table qualifies for webhook sync only when its events
+# carry the full object on create and update. Delete events are not stored: `user.deleted` and
+# `organization.deleted` carry only the object id, and the merge also matches on the `created_at`
+# partition key, so a tombstone row could not replace the stored row. A deleted record stays in the
+# table until the next full resync.
+WEBHOOK_EVENTS_BY_SCHEMA: dict[str, tuple[str, ...]] = {
+ "users": ("user.created", "user.updated"),
+ "organizations": ("organization.created", "organization.updated"),
+ "organization_memberships": ("organizationMembership.created", "organizationMembership.updated"),
+}
+
+WEBHOOK_EVENT_TO_SCHEMA = {event: schema for schema, events in WEBHOOK_EVENTS_BY_SCHEMA.items() for event in events}
+WEBHOOK_SCHEMA_NAMES = frozenset(WEBHOOK_EVENTS_BY_SCHEMA)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py
index 16c2e390014c..3c9485ffa9d3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py
@@ -1,4 +1,7 @@
-from typing import Optional, cast
+from typing import TYPE_CHECKING, Optional, cast
+
+if TYPE_CHECKING:
+ from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -15,8 +18,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.cler
clerk_source,
validate_credentials as validate_clerk_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import ENDPOINTS, RETIRED_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (
+ ENDPOINTS,
+ RETIRED_ENDPOINTS,
+ WEBHOOK_SCHEMA_NAMES,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+ FieldType,
+ ResumableSource,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
CanonicalDescriptions,
)
@@ -24,12 +37,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.reg
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.clerk import ClerkSourceConfig
from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
-class ClerkSource(ResumableSource[ClerkSourceConfig, ClerkResumeConfig]):
+class ClerkSource(
+ ResumableSource[ClerkSourceConfig, ClerkResumeConfig],
+ WebhookSource[ClerkSourceConfig],
+):
codebase_detection = detect(
env=(
"^CLERK_SECRET_KEY$",
@@ -150,6 +167,16 @@ class ClerkSource(ResumableSource[ClerkSourceConfig, ClerkResumeConfig]):
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.CLERK
+ @property
+ def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.webhook_template import template
+
+ return template
+
+ @property
+ def webhook_resource_map(self) -> dict[str, str]:
+ return {name: name for name in WEBHOOK_SCHEMA_NAMES}
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
@@ -177,6 +204,28 @@ The secret key starts with `sk_live_`.
),
],
),
+ webhookSetupCaption="""Webhook sync keeps the users, organizations, and organization memberships tables up to date. Set it up in Clerk:
+
+1. In the [Clerk Dashboard](https://dashboard.clerk.com/), go to **Configure** > **Webhooks** and select **Add Endpoint**.
+2. Paste the webhook URL shown below into **Endpoint URL**.
+3. Subscribe to the `user`, `organization`, and `organizationMembership` created and updated events.
+4. Open the endpoint, copy its **Signing Secret**, and paste it into the field below.
+
+Clerk delete events only carry the record ID, so webhook sync doesn't remove deleted records. Resync a table to remove them.
+""",
+ webhookFields=cast(
+ list[FieldType],
+ [
+ SourceFieldInputConfig(
+ name="signing_secret",
+ label="Signing secret",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=True,
+ placeholder="whsec_...",
+ secret=True,
+ )
+ ],
+ ),
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
@@ -195,12 +244,14 @@ The secret key starts with `sk_live_`.
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Clerk only supports full refresh - the API doesn't support filtering by updated_at
+ # The Clerk API can't filter by updated_at, so a polled sync is always a full refresh. The
+ # tables in WEBHOOK_SCHEMA_NAMES can switch to webhook sync after the initial backfill.
schemas = [
SourceSchema(
name=endpoint,
supports_incremental=False,
supports_append=False,
+ supports_webhooks=endpoint in WEBHOOK_SCHEMA_NAMES,
incremental_fields=[],
)
for endpoint in list(ENDPOINTS)
@@ -273,6 +324,23 @@ The secret key starts with `sk_live_`.
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[ClerkResumeConfig]:
return ResumableSourceManager[ClerkResumeConfig](inputs, ClerkResumeConfig)
+ def get_webhook_source_manager(self, inputs: SourceInputs) -> WebhookSourceManager:
+ return WebhookSourceManager(inputs, inputs.logger)
+
+ def create_webhook(
+ self, config: ClerkSourceConfig, webhook_url: str, team_id: int, api_version: str | None = None
+ ) -> WebhookCreationResult:
+ return WebhookCreationResult(
+ success=False,
+ error="Clerk does not support automatic webhook creation. Please follow the manual setup instructions.",
+ )
+
+ def delete_webhook(
+ self, config: ClerkSourceConfig, webhook_url: str, team_id: int, api_version: str | None = None
+ ) -> WebhookDeletionResult:
+ # PostHog never creates the Clerk endpoint, so there is nothing to remove on Clerk's side.
+ return WebhookDeletionResult(success=True)
+
def source_for_pipeline(
self,
config: ClerkSourceConfig,
@@ -286,4 +354,5 @@ The secret key starts with `sk_live_`.
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
logger=inputs.logger,
+ webhook_source_manager=self.get_webhook_source_manager(inputs),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
index c5d7200155b5..302f1e148fe2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
@@ -5,6 +5,7 @@ from typing import Any
import pytest
from unittest.mock import MagicMock
+import pyarrow as pa
from requests import Response
from requests.exceptions import HTTPError, RequestException
from structlog.testing import capture_logs
@@ -14,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.cler
ClerkResumeConfig,
_convert_timestamps,
_strip_sensitive_fields,
+ _webhook_table_transformer,
validate_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (
@@ -413,3 +415,26 @@ class TestClerkValidateCredentials:
assert "Couldn't reach Clerk" in (message or "")
assert "connection reset by peer" not in (message or "")
assert network.requests_log[0].path == "/v1/users"
+
+
+class TestClerkWebhookTransformer:
+ def test_keeps_latest_event_and_skips_deletes(self) -> None:
+ table = pa.Table.from_pylist(
+ [
+ {
+ "type": "user.updated",
+ "timestamp": 1767312000000,
+ "data": '{"id":"user_1","email":"[email protected]","created_at":1767225600000}',
+ },
+ {
+ "type": "user.created",
+ "timestamp": 1767225600000,
+ "data": '{"id":"user_1","email":"[email protected]","created_at":1767225600000}',
+ },
+ {"type": "user.deleted", "timestamp": 1767398400000, "data": '{"id":"user_1","deleted":true}'},
+ ]
+ )
+
+ assert _webhook_table_transformer(table).to_pylist() == [
+ {"id": "user_1", "email": "[email protected]", "created_at": 1767225600}
+ ]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py
new file mode 100644
index 000000000000..6bf51cbc06ff
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py
@@ -0,0 +1,102 @@
+import hmac
+import json
+import time
+import base64
+import hashlib
+
+from parameterized import parameterized
+
+from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
+from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import WEBHOOK_EVENT_TO_SCHEMA
+from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.webhook_template import template
+
+# Svix secrets are random bytes. These are not valid UTF-8, so a template that uses the secret as a
+# string key fails to verify.
+KEY_BYTES = bytes(range(0xE0, 0xF8))
+SECRET = "whsec_" + base64.b64encode(KEY_BYTES).decode()
+
+
+class TestClerkWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+ template = template
+
+ def createHogGlobals(self, globals=None) -> dict:
+ data = {"request": {"method": "POST", "headers": {}, "body": {}, "query": {}, "stringBody": ""}}
+ if globals and globals.get("request"):
+ data["request"].update(globals["request"])
+ return data
+
+ def _request(
+ self, body: dict, key: bytes = KEY_BYTES, timestamp: int | None = None, extra_signature: str = ""
+ ) -> dict:
+ payload = json.dumps(body)
+ message_id = "msg_test"
+ timestamp_string = str(timestamp if timestamp is not None else int(time.time()))
+ digest = hmac.new(key, f"{message_id}.{timestamp_string}.{payload}".encode(), hashlib.sha256).digest()
+ signature = f"{extra_signature}v1,{base64.b64encode(digest).decode()}"
+ return {
+ "request": {
+ "method": "POST",
+ "headers": {"svix-id": message_id, "svix-timestamp": timestamp_string, "svix-signature": signature},
+ "body": body,
+ "query": {},
+ "stringBody": payload,
+ }
+ }
+
+ def _run(self, globals: dict, schema_mapping: dict | None = None):
+ return self.run_function(
+ {"signing_secret": SECRET, "bypass_signature_check": False, "schema_mapping": schema_mapping or {}},
+ globals=globals,
+ )
+
+ @parameterized.expand(sorted(WEBHOOK_EVENT_TO_SCHEMA.items()))
+ def test_routes_supported_events(self, event_type: str, resource: str) -> None:
+ body = {"type": event_type, "object": "event", "data": {"id": "resource_1"}}
+ self._run(self._request(body), {resource: "schema_1"})
+ self.mock_produce_to_warehouse_webhooks.assert_called_once_with(body, "schema_1")
+
+ def test_accepts_any_matching_signature_in_the_header(self) -> None:
+ # Svix lists one signature per active secret while a secret rotates.
+ body = {"type": "user.created", "data": {"id": "user_1"}}
+ self._run(self._request(body, extra_signature="v1,c3RhbGU= "), {"users": "schema_1"})
+ self.mock_produce_to_warehouse_webhooks.assert_called_once_with(body, "schema_1")
+
+ @parameterized.expand(
+ [
+ ("delete_event", "user.deleted", {"users": "schema_1"}, "Unhandled event type"),
+ ("unhandled_event", "email.created", {"users": "schema_1"}, "Unhandled event type"),
+ ("unmapped_resource", "user.created", {}, "No schema mapping"),
+ ]
+ )
+ def test_acknowledges_events_it_does_not_store(
+ self, _label: str, event_type: str, schema_mapping: dict, expected_body: str
+ ) -> None:
+ # A non-200 makes Svix retry, then disable the endpoint, for events PostHog never stores.
+ result = self._run(self._request({"type": event_type, "data": {"id": "user_1"}}), schema_mapping)
+
+ assert result.result["httpResponse"]["status"] == 200
+ assert expected_body in result.result["httpResponse"]["body"]
+ self.mock_produce_to_warehouse_webhooks.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("wrong_key", {"key": bytes(range(24))}, "Bad signature"),
+ ("stale", {"timestamp": int(time.time()) - 301}, "Timestamp outside tolerance"),
+ ]
+ )
+ def test_rejects_unverified_delivery(self, _label: str, request_kwargs: dict, expected_body: str) -> None:
+ body = {"type": "user.created", "data": {"id": "user_1"}}
+ result = self._run(self._request(body, **request_kwargs), {"users": "schema_1"})
+
+ assert result.result == {"httpResponse": {"status": 400, "body": expected_body}}
+ self.mock_produce_to_warehouse_webhooks.assert_not_called()
+
+ def test_rejects_delivery_without_svix_headers(self) -> None:
+ body = {"type": "user.created", "data": {"id": "user_1"}}
+ result = self._run(
+ {"request": {"method": "POST", "headers": {}, "body": body, "stringBody": json.dumps(body)}},
+ {"users": "schema_1"},
+ )
+
+ assert result.result == {"httpResponse": {"status": 400, "body": "Missing signature"}}
+ self.mock_produce_to_warehouse_webhooks.assert_not_called()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py
new file mode 100644
index 000000000000..931ab307e90b
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py
@@ -0,0 +1,112 @@
+from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+
+# Clerk sends webhooks through Svix. Svix signs "{svix-id}.{svix-timestamp}.{raw body}" with
+# HMAC-SHA256. The key is the base64 part of the whsec_ signing secret, decoded to bytes. The
+# svix-signature header holds one or more space-separated "v1,<base64 signature>" entries.
+template: HogFunctionTemplateDC = HogFunctionTemplateDC(
+ status="alpha",
+ free=False,
+ type="warehouse_source_webhook",
+ id="template-warehouse-source-clerk",
+ name="Clerk warehouse source webhook",
+ description="Receive Clerk webhook events for data warehouse ingestion",
+ icon_url="/static/services/clerk.png",
+ category=["Data warehouse"],
+ code_language="hog",
+ code="""\
+if (request.method != 'POST') {
+ return {'httpResponse': {'status': 405, 'body': 'Method not allowed'}}
+}
+
+if (not inputs.bypass_signature_check) {
+ if (empty(inputs.signing_secret)) {
+ return {
+ 'httpResponse': {'status': 200, 'body': 'Signing secret not configured, delivery dropped'},
+ 'appMetric': 'missing_credential'
+ }
+ }
+
+ let messageId := request.headers['svix-id']
+ let timestamp := request.headers['svix-timestamp']
+ let signatureHeader := request.headers['svix-signature']
+ if (empty(messageId) or empty(timestamp) or empty(signatureHeader)) {
+ return {'httpResponse': {'status': 400, 'body': 'Missing signature'}}
+ }
+
+ // The standard base64 alphabet has no underscore, so 'whsec_' can only match the prefix.
+ let key := replaceOne(trim(inputs.signing_secret), 'whsec_', '')
+ let expected := sha256HmacBase64Key(key, concat(messageId, '.', timestamp, '.', request.stringBody), 'base64')
+ let matched := false
+ for (let _, entry in splitByString(' ', signatureHeader)) {
+ let pair := splitByString(',', trim(entry), 2)
+ if (length(pair) = 2 and pair[1] = 'v1' and pair[2] = expected) {
+ matched := true
+ }
+ }
+ if (not matched) {
+ return {'httpResponse': {'status': 400, 'body': 'Bad signature'}}
+ }
+
+ let age := toInt(toUnixTimestamp(now())) - toInt(timestamp)
+ if (age > 300 or age < -300) {
+ return {'httpResponse': {'status': 400, 'body': 'Timestamp outside tolerance'}}
+ }
+}
+
+let eventType := request.body.type
+// Several event types feed one warehouse table, so the event is collapsed to a resource key
+// before the schema lookup. Keep in sync with WEBHOOK_EVENTS_BY_SCHEMA in settings.py.
+let resourceByEvent := {
+ 'user.created': 'users', 'user.updated': 'users',
+ 'organization.created': 'organizations', 'organization.updated': 'organizations',
+ 'organizationMembership.created': 'organization_memberships',
+ 'organizationMembership.updated': 'organization_memberships',
+}
+let resource := resourceByEvent?.[eventType]
+if (empty(resource)) {
+ return {'httpResponse': {'status': 200, 'body': f'Unhandled event type: {eventType}, skipping'}}
+}
+let schemaId := inputs.schema_mapping?.[resource]
+if (empty(schemaId)) {
+ return {'httpResponse': {'status': 200, 'body': f'No schema mapping for event type: {eventType}, skipping'}}
+}
+produceToWarehouseWebhooks(request.body, schemaId)""",
+ inputs_schema=[
+ {
+ "type": "string",
+ "key": "signing_secret",
+ "label": "Signing secret",
+ "required": False,
+ "secret": True,
+ "hidden": False,
+ "description": "Used to verify Clerk webhook deliveries",
+ },
+ {
+ "type": "boolean",
+ "key": "bypass_signature_check",
+ "label": "Bypass signature check",
+ "description": "If set, the svix-signature header will not be checked. This is not recommended.",
+ "default": False,
+ "required": False,
+ "secret": False,
+ },
+ {
+ "type": "json",
+ "key": "schema_mapping",
+ "label": "Schema mapping",
+ "description": "Maps Clerk resources to ExternalDataSchema IDs",
+ "required": True,
+ "secret": False,
+ "hidden": True,
+ },
+ {
+ "type": "string",
+ "key": "source_id",
+ "label": "Source ID",
+ "description": "The ExternalDataSource ID this webhook is associated with",
+ "required": True,
+ "secret": False,
+ "hidden": True,
+ },
+ ],
+)
diff --git a/rust/common/hogvm/src/stl.rs b/rust/common/hogvm/src/stl.rs
index d134e60a8a49..412aaf425e69 100644
--- a/rust/common/hogvm/src/stl.rs
+++ b/rust/common/hogvm/src/stl.rs
@@ -1204,6 +1204,24 @@ pub fn stl() -> Vec<(String, NativeFunction)> {
Ok(HogLiteral::from(encode_digest(&digest, &encoding)?).into())
}),
),
+ (
+ "sha256HmacBase64Key",
+ native_func(|vm, args| {
+ if args.len() < 2 || args.len() > 3 {
+ return Err(VmError::NativeCallFailed(
+ "sha256HmacBase64Key takes 2 or 3 arguments".to_string(),
+ ));
+ }
+ let key: &str = args[0].deref(&vm.heap)?.try_as()?;
+ let data: &str = args[1].deref(&vm.heap)?.try_as()?;
+ let mut mac = <HmacSha256 as Mac>::new_from_slice(&base64_decode_bytes(key)?)
+ .map_err(|e| VmError::NativeCallFailed(e.to_string()))?;
+ mac.update(data.as_bytes());
+ let digest = mac.finalize().into_bytes().to_vec();
+ let encoding = encoding_arg(vm, &args, 2)?;
+ Ok(HogLiteral::from(encode_digest(&digest, &encoding)?).into())
+ }),
+ ),
(
"base64Encode",
native_func(|vm, args| {
@@ -2723,17 +2741,20 @@ fn to_hex(bytes: &[u8]) -> String {
}
// Node's `Buffer.from(s, 'base64').toString()` — lenient about padding, UTF-8 decode lossily.
-fn base64_decode_to_string(s: &str) -> Result<String, VmError> {
+fn base64_decode_bytes(s: &str) -> Result<Vec<u8>, VmError> {
let engine = base64::engine::GeneralPurpose::new(
&base64::alphabet::STANDARD,
base64::engine::GeneralPurposeConfig::new()
.with_decode_padding_mode(base64::engine::DecodePaddingMode::Indifferent)
.with_decode_allow_trailing_bits(true),
);
- let bytes = engine
+ engine
.decode(s.trim())
- .map_err(|e| VmError::NativeCallFailed(format!("base64 decode failed: {e}")))?;
- Ok(String::from_utf8_lossy(&bytes).into_owned())
+ .map_err(|e| VmError::NativeCallFailed(format!("base64 decode failed: {e}")))
+}
+
+fn base64_decode_to_string(s: &str) -> Result<String, VmError> {
+ Ok(String::from_utf8_lossy(&base64_decode_bytes(s)?).into_owned())
}
// JS `encodeURIComponent`: keep the unreserved set, percent-encode every other UTF-8 byte.
diff --git a/rust/common/hogvm/tests/stl.rs b/rust/common/hogvm/tests/stl.rs
index 779771dd0ba2..648916c423c9 100644
--- a/rust/common/hogvm/tests/stl.rs
+++ b/rust/common/hogvm/tests/stl.rs
@@ -194,6 +194,31 @@ fn hmac_sha1_chain_rekeys_and_encodes() {
);
}
+#[test]
+fn hmac_sha256_base64_key_matches_svix() {
+ // Known answer from the Svix webhook verification docs. The key decodes to bytes that are not
+ // valid UTF-8, so this fails if the key is read as a string.
+ let key = || str_lit("MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw");
+ let payload = || str_lit("msg_p5jXN8AQM9LWM0D4loKWxJek.1614265330.{\"test\": 2432232314}");
+ assert_eq!(
+ run(call(
+ "sha256HmacBase64Key",
+ &[key(), payload(), str_lit("base64")]
+ )),
+ json!("g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=")
+ );
+ assert_eq!(
+ run(call(
+ "sha256HmacBase64Key",
+ &[str_lit("YWI"), str_lit("data")]
+ )),
+ run(call(
+ "sha256HmacBase64Key",
+ &[str_lit("YWI="), str_lit("data")]
+ ))
+ );
+}
+
#[test]
fn base64_round_trips() {
let encoded = run(call("base64Encode", &[str_lit("Hello, World! 🦔")]));