Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add credit balances table to the Clay source (#118996)

Tom Owersa8deb438d386
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index a6da29435d47..a54f97b66486 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -2650,10 +2650,10 @@ Today: tables are discovered from the user's account at sync time, so there is n
Diffed against: <https://developers.clay.com/openapi.json>
-- [ ] `Workspace credit balances (GET /public/v0/credits/balance)` — monitor remaining data credits and action balances to forecast enrichment costs (high)
-- [ ] `Workflow runs (POST /public/v0/workflows/runs/query)` — measure workflow volume, failures, and execution history across enrichment processes (medium)
-- [ ] `Routine run results (GET /public/v0/routines/run/{routine_run_id}/results)` — analyze enrichment results and completion status for known routine runs (medium)
-- [ ] `Batch routine results (GET /public/v0/routines/run-batch/{routine_run_id}/results)` — measure batch completion and collect enrichment outputs from known batch runs (medium)
+- [x] `Workspace credit balances (GET /public/v0/credits/balance)` — monitor remaining data credits and action balances to forecast enrichment costs (high). Added as `credit_balances`: one row per sync stamped with `retrieved_at`, so the append sync method keeps a balance history.
+- ~~`Workflow runs (POST /public/v0/workflows/runs/query)` — measure workflow volume, failures, and execution history across enrichment processes (medium)~~ — skipped: the endpoint is beta and needs a query string whose grammar is only served behind authentication (`GET /workflows/runs/query/reference`), so the request can't be built against a published contract
+- ~~`Routine run results (GET /public/v0/routines/run/{routine_run_id}/results)` — analyze enrichment results and completion status for known routine runs (medium)~~ — skipped: it needs a routine run ID and the public API has no call that lists runs, and the `result` shape depends on the routine
+- ~~`Batch routine results (GET /public/v0/routines/run-batch/{routine_run_id}/results)` — measure batch completion and collect enrichment outputs from known batch runs (medium)~~ — skipped: it needs a batch run ID with no list call, and the results come back as a presigned file URL rather than rows
Note: The official OpenAPI specification includes read operations and a beta query endpoint for workflow history. Tables use IDs from config.table_ids, so no fixed table names exist for today. Routine results require known run IDs; credit balances need snapshots over time. Authentication, searches of external prospects, and upload operations were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/clay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/clay.py
index 50c3da3cc7d7..a71990d80e46 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/clay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/clay.py
@@ -1,5 +1,6 @@
import re
import dataclasses
+from datetime import UTC, datetime
from typing import Any, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -10,13 +11,27 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponseCursorPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
CLAY_BASE_URL = "https://api.clay.com/public/v0"
PAGE_SIZE = 100 # API maximum
REQUEST_TIMEOUT_SECONDS = 60
+CREDIT_BALANCES_SCHEMA = "credit_balances"
+# The balance endpoint returns only the current balance, so each sync stamps the row with the time it
+# was read. That timestamp is the append cursor, which builds a balance history across syncs.
+CREDIT_BALANCES_INCREMENTAL_FIELDS: list[IncrementalField] = [
+ {
+ "label": "retrieved_at",
+ "type": IncrementalFieldType.DateTime,
+ "field": "retrieved_at",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+]
+
# Clay table IDs look like `t_0te9i4tZEHwc9hihBXu`, and appear after `/tables/` in table URLs.
_TABLE_ID_RE = re.compile(r"(?<![A-Za-z0-9_])t_[A-Za-z0-9]+")
@@ -55,6 +70,19 @@ def _query_body(table_id: str, limit: int = PAGE_SIZE) -> dict[str, Any]:
return {"query": {"tables": [{"id": table_id}], "field_mode": "names"}, "limit": limit}
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": CLAY_BASE_URL,
+ "request_timeout": REQUEST_TIMEOUT_SECONDS,
+ "auth": {
+ "type": "api_key",
+ "name": "clay-api-key",
+ "api_key": api_key,
+ "location": "header",
+ },
+ }
+
+
def clay_source(
api_key: str,
table_id: str,
@@ -63,16 +91,7 @@ def clay_source(
resumable_source_manager: ResumableSourceManager[ClayResumeConfig],
) -> SourceResponse:
config: RESTAPIConfig = {
- "client": {
- "base_url": CLAY_BASE_URL,
- "request_timeout": REQUEST_TIMEOUT_SECONDS,
- "auth": {
- "type": "api_key",
- "name": "clay-api-key",
- "api_key": api_key,
- "location": "header",
- },
- },
+ "client": _client_config(api_key),
"resource_defaults": {},
"resources": [
{
@@ -118,6 +137,50 @@ def clay_source(
return SourceResponse(name=table_id, items=lambda: resource, primary_keys=None)
+def credit_balance_row(balance: dict[str, Any], retrieved_at: datetime) -> dict[str, Any]:
+ # Clay sends whole balances as JSON integers. Cast to float so the first snapshot does not
+ # create an int64 column that a later fractional balance cannot be written to.
+ row: dict[str, Any] = {**balance, "retrieved_at": retrieved_at}
+ for field in ("balance", "action_execution_balance"):
+ if row.get(field) is not None:
+ row[field] = float(row[field])
+ return row
+
+
+def clay_credit_balances_source(api_key: str, team_id: int, job_id: str) -> SourceResponse:
+ retrieved_at = datetime.now(UTC)
+ config: RESTAPIConfig = {
+ "client": _client_config(api_key),
+ "resource_defaults": {},
+ "resources": [
+ {
+ "name": CREDIT_BALANCES_SCHEMA,
+ "table_name": CREDIT_BALANCES_SCHEMA,
+ "write_disposition": "replace",
+ "endpoint": {
+ "path": "/credits/balance",
+ "data_selector": "$",
+ "paginator": "single_page",
+ },
+ "data_map": lambda balance: credit_balance_row(balance, retrieved_at),
+ "table_format": "delta",
+ }
+ ],
+ }
+ resource = rest_api_resource(config, team_id, job_id, None)
+
+ return SourceResponse(
+ name=CREDIT_BALANCES_SCHEMA,
+ items=lambda: resource,
+ primary_keys=["retrieved_at"],
+ partition_count=1,
+ partition_size=1,
+ partition_mode="datetime",
+ partition_format="month",
+ partition_keys=["retrieved_at"],
+ )
+
+
_CLAY_REJECTED_KEY_MESSAGE = (
"Clay rejected PostHog's request while checking your API key. Create a new key under "
"Settings > Account > API keys in Clay, then reconnect."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/source.py
index 1a399caad7a1..f192dddaf4cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/source.py
@@ -9,7 +9,10 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldInputConfigType,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.clay.clay import (
+ CREDIT_BALANCES_INCREMENTAL_FIELDS,
+ CREDIT_BALANCES_SCHEMA,
ClayResumeConfig,
+ clay_credit_balances_source,
clay_source,
parse_table_ids,
validate_credentials as validate_clay_credentials,
@@ -37,6 +40,11 @@ class ClaySource(ResumableSource[ClaySourceConfig, ClayResumeConfig]):
def get_non_retryable_errors(self) -> dict[str, str | None]:
return {
+ # Listed before the generic 403 below, which names tables: the first matching key wins.
+ "403 Client Error: Forbidden for url: https://api.clay.com/public/v0/credits/balance": (
+ "Clay denied access to the workspace's credit balances. Check that the API key's user has "
+ "access to the workspace in Clay, or turn off the credit_balances table."
+ ),
"401 Client Error: Unauthorized for url: https://api.clay.com": (
"Clay rejected the API key. Create a new key under Settings > Account > API keys in Clay "
"and update the source."
@@ -60,12 +68,22 @@ class ClaySource(ResumableSource[ClaySourceConfig, ClayResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- table_ids = parse_table_ids(config.table_ids)
- if names is not None:
- table_ids = [table_id for table_id in table_ids if table_id in names]
- return [
- SourceSchema(name=table_id, supports_incremental=False, supports_append=False) for table_id in table_ids
+ schemas = [
+ SourceSchema(name=table_id, supports_incremental=False, supports_append=False)
+ for table_id in parse_table_ids(config.table_ids)
]
+ schemas.append(
+ SourceSchema(
+ name=CREDIT_BALANCES_SCHEMA,
+ supports_incremental=False,
+ supports_append=True,
+ incremental_fields=CREDIT_BALANCES_INCREMENTAL_FIELDS,
+ description="Remaining data credits and actions in the workspace. Use the append sync method to keep a history.",
+ )
+ )
+ if names is not None:
+ schemas = [schema for schema in schemas if schema.name in names]
+ return schemas
def validate_credentials(
self,
@@ -74,7 +92,11 @@ class ClaySource(ResumableSource[ClaySourceConfig, ClayResumeConfig]):
schema_name: Optional[str] = None,
api_version: str | None = None,
) -> tuple[bool, str | None]:
- table_ids = [schema_name] if schema_name else parse_table_ids(config.table_ids)
+ table_ids = (
+ [schema_name]
+ if schema_name and schema_name != CREDIT_BALANCES_SCHEMA
+ else parse_table_ids(config.table_ids)
+ )
return validate_clay_credentials(config.api_key, table_ids)
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[ClayResumeConfig]:
@@ -86,6 +108,8 @@ class ClaySource(ResumableSource[ClaySourceConfig, ClayResumeConfig]):
resumable_source_manager: ResumableSourceManager[ClayResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
+ if inputs.schema_name == CREDIT_BALANCES_SCHEMA:
+ return clay_credit_balances_source(api_key=config.api_key, team_id=inputs.team_id, job_id=inputs.job_id)
return clay_source(
api_key=config.api_key,
table_id=inputs.schema_name,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay.py
index 767b6c056120..0484c6f5c092 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay.py
@@ -1,6 +1,8 @@
+from datetime import UTC, datetime
from typing import Optional
import pytest
+import time_machine
from products.warehouse_sources.backend.temporal.data_imports.sources.clay.clay import (
REQUEST_TIMEOUT_SECONDS,
@@ -100,6 +102,27 @@ class TestClaySourceBehavior:
assert result.raised is None
assert [request.json().get("cursor") for request in result.requests] == ["resumed"]
+ @time_machine.travel("2026-03-04T05:06:07Z", tick=False)
+ def test_credit_balances_reads_balance_endpoint_and_stamps_snapshot_time(self) -> None:
+ result = self._driver().run(
+ "credit_balances",
+ [ScriptedResponse(json={"balance": 1250, "action_execution_balance": 300.5})],
+ )
+
+ assert result.raised is None
+ assert [(request.method, request.path) for request in result.requests] == [
+ ("GET", "/public/v0/credits/balance")
+ ]
+ assert result.requests[0].headers["clay-api-key"] == "clay_test_key"
+ assert result.rows == [
+ {
+ "balance": 1250.0,
+ "action_execution_balance": 300.5,
+ "retrieved_at": datetime(2026, 3, 4, 5, 6, 7, tzinfo=UTC),
+ }
+ ]
+ assert isinstance(result.rows[0]["balance"], float)
+
@pytest.mark.parametrize(
("me_status", "table_status", "expected_valid", "expected_message_part"),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay_source.py
index 6c3e86c01007..0e6e10c1c863 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clay/tests/test_clay_source.py
@@ -3,14 +3,19 @@ from typing import Optional
import pytest
from products.warehouse_sources.backend.temporal.data_imports.sources.clay.source import ClaySource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ ScriptedResponse,
+ scripted_network,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.clay import ClaySourceConfig
@pytest.mark.parametrize(
("names", "expected"),
[
- (None, ["t_first", "t_second"]),
+ (None, ["t_first", "t_second", "credit_balances"]),
(["t_second"], ["t_second"]),
+ (["credit_balances"], ["credit_balances"]),
(["t_unknown"], []),
],
)
@@ -20,4 +25,32 @@ def test_get_schemas_lists_configured_tables(names: Optional[list[str]], expecte
schemas = ClaySource().get_schemas(config, team_id=1, names=names)
assert [schema.name for schema in schemas] == expected
- assert all(not schema.supports_incremental and not schema.supports_append for schema in schemas)
+ assert all(
+ not schema.supports_incremental and schema.supports_append == (schema.name == "credit_balances")
+ for schema in schemas
+ )
+
+
+ ("schema_name", "expected_table_ids"),
+ [
+ (None, ["t_first", "t_second"]),
+ ("t_second", ["t_second"]),
+ ("credit_balances", ["t_first", "t_second"]),
+ ],
+)
+def test_validate_credentials_probes_only_real_tables(
+ schema_name: Optional[str], expected_table_ids: list[str]
+) -> None:
+ config = ClaySourceConfig(api_key="clay_test_key", table_ids="t_first t_second")
+
+ with scripted_network(lambda request: ScriptedResponse(json={"data": []})) as network:
+ valid, _ = ClaySource().validate_credentials(config, team_id=1, schema_name=schema_name)
+
+ assert valid is True
+ queried_tables = [
+ request.json()["query"]["tables"][0]["id"]
+ for request in network.requests_log
+ if request.path == "/public/v0/tables/query"
+ ]
+ assert queried_tables == expected_table_ids
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
index fa989654a162..ef25f449b902 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
@@ -146,7 +146,7 @@ CiscoDuo not-checkable:no-request
CiscoMeraki fail:stall-budget
Clari fail:stall-budget
Clarifai fail:stall-budget
-Clay not-checkable:schemas
+Clay fail:stall-budget
Clerk fail:stall-budget
Clever fail:stall-budget
ClickHouse not-checkable:schemas