:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from typing import Any, Optional
from requests import Request, Response
from products.warehouse_sources.sources.codefresh.settings import (
ACCOUNT_ID_PLACEHOLDER,
CODEFRESH_ENDPOINTS,
CodefreshEndpointConfig,
)
from products.warehouse_sources.sources.sdk import (
BasePaginator,
OffsetPaginator,
RESTAPIConfig,
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
make_tracked_session,
rest_api_resource,
validate_via_probe,
)
# Only the US SaaS host is supported. EU / self-hosted installs use a different host, which we don't
# let the user retarget yet (it would mean sending the stored API key to an arbitrary host).
CODEFRESH_BASE_URL = "https://g.codefresh.io/api"
REQUEST_TIMEOUT_SECONDS = 10.0
# Prefix of the error raised when the account id cannot be resolved. Shared with the source's
# non-retryable error map so the user reads one explanation instead of a raw exception.
ACCOUNT_LOOKUP_FAILED = "Could not resolve the Codefresh account for this API key"
ACCOUNT_LOOKUP_MESSAGE = (
"PostHog could not work out which Codefresh account this API key belongs to, so the users table "
"cannot sync. Give the key access to your teams, then try again."
)
@dataclasses.dataclass(frozen=True)
class CodefreshResumeConfig:
# Offset pagination position (projects, pipelines, images, step_types, environments).
offset: int | None = None
# Page pagination position (builds) plus the stable pagination session cursor so resumed pages
# read against the same snapshot the first page opened.
page: int | None = None
session_id: str | None = None
class CodefreshPagePaginator(BasePaginator):
"""Codefresh's builds pagination: a 1-indexed ``page`` param, a ``pagination.nextPage`` flag,
and a stable ``pagination.sessionId`` snapshot cursor that every page after the first must pin
via the ``X-Pagination-Session-Id`` header, so builds created mid-sync can't shift the window
underneath us. No built-in paginator covers the flag + header pair, hence this local subclass."""
def __init__(self, page: int = 1, session_id: str | None = None) -> None:
super().__init__()
self.page = page
self.session_id = session_id
def _apply(self, request: Request) -> None:
if request.params is None:
request.params = {}
request.params["page"] = self.page
if self.session_id:
# Pin every page to the snapshot the first page opened.
request.headers = {**(request.headers or {}), "X-Pagination-Session-Id": self.session_id}
def init_request(self, request: Request) -> None:
self._apply(request)
def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
try:
body = response.json()
except Exception:
body = None
pagination = body.get("pagination") or {} if isinstance(body, dict) else {}
self.session_id = pagination.get("sessionId") or self.session_id
# An empty page terminates the stream even if the API keeps advertising nextPage. Without
# this, a server-side cursor bug that streams empty pages forever would loop indefinitely.
if not data or not pagination.get("nextPage"):
self._has_next_page = False
return
self.page += 1
self._has_next_page = True
def update_request(self, request: Request) -> None:
self._apply(request)
def get_resume_state(self) -> Optional[dict[str, Any]]:
# self.page already points at the next page to fetch (update_state incremented it).
return {"page": self.page, "session_id": self.session_id} if self._has_next_page else None
def set_resume_state(self, state: dict[str, Any]) -> None:
page = state.get("page")
if page is not None:
self.page = int(page)
self._has_next_page = True
session_id = state.get("session_id")
if session_id:
self.session_id = session_id
def __str__(self) -> str:
return f"CodefreshPagePaginator(page={self.page})"
def _flatten(item: dict[str, Any], flatten_key: Optional[str]) -> dict[str, Any]:
"""Lift the fields of a nested object (e.g. a pipeline's ``metadata``) to the row top level so
the primary key / partition columns resolve against real top-level fields. Top-level fields win
on a name clash."""
if flatten_key and isinstance(item.get(flatten_key), dict):
item = dict(item)
nested = item.pop(flatten_key)
return {**nested, **item}
return item
def _redact_key(row: dict[str, Any], dotted_key: str) -> dict[str, Any]:
"""Return ``row`` with a possibly-nested field removed. ``"variables"`` drops a top-level field;
``"spec.variables"`` walks into ``spec`` and drops its ``variables``. Only the nodes on the path
are copied, so the upstream item is left unmodified; a missing or non-dict node is a no-op."""
head, _, rest = dotted_key.partition(".")
if head not in row:
return row
if not rest:
return {k: v for k, v in row.items() if k != head}
nested = row[head]
if not isinstance(nested, dict):
return row
return {**row, head: _redact_key(nested, rest)}
def _transform_row(item: dict[str, Any], config: CodefreshEndpointConfig) -> dict[str, Any]:
"""Flatten the row, then drop any redacted fields. Redaction runs after flattening so a key that
only surfaces once a nested object is lifted (and a top-level key of the same name) are both
caught. Redact keys may be dotted paths (e.g. ``spec.variables``) to reach nested fields."""
row = _flatten(item, config.flatten_key)
for key in config.redact_keys:
row = _redact_key(row, key)
return row
def _resolve_account_id(api_key: str) -> str:
"""Return the id of the account the API key belongs to.
``/accounts/{accountId}/users`` needs that id in its path, and Codefresh publishes no endpoint
that reports the account a key is scoped to. ``/team`` is scoped to that same account, is not
paginated, and every team it returns carries the account it belongs to, so one request answers
it.
"""
session = make_tracked_session(redact_values=(api_key,))
response = session.get(
f"{CODEFRESH_BASE_URL}/team",
headers={"Authorization": api_key, "Accept": "application/json"},
timeout=REQUEST_TIMEOUT_SECONDS,
)
response.raise_for_status()
body = response.json()
for team in body if isinstance(body, list) else []:
account_id = team.get("account") if isinstance(team, dict) else None
if account_id:
return str(account_id)
raise ValueError(f"{ACCOUNT_LOOKUP_FAILED}: no team returned by /team names an account.")
def _build_paginator_and_params(config: CodefreshEndpointConfig) -> tuple[BasePaginator, dict[str, Any]]:
if config.pagination == "offset":
# No usable body total; termination is short/empty page. The paginator injects limit+offset.
return OffsetPaginator(limit=config.page_size, total_path=None), {}
if config.pagination == "page":
return CodefreshPagePaginator(), {"limit": config.page_size}
# "none": single request, no pagination params (triggers).
return SinglePagePaginator(), {}
def codefresh_source(
api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[CodefreshResumeConfig],
) -> SourceResponse:
config = CODEFRESH_ENDPOINTS[endpoint]
paginator, params = _build_paginator_and_params(config)
path = config.path
if ACCOUNT_ID_PLACEHOLDER in path:
path = path.replace(ACCOUNT_ID_PLACEHOLDER, _resolve_account_id(api_key))
rest_config: RESTAPIConfig = {
"client": {
"base_url": CODEFRESH_BASE_URL,
# Auth is supplied via the framework auth config so its value is redacted from logs.
# Codefresh expects the raw token as the Authorization header value — no "Bearer " prefix.
"auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},
"headers": {"Accept": "application/json"},
"paginator": paginator,
},
"resources": [
{
"name": endpoint,
"endpoint": {
"path": path,
"params": params,
# Codefresh returns either a bare array or an envelope ({docs: [...]},
# {workflows: {docs: [...]}}); data_key is the path to walk.
"data_selector": ".".join(config.data_key) if config.data_key else None,
# For bare-array endpoints a 200 body that isn't a list means the response
# shape changed, so fail loud instead of syncing a stray object as a row.
# ``require_data_key`` extends the same treatment to an envelope endpoint whose
# body shape the vendor does not document.
"data_selector_required": config.data_key is None or config.require_data_key,
},
"data_map": lambda item, config=config: _transform_row(item, config),
}
],
}
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None:
if config.pagination == "offset" and resume.offset is not None:
initial_paginator_state = {"offset": resume.offset}
elif config.pagination == "page" and (resume.page is not None or resume.session_id is not None):
initial_paginator_state = {"page": resume.page, "session_id": resume.session_id}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when a next page remains; the hook fires AFTER a page is yielded so a crash
# re-pulls the page we just emitted rather than skipping it — merge dedupes the re-pulled
# rows on the primary key. A short/last page passes None: nothing left to resume to.
if not state:
return
if config.pagination == "offset" and state.get("offset") is not None:
resumable_source_manager.save_state(CodefreshResumeConfig(offset=int(state["offset"])))
elif config.pagination == "page" and state.get("page") is not None:
resumable_source_manager.save_state(
CodefreshResumeConfig(page=int(state["page"]), session_id=state.get("session_id"))
)
resource = rest_api_resource(
rest_config,
team_id,
job_id,
None,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
return SourceResponse(
name=endpoint,
items=lambda: resource,
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
column_hints=resource.column_hints,
)
def validate_credentials(api_key: str, schema_name: Optional[str] = None) -> tuple[bool, str | None]:
"""Probe the token. Codefresh keys are scoped per resource, so at source-create (``schema_name``
is ``None``) a 403 means the token is genuine but lacks scope for the probed resource — accept it
and let the user pick the tables their key can reach. A 401 always means a bad token."""
config = CODEFRESH_ENDPOINTS.get(schema_name) if schema_name else None
path = config.path if config is not None else "/projects"
if ACCOUNT_ID_PLACEHOLDER in path:
# The path is only knowable once /team resolves the account id, and that lookup is the first
# request the sync makes, so run it here too. A 200 from /team is not enough on its own: a
# response that names no account fails the sync later, so reject it now.
try:
path = path.replace(ACCOUNT_ID_PLACEHOLDER, _resolve_account_id(api_key))
except ValueError:
return False, ACCOUNT_LOOKUP_MESSAGE
except Exception:
# The lookup failed for a reason that is not an answer about the account, such as a bad
# token, a missing scope, or a transport error. Probe /team so the status mapping below
# reports which of those it was.
path = CODEFRESH_ENDPOINTS["teams"].path
ok, status = validate_via_probe(
lambda: make_tracked_session(redact_values=(api_key,)),
f"{CODEFRESH_BASE_URL}{path}?limit=1",
headers={"Authorization": api_key, "Accept": "application/json"},
)
if ok:
return True, None
if status is None:
return False, "Could not connect to Codefresh. Please try again."
if status == 401:
return False, "Your Codefresh API key is invalid or has been revoked."
if status == 403:
if schema_name:
return False, f"Your Codefresh API key is missing the access scope required to sync '{schema_name}'."
# Valid token, but it lacks scope for the probe resource — don't block source creation.
return True, None
if status == 429 or status >= 500:
# Transient: a rate-limit or server error doesn't mean the key is bad. Surface it as a
# retryable failure rather than telling the user their credentials are invalid.
return False, "Codefresh is temporarily unavailable. Please try again in a moment."
return False, f"Codefresh API returned an unexpected status ({status})."