:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import json
import threading
import dataclasses
from collections.abc import Callable, Iterator
from typing import Any
from urllib.parse import parse_qsl, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt
from urllib3.util.retry import Retry
from products.warehouse_sources.sources.flagsmith.settings import (
FLAGSMITH_ENDPOINTS,
FlagsmithEndpointConfig,
ParentResource,
)
from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
reach_safe_point,
retry_with_deadline,
)
DEFAULT_BASE_URL = "https://api.flagsmith.com"
REQUEST_TIMEOUT_SECONDS = 60
MAX_RETRY_ATTEMPTS = 5
MAX_RETRY_WAIT_SECONDS = 60
# Cap on the total pages walked in one sync invocation, shared across parent enumeration and
# every fan-out child listing. `next` links come from a user-supplied (self-hosted) host, so a
# hostile or misconfigured server can return a non-empty cyclic `next` forever — and in a fan-out
# multiply that across thousands of parents. A single shared budget makes the whole sync
# self-terminate instead of issuing credentialed requests until the activity is cancelled.
# Generous enough not to truncate real data.
MAX_PAGES_PER_SYNC = 10_000
# Cap on how many fan-out parents (organisation ids, project ids, or environment api_keys) a single
# sync enumerates and holds in memory before child processing begins. Even within the page budget a
# hostile host could return enough parents to balloon this list and exhaust a warehouse worker.
# Generous enough to cover any real org.
MAX_FANOUT_PARENTS = 50_000
# Cap on the length of a single fan-out parent key. Real ids and api_keys are short identifiers, so
# the count cap alone doesn't bound retained bytes — a hostile host could return an arbitrarily long
# id/api_key (up to the response byte cap) that balloons the retained list and gets interpolated into
# child request URLs. Keys longer than this are skipped rather than trusted.
MAX_FANOUT_KEY_LENGTH = 256
class _PageBudget:
"""Mutable page allowance drawn down across a whole sync (all parents and child listings)."""
def __init__(self, limit: int) -> None:
self.remaining = limit
def take(self) -> bool:
"""Consume one page; return False once the shared budget is exhausted."""
if self.remaining <= 0:
return False
self.remaining -= 1
return True
# base_url can be a customer-controlled self-hosted host, so response bodies are streamed and
# read under a byte cap and a total-transfer deadline: an arbitrarily large (or endlessly
# dripped) 200 body must fail the sync instead of exhausting worker memory or holding a worker.
# The cap counts decoded bytes, so a gzip bomb can't slip past it.
MAX_RESPONSE_BYTES = 256 * 1024 * 1024
_READ_CHUNK_BYTES = 1024 * 1024
# The per-read socket timeout only bounds the gap between bytes, so a host that trickles one byte
# just before each read timeout could keep a worker busy until the activity's timeout. This
# absolute wall-clock deadline (enforced in `_read_bounded`) caps how long a single body read may
# take end to end, even while a read is blocked mid-chunk.
MAX_RESPONSE_SECONDS = 600
# Bytes pulled from an error body for a diagnostic message — error bodies are never needed in full.
_ERROR_SNIPPET_BYTES = 2048
class FlagsmithRetryableError(Exception):
def __init__(self, message: str, retry_after: float | None = None):
super().__init__(message)
self.retry_after = retry_after
class FlagsmithResponseTooLargeError(Exception):
pass
class FlagsmithResponseTimeoutError(Exception):
pass
def _read_bounded(
response: requests.Response, max_bytes: int = MAX_RESPONSE_BYTES, max_seconds: float = MAX_RESPONSE_SECONDS
) -> bytes:
"""Read a streamed response body under both a byte cap and a total-transfer deadline.
The read runs on a worker thread bounded by ``join(timeout=max_seconds)``, so the deadline is
enforced as absolute wall-clock time even while a read is blocked: ``iter_content`` fills a
whole chunk before it yields, so a host that trickles bytes just under the per-read socket
timeout could otherwise never let an in-loop deadline check run. On timeout we close the
response to unblock the pending socket read and let the daemon thread unwind.
"""
box: dict[str, Any] = {}
def _reader() -> None:
try:
total = 0
chunks: list[bytes] = []
for chunk in response.iter_content(chunk_size=_READ_CHUNK_BYTES):
total += len(chunk)
if total > max_bytes:
box["error"] = FlagsmithResponseTooLargeError(
f"Flagsmith API response exceeded the size limit ({max_bytes} bytes)"
)
return
chunks.append(chunk)
box["data"] = b"".join(chunks)
except Exception as exc: # surfaced on the calling thread below
box["error"] = exc
thread = threading.Thread(target=_reader, name="flagsmith-read-bounded", daemon=True)
thread.start()
thread.join(timeout=max_seconds)
if thread.is_alive():
# Close the socket so the blocked read raises and the daemon thread can exit.
response.close()
raise FlagsmithResponseTimeoutError(
f"Flagsmith API response exceeded the download time limit ({max_seconds:g}s)"
)
if "error" in box:
raise box["error"]
return box.get("data", b"")
def _error_snippet(response: requests.Response) -> str:
try:
chunk = next(response.iter_content(chunk_size=_ERROR_SNIPPET_BYTES), b"")
return chunk[:_ERROR_SNIPPET_BYTES].decode("utf-8", errors="replace")
except Exception:
return ""
@dataclasses.dataclass(frozen=True)
class FlagsmithResumeConfig:
# Full URL of the next page to fetch ("" once a resource is exhausted).
next_url: str = ""
# For fan-out endpoints, the parent (organisation id, project id, or environment
# api_key) currently being paginated ("" for top-level endpoints or before the first
# parent starts). Two-level parents use ``FanoutParent.resume_key``.
parent_key: str = ""
@dataclasses.dataclass(frozen=True)
class FanoutParent:
"""One fan-out parent: the identifiers substituted into a child endpoint's path or query.
``child`` is empty for the single-level parents (organisation, project, environment) and
carries the inner identifier for the two-level ones (an identity id, or a feature id paired
with an environment id). ``extra`` carries the third identifier of a three-level parent (an
environment paired with a project and a resource inside it).
"""
key: str
child: str = ""
extra: str = ""
@property
def resume_key(self) -> str:
# Identifiers are numeric ids or api_keys, so a "/" join stays unambiguous.
return "/".join(part for part in (self.key, self.child, self.extra) if part)
def normalize_base_url(base_url: str | None) -> str:
"""Normalize the (optional) instance URL and reject anything that isn't plain http(s).
Defaults to Flagsmith SaaS; self-hosted deployments pass their own API URL.
Rejects URLs where the host ``urlparse`` reports could diverge from the host the HTTP
client actually dials, which would let a caller slip past the downstream SSRF allowlist
(which validates the parsed hostname): backslashes (treated as path separators by some
clients but not by ``urlparse``), userinfo (``user@host`` hides the real host after an
``@``), and query/fragment noise that has no place in a base URL.
"""
host = (base_url or "").strip() or DEFAULT_BASE_URL
if "://" not in host:
host = f"https://{host}"
# Catch raw and percent-encoded backslashes before urlparse silently keeps them.
if "\\" in host or "%5c" in host.lower():
raise ValueError(f"Invalid Flagsmith base URL: {base_url}")
host = host.rstrip("/")
parsed = urlparse(host)
if parsed.scheme not in ("http", "https") or not parsed.hostname:
raise ValueError(f"Invalid Flagsmith base URL: {base_url}")
if parsed.username is not None or parsed.password is not None or "@" in parsed.netloc:
raise ValueError(f"Invalid Flagsmith base URL: {base_url}")
if parsed.query or parsed.fragment or parsed.params:
raise ValueError(f"Invalid Flagsmith base URL: {base_url}")
return host
def hostname_of(base_url: str | None) -> str:
return urlparse(normalize_base_url(base_url)).hostname or ""
def scheme_of(base_url: str | None) -> str:
return urlparse(normalize_base_url(base_url)).scheme
def _api_base(base: str) -> str:
return f"{base}/api/v1"
def _headers(api_key: str) -> dict[str, str]:
# Organisation API keys authenticate with an "Api-Key" prefix
# (see https://docs.flagsmith.com/clients/rest#private-admin-api-endpoints).
return {
"Authorization": f"Api-Key {api_key}",
"Accept": "application/json",
}
def _initial_url(base: str, path: str, params: dict[str, Any]) -> str:
url = f"{_api_base(base)}{path}"
if params:
sep = "&" if "?" in path else "?"
url = f"{url}{sep}{urlencode(params)}"
return url
def _pinned_next_url(base: str, next_link: Any) -> str | None:
"""Rebuild a DRF ``next`` link onto the configured base URL.
The API returns absolute URLs derived from the request host, which a misconfigured
self-hosted proxy (or a hostile server) could point elsewhere. Keeping only the
path + query pins pagination to the host that credential validation approved.
"""
if not next_link or not isinstance(next_link, str):
return None
parsed = urlparse(next_link)
query = f"?{parsed.query}" if parsed.query else ""
return f"{base}{parsed.path}{query}"
def _cursor_next_url(base: str, current_url: str, cursor: Any) -> str | None:
"""Next-page URL for a listing that returns an opaque body cursor instead of a ``next`` link.
The next request repeats the current one with ``cursor`` replaced, rebuilt onto the configured
base for the same host-pinning reason as ``_pinned_next_url``.
"""
if not cursor or not isinstance(cursor, str):
return None
parsed = urlparse(current_url)
query = [(key, value) for key, value in parse_qsl(parsed.query) if key != "cursor"]
query.append(("cursor", cursor))
return f"{base}{parsed.path}?{urlencode(query)}"
def _extract_rows(base: str, data: Any, current_url: str) -> tuple[list[dict[str, Any]], str | None]:
"""Rows + next-page URL from a response body: a DRF pagination envelope
(``{count, next, previous, results}``), a cursor envelope (``{results, next_cursor}``), or a
plain JSON array (projects, users, evaluation data)."""
if isinstance(data, list):
return [row for row in data if isinstance(row, dict)], None
if isinstance(data, dict):
rows = [row for row in data.get("results") or [] if isinstance(row, dict)]
next_url = _pinned_next_url(base, data.get("next")) or _cursor_next_url(
base, current_url, data.get("next_cursor")
)
return rows, next_url
return [], None
def _wait_strategy(retry_state: Any) -> float:
exc = retry_state.outcome.exception() if retry_state.outcome else None
if isinstance(exc, FlagsmithRetryableError) and exc.retry_after is not None:
return min(exc.retry_after, MAX_RETRY_WAIT_SECONDS)
return min(2.0**retry_state.attempt_number, MAX_RETRY_WAIT_SECONDS)
@retry_with_deadline(
retry=retry_if_exception_type((FlagsmithRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=_wait_strategy,
reraise=True,
)
def _fetch_page(session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger) -> Any:
# stream=True keeps the body off the wire until it's read through `_read_bounded` under a byte
# cap and a transfer deadline — a customer-controlled host must not exhaust worker memory or
# hold a worker with an endless body.
with session.get(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS, stream=True) as response:
if response.status_code == 429:
retry_after_header = response.headers.get("Retry-After")
try:
retry_after = float(retry_after_header) if retry_after_header else None
except ValueError:
# A non-numeric value (e.g. an HTTP-date) falls back to exponential backoff.
retry_after = None
logger.warning(f"Flagsmith rate limited (429), retrying. retry_after={retry_after_header}, url={url}")
raise FlagsmithRetryableError(f"Flagsmith rate limited: url={url}", retry_after=retry_after)
if response.status_code >= 500:
raise FlagsmithRetryableError(f"Flagsmith server error: status={response.status_code}, url={url}")
if not response.ok:
logger.error(
f"Flagsmith API error: status={response.status_code}, body={_error_snippet(response)}, url={url}"
)
response.raise_for_status()
return json.loads(_read_bounded(response))
def validate_credentials(api_key: str, base_url: str | None, path: str = "/organisations/") -> int | None:
"""Probe an endpoint and return the HTTP status code (or None on transport failure)."""
try:
# base_url is user-supplied (self-hosted), so pin redirects off: validation and the
# outbound request must stay on the same target (SSRF defense-in-depth). capture=False
# keeps the probe's customer-content response body out of HTTP sample storage.
# retry=Retry(total=0): the probe runs inline on an API worker, so it takes a single
# attempt — a hostile base_url must not be able to hold the worker via adapter retries
# that honour an unbounded server-controlled Retry-After.
session = make_tracked_session(
redact_values=(api_key,), allow_redirects=False, capture=False, retry=Retry(total=0)
)
# stream=True and a context manager so we read only the status line/headers and never
# download the body: a hostile base_url could otherwise trickle an endless body and hold
# this inline API worker open well past the read timeout (which only bounds byte gaps).
with session.get(
f"{_api_base(normalize_base_url(base_url))}{path}", headers=_headers(api_key), timeout=10, stream=True
) as response:
return response.status_code
except Exception:
return None
def _iter_pages(
session: requests.Session,
base: str,
start_url: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
budget: _PageBudget,
) -> Iterator[list[dict[str, Any]]]:
"""Walk a listing (paginated or plain-array) without touching resume state — used to
enumerate fan-out parents, which must be re-listed from scratch on every run. Parents are
enumerated before any row is yielded, so each page is a safe point."""
url: str | None = start_url
while url:
if not budget.take():
logger.warning(f"Flagsmith: sync page budget ({MAX_PAGES_PER_SYNC}) exhausted at {start_url}, truncating")
return
data = _fetch_page(session, url, headers, logger)
rows, url = _extract_rows(base, data, url)
if rows:
yield rows
reach_safe_point()
def _extend_capped(
keys: list[str],
listing: Iterator[list[dict[str, Any]]],
field: str,
logger: FilteringBoundLogger,
limit: int | None = None,
predicate: Callable[[dict[str, Any]], bool] | None = None,
) -> bool:
"""Append ``row[field]`` from each row of a parent listing into ``keys``, stopping once the
retained list hits ``limit`` so a hostile host can't balloon it and exhaust worker memory. A
two-level enumeration passes the allowance left over from the outer level, so the cap bounds
the parents retained across the whole fan-out rather than per inner listing. Keys longer than
MAX_FANOUT_KEY_LENGTH are skipped so an oversized id/api_key can't balloon retained bytes or
poison a child request URL. ``predicate`` drops parents whose child endpoint would not serve
them. Returns True if the count cap was reached (the caller should stop enumerating)."""
# Resolved at call time, not bound as a default, so the module-level cap stays patchable.
limit = MAX_FANOUT_PARENTS if limit is None else limit
if limit <= 0:
return True
for rows in listing:
for row in rows:
if predicate is not None and not predicate(row):
continue
key = str(row[field])
if len(key) > MAX_FANOUT_KEY_LENGTH:
logger.warning(f"Flagsmith: skipping fan-out parent with oversized {field} ({len(key)} chars)")
continue
keys.append(key)
if len(keys) >= limit:
logger.warning(f"Flagsmith: fan-out parent cap ({MAX_FANOUT_PARENTS}) reached, truncating enumeration")
return True
return False
def _fetch_identity_parents(
session: requests.Session,
base: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
environment_keys: list[str],
budget: _PageBudget,
) -> list[FanoutParent]:
"""Pair every environment with each of its identities. Flagsmith has no bulk traits listing,
so a traits sync fans out once per identity and both caps apply: the shared page budget bounds
the requests and MAX_FANOUT_PARENTS the identities retained."""
parents: list[FanoutParent] = []
for environment_key in environment_keys:
listing = _iter_pages(
session,
base,
_initial_url(base, f"/environments/{environment_key}/identities/", {"page_size": 100}),
headers,
logger,
budget,
)
identity_ids: list[str] = []
capped = _extend_capped(identity_ids, listing, "id", logger, limit=MAX_FANOUT_PARENTS - len(parents))
parents.extend(FanoutParent(environment_key, identity_id) for identity_id in identity_ids)
if capped:
break
return parents
def _uses_v2_versioning(environment: dict[str, Any]) -> bool:
return bool(environment.get("use_v2_feature_versioning"))
def _fetch_environment_feature_parents(
session: requests.Session,
base: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
project_ids: list[str],
budget: _PageBudget,
require_v2_versioning: bool = False,
) -> list[FanoutParent]:
"""Pair every environment with every feature of the same project. The feature-segments and
version listings require both filters, and a pair only resolves within the project that owns
them. ``require_v2_versioning`` keeps only the environments whose version listing exists,
because Flagsmith rejects the request outright on an environment still using unversioned
feature states."""
parents: list[FanoutParent] = []
for project_id in project_ids:
environment_ids: list[str] = []
_extend_capped(
environment_ids,
_iter_pages(
session, base, _initial_url(base, f"/environments/?project={project_id}", {}), headers, logger, budget
),
"id",
logger,
predicate=_uses_v2_versioning if require_v2_versioning else None,
)
feature_ids: list[str] = []
_extend_capped(
feature_ids,
_iter_pages(
session,
base,
_initial_url(base, f"/projects/{project_id}/features/", {"page_size": 100}),
headers,
logger,
budget,
),
"id",
logger,
)
for environment_id in environment_ids:
for feature_id in feature_ids:
if len(parents) >= MAX_FANOUT_PARENTS:
logger.warning(
f"Flagsmith: fan-out parent cap ({MAX_FANOUT_PARENTS}) reached, truncating enumeration"
)
return parents
parents.append(FanoutParent(environment_id, feature_id))
return parents
def _fetch_project_resource_environment_parents(
session: requests.Session,
base: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
project_ids: list[str],
budget: _PageBudget,
resource_path: str,
) -> list[FanoutParent]:
"""Build the (project, project resource, environment) triples the evaluation-data and
segment-members listings need: the project and the resource sit in the path, the environment
is a required query filter, and a triple only resolves within the project that owns it."""
parents: list[FanoutParent] = []
for project_id in project_ids:
environment_ids: list[str] = []
_extend_capped(
environment_ids,
_iter_pages(
session, base, _initial_url(base, f"/environments/?project={project_id}", {}), headers, logger, budget
),
"id",
logger,
)
resource_ids: list[str] = []
_extend_capped(
resource_ids,
_iter_pages(
session,
base,
_initial_url(base, resource_path.format(project=project_id), {"page_size": 100}),
headers,
logger,
budget,
),
"id",
logger,
)
for resource_id in resource_ids:
for environment_id in environment_ids:
if len(parents) >= MAX_FANOUT_PARENTS:
logger.warning(
f"Flagsmith: fan-out parent cap ({MAX_FANOUT_PARENTS}) reached, truncating enumeration"
)
return parents
parents.append(FanoutParent(project_id, resource_id, environment_id))
return parents
def _fetch_parent_keys(
session: requests.Session,
base: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
parent: ParentResource,
budget: _PageBudget,
) -> list[FanoutParent]:
if parent == "organisation":
listing = _iter_pages(session, base, _initial_url(base, "/organisations/", {}), headers, logger, budget)
org_ids: list[str] = []
_extend_capped(org_ids, listing, "id", logger)
return [FanoutParent(org_id) for org_id in org_ids]
project_listing = _iter_pages(session, base, _initial_url(base, "/projects/", {}), headers, logger, budget)
project_ids: list[str] = []
_extend_capped(project_ids, project_listing, "id", logger)
if parent == "project":
return [FanoutParent(project_id) for project_id in project_ids]
if parent == "environment_feature":
return _fetch_environment_feature_parents(session, base, headers, logger, project_ids, budget)
if parent == "versioned_environment_feature":
return _fetch_environment_feature_parents(
session, base, headers, logger, project_ids, budget, require_v2_versioning=True
)
if parent == "project_feature_environment":
return _fetch_project_resource_environment_parents(
session, base, headers, logger, project_ids, budget, "/projects/{project}/features/"
)
if parent == "project_segment_environment":
return _fetch_project_resource_environment_parents(
session, base, headers, logger, project_ids, budget, "/projects/{project}/segments/"
)
# Environments are addressed by their (non-secret, client-side) api_key and listed
# per project; keep API order so the resume bookmark resolves deterministically.
keys: list[str] = []
for project_id in project_ids:
env_listing = _iter_pages(
session, base, _initial_url(base, f"/environments/?project={project_id}", {}), headers, logger, budget
)
if _extend_capped(keys, env_listing, "api_key", logger):
break
if parent == "environment":
return [FanoutParent(key) for key in keys]
return _fetch_identity_parents(session, base, headers, logger, keys, budget)
def _paginate_resource(
session: requests.Session,
base: str,
start_url: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[FlagsmithResumeConfig],
parent: FanoutParent,
parent_field: str | None,
child_field: str | None,
extra_field: str | None,
budget: _PageBudget,
) -> Iterator[list[dict[str, Any]]]:
url: str | None = start_url
while url:
if not budget.take():
logger.warning(f"Flagsmith: sync page budget ({MAX_PAGES_PER_SYNC}) exhausted at {start_url}, truncating")
# Terminal resume state so a resume advances past this parent instead of re-entering
# the same (possibly cyclic) chain and re-spending the budget on it.
resumable_source_manager.save_state(FlagsmithResumeConfig(next_url="", parent_key=parent.resume_key))
return
data = _fetch_page(session, url, headers, logger)
rows, next_url = _extract_rows(base, data, url)
for row in rows:
if parent_field:
row[parent_field] = parent.key
if child_field:
row[child_field] = parent.child
if extra_field:
row[extra_field] = parent.extra
if rows:
yield rows
# Save state AFTER yielding so a heartbeat-timeout crash re-fetches from the next
# page rather than re-emitting the page we just yielded (merge dedupes regardless).
resumable_source_manager.save_state(
FlagsmithResumeConfig(next_url=next_url or "", parent_key=parent.resume_key)
)
resumable_source_manager.safe_point()
url = next_url
def _get_fan_out_rows(
session: requests.Session,
base: str,
config: FlagsmithEndpointConfig,
headers: dict[str, str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[FlagsmithResumeConfig],
resume: FlagsmithResumeConfig | None,
budget: _PageBudget,
) -> Iterator[list[dict[str, Any]]]:
assert config.parent is not None
parent_keys = _fetch_parent_keys(session, base, headers, logger, config.parent, budget)
if not parent_keys:
logger.warning(f"Flagsmith: no {config.parent}s found, nothing to sync for endpoint={config.name}")
return
start_idx = 0
resume_url: str | None = None
resume_keys = [parent.resume_key for parent in parent_keys]
if resume is not None and resume.parent_key and resume.parent_key in resume_keys:
idx = resume_keys.index(resume.parent_key)
if resume.next_url:
# Mid-parent: pick up at the saved page within that parent. Re-pin onto the current base
# so a stale URL from a since-retargeted source can't send the current API key to the
# previously configured host.
start_idx = idx
resume_url = _pinned_next_url(base, resume.next_url)
else:
# The saved parent finished (empty next_url marker); start at the next one.
start_idx = idx + 1
for i in range(start_idx, len(parent_keys)):
parent = parent_keys[i]
if i == start_idx and resume_url:
start_url = resume_url
else:
start_url = _initial_url(
base,
config.path.format(parent=parent.key, child=parent.child, extra=parent.extra),
config.params,
)
yield from _paginate_resource(
session,
base,
start_url,
headers,
logger,
resumable_source_manager,
parent,
config.parent_field,
config.child_field,
config.extra_field,
budget,
)
def get_rows(
api_key: str,
base_url: str | None,
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[FlagsmithResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
config = FLAGSMITH_ENDPOINTS[endpoint]
base = normalize_base_url(base_url)
headers = _headers(api_key)
# capture=False: Flagsmith bodies carry customer-authored content the name-based scrubbers
# can't recognise — feature values, segment rules, audit records, and member PII (names/emails).
# retry=Retry(total=0): _fetch_page owns the retry budget via tenacity with a bounded wait
# (_wait_strategy caps at MAX_RETRY_WAIT_SECONDS); leaving the adapter's DEFAULT_RETRY on would
# let a hostile base_url stall a sync worker with an unbounded, server-controlled Retry-After.
session = make_tracked_session(redact_values=(api_key,), allow_redirects=False, capture=False, retry=Retry(total=0))
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
# One budget for the whole invocation so a fan-out can't multiply the per-listing cap across
# parents into unbounded credentialed requests.
budget = _PageBudget(MAX_PAGES_PER_SYNC)
if config.parent is not None:
yield from _get_fan_out_rows(session, base, config, headers, logger, resumable_source_manager, resume, budget)
return
if resume is not None and resume.next_url:
logger.debug(f"Flagsmith: resuming endpoint={endpoint} from saved page")
# Re-pin onto the current base: a stale resume URL from a since-retargeted source must not
# send the current API key to the previously configured host.
start_url = _pinned_next_url(base, resume.next_url) or _initial_url(base, config.path, config.params)
else:
start_url = _initial_url(base, config.path, config.params)
yield from _paginate_resource(
session,
base,
start_url,
headers,
logger,
resumable_source_manager,
parent=FanoutParent(""),
parent_field=None,
child_field=None,
extra_field=None,
budget=budget,
)
def flagsmith_source(
api_key: str,
base_url: str | None,
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[FlagsmithResumeConfig],
) -> SourceResponse:
config = FLAGSMITH_ENDPOINTS[endpoint]
return SourceResponse(
name=endpoint,
items=lambda: get_rows(
api_key=api_key,
base_url=base_url,
endpoint=endpoint,
logger=logger,
resumable_source_manager=resumable_source_manager,
),
primary_keys=list(config.primary_keys),
partition_count=1 if config.partition_key else None,
partition_size=1 if config.partition_key else None,
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
)