:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import re
import json
import time
from collections.abc import Callable
from typing import Any, Optional, cast
from django.conf import settings
import dagster
import structlog
from posthog.dags.common import JobOwners
from posthog.dags.common.resources import redis
from posthog.egress.github.transport import github_request
from posthog.egress.limiter.policies import Priority
from posthog.exceptions_capture import capture_exception
from products.growth.backend.constants import SDK_CACHE_EXPIRY, SDK_TYPES, SdkTypes, github_sdk_versions_key
from products.growth.backend.sdk_health import SemanticVersion, try_parse_version
# Re-exported for backward compatibility with callers that still import from here.
__all__ = ["SDK_CACHE_EXPIRY", "SDK_TYPES", "SdkTypes", "github_sdk_versions_key"]
logger = structlog.get_logger(__name__)
UNPREFIXED_SEMVER_TAG = re.compile(r"\d+\.\d+(?:\.\d+)*$")
# Using lambda here to be able to define this before defining the functions
SDK_FETCH_FUNCTIONS: dict[SdkTypes, Callable[[], dict[str, Any]]] = {
"web": lambda: fetch_web_sdk_data(),
"posthog-python": lambda: fetch_python_sdk_data(),
"posthog-node": lambda: fetch_node_sdk_data(),
"posthog-react-native": lambda: fetch_react_native_sdk_data(),
"posthog-flutter": lambda: fetch_flutter_sdk_data(),
"posthog-kmp": lambda: fetch_kmp_sdk_data(),
"posthog-ios": lambda: fetch_ios_sdk_data(),
"posthog-android": lambda: fetch_android_sdk_data(),
"posthog-java": lambda: fetch_java_sdk_data(),
"posthog-server": lambda: fetch_java_server_sdk_data(),
"posthog-go": lambda: fetch_go_sdk_data(),
"posthog-php": lambda: fetch_php_sdk_data(),
"posthog-laravel": lambda: fetch_laravel_sdk_data(),
"posthog-ruby": lambda: fetch_ruby_sdk_data(),
"posthog-elixir": lambda: fetch_elixir_sdk_data(),
"posthog-dotnet": lambda: fetch_dotnet_sdk_data(),
"posthog-unity": lambda: fetch_sdk_data_from_releases(
"PostHog/posthog-unity", tag_prefixes=[UNPREFIXED_SEMVER_TAG]
),
"posthog-node-mcp": lambda: fetch_sdk_data_from_releases("PostHog/posthog-js", tag_prefixes=["@posthog/mcp@"]),
"posthog-python-mcp": lambda: fetch_python_sdk_data(),
"posthog-edge": lambda: fetch_node_sdk_data(),
"posthog-convex": lambda: fetch_sdk_data_from_releases("PostHog/posthog-js", tag_prefixes=["@posthog/convex@"]),
"posthog-rails": lambda: fetch_sdk_data_from_releases("PostHog/posthog-ruby", tag_prefixes=["posthog-rails-v"]),
"posthog-aspnetcore": lambda: fetch_sdk_data_from_releases(
"PostHog/posthog-dotnet", tag_prefixes=["PostHog.AspNetCore-v"]
),
"posthog-rs": lambda: fetch_rust_sdk_data(),
"posthog-rs-mcp": lambda: fetch_rust_sdk_data(),
"posthog-godot": lambda: fetch_sdk_data_from_releases(
"PostHog/posthog-godot", tag_prefixes=[UNPREFIXED_SEMVER_TAG]
),
"posthog-unreal": lambda: fetch_sdk_data_from_releases(
"PostHog/posthog-unreal", tag_prefixes=[UNPREFIXED_SEMVER_TAG]
),
"posthog-go-mcp": lambda: fetch_go_sdk_data(),
"posthog-ruby-mcp": lambda: fetch_ruby_sdk_data(),
"posthog-roblox": lambda: fetch_roblox_sdk_data(),
"js": lambda: fetch_js_lite_sdk_data(),
}
def fetch_github_data_for_sdk(lib_name: str) -> Optional[dict[str, Any]]:
"""Fetch GitHub data for specific SDK type using ClickHouse $lib value."""
fetch_fn = SDK_FETCH_FUNCTIONS.get(cast(SdkTypes, lib_name))
if fetch_fn:
return fetch_fn()
return None
def prefixed_or_unprefixed_semver_tags(*prefixes: str) -> list[str | re.Pattern]:
"""Match semver-style release tags with optional string prefixes."""
return [*prefixes, UNPREFIXED_SEMVER_TAG]
# nosemgrep: tuple-return-prefer-dataclass -- opaque sort key, only ever compared whole
def _semver_order_key(version: SemanticVersion) -> tuple[int, int, int, bool]:
# Stable releases sort above prereleases with the same core version
return (version.major, version.minor or 0, version.patch or 0, version.extra is None)
def fetch_sdk_data_from_releases(repo: str, tag_prefixes: list[str | re.Pattern] | None = None) -> dict[str, Any]:
"""Helper function to fetch SDK data from GitHub releases API."""
# By default we'll include anything in the list if not specified
if tag_prefixes is None:
tag_prefixes = [""]
releases = fetch_releases_from_repo(repo)
if not releases:
return {}
latest_version = None
latest_parsed: Optional[SemanticVersion] = None
release_dates = {}
for release in releases:
if release.get("draft") or release.get("prerelease"):
continue
tag = release.get("tag_name", "")
# Only process tags that match any of the tag prefixes
# We also support using regex here (used to match text that starts with a number)
version = None
for tag_prefix in tag_prefixes:
if isinstance(tag_prefix, re.Pattern):
if tag_prefix.match(tag):
version = tag # For regex matches we return the full tag
break
else:
if tag.startswith(tag_prefix):
version = tag[len(tag_prefix) :]
break
if not version:
continue
# GitHub orders /releases by creation date, not by version, so the first match is
# wrong whenever a hotfix ships on an older release line (or a release is backfilled).
# Pick the highest version that parses instead; a non-parseable "latest" would make
# the assessor drop the SDK from every report.
parsed = try_parse_version(version)
if parsed is not None and (
latest_parsed is None or _semver_order_key(parsed) > _semver_order_key(latest_parsed)
):
latest_version = version
latest_parsed = parsed
# Unintuitively we need to use `created_at` rather than `published_at`
# because the former represents when the tag was created while the latter is when the release was created
# and since some GitHub releases were backfilled we need the actual tag date
if created_at := release.get("created_at"):
release_dates[version] = created_at
if not latest_version:
return {}
return {"latestVersion": latest_version, "releaseDates": release_dates}
# Dedupes fetches within a run for repos shared by several fetchers (PostHog/posthog-js serves
# web, node, and react-native). Entries must expire well before the next hourly run: this module
# lives in a long-running process, so an eternal cache would re-write Redis with the same
# snapshot every run and freeze "latest version" until a redeploy.
LOCAL_RELEASES_CACHE_TTL_SECONDS = 30 * 60
local_releases_cache: dict[str, tuple[float, list[Any]]] = {}
def fetch_releases_from_repo(repo: str, skip_cache: bool = False) -> list[Any]:
"""Fetch releases from a GitHub repository.
Returns [] when any page fails, so callers skip the Redis write and the last known-good
data keeps serving. A partial page set must never be treated as the full release list:
it can miss versions entirely and gets a stale "latest" cached for days.
"""
global local_releases_cache
# We don't wanna have to fight against the local cache when running tests
# so we just skip it since the cache is only here to avoid hitting GitHub's rate limit
# and we fully mock the requests during tests anyway
if settings.TEST:
skip_cache = True
if not skip_cache:
cached = local_releases_cache.get(repo)
if cached is not None and time.monotonic() - cached[0] < LOCAL_RELEASES_CACHE_TTL_SECONDS:
logger.info(f"[SDK Health] Returning cached releases for {repo}")
return cached[1]
releases = []
page = 1
while page <= 10: # Github only permits us to list the first 1000 items, so that's 100 items * 10 pages
try:
url = f"https://api.github.com/repos/{repo}/releases?per_page=100&page={page}"
logger.info(f"[SDK Health] Fetching releases from {url}")
# Identity-blind, unauthenticated call — records volume telemetry, no installation budget.
response = github_request("GET", url, source="growth", timeout=10, priority=Priority.BATCH)
if not response.ok:
logger.error(f"[SDK Health] Failed to fetch releases for {repo}", status_code=response.status_code)
return []
releases_json = response.json()
if releases_json is None:
logger.error(f"[SDK Health] Expected list of releases, got empty response", repo=repo)
return []
if not isinstance(releases_json, list):
logger.error(f"[SDK Health] Expected list of releases, got {type(releases_json)}", repo=repo)
return []
if len(releases_json) == 0:
break
releases.extend(releases_json)
page += 1
except Exception as e:
logger.exception(f"[SDK Health] Failed to fetch releases for {repo}", repo=repo)
capture_exception(e, additional_properties={"repo": repo, "page": page, "url": url})
return []
# Only complete fetches are cached; a failed fetch must be retried, not served for 30 minutes
local_releases_cache[repo] = (time.monotonic(), releases)
return releases
def fetch_web_sdk_data() -> dict[str, Any]:
"""Fetch Web SDK data from GitHub releases API"""
# Newer versions in `posthog-js` use a monorepo approach where we prefix tags with `posthog-js@`
# while older versions before the monorepo used simple `v`-prefixed tags
return fetch_sdk_data_from_releases("PostHog/posthog-js", tag_prefixes=["posthog-js@", "v"])
def fetch_python_sdk_data() -> dict[str, Any]:
"""Fetch Python SDK data from GitHub releases API"""
# The repo now tags monorepo-style (`posthog-v7.38.0`), with sibling packages like
# `openfeature-provider-posthog-v*` that must not match; `v`-prefixed and bare tags are historical
return fetch_sdk_data_from_releases(
"PostHog/posthog-python", tag_prefixes=prefixed_or_unprefixed_semver_tags("posthog-v", "v")
)
def fetch_node_sdk_data() -> dict[str, Any]:
"""Fetch Node.js SDK data from GitHub releases API"""
# `posthog-node` was originally developed on the `posthog-js-lite` repo, but was later moved to the `posthog-js` monorepo
# We fetch the latest version from both repos and join them together.
posthog_js = fetch_sdk_data_from_releases("PostHog/posthog-js", tag_prefixes=["posthog-node@"])
posthog_js_lite = fetch_sdk_data_from_releases("PostHog/posthog-js-lite", tag_prefixes=["posthog-node-v"])
# Shouldn't happen, but just in case
if not posthog_js:
return {}
# The latest date is always from `posthog-js` since this is the only active repo,
# so its dates win if a version somehow exists in both repos
return {
"latestVersion": posthog_js["latestVersion"],
"releaseDates": {
**posthog_js_lite.get("releaseDates", {}),
**posthog_js["releaseDates"],
},
}
def fetch_js_lite_sdk_data() -> dict[str, Any]:
posthog_js = fetch_sdk_data_from_releases("PostHog/posthog-js", tag_prefixes=["posthog-js-lite@"])
posthog_js_lite = fetch_sdk_data_from_releases("PostHog/posthog-js-lite", tag_prefixes=["posthog-web-v"])
if not posthog_js or not posthog_js_lite:
return {}
return {
"latestVersion": posthog_js["latestVersion"],
"releaseDates": {
**posthog_js_lite.get("releaseDates", {}),
**posthog_js["releaseDates"],
},
}
def fetch_react_native_sdk_data() -> dict[str, Any]:
"""Fetch React Native SDK data from GitHub releases API"""
# `posthog-react-native` was originally developed on the `posthog-js-lite` repo, but was later moved to the `posthog-js` monorepo
# We fetch the latest version from both repos and join them together.
posthog_js = fetch_sdk_data_from_releases("PostHog/posthog-js", tag_prefixes=["posthog-react-native@"])
posthog_js_lite = fetch_sdk_data_from_releases("PostHog/posthog-js-lite", tag_prefixes=["posthog-react-native-v"])
# Shouldn't happen, but just in case
if not posthog_js:
return {}
# The latest date is always from `posthog-js` since this is the only active repo,
# so its dates win if a version somehow exists in both repos
return {
"latestVersion": posthog_js["latestVersion"],
"releaseDates": {
**posthog_js_lite.get("releaseDates", {}),
**posthog_js["releaseDates"],
},
}
def fetch_flutter_sdk_data() -> dict[str, Any]:
"""Fetch Flutter SDK data from GitHub releases API"""
# First attempt to cut the trailing `v` prefix and then just fallback to the full tag
return fetch_sdk_data_from_releases("PostHog/posthog-flutter", tag_prefixes=["v", ""])
def fetch_ios_sdk_data() -> dict[str, Any]:
"""Fetch iOS SDK data from GitHub releases API"""
return fetch_sdk_data_from_releases("PostHog/posthog-ios")
def fetch_kmp_sdk_data() -> dict[str, Any]:
"""Fetch Kotlin Multiplatform SDK data from GitHub releases API"""
# Early releases were tagged `v0.1.0`; the repo has since moved to bare tags like `0.2.2`
return fetch_sdk_data_from_releases("PostHog/posthog-kmp", tag_prefixes=prefixed_or_unprefixed_semver_tags("v"))
def fetch_android_sdk_data() -> dict[str, Any]:
"""Fetch Android SDK data from GitHub releases API"""
return fetch_sdk_data_from_releases(
"PostHog/posthog-android", tag_prefixes=prefixed_or_unprefixed_semver_tags("android-v")
)
def fetch_java_sdk_data() -> dict[str, Any]:
"""Fetch releases for the archived Java SDK."""
return fetch_sdk_data_from_releases("PostHog/posthog-java", tag_prefixes=[UNPREFIXED_SEMVER_TAG])
def fetch_java_server_sdk_data() -> dict[str, Any]:
"""Fetch Java server SDK data from its dedicated tags in the Android monorepo."""
return fetch_sdk_data_from_releases("PostHog/posthog-android", tag_prefixes=["server-v"])
def fetch_rust_sdk_data() -> dict[str, Any]:
return fetch_sdk_data_from_releases("PostHog/posthog-rs", tag_prefixes=prefixed_or_unprefixed_semver_tags("v"))
def fetch_go_sdk_data() -> dict[str, Any]:
"""Fetch Go SDK data from GitHub releases API"""
return fetch_sdk_data_from_releases("PostHog/posthog-go", tag_prefixes=prefixed_or_unprefixed_semver_tags("v"))
def fetch_php_sdk_data() -> dict[str, Any]:
"""Fetch PHP SDK data from GitHub releases API"""
return fetch_sdk_data_from_releases("PostHog/posthog-php", tag_prefixes=prefixed_or_unprefixed_semver_tags("v"))
def fetch_laravel_sdk_data() -> dict[str, Any]:
"""Use Laravel's independent source tags, not PHP or distribution release tags."""
return fetch_sdk_data_from_releases("PostHog/posthog-php", tag_prefixes=["laravel-v"])
def fetch_ruby_sdk_data() -> dict[str, Any]:
"""Fetch Ruby SDK data from GitHub releases API"""
# The repo now tags monorepo-style, releasing both `posthog-ruby-v*` and `posthog-rails-v*`;
# only the former is this SDK. `v`-prefixed and bare tags are historical
return fetch_sdk_data_from_releases(
"PostHog/posthog-ruby", tag_prefixes=prefixed_or_unprefixed_semver_tags("posthog-ruby-v", "v")
)
def fetch_elixir_sdk_data() -> dict[str, Any]:
"""Fetch Elixir SDK data from GitHub releases API"""
return fetch_sdk_data_from_releases("PostHog/posthog-elixir", tag_prefixes=prefixed_or_unprefixed_semver_tags("v"))
def fetch_dotnet_sdk_data() -> dict[str, Any]:
"""Fetch .NET SDK data from GitHub releases API"""
# The repo now tags monorepo-style (`PostHog-v2.13.0`), with sibling packages
# `PostHog.AspNetCore-v*` and `PostHog.AI-v*` that must not match; `v`-prefixed and
# bare tags are historical
return fetch_sdk_data_from_releases(
"PostHog/posthog-dotnet", tag_prefixes=prefixed_or_unprefixed_semver_tags("PostHog-v", "v")
)
def fetch_roblox_sdk_data() -> dict[str, Any]:
"""Fetch Roblox SDK data from GitHub releases API"""
return fetch_sdk_data_from_releases("PostHog/posthog-roblox", tag_prefixes=prefixed_or_unprefixed_semver_tags("v"))
# ---- Dagster defs
retry_policy = dagster.RetryPolicy(
max_retries=3,
delay=1,
backoff=dagster.Backoff.EXPONENTIAL,
jitter=dagster.Jitter.FULL,
)
@dagster.op(retry_policy=retry_policy)
def fetch_github_sdk_versions_op(context: dagster.OpExecutionContext) -> dict[str, Optional[dict[str, Any]]]:
"""Fetch GitHub SDK version data for all SDK types."""
sdk_data = {}
fetched_count = 0
failed_count = 0
for lib_name in SDK_TYPES:
try:
context.log.info(f"Fetching {lib_name} SDK data from GitHub")
github_data = fetch_github_data_for_sdk(lib_name)
if github_data:
sdk_data[lib_name] = github_data
fetched_count += 1
context.log.info(f"Successfully fetched {lib_name} SDK data")
else:
failed_count += 1
context.log.warning(f"No data received from GitHub for {lib_name}")
except Exception as e:
failed_count += 1
context.log.exception(f"Failed to fetch {lib_name} SDK data")
capture_exception(e)
context.log.info(f"Fetched {fetched_count} SDK versions")
context.log.info(f"Failed to fetch {failed_count} SDK versions")
context.log.info(f"Total SDKs: {len(SDK_TYPES)}")
context.log.info(f"SDK data: {sdk_data}")
context.add_output_metadata(
{
"fetched_count": dagster.MetadataValue.int(fetched_count),
"failed_count": dagster.MetadataValue.int(failed_count),
"total_sdks": dagster.MetadataValue.int(len(SDK_TYPES)),
}
)
return sdk_data # type: ignore
@dagster.op(retry_policy=retry_policy)
def cache_github_sdk_versions_op(
context: dagster.OpExecutionContext,
sdk_data: dict[str, Optional[dict[str, Any]]],
redis_client: dagster.ResourceParam[redis.Redis],
) -> None:
"""Cache GitHub SDK version data to Redis."""
cached_count = 0
skipped_count = 0
for lib_name, github_data in sdk_data.items():
if github_data is None:
skipped_count += 1
continue
cache_key = github_sdk_versions_key(lib_name)
try:
redis_client.setex(cache_key, SDK_CACHE_EXPIRY, json.dumps(github_data))
cached_count += 1
context.log.info(f"Successfully cached {lib_name} SDK data")
except Exception as e:
context.log.exception(f"Failed to cache {lib_name} SDK data")
capture_exception(e)
context.log.info(f"Cached {cached_count} SDK versions")
context.log.info(f"Skipped {skipped_count} SDK versions")
context.log.info(f"Total SDKs: {len(sdk_data)}")
context.log.info(f"SDK data: {sdk_data}")
context.add_output_metadata(
{
"cached_count": dagster.MetadataValue.int(cached_count),
"skipped_count": dagster.MetadataValue.int(skipped_count),
"total_sdks": dagster.MetadataValue.int(len(sdk_data)),
}
)
@dagster.job(
description="Queries GitHub for most recent SDK versions and caches them in Redis",
tags={"owner": JobOwners.TEAM_GROWTH.value},
)
def cache_github_sdk_versions_job():
sdk_data = fetch_github_sdk_versions_op()
cache_github_sdk_versions_op(sdk_data)
cache_github_sdk_versions_schedule = dagster.ScheduleDefinition(
job=cache_github_sdk_versions_job,
cron_schedule="30 * * * *", # Every hour at half past the hour
execution_timezone="UTC",
name="cache_github_sdk_versions_schedule",
)