:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replicate): add model versions, collections and files tables (#127356)
Tom Owers25c98cfce0ba
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex e106358f8750..1ffcf812c1c2 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -897,7 +897,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/rentcast.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py1 products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/reply_io/reply_io.py1 products/warehouse_sources/backend/temporal/data_imports/sources/reply_io/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 0f968278559a..97c77ed33b46 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10220,9 +10220,9 @@ Today (6): `account`, `deployments`, `hardware`, `models`, `predictions`, `trainDiffed against: <https://api.replicate.com/openapi.json>-- [ ] `/models/{model_owner}/{model_name}/versions` — lookup resolving the version ID stamped on every prediction and training row already synced (high)-- [ ] `/collections and /collections/{collection_slug}` — the vendor's own model taxonomy - gives a category dimension for grouping model and prediction usage (medium)-- [ ] `/files` — uploaded input/output file records with size and expiry, joinable to predictions (low)+- [x] `/models/{model_owner}/{model_name}/versions` — lookup resolving the version ID stamped on every prediction and training row already synced (high) → `model_versions` (full refresh, fan-out over models referenced by `deployments` and `trainings`)+- [x] `/collections and /collections/{collection_slug}` — the vendor's own model taxonomy - gives a category dimension for grouping model and prediction usage (medium) → `collections` and `collection_models` (full refresh, fan-out over `collections`)+- [x] `/files` — uploaded input/output file records with size and expiry, joinable to predictions (low) → `files` (full refresh)## ReplyIo — gapsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.pyindex d5880843af39..8fd6cb49c8de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.py@@ -77,6 +77,55 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"sku": "The hardware SKU identifier (e.g. gpu-t4).",},},+ "model_versions": {+ "description": "A version of a model that the account deploys or trains. Join to predictions and trainings on version.",+ "docs_url": "https://replicate.com/docs/reference/http#models.versions.list",+ "columns": {+ "id": "The ID of the version.",+ "model_owner": "The account that owns the model.",+ "model_name": "The model's name.",+ "created_at": "The date and time the version was created.",+ "cog_version": "The version of Cog used to create this version.",+ "openapi_schema": "The OpenAPI schema for the model's inputs and outputs.",+ },+ },+ "collections": {+ "description": "A curated collection of models on Replicate, such as super resolution or image generation.",+ "docs_url": "https://replicate.com/docs/reference/http#collections.list",+ "columns": {+ "slug": "The slug of the collection (lowercase with dashes).",+ "name": "The name of the collection.",+ "description": "A description of the collection.",+ },+ },+ "collection_models": {+ "description": "A model that belongs to a Replicate collection. One row per collection and model.",+ "docs_url": "https://replicate.com/docs/reference/http#collections.get",+ "columns": {+ "collection_slug": "The slug of the collection the model belongs to.",+ "url": "The web URL of the model.",+ "owner": "The account that owns the model.",+ "name": "The model's name.",+ "description": "A description of what the model does.",+ "visibility": "Whether the model is public or private.",+ "run_count": "The number of times the model has been run.",+ "latest_version": "The most recent version of the model.",+ },+ },+ "files": {+ "description": "A file uploaded to Replicate by the account, for example as a prediction input.",+ "docs_url": "https://replicate.com/docs/reference/http#files.list",+ "columns": {+ "id": "A unique, randomly generated identifier for the file.",+ "content_type": "The content (MIME) type of the file.",+ "size": "The length of the file in bytes.",+ "checksums": "Checksums for the file, keyed by algorithm name.",+ "metadata": "Metadata provided by the user when the file was created.",+ "created_at": "When the file was created.",+ "expires_at": "When the file expires.",+ "urls": "URLs associated with the file.",+ },+ },"account": {"description": "The authenticated Replicate account.","docs_url": "https://replicate.com/docs/reference/http#accounts.get",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.pyindex 388319fb6317..4e179bee0373 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py@@ -2,12 +2,14 @@ import dataclassesfrom collections.abc import Iteratorfrom datetime import UTC, date, datetimefrom typing import Any, Optional-from urllib.parse import urlencode, urlsplit, urlunsplit+from urllib.parse import quote, urlencode, urlsplit, urlunsplitimport requestsfrom structlog.types import FilteringBoundLoggerfrom tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (make_tracked_session,retry_with_deadline,@@ -27,7 +29,7 @@ class ReplicateRetryableError(Exception):pass[email protected](frozen=True)class ReplicateResumeConfig:# Full next-page URL returned by the API (carries the opaque cursor and any created_after filter).# None means "start the endpoint from its first page".@@ -35,6 +37,8 @@ class ReplicateResumeConfig:# The `created_after` watermark this cursor was built against. A run whose watermark has since# advanced must not resume from this cursor (see `get_rows`), so we can tell the two apart.created_after: str | None = None+ # Fan-out endpoints only: the parent (`owner/name` model or collection slug) the cursor belongs to.+ fan_out_parent: str | None = Nonedef _get_headers(api_key: str) -> dict[str, str]:@@ -175,6 +179,143 @@ def _page_predates_cutoff(items: list[dict[str, Any]], incremental_field: str, creturn max(dated) <= cutoff+def _walk_pages(+ session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger+) -> Iterator[tuple[list[dict[str, Any]], str | None]]:+ """Yield each page's rows with the URL of the page after it."""+ next_url: str | None = url+ while next_url:+ try:+ data = _fetch_page(session, next_url, headers, logger)+ except requests.HTTPError as e:+ # Same 416-as-empty-page handling as the main pagination loop in `get_rows`.+ if e.response is not None and e.response.status_code == 416:+ return+ raise+ next_url = _next_url(data)+ yield (data.get("results", []) if isinstance(data, dict) else []), next_url+++def _is_skippable_child_error(error: requests.HTTPError) -> bool:+ # A referenced model can be deleted or made private after a deployment or training used it.+ return error.response is not None and error.response.status_code in (403, 404)+++@frozen(order=True)+class ModelRef:+ owner: str+ name: str+++def _parse_model_ref(value: Any) -> ModelRef | None:+ """Parse `owner/name` or `owner/name:version_id` into a model reference."""+ if not isinstance(value, str):+ return None+ owner, sep, name = value.split(":", 1)[0].partition("/")+ if not sep or not owner or not name or "/" in name:+ return None+ return ModelRef(owner=owner, name=name)+++def _referenced_models(row: dict[str, Any]) -> list[ModelRef]:+ release = row.get("current_release")+ output = row.get("output")+ candidates = [+ release.get("model") if isinstance(release, dict) else None,+ row.get("model"),+ # A finished training's output.version is the destination version, as owner/name:version_id.+ output.get("version") if isinstance(output, dict) else None,+ ]+ return [ref for ref in map(_parse_model_ref, candidates) if ref is not None]+++def _get_model_version_rows(+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ models: set[ModelRef] = set()+ for parent_path in ("/deployments", "/trainings"):+ for page, _ in _walk_pages(session, f"{REPLICATE_BASE_URL}{parent_path}", headers, logger):+ for item in page:+ models.update(_referenced_models(item))+ resumable_source_manager.safe_point()++ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ resume_parent = resume.fan_out_parent if resume is not None else None+ resume_url = _sanitize_next_url(resume.next_url) if resume is not None else None++ # Sorted, so a resumed run can skip every model that sorts before the saved one.+ for model in sorted(models):+ owner, name = model.owner, model.name+ parent = f"{owner}/{name}"+ url = f"{REPLICATE_BASE_URL}/models/{quote(owner, safe='')}/{quote(name, safe='')}/versions"+ if resume_parent is not None and parent <= resume_parent:+ if parent < resume_parent or resume_url is None:+ continue+ url = resume_url+ try:+ for page, next_url in _walk_pages(session, url, headers, logger):+ if not page:+ resumable_source_manager.safe_point()+ continue+ resumable_source_manager.save_state(ReplicateResumeConfig(next_url=next_url, fan_out_parent=parent))+ yield [{**version, "model_owner": owner, "model_name": name} for version in page]+ except requests.HTTPError as e:+ if not _is_skippable_child_error(e):+ raise+ logger.info(f"Replicate: skipping versions of {owner}/{name}: {e}")+ resumable_source_manager.safe_point()+++def _get_collection_model_rows(+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ resumed_url = _sanitize_next_url(resume.next_url) if resume is not None else None+ page_url = resumed_url or f"{REPLICATE_BASE_URL}/collections"+ # The cursor names the collections page and the last slug on it whose models were yielded.+ skip_through = resume.fan_out_parent if resume is not None else None++ for page, next_url in _walk_pages(session, page_url, headers, logger):+ slugs = [c["slug"] for c in page if isinstance(c.get("slug"), str) and c["slug"]]+ if skip_through in slugs:+ slugs = slugs[slugs.index(skip_through) + 1 :]+ skip_through = None+ for slug in slugs:+ try:+ detail = _fetch_page(+ session, f"{REPLICATE_BASE_URL}/collections/{quote(slug, safe='')}", headers, logger+ )+ except requests.HTTPError as e:+ if not _is_skippable_child_error(e):+ raise+ logger.info(f"Replicate: skipping collection {slug}: {e}")+ resumable_source_manager.safe_point()+ continue+ models = detail.get("models") if isinstance(detail, dict) else None+ rows = [{**model, "collection_slug": slug} for model in models or [] if isinstance(model, dict)]+ if rows:+ resumable_source_manager.save_state(ReplicateResumeConfig(next_url=page_url, fan_out_parent=slug))+ yield rows+ else:+ resumable_source_manager.safe_point()+ resumable_source_manager.safe_point()+ if next_url is None:+ break+ page_url = next_url+++_FAN_OUT_ITERATORS = {+ "model_versions": _get_model_version_rows,+ "collection_models": _get_collection_model_rows,+}++def get_rows(api_key: str,endpoint: str,@@ -191,6 +332,11 @@ def get_rows(# user-authored model inputs, outputs, and logs that the generic scrubber can't reliably sanitize.session = make_tracked_session(redact_values=(api_key,), capture=False)+ fan_out = _FAN_OUT_ITERATORS.get(endpoint)+ if fan_out is not None:+ yield from fan_out(session, headers, logger, resumable_source_manager)+ return+# Single-request endpoints (bare array / single object) have no pagination or resume state.if config.response_shape != "paginated":data = _fetch_page(session, f"{REPLICATE_BASE_URL}{config.path}", headers, logger)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.pyindex cb5498c17fe2..b04daaf90478 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py@@ -85,6 +85,33 @@ REPLICATE_ENDPOINTS: dict[str, ReplicateEndpointConfig] = {response_shape="object",primary_keys=["username"],),+ # Versions of the models the account deploys or trains (base and destination models). Fanning out+ # over the full public catalog would cost one request series per public model, so the parents+ # come from deployments and trainings instead. Full refresh.+ "model_versions": ReplicateEndpointConfig(+ name="model_versions",+ path="/models/{model_owner}/{model_name}/versions",+ primary_keys=["model_owner", "model_name", "id"],+ partition_key="created_at",+ ),+ # Replicate's curated model collections. Small, no timestamp filter — full refresh.+ "collections": ReplicateEndpointConfig(+ name="collections",+ path="/collections",+ primary_keys=["slug"],+ ),+ # One row per model in each collection, from a fan-out over /collections/{collection_slug}.+ "collection_models": ReplicateEndpointConfig(+ name="collection_models",+ path="/collections/{collection_slug}",+ primary_keys=["collection_slug", "owner", "name"],+ ),+ # Files the account uploaded. No timestamp filter — full refresh.+ "files": ReplicateEndpointConfig(+ name="files",+ path="/files",+ partition_key="created_at",+ ),}ENDPOINTS = tuple(REPLICATE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.pyindex 950fcf0ac84b..7885816f72b7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.py@@ -110,6 +110,8 @@ You can create an API token in your [Replicate account settings](https://replica)if endpoint == "models":return "The full public Replicate model catalog, not only your own models. Off by default."+ if endpoint == "model_versions":+ return "Versions of the models your deployments and trainings use, including fine-tuned destination models."return Nonedef _build_schema(endpoint: str) -> SourceSchema:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.pyindex 85b69eb6c1e9..d659972d3d71 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.py@@ -36,6 +36,9 @@ class _FakeResumableManager:def save_state(self, data: ReplicateResumeConfig) -> None:self.saved.append(data)+ def safe_point(self) -> None:+ pass+class TestFormatIncrementalValue:@parameterized.expand(@@ -276,6 +279,146 @@ class TestGetRows:assert manager.saved == []+def _http_error(status: int, url: str) -> requests.HTTPError:+ response = requests.Response()+ response.status_code = status+ response.url = url+ return requests.HTTPError(f"{status} Client Error for url: {url}", response=response)+++class TestFanOut:+ def test_model_versions_fan_out_over_deployed_and_trained_models(self, monkeypatch: Any) -> None:+ base = "https://api.replicate.com/v1"+ trainings_page2 = f"{base}/trainings?cursor=t2"+ sdxl_page2 = f"{base}/models/stability-ai/sdxl/versions?cursor=v2"+ gone = f"{base}/models/acme/deleted-model/versions"+ pages: dict[str, Any] = {+ f"{base}/deployments": {+ "results": [{"owner": "acme", "name": "prod", "current_release": {"model": "acme/my-model"}}],+ "next": None,+ },+ f"{base}/trainings": {+ "results": [+ {"id": "t1", "model": "stability-ai/sdxl", "output": {"version": "acme/my-model:abc"}},+ {"id": "t2", "model": "acme/deleted-model", "output": None},+ ],+ "next": trainings_page2,+ },+ trainings_page2: {"results": [{"id": "t3", "model": "stability-ai/sdxl", "output": {}}], "next": None},+ f"{base}/models/acme/my-model/versions": {"results": [{"id": "abc"}], "next": None},+ f"{base}/models/stability-ai/sdxl/versions": {"results": [{"id": "v1"}], "next": sdxl_page2},+ sdxl_page2: {"results": [{"id": "v0"}], "next": None},+ }+ fetched: list[str] = []++ def fake_fetch(session: Any, url: str, headers: dict[str, str], logger: Any) -> Any:+ fetched.append(url)+ if url == gone:+ raise _http_error(404, url)+ return pages[url]++ monkeypatch.setattr(replicate, "_fetch_page", fake_fetch)+ rows = [+ row+ for page in get_rows(+ api_key="r8_test",+ endpoint="model_versions",+ logger=MagicMock(),+ resumable_source_manager=_FakeResumableManager(), # type: ignore[arg-type]+ )+ for row in page+ ]++ assert rows == [+ {"id": "abc", "model_owner": "acme", "model_name": "my-model"},+ {"id": "v1", "model_owner": "stability-ai", "model_name": "sdxl"},+ {"id": "v0", "model_owner": "stability-ai", "model_name": "sdxl"},+ ]+ # Each referenced model is fetched once, however many deployments or trainings use it.+ assert fetched.count(f"{base}/models/stability-ai/sdxl/versions") == 1+ assert gone in fetched++ def test_model_versions_raise_non_skippable_child_errors(self, monkeypatch: Any) -> None:+ base = "https://api.replicate.com/v1"+ pages: dict[str, Any] = {+ f"{base}/deployments": {"results": [], "next": None},+ f"{base}/trainings": {"results": [{"id": "t1", "model": "acme/my-model"}], "next": None},+ }++ def fake_fetch(session: Any, url: str, headers: dict[str, str], logger: Any) -> Any:+ if url in pages:+ return pages[url]+ raise _http_error(401, url)++ monkeypatch.setattr(replicate, "_fetch_page", fake_fetch)+ with pytest.raises(requests.HTTPError):+ list(+ get_rows(+ api_key="r8_test",+ endpoint="model_versions",+ logger=MagicMock(),+ resumable_source_manager=_FakeResumableManager(), # type: ignore[arg-type]+ )+ )++ def test_collection_models_fan_out_tags_each_model_with_its_collection(self, monkeypatch: Any) -> None:+ base = "https://api.replicate.com/v1"+ collections_page2 = f"{base}/collections?cursor=c2"+ pages = {+ f"{base}/collections": {"results": [{"slug": "super-resolution"}], "next": collections_page2},+ collections_page2: {"results": [{"slug": "empty"}, {"slug": "image-to-text"}], "next": None},+ f"{base}/collections/super-resolution": {+ "slug": "super-resolution",+ "models": [{"owner": "acme", "name": "upscaler"}, {"owner": "acme", "name": "esrgan"}],+ },+ f"{base}/collections/empty": {"slug": "empty", "models": []},+ f"{base}/collections/image-to-text": {+ "slug": "image-to-text",+ "models": [{"owner": "acme", "name": "upscaler"}],+ },+ }+ rows, _ = _collect(_FakeResumableManager(), monkeypatch, pages, "collection_models")++ assert rows == [+ {"owner": "acme", "name": "upscaler", "collection_slug": "super-resolution"},+ {"owner": "acme", "name": "esrgan", "collection_slug": "super-resolution"},+ {"owner": "acme", "name": "upscaler", "collection_slug": "image-to-text"},+ ]++ def test_model_versions_resume_from_the_saved_model_and_page(self, monkeypatch: Any) -> None:+ base = "https://api.replicate.com/v1"+ resume_url = f"{base}/models/acme/b-model/versions?cursor=v2"+ pages: dict[str, Any] = {+ f"{base}/deployments": {"results": [], "next": None},+ f"{base}/trainings": {+ "results": [{"model": "acme/c-model"}, {"model": "acme/b-model"}, {"model": "acme/a-model"}],+ "next": None,+ },+ resume_url: {"results": [{"id": "b1"}], "next": None},+ f"{base}/models/acme/c-model/versions": {"results": [{"id": "c1"}], "next": None},+ }+ manager = _FakeResumableManager(ReplicateResumeConfig(next_url=resume_url, fan_out_parent="acme/b-model"))+ rows, fetched = _collect(manager, monkeypatch, pages, "model_versions")++ assert [r["id"] for r in rows] == ["b1", "c1"]+ assert f"{base}/models/acme/a-model/versions" not in fetched+ assert manager.saved[-1] == ReplicateResumeConfig(next_url=None, fan_out_parent="acme/c-model")++ def test_collection_models_resume_after_the_saved_collection(self, monkeypatch: Any) -> None:+ base = "https://api.replicate.com/v1"+ page2 = f"{base}/collections?cursor=c2"+ pages = {+ page2: {"results": [{"slug": "done"}, {"slug": "next"}], "next": None},+ f"{base}/collections/next": {"slug": "next", "models": [{"owner": "acme", "name": "m"}]},+ }+ manager = _FakeResumableManager(ReplicateResumeConfig(next_url=page2, fan_out_parent="done"))+ rows, fetched = _collect(manager, monkeypatch, pages, "collection_models")++ assert rows == [{"owner": "acme", "name": "m", "collection_slug": "next"}]+ assert fetched == [page2, f"{base}/collections/next"]+ assert manager.saved == [ReplicateResumeConfig(next_url=page2, fan_out_parent="next")]++class TestFetchPageRetries:@parameterized.expand([("rate_limited", 429), ("server_error", 503)])def test_retryable_status_codes_retry_then_succeed(self, _name: str, status: int) -> None: