Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replicate): add model versions, collections and files tables (#127356)

Tom Owers25c98cfce0ba
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index e106358f8750..1ffcf812c1c2 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -897,7 +897,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/rentcast.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/reply_io/reply_io.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/reply_io/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 0f968278559a..97c77ed33b46 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10220,9 +10220,9 @@ Today (6): `account`, `deployments`, `hardware`, `models`, `predictions`, `train
Diffed against: <https://api.replicate.com/openapi.json>
-- [ ] `/models/{model_owner}/{model_name}/versions` — lookup resolving the version ID stamped on every prediction and training row already synced (high)
-- [ ] `/collections and /collections/{collection_slug}` — the vendor's own model taxonomy - gives a category dimension for grouping model and prediction usage (medium)
-- [ ] `/files` — uploaded input/output file records with size and expiry, joinable to predictions (low)
+- [x] `/models/{model_owner}/{model_name}/versions` — lookup resolving the version ID stamped on every prediction and training row already synced (high) → `model_versions` (full refresh, fan-out over models referenced by `deployments` and `trainings`)
+- [x] `/collections and /collections/{collection_slug}` — the vendor's own model taxonomy - gives a category dimension for grouping model and prediction usage (medium) → `collections` and `collection_models` (full refresh, fan-out over `collections`)
+- [x] `/files` — uploaded input/output file records with size and expiry, joinable to predictions (low) → `files` (full refresh)
## ReplyIo — gaps
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.py
index d5880843af39..8fd6cb49c8de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/canonical_descriptions.py
@@ -77,6 +77,55 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"sku": "The hardware SKU identifier (e.g. gpu-t4).",
},
},
+ "model_versions": {
+ "description": "A version of a model that the account deploys or trains. Join to predictions and trainings on version.",
+ "docs_url": "https://replicate.com/docs/reference/http#models.versions.list",
+ "columns": {
+ "id": "The ID of the version.",
+ "model_owner": "The account that owns the model.",
+ "model_name": "The model's name.",
+ "created_at": "The date and time the version was created.",
+ "cog_version": "The version of Cog used to create this version.",
+ "openapi_schema": "The OpenAPI schema for the model's inputs and outputs.",
+ },
+ },
+ "collections": {
+ "description": "A curated collection of models on Replicate, such as super resolution or image generation.",
+ "docs_url": "https://replicate.com/docs/reference/http#collections.list",
+ "columns": {
+ "slug": "The slug of the collection (lowercase with dashes).",
+ "name": "The name of the collection.",
+ "description": "A description of the collection.",
+ },
+ },
+ "collection_models": {
+ "description": "A model that belongs to a Replicate collection. One row per collection and model.",
+ "docs_url": "https://replicate.com/docs/reference/http#collections.get",
+ "columns": {
+ "collection_slug": "The slug of the collection the model belongs to.",
+ "url": "The web URL of the model.",
+ "owner": "The account that owns the model.",
+ "name": "The model's name.",
+ "description": "A description of what the model does.",
+ "visibility": "Whether the model is public or private.",
+ "run_count": "The number of times the model has been run.",
+ "latest_version": "The most recent version of the model.",
+ },
+ },
+ "files": {
+ "description": "A file uploaded to Replicate by the account, for example as a prediction input.",
+ "docs_url": "https://replicate.com/docs/reference/http#files.list",
+ "columns": {
+ "id": "A unique, randomly generated identifier for the file.",
+ "content_type": "The content (MIME) type of the file.",
+ "size": "The length of the file in bytes.",
+ "checksums": "Checksums for the file, keyed by algorithm name.",
+ "metadata": "Metadata provided by the user when the file was created.",
+ "created_at": "When the file was created.",
+ "expires_at": "When the file expires.",
+ "urls": "URLs associated with the file.",
+ },
+ },
"account": {
"description": "The authenticated Replicate account.",
"docs_url": "https://replicate.com/docs/reference/http#accounts.get",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
index 388319fb6317..4e179bee0373 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
@@ -2,12 +2,14 @@ import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
-from urllib.parse import urlencode, urlsplit, urlunsplit
+from urllib.parse import quote, urlencode, urlsplit, urlunsplit
import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
make_tracked_session,
retry_with_deadline,
@@ -27,7 +29,7 @@ class ReplicateRetryableError(Exception):
pass
[email protected](frozen=True)
class ReplicateResumeConfig:
# Full next-page URL returned by the API (carries the opaque cursor and any created_after filter).
# None means "start the endpoint from its first page".
@@ -35,6 +37,8 @@ class ReplicateResumeConfig:
# The `created_after` watermark this cursor was built against. A run whose watermark has since
# advanced must not resume from this cursor (see `get_rows`), so we can tell the two apart.
created_after: str | None = None
+ # Fan-out endpoints only: the parent (`owner/name` model or collection slug) the cursor belongs to.
+ fan_out_parent: str | None = None
def _get_headers(api_key: str) -> dict[str, str]:
@@ -175,6 +179,143 @@ def _page_predates_cutoff(items: list[dict[str, Any]], incremental_field: str, c
return max(dated) <= cutoff
+def _walk_pages(
+ session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger
+) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
+ """Yield each page's rows with the URL of the page after it."""
+ next_url: str | None = url
+ while next_url:
+ try:
+ data = _fetch_page(session, next_url, headers, logger)
+ except requests.HTTPError as e:
+ # Same 416-as-empty-page handling as the main pagination loop in `get_rows`.
+ if e.response is not None and e.response.status_code == 416:
+ return
+ raise
+ next_url = _next_url(data)
+ yield (data.get("results", []) if isinstance(data, dict) else []), next_url
+
+
+def _is_skippable_child_error(error: requests.HTTPError) -> bool:
+ # A referenced model can be deleted or made private after a deployment or training used it.
+ return error.response is not None and error.response.status_code in (403, 404)
+
+
+@frozen(order=True)
+class ModelRef:
+ owner: str
+ name: str
+
+
+def _parse_model_ref(value: Any) -> ModelRef | None:
+ """Parse `owner/name` or `owner/name:version_id` into a model reference."""
+ if not isinstance(value, str):
+ return None
+ owner, sep, name = value.split(":", 1)[0].partition("/")
+ if not sep or not owner or not name or "/" in name:
+ return None
+ return ModelRef(owner=owner, name=name)
+
+
+def _referenced_models(row: dict[str, Any]) -> list[ModelRef]:
+ release = row.get("current_release")
+ output = row.get("output")
+ candidates = [
+ release.get("model") if isinstance(release, dict) else None,
+ row.get("model"),
+ # A finished training's output.version is the destination version, as owner/name:version_id.
+ output.get("version") if isinstance(output, dict) else None,
+ ]
+ return [ref for ref in map(_parse_model_ref, candidates) if ref is not None]
+
+
+def _get_model_version_rows(
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ models: set[ModelRef] = set()
+ for parent_path in ("/deployments", "/trainings"):
+ for page, _ in _walk_pages(session, f"{REPLICATE_BASE_URL}{parent_path}", headers, logger):
+ for item in page:
+ models.update(_referenced_models(item))
+ resumable_source_manager.safe_point()
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ resume_parent = resume.fan_out_parent if resume is not None else None
+ resume_url = _sanitize_next_url(resume.next_url) if resume is not None else None
+
+ # Sorted, so a resumed run can skip every model that sorts before the saved one.
+ for model in sorted(models):
+ owner, name = model.owner, model.name
+ parent = f"{owner}/{name}"
+ url = f"{REPLICATE_BASE_URL}/models/{quote(owner, safe='')}/{quote(name, safe='')}/versions"
+ if resume_parent is not None and parent <= resume_parent:
+ if parent < resume_parent or resume_url is None:
+ continue
+ url = resume_url
+ try:
+ for page, next_url in _walk_pages(session, url, headers, logger):
+ if not page:
+ resumable_source_manager.safe_point()
+ continue
+ resumable_source_manager.save_state(ReplicateResumeConfig(next_url=next_url, fan_out_parent=parent))
+ yield [{**version, "model_owner": owner, "model_name": name} for version in page]
+ except requests.HTTPError as e:
+ if not _is_skippable_child_error(e):
+ raise
+ logger.info(f"Replicate: skipping versions of {owner}/{name}: {e}")
+ resumable_source_manager.safe_point()
+
+
+def _get_collection_model_rows(
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ resumed_url = _sanitize_next_url(resume.next_url) if resume is not None else None
+ page_url = resumed_url or f"{REPLICATE_BASE_URL}/collections"
+ # The cursor names the collections page and the last slug on it whose models were yielded.
+ skip_through = resume.fan_out_parent if resume is not None else None
+
+ for page, next_url in _walk_pages(session, page_url, headers, logger):
+ slugs = [c["slug"] for c in page if isinstance(c.get("slug"), str) and c["slug"]]
+ if skip_through in slugs:
+ slugs = slugs[slugs.index(skip_through) + 1 :]
+ skip_through = None
+ for slug in slugs:
+ try:
+ detail = _fetch_page(
+ session, f"{REPLICATE_BASE_URL}/collections/{quote(slug, safe='')}", headers, logger
+ )
+ except requests.HTTPError as e:
+ if not _is_skippable_child_error(e):
+ raise
+ logger.info(f"Replicate: skipping collection {slug}: {e}")
+ resumable_source_manager.safe_point()
+ continue
+ models = detail.get("models") if isinstance(detail, dict) else None
+ rows = [{**model, "collection_slug": slug} for model in models or [] if isinstance(model, dict)]
+ if rows:
+ resumable_source_manager.save_state(ReplicateResumeConfig(next_url=page_url, fan_out_parent=slug))
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+ resumable_source_manager.safe_point()
+ if next_url is None:
+ break
+ page_url = next_url
+
+
+_FAN_OUT_ITERATORS = {
+ "model_versions": _get_model_version_rows,
+ "collection_models": _get_collection_model_rows,
+}
+
+
def get_rows(
api_key: str,
endpoint: str,
@@ -191,6 +332,11 @@ def get_rows(
# user-authored model inputs, outputs, and logs that the generic scrubber can't reliably sanitize.
session = make_tracked_session(redact_values=(api_key,), capture=False)
+ fan_out = _FAN_OUT_ITERATORS.get(endpoint)
+ if fan_out is not None:
+ yield from fan_out(session, headers, logger, resumable_source_manager)
+ return
+
# Single-request endpoints (bare array / single object) have no pagination or resume state.
if config.response_shape != "paginated":
data = _fetch_page(session, f"{REPLICATE_BASE_URL}{config.path}", headers, logger)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py
index cb5498c17fe2..b04daaf90478 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/settings.py
@@ -85,6 +85,33 @@ REPLICATE_ENDPOINTS: dict[str, ReplicateEndpointConfig] = {
response_shape="object",
primary_keys=["username"],
),
+ # Versions of the models the account deploys or trains (base and destination models). Fanning out
+ # over the full public catalog would cost one request series per public model, so the parents
+ # come from deployments and trainings instead. Full refresh.
+ "model_versions": ReplicateEndpointConfig(
+ name="model_versions",
+ path="/models/{model_owner}/{model_name}/versions",
+ primary_keys=["model_owner", "model_name", "id"],
+ partition_key="created_at",
+ ),
+ # Replicate's curated model collections. Small, no timestamp filter — full refresh.
+ "collections": ReplicateEndpointConfig(
+ name="collections",
+ path="/collections",
+ primary_keys=["slug"],
+ ),
+ # One row per model in each collection, from a fan-out over /collections/{collection_slug}.
+ "collection_models": ReplicateEndpointConfig(
+ name="collection_models",
+ path="/collections/{collection_slug}",
+ primary_keys=["collection_slug", "owner", "name"],
+ ),
+ # Files the account uploaded. No timestamp filter — full refresh.
+ "files": ReplicateEndpointConfig(
+ name="files",
+ path="/files",
+ partition_key="created_at",
+ ),
}
ENDPOINTS = tuple(REPLICATE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.py
index 950fcf0ac84b..7885816f72b7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/source.py
@@ -110,6 +110,8 @@ You can create an API token in your [Replicate account settings](https://replica
)
if endpoint == "models":
return "The full public Replicate model catalog, not only your own models. Off by default."
+ if endpoint == "model_versions":
+ return "Versions of the models your deployments and trainings use, including fine-tuned destination models."
return None
def _build_schema(endpoint: str) -> SourceSchema:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.py
index 85b69eb6c1e9..d659972d3d71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/tests/test_replicate.py
@@ -36,6 +36,9 @@ class _FakeResumableManager:
def save_state(self, data: ReplicateResumeConfig) -> None:
self.saved.append(data)
+ def safe_point(self) -> None:
+ pass
+
class TestFormatIncrementalValue:
@parameterized.expand(
@@ -276,6 +279,146 @@ class TestGetRows:
assert manager.saved == []
+def _http_error(status: int, url: str) -> requests.HTTPError:
+ response = requests.Response()
+ response.status_code = status
+ response.url = url
+ return requests.HTTPError(f"{status} Client Error for url: {url}", response=response)
+
+
+class TestFanOut:
+ def test_model_versions_fan_out_over_deployed_and_trained_models(self, monkeypatch: Any) -> None:
+ base = "https://api.replicate.com/v1"
+ trainings_page2 = f"{base}/trainings?cursor=t2"
+ sdxl_page2 = f"{base}/models/stability-ai/sdxl/versions?cursor=v2"
+ gone = f"{base}/models/acme/deleted-model/versions"
+ pages: dict[str, Any] = {
+ f"{base}/deployments": {
+ "results": [{"owner": "acme", "name": "prod", "current_release": {"model": "acme/my-model"}}],
+ "next": None,
+ },
+ f"{base}/trainings": {
+ "results": [
+ {"id": "t1", "model": "stability-ai/sdxl", "output": {"version": "acme/my-model:abc"}},
+ {"id": "t2", "model": "acme/deleted-model", "output": None},
+ ],
+ "next": trainings_page2,
+ },
+ trainings_page2: {"results": [{"id": "t3", "model": "stability-ai/sdxl", "output": {}}], "next": None},
+ f"{base}/models/acme/my-model/versions": {"results": [{"id": "abc"}], "next": None},
+ f"{base}/models/stability-ai/sdxl/versions": {"results": [{"id": "v1"}], "next": sdxl_page2},
+ sdxl_page2: {"results": [{"id": "v0"}], "next": None},
+ }
+ fetched: list[str] = []
+
+ def fake_fetch(session: Any, url: str, headers: dict[str, str], logger: Any) -> Any:
+ fetched.append(url)
+ if url == gone:
+ raise _http_error(404, url)
+ return pages[url]
+
+ monkeypatch.setattr(replicate, "_fetch_page", fake_fetch)
+ rows = [
+ row
+ for page in get_rows(
+ api_key="r8_test",
+ endpoint="model_versions",
+ logger=MagicMock(),
+ resumable_source_manager=_FakeResumableManager(), # type: ignore[arg-type]
+ )
+ for row in page
+ ]
+
+ assert rows == [
+ {"id": "abc", "model_owner": "acme", "model_name": "my-model"},
+ {"id": "v1", "model_owner": "stability-ai", "model_name": "sdxl"},
+ {"id": "v0", "model_owner": "stability-ai", "model_name": "sdxl"},
+ ]
+ # Each referenced model is fetched once, however many deployments or trainings use it.
+ assert fetched.count(f"{base}/models/stability-ai/sdxl/versions") == 1
+ assert gone in fetched
+
+ def test_model_versions_raise_non_skippable_child_errors(self, monkeypatch: Any) -> None:
+ base = "https://api.replicate.com/v1"
+ pages: dict[str, Any] = {
+ f"{base}/deployments": {"results": [], "next": None},
+ f"{base}/trainings": {"results": [{"id": "t1", "model": "acme/my-model"}], "next": None},
+ }
+
+ def fake_fetch(session: Any, url: str, headers: dict[str, str], logger: Any) -> Any:
+ if url in pages:
+ return pages[url]
+ raise _http_error(401, url)
+
+ monkeypatch.setattr(replicate, "_fetch_page", fake_fetch)
+ with pytest.raises(requests.HTTPError):
+ list(
+ get_rows(
+ api_key="r8_test",
+ endpoint="model_versions",
+ logger=MagicMock(),
+ resumable_source_manager=_FakeResumableManager(), # type: ignore[arg-type]
+ )
+ )
+
+ def test_collection_models_fan_out_tags_each_model_with_its_collection(self, monkeypatch: Any) -> None:
+ base = "https://api.replicate.com/v1"
+ collections_page2 = f"{base}/collections?cursor=c2"
+ pages = {
+ f"{base}/collections": {"results": [{"slug": "super-resolution"}], "next": collections_page2},
+ collections_page2: {"results": [{"slug": "empty"}, {"slug": "image-to-text"}], "next": None},
+ f"{base}/collections/super-resolution": {
+ "slug": "super-resolution",
+ "models": [{"owner": "acme", "name": "upscaler"}, {"owner": "acme", "name": "esrgan"}],
+ },
+ f"{base}/collections/empty": {"slug": "empty", "models": []},
+ f"{base}/collections/image-to-text": {
+ "slug": "image-to-text",
+ "models": [{"owner": "acme", "name": "upscaler"}],
+ },
+ }
+ rows, _ = _collect(_FakeResumableManager(), monkeypatch, pages, "collection_models")
+
+ assert rows == [
+ {"owner": "acme", "name": "upscaler", "collection_slug": "super-resolution"},
+ {"owner": "acme", "name": "esrgan", "collection_slug": "super-resolution"},
+ {"owner": "acme", "name": "upscaler", "collection_slug": "image-to-text"},
+ ]
+
+ def test_model_versions_resume_from_the_saved_model_and_page(self, monkeypatch: Any) -> None:
+ base = "https://api.replicate.com/v1"
+ resume_url = f"{base}/models/acme/b-model/versions?cursor=v2"
+ pages: dict[str, Any] = {
+ f"{base}/deployments": {"results": [], "next": None},
+ f"{base}/trainings": {
+ "results": [{"model": "acme/c-model"}, {"model": "acme/b-model"}, {"model": "acme/a-model"}],
+ "next": None,
+ },
+ resume_url: {"results": [{"id": "b1"}], "next": None},
+ f"{base}/models/acme/c-model/versions": {"results": [{"id": "c1"}], "next": None},
+ }
+ manager = _FakeResumableManager(ReplicateResumeConfig(next_url=resume_url, fan_out_parent="acme/b-model"))
+ rows, fetched = _collect(manager, monkeypatch, pages, "model_versions")
+
+ assert [r["id"] for r in rows] == ["b1", "c1"]
+ assert f"{base}/models/acme/a-model/versions" not in fetched
+ assert manager.saved[-1] == ReplicateResumeConfig(next_url=None, fan_out_parent="acme/c-model")
+
+ def test_collection_models_resume_after_the_saved_collection(self, monkeypatch: Any) -> None:
+ base = "https://api.replicate.com/v1"
+ page2 = f"{base}/collections?cursor=c2"
+ pages = {
+ page2: {"results": [{"slug": "done"}, {"slug": "next"}], "next": None},
+ f"{base}/collections/next": {"slug": "next", "models": [{"owner": "acme", "name": "m"}]},
+ }
+ manager = _FakeResumableManager(ReplicateResumeConfig(next_url=page2, fan_out_parent="done"))
+ rows, fetched = _collect(manager, monkeypatch, pages, "collection_models")
+
+ assert rows == [{"owner": "acme", "name": "m", "collection_slug": "next"}]
+ assert fetched == [page2, f"{base}/collections/next"]
+ assert manager.saved == [ReplicateResumeConfig(next_url=page2, fan_out_parent="next")]
+
+
class TestFetchPageRetries:
@parameterized.expand([("rate_limited", 429), ("server_error", 503)])
def test_retryable_status_codes_retry_then_succeed(self, _name: str, status: int) -> None: