Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Clarifai source (#118997)

Tom Owersba78761879c2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index bf575f92954e..0a0c9bd61ca0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -2635,10 +2635,10 @@ Today (4): `concepts`, `datasets`, `models`, `workflows`
Diffed against: <https://raw.githubusercontent.com/Clarifai/clarifai-python-grpc/master/clarifai_grpc/grpc/api/service_pb2.py>
- [ ] `Inputs (ListInputs, GET /v2/users/{user_id}/apps/{app_id}/inputs)` — analyze input volumes, metadata, and processing status within each app (high)
-- [ ] `Model versions (ListModelVersions, GET /v2/users/{user_id}/apps/{app_id}/models/{model_id}/versions)` — track training status and version history beyond each model's latest version (high)
-- [ ] `Evaluations (ListEvaluations, GET /v2/users/{user_id}/apps/{app_id}/evaluations)` — compare model accuracy and evaluation results across versions and datasets (high)
-- [ ] `Dataset inputs (ListDatasetInputs, GET /v2/users/{user_id}/apps/{app_id}/datasets/{dataset_id}/inputs)` — join datasets with their input memberships to compare data coverage and composition (high)
-- [ ] `Annotations (ListAnnotations, GET /v2/users/{user_id}/apps/{app_id}/annotations)` — analyze labels, annotation coverage, and review outcomes for stored inputs (high)
+- [x] `Model versions (ListModelVersions, GET /v2/users/{user_id}/apps/{app_id}/models/{model_id}/versions)` — track training status and version history beyond each model's latest version (high)
+- [x] `Evaluations (ListEvaluations, GET /v2/users/{user_id}/apps/{app_id}/evaluations)` — compare model accuracy and evaluation results across versions and datasets (high)
+- [x] `Dataset inputs (ListDatasetInputs, GET /v2/users/{user_id}/apps/{app_id}/datasets/{dataset_id}/inputs)` — join datasets with their input memberships to compare data coverage and composition (high)
+- [x] `Annotations (ListAnnotations, GET /v2/users/{user_id}/apps/{app_id}/annotations)` — analyze labels, annotation coverage, and review outcomes for stored inputs (high)
- [ ] `Dataset versions (ListDatasetVersions, GET /v2/users/{user_id}/apps/{app_id}/datasets/{dataset_id}/versions)` — track dataset snapshots and connect evaluation outcomes with training data changes (medium)
- [ ] `Workflow versions (ListWorkflowVersions, GET /v2/users/{user_id}/apps/{app_id}/workflows/{workflow_id}/versions)` — compare workflow revisions and their model dependencies over time (medium)
- [ ] `Concept relations (ListConceptRelations, GET /v2/users/{user_id}/apps/{app_id}/concepts/{concept_id}/relations)` — analyze concept hierarchies and aggregate labels through parent relationships (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
index f3a8d2a09d7c..61ecbfcaded5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
@@ -42,4 +42,54 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"app_id": "Identifier of the app that contains the concept.",
},
},
+ "annotations": {
+ "description": "Annotations on inputs in the selected Clarifai app, including untrusted annotations.",
+ "docs_url": "https://docs.clarifai.com/resources/api-overview/",
+ "columns": {
+ "id": "Annotation identifier within the app.",
+ "input_id": "Identifier of the annotated input.",
+ "data": "Annotation content, such as concepts, regions, or frames.",
+ "user_id": "Identifier of the user who created the annotation.",
+ "model_version_id": "Identifier of the model version that created the annotation, if any.",
+ "status": "Annotation status, such as success, pending, or awaiting review.",
+ "trusted": "Whether the annotation is trusted.",
+ "task_id": "Identifier of the labeling task that produced the annotation, if any.",
+ "created_at": "Time when the annotation was created.",
+ "modified_at": "Time when the annotation was last modified.",
+ },
+ },
+ "evaluations": {
+ "description": "Model evaluation results in the selected Clarifai app.",
+ "docs_url": "https://docs.clarifai.com/resources/api-overview/",
+ "columns": {
+ "id": "Evaluation identifier within the app.",
+ "status": "Status of the evaluation run.",
+ "model": "Model and model version that were evaluated.",
+ "ground_truth_dataset": "Dataset version used as ground truth.",
+ "summary": "Summary metrics of the evaluation, such as accuracy, precision, and recall.",
+ },
+ },
+ "dataset_inputs": {
+ "description": "Inputs that belong to each dataset in the selected Clarifai app.",
+ "docs_url": "https://docs.clarifai.com/create/datasets/manage/",
+ "columns": {
+ "dataset_id": "Identifier of the dataset that contains the input.",
+ "input_id": "Identifier of the input.",
+ "input": "The input, including its data and metadata.",
+ "created_at": "Time when the input was added to the dataset.",
+ },
+ },
+ "model_versions": {
+ "description": "All versions of each model in the selected Clarifai app.",
+ "docs_url": "https://docs.clarifai.com/create/models/manage/",
+ "columns": {
+ "model_id": "Identifier of the model that the version belongs to.",
+ "id": "Model version identifier.",
+ "status": "Training status of the model version.",
+ "created_at": "Time when the model version was created.",
+ "completed_at": "Time when training of the model version completed.",
+ "total_input_count": "Number of inputs used to train the model version.",
+ "metrics": "Evaluation metrics of the model version.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/clarifai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/clarifai.py
index 4f60a39b46aa..96a48c9224d9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/clarifai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/clarifai.py
@@ -9,13 +9,12 @@ from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.clarifai.settings import (
AUTH_ERROR,
+ CLARIFAI_ENDPOINTS,
DEFAULT_API_HOST,
- ENDPOINTS,
HOST_ERROR,
- PAGE_SIZE,
PERMISSION_ERROR,
- PRIMARY_KEYS,
RESOURCE_ERROR,
+ ClarifaiEndpointConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import ValidateDatabaseHostMixin
@@ -23,10 +22,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
RESTClientRetryableError,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import PaginatorConfig
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ PaginatorConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
UNKNOWN_RESOURCE_PREFIX,
@@ -40,7 +46,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
@frozen
class ClarifaiResumeConfig:
- page: int
+ page: int | None = None
+ # Fan-out endpoints store the state that `build_dependent_resource` checkpoints per parent.
+ fanout_state: dict[str, Any] | None = None
+
+
+def _add_input_id(row: dict[str, Any]) -> dict[str, Any]:
+ input_data = row.get("input")
+ row["input_id"] = input_data.get("id") if isinstance(input_data, dict) else None
+ return row
+
+
+ROW_MAPS = {"dataset_inputs": _add_input_id}
class ClarifaiAPIError(ValueError):
@@ -108,65 +125,120 @@ class ClarifaiClient(ValidateDatabaseHostMixin):
)
raise ValueError("Clarifai returned an unsuccessful response. Check the API status and try again.")
- def resource_config(self, endpoint: str, *, probe: bool = False) -> RESTAPIConfig:
- if endpoint not in ENDPOINTS:
- raise UnknownResourceError(f"{UNKNOWN_RESOURCE_PREFIX} {endpoint}")
+ def client_config(self, *, probe: bool = False) -> ClientConfig:
session = make_tracked_session(redact_values=(self.config.personal_access_token,), allow_redirects=False)
# Clarifai reports API failures in the JSON status, including responses with HTTP 200.
session.hooks["response"].append(self.check_response)
return {
- "client": {
- "base_url": self.base_url,
- "auth": {
- "type": "api_key",
- "name": "Authorization",
- "api_key": f"Key {self.config.personal_access_token}",
- },
- "session": session,
- "allowed_hosts": [],
- "allow_redirects": False,
- "request_timeout": (10, 60),
- # PageNumberPaginatorConfig and the runtime constructor currently disagree on this field name.
- "paginator": "single_page"
- if probe
- else cast(PaginatorConfig, {"type": "page_number", "base_page": 1, "total_path": None}),
+ "base_url": self.base_url,
+ "auth": {
+ "type": "api_key",
+ "name": "Authorization",
+ "api_key": f"Key {self.config.personal_access_token}",
},
+ "session": session,
+ "allowed_hosts": [],
+ "allow_redirects": False,
+ "request_timeout": (10, 60),
+ # PageNumberPaginatorConfig and the runtime constructor currently disagree on this field name.
+ "paginator": "single_page"
+ if probe
+ else cast(PaginatorConfig, {"type": "page_number", "base_page": 1, "total_path": None}),
+ }
+
+ @staticmethod
+ def endpoint_config(endpoint: str) -> ClarifaiEndpointConfig:
+ config = CLARIFAI_ENDPOINTS.get(endpoint)
+ if config is None:
+ raise UnknownResourceError(f"{UNKNOWN_RESOURCE_PREFIX} {endpoint}")
+ return config
+
+ def resource_config(self, endpoint: str, *, probe: bool = False) -> RESTAPIConfig:
+ config = self.endpoint_config(endpoint)
+ if config.fanout is not None:
+ if not probe:
+ raise ValueError(f"{endpoint} fans out over {config.fanout.parent_name} and has no flat resource.")
+ # A child list needs a parent ID, so the probe checks the parent list instead.
+ config = self.endpoint_config(config.fanout.parent_name)
+ return {
+ "client": self.client_config(probe=probe),
"resources": [
{
- "name": endpoint,
+ "name": config.name,
"endpoint": {
- "path": endpoint,
- "data_selector": endpoint,
- "params": {"page": 1, "per_page": 1 if probe else PAGE_SIZE},
+ "path": config.path,
+ "data_selector": config.data_selector,
+ "params": {"page": 1, "per_page": 1 if probe else config.page_size, **config.params},
},
}
],
}
+ def own_app_rows(self, row: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]:
+ # Listings scoped to an app can also return Clarifai main models, whose children 404 under this app's path.
+ if row.get("user_id", self.config.user_id) != self.config.user_id:
+ return []
+ if row.get("app_id", self.config.app_id) != self.config.app_id:
+ return []
+ return row
+
def source(
self,
endpoint: str,
job_id: str,
manager: ResumableSourceManager[ClarifaiResumeConfig],
) -> SourceResponse:
+ config = self.endpoint_config(endpoint)
resume = manager.load_state() if manager.can_resume() else None
- def save_checkpoint(state: dict[str, Any] | None) -> None:
- if state is not None:
- manager.save_state(ClarifaiResumeConfig(page=int(state["page"])))
+ resource: Resource
+ if config.fanout is not None:
+ parent = self.endpoint_config(config.fanout.parent_name)
- resource = rest_api_resource(
- self.resource_config(endpoint),
- team_id=self.team_id,
- job_id=job_id,
- db_incremental_field_last_value=None,
- initial_paginator_state={"page": resume.page} if resume else None,
- resume_hook=save_checkpoint,
- )
+ def save_fanout_state(state: dict[str, Any] | None) -> None:
+ if state:
+ manager.save_state(ClarifaiResumeConfig(fanout_state=state))
+
+ resource = cast(
+ Resource,
+ build_dependent_resource(
+ endpoint_configs=CLARIFAI_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ client_config=self.client_config(),
+ path_format_values={},
+ team_id=self.team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ page_size_param="per_page",
+ parent_endpoint_extra={"data_selector": parent.data_selector},
+ parent_data_map=self.own_app_rows,
+ child_endpoint_extra={"data_selector": config.data_selector},
+ resume_hook=save_fanout_state,
+ initial_paginator_state=resume.fanout_state if resume else None,
+ ),
+ )
+ else:
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state is not None:
+ manager.save_state(ClarifaiResumeConfig(page=int(state["page"])))
+
+ resource = rest_api_resource(
+ self.resource_config(endpoint),
+ team_id=self.team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ initial_paginator_state={"page": resume.page} if resume and resume.page is not None else None,
+ resume_hook=save_checkpoint,
+ )
+ row_map = ROW_MAPS.get(endpoint)
+ if row_map is not None:
+ resource.add_map(row_map)
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=PRIMARY_KEYS,
+ primary_keys=config.primary_keys,
sort_mode=None,
on_complete=manager.clear_state,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
index 4c406b774ee8..5c742f668fe0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
@@ -1,7 +1,80 @@
+from dataclasses import field
+from typing import Any
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
+
DEFAULT_API_HOST = "https://api.clarifai.com"
PAGE_SIZE = 128
-ENDPOINTS = ("models", "workflows", "datasets", "concepts")
-PRIMARY_KEYS = ["id"]
+
+
+@frozen
+class ClarifaiEndpointConfig:
+ name: str
+ path: str
+ data_selector: str
+ primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ params: dict[str, Any] = field(default_factory=dict)
+ fanout: DependentEndpointConfig | None = None
+ page_size: int = PAGE_SIZE
+ incremental_fields: list[Any] = field(default_factory=list)
+ default_incremental_field: str | None = None
+
+
+CLARIFAI_ENDPOINTS: dict[str, ClarifaiEndpointConfig] = {
+ endpoint.name: endpoint
+ for endpoint in (
+ ClarifaiEndpointConfig(name="models", path="models", data_selector="models"),
+ ClarifaiEndpointConfig(name="workflows", path="workflows", data_selector="workflows"),
+ ClarifaiEndpointConfig(name="datasets", path="datasets", data_selector="datasets"),
+ ClarifaiEndpointConfig(name="concepts", path="concepts", data_selector="concepts"),
+ # Without this flag the API returns only trusted annotations, which hides pending and rejected reviews.
+ ClarifaiEndpointConfig(
+ name="annotations",
+ path="annotations",
+ data_selector="annotations",
+ params={"list_all_annotations": "true"},
+ ),
+ # Without this flag the API leaves out failed evaluation runs.
+ ClarifaiEndpointConfig(
+ name="evaluations",
+ path="evaluations",
+ data_selector="eval_metrics",
+ params={"show_failed_metrics": "true"},
+ ),
+ # Dataset inputs carry no ID of their own, so the key combines the dataset ID and the input ID.
+ ClarifaiEndpointConfig(
+ name="dataset_inputs",
+ path="datasets/{dataset_id}/inputs",
+ data_selector="dataset_inputs",
+ primary_keys=["dataset_id", "input_id"],
+ fanout=DependentEndpointConfig(
+ parent_name="datasets",
+ resolve_param="dataset_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "dataset_id"},
+ ),
+ ),
+ ClarifaiEndpointConfig(
+ name="model_versions",
+ path="models/{model_id}/versions",
+ data_selector="model_versions",
+ primary_keys=["model_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="models",
+ resolve_param="model_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "model_id"},
+ ),
+ ),
+ )
+}
+ENDPOINTS = tuple(CLARIFAI_ENDPOINTS)
AUTH_ERROR = "Clarifai rejected the token. Check your personal access token and app access."
PERMISSION_ERROR = (
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
index 697eccdaf074..c01e9cfddbf2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
@@ -124,8 +124,9 @@ class ClarifaiSource(ResumableSource[ClarifaiSourceConfig, ClarifaiResumeConfig]
releaseStatus=ReleaseStatus.ALPHA,
caption=(
"Create a personal access token in Clarifai under Settings > Secrets. "
- "Grant read access for models, workflows, datasets, and concepts that you want to sync. "
- "Allow their ListModels, ListWorkflows, ListDatasets, and ListConcepts endpoints. "
+ "Grant read access for the models, workflows, datasets, concepts, annotations, and evaluations "
+ "that you want to sync. Allow the ListModels, ListModelVersions, ListWorkflows, ListDatasets, "
+ "ListDatasetInputs, ListConcepts, ListAnnotations, and ListEvaluations endpoints for those tables. "
"Enter the user ID and app ID of the app owner."
),
fields=cast(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
index 841848da4d53..6a0f44b83ea5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.clarifai import (
@@ -47,10 +48,26 @@ class TestClarifaiTransport:
def _driver(config: ClarifaiSourceConfig) -> SourceDriver:
return SourceDriver(ClarifaiSource(), config)
- @pytest.mark.parametrize("endpoint", ["models", "workflows", "datasets", "concepts"])
+ @pytest.mark.parametrize(
+ ("endpoint", "data_key", "extra_params"),
+ [
+ ("models", "models", {}),
+ ("workflows", "workflows", {}),
+ ("datasets", "datasets", {}),
+ ("concepts", "concepts", {}),
+ ("annotations", "annotations", {"list_all_annotations": ["true"]}),
+ ("evaluations", "eval_metrics", {"show_failed_metrics": ["true"]}),
+ ],
+ )
@pytest.mark.parametrize("resume_page", [None, 4])
def test_pagination_and_resume(
- self, config: ClarifaiSourceConfig, manager: MagicMock, endpoint: str, resume_page: int | None
+ self,
+ config: ClarifaiSourceConfig,
+ manager: MagicMock,
+ endpoint: str,
+ data_key: str,
+ extra_params: dict[str, list[str]],
+ resume_page: int | None,
) -> None:
if resume_page is not None:
manager.can_resume.return_value = True
@@ -60,16 +77,16 @@ class TestClarifaiTransport:
http.get(
BASE_URL + endpoint,
[
- {"json": {"status": {"code": 10000}, endpoint: [{"id": "row-1"}]}},
- {"json": {"status": {"code": 10000}, endpoint: [{"id": "row-2"}]}},
- {"json": {"status": {"code": 10000}, endpoint: []}},
+ {"json": {"status": {"code": 10000}, data_key: [{"id": "row-1"}]}},
+ {"json": {"status": {"code": 10000}, data_key: [{"id": "row-2"}]}},
+ {"json": {"status": {"code": 10000}, data_key: []}},
],
)
response = ClarifaiClient(config, 42, "v2").source(endpoint, "test-job", manager)
rows = [row for page in cast(Iterable[list[dict[str, Any]]], response.items()) for row in page]
assert rows == [{"id": "row-1"}, {"id": "row-2"}]
assert [request.qs for request in http.request_history] == [
- {"page": [str(page)], "per_page": ["128"]} for page in range(first_page, first_page + 3)
+ {"page": [str(page)], "per_page": ["128"], **extra_params} for page in range(first_page, first_page + 3)
]
assert all(request.headers["Authorization"] == "Key fake-clarifai-pat" for request in http.request_history)
assert all(request.timeout == (10, 60) for request in http.request_history)
@@ -81,6 +98,77 @@ class TestClarifaiTransport:
response.on_complete()
manager.clear_state.assert_called_once_with()
+ @pytest.mark.parametrize(
+ ("endpoint", "parent", "data_key", "child_row", "expected_row"),
+ [
+ (
+ "dataset_inputs",
+ "datasets",
+ "dataset_inputs",
+ {"input": {"id": "input-1"}},
+ {"input": {"id": "input-1"}, "input_id": "input-1"},
+ ),
+ ("model_versions", "models", "model_versions", {"id": "version-1"}, {"id": "version-1"}),
+ ],
+ )
+ @pytest.mark.parametrize("resumed", [False, True])
+ def test_fan_out_pagination_and_resume(
+ self,
+ config: ClarifaiSourceConfig,
+ endpoint: str,
+ parent: str,
+ data_key: str,
+ child_row: dict[str, Any],
+ expected_row: dict[str, Any],
+ resumed: bool,
+ ) -> None:
+ parent_key = "dataset_id" if parent == "datasets" else "model_id"
+ child_suffix = "inputs" if parent == "datasets" else "versions"
+ prefix = "/v2/users/example-user/apps/example-app/"
+
+ def ok(key: str, rows: list[dict[str, Any]]) -> ScriptedResponse:
+ return ScriptedResponse(json={"status": {"code": 10000}, key: rows})
+
+ script = route(
+ {
+ f"{prefix}{parent}": [
+ ok(
+ parent,
+ [
+ {"id": "parent-1"},
+ {"id": "main-row", "user_id": "clarifai", "app_id": "main"},
+ {"id": "parent-2", "user_id": "example-user", "app_id": "example-app"},
+ ],
+ ),
+ ok(parent, []),
+ ],
+ f"{prefix}{parent}/parent-1/{child_suffix}": [ok(data_key, [child_row]), ok(data_key, [])],
+ f"{prefix}{parent}/parent-2/{child_suffix}": [
+ ok(data_key, [child_row]),
+ ok(data_key, [child_row]),
+ ok(data_key, []),
+ ],
+ }
+ )
+ resume_state = (
+ ClarifaiResumeConfig(
+ fanout_state={"completed": [f"{parent}/parent-1/{child_suffix}"], "current": None, "child_state": None}
+ )
+ if resumed
+ else None
+ )
+ result = self._driver(config).run(endpoint, script, resume_state=resume_state)
+
+ assert result.raised is None
+ expected_parents = ["parent-2"] * 2 if resumed else ["parent-1", "parent-2", "parent-2"]
+ assert result.rows == [{**expected_row, parent_key: parent_id} for parent_id in expected_parents]
+ child_requests = [request for request in result.requests if request.path.endswith(child_suffix)]
+ assert [(request.path.split("/")[-2], request.param("page")) for request in child_requests] == (
+ [] if resumed else [("parent-1", "1"), ("parent-1", "2")]
+ ) + [("parent-2", "1"), ("parent-2", "2"), ("parent-2", "3")]
+ assert all(request.param("per_page") == "128" for request in result.requests)
+ assert result.final_state is None or result.final_state.page is None
+
@pytest.mark.parametrize(
("http_status", "api_status", "message"),
[
@@ -161,6 +249,8 @@ class TestClarifaiCredentials:
(200, 11007, None, True, None),
(200, 11007, "datasets", False, "The Clarifai token lacks permission"),
(404, 11101, None, False, "Clarifai could not find the resource"),
+ (200, 10000, "model_versions", True, None),
+ (200, 11007, "dataset_inputs", False, "The Clarifai token lacks permission"),
],
)
def test_probe_status_mapping(
@@ -172,7 +262,10 @@ class TestClarifaiCredentials:
valid: bool,
message: str | None,
) -> None:
- endpoint = schema_name or "models"
+ # Fan-out tables have no parent ID at probe time, so the probe lists their parent.
+ endpoint = {"model_versions": "models", "dataset_inputs": "datasets"}.get(
+ schema_name or "", schema_name or "models"
+ )
with scripted_network(
[ScriptedResponse(status=http_status, json={"status": {"code": api_status}, endpoint: []})]
) as network: