Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 more tables to the Clarifai source (#119530)

Tom Owers3cb0e54272ed
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index e7fd8d401c00..d433001e28bd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -2634,14 +2634,14 @@ Today (4): `concepts`, `datasets`, `models`, `workflows`
Diffed against: <https://raw.githubusercontent.com/Clarifai/clarifai-python-grpc/master/clarifai_grpc/grpc/api/service_pb2.py>
-- [ ] `Inputs (ListInputs, GET /v2/users/{user_id}/apps/{app_id}/inputs)` — analyze input volumes, metadata, and processing status within each app (high)
+- [x] `Inputs (ListInputs, GET /v2/users/{user_id}/apps/{app_id}/inputs)` — analyze input volumes, metadata, and processing status within each app (high)
- [x] `Model versions (ListModelVersions, GET /v2/users/{user_id}/apps/{app_id}/models/{model_id}/versions)` — track training status and version history beyond each model's latest version (high)
- [x] `Evaluations (ListEvaluations, GET /v2/users/{user_id}/apps/{app_id}/evaluations)` — compare model accuracy and evaluation results across versions and datasets (high)
- [x] `Dataset inputs (ListDatasetInputs, GET /v2/users/{user_id}/apps/{app_id}/datasets/{dataset_id}/inputs)` — join datasets with their input memberships to compare data coverage and composition (high)
- [x] `Annotations (ListAnnotations, GET /v2/users/{user_id}/apps/{app_id}/annotations)` — analyze labels, annotation coverage, and review outcomes for stored inputs (high)
-- [ ] `Dataset versions (ListDatasetVersions, GET /v2/users/{user_id}/apps/{app_id}/datasets/{dataset_id}/versions)` — track dataset snapshots and connect evaluation outcomes with training data changes (medium)
-- [ ] `Workflow versions (ListWorkflowVersions, GET /v2/users/{user_id}/apps/{app_id}/workflows/{workflow_id}/versions)` — compare workflow revisions and their model dependencies over time (medium)
-- [ ] `Concept relations (ListConceptRelations, GET /v2/users/{user_id}/apps/{app_id}/concepts/{concept_id}/relations)` — analyze concept hierarchies and aggregate labels through parent relationships (medium)
+- [x] `Dataset versions (ListDatasetVersions, GET /v2/users/{user_id}/apps/{app_id}/datasets/{dataset_id}/versions)` — track dataset snapshots and connect evaluation outcomes with training data changes (medium)
+- [x] `Workflow versions (ListWorkflowVersions, GET /v2/users/{user_id}/apps/{app_id}/workflows/{workflow_id}/versions)` — compare workflow revisions and their model dependencies over time (medium)
+- [x] `Concept relations (ListConceptRelations, GET /v2/users/{user_id}/apps/{app_id}/concepts/{concept_id}/relations)` — analyze concept hierarchies and aggregate labels through parent relationships (medium)
- [ ] `Tasks (ListTasks, GET /v2/users/{user_id}/apps/{app_id}/tasks)` — measure labeling work, task progress, and annotation workload (medium)
- [ ] `Collaborators (ListCollaborators, GET /v2/users/{user_id}/apps/{app_id}/collaborators)` — analyze app memberships and collaborator access (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
index 61ecbfcaded5..c530c9a77ccb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
@@ -92,4 +92,53 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"metrics": "Evaluation metrics of the model version.",
},
},
+ "inputs": {
+ "description": "Inputs, such as images, videos, and text, stored in the selected Clarifai app.",
+ "docs_url": "https://docs.clarifai.com/resources/api-overview/",
+ "columns": {
+ "id": "Input identifier within the app.",
+ "data": "Input content and metadata, such as the media URL and custom metadata.",
+ "status": "Processing status of the input.",
+ "dataset_ids": "Identifiers of the datasets that contain the input.",
+ "created_at": "Time when the input was added.",
+ "modified_at": "Time when the input was last modified.",
+ },
+ },
+ "concept_relations": {
+ "description": "Relations between concepts in the knowledge graph of the selected Clarifai app.",
+ "docs_url": "https://docs.clarifai.com/create/concepts/manage/",
+ "columns": {
+ "id": "Concept relation identifier within the app.",
+ "subject_concept": "Concept that is the subject of the relation.",
+ "object_concept": "Concept that is the object of the relation.",
+ "predicate": "Type of relation, such as hypernym, hyponym, or synonym.",
+ "knowledge_graph_id": "Identifier of the knowledge graph that contains the relation.",
+ },
+ },
+ "dataset_versions": {
+ "description": "Snapshots of each dataset in the selected Clarifai app.",
+ "docs_url": "https://docs.clarifai.com/create/datasets/manage/",
+ "columns": {
+ "dataset_id": "Identifier of the dataset that the version belongs to.",
+ "id": "Dataset version identifier.",
+ "status": "Processing status of the dataset version.",
+ "description": "Description of the dataset version.",
+ "metrics": "Metrics of the dataset version, such as input and annotation counts.",
+ "created_at": "Time when the dataset version was created.",
+ "modified_at": "Time when the dataset version was last modified.",
+ },
+ },
+ "workflow_versions": {
+ "description": "All versions of each workflow in the selected Clarifai app.",
+ "docs_url": "https://docs.clarifai.com/create/workflows/manage/",
+ "columns": {
+ "workflow_id": "Identifier of the workflow that the version belongs to.",
+ "id": "Workflow version identifier.",
+ "nodes": "Model nodes and their connections in the workflow version.",
+ "description": "Description of the workflow version.",
+ "is_deprecated": "Whether the workflow version is deprecated.",
+ "created_at": "Time when the workflow version was created.",
+ "modified_at": "Time when the workflow version was last modified.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
index 5c742f668fe0..70eac8623f26 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
@@ -31,6 +31,9 @@ CLARIFAI_ENDPOINTS: dict[str, ClarifaiEndpointConfig] = {
ClarifaiEndpointConfig(name="workflows", path="workflows", data_selector="workflows"),
ClarifaiEndpointConfig(name="datasets", path="datasets", data_selector="datasets"),
ClarifaiEndpointConfig(name="concepts", path="concepts", data_selector="concepts"),
+ ClarifaiEndpointConfig(name="inputs", path="inputs", data_selector="inputs"),
+ # Without a concept ID in the path, ListConceptRelations returns every relation in the app.
+ ClarifaiEndpointConfig(name="concept_relations", path="concepts/relations", data_selector="concept_relations"),
# Without this flag the API returns only trusted annotations, which hides pending and rejected reviews.
ClarifaiEndpointConfig(
name="annotations",
@@ -72,6 +75,32 @@ CLARIFAI_ENDPOINTS: dict[str, ClarifaiEndpointConfig] = {
parent_field_renames={"id": "model_id"},
),
),
+ ClarifaiEndpointConfig(
+ name="dataset_versions",
+ path="datasets/{dataset_id}/versions",
+ data_selector="dataset_versions",
+ primary_keys=["dataset_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="datasets",
+ resolve_param="dataset_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "dataset_id"},
+ ),
+ ),
+ ClarifaiEndpointConfig(
+ name="workflow_versions",
+ path="workflows/{workflow_id}/versions",
+ data_selector="workflow_versions",
+ primary_keys=["workflow_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="workflows",
+ resolve_param="workflow_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "workflow_id"},
+ ),
+ ),
)
}
ENDPOINTS = tuple(CLARIFAI_ENDPOINTS)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
index c01e9cfddbf2..9d33aeeeffe3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
@@ -124,9 +124,10 @@ class ClarifaiSource(ResumableSource[ClarifaiSourceConfig, ClarifaiResumeConfig]
releaseStatus=ReleaseStatus.ALPHA,
caption=(
"Create a personal access token in Clarifai under Settings > Secrets. "
- "Grant read access for the models, workflows, datasets, concepts, annotations, and evaluations "
- "that you want to sync. Allow the ListModels, ListModelVersions, ListWorkflows, ListDatasets, "
- "ListDatasetInputs, ListConcepts, ListAnnotations, and ListEvaluations endpoints for those tables. "
+ "Grant read access for the models, workflows, datasets, concepts, inputs, annotations, and "
+ "evaluations that you want to sync. Allow the ListModels, ListModelVersions, ListWorkflows, "
+ "ListWorkflowVersions, ListDatasets, ListDatasetVersions, ListDatasetInputs, ListConcepts, "
+ "ListConceptRelations, ListInputs, ListAnnotations, and ListEvaluations endpoints for those tables. "
"Enter the user ID and app ID of the app owner."
),
fields=cast(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
index 6a0f44b83ea5..068078bc4e23 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
@@ -49,14 +49,16 @@ class TestClarifaiTransport:
return SourceDriver(ClarifaiSource(), config)
@pytest.mark.parametrize(
- ("endpoint", "data_key", "extra_params"),
+ ("endpoint", "path", "data_key", "extra_params"),
[
- ("models", "models", {}),
- ("workflows", "workflows", {}),
- ("datasets", "datasets", {}),
- ("concepts", "concepts", {}),
- ("annotations", "annotations", {"list_all_annotations": ["true"]}),
- ("evaluations", "eval_metrics", {"show_failed_metrics": ["true"]}),
+ ("models", "models", "models", {}),
+ ("workflows", "workflows", "workflows", {}),
+ ("datasets", "datasets", "datasets", {}),
+ ("concepts", "concepts", "concepts", {}),
+ ("inputs", "inputs", "inputs", {}),
+ ("concept_relations", "concepts/relations", "concept_relations", {}),
+ ("annotations", "annotations", "annotations", {"list_all_annotations": ["true"]}),
+ ("evaluations", "evaluations", "eval_metrics", {"show_failed_metrics": ["true"]}),
],
)
@pytest.mark.parametrize("resume_page", [None, 4])
@@ -65,6 +67,7 @@ class TestClarifaiTransport:
config: ClarifaiSourceConfig,
manager: MagicMock,
endpoint: str,
+ path: str,
data_key: str,
extra_params: dict[str, list[str]],
resume_page: int | None,
@@ -75,7 +78,7 @@ class TestClarifaiTransport:
first_page = resume_page or 1
with requests_mock.Mocker() as http:
http.get(
- BASE_URL + endpoint,
+ BASE_URL + path,
[
{"json": {"status": {"code": 10000}, data_key: [{"id": "row-1"}]}},
{"json": {"status": {"code": 10000}, data_key: [{"id": "row-2"}]}},