Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 2 missing tables to the Clarifai source (#119881)

Tom Owers36bfe2f3ee34
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index e7e7a5d15a9f..2154ba8fd78f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -2642,8 +2642,8 @@ Diffed against: <https://raw.githubusercontent.com/Clarifai/clarifai-python-grpc
- [x] `Dataset versions (ListDatasetVersions, GET /v2/users/{user_id}/apps/{app_id}/datasets/{dataset_id}/versions)` — track dataset snapshots and connect evaluation outcomes with training data changes (medium)
- [x] `Workflow versions (ListWorkflowVersions, GET /v2/users/{user_id}/apps/{app_id}/workflows/{workflow_id}/versions)` — compare workflow revisions and their model dependencies over time (medium)
- [x] `Concept relations (ListConceptRelations, GET /v2/users/{user_id}/apps/{app_id}/concepts/{concept_id}/relations)` — analyze concept hierarchies and aggregate labels through parent relationships (medium)
-- [ ] `Tasks (ListTasks, GET /v2/users/{user_id}/apps/{app_id}/tasks)` — measure labeling work, task progress, and annotation workload (medium)
-- [ ] `Collaborators (ListCollaborators, GET /v2/users/{user_id}/apps/{app_id}/collaborators)` — analyze app memberships and collaborator access (medium)
+- [x] `Tasks (ListTasks, GET /v2/users/{user_id}/apps/{app_id}/tasks)` — measure labeling work, task progress, and annotation workload (medium)
+- [x] `Collaborators (ListCollaborators, GET /v2/users/{user_id}/apps/{app_id}/collaborators)` — analyze app memberships and collaborator access (medium)
Note: Vendor Swagger and documentation hosts failed DNS resolution, so the official Python SDK provided the reference. Decoded its service descriptor to read list operations and HTTP GET bindings, then checked the official Go SDK descriptions. Placeholder names are simplified from descriptor field names; nested collections require parent IDs, and models currently include only their latest version. Deprecated evaluation routes, authentication, uploads, and prediction writes are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
index c530c9a77ccb..1165bbaab587 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/canonical_descriptions.py
@@ -69,6 +69,40 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"summary": "Summary metrics of the evaluation, such as accuracy, precision, and recall.",
},
},
+ "tasks": {
+ "description": "Labeling and auto-annotation tasks in the selected Clarifai app, including label order tasks.",
+ "docs_url": "https://docs.clarifai.com/resources/api-overview/",
+ "columns": {
+ "id": "Task identifier within the app.",
+ "name": "Title of the task.",
+ "description": "Description of the task.",
+ "type": "Task type, such as concepts classification or bounding box detection.",
+ "status": "Status of the task.",
+ "worker": "Workers that the task is assigned to.",
+ "review": "Review settings of the task, such as the review strategy and reviewers.",
+ "input_source": "Source of the inputs that the task works on, such as a dataset or all inputs.",
+ "concepts": "Concepts that are used in the work of the task.",
+ "label_order_id": "Identifier of the label order that the task belongs to, if any.",
+ "priority": "Priority of the task.",
+ "metrics": "Task metrics, which the API fills in only on request.",
+ "created_at": "Time when the task was created.",
+ "modified_at": "Time when the task was last updated.",
+ },
+ },
+ "collaborators": {
+ "description": "Users who have access to the selected Clarifai app, including removed collaborators.",
+ "docs_url": "https://docs.clarifai.com/resources/api-overview/",
+ "columns": {
+ "id": "Collaborator identifier.",
+ "app": "App that the collaborator has access to.",
+ "user": "User who is the collaborator.",
+ "scopes": "Permission scopes of the collaborator.",
+ "endpoints": "API endpoints that the collaborator can call.",
+ "created_at": "Time when the app was shared with the collaborator.",
+ "modified_at": "Time when the collaborator was last updated.",
+ "deleted_at": "Time when the collaborator was removed from the app, if removed.",
+ },
+ },
"dataset_inputs": {
"description": "Inputs that belong to each dataset in the selected Clarifai app.",
"docs_url": "https://docs.clarifai.com/create/datasets/manage/",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
index 70eac8623f26..6844dd2a264d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/settings.py
@@ -48,6 +48,20 @@ CLARIFAI_ENDPOINTS: dict[str, ClarifaiEndpointConfig] = {
data_selector="eval_metrics",
params={"show_failed_metrics": "true"},
),
+ # Without this flag the API leaves out label order tasks.
+ ClarifaiEndpointConfig(
+ name="tasks",
+ path="tasks",
+ data_selector="tasks",
+ params={"including_label_order_tasks": "true"},
+ ),
+ # Without this flag the API leaves out deleted collaborators, which hides past access.
+ ClarifaiEndpointConfig(
+ name="collaborators",
+ path="collaborators",
+ data_selector="collaborators",
+ params={"list_all_collaborators": "true"},
+ ),
# Dataset inputs carry no ID of their own, so the key combines the dataset ID and the input ID.
ClarifaiEndpointConfig(
name="dataset_inputs",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
index 9d33aeeeffe3..9a16d85033ec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/source.py
@@ -124,10 +124,11 @@ class ClarifaiSource(ResumableSource[ClarifaiSourceConfig, ClarifaiResumeConfig]
releaseStatus=ReleaseStatus.ALPHA,
caption=(
"Create a personal access token in Clarifai under Settings > Secrets. "
- "Grant read access for the models, workflows, datasets, concepts, inputs, annotations, and "
- "evaluations that you want to sync. Allow the ListModels, ListModelVersions, ListWorkflows, "
- "ListWorkflowVersions, ListDatasets, ListDatasetVersions, ListDatasetInputs, ListConcepts, "
- "ListConceptRelations, ListInputs, ListAnnotations, and ListEvaluations endpoints for those tables. "
+ "Grant read access for the models, workflows, datasets, concepts, inputs, annotations, "
+ "evaluations, tasks, and collaborators that you want to sync. Allow the ListModels, "
+ "ListModelVersions, ListWorkflows, ListWorkflowVersions, ListDatasets, ListDatasetVersions, "
+ "ListDatasetInputs, ListConcepts, ListConceptRelations, ListInputs, ListAnnotations, "
+ "ListEvaluations, ListTasks, and ListCollaborators endpoints for those tables. "
"Enter the user ID and app ID of the app owner."
),
fields=cast(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
index 6123ad671fab..c3d0778fc581 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clarifai/tests/test_clarifai.py
@@ -58,6 +58,8 @@ class TestClarifaiTransport:
("concept_relations", "concepts/relations", "concept_relations", {}),
("annotations", "annotations", "annotations", {"list_all_annotations": ["true"]}),
("evaluations", "evaluations", "eval_metrics", {"show_failed_metrics": ["true"]}),
+ ("tasks", "tasks", "tasks", {"including_label_order_tasks": ["true"]}),
+ ("collaborators", "collaborators", "collaborators", {"list_all_collaborators": ["true"]}),
],
)
@pytest.mark.parametrize("resume_page", [None, 4])