Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse_sources): add companycam project and photo fan-out tables (#119242)

Tom Owersb897365f57ce
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8c9c86a6c3f1..d1f9da8a328a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -3095,13 +3095,13 @@ Today (8): `Checklists`, `ChecklistTemplates`, `Groups`, `Photos`, `Projects`, `
Diffed against: <https://raw.githubusercontent.com/CompanyCam/openapi-spec/main/openapi.yaml>
-- [ ] `Assigned users (GET /v2/projects/{project_id}/assigned_users)` — connect projects to assigned staff and compare project workload by person (high)
-- [ ] `Photo tags (GET /v2/photos/{photo_id}/tags)` — connect synced photos to synced tags for image classification analysis (high)
-- [ ] `Project labels (GET /v2/projects/{project_id}/labels)` — add project classifications that are absent from the project list for reporting (medium)
+- [x] `Assigned users (GET /v2/projects/{project_id}/assigned_users)` — connect projects to assigned staff and compare project workload by person (high)
+- [x] `Photo tags (GET /v2/photos/{photo_id}/tags)` — connect synced photos to synced tags for image classification analysis (high)
+- [x] `Project labels (GET /v2/projects/{project_id}/labels)` — add project classifications that are absent from the project list for reporting (medium)
- [ ] `Project collaborators (GET /v2/projects/{project_id}/collaborators)` — analyze external collaboration and company participation on projects (medium)
- [ ] `Project comments (GET /v2/projects/{project_id}/comments)` — analyze project communication and identify unresolved work discussions (medium)
- [ ] `Photo comments (GET /v2/photos/{photo_id}/comments)` — connect field discussions to documented photos and identify recurring issues (medium)
-- [ ] `Project documents (GET /v2/projects/{project_id}/documents)` — measure project documentation and connect document metadata to project activity (medium)
+- [x] `Project documents (GET /v2/projects/{project_id}/documents)` — measure project documentation and connect document metadata to project activity (medium)
Note: The official OpenAPI specification was parsed for all GET operations and project, photo, group, and checklist schemas. The project list lacks assigned users and labels; photos lack their tag relationships. Groups already embed users, and checklists already embed tasks and sections. Missing resources need requests per project or photo; uploads and webhooks were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/canonical_descriptions.py
index 1b8f4177043b..fe283e36a4dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/canonical_descriptions.py
@@ -134,4 +134,65 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "Timestamp when the ChecklistTemplate was last updated.",
},
},
+ "ProjectAssignedUsers": {
+ "description": "A user assigned to a Project. One row per Project and assigned User.",
+ "docs_url": "https://docs.companycam.com/reference/listprojectassignedusers",
+ "columns": {
+ "project_id": "The id of the Project the User is assigned to.",
+ "id": "A unique identifier for the User.",
+ "company_id": "A unique identifier for the Company the User belongs to.",
+ "email_address": "The User's email address.",
+ "status": "Indicates the status of the User.",
+ "first_name": "The User's first name.",
+ "last_name": "The User's last name.",
+ "profile_image": "A list of images for the different variants of the User's profile image.",
+ "phone_number": "The User's phone number.",
+ "created_at": "Timestamp when the User was created.",
+ "updated_at": "Timestamp when the User was last updated.",
+ "user_url": "The link to the User in the web app.",
+ },
+ },
+ "ProjectLabels": {
+ "description": "A label applied to a Project. One row per Project and Label.",
+ "docs_url": "https://docs.companycam.com/reference/listprojectlabels",
+ "columns": {
+ "project_id": "The id of the Project the Label is applied to.",
+ "id": "A unique ID for the Label.",
+ "company_id": "A unique ID for the Company the Label belongs to.",
+ "display_value": "The display value of the Label, used to display to the user.",
+ "value": "The lowercase version of display_value, useful for searching and sorting.",
+ "created_at": "Timestamp when the Label was created on the server.",
+ "updated_at": "Timestamp when the Label was last updated.",
+ },
+ },
+ "ProjectDocuments": {
+ "description": "A document (for example a PDF) attached to a Project.",
+ "docs_url": "https://docs.companycam.com/reference/listprojectdocuments",
+ "columns": {
+ "id": "The unique ID of the Document.",
+ "project_id": "The unique ID of the Project the Document is attached to.",
+ "creator_id": "The unique ID of the entity that created the Document.",
+ "creator_type": "The type of the entity that created the Document.",
+ "creator_name": "The display name of the entity that created the Document.",
+ "name": "The filename of the Document.",
+ "url": "The URL where the Document can be downloaded or viewed.",
+ "content_type": "The content type of the Document (for example application/pdf).",
+ "byte_size": "The size of the Document in bytes.",
+ "created_at": "Timestamp when the Document was created on the server.",
+ "updated_at": "Timestamp when the Document was last updated.",
+ },
+ },
+ "PhotoTags": {
+ "description": "A Tag applied to a Photo. One row per Photo and Tag.",
+ "docs_url": "https://docs.companycam.com/reference/listphototags",
+ "columns": {
+ "photo_id": "The id of the Photo the Tag is applied to.",
+ "id": "A unique ID for the Tag.",
+ "company_id": "A unique ID for the Company the Tag belongs to.",
+ "display_value": "The display value of the Tag, used to display to the user.",
+ "value": "The lowercase version of display_value, useful for searching and sorting.",
+ "created_at": "Timestamp when the Tag was created on the server.",
+ "updated_at": "Timestamp when the Tag was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/companycam.py b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/companycam.py
index a31827e90042..29c6d23174ba 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/companycam.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/companycam.py
@@ -1,6 +1,7 @@
import dataclasses
+from collections.abc import Iterable
from datetime import UTC, date, datetime
-from typing import Any, Optional
+from typing import Any, Optional, cast
from requests import Request, Response
@@ -9,12 +10,19 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
BasePaginator,
PageNumberPaginator,
SinglePagePaginator,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ Endpoint,
+ EndpointResource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -162,6 +170,54 @@ def get_resource(
}
+def _client_config(api_key: str, api_version: str, endpoint: str) -> ClientConfig:
+ return {
+ "base_url": base_url(api_version),
+ "auth": {"type": "bearer", "token": api_key},
+ "paginator": _paginator_for(endpoint),
+ # capture=False: project notes, photo/video metadata, and other free-text fields
+ # can carry customer-authored (potentially sensitive) content the name-based HTTP
+ # sample scrubbers aren't guaranteed to catch, same as the other PII-heavy sources.
+ "capture": False,
+ }
+
+
+def _fanout_source(api_key: str, endpoint: str, team_id: int, job_id: str, api_version: str) -> SourceResponse:
+ endpoint_config = COMPANYCAM_ENDPOINTS[endpoint]
+ assert endpoint_config.fanout is not None
+
+ # The client paginator is the child's; the parent may paginate differently (Photos uses a cursor).
+ parent_endpoint_extra: Endpoint = {"paginator": _paginator_for(endpoint_config.fanout.parent_name)}
+
+ # No resume hook: per-parent resume state lists every completed parent path and is rewritten
+ # after each child page, which grows quadratically over large photo or project libraries.
+ resource = cast(
+ Iterable[Any],
+ build_dependent_resource(
+ endpoint_configs=COMPANYCAM_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=endpoint_config.fanout,
+ client_config=_client_config(api_key, api_version, endpoint),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra=parent_endpoint_extra,
+ page_size_param="per_page",
+ ),
+ )
+
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: resource,
+ primary_keys=endpoint_config.primary_keys,
+ partition_mode="datetime" if endpoint_config.partition_key else None,
+ partition_format="month" if endpoint_config.partition_key else None,
+ partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
+ sort_mode="asc",
+ )
+
+
def companycam_source(
api_key: str,
endpoint: str,
@@ -174,16 +230,11 @@ def companycam_source(
) -> SourceResponse:
endpoint_config = COMPANYCAM_ENDPOINTS[endpoint]
+ if endpoint_config.fanout is not None:
+ return _fanout_source(api_key, endpoint, team_id, job_id, api_version)
+
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": base_url(api_version),
- "auth": {"type": "bearer", "token": api_key},
- "paginator": _paginator_for(endpoint),
- # capture=False: project notes, photo/video metadata, and other free-text fields
- # can carry customer-authored (potentially sensitive) content the name-based HTTP
- # sample scrubbers aren't guaranteed to catch, same as the other PII-heavy sources.
- "capture": False,
- },
+ "client": _client_config(api_key, api_version, endpoint),
"resources": [get_resource(endpoint, should_use_incremental_field)],
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/settings.py
index 6e52b15d310b..f4ce9d216b6c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/settings.py
@@ -1,6 +1,9 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortMode
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
@@ -29,6 +32,14 @@ class CompanycamEndpointConfig:
# False for endpoints the docs show no page/per_page params for (ChecklistTemplates returns
# a company's full template list in one response).
paginated: bool = True
+ # Set for child resources CompanyCam only exposes per parent (e.g. labels under a project).
+ fanout: Optional[DependentEndpointConfig] = None
+ # Off by default for fan-outs that cost one request per parent row on every sync.
+ should_sync_default: bool = True
+ page_size: int = PER_PAGE
+ # Read by the fan-out helper's FanoutEndpointLike protocol. Every fan-out child here is full
+ # refresh, so it is never used to build an incremental request.
+ default_incremental_field: Optional[str] = None
def _incremental_field(field_name: str) -> list[IncrementalField]:
@@ -42,6 +53,16 @@ def _incremental_field(field_name: str) -> list[IncrementalField]:
]
+def _project_fanout() -> DependentEndpointConfig:
+ return DependentEndpointConfig(
+ parent_name="Projects",
+ resolve_param="project_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "project_id"},
+ )
+
+
COMPANYCAM_ENDPOINTS: dict[str, CompanycamEndpointConfig] = {
"Projects": CompanycamEndpointConfig(
name="Projects",
@@ -85,6 +106,39 @@ COMPANYCAM_ENDPOINTS: dict[str, CompanycamEndpointConfig] = {
path="/templates/checklists",
paginated=False,
),
+ # Fan-out children. The child rows carry no parent reference, so the parent id is injected and
+ # kept in the primary key: the same user, label, or tag appears under many parents.
+ "ProjectAssignedUsers": CompanycamEndpointConfig(
+ name="ProjectAssignedUsers",
+ path="/projects/{project_id}/assigned_users",
+ primary_keys=["project_id", "id"],
+ fanout=_project_fanout(),
+ ),
+ "ProjectLabels": CompanycamEndpointConfig(
+ name="ProjectLabels",
+ path="/projects/{project_id}/labels",
+ primary_keys=["project_id", "id"],
+ fanout=_project_fanout(),
+ ),
+ "ProjectDocuments": CompanycamEndpointConfig(
+ name="ProjectDocuments",
+ path="/projects/{project_id}/documents",
+ primary_keys=["project_id", "id"],
+ fanout=_project_fanout(),
+ ),
+ "PhotoTags": CompanycamEndpointConfig(
+ name="PhotoTags",
+ path="/photos/{photo_id}/tags",
+ primary_keys=["photo_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="Photos",
+ resolve_param="photo_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "photo_id"},
+ ),
+ should_sync_default=False,
+ ),
}
ENDPOINTS = tuple(COMPANYCAM_ENDPOINTS.keys())
@@ -92,3 +146,7 @@ ENDPOINTS = tuple(COMPANYCAM_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in COMPANYCAM_ENDPOINTS.items()
}
+
+SHOULD_SYNC_DEFAULT: dict[str, bool] = {
+ name: config.should_sync_default for name, config in COMPANYCAM_ENDPOINTS.items()
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/source.py
index 4ae9219a5cf4..2824431f94fb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/source.py
@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.companycam
from products.warehouse_sources.backend.temporal.data_imports.sources.companycam.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ SHOULD_SYNC_DEFAULT,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.companycam import (
CompanycamSourceConfig,
@@ -71,7 +72,7 @@ class CompanycamSource(ResumableSource[CompanycamSourceConfig, CompanycamResumeC
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, should_sync_default=SHOULD_SYNC_DEFAULT)
def validate_credentials(
self,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/tests/test_companycam.py b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/tests/test_companycam.py
index 8e1e2ec4cd74..d1ba21b34d56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/tests/test_companycam.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/companycam/tests/test_companycam.py
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.companycam.companycam import (
@@ -153,6 +154,65 @@ class TestCompanycamSourceResumeBehavior:
assert result.params("modified_since") == ["2023-11-14T22:13:20+00:00"]
+class TestCompanycamFanout:
+ @staticmethod
+ def _driver() -> SourceDriver:
+ return SourceDriver(CompanycamSource(), CompanycamSourceConfig(api_key="test-key"))
+
+ @pytest.mark.parametrize(
+ ("endpoint", "parent_path", "parent_pages", "child_template", "parent_key"),
+ [
+ (
+ "ProjectLabels",
+ "/projects",
+ [ScriptedResponse(json=[{"id": "a"}]), ScriptedResponse(json=[{"id": "b"}]), ScriptedResponse(json=[])],
+ "/projects/{}/labels",
+ "project_id",
+ ),
+ (
+ "PhotoTags",
+ "/photos",
+ [
+ ScriptedResponse(json=[{"id": "a"}], headers={"X-Has-Next": "true", "X-Next-Cursor": "c1"}),
+ ScriptedResponse(json=[{"id": "b"}], headers={"X-Has-Next": "false", "X-Next-Cursor": ""}),
+ ],
+ "/photos/{}/tags",
+ "photo_id",
+ ),
+ ],
+ )
+ def test_walks_every_parent_and_tags_child_rows_with_parent_id(
+ self,
+ endpoint: str,
+ parent_path: str,
+ parent_pages: list[ScriptedResponse],
+ child_template: str,
+ parent_key: str,
+ ) -> None:
+ result = self._driver().run(
+ endpoint,
+ route(
+ {
+ parent_path: parent_pages,
+ child_template.format("a"): [
+ ScriptedResponse(json=[{"id": "t1"}]),
+ ScriptedResponse(json=[{"id": "t2"}]),
+ ScriptedResponse(json=[]),
+ ],
+ child_template.format("b"): [ScriptedResponse(json=[{"id": "t1"}]), ScriptedResponse(json=[])],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert sorted((row[parent_key], row["id"]) for row in result.rows) == [("a", "t1"), ("a", "t2"), ("b", "t1")]
+ child_pages = [
+ request.param("page") for request in result.requests if request.path.endswith(child_template.format("a"))
+ ]
+ assert child_pages == ["1", "2", "3"]
+ assert result.saved_states == []
+
+
class TestValidateCredentials:
@pytest.mark.parametrize(("status_code", "expected"), [(200, True), (401, False), (403, False)])
def test_validate_credentials(self, status_code: int, expected: bool) -> None: