Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add kommo catalog elements and field group tables (#122647)

Tom Owers02e5ac26ecd2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index ac06a9126869..9f68414749e8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6950,12 +6950,12 @@ Today (17): `Catalogs`, `Companies`, `CompanyCustomFields`, `CompanyNotes`, `Com
Diffed against: <https://developers.kommo.com/llms.txt>
-- [ ] `Entity field groups (GET /api/v4/{entity_type}/custom_fields/groups)` — resolve group_id in existing custom field tables and organize analysis by field group (high)
-- [ ] `List elements (GET /api/v4/catalogs/{list_id}/elements)` — import product and other catalog records; Catalogs currently contains only list definitions (high)
+- [x] `Entity field groups (GET /api/v4/{entity_type}/custom_fields/groups)` — resolve group_id in existing custom field tables and organize analysis by field group (high). Added as `LeadCustomFieldGroups`, `ContactCustomFieldGroups`, and `CompanyCustomFieldGroups`.
+- [x] `List elements (GET /api/v4/catalogs/{list_id}/elements)` — import product and other catalog records; Catalogs currently contains only list definitions (high). Added as `CatalogElements` (fan-out over `Catalogs`).
- [ ] `Linked entities (GET /api/v4/{entity}/{entity_id}/links)` — preserve catalog relationships, quantities, and link metadata beyond embedded contact and lead IDs (medium)
- [ ] `Incoming leads (GET /api/v4/leads/unsorted)` — measure intake and unprocessed leads by source before they enter the sales pipeline (medium)
-- [ ] `Chat history (GET /v2/origin/custom/{scope_id}/chats/{conversation_id}/history)` — analyze message volume, response times, and customer conversations beyond CRM notes (medium)
-- [ ] `Catalog custom fields (GET /api/v4/catalogs/{list_id}/custom_fields)` — interpret custom fields and enum values when importing catalog records (medium)
+- [ ] `Chat history (GET /v2/origin/custom/{scope_id}/chats/{conversation_id}/history)` — analyze message volume, response times, and customer conversations beyond CRM notes (medium); skipped, the Chats API only answers requests signed with a registered chat channel's secret, and only for chats of that channel, so the account's long-lived token cannot read it
+- [x] `Catalog custom fields (GET /api/v4/catalogs/{list_id}/custom_fields)` — interpret custom fields and enum values when importing catalog records (medium). Added as `CatalogCustomFields` (fan-out over `Catalogs`).
Note: Parsed the official llms.txt operation index and OpenAPI blocks in linked Markdown references. Pipelines embed stages; Leads request loss reasons and source names; Users request roles and groups. Catalog records and entity links need requests for each parent ID. Chat history uses the separate Chats API and requires channel credentials.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/canonical_descriptions.py
index 45137746799d..cc308319d438 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/canonical_descriptions.py
@@ -56,6 +56,16 @@ _CUSTOM_FIELD_COLUMNS = {
"account_id": "Identifier of the Kommo account the field belongs to.",
}
+_FIELD_GROUP_COLUMNS = {
+ "id": "Identifier of the field group. The predefined main group is `default` on every entity.",
+ "name": "Field group name as shown in the entity card.",
+ "sort": "Position of the group in the entity card.",
+ "entity_type": "Entity the group belongs to (leads, contacts, or companies).",
+ "is_predefined": "Whether the group is a built-in group. Built-in groups cannot be deleted.",
+ "type": "Group type: custom_field_group for field groups, linked_group for product groups.",
+ "fields": "Identifiers of the custom fields shown in the group.",
+}
+
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"Leads": {
"description": "A sales opportunity moving through a Kommo pipeline, with its stage, price, and owner.",
@@ -187,6 +197,32 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"account_id": "Identifier of the Kommo account the list belongs to.",
},
},
+ "CatalogElements": {
+ "description": "An element of a Kommo list (catalog), such as a product in a product list.",
+ "docs_url": "https://developers.kommo.com/reference/get-list-elements",
+ "columns": {
+ "id": "Unique identifier of the list element.",
+ "catalog_id": "Identifier of the list the element belongs to.",
+ "name": "Element name.",
+ "created_by": "Identifier of the user who created the element.",
+ "updated_by": "Identifier of the user who last changed the element.",
+ "created_at": "When the element was created, as a Unix timestamp.",
+ "updated_at": "When the element was last changed, as a Unix timestamp.",
+ "is_deleted": "Whether the element is deleted.",
+ "custom_fields_values": "Custom field values as a list of {field_id, field_name, field_code, field_type, values}.",
+ "account_id": "Identifier of the Kommo account the element belongs to.",
+ },
+ },
+ "CatalogCustomFields": {
+ "description": "Custom field definitions available on the elements of a Kommo list (catalog).",
+ "docs_url": "https://developers.kommo.com/reference/custom-fields-list-of-list",
+ "columns": {
+ **_CUSTOM_FIELD_COLUMNS,
+ "catalog_id": "Identifier of the list the field belongs to.",
+ "entity_type": "Entity the field belongs to. Always catalogs for list fields.",
+ "remind": "Reminder setting for birthday fields: never, day, week, or month before the date.",
+ },
+ },
"LeadCustomFields": {
"description": "Custom field definitions available on leads.",
"docs_url": "https://developers.kommo.com/reference/custom-field-by-entity",
@@ -202,6 +238,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://developers.kommo.com/reference/custom-field-by-entity",
"columns": _CUSTOM_FIELD_COLUMNS,
},
+ "LeadCustomFieldGroups": {
+ "description": "Groups that organize the custom fields of leads into tabs of the lead card.",
+ "docs_url": "https://developers.kommo.com/reference/list-of-entity-field-groups",
+ "columns": _FIELD_GROUP_COLUMNS,
+ },
+ "ContactCustomFieldGroups": {
+ "description": "Groups that organize the custom fields of contacts into tabs of the contact card.",
+ "docs_url": "https://developers.kommo.com/reference/list-of-entity-field-groups",
+ "columns": _FIELD_GROUP_COLUMNS,
+ },
+ "CompanyCustomFieldGroups": {
+ "description": "Groups that organize the custom fields of companies into tabs of the company card.",
+ "docs_url": "https://developers.kommo.com/reference/list-of-entity-field-groups",
+ "columns": _FIELD_GROUP_COLUMNS,
+ },
"LeadTags": {
"description": "Tags that can be applied to leads.",
"docs_url": "https://developers.kommo.com/reference/list-of-entity-tags",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/kommo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/kommo.py
index 75227148813e..428fb8d443bd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/kommo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/kommo.py
@@ -5,7 +5,10 @@ from typing import Any, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
- rest_api_resource,
+ rest_api_resources,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ rename_parent_fields,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
BasePaginator,
@@ -13,7 +16,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ Endpoint,
+ EndpointResource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.kommo.settings import (
ENDPOINT_CONFIG,
@@ -27,13 +33,17 @@ REQUEST_TIMEOUT = (10.0, 60.0)
KOMMO_DOMAIN = "kommo.com"
+CATALOG_PARENT = "catalogs"
+
# A single DNS label: Kommo accounts live at https://<subdomain>.kommo.com.
_SUBDOMAIN_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
[email protected](frozen=True)
class KommoResumeConfig:
- page: int
+ page: int | None = None
+ # Per-catalog progress of a fan-out endpoint, in the framework's dependent-resource shape.
+ fanout_state: dict[str, Any] | None = None
def normalize_subdomain(raw: str) -> str | None:
@@ -80,21 +90,44 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResou
"initial_value": 0,
}
- return {
+ endpoint_config: Endpoint = {
+ "path": endpoint.path,
+ "data_selector": endpoint.data_selector,
+ "params": params,
+ "paginator": _build_paginator(endpoint),
+ }
+ if endpoint.per_catalog:
+ params["catalog_id"] = {"type": "resolve", "resource": CATALOG_PARENT, "field": "id"}
+ # A catalog deleted after the Catalogs listing answers 404; skip it rather than fail the run.
+ endpoint_config["response_actions"] = [{"status_code": 404, "action": "ignore"}]
+
+ resource: EndpointResource = {
"name": name,
"table_name": name.lower(),
"primary_key": endpoint.primary_key,
"write_disposition": {"disposition": "merge", "strategy": "upsert"}
if incremental_param is not None
else "replace",
- "endpoint": {
- "path": endpoint.path,
- "data_selector": endpoint.data_selector,
- "params": params,
- "paginator": _build_paginator(endpoint),
- },
+ "endpoint": endpoint_config,
"table_format": "delta",
}
+ if endpoint.per_catalog:
+ # Catalog custom fields don't name their catalog, and both ids are only unique per catalog
+ # as far as the docs go, so every row carries the catalog it was read from.
+ resource["include_from_parent"] = ["id"]
+ resource["data_map"] = rename_parent_fields(CATALOG_PARENT, {"id": "catalog_id"})
+ return resource
+
+
+def get_resources(name: str, should_use_incremental_field: bool) -> list[EndpointResource | str]:
+ resource = get_resource(name, should_use_incremental_field)
+ if not ENDPOINT_CONFIG[name].per_catalog:
+ return [resource]
+
+ parent = get_resource("Catalogs", should_use_incremental_field=False)
+ parent["name"] = CATALOG_PARENT
+ parent["table_name"] = CATALOG_PARENT
+ return [parent, resource]
def kommo_source(
@@ -116,8 +149,9 @@ def kommo_source(
"allow_redirects": False,
"request_timeout": REQUEST_TIMEOUT,
},
- "resources": [get_resource(endpoint, should_use_incremental_field)],
+ "resources": get_resources(endpoint, should_use_incremental_field),
}
+ per_catalog = ENDPOINT_CONFIG[endpoint].per_catalog
# A page number is only a safe resume point on a full-refresh run. Incremental runs advance
# `filter[updated_at][from]` after every batch, so page N of the resumed query is not page N
@@ -130,14 +164,21 @@ def kommo_source(
if resume_enabled and resumable_source_manager.can_resume():
resume_config = resumable_source_manager.load_state()
if resume_config is not None:
- initial_paginator_state = {"page": resume_config.page}
+ if per_catalog:
+ initial_paginator_state = resume_config.fanout_state
+ elif resume_config.page is not None:
+ initial_paginator_state = {"page": resume_config.page}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if not state:
+ return
+ if per_catalog:
+ resumable_source_manager.save_state(KommoResumeConfig(fanout_state=state))
# Only persist while there's a next page to come back to; the Redis TTL cleans up.
- if state and state.get("page") is not None:
+ elif state.get("page") is not None:
resumable_source_manager.save_state(KommoResumeConfig(page=int(state["page"])))
- return rest_api_resource(
+ resources = rest_api_resources(
config,
team_id,
job_id,
@@ -145,6 +186,7 @@ def kommo_source(
resume_hook=save_checkpoint if resume_enabled else None,
initial_paginator_state=initial_paginator_state,
)
+ return next(resource for resource in resources if resource.name == endpoint)
def validate_credentials(api_key: str, subdomain: str) -> tuple[bool, str | None]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/settings.py
index 963fd4c29e6d..723440123145 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/settings.py
@@ -20,6 +20,9 @@ class KommoEndpoint:
params: dict[str, Any] = field(default_factory=dict)
# Endpoints that return their whole collection in one unpaginated response.
paginated: bool = True
+ # Endpoints read once per catalog: `path` carries a `{catalog_id}` placeholder that the
+ # fan-out over the Catalogs endpoint fills in.
+ per_catalog: bool = False
# `order[updated_at]=asc` is set on every endpoint that documents it, so rows arrive in the
@@ -98,6 +101,18 @@ ENDPOINT_CONFIG: dict[str, KommoEndpoint] = {
data_selector="_embedded.catalogs",
primary_key=["id"],
),
+ "CatalogElements": KommoEndpoint(
+ path=f"{API_BASE_PATH}/catalogs/{{catalog_id}}/elements",
+ data_selector="_embedded.elements",
+ primary_key=["catalog_id", "id"],
+ per_catalog=True,
+ ),
+ "CatalogCustomFields": KommoEndpoint(
+ path=f"{API_BASE_PATH}/catalogs/{{catalog_id}}/custom_fields",
+ data_selector="_embedded.custom_fields",
+ primary_key=["catalog_id", "id"],
+ per_catalog=True,
+ ),
"LeadCustomFields": KommoEndpoint(
path=f"{API_BASE_PATH}/leads/custom_fields",
data_selector="_embedded.custom_fields",
@@ -113,6 +128,25 @@ ENDPOINT_CONFIG: dict[str, KommoEndpoint] = {
data_selector="_embedded.custom_fields",
primary_key=["id"],
),
+ # Field group ids are strings, and the predefined group is `default` for every entity type.
+ "LeadCustomFieldGroups": KommoEndpoint(
+ path=f"{API_BASE_PATH}/leads/custom_fields/groups",
+ data_selector="_embedded.custom_field_groups",
+ primary_key=["id"],
+ paginated=False,
+ ),
+ "ContactCustomFieldGroups": KommoEndpoint(
+ path=f"{API_BASE_PATH}/contacts/custom_fields/groups",
+ data_selector="_embedded.custom_field_groups",
+ primary_key=["id"],
+ paginated=False,
+ ),
+ "CompanyCustomFieldGroups": KommoEndpoint(
+ path=f"{API_BASE_PATH}/companies/custom_fields/groups",
+ data_selector="_embedded.custom_field_groups",
+ primary_key=["id"],
+ paginated=False,
+ ),
"LeadTags": KommoEndpoint(
path=f"{API_BASE_PATH}/leads/tags",
data_selector="_embedded.tags",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/tests/test_kommo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/tests/test_kommo.py
index 4e5602a576a0..c2c06712dc00 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/tests/test_kommo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kommo/tests/test_kommo.py
@@ -123,6 +123,89 @@ class TestKommoSourceTransport:
assert result.params("page") == ["7"]
assert [row["id"] for row in result.rows] == [9]
+ @staticmethod
+ def _catalogs_page(ids: list[int]) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={"_page": 1, "_page_count": 1, "_embedded": {"catalogs": [{"id": cid} for cid in ids]}}
+ )
+
+ @staticmethod
+ def _elements_page(ids: list[int], page: int, page_count: int) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={
+ "_page": page,
+ "_page_count": page_count,
+ "_embedded": {"elements": [{"id": element_id, "name": f"Item {element_id}"} for element_id in ids]},
+ }
+ )
+
+ def test_catalog_elements_fan_out_over_every_catalog_and_tag_rows_with_it(self) -> None:
+ result = self._driver().run(
+ "CatalogElements",
+ [
+ self._catalogs_page([11, 22, 33]),
+ self._elements_page([1], page=1, page_count=2),
+ self._elements_page([2], page=2, page_count=2),
+ # Kommo answers an empty catalog with 204 No Content.
+ ScriptedResponse(status=204),
+ # A catalog deleted after the listing must not fail the whole fan-out.
+ ScriptedResponse(status=404, json={"title": "Not found"}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/api/v4/catalogs",
+ "/api/v4/catalogs/11/elements",
+ "/api/v4/catalogs/11/elements",
+ "/api/v4/catalogs/22/elements",
+ "/api/v4/catalogs/33/elements",
+ ]
+ assert result.params("page") == ["1", "1", "2", "1", "1"]
+ assert [(row["catalog_id"], row["id"]) for row in result.rows] == [(11, 1), (11, 2)]
+ assert all("_catalogs_id" not in row for row in result.rows)
+ assert result.final_state == KommoResumeConfig(
+ fanout_state={
+ "completed": [
+ "/api/v4/catalogs/11/elements",
+ "/api/v4/catalogs/22/elements",
+ "/api/v4/catalogs/33/elements",
+ ],
+ "current": None,
+ "child_state": None,
+ }
+ )
+ assert result.response is not None
+ assert result.response.primary_keys == ["catalog_id", "id"]
+
+ def test_catalog_fan_out_resumes_mid_catalog_and_skips_finished_ones(self) -> None:
+ result = self._driver().run(
+ "CatalogCustomFields",
+ [
+ self._catalogs_page([11, 22, 33]),
+ ScriptedResponse(
+ json={"_page": 3, "_page_count": 3, "_embedded": {"custom_fields": [{"id": 7, "type": "text"}]}}
+ ),
+ ScriptedResponse(status=204),
+ ],
+ resume_state=KommoResumeConfig(
+ fanout_state={
+ "completed": ["/api/v4/catalogs/11/custom_fields"],
+ "current": "/api/v4/catalogs/22/custom_fields",
+ "child_state": {"page": 3},
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/api/v4/catalogs",
+ "/api/v4/catalogs/22/custom_fields",
+ "/api/v4/catalogs/33/custom_fields",
+ ]
+ assert result.params("page") == ["1", "3", "1"]
+ assert [(row["catalog_id"], row["id"]) for row in result.rows] == [(22, 7)]
+
class TestValidateCredentials:
@pytest.mark.parametrize(