Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 6 missing tables to the Freshdesk source (#125173)

Tom Owers9875ce8ee9e7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8c20831aa832..c97e487e4e3e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -4967,10 +4967,10 @@ Diffed against: <https://developers.freshdesk.com/api/>
- [x] `/api/v2/contact_fields` — lookup resolving custom contact field IDs and their dropdown choices (high)
- [x] `/api/v2/canned_response_folders/{id}/responses` — the canned responses themselves; only their folders are synced today (high)
- [x] `/api/v2/solutions/categories (+ /categories/{id}/folders, /folders/{id}/articles)` — knowledge base hierarchy and article stats (hits, thumbs up/down) for deflection analysis (high)
-- [ ] `/api/v2/company_fields` — lookup resolving custom company field IDs and choices (medium)
-- [ ] `/api/v2/custom_objects/schemas (+ /schemas/{id}/records)` — customer-defined objects linked to tickets; schemas act as the lookup for the records (medium)
-- [ ] `/api/v2/customer-satisfaction/surveys/{survey_id}/responses` — new-style CSAT responses; only legacy satisfaction_ratings are synced (medium)
-- [ ] `/api/v2/ticket-forms` — lookup resolving the form ID on tickets in multi-form portals (medium)
+- [x] `/api/v2/company_fields` — lookup resolving custom company field IDs and choices (medium)
+- [x] `/api/v2/custom_objects/schemas (+ /schemas/{id}/records)` — customer-defined objects linked to tickets; schemas act as the lookup for the records (medium)
+- [x] `/api/v2/customer-satisfaction/surveys/{survey_id}/responses` — new-style CSAT responses; only legacy satisfaction_ratings are synced (medium)
+- [x] `/api/v2/ticket-forms` — lookup resolving the form ID on tickets in multi-form portals (medium)
- [ ] `/api/v2/collaboration/threads (+ /collaboration/messages)` — internal discussion threads attached to tickets - collaboration effort per ticket (medium)
- [ ] `/api/v2/discussions/categories (+ /forums, /topics, /comments)` — community forum activity, another deflection surface (low)
- [ ] `/api/v2/tickets/{id}/watchers` — who is following each ticket - escalation/involvement signal (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/canonical_descriptions.py
index a94077ada5cd..86b7cd7f11ed 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/canonical_descriptions.py
@@ -287,4 +287,84 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
seo_data="Meta title, description and keywords for the article.",
),
},
+ "company_fields": {
+ "description": "A field on the company form, including the choices of a dropdown field.",
+ "docs_url": "https://developers.freshdesk.com/api/#list_all_company_fields",
+ "columns": _columns(
+ name="Internal name of the company field.",
+ label="Display name for the field.",
+ type="Type of value the field holds (e.g. custom_text, custom_dropdown).",
+ position="Position of the field on the form.",
+ default="Whether this is a default field rather than a custom one.",
+ required_for_agents="Whether the field is mandatory for agents.",
+ choices="Available choices, for a dropdown field.",
+ ),
+ },
+ "ticket_forms": {
+ "description": "A ticket form that shows customers the fields for one kind of request.",
+ "docs_url": "https://developers.freshdesk.com/api/#list_all_ticket_forms",
+ "columns": _columns(
+ name="Internal name of the ticket form.",
+ title="Name of the ticket form.",
+ default="Whether this is the default form.",
+ description="Description of the ticket form.",
+ portals="Portals the ticket form is associated with.",
+ last_updated_by="ID of the agent who last updated the form.",
+ ),
+ },
+ "custom_object_schemas": {
+ "description": "A custom object type defined in the account, with the fields its records hold.",
+ "docs_url": "https://developers.freshdesk.com/api/#retrieve_existing_custom_objects",
+ "columns": {
+ "id": "Unique identifier for the custom object schema.",
+ "name": "Name of the custom object.",
+ "prefix": "Prefix of the display IDs of the object's records.",
+ "description": "Description of the custom object.",
+ "version": "Version of the schema.",
+ "deleted": "Whether the custom object has been deleted.",
+ "fields": "Fields defined on the custom object, with their types and choices.",
+ "created_time": "Time at which the schema was created, in epoch milliseconds.",
+ "updated_time": "Time at which the schema was last updated, in epoch milliseconds.",
+ },
+ },
+ "custom_object_records": {
+ "description": "A record of a custom object, such as a booking or an order linked to tickets.",
+ "docs_url": "https://developers.freshdesk.com/api/#retrieve_all_the_records_of_a_custom_object",
+ "columns": {
+ "schema_id": "ID of the custom object schema the record belongs to.",
+ "display_id": "Unique identifier for the record within its custom object.",
+ "data": "Values of the record's fields, keyed by field name.",
+ "metadata": "Metadata about the record, such as the name of its primary field.",
+ "created_time": "Time at which the record was created, in epoch milliseconds.",
+ "updated_time": "Time at which the record was last updated, in epoch milliseconds.",
+ },
+ },
+ "csat_surveys": {
+ "description": "A customer satisfaction survey of the New CSAT feature, with its questions.",
+ "docs_url": "https://developers.freshdesk.com/api/#csat_surveys",
+ "columns": {
+ "id": "Unique identifier for the survey (UUID).",
+ "title": "Title of the survey.",
+ "description": "Description of the survey.",
+ "header_message": "Message shown at the start of the survey.",
+ "state": "State of the survey (ACTIVE, DRAFT, PAUSE or IN_PROGRESS).",
+ "language": "Language code of the survey.",
+ "questions": "Questions in the survey, with their types, scales and choices.",
+ },
+ },
+ "csat_survey_responses": {
+ "description": "A customer's response to a New CSAT survey for a ticket.",
+ "docs_url": "https://developers.freshdesk.com/api/#view_survey_responses",
+ "columns": _columns(
+ id="Identifier for the survey response.",
+ survey_id="ID of the survey the response belongs to.",
+ survey_version_id="ID of the survey version that was answered.",
+ user_id="ID of the contact who responded.",
+ agent_id="ID of the agent the response is about.",
+ group_id="ID of the group the response is about.",
+ ticket_id="ID of the ticket the response is for.",
+ status="Completion status of the response (COMPLETE, PARTIAL_COMPLETE or PARTIAL).",
+ answers="Answers given, each with the question ID, question text and value.",
+ ),
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/freshdesk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/freshdesk.py
index 1c1816c44bcb..f92472bd3fc9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/freshdesk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/freshdesk.py
@@ -2,7 +2,9 @@ import dataclasses
from collections.abc import Callable, Iterable
from datetime import UTC, date, datetime
from typing import Any, Optional, cast
+from urllib.parse import parse_qs, urlsplit
+from requests import Response
from requests.auth import HTTPBasicAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -16,7 +18,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
build_dependent_resource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BaseNextUrlPaginator,
+ BasePaginator,
HeaderLinkPaginator,
+ JSONResponsePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
@@ -29,7 +34,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.temporal.data_imports.sources.freshdesk.settings import (
FRESHDESK_ENDPOINTS,
IGNORE_DELETED_PARENT,
- PER_PAGE,
FreshdeskEndpointConfig,
)
@@ -52,6 +56,52 @@ def _base_url(subdomain: str) -> str:
return f"https://{normalize_subdomain(subdomain)}.freshdesk.com"
+class FreshdeskCustomObjectsPaginator(BaseNextUrlPaginator):
+ """Follows `_links.next.href`, which is relative to `/api/v2/custom_objects`, not to the host."""
+
+ def __init__(self, base_url: str) -> None:
+ super().__init__()
+ self.links_base_url = f"{base_url}/api/v2/custom_objects"
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ try:
+ next_link = response.json().get("_links", {}).get("next")
+ except Exception:
+ next_link = None
+ href = next_link.get("href") if isinstance(next_link, dict) else next_link
+ # The last page can still carry a next link, with an empty `next_token`.
+ if not isinstance(href, str) or not parse_qs(urlsplit(href).query).get("next_token"):
+ self._advance_to(None)
+ return
+ self._advance_to(href if href.startswith("http") else f"{self.links_base_url}{href}")
+
+ def __str__(self) -> str:
+ return "FreshdeskCustomObjectsPaginator"
+
+
+def _paginator(config: FreshdeskEndpointConfig, base_url: str) -> Optional[BasePaginator]:
+ """Returns ``None`` for the client default, the `Link` header paginator."""
+ if config.pagination == "paging_next":
+ return JSONResponsePaginator(next_url_path="paging.next")
+ if config.pagination == "custom_objects":
+ return FreshdeskCustomObjectsPaginator(base_url)
+ return None
+
+
+def _page_size_params(config: FreshdeskEndpointConfig) -> dict[str, Any]:
+ return {config.page_size_param: config.page_size} if config.page_size_param else {}
+
+
+def _endpoint_extra(config: FreshdeskEndpointConfig, base_url: str) -> Endpoint:
+ extra: Endpoint = {}
+ paginator = _paginator(config, base_url)
+ if paginator is not None:
+ extra["paginator"] = paginator
+ if config.data_key:
+ extra["data_selector"] = config.data_key
+ return extra
+
+
def _format_updated_since(value: Any) -> str:
"""Format an incremental cursor value as the ISO 8601 UTC string Freshdesk expects."""
if isinstance(value, datetime):
@@ -67,7 +117,7 @@ def _build_params(
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
) -> dict[str, Any]:
- params: dict[str, Any] = {"per_page": PER_PAGE}
+ params = _page_size_params(config)
params.update(config.extra_params)
if should_use_incremental_field and config.updated_since_param and db_incremental_field_last_value:
@@ -127,12 +177,18 @@ def _fanout_resource(
initial_paginator_state: Optional[dict[str, Any]],
) -> Iterable[Any]:
assert config.fanout is not None
+ parent = FRESHDESK_ENDPOINTS[config.fanout.parent_name]
+ fanout = _windowed_fanout(config.fanout, should_use_incremental_field, db_incremental_field_last_value)
+ # Parent and child can name their page-size param differently, so each side is sized here
+ # instead of through the helper's single shared `page_size_param`.
+ fanout = dataclasses.replace(fanout, parent_params={**_page_size_params(parent), **fanout.parent_params})
+ base_url = client_config["base_url"]
return cast(
Iterable[Any],
build_dependent_resource(
endpoint_configs=FRESHDESK_ENDPOINTS,
child_endpoint=endpoint,
- fanout=_windowed_fanout(config.fanout, should_use_incremental_field, db_incremental_field_last_value),
+ fanout=fanout,
client_config=client_config,
path_format_values={},
team_id=team_id,
@@ -143,7 +199,10 @@ def _fanout_resource(
# No Freshdesk sub-resource takes a timestamp filter, so the cursor binds to no
# request param. The child still merges on its primary key.
incremental_config_factory=lambda _field: None,
- page_size_param="per_page",
+ page_size_param=None,
+ child_params_extra=_page_size_params(config),
+ parent_endpoint_extra=_endpoint_extra(parent, base_url) or None,
+ child_endpoint_extra=_endpoint_extra(config, base_url) or None,
resume_hook=resume_hook,
initial_paginator_state=initial_paginator_state,
),
@@ -265,19 +324,18 @@ def freshdesk_source(
initial_paginator_state,
)
else:
+ endpoint_config: Endpoint = {
+ "path": config.path,
+ "params": _build_params(config, should_use_incremental_field, db_incremental_field_last_value),
+ # Most list endpoints return a bare array; a few (e.g. skills) wrap it in an object.
+ "data_selector": config.data_key,
+ }
+ paginator = _paginator(config, client_config["base_url"])
+ if paginator is not None:
+ endpoint_config["paginator"] = paginator
rest_config: RESTAPIConfig = {
"client": client_config,
- "resources": [
- {
- "name": endpoint,
- "endpoint": {
- "path": config.path,
- "params": _build_params(config, should_use_incremental_field, db_incremental_field_last_value),
- # Most list endpoints return a bare array; a few (e.g. skills) wrap it in an object.
- "data_selector": config.data_key,
- },
- }
- ],
+ "resources": [{"name": endpoint, "endpoint": endpoint_config}],
}
resource = rest_api_resource(
rest_config,
@@ -291,7 +349,7 @@ def freshdesk_source(
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=["id"],
+ primary_keys=config.primary_keys,
partition_count=1 if config.partition_key else None,
partition_size=1 if config.partition_key else None,
partition_mode="datetime" if config.partition_key else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/settings.py
index 0fbffb6e16a9..f55592929561 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/settings.py
@@ -1,7 +1,7 @@
"""Freshdesk source settings and endpoint catalog."""
from dataclasses import dataclass, field
-from typing import Optional
+from typing import Literal, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
DependentEndpointConfig,
@@ -11,6 +11,11 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
PER_PAGE = 100
+# "link_header" is the RFC 5988 `Link` header most v2 endpoints use. "paging_next" is the full
+# next-page URL in the body of the new CSAT APIs. "custom_objects" is the `_links.next.href` of
+# the custom objects API, which is relative to `/api/v2/custom_objects` rather than to the host.
+FreshdeskPagination = Literal["link_header", "paging_next", "custom_objects"]
+
@dataclass(frozen=True)
class FreshdeskChainedFanoutConfig:
@@ -44,6 +49,10 @@ class FreshdeskEndpointConfig:
data_key: Optional[str] = None
# Read by the shared fan-out helper to size parent and child pages.
page_size: int = PER_PAGE
+ # ``None`` for an endpoint that documents no page-size param.
+ page_size_param: Optional[str] = "per_page"
+ pagination: FreshdeskPagination = "link_header"
+ primary_keys: list[str] = field(default_factory=lambda: ["id"])
# Set on a sub-resource that is fetched once per row of a top-level endpoint.
fanout: Optional[DependentEndpointConfig] = None
chained_fanout: Optional[FreshdeskChainedFanoutConfig] = None
@@ -167,6 +176,56 @@ FRESHDESK_ENDPOINTS: dict[str, FreshdeskEndpointConfig] = {
parent_name="solution_folders", resolve_param="folder_id", resolve_field="id"
),
),
+ "company_fields": FreshdeskEndpointConfig(name="company_fields", path="/api/v2/company_fields"),
+ "ticket_forms": FreshdeskEndpointConfig(name="ticket_forms", path="/api/v2/ticket-forms"),
+ "custom_object_schemas": FreshdeskEndpointConfig(
+ name="custom_object_schemas",
+ path="/api/v2/custom_objects/schemas",
+ data_key="schemas",
+ page_size_param=None,
+ ),
+ "custom_object_records": FreshdeskEndpointConfig(
+ name="custom_object_records",
+ path="/api/v2/custom_objects/schemas/{schema_id}/records",
+ data_key="records",
+ page_size_param="page_size",
+ pagination="custom_objects",
+ # `display_id` carries the schema's prefix, but nothing documents it as unique across schemas.
+ primary_keys=["schema_id", "display_id"],
+ fanout=DependentEndpointConfig(
+ parent_name="custom_object_schemas",
+ resolve_param="schema_id",
+ resolve_field="id",
+ # Record rows do not say which schema they belong to.
+ include_from_parent=["id"],
+ parent_field_renames={"id": "schema_id"},
+ child_response_actions=IGNORE_DELETED_PARENT,
+ ),
+ ),
+ "csat_surveys": FreshdeskEndpointConfig(
+ name="csat_surveys",
+ path="/api/v2/customer-satisfaction/surveys",
+ data_key="data",
+ pagination="paging_next",
+ ),
+ "csat_survey_responses": FreshdeskEndpointConfig(
+ name="csat_survey_responses",
+ path="/api/v2/customer-satisfaction/surveys/{survey_id}/responses",
+ data_key="data",
+ pagination="paging_next",
+ partition_key="created_at",
+ primary_keys=["survey_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="csat_surveys",
+ resolve_param="survey_id",
+ resolve_field="id",
+ # Response rows carry `survey_id` too, but it is part of the primary key, so take it
+ # from the parent where it can never be missing.
+ include_from_parent=["id"],
+ parent_field_renames={"id": "survey_id"},
+ child_response_actions=IGNORE_DELETED_PARENT,
+ ),
+ ),
}
ENDPOINTS = tuple(FRESHDESK_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/tests/test_freshdesk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/tests/test_freshdesk.py
index 93880554a279..c4c63096088e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/tests/test_freshdesk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/tests/test_freshdesk.py
@@ -54,19 +54,40 @@ class TestRequestParams:
assert result.requests[0].param("_updated_since") == "2026-03-04T00:00:00Z"
+TICKETS_PAGE_2 = "https://acme.freshdesk.com/api/v2/tickets?per_page=100&page=2"
+SURVEYS_PAGE_2 = "https://acme.freshdesk.com/api/v2/customer-satisfaction/surveys?per_page=100&next_token=t2"
+
+
class TestPagination:
- def test_paginates_via_link_header_and_saves_state(self) -> None:
- next_url = "https://acme.freshdesk.com/api/v2/tickets?per_page=100&page=2"
- result = _driver().run(
- "tickets",
- [
- ScriptedResponse(json=[{"id": 1}], headers={"Link": f'<{next_url}>; rel="next"'}),
- ScriptedResponse(json=[{"id": 2}]),
- ],
- )
+ @pytest.mark.parametrize(
+ "endpoint, script, next_url",
+ [
+ (
+ "tickets",
+ [
+ ScriptedResponse(json=[{"id": 1}], headers={"Link": f'<{TICKETS_PAGE_2}>; rel="next"'}),
+ ScriptedResponse(json=[{"id": 2}]),
+ ],
+ TICKETS_PAGE_2,
+ ),
+ (
+ "csat_surveys",
+ [
+ ScriptedResponse(json={"data": [{"id": 1}], "paging": {"next": SURVEYS_PAGE_2, "previous": None}}),
+ ScriptedResponse(json={"data": [{"id": 2}], "paging": {"next": None, "previous": SURVEYS_PAGE_2}}),
+ ],
+ SURVEYS_PAGE_2,
+ ),
+ ],
+ )
+ def test_paginates_to_the_last_page_and_saves_state(
+ self, endpoint: str, script: list[ScriptedResponse], next_url: str
+ ) -> None:
+ result = _driver().run(endpoint, script)
assert result.raised is None
assert [row["id"] for row in result.rows] == [1, 2]
+ assert result.urls[1] == next_url
# State saved once, after the first (only non-terminal) page.
assert result.saved_states == [FreshdeskResumeConfig(next_url=next_url)]
@@ -144,3 +165,80 @@ class TestFanout:
"/api/v2/solutions/categories/3/folders",
"/api/v2/solutions/folders/4/articles",
]
+
+ @pytest.mark.parametrize(
+ "endpoint, script, expected_paths, parent_key, parent_value",
+ [
+ (
+ "custom_object_records",
+ [
+ ScriptedResponse(json={"schemas": [{"id": 30}]}),
+ ScriptedResponse(
+ json={
+ "records": [{"display_id": "BKG-1"}],
+ "_links": {"next": {"href": "/schemas/30/records?page_size=100&next_token=t2"}},
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "records": [{"display_id": "BKG-2"}],
+ "_links": {"next": {"href": "/schemas/30/records?page_size=100&prev_token=t2&next_token="}},
+ }
+ ),
+ ],
+ ["/api/v2/custom_objects/schemas"] + ["/api/v2/custom_objects/schemas/30/records"] * 2,
+ "schema_id",
+ 30,
+ ),
+ (
+ "custom_object_records",
+ [
+ ScriptedResponse(json={"schemas": [{"id": 30}]}),
+ ScriptedResponse(
+ json={
+ "records": [{"display_id": "BKG-1"}],
+ "_links": {"next": {"href": "/schemas/30/records?page_size=100&next_token=t2"}},
+ }
+ ),
+ ScriptedResponse(json={"records": [{"display_id": "BKG-2"}], "_links": {}}),
+ ],
+ ["/api/v2/custom_objects/schemas"] + ["/api/v2/custom_objects/schemas/30/records"] * 2,
+ "schema_id",
+ 30,
+ ),
+ (
+ "csat_survey_responses",
+ [
+ ScriptedResponse(json={"data": [{"id": "s-1"}], "paging": {"next": None}}),
+ ScriptedResponse(
+ json={
+ "data": [{"id": "r-1"}],
+ "paging": {
+ "next": "https://acme.freshdesk.com/api/v2/customer-satisfaction/surveys/s-1/responses?per_page=100&next_token=t2"
+ },
+ }
+ ),
+ ScriptedResponse(json={"data": [{"id": "r-2", "survey_id": "s-1"}], "paging": {"next": None}}),
+ ],
+ ["/api/v2/customer-satisfaction/surveys"] + ["/api/v2/customer-satisfaction/surveys/s-1/responses"] * 2,
+ "survey_id",
+ "s-1",
+ ),
+ ],
+ )
+ def test_fan_out_pages_the_child_through_its_body_link(
+ self,
+ endpoint: str,
+ script: list[ScriptedResponse],
+ expected_paths: list[str],
+ parent_key: str,
+ parent_value: Any,
+ ) -> None:
+ result = _driver().run(endpoint, script)
+
+ assert result.raised is None
+ assert result.paths == expected_paths
+ assert result.requests[2].param("next_token") == "t2"
+ assert len(result.rows) == 2
+ # The parent id is part of the child's primary key, so every row must carry it.
+ assert {row[parent_key] for row in result.rows} == {parent_value}