Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the SolarwindsServiceDesk source (#120210)

Tom Owersfe64c8f5c5c5
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index dad3d1706c03..5291efa32790 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1050,8 +1050,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/snowplow/snowplow.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/snyk/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/snyk/snyk.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/solarwinds_service_desk.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/sonar_cloud/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/sonar_cloud/sonar_cloud.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/sonarqube/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index a771f64fd5a8..93dfd0b25fa9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11373,10 +11373,10 @@ Today (12): `catalog_items`, `changes`, `departments`, `groups`, `hardwares`, `i
Diffed against: <https://apidoc.samanage.com/redoc/schema/resolved_schema.json>
-- [ ] `categories` — lookup table resolving the category/subcategory IDs carried on incidents, changes and problems we already sync (high)
-- [ ] `{object_type}/{id}/audits (and /audits)` — per-ticket state/field transition history - the only way to compute time-in-state, reassignment and SLA breach analytics (high)
-- [ ] `{object_type}/{id}/time_tracks` — time logged per incident/change/problem; core effort and cost-per-ticket metric (high)
-- [ ] `configuration_items` — CMDB configuration items that incidents and changes reference; joins tickets to affected infrastructure (high)
+- [x] `categories` — lookup table resolving the category/subcategory IDs carried on incidents, changes and problems we already sync (high)
+- [x] `{object_type}/{id}/audits (and /audits)` — per-ticket state/field transition history - the only way to compute time-in-state, reassignment and SLA breach analytics (high)
+- [x] `{object_type}/{id}/time_tracks` — time logged per incident/change/problem; core effort and cost-per-ticket metric (high)
+- [x] `configuration_items` — CMDB configuration items that incidents and changes reference; joins tickets to affected infrastructure (high)
- [ ] `roles` — lookup resolving the role IDs on users we already sync (medium)
- [ ] `vendors` — lookup for the vendor IDs on hardwares, other_assets, contracts and purchase_orders (medium)
- [ ] `contracts` — asset contracts with cost and renewal dates; underpins spend and renewal reporting (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
index 82709ed1494a..e11fe8cf0e26 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
@@ -151,6 +151,7 @@ def _make_paginate_dependent_resource(
initial_state: Optional[dict[str, Any]] = None,
data_selector_required: bool = False,
data_selector_empty_ok: bool = False,
+ data_selector_malformed_retryable: bool = False,
on_parent_error: Optional[Callable[[str, Exception], None]] = None,
parent_source: FanoutParentSource = "api",
) -> Callable[..., Iterator[list[Any]]]:
@@ -243,6 +244,7 @@ def _make_paginate_dependent_resource(
initial_paginator_state=child_initial,
data_selector_required=data_selector_required,
data_selector_empty_ok=data_selector_empty_ok,
+ data_selector_malformed_retryable=data_selector_malformed_retryable,
):
if parent_record:
for child_record in child_page:
@@ -496,6 +498,7 @@ def create_resources(
initial_state=dependent_initial_state,
data_selector_required=bool(endpoint_config.get("data_selector_required")),
data_selector_empty_ok=bool(endpoint_config.get("data_selector_empty_ok")),
+ data_selector_malformed_retryable=bool(endpoint_config.get("data_selector_malformed_retryable")),
on_parent_error=on_parent_error,
parent_source=parent_source,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_fanout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_fanout.py
index a731891e53ae..af02df9acfa9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_fanout.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_fanout.py
@@ -466,6 +466,7 @@ class _FakeResumableClient:
initial_paginator_state=None,
data_selector_required=False,
data_selector_empty_ok=False,
+ data_selector_malformed_retryable=False,
):
pages = self.pages_by_path[path]
start = initial_paginator_state["page"] if initial_paginator_state else 0
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/canonical_descriptions.py
index 11f87a4ce7f7..32ef79a96b18 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/canonical_descriptions.py
@@ -166,4 +166,68 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "When the asset was last updated (ISO 8601).",
},
},
+ "categories": {
+ "description": "Categories and subcategories used to classify incidents, problems, and changes.",
+ "docs_url": "https://apidoc.samanage.com/#tag/Category",
+ "columns": {
+ "id": "Unique identifier for the category.",
+ "name": "Name of the category.",
+ "parent_id": "Id of the parent category. Set on subcategories only.",
+ "default_tags": "Tags applied by default to records in this category.",
+ "default_assignee_id": "Id of the user that records in this category are assigned to by default.",
+ "default_group_assignee_id": "Id of the group that records in this category are assigned to by default.",
+ },
+ },
+ "configuration_items": {
+ "description": "Configuration items (CMDB records) and the incidents, problems, changes, and releases that reference them.",
+ "docs_url": "https://apidoc.samanage.com/#tag/Configuration-Item",
+ "columns": {
+ "id": "Unique identifier for the configuration item.",
+ "name": "Name of the configuration item.",
+ "description": "Description of the configuration item.",
+ "type": "Configuration item type, with its id, name, and parent type.",
+ "type_id": "Id of the configuration item type.",
+ "state": "Current state of the configuration item.",
+ "manager_id": "Id of the user who manages the configuration item.",
+ "user_id": "Id of the user the configuration item belongs to.",
+ "site": "Site the configuration item is associated with.",
+ "department": "Department the configuration item is associated with.",
+ "tags": "Tags applied to the configuration item.",
+ "incidents": "Incidents linked to the configuration item.",
+ "changes": "Changes linked to the configuration item.",
+ "problems": "Problems linked to the configuration item.",
+ "releases": "Releases linked to the configuration item.",
+ "created_at": "When the configuration item was created (ISO 8601).",
+ "updated_at": "When the configuration item was last updated (ISO 8601).",
+ },
+ },
+ "audits": {
+ "description": "Audit trail entries that record each change to service desk records, such as state and assignment changes.",
+ "docs_url": "https://apidoc.samanage.com/#tag/Audit",
+ "columns": {
+ "id": "Unique identifier for the audit entry.",
+ "message": "Description of the change (e.g. State changed from New to On Hold).",
+ "action": "Type of action that was audited (e.g. Update).",
+ "source_type": "Type of the record that changed (e.g. Incident).",
+ "source_id": "Id of the record that changed.",
+ "user_id": "Id of the user who made the change.",
+ "user": "User who made the change.",
+ "note": "Note attached to the audit entry.",
+ "created_at": "When the change happened (ISO 8601).",
+ },
+ },
+ "time_tracks": {
+ "description": "Time logged against incidents, one row per time entry.",
+ "docs_url": "https://apidoc.samanage.com/#tag/Time-Track",
+ "columns": {
+ "id": "Unique identifier for the time entry.",
+ "incident_id": "Id of the incident the time was logged against.",
+ "name": "Name or note of the time entry.",
+ "minutes": "Number of minutes logged.",
+ "creator": "User who logged the time.",
+ "parent": "Record the time entry belongs to, with its id and class.",
+ "created_at": "When the time entry was created (ISO 8601).",
+ "updated_at": "When the time entry was last updated (ISO 8601).",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/settings.py
index b5e845c5b958..454ffc47206c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/settings.py
@@ -1,6 +1,11 @@
-from dataclasses import dataclass, field
+from dataclasses import field
from typing import Optional
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# The API docs' pagination examples request 100 rows per page; no hard maximum is documented, so we
@@ -20,7 +25,7 @@ def _updated_at_incremental_fields() -> list[IncrementalField]:
]
-@dataclass
+@frozen
class SolarwindsServiceDeskEndpointConfig:
name: str
path: str # Path on the regional API host, e.g. "/incidents.json"
@@ -34,8 +39,12 @@ class SolarwindsServiceDeskEndpointConfig:
# actually includes `created_at`.
partition_key: Optional[str] = None
incremental_fields: list[IncrementalField] = field(default_factory=list)
- # Record ids are account-unique integers, so `id` is a safe primary key on every endpoint.
+ # Record ids are account-unique integers, so `id` is a safe primary key on every top-level endpoint.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ # Set on child endpoints that only exist nested under a parent record.
+ fanout: Optional[DependentEndpointConfig] = None
+ page_size: int = PER_PAGE
+ default_incremental_field: Optional[str] = None
SOLARWINDS_SERVICE_DESK_ENDPOINTS: dict[str, SolarwindsServiceDeskEndpointConfig] = {
@@ -105,6 +114,42 @@ SOLARWINDS_SERVICE_DESK_ENDPOINTS: dict[str, SolarwindsServiceDeskEndpointConfig
wrapper_key="other_asset",
partition_key="created_at",
),
+ "categories": SolarwindsServiceDeskEndpointConfig(
+ name="categories",
+ path="/categories.json",
+ wrapper_key="category",
+ ),
+ "configuration_items": SolarwindsServiceDeskEndpointConfig(
+ name="configuration_items",
+ path="/configuration_items.json",
+ wrapper_key="configuration_item",
+ partition_key="created_at",
+ ),
+ # The account-wide audit list covers every record type, so the per-record
+ # /{object_type}/{id}/audits endpoint is not needed.
+ "audits": SolarwindsServiceDeskEndpointConfig(
+ name="audits",
+ path="/audits.json",
+ wrapper_key="audit",
+ partition_key="created_at",
+ ),
+ # Time tracks have no account-wide list; they are only reachable under each incident.
+ "time_tracks": SolarwindsServiceDeskEndpointConfig(
+ name="time_tracks",
+ path="/incidents/{incident_id}/time_tracks.json",
+ wrapper_key="time_track",
+ partition_key="created_at",
+ primary_keys=["incident_id", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="incidents",
+ resolve_param="incident_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "incident_id"},
+ # An incident deleted between the parent listing and its child fetch has no time tracks to sync.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(SOLARWINDS_SERVICE_DESK_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/solarwinds_service_desk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/solarwinds_service_desk.py
index 816230cd27a6..c01dd3cbe4b0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/solarwinds_service_desk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/solarwinds_service_desk.py
@@ -1,4 +1,3 @@
-import dataclasses
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import urlencode
@@ -6,14 +5,21 @@ from urllib.parse import urlencode
from dateutil import parser as dateutil_parser
from requests import Response
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
PageNumberPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -38,13 +44,15 @@ DEFAULT_PROBE_PATH = "/users.json"
TOTAL_PAGES_HEADER = "X-Total-Pages"
+@frozen
class SolarwindsServiceDeskResumeConfig:
# Next 1-indexed page to fetch. None means start from page 1.
next_page: int | None = None
# `updated_from` filter computed when the run started. Persisted so a resumed run issues the
# exact same query instead of recomputing a window whose page boundaries would shift mid-crawl.
updated_from: str | None = None
+ # Fan-out progress for child endpoints: completed parent paths plus the in-progress child page.
+ fanout_state: dict[str, Any] | None = None
def base_url(region: Optional[str]) -> str:
@@ -82,15 +90,17 @@ def _format_updated_from(value: Any) -> Optional[str]:
return aware.astimezone(UTC).strftime("%Y-%m-%dT%H:%M")
-def _unwrap_row(item: dict[str, Any], wrapper_key: str) -> dict[str, Any]:
+def _unwrap_row(item: dict[str, Any], wrapper_key: str, parent_keys: frozenset[str] = frozenset()) -> dict[str, Any]:
"""Normalize a list item to a bare record dict.
The official response samples are inconsistent: some list endpoints show bare records while
others wrap each row under its singular resource name (e.g. ``{"problem": {...}}``). A real
record is never a single-key dict of its own singular name, so unwrapping is unambiguous.
+ ``parent_keys`` are the fields a fan-out copies onto each child row, which sit beside the wrapper.
"""
- if set(item.keys()) == {wrapper_key} and isinstance(item[wrapper_key], dict):
- return item[wrapper_key]
+ wrapped = item.get(wrapper_key)
+ if isinstance(wrapped, dict) and set(item.keys()) - parent_keys == {wrapper_key}:
+ return {**wrapped, **{key: item[key] for key in parent_keys if key in item}}
return item
@@ -128,6 +138,9 @@ def solarwinds_service_desk_source(
config = SOLARWINDS_SERVICE_DESK_ENDPOINTS[endpoint]
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if config.fanout is not None:
+ return _fanout_source_response(region, api_token, endpoint, team_id, job_id, resumable_source_manager, resume)
+
initial_paginator_state: Optional[dict[str, Any]] = None
if resume is not None and resume.next_page:
initial_paginator_state = {"page": resume.next_page}
@@ -140,18 +153,7 @@ def solarwinds_service_desk_source(
updated_from = _format_updated_from(db_incremental_field_last_value)
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": base_url(region),
- # Only the non-secret Accept header goes here; auth rides the framework auth config so
- # its value is redacted from logs and raised error messages.
- "headers": {"Accept": ACCEPT_HEADER},
- "auth": {
- "type": "api_key",
- "api_key": f"Bearer {api_token}",
- "name": "X-Samanage-Authorization",
- "location": "header",
- },
- },
+ "client": _client_config(region, api_token),
"resource_defaults": {},
"resources": [
{
@@ -190,6 +192,26 @@ def solarwinds_service_desk_source(
initial_paginator_state=initial_paginator_state,
)
+ return _source_response(endpoint, resource)
+
+
+def _client_config(region: Optional[str], api_token: str) -> ClientConfig:
+ return {
+ "base_url": base_url(region),
+ # Only the non-secret Accept header goes here; auth rides the framework auth config so
+ # its value is redacted from logs and raised error messages.
+ "headers": {"Accept": ACCEPT_HEADER},
+ "auth": {
+ "type": "api_key",
+ "api_key": f"Bearer {api_token}",
+ "name": "X-Samanage-Authorization",
+ "location": "header",
+ },
+ }
+
+
+def _source_response(endpoint: str, resource: Any) -> SourceResponse:
+ config = SOLARWINDS_SERVICE_DESK_ENDPOINTS[endpoint]
return SourceResponse(
name=endpoint,
items=lambda: resource,
@@ -206,6 +228,56 @@ def solarwinds_service_desk_source(
)
+def _fanout_source_response(
+ region: Optional[str],
+ api_token: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[SolarwindsServiceDeskResumeConfig],
+ resume: Optional[SolarwindsServiceDeskResumeConfig],
+) -> SourceResponse:
+ config = SOLARWINDS_SERVICE_DESK_ENDPOINTS[endpoint]
+ fanout = config.fanout
+ assert fanout is not None
+ parent_wrapper_key = SOLARWINDS_SERVICE_DESK_ENDPOINTS[fanout.parent_name].wrapper_key
+ # Renames run before the child unwrap, so the unwrap must keep the renamed parent fields.
+ parent_keys = frozenset(
+ fanout.parent_field_renames.get(field, f"_{fanout.parent_name}_{field}") for field in fanout.include_from_parent
+ )
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state is not None:
+ resumable_source_manager.save_state(SolarwindsServiceDeskResumeConfig(fanout_state=state))
+
+ dependent = build_dependent_resource(
+ endpoint_configs=SOLARWINDS_SERVICE_DESK_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=fanout,
+ client_config=_client_config(region, api_token),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={
+ "paginator": SolarwindsPageNumberPaginator(base_page=1),
+ "data_selector_malformed_retryable": True,
+ },
+ child_endpoint_extra={
+ "paginator": SolarwindsPageNumberPaginator(base_page=1),
+ "data_selector_malformed_retryable": True,
+ },
+ parent_data_map=lambda item: _unwrap_row(item, parent_wrapper_key),
+ page_size_param="per_page",
+ resume_hook=save_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume is not None else None,
+ )
+ assert isinstance(dependent, Resource)
+ child = dependent.add_map(lambda item: _unwrap_row(item, config.wrapper_key, parent_keys))
+
+ return _source_response(endpoint, child)
+
+
def validate_credentials(region: Optional[str], api_token: str, path: str | None = None) -> tuple[bool, str | None]:
"""Probe a single list endpoint to validate the API token.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/source.py
index 3b0c6a0c6515..911965b8d339 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/source.py
@@ -154,6 +154,9 @@ SolarWinds Service Desk runs independent regional stacks that do not share data
# At source-create only the token is probed (a 403 there can just mean the token's role
# doesn't cover the probe resource); with a schema_name we probe that endpoint's own path.
endpoint = SOLARWINDS_SERVICE_DESK_ENDPOINTS.get(schema_name) if schema_name else None
+ if endpoint is not None and endpoint.fanout is not None:
+ # A child path needs a parent id, so probe the parent list the fan-out reads.
+ endpoint = SOLARWINDS_SERVICE_DESK_ENDPOINTS[endpoint.fanout.parent_name]
return validate_credentials(config.region, config.api_token, endpoint.path if endpoint else None)
def get_resumable_source_manager(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk.py
index f05a99c93fc9..80d5e2defc83 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk.py
@@ -5,9 +5,11 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
DriveResult,
+ Script,
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.solarwindsservicedesk import (
@@ -35,7 +37,7 @@ def _response(items: Optional[list[Any]], *, total_pages: Optional[int] = None)
def _run(
- responses: list[ScriptedResponse],
+ responses: Script,
endpoint: str = "incidents",
resume_state: Optional[SolarwindsServiceDeskResumeConfig] = None,
**inputs: Any,
@@ -85,6 +87,96 @@ class TestPagination:
assert result.requests[0].param("updated_from") is None
+class TestTimeTracksFanout:
+ def test_walks_every_incident_and_tags_rows_with_the_incident(self) -> None:
+ result = _run(
+ route(
+ {
+ "/incidents.json": [
+ _response([{"incident": {"id": 1}}, {"id": 2}, {"id": 3}], total_pages=2),
+ _response([{"id": 4}], total_pages=2),
+ ],
+ "/incidents/1/time_tracks.json": [
+ _response([{"time_track": {"id": 10, "minutes": "30"}}], total_pages=2),
+ _response([{"id": 11, "minutes": "5"}], total_pages=2),
+ ],
+ # Deleted between the incident listing and the child fetch.
+ "/incidents/2/time_tracks.json": [ScriptedResponse(status=404, json={})],
+ "/incidents/3/time_tracks.json": [_response([])],
+ "/incidents/4/time_tracks.json": [
+ _response([{"time_track": {"id": 12, "minutes": "60"}}], total_pages=1)
+ ],
+ }
+ ),
+ endpoint="time_tracks",
+ )
+ assert result.raised is None
+ assert result.rows == [
+ {"id": 10, "minutes": "30", "incident_id": 1},
+ {"id": 11, "minutes": "5", "incident_id": 1},
+ {"id": 12, "minutes": "60", "incident_id": 4},
+ ]
+ assert result.paths == [
+ "/incidents.json",
+ "/incidents/1/time_tracks.json",
+ "/incidents/1/time_tracks.json",
+ "/incidents/2/time_tracks.json",
+ "/incidents/3/time_tracks.json",
+ "/incidents.json",
+ "/incidents/4/time_tracks.json",
+ ]
+ assert result.final_state.fanout_state["completed"] == [
+ "/incidents/1/time_tracks.json",
+ "/incidents/2/time_tracks.json",
+ "/incidents/3/time_tracks.json",
+ "/incidents/4/time_tracks.json",
+ ]
+
+ def test_resume_skips_completed_incidents_and_continues_the_current_one(self) -> None:
+ result = _run(
+ route(
+ {
+ "/incidents.json": [_response([{"id": 1}, {"id": 2}, {"id": 3}], total_pages=1)],
+ "/incidents/2/time_tracks.json": [_response([{"id": 21}], total_pages=2)],
+ "/incidents/3/time_tracks.json": [_response([{"id": 31}], total_pages=1)],
+ }
+ ),
+ endpoint="time_tracks",
+ resume_state=SolarwindsServiceDeskResumeConfig(
+ fanout_state={
+ "completed": ["/incidents/1/time_tracks.json"],
+ "current": "/incidents/2/time_tracks.json",
+ "child_state": {"page": 2},
+ }
+ ),
+ )
+ assert result.raised is None
+ assert result.rows == [{"id": 21, "incident_id": 2}, {"id": 31, "incident_id": 3}]
+ assert [r.path for r in result.requests if r.path != "/incidents.json"] == [
+ "/incidents/2/time_tracks.json",
+ "/incidents/3/time_tracks.json",
+ ]
+ assert result.requests[1].param("page") == "2"
+
+ def test_retries_a_child_response_that_is_not_a_list(self) -> None:
+ # A 200 error object carries no X-Total-Pages header; wrapped as a row it would keep the
+ # crawl paging, so it must be retried instead.
+ result = _run(
+ route(
+ {
+ "/incidents.json": [_response([{"id": 1}], total_pages=1)],
+ "/incidents/1/time_tracks.json": [
+ ScriptedResponse(json={"error": "temporarily unavailable"}),
+ _response([{"id": 10}], total_pages=1),
+ ],
+ }
+ ),
+ endpoint="time_tracks",
+ )
+ assert result.raised is None
+ assert result.rows == [{"id": 10, "incident_id": 1}]
+
+
class TestFormatUpdatedFrom:
@parameterized.expand(
[
@@ -109,12 +201,26 @@ class TestUnwrapRow:
("bare_record", {"id": 2, "name": "B"}, "user", {"id": 2, "name": "B"}),
# A single-key dict whose value isn't a record is a real record that merely has one field.
("wrapper_key_not_a_record", {"user": "not a record", "id": 3}, "user", {"user": "not a record", "id": 3}),
+ (
+ "wrapped_with_parent_field",
+ {"time_track": {"id": 4}, "incident_id": 9},
+ "time_track",
+ {"id": 4, "incident_id": 9},
+ ),
+ ("bare_with_parent_field", {"id": 5, "incident_id": 9}, "time_track", {"id": 5, "incident_id": 9}),
+ # Only the declared parent fields may sit beside the wrapper; anything else means a real record.
+ (
+ "wrapper_beside_other_field",
+ {"time_track": {"id": 6}, "note": "x"},
+ "time_track",
+ {"time_track": {"id": 6}, "note": "x"},
+ ),
]
)
def test_normalizes_documented_shapes(
self, _name: str, item: dict[str, Any], wrapper_key: str, expected: dict[str, Any]
) -> None:
- assert _unwrap_row(item, wrapper_key) == expected
+ assert _unwrap_row(item, wrapper_key, frozenset({"incident_id"})) == expected
class TestBaseUrl:
@@ -173,7 +279,7 @@ class TestSourceResponse:
response = result.response
assert response is not None
assert response.name == endpoint
- assert response.primary_keys == ["id"]
+ assert response.primary_keys == config.primary_keys
# Ordering is undocumented, so the watermark must only commit after a completed sync.
assert response.sort_mode == "desc"
if config.partition_key:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk_source.py
index 11f0f0a62b93..1c85e3208ac1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/solarwinds_service_desk/tests/test_solarwinds_service_desk_source.py
@@ -49,6 +49,8 @@ class TestSolarwindsServiceDeskSource:
# At create only the token is probed; per-schema validation probes that endpoint's path.
("at_create", None, None),
("for_schema", "incidents", "/incidents.json"),
+ # A fan-out child path needs a parent id, so its probe reads the parent list instead.
+ ("fanout_schema", "time_tracks", "/incidents.json"),
("unknown_schema", "not_a_table", None),
]
)