Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 5 missing tables to the SumoLogic source (#124003)

Tom Owers5aae83a55a0e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c8efee260ab8..9e7058961e90 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11816,10 +11816,10 @@ Today (13): `collector_sources`, `collectors`, `connections`, `dashboards`, `fie
Diffed against: <https://api.sumologic.com/docs/sumologic-api.yaml>
-- [ ] `/v1/slos (and /v1/slos/sli)` — SLO definitions, SLIs and error budgets - the headline reliability objects, with no equivalent among the synced monitors (high)
-- [ ] `/v1/fields (plus /v1/fields/builtin, /v1/fields/dropped)` — field metadata lookup resolving the field names that appear across logs, partitions and extraction rules (high)
-- [ ] `/v1/mutingSchedules` — muting windows that explain gaps in monitor alerting - required to interpret the health_events already synced (medium)
-- [ ] `/v2/content/folders (with /v2/content/{id}/path)` — content folder tree, the lookup that resolves parentId on dashboards, searches and monitors (medium)
+- [x] `/v1/slos (and /v1/slos/sli)` — SLO definitions, SLIs and error budgets - the headline reliability objects, with no equivalent among the synced monitors (high)
+- [x] `/v1/fields (plus /v1/fields/builtin, /v1/fields/dropped)` — field metadata lookup resolving the field names that appear across logs, partitions and extraction rules (high)
+- [x] `/v1/mutingSchedules` — muting windows that explain gaps in monitor alerting - required to interpret the health_events already synced (medium)
+- [x] `/v2/content/folders (with /v2/content/{id}/path)` — content folder tree, the lookup that resolves parentId on dashboards, searches and monitors (medium)
- [ ] `/v1/otCollectors` — OpenTelemetry collector inventory; the synced collectors table comes from the legacy Collector Management API and misses these entirely (medium)
- [ ] `/v1/logSearches` — saved log searches, the reusable query definitions behind scheduled views and dashboards (medium)
- [ ] `/v2/metricsSearches` — saved metrics searches, the metrics-side counterpart (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.py
index d1c0ffd75952..b1a6e37d9a22 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.py
@@ -171,4 +171,79 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"severityLevel": "Severity of the event.",
},
},
+ "slos": {
+ "description": "Service level objectives (SLOs) defined in your Sumo Logic organization.",
+ "docs_url": "https://help.sumologic.com/docs/api/slo-management/",
+ "columns": {
+ "id": "Unique identifier for the SLO.",
+ "name": "Name of the SLO.",
+ "description": "Description of the SLO.",
+ "parentId": "Identifier of the SLO library folder that holds the SLO.",
+ "contentType": "Type of the content item (Slo).",
+ "signalType": "Type of SLI signal (Latency, Error, Throughput, Availability, or Other).",
+ "compliance": "Compliance target and period of the SLO.",
+ "indicator": "Definition of the service level indicator (SLI) the SLO evaluates.",
+ "service": "Name of the service the SLO covers.",
+ "application": "Name of the application the SLO covers.",
+ "tags": "Tags associated with the SLO.",
+ "path": "SLO library path of the SLO.",
+ "createdAt": "When the SLO was created.",
+ "modifiedAt": "When the SLO was last modified.",
+ },
+ },
+ "slo_sli_statuses": {
+ "description": "Current SLI value and error budget remaining of each SLO, for the current compliance period.",
+ "docs_url": "https://help.sumologic.com/docs/api/slo-management/",
+ "columns": {
+ "slo_id": "Identifier of the SLO the status belongs to.",
+ "status": "Status of the SLI computation (Success, Error, or InProgress).",
+ "sliPercentage": "SLI percentage for the compliance period. Set when the status is Success.",
+ "errorBudgetRemainingPercentage": "Percentage of the error budget remaining for the compliance period.",
+ "absoluteErrorBudgetRemaining": "Absolute error budget remaining, as a time duration or a request count.",
+ "progressPercentage": "Progress of the SLI computation, in percent.",
+ },
+ },
+ "fields": {
+ "description": "Custom fields configured in your Sumo Logic account. Field extraction rules, partitions, and log metadata refer to these field names.",
+ "docs_url": "https://help.sumologic.com/docs/api/field-management/",
+ "columns": {
+ "fieldId": "Unique identifier for the field.",
+ "fieldName": "Name of the field.",
+ "dataType": "Data type of the field (String, Long, Int, Double, or Boolean).",
+ "state": "Whether the field is enabled and its values are accepted (Enabled or Disabled).",
+ },
+ },
+ "muting_schedules": {
+ "description": "Muting schedules that pause monitor alert notifications for a time window.",
+ "docs_url": "https://help.sumologic.com/docs/api/muting-schedules/",
+ "columns": {
+ "id": "Unique identifier for the muting schedule.",
+ "name": "Name of the muting schedule.",
+ "description": "Description of the muting schedule.",
+ "parentId": "Identifier of the muting schedules library folder that holds the schedule.",
+ "contentType": "Type of the content item (MutingSchedule).",
+ "schedule": "Time zone, start, duration, and recurrence rule of the muting window.",
+ "monitor": "Monitors the schedule applies to.",
+ "notificationGroups": "Alert groups the schedule applies to.",
+ "path": "Muting schedules library path of the schedule.",
+ "createdAt": "When the muting schedule was created.",
+ "modifiedAt": "When the muting schedule was last modified.",
+ },
+ },
+ "content_folders": {
+ "description": "Content library folders under the personal folder of the access key's user. Dashboards, searches, and other content refer to these folders through parentId.",
+ "docs_url": "https://help.sumologic.com/docs/api/content-management/",
+ "columns": {
+ "id": "Unique identifier for the folder.",
+ "name": "Name of the folder.",
+ "description": "Description of the folder.",
+ "itemType": "Type of the content item (Folder).",
+ "parentId": "Identifier of the parent folder.",
+ "permissions": "Permissions the access key's user has on the folder.",
+ "createdAt": "When the folder was created.",
+ "createdBy": "Identifier of the user who created the folder.",
+ "modifiedAt": "When the folder was last modified.",
+ "modifiedBy": "Identifier of the user who last modified the folder.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.py
index 596506446242..6d9ef1a0f9fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.py
@@ -6,8 +6,11 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment
# "token": Sumo Logic management APIs paginated with `limit` + `token` params; the response body
# carries the record list under `data_key` plus a `next` continuation token.
# "offset": `limit` + `offset` params; a short page signals the end.
+# "single": one unpaginated GET that returns every record.
# "search_job": the async Search Job API (submit / poll / fetch) — bespoke path in sumo_logic.py.
-PaginationStyle = Literal["token", "offset", "search_job"]
+# "slo_sli": walks the SLO search pages and bulk-fetches the SLI status of each page's SLOs.
+# "content_tree": walks the content library folder tree from the personal folder down.
+PaginationStyle = Literal["token", "offset", "single", "search_job", "slo_sli", "content_tree"]
# First sync of the logs table reaches back this far. Log volume through the Search Job API is
# expensive (one async job per time window), so this is deliberately shorter than typical retention.
@@ -120,6 +123,38 @@ SUMO_LOGIC_ENDPOINTS: dict[str, SumoLogicEndpointConfig] = {
name="field_extraction_rules", path="/v1/extractionRules", data_key="data"
),
"scheduled_views": SumoLogicEndpointConfig(name="scheduled_views", path="/v1/scheduledViews", data_key="data"),
+ "slos": SumoLogicEndpointConfig(
+ name="slos",
+ path="/v1/slos/search",
+ pagination="offset",
+ nest_key="item",
+ extra_params={"query": "type:slo"},
+ ),
+ "slo_sli_statuses": SumoLogicEndpointConfig(
+ name="slo_sli_statuses",
+ path="/v1/slos/sli",
+ pagination="slo_sli",
+ primary_keys=["slo_id"],
+ ),
+ "fields": SumoLogicEndpointConfig(
+ name="fields",
+ path="/v1/fields",
+ data_key="data",
+ pagination="single",
+ primary_keys=["fieldId"],
+ ),
+ "muting_schedules": SumoLogicEndpointConfig(
+ name="muting_schedules",
+ path="/v1/mutingSchedules/search",
+ pagination="offset",
+ nest_key="item",
+ extra_params={"query": "type:mutingschedule"},
+ ),
+ "content_folders": SumoLogicEndpointConfig(
+ name="content_folders",
+ path="/v2/content/folders/{folder_id}",
+ pagination="content_tree",
+ ),
"health_events": SumoLogicEndpointConfig(
name="health_events",
path="/v1/healthEvents",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py
index 58f33f888443..8763cdf3cc49 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py
@@ -277,6 +277,94 @@ def _get_collector_fan_out_rows(
resumable_source_manager.save_state(SumoLogicResumeConfig(offset=offset))
+def _get_single_page_rows(fetch: Any, host: str, config: SumoLogicEndpointConfig) -> Iterator[list[dict[str, Any]]]:
+ items = _extract_items(fetch("GET", _build_url(host, config.path, dict(config.extra_params))), config)
+ if items:
+ yield items
+
+
+def _get_slo_sli_rows(
+ fetch: Any,
+ host: str,
+ config: SumoLogicEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[SumoLogicResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ """Walk the SLO search pages and bulk-fetch the current SLI status of each page's SLOs.
+
+ The SLI endpoint only reads SLOs by id, so the SLO search supplies the ids. Resume state tracks
+ the SLO search offset.
+ """
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ offset = resume.offset if resume and resume.offset is not None else 0
+ slos_config = SUMO_LOGIC_ENDPOINTS["slos"]
+
+ while True:
+ params: dict[str, Any] = {"limit": slos_config.page_size, "offset": offset, **slos_config.extra_params}
+ items = _extract_items(fetch("GET", _build_url(host, slos_config.path, params)), slos_config)
+ slo_ids = [
+ str(slo["id"])
+ for slo in (_unnest_item(item, slos_config) for item in items)
+ if slo.get("id") is not None and slo.get("contentType") != "Folder"
+ ]
+
+ rows: list[dict[str, Any]] = []
+ if slo_ids:
+ statuses = fetch("GET", _build_url(host, config.path, {"ids": ",".join(slo_ids)}))
+ if isinstance(statuses, dict):
+ rows = [{"slo_id": slo_id, **status} for slo_id, status in statuses.items() if isinstance(status, dict)]
+
+ is_last_page = len(items) < slos_config.page_size
+ if not is_last_page:
+ resumable_source_manager.save_state(SumoLogicResumeConfig(offset=offset + slos_config.page_size))
+
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+ if is_last_page:
+ break
+ offset += slos_config.page_size
+
+
+def _get_content_folder_rows(
+ fetch: Any,
+ host: str,
+ config: SumoLogicEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[SumoLogicResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ """Walk the content library folder tree breadth-first from the personal folder of the key's user.
+
+ Each folder read lists its direct children, so every subfolder costs one request. The child list
+ is dropped from the row: child items point back at their folder through ``parentId``.
+ """
+ pending: deque[str] = deque(["personal"])
+ seen: set[str] = set()
+
+ while pending:
+ folder_id = pending.popleft()
+ try:
+ folder = fetch("GET", _build_url(host, config.path.format(folder_id=folder_id), {}))
+ except requests.HTTPError as e:
+ # A folder deleted after its parent was read 404s; the rest of the tree is still valid.
+ if e.response is not None and e.response.status_code == 404 and folder_id != "personal":
+ resumable_source_manager.safe_point()
+ continue
+ raise
+ if not isinstance(folder, dict):
+ continue
+
+ if folder.get("id") is not None:
+ seen.add(str(folder["id"]))
+ for child in folder.get("children") or []:
+ child_id = child.get("id") if isinstance(child, dict) else None
+ if child_id is not None and child.get("itemType") == "Folder" and str(child_id) not in seen:
+ seen.add(str(child_id))
+ pending.append(str(child_id))
+
+ yield [{key: value for key, value in folder.items() if key != "children"}]
+
+
def _message_row(message: dict[str, Any]) -> dict[str, Any]:
"""Flatten a Search Job message's ``map`` and derive a typed ``message_time`` datetime."""
row = dict(message.get("map", {}))
@@ -450,6 +538,12 @@ def get_rows(
)
elif config.fan_out_over_collectors:
batches = _get_collector_fan_out_rows(fetch, host, config, resumable_source_manager)
+ elif config.pagination == "slo_sli":
+ batches = _get_slo_sli_rows(fetch, host, config, resumable_source_manager)
+ elif config.pagination == "content_tree":
+ batches = _get_content_folder_rows(fetch, host, config, resumable_source_manager)
+ elif config.pagination == "single":
+ batches = _get_single_page_rows(fetch, host, config)
elif config.pagination == "offset":
batches = _get_offset_paginated_rows(fetch, host, config, resumable_source_manager)
else:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.py
index 2acd786a7004..15f88bce8c16 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.py
@@ -247,6 +247,104 @@ class TestCollectorFanOut:
]
+class TestSloSliStatuses:
+ def test_walks_slo_pages_and_fetches_sli_for_slos_only(self) -> None:
+ page_size = SUMO_LOGIC_ENDPOINTS["slos"].page_size
+ first_page = [{"item": {"id": "folder-1", "contentType": "Folder"}, "path": "/slos/f"}] + [
+ {"item": {"id": f"slo-{i}", "contentType": "Slo"}, "path": f"/slos/{i}"} for i in range(page_size - 1)
+ ]
+ # The last page holds only a folder, so it needs no SLI request.
+ pages = {"0": first_page, str(page_size): [{"item": {"id": "folder-2", "contentType": "Folder"}}]}
+ sli_ids: list[list[str]] = []
+
+ def handler(request: RecordedRequest) -> ScriptedResponse:
+ if request.path.endswith("/v1/slos/search"):
+ assert request.param("query") == "type:slo"
+ return _response(pages[request.param("offset") or "0"])
+ if request.path.endswith("/v1/slos/sli"):
+ ids = (request.param("ids") or "").split(",")
+ sli_ids.append(ids)
+ return _response({slo_id: {"status": "Success", "sliPercentage": 99.5} for slo_id in ids})
+ raise UnexpectedRequest(f"unexpected request {request.method} {request.url}")
+
+ result = _driver().run("slo_sli_statuses", handler)
+
+ assert result.raised is None
+ assert sli_ids == [[f"slo-{i}" for i in range(page_size - 1)]]
+ assert len(result.rows) == page_size - 1
+ assert result.rows[0] == {"slo_id": "slo-0", "status": "Success", "sliPercentage": 99.5}
+ assert [s.offset for s in result.saved_states] == [page_size]
+
+ def test_resumes_from_saved_offset(self) -> None:
+ result = _driver().run(
+ "slo_sli_statuses",
+ route(
+ {
+ "/v1/slos/search": [_response([{"item": {"id": "slo-9", "contentType": "Slo"}}])],
+ "/v1/slos/sli": [_response({"slo-9": {"status": "InProgress", "progressPercentage": 40.0}})],
+ }
+ ),
+ resume_state=SumoLogicResumeConfig(offset=200),
+ )
+
+ assert result.requests[0].param("offset") == "200"
+ assert result.rows == [{"slo_id": "slo-9", "status": "InProgress", "progressPercentage": 40.0}]
+
+
+class TestContentFolders:
+ def _folder(self, folder_id: str, parent_id: str, children: list[dict[str, Any]]) -> ScriptedResponse:
+ return _response({"id": folder_id, "name": f"name-{folder_id}", "parentId": parent_id, "children": children})
+
+ @pytest.mark.parametrize(
+ ("deleted_folder", "expected_ids", "expect_raise"),
+ [
+ (None, ["p", "a", "b", "a1"], False),
+ # A subfolder deleted after its parent was read is skipped; the walk continues.
+ ("a", ["p", "b"], False),
+ # The root of the walk missing is a real failure, not a race.
+ ("personal", [], True),
+ ],
+ )
+ def test_walks_folder_tree_breadth_first(
+ self, deleted_folder: str | None, expected_ids: list[str], expect_raise: bool
+ ) -> None:
+ folders = {
+ "personal": self._folder(
+ "p",
+ "root",
+ [
+ {"id": "a", "itemType": "Folder"},
+ {"id": "dash-1", "itemType": "Dashboard"},
+ {"id": "b", "itemType": "Folder"},
+ ],
+ ),
+ # `p` points back at the personal folder; it must not be read twice.
+ "a": self._folder("a", "p", [{"id": "a1", "itemType": "Folder"}, {"id": "p", "itemType": "Folder"}]),
+ "b": self._folder("b", "p", [{"id": "search-1", "itemType": "Search"}]),
+ "a1": self._folder("a1", "a", []),
+ }
+ fetched: list[str] = []
+
+ def handler(request: RecordedRequest) -> ScriptedResponse:
+ folder_id = request.path.rsplit("/", 1)[-1]
+ fetched.append(folder_id)
+ if folder_id == deleted_folder:
+ return _response({"errors": [{"code": "content:not_found"}]}, status=404)
+ if folder_id not in folders:
+ raise UnexpectedRequest(f"unexpected request {request.method} {request.url}")
+ return folders[folder_id]
+
+ result = _driver().run("content_folders", handler)
+
+ assert [row["id"] for row in result.rows] == expected_ids
+ assert all("children" not in row for row in result.rows)
+ assert fetched.count("personal") == 1
+ if expect_raise:
+ assert result.raised is not None
+ else:
+ assert result.raised is None
+
+
class FakeSearchJobApi:
"""Fakes the submit/poll/fetch/delete Search Job flow, recording created jobs and deletions."""