:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 5 missing tables to the SumoLogic source (#124003)
Tom Owers5aae83a55a0e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex c8efee260ab8..9e7058961e90 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11816,10 +11816,10 @@ Today (13): `collector_sources`, `collectors`, `connections`, `dashboards`, `fieDiffed against: <https://api.sumologic.com/docs/sumologic-api.yaml>-- [ ] `/v1/slos (and /v1/slos/sli)` — SLO definitions, SLIs and error budgets - the headline reliability objects, with no equivalent among the synced monitors (high)-- [ ] `/v1/fields (plus /v1/fields/builtin, /v1/fields/dropped)` — field metadata lookup resolving the field names that appear across logs, partitions and extraction rules (high)-- [ ] `/v1/mutingSchedules` — muting windows that explain gaps in monitor alerting - required to interpret the health_events already synced (medium)-- [ ] `/v2/content/folders (with /v2/content/{id}/path)` — content folder tree, the lookup that resolves parentId on dashboards, searches and monitors (medium)+- [x] `/v1/slos (and /v1/slos/sli)` — SLO definitions, SLIs and error budgets - the headline reliability objects, with no equivalent among the synced monitors (high)+- [x] `/v1/fields (plus /v1/fields/builtin, /v1/fields/dropped)` — field metadata lookup resolving the field names that appear across logs, partitions and extraction rules (high)+- [x] `/v1/mutingSchedules` — muting windows that explain gaps in monitor alerting - required to interpret the health_events already synced (medium)+- [x] `/v2/content/folders (with /v2/content/{id}/path)` — content folder tree, the lookup that resolves parentId on dashboards, searches and monitors (medium)- [ ] `/v1/otCollectors` — OpenTelemetry collector inventory; the synced collectors table comes from the legacy Collector Management API and misses these entirely (medium)- [ ] `/v1/logSearches` — saved log searches, the reusable query definitions behind scheduled views and dashboards (medium)- [ ] `/v2/metricsSearches` — saved metrics searches, the metrics-side counterpart (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.pyindex d1c0ffd75952..b1a6e37d9a22 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/canonical_descriptions.py@@ -171,4 +171,79 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"severityLevel": "Severity of the event.",},},+ "slos": {+ "description": "Service level objectives (SLOs) defined in your Sumo Logic organization.",+ "docs_url": "https://help.sumologic.com/docs/api/slo-management/",+ "columns": {+ "id": "Unique identifier for the SLO.",+ "name": "Name of the SLO.",+ "description": "Description of the SLO.",+ "parentId": "Identifier of the SLO library folder that holds the SLO.",+ "contentType": "Type of the content item (Slo).",+ "signalType": "Type of SLI signal (Latency, Error, Throughput, Availability, or Other).",+ "compliance": "Compliance target and period of the SLO.",+ "indicator": "Definition of the service level indicator (SLI) the SLO evaluates.",+ "service": "Name of the service the SLO covers.",+ "application": "Name of the application the SLO covers.",+ "tags": "Tags associated with the SLO.",+ "path": "SLO library path of the SLO.",+ "createdAt": "When the SLO was created.",+ "modifiedAt": "When the SLO was last modified.",+ },+ },+ "slo_sli_statuses": {+ "description": "Current SLI value and error budget remaining of each SLO, for the current compliance period.",+ "docs_url": "https://help.sumologic.com/docs/api/slo-management/",+ "columns": {+ "slo_id": "Identifier of the SLO the status belongs to.",+ "status": "Status of the SLI computation (Success, Error, or InProgress).",+ "sliPercentage": "SLI percentage for the compliance period. Set when the status is Success.",+ "errorBudgetRemainingPercentage": "Percentage of the error budget remaining for the compliance period.",+ "absoluteErrorBudgetRemaining": "Absolute error budget remaining, as a time duration or a request count.",+ "progressPercentage": "Progress of the SLI computation, in percent.",+ },+ },+ "fields": {+ "description": "Custom fields configured in your Sumo Logic account. Field extraction rules, partitions, and log metadata refer to these field names.",+ "docs_url": "https://help.sumologic.com/docs/api/field-management/",+ "columns": {+ "fieldId": "Unique identifier for the field.",+ "fieldName": "Name of the field.",+ "dataType": "Data type of the field (String, Long, Int, Double, or Boolean).",+ "state": "Whether the field is enabled and its values are accepted (Enabled or Disabled).",+ },+ },+ "muting_schedules": {+ "description": "Muting schedules that pause monitor alert notifications for a time window.",+ "docs_url": "https://help.sumologic.com/docs/api/muting-schedules/",+ "columns": {+ "id": "Unique identifier for the muting schedule.",+ "name": "Name of the muting schedule.",+ "description": "Description of the muting schedule.",+ "parentId": "Identifier of the muting schedules library folder that holds the schedule.",+ "contentType": "Type of the content item (MutingSchedule).",+ "schedule": "Time zone, start, duration, and recurrence rule of the muting window.",+ "monitor": "Monitors the schedule applies to.",+ "notificationGroups": "Alert groups the schedule applies to.",+ "path": "Muting schedules library path of the schedule.",+ "createdAt": "When the muting schedule was created.",+ "modifiedAt": "When the muting schedule was last modified.",+ },+ },+ "content_folders": {+ "description": "Content library folders under the personal folder of the access key's user. Dashboards, searches, and other content refer to these folders through parentId.",+ "docs_url": "https://help.sumologic.com/docs/api/content-management/",+ "columns": {+ "id": "Unique identifier for the folder.",+ "name": "Name of the folder.",+ "description": "Description of the folder.",+ "itemType": "Type of the content item (Folder).",+ "parentId": "Identifier of the parent folder.",+ "permissions": "Permissions the access key's user has on the folder.",+ "createdAt": "When the folder was created.",+ "createdBy": "Identifier of the user who created the folder.",+ "modifiedAt": "When the folder was last modified.",+ "modifiedBy": "Identifier of the user who last modified the folder.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.pyindex 596506446242..6d9ef1a0f9fe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/settings.py@@ -6,8 +6,11 @@ from products.warehouse_sources.backend.types import IncrementalField, Increment# "token": Sumo Logic management APIs paginated with `limit` + `token` params; the response body# carries the record list under `data_key` plus a `next` continuation token.# "offset": `limit` + `offset` params; a short page signals the end.+# "single": one unpaginated GET that returns every record.# "search_job": the async Search Job API (submit / poll / fetch) — bespoke path in sumo_logic.py.-PaginationStyle = Literal["token", "offset", "search_job"]+# "slo_sli": walks the SLO search pages and bulk-fetches the SLI status of each page's SLOs.+# "content_tree": walks the content library folder tree from the personal folder down.+PaginationStyle = Literal["token", "offset", "single", "search_job", "slo_sli", "content_tree"]# First sync of the logs table reaches back this far. Log volume through the Search Job API is# expensive (one async job per time window), so this is deliberately shorter than typical retention.@@ -120,6 +123,38 @@ SUMO_LOGIC_ENDPOINTS: dict[str, SumoLogicEndpointConfig] = {name="field_extraction_rules", path="/v1/extractionRules", data_key="data"),"scheduled_views": SumoLogicEndpointConfig(name="scheduled_views", path="/v1/scheduledViews", data_key="data"),+ "slos": SumoLogicEndpointConfig(+ name="slos",+ path="/v1/slos/search",+ pagination="offset",+ nest_key="item",+ extra_params={"query": "type:slo"},+ ),+ "slo_sli_statuses": SumoLogicEndpointConfig(+ name="slo_sli_statuses",+ path="/v1/slos/sli",+ pagination="slo_sli",+ primary_keys=["slo_id"],+ ),+ "fields": SumoLogicEndpointConfig(+ name="fields",+ path="/v1/fields",+ data_key="data",+ pagination="single",+ primary_keys=["fieldId"],+ ),+ "muting_schedules": SumoLogicEndpointConfig(+ name="muting_schedules",+ path="/v1/mutingSchedules/search",+ pagination="offset",+ nest_key="item",+ extra_params={"query": "type:mutingschedule"},+ ),+ "content_folders": SumoLogicEndpointConfig(+ name="content_folders",+ path="/v2/content/folders/{folder_id}",+ pagination="content_tree",+ ),"health_events": SumoLogicEndpointConfig(name="health_events",path="/v1/healthEvents",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.pyindex 58f33f888443..8763cdf3cc49 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py@@ -277,6 +277,94 @@ def _get_collector_fan_out_rows(resumable_source_manager.save_state(SumoLogicResumeConfig(offset=offset))+def _get_single_page_rows(fetch: Any, host: str, config: SumoLogicEndpointConfig) -> Iterator[list[dict[str, Any]]]:+ items = _extract_items(fetch("GET", _build_url(host, config.path, dict(config.extra_params))), config)+ if items:+ yield items+++def _get_slo_sli_rows(+ fetch: Any,+ host: str,+ config: SumoLogicEndpointConfig,+ resumable_source_manager: ResumableSourceManager[SumoLogicResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ """Walk the SLO search pages and bulk-fetch the current SLI status of each page's SLOs.++ The SLI endpoint only reads SLOs by id, so the SLO search supplies the ids. Resume state tracks+ the SLO search offset.+ """+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ offset = resume.offset if resume and resume.offset is not None else 0+ slos_config = SUMO_LOGIC_ENDPOINTS["slos"]++ while True:+ params: dict[str, Any] = {"limit": slos_config.page_size, "offset": offset, **slos_config.extra_params}+ items = _extract_items(fetch("GET", _build_url(host, slos_config.path, params)), slos_config)+ slo_ids = [+ str(slo["id"])+ for slo in (_unnest_item(item, slos_config) for item in items)+ if slo.get("id") is not None and slo.get("contentType") != "Folder"+ ]++ rows: list[dict[str, Any]] = []+ if slo_ids:+ statuses = fetch("GET", _build_url(host, config.path, {"ids": ",".join(slo_ids)}))+ if isinstance(statuses, dict):+ rows = [{"slo_id": slo_id, **status} for slo_id, status in statuses.items() if isinstance(status, dict)]++ is_last_page = len(items) < slos_config.page_size+ if not is_last_page:+ resumable_source_manager.save_state(SumoLogicResumeConfig(offset=offset + slos_config.page_size))++ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()++ if is_last_page:+ break+ offset += slos_config.page_size+++def _get_content_folder_rows(+ fetch: Any,+ host: str,+ config: SumoLogicEndpointConfig,+ resumable_source_manager: ResumableSourceManager[SumoLogicResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ """Walk the content library folder tree breadth-first from the personal folder of the key's user.++ Each folder read lists its direct children, so every subfolder costs one request. The child list+ is dropped from the row: child items point back at their folder through ``parentId``.+ """+ pending: deque[str] = deque(["personal"])+ seen: set[str] = set()++ while pending:+ folder_id = pending.popleft()+ try:+ folder = fetch("GET", _build_url(host, config.path.format(folder_id=folder_id), {}))+ except requests.HTTPError as e:+ # A folder deleted after its parent was read 404s; the rest of the tree is still valid.+ if e.response is not None and e.response.status_code == 404 and folder_id != "personal":+ resumable_source_manager.safe_point()+ continue+ raise+ if not isinstance(folder, dict):+ continue++ if folder.get("id") is not None:+ seen.add(str(folder["id"]))+ for child in folder.get("children") or []:+ child_id = child.get("id") if isinstance(child, dict) else None+ if child_id is not None and child.get("itemType") == "Folder" and str(child_id) not in seen:+ seen.add(str(child_id))+ pending.append(str(child_id))++ yield [{key: value for key, value in folder.items() if key != "children"}]++def _message_row(message: dict[str, Any]) -> dict[str, Any]:"""Flatten a Search Job message's ``map`` and derive a typed ``message_time`` datetime."""row = dict(message.get("map", {}))@@ -450,6 +538,12 @@ def get_rows()elif config.fan_out_over_collectors:batches = _get_collector_fan_out_rows(fetch, host, config, resumable_source_manager)+ elif config.pagination == "slo_sli":+ batches = _get_slo_sli_rows(fetch, host, config, resumable_source_manager)+ elif config.pagination == "content_tree":+ batches = _get_content_folder_rows(fetch, host, config, resumable_source_manager)+ elif config.pagination == "single":+ batches = _get_single_page_rows(fetch, host, config)elif config.pagination == "offset":batches = _get_offset_paginated_rows(fetch, host, config, resumable_source_manager)else:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.pyindex 2acd786a7004..15f88bce8c16 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/tests/test_sumo_logic.py@@ -247,6 +247,104 @@ class TestCollectorFanOut:]+class TestSloSliStatuses:+ def test_walks_slo_pages_and_fetches_sli_for_slos_only(self) -> None:+ page_size = SUMO_LOGIC_ENDPOINTS["slos"].page_size+ first_page = [{"item": {"id": "folder-1", "contentType": "Folder"}, "path": "/slos/f"}] + [+ {"item": {"id": f"slo-{i}", "contentType": "Slo"}, "path": f"/slos/{i}"} for i in range(page_size - 1)+ ]+ # The last page holds only a folder, so it needs no SLI request.+ pages = {"0": first_page, str(page_size): [{"item": {"id": "folder-2", "contentType": "Folder"}}]}+ sli_ids: list[list[str]] = []++ def handler(request: RecordedRequest) -> ScriptedResponse:+ if request.path.endswith("/v1/slos/search"):+ assert request.param("query") == "type:slo"+ return _response(pages[request.param("offset") or "0"])+ if request.path.endswith("/v1/slos/sli"):+ ids = (request.param("ids") or "").split(",")+ sli_ids.append(ids)+ return _response({slo_id: {"status": "Success", "sliPercentage": 99.5} for slo_id in ids})+ raise UnexpectedRequest(f"unexpected request {request.method} {request.url}")++ result = _driver().run("slo_sli_statuses", handler)++ assert result.raised is None+ assert sli_ids == [[f"slo-{i}" for i in range(page_size - 1)]]+ assert len(result.rows) == page_size - 1+ assert result.rows[0] == {"slo_id": "slo-0", "status": "Success", "sliPercentage": 99.5}+ assert [s.offset for s in result.saved_states] == [page_size]++ def test_resumes_from_saved_offset(self) -> None:+ result = _driver().run(+ "slo_sli_statuses",+ route(+ {+ "/v1/slos/search": [_response([{"item": {"id": "slo-9", "contentType": "Slo"}}])],+ "/v1/slos/sli": [_response({"slo-9": {"status": "InProgress", "progressPercentage": 40.0}})],+ }+ ),+ resume_state=SumoLogicResumeConfig(offset=200),+ )++ assert result.requests[0].param("offset") == "200"+ assert result.rows == [{"slo_id": "slo-9", "status": "InProgress", "progressPercentage": 40.0}]+++class TestContentFolders:+ def _folder(self, folder_id: str, parent_id: str, children: list[dict[str, Any]]) -> ScriptedResponse:+ return _response({"id": folder_id, "name": f"name-{folder_id}", "parentId": parent_id, "children": children})++ @pytest.mark.parametrize(+ ("deleted_folder", "expected_ids", "expect_raise"),+ [+ (None, ["p", "a", "b", "a1"], False),+ # A subfolder deleted after its parent was read is skipped; the walk continues.+ ("a", ["p", "b"], False),+ # The root of the walk missing is a real failure, not a race.+ ("personal", [], True),+ ],+ )+ def test_walks_folder_tree_breadth_first(+ self, deleted_folder: str | None, expected_ids: list[str], expect_raise: bool+ ) -> None:+ folders = {+ "personal": self._folder(+ "p",+ "root",+ [+ {"id": "a", "itemType": "Folder"},+ {"id": "dash-1", "itemType": "Dashboard"},+ {"id": "b", "itemType": "Folder"},+ ],+ ),+ # `p` points back at the personal folder; it must not be read twice.+ "a": self._folder("a", "p", [{"id": "a1", "itemType": "Folder"}, {"id": "p", "itemType": "Folder"}]),+ "b": self._folder("b", "p", [{"id": "search-1", "itemType": "Search"}]),+ "a1": self._folder("a1", "a", []),+ }+ fetched: list[str] = []++ def handler(request: RecordedRequest) -> ScriptedResponse:+ folder_id = request.path.rsplit("/", 1)[-1]+ fetched.append(folder_id)+ if folder_id == deleted_folder:+ return _response({"errors": [{"code": "content:not_found"}]}, status=404)+ if folder_id not in folders:+ raise UnexpectedRequest(f"unexpected request {request.method} {request.url}")+ return folders[folder_id]++ result = _driver().run("content_folders", handler)++ assert [row["id"] for row in result.rows] == expected_ids+ assert all("children" not in row for row in result.rows)+ assert fetched.count("personal") == 1+ if expect_raise:+ assert result.raised is not None+ else:+ assert result.raised is None++class FakeSearchJobApi:"""Fakes the submit/poll/fetch/delete Search Job flow, recording created jobs and deletions."""