:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signoz): stop retrying when the host returns a non-JSON body (#123532)
Tom Owersa5e5edf92de1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.pyindex 11b9acd9a26a..3598fa950f96 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py@@ -13,6 +13,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httretry_with_deadline,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (+ _looks_like_json,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindowfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -37,6 +40,7 @@ SIGNOZ_API_VERSION_V5 = "v5"QUERY_RANGE_PATH = "/api/v5/query_range"HOST_NOT_ALLOWED_ERROR = "SigNoz host is not allowed"+NON_JSON_RESPONSE_ERROR = "SigNoz host returned a non-JSON response"class SigNozRetryableError(Exception):@@ -49,6 +53,10 @@ class SigNozHostNotAllowedError(Exception):pass+class SigNozNonJSONResponseError(Exception):+ pass++@dataclasses.dataclassclass SigNozResumeConfig:"""Position within a telemetry sync: the current time window plus the row offset into it.@@ -146,6 +154,11 @@ def validate_credentials(return False, HOST_NOT_ALLOWED_ERRORif response.status_code == 200:+ if not _looks_like_json(response.content):+ return (+ False,+ "This host didn't return SigNoz API data. Check that it points at your SigNoz instance, not a login or proxy page.",+ )return True, Noneif response.status_code == 401:@@ -320,7 +333,14 @@ def get_rows(logger.error(f"SigNoz API error: status={response.status_code}, body={response.text}, url={url}")response.raise_for_status()- return response.json()+ try:+ return response.json()+ except requests.exceptions.JSONDecodeError as e:+ # An HTML or empty 2xx body means the host isn't a SigNoz API (a landing, login, or+ # proxy page), so retrying won't help. A truncated JSON body stays retryable.+ if not _looks_like_json(response.content):+ raise SigNozNonJSONResponseError(NON_JSON_RESPONSE_ERROR) from e+ raiseif config.kind == "config":items = _extract_config_items(fetch(f"{base}{config.path}"), config)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.pyindex 191b76b87427..cb2649eccc22 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.py@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.signoz.set)from products.warehouse_sources.backend.temporal.data_imports.sources.signoz.signoz import (HOST_NOT_ALLOWED_ERROR,+ NON_JSON_RESPONSE_ERROR,SIGNOZ_API_VERSION_V5,SigNozResumeConfig,signoz_source,@@ -97,6 +98,7 @@ Create the API key in SigNoz under **Settings > Service Accounts**: create a ser"401 Client Error": "Invalid SigNoz API key. Generate a new key from a service account and reconnect.","403 Client Error": "Your SigNoz API key lacks the required permissions for this data. Check the service account's role and reconnect.",HOST_NOT_ALLOWED_ERROR: "The configured SigNoz host is not allowed. Check the host and try again.",+ NON_JSON_RESPONSE_ERROR: "Your SigNoz host didn't return API data, for example it returned a web page. Check that the host points at your SigNoz instance, not a login or proxy page, then try again.",}def get_retryable_errors(self) -> set[str]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.py b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.pyindex ef751a7fcd45..ccfb40253f3a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.py@@ -102,19 +102,24 @@ class TestExtractConfigItems:class TestValidateCredentials:@pytest.mark.parametrize(- ("status_code", "schema_name", "expected_valid"),+ ("status_code", "body", "schema_name", "expected_valid"),[- (200, None, True),- (401, None, False),+ (200, b'{"status": "success", "data": []}', None, True),+ # A host serving a web page for every path is not a SigNoz API.+ (200, b"<!doctype html><html></html>", None, False),+ (200, b"", None, False),+ (401, None, None, False),# 403 at source-create is a valid key without the probe's role -> accept.- (403, None, True),+ (403, None, None, True),# 403 on a scoped probe is a hard failure.- (403, "logs", False),- (500, None, False),+ (403, None, "logs", False),+ (500, None, None, False),],)- def test_status_mapping(self, status_code: int, schema_name: str | None, expected_valid: bool) -> None:- with scripted_network(always(ScriptedResponse(status=status_code))):+ def test_status_mapping(+ self, status_code: int, body: bytes | None, schema_name: str | None, expected_valid: bool+ ) -> None:+ with scripted_network(always(ScriptedResponse(status=status_code, body=body))):is_valid, error = validate_credentials("example.signoz.io", "key", schema_name=schema_name)assert is_valid is expected_valid@@ -212,6 +217,22 @@ class TestGetRows:assert result.raised is Noneassert result.items == [[{"id": "c1", "name": "oncall", "type": "slack"}]]+ @pytest.mark.parametrize(+ ("body", "expected_non_retryable"),+ [+ (b"<!doctype html><html></html>", True),+ (b"", True),+ # A truncated JSON body is a transient upstream blip, so it stays retryable.+ (b'{"status": "succ', False),+ ],+ )+ def test_non_json_body_is_non_retryable_only_when_not_json(self, body: bytes, expected_non_retryable: bool) -> None:+ result = SourceDriver(SigNozSource(), CONFIG).run("alert_rules", [ScriptedResponse(body=body)])++ assert result.raised is not None+ non_retryable = SigNozSource().get_non_retryable_errors()+ assert any(pattern in str(result.raised) for pattern in non_retryable) is expected_non_retryable+def test_full_page_advances_window_start_and_saves_state_after_yield(self) -> None:responses = [_page([_log_row("2026-01-01T00:00:00Z", "1"), _log_row("2026-01-01T00:00:01Z", "2")]),