Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signoz): stop retrying when the host returns a non-JSON body (#123532)

Tom Owersa5e5edf92de1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py
index 11b9acd9a26a..3598fa950f96 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py
@@ -13,6 +13,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
retry_with_deadline,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
+ _looks_like_json,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -37,6 +40,7 @@ SIGNOZ_API_VERSION_V5 = "v5"
QUERY_RANGE_PATH = "/api/v5/query_range"
HOST_NOT_ALLOWED_ERROR = "SigNoz host is not allowed"
+NON_JSON_RESPONSE_ERROR = "SigNoz host returned a non-JSON response"
class SigNozRetryableError(Exception):
@@ -49,6 +53,10 @@ class SigNozHostNotAllowedError(Exception):
pass
+class SigNozNonJSONResponseError(Exception):
+ pass
+
+
@dataclasses.dataclass
class SigNozResumeConfig:
"""Position within a telemetry sync: the current time window plus the row offset into it.
@@ -146,6 +154,11 @@ def validate_credentials(
return False, HOST_NOT_ALLOWED_ERROR
if response.status_code == 200:
+ if not _looks_like_json(response.content):
+ return (
+ False,
+ "This host didn't return SigNoz API data. Check that it points at your SigNoz instance, not a login or proxy page.",
+ )
return True, None
if response.status_code == 401:
@@ -320,7 +333,14 @@ def get_rows(
logger.error(f"SigNoz API error: status={response.status_code}, body={response.text}, url={url}")
response.raise_for_status()
- return response.json()
+ try:
+ return response.json()
+ except requests.exceptions.JSONDecodeError as e:
+ # An HTML or empty 2xx body means the host isn't a SigNoz API (a landing, login, or
+ # proxy page), so retrying won't help. A truncated JSON body stays retryable.
+ if not _looks_like_json(response.content):
+ raise SigNozNonJSONResponseError(NON_JSON_RESPONSE_ERROR) from e
+ raise
if config.kind == "config":
items = _extract_config_items(fetch(f"{base}{config.path}"), config)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.py
index 191b76b87427..cb2649eccc22 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/source.py
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.signoz.set
)
from products.warehouse_sources.backend.temporal.data_imports.sources.signoz.signoz import (
HOST_NOT_ALLOWED_ERROR,
+ NON_JSON_RESPONSE_ERROR,
SIGNOZ_API_VERSION_V5,
SigNozResumeConfig,
signoz_source,
@@ -97,6 +98,7 @@ Create the API key in SigNoz under **Settings > Service Accounts**: create a ser
"401 Client Error": "Invalid SigNoz API key. Generate a new key from a service account and reconnect.",
"403 Client Error": "Your SigNoz API key lacks the required permissions for this data. Check the service account's role and reconnect.",
HOST_NOT_ALLOWED_ERROR: "The configured SigNoz host is not allowed. Check the host and try again.",
+ NON_JSON_RESPONSE_ERROR: "Your SigNoz host didn't return API data, for example it returned a web page. Check that the host points at your SigNoz instance, not a login or proxy page, then try again.",
}
def get_retryable_errors(self) -> set[str]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.py b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.py
index ef751a7fcd45..ccfb40253f3a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/tests/test_signoz.py
@@ -102,19 +102,24 @@ class TestExtractConfigItems:
class TestValidateCredentials:
@pytest.mark.parametrize(
- ("status_code", "schema_name", "expected_valid"),
+ ("status_code", "body", "schema_name", "expected_valid"),
[
- (200, None, True),
- (401, None, False),
+ (200, b'{"status": "success", "data": []}', None, True),
+ # A host serving a web page for every path is not a SigNoz API.
+ (200, b"<!doctype html><html></html>", None, False),
+ (200, b"", None, False),
+ (401, None, None, False),
# 403 at source-create is a valid key without the probe's role -> accept.
- (403, None, True),
+ (403, None, None, True),
# 403 on a scoped probe is a hard failure.
- (403, "logs", False),
- (500, None, False),
+ (403, None, "logs", False),
+ (500, None, None, False),
],
)
- def test_status_mapping(self, status_code: int, schema_name: str | None, expected_valid: bool) -> None:
- with scripted_network(always(ScriptedResponse(status=status_code))):
+ def test_status_mapping(
+ self, status_code: int, body: bytes | None, schema_name: str | None, expected_valid: bool
+ ) -> None:
+ with scripted_network(always(ScriptedResponse(status=status_code, body=body))):
is_valid, error = validate_credentials("example.signoz.io", "key", schema_name=schema_name)
assert is_valid is expected_valid
@@ -212,6 +217,22 @@ class TestGetRows:
assert result.raised is None
assert result.items == [[{"id": "c1", "name": "oncall", "type": "slack"}]]
+ @pytest.mark.parametrize(
+ ("body", "expected_non_retryable"),
+ [
+ (b"<!doctype html><html></html>", True),
+ (b"", True),
+ # A truncated JSON body is a transient upstream blip, so it stays retryable.
+ (b'{"status": "succ', False),
+ ],
+ )
+ def test_non_json_body_is_non_retryable_only_when_not_json(self, body: bytes, expected_non_retryable: bool) -> None:
+ result = SourceDriver(SigNozSource(), CONFIG).run("alert_rules", [ScriptedResponse(body=body)])
+
+ assert result.raised is not None
+ non_retryable = SigNozSource().get_non_retryable_errors()
+ assert any(pattern in str(result.raised) for pattern in non_retryable) is expected_non_retryable
+
def test_full_page_advances_window_start_and_saves_state_after_yield(self) -> None:
responses = [
_page([_log_row("2026-01-01T00:00:00Z", "1"), _log_row("2026-01-01T00:00:01Z", "2")]),