Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(azure_cost_management): retry cut-off json bodies on 200 responses (#130447)

Tom Owers07d6767b9720
diff --git a/products/warehouse_sources/sources/azure_cost_management/azure_cost_management.py b/products/warehouse_sources/sources/azure_cost_management/azure_cost_management.py
index 6708ed2ad0c7..dfe751c76082 100644
--- a/products/warehouse_sources/sources/azure_cost_management/azure_cost_management.py
+++ b/products/warehouse_sources/sources/azure_cost_management/azure_cost_management.py
@@ -368,7 +368,17 @@ class AzureCostManagementClient:
self.mint_token()
continue
- if response.status_code == 429 or response.status_code >= 500:
+ if response.ok:
+ try:
+ return response.json()
+ except requests.exceptions.JSONDecodeError:
+ # A large result page cut off mid-stream arrives as a 200 with malformed JSON,
+ # and the same request normally succeeds when sent again.
+ retryable = True
+ else:
+ retryable = response.status_code == 429 or response.status_code >= 500
+
+ if retryable:
attempt += 1
if attempt >= MAX_RETRY_ATTEMPTS:
raise AzureCostManagementRetryableError(
@@ -376,21 +386,18 @@ class AzureCostManagementClient:
)
delay = _backoff_seconds(response, attempt)
self._logger.debug(
- f"Azure Cost Management throttled: status={response.status_code}, sleeping {delay}s (attempt {attempt})"
+ f"Azure Cost Management transient response: status={response.status_code}, sleeping {delay}s (attempt {attempt})"
)
self._sleep(delay)
continue
- if not response.ok:
- self._logger.error(
- f"Azure Cost Management error: status={response.status_code}, body={response.text}, url={url}"
- )
- raise requests.HTTPError(
- f"{response.status_code} Client Error: {response.reason} for url: {MANAGEMENT_HOST}{_error_description(response)}",
- response=response,
- )
-
- return response.json()
+ self._logger.error(
+ f"Azure Cost Management error: status={response.status_code}, body={response.text}, url={url}"
+ )
+ raise requests.HTTPError(
+ f"{response.status_code} Client Error: {response.reason} for url: {MANAGEMENT_HOST}{_error_description(response)}",
+ response=response,
+ )
def _scope_probe_failure_message(error: Exception) -> str:
diff --git a/products/warehouse_sources/sources/azure_cost_management/tests/test_azure_cost_management.py b/products/warehouse_sources/sources/azure_cost_management/tests/test_azure_cost_management.py
index 36cd36909794..77497e5f8d61 100644
--- a/products/warehouse_sources/sources/azure_cost_management/tests/test_azure_cost_management.py
+++ b/products/warehouse_sources/sources/azure_cost_management/tests/test_azure_cost_management.py
@@ -385,13 +385,22 @@ class TestClientRequest:
assert MANAGEMENT_HOST in str(error.value)
assert len(network.requests_log) == 4
- @parameterized.expand([(429,), (500,), (503,)])
- def test_retries_transient_statuses_then_succeeds(self, status: int) -> None:
+ @parameterized.expand(
+ [
+ ("throttled", 429, None),
+ ("server_error", 500, None),
+ ("unavailable", 503, None),
+ ("truncated_body", 200, b'{"properties": {"rows": [[1.5, 20240101'),
+ ]
+ )
+ def test_retries_transient_responses_then_succeeds(self, _name: str, status: int, body: Optional[bytes]) -> None:
sleeps: list[float] = []
with scripted_network(
[
_token_response(),
- ScriptedResponse(status=status, headers={"x-ms-ratelimit-microsoft.consumption-retry-after": "9"}),
+ ScriptedResponse(
+ status=status, body=body, headers={"x-ms-ratelimit-microsoft.consumption-retry-after": "9"}
+ ),
_query_response([]),
]
) as network:
@@ -400,9 +409,15 @@ class TestClientRequest:
assert sleeps == [9.0]
assert [request.path for request in network.requests_log[1:]] == ["/q", "/q"]
- def test_exhausted_retries_raise_retryable_error(self) -> None:
+ @parameterized.expand(
+ [
+ ("throttled", ScriptedResponse(status=429)),
+ ("truncated_body", ScriptedResponse(body=b'{"properties": {"rows": [')),
+ ]
+ )
+ def test_exhausted_retries_raise_retryable_error(self, _name: str, answer: ScriptedResponse) -> None:
sleeps: list[float] = []
- with scripted_network([_token_response(), *[ScriptedResponse(status=429) for _ in range(6)]]) as network:
+ with scripted_network([_token_response(), *[answer for _ in range(6)]]) as network:
with pytest.raises(AzureCostManagementRetryableError):
_client(sleeps.append).request("POST", f"{MANAGEMENT_HOST}/q")