Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(workers): retry dropped image fetches and name unusable inputs instead of failing as internal errors

nirholas2fa0763fee17
diff --git a/workers/avatar-pipeline-controller/worker_security.py b/workers/avatar-pipeline-controller/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/avatar-pipeline-controller/worker_security.py
+++ b/workers/avatar-pipeline-controller/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/avatar-reconstruction/worker_security.py b/workers/avatar-reconstruction/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/avatar-reconstruction/worker_security.py
+++ b/workers/avatar-reconstruction/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/cad-forge/worker_security.py b/workers/cad-forge/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/cad-forge/worker_security.py
+++ b/workers/cad-forge/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/garment-forge/worker_security.py b/workers/garment-forge/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/garment-forge/worker_security.py
+++ b/workers/garment-forge/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/longcat/worker_security.py b/workers/longcat/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/longcat/worker_security.py
+++ b/workers/longcat/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-hunyuan3d/worker_security.py b/workers/model-hunyuan3d/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-hunyuan3d/worker_security.py
+++ b/workers/model-hunyuan3d/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-text2motion/worker_security.py b/workers/model-text2motion/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-text2motion/worker_security.py
+++ b/workers/model-text2motion/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-trellis/worker_security.py b/workers/model-trellis/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-trellis/worker_security.py
+++ b/workers/model-trellis/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-trellis2/worker_security.py b/workers/model-trellis2/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-trellis2/worker_security.py
+++ b/workers/model-trellis2/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-triposg/main.py b/workers/model-triposg/main.py
index 1aae1989b..4f5e9f52f 100644
--- a/workers/model-triposg/main.py
+++ b/workers/model-triposg/main.py
@@ -63,6 +63,7 @@ import uuid
from contextlib import asynccontextmanager
from typing import Optional
+import httpx
import numpy as np
import torch
import trimesh
@@ -260,17 +261,32 @@ def _require_api_key(authorization: str) -> None:
raise HTTPException(status_code=401, detail=str(exc)) from exc
+class InputImageError(ValueError):
+ """The caller's image cannot be used. Its message is ours, never an
+ upstream string, so it is safe to return verbatim on the task record and
+ lets an operator tell a bad input from a worker fault without the logs."""
+
+
def _decode_image_bytes(src: str) -> bytes:
if src.startswith("data:image"):
return base64.b64decode(src.split(",", 1)[1])
if src.startswith("https://"):
# SSRF-hardened: https-only, private/loopback/link-local/metadata IPs
- # rejected after DNS resolution, redirects re-validated per hop, bounded.
+ # rejected after DNS resolution, redirects re-validated per hop, bounded,
+ # transient failures retried.
try:
return fetch_remote_bytes(src, timeout=30)
except UnsafeUrlError as exc:
- raise ValueError(f"refused to fetch image source: {exc}") from exc
- raise ValueError(f"unsupported image source: {src[:60]}")
+ raise InputImageError(f"refused to fetch image source: {exc}") from exc
+ except httpx.HTTPStatusError as exc:
+ raise InputImageError(
+ f"image host answered HTTP {exc.response.status_code}"
+ ) from exc
+ except httpx.TransportError as exc:
+ raise InputImageError(
+ f"image host unreachable after retries ({type(exc).__name__})"
+ ) from exc
+ raise InputImageError("unsupported image source: expected https or data:image")
def _prepare_photo(src: str) -> Image.Image:
@@ -284,9 +300,19 @@ def _prepare_photo(src: str) -> Image.Image:
Image.open(io.BytesIO(data)).convert("RGBA").save(tmp, format="PNG")
tmp_path = tmp.name
try:
- return prepare_image(tmp_path, bg_color=np.array([1.0, 1.0, 1.0]), rmbg_net=_rmbg_net)
+ prepared = prepare_image(tmp_path, bg_color=np.array([1.0, 1.0, 1.0]), rmbg_net=_rmbg_net)
finally:
os.unlink(tmp_path)
+ # Upstream's load_image reports an unusable input by RETURNING a string
+ # ("invalid image: pure black image", ...) which prepare_image then calls
+ # .permute on, so a blank or fully transparent upload used to surface as an
+ # AttributeError and an opaque internal error (11 of them in the three days
+ # to 2026-10-09). Name the input as the problem instead.
+ if not isinstance(prepared, Image.Image):
+ raise InputImageError(
+ "image has no visible subject (blank, black, or fully transparent)"
+ )
+ return prepared
def _prepare_sketch(src: str) -> Image.Image:
@@ -459,6 +485,14 @@ async def _run_inference(
task_id, mode, elapsed, len(glb_bytes), gcs_url,
)
+ except InputImageError as exc:
+ log.warning("[%s] input rejected: %s", task_id, exc)
+ await _update_task(
+ task_id,
+ status="failed",
+ error=f"input image unusable: {exc}",
+ elapsed_ms=int((time.time() - t0) * 1000),
+ )
except Exception as exc:
await _update_task(
task_id,
diff --git a/workers/model-triposg/test_fetch_retry.py b/workers/model-triposg/test_fetch_retry.py
new file mode 100644
index 000000000..193024a74
--- /dev/null
+++ b/workers/model-triposg/test_fetch_retry.py
@@ -0,0 +1,109 @@
+"""Unit tests for the transient-failure retry in worker_security.fetch_remote_bytes.
+No network, no torch. Runs anywhere:
+
+ python3 workers/model-triposg/test_fetch_retry.py
+
+(also collectable by pytest if it is installed). worker_security.py is vendored
+byte-identical into every worker, so these pin the behavior for all of them: a
+dropped connection or a 5xx/429 is retried, while a refused URL, an oversize
+body, or any other 4xx fails on the first attempt.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+
+import httpx
+
+import worker_security as ws
+
+
+def _status_error(code: int) -> httpx.HTTPStatusError:
+ request = httpx.Request("GET", "https://img.example/a.png")
+ return httpx.HTTPStatusError("x", request=request, response=httpx.Response(code, request=request))
+
+
+def _run_sync(failures: list[BaseException]) -> tuple[object, int]:
+ calls = {"n": 0}
+
+ def fake_once(url, **_kwargs):
+ calls["n"] += 1
+ if failures:
+ raise failures.pop(0)
+ return b"png-bytes"
+
+ original_once, original_sleep = ws._fetch_once, ws.time.sleep
+ ws._fetch_once, ws.time.sleep = fake_once, lambda _s: None
+ try:
+ try:
+ return ws.fetch_remote_bytes("https://img.example/a.png"), calls["n"]
+ except Exception as exc: # noqa: BLE001, the outcome under test
+ return exc, calls["n"]
+ finally:
+ ws._fetch_once, ws.time.sleep = original_once, original_sleep
+
+
+def _run_async(failures: list[BaseException]) -> tuple[object, int]:
+ calls = {"n": 0}
+
+ async def fake_once(client, url, **_kwargs):
+ calls["n"] += 1
+ if failures:
+ raise failures.pop(0)
+ return b"png-bytes"
+
+ async def no_sleep(_s):
+ return None
+
+ original_once, original_sleep = ws._fetch_once_async, ws.asyncio.sleep
+ ws._fetch_once_async, ws.asyncio.sleep = fake_once, no_sleep
+ try:
+ try:
+ result = asyncio.run(ws.fetch_remote_bytes_async(None, "https://img.example/a.png"))
+ return result, calls["n"]
+ except Exception as exc: # noqa: BLE001, the outcome under test
+ return exc, calls["n"]
+ finally:
+ ws._fetch_once_async, ws.asyncio.sleep = original_once, original_sleep
+
+
+def test_dropped_connection_is_retried_then_succeeds():
+ drop = httpx.RemoteProtocolError("Server disconnected without sending a response.")
+ for run in (_run_sync, _run_async):
+ result, calls = run([drop, drop])
+ assert result == b"png-bytes", result
+ assert calls == 3
+
+
+def test_server_errors_and_429_are_retried():
+ for code in (429, 500, 503):
+ for run in (_run_sync, _run_async):
+ result, calls = run([_status_error(code)])
+ assert result == b"png-bytes", (code, result)
+ assert calls == 2
+
+
+def test_gives_up_after_the_attempt_budget():
+ drops = [httpx.ReadTimeout("slow")] * ws._FETCH_ATTEMPTS
+ for run in (_run_sync, _run_async):
+ result, calls = run(list(drops))
+ assert isinstance(result, httpx.ReadTimeout), result
+ assert calls == ws._FETCH_ATTEMPTS
+
+
+def test_final_answers_are_not_retried():
+ for failure in (_status_error(403), _status_error(404), ws.UnsafeUrlError("private"), ValueError("too large")):
+ for run in (_run_sync, _run_async):
+ result, calls = run([failure])
+ assert result is failure, result
+ assert calls == 1
+
+
+if __name__ == "__main__":
+ tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
+ for t in tests:
+ t()
+ print(f"ok {t.__name__}")
+ print(f"{len(tests)} passed")
+ sys.exit(0)
diff --git a/workers/model-triposg/worker_security.py b/workers/model-triposg/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-triposg/worker_security.py
+++ b/workers/model-triposg/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-triposr/worker_security.py b/workers/model-triposr/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-triposr/worker_security.py
+++ b/workers/model-triposr/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-video2motion/worker_security.py b/workers/model-video2motion/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-video2motion/worker_security.py
+++ b/workers/model-video2motion/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/model-video2scene/worker_security.py b/workers/model-video2scene/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/model-video2scene/worker_security.py
+++ b/workers/model-video2scene/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/rembg/worker_security.py b/workers/rembg/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/rembg/worker_security.py
+++ b/workers/rembg/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/remesh/worker_security.py b/workers/remesh/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/remesh/worker_security.py
+++ b/workers/remesh/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/rig/worker_security.py b/workers/rig/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/rig/worker_security.py
+++ b/workers/rig/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/segment/worker_security.py b/workers/segment/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/segment/worker_security.py
+++ b/workers/segment/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/stylize/worker_security.py b/workers/stylize/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/stylize/worker_security.py
+++ b/workers/stylize/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp:
diff --git a/workers/texture/worker_security.py b/workers/texture/worker_security.py
index 21951efcd..ed5a528c2 100644
--- a/workers/texture/worker_security.py
+++ b/workers/texture/worker_security.py
@@ -21,7 +21,9 @@ from __future__ import annotations
import hmac
import ipaddress
import logging
+import asyncio
import socket
+import time
import uuid
from typing import Iterable, Optional
from urllib.parse import urlsplit
@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:
DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB
_MAX_REDIRECTS = 4
+# Transient fetch failures are retried: a dropped connection, a timeout, or a
+# 429/5xx from the image host. The public R2 dev endpoint that serves every
+# forge reference image drops connections under load ("Server disconnected
+# without sending a response"), and before this retry each drop ended a
+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a
+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and
+# other 4xx answers are final and never retried.
+_FETCH_ATTEMPTS = 3
+_FETCH_BACKOFF_SECS = (0.5, 1.5)
+
class UnsafeUrlError(ValueError):
"""Raised when a URL targets a disallowed scheme or a private/internal IP."""
@@ -146,10 +158,44 @@ def fetch_remote_bytes(
* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop is
re-validated; httpx's own redirect following is disabled.
* the response body is streamed and aborted once it exceeds ``max_bytes``.
+ * transient failures (dropped connection, timeout, 429/5xx) are retried
+ with a short backoff; see ``_FETCH_ATTEMPTS``.
Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /
``ValueError`` for transport or size failures.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ time.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return _fetch_once(
+ url, timeout=timeout, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+def _is_transient(exc: httpx.HTTPError) -> bool:
+ if isinstance(exc, httpx.HTTPStatusError):
+ code = exc.response.status_code
+ return code == 429 or code >= 500
+ return isinstance(exc, httpx.TransportError)
+
+
+def _fetch_once(
+ url: str,
+ *,
+ timeout: float,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
with httpx.Client(follow_redirects=False, timeout=timeout) as client:
for _ in range(_MAX_REDIRECTS + 1):
@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async(
"""Async variant of ``fetch_remote_bytes`` using a caller-provided client.
The provided ``client`` MUST be constructed with ``follow_redirects=False``
- so this function can re-validate each redirect hop itself.
+ so this function can re-validate each redirect hop itself. Transient
+ failures are retried exactly as in the sync variant.
"""
+ for attempt in range(_FETCH_ATTEMPTS - 1):
+ try:
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+ except httpx.HTTPError as exc:
+ if not _is_transient(exc):
+ raise
+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)
+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])
+ return await _fetch_once_async(
+ client, url, max_bytes=max_bytes,
+ allow_http=allow_http, headers=headers,
+ )
+
+
+async def _fetch_once_async(
+ client: "httpx.AsyncClient",
+ url: str,
+ *,
+ max_bytes: int,
+ allow_http: bool,
+ headers: Optional[dict],
+) -> bytes:
current = assert_safe_url(url, allow_http=allow_http)
for _ in range(_MAX_REDIRECTS + 1):
async with client.stream("GET", current, headers=headers) as resp: