Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(workers): retry dropped image fetches and name unusable inputs instead of failing as internal errors
nirholas2fa0763fee17
diff --git a/workers/avatar-pipeline-controller/worker_security.py b/workers/avatar-pipeline-controller/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/avatar-pipeline-controller/worker_security.py+++ b/workers/avatar-pipeline-controller/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/avatar-reconstruction/worker_security.py b/workers/avatar-reconstruction/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/avatar-reconstruction/worker_security.py+++ b/workers/avatar-reconstruction/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/cad-forge/worker_security.py b/workers/cad-forge/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/cad-forge/worker_security.py+++ b/workers/cad-forge/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/garment-forge/worker_security.py b/workers/garment-forge/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/garment-forge/worker_security.py+++ b/workers/garment-forge/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/longcat/worker_security.py b/workers/longcat/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/longcat/worker_security.py+++ b/workers/longcat/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-hunyuan3d/worker_security.py b/workers/model-hunyuan3d/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-hunyuan3d/worker_security.py+++ b/workers/model-hunyuan3d/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-text2motion/worker_security.py b/workers/model-text2motion/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-text2motion/worker_security.py+++ b/workers/model-text2motion/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-trellis/worker_security.py b/workers/model-trellis/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-trellis/worker_security.py+++ b/workers/model-trellis/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-trellis2/worker_security.py b/workers/model-trellis2/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-trellis2/worker_security.py+++ b/workers/model-trellis2/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-triposg/main.py b/workers/model-triposg/main.pyindex 1aae1989b..4f5e9f52f 100644--- a/workers/model-triposg/main.py+++ b/workers/model-triposg/main.py@@ -63,6 +63,7 @@ import uuidfrom contextlib import asynccontextmanagerfrom typing import Optional+import httpximport numpy as npimport torchimport trimesh@@ -260,17 +261,32 @@ def _require_api_key(authorization: str) -> None:raise HTTPException(status_code=401, detail=str(exc)) from exc+class InputImageError(ValueError):+ """The caller's image cannot be used. Its message is ours, never an+ upstream string, so it is safe to return verbatim on the task record and+ lets an operator tell a bad input from a worker fault without the logs."""++def _decode_image_bytes(src: str) -> bytes:if src.startswith("data:image"):return base64.b64decode(src.split(",", 1)[1])if src.startswith("https://"):# SSRF-hardened: https-only, private/loopback/link-local/metadata IPs- # rejected after DNS resolution, redirects re-validated per hop, bounded.+ # rejected after DNS resolution, redirects re-validated per hop, bounded,+ # transient failures retried.try:return fetch_remote_bytes(src, timeout=30)except UnsafeUrlError as exc:- raise ValueError(f"refused to fetch image source: {exc}") from exc- raise ValueError(f"unsupported image source: {src[:60]}")+ raise InputImageError(f"refused to fetch image source: {exc}") from exc+ except httpx.HTTPStatusError as exc:+ raise InputImageError(+ f"image host answered HTTP {exc.response.status_code}"+ ) from exc+ except httpx.TransportError as exc:+ raise InputImageError(+ f"image host unreachable after retries ({type(exc).__name__})"+ ) from exc+ raise InputImageError("unsupported image source: expected https or data:image")def _prepare_photo(src: str) -> Image.Image:@@ -284,9 +300,19 @@ def _prepare_photo(src: str) -> Image.Image:Image.open(io.BytesIO(data)).convert("RGBA").save(tmp, format="PNG")tmp_path = tmp.nametry:- return prepare_image(tmp_path, bg_color=np.array([1.0, 1.0, 1.0]), rmbg_net=_rmbg_net)+ prepared = prepare_image(tmp_path, bg_color=np.array([1.0, 1.0, 1.0]), rmbg_net=_rmbg_net)finally:os.unlink(tmp_path)+ # Upstream's load_image reports an unusable input by RETURNING a string+ # ("invalid image: pure black image", ...) which prepare_image then calls+ # .permute on, so a blank or fully transparent upload used to surface as an+ # AttributeError and an opaque internal error (11 of them in the three days+ # to 2026-10-09). Name the input as the problem instead.+ if not isinstance(prepared, Image.Image):+ raise InputImageError(+ "image has no visible subject (blank, black, or fully transparent)"+ )+ return prepareddef _prepare_sketch(src: str) -> Image.Image:@@ -459,6 +485,14 @@ async def _run_inference(task_id, mode, elapsed, len(glb_bytes), gcs_url,)+ except InputImageError as exc:+ log.warning("[%s] input rejected: %s", task_id, exc)+ await _update_task(+ task_id,+ status="failed",+ error=f"input image unusable: {exc}",+ elapsed_ms=int((time.time() - t0) * 1000),+ )except Exception as exc:await _update_task(task_id,diff --git a/workers/model-triposg/test_fetch_retry.py b/workers/model-triposg/test_fetch_retry.pynew file mode 100644index 000000000..193024a74--- /dev/null+++ b/workers/model-triposg/test_fetch_retry.py@@ -0,0 +1,109 @@+"""Unit tests for the transient-failure retry in worker_security.fetch_remote_bytes.+No network, no torch. Runs anywhere:++ python3 workers/model-triposg/test_fetch_retry.py++(also collectable by pytest if it is installed). worker_security.py is vendored+byte-identical into every worker, so these pin the behavior for all of them: a+dropped connection or a 5xx/429 is retried, while a refused URL, an oversize+body, or any other 4xx fails on the first attempt.+"""++from __future__ import annotations++import asyncio+import sys++import httpx++import worker_security as ws+++def _status_error(code: int) -> httpx.HTTPStatusError:+ request = httpx.Request("GET", "https://img.example/a.png")+ return httpx.HTTPStatusError("x", request=request, response=httpx.Response(code, request=request))+++def _run_sync(failures: list[BaseException]) -> tuple[object, int]:+ calls = {"n": 0}++ def fake_once(url, **_kwargs):+ calls["n"] += 1+ if failures:+ raise failures.pop(0)+ return b"png-bytes"++ original_once, original_sleep = ws._fetch_once, ws.time.sleep+ ws._fetch_once, ws.time.sleep = fake_once, lambda _s: None+ try:+ try:+ return ws.fetch_remote_bytes("https://img.example/a.png"), calls["n"]+ except Exception as exc: # noqa: BLE001, the outcome under test+ return exc, calls["n"]+ finally:+ ws._fetch_once, ws.time.sleep = original_once, original_sleep+++def _run_async(failures: list[BaseException]) -> tuple[object, int]:+ calls = {"n": 0}++ async def fake_once(client, url, **_kwargs):+ calls["n"] += 1+ if failures:+ raise failures.pop(0)+ return b"png-bytes"++ async def no_sleep(_s):+ return None++ original_once, original_sleep = ws._fetch_once_async, ws.asyncio.sleep+ ws._fetch_once_async, ws.asyncio.sleep = fake_once, no_sleep+ try:+ try:+ result = asyncio.run(ws.fetch_remote_bytes_async(None, "https://img.example/a.png"))+ return result, calls["n"]+ except Exception as exc: # noqa: BLE001, the outcome under test+ return exc, calls["n"]+ finally:+ ws._fetch_once_async, ws.asyncio.sleep = original_once, original_sleep+++def test_dropped_connection_is_retried_then_succeeds():+ drop = httpx.RemoteProtocolError("Server disconnected without sending a response.")+ for run in (_run_sync, _run_async):+ result, calls = run([drop, drop])+ assert result == b"png-bytes", result+ assert calls == 3+++def test_server_errors_and_429_are_retried():+ for code in (429, 500, 503):+ for run in (_run_sync, _run_async):+ result, calls = run([_status_error(code)])+ assert result == b"png-bytes", (code, result)+ assert calls == 2+++def test_gives_up_after_the_attempt_budget():+ drops = [httpx.ReadTimeout("slow")] * ws._FETCH_ATTEMPTS+ for run in (_run_sync, _run_async):+ result, calls = run(list(drops))+ assert isinstance(result, httpx.ReadTimeout), result+ assert calls == ws._FETCH_ATTEMPTS+++def test_final_answers_are_not_retried():+ for failure in (_status_error(403), _status_error(404), ws.UnsafeUrlError("private"), ValueError("too large")):+ for run in (_run_sync, _run_async):+ result, calls = run([failure])+ assert result is failure, result+ assert calls == 1+++if __name__ == "__main__":+ tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]+ for t in tests:+ t()+ print(f"ok {t.__name__}")+ print(f"{len(tests)} passed")+ sys.exit(0)diff --git a/workers/model-triposg/worker_security.py b/workers/model-triposg/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-triposg/worker_security.py+++ b/workers/model-triposg/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-triposr/worker_security.py b/workers/model-triposr/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-triposr/worker_security.py+++ b/workers/model-triposr/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-video2motion/worker_security.py b/workers/model-video2motion/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-video2motion/worker_security.py+++ b/workers/model-video2motion/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/model-video2scene/worker_security.py b/workers/model-video2scene/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/model-video2scene/worker_security.py+++ b/workers/model-video2scene/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/rembg/worker_security.py b/workers/rembg/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/rembg/worker_security.py+++ b/workers/rembg/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/remesh/worker_security.py b/workers/remesh/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/remesh/worker_security.py+++ b/workers/remesh/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/rig/worker_security.py b/workers/rig/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/rig/worker_security.py+++ b/workers/rig/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/segment/worker_security.py b/workers/segment/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/segment/worker_security.py+++ b/workers/segment/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/stylize/worker_security.py b/workers/stylize/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/stylize/worker_security.py+++ b/workers/stylize/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp:diff --git a/workers/texture/worker_security.py b/workers/texture/worker_security.pyindex 21951efcd..ed5a528c2 100644--- a/workers/texture/worker_security.py+++ b/workers/texture/worker_security.py@@ -21,7 +21,9 @@ from __future__ import annotationsimport hmacimport ipaddressimport logging+import asyncioimport socket+import timeimport uuidfrom typing import Iterable, Optionalfrom urllib.parse import urlsplit@@ -61,6 +63,16 @@ def require_api_key(authorization: Optional[str], api_key: str) -> None:DEFAULT_MAX_BYTES = 64 * 1024 * 1024 # 64 MiB_MAX_REDIRECTS = 4+# Transient fetch failures are retried: a dropped connection, a timeout, or a+# 429/5xx from the image host. The public R2 dev endpoint that serves every+# forge reference image drops connections under load ("Server disconnected+# without sending a response"), and before this retry each drop ended a+# generation as an opaque internal error on TripoSG and Hunyuan3D (about 20 a+# week measured 2026-10-09). Refusals (UnsafeUrlError), oversize bodies, and+# other 4xx answers are final and never retried.+_FETCH_ATTEMPTS = 3+_FETCH_BACKOFF_SECS = (0.5, 1.5)+class UnsafeUrlError(ValueError):"""Raised when a URL targets a disallowed scheme or a private/internal IP."""@@ -146,10 +158,44 @@ def fetch_remote_bytes(* redirects are followed manually (max ``_MAX_REDIRECTS``) so each hop isre-validated; httpx's own redirect following is disabled.* the response body is streamed and aborted once it exceeds ``max_bytes``.+ * transient failures (dropped connection, timeout, 429/5xx) are retried+ with a short backoff; see ``_FETCH_ATTEMPTS``.Raises ``UnsafeUrlError`` for disallowed targets and ``httpx.HTTPError`` /``ValueError`` for transport or size failures."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ time.sleep(_FETCH_BACKOFF_SECS[attempt])+ return _fetch_once(+ url, timeout=timeout, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++def _is_transient(exc: httpx.HTTPError) -> bool:+ if isinstance(exc, httpx.HTTPStatusError):+ code = exc.response.status_code+ return code == 429 or code >= 500+ return isinstance(exc, httpx.TransportError)+++def _fetch_once(+ url: str,+ *,+ timeout: float,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)with httpx.Client(follow_redirects=False, timeout=timeout) as client:for _ in range(_MAX_REDIRECTS + 1):@@ -188,8 +234,34 @@ async def fetch_remote_bytes_async("""Async variant of ``fetch_remote_bytes`` using a caller-provided client.The provided ``client`` MUST be constructed with ``follow_redirects=False``- so this function can re-validate each redirect hop itself.+ so this function can re-validate each redirect hop itself. Transient+ failures are retried exactly as in the sync variant."""+ for attempt in range(_FETCH_ATTEMPTS - 1):+ try:+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+ except httpx.HTTPError as exc:+ if not _is_transient(exc):+ raise+ log.warning("transient fetch failure (attempt %d): %s", attempt + 1, type(exc).__name__)+ await asyncio.sleep(_FETCH_BACKOFF_SECS[attempt])+ return await _fetch_once_async(+ client, url, max_bytes=max_bytes,+ allow_http=allow_http, headers=headers,+ )+++async def _fetch_once_async(+ client: "httpx.AsyncClient",+ url: str,+ *,+ max_bytes: int,+ allow_http: bool,+ headers: Optional[dict],+) -> bytes:current = assert_safe_url(url, allow_http=allow_http)for _ in range(_MAX_REDIRECTS + 1):async with client.stream("GET", current, headers=headers) as resp: