Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

fix: retry wrapped Hugging Face 429s during Docker model preload

Christian2fe306f0cc3e
diff --git a/Dockerfile b/Dockerfile
index 9bd9098..e27db4a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -36,7 +36,8 @@ COPY src/ ./src/
COPY scripts/preload_docker_models.py ./scripts/
RUN --mount=type=secret,id=HF_TOKEN,required=false \
- export HUGGING_FACE_HUB_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
+ export HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
+ && export HUGGING_FACE_HUB_TOKEN="$HF_TOKEN" \
&& /app/.venv/bin/python scripts/preload_docker_models.py \
&& rm -rf /root/.cache /tmp/*
diff --git a/pyproject.toml b/pyproject.toml
index 942af3e..eeeedc7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "brainapi2"
-version = "2.14.3-dev"
+version = "2.14.4-dev"
description = "Version 2.x.x of the BrainAPI memory layer."
authors = [
{name = "Christian",email = "[email protected]"}
diff --git a/scripts/preload_docker_models.py b/scripts/preload_docker_models.py
index 70f272c..ab86347 100644
--- a/scripts/preload_docker_models.py
+++ b/scripts/preload_docker_models.py
@@ -11,15 +11,64 @@ SENTENCE_TRANSFORMER_MODELS = (
"intfloat/e5-small",
)
-MAX_ATTEMPTS = 8
-BASE_DELAY_SECONDS = 5
+MAX_ATTEMPTS = 10
+BASE_DELAY_SECONDS = 10
+MAX_DELAY_SECONDS = 120
+
+
+def _exception_chain(exc: BaseException):
+ seen: set[int] = set()
+ current: BaseException | None = exc
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ yield current
+ current = current.__cause__ or current.__context__
def _is_rate_limited(exc: BaseException) -> bool:
- if "429" in str(exc):
+ for item in _exception_chain(exc):
+ response = getattr(item, "response", None)
+ if getattr(response, "status_code", None) == 429:
+ return True
+ text = str(item).lower()
+ if (
+ "429" in text
+ or "too many requests" in text
+ or "rate limit" in text
+ or "ratelimit" in text
+ ):
+ return True
+ return False
+
+
+def _is_transient_hub_error(exc: BaseException) -> bool:
+ if _is_rate_limited(exc):
return True
- response = getattr(exc, "response", None)
- return getattr(response, "status_code", None) == 429
+ for item in _exception_chain(exc):
+ text = str(item).lower()
+ if (
+ "couldn't connect to 'https://huggingface.co'" in text
+ or "localentrynotfounderror" in type(item).__name__.lower()
+ or "temporarily unavailable" in text
+ or "service unavailable" in text
+ or "gateway timeout" in text
+ ):
+ return True
+ return False
+
+
+def _retry_after_seconds(exc: BaseException) -> float | None:
+ for item in _exception_chain(exc):
+ response = getattr(item, "response", None)
+ headers = getattr(response, "headers", None) or {}
+ raw = headers.get("Retry-After") or headers.get("retry-after")
+ if raw is None:
+ continue
+ try:
+ return float(raw)
+ except (TypeError, ValueError):
+ continue
+ return None
def _retry(action, label: str) -> None:
@@ -28,29 +77,53 @@ def _retry(action, label: str) -> None:
action()
return
except Exception as exc:
- if attempt == MAX_ATTEMPTS or not _is_rate_limited(exc):
+ if attempt == MAX_ATTEMPTS or not _is_transient_hub_error(exc):
raise
- delay = BASE_DELAY_SECONDS * (2 ** (attempt - 1))
+ retry_after = _retry_after_seconds(exc)
+ exponential = min(
+ BASE_DELAY_SECONDS * (2 ** (attempt - 1)),
+ MAX_DELAY_SECONDS,
+ )
+ delay = max(retry_after or 0.0, float(exponential))
+ kind = "rate limited" if _is_rate_limited(exc) else "hub error"
print(
- f"[preload] {label}: rate limited, "
- f"retry {attempt}/{MAX_ATTEMPTS} in {delay}s",
+ f"[preload] {label}: {kind}, "
+ f"retry {attempt}/{MAX_ATTEMPTS} in {delay:.0f}s "
+ f"({type(exc).__name__}: {exc})",
flush=True,
)
time.sleep(delay)
def _configure_hf_token() -> None:
- token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
+ token = (
+ os.environ.get("HF_TOKEN")
+ or os.environ.get("HUGGING_FACE_HUB_TOKEN")
+ or ""
+ ).strip()
if not token:
+ print(
+ "[preload] warning: no HF_TOKEN/HUGGING_FACE_HUB_TOKEN set; "
+ "anonymous Hugging Face downloads are frequently rate-limited in CI",
+ flush=True,
+ )
return
os.environ["HF_TOKEN"] = token
os.environ["HUGGING_FACE_HUB_TOKEN"] = token
+ print("[preload] Hugging Face token configured", flush=True)
+ try:
+ from huggingface_hub import login
+
+ login(token=token, add_to_git_credential=False)
+ except Exception as exc:
+ print(f"[preload] huggingface_hub.login skipped: {exc}", flush=True)
def _preload_sentence_transformers() -> None:
from sentence_transformers import SentenceTransformer
for model in SENTENCE_TRANSFORMER_MODELS:
+ print(f"[preload] loading sentence-transformers model: {model}", flush=True)
_retry(lambda model=model: SentenceTransformer(model), model)
@@ -72,6 +145,7 @@ def _preload_spacy() -> None:
f"spacy download {model_name} failed with exit code {result.returncode}"
)
+ print(f"[preload] downloading spaCy model: {model}", flush=True)
_retry(download, model)
diff --git a/tests/test_spacy_fixes_verification.py b/tests/test_spacy_fixes_verification.py
index abdcf85..8704fad 100644
--- a/tests/test_spacy_fixes_verification.py
+++ b/tests/test_spacy_fixes_verification.py
@@ -44,6 +44,27 @@ class TestDockerfileSpacyModels(unittest.TestCase):
self.assertIn("spacy download", preload)
self.assertIn("src.constants.spacy_models", preload)
+ def test_preload_detects_wrapped_huggingface_rate_limits(self):
+ import scripts.preload_docker_models as preload
+
+ class FakeResponse:
+ status_code = 429
+ headers = {"Retry-After": "7"}
+
+ class HubError(Exception):
+ def __init__(self):
+ super().__init__("429 Client Error: Too Many Requests")
+ self.response = FakeResponse()
+
+ wrapped = OSError(
+ "We couldn't connect to 'https://huggingface.co' to load the files"
+ )
+ wrapped.__cause__ = HubError()
+
+ self.assertTrue(preload._is_rate_limited(wrapped))
+ self.assertTrue(preload._is_transient_hub_error(wrapped))
+ self.assertEqual(preload._retry_after_seconds(wrapped), 7.0)
+
def test_chinese_uses_web_pipeline_name(self):
from src.constants.spacy_models import SPACY_MODEL_NAMES