BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
fix: retry wrapped Hugging Face 429s during Docker model preload
Christian2fe306f0cc3e
diff --git a/Dockerfile b/Dockerfileindex 9bd9098..e27db4a 100644--- a/Dockerfile+++ b/Dockerfile@@ -36,7 +36,8 @@ COPY src/ ./src/COPY scripts/preload_docker_models.py ./scripts/RUN --mount=type=secret,id=HF_TOKEN,required=false \- export HUGGING_FACE_HUB_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \+ export HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \+ && export HUGGING_FACE_HUB_TOKEN="$HF_TOKEN" \&& /app/.venv/bin/python scripts/preload_docker_models.py \&& rm -rf /root/.cache /tmp/*diff --git a/pyproject.toml b/pyproject.tomlindex 942af3e..eeeedc7 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "brainapi2"-version = "2.14.3-dev"+version = "2.14.4-dev"description = "Version 2.x.x of the BrainAPI memory layer."authors = [{name = "Christian",email = "[email protected]"}diff --git a/scripts/preload_docker_models.py b/scripts/preload_docker_models.pyindex 70f272c..ab86347 100644--- a/scripts/preload_docker_models.py+++ b/scripts/preload_docker_models.py@@ -11,15 +11,64 @@ SENTENCE_TRANSFORMER_MODELS = ("intfloat/e5-small",)-MAX_ATTEMPTS = 8-BASE_DELAY_SECONDS = 5+MAX_ATTEMPTS = 10+BASE_DELAY_SECONDS = 10+MAX_DELAY_SECONDS = 120+++def _exception_chain(exc: BaseException):+ seen: set[int] = set()+ current: BaseException | None = exc+ while current is not None and id(current) not in seen:+ seen.add(id(current))+ yield current+ current = current.__cause__ or current.__context__def _is_rate_limited(exc: BaseException) -> bool:- if "429" in str(exc):+ for item in _exception_chain(exc):+ response = getattr(item, "response", None)+ if getattr(response, "status_code", None) == 429:+ return True+ text = str(item).lower()+ if (+ "429" in text+ or "too many requests" in text+ or "rate limit" in text+ or "ratelimit" in text+ ):+ return True+ return False+++def _is_transient_hub_error(exc: BaseException) -> bool:+ if _is_rate_limited(exc):return True- response = getattr(exc, "response", None)- return getattr(response, "status_code", None) == 429+ for item in _exception_chain(exc):+ text = str(item).lower()+ if (+ "couldn't connect to 'https://huggingface.co'" in text+ or "localentrynotfounderror" in type(item).__name__.lower()+ or "temporarily unavailable" in text+ or "service unavailable" in text+ or "gateway timeout" in text+ ):+ return True+ return False+++def _retry_after_seconds(exc: BaseException) -> float | None:+ for item in _exception_chain(exc):+ response = getattr(item, "response", None)+ headers = getattr(response, "headers", None) or {}+ raw = headers.get("Retry-After") or headers.get("retry-after")+ if raw is None:+ continue+ try:+ return float(raw)+ except (TypeError, ValueError):+ continue+ return Nonedef _retry(action, label: str) -> None:@@ -28,29 +77,53 @@ def _retry(action, label: str) -> None:action()returnexcept Exception as exc:- if attempt == MAX_ATTEMPTS or not _is_rate_limited(exc):+ if attempt == MAX_ATTEMPTS or not _is_transient_hub_error(exc):raise- delay = BASE_DELAY_SECONDS * (2 ** (attempt - 1))+ retry_after = _retry_after_seconds(exc)+ exponential = min(+ BASE_DELAY_SECONDS * (2 ** (attempt - 1)),+ MAX_DELAY_SECONDS,+ )+ delay = max(retry_after or 0.0, float(exponential))+ kind = "rate limited" if _is_rate_limited(exc) else "hub error"print(- f"[preload] {label}: rate limited, "- f"retry {attempt}/{MAX_ATTEMPTS} in {delay}s",+ f"[preload] {label}: {kind}, "+ f"retry {attempt}/{MAX_ATTEMPTS} in {delay:.0f}s "+ f"({type(exc).__name__}: {exc})",flush=True,)time.sleep(delay)def _configure_hf_token() -> None:- token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")+ token = (+ os.environ.get("HF_TOKEN")+ or os.environ.get("HUGGING_FACE_HUB_TOKEN")+ or ""+ ).strip()if not token:+ print(+ "[preload] warning: no HF_TOKEN/HUGGING_FACE_HUB_TOKEN set; "+ "anonymous Hugging Face downloads are frequently rate-limited in CI",+ flush=True,+ )returnos.environ["HF_TOKEN"] = tokenos.environ["HUGGING_FACE_HUB_TOKEN"] = token+ print("[preload] Hugging Face token configured", flush=True)+ try:+ from huggingface_hub import login++ login(token=token, add_to_git_credential=False)+ except Exception as exc:+ print(f"[preload] huggingface_hub.login skipped: {exc}", flush=True)def _preload_sentence_transformers() -> None:from sentence_transformers import SentenceTransformerfor model in SENTENCE_TRANSFORMER_MODELS:+ print(f"[preload] loading sentence-transformers model: {model}", flush=True)_retry(lambda model=model: SentenceTransformer(model), model)@@ -72,6 +145,7 @@ def _preload_spacy() -> None:f"spacy download {model_name} failed with exit code {result.returncode}")+ print(f"[preload] downloading spaCy model: {model}", flush=True)_retry(download, model)diff --git a/tests/test_spacy_fixes_verification.py b/tests/test_spacy_fixes_verification.pyindex abdcf85..8704fad 100644--- a/tests/test_spacy_fixes_verification.py+++ b/tests/test_spacy_fixes_verification.py@@ -44,6 +44,27 @@ class TestDockerfileSpacyModels(unittest.TestCase):self.assertIn("spacy download", preload)self.assertIn("src.constants.spacy_models", preload)+ def test_preload_detects_wrapped_huggingface_rate_limits(self):+ import scripts.preload_docker_models as preload++ class FakeResponse:+ status_code = 429+ headers = {"Retry-After": "7"}++ class HubError(Exception):+ def __init__(self):+ super().__init__("429 Client Error: Too Many Requests")+ self.response = FakeResponse()++ wrapped = OSError(+ "We couldn't connect to 'https://huggingface.co' to load the files"+ )+ wrapped.__cause__ = HubError()++ self.assertTrue(preload._is_rate_limited(wrapped))+ self.assertTrue(preload._is_transient_hub_error(wrapped))+ self.assertEqual(preload._retry_after_seconds(wrapped), 7.0)+def test_chinese_uses_web_pipeline_name(self):from src.constants.spacy_models import SPACY_MODEL_NAMES