Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(ingest): lock the repo host DNS resolve cache against concurrent mutation (#979)

manjunath bhaskar80b9bea0d58d
diff --git a/semantica/ingest/repo_ingestor.py b/semantica/ingest/repo_ingestor.py
index 4b4f88c1..ad0a9b88 100644
--- a/semantica/ingest/repo_ingestor.py
+++ b/semantica/ingest/repo_ingestor.py
@@ -35,6 +35,7 @@ import re
import shutil
import socket
import tempfile
+import threading
import time
from collections import OrderedDict
from dataclasses import dataclass, field
@@ -67,6 +68,14 @@ _REPO_HOST_RESOLVE_CACHE: "OrderedDict[str, Tuple[float, Tuple[str, ...]]]" = (
)
_REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = 60.0
_REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = 1024
+# Guards all reads/writes/prunes of _REPO_HOST_RESOLVE_CACHE. The cache is a
+# module-level OrderedDict shared by every RepoIngestor instance and every
+# thread; without a lock, concurrent ingest_repository() calls can mutate the
+# dict while another thread is iterating it (e.g. during pruning), raising
+# "RuntimeError: OrderedDict mutated during iteration". The blocking
+# socket.getaddrinfo() call is intentionally kept outside this lock so a slow
+# DNS lookup for one host cannot stall cache access for other hosts.
+_REPO_HOST_RESOLVE_CACHE_LOCK = threading.Lock()
@dataclass
@@ -597,15 +606,19 @@ class RepoIngestor:
"""
cache_key = host.lower().rstrip(".")
now = time.monotonic()
- RepoIngestor._prune_repo_host_resolve_cache(now)
- cached = _REPO_HOST_RESOLVE_CACHE.get(cache_key)
- if cached is not None:
- expires_at, ips = cached
- if now < expires_at:
- _REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
- return ips
- _REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
-
+ with _REPO_HOST_RESOLVE_CACHE_LOCK:
+ RepoIngestor._prune_repo_host_resolve_cache_locked(now)
+ cached = _REPO_HOST_RESOLVE_CACHE.get(cache_key)
+ if cached is not None:
+ expires_at, ips = cached
+ if now < expires_at:
+ _REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
+ return ips
+ _REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
+
+ # DNS resolution is blocking I/O; keep it outside the lock so a slow
+ # or hanging lookup for one host cannot stall cache access for
+ # concurrent lookups of other hosts.
try:
addrinfos = socket.getaddrinfo(
host, None, type=socket.SOCK_STREAM
@@ -629,17 +642,32 @@ class RepoIngestor:
)
result = tuple(ips)
- _REPO_HOST_RESOLVE_CACHE[cache_key] = (
- now + _REPO_HOST_RESOLVE_CACHE_TTL_SECONDS,
- result,
- )
- _REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
- RepoIngestor._prune_repo_host_resolve_cache(now)
+ with _REPO_HOST_RESOLVE_CACHE_LOCK:
+ now = time.monotonic()
+ _REPO_HOST_RESOLVE_CACHE[cache_key] = (
+ now + _REPO_HOST_RESOLVE_CACHE_TTL_SECONDS,
+ result,
+ )
+ _REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
+ RepoIngestor._prune_repo_host_resolve_cache_locked(now)
return result
@staticmethod
def _prune_repo_host_resolve_cache(now: Optional[float] = None) -> None:
- """Remove expired host entries and enforce a hard cache size cap."""
+ """Remove expired host entries and enforce a hard cache size cap.
+
+ Acquires ``_REPO_HOST_RESOLVE_CACHE_LOCK``. Callers that already hold
+ the lock must use ``_prune_repo_host_resolve_cache_locked`` instead to
+ avoid deadlocking on the (non-reentrant) lock.
+ """
+ if now is None:
+ now = time.monotonic()
+ with _REPO_HOST_RESOLVE_CACHE_LOCK:
+ RepoIngestor._prune_repo_host_resolve_cache_locked(now)
+
+ @staticmethod
+ def _prune_repo_host_resolve_cache_locked(now: Optional[float] = None) -> None:
+ """Prune implementation; caller must already hold the cache lock."""
if now is None:
now = time.monotonic()
diff --git a/tests/ingest/test_repo_ingestor_security.py b/tests/ingest/test_repo_ingestor_security.py
index 738c0f76..67494abc 100644
--- a/tests/ingest/test_repo_ingestor_security.py
+++ b/tests/ingest/test_repo_ingestor_security.py
@@ -1,6 +1,7 @@
"""Security-focused tests for RepoIngestor (issue #868)."""
import socket
+import threading
from unittest.mock import MagicMock, patch
import pytest
@@ -480,3 +481,75 @@ class TestLocalPathSupport:
assert not isinstance(exc, ValidationError), (
f"Local path must not raise ValidationError; got: {exc}"
)
+
+
+class TestRepoHostResolveCacheThreadSafety:
+ """Regression test: _REPO_HOST_RESOLVE_CACHE must survive concurrent use.
+
+ _REPO_HOST_RESOLVE_CACHE is a module-level OrderedDict shared across every
+ RepoIngestor instance and thread. Before the fix, _resolve_repo_host_ips
+ and _prune_repo_host_resolve_cache read, wrote, and iterated the dict with
+ no lock. Under concurrent host validation (e.g. multiple
+ ingest_repository() calls running in a thread pool), one thread's
+ insert/evict during another thread's iteration reliably raised
+ RuntimeError: OrderedDict mutated during iteration.
+ """
+
+ def test_concurrent_resolve_repo_host_ips_does_not_raise(self):
+ def fake_getaddrinfo(host, *args, **kwargs):
+ return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]
+
+ orig_ttl = repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_TTL_SECONDS
+ orig_max = repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES
+ # Small TTL/cap so eviction and pruning happen on nearly every call,
+ # keeping the dict under constant mutation without needing an
+ # unreasonably large iteration count.
+ repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = 0.001
+ repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = 8
+
+ errors = []
+ errors_lock = threading.Lock()
+
+ def worker(worker_id):
+ for i in range(500):
+ host = f"race-host-{worker_id}-{i}.example.com"
+ try:
+ repo_ingestor_mod.RepoIngestor._resolve_repo_host_ips(host)
+ except Exception as exc: # pragma: no cover - failure path
+ with errors_lock:
+ errors.append(exc)
+
+ try:
+ with patch(
+ "semantica.ingest.repo_ingestor.socket.getaddrinfo",
+ side_effect=fake_getaddrinfo,
+ ):
+ # daemon=True so a hung worker cannot also block the test
+ # process from exiting once it's reported below.
+ threads = [
+ threading.Thread(target=worker, args=(n,), daemon=True)
+ for n in range(32)
+ ]
+ for t in threads:
+ t.start()
+ # Assert right after each join, not after the whole loop:
+ # join(timeout=30) alone does not fail the test if a thread
+ # hangs, and checking only once every thread has been
+ # joined means a mass hang costs up to 32*30s = 16 minutes
+ # before the test even reaches the check -- the exact
+ # CI-reliability problem this guards against. Failing on
+ # the first hung thread caps the worst case at ~30s.
+ for t in threads:
+ t.join(timeout=30)
+ assert not t.is_alive(), (
+ f"worker thread {t.name} did not finish within "
+ f"the 30s join timeout (still running)"
+ )
+ finally:
+ repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = orig_ttl
+ repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = orig_max
+
+ assert not errors, (
+ f"Concurrent host resolution raised {len(errors)} error(s); "
+ f"first: {errors[0]!r}"
+ )