Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

refactor(server): make template DB rows the read authority for phase gates

高然b816aad1023c
diff --git a/server/opensandbox_server/api/templates.py b/server/opensandbox_server/api/templates.py
index 193f28621..546ba7dd2 100644
--- a/server/opensandbox_server/api/templates.py
+++ b/server/opensandbox_server/api/templates.py
@@ -68,8 +68,9 @@ def _get_template_service() -> FastSandboxTemplateService:
detail=_TEMPLATE_NOT_K8S_DETAIL,
)
_service = FastSandboxTemplateService(config)
- # Watch-driven convergence: DB rows track the CRD build phase even when
- # nobody polls the read API.
+ # Watch-driven convergence: the watch reactor is the only path that moves
+ # CR build phase into the DB rows; reads (including the create gate) trust
+ # the DB and never re-read the CR.
_service.start_background_sync()
return _service
diff --git a/server/opensandbox_server/services/fast_sandbox/service.py b/server/opensandbox_server/services/fast_sandbox/service.py
index f2a962d19..e10fd9b10 100644
--- a/server/opensandbox_server/services/fast_sandbox/service.py
+++ b/server/opensandbox_server/services/fast_sandbox/service.py
@@ -268,7 +268,7 @@ class FastSandboxService(SandboxService, ExtensionService):
if self._template_service is None:
self._template_service = FastSandboxTemplateService(self._app_config)
# Keep template rows converged even without /templates traffic
- # (template-mode create resolves against the synced row).
+ # (template-mode create resolves against the watch-converged row).
self._template_service.start_background_sync()
return self._template_service
diff --git a/server/opensandbox_server/services/templates/template_service.py b/server/opensandbox_server/services/templates/template_service.py
index 2b21d80e9..8c1e34b94 100644
--- a/server/opensandbox_server/services/templates/template_service.py
+++ b/server/opensandbox_server/services/templates/template_service.py
@@ -301,7 +301,7 @@ class FastSandboxTemplateService:
record = self._repo().get(template_id, namespace)
if record is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_TEMPLATE_NOT_FOUND)
- return self._sync_status(record)
+ return record
def list_templates(
self,
@@ -319,7 +319,6 @@ class FastSandboxTemplateService:
page_size=page_size,
)
)
- self._sync_status_bulk(namespace, result.items)
return result.items, result.total_items
def delete_template(self, template_id: str) -> None:
@@ -354,12 +353,22 @@ class FastSandboxTemplateService:
warmImages and older clients. Template-mode Create uses this:
unknown or non-Succeeded templates both yield 404, so no existence
information leaks.
+
+ The DB row is the read authority: the watch reactor is the only
+ writer of build-phase transitions (CR status -> watch event -> row),
+ so this gate is a pure DB read on the sandbox-create hot path.
"""
namespace = self._resolve_namespace()
record = self._repo().get(template_id, namespace)
if record is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_TEMPLATE_NOT_FOUND)
- record = self._sync_status(record)
+ if record.phase in (FastSandboxTemplatePhase.PENDING, FastSandboxTemplatePhase.BUILDING):
+ # Non-terminal row on the create gate: make sure the namespace
+ # watch is alive so the row still converges (e.g. the watch
+ # failed to start when the template was created). Healthy
+ # deployments pay a set-membership check; start() is
+ # non-blocking either way.
+ self._ensure_namespace_watch(namespace)
if record.phase is not FastSandboxTemplatePhase.SUCCEEDED:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_TEMPLATE_NOT_FOUND)
entrypoint = list(record.spec.get("entrypoint") or DEFAULT_ENTRYPOINT)
@@ -432,62 +441,12 @@ class FastSandboxTemplateService:
}
# -- status sync -------------------------------------------------------------
-
- def _sync_status(self, record: FastSandboxTemplateRecord) -> FastSandboxTemplateRecord:
- crd = self._read_crd(record.namespace, record.crd_name)
- if crd is None:
- # A build that never finished cannot finish once the CRD is gone;
- # a Succeeded row keeps its artifact reference.
- if record.phase in (FastSandboxTemplatePhase.PENDING, FastSandboxTemplatePhase.BUILDING):
- self._update_phase(
- record, FastSandboxTemplatePhase.FAILED, None, "SandboxTemplate CRD is gone."
- )
- return record
- crd_status = crd.get("status") or {}
- phase = _map_phase(crd_status.get("phase"))
- if phase is None:
- return record
- manifest_ref = crd_status.get("manifestRef") or None
- message = _failure_message(crd_status)
- if (
- phase is not record.phase
- or manifest_ref != record.manifest_ref
- or (phase is FastSandboxTemplatePhase.FAILED and message != record.message)
- ):
- self._update_phase(record, phase, manifest_ref, message)
- return record
-
- def _sync_status_bulk(self, namespace: str, records: list[FastSandboxTemplateRecord]) -> None:
- if not records:
- return
- try:
- crds = self._kubernetes().list_custom_objects(
- GROUP, VERSION, namespace, PLURAL, ignore_not_found=True
- ) or []
- except Exception as exc: # noqa: BLE001 - reads converge on the next request
- logger.warning(f"SandboxTemplate CR list failed during sync: {exc}")
- return
- by_name = {crd.get("metadata", {}).get("name", ""): crd for crd in crds}
- for record in records:
- crd = by_name.get(record.crd_name)
- if crd is None:
- if record.phase in (FastSandboxTemplatePhase.PENDING, FastSandboxTemplatePhase.BUILDING):
- self._update_phase(
- record, FastSandboxTemplatePhase.FAILED, None, "SandboxTemplate CRD is gone."
- )
- continue
- crd_status = crd.get("status") or {}
- phase = _map_phase(crd_status.get("phase"))
- if phase is None:
- continue
- manifest_ref = crd_status.get("manifestRef") or None
- message = _failure_message(crd_status)
- if (
- phase is not record.phase
- or manifest_ref != record.manifest_ref
- or (phase is FastSandboxTemplatePhase.FAILED and message != record.message)
- ):
- self._update_phase(record, phase, manifest_ref, message)
+ #
+ # The watch reactor (start_background_sync / _on_template_event) is the
+ # ONLY path that moves CR build phase into the DB rows; reads (including
+ # the sandbox-create gate) trust the DB projection and never re-read the
+ # CR. "CRD is gone" is likewise event-driven: the DELETED watch event
+ # fails the row (template_service watch handler below).
def _update_phase(
self,
@@ -511,21 +470,6 @@ class FastSandboxTemplateService:
except Exception as exc: # noqa: BLE001 - reads converge on the next request
logger.warning(f"Template status persist failed: {exc}")
- def _read_crd(self, namespace: str, crd_name: str) -> Optional[dict]:
- try:
- return self._kubernetes().get_custom_object(
- GROUP, VERSION, namespace, PLURAL, crd_name
- )
- except Exception as exc: # noqa: BLE001
- logger.warning(f"SandboxTemplate CR read failed: {exc}")
- raise HTTPException(
- status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
- detail={
- "code": SandboxErrorCodes.FSB_API_ERROR,
- "message": "SandboxTemplate CRDs are unavailable.",
- },
- ) from exc
-
def _map_phase(value) -> Optional[FastSandboxTemplatePhase]:
try:
diff --git a/server/tests/test_templates.py b/server/tests/test_templates.py
index 279439775..451c4d5d6 100644
--- a/server/tests/test_templates.py
+++ b/server/tests/test_templates.py
@@ -122,6 +122,17 @@ class _FakeTemplateCRs:
self.crs[(namespace, name)]["status"] = deepcopy(status)
+def _converge(service, crs, namespace: str, name: str, event: str = "MODIFIED") -> None:
+ """Deliver the CR store's current state through the watch reactor — the
+ only path that moves build phase into DB rows (reads never re-read the
+ CR, so tests must simulate the event to converge a row)."""
+ handler = service._on_template_event(namespace)
+ if event == "DELETED":
+ handler("DELETED", {"metadata": {"name": name}})
+ return
+ handler("MODIFIED", deepcopy(crs.crs[(namespace, name)]))
+
+
@pytest.fixture
def repo(tmp_path):
repository = SQLiteFastSandboxTemplateRepository(tmp_path / "templates.db")
@@ -324,13 +335,14 @@ def test_create_rolls_back_row_on_crd_conflict(service, crs, repo):
assert excinfo.value.status_code == 409
-def test_get_syncs_phase_from_crd(service, crs, repo):
+def test_get_reads_watch_converged_row(service, crs, repo):
record = service.create_template(_create_request())
crs.set_status(
"ns-1",
record.crd_name,
{"phase": "Succeeded", "manifestRef": "s3://sandbox-images/publish/<build>/manifest.json"},
)
+ _converge(service, crs, "ns-1", record.crd_name)
synced = service.get_template(record.template_id)
assert synced.phase is FastSandboxTemplatePhase.SUCCEEDED
assert synced.manifest_ref.startswith("s3://")
@@ -340,6 +352,7 @@ def test_get_syncs_phase_from_crd(service, crs, repo):
def test_get_marks_failed_when_crd_gone(service, crs, repo):
record = service.create_template(_create_request())
crs.crs.pop(("ns-1", record.crd_name))
+ _converge(service, crs, "ns-1", record.crd_name, event="DELETED")
synced = service.get_template(record.template_id)
assert synced.phase is FastSandboxTemplatePhase.FAILED
assert synced.manifest_ref is None
@@ -352,15 +365,17 @@ def test_failed_phase_carries_message(service, crs):
record.crd_name,
{"phase": "Failed", "conditions": [{"type": "Failed", "message": "kernel panic"}]},
)
+ _converge(service, crs, "ns-1", record.crd_name)
synced = service.get_template(record.template_id)
assert synced.phase is FastSandboxTemplatePhase.FAILED
assert synced.message == "kernel panic"
-def test_list_bulk_syncs_and_filters(service, crs):
+def test_list_shows_watch_converged_phases(service, crs):
first = service.create_template(_create_request(metadata={"env": "prod"}))
second = service.create_template(_create_request(metadata={"env": "dev"}))
crs.set_status("ns-1", first.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})
+ _converge(service, crs, "ns-1", first.crd_name)
items, total = service.list_templates(metadata={"env": "prod"})
assert total == 1
assert items[0].template_id == first.template_id
@@ -425,11 +440,46 @@ def test_resolve_artifact_requires_succeeded(service, crs):
assert excinfo.value.status_code == 404
crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})
+ _converge(service, crs, "ns-1", record.crd_name)
artifact_ref, entrypoint = service.resolve_template_artifact(record.template_id)
assert artifact_ref == record.template_id
assert entrypoint == ["sleep", "1"]
+def test_resolve_gate_trusts_db_row_not_crd(service, crs):
+ """The create gate reads the DB projection only; CR state reaches it
+ exclusively through the watch reactor."""
+ record = service.create_template(_create_request(entrypoint=["sleep", "1"]))
+
+ # CR already says Succeeded but the event has not been delivered: 404.
+ crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})
+ with pytest.raises(Exception) as excinfo:
+ service.resolve_template_artifact(record.template_id)
+ assert excinfo.value.status_code == 404
+
+ # Event lands -> row converges -> gate passes, and it keeps passing even
+ # if the CR store regresses afterwards (the gate never re-reads the CR).
+ _converge(service, crs, "ns-1", record.crd_name)
+ crs.set_status("ns-1", record.crd_name, {"phase": "Failed"})
+ artifact_ref, entrypoint = service.resolve_template_artifact(record.template_id)
+ assert artifact_ref == record.template_id
+ assert entrypoint == ["sleep", "1"]
+
+
+def test_resolve_restarts_watch_for_non_terminal_rows(service, crs, repo):
+ """A row that never converged (its namespace watch failed to start when
+ the template was created) still gets a watch retry on the create gate,
+ so PENDING rows cannot get stuck permanently."""
+ record = service_record("tpl-stuck", "ns-1")
+ repo.create(record)
+ assert "ns-1" not in service._watched_namespaces
+
+ with pytest.raises(Exception) as excinfo:
+ service.resolve_template_artifact(record.template_id)
+ assert excinfo.value.status_code == 404
+ assert "ns-1" in service._watched_namespaces
+
+
@pytest.fixture
def client(service, monkeypatch):
app = FastAPI()
@@ -439,7 +489,7 @@ def client(service, monkeypatch):
yield test_client
-def test_routes_template_lifecycle(client, crs):
+def test_routes_template_lifecycle(client, crs, service):
response = client.post(
"/v1/templates",
json={
@@ -457,6 +507,7 @@ def test_routes_template_lifecycle(client, crs):
template_id = body["templateId"]
crs.set_status("ns-1", template_id, {"phase": "Succeeded", "manifestRef": "s3://b/m"})
+ _converge(service, crs, "ns-1", template_id)
detail = client.get(f"/v1/templates/{template_id}")
assert detail.status_code == 200
@@ -553,6 +604,7 @@ def test_template_mode_create_maps_artifact_and_entrypoint(service, crs):
record = service.create_template(_create_request(entrypoint=["python", "app.py"]))
crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})
+ _converge(service, crs, "ns-1", record.crd_name)
with patch.object(K8sClient, "_load_config"):
k8s = K8sClient(KubernetesRuntimeConfig())
@@ -607,6 +659,7 @@ def test_composite_routes_template_id_create_to_fsb(service, crs):
record = service.create_template(_create_request(entrypoint=["python", "app.py"]))
crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})
+ _converge(service, crs, "ns-1", record.crd_name)
with patch.object(K8sClient, "_load_config"):
k8s = K8sClient(KubernetesRuntimeConfig())