Secure, Fast, and Extensible Sandbox runtime for AI agents.
refactor(server): make template DB rows the read authority for phase gates
高然b816aad1023c
diff --git a/server/opensandbox_server/api/templates.py b/server/opensandbox_server/api/templates.pyindex 193f28621..546ba7dd2 100644--- a/server/opensandbox_server/api/templates.py+++ b/server/opensandbox_server/api/templates.py@@ -68,8 +68,9 @@ def _get_template_service() -> FastSandboxTemplateService:detail=_TEMPLATE_NOT_K8S_DETAIL,)_service = FastSandboxTemplateService(config)- # Watch-driven convergence: DB rows track the CRD build phase even when- # nobody polls the read API.+ # Watch-driven convergence: the watch reactor is the only path that moves+ # CR build phase into the DB rows; reads (including the create gate) trust+ # the DB and never re-read the CR._service.start_background_sync()return _servicediff --git a/server/opensandbox_server/services/fast_sandbox/service.py b/server/opensandbox_server/services/fast_sandbox/service.pyindex f2a962d19..e10fd9b10 100644--- a/server/opensandbox_server/services/fast_sandbox/service.py+++ b/server/opensandbox_server/services/fast_sandbox/service.py@@ -268,7 +268,7 @@ class FastSandboxService(SandboxService, ExtensionService):if self._template_service is None:self._template_service = FastSandboxTemplateService(self._app_config)# Keep template rows converged even without /templates traffic- # (template-mode create resolves against the synced row).+ # (template-mode create resolves against the watch-converged row).self._template_service.start_background_sync()return self._template_servicediff --git a/server/opensandbox_server/services/templates/template_service.py b/server/opensandbox_server/services/templates/template_service.pyindex 2b21d80e9..8c1e34b94 100644--- a/server/opensandbox_server/services/templates/template_service.py+++ b/server/opensandbox_server/services/templates/template_service.py@@ -301,7 +301,7 @@ class FastSandboxTemplateService:record = self._repo().get(template_id, namespace)if record is None:raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_TEMPLATE_NOT_FOUND)- return self._sync_status(record)+ return recorddef list_templates(self,@@ -319,7 +319,6 @@ class FastSandboxTemplateService:page_size=page_size,))- self._sync_status_bulk(namespace, result.items)return result.items, result.total_itemsdef delete_template(self, template_id: str) -> None:@@ -354,12 +353,22 @@ class FastSandboxTemplateService:warmImages and older clients. Template-mode Create uses this:unknown or non-Succeeded templates both yield 404, so no existenceinformation leaks.++ The DB row is the read authority: the watch reactor is the only+ writer of build-phase transitions (CR status -> watch event -> row),+ so this gate is a pure DB read on the sandbox-create hot path."""namespace = self._resolve_namespace()record = self._repo().get(template_id, namespace)if record is None:raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_TEMPLATE_NOT_FOUND)- record = self._sync_status(record)+ if record.phase in (FastSandboxTemplatePhase.PENDING, FastSandboxTemplatePhase.BUILDING):+ # Non-terminal row on the create gate: make sure the namespace+ # watch is alive so the row still converges (e.g. the watch+ # failed to start when the template was created). Healthy+ # deployments pay a set-membership check; start() is+ # non-blocking either way.+ self._ensure_namespace_watch(namespace)if record.phase is not FastSandboxTemplatePhase.SUCCEEDED:raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=_TEMPLATE_NOT_FOUND)entrypoint = list(record.spec.get("entrypoint") or DEFAULT_ENTRYPOINT)@@ -432,62 +441,12 @@ class FastSandboxTemplateService:}# -- status sync --------------------------------------------------------------- def _sync_status(self, record: FastSandboxTemplateRecord) -> FastSandboxTemplateRecord:- crd = self._read_crd(record.namespace, record.crd_name)- if crd is None:- # A build that never finished cannot finish once the CRD is gone;- # a Succeeded row keeps its artifact reference.- if record.phase in (FastSandboxTemplatePhase.PENDING, FastSandboxTemplatePhase.BUILDING):- self._update_phase(- record, FastSandboxTemplatePhase.FAILED, None, "SandboxTemplate CRD is gone."- )- return record- crd_status = crd.get("status") or {}- phase = _map_phase(crd_status.get("phase"))- if phase is None:- return record- manifest_ref = crd_status.get("manifestRef") or None- message = _failure_message(crd_status)- if (- phase is not record.phase- or manifest_ref != record.manifest_ref- or (phase is FastSandboxTemplatePhase.FAILED and message != record.message)- ):- self._update_phase(record, phase, manifest_ref, message)- return record-- def _sync_status_bulk(self, namespace: str, records: list[FastSandboxTemplateRecord]) -> None:- if not records:- return- try:- crds = self._kubernetes().list_custom_objects(- GROUP, VERSION, namespace, PLURAL, ignore_not_found=True- ) or []- except Exception as exc: # noqa: BLE001 - reads converge on the next request- logger.warning(f"SandboxTemplate CR list failed during sync: {exc}")- return- by_name = {crd.get("metadata", {}).get("name", ""): crd for crd in crds}- for record in records:- crd = by_name.get(record.crd_name)- if crd is None:- if record.phase in (FastSandboxTemplatePhase.PENDING, FastSandboxTemplatePhase.BUILDING):- self._update_phase(- record, FastSandboxTemplatePhase.FAILED, None, "SandboxTemplate CRD is gone."- )- continue- crd_status = crd.get("status") or {}- phase = _map_phase(crd_status.get("phase"))- if phase is None:- continue- manifest_ref = crd_status.get("manifestRef") or None- message = _failure_message(crd_status)- if (- phase is not record.phase- or manifest_ref != record.manifest_ref- or (phase is FastSandboxTemplatePhase.FAILED and message != record.message)- ):- self._update_phase(record, phase, manifest_ref, message)+ #+ # The watch reactor (start_background_sync / _on_template_event) is the+ # ONLY path that moves CR build phase into the DB rows; reads (including+ # the sandbox-create gate) trust the DB projection and never re-read the+ # CR. "CRD is gone" is likewise event-driven: the DELETED watch event+ # fails the row (template_service watch handler below).def _update_phase(self,@@ -511,21 +470,6 @@ class FastSandboxTemplateService:except Exception as exc: # noqa: BLE001 - reads converge on the next requestlogger.warning(f"Template status persist failed: {exc}")- def _read_crd(self, namespace: str, crd_name: str) -> Optional[dict]:- try:- return self._kubernetes().get_custom_object(- GROUP, VERSION, namespace, PLURAL, crd_name- )- except Exception as exc: # noqa: BLE001- logger.warning(f"SandboxTemplate CR read failed: {exc}")- raise HTTPException(- status_code=status.HTTP_503_SERVICE_UNAVAILABLE,- detail={- "code": SandboxErrorCodes.FSB_API_ERROR,- "message": "SandboxTemplate CRDs are unavailable.",- },- ) from exc-def _map_phase(value) -> Optional[FastSandboxTemplatePhase]:try:diff --git a/server/tests/test_templates.py b/server/tests/test_templates.pyindex 279439775..451c4d5d6 100644--- a/server/tests/test_templates.py+++ b/server/tests/test_templates.py@@ -122,6 +122,17 @@ class _FakeTemplateCRs:self.crs[(namespace, name)]["status"] = deepcopy(status)+def _converge(service, crs, namespace: str, name: str, event: str = "MODIFIED") -> None:+ """Deliver the CR store's current state through the watch reactor — the+ only path that moves build phase into DB rows (reads never re-read the+ CR, so tests must simulate the event to converge a row)."""+ handler = service._on_template_event(namespace)+ if event == "DELETED":+ handler("DELETED", {"metadata": {"name": name}})+ return+ handler("MODIFIED", deepcopy(crs.crs[(namespace, name)]))++@pytest.fixturedef repo(tmp_path):repository = SQLiteFastSandboxTemplateRepository(tmp_path / "templates.db")@@ -324,13 +335,14 @@ def test_create_rolls_back_row_on_crd_conflict(service, crs, repo):assert excinfo.value.status_code == 409-def test_get_syncs_phase_from_crd(service, crs, repo):+def test_get_reads_watch_converged_row(service, crs, repo):record = service.create_template(_create_request())crs.set_status("ns-1",record.crd_name,{"phase": "Succeeded", "manifestRef": "s3://sandbox-images/publish/<build>/manifest.json"},)+ _converge(service, crs, "ns-1", record.crd_name)synced = service.get_template(record.template_id)assert synced.phase is FastSandboxTemplatePhase.SUCCEEDEDassert synced.manifest_ref.startswith("s3://")@@ -340,6 +352,7 @@ def test_get_syncs_phase_from_crd(service, crs, repo):def test_get_marks_failed_when_crd_gone(service, crs, repo):record = service.create_template(_create_request())crs.crs.pop(("ns-1", record.crd_name))+ _converge(service, crs, "ns-1", record.crd_name, event="DELETED")synced = service.get_template(record.template_id)assert synced.phase is FastSandboxTemplatePhase.FAILEDassert synced.manifest_ref is None@@ -352,15 +365,17 @@ def test_failed_phase_carries_message(service, crs):record.crd_name,{"phase": "Failed", "conditions": [{"type": "Failed", "message": "kernel panic"}]},)+ _converge(service, crs, "ns-1", record.crd_name)synced = service.get_template(record.template_id)assert synced.phase is FastSandboxTemplatePhase.FAILEDassert synced.message == "kernel panic"-def test_list_bulk_syncs_and_filters(service, crs):+def test_list_shows_watch_converged_phases(service, crs):first = service.create_template(_create_request(metadata={"env": "prod"}))second = service.create_template(_create_request(metadata={"env": "dev"}))crs.set_status("ns-1", first.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})+ _converge(service, crs, "ns-1", first.crd_name)items, total = service.list_templates(metadata={"env": "prod"})assert total == 1assert items[0].template_id == first.template_id@@ -425,11 +440,46 @@ def test_resolve_artifact_requires_succeeded(service, crs):assert excinfo.value.status_code == 404crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})+ _converge(service, crs, "ns-1", record.crd_name)artifact_ref, entrypoint = service.resolve_template_artifact(record.template_id)assert artifact_ref == record.template_idassert entrypoint == ["sleep", "1"]+def test_resolve_gate_trusts_db_row_not_crd(service, crs):+ """The create gate reads the DB projection only; CR state reaches it+ exclusively through the watch reactor."""+ record = service.create_template(_create_request(entrypoint=["sleep", "1"]))++ # CR already says Succeeded but the event has not been delivered: 404.+ crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})+ with pytest.raises(Exception) as excinfo:+ service.resolve_template_artifact(record.template_id)+ assert excinfo.value.status_code == 404++ # Event lands -> row converges -> gate passes, and it keeps passing even+ # if the CR store regresses afterwards (the gate never re-reads the CR).+ _converge(service, crs, "ns-1", record.crd_name)+ crs.set_status("ns-1", record.crd_name, {"phase": "Failed"})+ artifact_ref, entrypoint = service.resolve_template_artifact(record.template_id)+ assert artifact_ref == record.template_id+ assert entrypoint == ["sleep", "1"]+++def test_resolve_restarts_watch_for_non_terminal_rows(service, crs, repo):+ """A row that never converged (its namespace watch failed to start when+ the template was created) still gets a watch retry on the create gate,+ so PENDING rows cannot get stuck permanently."""+ record = service_record("tpl-stuck", "ns-1")+ repo.create(record)+ assert "ns-1" not in service._watched_namespaces++ with pytest.raises(Exception) as excinfo:+ service.resolve_template_artifact(record.template_id)+ assert excinfo.value.status_code == 404+ assert "ns-1" in service._watched_namespaces++@pytest.fixturedef client(service, monkeypatch):app = FastAPI()@@ -439,7 +489,7 @@ def client(service, monkeypatch):yield test_client-def test_routes_template_lifecycle(client, crs):+def test_routes_template_lifecycle(client, crs, service):response = client.post("/v1/templates",json={@@ -457,6 +507,7 @@ def test_routes_template_lifecycle(client, crs):template_id = body["templateId"]crs.set_status("ns-1", template_id, {"phase": "Succeeded", "manifestRef": "s3://b/m"})+ _converge(service, crs, "ns-1", template_id)detail = client.get(f"/v1/templates/{template_id}")assert detail.status_code == 200@@ -553,6 +604,7 @@ def test_template_mode_create_maps_artifact_and_entrypoint(service, crs):record = service.create_template(_create_request(entrypoint=["python", "app.py"]))crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})+ _converge(service, crs, "ns-1", record.crd_name)with patch.object(K8sClient, "_load_config"):k8s = K8sClient(KubernetesRuntimeConfig())@@ -607,6 +659,7 @@ def test_composite_routes_template_id_create_to_fsb(service, crs):record = service.create_template(_create_request(entrypoint=["python", "app.py"]))crs.set_status("ns-1", record.crd_name, {"phase": "Succeeded", "manifestRef": "s3://b/m"})+ _converge(service, crs, "ns-1", record.crd_name)with patch.object(K8sClient, "_load_config"):k8s = K8sClient(KubernetesRuntimeConfig())