Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(server): record a recovered fsb snapshot even if it already failed

Maohaodc86d5bb6fb0
diff --git a/server/opensandbox_server/services/fast_sandbox/snapshot_runtime.py b/server/opensandbox_server/services/fast_sandbox/snapshot_runtime.py
index 10f51c7e2..4bf1b9180 100644
--- a/server/opensandbox_server/services/fast_sandbox/snapshot_runtime.py
+++ b/server/opensandbox_server/services/fast_sandbox/snapshot_runtime.py
@@ -171,16 +171,17 @@ class FastSandboxSnapshotRuntime:
)
# The request is durable and idempotent by request_id, so it may
# have been persisted before the error (e.g. a lost reply). Report
- # an existing snapshot, or it would be left without a record.
+ # an existing snapshot as submitted whatever its phase: a failed
+ # status would leave it without a record, and convergence records
+ # its terminal state once the row exists.
try:
- existing = self._fastpath.get_sandbox_snapshot(ns, snapshot_name)
+ self._fastpath.get_sandbox_snapshot(ns, snapshot_name)
except FastPathError:
return SnapshotRuntimeStatus(
state=SnapshotState.FAILED,
reason="snapshot_runtime_create_failed",
message=f"Failed to create fsb snapshot {snapshot_name}: {exc}",
)
- return self._status_from_info(existing.snapshot)
return SnapshotRuntimeStatus(
state=SnapshotState.CREATING,
reason="snapshot_runtime_submitted",
diff --git a/server/tests/test_fsb_snapshot_runtime.py b/server/tests/test_fsb_snapshot_runtime.py
index 706aba1eb..97f93fa43 100644
--- a/server/tests/test_fsb_snapshot_runtime.py
+++ b/server/tests/test_fsb_snapshot_runtime.py
@@ -20,6 +20,8 @@ from __future__ import annotations
from typing import cast
import pytest
+from opensandbox_server.api.schema import CreateSnapshotRequest
+from opensandbox_server.repositories.snapshots.sqlite import SQLiteSnapshotRepository
from opensandbox_server.services.fast_sandbox.fastpath_client import (
FastPathClient,
FastPathError,
@@ -39,6 +41,7 @@ from opensandbox_server.services.snapshot_runtime import (
SNAPSHOT_CREATE_CONFLICT_REASON,
SnapshotRuntimePreflightError,
)
+from opensandbox_server.services.snapshot_service import PersistedSnapshotService
SNAPSHOT_ID = "11111111-2222-4333-8444-555555555555"
SANDBOX_ID = "fsb-aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"
@@ -181,20 +184,58 @@ def test_create_snapshot_maps_fastpath_errors_to_failed() -> None:
assert status.reason == "snapshot_runtime_create_failed"
-def test_create_snapshot_reports_snapshot_persisted_before_the_error() -> None:
+ "phase",
+ [pb2.SNAPSHOT_PHASE_PUBLISHING, pb2.SNAPSHOT_PHASE_FAILED],
+ ids=["publishing", "failed"],
+)
+def test_create_snapshot_reports_snapshot_persisted_before_the_error(phase: int) -> None:
# The request is durable and idempotent by request_id: when the reply is
# lost (e.g. the client deadline expires), the snapshot may already exist.
+ # It is reported as submitted even if it already failed, so the service
+ # writes its row instead of leaving it without a record.
fastpath = FakeFastPathClient()
fastpath.create_error = FastPathError("DEADLINE_EXCEEDED", "deadline exceeded")
fastpath.get_by_name[("tenant-a", build_public_snapshot_name(SNAPSHOT_ID))] = _Response(
- _Info(pb2.SNAPSHOT_PHASE_PUBLISHING)
+ _Info(phase)
)
runtime, _, _ = _runtime(fastpath=fastpath)
status = runtime.create_snapshot(SNAPSHOT_ID, SANDBOX_ID, namespace="tenant-a")
assert status.state == SnapshotState.CREATING
- assert status.reason == "snapshot_runtime_in_progress"
+ assert status.reason == "snapshot_runtime_submitted"
+
+
+def test_service_records_a_recovered_snapshot_that_already_failed(tmp_path) -> None:
+ fastpath = FakeFastPathClient()
+ fastpath.create_error = FastPathError("DEADLINE_EXCEEDED", "deadline exceeded")
+ # The snapshot id is generated by the service, so answer for any name.
+ fastpath.get_sandbox_snapshot = lambda namespace, snapshot_name, **_: _Response( # type: ignore[method-assign]
+ _Info(pb2.SNAPSHOT_PHASE_FAILED, message="commit job crashed")
+ )
+ runtime, _, _ = _runtime(fastpath=fastpath)
+
+ class _SandboxService:
+ @staticmethod
+ def get_sandbox(sandbox_id: str):
+ return {"id": sandbox_id, "status": {"state": "Running"}}
+
+ service = PersistedSnapshotService(
+ SQLiteSnapshotRepository(tmp_path / "snapshots.db"),
+ _SandboxService(),
+ snapshot_runtime=runtime,
+ )
+ # The create succeeds and writes the row; reading it converges the row to
+ # the snapshot's real state, and the snapshot is kept for delete to clean up.
+ try:
+ created = service.create_snapshot(SANDBOX_ID, CreateSnapshotRequest(name="lost-reply"))
+ snapshot = service.get_snapshot(created.id)
+ finally:
+ service.close()
+
+ assert snapshot.status.state == SnapshotState.FAILED.value
+ assert fastpath.deleted == []
def test_create_snapshot_fails_when_no_snapshot_was_persisted() -> None: