Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix: address review findings across env, chart and server template path

高然8cd2fd1fe9b0
diff --git a/.github/workflows/fast-sandbox-test.yml b/.github/workflows/fast-sandbox-test.yml
index 2828c5d10..5a3b75df2 100644
--- a/.github/workflows/fast-sandbox-test.yml
+++ b/.github/workflows/fast-sandbox-test.yml
@@ -83,7 +83,10 @@ jobs:
run: sudo -E env "PATH=$PATH" ./scripts/fast-sandbox-env/integration-env.sh up --auto-clean
- name: Run base SDK e2e
- timeout-minutes: 30
+ # The suite runs once per template (native + overlaybd when
+ # OCI_STACK=1), so the step needs roughly double the single-pass
+ # budget.
+ timeout-minutes: 60
run: sudo -E env "PATH=$PATH" ./scripts/fast-sandbox-env/integration-env.sh sdk-e2e
- name: Run idle-pause SDK e2e
diff --git a/manifests/charts/fast-sandbox/templates/runtime-daemonset.yaml b/manifests/charts/fast-sandbox/templates/runtime-daemonset.yaml
index c7d48d442..b457170b3 100644
--- a/manifests/charts/fast-sandbox/templates/runtime-daemonset.yaml
+++ b/manifests/charts/fast-sandbox/templates/runtime-daemonset.yaml
@@ -213,7 +213,7 @@ spec:
[ -S /run/fast-sandbox/overlaybd/ublkd.sock ] && break
sleep 0.2
done
- exec ./firecracker-runtime
+ exec /workspace/firecracker-runtime
{{- end }}
# Only Downward API identities live in env; all tunables come from
# the mounted agent config (nodeReadiness hot-reloads per pass).
@@ -319,7 +319,23 @@ spec:
- /bin/sh
- -ec
- curl -fsS --max-time 2 --unix-socket /run/fast-sandbox/overlaybd/ublkd.sock http://localhost/v1/ping
+ # curl's own --max-time is 2s; the probe timeout must exceed it.
periodSeconds: 5
+ timeoutSeconds: 3
+ # The daemon sits on every sandbox's block-data path: a hung process
+ # (socket file alive, pings failing) must be restarted instead of
+ # stalling all node IO until manual intervention. Conservative
+ # thresholds: ~60s of consecutive failures before a restart.
+ livenessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -ec
+ - curl -fsS --max-time 4 --unix-socket /run/fast-sandbox/overlaybd/ublkd.sock http://localhost/v1/ping
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 6
volumeMounts:
- name: oci-tools
mountPath: /opt/fast-sandbox/oci
diff --git a/scripts/fast-sandbox-env/integration-env.sh b/scripts/fast-sandbox-env/integration-env.sh
index d0e37f9ff..00513745b 100755
--- a/scripts/fast-sandbox-env/integration-env.sh
+++ b/scripts/fast-sandbox-env/integration-env.sh
@@ -619,7 +619,9 @@ build_images() {
# build paths cannot drift.
FSB_SRC_DIR="$FSB_DIR" "$OSB_ROOT/manifests/release/build-fast-sandbox.sh" \
|| die "fast-sandbox image build failed"
- bake_registry_ca_into_images
+ if [[ "$OCI_STACK" == 1 ]]; then
+ bake_registry_ca_into_images
+ fi
log "building the OpenSandbox egress image ($IMG_EGRESS)"
# Context is the repo root (COPYs components/egress + components/internal).
# shellcheck disable=SC2086
@@ -1178,10 +1180,12 @@ template_up() {
--build-arg OCI_TOOLS_IMAGE="$IMG_OCI_TOOLS" -t "$IMG_BUILDER" \
-f "$FSB_DIR/build/Dockerfile.sandboxtemplate-builder" "$FSB_DIR" >/dev/null \
|| die "sandboxtemplate-builder image build failed"
- bake_registry_ca "$IMG_BUILDER"
- verify_image_ca_bake "$IMG_BUILDER"
+ if [[ "$OCI_STACK" == 1 ]]; then
+ bake_registry_ca "$IMG_BUILDER"
+ verify_image_ca_bake "$IMG_BUILDER"
+ probe_registry_tls
+ fi
kind load docker-image "$IMG_BUILDER" --name "$KIND_CLUSTER" >/dev/null
- probe_registry_tls
local created
local -a tmpl_args=(
template create
@@ -1205,7 +1209,9 @@ template_up() {
[[ -n "$manifest_ref" ]] || fail "template manifestRef is empty"
log "template manifestRef: $manifest_ref"
pass "SandboxTemplate Succeeded + artifacts published (via opensandbox CLI)"
- [[ "$OCI_STACK" == 1 ]] && template_up_overlaybd
+ if [[ "$OCI_STACK" == 1 ]]; then
+ template_up_overlaybd
+ fi
return 0
}
@@ -1231,16 +1237,18 @@ _overlaybd_template_failed() {
template_up_overlaybd() {
log "verify: creating the overlaybd template via the opensandbox CLI (publish=oci://$OCI_REGISTRY_HOST/$OCI_REGISTRY_REPO)"
- local created
- created="$(osb_api template create \
- --image "$SBX_IMAGE" \
- --publish "oci://$OCI_REGISTRY_HOST/$OCI_REGISTRY_REPO" \
- --format overlaybd \
- --resource cpu=1 --resource memory=512Mi --resource disk=2Gi \
- --warmup-seconds 15 \
- --metadata origin=fast-sandbox-env-oci \
- -o json 2>/dev/null)" \
- || fail "overlaybd template create failed against $SERVER_URL: $(osb_api template create --image x --publish y --format overlaybd 2>&1 >/dev/null | head -c 400)"
+ local created create_err="$WORK/overlaybd-create.err"
+ local -a ovl_args=(
+ template create
+ --image "$SBX_IMAGE"
+ --publish "oci://$OCI_REGISTRY_HOST/$OCI_REGISTRY_REPO"
+ --format overlaybd
+ --resource cpu=1 --resource memory=512Mi --resource disk=2Gi
+ --warmup-seconds 15
+ --metadata origin=fast-sandbox-env-oci
+ )
+ created="$(osb_api "${ovl_args[@]}" -o json 2>"$create_err")" \
+ || fail "overlaybd template create failed against $SERVER_URL: $(head -c 400 "$create_err")"
OCI_TEMPLATE_ID="$(printf '%s' "$created" | jq -r '.template_id')"
[[ -n "$OCI_TEMPLATE_ID" && "$OCI_TEMPLATE_ID" != "null" ]] || fail "overlaybd template create carried no templateId"
printf '%s' "$OCI_TEMPLATE_ID" > "$WORK/overlaybd-template-id"
@@ -1604,32 +1612,22 @@ warm_images_ready() {
# shared kernel.
ublk_module_up() {
[[ "$OCI_STACK" == 1 ]] || {
- log "OCI stack disabled (OCI_STACK=0); skipping the ublk check"
+ log "OCI_STACK=0: skipping the ublk check (overlaybd disabled)"
return 0
}
+ # The overlaybd stack is unusable without the kernel module: the node
+ # ublkd sidecar crash-loops without /dev/ublk-control and every build
+ # fails at its probe device, so fail here instead of ~16 minutes later.
if ! grep -qiE 'ublk-char|ublkc' /proc/devices; then
- log "ublk not registered yet; attempting modprobe ublk_drv"
+ log "ublk not registered; attempting modprobe ublk_drv"
sudo_ modprobe ublk_drv 2>/dev/null || true
fi
- # Control node numbers: prefer the host node (misc major 10, dynamic
- # minor); fall back to kernel conventions when the runner's /dev hides it.
- UBLK_CTRL_NODE="/dev/ublk-control"
- [[ -e "$UBLK_CTRL_NODE" ]] || UBLK_CTRL_NODE="/dev/ublkc"
- if [[ -e "$UBLK_CTRL_NODE" ]]; then
- UBLK_CTRL_MAJOR="$(ls -l "$UBLK_CTRL_NODE" | awk '{gsub(",", "", $5); print $5}')"
- UBLK_CTRL_MINOR="$(ls -l "$UBLK_CTRL_NODE" | awk '{print $6}')"
- else
- # misc major is statically 10; the minor cannot be derived — the kind
- # node check below will fail with this context in the message.
- UBLK_CTRL_MAJOR=10
- UBLK_CTRL_MINOR=""
- fi
- UBLK_CHAR_MAJOR="$(awk '$2 == "ublk-char" || $2 == "ublkc" {print $1; exit}' /proc/devices)"
- log "ublk numbers: ctrl=$UBLK_CTRL_NODE ($UBLK_CTRL_MAJOR,$UBLK_CTRL_MINOR) char-major=$UBLK_CHAR_MAJOR"
- pass "ublk registered in /proc/devices"
+ grep -qiE 'ublk-char|ublkc' /proc/devices \
+ || die "ublk is not registered in /proc/devices after modprobe: the kernel needs CONFIG_BLK_DEV_UBLK (and the runner must be able to load modules); overlaybd requires it when OCI_STACK=1"
+ log "ublk registered: $(grep -i ublk /proc/devices | tr '\n' ' ')"
+ pass "ublk module registered"
}
-
render_pool() { # > $GEN_DIR/firecracker-egress-pool.yaml
local src="$MANIFESTS_DIR/pool/firecracker-egress-pool.yaml" out="$GEN_DIR/firecracker-egress-pool.yaml"
mkdir -p "$GEN_DIR"
@@ -1806,8 +1804,15 @@ render_server_config() { # > $GEN_DIR/osb-server-config.toml
mkdir -p "$GEN_DIR"
awk -v api_key="$SERVER_API_KEY" \
-v fastpath="$FASTPATH_ENDPOINT" -v fsb_ns="$RESOURCE_NS" -v pool="$POOL_NAME" \
- -v execd="$EXECD" '
- { gsub(/@SERVER_API_KEY@/, api_key)
+ -v execd="$EXECD" -v oci_secret="$OCI_REGISTRY_SECRET" -v oci_stack="$OCI_STACK" '
+ { if ($0 == "# @OCI_PUBLISH_SECRET@") {
+ # OverlayBD templates publish through the in-cluster registry; the
+ # server bakes this Secret name as output.publishSecretRef. Emitted
+ # in-place so the key stays inside the [kubernetes] table.
+ if (oci_stack == "1")
+ printf "template_oci_publish_secret = \"%s\"\n", oci_secret
+ next }
+ gsub(/@SERVER_API_KEY@/, api_key)
gsub(/@SIGNING_KEY@/, signing_key)
gsub(/@FASTPATH_ENDPOINT@/, fastpath)
gsub(/@FSB_NAMESPACE@/, fsb_ns)
@@ -1836,14 +1841,8 @@ fastpath_endpoint = "@FASTPATH_ENDPOINT@"
fastpath_resource_pool = "@POOL_NAME@"
fastpath_wait_ready_seconds = 30.0
template_s3_publish_secret = "sandbox-oss-credentials"
+# @OCI_PUBLISH_SECRET@
TOML
- if [[ "$OCI_STACK" == 1 ]]; then
- # OverlayBD templates publish through the in-cluster registry; the
- # server bakes this Secret name as output.publishSecretRef.
- cat >> "$GEN_DIR/osb-server-config.toml" <<TOML
-template_oci_publish_secret = "$OCI_REGISTRY_SECRET"
-TOML
- fi
if [[ "$IDLE_PAUSE" == "1" ]]; then
# Dotted keys nest the redis sub-table.
cat >> "$GEN_DIR/osb-server-config.toml" <<TOML
diff --git a/server/opensandbox_server/api/schema.py b/server/opensandbox_server/api/schema.py
index fc7fec5cb..e00ca3fa4 100644
--- a/server/opensandbox_server/api/schema.py
+++ b/server/opensandbox_server/api/schema.py
@@ -1727,7 +1727,13 @@ class FsbTemplateStatus(BaseModel):
manifest_ref: Optional[str] = Field(
None,
alias="manifestRef",
- description="S3 manifest reference of the published artifacts; present when Succeeded",
+ description=(
+ "Artifact reference of the published snapshot set — "
+ "s3://.../manifest.json for format=native, or the immutable "
+ "oci://registry/repo@sha256:... manifest for format=overlaybd "
+ "(the runnable reference for template-mode creates); present "
+ "when Succeeded"
+ ),
)
message: Optional[str] = Field(
None,
@@ -1752,7 +1758,13 @@ class FsbTemplate(BaseModel):
)
metadata: Optional[Dict[str, str]] = Field(None, description="Custom metadata from the creation request")
readiness: Optional[FsbTemplateReadiness] = Field(None, description="Build-side readiness gate")
- publish: str = Field(..., description="S3-compatible publish target")
+ publish: str = Field(
+ ...,
+ description=(
+ "Artifact publish destination — oci://registry/repository for "
+ "format=overlaybd (default), s3://bucket/prefix for format=native"
+ ),
+ )
format: Literal["native", "overlaybd"] = Field(..., description="Snapshot storage encoding")
status: FsbTemplateStatus = Field(..., description="Build status")
created_at: datetime = Field(..., alias="createdAt", description="Creation timestamp")
diff --git a/server/opensandbox_server/services/templates/template_service.py b/server/opensandbox_server/services/templates/template_service.py
index 5a59882de..8b90b1e86 100644
--- a/server/opensandbox_server/services/templates/template_service.py
+++ b/server/opensandbox_server/services/templates/template_service.py
@@ -94,10 +94,12 @@ def _publish_validation_error(format: str, publish: str) -> Optional[str]:
Mirrors the SandboxTemplate builder's output validation so an invalid
destination fails as a 400 at creation instead of a failed build Pod.
- overlaybd requires ``oci://host/repository``: the host may carry a port,
- the repository path must be non-empty and free of tag (``:``) or digest
- (``@``) suffixes — the system generates the build tags itself. native
- requires an ``s3://`` object-store target.
+ overlaybd requires ``oci://host/repository``: the host may carry a port
+ but no userinfo (credentials must come from publishSecretRef, and the
+ publish value is echoed back by the template API), the repository path
+ must be non-empty, slash-terminated-free, and free of tag (``:``) or
+ digest (``@``) suffixes — the system generates the build tags itself.
+ native requires an ``s3://bucket`` target with a non-empty bucket.
"""
if format == "overlaybd":
if not publish.startswith("oci://"):
@@ -113,12 +115,23 @@ def _publish_validation_error(format: str, publish: str) -> Optional[str]:
"publish must be 'oci://registry/repository' with a "
"non-empty registry host and repository path."
)
+ if "@" in host:
+ return (
+ "publish must not embed credentials in the registry host "
+ "(userinfo like user:pass@host is rejected; publish "
+ "credentials come from publishSecretRef)."
+ )
if "@" in repository or ":" in repository:
return (
"publish must be repository-only: tags and digests are "
"rejected because the system generates an immutable "
"per-build tag pair."
)
+ if repository.endswith("/"):
+ return (
+ "publish must be 'oci://registry/repository' without a "
+ "trailing slash."
+ )
return None
if format == "native":
if not publish.startswith("s3://"):
@@ -126,6 +139,11 @@ def _publish_validation_error(format: str, publish: str) -> Optional[str]:
"native templates publish to an S3-compatible store: "
"publish must be 's3://bucket/prefix'."
)
+ if not publish[len("s3://"):].partition("/")[0]:
+ return (
+ "publish must be 's3://bucket/prefix' with a non-empty "
+ "bucket name."
+ )
return None
return (
f"Unsupported template format {format!r}; "
diff --git a/server/tests/test_templates.py b/server/tests/test_templates.py
index 7490c8867..16570bb93 100644
--- a/server/tests/test_templates.py
+++ b/server/tests/test_templates.py
@@ -575,6 +575,8 @@ def test_create_overlaybd_requires_oci_publish_secret(repo, crs, monkeypatch):
"oci:///templates",
"oci://registry.example.com/sandbox/templates:v1",
"oci://registry.example.com/sandbox/templates@sha256:abc",
+ "oci://registry.example.com/sandbox/templates/",
+ "oci://user:[email protected]/sandbox/templates",
],
)
def test_create_overlaybd_rejects_invalid_publish(service, publish):
@@ -601,6 +603,13 @@ def test_create_native_rejects_oci_publish(service):
assert "s3://" in str(excinfo.value.detail)
+def test_create_native_rejects_bucketless_publish(service):
+ with pytest.raises(HTTPException) as excinfo:
+ service.create_template(_create_request(publish="s3://"))
+ assert excinfo.value.status_code == 400
+ assert "bucket" in str(excinfo.value.detail)
+
+
@pytest.fixture
def client(service, monkeypatch):
app = FastAPI()