Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(kubernetes): read the commit result from the API server when the cache lags

JasonBuildAI48c17a73d96a
diff --git a/kubernetes/cmd/controller/main.go b/kubernetes/cmd/controller/main.go
index a80f815ed..287b7dad7 100644
--- a/kubernetes/cmd/controller/main.go
+++ b/kubernetes/cmd/controller/main.go
@@ -497,6 +497,7 @@ func main() {
}
if err := (&controller.SandboxSnapshotReconciler{
Client: mgr.GetClient(),
+ APIReader: mgr.GetAPIReader(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("sandboxsnapshot-controller"),
ImageCommitterImage: imageCommitterImage,
diff --git a/kubernetes/internal/controller/sandboxsnapshot_controller.go b/kubernetes/internal/controller/sandboxsnapshot_controller.go
index 4d3190f86..0d9d49704 100644
--- a/kubernetes/internal/controller/sandboxsnapshot_controller.go
+++ b/kubernetes/internal/controller/sandboxsnapshot_controller.go
@@ -70,6 +70,11 @@ type SandboxSnapshotReconciler struct {
Scheme *runtime.Scheme
Recorder record.EventRecorder
+ // APIReader reads straight from the API server, bypassing the informer
+ // cache. It is required when a cached read can be stale (for example the
+ // commit Pod status that lags behind the commit Job status).
+ APIReader client.Reader
+
// ImageCommitterImage is the image used for commit and unpause Jobs.
ImageCommitterImage string
@@ -169,6 +174,9 @@ func (r *SandboxSnapshotReconciler) Reconcile(ctx context.Context, req ctrl.Requ
// SetupWithManager sets up the controller with the Manager.
func (r *SandboxSnapshotReconciler) SetupWithManager(mgr ctrl.Manager) error {
+ if r.APIReader == nil {
+ r.APIReader = mgr.GetAPIReader()
+ }
return ctrl.NewControllerManagedBy(mgr).
For(&sandboxv1alpha1.SandboxSnapshot{}).
Owns(&batchv1.Job{}).
diff --git a/kubernetes/internal/controller/sandboxsnapshot_controller_test.go b/kubernetes/internal/controller/sandboxsnapshot_controller_test.go
index 7738ce9ed..2adebcd7a 100644
--- a/kubernetes/internal/controller/sandboxsnapshot_controller_test.go
+++ b/kubernetes/internal/controller/sandboxsnapshot_controller_test.go
@@ -36,6 +36,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/client/interceptor"
sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1"
snapshotcontract "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/snapshot"
@@ -54,9 +55,10 @@ func newTestSnapshotReconciler(objs ...client.Object) *SandboxSnapshotReconciler
Build()
return &SandboxSnapshotReconciler{
- Client: fakeClient,
- Scheme: scheme,
- Recorder: record.NewFakeRecorder(10),
+ Client: fakeClient,
+ APIReader: fakeClient,
+ Scheme: scheme,
+ Recorder: record.NewFakeRecorder(10),
}
}
@@ -332,6 +334,71 @@ func TestSandboxSnapshotHandleCommitting_RootfsCommitWithAllDigestsSucceeds(t *t
assert.Equal(t, "sha256:main", updated.Status.Containers[0].ImageDigest)
}
+func TestSandboxSnapshotHandleCommitting_StaleCacheReadsResultFromAPIServer(t *testing.T) {
+ snapshot := &sandboxv1alpha1.SandboxSnapshot{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-snapshot",
+ Namespace: "default",
+ },
+ Status: sandboxv1alpha1.SandboxSnapshotStatus{
+ Phase: sandboxv1alpha1.SandboxSnapshotPhaseCommitting,
+ Format: sandboxv1alpha1.SandboxSnapshotFormatRootfsV1,
+ Containers: []sandboxv1alpha1.ContainerSnapshot{
+ {ContainerName: "main", ImageURI: "registry/test-main:tag"},
+ },
+ },
+ }
+ job := &batchv1.Job{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-snapshot-commit",
+ Namespace: "default",
+ },
+ Status: batchv1.JobStatus{
+ Succeeded: 1,
+ },
+ }
+ commitPod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-snapshot-commit-abcde",
+ Namespace: "default",
+ Labels: map[string]string{
+ "job-name": "test-snapshot-commit",
+ },
+ },
+ Status: corev1.PodStatus{
+ ContainerStatuses: []corev1.ContainerStatus{
+ {
+ Name: commitJobContainerName,
+ State: corev1.ContainerState{
+ Terminated: &corev1.ContainerStateTerminated{
+ ExitCode: 0,
+ Message: `{"containers":[{"name":"main","image":"registry/test-main:tag","digest":"sha256:main"}]}`,
+ },
+ },
+ },
+ },
+ },
+ }
+
+ r := newTestSnapshotReconciler(snapshot, job, commitPod)
+ live := r.Client.(client.WithWatch)
+ // The informer cache has not observed the commit Pod yet, while the API
+ // server already holds its termination message.
+ r.Client = interceptor.NewClient(live, interceptor.Funcs{
+ List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { return nil },
+ })
+
+ result, err := r.handleCommitting(context.Background(), snapshot)
+ require.NoError(t, err)
+ assert.Equal(t, ctrl.Result{}, result)
+
+ updated := &sandboxv1alpha1.SandboxSnapshot{}
+ require.NoError(t, live.Get(context.Background(), types.NamespacedName{Name: "test-snapshot", Namespace: "default"}, updated))
+ assert.Equal(t, sandboxv1alpha1.SandboxSnapshotPhaseSucceed, updated.Status.Phase)
+ require.Len(t, updated.Status.Containers, 1)
+ assert.Equal(t, "sha256:main", updated.Status.Containers[0].ImageDigest)
+}
+
func findSnapshotCondition(conditions []sandboxv1alpha1.SandboxSnapshotCondition, conditionType sandboxv1alpha1.SandboxSnapshotConditionType) *sandboxv1alpha1.SandboxSnapshotCondition {
for i := range conditions {
if conditions[i].Type == conditionType {
diff --git a/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go b/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go
index 2e07fc788..00ee9d87d 100644
--- a/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go
+++ b/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go
@@ -882,9 +882,25 @@ func containersWithoutDigest(containers []sandboxv1alpha1.ContainerSnapshot) []s
}
func (r *SandboxSnapshotReconciler) getCommitJobResult(ctx context.Context, namespace, jobName string) (*commitJobResult, bool, error) {
+ result, found, err := findCommitJobResult(ctx, r.Client, namespace, jobName)
+ if err != nil || found {
+ return result, found, err
+ }
+ // The informer cache can lag behind the commit Job status: a Job is
+ // reported Succeeded only after its Pod reached a terminal state, but the
+ // Pod update may not have reached the cache yet. Reading the Pod from the
+ // API server keeps a cache miss from being mistaken for a lost result,
+ // which would be an unrecoverable pause failure.
+ if r.APIReader == nil {
+ return nil, false, nil
+ }
+ return findCommitJobResult(ctx, r.APIReader, namespace, jobName)
+}
+
+func findCommitJobResult(ctx context.Context, reader client.Reader, namespace, jobName string) (*commitJobResult, bool, error) {
for _, labelKey := range []string{"job-name", "batch.kubernetes.io/job-name"} {
podList := &corev1.PodList{}
- if err := r.List(ctx, podList,
+ if err := reader.List(ctx, podList,
client.InNamespace(namespace),
client.MatchingLabels{labelKey: jobName},
); err != nil {