Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(kubernetes): read the commit result from the API server when the cache lags
JasonBuildAI48c17a73d96a
diff --git a/kubernetes/cmd/controller/main.go b/kubernetes/cmd/controller/main.goindex a80f815ed..287b7dad7 100644--- a/kubernetes/cmd/controller/main.go+++ b/kubernetes/cmd/controller/main.go@@ -497,6 +497,7 @@ func main() {}if err := (&controller.SandboxSnapshotReconciler{Client: mgr.GetClient(),+ APIReader: mgr.GetAPIReader(),Scheme: mgr.GetScheme(),Recorder: mgr.GetEventRecorderFor("sandboxsnapshot-controller"),ImageCommitterImage: imageCommitterImage,diff --git a/kubernetes/internal/controller/sandboxsnapshot_controller.go b/kubernetes/internal/controller/sandboxsnapshot_controller.goindex 4d3190f86..0d9d49704 100644--- a/kubernetes/internal/controller/sandboxsnapshot_controller.go+++ b/kubernetes/internal/controller/sandboxsnapshot_controller.go@@ -70,6 +70,11 @@ type SandboxSnapshotReconciler struct {Scheme *runtime.SchemeRecorder record.EventRecorder+ // APIReader reads straight from the API server, bypassing the informer+ // cache. It is required when a cached read can be stale (for example the+ // commit Pod status that lags behind the commit Job status).+ APIReader client.Reader+// ImageCommitterImage is the image used for commit and unpause Jobs.ImageCommitterImage string@@ -169,6 +174,9 @@ func (r *SandboxSnapshotReconciler) Reconcile(ctx context.Context, req ctrl.Requ// SetupWithManager sets up the controller with the Manager.func (r *SandboxSnapshotReconciler) SetupWithManager(mgr ctrl.Manager) error {+ if r.APIReader == nil {+ r.APIReader = mgr.GetAPIReader()+ }return ctrl.NewControllerManagedBy(mgr).For(&sandboxv1alpha1.SandboxSnapshot{}).Owns(&batchv1.Job{}).diff --git a/kubernetes/internal/controller/sandboxsnapshot_controller_test.go b/kubernetes/internal/controller/sandboxsnapshot_controller_test.goindex 7738ce9ed..2adebcd7a 100644--- a/kubernetes/internal/controller/sandboxsnapshot_controller_test.go+++ b/kubernetes/internal/controller/sandboxsnapshot_controller_test.go@@ -36,6 +36,7 @@ import (ctrl "sigs.k8s.io/controller-runtime""sigs.k8s.io/controller-runtime/pkg/client""sigs.k8s.io/controller-runtime/pkg/client/fake"+ "sigs.k8s.io/controller-runtime/pkg/client/interceptor"sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1"snapshotcontract "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/snapshot"@@ -54,9 +55,10 @@ func newTestSnapshotReconciler(objs ...client.Object) *SandboxSnapshotReconcilerBuild()return &SandboxSnapshotReconciler{- Client: fakeClient,- Scheme: scheme,- Recorder: record.NewFakeRecorder(10),+ Client: fakeClient,+ APIReader: fakeClient,+ Scheme: scheme,+ Recorder: record.NewFakeRecorder(10),}}@@ -332,6 +334,71 @@ func TestSandboxSnapshotHandleCommitting_RootfsCommitWithAllDigestsSucceeds(t *tassert.Equal(t, "sha256:main", updated.Status.Containers[0].ImageDigest)}+func TestSandboxSnapshotHandleCommitting_StaleCacheReadsResultFromAPIServer(t *testing.T) {+ snapshot := &sandboxv1alpha1.SandboxSnapshot{+ ObjectMeta: metav1.ObjectMeta{+ Name: "test-snapshot",+ Namespace: "default",+ },+ Status: sandboxv1alpha1.SandboxSnapshotStatus{+ Phase: sandboxv1alpha1.SandboxSnapshotPhaseCommitting,+ Format: sandboxv1alpha1.SandboxSnapshotFormatRootfsV1,+ Containers: []sandboxv1alpha1.ContainerSnapshot{+ {ContainerName: "main", ImageURI: "registry/test-main:tag"},+ },+ },+ }+ job := &batchv1.Job{+ ObjectMeta: metav1.ObjectMeta{+ Name: "test-snapshot-commit",+ Namespace: "default",+ },+ Status: batchv1.JobStatus{+ Succeeded: 1,+ },+ }+ commitPod := &corev1.Pod{+ ObjectMeta: metav1.ObjectMeta{+ Name: "test-snapshot-commit-abcde",+ Namespace: "default",+ Labels: map[string]string{+ "job-name": "test-snapshot-commit",+ },+ },+ Status: corev1.PodStatus{+ ContainerStatuses: []corev1.ContainerStatus{+ {+ Name: commitJobContainerName,+ State: corev1.ContainerState{+ Terminated: &corev1.ContainerStateTerminated{+ ExitCode: 0,+ Message: `{"containers":[{"name":"main","image":"registry/test-main:tag","digest":"sha256:main"}]}`,+ },+ },+ },+ },+ },+ }++ r := newTestSnapshotReconciler(snapshot, job, commitPod)+ live := r.Client.(client.WithWatch)+ // The informer cache has not observed the commit Pod yet, while the API+ // server already holds its termination message.+ r.Client = interceptor.NewClient(live, interceptor.Funcs{+ List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { return nil },+ })++ result, err := r.handleCommitting(context.Background(), snapshot)+ require.NoError(t, err)+ assert.Equal(t, ctrl.Result{}, result)++ updated := &sandboxv1alpha1.SandboxSnapshot{}+ require.NoError(t, live.Get(context.Background(), types.NamespacedName{Name: "test-snapshot", Namespace: "default"}, updated))+ assert.Equal(t, sandboxv1alpha1.SandboxSnapshotPhaseSucceed, updated.Status.Phase)+ require.Len(t, updated.Status.Containers, 1)+ assert.Equal(t, "sha256:main", updated.Status.Containers[0].ImageDigest)+}+func findSnapshotCondition(conditions []sandboxv1alpha1.SandboxSnapshotCondition, conditionType sandboxv1alpha1.SandboxSnapshotConditionType) *sandboxv1alpha1.SandboxSnapshotCondition {for i := range conditions {if conditions[i].Type == conditionType {diff --git a/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go b/kubernetes/internal/controller/sandboxsnapshot_lifecycle.goindex 2e07fc788..00ee9d87d 100644--- a/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go+++ b/kubernetes/internal/controller/sandboxsnapshot_lifecycle.go@@ -882,9 +882,25 @@ func containersWithoutDigest(containers []sandboxv1alpha1.ContainerSnapshot) []s}func (r *SandboxSnapshotReconciler) getCommitJobResult(ctx context.Context, namespace, jobName string) (*commitJobResult, bool, error) {+ result, found, err := findCommitJobResult(ctx, r.Client, namespace, jobName)+ if err != nil || found {+ return result, found, err+ }+ // The informer cache can lag behind the commit Job status: a Job is+ // reported Succeeded only after its Pod reached a terminal state, but the+ // Pod update may not have reached the cache yet. Reading the Pod from the+ // API server keeps a cache miss from being mistaken for a lost result,+ // which would be an unrecoverable pause failure.+ if r.APIReader == nil {+ return nil, false, nil+ }+ return findCommitJobResult(ctx, r.APIReader, namespace, jobName)+}++func findCommitJobResult(ctx context.Context, reader client.Reader, namespace, jobName string) (*commitJobResult, bool, error) {for _, labelKey := range []string{"job-name", "batch.kubernetes.io/job-name"} {podList := &corev1.PodList{}- if err := r.List(ctx, podList,+ if err := reader.List(ctx, podList,client.InNamespace(namespace),client.MatchingLabels{labelKey: jobName},); err != nil {