Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

105 KB
Raw Blame
// Copyright 2025 The OpenSandbox Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/tools/record"
	"k8s.io/utils/ptr"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"

	sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1"
	taskscheduler "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/scheduler"
	"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils"
	"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/expectations"
	"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex"
	taskexecutor "github.com/alibaba/OpenSandbox/sandbox-k8s/pkg/task-executor"
)

// newTestReconciler creates a BatchSandboxReconciler with a fake client for testing.
func newTestReconciler(objs ...client.Object) *BatchSandboxReconciler {
	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithIndex(&corev1.Pod{}, fieldindex.IndexNameForOwnerRefUID, fieldindex.OwnerIndexFunc).
		WithStatusSubresource(
			&sandboxv1alpha1.BatchSandbox{},
			&sandboxv1alpha1.SandboxSnapshot{},
		).
		WithObjects(objs...).
		Build()
	return &BatchSandboxReconciler{
		Client:              fakeClient,
		APIReader:           fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}
}

func setSandboxAllocation(obj metav1.Object, alloc sandboxAllocation) {
	if obj.GetAnnotations() == nil {
		obj.SetAnnotations(map[string]string{})
	}
	obj.GetAnnotations()[annoAllocStatusKey] = utils.DumpJSON(alloc)
}

type forbiddenTaskScheduler struct {
	t *testing.T
}

func (f *forbiddenTaskScheduler) Schedule() error {
	f.t.Fatalf("task scheduler should not be invoked while sandbox is paused")
	return nil
}

func (f *forbiddenTaskScheduler) UpdatePods(_ []*corev1.Pod) {
	f.t.Fatalf("task scheduler should not receive pod updates while sandbox is paused")
}

func (f *forbiddenTaskScheduler) ListTask() []taskscheduler.Task {
	f.t.Fatalf("task scheduler should not list tasks while sandbox is paused")
	return nil
}

func (f *forbiddenTaskScheduler) StopTask() []taskscheduler.Task {
	f.t.Fatalf("task scheduler should not stop tasks while sandbox is paused")
	return nil
}

func (f *forbiddenTaskScheduler) AddTasks(_ []*taskexecutor.Task) error {
	f.t.Fatalf("task scheduler should not add tasks while sandbox is paused")
	return nil
}

type fakeSchedulerTask struct {
	name     string
	state    taskscheduler.TaskState
	podName  string
	released bool
}

func (f fakeSchedulerTask) GetName() string {
	return f.name
}

func (f fakeSchedulerTask) GetState() taskscheduler.TaskState {
	return f.state
}

func (f fakeSchedulerTask) GetPodName() string {
	return f.podName
}

func (f fakeSchedulerTask) IsResourceReleased() bool {
	return f.released
}

func (f fakeSchedulerTask) GetTerminatedMessage() string {
	return ""
}

type recordingTaskScheduler struct {
	updatePodsCalls int
	scheduleCalls   int
	stopCalls       int
	tasks           []taskscheduler.Task
}

func (r *recordingTaskScheduler) Schedule() error {
	r.scheduleCalls++
	return nil
}

func (r *recordingTaskScheduler) UpdatePods(_ []*corev1.Pod) {
	r.updatePodsCalls++
}

func (r *recordingTaskScheduler) ListTask() []taskscheduler.Task {
	return r.tasks
}

func (r *recordingTaskScheduler) StopTask() []taskscheduler.Task {
	r.stopCalls++
	return r.tasks
}

func (r *recordingTaskScheduler) AddTasks(_ []*taskexecutor.Task) error {
	return nil
}

// ---------- dispatchPauseResume 5-case tests ----------

func TestDispatchPauseResume_Case1_PauseTrue(t *testing.T) {
	// gen > pauseObservedGen, pause=true → handlePause dispatched
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
		},
	}
	r := newTestReconciler(bs)
	result, handled, err := r.dispatchPauseResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, handled, "should be handled by handlePause")
	assert.True(t, result.RequeueAfter > 0, "handlePause should requeue")

	// Verify ACK: phase should be Pausing
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePausing, updated.Status.Phase)
	assert.Equal(t, int64(2), updated.Status.PauseObservedGeneration)
}

func TestDispatchPauseResume_Case1_PauseTrue_AlreadyPaused(t *testing.T) {
	// Paused, pause=true, gen > pauseObservedGen (for example a renew-only spec change)
	// → the paused state is left alone; handlePause must not run again.
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePaused,
			PauseObservedGeneration: 2,
		},
	}
	r := newTestReconciler(bs)
	_, handled, err := r.dispatchPauseResume(context.Background(), bs)
	require.NoError(t, err)
	assert.False(t, handled, "an already Paused sandbox must not be re-paused")

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)

	snapshot := &sandboxv1alpha1.SandboxSnapshot{}
	err = r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: internalPauseSnapshotName("test-bs")}, snapshot)
	assert.True(t, apierrors.IsNotFound(err), "no new pause snapshot should be created")
}

func TestDispatchPauseResume_Case2_PauseFalse(t *testing.T) {
	// gen > pauseObservedGen, pause=false → handleResume dispatched
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase:  sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
			Format: sandboxv1alpha1.SandboxSnapshotFormatRootfsV1,
			Containers: []sandboxv1alpha1.ContainerSnapshot{
				{ContainerName: "main", ImageURI: "registry/test-bs-main:snap-gen1", ImageDigest: "sha256:" + strings.Repeat("a", 64)},
			},
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePaused,
		},
	}
	r := newTestReconciler(bs, snapshot)
	result, handled, err := r.dispatchPauseResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, handled, "should be handled by handleResume")
	assert.True(t, result.RequeueAfter > 0, "handleResume should requeue")

	// Verify ACK: phase=Resuming
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseResuming, updated.Status.Phase)
}

func TestDispatchPauseResume_Case3_PauseNil_ACKOnly(t *testing.T) {
	// gen > pauseObservedGen, pause=nil → no dedicated ACK API call, continue normal flow (handled=false).
	// The ACK is deferred to persistRuntimeView in the main reconcile loop.
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
		},
	}
	r := newTestReconciler(bs)
	result, handled, err := r.dispatchPauseResume(context.Background(), bs)
	require.NoError(t, err)
	assert.False(t, handled, "ACK only should not block normal flow")
	assert.Equal(t, ctrl.Result{}, result)

	// Verify ACK is NOT written to server by dispatch (deferred to persistRuntimeView).
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, int64(1), updated.Status.PauseObservedGeneration, "server should not be updated by dispatch; ACK is deferred")
}

func TestDispatchPauseResume_Case4_GenEqual_PauseSet(t *testing.T) {
	// gen == pauseObservedGen, pause != nil → syncPauseOrClear
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseCommitting,
		},
	}
	r := newTestReconciler(bs, snapshot)
	result, handled, err := r.dispatchPauseResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, handled, "syncPauseOrClear should handle this")
	assert.True(t, result.RequeueAfter > 0, "committing snapshot should requeue")
}

func TestDispatchPauseResume_Case5_GenEqual_PauseNil(t *testing.T) {
	// gen == pauseObservedGen, pause == nil → normal flow (handled=false)
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
		},
	}
	r := newTestReconciler(bs)
	result, handled, err := r.dispatchPauseResume(context.Background(), bs)
	require.NoError(t, err)
	assert.False(t, handled, "normal flow should not be blocked")
	assert.Equal(t, ctrl.Result{}, result)
}

// ---------- handlePause tests ----------

func TestHandlePause_NormalFlow(t *testing.T) {
	// Normal pause: ACK, create SandboxSnapshot, verify phase=Pausing
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
		},
	}
	r := newTestReconciler(bs)

	result, err := r.handlePause(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	// Verify ACK: phase=Pausing, pauseObservedGeneration=2
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePausing, updated.Status.Phase)
	assert.Equal(t, int64(2), updated.Status.PauseObservedGeneration)

	// Verify SandboxSnapshot was created under the reserved internal name
	snap := &sandboxv1alpha1.SandboxSnapshot{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap))
	assert.Equal(t, "test-bs", snap.Spec.SandboxName)
	// Verify OwnerRef
	assert.Equal(t, "test-bs", snap.OwnerReferences[0].Name)
}

func TestHandlePause_WaitsForTaskCleanupBeforeCreatingSnapshot(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
			TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{
				Spec: sandboxv1alpha1.TaskSpec{
					Process: &sandboxv1alpha1.ProcessTask{Command: []string{"sleep", "3600"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-0",
			Namespace: "default",
			OwnerReferences: []metav1.OwnerReference{{
				APIVersion: sandboxv1alpha1.GroupVersion.String(),
				Kind:       "BatchSandbox",
				Name:       "test-bs",
				UID:        "test-uid",
			}},
		},
		Status: corev1.PodStatus{PodIP: "10.0.0.1"},
	}
	r := newTestReconciler(bs, pod)
	scheduler := &recordingTaskScheduler{
		tasks: []taskscheduler.Task{fakeSchedulerTask{
			name:     "test-bs-0",
			state:    taskscheduler.RunningTaskState,
			podName:  "test-bs-0",
			released: false,
		}},
	}
	r.taskSchedulers.Store(types.NamespacedName{Namespace: "default", Name: "test-bs"}.String(), scheduler)

	result, err := r.handlePause(context.Background(), bs)

	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)
	assert.Equal(t, 1, scheduler.updatePodsCalls)
	assert.Equal(t, 1, scheduler.stopCalls)
	assert.Equal(t, 1, scheduler.scheduleCalls)
	snap := &sandboxv1alpha1.SandboxSnapshot{}
	err = r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap)
	assert.True(t, apierrors.IsNotFound(err), "snapshot must wait until tasks are released")
}

func TestHandlePause_CreatesSnapshotAfterTaskCleanup(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
			TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{
				Spec: sandboxv1alpha1.TaskSpec{
					Process: &sandboxv1alpha1.ProcessTask{Command: []string{"sleep", "3600"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-0",
			Namespace: "default",
			OwnerReferences: []metav1.OwnerReference{{
				APIVersion: sandboxv1alpha1.GroupVersion.String(),
				Kind:       "BatchSandbox",
				Name:       "test-bs",
				UID:        "test-uid",
			}},
		},
		Status: corev1.PodStatus{PodIP: "10.0.0.1"},
	}
	r := newTestReconciler(bs, pod)
	scheduler := &recordingTaskScheduler{
		tasks: []taskscheduler.Task{fakeSchedulerTask{
			name:     "test-bs-0",
			state:    taskscheduler.RunningTaskState,
			podName:  "test-bs-0",
			released: true,
		}},
	}
	r.taskSchedulers.Store(types.NamespacedName{Namespace: "default", Name: "test-bs"}.String(), scheduler)

	result, err := r.handlePause(context.Background(), bs)

	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)
	assert.Equal(t, 1, scheduler.stopCalls)
	assert.Equal(t, 1, scheduler.scheduleCalls)
	snap := &sandboxv1alpha1.SandboxSnapshot{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap))
	assert.Equal(t, "test-bs", snap.Spec.SandboxName)
}

func TestHandlePause_RejectsUnsupportedReplicas(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(2)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
		},
	}
	r := newTestReconciler(bs)

	result, err := r.handlePause(context.Background(), bs)
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseSucceed, updated.Status.Phase)
	assert.Equal(t, int64(2), updated.Status.PauseObservedGeneration)

	var pauseFailed *sandboxv1alpha1.BatchSandboxCondition
	for i := range updated.Status.Conditions {
		if updated.Status.Conditions[i].Type == sandboxv1alpha1.BatchSandboxConditionPauseFailed {
			pauseFailed = &updated.Status.Conditions[i]
			break
		}
	}
	require.NotNil(t, pauseFailed)
	assert.Equal(t, sandboxv1alpha1.ConditionTrue, pauseFailed.Status)
	assert.Equal(t, "UnsupportedReplicas", pauseFailed.Reason)
	assert.Contains(t, pauseFailed.Message, "spec.replicas=1")

	snap := &sandboxv1alpha1.SandboxSnapshot{}
	err = r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap)
	assert.True(t, apierrors.IsNotFound(err), "unsupported replicas should be rejected before creating a snapshot")
}

func TestHandlePause_PoolMode(t *testing.T) {
	// Pool mode: pause should snapshot the allocated pod without mutating poolRef/template first.
	pool := &sandboxv1alpha1.Pool{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-pool",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.PoolSpec{
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{
						{Name: "main", Image: "pool-image:latest"},
					},
				},
			},
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:   ptr.To(true),
			PoolRef: "test-pool",
			// Template is nil (pool mode)
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
		},
	}
	r := newTestReconciler(bs, pool)

	result, err := r.handlePause(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	// Verify template was solidified from Pool
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Nil(t, updated.Spec.Template)
	assert.Equal(t, "test-pool", updated.Spec.PoolRef)

	// Verify SandboxSnapshot was created under the reserved internal name
	snap := &sandboxv1alpha1.SandboxSnapshot{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap))
	assert.Equal(t, "test-bs", snap.Spec.SandboxName)
}

func TestHandlePause_PoolModeDoesNotRequirePoolCR(t *testing.T) {
	// The source pod allocation is enough to snapshot a pooled BatchSandbox.
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			PoolRef:  "nonexistent-pool",
			Replicas: ptr.To(int32(1)),
			// Template is nil - pool mode
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
		},
	}
	setSandboxAllocation(bs, sandboxAllocation{Pods: []string{"pool-pod"}})
	poolPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pool-pod",
			Namespace: "default",
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{{Name: "main", Image: "pool-image:latest"}},
		},
	}
	r := newTestReconciler(bs, poolPod)

	result, err := r.handlePause(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePausing, updated.Status.Phase)
	assert.Nil(t, updated.Spec.Template)
	assert.Equal(t, "nonexistent-pool", updated.Spec.PoolRef)

	snap := &sandboxv1alpha1.SandboxSnapshot{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap))
	assert.Equal(t, "test-bs", snap.Spec.SandboxName)
}

func TestHandlePause_FailedRetry(t *testing.T) {
	// Old snapshot is Failed → delete it first, but do not ACK a new pause attempt
	// until the stale failed snapshot is gone.
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
			Conditions: []sandboxv1alpha1.BatchSandboxCondition{
				{
					Type:               sandboxv1alpha1.BatchSandboxConditionPauseFailed,
					Status:             sandboxv1alpha1.ConditionTrue,
					Reason:             "SnapshotFailed",
					Message:            "previous attempt failed",
					LastTransitionTime: ptr.To(metav1.Now()),
				},
			},
		},
	}
	oldSnapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseFailed,
		},
	}
	r := newTestReconciler(bs, oldSnapshot)

	result, err := r.handlePause(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	// Verify old snapshot was deleted
	snap := &sandboxv1alpha1.SandboxSnapshot{}
	err = r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap)
	assert.True(t, err == nil || len(snap.UID) == 0 || snap.DeletionTimestamp != nil,
		"old Failed snapshot should be deleted or being deleted")

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, int64(1), updated.Status.PauseObservedGeneration, "retry should not ACK until stale failed snapshot is removed")
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseSucceed, updated.Status.Phase, "retry cleanup should not transition to Pausing yet")
}

func TestHandlePause_SucceededSnapshotRetry(t *testing.T) {
	// Old snapshot is Succeed from a previous resume cleanup failure. It must be
	// deleted before ACKing a new pause attempt, otherwise pause reuses stale state.
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
		},
	}
	oldSnapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
		},
	}
	r := newTestReconciler(bs, oldSnapshot)

	result, err := r.handlePause(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	snap := &sandboxv1alpha1.SandboxSnapshot{}
	err = r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap)
	assert.True(t, apierrors.IsNotFound(err), "old Succeed snapshot should be deleted before retrying pause")

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, int64(1), updated.Status.PauseObservedGeneration, "retry cleanup should not ACK until stale succeeded snapshot is removed")
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseSucceed, updated.Status.Phase, "retry cleanup should not transition to Pausing yet")
}

// ---------- handleResume tests ----------

func TestHandleResume_NormalFlow(t *testing.T) {
	// handleResume now only ACKs Resuming phase and requeues
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(0)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePaused,
		},
	}
	r := newTestReconciler(bs)

	result, err := r.handleResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0, "should requeue after ACK")

	// Verify ACK: phase=Resuming
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseResuming, updated.Status.Phase)
	assert.Equal(t, int64(2), updated.Status.PauseObservedGeneration)
}

func TestContinueResume_NormalFlow(t *testing.T) {
	// continueResume does the actual work: read snapshot and replace images without changing replicas.
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase:  sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
			Format: sandboxv1alpha1.SandboxSnapshotFormatRootfsV1,
			Containers: []sandboxv1alpha1.ContainerSnapshot{
				{ContainerName: "main", ImageURI: "registry/test-bs-main:snap-gen1", ImageDigest: "sha256:" + strings.Repeat("a", 64)},
			},
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	r := newTestReconciler(bs, snapshot)
	r.ResumePullSecret = "my-pull-secret"

	result, err := r.continueResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	// Verify images replaced
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, "registry/test-bs-main@sha256:"+strings.Repeat("a", 64), updated.Spec.Template.Spec.Containers[0].Image)
	// Verify replicas are preserved.
	assert.Equal(t, int32(1), *updated.Spec.Replicas)
	// Verify controller does not clear spec.pause
	require.NotNil(t, updated.Spec.Pause)
	assert.False(t, *updated.Spec.Pause)
	// Verify imagePullSecrets added
	found := false
	for _, s := range updated.Spec.Template.Spec.ImagePullSecrets {
		if s.Name == "my-pull-secret" {
			found = true
		}
	}
	assert.True(t, found, "imagePullSecrets should contain resume-pull-secret")
}

func TestContinueResume_PreservesExistingReplicas(t *testing.T) {
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
			Containers: []sandboxv1alpha1.ContainerSnapshot{
				{ContainerName: "main", ImageURI: "registry/test-bs-main:snap-gen1"},
			},
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(2)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	r := newTestReconciler(bs, snapshot)

	result, err := r.continueResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, int32(2), *updated.Spec.Replicas)
	assert.Equal(t, "registry/test-bs-main:snap-gen1", updated.Spec.Template.Spec.Containers[0].Image)
}

func TestHandleResume_PoolMode(t *testing.T) {
	// handleResume only ACKs Resuming phase, poolRef is cleared by continueResume
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(0)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 1,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePaused,
		},
	}
	r := newTestReconciler(bs)

	result, err := r.handleResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	// Verify ACK: phase=Resuming
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseResuming, updated.Status.Phase)
}

func TestContinueResume_PoolMode(t *testing.T) {
	// continueResume clears poolRef without changing replicas.
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
			Containers: []sandboxv1alpha1.ContainerSnapshot{
				{ContainerName: "main", ImageURI: "registry/test-bs-main:snap-gen1"},
			},
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
			Finalizers: []string{finalizerPoolAllocation},
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	r := newTestReconciler(bs, snapshot)

	result, err := r.continueResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	// Verify poolRef cleared
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, "", updated.Spec.PoolRef)
	assert.NotContains(t, updated.Finalizers, finalizerPoolAllocation)
}

func TestContinueResume_UsesPatchedTemplateWhenCacheReturnsStaleObject(t *testing.T) {
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
			Containers: []sandboxv1alpha1.ContainerSnapshot{
				{ContainerName: "main", ImageURI: "registry/test-bs-main:snap-gen1"},
			},
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	staleBatchSandbox := bs.DeepCopy()
	returnStaleBatchSandbox := false
	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithStatusSubresource(&sandboxv1alpha1.BatchSandbox{}, &sandboxv1alpha1.SandboxSnapshot{}).
		WithObjects(bs.DeepCopy(), snapshot).
		WithInterceptorFuncs(interceptor.Funcs{
			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
				err := c.Patch(ctx, obj, patch, opts...)
				if err == nil && obj.GetNamespace() == "default" && obj.GetName() == "test-bs" {
					returnStaleBatchSandbox = true
				}
				return err
			},
			Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
				if returnStaleBatchSandbox && key.Namespace == "default" && key.Name == "test-bs" {
					if target, ok := obj.(*sandboxv1alpha1.BatchSandbox); ok {
						staleBatchSandbox.DeepCopyInto(target)
						return nil
					}
				}
				return c.Get(ctx, key, obj, opts...)
			},
		}).
		Build()
	r := &BatchSandboxReconciler{
		Client:              fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}

	result, err := r.continueResume(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)
	require.NotNil(t, bs.Spec.Template)
	assert.Equal(t, "registry/test-bs-main:snap-gen1", bs.Spec.Template.Spec.Containers[0].Image)
	assert.Equal(t, "", bs.Spec.PoolRef)
	require.NotNil(t, bs.Spec.Replicas)
	assert.Equal(t, int32(1), *bs.Spec.Replicas)
}

func TestContinueResume_SnapshotNotFound(t *testing.T) {
	// continueResume without snapshot → rollback to Paused with ResumeFailed, spec.pause unchanged
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(0)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	r := newTestReconciler(bs)

	result, err := r.continueResume(context.Background(), bs)
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	// Verify rollback keeps spec.pause and records retryable failure
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
	require.NotNil(t, updated.Spec.Pause)
	assert.False(t, *updated.Spec.Pause)
}

func TestContinueResume_SnapshotMissingButReadyPodTreatsResumeAsComplete(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-0",
			Namespace: "default",
			Labels: map[string]string{
				labelBatchSandboxNameKey: "test-bs",
			},
		},
		Status: corev1.PodStatus{
			Phase: corev1.PodRunning,
			Conditions: []corev1.PodCondition{
				{Type: corev1.PodReady, Status: corev1.ConditionTrue},
			},
		},
	}
	r := newTestReconciler(bs, pod)

	result, err := r.continueResume(context.Background(), bs)
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseSucceed, updated.Status.Phase)
	require.NotNil(t, updated.Spec.Pause)
	assert.False(t, *updated.Spec.Pause)

	for _, cond := range updated.Status.Conditions {
		assert.NotEqual(t, sandboxv1alpha1.BatchSandboxConditionResumeFailed, cond.Type)
	}
}

func TestContinueResume_SnapshotNotReady(t *testing.T) {
	// continueResume with snapshot still Committing → Phase=Paused + ResumeFailed condition, spec.pause unchanged
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseCommitting,
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(0)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	r := newTestReconciler(bs, snapshot)

	result, err := r.continueResume(context.Background(), bs)
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	// Verify Phase=Paused with ResumeFailed condition (retryable)
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
	require.NotNil(t, updated.Spec.Pause)
	assert.False(t, *updated.Spec.Pause)

	// Verify ResumeFailed condition is set
	foundCondition := false
	for _, cond := range updated.Status.Conditions {
		if cond.Type == sandboxv1alpha1.BatchSandboxConditionResumeFailed {
			foundCondition = true
			assert.Equal(t, sandboxv1alpha1.ConditionTrue, cond.Status)
			assert.Equal(t, "SnapshotNotReady", cond.Reason)
			assert.Contains(t, cond.Message, "snapshot not ready")
			break
		}
	}
	assert.True(t, foundCondition, "ResumeFailed condition should be set")
}

func TestReconcile_ResumingPoolResumeRebuildsStrategyAfterContinueResume(t *testing.T) {
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
			Containers: []sandboxv1alpha1.ContainerSnapshot{
				{ContainerName: "main", ImageURI: "registry/test-bs-main:snap-gen1"},
			},
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 5,
			UID:        "test-uid",
			Annotations: map[string]string{
				annoAllocStatusKey: `{"pods":["pool-pod-0"]}`,
			},
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 5,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	poolPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pool-pod-0",
			Namespace: "default",
		},
		Status: corev1.PodStatus{
			Phase: corev1.PodRunning,
			Conditions: []corev1.PodCondition{
				{Type: corev1.PodReady, Status: corev1.ConditionTrue},
			},
		},
	}
	r := newTestReconciler(bs, snapshot, poolPod)

	result, err := r.Reconcile(context.Background(), ctrl.Request{
		NamespacedName: types.NamespacedName{Namespace: "default", Name: "test-bs"},
	})
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, "", updated.Spec.PoolRef, "resume should detach the sandbox from the pool")
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseResuming, updated.Status.Phase, "old pooled pod must not short-circuit resume completion")

	resumedPod := &corev1.Pod{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-0"}, resumedPod))
	assert.Equal(t, "test-bs", resumedPod.Labels[labelBatchSandboxNameKey])

	stillPresent := &sandboxv1alpha1.SandboxSnapshot{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, stillPresent))
}

func TestReconcile_ResumingIgnoresDeletingReadyPodWhenCompletingResume(t *testing.T) {
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
			Containers: []sandboxv1alpha1.ContainerSnapshot{
				{ContainerName: "main", ImageURI: "registry/test-bs-main:snap-gen1"},
			},
		},
	}
	now := metav1.Now()
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 3,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}
	terminatingPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:              "test-bs-0",
			Namespace:         "default",
			DeletionTimestamp: &now,
			Finalizers:        []string{"test/finalizer"},
			Labels: map[string]string{
				labelBatchSandboxNameKey:     "test-bs",
				labelBatchSandboxPodIndexKey: "0",
			},
			OwnerReferences: []metav1.OwnerReference{{
				APIVersion: sandboxv1alpha1.GroupVersion.String(),
				Kind:       "BatchSandbox",
				Name:       "test-bs",
				UID:        "test-uid",
				Controller: ptr.To(true),
			}},
		},
		Status: corev1.PodStatus{
			Phase: corev1.PodRunning,
			Conditions: []corev1.PodCondition{
				{Type: corev1.PodReady, Status: corev1.ConditionTrue},
			},
		},
	}
	r := newTestReconciler(bs, snapshot, terminatingPod)

	result, err := r.Reconcile(context.Background(), ctrl.Request{
		NamespacedName: types.NamespacedName{Namespace: "default", Name: "test-bs"},
	})
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseResuming, updated.Status.Phase)

	stillPresent := &sandboxv1alpha1.SandboxSnapshot{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, stillPresent))
}

func TestReconcile_PausedNonPoolDoesNotScalePods(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "old-img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 3,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePaused,
		},
	}
	r := newTestReconciler(bs)

	result, err := r.Reconcile(context.Background(), ctrl.Request{
		NamespacedName: types.NamespacedName{Namespace: "default", Name: "test-bs"},
	})
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)

	pod := &corev1.Pod{}
	err = r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-0"}, pod)
	assert.Error(t, err, "paused non-pooled sandbox should not recreate pods")
}

func TestReconcile_PausedSkipsTaskSchedulingAndClearsScheduler(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 4,
			UID:        "test-uid",
			Finalizers: []string{finalizerTaskCleanup},
			Annotations: map[string]string{
				annoAllocStatusKey:  `{"pods":["pool-pod"]}`,
				annoAllocReleaseKey: `{"pods":["pool-pod"]}`,
			},
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			PoolRef:  "pool",
			Replicas: ptr.To(int32(1)),
			TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{
				Spec: sandboxv1alpha1.TaskSpec{
					Process: &sandboxv1alpha1.ProcessTask{
						Command: []string{"/bin/sh", "-c", "sleep 3600"},
					},
				},
			},
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "sandbox", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 4,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePaused,
			ObservedGeneration:      4,
		},
	}
	r := newTestReconciler(bs)
	key := types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name}.String()
	r.taskSchedulers.Store(key, &forbiddenTaskScheduler{t: t})

	result, err := r.Reconcile(context.Background(), ctrl.Request{
		NamespacedName: types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name},
	})
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	_, ok := r.taskSchedulers.Load(key)
	assert.False(t, ok, "paused reconcile should clear stale task scheduler state")
}

func TestCompletePause_PooledModeClearsTaskScheduler(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
			UID:        "test-uid",
			Annotations: map[string]string{
				annoAllocStatusKey: `{"pods":["pool-pod"]}`,
			},
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			PoolRef:  "pool",
			Replicas: ptr.To(int32(1)),
			TaskTemplate: &sandboxv1alpha1.TaskTemplateSpec{
				Spec: sandboxv1alpha1.TaskSpec{
					Process: &sandboxv1alpha1.ProcessTask{
						Command: []string{"/bin/sh", "-c", "sleep 3600"},
					},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 3,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	poolPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pool-pod",
			Namespace: "default",
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{{Name: "main", Image: "pool-image:latest"}},
		},
		Status: corev1.PodStatus{
			PodIP: "10.0.0.8",
		},
	}
	r := newTestReconciler(bs, poolPod)
	key := types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name}.String()
	r.taskSchedulers = sync.Map{}
	r.taskSchedulers.Store(key, fmt.Sprintf("scheduler:%s", key))

	err := r.completePause(context.Background(), bs)
	require.NoError(t, err)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
	assert.Equal(t, "", updated.Spec.PoolRef)
	require.NotNil(t, updated.Spec.Template)
	assert.Equal(t, "pool-image:latest", updated.Spec.Template.Spec.Containers[0].Image)
	assert.NotContains(t, updated.Annotations, annoAllocReleaseKey)

	_, ok := r.taskSchedulers.Load(key)
	assert.False(t, ok, "completePause should clear the stale in-memory task scheduler")
}

func TestPersistRuntimeView_PreservesPauseFailedConditionFromLatestStatus(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			ObservedGeneration:      1,
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
			Replicas:                1,
			Allocated:               1,
			Ready:                   1,
			Conditions: []sandboxv1alpha1.BatchSandboxCondition{
				{
					Type:               sandboxv1alpha1.BatchSandboxConditionReady,
					Status:             sandboxv1alpha1.ConditionTrue,
					Reason:             "PodsReady",
					Message:            "Sandbox is running",
					LastTransitionTime: ptr.To(metav1.Now()),
				},
			},
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-0",
			Namespace: "default",
			Labels: map[string]string{
				labelBatchSandboxNameKey:     "test-bs",
				labelBatchSandboxPodIndexKey: "0",
			},
			OwnerReferences: []metav1.OwnerReference{{
				APIVersion: sandboxv1alpha1.GroupVersion.String(),
				Kind:       "BatchSandbox",
				Name:       "test-bs",
				UID:        "test-uid",
				Controller: ptr.To(true),
			}},
		},
		Status: corev1.PodStatus{
			Phase: corev1.PodRunning,
			PodIP: "10.0.0.10",
			Conditions: []corev1.PodCondition{
				{Type: corev1.PodReady, Status: corev1.ConditionTrue},
			},
		},
	}
	r := newTestReconciler(bs, pod)

	// Simulate pause handler writing PauseFailed condition to API server.
	latest := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, latest))
	latest.Status.Conditions = append(latest.Status.Conditions, sandboxv1alpha1.BatchSandboxCondition{
		Type:               sandboxv1alpha1.BatchSandboxConditionPauseFailed,
		Status:             sandboxv1alpha1.ConditionTrue,
		Reason:             "SnapshotFailed",
		Message:            "Commit job failed",
		LastTransitionTime: ptr.To(metav1.Now()),
	})
	require.NoError(t, r.Status().Update(context.Background(), latest))

	// Simulate second reconcile: informer has caught up, so we read latest state.
	freshBS := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, freshBS))

	view := buildRuntimeView(freshBS, []*corev1.Pod{pod})
	_, errs := r.persistRuntimeView(context.Background(), freshBS, view)
	require.Empty(t, errs)

	// Verify PauseFailed is preserved after reconcile with fresh cache.
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))

	foundPauseFailed := false
	for _, cond := range updated.Status.Conditions {
		if cond.Type == sandboxv1alpha1.BatchSandboxConditionPauseFailed {
			foundPauseFailed = true
			assert.Equal(t, sandboxv1alpha1.ConditionTrue, cond.Status)
			assert.Equal(t, "SnapshotFailed", cond.Reason)
			assert.Equal(t, "Commit job failed", cond.Message)
		}
	}
	assert.True(t, foundPauseFailed, "persistRuntimeView should preserve PauseFailed condition once informer cache catches up")
}

func TestPersistRuntimeView_SkipsStatusUpdateWhenRuntimeStatusUnchanged(t *testing.T) {
	transitionTime := metav1.NewTime(time.Now().Add(-5 * time.Minute))
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
			UID:        "test-uid",
			Annotations: map[string]string{
				annotationSandboxEndpoints: `["10.0.0.10"]`,
			},
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			ObservedGeneration: 3,
			Phase:              sandboxv1alpha1.BatchSandboxPhaseSucceed,
			Replicas:           1,
			Allocated:          1,
			Ready:              1,
			Conditions: []sandboxv1alpha1.BatchSandboxCondition{
				{
					Type:               sandboxv1alpha1.BatchSandboxConditionReady,
					Status:             sandboxv1alpha1.ConditionTrue,
					Reason:             "PodsReady",
					Message:            "Sandbox is running",
					LastTransitionTime: &transitionTime,
				},
			},
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-0",
			Namespace: "default",
		},
		Status: corev1.PodStatus{
			Phase: corev1.PodRunning,
			PodIP: "10.0.0.10",
			Conditions: []corev1.PodCondition{
				{Type: corev1.PodReady, Status: corev1.ConditionTrue},
			},
		},
	}

	statusUpdates := 0
	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithStatusSubresource(&sandboxv1alpha1.BatchSandbox{}).
		WithObjects(bs).
		WithInterceptorFuncs(interceptor.Funcs{
			SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
				if subResourceName == "status" {
					statusUpdates++
				}
				return c.SubResource(subResourceName).Update(ctx, obj, opts...)
			},
		}).
		Build()
	r := &BatchSandboxReconciler{
		Client:              fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}

	view := buildRuntimeView(bs.DeepCopy(), []*corev1.Pod{pod})
	_, errs := r.persistRuntimeView(context.Background(), bs.DeepCopy(), view)
	require.Empty(t, errs)
	assert.Equal(t, 0, statusUpdates, "unchanged runtime status should not be persisted again")
}

func TestPersistRuntimeView_RetriesSucceededPauseSnapshotCleanup(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
			UID:        "test-uid",
			Annotations: map[string]string{
				annotationSandboxEndpoints: "null",
			},
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(false),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			ObservedGeneration:      3,
			PauseObservedGeneration: 3,
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
			Replicas:                1,
			Allocated:               1,
			Ready:                   1,
		},
	}
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
		},
	}
	r := newTestReconciler(bs, snapshot)

	status := bs.Status
	view := runtimeView{status: &status}
	_, errs := r.persistRuntimeView(context.Background(), bs.DeepCopy(), view)
	require.Empty(t, errs)

	stillPresent := &sandboxv1alpha1.SandboxSnapshot{}
	err := r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, stillPresent)
	assert.True(t, apierrors.IsNotFound(err), "successful internal pause snapshot cleanup should be retried after resume")
}

func TestBuildRuntimeView_AggregatesPodFailuresInSteadyState(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs",
			Namespace: "default",
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase: sandboxv1alpha1.BatchSandboxPhaseSucceed,
		},
	}

	pods := []*corev1.Pod{
		{
			ObjectMeta: metav1.ObjectMeta{Name: "err-image-0", Namespace: "default"},
			Status: corev1.PodStatus{
				ContainerStatuses: []corev1.ContainerStatus{{
					State: corev1.ContainerState{
						Waiting: &corev1.ContainerStateWaiting{
							Reason:  "ErrImagePull",
							Message: "image not found",
						},
					},
				}},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{Name: "crash-1", Namespace: "default"},
			Status: corev1.PodStatus{
				ContainerStatuses: []corev1.ContainerStatus{{
					State: corev1.ContainerState{
						Waiting: &corev1.ContainerStateWaiting{
							Reason:  "CrashLoopBackOff",
							Message: "back-off restarting failed container",
						},
					},
				}},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{Name: "err-image-2", Namespace: "default"},
			Status: corev1.PodStatus{
				ContainerStatuses: []corev1.ContainerStatus{{
					State: corev1.ContainerState{
						Waiting: &corev1.ContainerStateWaiting{
							Reason:  "ErrImagePull",
							Message: "image still not found",
						},
					},
				}},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{Name: "healthy-3", Namespace: "default"},
			Status: corev1.PodStatus{
				Phase: corev1.PodRunning,
				Conditions: []corev1.PodCondition{
					{Type: corev1.PodReady, Status: corev1.ConditionTrue},
				},
			},
		},
	}

	view := buildRuntimeView(bs, pods)
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)

	var podFailed *sandboxv1alpha1.BatchSandboxCondition
	for i := range view.status.Conditions {
		if view.status.Conditions[i].Type == sandboxv1alpha1.BatchSandboxConditionPodFailed {
			podFailed = &view.status.Conditions[i]
			break
		}
	}
	require.NotNil(t, podFailed)
	assert.Equal(t, sandboxv1alpha1.ConditionTrue, podFailed.Status)
	assert.Equal(t, "ErrImagePull", podFailed.Reason)
	assert.Equal(t, "3/4 observed pods failed; primary reason=ErrImagePull; sample pod=err-image-0", podFailed.Message)
}

func TestBuildRuntimeView_MarksTerminalInitContainerFailure(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase:      sandboxv1alpha1.BatchSandboxPhasePending,
			TaskFailed: 2,
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{Name: "sandbox-0", Namespace: "default"},
		Status: corev1.PodStatus{
			Phase: corev1.PodFailed,
			InitContainerStatuses: []corev1.ContainerStatus{{
				Name: "bootstrap-execd",
				State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
					ExitCode: 1,
					Reason:   "Error",
					Message:  "install failed",
				}},
			}},
		},
	}

	view := buildRuntimeView(bs, []*corev1.Pod{pod})

	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)
	assert.Equal(t, int32(2), view.status.TaskFailed, "pod failures must not overwrite the failed-task count")
	for i := range view.status.Conditions {
		condition := view.status.Conditions[i]
		if condition.Type != sandboxv1alpha1.BatchSandboxConditionPodFailed {
			continue
		}
		assert.Equal(t, sandboxv1alpha1.ConditionTrue, condition.Status)
		assert.Equal(t, "InitContainerFailed", condition.Reason)
		assert.Equal(
			t,
			"1/1 observed pods failed; primary reason=InitContainerFailed; sample pod=sandbox-0; sample detail=Pod sandbox-0 init container bootstrap-execd exited with code 1 (Error): install failed",
			condition.Message,
		)
		return
	}
	t.Fatal("expected PodFailed condition")
}

func TestBuildRuntimeView_MarksTerminatedMainContainerFailureWhilePodRunning(t *testing.T) {
	// restartPolicy=Never multi-container pod: the main container (containers[0],
	// by server convention the "sandbox" container) exits non-zero while an egress
	// sidecar keeps the pod Running forever. The sandbox must fail instead of
	// regressing to Pending (the pod phase will never become Failed).
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{Name: "test-bs", Namespace: "default"},
		Status:     sandboxv1alpha1.BatchSandboxStatus{Phase: sandboxv1alpha1.BatchSandboxPhaseSucceed},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{Name: "sbx-0", Namespace: "default"},
		Spec: corev1.PodSpec{
			RestartPolicy: corev1.RestartPolicyNever,
			Containers: []corev1.Container{
				{Name: "sandbox"},
				{Name: "egress"},
			},
		},
		Status: corev1.PodStatus{
			Phase: corev1.PodRunning,
			ContainerStatuses: []corev1.ContainerStatus{
				{
					Name: "sandbox",
					State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
						ExitCode: 1,
						Reason:   "Error",
						Message:  "main process crashed",
					}},
				},
				{
					Name:  "egress",
					State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}},
				},
			},
		},
	}

	view := buildRuntimeView(bs, []*corev1.Pod{pod})

	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)
	for i := range view.status.Conditions {
		condition := view.status.Conditions[i]
		if condition.Type != sandboxv1alpha1.BatchSandboxConditionPodFailed {
			continue
		}
		assert.Equal(t, sandboxv1alpha1.ConditionTrue, condition.Status)
		assert.Equal(t, "ContainerFailed", condition.Reason)
		assert.Equal(
			t,
			"1/1 observed pods failed; primary reason=ContainerFailed; sample pod=sbx-0",
			condition.Message,
		)
		return
	}
	t.Fatal("expected PodFailed condition")
}

func TestBuildRuntimeView_TerminatedMainContainerWithRestartablePolicyIsNotTerminal(t *testing.T) {
	// With the default (Always) restart policy the kubelet restarts a terminated
	// main container; a transient termination must not brick the sandbox in the
	// sticky Failed phase. Same pod as above but restart policy left unset.
	tests := []struct {
		name          string
		restartPolicy corev1.RestartPolicy
	}{
		{name: "unset defaults to Always", restartPolicy: ""},
		{name: "Always", restartPolicy: corev1.RestartPolicyAlways},
		{name: "OnFailure restarts on non-zero exit", restartPolicy: corev1.RestartPolicyOnFailure},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			bs := &sandboxv1alpha1.BatchSandbox{
				ObjectMeta: metav1.ObjectMeta{Name: "test-bs", Namespace: "default"},
				Status:     sandboxv1alpha1.BatchSandboxStatus{Phase: sandboxv1alpha1.BatchSandboxPhaseSucceed},
			}
			pod := &corev1.Pod{
				ObjectMeta: metav1.ObjectMeta{Name: "sbx-0", Namespace: "default"},
				Spec: corev1.PodSpec{
					RestartPolicy: tt.restartPolicy,
					Containers: []corev1.Container{
						{Name: "sandbox"},
						{Name: "egress"},
					},
				},
				Status: corev1.PodStatus{
					Phase: corev1.PodRunning,
					ContainerStatuses: []corev1.ContainerStatus{
						{
							Name: "sandbox",
							State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
								ExitCode: 1,
								Reason:   "Error",
							}},
						},
						{
							Name:  "egress",
							State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}},
						},
					},
				},
			}

			view := buildRuntimeView(bs, []*corev1.Pod{pod})

			assert.NotEqual(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase,
				"restartable policies must not treat the termination as terminal")
			for i := range view.status.Conditions {
				assert.NotEqual(t, sandboxv1alpha1.BatchSandboxConditionPodFailed, view.status.Conditions[i].Type)
			}
		})
	}
}

func TestBuildRuntimeView_TerminatingPodIsNotMainContainerTerminalFailure(t *testing.T) {
	// Deletion, eviction, or node drain signal-kills the main container while the
	// pod still exists (deletionTimestamp set, sidecar winding down). The signal
	// exit must not record a sticky terminal failure that would block the
	// replacement of the deleted pod.
	now := metav1.Now()
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{Name: "test-bs", Namespace: "default"},
		Status:     sandboxv1alpha1.BatchSandboxStatus{Phase: sandboxv1alpha1.BatchSandboxPhaseSucceed},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:              "sbx-0",
			Namespace:         "default",
			DeletionTimestamp: &now,
		},
		Spec: corev1.PodSpec{
			RestartPolicy: corev1.RestartPolicyNever,
			Containers: []corev1.Container{
				{Name: "sandbox"},
				{Name: "egress"},
			},
		},
		Status: corev1.PodStatus{
			Phase: corev1.PodRunning,
			ContainerStatuses: []corev1.ContainerStatus{
				{
					Name: "sandbox",
					State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
						ExitCode: 137,
						Reason:   "Error",
					}},
				},
				{
					Name:  "egress",
					State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}},
				},
			},
		},
	}

	view := buildRuntimeView(bs, []*corev1.Pod{pod})

	assert.NotEqual(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase,
		"terminating pods must not turn into a sticky terminal failure")
	for i := range view.status.Conditions {
		assert.NotEqual(t, sandboxv1alpha1.BatchSandboxConditionPodFailed, view.status.Conditions[i].Type)
	}
}

func TestBuildRuntimeView_InitialRetryablePodStatesRemainPending(t *testing.T) {
	tests := []struct {
		name string
		pod  *corev1.Pod
	}{
		{
			name: "ordinary pending",
			pod: &corev1.Pod{
				ObjectMeta: metav1.ObjectMeta{Name: "pending-0"},
				Status:     corev1.PodStatus{Phase: corev1.PodPending},
			},
		},
		{
			name: "waiting for scheduling",
			pod: &corev1.Pod{
				ObjectMeta: metav1.ObjectMeta{Name: "unscheduled-0"},
				Status: corev1.PodStatus{
					Phase: corev1.PodPending,
					Conditions: []corev1.PodCondition{{
						Type:    corev1.PodScheduled,
						Status:  corev1.ConditionFalse,
						Reason:  corev1.PodReasonUnschedulable,
						Message: "insufficient cpu",
					}},
				},
			},
		},
		{
			name: "retryable image pull",
			pod: &corev1.Pod{
				ObjectMeta: metav1.ObjectMeta{Name: "image-pull-0"},
				Status: corev1.PodStatus{
					Phase: corev1.PodPending,
					ContainerStatuses: []corev1.ContainerStatus{{
						State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{
							Reason:  "ImagePullBackOff",
							Message: "back-off pulling image",
						}},
					}},
				},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			bs := &sandboxv1alpha1.BatchSandbox{
				Status: sandboxv1alpha1.BatchSandboxStatus{Phase: sandboxv1alpha1.BatchSandboxPhasePending},
			}

			view := buildRuntimeView(bs, []*corev1.Pod{tt.pod})

			assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePending, view.status.Phase)
			for _, condition := range view.status.Conditions {
				assert.NotEqual(t, sandboxv1alpha1.BatchSandboxConditionPodFailed, condition.Type)
			}
		})
	}
}

func TestReconcile_DoesNotReplacePodAfterTerminalPodFailure(t *testing.T) {
	previousExpectations := batchSandboxScaleExpectations
	batchSandboxScaleExpectations = expectations.NewScaleExpectations()
	t.Cleanup(func() { batchSandboxScaleExpectations = previousExpectations })

	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{Name: "terminal-failure", Namespace: "default", UID: "terminal-failure-uid"},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{
				Containers: []corev1.Container{{Name: "main", Image: "example.invalid/sandbox:test"}},
			}},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase: sandboxv1alpha1.BatchSandboxPhaseFailed,
			Conditions: []sandboxv1alpha1.BatchSandboxCondition{{
				Type:   sandboxv1alpha1.BatchSandboxConditionPodFailed,
				Status: sandboxv1alpha1.ConditionTrue,
				Reason: "InitContainerFailed",
			}},
		},
	}
	reconciler := newTestReconciler(bs)

	_, err := reconciler.Reconcile(context.Background(), ctrl.Request{
		NamespacedName: types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name},
	})
	require.NoError(t, err)

	pod := &corev1.Pod{}
	err = reconciler.Get(context.Background(), types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name + "-0"}, pod)
	assert.True(t, apierrors.IsNotFound(err), "a recorded terminal Pod failure must not be hidden by an automatic replacement")
}

func TestReconcile_ReplacesMissingPodWithoutTerminalFailure(t *testing.T) {
	previousExpectations := batchSandboxScaleExpectations
	batchSandboxScaleExpectations = expectations.NewScaleExpectations()
	t.Cleanup(func() { batchSandboxScaleExpectations = previousExpectations })

	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{Name: "retryable-pending", Namespace: "default", UID: "retryable-pending-uid"},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{
				Containers: []corev1.Container{{Name: "main", Image: "example.invalid/sandbox:test"}},
			}},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{Phase: sandboxv1alpha1.BatchSandboxPhasePending},
	}
	reconciler := newTestReconciler(bs)

	_, err := reconciler.Reconcile(context.Background(), ctrl.Request{
		NamespacedName: types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name},
	})
	require.NoError(t, err)

	pod := &corev1.Pod{}
	err = reconciler.Get(context.Background(), types.NamespacedName{Namespace: bs.Namespace, Name: bs.Name + "-0"}, pod)
	require.NoError(t, err, "ordinary expected-state reconciliation must still replace a missing Pod")
}

func TestBuildRuntimeView_AggregatesResumeFailures(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs",
			Namespace: "default",
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase: sandboxv1alpha1.BatchSandboxPhaseResuming,
		},
	}

	pods := []*corev1.Pod{
		{
			ObjectMeta: metav1.ObjectMeta{Name: "imgpull-0", Namespace: "default"},
			Status: corev1.PodStatus{
				ContainerStatuses: []corev1.ContainerStatus{{
					State: corev1.ContainerState{
						Waiting: &corev1.ContainerStateWaiting{
							Reason:  "ImagePullBackOff",
							Message: "pull back off",
						},
					},
				}},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{Name: "imgpull-1", Namespace: "default"},
			Status: corev1.PodStatus{
				ContainerStatuses: []corev1.ContainerStatus{{
					State: corev1.ContainerState{
						Waiting: &corev1.ContainerStateWaiting{
							Reason:  "ImagePullBackOff",
							Message: "pull back off again",
						},
					},
				}},
			},
		},
		{
			ObjectMeta: metav1.ObjectMeta{Name: "healthy-2", Namespace: "default"},
			Status: corev1.PodStatus{
				Phase: corev1.PodPending,
			},
		},
	}

	view := buildRuntimeView(bs, pods)
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)

	var resumeFailed *sandboxv1alpha1.BatchSandboxCondition
	var podFailed *sandboxv1alpha1.BatchSandboxCondition
	for i := range view.status.Conditions {
		switch view.status.Conditions[i].Type {
		case sandboxv1alpha1.BatchSandboxConditionResumeFailed:
			resumeFailed = &view.status.Conditions[i]
		case sandboxv1alpha1.BatchSandboxConditionPodFailed:
			podFailed = &view.status.Conditions[i]
		}
	}
	require.NotNil(t, resumeFailed)
	require.NotNil(t, podFailed)
	assert.Equal(t, sandboxv1alpha1.ConditionTrue, resumeFailed.Status)
	assert.Equal(t, "ImagePullBackOff", resumeFailed.Reason)
	assert.Equal(t, "2/3 observed pods failed during resume; primary reason=ImagePullBackOff; sample pod=imgpull-0", resumeFailed.Message)
	assert.Equal(t, "ImagePullBackOff", podFailed.Reason)
	assert.Equal(t, "2/3 observed pods failed; primary reason=ImagePullBackOff; sample pod=imgpull-0", podFailed.Message)
}

func TestBuildRuntimeView_MarksResumeFailedWhenStaleCacheMissesResumingPhase(t *testing.T) {
	// Under informer lag the cached phase may still be the pre-pause steady phase
	// while a resume request is already in flight; pod failures must still surface
	// ResumeFailed instead of only PodFailed.
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 4,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause: ptr.To(false),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
			PauseObservedGeneration: 3,
		},
	}

	pods := []*corev1.Pod{{
		ObjectMeta: metav1.ObjectMeta{Name: "imgpull-0", Namespace: "default"},
		Status: corev1.PodStatus{
			ContainerStatuses: []corev1.ContainerStatus{{
				State: corev1.ContainerState{
					Waiting: &corev1.ContainerStateWaiting{
						Reason:  "ErrImagePull",
						Message: "image not found",
					},
				},
			}},
		},
	}}

	view := buildRuntimeView(bs, pods)
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)

	var resumeFailed *sandboxv1alpha1.BatchSandboxCondition
	for i := range view.status.Conditions {
		if view.status.Conditions[i].Type == sandboxv1alpha1.BatchSandboxConditionResumeFailed {
			resumeFailed = &view.status.Conditions[i]
			break
		}
	}
	require.NotNil(t, resumeFailed, "resume in flight must mark ResumeFailed even when the cached phase lags")
	assert.Equal(t, sandboxv1alpha1.ConditionTrue, resumeFailed.Status)
	assert.Equal(t, "ErrImagePull", resumeFailed.Reason)
}

func TestBuildRuntimeView_SteadyFailureWithoutResumeInFlightKeepsResumeFailedAbsent(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause: ptr.To(false),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase:                   sandboxv1alpha1.BatchSandboxPhaseSucceed,
			PauseObservedGeneration: 3,
		},
	}

	pods := []*corev1.Pod{{
		ObjectMeta: metav1.ObjectMeta{Name: "crash-0", Namespace: "default"},
		Status: corev1.PodStatus{
			ContainerStatuses: []corev1.ContainerStatus{{
				State: corev1.ContainerState{
					Waiting: &corev1.ContainerStateWaiting{
						Reason:  "CrashLoopBackOff",
						Message: "crash",
					},
				},
			}},
		},
	}}

	view := buildRuntimeView(bs, pods)
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhaseFailed, view.status.Phase)
	for i := range view.status.Conditions {
		assert.NotEqual(t, sandboxv1alpha1.BatchSandboxConditionResumeFailed, view.status.Conditions[i].Type,
			"steady-state pod failures must not set ResumeFailed")
	}
}

func TestBuildRuntimeView_PreservesConditionTransitionTimeWhenUnchanged(t *testing.T) {
	transitionTime := metav1.NewTime(time.Now().Add(-5 * time.Minute))
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 3,
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase: sandboxv1alpha1.BatchSandboxPhaseSucceed,
			Conditions: []sandboxv1alpha1.BatchSandboxCondition{
				{
					Type:               sandboxv1alpha1.BatchSandboxConditionReady,
					Status:             sandboxv1alpha1.ConditionTrue,
					Reason:             "PodsReady",
					Message:            "Sandbox is running",
					LastTransitionTime: &transitionTime,
				},
			},
		},
	}

	pods := []*corev1.Pod{
		{
			ObjectMeta: metav1.ObjectMeta{Name: "healthy-0", Namespace: "default"},
			Status: corev1.PodStatus{
				Phase: corev1.PodRunning,
				PodIP: "10.0.0.10",
				Conditions: []corev1.PodCondition{
					{Type: corev1.PodReady, Status: corev1.ConditionTrue},
				},
			},
		},
	}

	view := buildRuntimeView(bs, pods)

	var ready *sandboxv1alpha1.BatchSandboxCondition
	for i := range view.status.Conditions {
		if view.status.Conditions[i].Type == sandboxv1alpha1.BatchSandboxConditionReady {
			ready = &view.status.Conditions[i]
			break
		}
	}
	require.NotNil(t, ready)
	require.NotNil(t, ready.LastTransitionTime)
	assert.Equal(t, transitionTime, *ready.LastTransitionTime, "unchanged condition should keep its original transition time")
}

func TestBuildRuntimeView_AddsPausedConditionFromEmptyConditions(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 5,
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			Phase: sandboxv1alpha1.BatchSandboxPhasePaused,
		},
	}

	view := buildRuntimeView(bs, nil)

	var paused *sandboxv1alpha1.BatchSandboxCondition
	for i := range view.status.Conditions {
		if view.status.Conditions[i].Type == sandboxv1alpha1.BatchSandboxConditionPaused {
			paused = &view.status.Conditions[i]
			break
		}
	}

	require.NotNil(t, paused)
	require.NotNil(t, paused.LastTransitionTime)
	assert.Equal(t, sandboxv1alpha1.ConditionTrue, paused.Status)
	assert.Equal(t, "Paused", paused.Reason)
	assert.Equal(t, "Sandbox is paused", paused.Message)
}

// ---------- completePause test ----------

func TestCompletePause(t *testing.T) {
	// completePause: status.phase=Paused, controller does not clear spec.pause
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	r := newTestReconciler(bs)

	err := r.completePause(context.Background(), bs)
	require.NoError(t, err)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))

	// Phase should be Paused
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
	var pausedCondition *sandboxv1alpha1.BatchSandboxCondition
	for i := range updated.Status.Conditions {
		if updated.Status.Conditions[i].Type == sandboxv1alpha1.BatchSandboxConditionType("Paused") {
			pausedCondition = &updated.Status.Conditions[i]
			break
		}
	}
	require.NotNil(t, pausedCondition, "Paused phase should publish an explicit Paused condition")
	assert.Equal(t, sandboxv1alpha1.ConditionTrue, pausedCondition.Status)
	// Replicas should remain unchanged (not set to 0 per design doc)
	assert.Equal(t, int32(1), *updated.Spec.Replicas)
	// Pause remains true until the next external request changes it
	require.NotNil(t, updated.Spec.Pause)
	assert.True(t, *updated.Spec.Pause)
}

func TestCompletePause_DeleteFailureLeavesPhasePausing(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-0",
			Namespace: "default",
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion: sandboxv1alpha1.SchemeBuilder.GroupVersion.String(),
					Kind:       "BatchSandbox",
					Name:       "test-bs",
					UID:        "test-uid",
				},
			},
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{{Name: "main", Image: "img"}},
		},
	}
	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithIndex(&corev1.Pod{}, fieldindex.IndexNameForOwnerRefUID, fieldindex.OwnerIndexFunc).
		WithStatusSubresource(&sandboxv1alpha1.BatchSandbox{}).
		WithObjects(bs, pod).
		WithInterceptorFuncs(interceptor.Funcs{
			Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error {
				if obj.GetNamespace() == "default" && obj.GetName() == "test-bs-0" {
					return fmt.Errorf("delete failed")
				}
				return c.Delete(ctx, obj, opts...)
			},
		}).
		Build()
	r := &BatchSandboxReconciler{
		Client:              fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}

	err := r.completePause(context.Background(), bs)
	require.ErrorContains(t, err, "delete failed")

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePausing, updated.Status.Phase)
}

func TestCompletePause_PooledSandboxDetachesForPoolGC(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-bs-uid",
			Finalizers: []string{finalizerPoolAllocation},
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	setSandboxAllocation(bs, sandboxAllocation{Pods: []string{"pool-pod-1"}})

	poolPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pool-pod-1",
			Namespace: "default",
			Labels: map[string]string{
				labelPoolName:     "test-pool",
				labelPoolRevision: "rev-1",
				"app":             "demo",
			},
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion: sandboxv1alpha1.SchemeBuilder.GroupVersion.String(),
					Kind:       "Pool",
					Name:       "test-pool",
					UID:        "test-pool-uid",
				},
			},
		},
		Spec: corev1.PodSpec{
			NodeName:   "node-a",
			Containers: []corev1.Container{{Name: "main", Image: "pool-image:latest"}},
		},
	}

	r := newTestReconciler(bs, poolPod)

	err := r.completePause(context.Background(), bs)
	require.NoError(t, err)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
	assert.Equal(t, "", updated.Spec.PoolRef)
	require.NotNil(t, updated.Spec.Template)
	assert.Equal(t, "pool-image:latest", updated.Spec.Template.Spec.Containers[0].Image)
	assert.Equal(t, "demo", updated.Spec.Template.Labels["app"])
	assert.NotContains(t, updated.Spec.Template.Labels, labelPoolName)
	assert.NotContains(t, updated.Spec.Template.Labels, labelPoolRevision)
	assert.Equal(t, "", updated.Spec.Template.Spec.NodeName)
	assert.NotContains(t, updated.Annotations, annoAllocReleaseKey)
	assert.NotContains(t, updated.Finalizers, finalizerPoolAllocation)
}

func TestCompletePause_PooledSandboxDoesNotDeleteSourcePod(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-bs-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	setSandboxAllocation(bs, sandboxAllocation{Pods: []string{"pool-pod-1"}})

	poolPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pool-pod-1",
			Namespace: "default",
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion: sandboxv1alpha1.SchemeBuilder.GroupVersion.String(),
					Kind:       "Pool",
					Name:       "test-pool",
					UID:        "test-pool-uid",
				},
			},
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{{Name: "main", Image: "pool-image:latest"}},
		},
	}
	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithStatusSubresource(&sandboxv1alpha1.BatchSandbox{}).
		WithObjects(bs, poolPod).
		WithInterceptorFuncs(interceptor.Funcs{
			Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error {
				if obj.GetNamespace() == "default" && obj.GetName() == "pool-pod-1" {
					return fmt.Errorf("batchsandbox reconciler must not delete pooled source pod")
				}
				return c.Delete(ctx, obj, opts...)
			},
		}).
		Build()
	r := &BatchSandboxReconciler{
		Client:              fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}

	err := r.completePause(context.Background(), bs)
	require.NoError(t, err)

	stillPresent := &corev1.Pod{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "pool-pod-1"}, stillPresent))
}

func TestCompletePause_PooledSandboxAcknowledgesSpecPatchGeneration(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-bs-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	setSandboxAllocation(bs, sandboxAllocation{Pods: []string{"pool-pod-1"}})

	poolPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pool-pod-1",
			Namespace: "default",
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion: sandboxv1alpha1.SchemeBuilder.GroupVersion.String(),
					Kind:       "Pool",
					Name:       "test-pool",
					UID:        "test-pool-uid",
				},
			},
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{{Name: "main", Image: "pool-image:latest"}},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithStatusSubresource(&sandboxv1alpha1.BatchSandbox{}).
		WithObjects(bs, poolPod).
		WithInterceptorFuncs(interceptor.Funcs{
			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
				if err := c.Patch(ctx, obj, patch, opts...); err != nil {
					return err
				}
				if obj.GetNamespace() == "default" && obj.GetName() == "test-bs" {
					latest := &sandboxv1alpha1.BatchSandbox{}
					if err := c.Get(ctx, types.NamespacedName{Namespace: "default", Name: "test-bs"}, latest); err != nil {
						return err
					}
					latest.Generation = 3
					return c.Update(ctx, latest)
				}
				return nil
			},
		}).
		Build()
	r := &BatchSandboxReconciler{
		Client:              fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}

	require.NoError(t, r.completePause(context.Background(), bs))

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, int64(3), updated.Generation)
	assert.Equal(t, updated.Generation, updated.Status.PauseObservedGeneration)
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
}

func TestCompletePause_DoesNotAcknowledgeQueuedResumeGeneration(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
			UID:        "test-bs-uid",
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			PoolRef:  "test-pool",
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	setSandboxAllocation(bs, sandboxAllocation{Pods: []string{"pool-pod-1"}})

	poolPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "pool-pod-1",
			Namespace: "default",
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion: sandboxv1alpha1.SchemeBuilder.GroupVersion.String(),
					Kind:       "Pool",
					Name:       "test-pool",
					UID:        "test-pool-uid",
				},
			},
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{{Name: "main", Image: "pool-image:latest"}},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithStatusSubresource(&sandboxv1alpha1.BatchSandbox{}).
		WithObjects(bs, poolPod).
		WithInterceptorFuncs(interceptor.Funcs{
			Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
				if err := c.Patch(ctx, obj, patch, opts...); err != nil {
					return err
				}
				if obj.GetNamespace() == "default" && obj.GetName() == "test-bs" {
					latest := &sandboxv1alpha1.BatchSandbox{}
					if err := c.Get(ctx, types.NamespacedName{Namespace: "default", Name: "test-bs"}, latest); err != nil {
						return err
					}
					latest.Generation = 3
					latest.Spec.Pause = ptr.To(false)
					return c.Update(ctx, latest)
				}
				return nil
			},
		}).
		Build()
	r := &BatchSandboxReconciler{
		Client:              fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}

	require.NoError(t, r.completePause(context.Background(), bs))

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, int64(3), updated.Generation)
	assert.NotNil(t, updated.Spec.Pause)
	assert.False(t, *updated.Spec.Pause)
	assert.Equal(t, int64(2), updated.Status.PauseObservedGeneration)
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
}

// ---------- syncPauseOrClear tests ----------

func TestSyncPauseOrClear_SnapshotReady(t *testing.T) {
	// Snapshot Succeed → completePause
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseSucceed,
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	r := newTestReconciler(bs, snapshot)

	result, err := r.syncPauseOrClear(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0)

	// Verify completePause was called
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePaused, updated.Status.Phase)
	// Replicas should remain unchanged (not set to 0 per design doc)
	assert.Equal(t, int32(1), *updated.Spec.Replicas)
	require.NotNil(t, updated.Spec.Pause)
	assert.True(t, *updated.Spec.Pause)
}

func TestSyncPauseOrClear_SnapshotFailed(t *testing.T) {
	// Snapshot Failed → retryable or terminal failure depending on Pod existence, spec.pause unchanged
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseFailed,
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	r := newTestReconciler(bs, snapshot)

	result, err := r.syncPauseOrClear(context.Background(), bs)
	require.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, result)

	// Verify retryable PauseFailed handling. In this fake-client setup no source Pod exists,
	// so the controller escalates the phase to Failed instead of returning to Succeed.
	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	require.NotNil(t, updated.Spec.Pause)
	assert.True(t, *updated.Spec.Pause)

	// Verify PauseFailed condition is set
	foundCondition := false
	for _, cond := range updated.Status.Conditions {
		if cond.Type == sandboxv1alpha1.BatchSandboxConditionPauseFailed {
			foundCondition = true
			assert.Equal(t, sandboxv1alpha1.ConditionTrue, cond.Status)
			assert.Equal(t, "PodNotFound", cond.Reason) // Pod not found in fake client
			break
		}
	}
	assert.True(t, foundCondition, "PauseFailed condition should be set")
}

func TestSyncPauseOrClear_SnapshotFailedReturnsStatusUpdateError(t *testing.T) {
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseFailed,
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
			Template: &corev1.PodTemplateSpec{
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{Name: "main", Image: "img"}},
				},
			},
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	fakeClient := fake.NewClientBuilder().
		WithScheme(testscheme).
		WithStatusSubresource(&sandboxv1alpha1.BatchSandbox{}, &sandboxv1alpha1.SandboxSnapshot{}).
		WithObjects(bs, snapshot).
		WithInterceptorFuncs(interceptor.Funcs{
			SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
				if subResourceName == "status" && obj.GetNamespace() == "default" && obj.GetName() == "test-bs" {
					return fmt.Errorf("status update failed")
				}
				return c.SubResource(subResourceName).Update(ctx, obj, opts...)
			},
		}).
		Build()
	r := &BatchSandboxReconciler{
		Client:              fakeClient,
		Scheme:              testscheme,
		Recorder:            record.NewFakeRecorder(10),
		StatusRVExpectation: expectations.NewResourceVersionExpectation(),
	}

	result, err := r.syncPauseOrClear(context.Background(), bs)
	require.ErrorContains(t, err, "status update failed")
	assert.Equal(t, ctrl.Result{}, result)
}

func TestSyncPauseOrClear_SnapshotCommitting(t *testing.T) {
	// Snapshot Committing → requeue
	snapshot := &sandboxv1alpha1.SandboxSnapshot{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-bs-pause",
			Namespace: "default",
		},
		Spec: sandboxv1alpha1.SandboxSnapshotSpec{SandboxName: "test-bs"},
		Status: sandboxv1alpha1.SandboxSnapshotStatus{
			Phase: sandboxv1alpha1.SandboxSnapshotPhaseCommitting,
		},
	}
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
			Phase:                   sandboxv1alpha1.BatchSandboxPhasePausing,
		},
	}
	r := newTestReconciler(bs, snapshot)

	result, err := r.syncPauseOrClear(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0, "committing snapshot should requeue")
}

func TestSyncPauseOrClear_NoSnapshot(t *testing.T) {
	// No snapshot while already Pausing → create it and requeue.
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{
			PauseObservedGeneration: 2,
		},
	}
	r := newTestReconciler(bs)

	result, err := r.syncPauseOrClear(context.Background(), bs)
	require.NoError(t, err)
	assert.True(t, result.RequeueAfter > 0, "no snapshot should requeue")

	snap := &sandboxv1alpha1.SandboxSnapshot{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs-pause"}, snap))
	assert.Equal(t, "test-bs", snap.Spec.SandboxName)
}

// ---------- ackPauseWithPhase behavior ----------

func TestAckPauseWithPhase_DoesNotMutateSpecPause(t *testing.T) {
	bs := &sandboxv1alpha1.BatchSandbox{
		ObjectMeta: metav1.ObjectMeta{
			Name:       "test-bs",
			Namespace:  "default",
			Generation: 2,
		},
		Spec: sandboxv1alpha1.BatchSandboxSpec{
			Pause:    ptr.To(true),
			Replicas: ptr.To(int32(1)),
		},
		Status: sandboxv1alpha1.BatchSandboxStatus{},
	}
	r := newTestReconciler(bs)

	err := r.ackPauseWithPhase(context.Background(), bs, sandboxv1alpha1.BatchSandboxPhasePausing, "")
	require.NoError(t, err)

	updated := &sandboxv1alpha1.BatchSandbox{}
	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "test-bs"}, updated))
	assert.Equal(t, sandboxv1alpha1.BatchSandboxPhasePausing, updated.Status.Phase)
	require.NotNil(t, updated.Spec.Pause)
	assert.True(t, *updated.Spec.Pause)
}

func TestInjectQEMURestorePreservesUserInitContainersAndIsIdempotent(t *testing.T) {
	template := &corev1.PodTemplateSpec{
		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
			"sandbox.opensandbox.io/qemu-container": "main",
		}},
		Spec: corev1.PodSpec{
			// A pod-level runAsNonRoot would be inherited by the injected init
			// container unless it overrides the value explicitly.
			SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: ptr.To(true)},
			InitContainers:  []corev1.Container{{Name: "download-ubuntu", Image: "ubuntu-seed:test"}},
			Containers:      []corev1.Container{{Name: "main", Image: "qemu:test"}},
		},
	}
	snapshotObject := &sandboxv1alpha1.SandboxSnapshot{Status: sandboxv1alpha1.SandboxSnapshotStatus{
		Format: sandboxv1alpha1.SandboxSnapshotFormatQEMUV1,
		VirtualMachine: &sandboxv1alpha1.VirtualMachineSnapshot{
			ImageURI:       "registry.example/sandbox-vmstate:snapshot",
			ImageDigest:    "sha256:" + strings.Repeat("1", 64),
			PayloadDigest:  "sha256:" + strings.Repeat("2", 64),
			SizeBytes:      1024,
			Compression:    "zstd",
			ManifestDigest: "sha256:" + strings.Repeat("3", 64),
			Compatibility: sandboxv1alpha1.QEMUCompatibility{
				Architecture:      "amd64",
				QEMUVersion:       "9.1.0",
				MachineType:       "pc-q35-9.1",
				CPUModel:          "host",
				VCPUs:             2,
				MemoryBytes:       1 << 30,
				QEMUConfigDigest:  "sha256:config",
				RequiredNodeClass: "shenlong-v1",
			},
		},
	}}

	require.NoError(t, injectQEMURestore(template, snapshotObject))
	require.NoError(t, injectQEMURestore(template, snapshotObject))
	require.Len(t, template.Spec.InitContainers, 2)
	assert.Equal(t, "download-ubuntu", template.Spec.InitContainers[0].Name)
	restore := template.Spec.InitContainers[1]
	assert.Equal(t, vmStateRestoreInitName, restore.Name)
	assert.Equal(t, "registry.example/sandbox-vmstate@sha256:"+strings.Repeat("1", 64), restore.Image)
	assert.Contains(t, restore.Args, "sha256:"+strings.Repeat("3", 64))
	require.NotNil(t, restore.SecurityContext)
	require.NotNil(t, restore.SecurityContext.RunAsUser)
	assert.Equal(t, int64(0), *restore.SecurityContext.RunAsUser)
	require.NotNil(t, restore.SecurityContext.RunAsNonRoot, "RunAsNonRoot must be set explicitly so a pod-level runAsNonRoot=true does not conflict with UID 0")
	assert.False(t, *restore.SecurityContext.RunAsNonRoot)
	assert.Equal(t, "shenlong-v1", template.Spec.NodeSelector["sandbox.opensandbox.io/qemu-node-class"])
	require.Len(t, template.Spec.Volumes, 1)
	require.Len(t, template.Spec.Containers[0].VolumeMounts, 1)
	assert.Equal(t, vmStateRestoreMountPath, template.Spec.Containers[0].VolumeMounts[0].MountPath)
	assert.Contains(t, template.Spec.Containers[0].Env, corev1.EnvVar{Name: "OPENSANDBOX_RESTORE_MODE", Value: "qemu-v1"})
	assert.Contains(t, template.Spec.Containers[0].Env, corev1.EnvVar{Name: "OPENSANDBOX_VMSTATE_DIR", Value: vmStateRestoreMountPath})

	expectedStorage := vmStateRestoreStorageSize(1024)
	require.NotNil(t, template.Spec.Volumes[0].EmptyDir)
	require.NotNil(t, template.Spec.Volumes[0].EmptyDir.SizeLimit, "restore emptyDir must reserve a size limit based on the recorded VM state size")
	assert.Equal(t, expectedStorage.Value(), template.Spec.Volumes[0].EmptyDir.SizeLimit.Value())
	assert.Equal(t, expectedStorage, restore.Resources.Requests[corev1.ResourceEphemeralStorage])
	assert.Equal(t, expectedStorage, restore.Resources.Limits[corev1.ResourceEphemeralStorage])
}

// Ensure ctrl.Result type is used
var _ = ctrl.Result{}