Secure, Fast, and Extensible Sandbox runtime for AI agents.
// Copyright 2025 The OpenSandbox Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"context"
"crypto/sha256"
"encoding/hex"
stdjson "encoding/json"
gerrors "errors"
"fmt"
"os"
"sort"
"strconv"
"sync"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/retry"
"k8s.io/client-go/util/workqueue"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
sandboxv1alpha1 "github.com/alibaba/OpenSandbox/sandbox-k8s/apis/sandbox/v1alpha1"
"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/controller/eviction"
"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/controller/recycle"
"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils"
controllerutils "github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/controller"
"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/expectations"
"github.com/alibaba/OpenSandbox/sandbox-k8s/internal/utils/fieldindex"
)
const (
defaultRetryTime = 5 * time.Second
)
const (
labelPoolName = "sandbox.opensandbox.io/pool-name"
labelPoolRevision = "sandbox.opensandbox.io/pool-revision"
)
const (
defaultSyncSandboxAllocConcurrency = 256
envSyncSandboxAllocConcurrency = "SYNC_SANDBOX_ALLOC_CONCURRENCY"
defaultRecyclePodConcurrency = 64
envRecyclePodConcurrency = "RECYCLE_POD_CONCURRENCY"
)
var (
poolScaleExpectations = expectations.NewScaleExpectations()
syncSandboxAllocConcurrency int
recyclePodConcurrency int
)
func init() {
syncSandboxAllocConcurrency = defaultSyncSandboxAllocConcurrency
if val := os.Getenv(envSyncSandboxAllocConcurrency); val != "" {
if n, err := strconv.Atoi(val); err == nil && n > 0 {
syncSandboxAllocConcurrency = n
}
}
recyclePodConcurrency = defaultRecyclePodConcurrency
if val := os.Getenv(envRecyclePodConcurrency); val != "" {
if n, err := strconv.Atoi(val); err == nil && n > 0 {
recyclePodConcurrency = n
}
}
}
// PoolReconciler reconciles a Pool object
type PoolReconciler struct {
client.Client
APIReader client.Reader
Scheme *runtime.Scheme
Recorder record.EventRecorder
Allocator Allocator
RestConfig *rest.Config
}
// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=pools,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=pools/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=pools/finalizers,verbs=update
// +kubebuilder:rbac:groups=sandbox.opensandbox.io,resources=batchsandboxes,verbs=get;list;watch;patch
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create
// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
func (r *PoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, retErr error) {
log := logf.FromContext(ctx)
start := time.Now()
defer func() {
log.Info("Reconcile finished", "duration", time.Since(start).String(), "requeueAfter", result.RequeueAfter.String(), "error", retErr)
}()
pool := &sandboxv1alpha1.Pool{}
if err := r.Get(ctx, req.NamespacedName, pool); err != nil {
if errors.IsNotFound(err) {
poolUnavailable, checkErr := r.confirmPoolUnavailable(ctx, req.NamespacedName, "")
if checkErr != nil {
return ctrl.Result{}, checkErr
}
if !poolUnavailable {
log.Info("Pool exists in API server but has not reached the cache yet", "pool", req.NamespacedName.String())
return ctrl.Result{RequeueAfter: defaultRetryTime}, nil
}
// Pool resource is confirmed absent from the API server.
controllerKey := req.NamespacedName.String()
poolScaleExpectations.DeleteExpectations(controllerKey)
r.Allocator.ClearPoolAllocation(ctx, req.Namespace, req.Name)
cleanupComplete, cleanupErr := r.cleanupTerminatingSandboxesForUnavailablePool(ctx, req.Namespace, req.Name, "")
if cleanupErr != nil {
return ctrl.Result{}, cleanupErr
}
if !cleanupComplete {
return ctrl.Result{RequeueAfter: defaultRetryTime}, nil
}
log.Info("Pool resource not found, cleaned up scale expectations", "pool", controllerKey)
return ctrl.Result{}, nil
}
log.Error(err, "Failed to get Pool")
return ctrl.Result{}, err
}
if !pool.DeletionTimestamp.IsZero() {
poolUnavailable, checkErr := r.confirmPoolUnavailable(ctx, req.NamespacedName, pool.UID)
if checkErr != nil {
return ctrl.Result{}, checkErr
}
if !poolUnavailable {
log.Info("Pool was recreated or is no longer terminating", "pool", req.NamespacedName.String())
return ctrl.Result{RequeueAfter: defaultRetryTime}, nil
}
controllerKey := controllerutils.GetControllerKey(pool)
poolScaleExpectations.DeleteExpectations(controllerKey)
r.Allocator.ClearPoolAllocation(ctx, req.Namespace, req.Name)
cleanupComplete, cleanupErr := r.cleanupTerminatingSandboxesForUnavailablePool(ctx, req.Namespace, req.Name, pool.UID)
if cleanupErr != nil {
return ctrl.Result{}, cleanupErr
}
if !cleanupComplete {
return ctrl.Result{RequeueAfter: defaultRetryTime}, nil
}
log.Info("Pool resource is being deleted, cleaned up scale expectations", "pool", controllerKey)
return ctrl.Result{}, nil
}
podList := &corev1.PodList{}
if err := r.List(ctx, podList, &client.ListOptions{
Namespace: pool.Namespace,
FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(pool.UID)}),
}); err != nil {
log.Error(err, "Failed to list pods")
return reconcile.Result{}, err
}
controllerKey := controllerutils.GetControllerKey(pool)
podNames := make(map[string]struct{}, len(podList.Items))
pods := make([]*corev1.Pod, 0, len(podList.Items))
for i := range podList.Items {
pod := podList.Items[i]
podNames[pod.Name] = struct{}{}
poolScaleExpectations.ObserveScale(controllerKey, expectations.Create, pod.Name)
if pod.DeletionTimestamp.IsZero() {
pods = append(pods, &pod)
}
}
observeDeletedPods(controllerKey, podNames)
batchSandboxList := &sandboxv1alpha1.BatchSandboxList{}
if err := r.List(ctx, batchSandboxList, &client.ListOptions{
Namespace: pool.Namespace,
FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForPoolRef: pool.Name}),
}); err != nil {
log.Error(err, "Failed to list batch sandboxes")
return reconcile.Result{}, err
}
batchSandboxes := make([]*sandboxv1alpha1.BatchSandbox, 0, len(batchSandboxList.Items))
for i := range batchSandboxList.Items {
batchSandbox := batchSandboxList.Items[i]
if batchSandbox.Spec.Template != nil {
continue
}
batchSandboxes = append(batchSandboxes, &batchSandbox)
}
log.Info("Pool reconcile", "pool", pool.Name, "pods", len(pods), "batchSandboxes", len(batchSandboxes))
return r.reconcilePool(ctx, pool, batchSandboxes, pods, int32(len(podList.Items)))
}
func observeDeletedPods(controllerKey string, existingPodNames map[string]struct{}) {
for name := range poolScaleExpectations.GetExpectations(controllerKey)[expectations.Delete] {
if _, exists := existingPodNames[name]; !exists {
poolScaleExpectations.ObserveScale(controllerKey, expectations.Delete, name)
}
}
}
// confirmPoolUnavailable reads directly from the API server so a stale informer
// cache cannot make an active or recreated Pool look absent. An empty expectedUID
// only confirms API-level absence; a non-empty UID also accepts that same Pool
// while it remains terminating.
func (r *PoolReconciler) confirmPoolUnavailable(ctx context.Context, poolKey client.ObjectKey, expectedUID types.UID) (bool, error) {
if r.APIReader == nil {
return false, fmt.Errorf("uncached API reader is not configured")
}
latestPool := &sandboxv1alpha1.Pool{}
if err := r.APIReader.Get(ctx, poolKey, latestPool); err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, fmt.Errorf("failed to verify pool %s from API server: %w", poolKey.String(), err)
}
if expectedUID == "" {
return false, nil
}
return latestPool.UID == expectedUID && !latestPool.DeletionTimestamp.IsZero(), nil
}
// cleanupTerminatingSandboxesForUnavailablePool removes Pool-owned finalizers
// once pending allocated pods are gone. It returns false while cleanup is pending
// or the Pool becomes available again, so the caller retries.
func (r *PoolReconciler) cleanupTerminatingSandboxesForUnavailablePool(ctx context.Context, namespace, poolName string, expectedPoolUID types.UID) (bool, error) {
log := logf.FromContext(ctx)
batchSandboxList := &sandboxv1alpha1.BatchSandboxList{}
if err := r.List(ctx, batchSandboxList, &client.ListOptions{
Namespace: namespace,
FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForPoolRef: poolName}),
}); err != nil {
return false, fmt.Errorf("failed to list batch sandboxes for unavailable pool %s/%s: %w", namespace, poolName, err)
}
complete := true
var errs []error
for i := range batchSandboxList.Items {
sandbox := &batchSandboxList.Items[i]
if sandbox.DeletionTimestamp.IsZero() || !controllerutil.ContainsFinalizer(sandbox, finalizerPoolAllocation) {
continue
}
poolUnavailable, removed, err := r.removePoolAllocationFinalizerIfUnavailable(
ctx,
client.ObjectKeyFromObject(sandbox),
client.ObjectKey{Namespace: namespace, Name: poolName},
expectedPoolUID,
)
if err != nil {
if errors.IsNotFound(err) {
continue
}
log.Error(err, "Failed to remove stale pool allocation finalizer", "pool", poolName, "sandbox", sandbox.Name)
errs = append(errs, err)
continue
}
if !poolUnavailable {
log.Info("Stopped finalizer cleanup because Pool is available", "pool", poolName)
return false, gerrors.Join(errs...)
}
if removed {
log.Info("Removed stale pool allocation finalizer", "pool", poolName, "sandbox", sandbox.Name)
} else {
complete = false
}
}
return complete, gerrors.Join(errs...)
}
func (r *PoolReconciler) removePoolAllocationFinalizerIfUnavailable(
ctx context.Context,
sandboxKey client.ObjectKey,
poolKey client.ObjectKey,
expectedPoolUID types.UID,
) (poolUnavailable bool, removed bool, retErr error) {
if r.APIReader == nil {
return false, false, fmt.Errorf("uncached API reader is not configured")
}
poolUnavailable = true
retErr = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
removed = false
latestSandbox := &sandboxv1alpha1.BatchSandbox{}
if err := r.APIReader.Get(ctx, sandboxKey, latestSandbox); err != nil {
return err
}
if latestSandbox.Spec.PoolRef != poolKey.Name ||
latestSandbox.DeletionTimestamp.IsZero() ||
!controllerutil.ContainsFinalizer(latestSandbox, finalizerPoolAllocation) {
return nil
}
var err error
poolUnavailable, err = r.confirmPoolUnavailable(ctx, poolKey, expectedPoolUID)
if err != nil || !poolUnavailable {
return err
}
pending, err := r.pendingPoolPods(ctx, latestSandbox)
if err != nil {
return err
}
for _, podName := range pending {
pod := &corev1.Pod{}
err := r.APIReader.Get(ctx, client.ObjectKey{Namespace: sandboxKey.Namespace, Name: podName}, pod)
if errors.IsNotFound(err) {
continue
}
if err != nil {
return fmt.Errorf("failed to verify allocated pod %s/%s: %w", sandboxKey.Namespace, podName, err)
}
return nil
}
base := latestSandbox.DeepCopy()
controllerutil.RemoveFinalizer(latestSandbox, finalizerPoolAllocation)
if err := r.Patch(ctx, latestSandbox, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil {
return err
}
removed = true
return nil
})
return poolUnavailable, removed, retErr
}
// reconcilePool contains the main reconciliation logic
func (r *PoolReconciler) reconcilePool(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod, totalPodCnt int32) (ctrl.Result, error) {
var result ctrl.Result
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
// 1. Get latest Pool CR
latestPool := &sandboxv1alpha1.Pool{}
if err := r.Get(ctx, client.ObjectKeyFromObject(pool), latestPool); err != nil {
return err
}
// 2. Handle pod eviction
schedulePods, evictionErr := r.handleEviction(ctx, latestPool, pods)
if schedulePods == nil {
return evictionErr
}
// 3. Schedule sandbox (compute + persist + sync)
schedResult, err := r.scheduleSandbox(ctx, latestPool, batchSandboxes, schedulePods)
if err != nil {
return err
}
// Requeue if there are pending sandboxes waiting for scheduling
if schedResult.SupplyCnt > 0 {
result = ctrl.Result{RequeueAfter: defaultRetryTime}
}
// Best-effort compatibility migration for allocations written before PoolRef
// was included in the alloc-status annotation. Do not let a patch failure
// interrupt scheduling, releasing, or scaling.
for _, sandbox := range batchSandboxes {
if err := r.backfillLegacyPoolAllocation(ctx, latestPool, sandbox, schedulePods, schedResult.LatestAllocation); err != nil {
logf.FromContext(ctx).Error(err, "Failed to backfill legacy pool allocation", "pool", latestPool.Name, "sandbox", sandbox.Name)
if result.RequeueAfter == 0 || result.RequeueAfter > defaultRetryTime {
result.RequeueAfter = defaultRetryTime
}
}
}
// 4. Handle pool upgrade
updateResult, err := r.updatePool(ctx, latestPool, schedulePods, schedResult.IdlePods)
if err != nil {
return err
}
// 5. Handle pool scale
toDeletePods := append(updateResult.ToDeletePods, schedResult.ToDelete...)
args := &scaleArgs{
updateRevision: updateResult.UpdateRevision,
pods: schedulePods,
totalPodCnt: totalPodCnt,
allocatedCnt: int32(len(schedResult.LatestAllocation)),
idlePods: updateResult.IdlePods,
toDeletePods: toDeletePods,
supplyCnt: schedResult.SupplyCnt + updateResult.SupplyUpdateRevision,
}
scalePending, err := r.scalePool(ctx, latestPool, args)
if err != nil {
return err
}
if scalePending {
result = ctrl.Result{RequeueAfter: defaultRetryTime}
}
// 6. Update pool status
if err := r.updatePoolStatus(ctx, updateResult.UpdateRevision, latestPool, pods, schedulePods, schedResult.LatestAllocation); err != nil {
return err
}
if evictionErr != nil {
return evictionErr
}
return nil
})
return result, err
}
// backfillLegacyPoolAllocation adds PoolRef and Generation to a verified legacy
// allocation annotation. It intentionally leaves all newer or ambiguous records
// untouched.
func (r *PoolReconciler) backfillLegacyPoolAllocation(ctx context.Context, pool *sandboxv1alpha1.Pool, sandbox *sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod, latestAllocation map[string]string) error {
if sandbox.Spec.PoolRef == "" || sandbox.Spec.PoolRef != pool.Name ||
!sandbox.DeletionTimestamp.IsZero() ||
!controllerutil.ContainsFinalizer(sandbox, finalizerPoolAllocation) {
return nil
}
allocation, valid := parseLegacySandboxAllocation(sandbox)
if !valid {
return nil
}
releasePods, valid := validLegacyAllocationRelease(sandbox.GetAnnotations(), annoAllocReleaseKey)
if !valid {
return nil
}
releasedPods, valid := validLegacyAllocationRelease(sandbox.GetAnnotations(), annoAllocReleasedKey)
if !valid {
return nil
}
if allocationIntersects(allocation.Pods, releasePods) || allocationIntersects(allocation.Pods, releasedPods) {
return nil
}
poolPods := make(map[string]struct{}, len(pods))
for _, pod := range pods {
if pod != nil && pod.DeletionTimestamp.IsZero() && pod.Labels[labelPoolName] == pool.Name {
poolPods[pod.Name] = struct{}{}
}
}
for _, podName := range allocation.Pods {
if _, ok := poolPods[podName]; !ok {
return nil
}
if latestAllocation[podName] != sandbox.Name {
return nil
}
}
allocation.PoolRef = pool.Name
allocation.Generation = sandbox.Generation
rawAllocation, err := json.Marshal(allocation)
if err != nil {
return err
}
patchData, err := json.Marshal(map[string]any{
"metadata": map[string]any{
"annotations": map[string]string{annoAllocStatusKey: string(rawAllocation)},
},
})
if err != nil {
return err
}
obj := &sandboxv1alpha1.BatchSandbox{}
obj.Name = sandbox.Name
obj.Namespace = sandbox.Namespace
return r.Patch(ctx, obj, client.RawPatch(types.MergePatchType, patchData))
}
// parseLegacySandboxAllocation accepts only the historical pods-only JSON
// shape. Explicit or partial newer evidence is never rewritten by migration.
func parseLegacySandboxAllocation(sandbox *sandboxv1alpha1.BatchSandbox) (sandboxAllocation, bool) {
raw, ok := sandbox.GetAnnotations()[annoAllocStatusKey]
if !ok {
return sandboxAllocation{}, false
}
var payload map[string]stdjson.RawMessage
if err := json.Unmarshal([]byte(raw), &payload); err != nil || payload == nil {
return sandboxAllocation{}, false
}
if _, exists := payload["poolRef"]; exists {
return sandboxAllocation{}, false
}
if _, exists := payload["generation"]; exists {
return sandboxAllocation{}, false
}
if len(payload) != 1 {
return sandboxAllocation{}, false
}
rawPods, ok := payload["pods"]
if !ok {
return sandboxAllocation{}, false
}
var pods []string
if err := json.Unmarshal(rawPods, &pods); err != nil || !validLegacyAllocationPods(pods) {
return sandboxAllocation{}, false
}
return sandboxAllocation{Pods: pods}, true
}
func validLegacyAllocationPods(pods []string) bool {
if len(pods) == 0 {
return false
}
return validUniqueDNS1123PodNames(pods)
}
// validLegacyAllocationRelease verifies that an optional release annotation has
// an explicit, non-null pods list whose pod names are valid and unique.
func validLegacyAllocationRelease(annotations map[string]string, key string) ([]string, bool) {
raw, ok := annotations[key]
if !ok {
return nil, true
}
var payload map[string]stdjson.RawMessage
if err := json.Unmarshal([]byte(raw), &payload); err != nil || payload == nil {
return nil, false
}
rawPods, ok := payload["pods"]
if !ok {
return nil, false
}
var pods []string
if err := json.Unmarshal(rawPods, &pods); err != nil || pods == nil || !validUniqueDNS1123PodNames(pods) {
return nil, false
}
return pods, true
}
func validUniqueDNS1123PodNames(pods []string) bool {
seen := make(map[string]struct{}, len(pods))
for _, podName := range pods {
if podName == "" || len(validation.IsDNS1123Subdomain(podName)) != 0 {
return false
}
if _, ok := seen[podName]; ok {
return false
}
seen[podName] = struct{}{}
}
return true
}
func allocationIntersects(allocationPods, otherPods []string) bool {
allocationSet := make(map[string]struct{}, len(allocationPods))
for _, podName := range allocationPods {
allocationSet[podName] = struct{}{}
}
for _, podName := range otherPods {
if _, ok := allocationSet[podName]; ok {
return true
}
}
return false
}
func (r *PoolReconciler) calculateRevision(pool *sandboxv1alpha1.Pool) (string, error) {
template, err := json.Marshal(pool.Spec.Template)
if err != nil {
return "", err
}
revision := sha256.Sum256(template)
return hex.EncodeToString(revision[:8]), nil
}
// shouldReconcilePoolForBatchSandboxUpdate filters updates that affect pool allocation.
func shouldReconcilePoolForBatchSandboxUpdate(e event.UpdateEvent) bool {
oldObj, okOld := e.ObjectOld.(*sandboxv1alpha1.BatchSandbox)
newObj, okNew := e.ObjectNew.(*sandboxv1alpha1.BatchSandbox)
if !okOld || !okNew {
return false
}
if newObj.Spec.PoolRef == "" {
return false
}
oldVal := oldObj.Annotations[annoAllocReleaseKey]
newVal := newObj.Annotations[annoAllocReleaseKey]
if oldVal != newVal {
return true
}
// Trigger reconcile when PoolRef changes, e.g. auto-assign resolves "*" to a concrete Pool.
if oldObj.Spec.PoolRef != newObj.Spec.PoolRef {
return true
}
if !equality.Semantic.DeepEqual(oldObj.Spec.Replicas, newObj.Spec.Replicas) {
return true
}
// Trigger reconcile when sandbox enters terminating state (DeletionTimestamp is set).
if oldObj.DeletionTimestamp.IsZero() && !newObj.DeletionTimestamp.IsZero() {
return true
}
// Task cleanup completion hands the remaining pods back to this controller.
return !newObj.DeletionTimestamp.IsZero() &&
controllerutil.ContainsFinalizer(oldObj, finalizerTaskCleanup) &&
!controllerutil.ContainsFinalizer(newObj, finalizerTaskCleanup)
}
// SetupWithManager sets up the controller with the Manager.
func (r *PoolReconciler) SetupWithManager(mgr ctrl.Manager, maxConcurrentReconciles int) error {
if r.APIReader == nil {
r.APIReader = mgr.GetAPIReader()
}
filterBatchSandbox := predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool {
bsb, ok := e.Object.(*sandboxv1alpha1.BatchSandbox)
if !ok {
return false
}
return bsb.Spec.PoolRef != ""
},
UpdateFunc: shouldReconcilePoolForBatchSandboxUpdate,
DeleteFunc: func(e event.DeleteEvent) bool {
bsb, ok := e.Object.(*sandboxv1alpha1.BatchSandbox)
if !ok {
return false
}
return bsb.Spec.PoolRef != ""
},
GenericFunc: func(e event.GenericEvent) bool {
bsb, ok := e.Object.(*sandboxv1alpha1.BatchSandbox)
if !ok {
return false
}
return bsb.Spec.PoolRef != ""
},
}
findPoolForBatchSandbox := func(ctx context.Context, obj client.Object) []reconcile.Request {
log := logf.FromContext(ctx)
batchSandbox, ok := obj.(*sandboxv1alpha1.BatchSandbox)
if !ok {
log.Error(nil, "Invalid object type, expected BatchSandbox")
return nil
}
return []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Namespace: batchSandbox.Namespace,
Name: batchSandbox.Spec.PoolRef,
},
},
}
}
filterBatchSandboxDetached := predicate.Funcs{
UpdateFunc: func(e event.UpdateEvent) bool {
oldObj, okOld := e.ObjectOld.(*sandboxv1alpha1.BatchSandbox)
newObj, okNew := e.ObjectNew.(*sandboxv1alpha1.BatchSandbox)
if !okOld || !okNew {
return false
}
return oldObj.Spec.PoolRef != "" && newObj.Spec.PoolRef == ""
},
}
enqueueOldPoolForDetachedBatchSandbox := handler.Funcs{
UpdateFunc: func(_ context.Context, e event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) {
oldObj, ok := e.ObjectOld.(*sandboxv1alpha1.BatchSandbox)
if !ok || oldObj.Spec.PoolRef == "" {
return
}
q.Add(reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: oldObj.Namespace,
Name: oldObj.Spec.PoolRef,
},
})
},
}
return ctrl.NewControllerManagedBy(mgr).
For(&sandboxv1alpha1.Pool{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Owns(&corev1.Pod{}).
Watches(
&sandboxv1alpha1.BatchSandbox{},
handler.EnqueueRequestsFromMapFunc(findPoolForBatchSandbox),
builder.WithPredicates(filterBatchSandbox),
).
Watches(
&sandboxv1alpha1.BatchSandbox{},
enqueueOldPoolForDetachedBatchSandbox,
builder.WithPredicates(filterBatchSandboxDetached),
).
Named("pool").
WithOptions(controller.Options{MaxConcurrentReconciles: maxConcurrentReconciles}).
Complete(r)
}
func (r *PoolReconciler) doAllocate(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod, toAllocate map[string][]string) error {
// 1. Compute latest allocated pods per sandbox (merge current + newly allocated).
toSyncMap := r.getLatestAllocated(ctx, pool, batchSandboxes, toAllocate)
// 2. Concurrently sync each sandbox's Allocated annotation (AddFinalizer is called inside SyncSandboxAllocation).
syncStart := time.Now()
err := r.syncSandboxConcurrently(ctx, batchSandboxes, toSyncMap, r.Allocator.SyncSandboxAllocation, "allocated")
// Skip no-op rounds (nothing to sync) so idle reconciles don't skew the histogram.
if len(toSyncMap) > 0 {
recordAllocatorSyncAllocResultDuration(ctx, pool, time.Since(syncStart), err)
}
return err
}
// getLatestAllocated computes the latest allocated pods for each sandbox by merging current allocation with new pods to allocate.
func (r *PoolReconciler) getLatestAllocated(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, toAllocate map[string][]string) map[string][]string {
log := logf.FromContext(ctx)
sandboxByName := make(map[string]*sandboxv1alpha1.BatchSandbox, len(batchSandboxes))
for _, bs := range batchSandboxes {
sandboxByName[bs.Name] = bs
}
toSyncMap := make(map[string][]string, len(toAllocate))
for sandboxName, allocPods := range toAllocate {
if len(allocPods) == 0 {
continue
}
sandbox, ok := sandboxByName[sandboxName]
if !ok {
log.Error(nil, "Sandbox not found for allocate", "sandbox", sandboxName)
continue
}
currentAllocated, err := r.Allocator.GetSandboxAllocation(ctx, sandbox)
if err != nil {
log.Error(err, "Failed to get sandbox allocated", "sandbox", sandboxName)
continue
}
toSyncMap[sandboxName] = append(currentAllocated, allocPods...)
}
return toSyncMap
}
// syncSandboxConcurrently syncs allocation or released state for each sandbox concurrently.
// Each sandbox is an independent resource, so concurrent writes are safe.
func (r *PoolReconciler) syncSandboxConcurrently(ctx context.Context, batchSandboxes []*sandboxv1alpha1.BatchSandbox, toSyncMap map[string][]string, syncFn func(context.Context, *sandboxv1alpha1.BatchSandbox, []string) error, label string) error {
log := logf.FromContext(ctx)
sandboxByName := make(map[string]*sandboxv1alpha1.BatchSandbox, len(batchSandboxes))
for _, bs := range batchSandboxes {
sandboxByName[bs.Name] = bs
}
errCh := make(chan error, len(toSyncMap))
sem := make(chan struct{}, syncSandboxAllocConcurrency)
var wg sync.WaitGroup
for sandboxName, pods := range toSyncMap {
sandbox, ok := sandboxByName[sandboxName]
if !ok {
log.Error(nil, "Sandbox not found for sync "+label, "sandbox", sandboxName)
continue
}
wg.Add(1)
go func() {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
syncStart := time.Now()
syncErr := syncFn(ctx, sandbox, pods)
recordAllocatorSyncSingleAllocResultDuration(ctx, sandbox, time.Since(syncStart), syncErr)
if syncErr != nil {
log.Error(syncErr, "Failed to sync sandbox "+label, "sandbox", sandbox.Name)
errCh <- syncErr
}
}()
}
wg.Wait()
close(errCh)
var errs []error
for err := range errCh {
errs = append(errs, err)
}
return gerrors.Join(errs...)
}
func (r *PoolReconciler) doRecycle(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod, toRecycle map[string][]string) (map[string][]string, []string, error) {
if len(toRecycle) == 0 {
return nil, nil, nil
}
if r.APIReader == nil {
return nil, nil, fmt.Errorf("uncached API reader is not configured")
}
handler, err := recycle.NewHandler(r.Client, r.RestConfig, pool)
if err != nil {
return nil, nil, fmt.Errorf("failed to get recycle handler for pool %s: %w", pool.Name, err)
}
results := r.runRecycleTasks(ctx, pool, pods, toRecycle, handler)
return collectRecycleResults(ctx, results)
}
type recycleResult struct {
sandboxName string
podName string
status *recycle.Status
err error
}
// runRecycleTasks executes TryRecycle concurrently for each (sandbox, pod) pair and
// returns one result per task.
func (r *PoolReconciler) runRecycleTasks(ctx context.Context, pool *sandboxv1alpha1.Pool, pods []*corev1.Pod, toRecycle map[string][]string, handler recycle.Handler) []recycleResult {
podByName := make(map[string]*corev1.Pod, len(pods))
for _, p := range pods {
podByName[p.Name] = p
}
// Flatten the map into an ordered slice so goroutines can write by index.
type task struct {
sandboxName string
podName string
}
var tasks []task
for sandboxName, podNames := range toRecycle {
for _, podName := range podNames {
tasks = append(tasks, task{sandboxName: sandboxName, podName: podName})
}
}
// Results are written by index so each goroutine writes to a unique slot without synchronization.
results := make([]recycleResult, len(tasks))
sem := make(chan struct{}, recyclePodConcurrency)
var wg sync.WaitGroup
for idx, task := range tasks {
localIdx, localTask := idx, task
wg.Add(1)
go func() {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
pod := podByName[localTask.podName]
if pod == nil {
// Scheduling excludes terminating pods. Confirm absence before
// treating a missing entry as completed recycling.
pod = &corev1.Pod{}
err := r.APIReader.Get(ctx, client.ObjectKey{Namespace: pool.Namespace, Name: localTask.podName}, pod)
if errors.IsNotFound(err) {
pod = nil
} else if err != nil {
results[localIdx] = recycleResult{sandboxName: localTask.sandboxName, podName: localTask.podName, err: err}
return
}
}
status, err := handler.TryRecycle(ctx, pool, pod, &recycle.Spec{ID: localTask.sandboxName})
results[localIdx] = recycleResult{sandboxName: localTask.sandboxName, podName: localTask.podName, status: status, err: err}
}()
}
wg.Wait()
return results
}
// collectRecycleResults aggregates per-task results into the succeed map and delete list.
func collectRecycleResults(ctx context.Context, results []recycleResult) (map[string][]string, []string, error) {
log := logf.FromContext(ctx)
succeedMap := make(map[string][]string)
var toDeletePods []string
var errs []error
for _, res := range results {
if res.err != nil {
log.Error(res.err, "Failed to recycle pod", "pod", res.podName, "sandbox", res.sandboxName)
errs = append(errs, res.err)
continue
}
if res.status.State == recycle.StateSucceeded {
succeedMap[res.sandboxName] = append(succeedMap[res.sandboxName], res.podName)
}
if res.status.NeedDelete {
toDeletePods = append(toDeletePods, res.podName)
}
}
return succeedMap, toDeletePods, gerrors.Join(errs...)
}
// doRelease runs the recycle operation for pods to be returned to the pool,
// then persists the released state to each sandbox's annotation.
func (r *PoolReconciler) doRelease(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod, toRelease map[string][]string) ([]string, error) {
log := logf.FromContext(ctx)
// 1. Recycle pods.
succeedMap, toDeletePods, err := r.doRecycle(ctx, pool, batchSandboxes, pods, toRelease)
if err != nil {
log.Error(err, "Some errors occurred during recycle")
r.Recorder.Eventf(pool, corev1.EventTypeWarning, eventReasonFailedRecyclePod, "Failed to recycle some pods: %v", err)
}
// Emit recycle success events.
var allRecycled []string
for _, recycledPods := range succeedMap {
allRecycled = append(allRecycled, recycledPods...)
}
if len(allRecycled) > 0 {
r.Recorder.Eventf(pool, corev1.EventTypeNormal, eventReasonPodRecycled, "Recycled %d pod(s): %v", len(allRecycled), allRecycled)
}
// 2. Compute latest released pods per sandbox (merge current + recycle-succeeded).
// Also collect orphan pods whose sandboxes no longer exist.
toSyncMap, orphanPods := r.getLatestReleased(ctx, batchSandboxes, succeedMap)
// 3. Concurrently sync each sandbox's Released annotation.
syncErr := r.syncSandboxConcurrently(ctx, batchSandboxes, toSyncMap, r.Allocator.SyncSandboxReleased, "released")
if syncErr != nil {
log.Error(syncErr, "Failed to sync released")
}
// 4. A terminating sandbox may already have no unreleased allocations
// (including legacy objects whose alloc-status is empty). No recycle result is
// produced in that case, so remove only the stale finalizer. Do not call
// SyncSandboxReleased again: historical released Pods may already belong to a
// different sandbox in the in-memory allocation store.
finalizeErr := r.finalizeTerminatingSandboxes(ctx, batchSandboxes)
if finalizeErr != nil {
log.Error(finalizeErr, "Failed to finalize terminating sandboxes")
}
// 5. Release in-memory allocations for orphan pods directly (no annotation to persist).
if len(orphanPods) > 0 {
r.Allocator.ReleasePodsAllocation(ctx, pool.Namespace, pool.Name, orphanPods)
}
return toDeletePods, gerrors.Join(err, syncErr, finalizeErr)
}
// finalizeTerminatingSandboxes removes the Pool finalizer once every allocated
// Pod is already recorded as released. It intentionally does not persist the
// released annotation or mutate the allocation store: released Pods may have
// been reassigned to another sandbox by the time this cleanup runs.
func (r *PoolReconciler) finalizeTerminatingSandboxes(ctx context.Context, batchSandboxes []*sandboxv1alpha1.BatchSandbox) error {
log := logf.FromContext(ctx)
var errs []error
for _, sandbox := range batchSandboxes {
if sandbox.DeletionTimestamp.IsZero() ||
!controllerutil.ContainsFinalizer(sandbox, finalizerPoolAllocation) {
continue
}
pending, err := r.pendingPoolPods(ctx, sandbox)
if err != nil {
errs = append(errs, fmt.Errorf("get pending allocations for %s: %w", client.ObjectKeyFromObject(sandbox), err))
continue
}
if len(pending) > 0 {
continue
}
if err := utils.UpdateFinalizer(r.Client, sandbox, utils.RemoveFinalizerOpType, finalizerPoolAllocation); err != nil {
if errors.IsNotFound(err) {
continue
}
err = fmt.Errorf("failed to remove pool allocation finalizer from sandbox %s: %w", sandbox.Name, err)
log.Error(err, "Cannot finalize terminating sandbox", "sandbox", sandbox.Name)
errs = append(errs, err)
continue
}
controllerutil.RemoveFinalizer(sandbox, finalizerPoolAllocation)
log.Info("Finalized terminating sandbox with no pending pool allocations", "sandbox", sandbox.Name)
}
return gerrors.Join(errs...)
}
// pendingPoolPods excludes completed recycling. Historical released pods may
// already belong to another sandbox and must not delay this finalizer.
func (r *PoolReconciler) pendingPoolPods(ctx context.Context, sandbox *sandboxv1alpha1.BatchSandbox) ([]string, error) {
allocated, err := r.Allocator.GetSandboxAllocation(ctx, sandbox)
if err != nil {
return nil, err
}
released, err := r.Allocator.GetSandboxReleased(ctx, sandbox)
if err != nil {
return nil, err
}
return sets.List(sets.New(allocated...).Difference(sets.New(released...))), nil
}
// getLatestReleased computes the latest released pods for each sandbox by merging current released with recycle-succeeded pods.
// It also returns orphanPods: pods from succeedMap whose sandbox no longer exists and should be released directly by the caller.
func (r *PoolReconciler) getLatestReleased(ctx context.Context, batchSandboxes []*sandboxv1alpha1.BatchSandbox, succeedMap map[string][]string) (map[string][]string, []string) {
log := logf.FromContext(ctx)
sandboxByName := make(map[string]*sandboxv1alpha1.BatchSandbox, len(batchSandboxes))
for _, bs := range batchSandboxes {
sandboxByName[bs.Name] = bs
}
toSyncMap := make(map[string][]string, len(succeedMap))
orphanPods := make([]string, 0)
for sandboxName, succeedPods := range succeedMap {
if len(succeedPods) == 0 {
continue
}
sandbox, ok := sandboxByName[sandboxName]
if !ok {
// Orphan sandbox: deleted before recycle completed. Collect its pods for direct release.
log.Info("GC: sandbox not found for recycle result, collecting orphan pods", "sandbox", sandboxName, "pods", succeedPods)
orphanPods = append(orphanPods, succeedPods...)
continue
}
currentReleased, err := r.Allocator.GetSandboxReleased(ctx, sandbox)
if err != nil {
log.Error(err, "Failed to get sandbox released", "sandbox", sandboxName)
continue
}
toSyncMap[sandboxName] = append(currentReleased, succeedPods...)
}
return toSyncMap, orphanPods
}
func (r *PoolReconciler) scheduleSandbox(ctx context.Context, pool *sandboxv1alpha1.Pool, batchSandboxes []*sandboxv1alpha1.BatchSandbox, pods []*corev1.Pod) (*scheduleResult, error) {
log := logf.FromContext(ctx)
// 1. Compute scheduling actions.
spec := &allocSpec{
Sandboxes: batchSandboxes,
Pool: pool,
Pods: pods,
}
scheduleStart := time.Now()
allocAction, err := r.Allocator.Schedule(ctx, spec)
recordAllocatorScheduleDuration(ctx, pool, time.Since(scheduleStart), err)
if err != nil {
r.Recorder.Eventf(pool, corev1.EventTypeWarning, eventReasonAllocationFailed, "Failed to schedule sandboxes: %v", err)
return nil, err
}
log.Info("Allocate action", "pool", pool.Name, "toAllocate", allocAction.ToAllocate, "toRelease", allocAction.ToRelease)
// 2. Execute scheduling actions.
// 2.1 Execute ToAllocate / update in-memory store.
err = r.doAllocate(ctx, pool, batchSandboxes, pods, allocAction.ToAllocate)
if err != nil {
return nil, err
}
// Emit allocation events.
sandboxByName := make(map[string]*sandboxv1alpha1.BatchSandbox, len(batchSandboxes))
for _, bs := range batchSandboxes {
sandboxByName[bs.Name] = bs
}
for sandboxName, allocPods := range allocAction.ToAllocate {
if len(allocPods) == 0 {
continue
}
r.Recorder.Eventf(pool, corev1.EventTypeNormal, eventReasonAllocationSucceeded,
"Allocated %d pod(s) to sandbox %s: %v", len(allocPods), sandboxName, allocPods)
if sbx, ok := sandboxByName[sandboxName]; ok {
r.Recorder.Eventf(sbx, corev1.EventTypeNormal, eventReasonScheduled,
"Successfully assigned %d pod(s) from pool %s: %v", len(allocPods), pool.Name, allocPods)
}
}
// 2.2 Execute ToRelease / release in-memory store.
toDeletePods, err := r.doRelease(ctx, pool, batchSandboxes, pods, allocAction.ToRelease)
if err != nil {
return nil, err
}
// 3. Return schedule result
latestAllocation, err := r.Allocator.GetPoolAllocation(ctx, pool)
if err != nil {
return nil, err
}
idlePods := make([]string, 0)
for _, pod := range pods {
if _, ok := latestAllocation[pod.Name]; !ok {
idlePods = append(idlePods, pod.Name)
}
}
result := &scheduleResult{
LatestAllocation: latestAllocation,
IdlePods: idlePods,
ToDelete: toDeletePods,
SupplyCnt: allocAction.PodSupplement,
}
log.Info("Schedule result", "pool", pool.Name, "toDeletePods", toDeletePods, "supplyCnt", allocAction.PodSupplement)
return result, nil
}
func (r *PoolReconciler) updatePool(ctx context.Context, pool *sandboxv1alpha1.Pool, pods []*corev1.Pod, idlePods []string) (*updateResult, error) {
updateRevision, err := r.calculateRevision(pool)
if err != nil {
return nil, err
}
strategy := newPoolUpdateStrategy(pool)
result := strategy.Compute(ctx, updateRevision, pods, idlePods)
result.UpdateRevision = updateRevision
if len(result.ToDeletePods) > 0 {
r.Recorder.Eventf(pool, corev1.EventTypeNormal, eventReasonPodUpdated,
"Rolling update: deleting %d pod(s) for revision %s: %v", len(result.ToDeletePods), updateRevision, result.ToDeletePods)
}
return result, nil
}
type scaleArgs struct {
updateRevision string
pods []*corev1.Pod
totalPodCnt int32 // all pods including evicting ones, for PoolMax enforcement
allocatedCnt int32
supplyCnt int32 // to create
idlePods []string
toDeletePods []string
}
type scheduleResult struct {
// LatestAllocation is the most recent pod-to-sandbox allocation map.
LatestAllocation map[string]string
// IdlePods contains pods that are not currently allocated to any sandbox.
IdlePods []string
// ToDelete contains pods that the recycle handler has decided to delete
// (e.g. direct deletion or restart failure fallback).
ToDelete []string
// SupplyCnt is the number of additional pods the allocator needs but are not yet available.
SupplyCnt int32
}
type updateResult struct {
UpdateRevision string
IdlePods []string
ToDeletePods []string
// Supply Pods with update revision
SupplyUpdateRevision int32
}
func (r *PoolReconciler) scalePool(ctx context.Context, pool *sandboxv1alpha1.Pool, args *scaleArgs) (bool, error) {
log := logf.FromContext(ctx)
errs := make([]error, 0)
pods := args.pods
if satisfied, unsatisfiedDuration, dirtyPods := poolScaleExpectations.SatisfiedExpectations(controllerutils.GetControllerKey(pool)); !satisfied {
if unsatisfiedDuration >= expectations.ExpectationTimeout {
log.Info("Pool scale expectations timed out, clearing stale expectations",
"unsatisfiedDuration", unsatisfiedDuration, "dirtyPods", dirtyPods)
poolScaleExpectations.DeleteExpectations(controllerutils.GetControllerKey(pool))
} else {
log.Info("Pool scale is not ready, requeue", "unsatisfiedDuration", unsatisfiedDuration, "dirtyPods", dirtyPods)
return true, nil
}
}
schedulableCnt := int32(len(args.pods))
totalPodCnt := args.totalPodCnt
allocatedCnt := args.allocatedCnt
supplyCnt := args.supplyCnt
toDeletePods := args.toDeletePods
bufferCnt := r.countReadyIdlePods(pods, args.idlePods)
// Calculate desired buffer cnt.
desiredBufferCnt := desiredBufferCount(bufferCnt, pool.Spec.CapacitySpec.BufferMin, pool.Spec.CapacitySpec.BufferMax)
// Calculate desired schedulable cnt.
desiredSchedulableCnt := max(allocatedCnt+supplyCnt+desiredBufferCnt, pool.Spec.CapacitySpec.PoolMin)
// Enforce PoolMax: limit new pods based on total running pods (including evicting).
maxNewPods := max(pool.Spec.CapacitySpec.PoolMax-totalPodCnt, 0)
log.Info("Scale pool decision", "pool", pool.Name,
"totalPodCnt", totalPodCnt, "schedulableCnt", schedulableCnt,
"allocatedCnt", allocatedCnt, "bufferCnt", bufferCnt,
"desiredBufferCnt", desiredBufferCnt, "supplyCnt", supplyCnt,
"desiredSchedulableCnt", desiredSchedulableCnt, "maxNewPods", maxNewPods,
"toDeletePods", len(toDeletePods), "idlePods", len(args.idlePods))
// Scale-up: create new pods if needed and allowed by PoolMax
if desiredSchedulableCnt > schedulableCnt && maxNewPods > 0 {
createCnt := min(desiredSchedulableCnt-schedulableCnt, maxNewPods)
scaleMaxUnavailable := r.getScaleMaxUnavailable(pool, desiredSchedulableCnt)
notReadyCnt := r.countNotReadyPods(pods)
limitedCreateCnt := scaleMaxUnavailable - notReadyCnt
createCnt = max(0, min(createCnt, limitedCreateCnt))
if createCnt > 0 {
log.Info("Scaling up pool with constraint", "pool", pool.Name,
"createCnt", createCnt, "scaleMaxUnavailable", scaleMaxUnavailable,
"notReadyCnt", notReadyCnt, "desiredSchedulableCnt", desiredSchedulableCnt, "limitedCreateCnt", limitedCreateCnt)
for range createCnt {
if err := r.createPoolPod(ctx, pool, args.updateRevision); err != nil {
log.Error(err, "Failed to create pool pod")
errs = append(errs, err)
}
}
}
}
// Scale-down: delete redundant or excess pods
scaleIn := int32(0)
if desiredSchedulableCnt < schedulableCnt {
scaleIn = schedulableCnt - desiredSchedulableCnt
}
if scaleIn > 0 || len(toDeletePods) > 0 {
podsToDelete := r.pickPodsToDelete(pods, args.idlePods, args.toDeletePods, scaleIn)
maxDeleteCnt := r.getScaleMaxUnavailable(pool, desiredSchedulableCnt)
if int32(len(podsToDelete)) > maxDeleteCnt {
podsToDelete = podsToDelete[:maxDeleteCnt]
}
log.Info("Scaling down pool", "pool", pool.Name, "scaleIn", scaleIn, "toDeletePods", len(toDeletePods), "podsToDelete", len(podsToDelete))
for _, pod := range podsToDelete {
log.Info("Deleting pool pod", "pool", pool.Name, "pod", pod.Name)
controllerKey := controllerutils.GetControllerKey(pool)
poolScaleExpectations.ExpectScale(controllerKey, expectations.Delete, pod.Name)
if err := r.Delete(ctx, pod); err != nil {
poolScaleExpectations.ObserveScale(controllerKey, expectations.Delete, pod.Name)
if errors.IsNotFound(err) {
continue
}
log.Error(err, "Failed to delete pool pod", "pod", pod.Name)
r.Recorder.Eventf(pool, corev1.EventTypeWarning, eventReasonFailedDelete, "Failed to delete pool pod %s: %v", pod.Name, err)
errs = append(errs, err)
} else {
r.Recorder.Eventf(pool, corev1.EventTypeNormal, eventReasonSuccessfulDelete, "Deleted pool pod %s (scale-down)", pod.Name)
}
}
}
return false, gerrors.Join(errs...)
}
func (r *PoolReconciler) updatePoolStatus(ctx context.Context, updateRevision string, pool *sandboxv1alpha1.Pool, pods []*corev1.Pod, schedulePods []*corev1.Pod, podAllocation map[string]string) error {
oldStatus := pool.Status.DeepCopy()
availableCnt := int32(0)
for _, pod := range schedulePods {
if _, ok := podAllocation[pod.Name]; ok {
continue
}
if !utils.IsPodReady(pod) {
continue
}
availableCnt++
}
updatedCnt := int32(0)
for _, pod := range pods {
if pod.Labels[labelPoolRevision] == updateRevision {
updatedCnt++
}
}
pool.Status.ObservedGeneration = pool.Generation
pool.Status.Total = int32(len(pods))
pool.Status.Allocated = int32(len(podAllocation))
pool.Status.Available = availableCnt
pool.Status.Revision = updateRevision
pool.Status.Updated = updatedCnt
if equality.Semantic.DeepEqual(*oldStatus, pool.Status) {
return nil
}
log := logf.FromContext(ctx)
log.Info("Update pool status", "ObservedGeneration", pool.Status.ObservedGeneration, "Total", pool.Status.Total,
"Allocated", pool.Status.Allocated, "Available", pool.Status.Available, "Revision", pool.Status.Revision, "Updated", pool.Status.Updated)
if err := r.Status().Update(ctx, pool); err != nil {
return err
}
return nil
}
func (r *PoolReconciler) pickPodsToDelete(pods []*corev1.Pod, idlePodNames []string, toDeletePodNames []string, scaleIn int32) []*corev1.Pod {
podMap := make(map[string]*corev1.Pod)
for _, pod := range pods {
podMap[pod.Name] = pod
}
selected := make(map[string]struct{})
var podsToDelete []*corev1.Pod
for _, name := range toDeletePodNames {
pod, ok := podMap[name]
if !ok || !pod.DeletionTimestamp.IsZero() {
continue
}
if _, exists := selected[name]; exists {
continue
}
podsToDelete = append(podsToDelete, pod)
selected[name] = struct{}{}
}
var idlePods []*corev1.Pod
for _, name := range idlePodNames {
pod, ok := podMap[name]
if !ok || !pod.DeletionTimestamp.IsZero() {
continue
}
idlePods = append(idlePods, pod)
}
sort.Slice(idlePods, func(i, j int) bool {
iReady := utils.IsPodReady(idlePods[i])
jReady := utils.IsPodReady(idlePods[j])
if iReady != jReady {
return !iReady
}
if idlePods[i].CreationTimestamp.Equal(&idlePods[j].CreationTimestamp) {
return idlePods[i].Name < idlePods[j].Name
}
return idlePods[i].CreationTimestamp.After(idlePods[j].CreationTimestamp.Time)
})
for _, pod := range idlePods {
if scaleIn <= 0 {
break
}
if _, exists := selected[pod.Name]; exists {
continue
}
podsToDelete = append(podsToDelete, pod)
selected[pod.Name] = struct{}{}
scaleIn -= 1
}
return podsToDelete
}
func desiredBufferCount(readyBufferCnt, bufferMin, bufferMax int32) int32 {
if readyBufferCnt < bufferMin {
return bufferMin
}
if readyBufferCnt > bufferMax {
return bufferMax
}
return readyBufferCnt
}
func (r *PoolReconciler) countReadyIdlePods(pods []*corev1.Pod, idlePodNames []string) int32 {
idleNames := make(map[string]struct{}, len(idlePodNames))
for _, name := range idlePodNames {
idleNames[name] = struct{}{}
}
var count int32
for _, pod := range pods {
if _, idle := idleNames[pod.Name]; idle && utils.IsPodReady(pod) {
count++
}
}
return count
}
// getScaleMaxUnavailable returns the resolved maxUnavailable value.
// If not specified, defaults to 25% of desiredTotal.
// Minimum return value is 1 to ensure scaling progress.
func (r *PoolReconciler) getScaleMaxUnavailable(pool *sandboxv1alpha1.Pool, desiredTotal int32) int32 {
defaultPercentage := intstr.FromString("25%")
maxUnavailable := &defaultPercentage
if pool.Spec.ScaleStrategy != nil && pool.Spec.ScaleStrategy.MaxUnavailable != nil {
maxUnavailable = pool.Spec.ScaleStrategy.MaxUnavailable
}
result, err := intstr.GetScaledValueFromIntOrPercent(maxUnavailable, int(desiredTotal), true)
if err != nil || result < 1 {
result = 1
}
return int32(result)
}
// countNotReadyPods returns the count of pods that are not ready.
// A pod is considered not ready if it doesn't have a Ready condition
// with status True.
func (r *PoolReconciler) countNotReadyPods(pods []*corev1.Pod) int32 {
var count int32
for _, pod := range pods {
if !utils.IsPodReady(pod) {
count++
}
}
return count
}
func (r *PoolReconciler) createPoolPod(ctx context.Context, pool *sandboxv1alpha1.Pool, updateRevision string) error {
log := logf.FromContext(ctx)
pod, err := utils.GetPodFromTemplate(pool.Spec.Template, pool, metav1.NewControllerRef(pool, sandboxv1alpha1.SchemeBuilder.GroupVersion.WithKind("Pool")))
if err != nil {
return err
}
pod.Namespace = pool.Namespace
pod.Name = ""
pod.GenerateName = pool.Name + "-"
pod.Labels[labelPoolName] = pool.Name
pod.Labels[labelPoolRevision] = updateRevision
if err := ctrl.SetControllerReference(pool, pod, r.Scheme); err != nil {
return err
}
if err := r.Create(ctx, pod); err != nil {
r.Recorder.Eventf(pool, corev1.EventTypeWarning, eventReasonFailedCreate, "Failed to create pool pod: %v", err)
return err
}
poolScaleExpectations.ExpectScale(controllerutils.GetControllerKey(pool), expectations.Create, pod.Name)
log.Info("Created pool pod", "pool", pool.Name, "pod", pod.Name, "revision", updateRevision)
r.Recorder.Eventf(pool, corev1.EventTypeNormal, eventReasonSuccessfulCreate, "Created pool pod: %v", pod.Name)
return nil
}
// handleEviction fetches the current allocation, evicts idle pods marked for eviction,
// and returns the schedulable pods (excluding evicting idle pods) along with any eviction error.
// Eviction errors are non-fatal: they are returned to trigger a requeue but do not block the current reconcile.
func (r *PoolReconciler) handleEviction(ctx context.Context, pool *sandboxv1alpha1.Pool, pods []*corev1.Pod) ([]*corev1.Pod, error) {
log := logf.FromContext(ctx)
podAllocation, err := r.Allocator.GetPoolAllocation(ctx, pool)
if err != nil {
log.Error(err, "Failed to get pool allocation")
return nil, err
}
handler := eviction.NewEvictionHandler(ctx, r.Client, pool)
var evictionErrs []error
filtered := make([]*corev1.Pod, 0, len(pods))
for _, pod := range pods {
if !handler.NeedsEviction(pod) {
filtered = append(filtered, pod)
continue
}
if sandboxName, allocated := podAllocation[pod.Name]; allocated {
log.V(1).Info("Skipping eviction for allocated pod", "pod", pod.Name, "sandbox", sandboxName)
filtered = append(filtered, pod)
continue
}
// Idle pod marked for eviction: evict and exclude from scheduling
log.Info("Evicting idle pool pod", "pool", pool.Name, "pod", pod.Name)
if err := handler.Evict(ctx, pod); err != nil {
log.Error(err, "Failed to evict pod", "pod", pod.Name)
evictionErrs = append(evictionErrs, fmt.Errorf("failed to evict pod %s: %w", pod.Name, err))
} else {
r.Recorder.Eventf(pool, corev1.EventTypeNormal, eventReasonPodEvicted, "Evicted idle pod: %s", pod.Name)
}
}
return filtered, gerrors.Join(evictionErrs...)
}