Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: prevent invalidated fetches from repopulating caches (#3229)
Kyle Mendell05551e86c40f
diff --git a/backend/internal/common/errors.go b/backend/internal/common/errors.goindex 451ecfd66..1a3a46441 100644--- a/backend/internal/common/errors.go+++ b/backend/internal/common/errors.go@@ -1937,6 +1937,18 @@ func (e *DefaultTransportTypeError) Error() string {return "http.DefaultTransport is not *http.Transport"}+// CacheInvalidatedDuringFetchError restarts a cache lookup when invalidation+// makes an in-flight fetch ineligible to publish its result.+type CacheInvalidatedDuringFetchError struct{}++func (e *CacheInvalidatedDuringFetchError) Error() string {+ return "cache invalidated during fetch"+}++func IsCacheInvalidatedDuringFetchError(err error) bool {+ return isErrorTypeInternal[*CacheInvalidatedDuringFetchError](err)+}+// ManagerCALockTypeError is returned when a cached manager CA lock value is not// the expected *sync.Mutex.type ManagerCALockTypeError struct{}diff --git a/backend/pkg/utils/cache/cache_util.go b/backend/pkg/utils/cache/cache_util.goindex 54fe7e5bb..972aa2705 100644--- a/backend/pkg/utils/cache/cache_util.go+++ b/backend/pkg/utils/cache/cache_util.go@@ -6,6 +6,7 @@ import ("sync""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""golang.org/x/sync/singleflight")@@ -19,18 +20,28 @@ func (e *StaleError) Unwrap() error { return e.Err }type Cache[T any] struct {ttl time.Duration- mu sync.RWMutex- val T- exp time.Time- set bool+ mu sync.RWMutex+ val T+ exp time.Time+ set bool+ generation uint64sf singleflight.Group}+type cacheSnapshotInternal[T any] struct {+ value T+ set bool+ fresh bool+ generation uint64+}+type KeyedCache[K comparable, T any] struct {- mu sync.RWMutex- entries map[K]T- sf singleflight.Group+ mu sync.RWMutex+ entries map[K]T+ keyGenerations map[K]uint64+ allGeneration uint64+ sf singleflight.Group}func New[T any](ttl time.Duration) *Cache[T] {@@ -39,10 +50,32 @@ func New[T any](ttl time.Duration) *Cache[T] {func NewKeyed[K comparable, T any]() *KeyedCache[K, T] {return &KeyedCache[K, T]{- entries: make(map[K]T),+ entries: make(map[K]T),+ keyGenerations: make(map[K]uint64),+ }+}++func (c *Cache[T]) snapshotInternal() cacheSnapshotInternal[T] {+ c.mu.RLock()+ defer c.mu.RUnlock()+ return cacheSnapshotInternal[T]{+ value: c.val,+ set: c.set,+ fresh: c.set && (c.ttl <= 0 || time.Now().Before(c.exp)),+ generation: c.generation,}}+func retryInvalidatedFetchInternal(ctx context.Context, err error) (bool, error) {+ if !common.IsCacheInvalidatedDuringFetchError(err) {+ return false, err+ }+ if ctxErr := ctx.Err(); ctxErr != nil {+ return false, ctxErr+ }+ return true, nil+}+// Get returns the cached value for key, if present.func (c *KeyedCache[K, T]) Get(key K) (T, bool) {c.mu.RLock()@@ -64,94 +97,131 @@ func (c *KeyedCache[K, T]) GetOrFetch(valid func(cached T) bool,fetch func(ctx context.Context) (T, error),) (T, error) {- c.mu.RLock()- cached, ok := c.entries[key]- c.mu.RUnlock()- if ok && (valid == nil || valid(cached)) {- return cached, nil- }-- res, err, _ := c.sf.Do(fmt.Sprint(key), func() (any, error) {+ for {c.mu.RLock()cached, ok := c.entries[key]+ allGeneration := c.allGeneration+ keyGeneration := c.keyGenerations[key]c.mu.RUnlock()if ok && (valid == nil || valid(cached)) {return cached, nil}- v, err := fetch(ctx)+ flightKey := fmt.Sprintf("%#v:%d:%d", key, allGeneration, keyGeneration)+ res, err, _ := c.sf.Do(flightKey, func() (any, error) {+ c.mu.RLock()+ cached, ok := c.entries[key]+ generationCurrent := c.allGeneration == allGeneration && c.keyGenerations[key] == keyGeneration+ c.mu.RUnlock()+ if !generationCurrent {+ return nil, &common.CacheInvalidatedDuringFetchError{}+ }+ if ok && (valid == nil || valid(cached)) {+ return cached, nil+ }++ v, err := fetch(ctx)+ if err != nil {+ return nil, err+ }++ c.mu.Lock()+ generationCurrent = c.allGeneration == allGeneration && c.keyGenerations[key] == keyGeneration+ if generationCurrent {+ c.entries[key] = v+ }+ c.mu.Unlock()+ if !generationCurrent {+ return nil, &common.CacheInvalidatedDuringFetchError{}+ }+ return v, nil+ })+ retry, err := retryInvalidatedFetchInternal(ctx, err)+ if retry {+ continue+ }if err != nil {var zero Treturn zero, err}- c.mu.Lock()- c.entries[key] = v- c.mu.Unlock()+ v, _ := res.(T)return v, nil- })- if err != nil {- var zero T- return zero, err}-- v, _ := res.(T)- return v, nil}func (c *KeyedCache[K, T]) Invalidate(key K) {c.mu.Lock()delete(c.entries, key)+ c.keyGenerations[key]++c.mu.Unlock()}func (c *KeyedCache[K, T]) InvalidateAll() {c.mu.Lock()c.entries = make(map[K]T)+ c.keyGenerations = make(map[K]uint64)+ c.allGeneration++c.mu.Unlock()}func (c *Cache[T]) GetOrFetch(ctx context.Context, fetch func(ctx context.Context) (T, error)) (T, error) {- c.mu.RLock()- if c.set && (c.ttl <= 0 || time.Now().Before(c.exp)) {- v := c.val- c.mu.RUnlock()- return v, nil- }-- hasStale := c.set- stale := c.val- c.mu.RUnlock()+ for {+ snapshot := c.snapshotInternal()+ if snapshot.fresh {+ return snapshot.value, nil+ }- res, err, _ := c.sf.Do("singleton", func() (any, error) {- v, err := fetch(ctx)+ res, err, _ := c.sf.Do(fmt.Sprintf("singleton:%d", snapshot.generation), func() (any, error) {+ current := c.snapshotInternal()+ if current.fresh {+ return current.value, nil+ }+ if current.generation != snapshot.generation {+ return nil, &common.CacheInvalidatedDuringFetchError{}+ }++ v, err := fetch(ctx)+ if err != nil {+ return nil, err+ }++ c.mu.Lock()+ generationCurrent := c.generation == snapshot.generation+ if generationCurrent {+ c.val = v+ c.set = true+ if c.ttl > 0 {+ c.exp = time.Now().Add(c.ttl)+ }+ }+ c.mu.Unlock()+ if !generationCurrent {+ return nil, &common.CacheInvalidatedDuringFetchError{}+ }+ return v, nil+ })+ retry, err := retryInvalidatedFetchInternal(ctx, err)+ if retry {+ continue+ }if err != nil {- return nil, err+ current := c.snapshotInternal()+ if snapshot.set && current.generation == snapshot.generation {+ return snapshot.value, &StaleError{Err: err}+ }+ var zero T+ return zero, err}- c.mu.Lock()- c.val = v- c.set = true- if c.ttl > 0 {- c.exp = time.Now().Add(c.ttl)- }- c.mu.Unlock()+ v, _ := res.(T)return v, nil- })- if err != nil {- if hasStale {- return stale, &StaleError{Err: err}- }- var zero T- return zero, err}-- v, _ := res.(T)- return v, nil}func (c *Cache[T]) Invalidate() {c.mu.Lock()+ c.generation++c.set = falsevar zero Tc.val = zerodiff --git a/backend/pkg/utils/cache/cache_util_test.go b/backend/pkg/utils/cache/cache_util_test.goindex 349b1dc3d..4c119eb5e 100644--- a/backend/pkg/utils/cache/cache_util_test.go+++ b/backend/pkg/utils/cache/cache_util_test.go@@ -1,6 +1,14 @@package cache-import "testing"+import (+ "context"+ "errors"+ "fmt"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)func TestKeyedCacheGetSet(t *testing.T) {c := NewKeyed[string, *int]()@@ -27,3 +35,386 @@ func TestKeyedCacheGetSet(t *testing.T) {t.Fatal("expected miss after invalidate")}}++func TestCacheInvalidationDoesNotPublishInFlightFetch(t *testing.T) {+ c := New[string](time.Minute)+ firstFetchStarted := make(chan struct{})+ releaseFirstFetch := make(chan struct{})+ var fetchCalls atomic.Int32++ fetch := func(ctx context.Context) (string, error) {+ switch fetchCalls.Add(1) {+ case 1:+ close(firstFetchStarted)+ select {+ case <-releaseFirstFetch:+ return "obsolete", nil+ case <-ctx.Done():+ return "", ctx.Err()+ }+ case 2:+ return "fresh", nil+ default:+ return "", fmt.Errorf("unexpected fetch call")+ }+ }++ type result struct {+ value string+ err error+ }+ firstResult := make(chan result, 1)+ go func() {+ value, err := c.GetOrFetch(context.Background(), fetch)+ firstResult <- result{value: value, err: err}+ }()++ select {+ case <-firstFetchStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("first fetch did not start")+ }++ c.Invalidate()+ currentResult := make(chan result, 1)+ go func() {+ value, err := c.GetOrFetch(context.Background(), fetch)+ currentResult <- result{value: value, err: err}+ }()++ var current result+ select {+ case current = <-currentResult:+ case <-time.After(5 * time.Second):+ t.Fatal("post-invalidation fetch waited for the obsolete singleflight call")+ }+ if current.err != nil || current.value != "fresh" {+ t.Fatalf("expected fresh post-invalidation result, got %q (err=%v)", current.value, current.err)+ }++ close(releaseFirstFetch)+ var first result+ select {+ case first = <-firstResult:+ case <-time.After(5 * time.Second):+ t.Fatal("first caller did not finish")+ }+ if first.err != nil || first.value != "fresh" {+ t.Fatalf("expected first caller to observe the current generation, got %q (err=%v)", first.value, first.err)+ }+ if got := fetchCalls.Load(); got != 2 {+ t.Fatalf("expected two generation-specific fetches, got %d", got)+ }++ value, err := c.GetOrFetch(context.Background(), fetch)+ if err != nil || value != "fresh" {+ t.Fatalf("expected cache to retain fresh value, got %q (err=%v)", value, err)+ }+}++func TestCacheCanceledCallerDoesNotRetryAfterInvalidation(t *testing.T) {+ c := New[string](time.Minute)+ ctx, cancel := context.WithCancel(context.Background())+ fetchStarted := make(chan struct{})+ releaseFetch := make(chan struct{})+ var fetchCalls atomic.Int32++ resultChannel := make(chan error, 1)+ go func() {+ _, err := c.GetOrFetch(ctx, func(context.Context) (string, error) {+ if fetchCalls.Add(1) == 1 {+ close(fetchStarted)+ <-releaseFetch+ return "obsolete", nil+ }+ return "retry", nil+ })+ resultChannel <- err+ }()++ select {+ case <-fetchStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("singleton fetch did not start")+ }++ c.Invalidate()+ cancel()+ close(releaseFetch)++ select {+ case err := <-resultChannel:+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("expected canceled caller to return context cancellation, got %v", err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("canceled singleton caller did not finish")+ }+ if got := fetchCalls.Load(); got != 1 {+ t.Fatalf("expected canceled singleton caller not to retry, got %d fetches", got)+ }+}++func TestCacheConcurrentMissesUseSingleflight(t *testing.T) {+ const callerCount = 32++ c := New[int](time.Minute)+ start := make(chan struct{})+ fetchStarted := make(chan struct{})+ releaseFetch := make(chan struct{})+ var fetchStartedOnce sync.Once+ var fetchCalls atomic.Int32+ var callersReady sync.WaitGroup+ var callersDone sync.WaitGroup+ callersReady.Add(callerCount)+ callersDone.Add(callerCount)++ type result struct {+ value int+ err error+ }+ results := make(chan result, callerCount)+ for range callerCount {+ go func() {+ defer callersDone.Done()+ callersReady.Done()+ <-start+ value, err := c.GetOrFetch(context.Background(), func(context.Context) (int, error) {+ fetchCalls.Add(1)+ fetchStartedOnce.Do(func() { close(fetchStarted) })+ <-releaseFetch+ return 42, nil+ })+ results <- result{value: value, err: err}+ }()+ }++ callersReady.Wait()+ close(start)+ select {+ case <-fetchStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("singleflight fetch did not start")+ }+ close(releaseFetch)+ callersDone.Wait()+ close(results)++ for result := range results {+ if result.err != nil || result.value != 42 {+ t.Fatalf("expected shared value 42, got %d (err=%v)", result.value, result.err)+ }+ }+ if got := fetchCalls.Load(); got != 1 {+ t.Fatalf("expected one shared fetch, got %d", got)+ }+}++func TestKeyedCacheInvalidationDoesNotPublishInFlightFetch(t *testing.T) {+ tests := []struct {+ name string+ invalidateAll bool+ otherEntryPresent bool+ }{+ {name: "key", otherEntryPresent: true},+ {name: "all", invalidateAll: true},+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ c := NewKeyed[string, string]()+ c.Set("other", "preserved")+ firstFetchStarted := make(chan struct{})+ releaseFirstFetch := make(chan struct{})+ var fetchCalls atomic.Int32++ fetch := func(ctx context.Context) (string, error) {+ switch fetchCalls.Add(1) {+ case 1:+ close(firstFetchStarted)+ select {+ case <-releaseFirstFetch:+ return "obsolete", nil+ case <-ctx.Done():+ return "", ctx.Err()+ }+ case 2:+ return "fresh", nil+ default:+ return "", fmt.Errorf("unexpected fetch call")+ }+ }++ type result struct {+ value string+ err error+ }+ firstResult := make(chan result, 1)+ go func() {+ value, err := c.GetOrFetch(context.Background(), "target", nil, fetch)+ firstResult <- result{value: value, err: err}+ }()++ select {+ case <-firstFetchStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("first keyed fetch did not start")+ }++ if test.invalidateAll {+ c.InvalidateAll()+ } else {+ c.Invalidate("target")+ }++ currentResult := make(chan result, 1)+ go func() {+ value, err := c.GetOrFetch(context.Background(), "target", nil, fetch)+ currentResult <- result{value: value, err: err}+ }()++ var current result+ select {+ case current = <-currentResult:+ case <-time.After(5 * time.Second):+ t.Fatal("post-invalidation keyed fetch waited for the obsolete singleflight call")+ }+ if current.err != nil || current.value != "fresh" {+ t.Fatalf("expected fresh keyed result, got %q (err=%v)", current.value, current.err)+ }++ close(releaseFirstFetch)+ var first result+ select {+ case first = <-firstResult:+ case <-time.After(5 * time.Second):+ t.Fatal("first keyed caller did not finish")+ }+ if first.err != nil || first.value != "fresh" {+ t.Fatalf("expected first keyed caller to observe the current generation, got %q (err=%v)", first.value, first.err)+ }+ if got := fetchCalls.Load(); got != 2 {+ t.Fatalf("expected two generation-specific keyed fetches, got %d", got)+ }++ value, ok := c.Get("target")+ if !ok || value != "fresh" {+ t.Fatalf("expected keyed cache to retain fresh value, got %q (ok=%v)", value, ok)+ }+ _, otherPresent := c.Get("other")+ if otherPresent != test.otherEntryPresent {+ t.Fatalf("unexpected unrelated entry state: got present=%v, want %v", otherPresent, test.otherEntryPresent)+ }+ })+ }+}++func TestKeyedCacheCanceledCallerDoesNotRetryAfterInvalidation(t *testing.T) {+ c := NewKeyed[string, string]()+ ctx, cancel := context.WithCancel(context.Background())+ fetchStarted := make(chan struct{})+ releaseFetch := make(chan struct{})+ var fetchCalls atomic.Int32++ resultChannel := make(chan error, 1)+ go func() {+ _, err := c.GetOrFetch(ctx, "target", nil, func(context.Context) (string, error) {+ if fetchCalls.Add(1) == 1 {+ close(fetchStarted)+ <-releaseFetch+ return "obsolete", nil+ }+ return "retry", nil+ })+ resultChannel <- err+ }()++ select {+ case <-fetchStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("keyed fetch did not start")+ }++ c.Invalidate("target")+ cancel()+ close(releaseFetch)++ select {+ case err := <-resultChannel:+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("expected canceled caller to return context cancellation, got %v", err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("canceled keyed caller did not finish")+ }+ if got := fetchCalls.Load(); got != 1 {+ t.Fatalf("expected canceled keyed caller not to retry, got %d fetches", got)+ }+}++func TestCacheStaleFallbackRequiresCurrentGeneration(t *testing.T) {+ upstreamErr := errors.New("upstream unavailable")++ t.Run("current generation", func(t *testing.T) {+ c := New[string](time.Minute)+ c.val = "stale"+ c.set = true+ c.exp = time.Now().Add(-time.Minute)++ value, err := c.GetOrFetch(context.Background(), func(context.Context) (string, error) {+ return "", upstreamErr+ })+ if value != "stale" {+ t.Fatalf("expected stale fallback, got %q", value)+ }+ var staleErr *StaleError+ if !errors.As(err, &staleErr) || !errors.Is(err, upstreamErr) {+ t.Fatalf("expected stale error wrapping upstream failure, got %v", err)+ }+ })++ t.Run("invalidated generation", func(t *testing.T) {+ c := New[string](time.Minute)+ c.val = "stale"+ c.set = true+ c.exp = time.Now().Add(-time.Minute)+ fetchStarted := make(chan struct{})+ releaseFetch := make(chan struct{})++ type result struct {+ value string+ err error+ }+ resultChannel := make(chan result, 1)+ go func() {+ value, err := c.GetOrFetch(context.Background(), func(context.Context) (string, error) {+ close(fetchStarted)+ <-releaseFetch+ return "", upstreamErr+ })+ resultChannel <- result{value: value, err: err}+ }()++ select {+ case <-fetchStarted:+ case <-time.After(5 * time.Second):+ t.Fatal("stale fallback fetch did not start")+ }+ c.Invalidate()+ close(releaseFetch)++ var got result+ select {+ case got = <-resultChannel:+ case <-time.After(5 * time.Second):+ t.Fatal("invalidated stale fallback did not finish")+ }+ if got.value != "" || !errors.Is(got.err, upstreamErr) {+ t.Fatalf("expected upstream error without stale data, got %q (err=%v)", got.value, got.err)+ }+ var staleErr *StaleError+ if errors.As(got.err, &staleErr) {+ t.Fatalf("invalidated stale generation must not be returned: %v", got.err)+ }+ })+}