Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(models): prevent heartbeats from restoring released concurrency leases (#4755)
ydflowaa58192f8104
diff --git a/internal/models/limiter/heartbeat_test.go b/internal/models/limiter/heartbeat_test.gonew file mode 100644index 000000000..1d79b5b2e--- /dev/null+++ b/internal/models/limiter/heartbeat_test.go@@ -0,0 +1,115 @@+package limiter++import (+ "context"+ "sync"+ "testing"+ "time"++ "github.com/redis/go-redis/v9"+)++// Pause a heartbeat before it reaches Redis so release can remove its lease+// first. This reproduces the command ordering without relying on a race.+type heartbeatHook struct {+ started chan struct{}+ resume chan struct{}+ finished chan struct{}+ once sync.Once+}++func (h *heartbeatHook) DialHook(next redis.DialHook) redis.DialHook { return next }++func (h *heartbeatHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {+ return next+}++func (h *heartbeatHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {+ return func(ctx context.Context, cmd redis.Cmder) error {+ paused := false+ if cmd.Name() == "zadd" {+ h.once.Do(func() {+ paused = true+ close(h.started)+ <-h.resume+ })+ }+ err := next(ctx, cmd)+ if paused {+ close(h.finished)+ }+ return err+ }+}++func TestRedisLimiterHeartbeatDoesNotRestoreReleasedLease(t *testing.T) {+ lim, _, rdb := newTestLimiter(t, 300*time.Millisecond, 10*time.Millisecond)+ hook := &heartbeatHook{+ started: make(chan struct{}), resume: make(chan struct{}), finished: make(chan struct{}),+ }+ rdb.AddHook(hook)+ var resumeOnce sync.Once+ resume := func() { resumeOnce.Do(func() { close(hook.resume) }) }+ t.Cleanup(resume)++ const key = "released-heartbeat"+ release, err := lim.Acquire(context.Background(), key, 1)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(release)+ select {+ case <-hook.started:+ case <-time.After(2 * time.Second):+ t.Fatal("heartbeat did not start")+ }++ release()+ if got := zcard(t, rdb, key); got != 0 {+ t.Fatalf("holders after release = %d, want 0", got)+ }+ resume()+ select {+ case <-hook.finished:+ case <-time.After(2 * time.Second):+ t.Fatal("heartbeat did not finish")+ }+ if got := zcard(t, rdb, key); got != 0 {+ t.Fatalf("late heartbeat restored a released lease: holders = %d, want 0", got)+ }+}++func TestRedisLimiterHeartbeatRenewsLiveLease(t *testing.T) {+ lim, server, rdb := newTestLimiter(t, 300*time.Millisecond, 10*time.Millisecond)+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)+ defer cancel()+ const key = "live-heartbeat"+ release, err := lim.Acquire(ctx, key, 1)+ if err != nil {+ t.Fatal(err)+ }+ defer release()+ initial, err := rdb.ZRangeWithScores(ctx, keyPrefix+key, 0, -1).Result()+ if err != nil || len(initial) != 1 {+ t.Fatalf("initial leases = %v, err = %v", initial, err)+ }+ // Advance the Redis key TTL without changing the wall clock used for lease+ // scores. A live heartbeat must renew both independently.+ server.FastForward(lim.ttl)+ tick := time.NewTicker(10 * time.Millisecond)+ defer tick.Stop()+ for {+ select {+ case <-ctx.Done():+ t.Fatal("heartbeat did not renew the live lease and key TTL")+ case <-tick.C:+ score, err := rdb.ZScore(ctx, keyPrefix+key, initial[0].Member.(string)).Result()+ if err != nil {+ t.Fatal(err)+ }+ if score > initial[0].Score && server.TTL(keyPrefix+key) > lim.ttl {+ return+ }+ }+ }+}diff --git a/internal/models/limiter/limiter.go b/internal/models/limiter/limiter.goindex b475ee0aa..3507c7971 100644--- a/internal/models/limiter/limiter.go+++ b/internal/models/limiter/limiter.go@@ -220,7 +220,9 @@ func (l *redisLimiter) hold(zkey, token string) func() {// dropping every live lease and admitting over the limit. The// heartbeat pushes the key TTL out in lockstep with the lease.bg := context.Background()- _ = l.rdb.ZAdd(bg, zkey, redis.Z{+ // A release or expiry may remove the token while this heartbeat+ // is in flight. Refresh existing leases only; never recreate one.+ _ = l.rdb.ZAddXX(bg, zkey, redis.Z{Score: float64(now + l.ttl.Milliseconds()),Member: token,}).Err()