RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): unify memory task execution with heartbeat leases (#19189)
jay77721363f77ef80d5
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.goindex 9117bb19e2..7fceec5727 100644--- a/internal/engine/nats/nats.go+++ b/internal/engine/nats/nats.go@@ -120,7 +120,8 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {// by any consumer and un-reparsable ("already exists, status: SCHEDULED").// Duplicate delivery is instead made safe at the consumer level:// StartRunning's CREATED/SCHEDULED→RUNNING CAS plus the in-process claim guard- // ack-skip any second copy (see Ingestor.processMessage).+ // prevent a second copy from executing while the first owner is active (see+ // Ingestor.processMessage).ack, err := n.jetStream.Publish(ctx, subject, payload)if err != nil {return err@@ -225,8 +226,7 @@ func (n *NatsEngine) InitConsumer(subject string) error {// 60s AckWait is the effective schedule if BackOff is ever dropped.// INVARIANT: the worker's InProgress heartbeat (Ingestor// defaultHeartbeatInterval) must stay below BackOff[0] = 5s, or in-flight- // messages get redelivered mid-run and the claim-guard ack-skip acks the- // only live copy.+ // messages get redelivered mid-run before the owning worker can settle them.// Note: CreateOrUpdateConsumer is atomic. MaxWaiting is immutable after// creation; if it mismatches the whole update fails (error "max waiting// can not be updated") and NONE of AckWait/BackOff/MaxAckPending arediff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.goindex 9a4be0556a..ecb21e6806 100644--- a/internal/ingestion/service/execute_memory_task_panic_test.go+++ b/internal/ingestion/service/execute_memory_task_panic_test.go@@ -35,13 +35,13 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))// Inject a panicking memory runner through the same seam used in production// (defaultRunMemoryTask calls memorySvc.HandleSaveToMemoryTask).- ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {panic("simulated memory extraction panic")}handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-1", TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{- "id": "mem-panic-1", "task_type": "memory", "memory_id": "mem-p", "source_id": 1,+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-1", map[string]any{+ "memory_id": "mem-p", "source_id": 1,"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},}, handle)@@ -82,7 +82,7 @@ func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {ingestor := NewIngestor("test", 1, []string{"pdf"})ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))- ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {panic("simulated memory extraction panic")}ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {@@ -90,8 +90,8 @@ func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {}memHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-2", TaskType: common.TaskTypeMemory}}- memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{- "id": "mem-panic-2", "task_type": "memory", "memory_id": "mem-p2", "source_id": 1,+ memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-2", map[string]any{+ "memory_id": "mem-p2", "source_id": 1,"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},}, memHandle)diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.goindex 3cd8232f54..e658601591 100644--- a/internal/ingestion/service/execute_task_ack_test.go+++ b/internal/ingestion/service/execute_task_ack_test.go@@ -3,6 +3,7 @@ package serviceimport ("context""errors"+ "sync""sync/atomic""testing""time"@@ -19,17 +20,62 @@ import (// counters are atomic because executeTask drives them from multiple goroutines// (the heartbeat goroutine calls InProgress; the defer calls Ack/Nack) while the// test goroutine reads them - plain ints would be a data race under -race.+//+// settledWithInProgress is set when Ack or Nack runs while an InProgress is+// still in flight on the same message — a violation of Heartbeat's contract+// that the heartbeat must be stopped and waited on before settling.type fakeTaskHandle struct {msg common.TaskMessageacks atomic.Int64nacks atomic.Int64inProgress atomic.Int64++ mu sync.Mutex+ inProgressActive bool+ settledWithInProgress bool}func (f *fakeTaskHandle) GetMessage() common.TaskMessage { return f.msg }-func (f *fakeTaskHandle) Ack() error { f.acks.Add(1); return nil }-func (f *fakeTaskHandle) Nack() error { f.nacks.Add(1); return nil }-func (f *fakeTaskHandle) InProgress() error { f.inProgress.Add(1); return nil }++func (f *fakeTaskHandle) Ack() error {+ f.mu.Lock()+ if f.inProgressActive {+ f.settledWithInProgress = true+ }+ f.mu.Unlock()+ f.acks.Add(1)+ return nil+}++func (f *fakeTaskHandle) Nack() error {+ f.mu.Lock()+ if f.inProgressActive {+ f.settledWithInProgress = true+ }+ f.mu.Unlock()+ f.nacks.Add(1)+ return nil+}++func (f *fakeTaskHandle) InProgress() error {+ f.mu.Lock()+ f.inProgressActive = true+ f.mu.Unlock()+ f.inProgress.Add(1)+ // Simulate a real network call that takes a moment, so a racing Ack/Nack+ // would observe inProgressActive=true.+ time.Sleep(2 * time.Millisecond)+ f.mu.Lock()+ f.inProgressActive = false+ f.mu.Unlock()+ return nil+}++func (f *fakeTaskHandle) wasSettledWithInProgress() bool {+ f.mu.Lock()+ defer f.mu.Unlock()+ return f.settledWithInProgress+}func newAckTaskCtx(ctx context.Context, taskID, docID string, handle *fakeTaskHandle) *taskpkg.TaskContext {taskCtx := taskpkg.NewTaskContextForScheduling(diff --git a/internal/ingestion/service/heartbeat.go b/internal/ingestion/service/heartbeat.gonew file mode 100644index 0000000000..5ef6ff6383--- /dev/null+++ b/internal/ingestion/service/heartbeat.go@@ -0,0 +1,129 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "fmt"+ "sync"+ "time"++ "ragflow/internal/common"+)++// Heartbeat renews one broker message's AckWait deadline by calling+// Handle.InProgress periodically until Stop. It is task-kind agnostic: the log+// label comes from the explicit id passed at construction, never from+// TaskContext.IngestionTask (which is nil for memory tasks), so the same type+// serves both the ingestion and memory execution paths.+//+// Contract: Stop blocks until the renewal goroutine has exited, guaranteeing no+// InProgress call is in flight when it returns. Callers must Stop before+// Acking/Nacking the message so settlement never races an InProgress on the+// same message.+type Heartbeat struct {+ id string+ handle common.TaskHandle+ interval time.Duration+ ctx context.Context++ mu sync.Mutex+ started bool+ stopped bool+ stopCh chan struct{}+ doneCh chan struct{}+}++// NewHeartbeat builds a Heartbeat for the given message handle. A nil handle or+// non-positive interval makes Start a no-op (standalone/test path).+func NewHeartbeat(id string, handle common.TaskHandle, interval time.Duration) *Heartbeat {+ return &Heartbeat{+ id: id,+ handle: handle,+ interval: interval,+ ctx: context.Background(),+ stopCh: make(chan struct{}),+ doneCh: make(chan struct{}),+ }+}++// WithContext binds an external cancellation signal (e.g. ingestor shutdown) so+// the renewal goroutine also exits when ctx is done. It must be called before+// Start.+func (h *Heartbeat) WithContext(ctx context.Context) *Heartbeat {+ if h != nil && ctx != nil {+ h.ctx = ctx+ }+ return h+}++// Start launches the renewal goroutine. It is idempotent: repeated calls and a+// call after Stop are no-ops. With a nil handle or non-positive interval it+// starts nothing, so the corresponding Stop returns immediately.+func (h *Heartbeat) Start() {+ if h == nil || h.handle == nil || h.interval <= 0 {+ return+ }+ h.mu.Lock()+ if h.started || h.stopped {+ h.mu.Unlock()+ return+ }+ h.started = true+ h.mu.Unlock()+ go h.loop()+}++// Stop signals the renewal goroutine to exit and BLOCKS until it has, so no+// InProgress is in flight when it returns. It is idempotent; calling Stop when+// no goroutine was started (or before Start) returns immediately and prevents a+// later Start from launching a renewal goroutine.+func (h *Heartbeat) Stop() {+ if h == nil {+ return+ }+ h.mu.Lock()+ started := h.started+ if !h.stopped {+ h.stopped = true+ if started {+ close(h.stopCh)+ }+ }+ h.mu.Unlock()+ if started {+ <-h.doneCh+ }+}++func (h *Heartbeat) loop() {+ defer close(h.doneCh)+ ticker := time.NewTicker(h.interval)+ defer ticker.Stop()+ for {+ select {+ case <-ticker.C:+ if err := h.handle.InProgress(); err != nil {+ common.Error(fmt.Sprintf("heartbeat task %s", h.id), err)+ }+ case <-h.stopCh:+ return+ case <-h.ctx.Done():+ return+ }+ }+}diff --git a/internal/ingestion/service/heartbeat_test.go b/internal/ingestion/service/heartbeat_test.goindex 5b19b7381d..acb1a29718 100644--- a/internal/ingestion/service/heartbeat_test.go+++ b/internal/ingestion/service/heartbeat_test.go@@ -6,8 +6,6 @@ import ("time""ragflow/internal/common"- "ragflow/internal/entity"- taskpkg "ragflow/internal/ingestion/task")// controllableHandle is a TaskHandle whose InProgress behavior is delegated,@@ -21,32 +19,34 @@ func (h *controllableHandle) Ack() error { return nil }func (h *controllableHandle) Nack() error { return nil }func (h *controllableHandle) InProgress() error { return h.inProgressFn() }-// TestStartHeartbeat_TicksInProgressUntilStop: with a short interval the-// heartbeat goroutine calls InProgress repeatedly; stop() halts it.-func TestStartHeartbeat_TicksInProgressUntilStop(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.heartbeatInterval = 2 * time.Millisecond-+// TestHeartbeat_TicksInProgressUntilStop: with a short interval the heartbeat+// goroutine calls InProgress repeatedly; Stop halts it.+func TestHeartbeat_TicksInProgressUntilStop(t *testing.T) {handle := &fakeTaskHandle{}- taskCtx := newAckTaskCtx(t.Context(), "task-1", "doc-1", handle)-- stop := ingestor.startHeartbeat(taskCtx)- time.Sleep(15 * time.Millisecond)- stop()+ hb := NewHeartbeat("task-1", handle, 2*time.Millisecond)+ hb.Start()+ deadline := time.Now().Add(2 * time.Second)+ for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {+ time.Sleep(2 * time.Millisecond)+ }if handle.inProgress.Load() == 0 {t.Fatal("expected InProgress heartbeats, got 0")}++ hb.Stop()+ after := handle.inProgress.Load()+ time.Sleep(10 * time.Millisecond)+ if got := handle.inProgress.Load(); got != after {+ t.Fatalf("InProgress calls continued after Stop: before=%d after=%d", after, got)+ }}-// TestStartHeartbeat_StopWaitsForInFlightInProgress: stop() must block until an+// TestHeartbeat_StopWaitsForInFlightInProgress: Stop must block until an// in-flight InProgress call returns, so the caller can ack/nack with no// concurrent InProgress on the same message. Regression guard for the-// close-without-wait heartbeat shutdown (problem 1).-func TestStartHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.heartbeatInterval = time.Millisecond-+// close-without-wait heartbeat shutdown.+func TestHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {started := make(chan struct{})release := make(chan struct{})var startedOnce sync.Once@@ -57,21 +57,16 @@ func TestStartHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {return nil},}- taskCtx := taskpkg.NewTaskContextForScheduling(- t.Context(),- &entity.IngestionTask{ID: "task-1", DocumentID: "doc-1", DatasetID: "kb-1", Status: common.RUNNING},- )- taskCtx.Handle = h-- stop := ingestor.startHeartbeat(taskCtx)+ hb := NewHeartbeat("task-1", h, time.Millisecond)+ hb.Start()<-started // heartbeat goroutine is inside InProgressstopDone := make(chan struct{})- go func() { stop(); close(stopDone) }()+ go func() { hb.Stop(); close(stopDone) }()select {case <-stopDone:- t.Fatal("stop() returned before in-flight InProgress completed")+ t.Fatal("Stop returned before in-flight InProgress completed")case <-time.After(20 * time.Millisecond):}@@ -79,23 +74,46 @@ func TestStartHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {select {case <-stopDone:- // good: stop() returned only after InProgress completed+ // good: Stop returned only after InProgress completed+ case <-time.After(time.Second):+ t.Fatal("Stop did not return after in-flight InProgress completed")+ }+}++// TestHeartbeat_NoOpWhenNoHandle: with no MQ handle, Start starts no goroutine+// and Stop returns immediately (never blocks).+func TestHeartbeat_NoOpWhenNoHandle(t *testing.T) {+ hb := NewHeartbeat("task-1", nil, time.Millisecond)+ hb.Start()+ hb.Stop() // must not block or panic+}++// TestHeartbeat_StopBeforeStart: Stop before Start is a no-op and never blocks,+// covering the "lease never started" path (e.g. a task context that admission+// rejected before starting a heartbeat).+func TestHeartbeat_StopBeforeStart(t *testing.T) {+ hb := NewHeartbeat("task-1", &fakeTaskHandle{}, time.Millisecond)+ done := make(chan struct{})+ go func() { hb.Stop(); close(done) }()+ select {+ case <-done:+ // good: returned immediatelycase <-time.After(time.Second):- t.Fatal("stop() did not return after in-flight InProgress completed")+ t.Fatal("Stop before Start blocked")}}-// TestStartHeartbeat_NoOpWhenNoHandle: with no MQ handle (standalone/test path)-// startHeartbeat returns a no-op stop and starts no goroutine.-func TestStartHeartbeat_NoOpWhenNoHandle(t *testing.T) {- ingestor := newUnitIngestor("test", 1, []string{"pdf"})- ingestor.heartbeatInterval = time.Millisecond-- taskCtx := taskpkg.NewTaskContextForScheduling(- t.Context(),- &entity.IngestionTask{ID: "task-1"},- )- // taskCtx.Handle is nil- stop := ingestor.startHeartbeat(taskCtx)- stop() // must not block or panic+// TestHeartbeat_StopBeforeStartPreventsLaterStart ensures a stopped heartbeat+// cannot launch a renewal goroutine afterwards.+func TestHeartbeat_StopBeforeStartPreventsLaterStart(t *testing.T) {+ handle := &fakeTaskHandle{}+ hb := NewHeartbeat("task-1", handle, time.Millisecond)++ hb.Stop()+ hb.Start()+ time.Sleep(10 * time.Millisecond)++ if got := handle.inProgress.Load(); got != 0 {+ t.Fatalf("InProgress calls after Stop then Start = %d, want 0", got)+ }}diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex d82def8cfd..b0e1d10234 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -47,9 +47,10 @@ import (// in-flight message's ack deadline renewed while a worker parses. It MUST stay// comfortably below the consumer's ack deadline, which the server normalizes// to BackOff[0] = 5s (see NatsEngine.InitConsumer): a pulse slower than the-// deadline lets the broker redeliver mid-run, and the redelivered copy is-// ack-skipped by the claim guard. If the worker or process stops, the broker-// can redeliver the unsettled message after the ack deadline.+// deadline lets the broker redeliver mid-run. A duplicate delivery renews its+// lease but remains unsettled until the owning worker reaches a durable+// outcome. If the worker or process stops, the broker can redeliver the+// unsettled message after the ack deadline.const defaultHeartbeatInterval = 2 * time.Secondtype Ingestor struct {@@ -111,8 +112,8 @@ type Ingestor struct {// runMemoryTask dispatches one async memory-extraction task. Tests may// override this to inject a panicking/failing runner without a live DB or// real MemoryMessageService. Defaults to defaultRunMemoryTask, which calls- // memorySvc.HandleSaveToMemoryTask.- runMemoryTask func(ctx context.Context, payload map[string]any) error+ // memorySvc.HandleSaveToMemoryTask with the envelope task id.+ runMemoryTask func(ctx context.Context, taskID string, payload map[string]any) error// cancelCheck is polled periodically (every 3s) during task execution.// When it returns true the task's context is cancelled, which causes the@@ -368,6 +369,16 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {}return}+ // A memory task without an envelope id has no valid identity: claiming+ // an empty key would strand a no-op claim in currentTasks and block+ // nothing useful, so Ack-skip it instead.+ if taskMessage.TaskID == "" {+ common.Warn("memory task with empty task id received, ack")+ if err := handle.Ack(); err != nil {+ common.Error("error ack memory task with empty task id", err)+ }+ return+ }var payload map[string]anyif len(taskMessage.Payload) == 0 || json.Unmarshal(taskMessage.Payload, &payload) != nil {common.Warn(fmt.Sprintf("memory task %s has no parseable payload, ack", taskMessage.TaskID))@@ -376,13 +387,39 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {}return}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, payload, handle)+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, payload, handle)+ // Claim the task before enqueueing so a redelivered copy of the same+ // memory task (NATS AckWait/BackOff redelivery, or a restart replay) is+ // not executed again while its owner is still running. Memory tasks have no+ // ingestion_task row / status CAS to dedupe against, so this claim+ // guards duplicate execution within this ingestor. The claim is released+ // by executeMemoryTask when the worker finishes.+ if !e.claimTask(taskMessage.TaskID) {+ common.Warn(fmt.Sprintf("memory task %s redelivered while worker still processing, renew lease", taskMessage.TaskID))+ if err := handle.InProgress(); err != nil {+ common.Error(fmt.Sprintf("renew redelivered memory task %s", taskMessage.TaskID), err)+ }+ return+ }+ claimedTaskID = taskMessage.TaskID+ // Begin renewing the broker lease before the task can wait in taskChan.+ // Starting only in executeMemoryTask would leave queued work unprotected+ // until a worker becomes available: a task waiting past AckWait would be+ // redelivered before its owner can begin running it.+ // The stop function rides on the task context so the worker can stop it+ // before settlement (the lease is owned by the task, not by the map).+ hb := NewHeartbeat(taskMessage.TaskID, handle, e.heartbeatInterval).WithContext(e.ctx)+ hb.Start()+ taskCtx.SetStopLease(hb.Stop)select {case e.taskChan <- taskCtx:+ claimedTaskID = "" // executeMemoryTask owns the release nowcommon.Info(fmt.Sprintf("Memory task %s queued (channel: %d/%d)", taskMessage.TaskID, len(e.taskChan), cap(e.taskChan)))case <-e.ctx.Done():- // Shutdown won the race: return without settling so the broker- // redelivers the memory task after restart.+ // Shutdown won the race: stop the lease, release the claim, and+ // return without settling so the broker redelivers the memory task+ // after restart.+ taskCtx.StopLease()common.Info(fmt.Sprintf("Ingestor shutting down; memory task %s not enqueued", taskMessage.TaskID))return}@@ -432,13 +469,14 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {returncase common.RUNNING:// Guard against MQ redelivery: if another worker in this- // process is already processing this task, ack the redelivered- // message and skip instead of scheduling it again.+ // process is already processing this task, renew the redelivered+ // message's lease and skip instead of scheduling it again. The+ // owning worker remains the only code path that may Ack/Nack.if !e.claimTask(task.ID) {- common.Warn(fmt.Sprintf("task %s redelivered while worker still processing, ack skip (task_id=%s doc_id=%s kb_id=%s)",+ common.Warn(fmt.Sprintf("task %s redelivered while worker still processing, renew lease (task_id=%s doc_id=%s kb_id=%s)",taskMessage.TaskID, task.ID, task.DocumentID, task.DatasetID))- if ackErr := handle.Ack(); ackErr != nil {- common.Error(fmt.Sprintf("error ack redelivered task %s", taskMessage.TaskID), ackErr)+ if err := handle.InProgress(); err != nil {+ common.Error(fmt.Sprintf("renew redelivered task %s", taskMessage.TaskID), err)}return}@@ -470,8 +508,8 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {// first few parse" defect.//// The in-flight claim (set just above) guards against a redelivery racing- // the blocked send: a duplicate delivery sees claimTask fail and is- // ack-skipped, so a blocking send cannot double-execute a task.+ // the blocked send: a duplicate delivery sees claimTask fail, renews its+ // lease, and cannot double-execute a task.select {case e.taskChan <- taskCtx:claimedTaskID = "" // executeTask owns the release now@@ -530,10 +568,26 @@ func (e *Ingestor) workerLoop(id int32) {// ingestion_task row / state machine: the runner persists the extracted// messages and settles task progress on the way out.//+// The task id comes exclusively from taskCtx.ID() (the envelope TaskID set by+// the scheduler). It is never re-derived from MemoryPayload — the payload no+// longer carries identity.+//+// Settlement runs in one deferred closed loop on every exit path (success,+// terminal/transient failure, panic) in a fixed order: stop the admission+// heartbeat and wait for it to exit, then Ack/Nack exactly once, then release+// the in-process claim. Stopping the heartbeat before the Ack/Nack is required+// by Heartbeat's contract — no InProgress may be in flight on the same message+// while it is being settled. The claim is released after settlement so a+// redelivery that races the settlement stays owned by this worker; a redelivery+// after the function returns (e.g. a lost Ack) re-claims and re-runs the task+// as normal at-least-once delivery.+//// Settlement is error-category aware:// - Terminal failure (task row absent, already-failed, or progress=-1 already// persisted) is Acked so an already-consumed message is never redelivered// into an infinite nack loop.+// - A task row already at progress>=1.0 is treated as success (the completed+// short-circuit in HandleSaveToMemoryTask) and Acked.// - Transient failure (a task-load DB error before any durable marker, or an// LLM/network failure that did not reach progress=-1) is Nacked so the// message is redelivered and retried instead of being silently dropped.@@ -545,22 +599,41 @@ func (e *Ingestor) workerLoop(id int32) {// stall every subsequent document parse. The recovered panic is treated as a// transient failure and Nacked for redelivery.func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {- taskID, _ := taskCtx.MemoryPayload["id"].(string)- if taskID == "" {- taskID, _ = taskCtx.MemoryPayload["task_id"].(string)- }-- // Recover a panic so a single poison memory task never crashes the worker- // (and, at max_concurrent_workers=1, the whole ingestor's only slot).+ taskID := taskCtx.ID()++ // The admission path (processMessage) starts the heartbeat before queueing+ // and attaches its stop function via SetStopLease. Direct-execution callers+ // (unit tests, non-queued paths) carry no lease, so start a local heartbeat+ // as a fallback. The two are mutually exclusive — never both running.+ if taskCtx.StopLeaseFn() == nil && taskCtx.Handle != nil && e.heartbeatInterval > 0 {+ hb := NewHeartbeat(taskID, taskCtx.Handle, e.heartbeatInterval).WithContext(e.ctx)+ hb.Start()+ taskCtx.SetStopLease(hb.Stop)+ }++ var (+ settleAck bool+ settleNack bool+ )defer func() {+ // Stop the lease (and wait for any in-flight InProgress) before settling.+ taskCtx.StopLease()if r := recover(); r != nil {+ // A panic is treated as a transient failure: Nack so the broker+ // redelivers and retries the message.common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))- if taskCtx != nil && taskCtx.Handle != nil {- if nackErr := taskCtx.Handle.Nack(); nackErr != nil {- common.Error(fmt.Sprintf("nack memory task %s after panic", taskID), nackErr)- }+ settleNack = true+ }+ if settleAck {+ if err := taskCtx.Handle.Ack(); err != nil {+ common.Error(fmt.Sprintf("ack memory task %s", taskID), err)+ }+ } else if settleNack {+ if err := taskCtx.Handle.Nack(); err != nil {+ common.Error(fmt.Sprintf("nack memory task %s", taskID), err)}}+ e.releaseTask(taskID)}()common.Info(fmt.Sprintf("Starting memory task %s", taskID))@@ -570,39 +643,32 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskC}if e.memorySvc == nil {common.Warn(fmt.Sprintf("memory task %s: memory extractor disabled, ack", taskID))- if err := taskCtx.Handle.Ack(); err != nil {- common.Error(fmt.Sprintf("ack memory task %s", taskID), err)- }+ settleAck = truereturn}- if err := e.runMemoryTask(ctx, taskCtx.MemoryPayload); err != nil {+ if err := e.runMemoryTask(ctx, taskID, taskCtx.MemoryPayload); err != nil {// defaultRunMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal- // (durable progress=-1 written, or no row to retry). Everything else is- // transient and must be redelivered rather than dropped.+ // (durable progress=-1 written, completed progress>=1.0, or no row to+ // retry). Everything else is transient and must be redelivered rather+ // than dropped.if errors.Is(err, servicepkg.ErrMemoryTaskTerminal) {common.Error(fmt.Sprintf("memory task %s failed terminally, ack", taskID), err)- if ackErr := taskCtx.Handle.Ack(); ackErr != nil {- common.Error(fmt.Sprintf("ack failed memory task %s", taskID), ackErr)- }+ settleAck = truereturn}common.Error(fmt.Sprintf("memory task %s failed transiently, nack for redelivery", taskID), err)- if nackErr := taskCtx.Handle.Nack(); nackErr != nil {- common.Error(fmt.Sprintf("nack memory task %s", taskID), nackErr)- }+ settleNack = truereturn}common.Info(fmt.Sprintf("Memory task %s completed", taskID))- if err := taskCtx.Handle.Ack(); err != nil {- common.Error(fmt.Sprintf("ack memory task %s", taskID), err)- }+ settleAck = true}// defaultRunMemoryTask is the production memory-task runner. It is held behind// the runMemoryTask field so tests can substitute a panicking/failing runner// without a live DB or real MemoryMessageService.-func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, payload map[string]any) error {- return e.memorySvc.HandleSaveToMemoryTask(ctx, payload)+func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {+ return e.memorySvc.HandleSaveToMemoryTask(ctx, taskID, payload)}func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {@@ -853,16 +919,17 @@ func (e *Ingestor) settleToTerminal(ctx context.Context, taskID string) error {}// settleMessage runs body under a heartbeat, then settles the MQ message. The-// heartbeat is stopped (and waited on) before ack/nack — see startHeartbeat.+// heartbeat is stopped (and waited on) before ack/nack — see Heartbeat.Stop.// A panic in body is recovered: the task is marked FAILED and the message is// Nacked for redelivery, so a single task's panic never crashes the worker.// Settlement queries the DB for the task's actual status: a terminal state// (COMPLETED/STOPPED/FAILED) means Ack; anything else means Nack. The body's// return value is advisory only — DB truth is authoritative (BP1).func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskContext, body func(context.Context) bool) (terminal bool) {- stop := e.startHeartbeat(taskCtx)+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(taskCtx.Ctx)+ hb.Start()defer func() {- stop() // stop heartbeat (and wait) before ack/nack+ hb.Stop() // stop heartbeat (and wait) before ack/nackif r := recover(); r != nil {// Recover the panic so the worker process survives. Mark the// task FAILED so a redelivery does not re-run a poison message@@ -1024,42 +1091,10 @@ func (e *Ingestor) markTimeoutProgress(task *entity.IngestionTask) {// claimTask registers a worker claim on a task ID. Returns false if another// worker has already claimed it (e.g. MQ redelivery), true on first claim.-// startHeartbeat launches a goroutine that calls Handle.InProgress every-// heartbeatInterval to keep the broker AckWait timer fresh during long tasks.-// It returns a stop function that signals the goroutine to exit and BLOCKS-// until it has, so the caller can ack/nack with no in-flight InProgress on the-// same message. Returns a no-op stop when there is no handle or no interval-// (standalone/test path).-func (e *Ingestor) startHeartbeat(taskCtx *taskpkg.TaskContext) func() {- if taskCtx.Handle == nil || e.heartbeatInterval <= 0 {- return func() {}- }- var wg sync.WaitGroup- wg.Add(1)- done := make(chan struct{})- go func() {- defer wg.Done()- ticker := time.NewTicker(e.heartbeatInterval)- defer ticker.Stop()- for {- select {- case <-ticker.C:- if err := taskCtx.Handle.InProgress(); err != nil {- common.Error(fmt.Sprintf("heartbeat task %s", taskCtx.IngestionTask.ID), err)- }- case <-done:- return- case <-taskCtx.Ctx.Done():- return- }- }- }()- return func() {- close(done)- wg.Wait()- }-}-+// The claim is released by releaseTask when the worker finishes, so a future+// redelivery (after restart) can re-claim the task. Broker lease renewal is a+// separate concern handled by Heartbeat (doc: settleMessage; memory: admission+// in processMessage + stop via TaskContext.StopLease).func (e *Ingestor) claimTask(taskID string) bool {e.tasksMu.Lock()defer e.tasksMu.Unlock()diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.goindex e80f462c49..0d5e5383c9 100644--- a/internal/ingestion/service/process_message_test.go+++ b/internal/ingestion/service/process_message_test.go@@ -33,8 +33,6 @@ func TestProcessMessage_MemoryTaskDispatches(t *testing.T) {ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))payload, err := json.Marshal(map[string]any{- "id": "mem-task-1",- "task_type": "memory","memory_id": "mem-1","source_id": 42,"message_dict": map[string]any{@@ -114,8 +112,8 @@ func TestExecuteMemoryTaskAlreadyFailedAcks(t *testing.T) {ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{- "id": taskID, "task_type": "memory", "memory_id": "mem-3", "source_id": 7,+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{+ "memory_id": "mem-3", "source_id": 7,"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},}, handle)@@ -123,7 +121,7 @@ func TestExecuteMemoryTaskAlreadyFailedAcks(t *testing.T) {// HandleSaveToMemoryTask return the "already failed" error. Otherwise the// Ack below would only reflect the success path and prove nothing about// the Ack-on-failure contract.- if err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskCtx.MemoryPayload); err == nil {+ if err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskID, taskCtx.MemoryPayload); err == nil {t.Fatal("expected HandleSaveToMemoryTask to fail on an already-failed (progress=-1) task, got nil")}@@ -149,14 +147,14 @@ func TestExecuteMemoryTaskTransientFailureNacks(t *testing.T) {ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-task-x", TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{- "id": "mem-task-x", "task_type": "memory", "memory_id": "mem-x", "source_id": 1,+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-task-x", map[string]any{+ "memory_id": "mem-x", "source_id": 1,"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},}, handle)// Precondition: with no tasks table, HandleSaveToMemoryTask must fail with a// transient error that is NOT wrapped in ErrMemoryTaskTerminal.- err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskCtx.MemoryPayload)+ err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), "mem-task-x", taskCtx.MemoryPayload)if err == nil {t.Fatal("expected HandleSaveToMemoryTask to fail with missing tasks table, got nil")}@@ -291,10 +289,11 @@ func TestProcessMessage_AlreadyCompletedAcks(t *testing.T) {}}-// TestProcessMessage_ClaimFailsAcks: a RUNNING task whose claim fails-// (redelivery guard) is acked without enqueuing — another worker is already-// processing it.-func TestProcessMessage_ClaimFailsAcks(t *testing.T) {+// TestProcessMessage_ClaimFailsRenewsLease: a RUNNING task whose claim fails+// is already owned by another worker. The duplicate must not Ack the broker+// message before that owner reaches a durable outcome; it only renews the+// delivery lease and stays out of the worker queue.+func TestProcessMessage_ClaimFailsRenewsLease(t *testing.T) {db := testutil.SetupTestDB(t)cleanup := testutil.ReplaceDBForTest(t, db)defer cleanup()@@ -307,8 +306,11 @@ func TestProcessMessage_ClaimFailsAcks(t *testing.T) {handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)ingestor.processMessage(handle)- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {- t.Fatalf("claim-fail: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("claim-fail: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+ if handle.inProgress.Load() != 1 {+ t.Fatalf("claim-fail: expected duplicate lease renewal, got InProgress=%d", handle.inProgress.Load())}if len(ingestor.taskChan) != 0 {t.Fatal("expected no task enqueued when claim fails")@@ -499,3 +501,265 @@ func TestProcessMessage_StartRunningErrorNacks(t *testing.T) {t.Fatal("expected no task enqueued on activation failure")}}++// TestProcessMessage_MemoryTaskHeartbeatsWhileQueued verifies that a claimed+// memory message renews its broker lease before a worker starts it. Without+// this, a task waiting in taskChan can exceed AckWait and consume unnecessary+// broker deliveries before a worker starts it.+func TestProcessMessage_MemoryTaskHeartbeatsWhileQueued(t *testing.T) {+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))+ ingestor.heartbeatInterval = 5 * time.Millisecond++ payload, err := json.Marshal(map[string]any{+ "memory_id": "mem-1", "source_id": 42,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ })+ if err != nil {+ t.Fatalf("marshal payload: %v", err)+ }+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-queued-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}++ ingestor.processMessage(handle)++ deadline := time.Now().Add(2 * time.Second)+ for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {+ time.Sleep(5 * time.Millisecond)+ }+ if handle.inProgress.Load() == 0 {+ t.Fatal("expected InProgress heartbeat while memory task was queued without a worker")+ }++ // Drain and stop the lease so the heartbeat goroutine does not leak.+ taskCtx := <-ingestor.taskChan+ taskCtx.StopLease()+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {+ t.Fatalf("queued memory task must not be settled by admission, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}++// TestProcessMessage_MemoryTaskRedeliveryRenewsLease verifies the claim guard on+// the memory dispatch path: when a memory task is redelivered by the broker+// while the first copy is still queued/in-flight, the duplicate must renew its+// lease without Acking it or enqueuing a second worker. A different task id+// must still be accepted (claim is per-task-id).+func TestProcessMessage_MemoryTaskRedeliveryRenewsLease(t *testing.T) {+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))++ payload, err := json.Marshal(map[string]any{+ "memory_id": "mem-1", "source_id": 42,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ })+ if err != nil {+ t.Fatalf("marshal payload: %v", err)+ }++ // First delivery: claim succeeds, task is enqueued, message unsettled.+ first := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}+ ingestor.processMessage(first)+ if first.acks.Load() != 0 || first.nacks.Load() != 0 {+ t.Fatalf("first delivery: expected 0 Ack/0 Nack (settled by worker), got acks=%d nacks=%d", first.acks.Load(), first.nacks.Load())+ }+ if len(ingestor.taskChan) != 1 {+ t.Fatalf("first delivery: expected 1 memory task enqueued, got %d", len(ingestor.taskChan))+ }++ // Redelivery while the first copy is still queued: claim fails, the+ // duplicate must renew its lease and NOT be enqueued.+ dup := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}+ ingestor.processMessage(dup)+ if dup.acks.Load() != 0 || dup.nacks.Load() != 0 {+ t.Fatalf("redelivered copy: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", dup.acks.Load(), dup.nacks.Load())+ }+ if dup.inProgress.Load() != 1 {+ t.Fatalf("redelivered copy: expected 1 lease renewal, got InProgress=%d", dup.inProgress.Load())+ }+ if len(ingestor.taskChan) != 1 {+ t.Fatalf("redelivered copy must not be enqueued, got %d tasks in channel", len(ingestor.taskChan))+ }++ // A different memory task must still be accepted (claim is per-task-id).+ otherPayload, err := json.Marshal(map[string]any{+ "memory_id": "mem-2", "source_id": 43,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ })+ if err != nil {+ t.Fatalf("marshal other payload: %v", err)+ }+ other := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-2", TaskType: common.TaskTypeMemory, Payload: otherPayload}}+ ingestor.processMessage(other)+ if len(ingestor.taskChan) != 2 {+ t.Fatalf("different memory task should be enqueued, got %d tasks in channel", len(ingestor.taskChan))+ }++ // Drain both and stop leases so heartbeat goroutines do not leak.+ firstCtx := <-ingestor.taskChan+ firstCtx.StopLease()+ secondCtx := <-ingestor.taskChan+ secondCtx.StopLease()+}++// TestProcessMessage_MemoryTaskEmptyIDAcks verifies that a memory task with an+// empty envelope task id is Acked and skipped before claiming — an empty claim+// key would otherwise strand a no-op entry in currentTasks.+func TestProcessMessage_MemoryTaskEmptyIDAcks(t *testing.T) {+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))++ payload, err := json.Marshal(map[string]any{+ "memory_id": "mem-1", "source_id": 42,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ })+ if err != nil {+ t.Fatalf("marshal payload: %v", err)+ }+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "", TaskType: common.TaskTypeMemory, Payload: payload}}++ ingestor.processMessage(handle)+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("empty-id memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+ if len(ingestor.taskChan) != 0 {+ t.Fatalf("empty-id memory task must not be enqueued, got %d tasks", len(ingestor.taskChan))+ }+ if _, claimed := ingestor.currentTasks[""]; claimed {+ t.Fatal("empty-id memory task must not claim the empty key")+ }+}++// TestExecuteMemoryTask_ReleasesClaim verifies that executeMemoryTask releases+// the claim taken by processMessage once the worker finishes, so a later+// redelivery (e.g. after restart) can re-claim and re-run the task instead of+// being permanently ack-skipped.+func TestExecuteMemoryTask_ReleasesClaim(t *testing.T) {+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))+ // Stub the runner so no DB / LLM is touched.+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {+ return nil+ }++ if !ingestor.claimTask("mem-release-1") {+ t.Fatal("expected claim to succeed")+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-release-1", TaskType: common.TaskTypeMemory}}+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-release-1", map[string]any{+ "memory_id": "mem-r", "source_id": 1,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ }, handle)++ ingestor.executeMemoryTask(context.Background(), taskCtx)++ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack on success, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+ if _, stillClaimed := ingestor.currentTasks["mem-release-1"]; stillClaimed {+ t.Fatal("expected claim released after executeMemoryTask finished")+ }+ if !ingestor.claimTask("mem-release-1") {+ t.Fatal("expected re-claim to succeed after release (future redelivery can re-run)")+ }+ ingestor.releaseTask("mem-release-1")+}++// TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask drives the real+// admission path (processMessage claim + lease) then executes the task: a+// long-running memory task (LLM extraction can take 10-65s) must renew the+// broker lease via InProgress, and settlement must stop the heartbeat before+// Acking (no Ack while an InProgress is in flight).+func TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask(t *testing.T) {+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))+ ingestor.heartbeatInterval = 5 * time.Millisecond++ started := make(chan struct{})+ proceed := make(chan struct{})+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {+ close(started) // admission heartbeat is running by now+ <-proceed // simulate a long LLM extraction+ return nil+ }++ payload, err := json.Marshal(map[string]any{+ "memory_id": "mem-h", "source_id": 1,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ })+ if err != nil {+ t.Fatalf("marshal payload: %v", err)+ }+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}++ // Admit through processMessage so the claim + lease are real, then drain+ // and execute like a worker would. executeMemoryTask's deferred settlement+ // (stop lease → Ack → release claim) runs before the goroutine exits, so+ // waiting on done before reading currentTasks is race-free.+ ingestor.processMessage(handle)+ taskCtx := <-ingestor.taskChan+ done := make(chan struct{})+ go func() {+ defer close(done)+ ingestor.executeMemoryTask(context.Background(), taskCtx)+ }()+ <-started++ // Poll for heartbeats with a generous deadline so the test is resilient+ // to slow CI schedulers.+ heartbeatDeadline := time.Now().Add(2 * time.Second)+ for handle.inProgress.Load() == 0 && time.Now().Before(heartbeatDeadline) {+ time.Sleep(5 * time.Millisecond)+ }+ if handle.inProgress.Load() == 0 {+ t.Fatal("expected InProgress heartbeats while runMemoryTask was blocked, got 0")+ }++ close(proceed) // release the long task — only after confirming heartbeats++ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("executeMemoryTask did not return after the long task was released")+ }+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack on completion after heartbeat, got acks=%d nacks=%d inProgress=%d",+ handle.acks.Load(), handle.nacks.Load(), handle.inProgress.Load())+ }+ if handle.wasSettledWithInProgress() {+ t.Fatal("Ack ran while an InProgress was still in flight — heartbeat must stop before settlement")+ }+ if _, stillClaimed := ingestor.currentTasks["mem-hb-1"]; stillClaimed {+ t.Fatal("expected claim released after executeMemoryTask finished")+ }+}++// TestExecuteMemoryTaskAlreadyCompletedAcks verifies the idempotent+// short-circuit for a task row already extracted to completion (progress=1.0):+// a redelivery after restart (or a duplicate copy) must NOT re-run the LLM+// extraction — which would insert duplicate memory entries — but must Ack the+// message. The production runner short-circuits on the progress=1.0 row.+func TestExecuteMemoryTaskAlreadyCompletedAcks(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ taskID := "mem-completed-1"+ if err := dao.DB.Create(&entity.Task{ID: taskID, DocID: "doc-mem-c", Progress: 1.0}).Error; err != nil {+ t.Fatalf("insert already-completed task: %v", err)+ }++ ingestor := newUnitIngestor("test", 1, []string{"pdf"})+ // Real memory service so the production runner path is exercised.+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))++ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{+ "memory_id": "mem-c", "source_id": 7,+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},+ }, handle)++ ingestor.executeMemoryTask(context.Background(), taskCtx)+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("already-completed memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.goindex 9ca468a63e..15f88d290d 100644--- a/internal/ingestion/task/task_context.go+++ b/internal/ingestion/task/task_context.go@@ -58,10 +58,26 @@ type TaskContext struct {File any// MemoryPayload carries the raw task_type="memory" message body for- // memory tasks (id/memory_id/source_id/message_dict). Only set for+ // memory tasks (memory_id/source_id/message_dict). Only set for// TaskKindMemory.MemoryPayload map[string]any+ // taskID is the envelope task identifier (TaskMessage.TaskID) that the+ // scheduler claims and later releases. It is the authoritative identity+ // for claim, release, and logging across both task kinds. Unexported so+ // the claim key cannot be mutated between admission (claim) and+ // settlement (release), which would leak the claim and permanently block+ // redelivery. There is deliberately no fallback chain in ID(): identity+ // lives only here, never re-derived from IngestionTask or MemoryPayload.+ taskID string++ // stopLease stops the broker-lease heartbeat started at admission+ // (processMessage) for a queued/running memory task. It is set by the+ // memory dispatch path before enqueueing and invoked by the worker's+ // settlement defer before Ack/Nack. Doc (ingestion) tasks leave it nil —+ // their heartbeat is started and stopped entirely inside settleMessage.+ stopLease func()+// Handle is the message-queue ack handle for the task message that scheduled// this context. The scheduler sets it before queueing; the worker decides// the terminal Ack/Nack:@@ -77,12 +93,15 @@ type TaskContext struct {}// NewMemoryTaskContextForScheduling creates a lightweight TaskContext for a-// memory-extraction task. It only sets the scheduling-related fields, not the-// full ingestion business data.-func NewMemoryTaskContextForScheduling(ctx context.Context, payload map[string]any, handle common.TaskHandle) *TaskContext {+// memory-extraction task. taskID is the envelope TaskMessage.TaskID that the+// scheduler claims and will release; it is the authoritative identity for the+// whole memory task lifecycle. Only the scheduling-related fields are set, not+// the full ingestion business data.+func NewMemoryTaskContextForScheduling(ctx context.Context, taskID string, payload map[string]any, handle common.TaskHandle) *TaskContext {return &TaskContext{Ctx: ctx,Kind: TaskKindMemory,+ taskID: taskID,MemoryPayload: payload,Handle: handle,}@@ -94,10 +113,50 @@ func NewTaskContextForScheduling(ctx context.Context, task *entity.IngestionTaskreturn &TaskContext{Ctx: ctx,Kind: TaskKindIngestion,+ taskID: task.ID,IngestionTask: task,}}+// SetStopLease attaches the admission-started broker-lease stop function to a+// memory task context so the worker can stop the heartbeat before settlement.+// A nil stop is a no-op. Doc (ingestion) contexts never call this.+func (c *TaskContext) SetStopLease(stop func()) {+ if c != nil {+ c.stopLease = stop+ }+}++// StopLease invokes the attached stop function, if any, blocking until no+// InProgress is in flight (Heartbeat.Stop semantics). It is safe to call with+// no attached lease (doc path or direct-execution test path).+func (c *TaskContext) StopLease() {+ if c != nil && c.stopLease != nil {+ c.stopLease()+ }+}++// StopLeaseFn returns the attached stop function (nil when no admission+// heartbeat was started), so callers can detect whether a lease exists without+// invoking it.+func (c *TaskContext) StopLeaseFn() func() {+ if c == nil {+ return nil+ }+ return c.stopLease+}++// ID returns the task identifier for claim/release and logging. It is the+// envelope TaskMessage.TaskID captured at construction — there is deliberately+// no fallback to IngestionTask or MemoryPayload, so identity is never+// re-derived from a source that could disagree with the claim key.+func (c *TaskContext) ID() string {+ if c == nil {+ return ""+ }+ return c.taskID+}+// LoadFromIngestionTask loads the full task context from an IngestionTask.// It follows the FK chain: ingestion task -> document -> knowledgebase -> tenant.func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionTask) (*TaskContext, error) {@@ -123,6 +182,7 @@ func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionTreturn &TaskContext{Ctx: ctx,+ taskID: ingestionTask.ID,IngestionTask: ingestionTask,PipelineID: pipelineID,Doc: *doc,diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.goindex 4ece2332f4..67273cbac5 100644--- a/internal/service/memory_extractor.go+++ b/internal/service/memory_extractor.go@@ -36,6 +36,7 @@ package serviceimport ("context"+ "crypto/sha256""encoding/json""errors""fmt"@@ -67,6 +68,12 @@ var memoryNow = time.Now// errors so executeMemoryTask can Nack and let the message be redelivered.var ErrMemoryTaskTerminal = errors.New("memory: terminal task failure, do not redeliver")+// errMemoryTaskRetryable marks a failure that must leave the broker message+// unsettled. It is used when extracted messages may already be stored but the+// durable task state could not be recorded; stable chunk ids make that retry+// safe.+var errMemoryTaskRetryable = errors.New("memory: retryable task failure")+// extractedMemory is one LLM-extracted memory item ready for persistence.type extractedMemory struct {MessageType string@@ -79,19 +86,19 @@ type extractedMemory struct {// Python handle_save_to_memory_task: validate the task row, then// extract + persist, settling task progress on the way out.//+// taskID is the authoritative identity passed explicitly by the Ingestor (the+// envelope TaskID); it is never re-derived from the payload. The payload+// carries only business parameters (memory_id / source_id / message_dict).+//// The returned error is wrapped in ErrMemoryTaskTerminal when the failure has// already produced a durable terminal outcome (dependency/config error, task// row absent, task already failed, or extraction failed after progress=-1 was// persisted). Transient failures (a task-load DB error before any marker was// written) return an unwrapped error so the caller can Nack and redeliver.-func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, payload map[string]any) error {+func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {if s == nil || s.taskDAO == nil || s.memories == nil {return fmt.Errorf("%w: memory: nil MemoryMessageService or memory dependency", ErrMemoryTaskTerminal)}- taskID, _ := payload["id"].(string)- if taskID == "" {- taskID, _ = payload["task_id"].(string)- }memoryID, _ := payload["memory_id"].(string)sourceID := payloadInt64(payload["source_id"])msgDict, _ := payload["message_dict"].(map[string]any)@@ -116,9 +123,22 @@ func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, payloif task.Progress == -1 {return fmt.Errorf("%w: memory: task %s is already failed", ErrMemoryTaskTerminal, taskID)}+ // A task already extracted to completion (progress>=1.0) is a durable+ // terminal outcome: a redelivery after restart (or a duplicate copy from+ // another consumer) must not re-run the LLM extraction, which would insert+ // duplicate memory entries. Short-circuit to success so the Ingestor Acks+ // the message instead of re-executing the task.+ if task.Progress >= 1.0 {+ return nil+ }if err = s.saveExtractedToMemory(ctx, memoryID, msg, sourceID, taskID); err != nil {- s.updateTaskProgress(ctx, taskID, -1, err.Error())+ if errors.Is(err, errMemoryTaskRetryable) {+ return err+ }+ if progressErr := s.updateTaskProgress(ctx, taskID, -1, err.Error()); progressErr != nil {+ return fmt.Errorf("%w: memory: mark task %s failed: %v", errMemoryTaskRetryable, taskID, progressErr)+ }return fmt.Errorf("%w: %w", ErrMemoryTaskTerminal, err)}return nil@@ -138,7 +158,9 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memory}extractTypes := getTypesToExtract(memoryTypes)if len(extractTypes) == 0 {- s.updateTaskProgress(ctx, taskID, 1.0, fmt.Sprintf("Memory '%s' don't need to extract.", memoryID))+ if err := s.updateTaskProgress(ctx, taskID, 1.0, fmt.Sprintf("Memory '%s' don't need to extract.", memoryID)); err != nil {+ return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)+ }return nil}@@ -147,10 +169,12 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memoryreturn err}if len(extracted) == 0 {- s.updateTaskProgress(ctx, taskID, 1.0, "No memory extracted from raw message.")+ if err := s.updateTaskProgress(ctx, taskID, 1.0, "No memory extracted from raw message."); err != nil {+ return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)+ }return nil}- s.updateTaskProgress(ctx, taskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extracted)))+ _ = s.updateTaskProgress(ctx, taskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extracted)))// conversation_time is stamped as server-local wall clock, not UTC.now := memoryNow()@@ -161,7 +185,9 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memoryif err = s.embedAndSaveMessages(ctx, mem, messages); err != nil {return err}- s.updateTaskProgress(ctx, taskID, 1.0, "Message saved successfully.")+ if err := s.updateTaskProgress(ctx, taskID, 1.0, "Message saved successfully."); err != nil {+ return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)+ }return nil}@@ -201,7 +227,7 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo}chatModel := models.NewChatModel(driver, &modelName, apiConfig)- s.updateTaskProgress(ctx, taskID, 0.15, "Prepared prompts and LLM.")+ _ = s.updateTaskProgress(ctx, taskID, 0.15, "Prepared prompts and LLM.")temperature := mem.Temperatureresp, err := chatModel.ModelDriver.ChatWithMessages(ctx, modelName, messages, apiConfig, &models.ChatConfig{Temperature: &temperature}, nil)if err != nil {@@ -210,7 +236,7 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemoif resp == nil || resp.Answer == nil {return nil, errors.New("empty response from chat model")}- s.updateTaskProgress(ctx, taskID, 0.35, "Get extracted result from LLM.")+ _ = s.updateTaskProgress(ctx, taskID, 0.35, "Get extracted result from LLM.")return parseMemoryExtraction(*resp.Answer, extractTypes), nil}@@ -221,11 +247,30 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo// logical message fields are set here; the doc engine maps them to// storage fields (including tokenization) at insert time.func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory, now time.Time) map[string]any {- var invalidAt any+ validAt, hasExplicitValidAt := normalizeMemoryTime(item.ValidAt)+ if !hasExplicitValidAt {+ validAt = now.Format(memoryTimeLayout)+ }+ invalidAt, hasExplicitInvalidAt := normalizeMemoryTime(item.InvalidAt)if strings.TrimSpace(item.InvalidAt) != "" {- invalidAt = formatMemoryTime(item.InvalidAt, now)+ if !hasExplicitInvalidAt {+ invalidAt = now.Format(memoryTimeLayout)+ }+ }+ var storedInvalidAt any+ if invalidAt != "" {+ storedInvalidAt = invalidAt+ }+ fingerprintValidAt := ""+ if hasExplicitValidAt {+ fingerprintValidAt = validAt+ }+ fingerprintInvalidAt := ""+ if hasExplicitInvalidAt {+ fingerprintInvalidAt = invalidAt}return map[string]any{+ "id": extractedMessageDocumentID(memoryID, sourceID, item.MessageType, item.Content, fingerprintValidAt, fingerprintInvalidAt),"message_id": messageID,"message_type": item.MessageType,"source_id": sourceID,@@ -234,13 +279,24 @@ func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg Memor"agent_id": msg.AgentID,"session_id": msg.SessionID,"content": item.Content,- "valid_at": formatMemoryTime(item.ValidAt, now),- "invalid_at": invalidAt,+ "valid_at": validAt,+ "invalid_at": storedInvalidAt,"forget_at": nil,"status": true,}}+// extractedMessageDocumentID returns the chunk-store id for an extracted item.+// sourceID identifies the immutable raw message, while the normalized content+// fingerprint distinguishes multiple extracts from that source. Only explicit,+// parseable timestamps participate, so a runtime fallback cannot change the+// identity on retry. Repeated task executions therefore upsert the same chunk+// instead of creating duplicates.+func extractedMessageDocumentID(memoryID string, sourceID int64, messageType, content, validAt, invalidAt string) string {+ fingerprint := sha256.Sum256([]byte(strings.Join([]string{messageType, content, validAt, invalidAt}, "\x00")))+ return fmt.Sprintf("%s_%d_%x", memoryID, sourceID, fingerprint[:8])+}+// parseMemoryExtraction ports memory.utils.msg_util.get_json_result_from_llm_response// plus the per-type flattening in extract_by_llm. Only the configured// extract types are collected; unparseable responses yield an empty list.@@ -288,28 +344,43 @@ func parseMemoryExtraction(answer string, extractTypes []string) []extractedMemo// back to the supplied time formatted in its own location (server-local// when callers pass time.Now()).func formatMemoryTime(value string, fallback time.Time) string {+ if normalized, ok := normalizeMemoryTime(value); ok {+ return normalized+ }+ return fallback.Format(memoryTimeLayout)+}++// normalizeMemoryTime parses an explicit LLM-supplied timestamp without a+// runtime fallback. Its result is suitable for a stable document fingerprint.+func normalizeMemoryTime(value string) (string, bool) {v := strings.TrimSpace(value)- if v != "" {- for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"} {- if t, err := time.Parse(layout, v); err == nil {- return t.Format(memoryTimeLayout)- }+ if v == "" {+ return "", false+ }+ for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"} {+ if t, err := time.Parse(layout, v); err == nil {+ return t.Format(memoryTimeLayout), true}}- return fallback.Format(memoryTimeLayout)+ return "", false}// updateTaskProgress stamps and persists task progress, mirroring-// Python TaskService.update_progress call sites. Failures are logged-// and swallowed so progress reporting never breaks extraction.-func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID string, progress float64, msg string) {- if s == nil || s.taskDAO == nil || taskID == "" {- return+// Python TaskService.update_progress call sites. Callers that establish a+// terminal task state must handle an error so the message can be retried.+func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID string, progress float64, msg string) error {+ if s == nil || s.taskDAO == nil {+ return errors.New("memory: nil task DAO")+ }+ if taskID == "" {+ return errors.New("memory: empty task id")}stamped := time.Now().Format(memoryTimeLayout) + " " + msgif err := s.taskDAO.UpdateProgress(ctx, dao.DB, taskID, progress, stamped); err != nil {common.Warn("memory: update task progress failed", zap.Error(err))+ return err}+ return nil}func payloadInt64(v any) int64 {diff --git a/internal/service/memory_extractor_test.go b/internal/service/memory_extractor_test.goindex 8ea40384df..8357b10774 100644--- a/internal/service/memory_extractor_test.go+++ b/internal/service/memory_extractor_test.go@@ -23,6 +23,10 @@ import ("strings""testing""time"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/testutil")// pinMemoryNow fixes the memory wall clock at the given instant for the@@ -95,6 +99,61 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {}}+// TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries ensures fallback+// timestamps do not become part of the chunk-store identity. A retry may run+// at a different time and allocate a new logical message id, but it must still+// overwrite the same extracted document.+func TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries(t *testing.T) {+ msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}+ firstAttempt := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)+ retryAttempt := firstAttempt.Add(10 * time.Second)++ for _, test := range []struct {+ name string+ item extractedMemory+ }{+ {+ name: "omitted valid at",+ item: extractedMemory{MessageType: "fact", Content: "likes coffee"},+ },+ {+ name: "invalid timestamps",+ item: extractedMemory{+ MessageType: "fact",+ Content: "likes coffee",+ ValidAt: "not a timestamp",+ InvalidAt: "also not a timestamp",+ },+ },+ } {+ t.Run(test.name, func(t *testing.T) {+ first := buildExtractedMessage(8, 42, "mem-1", msg, test.item, firstAttempt)+ retry := buildExtractedMessage(9, 42, "mem-1", msg, test.item, retryAttempt)++ firstID, ok := first["id"].(string)+ if !ok || firstID == "" {+ t.Fatalf("first extracted message id = %#v, want a stable non-empty string", first["id"])+ }+ if got, ok := retry["id"].(string); !ok || got != firstID {+ t.Fatalf("retry extracted message id = %#v, want %q", retry["id"], firstID)+ }+ })+ }+}++// TestUpdateTaskProgressReturnsPersistenceError ensures callers can keep the+// broker message retryable when they cannot persist the terminal task state.+func TestUpdateTaskProgressReturnsPersistenceError(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.IngestionTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ svc := &MemoryMessageService{taskDAO: dao.NewTaskDAO()}+ if err := svc.updateTaskProgress(t.Context(), "mem-task-1", 1.0, "completed"); err == nil {+ t.Fatal("updateTaskProgress() error = nil, want persistence error")+ }+}+// TestTypeInstructionsStayInLockstepWithPython pins the rules that// memory/utils/prompt_util.py carries and the Go map historically dropped:// per-type Examples lines, the full "Timestamp Rules for <Type> Knowledge:"diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.goindex adb1718d82..201c712b82 100644--- a/internal/service/memory_message_service.go+++ b/internal/service/memory_message_service.go@@ -170,7 +170,7 @@ func (s *MemoryMessageService) QueueSaveToMemoryTask(ctx context.Context, memory})continue}- if err = queueMemoryTask(ctx, memoryID, mem.TenantID, rawMessageID, task, msg); err != nil {+ if err = queueMemoryTask(ctx, fmt.Sprint(task["id"]), memoryID, mem.TenantID, rawMessageID, msg); err != nil {res.Failed = append(res.Failed, MemoryFailure{MemoryID: memoryID,FailMsg: err.Error(),@@ -278,7 +278,9 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr}vectorDim = len(vector)message[fmt.Sprintf("q_%d_vec", len(vector))] = vector- message["id"] = fmt.Sprintf("%s_%v", message["memory_id"], message["message_id"])+ if id, ok := message["id"].(string); !ok || id == "" {+ message["id"] = fmt.Sprintf("%s_%v", message["memory_id"], message["message_id"])+ }message["doc_id"] = message["memory_id"]}@@ -339,12 +341,14 @@ func taskFromRow(row map[string]any) *entity.Task {}}-func queueMemoryTask(ctx context.Context, memoryID, tenantID string, rawMessageID int64, task map[string]any, msg MemoryMessage) error {- taskID := fmt.Sprint(task["id"])+// queueMemoryTask publishes a memory-extraction task to NATS (tasks.RAGFLOW).+// taskID is the durable task row's id (buildTaskRow) and the sole identity on+// the envelope; the payload carries only business parameters — memory_id,+// source_id, and the dialogue to extract. Identity is deliberately NOT+// duplicated into the payload so the consumer never has two ids that could+// disagree (the envelope TaskID is authoritative end to end).+func queueMemoryTask(ctx context.Context, taskID, memoryID, tenantID string, rawMessageID int64, msg MemoryMessage) error {message := map[string]any{- "id": taskID,- "task_id": taskID,- "task_type": task["task_type"],"memory_id": memoryID,"tenant_id": tenantID,"source_id": rawMessageID,diff --git a/internal/service/memory_queue_task_test.go b/internal/service/memory_queue_task_test.goindex 771e84b176..37f1e86d64 100644--- a/internal/service/memory_queue_task_test.go+++ b/internal/service/memory_queue_task_test.go@@ -19,10 +19,10 @@ func TestQueueMemoryTaskUninitializedQueueReturnsError(t *testing.T) {err := queueMemoryTask(t.Context(),+ "task-1","mem-1","tenant-1",1,- map[string]any{"id": "task-1", "task_type": "memory"},MemoryMessage{UserID: "u1", AgentID: "agent-1", SessionID: "s1", UserInput: "hi", AgentResponse: "hello"},)if err == nil {