Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): unify memory task execution with heartbeat leases (#19189)

jay77721363f77ef80d5
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.go
index 9117bb19e2..7fceec5727 100644
--- a/internal/engine/nats/nats.go
+++ b/internal/engine/nats/nats.go
@@ -120,7 +120,8 @@ func (n *NatsEngine) PublishTask(subject string, payload []byte) error {
// by any consumer and un-reparsable ("already exists, status: SCHEDULED").
// Duplicate delivery is instead made safe at the consumer level:
// StartRunning's CREATED/SCHEDULED→RUNNING CAS plus the in-process claim guard
- // ack-skip any second copy (see Ingestor.processMessage).
+ // prevent a second copy from executing while the first owner is active (see
+ // Ingestor.processMessage).
ack, err := n.jetStream.Publish(ctx, subject, payload)
if err != nil {
return err
@@ -225,8 +226,7 @@ func (n *NatsEngine) InitConsumer(subject string) error {
// 60s AckWait is the effective schedule if BackOff is ever dropped.
// INVARIANT: the worker's InProgress heartbeat (Ingestor
// defaultHeartbeatInterval) must stay below BackOff[0] = 5s, or in-flight
- // messages get redelivered mid-run and the claim-guard ack-skip acks the
- // only live copy.
+ // messages get redelivered mid-run before the owning worker can settle them.
// Note: CreateOrUpdateConsumer is atomic. MaxWaiting is immutable after
// creation; if it mismatches the whole update fails (error "max waiting
// can not be updated") and NONE of AckWait/BackOff/MaxAckPending are
diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.go
index 9a4be0556a..ecb21e6806 100644
--- a/internal/ingestion/service/execute_memory_task_panic_test.go
+++ b/internal/ingestion/service/execute_memory_task_panic_test.go
@@ -35,13 +35,13 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {
ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
// Inject a panicking memory runner through the same seam used in production
// (defaultRunMemoryTask calls memorySvc.HandleSaveToMemoryTask).
- ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {
+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
panic("simulated memory extraction panic")
}
handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-1", TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{
- "id": "mem-panic-1", "task_type": "memory", "memory_id": "mem-p", "source_id": 1,
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-1", map[string]any{
+ "memory_id": "mem-p", "source_id": 1,
"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
}, handle)
@@ -82,7 +82,7 @@ func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
ingestor := NewIngestor("test", 1, []string{"pdf"})
ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
- ingestor.runMemoryTask = func(_ context.Context, _ map[string]any) error {
+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
panic("simulated memory extraction panic")
}
ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {
@@ -90,8 +90,8 @@ func TestWorkerLoop_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
}
memHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-2", TaskType: common.TaskTypeMemory}}
- memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{
- "id": "mem-panic-2", "task_type": "memory", "memory_id": "mem-p2", "source_id": 1,
+ memCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-2", map[string]any{
+ "memory_id": "mem-p2", "source_id": 1,
"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
}, memHandle)
diff --git a/internal/ingestion/service/execute_task_ack_test.go b/internal/ingestion/service/execute_task_ack_test.go
index 3cd8232f54..e658601591 100644
--- a/internal/ingestion/service/execute_task_ack_test.go
+++ b/internal/ingestion/service/execute_task_ack_test.go
@@ -3,6 +3,7 @@ package service
import (
"context"
"errors"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -19,17 +20,62 @@ import (
// counters are atomic because executeTask drives them from multiple goroutines
// (the heartbeat goroutine calls InProgress; the defer calls Ack/Nack) while the
// test goroutine reads them - plain ints would be a data race under -race.
+//
+// settledWithInProgress is set when Ack or Nack runs while an InProgress is
+// still in flight on the same message — a violation of Heartbeat's contract
+// that the heartbeat must be stopped and waited on before settling.
type fakeTaskHandle struct {
msg common.TaskMessage
acks atomic.Int64
nacks atomic.Int64
inProgress atomic.Int64
+
+ mu sync.Mutex
+ inProgressActive bool
+ settledWithInProgress bool
}
func (f *fakeTaskHandle) GetMessage() common.TaskMessage { return f.msg }
-func (f *fakeTaskHandle) Ack() error { f.acks.Add(1); return nil }
-func (f *fakeTaskHandle) Nack() error { f.nacks.Add(1); return nil }
-func (f *fakeTaskHandle) InProgress() error { f.inProgress.Add(1); return nil }
+
+func (f *fakeTaskHandle) Ack() error {
+ f.mu.Lock()
+ if f.inProgressActive {
+ f.settledWithInProgress = true
+ }
+ f.mu.Unlock()
+ f.acks.Add(1)
+ return nil
+}
+
+func (f *fakeTaskHandle) Nack() error {
+ f.mu.Lock()
+ if f.inProgressActive {
+ f.settledWithInProgress = true
+ }
+ f.mu.Unlock()
+ f.nacks.Add(1)
+ return nil
+}
+
+func (f *fakeTaskHandle) InProgress() error {
+ f.mu.Lock()
+ f.inProgressActive = true
+ f.mu.Unlock()
+ f.inProgress.Add(1)
+ // Simulate a real network call that takes a moment, so a racing Ack/Nack
+ // would observe inProgressActive=true.
+ time.Sleep(2 * time.Millisecond)
+ f.mu.Lock()
+ f.inProgressActive = false
+ f.mu.Unlock()
+ return nil
+}
+
+func (f *fakeTaskHandle) wasSettledWithInProgress() bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return f.settledWithInProgress
+}
func newAckTaskCtx(ctx context.Context, taskID, docID string, handle *fakeTaskHandle) *taskpkg.TaskContext {
taskCtx := taskpkg.NewTaskContextForScheduling(
diff --git a/internal/ingestion/service/heartbeat.go b/internal/ingestion/service/heartbeat.go
new file mode 100644
index 0000000000..5ef6ff6383
--- /dev/null
+++ b/internal/ingestion/service/heartbeat.go
@@ -0,0 +1,129 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "time"
+
+ "ragflow/internal/common"
+)
+
+// Heartbeat renews one broker message's AckWait deadline by calling
+// Handle.InProgress periodically until Stop. It is task-kind agnostic: the log
+// label comes from the explicit id passed at construction, never from
+// TaskContext.IngestionTask (which is nil for memory tasks), so the same type
+// serves both the ingestion and memory execution paths.
+//
+// Contract: Stop blocks until the renewal goroutine has exited, guaranteeing no
+// InProgress call is in flight when it returns. Callers must Stop before
+// Acking/Nacking the message so settlement never races an InProgress on the
+// same message.
+type Heartbeat struct {
+ id string
+ handle common.TaskHandle
+ interval time.Duration
+ ctx context.Context
+
+ mu sync.Mutex
+ started bool
+ stopped bool
+ stopCh chan struct{}
+ doneCh chan struct{}
+}
+
+// NewHeartbeat builds a Heartbeat for the given message handle. A nil handle or
+// non-positive interval makes Start a no-op (standalone/test path).
+func NewHeartbeat(id string, handle common.TaskHandle, interval time.Duration) *Heartbeat {
+ return &Heartbeat{
+ id: id,
+ handle: handle,
+ interval: interval,
+ ctx: context.Background(),
+ stopCh: make(chan struct{}),
+ doneCh: make(chan struct{}),
+ }
+}
+
+// WithContext binds an external cancellation signal (e.g. ingestor shutdown) so
+// the renewal goroutine also exits when ctx is done. It must be called before
+// Start.
+func (h *Heartbeat) WithContext(ctx context.Context) *Heartbeat {
+ if h != nil && ctx != nil {
+ h.ctx = ctx
+ }
+ return h
+}
+
+// Start launches the renewal goroutine. It is idempotent: repeated calls and a
+// call after Stop are no-ops. With a nil handle or non-positive interval it
+// starts nothing, so the corresponding Stop returns immediately.
+func (h *Heartbeat) Start() {
+ if h == nil || h.handle == nil || h.interval <= 0 {
+ return
+ }
+ h.mu.Lock()
+ if h.started || h.stopped {
+ h.mu.Unlock()
+ return
+ }
+ h.started = true
+ h.mu.Unlock()
+ go h.loop()
+}
+
+// Stop signals the renewal goroutine to exit and BLOCKS until it has, so no
+// InProgress is in flight when it returns. It is idempotent; calling Stop when
+// no goroutine was started (or before Start) returns immediately and prevents a
+// later Start from launching a renewal goroutine.
+func (h *Heartbeat) Stop() {
+ if h == nil {
+ return
+ }
+ h.mu.Lock()
+ started := h.started
+ if !h.stopped {
+ h.stopped = true
+ if started {
+ close(h.stopCh)
+ }
+ }
+ h.mu.Unlock()
+ if started {
+ <-h.doneCh
+ }
+}
+
+func (h *Heartbeat) loop() {
+ defer close(h.doneCh)
+ ticker := time.NewTicker(h.interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ if err := h.handle.InProgress(); err != nil {
+ common.Error(fmt.Sprintf("heartbeat task %s", h.id), err)
+ }
+ case <-h.stopCh:
+ return
+ case <-h.ctx.Done():
+ return
+ }
+ }
+}
diff --git a/internal/ingestion/service/heartbeat_test.go b/internal/ingestion/service/heartbeat_test.go
index 5b19b7381d..acb1a29718 100644
--- a/internal/ingestion/service/heartbeat_test.go
+++ b/internal/ingestion/service/heartbeat_test.go
@@ -6,8 +6,6 @@ import (
"time"
"ragflow/internal/common"
- "ragflow/internal/entity"
- taskpkg "ragflow/internal/ingestion/task"
)
// controllableHandle is a TaskHandle whose InProgress behavior is delegated,
@@ -21,32 +19,34 @@ func (h *controllableHandle) Ack() error { return nil }
func (h *controllableHandle) Nack() error { return nil }
func (h *controllableHandle) InProgress() error { return h.inProgressFn() }
-// TestStartHeartbeat_TicksInProgressUntilStop: with a short interval the
-// heartbeat goroutine calls InProgress repeatedly; stop() halts it.
-func TestStartHeartbeat_TicksInProgressUntilStop(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.heartbeatInterval = 2 * time.Millisecond
-
+// TestHeartbeat_TicksInProgressUntilStop: with a short interval the heartbeat
+// goroutine calls InProgress repeatedly; Stop halts it.
+func TestHeartbeat_TicksInProgressUntilStop(t *testing.T) {
handle := &fakeTaskHandle{}
- taskCtx := newAckTaskCtx(t.Context(), "task-1", "doc-1", handle)
-
- stop := ingestor.startHeartbeat(taskCtx)
- time.Sleep(15 * time.Millisecond)
- stop()
+ hb := NewHeartbeat("task-1", handle, 2*time.Millisecond)
+ hb.Start()
+ deadline := time.Now().Add(2 * time.Second)
+ for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {
+ time.Sleep(2 * time.Millisecond)
+ }
if handle.inProgress.Load() == 0 {
t.Fatal("expected InProgress heartbeats, got 0")
}
+
+ hb.Stop()
+ after := handle.inProgress.Load()
+ time.Sleep(10 * time.Millisecond)
+ if got := handle.inProgress.Load(); got != after {
+ t.Fatalf("InProgress calls continued after Stop: before=%d after=%d", after, got)
+ }
}
-// TestStartHeartbeat_StopWaitsForInFlightInProgress: stop() must block until an
+// TestHeartbeat_StopWaitsForInFlightInProgress: Stop must block until an
// in-flight InProgress call returns, so the caller can ack/nack with no
// concurrent InProgress on the same message. Regression guard for the
-// close-without-wait heartbeat shutdown (problem 1).
-func TestStartHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.heartbeatInterval = time.Millisecond
-
+// close-without-wait heartbeat shutdown.
+func TestHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
var startedOnce sync.Once
@@ -57,21 +57,16 @@ func TestStartHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {
return nil
},
}
- taskCtx := taskpkg.NewTaskContextForScheduling(
- t.Context(),
- &entity.IngestionTask{ID: "task-1", DocumentID: "doc-1", DatasetID: "kb-1", Status: common.RUNNING},
- )
- taskCtx.Handle = h
-
- stop := ingestor.startHeartbeat(taskCtx)
+ hb := NewHeartbeat("task-1", h, time.Millisecond)
+ hb.Start()
<-started // heartbeat goroutine is inside InProgress
stopDone := make(chan struct{})
- go func() { stop(); close(stopDone) }()
+ go func() { hb.Stop(); close(stopDone) }()
select {
case <-stopDone:
- t.Fatal("stop() returned before in-flight InProgress completed")
+ t.Fatal("Stop returned before in-flight InProgress completed")
case <-time.After(20 * time.Millisecond):
}
@@ -79,23 +74,46 @@ func TestStartHeartbeat_StopWaitsForInFlightInProgress(t *testing.T) {
select {
case <-stopDone:
- // good: stop() returned only after InProgress completed
+ // good: Stop returned only after InProgress completed
+ case <-time.After(time.Second):
+ t.Fatal("Stop did not return after in-flight InProgress completed")
+ }
+}
+
+// TestHeartbeat_NoOpWhenNoHandle: with no MQ handle, Start starts no goroutine
+// and Stop returns immediately (never blocks).
+func TestHeartbeat_NoOpWhenNoHandle(t *testing.T) {
+ hb := NewHeartbeat("task-1", nil, time.Millisecond)
+ hb.Start()
+ hb.Stop() // must not block or panic
+}
+
+// TestHeartbeat_StopBeforeStart: Stop before Start is a no-op and never blocks,
+// covering the "lease never started" path (e.g. a task context that admission
+// rejected before starting a heartbeat).
+func TestHeartbeat_StopBeforeStart(t *testing.T) {
+ hb := NewHeartbeat("task-1", &fakeTaskHandle{}, time.Millisecond)
+ done := make(chan struct{})
+ go func() { hb.Stop(); close(done) }()
+ select {
+ case <-done:
+ // good: returned immediately
case <-time.After(time.Second):
- t.Fatal("stop() did not return after in-flight InProgress completed")
+ t.Fatal("Stop before Start blocked")
}
}
-// TestStartHeartbeat_NoOpWhenNoHandle: with no MQ handle (standalone/test path)
-// startHeartbeat returns a no-op stop and starts no goroutine.
-func TestStartHeartbeat_NoOpWhenNoHandle(t *testing.T) {
- ingestor := newUnitIngestor("test", 1, []string{"pdf"})
- ingestor.heartbeatInterval = time.Millisecond
-
- taskCtx := taskpkg.NewTaskContextForScheduling(
- t.Context(),
- &entity.IngestionTask{ID: "task-1"},
- )
- // taskCtx.Handle is nil
- stop := ingestor.startHeartbeat(taskCtx)
- stop() // must not block or panic
+// TestHeartbeat_StopBeforeStartPreventsLaterStart ensures a stopped heartbeat
+// cannot launch a renewal goroutine afterwards.
+func TestHeartbeat_StopBeforeStartPreventsLaterStart(t *testing.T) {
+ handle := &fakeTaskHandle{}
+ hb := NewHeartbeat("task-1", handle, time.Millisecond)
+
+ hb.Stop()
+ hb.Start()
+ time.Sleep(10 * time.Millisecond)
+
+ if got := handle.inProgress.Load(); got != 0 {
+ t.Fatalf("InProgress calls after Stop then Start = %d, want 0", got)
+ }
}
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index d82def8cfd..b0e1d10234 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -47,9 +47,10 @@ import (
// in-flight message's ack deadline renewed while a worker parses. It MUST stay
// comfortably below the consumer's ack deadline, which the server normalizes
// to BackOff[0] = 5s (see NatsEngine.InitConsumer): a pulse slower than the
-// deadline lets the broker redeliver mid-run, and the redelivered copy is
-// ack-skipped by the claim guard. If the worker or process stops, the broker
-// can redeliver the unsettled message after the ack deadline.
+// deadline lets the broker redeliver mid-run. A duplicate delivery renews its
+// lease but remains unsettled until the owning worker reaches a durable
+// outcome. If the worker or process stops, the broker can redeliver the
+// unsettled message after the ack deadline.
const defaultHeartbeatInterval = 2 * time.Second
type Ingestor struct {
@@ -111,8 +112,8 @@ type Ingestor struct {
// runMemoryTask dispatches one async memory-extraction task. Tests may
// override this to inject a panicking/failing runner without a live DB or
// real MemoryMessageService. Defaults to defaultRunMemoryTask, which calls
- // memorySvc.HandleSaveToMemoryTask.
- runMemoryTask func(ctx context.Context, payload map[string]any) error
+ // memorySvc.HandleSaveToMemoryTask with the envelope task id.
+ runMemoryTask func(ctx context.Context, taskID string, payload map[string]any) error
// cancelCheck is polled periodically (every 3s) during task execution.
// When it returns true the task's context is cancelled, which causes the
@@ -368,6 +369,16 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
}
return
}
+ // A memory task without an envelope id has no valid identity: claiming
+ // an empty key would strand a no-op claim in currentTasks and block
+ // nothing useful, so Ack-skip it instead.
+ if taskMessage.TaskID == "" {
+ common.Warn("memory task with empty task id received, ack")
+ if err := handle.Ack(); err != nil {
+ common.Error("error ack memory task with empty task id", err)
+ }
+ return
+ }
var payload map[string]any
if len(taskMessage.Payload) == 0 || json.Unmarshal(taskMessage.Payload, &payload) != nil {
common.Warn(fmt.Sprintf("memory task %s has no parseable payload, ack", taskMessage.TaskID))
@@ -376,13 +387,39 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
}
return
}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, payload, handle)
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, payload, handle)
+ // Claim the task before enqueueing so a redelivered copy of the same
+ // memory task (NATS AckWait/BackOff redelivery, or a restart replay) is
+ // not executed again while its owner is still running. Memory tasks have no
+ // ingestion_task row / status CAS to dedupe against, so this claim
+ // guards duplicate execution within this ingestor. The claim is released
+ // by executeMemoryTask when the worker finishes.
+ if !e.claimTask(taskMessage.TaskID) {
+ common.Warn(fmt.Sprintf("memory task %s redelivered while worker still processing, renew lease", taskMessage.TaskID))
+ if err := handle.InProgress(); err != nil {
+ common.Error(fmt.Sprintf("renew redelivered memory task %s", taskMessage.TaskID), err)
+ }
+ return
+ }
+ claimedTaskID = taskMessage.TaskID
+ // Begin renewing the broker lease before the task can wait in taskChan.
+ // Starting only in executeMemoryTask would leave queued work unprotected
+ // until a worker becomes available: a task waiting past AckWait would be
+ // redelivered before its owner can begin running it.
+ // The stop function rides on the task context so the worker can stop it
+ // before settlement (the lease is owned by the task, not by the map).
+ hb := NewHeartbeat(taskMessage.TaskID, handle, e.heartbeatInterval).WithContext(e.ctx)
+ hb.Start()
+ taskCtx.SetStopLease(hb.Stop)
select {
case e.taskChan <- taskCtx:
+ claimedTaskID = "" // executeMemoryTask owns the release now
common.Info(fmt.Sprintf("Memory task %s queued (channel: %d/%d)", taskMessage.TaskID, len(e.taskChan), cap(e.taskChan)))
case <-e.ctx.Done():
- // Shutdown won the race: return without settling so the broker
- // redelivers the memory task after restart.
+ // Shutdown won the race: stop the lease, release the claim, and
+ // return without settling so the broker redelivers the memory task
+ // after restart.
+ taskCtx.StopLease()
common.Info(fmt.Sprintf("Ingestor shutting down; memory task %s not enqueued", taskMessage.TaskID))
return
}
@@ -432,13 +469,14 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
return
case common.RUNNING:
// Guard against MQ redelivery: if another worker in this
- // process is already processing this task, ack the redelivered
- // message and skip instead of scheduling it again.
+ // process is already processing this task, renew the redelivered
+ // message's lease and skip instead of scheduling it again. The
+ // owning worker remains the only code path that may Ack/Nack.
if !e.claimTask(task.ID) {
- common.Warn(fmt.Sprintf("task %s redelivered while worker still processing, ack skip (task_id=%s doc_id=%s kb_id=%s)",
+ common.Warn(fmt.Sprintf("task %s redelivered while worker still processing, renew lease (task_id=%s doc_id=%s kb_id=%s)",
taskMessage.TaskID, task.ID, task.DocumentID, task.DatasetID))
- if ackErr := handle.Ack(); ackErr != nil {
- common.Error(fmt.Sprintf("error ack redelivered task %s", taskMessage.TaskID), ackErr)
+ if err := handle.InProgress(); err != nil {
+ common.Error(fmt.Sprintf("renew redelivered task %s", taskMessage.TaskID), err)
}
return
}
@@ -470,8 +508,8 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {
// first few parse" defect.
//
// The in-flight claim (set just above) guards against a redelivery racing
- // the blocked send: a duplicate delivery sees claimTask fail and is
- // ack-skipped, so a blocking send cannot double-execute a task.
+ // the blocked send: a duplicate delivery sees claimTask fail, renews its
+ // lease, and cannot double-execute a task.
select {
case e.taskChan <- taskCtx:
claimedTaskID = "" // executeTask owns the release now
@@ -530,10 +568,26 @@ func (e *Ingestor) workerLoop(id int32) {
// ingestion_task row / state machine: the runner persists the extracted
// messages and settles task progress on the way out.
//
+// The task id comes exclusively from taskCtx.ID() (the envelope TaskID set by
+// the scheduler). It is never re-derived from MemoryPayload — the payload no
+// longer carries identity.
+//
+// Settlement runs in one deferred closed loop on every exit path (success,
+// terminal/transient failure, panic) in a fixed order: stop the admission
+// heartbeat and wait for it to exit, then Ack/Nack exactly once, then release
+// the in-process claim. Stopping the heartbeat before the Ack/Nack is required
+// by Heartbeat's contract — no InProgress may be in flight on the same message
+// while it is being settled. The claim is released after settlement so a
+// redelivery that races the settlement stays owned by this worker; a redelivery
+// after the function returns (e.g. a lost Ack) re-claims and re-runs the task
+// as normal at-least-once delivery.
+//
// Settlement is error-category aware:
// - Terminal failure (task row absent, already-failed, or progress=-1 already
// persisted) is Acked so an already-consumed message is never redelivered
// into an infinite nack loop.
+// - A task row already at progress>=1.0 is treated as success (the completed
+// short-circuit in HandleSaveToMemoryTask) and Acked.
// - Transient failure (a task-load DB error before any durable marker, or an
// LLM/network failure that did not reach progress=-1) is Nacked so the
// message is redelivered and retried instead of being silently dropped.
@@ -545,22 +599,41 @@ func (e *Ingestor) workerLoop(id int32) {
// stall every subsequent document parse. The recovered panic is treated as a
// transient failure and Nacked for redelivery.
func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {
- taskID, _ := taskCtx.MemoryPayload["id"].(string)
- if taskID == "" {
- taskID, _ = taskCtx.MemoryPayload["task_id"].(string)
- }
-
- // Recover a panic so a single poison memory task never crashes the worker
- // (and, at max_concurrent_workers=1, the whole ingestor's only slot).
+ taskID := taskCtx.ID()
+
+ // The admission path (processMessage) starts the heartbeat before queueing
+ // and attaches its stop function via SetStopLease. Direct-execution callers
+ // (unit tests, non-queued paths) carry no lease, so start a local heartbeat
+ // as a fallback. The two are mutually exclusive — never both running.
+ if taskCtx.StopLeaseFn() == nil && taskCtx.Handle != nil && e.heartbeatInterval > 0 {
+ hb := NewHeartbeat(taskID, taskCtx.Handle, e.heartbeatInterval).WithContext(e.ctx)
+ hb.Start()
+ taskCtx.SetStopLease(hb.Stop)
+ }
+
+ var (
+ settleAck bool
+ settleNack bool
+ )
defer func() {
+ // Stop the lease (and wait for any in-flight InProgress) before settling.
+ taskCtx.StopLease()
if r := recover(); r != nil {
+ // A panic is treated as a transient failure: Nack so the broker
+ // redelivers and retries the message.
common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))
- if taskCtx != nil && taskCtx.Handle != nil {
- if nackErr := taskCtx.Handle.Nack(); nackErr != nil {
- common.Error(fmt.Sprintf("nack memory task %s after panic", taskID), nackErr)
- }
+ settleNack = true
+ }
+ if settleAck {
+ if err := taskCtx.Handle.Ack(); err != nil {
+ common.Error(fmt.Sprintf("ack memory task %s", taskID), err)
+ }
+ } else if settleNack {
+ if err := taskCtx.Handle.Nack(); err != nil {
+ common.Error(fmt.Sprintf("nack memory task %s", taskID), err)
}
}
+ e.releaseTask(taskID)
}()
common.Info(fmt.Sprintf("Starting memory task %s", taskID))
@@ -570,39 +643,32 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskC
}
if e.memorySvc == nil {
common.Warn(fmt.Sprintf("memory task %s: memory extractor disabled, ack", taskID))
- if err := taskCtx.Handle.Ack(); err != nil {
- common.Error(fmt.Sprintf("ack memory task %s", taskID), err)
- }
+ settleAck = true
return
}
- if err := e.runMemoryTask(ctx, taskCtx.MemoryPayload); err != nil {
+ if err := e.runMemoryTask(ctx, taskID, taskCtx.MemoryPayload); err != nil {
// defaultRunMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal
- // (durable progress=-1 written, or no row to retry). Everything else is
- // transient and must be redelivered rather than dropped.
+ // (durable progress=-1 written, completed progress>=1.0, or no row to
+ // retry). Everything else is transient and must be redelivered rather
+ // than dropped.
if errors.Is(err, servicepkg.ErrMemoryTaskTerminal) {
common.Error(fmt.Sprintf("memory task %s failed terminally, ack", taskID), err)
- if ackErr := taskCtx.Handle.Ack(); ackErr != nil {
- common.Error(fmt.Sprintf("ack failed memory task %s", taskID), ackErr)
- }
+ settleAck = true
return
}
common.Error(fmt.Sprintf("memory task %s failed transiently, nack for redelivery", taskID), err)
- if nackErr := taskCtx.Handle.Nack(); nackErr != nil {
- common.Error(fmt.Sprintf("nack memory task %s", taskID), nackErr)
- }
+ settleNack = true
return
}
common.Info(fmt.Sprintf("Memory task %s completed", taskID))
- if err := taskCtx.Handle.Ack(); err != nil {
- common.Error(fmt.Sprintf("ack memory task %s", taskID), err)
- }
+ settleAck = true
}
// defaultRunMemoryTask is the production memory-task runner. It is held behind
// the runMemoryTask field so tests can substitute a panicking/failing runner
// without a live DB or real MemoryMessageService.
-func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, payload map[string]any) error {
- return e.memorySvc.HandleSaveToMemoryTask(ctx, payload)
+func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {
+ return e.memorySvc.HandleSaveToMemoryTask(ctx, taskID, payload)
}
func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {
@@ -853,16 +919,17 @@ func (e *Ingestor) settleToTerminal(ctx context.Context, taskID string) error {
}
// settleMessage runs body under a heartbeat, then settles the MQ message. The
-// heartbeat is stopped (and waited on) before ack/nack — see startHeartbeat.
+// heartbeat is stopped (and waited on) before ack/nack — see Heartbeat.Stop.
// A panic in body is recovered: the task is marked FAILED and the message is
// Nacked for redelivery, so a single task's panic never crashes the worker.
// Settlement queries the DB for the task's actual status: a terminal state
// (COMPLETED/STOPPED/FAILED) means Ack; anything else means Nack. The body's
// return value is advisory only — DB truth is authoritative (BP1).
func (e *Ingestor) settleMessage(ctx context.Context, taskCtx *taskpkg.TaskContext, body func(context.Context) bool) (terminal bool) {
- stop := e.startHeartbeat(taskCtx)
+ hb := NewHeartbeat(taskCtx.ID(), taskCtx.Handle, e.heartbeatInterval).WithContext(taskCtx.Ctx)
+ hb.Start()
defer func() {
- stop() // stop heartbeat (and wait) before ack/nack
+ hb.Stop() // stop heartbeat (and wait) before ack/nack
if r := recover(); r != nil {
// Recover the panic so the worker process survives. Mark the
// task FAILED so a redelivery does not re-run a poison message
@@ -1024,42 +1091,10 @@ func (e *Ingestor) markTimeoutProgress(task *entity.IngestionTask) {
// claimTask registers a worker claim on a task ID. Returns false if another
// worker has already claimed it (e.g. MQ redelivery), true on first claim.
-// startHeartbeat launches a goroutine that calls Handle.InProgress every
-// heartbeatInterval to keep the broker AckWait timer fresh during long tasks.
-// It returns a stop function that signals the goroutine to exit and BLOCKS
-// until it has, so the caller can ack/nack with no in-flight InProgress on the
-// same message. Returns a no-op stop when there is no handle or no interval
-// (standalone/test path).
-func (e *Ingestor) startHeartbeat(taskCtx *taskpkg.TaskContext) func() {
- if taskCtx.Handle == nil || e.heartbeatInterval <= 0 {
- return func() {}
- }
- var wg sync.WaitGroup
- wg.Add(1)
- done := make(chan struct{})
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(e.heartbeatInterval)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- if err := taskCtx.Handle.InProgress(); err != nil {
- common.Error(fmt.Sprintf("heartbeat task %s", taskCtx.IngestionTask.ID), err)
- }
- case <-done:
- return
- case <-taskCtx.Ctx.Done():
- return
- }
- }
- }()
- return func() {
- close(done)
- wg.Wait()
- }
-}
-
+// The claim is released by releaseTask when the worker finishes, so a future
+// redelivery (after restart) can re-claim the task. Broker lease renewal is a
+// separate concern handled by Heartbeat (doc: settleMessage; memory: admission
+// in processMessage + stop via TaskContext.StopLease).
func (e *Ingestor) claimTask(taskID string) bool {
e.tasksMu.Lock()
defer e.tasksMu.Unlock()
diff --git a/internal/ingestion/service/process_message_test.go b/internal/ingestion/service/process_message_test.go
index e80f462c49..0d5e5383c9 100644
--- a/internal/ingestion/service/process_message_test.go
+++ b/internal/ingestion/service/process_message_test.go
@@ -33,8 +33,6 @@ func TestProcessMessage_MemoryTaskDispatches(t *testing.T) {
ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
payload, err := json.Marshal(map[string]any{
- "id": "mem-task-1",
- "task_type": "memory",
"memory_id": "mem-1",
"source_id": 42,
"message_dict": map[string]any{
@@ -114,8 +112,8 @@ func TestExecuteMemoryTaskAlreadyFailedAcks(t *testing.T) {
ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{
- "id": taskID, "task_type": "memory", "memory_id": "mem-3", "source_id": 7,
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{
+ "memory_id": "mem-3", "source_id": 7,
"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
}, handle)
@@ -123,7 +121,7 @@ func TestExecuteMemoryTaskAlreadyFailedAcks(t *testing.T) {
// HandleSaveToMemoryTask return the "already failed" error. Otherwise the
// Ack below would only reflect the success path and prove nothing about
// the Ack-on-failure contract.
- if err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskCtx.MemoryPayload); err == nil {
+ if err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskID, taskCtx.MemoryPayload); err == nil {
t.Fatal("expected HandleSaveToMemoryTask to fail on an already-failed (progress=-1) task, got nil")
}
@@ -149,14 +147,14 @@ func TestExecuteMemoryTaskTransientFailureNacks(t *testing.T) {
ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-task-x", TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), map[string]any{
- "id": "mem-task-x", "task_type": "memory", "memory_id": "mem-x", "source_id": 1,
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-task-x", map[string]any{
+ "memory_id": "mem-x", "source_id": 1,
"message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
}, handle)
// Precondition: with no tasks table, HandleSaveToMemoryTask must fail with a
// transient error that is NOT wrapped in ErrMemoryTaskTerminal.
- err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), taskCtx.MemoryPayload)
+ err := ingestor.memorySvc.HandleSaveToMemoryTask(context.Background(), "mem-task-x", taskCtx.MemoryPayload)
if err == nil {
t.Fatal("expected HandleSaveToMemoryTask to fail with missing tasks table, got nil")
}
@@ -291,10 +289,11 @@ func TestProcessMessage_AlreadyCompletedAcks(t *testing.T) {
}
}
-// TestProcessMessage_ClaimFailsAcks: a RUNNING task whose claim fails
-// (redelivery guard) is acked without enqueuing — another worker is already
-// processing it.
-func TestProcessMessage_ClaimFailsAcks(t *testing.T) {
+// TestProcessMessage_ClaimFailsRenewsLease: a RUNNING task whose claim fails
+// is already owned by another worker. The duplicate must not Ack the broker
+// message before that owner reaches a durable outcome; it only renews the
+// delivery lease and stays out of the worker queue.
+func TestProcessMessage_ClaimFailsRenewsLease(t *testing.T) {
db := testutil.SetupTestDB(t)
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
@@ -307,8 +306,11 @@ func TestProcessMessage_ClaimFailsAcks(t *testing.T) {
handle := newFakeHandle(taskID, common.TaskTypeIngestionTask)
ingestor.processMessage(handle)
- if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
- t.Fatalf("claim-fail: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("claim-fail: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+ if handle.inProgress.Load() != 1 {
+ t.Fatalf("claim-fail: expected duplicate lease renewal, got InProgress=%d", handle.inProgress.Load())
}
if len(ingestor.taskChan) != 0 {
t.Fatal("expected no task enqueued when claim fails")
@@ -499,3 +501,265 @@ func TestProcessMessage_StartRunningErrorNacks(t *testing.T) {
t.Fatal("expected no task enqueued on activation failure")
}
}
+
+// TestProcessMessage_MemoryTaskHeartbeatsWhileQueued verifies that a claimed
+// memory message renews its broker lease before a worker starts it. Without
+// this, a task waiting in taskChan can exceed AckWait and consume unnecessary
+// broker deliveries before a worker starts it.
+func TestProcessMessage_MemoryTaskHeartbeatsWhileQueued(t *testing.T) {
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
+ ingestor.heartbeatInterval = 5 * time.Millisecond
+
+ payload, err := json.Marshal(map[string]any{
+ "memory_id": "mem-1", "source_id": 42,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ })
+ if err != nil {
+ t.Fatalf("marshal payload: %v", err)
+ }
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-queued-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}
+
+ ingestor.processMessage(handle)
+
+ deadline := time.Now().Add(2 * time.Second)
+ for handle.inProgress.Load() == 0 && time.Now().Before(deadline) {
+ time.Sleep(5 * time.Millisecond)
+ }
+ if handle.inProgress.Load() == 0 {
+ t.Fatal("expected InProgress heartbeat while memory task was queued without a worker")
+ }
+
+ // Drain and stop the lease so the heartbeat goroutine does not leak.
+ taskCtx := <-ingestor.taskChan
+ taskCtx.StopLease()
+ if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
+ t.Fatalf("queued memory task must not be settled by admission, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
+// TestProcessMessage_MemoryTaskRedeliveryRenewsLease verifies the claim guard on
+// the memory dispatch path: when a memory task is redelivered by the broker
+// while the first copy is still queued/in-flight, the duplicate must renew its
+// lease without Acking it or enqueuing a second worker. A different task id
+// must still be accepted (claim is per-task-id).
+func TestProcessMessage_MemoryTaskRedeliveryRenewsLease(t *testing.T) {
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
+
+ payload, err := json.Marshal(map[string]any{
+ "memory_id": "mem-1", "source_id": 42,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ })
+ if err != nil {
+ t.Fatalf("marshal payload: %v", err)
+ }
+
+ // First delivery: claim succeeds, task is enqueued, message unsettled.
+ first := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}
+ ingestor.processMessage(first)
+ if first.acks.Load() != 0 || first.nacks.Load() != 0 {
+ t.Fatalf("first delivery: expected 0 Ack/0 Nack (settled by worker), got acks=%d nacks=%d", first.acks.Load(), first.nacks.Load())
+ }
+ if len(ingestor.taskChan) != 1 {
+ t.Fatalf("first delivery: expected 1 memory task enqueued, got %d", len(ingestor.taskChan))
+ }
+
+ // Redelivery while the first copy is still queued: claim fails, the
+ // duplicate must renew its lease and NOT be enqueued.
+ dup := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-1", TaskType: common.TaskTypeMemory, Payload: payload}}
+ ingestor.processMessage(dup)
+ if dup.acks.Load() != 0 || dup.nacks.Load() != 0 {
+ t.Fatalf("redelivered copy: expected 0 Ack/0 Nack while owner runs, got acks=%d nacks=%d", dup.acks.Load(), dup.nacks.Load())
+ }
+ if dup.inProgress.Load() != 1 {
+ t.Fatalf("redelivered copy: expected 1 lease renewal, got InProgress=%d", dup.inProgress.Load())
+ }
+ if len(ingestor.taskChan) != 1 {
+ t.Fatalf("redelivered copy must not be enqueued, got %d tasks in channel", len(ingestor.taskChan))
+ }
+
+ // A different memory task must still be accepted (claim is per-task-id).
+ otherPayload, err := json.Marshal(map[string]any{
+ "memory_id": "mem-2", "source_id": 43,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ })
+ if err != nil {
+ t.Fatalf("marshal other payload: %v", err)
+ }
+ other := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-redeliver-2", TaskType: common.TaskTypeMemory, Payload: otherPayload}}
+ ingestor.processMessage(other)
+ if len(ingestor.taskChan) != 2 {
+ t.Fatalf("different memory task should be enqueued, got %d tasks in channel", len(ingestor.taskChan))
+ }
+
+ // Drain both and stop leases so heartbeat goroutines do not leak.
+ firstCtx := <-ingestor.taskChan
+ firstCtx.StopLease()
+ secondCtx := <-ingestor.taskChan
+ secondCtx.StopLease()
+}
+
+// TestProcessMessage_MemoryTaskEmptyIDAcks verifies that a memory task with an
+// empty envelope task id is Acked and skipped before claiming — an empty claim
+// key would otherwise strand a no-op entry in currentTasks.
+func TestProcessMessage_MemoryTaskEmptyIDAcks(t *testing.T) {
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
+
+ payload, err := json.Marshal(map[string]any{
+ "memory_id": "mem-1", "source_id": 42,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ })
+ if err != nil {
+ t.Fatalf("marshal payload: %v", err)
+ }
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "", TaskType: common.TaskTypeMemory, Payload: payload}}
+
+ ingestor.processMessage(handle)
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("empty-id memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+ if len(ingestor.taskChan) != 0 {
+ t.Fatalf("empty-id memory task must not be enqueued, got %d tasks", len(ingestor.taskChan))
+ }
+ if _, claimed := ingestor.currentTasks[""]; claimed {
+ t.Fatal("empty-id memory task must not claim the empty key")
+ }
+}
+
+// TestExecuteMemoryTask_ReleasesClaim verifies that executeMemoryTask releases
+// the claim taken by processMessage once the worker finishes, so a later
+// redelivery (e.g. after restart) can re-claim and re-run the task instead of
+// being permanently ack-skipped.
+func TestExecuteMemoryTask_ReleasesClaim(t *testing.T) {
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
+ // Stub the runner so no DB / LLM is touched.
+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
+ return nil
+ }
+
+ if !ingestor.claimTask("mem-release-1") {
+ t.Fatal("expected claim to succeed")
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-release-1", TaskType: common.TaskTypeMemory}}
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-release-1", map[string]any{
+ "memory_id": "mem-r", "source_id": 1,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ }, handle)
+
+ ingestor.executeMemoryTask(context.Background(), taskCtx)
+
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack on success, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+ if _, stillClaimed := ingestor.currentTasks["mem-release-1"]; stillClaimed {
+ t.Fatal("expected claim released after executeMemoryTask finished")
+ }
+ if !ingestor.claimTask("mem-release-1") {
+ t.Fatal("expected re-claim to succeed after release (future redelivery can re-run)")
+ }
+ ingestor.releaseTask("mem-release-1")
+}
+
+// TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask drives the real
+// admission path (processMessage claim + lease) then executes the task: a
+// long-running memory task (LLM extraction can take 10-65s) must renew the
+// broker lease via InProgress, and settlement must stop the heartbeat before
+// Acking (no Ack while an InProgress is in flight).
+func TestExecuteMemoryTask_HeartbeatsInProgressDuringLongTask(t *testing.T) {
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(nil))
+ ingestor.heartbeatInterval = 5 * time.Millisecond
+
+ started := make(chan struct{})
+ proceed := make(chan struct{})
+ ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
+ close(started) // admission heartbeat is running by now
+ <-proceed // simulate a long LLM extraction
+ return nil
+ }
+
+ payload, err := json.Marshal(map[string]any{
+ "memory_id": "mem-h", "source_id": 1,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ })
+ if err != nil {
+ t.Fatalf("marshal payload: %v", err)
+ }
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-hb-1", TaskType: common.TaskTypeMemory, Payload: payload}}
+
+ // Admit through processMessage so the claim + lease are real, then drain
+ // and execute like a worker would. executeMemoryTask's deferred settlement
+ // (stop lease → Ack → release claim) runs before the goroutine exits, so
+ // waiting on done before reading currentTasks is race-free.
+ ingestor.processMessage(handle)
+ taskCtx := <-ingestor.taskChan
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ ingestor.executeMemoryTask(context.Background(), taskCtx)
+ }()
+ <-started
+
+ // Poll for heartbeats with a generous deadline so the test is resilient
+ // to slow CI schedulers.
+ heartbeatDeadline := time.Now().Add(2 * time.Second)
+ for handle.inProgress.Load() == 0 && time.Now().Before(heartbeatDeadline) {
+ time.Sleep(5 * time.Millisecond)
+ }
+ if handle.inProgress.Load() == 0 {
+ t.Fatal("expected InProgress heartbeats while runMemoryTask was blocked, got 0")
+ }
+
+ close(proceed) // release the long task — only after confirming heartbeats
+
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("executeMemoryTask did not return after the long task was released")
+ }
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack on completion after heartbeat, got acks=%d nacks=%d inProgress=%d",
+ handle.acks.Load(), handle.nacks.Load(), handle.inProgress.Load())
+ }
+ if handle.wasSettledWithInProgress() {
+ t.Fatal("Ack ran while an InProgress was still in flight — heartbeat must stop before settlement")
+ }
+ if _, stillClaimed := ingestor.currentTasks["mem-hb-1"]; stillClaimed {
+ t.Fatal("expected claim released after executeMemoryTask finished")
+ }
+}
+
+// TestExecuteMemoryTaskAlreadyCompletedAcks verifies the idempotent
+// short-circuit for a task row already extracted to completion (progress=1.0):
+// a redelivery after restart (or a duplicate copy) must NOT re-run the LLM
+// extraction — which would insert duplicate memory entries — but must Ack the
+// message. The production runner short-circuits on the progress=1.0 row.
+func TestExecuteMemoryTaskAlreadyCompletedAcks(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ taskID := "mem-completed-1"
+ if err := dao.DB.Create(&entity.Task{ID: taskID, DocID: "doc-mem-c", Progress: 1.0}).Error; err != nil {
+ t.Fatalf("insert already-completed task: %v", err)
+ }
+
+ ingestor := newUnitIngestor("test", 1, []string{"pdf"})
+ // Real memory service so the production runner path is exercised.
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeMemory}}
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), taskID, map[string]any{
+ "memory_id": "mem-c", "source_id": 7,
+ "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
+ }, handle)
+
+ ingestor.executeMemoryTask(context.Background(), taskCtx)
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("already-completed memory task: expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.go
index 9ca468a63e..15f88d290d 100644
--- a/internal/ingestion/task/task_context.go
+++ b/internal/ingestion/task/task_context.go
@@ -58,10 +58,26 @@ type TaskContext struct {
File any
// MemoryPayload carries the raw task_type="memory" message body for
- // memory tasks (id/memory_id/source_id/message_dict). Only set for
+ // memory tasks (memory_id/source_id/message_dict). Only set for
// TaskKindMemory.
MemoryPayload map[string]any
+ // taskID is the envelope task identifier (TaskMessage.TaskID) that the
+ // scheduler claims and later releases. It is the authoritative identity
+ // for claim, release, and logging across both task kinds. Unexported so
+ // the claim key cannot be mutated between admission (claim) and
+ // settlement (release), which would leak the claim and permanently block
+ // redelivery. There is deliberately no fallback chain in ID(): identity
+ // lives only here, never re-derived from IngestionTask or MemoryPayload.
+ taskID string
+
+ // stopLease stops the broker-lease heartbeat started at admission
+ // (processMessage) for a queued/running memory task. It is set by the
+ // memory dispatch path before enqueueing and invoked by the worker's
+ // settlement defer before Ack/Nack. Doc (ingestion) tasks leave it nil —
+ // their heartbeat is started and stopped entirely inside settleMessage.
+ stopLease func()
+
// Handle is the message-queue ack handle for the task message that scheduled
// this context. The scheduler sets it before queueing; the worker decides
// the terminal Ack/Nack:
@@ -77,12 +93,15 @@ type TaskContext struct {
}
// NewMemoryTaskContextForScheduling creates a lightweight TaskContext for a
-// memory-extraction task. It only sets the scheduling-related fields, not the
-// full ingestion business data.
-func NewMemoryTaskContextForScheduling(ctx context.Context, payload map[string]any, handle common.TaskHandle) *TaskContext {
+// memory-extraction task. taskID is the envelope TaskMessage.TaskID that the
+// scheduler claims and will release; it is the authoritative identity for the
+// whole memory task lifecycle. Only the scheduling-related fields are set, not
+// the full ingestion business data.
+func NewMemoryTaskContextForScheduling(ctx context.Context, taskID string, payload map[string]any, handle common.TaskHandle) *TaskContext {
return &TaskContext{
Ctx: ctx,
Kind: TaskKindMemory,
+ taskID: taskID,
MemoryPayload: payload,
Handle: handle,
}
@@ -94,10 +113,50 @@ func NewTaskContextForScheduling(ctx context.Context, task *entity.IngestionTask
return &TaskContext{
Ctx: ctx,
Kind: TaskKindIngestion,
+ taskID: task.ID,
IngestionTask: task,
}
}
+// SetStopLease attaches the admission-started broker-lease stop function to a
+// memory task context so the worker can stop the heartbeat before settlement.
+// A nil stop is a no-op. Doc (ingestion) contexts never call this.
+func (c *TaskContext) SetStopLease(stop func()) {
+ if c != nil {
+ c.stopLease = stop
+ }
+}
+
+// StopLease invokes the attached stop function, if any, blocking until no
+// InProgress is in flight (Heartbeat.Stop semantics). It is safe to call with
+// no attached lease (doc path or direct-execution test path).
+func (c *TaskContext) StopLease() {
+ if c != nil && c.stopLease != nil {
+ c.stopLease()
+ }
+}
+
+// StopLeaseFn returns the attached stop function (nil when no admission
+// heartbeat was started), so callers can detect whether a lease exists without
+// invoking it.
+func (c *TaskContext) StopLeaseFn() func() {
+ if c == nil {
+ return nil
+ }
+ return c.stopLease
+}
+
+// ID returns the task identifier for claim/release and logging. It is the
+// envelope TaskMessage.TaskID captured at construction — there is deliberately
+// no fallback to IngestionTask or MemoryPayload, so identity is never
+// re-derived from a source that could disagree with the claim key.
+func (c *TaskContext) ID() string {
+ if c == nil {
+ return ""
+ }
+ return c.taskID
+}
+
// LoadFromIngestionTask loads the full task context from an IngestionTask.
// It follows the FK chain: ingestion task -> document -> knowledgebase -> tenant.
func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionTask) (*TaskContext, error) {
@@ -123,6 +182,7 @@ func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionT
return &TaskContext{
Ctx: ctx,
+ taskID: ingestionTask.ID,
IngestionTask: ingestionTask,
PipelineID: pipelineID,
Doc: *doc,
diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.go
index 4ece2332f4..67273cbac5 100644
--- a/internal/service/memory_extractor.go
+++ b/internal/service/memory_extractor.go
@@ -36,6 +36,7 @@ package service
import (
"context"
+ "crypto/sha256"
"encoding/json"
"errors"
"fmt"
@@ -67,6 +68,12 @@ var memoryNow = time.Now
// errors so executeMemoryTask can Nack and let the message be redelivered.
var ErrMemoryTaskTerminal = errors.New("memory: terminal task failure, do not redeliver")
+// errMemoryTaskRetryable marks a failure that must leave the broker message
+// unsettled. It is used when extracted messages may already be stored but the
+// durable task state could not be recorded; stable chunk ids make that retry
+// safe.
+var errMemoryTaskRetryable = errors.New("memory: retryable task failure")
+
// extractedMemory is one LLM-extracted memory item ready for persistence.
type extractedMemory struct {
MessageType string
@@ -79,19 +86,19 @@ type extractedMemory struct {
// Python handle_save_to_memory_task: validate the task row, then
// extract + persist, settling task progress on the way out.
//
+// taskID is the authoritative identity passed explicitly by the Ingestor (the
+// envelope TaskID); it is never re-derived from the payload. The payload
+// carries only business parameters (memory_id / source_id / message_dict).
+//
// The returned error is wrapped in ErrMemoryTaskTerminal when the failure has
// already produced a durable terminal outcome (dependency/config error, task
// row absent, task already failed, or extraction failed after progress=-1 was
// persisted). Transient failures (a task-load DB error before any marker was
// written) return an unwrapped error so the caller can Nack and redeliver.
-func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, payload map[string]any) error {
+func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {
if s == nil || s.taskDAO == nil || s.memories == nil {
return fmt.Errorf("%w: memory: nil MemoryMessageService or memory dependency", ErrMemoryTaskTerminal)
}
- taskID, _ := payload["id"].(string)
- if taskID == "" {
- taskID, _ = payload["task_id"].(string)
- }
memoryID, _ := payload["memory_id"].(string)
sourceID := payloadInt64(payload["source_id"])
msgDict, _ := payload["message_dict"].(map[string]any)
@@ -116,9 +123,22 @@ func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, paylo
if task.Progress == -1 {
return fmt.Errorf("%w: memory: task %s is already failed", ErrMemoryTaskTerminal, taskID)
}
+ // A task already extracted to completion (progress>=1.0) is a durable
+ // terminal outcome: a redelivery after restart (or a duplicate copy from
+ // another consumer) must not re-run the LLM extraction, which would insert
+ // duplicate memory entries. Short-circuit to success so the Ingestor Acks
+ // the message instead of re-executing the task.
+ if task.Progress >= 1.0 {
+ return nil
+ }
if err = s.saveExtractedToMemory(ctx, memoryID, msg, sourceID, taskID); err != nil {
- s.updateTaskProgress(ctx, taskID, -1, err.Error())
+ if errors.Is(err, errMemoryTaskRetryable) {
+ return err
+ }
+ if progressErr := s.updateTaskProgress(ctx, taskID, -1, err.Error()); progressErr != nil {
+ return fmt.Errorf("%w: memory: mark task %s failed: %v", errMemoryTaskRetryable, taskID, progressErr)
+ }
return fmt.Errorf("%w: %w", ErrMemoryTaskTerminal, err)
}
return nil
@@ -138,7 +158,9 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memory
}
extractTypes := getTypesToExtract(memoryTypes)
if len(extractTypes) == 0 {
- s.updateTaskProgress(ctx, taskID, 1.0, fmt.Sprintf("Memory '%s' don't need to extract.", memoryID))
+ if err := s.updateTaskProgress(ctx, taskID, 1.0, fmt.Sprintf("Memory '%s' don't need to extract.", memoryID)); err != nil {
+ return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)
+ }
return nil
}
@@ -147,10 +169,12 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memory
return err
}
if len(extracted) == 0 {
- s.updateTaskProgress(ctx, taskID, 1.0, "No memory extracted from raw message.")
+ if err := s.updateTaskProgress(ctx, taskID, 1.0, "No memory extracted from raw message."); err != nil {
+ return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)
+ }
return nil
}
- s.updateTaskProgress(ctx, taskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extracted)))
+ _ = s.updateTaskProgress(ctx, taskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extracted)))
// conversation_time is stamped as server-local wall clock, not UTC.
now := memoryNow()
@@ -161,7 +185,9 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memory
if err = s.embedAndSaveMessages(ctx, mem, messages); err != nil {
return err
}
- s.updateTaskProgress(ctx, taskID, 1.0, "Message saved successfully.")
+ if err := s.updateTaskProgress(ctx, taskID, 1.0, "Message saved successfully."); err != nil {
+ return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)
+ }
return nil
}
@@ -201,7 +227,7 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo
}
chatModel := models.NewChatModel(driver, &modelName, apiConfig)
- s.updateTaskProgress(ctx, taskID, 0.15, "Prepared prompts and LLM.")
+ _ = s.updateTaskProgress(ctx, taskID, 0.15, "Prepared prompts and LLM.")
temperature := mem.Temperature
resp, err := chatModel.ModelDriver.ChatWithMessages(ctx, modelName, messages, apiConfig, &models.ChatConfig{Temperature: &temperature}, nil)
if err != nil {
@@ -210,7 +236,7 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo
if resp == nil || resp.Answer == nil {
return nil, errors.New("empty response from chat model")
}
- s.updateTaskProgress(ctx, taskID, 0.35, "Get extracted result from LLM.")
+ _ = s.updateTaskProgress(ctx, taskID, 0.35, "Get extracted result from LLM.")
return parseMemoryExtraction(*resp.Answer, extractTypes), nil
}
@@ -221,11 +247,30 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo
// logical message fields are set here; the doc engine maps them to
// storage fields (including tokenization) at insert time.
func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory, now time.Time) map[string]any {
- var invalidAt any
+ validAt, hasExplicitValidAt := normalizeMemoryTime(item.ValidAt)
+ if !hasExplicitValidAt {
+ validAt = now.Format(memoryTimeLayout)
+ }
+ invalidAt, hasExplicitInvalidAt := normalizeMemoryTime(item.InvalidAt)
if strings.TrimSpace(item.InvalidAt) != "" {
- invalidAt = formatMemoryTime(item.InvalidAt, now)
+ if !hasExplicitInvalidAt {
+ invalidAt = now.Format(memoryTimeLayout)
+ }
+ }
+ var storedInvalidAt any
+ if invalidAt != "" {
+ storedInvalidAt = invalidAt
+ }
+ fingerprintValidAt := ""
+ if hasExplicitValidAt {
+ fingerprintValidAt = validAt
+ }
+ fingerprintInvalidAt := ""
+ if hasExplicitInvalidAt {
+ fingerprintInvalidAt = invalidAt
}
return map[string]any{
+ "id": extractedMessageDocumentID(memoryID, sourceID, item.MessageType, item.Content, fingerprintValidAt, fingerprintInvalidAt),
"message_id": messageID,
"message_type": item.MessageType,
"source_id": sourceID,
@@ -234,13 +279,24 @@ func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg Memor
"agent_id": msg.AgentID,
"session_id": msg.SessionID,
"content": item.Content,
- "valid_at": formatMemoryTime(item.ValidAt, now),
- "invalid_at": invalidAt,
+ "valid_at": validAt,
+ "invalid_at": storedInvalidAt,
"forget_at": nil,
"status": true,
}
}
+// extractedMessageDocumentID returns the chunk-store id for an extracted item.
+// sourceID identifies the immutable raw message, while the normalized content
+// fingerprint distinguishes multiple extracts from that source. Only explicit,
+// parseable timestamps participate, so a runtime fallback cannot change the
+// identity on retry. Repeated task executions therefore upsert the same chunk
+// instead of creating duplicates.
+func extractedMessageDocumentID(memoryID string, sourceID int64, messageType, content, validAt, invalidAt string) string {
+ fingerprint := sha256.Sum256([]byte(strings.Join([]string{messageType, content, validAt, invalidAt}, "\x00")))
+ return fmt.Sprintf("%s_%d_%x", memoryID, sourceID, fingerprint[:8])
+}
+
// parseMemoryExtraction ports memory.utils.msg_util.get_json_result_from_llm_response
// plus the per-type flattening in extract_by_llm. Only the configured
// extract types are collected; unparseable responses yield an empty list.
@@ -288,28 +344,43 @@ func parseMemoryExtraction(answer string, extractTypes []string) []extractedMemo
// back to the supplied time formatted in its own location (server-local
// when callers pass time.Now()).
func formatMemoryTime(value string, fallback time.Time) string {
+ if normalized, ok := normalizeMemoryTime(value); ok {
+ return normalized
+ }
+ return fallback.Format(memoryTimeLayout)
+}
+
+// normalizeMemoryTime parses an explicit LLM-supplied timestamp without a
+// runtime fallback. Its result is suitable for a stable document fingerprint.
+func normalizeMemoryTime(value string) (string, bool) {
v := strings.TrimSpace(value)
- if v != "" {
- for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"} {
- if t, err := time.Parse(layout, v); err == nil {
- return t.Format(memoryTimeLayout)
- }
+ if v == "" {
+ return "", false
+ }
+ for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"} {
+ if t, err := time.Parse(layout, v); err == nil {
+ return t.Format(memoryTimeLayout), true
}
}
- return fallback.Format(memoryTimeLayout)
+ return "", false
}
// updateTaskProgress stamps and persists task progress, mirroring
-// Python TaskService.update_progress call sites. Failures are logged
-// and swallowed so progress reporting never breaks extraction.
-func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID string, progress float64, msg string) {
- if s == nil || s.taskDAO == nil || taskID == "" {
- return
+// Python TaskService.update_progress call sites. Callers that establish a
+// terminal task state must handle an error so the message can be retried.
+func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID string, progress float64, msg string) error {
+ if s == nil || s.taskDAO == nil {
+ return errors.New("memory: nil task DAO")
+ }
+ if taskID == "" {
+ return errors.New("memory: empty task id")
}
stamped := time.Now().Format(memoryTimeLayout) + " " + msg
if err := s.taskDAO.UpdateProgress(ctx, dao.DB, taskID, progress, stamped); err != nil {
common.Warn("memory: update task progress failed", zap.Error(err))
+ return err
}
+ return nil
}
func payloadInt64(v any) int64 {
diff --git a/internal/service/memory_extractor_test.go b/internal/service/memory_extractor_test.go
index 8ea40384df..8357b10774 100644
--- a/internal/service/memory_extractor_test.go
+++ b/internal/service/memory_extractor_test.go
@@ -23,6 +23,10 @@ import (
"strings"
"testing"
"time"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
)
// pinMemoryNow fixes the memory wall clock at the given instant for the
@@ -95,6 +99,61 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {
}
}
+// TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries ensures fallback
+// timestamps do not become part of the chunk-store identity. A retry may run
+// at a different time and allocate a new logical message id, but it must still
+// overwrite the same extracted document.
+func TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries(t *testing.T) {
+ msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}
+ firstAttempt := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)
+ retryAttempt := firstAttempt.Add(10 * time.Second)
+
+ for _, test := range []struct {
+ name string
+ item extractedMemory
+ }{
+ {
+ name: "omitted valid at",
+ item: extractedMemory{MessageType: "fact", Content: "likes coffee"},
+ },
+ {
+ name: "invalid timestamps",
+ item: extractedMemory{
+ MessageType: "fact",
+ Content: "likes coffee",
+ ValidAt: "not a timestamp",
+ InvalidAt: "also not a timestamp",
+ },
+ },
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ first := buildExtractedMessage(8, 42, "mem-1", msg, test.item, firstAttempt)
+ retry := buildExtractedMessage(9, 42, "mem-1", msg, test.item, retryAttempt)
+
+ firstID, ok := first["id"].(string)
+ if !ok || firstID == "" {
+ t.Fatalf("first extracted message id = %#v, want a stable non-empty string", first["id"])
+ }
+ if got, ok := retry["id"].(string); !ok || got != firstID {
+ t.Fatalf("retry extracted message id = %#v, want %q", retry["id"], firstID)
+ }
+ })
+ }
+}
+
+// TestUpdateTaskProgressReturnsPersistenceError ensures callers can keep the
+// broker message retryable when they cannot persist the terminal task state.
+func TestUpdateTaskProgressReturnsPersistenceError(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.IngestionTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ svc := &MemoryMessageService{taskDAO: dao.NewTaskDAO()}
+ if err := svc.updateTaskProgress(t.Context(), "mem-task-1", 1.0, "completed"); err == nil {
+ t.Fatal("updateTaskProgress() error = nil, want persistence error")
+ }
+}
+
// TestTypeInstructionsStayInLockstepWithPython pins the rules that
// memory/utils/prompt_util.py carries and the Go map historically dropped:
// per-type Examples lines, the full "Timestamp Rules for <Type> Knowledge:"
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index adb1718d82..201c712b82 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -170,7 +170,7 @@ func (s *MemoryMessageService) QueueSaveToMemoryTask(ctx context.Context, memory
})
continue
}
- if err = queueMemoryTask(ctx, memoryID, mem.TenantID, rawMessageID, task, msg); err != nil {
+ if err = queueMemoryTask(ctx, fmt.Sprint(task["id"]), memoryID, mem.TenantID, rawMessageID, msg); err != nil {
res.Failed = append(res.Failed, MemoryFailure{
MemoryID: memoryID,
FailMsg: err.Error(),
@@ -278,7 +278,9 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
}
vectorDim = len(vector)
message[fmt.Sprintf("q_%d_vec", len(vector))] = vector
- message["id"] = fmt.Sprintf("%s_%v", message["memory_id"], message["message_id"])
+ if id, ok := message["id"].(string); !ok || id == "" {
+ message["id"] = fmt.Sprintf("%s_%v", message["memory_id"], message["message_id"])
+ }
message["doc_id"] = message["memory_id"]
}
@@ -339,12 +341,14 @@ func taskFromRow(row map[string]any) *entity.Task {
}
}
-func queueMemoryTask(ctx context.Context, memoryID, tenantID string, rawMessageID int64, task map[string]any, msg MemoryMessage) error {
- taskID := fmt.Sprint(task["id"])
+// queueMemoryTask publishes a memory-extraction task to NATS (tasks.RAGFLOW).
+// taskID is the durable task row's id (buildTaskRow) and the sole identity on
+// the envelope; the payload carries only business parameters — memory_id,
+// source_id, and the dialogue to extract. Identity is deliberately NOT
+// duplicated into the payload so the consumer never has two ids that could
+// disagree (the envelope TaskID is authoritative end to end).
+func queueMemoryTask(ctx context.Context, taskID, memoryID, tenantID string, rawMessageID int64, msg MemoryMessage) error {
message := map[string]any{
- "id": taskID,
- "task_id": taskID,
- "task_type": task["task_type"],
"memory_id": memoryID,
"tenant_id": tenantID,
"source_id": rawMessageID,
diff --git a/internal/service/memory_queue_task_test.go b/internal/service/memory_queue_task_test.go
index 771e84b176..37f1e86d64 100644
--- a/internal/service/memory_queue_task_test.go
+++ b/internal/service/memory_queue_task_test.go
@@ -19,10 +19,10 @@ func TestQueueMemoryTaskUninitializedQueueReturnsError(t *testing.T) {
err := queueMemoryTask(
t.Context(),
+ "task-1",
"mem-1",
"tenant-1",
1,
- map[string]any{"id": "task-1", "task_type": "memory"},
MemoryMessage{UserID: "u1", AgentID: "agent-1", SessionID: "s1", UserInput: "hi", AgentResponse: "hello"},
)
if err == nil {