Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(nats): use DiscardNew for all JetStream work-queue streams (#19099)

jay77721b7ce6a3708b2
diff --git a/internal/engine/nats/knowledgecompile.go b/internal/engine/nats/knowledgecompile.go
index c2a7be41a4..330a199f94 100644
--- a/internal/engine/nats/knowledgecompile.go
+++ b/internal/engine/nats/knowledgecompile.go
@@ -58,6 +58,7 @@ func (n *NatsEngine) InitKnowledgeCompileStream() error {
Subjects: []string{knowledgeCompileSubjectPrefix},
Retention: jetstream.WorkQueuePolicy,
Storage: jetstream.FileStorage,
+ Discard: jetstream.DiscardNew,
MaxMsgs: 1024 * 1024,
MaxBytes: 1024 * 1024 * 64,
})
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.go
index 66700a67a2..9117bb19e2 100644
--- a/internal/engine/nats/nats.go
+++ b/internal/engine/nats/nats.go
@@ -79,6 +79,7 @@ func (n *NatsEngine) Init() error {
Subjects: []string{"tasks.>"},
Retention: jetstream.WorkQueuePolicy,
Storage: jetstream.FileStorage,
+ Discard: jetstream.DiscardNew,
MaxMsgs: 1024 * 128,
MaxBytes: 1024 * 1024 * 64,
// Server-side dedup window. Inert for task publishes: PublishTask
diff --git a/internal/engine/nats/stream_config_test.go b/internal/engine/nats/stream_config_test.go
index 331fb44ce6..d6baf63875 100644
--- a/internal/engine/nats/stream_config_test.go
+++ b/internal/engine/nats/stream_config_test.go
@@ -139,13 +139,17 @@ func TestInitMigratesLegacyStreamConfig(t *testing.T) {
t.Fatalf("legacy stream info: %v", err)
}
// Precondition: the legacy stream differs from the wanted config (server
- // fills a 2m default Duplicates when unspecified).
+ // fills a 2m default Duplicates when unspecified, and Discard defaults to
+ // DiscardOld).
if legacyInfo.Config.MaxBytes != int64(1024*1024) {
t.Fatalf("precondition: legacy MaxBytes = %d, want 1MB", legacyInfo.Config.MaxBytes)
}
if legacyInfo.Config.Duplicates == 10*time.Minute {
t.Fatalf("precondition: legacy Duplicates already migrated (%v)", legacyInfo.Config.Duplicates)
}
+ if legacyInfo.Config.Discard != jetstream.DiscardOld {
+ t.Fatalf("precondition: legacy Discard = %v, want DiscardOld", legacyInfo.Config.Discard)
+ }
engine := NewNatsEngine(host, port)
if err := engine.Init(); err != nil {
@@ -162,6 +166,9 @@ func TestInitMigratesLegacyStreamConfig(t *testing.T) {
if got := info.Config.Duplicates; got != 10*time.Minute {
t.Fatalf("Duplicates after migration = %v, want 10m", got)
}
+ if got := info.Config.Discard; got != jetstream.DiscardNew {
+ t.Fatalf("Discard after migration = %v, want DiscardNew", got)
+ }
// Non-owned fields must not be reset by the partial update.
if got := info.Config.Retention; got != jetstream.WorkQueuePolicy {
t.Fatalf("Retention after migration = %v, want WorkQueuePolicy (must not be reset)", got)
diff --git a/internal/engine/nats/stream_migrate.go b/internal/engine/nats/stream_migrate.go
index 8c7d91ff73..4dabf3222c 100644
--- a/internal/engine/nats/stream_migrate.go
+++ b/internal/engine/nats/stream_migrate.go
@@ -26,10 +26,10 @@ import (
// ensureStreamConfig creates the stream, or - when it already exists with an
// older configuration (e.g. created by a previous deployment) - migrates it in
// place via UpdateStream instead of failing with "stream name already in
-// use". Only the capacity/dedup fields (MaxBytes, MaxMsgs, Duplicates) are
-// migrated; the server-side current config is the merge base so a partial
-// update can never reset fields this helper does not own (Subjects, Retention,
-// Storage, ...).
+// use". Only the capacity/dedup/discard fields (MaxBytes, MaxMsgs, Duplicates,
+// Discard) are migrated; the server-side current config is the merge base so a
+// partial update can never reset fields this helper does not own (Subjects,
+// Retention, Storage, ...).
func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstream.StreamConfig) (jetstream.Stream, error) {
st, err := js.CreateStream(ctx, want)
if err == nil {
@@ -51,6 +51,9 @@ func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstr
if want.Duplicates != 0 && cur.Duplicates != want.Duplicates {
needUpdate = true
}
+ if want.Discard != jetstream.DiscardOld && cur.Discard != want.Discard {
+ needUpdate = true
+ }
if needUpdate {
merged := cur
merged.MaxBytes = want.MaxBytes
@@ -58,6 +61,9 @@ func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstr
if want.Duplicates != 0 {
merged.Duplicates = want.Duplicates
}
+ if want.Discard != jetstream.DiscardOld {
+ merged.Discard = want.Discard
+ }
st, err = js.UpdateStream(ctx, merged)
if err != nil {
return nil, err
diff --git a/internal/engine/nats/syncer.go b/internal/engine/nats/syncer.go
index 13a93baae1..e2253b900c 100644
--- a/internal/engine/nats/syncer.go
+++ b/internal/engine/nats/syncer.go
@@ -67,6 +67,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {
Subjects: []string{syncerSubjectPattern},
Retention: jetstream.WorkQueuePolicy,
Storage: jetstream.FileStorage,
+ Discard: jetstream.DiscardNew,
MaxMsgs: 1024 * 128,
MaxBytes: 1024 * 1024 * 64,
Duplicates: 10 * time.Minute,
diff --git a/internal/harness/events/nats.go b/internal/harness/events/nats.go
index a7470e3721..1e5d94d7e0 100644
--- a/internal/harness/events/nats.go
+++ b/internal/harness/events/nats.go
@@ -58,6 +58,7 @@ func NewNATSEventStore(conn *nats.Conn, stream string) (*NATSEventStore, error)
MaxAge: 7 * 24 * time.Hour, // 7 days retention
Storage: jetstream.FileStorage,
Retention: jetstream.LimitsPolicy,
+ Discard: jetstream.DiscardNew,
})
if err != nil {
return nil, fmt.Errorf("create jetstream stream: %w", err)