RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(nats): use DiscardNew for all JetStream work-queue streams (#19099)
jay77721b7ce6a3708b2
diff --git a/internal/engine/nats/knowledgecompile.go b/internal/engine/nats/knowledgecompile.goindex c2a7be41a4..330a199f94 100644--- a/internal/engine/nats/knowledgecompile.go+++ b/internal/engine/nats/knowledgecompile.go@@ -58,6 +58,7 @@ func (n *NatsEngine) InitKnowledgeCompileStream() error {Subjects: []string{knowledgeCompileSubjectPrefix},Retention: jetstream.WorkQueuePolicy,Storage: jetstream.FileStorage,+ Discard: jetstream.DiscardNew,MaxMsgs: 1024 * 1024,MaxBytes: 1024 * 1024 * 64,})diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.goindex 66700a67a2..9117bb19e2 100644--- a/internal/engine/nats/nats.go+++ b/internal/engine/nats/nats.go@@ -79,6 +79,7 @@ func (n *NatsEngine) Init() error {Subjects: []string{"tasks.>"},Retention: jetstream.WorkQueuePolicy,Storage: jetstream.FileStorage,+ Discard: jetstream.DiscardNew,MaxMsgs: 1024 * 128,MaxBytes: 1024 * 1024 * 64,// Server-side dedup window. Inert for task publishes: PublishTaskdiff --git a/internal/engine/nats/stream_config_test.go b/internal/engine/nats/stream_config_test.goindex 331fb44ce6..d6baf63875 100644--- a/internal/engine/nats/stream_config_test.go+++ b/internal/engine/nats/stream_config_test.go@@ -139,13 +139,17 @@ func TestInitMigratesLegacyStreamConfig(t *testing.T) {t.Fatalf("legacy stream info: %v", err)}// Precondition: the legacy stream differs from the wanted config (server- // fills a 2m default Duplicates when unspecified).+ // fills a 2m default Duplicates when unspecified, and Discard defaults to+ // DiscardOld).if legacyInfo.Config.MaxBytes != int64(1024*1024) {t.Fatalf("precondition: legacy MaxBytes = %d, want 1MB", legacyInfo.Config.MaxBytes)}if legacyInfo.Config.Duplicates == 10*time.Minute {t.Fatalf("precondition: legacy Duplicates already migrated (%v)", legacyInfo.Config.Duplicates)}+ if legacyInfo.Config.Discard != jetstream.DiscardOld {+ t.Fatalf("precondition: legacy Discard = %v, want DiscardOld", legacyInfo.Config.Discard)+ }engine := NewNatsEngine(host, port)if err := engine.Init(); err != nil {@@ -162,6 +166,9 @@ func TestInitMigratesLegacyStreamConfig(t *testing.T) {if got := info.Config.Duplicates; got != 10*time.Minute {t.Fatalf("Duplicates after migration = %v, want 10m", got)}+ if got := info.Config.Discard; got != jetstream.DiscardNew {+ t.Fatalf("Discard after migration = %v, want DiscardNew", got)+ }// Non-owned fields must not be reset by the partial update.if got := info.Config.Retention; got != jetstream.WorkQueuePolicy {t.Fatalf("Retention after migration = %v, want WorkQueuePolicy (must not be reset)", got)diff --git a/internal/engine/nats/stream_migrate.go b/internal/engine/nats/stream_migrate.goindex 8c7d91ff73..4dabf3222c 100644--- a/internal/engine/nats/stream_migrate.go+++ b/internal/engine/nats/stream_migrate.go@@ -26,10 +26,10 @@ import (// ensureStreamConfig creates the stream, or - when it already exists with an// older configuration (e.g. created by a previous deployment) - migrates it in// place via UpdateStream instead of failing with "stream name already in-// use". Only the capacity/dedup fields (MaxBytes, MaxMsgs, Duplicates) are-// migrated; the server-side current config is the merge base so a partial-// update can never reset fields this helper does not own (Subjects, Retention,-// Storage, ...).+// use". Only the capacity/dedup/discard fields (MaxBytes, MaxMsgs, Duplicates,+// Discard) are migrated; the server-side current config is the merge base so a+// partial update can never reset fields this helper does not own (Subjects,+// Retention, Storage, ...).func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstream.StreamConfig) (jetstream.Stream, error) {st, err := js.CreateStream(ctx, want)if err == nil {@@ -51,6 +51,9 @@ func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstrif want.Duplicates != 0 && cur.Duplicates != want.Duplicates {needUpdate = true}+ if want.Discard != jetstream.DiscardOld && cur.Discard != want.Discard {+ needUpdate = true+ }if needUpdate {merged := curmerged.MaxBytes = want.MaxBytes@@ -58,6 +61,9 @@ func ensureStreamConfig(ctx context.Context, js jetstream.JetStream, want jetstrif want.Duplicates != 0 {merged.Duplicates = want.Duplicates}+ if want.Discard != jetstream.DiscardOld {+ merged.Discard = want.Discard+ }st, err = js.UpdateStream(ctx, merged)if err != nil {return nil, errdiff --git a/internal/engine/nats/syncer.go b/internal/engine/nats/syncer.goindex 13a93baae1..e2253b900c 100644--- a/internal/engine/nats/syncer.go+++ b/internal/engine/nats/syncer.go@@ -67,6 +67,7 @@ func (n *NatsEngine) initSyncerStreamLocked() error {Subjects: []string{syncerSubjectPattern},Retention: jetstream.WorkQueuePolicy,Storage: jetstream.FileStorage,+ Discard: jetstream.DiscardNew,MaxMsgs: 1024 * 128,MaxBytes: 1024 * 1024 * 64,Duplicates: 10 * time.Minute,diff --git a/internal/harness/events/nats.go b/internal/harness/events/nats.goindex a7470e3721..1e5d94d7e0 100644--- a/internal/harness/events/nats.go+++ b/internal/harness/events/nats.go@@ -58,6 +58,7 @@ func NewNATSEventStore(conn *nats.Conn, stream string) (*NATSEventStore, error)MaxAge: 7 * 24 * time.Hour, // 7 days retentionStorage: jetstream.FileStorage,Retention: jetstream.LimitsPolicy,+ Discard: jetstream.DiscardNew,})if err != nil {return nil, fmt.Errorf("create jetstream stream: %w", err)