RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(nats): add shared NatsKVStore helper for cache namespaces (#19148)
xugangqiangbcbf96075d74
diff --git a/internal/engine/nats/kvstore.go b/internal/engine/nats/kvstore.gonew file mode 100644index 0000000000..d61000308d--- /dev/null+++ b/internal/engine/nats/kvstore.go@@ -0,0 +1,152 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package nats++import (+ "context"+ "errors"+ "fmt"+ "time"++ "github.com/nats-io/nats.go/jetstream"+)++// Default limits applied to a cache namespace bucket when the caller does not+// specify them. A cache entry only needs the latest revision, so History=1 and+// a 1 MiB value cap are sane defaults; Storage defaults to FileStorage so the+// cache survives restarts and is not bounded by RAM.+const (+ defaultKVHistory = 1+ defaultKVMaxValueSize = 1024 * 1024+)++// BucketConfig describes the policy for one cache namespace's KV bucket. It is a+// thin, ergonomic wrapper over jetstream.KeyValueConfig: zero values for+// History/MaxValueSize/Storage fall back to cache-appropriate defaults in+// NewNatsKVStore rather than to the JetStream library defaults.+type BucketConfig struct {+ Name string+ Description string+ History int // cache: 1 (keep only the latest revision)+ TTL time.Duration+ MaxValueSize int // OCR: 8 MiB; others: 1 MiB default+ Storage jetstream.StorageType+ Replicas int // cache: 1 (rebuildable)+ Compression bool+}++// NatsKVStore is a thin wrapper over a JetStream KV bucket. It normalizes the+// not-found and delete semantics used across the cache namespaces (ocr,+// embedding, checkpoint) so each caller shares one bucket-management path.+type NatsKVStore struct {+ kv jetstream.KeyValue+}++// NewNatsKVStore creates or updates the backing bucket and returns a handle.+// It uses CreateOrUpdateKeyValue so re-configuring an existing bucket is+// idempotent and safe to call on every process start.+func NewNatsKVStore(ctx context.Context, js jetstream.JetStream, cfg BucketConfig) (*NatsKVStore, error) {+ if cfg.Name == "" {+ return nil, fmt.Errorf("NewNatsKVStore: bucket Name must not be empty")+ }+ if cfg.History == 0 {+ cfg.History = defaultKVHistory+ }+ storage := cfg.Storage+ if storage == 0 {+ storage = jetstream.FileStorage+ }+ if cfg.MaxValueSize == 0 {+ cfg.MaxValueSize = defaultKVMaxValueSize+ }++ kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{+ Bucket: cfg.Name,+ Description: cfg.Description,+ History: uint8(cfg.History),+ TTL: cfg.TTL,+ MaxValueSize: int32(cfg.MaxValueSize),+ Storage: storage,+ Replicas: cfg.Replicas,+ Compression: cfg.Compression,+ })+ if err != nil {+ return nil, fmt.Errorf("NewNatsKVStore: create bucket %q: %w", cfg.Name, err)+ }+ return &NatsKVStore{kv: kv}, nil+}++// Put writes value for key, overwriting any existing value.+func (s *NatsKVStore) Put(ctx context.Context, key string, val []byte) error {+ if _, err := s.kv.Put(ctx, key, val); err != nil {+ return fmt.Errorf("NatsKVStore.Put: %w", err)+ }+ return nil+}++// Get returns (value, found, error). A missing key is surfaced as+// found=false with a nil error, never as ErrKeyNotFound.+func (s *NatsKVStore) Get(ctx context.Context, key string) ([]byte, bool, error) {+ entry, err := s.kv.Get(ctx, key)+ if err != nil {+ if errors.Is(err, jetstream.ErrKeyNotFound) {+ return nil, false, nil+ }+ return nil, false, fmt.Errorf("NatsKVStore.Get: %w", err)+ }+ return entry.Value(), true, nil+}++// Delete removes key. Deleting a missing key is treated as success (idempotent)+// and does not return ErrKeyNotFound.+func (s *NatsKVStore) Delete(ctx context.Context, key string) error {+ if err := s.kv.Delete(ctx, key); err != nil {+ if errors.Is(err, jetstream.ErrKeyNotFound) {+ return nil+ }+ return fmt.Errorf("NatsKVStore.Delete: %w", err)+ }+ return nil+}++// Keys lists all keys currently in the bucket. It returns a non-nil, empty+// slice when the bucket has no keys, so callers can safely range over or take+// len() of the result.+func (s *NatsKVStore) Keys(ctx context.Context) ([]string, error) {+ lister, err := s.kv.ListKeys(ctx)+ if err != nil {+ return nil, fmt.Errorf("NatsKVStore.Keys: %w", err)+ }+ defer lister.Stop()++ keys := make([]string, 0)+ for k := range lister.Keys() {+ keys = append(keys, k)+ }+ return keys, nil+}++// EnsureKVBucket creates or updates a cache bucket on this engine and returns a+// ready-to-use NatsKVStore. It does not reuse the engine's dedicated+// knowledge-compile lease handle (NatsEngine.kv); the two are independent+// buckets.+func (n *NatsEngine) EnsureKVBucket(ctx context.Context, cfg BucketConfig) (*NatsKVStore, error) {+ if n.jetStream == nil {+ return nil, fmt.Errorf("EnsureKVBucket: jetStream not initialized (call Init first)")+ }+ return NewNatsKVStore(ctx, n.jetStream, cfg)+}diff --git a/internal/engine/nats/kvstore_test.go b/internal/engine/nats/kvstore_test.gonew file mode 100644index 0000000000..3b0da9978e--- /dev/null+++ b/internal/engine/nats/kvstore_test.go@@ -0,0 +1,503 @@+package nats++import (+ "context"+ "errors"+ "slices"+ "sync"+ "testing"+ "time"++ "github.com/nats-io/nats.go/jetstream"+)++// setupNATSEngine is a thin alias for the white-box helper defined in+// syncer_test.go (it boots an embedded NATS server with JetStream and returns an+// initialized NatsEngine). We cannot use testutil.SetupNatsEngine here because+// that package imports this one, which would create an import cycle.+func setupNATSEngine(t *testing.T) *NatsEngine {+ t.Helper()+ return setupSyncerNATSEngine(t)+}++// newTestKVStore boots an embedded engine and creates a uniquely-named bucket+// for the calling test.+func newTestKVStore(t *testing.T, cfg BucketConfig) (*NatsKVStore, *NatsEngine) {+ t.Helper()+ engine := setupNATSEngine(t)+ if cfg.Name == "" {+ cfg.Name = "kvstore_test_" + sanitizeTestName(t.Name())+ }+ store, err := engine.EnsureKVBucket(t.Context(), cfg)+ if err != nil {+ t.Fatalf("EnsureKVBucket: %v", err)+ }+ return store, engine+}++func TestNatsKVStorePutGetHappyPath(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ const key = "ocr.page1.hash-a1b2c3"+ want := []byte("ocr-result-bytes")++ if err := store.Put(t.Context(), key, want); err != nil {+ t.Fatalf("Put: %v", err)+ }++ got, found, err := store.Get(t.Context(), key)+ if err != nil {+ t.Fatalf("Get: %v", err)+ }+ if !found {+ t.Fatal("Get returned found=false for an existing key")+ }+ if string(got) != string(want) {+ t.Fatalf("Get value = %q, want %q", got, want)+ }+}++func TestNatsKVStoreGetMissingKey(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ got, found, err := store.Get(t.Context(), "never.written.key")+ if err != nil {+ t.Fatalf("Get on missing key returned error: %v", err)+ }+ if found {+ t.Fatal("Get on missing key returned found=true")+ }+ if got != nil {+ t.Fatalf("Get on missing key returned non-nil value: %v", got)+ }+}++func TestNatsKVStoreDeleteMissingKeyIsIdempotent(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ if err := store.Delete(t.Context(), "never.written.key"); err != nil {+ t.Fatalf("Delete on missing key returned error (should be nil): %v", err)+ }+ // Deleting the same missing key a second time must also be nil.+ if err := store.Delete(t.Context(), "never.written.key"); err != nil {+ t.Fatalf("second Delete on missing key returned error: %v", err)+ }+}++func TestNatsKVStoreDeleteExistingKey(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})+ const key = "emb.model-x.4096.text-hash"++ if err := store.Put(t.Context(), key, []byte("v1")); err != nil {+ t.Fatalf("Put: %v", err)+ }+ if err := store.Delete(t.Context(), key); err != nil {+ t.Fatalf("Delete existing key: %v", err)+ }+ _, found, err := store.Get(t.Context(), key)+ if err != nil {+ t.Fatalf("Get after delete: %v", err)+ }+ if found {+ t.Fatal("key still found after Delete")+ }+}++func TestNatsKVStoreKeysListsWrittenKeys(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ want := []string{"a.1", "b.2", "c.3"}+ for _, k := range want {+ if err := store.Put(t.Context(), k, []byte("x")); err != nil {+ t.Fatalf("Put %q: %v", k, err)+ }+ }++ got, err := store.Keys(t.Context())+ if err != nil {+ t.Fatalf("Keys: %v", err)+ }+ if len(got) != len(want) {+ t.Fatalf("Keys returned %d keys, want %d (%v)", len(got), len(want), got)+ }+ slices.Sort(got)+ slices.Sort(want)+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("Keys mismatch: got %v want %v", got, want)+ }+ }+}++func TestNatsKVStoreKeysEmptyBucket(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ got, err := store.Keys(t.Context())+ if err != nil {+ t.Fatalf("Keys on empty bucket: %v", err)+ }+ if got == nil {+ t.Fatal("Keys on empty bucket returned nil slice; want non-nil empty slice")+ }+ if len(got) != 0 {+ t.Fatalf("Keys on empty bucket returned %v, want empty", got)+ }+}++func TestNatsKVStoreOverwriteKeepsLatest(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{History: 1})+ const key = "checkpoint.user-1"++ if err := store.Put(t.Context(), key, []byte("v1")); err != nil {+ t.Fatalf("Put v1: %v", err)+ }+ if err := store.Put(t.Context(), key, []byte("v2")); err != nil {+ t.Fatalf("Put v2: %v", err)+ }++ got, found, err := store.Get(t.Context(), key)+ if err != nil {+ t.Fatalf("Get: %v", err)+ }+ if !found {+ t.Fatal("overwritten key not found")+ }+ if string(got) != "v2" {+ t.Fatalf("Get = %q, want latest %q", got, "v2")+ }++ // With History=1 the bucket keeps exactly one revision, so Keys must show+ // the key exactly once, not once per revision.+ keys, err := store.Keys(t.Context())+ if err != nil {+ t.Fatalf("Keys: %v", err)+ }+ if len(keys) != 1 {+ t.Fatalf("Keys after overwrite = %v, want exactly 1 key", keys)+ }+}++func TestNatsKVStoreEmptyValue(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})+ const key = "ocr.empty"++ if err := store.Put(t.Context(), key, []byte{}); err != nil {+ t.Fatalf("Put empty value: %v", err)+ }+ got, found, err := store.Get(t.Context(), key)+ if err != nil {+ t.Fatalf("Get: %v", err)+ }+ if !found {+ t.Fatal("key with empty value not found")+ }+ if len(got) != 0 {+ t.Fatalf("Get empty value = %v (len %d), want len 0", got, len(got))+ }+}++func TestNatsKVStoreSpecialKeyNames(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ keys := []string{+ "ocr.parser-config.image-hash", // dots and dashes+ "emb_model-x_dim-4096_text-hash", // underscores+ "12345", // purely numeric+ "aVeryLongKey" + repeat("x", 200), // 200+ char key+ }+ for _, k := range keys {+ if err := store.Put(t.Context(), k, []byte("v")); err != nil {+ t.Fatalf("Put %q: %v", k, err)+ }+ }++ got, err := store.Keys(t.Context())+ if err != nil {+ t.Fatalf("Keys: %v", err)+ }+ if len(got) != len(keys) {+ t.Fatalf("Keys = %v, want %d keys", got, len(keys))+ }+}++func TestNatsKVStoreDeleteRemovesFromKeys(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})+ const keep = "keep.me"+ const drop = "drop.me"++ if err := store.Put(t.Context(), keep, []byte("k")); err != nil {+ t.Fatalf("Put keep: %v", err)+ }+ if err := store.Put(t.Context(), drop, []byte("d")); err != nil {+ t.Fatalf("Put drop: %v", err)+ }+ if err := store.Delete(t.Context(), drop); err != nil {+ t.Fatalf("Delete drop: %v", err)+ }++ got, err := store.Keys(t.Context())+ if err != nil {+ t.Fatalf("Keys: %v", err)+ }+ if slices.Contains(got, drop) {+ t.Fatalf("deleted key %q still present in Keys: %v", drop, got)+ }+ if !slices.Contains(got, keep) {+ t.Fatalf("kept key %q missing from Keys: %v", keep, got)+ }+}++func TestNatsKVStoreDeleteInvalidKeyReturnsError(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ // A key that is not a valid NATS subject (contains a space) must surface a+ // concrete error from the underlying store, NOT be silently treated as a+ // successful delete of a missing key.+ err := store.Delete(t.Context(), "bad key with space")+ if err == nil {+ t.Fatal("Delete with invalid key returned nil error; want error")+ }+ if errors.Is(err, jetstream.ErrKeyNotFound) {+ t.Fatalf("Delete with invalid key surfaced as ErrKeyNotFound: %v", err)+ }+}++func TestNatsKVStoreContextCancel(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})+ const key = "ctx.key"++ ctx, cancel := context.WithCancel(t.Context())+ cancel() // already canceled++ // A canceled context must surface as an error, NOT as a silent not-found.+ if _, _, err := store.Get(ctx, key); err == nil {+ t.Fatal("Get with canceled ctx returned nil error; want context error")+ }+ if err := store.Put(ctx, key, []byte("v")); err == nil {+ t.Fatal("Put with canceled ctx returned nil error; want context error")+ }+ if err := store.Delete(ctx, key); err == nil {+ t.Fatal("Delete with canceled ctx returned nil error; want context error")+ }+ if _, err := store.Keys(ctx); err == nil {+ t.Fatal("Keys with canceled ctx returned nil error; want context error")+ }+}++func TestNatsKVStoreNewIsIdempotent(t *testing.T) {+ engine := setupNATSEngine(t)+ cfg := BucketConfig{Name: "idempotent_bucket_" + sanitizeTestName(t.Name())}++ s1, err := engine.EnsureKVBucket(t.Context(), cfg)+ if err != nil {+ t.Fatalf("first EnsureKVBucket: %v", err)+ }+ s2, err := engine.EnsureKVBucket(t.Context(), cfg)+ if err != nil {+ t.Fatalf("second EnsureKVBucket (idempotent): %v", err)+ }++ // Both handles point at the same bucket; writing via s2 is visible via s1.+ if err := s2.Put(t.Context(), "k", []byte("v")); err != nil {+ t.Fatalf("Put via s2: %v", err)+ }+ if _, found, err := s1.Get(t.Context(), "k"); err != nil || !found {+ t.Fatalf("Get via s1: found=%v err=%v", found, err)+ }+}++func TestNatsKVStoreDefaultsApplied(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{Name: "defaults_bucket_" + sanitizeTestName(t.Name())})++ status, err := store.kv.Status(t.Context())+ if err != nil {+ t.Fatalf("Status: %v", err)+ }+ if got := status.History(); got != 1 {+ t.Fatalf("default History = %d, want 1", got)+ }+ if got := int(status.Config().MaxValueSize); got != 1024*1024 {+ t.Fatalf("default MaxValueSize = %d, want %d", got, 1024*1024)+ }+ if got := status.Config().Storage; got != jetstream.FileStorage {+ t.Fatalf("default Storage = %v, want FileStorage", got)+ }+}++func TestNatsKVStoreConfigApplied(t *testing.T) {+ cfg := BucketConfig{+ Name: "config_bucket_" + sanitizeTestName(t.Name()),+ Description: "unit-test bucket",+ History: 1,+ TTL: 7 * 24 * time.Hour,+ MaxValueSize: 8 * 1024 * 1024,+ Storage: jetstream.FileStorage,+ Replicas: 1,+ Compression: true,+ }+ store, _ := newTestKVStore(t, cfg)++ status, err := store.kv.Status(t.Context())+ if err != nil {+ t.Fatalf("Status: %v", err)+ }+ if got := status.History(); got != 1 {+ t.Fatalf("History = %d, want 1", got)+ }+ if got := status.TTL(); got != cfg.TTL {+ t.Fatalf("TTL = %v, want %v", got, cfg.TTL)+ }+ if got := int(status.Config().MaxValueSize); got != cfg.MaxValueSize {+ t.Fatalf("MaxValueSize = %d, want %d", got, cfg.MaxValueSize)+ }+ if got := status.Config().Storage; got != jetstream.FileStorage {+ t.Fatalf("Storage = %v, want FileStorage", got)+ }+ if got := status.Config().Replicas; got != 1 {+ t.Fatalf("Replicas = %d, want 1", got)+ }+ if !status.IsCompressed() {+ t.Fatal("Compression enabled but Status reports not compressed")+ }+ if got := status.Bucket(); got != cfg.Name {+ t.Fatalf("Bucket name = %q, want %q", got, cfg.Name)+ }+}++func TestNatsKVStoreMaxValueSizeEnforced(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{+ Name: "oversize_bucket_" + sanitizeTestName(t.Name()),+ MaxValueSize: 10,+ })++ // Exactly at the limit is allowed.+ if err := store.Put(t.Context(), "atlimit", make([]byte, 10)); err != nil {+ t.Fatalf("Put at limit (10 bytes): %v", err)+ }+ // One byte over the limit must be rejected.+ err := store.Put(t.Context(), "over", make([]byte, 11))+ if err == nil {+ t.Fatal("Put over MaxValueSize returned nil error; want rejection")+ }+ if errors.Is(err, jetstream.ErrKeyNotFound) {+ t.Fatalf("oversize Put surfaced as ErrKeyNotFound: %v", err)+ }+}++func TestNatsKVStoreTTLExpiry(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{+ Name: "ttl_bucket_" + sanitizeTestName(t.Name()),+ TTL: 2 * time.Second,+ })+ const key = "expiring.key"++ if err := store.Put(t.Context(), key, []byte("v")); err != nil {+ t.Fatalf("Put: %v", err)+ }++ deadline := time.Now().Add(15 * time.Second)+ for {+ _, found, err := store.Get(t.Context(), key)+ if err != nil {+ t.Fatalf("Get during TTL poll: %v", err)+ }+ if !found {+ return // expired as expected+ }+ if time.Now().After(deadline) {+ t.Fatal("key not expired within 15s despite 2s TTL")+ }+ time.Sleep(200 * time.Millisecond)+ }+}++func TestNatsKVStoreEnsureKVBucketRequiresInit(t *testing.T) {+ // NewNatsEngine without Init leaves jetStream nil.+ engine := NewNatsEngine("127.0.0.1", 4222)+ if _, err := engine.EnsureKVBucket(t.Context(), BucketConfig{Name: "x"}); err == nil {+ t.Fatal("EnsureKVBucket on uninitialized engine returned nil error; want error")+ }+}++func TestNatsKVStoreNewRejectsEmptyName(t *testing.T) {+ // Empty bucket name must be rejected before any JetStream call. A nil js is+ // therefore safe to pass here.+ if _, err := NewNatsKVStore(t.Context(), nil, BucketConfig{Name: ""}); err == nil {+ t.Fatal("NewNatsKVStore with empty Name returned nil error; want error")+ }+}++func TestNatsKVStoreConcurrentPuts(t *testing.T) {+ store, _ := newTestKVStore(t, BucketConfig{})++ const n = 50+ var wg sync.WaitGroup+ wg.Add(n)+ for i := 0; i < n; i++ {+ go func(i int) {+ defer wg.Done()+ key := "concurrent." + itoa(i)+ if err := store.Put(t.Context(), key, []byte(itoa(i))); err != nil {+ t.Errorf("Put %q: %v", key, err)+ }+ }(i)+ }+ wg.Wait()++ keys, err := store.Keys(t.Context())+ if err != nil {+ t.Fatalf("Keys: %v", err)+ }+ if len(keys) != n {+ t.Fatalf("after %d concurrent puts, Keys = %d keys: %v", n, len(keys), keys)+ }+}++// --- small local helpers (avoid pulling test-only deps) ---++func repeat(s string, n int) string {+ out := make([]byte, 0, n*len(s))+ for i := 0; i < n; i++ {+ out = append(out, s...)+ }+ return string(out)+}++func itoa(i int) string {+ if i == 0 {+ return "0"+ }+ neg := i < 0+ if neg {+ i = -i+ }+ var buf [20]byte+ pos := len(buf)+ for i > 0 {+ pos--+ buf[pos] = byte('0' + i%10)+ i /= 10+ }+ if neg {+ pos--+ buf[pos] = '-'+ }+ return string(buf[pos:])+}++func sanitizeTestName(name string) string {+ out := make([]byte, 0, len(name))+ for i := 0; i < len(name); i++ {+ c := name[i]+ switch {+ case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':+ out = append(out, c)+ default:+ out = append(out, '_')+ }+ }+ if len(out) > 40 {+ out = out[:40]+ }+ return string(out)+}