Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): consolidate checkpoint to Parser + per-chunk result cache (#19386)

xugangqiangcec02addf57e
diff --git a/internal/ingestion/chunkcache/chunkcache.go b/internal/ingestion/chunkcache/chunkcache.go
new file mode 100644
index 0000000000..5d8f62cc22
--- /dev/null
+++ b/internal/ingestion/chunkcache/chunkcache.go
@@ -0,0 +1,189 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package chunkcache owns the per-chunk result cache shared by the ingestion
+// components that call an expensive external model (Extractor → chat LLM,
+// Tokenizer → embedding model).
+//
+// Why it exists: the pipeline checkpoints only after the Parser, so a resumed
+// task re-executes every stage after it. That is affordable only if the
+// expensive per-chunk work is memoised, which is what this cache does — a
+// resumed run re-walks the same chunks, hits the cache, and issues no model
+// calls.
+//
+// Entry lifetime has two reclaim paths. The normal one is PurgeTask, called
+// once the task's chunks are durably indexed: from that point the cache can
+// never be re-read, so it is dropped immediately instead of occupying the
+// store. TTL is the backstop for tasks that never reach persist (failed,
+// cancelled, abandoned mid-resume) and for runs with no task scope.
+package chunkcache
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/cespare/xxhash/v2"
+
+ "ragflow/internal/engine/redis"
+ "ragflow/internal/ingestion/component/globals"
+)
+
+// TTL bounds how long a cached per-chunk result stays useful. It matches the
+// pipeline checkpoint TTL on purpose: a task that can still be resumed must
+// still find its cache, and one that can no longer be resumed has no reader.
+const TTL = 7 * 24 * time.Hour
+
+// keyNamespace prefixes every key this package writes, including the
+// manifests, so a live store can be inspected (or bulk-scanned) per concern.
+const keyNamespace = "kc"
+
+// Store is the slice of the Redis client this package needs.
+// *redis.Client satisfies it; tests supply an in-memory double.
+type Store interface {
+ Get(ctx context.Context, key string) (string, error)
+ Set(ctx context.Context, key, value string, exp time.Duration) bool
+ SAdd(ctx context.Context, key, member string) bool
+ SMembers(ctx context.Context, key string) ([]string, error)
+ Delete(ctx context.Context, key string) bool
+ Expire(ctx context.Context, key string, exp time.Duration) bool
+}
+
+// Client resolves the process-wide Redis client as a Store, or nil when Redis
+// is not configured. Always resolve through this function: passing redis.Get()
+// directly into a Store parameter would wrap a typed nil pointer in a non-nil
+// interface, defeating the nil guards below.
+func Client() Store {
+ if c := redis.Get(); c != nil {
+ return c
+ }
+ return nil
+}
+
+// Key builds the cache key for one (kind, model, chunk) triple, mixing any
+// extra config that changes the result (a prompt, a JSON schema, a top-N).
+//
+// modelID is the identity of the model that produced the cached value:
+// for the "emb" kind that is the dataset's embedding model id (kb.embd_id);
+// for the "extractor:*"/"meta"/"tagger" kinds it is the chat model id (llm_id,
+// or — when that is empty — the resolved tenant-default chat model). Callers
+// must resolve the model id to the value that actually generated the result
+// before calling Key, never a raw/possibly-empty override string: keying on an
+// empty modelID would collapse every model onto one bucket and serve a result
+// produced by a model the tenant no longer uses. The tokenizer keys on embd_id;
+// the extractor resolves its chat target (see extractorCacheModelID in the
+// component package) for the same reason.
+//
+// The chunk id — not the chunk text — is the identity input: it is the stable
+// per-chunk handle the chunker assigns (component.ChunkID), and it already
+// derives from the text, so keying on it keeps entries short and keeps the
+// pipeline and the persist stage agreeing on what "the same chunk" means.
+//
+// Either an empty chunk id or an empty modelID yields an empty key: in both
+// cases the caller must skip the cache rather than let every unidentified
+// chunk (or every model-less entry) share one bucket.
+func Key(kind, modelID, chunkID string, config ...string) string {
+ if chunkID == "" || modelID == "" {
+ return ""
+ }
+ h := xxhash.New()
+ // Length-prefix every field so no two different field splits can hash to
+ // the same digest (…"ab","c" vs …"a","bc").
+ writeField := func(s string) {
+ _, _ = fmt.Fprintf(h, "%d:%s", len(s), s)
+ }
+ writeField(kind)
+ writeField(modelID)
+ writeField(chunkID)
+ for _, c := range config {
+ writeField(c)
+ }
+ return fmt.Sprintf("%s:%s:%x", keyNamespace, kind, h.Sum64())
+}
+
+// manifestKey is the set listing every cache key produced for one task.
+func manifestKey(taskID string) string {
+ return fmt.Sprintf("%s:manifest:%s", keyNamespace, taskID)
+}
+
+// Get returns the cached value for key. The boolean distinguishes a miss from
+// a cached empty string; Set refuses empty values, so a hit is always
+// meaningful. Degrades to a miss when the store is unavailable.
+func Get(ctx context.Context, s Store, key string) (string, bool) {
+ if s == nil || key == "" {
+ return "", false
+ }
+ v, err := s.Get(ctx, key)
+ if err != nil || v == "" {
+ return "", false
+ }
+ return v, true
+}
+
+// Set caches value under key for TTL and records key on the current task's
+// manifest so PurgeTask can reclaim it at persist time. Best-effort
+// throughout: a store outage costs a recomputation, never a failed run.
+//
+// The manifest entry is only recorded once the value write succeeded, so the
+// manifest never lists a key that does not exist. When the run carries no task
+// id (canvas debug preview, headless test) the value is still cached but
+// nothing is recorded: no persist stage will ever reclaim it, so TTL is the
+// only reclaim path and a manifest would simply leak.
+func Set(ctx context.Context, s Store, key, value string) {
+ if s == nil || key == "" || value == "" {
+ return
+ }
+ if !s.Set(ctx, key, value, TTL) {
+ return
+ }
+ taskID := globals.TaskID(ctx)
+ if taskID == "" {
+ return
+ }
+ mk := manifestKey(taskID)
+ if s.SAdd(ctx, mk, key) {
+ // Bound the manifest's own lifetime: it must not outlive the entries
+ // it lists, or an abandoned task would leak one set per task forever.
+ // If the TTL cannot be applied, drop the manifest immediately rather
+ // than leave a TTL-less set that leaks one empty key per task.
+ if !s.Expire(ctx, mk, TTL) {
+ s.Delete(ctx, mk)
+ }
+ }
+}
+
+// PurgeTask drops every cache entry recorded for taskID, then the manifest
+// itself. Call it once the task's chunks are durably persisted: from that point
+// nothing can read the entries back, so holding them for the remaining TTL is
+// pure waste.
+//
+// The manifest is deleted last: losing it first would orphan the entries it
+// still lists, leaving them to expire. Best-effort — a failed reclaim only
+// means the entries wait out their TTL.
+func PurgeTask(ctx context.Context, s Store, taskID string) {
+ if s == nil || taskID == "" {
+ return
+ }
+ mk := manifestKey(taskID)
+ keys, err := s.SMembers(ctx, mk)
+ if err != nil {
+ return
+ }
+ for _, k := range keys {
+ s.Delete(ctx, k)
+ }
+ s.Delete(ctx, mk)
+}
diff --git a/internal/ingestion/chunkcache/chunkcache_test.go b/internal/ingestion/chunkcache/chunkcache_test.go
new file mode 100644
index 0000000000..dedc41165f
--- /dev/null
+++ b/internal/ingestion/chunkcache/chunkcache_test.go
@@ -0,0 +1,342 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunkcache
+
+import (
+ "context"
+ "errors"
+ "sort"
+ "testing"
+ "time"
+
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/globals"
+)
+
+// fakeStore is an in-memory Store double: enough Redis surface for the
+// manifest bookkeeping, with call counters so tests can assert the exact
+// command sequence (a real Redis is an integration-tier dependency).
+type fakeStore struct {
+ kv map[string]string
+ sets map[string]map[string]bool
+ ttl map[string]time.Duration
+ deleted []string
+ failSet bool
+ expireFails bool
+}
+
+func newFakeStore() *fakeStore {
+ return &fakeStore{
+ kv: map[string]string{},
+ sets: map[string]map[string]bool{},
+ ttl: map[string]time.Duration{},
+ }
+}
+
+func (f *fakeStore) Get(_ context.Context, key string) (string, error) {
+ v, ok := f.kv[key]
+ if !ok {
+ return "", errors.New("redis: nil")
+ }
+ return v, nil
+}
+
+func (f *fakeStore) Set(_ context.Context, key, value string, exp time.Duration) bool {
+ if f.failSet {
+ return false
+ }
+ f.kv[key] = value
+ f.ttl[key] = exp
+ return true
+}
+
+func (f *fakeStore) SAdd(_ context.Context, key, member string) bool {
+ if f.sets[key] == nil {
+ f.sets[key] = map[string]bool{}
+ }
+ f.sets[key][member] = true
+ return true
+}
+
+func (f *fakeStore) SMembers(_ context.Context, key string) ([]string, error) {
+ out := make([]string, 0, len(f.sets[key]))
+ for m := range f.sets[key] {
+ out = append(out, m)
+ }
+ sort.Strings(out)
+ return out, nil
+}
+
+func (f *fakeStore) Delete(_ context.Context, key string) bool {
+ f.deleted = append(f.deleted, key)
+ delete(f.kv, key)
+ delete(f.sets, key)
+ return true
+}
+
+func (f *fakeStore) Expire(_ context.Context, key string, exp time.Duration) bool {
+ if f.expireFails {
+ return false
+ }
+ f.ttl[key] = exp
+ return true
+}
+
+// taskCtx returns a ctx whose CanvasState carries taskID, mirroring what the
+// pipeline attaches for a real run.
+func taskCtx(taskID string) context.Context {
+ st := runtime.NewCanvasState("", "")
+ ctx := runtime.WithState(context.Background(), st)
+ globals.SetTaskID(ctx, taskID)
+ return ctx
+}
+
+// TestTTL_IsSevenDays pins the shared cache lifetime. Resume must be able to
+// reuse per-chunk results for as long as a checkpoint can live, so this value
+// and the pipeline checkpoint TTL are deliberately equal.
+func TestTTL_IsSevenDays(t *testing.T) {
+ if TTL != 7*24*time.Hour {
+ t.Errorf("TTL = %v, want 168h", TTL)
+ }
+}
+
+// TestSet_WritesValueAndRegistersManifest asserts a cache write both stores the
+// value under the shared TTL and records the key on the task manifest, so the
+// persist stage can reclaim exactly the keys this task produced.
+func TestSet_WritesValueAndRegistersManifest(t *testing.T) {
+ f := newFakeStore()
+ ctx := taskCtx("task-1")
+ Set(ctx, f, "kc:extractor:keywords:abc", "result")
+
+ if got := f.kv["kc:extractor:keywords:abc"]; got != "result" {
+ t.Errorf("cached value = %q, want %q", got, "result")
+ }
+ if got := f.ttl["kc:extractor:keywords:abc"]; got != TTL {
+ t.Errorf("cached TTL = %v, want %v", got, TTL)
+ }
+ members, _ := f.SMembers(ctx, "kc:manifest:task-1")
+ if len(members) != 1 || members[0] != "kc:extractor:keywords:abc" {
+ t.Errorf("manifest members = %v, want [kc:extractor:keywords:abc]", members)
+ }
+ if got := f.ttl["kc:manifest:task-1"]; got != TTL {
+ t.Errorf("manifest TTL = %v, want %v (must not outlive its entries)", got, TTL)
+ }
+}
+
+// TestSet_ManifestExpireFailureDeletesManifest asserts that when the manifest's
+// TTL cannot be applied (Expire returns false) Set removes the manifest key
+// rather than leaving a TTL-less set that leaks one empty key per abandoned task
+// forever. The value itself was already written and keeps its own TTL.
+func TestSet_ManifestExpireFailureDeletesManifest(t *testing.T) {
+ f := newFakeStore()
+ f.expireFails = true
+ ctx := taskCtx("task-1")
+ Set(ctx, f, "kc:extractor:keywords:abc", "result")
+
+ // Value is still cached (its own Set succeeded).
+ if got := f.kv["kc:extractor:keywords:abc"]; got != "result" {
+ t.Errorf("cached value = %q, want it kept", got)
+ }
+ // Manifest must NOT linger without a TTL.
+ mk := manifestKey("task-1")
+ if _, ok := f.sets[mk]; ok {
+ t.Errorf("manifest %q leaked after Expire failure; want it deleted", mk)
+ }
+ if len(f.deleted) == 0 || f.deleted[len(f.deleted)-1] != mk {
+ t.Errorf("expected manifest %q to be deleted on Expire failure, deleted=%v", mk, f.deleted)
+ }
+}
+
+// TestSet_SkipsManifestWithoutTaskScope asserts a run with no task id (canvas
+// debug preview, headless test) still caches but records nothing: there is no
+// task whose completion could reclaim the keys, so TTL is the only reclaim
+// path and a manifest would leak.
+func TestSet_SkipsManifestWithoutTaskScope(t *testing.T) {
+ f := newFakeStore()
+ Set(context.Background(), f, "kc:extractor:keywords:abc", "result")
+
+ if got := f.kv["kc:extractor:keywords:abc"]; got != "result" {
+ t.Errorf("cached value = %q, want it cached even without task scope", got)
+ }
+ if len(f.sets) != 0 {
+ t.Errorf("manifest sets = %v, want none without a task id", f.sets)
+ }
+}
+
+// TestSet_NoStoreIsNoop asserts a Redis-less deployment degrades silently
+// instead of panicking: callers pass the resolved client straight through.
+func TestSet_NoStoreIsNoop(t *testing.T) {
+ Set(taskCtx("task-1"), nil, "kc:extractor:keywords:abc", "result")
+ if got, ok := Get(taskCtx("task-1"), nil, "kc:extractor:keywords:abc"); ok || got != "" {
+ t.Errorf("Get(nil store) = (%q, %v), want (\"\", false)", got, ok)
+ }
+}
+
+// TestSet_SkipsEmptyKeyOrValue asserts we never cache under an empty key (the
+// caller could not derive a chunk id) and never cache an empty payload (which
+// Get cannot distinguish from a miss).
+func TestSet_SkipsEmptyKeyOrValue(t *testing.T) {
+ f := newFakeStore()
+ ctx := taskCtx("task-1")
+ Set(ctx, f, "", "result")
+ Set(ctx, f, "kc:extractor:keywords:abc", "")
+ if len(f.kv) != 0 {
+ t.Errorf("kv = %v, want no writes", f.kv)
+ }
+ if len(f.sets) != 0 {
+ t.Errorf("manifest sets = %v, want none", f.sets)
+ }
+}
+
+// TestSet_SkipsManifestWhenValueWriteFails asserts the manifest never lists a
+// key whose value write failed — otherwise PurgeTask would issue deletes for
+// keys that do not exist.
+func TestSet_SkipsManifestWhenValueWriteFails(t *testing.T) {
+ f := newFakeStore()
+ f.failSet = true
+ Set(taskCtx("task-1"), f, "kc:extractor:keywords:abc", "result")
+ if len(f.sets) != 0 {
+ t.Errorf("manifest sets = %v, want none when the value write failed", f.sets)
+ }
+}
+
+// TestGet_HitAndMiss asserts the miss signal is a boolean rather than an empty
+// string, so a caller can tell "not cached" from "cached empty".
+func TestGet_HitAndMiss(t *testing.T) {
+ f := newFakeStore()
+ ctx := taskCtx("task-1")
+ Set(ctx, f, "k", "v")
+ if got, ok := Get(ctx, f, "k"); !ok || got != "v" {
+ t.Errorf("Get(hit) = (%q, %v), want (\"v\", true)", got, ok)
+ }
+ if got, ok := Get(ctx, f, "absent"); ok || got != "" {
+ t.Errorf("Get(miss) = (%q, %v), want (\"\", false)", got, ok)
+ }
+ if got, ok := Get(ctx, f, ""); ok || got != "" {
+ t.Errorf("Get(empty key) = (%q, %v), want (\"\", false)", got, ok)
+ }
+}
+
+// TestPurgeTask_DeletesEveryEntryThenManifest asserts persist-time reclaim
+// removes all recorded entries and finally the manifest itself, leaving no
+// key behind that would otherwise wait out the 7-day TTL.
+func TestPurgeTask_DeletesEveryEntryThenManifest(t *testing.T) {
+ f := newFakeStore()
+ ctx := taskCtx("task-1")
+ Set(ctx, f, "kc:extractor:keywords:a", "1")
+ Set(ctx, f, "kc:meta:b", "2")
+ Set(ctx, f, "kc:emb:c", "3")
+
+ PurgeTask(context.Background(), f, "task-1")
+
+ want := []string{"kc:emb:c", "kc:extractor:keywords:a", "kc:meta:b", "kc:manifest:task-1"}
+ if len(f.deleted) != len(want) {
+ t.Fatalf("deleted = %v, want %v", f.deleted, want)
+ }
+ // Entry deletes may arrive in any order, but the manifest must be last:
+ // dropping it early would orphan the remaining entries if we crash midway.
+ if f.deleted[len(f.deleted)-1] != "kc:manifest:task-1" {
+ t.Errorf("last delete = %q, want the manifest key", f.deleted[len(f.deleted)-1])
+ }
+ entries := append([]string(nil), f.deleted[:len(f.deleted)-1]...)
+ sort.Strings(entries)
+ for i, w := range want[:len(want)-1] {
+ if entries[i] != w {
+ t.Errorf("entry deletes = %v, want %v", entries, want[:len(want)-1])
+ break
+ }
+ }
+ if len(f.kv) != 0 {
+ t.Errorf("kv after purge = %v, want empty", f.kv)
+ }
+}
+
+// TestPurgeTask_NoopWithoutTaskOrStore asserts the reclaim path is safe to
+// call unconditionally from the persist stage.
+func TestPurgeTask_NoopWithoutTaskOrStore(t *testing.T) {
+ f := newFakeStore()
+ PurgeTask(context.Background(), f, "")
+ if len(f.deleted) != 0 {
+ t.Errorf("deleted = %v, want none for an empty task id", f.deleted)
+ }
+ PurgeTask(context.Background(), nil, "task-1")
+}
+
+// TestKey_DiscriminatesEveryComponent asserts the key builder namespaces by
+// kind and mixes every identity input, so a chat-model swap, a prompt edit or
+// a different chunk can never read another entry's value.
+func TestKey_DiscriminatesEveryComponent(t *testing.T) {
+ base := Key("extractor:keywords", "gpt-4@openai", "chunk-1", "prompt-v1")
+ cases := []struct {
+ name string
+ key string
+ }{
+ {"different kind", Key("extractor:questions", "gpt-4@openai", "chunk-1", "prompt-v1")},
+ {"different model", Key("extractor:keywords", "gpt-5@openai", "chunk-1", "prompt-v1")},
+ {"different chunk", Key("extractor:keywords", "gpt-4@openai", "chunk-2", "prompt-v1")},
+ {"different config", Key("extractor:keywords", "gpt-4@openai", "chunk-1", "prompt-v2")},
+ }
+ for _, c := range cases {
+ if c.key == base {
+ t.Errorf("%s: key collides with base (%s)", c.name, base)
+ }
+ }
+ if same := Key("extractor:keywords", "gpt-4@openai", "chunk-1", "prompt-v1"); same != base {
+ t.Errorf("Key is not deterministic: %q vs %q", same, base)
+ }
+}
+
+// TestKey_IsPrefixedByKind asserts keys stay greppable/scannable per kind in a
+// live Redis, and that they share the kc: namespace the manifest uses.
+func TestKey_IsPrefixedByKind(t *testing.T) {
+ k := Key("emb", "bge-m3@builtin", "chunk-1")
+ if want := "kc:emb:"; len(k) <= len(want) || k[:len(want)] != want {
+ t.Errorf("Key = %q, want prefix %q", k, want)
+ }
+}
+
+// TestKey_EmptyChunkIDYieldsNoKey asserts a chunk with no stable id produces
+// no key at all, so callers skip the cache instead of sharing one bucket for
+// every unidentified chunk.
+func TestKey_EmptyChunkIDYieldsNoKey(t *testing.T) {
+ if k := Key("emb", "bge-m3@builtin", ""); k != "" {
+ t.Errorf("Key(empty chunk id) = %q, want \"\"", k)
+ }
+}
+
+// TestKey_EmptyModelIDYieldsNoKey asserts the symmetric contract: an empty
+// model id yields no key, so a caller that forgot to resolve the model (e.g.
+// keyed on a raw, empty llm_id override) cannot collapse every model onto one
+// bucket. Mirrors TestKey_EmptyChunkIDYieldsNoKey.
+func TestKey_EmptyModelIDYieldsNoKey(t *testing.T) {
+ if k := Key("emb", "", "chunk-1"); k != "" {
+ t.Errorf("Key(empty model id) = %q, want \"\"", k)
+ }
+ if k := Key("extractor:questions", "", "chunk-1", "prompt"); k != "" {
+ t.Errorf("Key(empty model id) = %q, want \"\"", k)
+ }
+}
+
+// TestKey_FieldsAreUnambiguous asserts the identity inputs are separated when
+// hashed: concatenating them differently must not collide.
+func TestKey_FieldsAreUnambiguous(t *testing.T) {
+ a := Key("extractor:keywords", "model", "chunk", "ab", "c")
+ b := Key("extractor:keywords", "model", "chunk", "a", "bc")
+ if a == b {
+ t.Errorf("ambiguous field packing: %q == %q", a, b)
+ }
+}
diff --git a/internal/ingestion/chunkcache/client_test.go b/internal/ingestion/chunkcache/client_test.go
new file mode 100644
index 0000000000..be68e751cc
--- /dev/null
+++ b/internal/ingestion/chunkcache/client_test.go
@@ -0,0 +1,47 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunkcache
+
+import (
+ "testing"
+
+ "ragflow/internal/engine/redis"
+)
+
+// Compile-time proof that the real client still satisfies Store. Without it a
+// signature change in the redis package would only surface at the call sites.
+var _ Store = (*redis.Client)(nil)
+
+// TestClient_NilInterfaceWhenRedisAbsent is the reason Client() exists. Handing
+// redis.Get() straight to a Store parameter would wrap a typed nil pointer in a
+// non-nil interface, defeating every `s == nil` guard and panicking on the
+// first call in a Redis-less deployment. Client() must return an interface that
+// compares equal to nil.
+func TestClient_NilInterfaceWhenRedisAbsent(t *testing.T) {
+ if redis.Get() != nil {
+ t.Skip("a global Redis client is configured in this process")
+ }
+ if got := Client(); got != nil {
+ t.Fatalf("Client() = %#v, want a nil interface", got)
+ }
+ // The guards must hold for the value Client() actually returns.
+ Set(taskCtx("task-1"), Client(), "k", "v")
+ if got, ok := Get(taskCtx("task-1"), Client(), "k"); ok || got != "" {
+ t.Errorf("Get = (%q, %v), want (\"\", false)", got, ok)
+ }
+ PurgeTask(taskCtx("task-1"), Client(), "task-1")
+}
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index f724cdae85..f8a83d063e 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -43,7 +43,6 @@ import (
"sync"
"time"
- "github.com/cespare/xxhash/v2"
eschema "github.com/cloudwego/eino/schema"
"go.uber.org/zap"
"gorm.io/gorm"
@@ -51,9 +50,9 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/engine/redis"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
+ "ragflow/internal/ingestion/chunkcache"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
"ragflow/internal/utility"
@@ -501,14 +500,27 @@ func toExtractorEinoMessages(msgs []eschema.Message) []*eschema.Message {
// input map. Computed once at the top of Invoke so the rest of
// the function reads as straight-line code.
type extractorInputs struct {
- llmID string
- lang string
- chunks []map[string]any
+ llmID string
+ // modelID is the resolved cache-key model identity (the llm_id, or — when
+ // that is empty — the tenant-default chat model resolved once per run in
+ // Invoke). Carrying it on the struct lets the per-chunk cache-key builders
+ // reuse the single run-level resolution instead of re-resolving the default
+ // model for every chunk (review finding #3). Empty means "not yet resolved"
+ // and triggers a per-call fallback so direct unit-test construction of
+ // extractorInputs keeps working.
+ modelID string
+ lang string
+ chunks []map[string]any
// temperature overrides the LLM temperature for this call. A
// nil value leaves the request's Temperature unset so the model
// (or the chat-model default) decides. The keyword/question helpers set it to
// extractorTemperature (0.2) to mirror generator.py.
temperature *float64
+ // cache memoises per-chunk extraction results so a resumed run (which
+ // re-executes every stage after the Parser checkpoint) pays no LLM cost
+ // for chunks it already extracted. nil disables caching: a Redis-less
+ // deployment, or a test that wants every call to reach the model.
+ cache chunkcache.Store
}
// resolveInputs overlays per-call inputs on top of the
@@ -518,6 +530,7 @@ type extractorInputs struct {
func (c *ExtractorComponent) resolveInputs(inputs map[string]any) extractorInputs {
out := extractorInputs{
llmID: c.Param.LLMID,
+ cache: chunkcache.Client(),
}
if inputs == nil {
return out
@@ -594,6 +607,10 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
return nil, fmt.Errorf("extractor: %w", err)
}
in := c.resolveInputs(inputs)
+ // Resolve the cache-key model identity once for the whole run and reuse it
+ // across every chunk, rather than re-resolving the (most common) default
+ // model per chunk when keying the per-chunk cache. See review finding #3.
+ in.modelID = extractorCacheModelID(ctx, db, in.llmID)
common.Debug("extractor stage",
zap.String("component", "Extractor"),
zap.Int("input_chunks", len(in.chunks)),
@@ -637,26 +654,51 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
}, nil
}
-// extractorLLMCacheKey builds a Redis key for textual extractions.
-func extractorLLMCacheKey(taskType, modelID, systemPrompt, text string) string {
- h := xxhash.New()
- h.WriteString(taskType)
- h.WriteString("\x00")
- h.WriteString(modelID)
- h.WriteString("\x00")
- h.WriteString(systemPrompt)
- h.WriteString("\x00")
- h.WriteString(text)
- return fmt.Sprintf("kc:extractor:%s:%x", taskType, h.Sum64())
+// extractorLLMCacheKey builds the cache key for one textual extraction. The
+// chunk id — not the chunk text — carries the chunk's identity: it already
+// derives from the text (component.ChunkID) and is the same handle the persist
+// stage uses, so the two stages agree on what "the same chunk" is. Returns ""
+// when the chunk has no id, which makes the cache a no-op for that chunk.
+func extractorLLMCacheKey(taskType, modelID, systemPrompt, chunkID string) string {
+ return chunkcache.Key("extractor:"+taskType, modelID, chunkID, systemPrompt)
}
-// callTextCached wraps callText with a 24-hour Redis cache.
-func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in extractorInputs, taskType, systemPrompt, chunkText string) (string, error) {
- key := extractorLLMCacheKey(taskType, in.llmID, systemPrompt, chunkText)
- if client := redis.Get(); client != nil {
- if data, err := client.Get(ctx, key); err == nil && data != "" {
- return data, nil
- }
+// extractorCacheModelID returns the model identity the per-chunk extraction
+// cache is keyed on. The cache must capture the model that actually produced
+// the text, not the raw llm_id override: when llm_id is empty the pipeline
+// falls back to the tenant's default chat model (resolveExtractorChatTarget,
+// extractor.go:1212), so keying on the empty override would collapse every
+// tenant-default run onto one bucket and serve extractions from a model the
+// tenant no longer uses. This mirrors the tokenizer's decision to key on the
+// dataset-bound embd_id rather than the raw DSL string.
+//
+// Non-empty llm_id is already a stable tenant-model id, so it is used as-is
+// (no behaviour change for the configured path). Only the empty case is
+// resolved to its real default-model identity. A resolution failure returns ""
+// — chunkcache.Key turns an empty modelID into an empty key, which disables
+// caching for that chunk rather than risking a cross-model hit.
+func extractorCacheModelID(ctx context.Context, db *gorm.DB, llmID string) string {
+ if llmID != "" {
+ return llmID
+ }
+ driver, modelName, _, _, err := resolveExtractorChatTarget(ctx, db, "")
+ if err != nil || (driver == "" && modelName == "") {
+ return ""
+ }
+ return driver + "/" + modelName
+}
+
+// callTextCached wraps callText with the per-chunk result cache.
+func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in extractorInputs, taskType, systemPrompt, chunkText, chunkID string) (string, error) {
+ // Prefer the run-level resolved identity; fall back only for callers that
+ // build extractorInputs directly in unit tests without pre-resolving it.
+ modelID := in.modelID
+ if modelID == "" {
+ modelID = extractorCacheModelID(ctx, db, in.llmID)
+ }
+ key := extractorLLMCacheKey(taskType, modelID, systemPrompt, chunkID)
+ if cached, hit := chunkcache.Get(ctx, in.cache, key); hit {
+ return cached, nil
}
res, err := c.callText(ctx, db, in, systemPrompt, chunkText)
if err != nil {
@@ -664,9 +706,7 @@ func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in
}
res = cleanExtractionResult(res)
if res != "" && !strings.Contains(res, "**ERROR**") {
- if client := redis.Get(); client != nil {
- client.Set(ctx, key, res, 24*time.Hour)
- }
+ chunkcache.Set(ctx, in.cache, key, res)
}
return res, nil
}
@@ -690,9 +730,11 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, i
kwTemp := extractorTemperature
kwIn := extractorInputs{
llmID: in.llmID,
+ modelID: in.modelID,
temperature: &kwTemp,
+ cache: in.cache,
}
- resultStr, err := c.callTextCached(ctx, db, kwIn, "keywords", systemPrompt, chunkText)
+ resultStr, err := c.callTextCached(ctx, db, kwIn, "keywords", systemPrompt, chunkText, chunkCacheID(ck))
if err != nil {
return err
}
@@ -730,9 +772,11 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,
qTemp := extractorTemperature
qIn := extractorInputs{
llmID: in.llmID,
+ modelID: in.modelID,
temperature: &qTemp,
+ cache: in.cache,
}
- resultStr, err := c.callTextCached(ctx, db, qIn, "questions", systemPrompt, chunkText)
+ resultStr, err := c.callTextCached(ctx, db, qIn, "questions", systemPrompt, chunkText, chunkCacheID(ck))
if err != nil {
return err
}
@@ -776,9 +820,11 @@ func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in
sumTemp := extractorTemperature
sumIn := extractorInputs{
llmID: in.llmID,
+ modelID: in.modelID,
temperature: &sumTemp,
+ cache: in.cache,
}
- resultStr, err := c.callTextCached(ctx, db, sumIn, "summary", systemPrompt, chunkText)
+ resultStr, err := c.callTextCached(ctx, db, sumIn, "summary", systemPrompt, chunkText, chunkCacheID(ck))
if err != nil {
return err
}
@@ -798,10 +844,7 @@ func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.D
futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))
for i, ck := range in.chunks {
i, ck := i, ck
- text, _ := ck["content_with_weight"].(string)
- if strings.TrimSpace(text) == "" {
- text, _ = ck["text"].(string)
- }
+ text := extractorChunkText(ck)
fn := func() error {
if err := c.runAutoKeywords(ctx, db, in, ck, text); err != nil {
return fmt.Errorf("chunk %d keywords: %w", i, err)
@@ -826,10 +869,7 @@ func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *go
futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))
for i, ck := range in.chunks {
i, ck := i, ck
- text, _ := ck["content_with_weight"].(string)
- if strings.TrimSpace(text) == "" {
- text, _ = ck["text"].(string)
- }
+ text := extractorChunkText(ck)
fn := c.remainingExtractionJob(ctx, db, in, i, ck, text)
f, err := extractorPool.Submit(ctx, fn)
if err != nil {
@@ -911,20 +951,28 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
}
schemaStr := string(schemaJSON)
- // LLM cache (mirrors Python get_llm_cache/set_llm_cache in
- // task_executor.py:543/550 gen_metadata_task): identical (model + chunk
- // text + schema) extractions are served from Redis within a 24h window so
- // repeated runs / identical chunks don't re-pay the LLM call.
- // Best-effort: a missing Redis client or any cache error falls through to
- // a live call instead of failing the extraction.
+ // Per-chunk result cache: identical (model + chunk + schema) extractions
+ // are served from the cache so a resumed run — which re-executes every
+ // stage after the Parser checkpoint — doesn't re-pay the LLM call.
+ // Best-effort: a missing client or any cache error falls through to a live
+ // call instead of failing the extraction.
+ chunkID := chunkCacheID(ck)
+ // Prefer the run-level resolved identity; fall back only for direct unit
+ // callers that did not pre-resolve it.
+ modelID := in.modelID
+ if modelID == "" {
+ modelID = extractorCacheModelID(ctx, db, in.llmID)
+ }
var parsed map[string]any
- if cached, hit := getMetadataLLMCache(ctx, in.llmID, schemaStr, chunkText); hit {
+ if cached, hit := getMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID); hit {
parsed = cached
} else {
metaTemp := extractorTemperature
metaIn := extractorInputs{
llmID: in.llmID,
+ modelID: in.modelID,
temperature: &metaTemp,
+ cache: in.cache,
}
systemPrompt := fmt.Sprintf(autoMetadataPrompt, schemaStr)
parsed, err = c.callStructured(ctx, db, metaIn, systemPrompt, chunkText)
@@ -935,7 +983,7 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
// Non-JSON or empty response — nothing to extract, not an error.
return nil
}
- setMetadataLLMCache(ctx, in.llmID, schemaStr, chunkText, parsed)
+ setMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID, parsed)
}
// Merge into the chunk metadata map, preserving existing keys.
var meta map[string]any
@@ -957,53 +1005,63 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
return nil
}
-// metadataLLMCacheTTL mirrors Python get_llm_cache/set_llm_cache 24h TTL.
-const metadataLLMCacheTTL = 24 * time.Hour
-
-// metadataLLMCacheKey builds a Redis key from (llm id, chunk text, "metadata",
-// schema), mirroring Python get_llm_cache's xxh64(llmnm + txt + history + genconf).
-func metadataLLMCacheKey(llmID, schemaJSON, chunkText string) string {
- h := xxhash.New()
- h.WriteString(llmID)
- h.WriteString("\x00")
- h.WriteString(chunkText)
- h.WriteString("\x00")
- h.WriteString("metadata")
- h.WriteString("\x00")
- h.WriteString(schemaJSON)
- return fmt.Sprintf("kc:meta:%x", h.Sum64())
+// extractorChunkText resolves the body an extraction is run against.
+//
+// "text" wins over "content_with_weight": every chunker writes the
+// authoritative body to "text" (and derives the chunk id from it), while
+// "content_with_weight" may survive as pass-through metadata from an upstream
+// parser block. Preferring the latter would send the LLM a different body than
+// the one the chunk id — and therefore the cache entry — stands for. This is
+// the same priority the Tokenizer applies (normalizeChunkTextFallback) and the
+// chunkers apply (itemText). The fallback keeps a chunk that only carries the
+// structured field extractable rather than sending an empty body.
+func extractorChunkText(ck map[string]any) string {
+ if v, _ := ck["text"].(string); strings.TrimSpace(v) != "" {
+ return v
+ }
+ v, _ := ck["content_with_weight"].(string)
+ return v
+}
+
+// chunkCacheID returns the chunk's stable per-chunk id, assigned by the chunker
+// (component.ChunkID over doc id + text). Returns "" when the chunk carries no
+// id — e.g. chunks that reached the Extractor without passing a chunker — in
+// which case callers must skip the cache instead of sharing one bucket across
+// every unidentified chunk.
+func chunkCacheID(ck map[string]any) string {
+ id, _ := ck["id"].(string)
+ return id
+}
+
+// metadataLLMCacheKey builds the cache key for one metadata extraction. The
+// schema is part of the key so editing the field set invalidates results
+// extracted against the previous one.
+func metadataLLMCacheKey(llmID, schemaJSON, chunkID string) string {
+ return chunkcache.Key("meta", llmID, chunkID, schemaJSON)
}
// getMetadataLLMCache returns a cached extraction for the given chunk, or
-// (nil, false) on miss / Redis unavailable / decode error. Best-effort.
-func getMetadataLLMCache(ctx context.Context, llmID, schemaJSON, chunkText string) (map[string]any, bool) {
- client := redis.Get()
- if client == nil {
- return nil, false
- }
- data, err := client.Get(ctx, metadataLLMCacheKey(llmID, schemaJSON, chunkText))
- if err != nil || data == "" {
+// (nil, false) on miss / cache unavailable / decode error. Best-effort.
+func getMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, schemaJSON, chunkID string) (map[string]any, bool) {
+ data, hit := chunkcache.Get(ctx, store, metadataLLMCacheKey(modelID, schemaJSON, chunkID))
+ if !hit {
return nil, false
}
var parsed map[string]any
- if err = json.Unmarshal([]byte(data), &parsed); err != nil {
+ if err := json.Unmarshal([]byte(data), &parsed); err != nil {
return nil, false
}
return parsed, true
}
-// setMetadataLLMCache stores an extraction result for 24h. Best-effort: a
-// missing Redis client or marshal error is silently ignored.
-func setMetadataLLMCache(ctx context.Context, llmID, schemaJSON, chunkText string, parsed map[string]any) {
- client := redis.Get()
- if client == nil {
- return
- }
+// setMetadataLLMCache stores an extraction result. Best-effort: an unavailable
+// cache or a marshal error is silently ignored.
+func setMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, schemaJSON, chunkID string, parsed map[string]any) {
data, err := json.Marshal(parsed)
if err != nil {
return
}
- client.Set(ctx, metadataLLMCacheKey(llmID, schemaJSON, chunkText), string(data), metadataLLMCacheTTL)
+ chunkcache.Set(ctx, store, metadataLLMCacheKey(modelID, schemaJSON, chunkID), string(data))
}
// callRaw runs one chat call against the LLM (per chunk in the normal path)
diff --git a/internal/ingestion/component/extractor_cache_test.go b/internal/ingestion/component/extractor_cache_test.go
new file mode 100644
index 0000000000..9e160168b8
--- /dev/null
+++ b/internal/ingestion/component/extractor_cache_test.go
@@ -0,0 +1,454 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/common"
+ "ragflow/internal/ingestion/chunkcache"
+ "ragflow/internal/ingestion/component/globals"
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// fakeCacheStore is an in-memory chunkcache.Store for the extractor cache
+// tests. A real Redis would push these into the integration tier; the point
+// under test is which key is read/written, not Redis itself.
+type fakeCacheStore struct {
+ mu sync.Mutex
+ kv map[string]string
+ ttl map[string]time.Duration
+ sets map[string]map[string]bool
+}
+
+func newFakeCacheStore() *fakeCacheStore {
+ return &fakeCacheStore{
+ kv: map[string]string{},
+ ttl: map[string]time.Duration{},
+ sets: map[string]map[string]bool{},
+ }
+}
+
+func (f *fakeCacheStore) Get(_ context.Context, key string) (string, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ v, ok := f.kv[key]
+ if !ok {
+ return "", errors.New("redis: nil")
+ }
+ return v, nil
+}
+
+func (f *fakeCacheStore) Set(_ context.Context, key, value string, exp time.Duration) bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.kv[key] = value
+ f.ttl[key] = exp
+ return true
+}
+
+func (f *fakeCacheStore) SAdd(_ context.Context, key, member string) bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if f.sets[key] == nil {
+ f.sets[key] = map[string]bool{}
+ }
+ f.sets[key][member] = true
+ return true
+}
+
+func (f *fakeCacheStore) SMembers(_ context.Context, key string) ([]string, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ out := make([]string, 0, len(f.sets[key]))
+ for m := range f.sets[key] {
+ out = append(out, m)
+ }
+ return out, nil
+}
+
+func (f *fakeCacheStore) Delete(_ context.Context, key string) bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ delete(f.kv, key)
+ delete(f.sets, key)
+ return true
+}
+
+func (f *fakeCacheStore) Expire(_ context.Context, key string, exp time.Duration) bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.ttl[key] = exp
+ return true
+}
+
+func (f *fakeCacheStore) get(key string) (string, bool) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ v, ok := f.kv[key]
+ return v, ok
+}
+
+func (f *fakeCacheStore) members(key string) []string {
+ out, _ := f.SMembers(context.Background(), key)
+ return out
+}
+
+// cacheTaskCtx returns a ctx carrying a CanvasState with taskID, mirroring the
+// pipeline so cache writes get recorded on the task manifest.
+func cacheTaskCtx(taskID string) context.Context {
+ ctx := runtime.WithState(context.Background(), runtime.NewCanvasState("", ""))
+ globals.SetTaskID(ctx, taskID)
+ return ctx
+}
+
+// TestCallTextCached_HitSkipsLLMCall asserts a cached extraction short-circuits
+// the model call. This is the property that makes the single Parser-only
+// checkpoint affordable: a resumed run re-walks every chunk but pays nothing.
+func TestCallTextCached_HitSkipsLLMCall(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "fresh"})
+ store := newFakeCacheStore()
+ key := chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys-prompt")
+ store.Set(context.Background(), key, "cached", chunkcache.TTL)
+
+ c := &ExtractorComponent{}
+ in := extractorInputs{llmID: "model-1", cache: store}
+ got, err := c.callTextCached(t.Context(), nil, in, "keywords", "sys-prompt", "chunk body", "chunk-1")
+ if err != nil {
+ t.Fatalf("callTextCached: %v", err)
+ }
+ if got != "cached" {
+ t.Errorf("result = %q, want %q", got, "cached")
+ }
+ if n := stub.Calls(); n != 0 {
+ t.Errorf("LLM calls = %d, want 0 on a cache hit", n)
+ }
+}
+
+// TestCallTextCached_KeyedByChunkIDNotText is the core of the key change. The
+// chunk id already derives from the text, so hashing the text again only makes
+// entries longer — and it forced every caller to keep the exact same text
+// around. A hit must depend on the chunk id alone.
+func TestCallTextCached_KeyedByChunkIDNotText(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "fresh"})
+ store := newFakeCacheStore()
+ store.Set(context.Background(), chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys"), "cached", chunkcache.TTL)
+
+ c := &ExtractorComponent{}
+ in := extractorInputs{llmID: "model-1", cache: store}
+ // Same chunk id, deliberately different text: still a hit.
+ got, err := c.callTextCached(t.Context(), nil, in, "keywords", "sys", "totally unrelated body", "chunk-1")
+ if err != nil {
+ t.Fatalf("callTextCached: %v", err)
+ }
+ if got != "cached" {
+ t.Errorf("result = %q, want the cached value (key must ignore chunk text)", got)
+ }
+ if n := stub.Calls(); n != 0 {
+ t.Errorf("LLM calls = %d, want 0", n)
+ }
+}
+
+// TestCallTextCached_DifferentChunkIDMisses asserts two distinct chunks never
+// share an entry.
+func TestCallTextCached_DifferentChunkIDMisses(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "fresh"})
+ store := newFakeCacheStore()
+ store.Set(context.Background(), chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys"), "cached", chunkcache.TTL)
+
+ c := &ExtractorComponent{}
+ in := extractorInputs{llmID: "model-1", cache: store}
+ got, err := c.callTextCached(t.Context(), nil, in, "keywords", "sys", "body", "chunk-2")
+ if err != nil {
+ t.Fatalf("callTextCached: %v", err)
+ }
+ if got != "fresh" {
+ t.Errorf("result = %q, want the freshly generated value", got)
+ }
+ if n := stub.Calls(); n != 1 {
+ t.Errorf("LLM calls = %d, want 1 on a miss", n)
+ }
+}
+
+// TestCallTextCached_WriteUsesSharedTTLAndManifest asserts a fresh extraction
+// is cached under the shared 7-day TTL and recorded on the task manifest, so
+// the persist stage can reclaim it immediately instead of leaving it to expire.
+func TestCallTextCached_WriteUsesSharedTTLAndManifest(t *testing.T) {
+ withStubChatInvoker(t, stubResponse{Content: "generated"})
+ store := newFakeCacheStore()
+ c := &ExtractorComponent{}
+ in := extractorInputs{llmID: "model-1", cache: store}
+
+ if _, err := c.callTextCached(cacheTaskCtx("task-9"), nil, in, "keywords", "sys", "body", "chunk-1"); err != nil {
+ t.Fatalf("callTextCached: %v", err)
+ }
+
+ key := chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys")
+ if got, ok := store.get(key); !ok || got != "generated" {
+ t.Errorf("cached value = %q (ok=%v), want %q", got, ok, "generated")
+ }
+ if got := store.ttl[key]; got != chunkcache.TTL {
+ t.Errorf("cache TTL = %v, want %v", got, chunkcache.TTL)
+ }
+ if members := store.members("kc:manifest:task-9"); len(members) != 1 || members[0] != key {
+ t.Errorf("manifest = %v, want [%s]", members, key)
+ }
+}
+
+// TestCallTextCached_NoChunkIDBypassesCache asserts a chunk with no stable id
+// (no chunker upstream) is neither read from nor written to the cache, rather
+// than every such chunk sharing one bucket.
+func TestCallTextCached_NoChunkIDBypassesCache(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "generated"})
+ store := newFakeCacheStore()
+ c := &ExtractorComponent{}
+ in := extractorInputs{llmID: "model-1", cache: store}
+
+ got, err := c.callTextCached(cacheTaskCtx("task-9"), nil, in, "keywords", "sys", "body", "")
+ if err != nil {
+ t.Fatalf("callTextCached: %v", err)
+ }
+ if got != "generated" {
+ t.Errorf("result = %q, want %q", got, "generated")
+ }
+ if n := stub.Calls(); n != 1 {
+ t.Errorf("LLM calls = %d, want 1", n)
+ }
+ if len(store.kv) != 0 {
+ t.Errorf("cache writes = %v, want none without a chunk id", store.kv)
+ }
+ if len(store.sets) != 0 {
+ t.Errorf("manifest writes = %v, want none without a chunk id", store.sets)
+ }
+}
+
+// TestCallTextCached_NoStoreStillCalls asserts a Redis-less deployment keeps
+// working: no cache, every call reaches the model.
+func TestCallTextCached_NoStoreStillCalls(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "generated"})
+ c := &ExtractorComponent{}
+ got, err := c.callTextCached(t.Context(), nil, extractorInputs{llmID: "model-1"}, "keywords", "sys", "body", "chunk-1")
+ if err != nil {
+ t.Fatalf("callTextCached: %v", err)
+ }
+ if got != "generated" || stub.Calls() != 1 {
+ t.Errorf("result = %q, calls = %d; want (%q, 1)", got, stub.Calls(), "generated")
+ }
+}
+
+// TestExtractor_ResolvesDefaultModelOncePerRun is the regression test for
+// review finding #3: on the default-model path (empty llm_id) the model
+// identity used to key the per-chunk cache must be resolved exactly ONCE per
+// run and reused across every chunk — not re-resolved per chunk. The per-chunk
+// re-resolution was a real performance regression introduced by this PR's
+// correctness fix (the cache key now derives from the resolved default model,
+// not the empty override), and is most visible when there is no cache at all
+// (every call reaches the resolver for nothing).
+//
+// The invariant: the chat-target resolver override is consulted once for each
+// real LLM call (callRaw resolves its target) and exactly once more for the
+// whole run (the run-level cache-key model resolution). So
+// resolverCalls == stub.Calls()+1. Before the fix the cache-key build
+// re-resolved the model per chunk, giving 2*stub.Calls().
+func TestExtractor_ResolvesDefaultModelOncePerRun(t *testing.T) {
+ var resolverCalls atomic.Int32
+ SetExtractorChatTargetResolverOverride(func(llmID string) (string, string, string, string, bool) {
+ resolverCalls.Add(1)
+ return "openai", "gpt-4o", "ak", "http://x", true
+ })
+ t.Cleanup(func() { SetExtractorChatTargetResolverOverride(nil) })
+
+ stub := withStubChatInvoker(t, stubResponse{Content: `{"category":"x"}`})
+
+ const nChunks = 5
+ chunks := make([]map[string]any, nChunks)
+ for i := range chunks {
+ chunks[i] = map[string]any{"text": fmt.Sprintf("chunk body %d", i)}
+ }
+
+ c := &ExtractorComponent{Param: schema.ExtractorParam{
+ LLMID: "", // default-model path — the one #3 is about
+ Keywords: schema.KeywordExtractConfig{TopN: 3},
+ Metadata: schema.MetadataExtractConfig{
+ Enabled: true,
+ Metadata: []common.MetadataFieldDef{{Key: "category", Type: "string"}},
+ },
+ }}
+
+ if _, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": chunks}); err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ // Sanity: the run actually exercised the LLM so the assertion below is
+ // not vacuously true.
+ if n := stub.Calls(); n == 0 {
+ t.Fatal("no LLM calls happened; test did not exercise the extraction path")
+ }
+
+ want := stub.Calls() + 1
+ if got := resolverCalls.Load(); got != want {
+ t.Errorf("resolver calls = %d, want %d (== stub.Calls()+1: one run-level model resolution, reused across all chunks)",
+ got, want)
+ }
+}
+
+// TestMetadataLLMCache_KeyedByChunkID asserts the metadata extraction cache
+// followed the same key change, and that a hit skips the model.
+func TestMetadataLLMCache_KeyedByChunkID(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: `{"category":"fresh"}`})
+ store := newFakeCacheStore()
+ c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
+ in := extractorInputs{llmID: "model-1", cache: store}
+
+ // Seed via the setter so the test pins the getter/setter agreement rather
+ // than duplicating the key format.
+ schemaJSON, err := json.Marshal(common.Turn2JSONSchema(c.Param.Metadata.Metadata))
+ if err != nil {
+ t.Fatalf("marshal metadata schema: %v", err)
+ }
+ setMetadataLLMCache(cacheTaskCtx("task-9"), in.cache, "model-1", string(schemaJSON), "chunk-1", map[string]any{"category": "cached"})
+
+ ck := map[string]any{"id": "chunk-1", "text": "a body that no longer participates in the key"}
+ if err := c.runEnableMetadata(t.Context(), nil, in, ck, "a body that no longer participates in the key"); err != nil {
+ t.Fatalf("runEnableMetadata: %v", err)
+ }
+ meta, _ := ck["metadata"].(map[string]any)
+ if meta["category"] != "cached" {
+ t.Errorf("metadata = %v, want category=cached from the chunk-id-keyed cache", meta)
+ }
+ if n := stub.Calls(); n != 0 {
+ t.Errorf("LLM calls = %d, want 0 on a cache hit", n)
+ }
+}
+
+// TestMetadataLLMCache_DistinctChunksDoNotShare asserts the metadata cache
+// discriminates chunks.
+func TestMetadataLLMCache_DistinctChunksDoNotShare(t *testing.T) {
+ store := newFakeCacheStore()
+ in := extractorInputs{llmID: "model-1", cache: store}
+ ctx := cacheTaskCtx("task-9")
+ setMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-1", map[string]any{"category": "one"})
+
+ if _, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-2"); hit {
+ t.Error("chunk-2 must not read chunk-1's metadata entry")
+ }
+ got, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-1")
+ if !hit || got["category"] != "one" {
+ t.Errorf("getMetadataLLMCache(chunk-1) = %v (hit=%v), want category=one", got, hit)
+ }
+}
+
+// TestMetadataLLMCache_SchemaChangeInvalidates asserts a metadata schema edit
+// does not silently reuse results extracted against the old field set.
+func TestMetadataLLMCache_SchemaChangeInvalidates(t *testing.T) {
+ store := newFakeCacheStore()
+ in := extractorInputs{llmID: "model-1", cache: store}
+ ctx := cacheTaskCtx("task-9")
+ setMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-1", map[string]any{"category": "one"})
+ if _, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"region":"string"}`, "chunk-1"); hit {
+ t.Error("a different metadata schema must not reuse the previous entry")
+ }
+}
+
+// TestMetadataLLMCache_NoChunkIDBypassesCache asserts the metadata path also
+// skips the cache when the chunk has no stable id.
+func TestMetadataLLMCache_NoChunkIDBypassesCache(t *testing.T) {
+ store := newFakeCacheStore()
+ in := extractorInputs{llmID: "model-1", cache: store}
+ ctx := cacheTaskCtx("task-9")
+ setMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "", map[string]any{"category": "one"})
+ if len(store.kv) != 0 {
+ t.Errorf("cache writes = %v, want none without a chunk id", store.kv)
+ }
+ if _, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, ""); hit {
+ t.Error("getMetadataLLMCache with no chunk id must miss")
+ }
+}
+
+// TestTaggerCacheKey_ScopedByChunkID_ModelAndText asserts the tagger cache key
+// is built from the model id, the chunk id, AND the chunk text fed to the model
+// (getChunkText folds in the body and important_kwd). The key must stay
+// deterministic for identical inputs, discriminate chunks/models/tag-sets/
+// few-shot examples/topN, treat a missing chunk id as a no-op, and bust when the
+// chunk text changes (a different important_kwd would otherwise serve stale tags).
+func TestTaggerCacheKey_ScopedByChunkID_ModelAndText(t *testing.T) {
+ allTags := map[string]float64{"a": 1, "b": 2}
+ ex := []schema.TaggedChunk{{Content: "example one", Tags: []string{"a"}}}
+
+ base := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, ex, 3)
+ if base == "" {
+ t.Fatal("taggerCacheKey returned empty for a valid chunk id")
+ }
+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, ex, 3); got != base {
+ t.Errorf("taggerCacheKey is not deterministic: %q vs %q", got, base)
+ }
+ if got := taggerCacheKey("llm-1", "chunk-2", "the body", allTags, ex, 3); got == base {
+ t.Error("a different chunk id must not collide")
+ }
+ if got := taggerCacheKey("llm-2", "chunk-1", "the body", allTags, ex, 3); got == base {
+ t.Error("a different model must not collide")
+ }
+ if got := taggerCacheKey("llm-1", "chunk-1", "a different body", allTags, ex, 3); got == base {
+ t.Error("a different chunk text (e.g. changed important_kwd) must not reuse the entry")
+ }
+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", map[string]float64{"a": 1}, ex, 3); got == base {
+ t.Error("a different tag set must not collide")
+ }
+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, []schema.TaggedChunk{{Content: "example two", Tags: []string{"a"}}}, 3); got == base {
+ t.Error("different few-shot examples must not collide")
+ }
+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, ex, 5); got == base {
+ t.Error("a different topN must not collide")
+ }
+ if got := taggerCacheKey("llm-1", "", "the body", allTags, ex, 3); got != "" {
+ t.Errorf("taggerCacheKey(no chunk id) = %q, want \"\"", got)
+ }
+}
+
+// TestTaggerLLMCache_RoundTripsUnderSharedTTL asserts the tagger cache uses the
+// same store/TTL/manifest contract as the other per-chunk caches.
+func TestTaggerLLMCache_RoundTripsUnderSharedTTL(t *testing.T) {
+ store := newFakeCacheStore()
+ ctx := cacheTaskCtx("task-9")
+ allTags := map[string]float64{"a": 1}
+ want := map[string]int{"a": 4}
+
+ setTaggerLLMCache(ctx, store, "llm-1", "chunk-1", "the body", allTags, nil, 3, want)
+
+ key := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, nil, 3)
+ if got := store.ttl[key]; got != chunkcache.TTL {
+ t.Errorf("tagger cache TTL = %v, want %v", got, chunkcache.TTL)
+ }
+ if members := store.members("kc:manifest:task-9"); len(members) != 1 || members[0] != key {
+ t.Errorf("manifest = %v, want [%s]", members, key)
+ }
+ got := getTaggerLLMCache(ctx, store, "llm-1", "chunk-1", "the body", allTags, nil, 3)
+ if got == nil || got["a"] != 4 {
+ t.Errorf("getTaggerLLMCache = %v, want %v", got, want)
+ }
+ if other := getTaggerLLMCache(ctx, store, "llm-1", "chunk-2", "the body", allTags, nil, 3); other != nil {
+ t.Errorf("chunk-2 read chunk-1's entry: %v", other)
+ }
+}
diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.go
index bc3192cd8c..d83cae3dbb 100644
--- a/internal/ingestion/component/extractor_tag.go
+++ b/internal/ingestion/component/extractor_tag.go
@@ -26,8 +26,8 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/engine/redis"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/chunkcache"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
@@ -611,7 +611,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
case <-ctx.Done():
return
}
- llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.llmID, driver, model, apiKey, baseURL, topN, indexed)
+ llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.cache, in.llmID, in.modelID, driver, model, apiKey, baseURL, topN, indexed)
}(i)
}
wg.Wait()
@@ -974,10 +974,13 @@ func getChunkText(chunk map[string]any) string {
var parts []string
- // 1. Extract main chunk content (prioritize content_with_weight, then text)
- if v, ok := chunk["content_with_weight"].(string); ok && strings.TrimSpace(v) != "" {
+ // 1. Main chunk content. "text" is preferred because every chunker writes
+ // the authoritative body there and derives the chunk id from it. Legacy
+ // chunks that only carry "content_with_weight" (parser-block output, many
+ // unit tests) are accepted as a fallback so detection/tagging still works.
+ if v, ok := chunk["text"].(string); ok && strings.TrimSpace(v) != "" {
parts = append(parts, strings.TrimSpace(v))
- } else if v, ok := chunk["text"].(string); ok && strings.TrimSpace(v) != "" {
+ } else if v, ok := chunk["content_with_weight"].(string); ok && strings.TrimSpace(v) != "" {
parts = append(parts, strings.TrimSpace(v))
}
@@ -1112,7 +1115,8 @@ func llmTagChunk(
chunk map[string]any,
allTags map[string]float64,
examples []schema.TaggedChunk,
- llmID, driver, model, apiKey, baseURL string,
+ cache chunkcache.Store,
+ llmID, modelID, driver, model, apiKey, baseURL string,
topN int,
idx *MemoryTagIndex,
) {
@@ -1120,6 +1124,12 @@ func llmTagChunk(
if text == "" {
return
}
+ chunkID := chunkCacheID(chunk)
+ // Prefer the run-level resolved identity passed from runAutoTags; fall back
+ // only for direct unit callers that did not pre-resolve it.
+ if modelID == "" {
+ modelID = extractorCacheModelID(ctx, db, llmID)
+ }
textHash := int64(xxhash.Sum64String(text))
var picked []schema.TaggedChunk
@@ -1147,7 +1157,7 @@ func llmTagChunk(
}
}
- if cached := getTaggerLLMCache(ctx, llmID, text, allTags, picked, topN); cached != nil {
+ if cached := getTaggerLLMCache(ctx, cache, modelID, chunkID, text, allTags, picked, topN); cached != nil {
chunk[common.TAG_FLD] = cached
chunk["tag_kwd"] = sortedTagWeightsKeys(cached)
return
@@ -1198,7 +1208,7 @@ func llmTagChunk(
if len(result) > 0 {
chunk[common.TAG_FLD] = result
chunk["tag_kwd"] = sortedTagWeightsKeys(result)
- setTaggerLLMCache(ctx, llmID, text, allTags, picked, topN, result)
+ setTaggerLLMCache(ctx, cache, modelID, chunkID, text, allTags, picked, topN, result)
}
}
@@ -1280,34 +1290,29 @@ func jsonRepairExtract(raw string) map[string]any {
return obj
}
-func taggerCacheKey(llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) string {
- hasher := xxhash.New()
- hasher.Write([]byte(llmID))
- hasher.Write([]byte("\x00"))
- hasher.Write([]byte(text))
- hasher.Write([]byte("\x00"))
- tagNames := sortedTagNames(allTags)
- hasher.Write([]byte(strings.Join(tagNames, ",")))
- hasher.Write([]byte("\x00"))
+// taggerCacheKey builds the cache key for one chunk's LLM tagging. Keyed on the
+// chunk id rather than the chunk text, like the other per-chunk caches. The tag
+// set, the few-shot examples and topN also participate: none of them is derived
+// from the chunk, so a change in any of them yields different tags. chunkText is
+// the exact text fed to the model (getChunkText folds in the chunk body and
+// important_kwd): it participates in the key so a change to the chunk's keywords
+// — which changes the tagging prompt — busts the cache instead of serving stale
+// tags for up to the cache TTL.
+func taggerCacheKey(modelID, chunkID, chunkText string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) string {
+ config := make([]string, 0, 2*len(examples)+3)
+ config = append(config, strings.Join(sortedTagNames(allTags), ","))
for _, ex := range examples {
- hasher.Write([]byte(ex.Content))
- hasher.Write([]byte("\x00"))
tagsJSON, _ := json.Marshal(ex.TagWeights)
- hasher.Write(tagsJSON)
- hasher.Write([]byte("\x00"))
+ config = append(config, ex.Content, string(tagsJSON))
}
- hasher.Write([]byte(fmt.Sprintf("%d", topN)))
- return fmt.Sprintf("tagger:%x", hasher.Sum64())
+ config = append(config, strconv.Itoa(topN))
+ config = append(config, chunkText)
+ return chunkcache.Key("tagger", modelID, chunkID, config...)
}
-func getTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) map[string]int {
- client := redis.Get()
- if client == nil {
- return nil
- }
- key := taggerCacheKey(llmID, text, allTags, examples, topN)
- data, err := client.Get(ctx, key)
- if err != nil || data == "" {
+func getTaggerLLMCache(ctx context.Context, store chunkcache.Store, modelID, chunkID, chunkText string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) map[string]int {
+ data, hit := chunkcache.Get(ctx, store, taggerCacheKey(modelID, chunkID, chunkText, allTags, examples, topN))
+ if !hit {
return nil
}
var result map[string]int
@@ -1317,20 +1322,15 @@ func getTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[stri
return result
}
-func setTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int, result map[string]int) {
+func setTaggerLLMCache(ctx context.Context, store chunkcache.Store, modelID, chunkID, chunkText string, allTags map[string]float64, examples []schema.TaggedChunk, topN int, result map[string]int) {
if result == nil {
return
}
- client := redis.Get()
- if client == nil {
- return
- }
- key := taggerCacheKey(llmID, text, allTags, examples, topN)
data, err := json.Marshal(result)
if err != nil {
return
}
- client.Set(ctx, key, string(data), 24*time.Hour)
+ chunkcache.Set(ctx, store, taggerCacheKey(modelID, chunkID, chunkText, allTags, examples, topN), string(data))
}
func sortedTagNames(allTags map[string]float64) []string {
diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.go
index 20229b7d23..7bd908c0cd 100644
--- a/internal/ingestion/component/extractor_tag_test.go
+++ b/internal/ingestion/component/extractor_tag_test.go
@@ -279,7 +279,7 @@ func TestLlmtagChunk_MessageFit(t *testing.T) {
allTags := map[string]float64{"RAG": 1, "database": 1, "AI": 1}
examples := []schema.TaggedChunk{{Content: "example one", TagWeights: map[string]int{"AI": 5}}}
- llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
if len(capt.req.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))
@@ -302,7 +302,7 @@ func TestLlmtagChunk_NoContextLength_SkipsFit(t *testing.T) {
allTags := map[string]float64{"RAG": 1}
examples := []schema.TaggedChunk{{Content: "example", TagWeights: map[string]int{"AI": 5}}}
- llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
if len(capt.req.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))
@@ -324,7 +324,7 @@ func TestLlmtagChunk_ColdStartFallback(t *testing.T) {
}
chunk := map[string]any{"content_with_weight": "some content"}
- llmTagChunk(t.Context(), nil, capt, chunk, idx.allTags, nil, "test@test", "test_driver", "test_model", "test_key", "", 3, idx)
+ llmTagChunk(t.Context(), nil, capt, chunk, idx.allTags, nil, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, idx)
if len(capt.req.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))
@@ -721,7 +721,7 @@ func TestPopulateTagKwd_LLMTagChunk(t *testing.T) {
chunk := map[string]any{"content_with_weight": "some content"}
allTags := map[string]float64{"RAG": 0.5, "vector database": 0.5}
- llmTagChunk(t.Context(), nil, capt, chunk, allTags, nil, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, nil, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
tagKwd, ok := chunk["tag_kwd"].([]string)
if !ok {
@@ -1166,10 +1166,10 @@ func TestTaggerCacheKey_IncludesFewShot(t *testing.T) {
{Content: "sample one", TagWeights: map[string]int{"TagA": 5}},
}
- k1 := taggerCacheKey("llm-1", "test text", allTags, ex1, 3)
- k2 := taggerCacheKey("llm-1", "test text", allTags, ex2, 3)
- k3 := taggerCacheKey("llm-1", "test text", allTags, ex3, 3)
- kEmpty := taggerCacheKey("llm-1", "test text", allTags, nil, 3)
+ k1 := taggerCacheKey("llm-1", "test text", "body", allTags, ex1, 3)
+ k2 := taggerCacheKey("llm-1", "test text", "body", allTags, ex2, 3)
+ k3 := taggerCacheKey("llm-1", "test text", "body", allTags, ex3, 3)
+ kEmpty := taggerCacheKey("llm-1", "test text", "body", allTags, nil, 3)
if k1 == k2 {
t.Fatalf("expected different cache keys for different few-shot examples: %s vs %s", k1, k2)
@@ -1182,7 +1182,7 @@ func TestTaggerCacheKey_IncludesFewShot(t *testing.T) {
}
// Identical few-shot examples produce identical key
- k1Dup := taggerCacheKey("llm-1", "test text", allTags, ex1, 3)
+ k1Dup := taggerCacheKey("llm-1", "test text", "body", allTags, ex1, 3)
if k1 != k1Dup {
t.Fatalf("expected identical cache keys for same few-shot examples: %s vs %s", k1, k1Dup)
}
diff --git a/internal/ingestion/component/globals/globals.go b/internal/ingestion/component/globals/globals.go
index cdf5c1cdbe..9b64fb6e63 100644
--- a/internal/ingestion/component/globals/globals.go
+++ b/internal/ingestion/component/globals/globals.go
@@ -53,10 +53,11 @@ func canvasStateFromContext(ctx context.Context) *runtime.CanvasState {
// depends on (e.g. TokenChunker drops `name`, which Tokenizer consumes for
// title embedding). Storing the shared fields in CanvasState.Globals restores
// the Python behaviour without mutating every component output.
-// The embedding-model id is intentionally NOT a global: it is a
-// Tokenizer-scoped setup (params["setups"]["embedding_model"]). Keeping it out
-// of the shared bag prevents another component (e.g. one expecting a chat
-// model) from misreading a generic "model_id" global as its own.
+// The embedding-model id is intentionally NOT a global: the Tokenizer resolves
+// it from the dataset's own embd_id (kb_id) via its injected resolver, never
+// from a run input. Keeping it out of the shared bag prevents another component
+// (e.g. one expecting a chat model) from misreading a generic "model_id" global
+// as its own.
var GlobalMetadataKeys = []string{
"name",
"doc_id",
@@ -73,6 +74,38 @@ var GlobalMetadataKeys = []string{
DebugChunkCapKey,
}
+// taskIDKey is the CanvasState.Globals slot carrying the ingestion task id of
+// the current run. It is deliberately unexported and absent from
+// GlobalMetadataKeys: the pipeline seeds it once via SetTaskID, so no run
+// input and no component output can overwrite the run's task scope.
+const taskIDKey = "task_id"
+
+// SetTaskID records the ingestion task id of the current run so components can
+// scope per-task bookkeeping (e.g. the per-chunk cache manifest). Called once
+// by the pipeline at run start. No-op when no CanvasState is attached.
+func SetTaskID(ctx context.Context, taskID string) {
+ if taskID == "" {
+ return
+ }
+ if st := canvasStateFromContext(ctx); st != nil {
+ st.SetGlobal(taskIDKey, taskID)
+ }
+}
+
+// TaskID returns the ingestion task id of the current run, or "" when the run
+// has no task scope (headless component tests, or a canvas invoked outside the
+// ingestion pipeline). Callers must treat "" as "skip per-task bookkeeping".
+func TaskID(ctx context.Context) string {
+ if st := canvasStateFromContext(ctx); st != nil {
+ if v, ok := st.GetGlobal(taskIDKey); ok {
+ if s, ok := v.(string); ok {
+ return s
+ }
+ }
+ }
+ return ""
+}
+
// DebugChunkCapKey is the run-input key (seeded into CanvasState.Globals)
// carrying the canvas-debug (dry-run) chunk cap: when >= 1, a chunker node in
// a debug run emits at most this many leading chunks for preview. 0 means "no
diff --git a/internal/ingestion/component/globals/globals_test.go b/internal/ingestion/component/globals/globals_test.go
index c418a42fc1..b4d83dca49 100644
--- a/internal/ingestion/component/globals/globals_test.go
+++ b/internal/ingestion/component/globals/globals_test.go
@@ -67,6 +67,56 @@ func TestDebugChunkCap_ReadsFromGlobals(t *testing.T) {
}
}
+// TestTaskID_EmptyWithoutState asserts TaskID degrades to "" when no
+// CanvasState is attached (headless unit tests) or the key was never seeded.
+// Callers use "" as "no task scope" and must skip per-task bookkeeping.
+func TestTaskID_EmptyWithoutState(t *testing.T) {
+ if got := TaskID(context.Background()); got != "" {
+ t.Errorf("TaskID(bare ctx) = %q, want \"\"", got)
+ }
+ ctx := runtime.WithState(context.Background(), &runtime.CanvasState{Globals: map[string]any{}})
+ if got := TaskID(ctx); got != "" {
+ t.Errorf("TaskID(empty globals) = %q, want \"\"", got)
+ }
+}
+
+// TestSetTaskID_RoundTrips asserts the pipeline-side seeder and the
+// component-side reader agree on the storage slot.
+func TestSetTaskID_RoundTrips(t *testing.T) {
+ st := &runtime.CanvasState{Globals: map[string]any{}}
+ ctx := runtime.WithState(context.Background(), st)
+ SetTaskID(ctx, "task-42")
+ if got := TaskID(ctx); got != "task-42" {
+ t.Errorf("TaskID = %q, want \"task-42\"", got)
+ }
+}
+
+// TestSetTaskID_NoStateIsNoop asserts seeding without a CanvasState does not
+// panic — headless component tests invoke stages with a bare context.
+func TestSetTaskID_NoStateIsNoop(t *testing.T) {
+ SetTaskID(context.Background(), "task-42")
+}
+
+// TestTaskIDKey_NotAWhitelistedMetadataKey asserts the task id is seeded
+// explicitly by the pipeline and is NOT part of GlobalMetadataKeys. Being on
+// that whitelist would let any component output (or run input) carrying a
+// "task_id" field overwrite the run's real task id, which would silently
+// redirect per-task cache bookkeeping to the wrong task.
+func TestTaskIDKey_NotAWhitelistedMetadataKey(t *testing.T) {
+ for _, k := range GlobalMetadataKeys {
+ if k == taskIDKey {
+ t.Fatalf("GlobalMetadataKeys must not contain %q", taskIDKey)
+ }
+ }
+ st := &runtime.CanvasState{Globals: map[string]any{}}
+ ctx := runtime.WithState(context.Background(), st)
+ SetTaskID(ctx, "task-real")
+ SeedIngestionGlobals(ctx, map[string]any{taskIDKey: "task-spoofed"})
+ if got := TaskID(ctx); got != "task-real" {
+ t.Errorf("TaskID after hostile seed = %q, want \"task-real\"", got)
+ }
+}
+
// TestSeedIngestionGlobals_PicksUpDebugChunkCap asserts that adding
// DebugChunkCapKey to GlobalMetadataKeys makes SeedIngestionGlobals copy the
// run-input value into CanvasState.Globals — the link the executor relies on
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index d6bbed6eee..b85bc88fbd 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -93,6 +93,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
+ "ragflow/internal/ingestion/chunkcache"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
@@ -153,11 +154,14 @@ type Embedder interface {
Encode(ctx context.Context, texts []string) ([]EmbeddingResult, error)
}
-// EmbedderResolver resolves the embedder for one tokenizer invocation.
-// embeddingModel is the Tokenizer-scoped embedding-model identifier (from the
-// component's setups); an empty value tells the resolver to fall back to the
-// dataset's configured model.
-type EmbedderResolver func(ctx context.Context, tenantID, kbID, embeddingModel string) (Embedder, error)
+// EmbedderResolver resolves the embedder and its dataset-bound embedding-model
+// id for one tokenizer invocation. The resolver derives the model exclusively
+// from the knowledgebase's configured embd_id (see internal/ingestion/task/
+// embedder.go); it never reads any embedding identifier from the DSL. The
+// returned embdID is the stable string the tokenizer keys its per-chunk
+// embedding cache on, so a KB whose embedding model changes yields a different
+// key and never serves a stale vector.
+type EmbedderResolver func(ctx context.Context, tenantID, kbID string) (Embedder, string, error)
// DefaultEmbedderResolver is the production embedder resolver. It is nil in
// this leaf package — which must not import internal/service (see the
@@ -174,8 +178,9 @@ var DefaultEmbedderResolver EmbedderResolver
// Inputs:
//
// tenant_id (string, optional) — used to resolve the embedding model
-// kb_id (string, optional) — dataset whose embd_id is used when the
-// setups embedding_model is unset
+// kb_id (string, optional) — dataset whose embd_id selects the embedding
+// model. The model always comes from the dataset;
+// the DSL never configures it.
// output_format (string) — one of json/markdown/text/html/chunks
// chunks (list[map]) — chunk list when output_format == "chunks"
// json (list[map]) — structured parser payload when output_format == "json" or unset
@@ -191,9 +196,8 @@ var DefaultEmbedderResolver EmbedderResolver
// output_format — always "chunks" (matches python set_output)
// _created_time / _elapsed_time — TrackElapsed bookkeeping
type TokenizerComponent struct {
- param schema.TokenizerParam
- resolver EmbedderResolver
- embeddingModel string
+ param schema.TokenizerParam
+ resolver EmbedderResolver
}
// NewTokenizerComponent constructs a production TokenizerComponent from DSL
@@ -215,7 +219,6 @@ func NewTokenizerComponentWithResolver(params map[string]any, resolver EmbedderR
func newTokenizerComponent(params map[string]any, resolver EmbedderResolver) (runtime.Component, error) {
p := schema.TokenizerParam{}.Defaults()
- embeddingModel := ""
if params != nil {
if v, ok := params["search_method"]; ok {
// Replace (not append) so a caller-supplied
@@ -256,35 +259,18 @@ func newTokenizerComponent(params map[string]any, resolver EmbedderResolver) (ru
p.Fields = append(p.Fields, t...)
}
}
- embeddingModel = embeddingModelFromSetups(params)
}
if err := p.Validate(); err != nil {
return nil, fmt.Errorf("tokenizer: param check: %w", err)
}
- return &TokenizerComponent{param: p, resolver: resolver, embeddingModel: embeddingModel}, nil
-}
-
-// embeddingModelFromSetups extracts the embedding-model identifier from the
-// component's setups map (params["setups"]["embedding_model"]). The embedding
-// model id is a Tokenizer-scoped setup rather than a run-level global so it is
-// never mistaken for, e.g., a chat model id shared across components. Empty
-// when unset — the resolver then falls back to the dataset's configured model.
-func embeddingModelFromSetups(params map[string]any) string {
- setups, ok := params["setups"].(map[string]any)
- if !ok {
- return ""
- }
- if v, ok := setups["embedding_model"].(string); ok {
- return strings.TrimSpace(v)
- }
- return ""
+ return &TokenizerComponent{param: p, resolver: resolver}, nil
}
// Inputs returns the parameter metadata.
func (c *TokenizerComponent) Inputs() map[string]string {
return map[string]string{
"tenant_id": "Tenant identifier used to resolve the embedding model (mirrors python self._canvas._tenant_id).",
- "kb_id": "Optional knowledgebase identifier used to resolve the bound embedding model when the setups embedding_model is unset.",
+ "kb_id": "Knowledgebase identifier used to resolve the bound embedding model (kb.embd_id). The embedding model is taken exclusively from the dataset; the DSL must not configure it.",
"output_format": "Upstream payload discriminator: json / markdown / text / html / chunks.",
"chunks": "List of chunk maps when output_format == \"chunks\".",
"json": "Structured parser payload when output_format == \"json\" or unset.",
@@ -327,10 +313,6 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
name := globals.GlobalOrInput(ctx, inputs, "name", "")
tenantID := globals.GlobalOrInput(ctx, inputs, "tenant_id", "")
kbID := globals.GlobalOrInput(ctx, inputs, "kb_id", "")
- // The embedding-model id is a Tokenizer-scoped setup (params["setups"]),
- // resolved at construction, not a run-level global — see
- // embeddingModelFromSetups.
- embeddingModel := c.embeddingModel
// decodeTokenizerFromUpstream validates `name`; carry the resolved
// name into the decode input so both a Globals-backed run and a
@@ -382,7 +364,7 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
// so embedding is skipped there — debug only exercises parse+chunk and
// must stay side-effect free.
if shouldHaveEmbedding(c.param.SearchMethod, kbID) {
- chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, embeddingModel, name, chunks)
+ chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, name, chunks, chunkcache.Client())
if err != nil {
return nil, err
}
@@ -408,7 +390,7 @@ func copyPipelineControlValues(output, input map[string]any) {
}
}
-func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, embeddingModel, name string, chunks []schema.ChunkDoc) ([]schema.ChunkDoc, int, error) {
+func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, name string, chunks []schema.ChunkDoc, store chunkcache.Store) ([]schema.ChunkDoc, int, error) {
if len(chunks) == 0 {
return chunks, 0, nil
}
@@ -421,7 +403,7 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, em
if resolver == nil {
return nil, 0, fmt.Errorf("tokenizer: embedding requested but no embedder resolver configured")
}
- embedder, err := resolver(ctx, tenantID, kbID, embeddingModel)
+ embedder, embdID, err := resolver(ctx, tenantID, kbID)
if err != nil {
return nil, 0, fmt.Errorf("tokenizer: resolve embedder: %w", err)
}
@@ -429,21 +411,60 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, em
return nil, 0, fmt.Errorf("tokenizer: embedding requested but encoder resolution returned nil")
}
+ // store may be nil (no redis configured / unit test); cache lookups then
+ // degrade to a no-op and every chunk is embedded normally.
+
+ // resolved[i] holds the content embedding vector for chunks[i] once known:
+ // either served from the per-chunk cache (isHit) or produced by the batch
+ // Encode below. nil means the chunk produced no embeddable text. trunc is the
+ // exact text fed to the model, carried so the Set-side key matches the
+ // Get-side key (the key includes the embedded text, not just the chunk id).
+ type resolvedVec struct {
+ content []float64
+ isHit bool
+ trunc string
+ }
+ resolved := make([]*resolvedVec, len(chunks))
texts := make([]string, 0, len(chunks))
pairs := make([]int, 0, len(chunks))
+ // truncs[i] is the embedded text for texts[i] / pairs[i]; carried so the
+ // Set-side key matches the Get-side key for freshly embedded content.
+ truncs := make([]string, 0, len(chunks))
for i, ck := range chunks {
raw := concatFields(ck, c.param.Fields)
txt := htmlTableRE.ReplaceAllString(raw, " ")
txt = strings.TrimSpace(txt)
- trunc := truncateForEmbedding(txt, embedder.MaxTokens())
if txt == "" {
continue
}
+ // Per-chunk embedding cache: identical (dataset embd_id, chunk, embedded
+ // text) triples reuse the previous content vector, skipping the embed
+ // round-trip on resume. The key includes the text that is actually
+ // embedded — after field selection (c.param.Fields) and truncation — not
+ // just the chunk id. A tokenizer-config or model change alters the
+ // embedded input; reusing the prior vector for that case would serve a
+ // stale embedding for up to the cache TTL. embdID must be non-empty or
+ // every model would collapse onto one key and served vectors could come
+ // from a different model.
+ trunc := truncateForEmbedding(txt, embedder.MaxTokens())
+ if chunkID, ok := ck.GetExtraString("id"); ok && embdID != "" && store != nil {
+ if cached, hit := chunkcache.Get(ctx, store, chunkcache.Key("emb", embdID, chunkID, trunc)); hit {
+ var vec []float64
+ if err := json.Unmarshal([]byte(cached), &vec); err == nil && len(vec) > 0 {
+ resolved[i] = &resolvedVec{content: vec, isHit: true, trunc: trunc}
+ continue
+ }
+ }
+ }
texts = append(texts, trunc)
+ truncs = append(truncs, trunc)
pairs = append(pairs, i)
}
if len(texts) == 0 {
- return chunks, 0, nil
+ // Nothing to embed — but cache hits may still need to be written through.
+ if store == nil {
+ return chunks, 0, nil
+ }
}
trimmedName := strings.TrimSpace(name)
@@ -502,17 +523,39 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, em
}
titleWeight := c.param.FilenameEmbdWeight
+ // Wire freshly embedded content into the resolved slice; cache hits already
+ // carry their vector from the loop above.
for i, idx := range pairs {
- merged := append([]float64(nil), contentResults[i].Vector...)
+ resolved[idx] = &resolvedVec{content: contentResults[i].Vector, trunc: truncs[i]}
+ }
+ for i, re := range resolved {
+ if re == nil {
+ continue
+ }
+ merged := append([]float64(nil), re.content...)
if hasTitleVec {
- merged, err = mergeEmbeddingVectors(titleVec, contentResults[i].Vector, titleWeight)
+ merged, err = mergeEmbeddingVectors(titleVec, re.content, titleWeight)
if err != nil {
return nil, 0, fmt.Errorf("tokenizer: merge vectors: %w", err)
}
}
- if err := chunks[idx].SetExtraValue(fmt.Sprintf("q_%d_vec", len(merged)), merged); err != nil {
+ if err := chunks[i].SetExtraValue(fmt.Sprintf("q_%d_vec", len(merged)), merged); err != nil {
return nil, 0, fmt.Errorf("tokenizer: vector marshal: %w", err)
}
+ // Backfill the per-chunk cache only for freshly embedded content; cache
+ // hits are left untouched. The key intentionally omits title weighting:
+ // the content vector is title-weight independent, so identical chunk
+ // content reuses across runs even when only the filename weight changes.
+ // The embedded text (trunc) is part of the key so a config/model change
+ // that alters the embedded input forces a fresh embed rather than serving
+ // a stale vector.
+ if !re.isHit {
+ if chunkID, ok := chunks[i].GetExtraString("id"); ok && embdID != "" && store != nil {
+ if b, merr := json.Marshal(re.content); merr == nil {
+ chunkcache.Set(ctx, store, chunkcache.Key("emb", embdID, chunkID, re.trunc), string(b))
+ }
+ }
+ }
}
return chunks, tokenCount, nil
}
diff --git a/internal/ingestion/component/tokenizer_cache_test.go b/internal/ingestion/component/tokenizer_cache_test.go
new file mode 100644
index 0000000000..df29a1e864
--- /dev/null
+++ b/internal/ingestion/component/tokenizer_cache_test.go
@@ -0,0 +1,313 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "encoding/json"
+ "sync"
+ "testing"
+ "time"
+
+ "ragflow/internal/ingestion/chunkcache"
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// vecOf reads a float64 vector stored by SetExtraValue back out of the chunk.
+func vecOf(t *testing.T, doc schema.ChunkDoc, key string) []float64 {
+ t.Helper()
+ raw, ok := doc.Extra[key]
+ if !ok {
+ t.Fatalf("missing extra %s", key)
+ }
+ var v []float64
+ if err := json.Unmarshal(raw, &v); err != nil {
+ t.Fatalf("unmarshal %s: %v", key, err)
+ }
+ return v
+}
+
+// memCacheStore is an in-memory chunkcache.Store used to exercise the
+// per-chunk embedding cache without Redis.
+type memCacheStore struct {
+ mu sync.Mutex
+ data map[string]string
+ sets map[string]map[string]struct{}
+}
+
+func newMemCacheStore() *memCacheStore {
+ return &memCacheStore{data: map[string]string{}, sets: map[string]map[string]struct{}{}}
+}
+
+func (m *memCacheStore) Get(_ context.Context, key string) (string, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return m.data[key], nil
+}
+
+func (m *memCacheStore) Set(_ context.Context, key, value string, _ time.Duration) bool {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.data[key] = value
+ return true
+}
+
+func (m *memCacheStore) SAdd(_ context.Context, key, member string) bool {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.sets[key] == nil {
+ m.sets[key] = map[string]struct{}{}
+ }
+ m.sets[key][member] = struct{}{}
+ return true
+}
+
+func (m *memCacheStore) SMembers(_ context.Context, key string) ([]string, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ out := make([]string, 0, len(m.sets[key]))
+ for k := range m.sets[key] {
+ out = append(out, k)
+ }
+ return out, nil
+}
+
+func (m *memCacheStore) Delete(_ context.Context, key string) bool {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ delete(m.data, key)
+ delete(m.sets, key)
+ return true
+}
+
+func (m *memCacheStore) Expire(_ context.Context, key string, _ time.Duration) bool {
+ return true
+}
+
+func (m *memCacheStore) countEmbKeys() int {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ n := 0
+ for k := range m.data {
+ if len(k) >= len("kc:emb:") && k[:len("kc:emb:")] == "kc:emb:" {
+ n++
+ }
+ }
+ return n
+}
+
+// chunkWithID builds a ChunkDoc carrying a chunk id in Extra (so the cache
+// key can be derived) plus the given body text.
+func chunkWithID(id, text string) schema.ChunkDoc {
+ doc, _ := schema.ChunkDocFromMap(map[string]any{"id": id, "text": text})
+ return doc
+}
+
+func TestEmbedChunks_PerChunkCacheMissThenHit(t *testing.T) {
+ comp, stub := withStubEmbedder(t, 4)
+ comp.param.Fields = []string{"text"}
+
+ store := newMemCacheStore()
+ chunks := []schema.ChunkDoc{chunkWithID("chunk-1", "hello world")}
+
+ // First pass: cache miss -> embedder is called once.
+ out1, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", chunks, store)
+ if err != nil {
+ t.Fatalf("embedChunks (miss): %v", err)
+ }
+ if got := stub.calls.Load(); got != 1 {
+ t.Fatalf("expected 1 embedder call on miss, got %d", got)
+ }
+ vec1 := vecOf(t, out1[0], "q_4_vec")
+ if store.countEmbKeys() != 1 {
+ t.Fatalf("expected 1 emb cache key after miss, got %d", store.countEmbKeys())
+ }
+
+ // Second pass with the same chunk id: cache hit -> embedder NOT called again.
+ out2, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", chunks, store)
+ if err != nil {
+ t.Fatalf("embedChunks (hit): %v", err)
+ }
+ if got := stub.calls.Load(); got != 1 {
+ t.Fatalf("expected embedder to be skipped on cache hit, but calls=%d", got)
+ }
+ vec2 := vecOf(t, out2[0], "q_4_vec")
+ if len(vec1) != len(vec2) {
+ t.Fatalf("cached vector differs from embedded vector: %v vs %v", vec1, vec2)
+ }
+ for i := range vec1 {
+ if vec1[i] != vec2[i] {
+ t.Fatalf("cached vector differs from embedded vector: %v vs %v", vec1, vec2)
+ }
+ }
+}
+
+func TestEmbedChunks_CacheKeyScopedByChunkAndEmbdID(t *testing.T) {
+ embdID := "embd-a"
+ comp, stub := withStubEmbedderEmbdID(t, 4, &embdID)
+ comp.param.Fields = []string{"text"}
+
+ store := newMemCacheStore()
+
+ // Two distinct chunks -> two embedder calls, two cache keys.
+ a := []schema.ChunkDoc{chunkWithID("chunk-a", "alpha"), chunkWithID("chunk-b", "beta")}
+ if _, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", a, store); err != nil {
+ t.Fatalf("embedChunks: %v", err)
+ }
+ if got := stub.calls.Load(); got != 1 {
+ t.Fatalf("expected 1 batch call for 2 chunks, got %d", got)
+ }
+ if n := store.countEmbKeys(); n != 2 {
+ t.Fatalf("expected 2 emb cache keys for 2 distinct chunks, got %d", n)
+ }
+
+ // Rebinding the dataset to a different embedding model must bypass the
+ // cache: the key is derived from the dataset's embd_id, so the entry
+ // written under embd-a is not reusable and a stale vector is never served.
+ embdID = "embd-b"
+ b := []schema.ChunkDoc{chunkWithID("chunk-a", "alpha")}
+ if _, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", b, store); err != nil {
+ t.Fatalf("embedChunks (rebound embd_id): %v", err)
+ }
+ if got := stub.calls.Load(); got != 2 {
+ t.Fatalf("expected a new embedder call after the dataset's embd_id changed, got calls=%d", got)
+ }
+ if n := store.countEmbKeys(); n != 3 {
+ t.Fatalf("expected 3 emb cache keys after embd_id change, got %d", n)
+ }
+}
+
+// The embedding model comes from the dataset, never from the DSL: two components
+// configured with different setups.embedding_model but the same dataset embd_id
+// must resolve to the same cache entry. This locks in that the DSL cannot
+// influence the cache key (and would fail if setups.embedding_model were read
+// again), which is what previously let a re-bound dataset serve a stale vector.
+func TestEmbedChunks_CacheIgnoresDSLEmbeddingModel(t *testing.T) {
+ embdID := "embd-a"
+ stub := newStubEmbedder(4)
+ build := func(dslModel string) *TokenizerComponent {
+ comp, err := NewTokenizerComponentWithResolver(
+ map[string]any{"setups": map[string]any{"embedding_model": dslModel}},
+ func(_ context.Context, _, _ string) (Embedder, string, error) { return stub, embdID, nil },
+ )
+ if err != nil {
+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
+ }
+ c := comp.(*TokenizerComponent)
+ c.param.Fields = []string{"text"}
+ return c
+ }
+
+ store := newMemCacheStore()
+ chunks := []schema.ChunkDoc{chunkWithID("chunk-1", "hello world")}
+
+ if _, _, err := build("dsl-model-a").embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {
+ t.Fatalf("embedChunks (dsl-model-a): %v", err)
+ }
+ if got := stub.calls.Load(); got != 1 {
+ t.Fatalf("expected 1 embedder call on miss, got %d", got)
+ }
+
+ // Same dataset embd_id, different DSL model -> must hit the same entry.
+ if _, _, err := build("dsl-model-b").embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {
+ t.Fatalf("embedChunks (dsl-model-b): %v", err)
+ }
+ if got := stub.calls.Load(); got != 1 {
+ t.Fatalf("DSL embedding_model must not affect the cache key: expected a hit, got calls=%d", got)
+ }
+ if n := store.countEmbKeys(); n != 1 {
+ t.Fatalf("expected 1 emb cache key across both DSL models, got %d", n)
+ }
+}
+
+// TestEmbedChunks_CacheKeyScopedByFields proves the embedding cache key captures
+// the text actually embedded — which depends on c.param.Fields — so changing the
+// tokenizer field selection forces a fresh embed instead of serving a stale
+// vector for up to the cache TTL. The chunk id is unchanged across the two runs,
+// isolating the variable to the embedded text.
+func TestEmbedChunks_CacheKeyScopedByFields(t *testing.T) {
+ embdID := "embd-test"
+ stub := newStubEmbedder(4)
+ build := func(fields []string) *TokenizerComponent {
+ comp, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) {
+ return stub, embdID, nil
+ })
+ if err != nil {
+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
+ }
+ c := comp.(*TokenizerComponent)
+ c.param.Fields = fields
+ return c
+ }
+
+ store := newMemCacheStore()
+ doc := schema.ChunkDoc{Text: "hello", Questions: "q1;q2"}
+ doc.Extra = map[string]json.RawMessage{}
+ if err := doc.SetExtraValue("id", "chunk-1"); err != nil {
+ t.Fatalf("SetExtraValue(id): %v", err)
+ }
+ chunks := []schema.ChunkDoc{doc}
+
+ // Fields=["text"]: only the body is embedded -> cache miss, 1 call.
+ if _, _, err := build([]string{"text"}).embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {
+ t.Fatalf("embedChunks (fields=text): %v", err)
+ }
+ if got := stub.calls.Load(); got != 1 {
+ t.Fatalf("expected 1 embedder call with fields=text, got %d", got)
+ }
+
+ // Fields=["text","questions"]: the embedded text now includes the questions,
+ // so the key must differ and the cache must be bypassed (another call).
+ if _, _, err := build([]string{"text", "questions"}).embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {
+ t.Fatalf("embedChunks (fields=text,questions): %v", err)
+ }
+ if got := stub.calls.Load(); got != 2 {
+ t.Fatalf("expected a fresh embed when Fields change the embedded text, got calls=%d (stale vector would otherwise be served)", got)
+ }
+ if n := store.countEmbKeys(); n != 2 {
+ t.Fatalf("expected 2 emb cache keys (one per embedded-text variant), got %d", n)
+ }
+}
+
+func TestEmbedChunks_MissingChunkIDSkipsCache(t *testing.T) {
+ comp, stub := withStubEmbedder(t, 4)
+ comp.param.Fields = []string{"text"}
+
+ store := newMemCacheStore()
+ // Chunk without an id: cannot be cached, always embedded.
+ noID, _ := schema.ChunkDocFromMap(map[string]any{"text": "no id here"})
+
+ first, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", []schema.ChunkDoc{noID}, store)
+ if err != nil {
+ t.Fatalf("embedChunks (no id): %v", err)
+ }
+ if stub.calls.Load() != 1 {
+ t.Fatalf("expected embedder call for uncacheable chunk")
+ }
+ vecOf(t, first[0], "q_4_vec")
+ if n := store.countEmbKeys(); n != 0 {
+ t.Fatalf("expected no emb cache key for id-less chunk, got %d", n)
+ }
+
+ // Second pass: still no id -> still embedded (calls increment).
+ comp.embedChunks(context.Background(), "tenant", "kb", "", []schema.ChunkDoc{noID}, store)
+ if stub.calls.Load() != 2 {
+ t.Fatalf("expected a second embedder call for id-less chunk (cache not used), got %d", stub.calls.Load())
+ }
+}
+
+var _ chunkcache.Store = (*memCacheStore)(nil)
diff --git a/internal/ingestion/component/tokenizer_nonpersist_test.go b/internal/ingestion/component/tokenizer_nonpersist_test.go
index 1d9a8c2457..7eee084374 100644
--- a/internal/ingestion/component/tokenizer_nonpersist_test.go
+++ b/internal/ingestion/component/tokenizer_nonpersist_test.go
@@ -42,7 +42,7 @@ func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {
stub := newStubEmbedder(4)
cIntf, err := NewTokenizerComponentWithResolver(
map[string]any{"search_method": []any{"embedding"}, "fields": []any{"text"}},
- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },
+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },
)
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.go
index ed6902400a..5703209b0e 100644
--- a/internal/ingestion/component/tokenizer_test.go
+++ b/internal/ingestion/component/tokenizer_test.go
@@ -385,7 +385,7 @@ func TestTokenizerComponent_Invoke_EncoderCountMismatch(t *testing.T) {
// Inject an embedder that returns the wrong number of vectors
// regardless of input.
wrong := &countMismatchedEmbedder{want: 1}
- cIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _, _ string) (Embedder, error) { return wrong, nil })
+ cIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) { return wrong, "", nil })
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
}
@@ -525,13 +525,13 @@ func TestTokenizerComponent_Embedding_UsesFilenameWeight(t *testing.T) {
requireTokenizerPool(t)
cIntf, err := NewTokenizerComponentWithResolver(map[string]any{
"filename_embd_weight": 0.25,
- }, func(_ context.Context, _, _, _ string) (Embedder, error) {
+ }, func(_ context.Context, _, _ string) (Embedder, string, error) {
stub := newStubEmbedder(2)
stub.resultsByCall = []embeddingCallResult{
{vectors: [][]float64{{8, 8}}, tokenCount: 3},
{vectors: [][]float64{{2, 2}}, tokenCount: 5},
}
- return stub, nil
+ return stub, "", nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
@@ -735,18 +735,18 @@ func floatSliceClose(got, want []float64) bool {
func TestTokenizerComponent_InstanceResolversDoNotLeakAcrossComponents(t *testing.T) {
requireTokenizerPool(t)
- compAIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _, _ string) (Embedder, error) {
+ compAIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) {
stub := newStubEmbedder(2)
stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{10, 10}}, tokenCount: 1}, {vectors: [][]float64{{1, 1}}, tokenCount: 1}}
- return stub, nil
+ return stub, "", nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver(A): %v", err)
}
- compBIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _, _ string) (Embedder, error) {
+ compBIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) {
stub := newStubEmbedder(2)
stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{20, 20}}, tokenCount: 1}, {vectors: [][]float64{{2, 2}}, tokenCount: 1}}
- return stub, nil
+ return stub, "", nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver(B): %v", err)
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index c60acb814c..3738df1d60 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -115,9 +115,23 @@ func newStubEmbedder(dim int) *stubEmbedder {
// (["full_text","embedding"]); callers that need a different mode construct
// the component directly via NewTokenizerComponent(NewTokenizerComponentWithResolver).
func withStubEmbedder(t *testing.T, dim int) (*TokenizerComponent, *stubEmbedder) {
+ // Non-empty: the per-chunk cache is skipped unless the dataset reports an
+ // embd_id, so an empty value here would silently disable the cache.
+ embdID := "embd-test"
+ return withStubEmbedderEmbdID(t, dim, &embdID)
+}
+
+// withStubEmbedderEmbdID builds a TokenizerComponent whose resolver reports the
+// value pointed to by embdID at call time. The per-chunk embedding cache keys on
+// that id, so mutating *embdID between two embedChunks passes models "the
+// dataset's embedding model was rebound" — the two passes must then share no
+// cache entry, which is exactly what a stale vector would otherwise cause.
+func withStubEmbedderEmbdID(t *testing.T, dim int, embdID *string) (*TokenizerComponent, *stubEmbedder) {
t.Helper()
stub := newStubEmbedder(dim)
- comp, err := NewTokenizerComponentWithResolver(nil, func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil })
+ comp, err := NewTokenizerComponentWithResolver(nil, func(ctx context.Context, _, _ string) (Embedder, string, error) {
+ return stub, *embdID, nil
+ })
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
}
@@ -196,8 +210,8 @@ func TestTokenizerComponent_Invoke_NilChunks(t *testing.T) {
func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {
cIntf, err := NewTokenizerComponentWithResolver(map[string]any{
"search_method": []any{"embedding"},
- }, func(ctx context.Context, _, _, _ string) (Embedder, error) {
- return newStubEmbedder(4), nil
+ }, func(ctx context.Context, _, _ string) (Embedder, string, error) {
+ return newStubEmbedder(4), "", nil
})
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
@@ -460,7 +474,7 @@ func TestChunkOrderInt_EmbeddingOnly(t *testing.T) {
stub := newStubEmbedder(3)
comp, err := NewTokenizerComponentWithResolver(
map[string]any{"search_method": []string{"embedding"}, "fields": []string{"text"}},
- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },
+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },
)
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
@@ -514,7 +528,7 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {
stub := newStubEmbedder(3)
comp, err := NewTokenizerComponentWithResolver(
map[string]any{"search_method": []string{"embedding"}, "fields": []string{"text"}},
- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },
+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },
)
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
@@ -584,7 +598,7 @@ func TestChunkOrderInt_AllPathsUnconditional(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
comp, err := NewTokenizerComponentWithResolver(
map[string]any{"search_method": tc.searchMethod, "fields": []string{"text"}},
- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },
+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },
)
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
@@ -627,7 +641,7 @@ func TestChunkOrderInt_A5FilterKeepsSequenceContiguous(t *testing.T) {
stub := newStubEmbedder(8)
comp, err := NewTokenizerComponentWithResolver(
map[string]any{"search_method": []string{"full_text", "embedding"}, "fields": []string{"text"}},
- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },
+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },
)
if err != nil {
t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index a73a081073..4a6740e8fa 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -30,6 +30,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
redis2 "ragflow/internal/engine/redis"
+ "ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/utility"
@@ -229,7 +230,10 @@ func cloneMapOrEmpty(m map[string]any) map[string]any {
// defaultCheckpointTTL is the expiry applied to the eino checkpoint payload
// and the RunTracker hash. A finished run's checkpoint is deleted on success;
// the TTL only guards against leaks from crashed runs that never clean up.
-var defaultCheckpointTTL = 24 * time.Hour
+//
+// It matches the per-chunk cache TTL (chunkcache.TTL) so a crashed run leaves
+// both its checkpoint and the chunk cache entries to expire on the same horizon.
+var defaultCheckpointTTL = 7 * 24 * time.Hour
// dslKeySuffix / ovfKeySuffix derive the two DSL-fingerprint keys from a
// checkpoint id. Both live in the same CheckPointStore as the eino payload
@@ -437,8 +441,18 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam
compileOpts = append(compileOpts,
canvas.WithCheckPointStore(store),
canvas.WithCheckPointID(p.taskID),
- canvas.WithInterruptAfterNonTerminalCpn(),
)
+ // Single interrupt point: immediately after the Parser component. The
+ // Extractor / Tokenizer stages do NOT take a checkpoint of their own —
+ // a resume re-runs them, but the per-chunk cache (chunkcache) absorbs
+ // the cost by serving prior LLM/embedding results. Interruption after
+ // every non-terminal component was dropped on purpose to avoid one
+ // needless ResumeWithData round per stage.
+ if parserCpnID := ExtractParserCpnID(p.rawDSL, component.ComponentNameParser); parserCpnID != "" {
+ compileOpts = append(compileOpts, canvas.WithInterruptAfter([]string{parserCpnID}))
+ } else {
+ compileOpts = append(compileOpts, canvas.WithInterruptAfterNonTerminalCpn())
+ }
}
// Run-level setups (keyed by cpnID) override the DSL-baked component
// setups at compile time (higher priority; see canvas.WithOverrideParams).
@@ -478,6 +492,11 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam
// component re-publishes `name` (and storage refs) as it derives
// them mid-run.
globals.SeedIngestionGlobals(runCtx, current)
+ // Expose the task id on the run context so every component can register the
+ // per-chunk cache keys it writes into the task manifest (chunkcache). Without
+ // this the manifest is never populated and PurgeTask on success becomes a
+ // no-op, leaving orphaned cache entries until TTL expiry.
+ globals.SetTaskID(runCtx, p.taskID)
if !resumable {
return p.runPlain(runCtx, current, compiled, tracker, runState)
diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.go
index a9359c7cd9..2bde20cb4c 100644
--- a/internal/ingestion/task/embedder.go
+++ b/internal/ingestion/task/embedder.go
@@ -63,31 +63,38 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.E
// newEmbedderResolver builds the production embedder resolver used by the
// Tokenizer component. It always resolves the embedder from the dataset's
-// configured embd_id (looked up by kbID). If the dataset has no embd_id
-// configured, it returns nil (no embedding). Kept as a constructor over
-// injectable deps so the resolution logic stays unit-testable without a live
-// model provider / DB.
+// configured embd_id (looked up by kbID) and returns that embd_id alongside the
+// embedder, so the Tokenizer can key its per-chunk cache on the dataset-bound
+// model. If the dataset has no embd_id configured, it returns a nil embedder and
+// an empty embd_id (no embedding). Kept as a constructor over injectable deps so
+// the resolution logic stays unit-testable without a live model provider / DB.
func newEmbedderResolver(
getKBEmbdID func(ctx context.Context, kbID string) (string, error),
getEmbeddingModel func(ctx context.Context, tenantID, embdID string) (*models.EmbeddingModel, error),
) componentpkg.EmbedderResolver {
- return func(ctx context.Context, tenantID, kbID, _ string) (componentpkg.Embedder, error) {
+ // The resolver derives the embedding model exclusively from the
+ // knowledgebase's configured embd_id — never from any DSL-supplied
+ // identifier. It returns embdID alongside the embedder so the tokenizer can
+ // key its per-chunk cache on the dataset-bound model: when a KB's embedding
+ // model changes, embdID changes, the cache key changes, and no stale vector
+ // is ever served.
+ return func(ctx context.Context, tenantID, kbID string) (componentpkg.Embedder, string, error) {
embdID, err := getKBEmbdID(ctx, kbID)
if err != nil {
- return nil, fmt.Errorf("embedder: resolve kb embd_id for kb_id=%s: %w", kbID, err)
+ return nil, "", fmt.Errorf("embedder: resolve kb embd_id for kb_id=%s: %w", kbID, err)
}
embdID = strings.TrimSpace(embdID)
if embdID == "" {
- return nil, nil
+ return nil, "", nil
}
model, err := getEmbeddingModel(ctx, tenantID, embdID)
if err != nil {
- return nil, err
+ return nil, "", err
}
if model == nil {
- return nil, fmt.Errorf("embedder: resolved embedding model is nil for embd_id=%s", embdID)
+ return nil, "", fmt.Errorf("embedder: resolved embedding model is nil for embd_id=%s", embdID)
}
- return &embedder{model: model}, nil
+ return &embedder{model: model}, embdID, nil
}
}
diff --git a/internal/ingestion/task/embedder_test.go b/internal/ingestion/task/embedder_test.go
index e3bbe5d730..ce288d27c4 100644
--- a/internal/ingestion/task/embedder_test.go
+++ b/internal/ingestion/task/embedder_test.go
@@ -40,7 +40,7 @@ func TestEmbedderResolver_UsesKBEmbdID(t *testing.T) {
},
)
ctx := t.Context()
- emb, err := resolver(ctx, "tenant-1", "kb-1", "should-be-ignored")
+ emb, embdID, err := resolver(ctx, "tenant-1", "kb-1")
if err != nil {
t.Fatalf("resolver: %v", err)
}
@@ -50,6 +50,9 @@ func TestEmbedderResolver_UsesKBEmbdID(t *testing.T) {
if gotTenantID != "tenant-1" || gotEmbdID != "kb-embd-1" {
t.Fatalf("resolver args = (%q, %q), want (tenant-1, kb-embd-1)", gotTenantID, gotEmbdID)
}
+ if embdID != "kb-embd-1" {
+ t.Fatalf("resolver embdID = %q, want kb-embd-1", embdID)
+ }
}
func TestEmbedderResolver_EmptyKBEmbdIDReturnsNil(t *testing.T) {
@@ -63,13 +66,16 @@ func TestEmbedderResolver_EmptyKBEmbdIDReturnsNil(t *testing.T) {
},
)
ctx := t.Context()
- emb, err := resolver(ctx, "tenant-1", "kb-1", "ignored")
+ emb, embdID, err := resolver(ctx, "tenant-1", "kb-1")
if err != nil {
t.Fatalf("resolver: %v", err)
}
if emb != nil {
t.Fatal("expected nil embedder when kb has no embd_id")
}
+ if embdID != "" {
+ t.Fatalf("expected empty embdID when kb has no embd_id, got %q", embdID)
+ }
}
// stubDriver records the texts passed to Embed for verification.
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 5dc6c75f66..46e3689622 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -31,6 +31,7 @@ import (
"ragflow/internal/engine"
enginetypes "ragflow/internal/engine/types"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/chunkcache"
"ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/component/globals"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
@@ -323,6 +324,18 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
s.taskCtx.Doc.ParserConfig,
)
+ // Persist is now fully durable: every failure-capable step above
+ // (index write, compiled-product reconcile, Wiki active-MAP state) has
+ // succeeded. Only now drop the per-chunk cache entries this task produced,
+ // so a failure in any of those steps leaves the cache intact for the retry —
+ // the retry resumes after the Parser checkpoint and would otherwise have to
+ // re-pay every embedding/LLM call, which is exactly the cost this cache
+ // exists to absorb. PurgeTask is best-effort and silent when no manifest
+ // exists (e.g. a Redis-less run, where chunkcache.Client() is nil).
+ if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" {
+ chunkcache.PurgeTask(ctx, chunkcache.Client(), s.taskCtx.IngestionTask.ID)
+ }
+
return &PipelineResult{
DocID: s.taskCtx.Doc.ID,
KbID: s.taskCtx.Doc.KbID,