RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(ingestion): consolidate checkpoint to Parser + per-chunk result cache (#19386)
xugangqiangcec02addf57e
diff --git a/internal/ingestion/chunkcache/chunkcache.go b/internal/ingestion/chunkcache/chunkcache.gonew file mode 100644index 0000000000..5d8f62cc22--- /dev/null+++ b/internal/ingestion/chunkcache/chunkcache.go@@ -0,0 +1,189 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package chunkcache owns the per-chunk result cache shared by the ingestion+// components that call an expensive external model (Extractor → chat LLM,+// Tokenizer → embedding model).+//+// Why it exists: the pipeline checkpoints only after the Parser, so a resumed+// task re-executes every stage after it. That is affordable only if the+// expensive per-chunk work is memoised, which is what this cache does — a+// resumed run re-walks the same chunks, hits the cache, and issues no model+// calls.+//+// Entry lifetime has two reclaim paths. The normal one is PurgeTask, called+// once the task's chunks are durably indexed: from that point the cache can+// never be re-read, so it is dropped immediately instead of occupying the+// store. TTL is the backstop for tasks that never reach persist (failed,+// cancelled, abandoned mid-resume) and for runs with no task scope.+package chunkcache++import (+ "context"+ "fmt"+ "time"++ "github.com/cespare/xxhash/v2"++ "ragflow/internal/engine/redis"+ "ragflow/internal/ingestion/component/globals"+)++// TTL bounds how long a cached per-chunk result stays useful. It matches the+// pipeline checkpoint TTL on purpose: a task that can still be resumed must+// still find its cache, and one that can no longer be resumed has no reader.+const TTL = 7 * 24 * time.Hour++// keyNamespace prefixes every key this package writes, including the+// manifests, so a live store can be inspected (or bulk-scanned) per concern.+const keyNamespace = "kc"++// Store is the slice of the Redis client this package needs.+// *redis.Client satisfies it; tests supply an in-memory double.+type Store interface {+ Get(ctx context.Context, key string) (string, error)+ Set(ctx context.Context, key, value string, exp time.Duration) bool+ SAdd(ctx context.Context, key, member string) bool+ SMembers(ctx context.Context, key string) ([]string, error)+ Delete(ctx context.Context, key string) bool+ Expire(ctx context.Context, key string, exp time.Duration) bool+}++// Client resolves the process-wide Redis client as a Store, or nil when Redis+// is not configured. Always resolve through this function: passing redis.Get()+// directly into a Store parameter would wrap a typed nil pointer in a non-nil+// interface, defeating the nil guards below.+func Client() Store {+ if c := redis.Get(); c != nil {+ return c+ }+ return nil+}++// Key builds the cache key for one (kind, model, chunk) triple, mixing any+// extra config that changes the result (a prompt, a JSON schema, a top-N).+//+// modelID is the identity of the model that produced the cached value:+// for the "emb" kind that is the dataset's embedding model id (kb.embd_id);+// for the "extractor:*"/"meta"/"tagger" kinds it is the chat model id (llm_id,+// or — when that is empty — the resolved tenant-default chat model). Callers+// must resolve the model id to the value that actually generated the result+// before calling Key, never a raw/possibly-empty override string: keying on an+// empty modelID would collapse every model onto one bucket and serve a result+// produced by a model the tenant no longer uses. The tokenizer keys on embd_id;+// the extractor resolves its chat target (see extractorCacheModelID in the+// component package) for the same reason.+//+// The chunk id — not the chunk text — is the identity input: it is the stable+// per-chunk handle the chunker assigns (component.ChunkID), and it already+// derives from the text, so keying on it keeps entries short and keeps the+// pipeline and the persist stage agreeing on what "the same chunk" means.+//+// Either an empty chunk id or an empty modelID yields an empty key: in both+// cases the caller must skip the cache rather than let every unidentified+// chunk (or every model-less entry) share one bucket.+func Key(kind, modelID, chunkID string, config ...string) string {+ if chunkID == "" || modelID == "" {+ return ""+ }+ h := xxhash.New()+ // Length-prefix every field so no two different field splits can hash to+ // the same digest (…"ab","c" vs …"a","bc").+ writeField := func(s string) {+ _, _ = fmt.Fprintf(h, "%d:%s", len(s), s)+ }+ writeField(kind)+ writeField(modelID)+ writeField(chunkID)+ for _, c := range config {+ writeField(c)+ }+ return fmt.Sprintf("%s:%s:%x", keyNamespace, kind, h.Sum64())+}++// manifestKey is the set listing every cache key produced for one task.+func manifestKey(taskID string) string {+ return fmt.Sprintf("%s:manifest:%s", keyNamespace, taskID)+}++// Get returns the cached value for key. The boolean distinguishes a miss from+// a cached empty string; Set refuses empty values, so a hit is always+// meaningful. Degrades to a miss when the store is unavailable.+func Get(ctx context.Context, s Store, key string) (string, bool) {+ if s == nil || key == "" {+ return "", false+ }+ v, err := s.Get(ctx, key)+ if err != nil || v == "" {+ return "", false+ }+ return v, true+}++// Set caches value under key for TTL and records key on the current task's+// manifest so PurgeTask can reclaim it at persist time. Best-effort+// throughout: a store outage costs a recomputation, never a failed run.+//+// The manifest entry is only recorded once the value write succeeded, so the+// manifest never lists a key that does not exist. When the run carries no task+// id (canvas debug preview, headless test) the value is still cached but+// nothing is recorded: no persist stage will ever reclaim it, so TTL is the+// only reclaim path and a manifest would simply leak.+func Set(ctx context.Context, s Store, key, value string) {+ if s == nil || key == "" || value == "" {+ return+ }+ if !s.Set(ctx, key, value, TTL) {+ return+ }+ taskID := globals.TaskID(ctx)+ if taskID == "" {+ return+ }+ mk := manifestKey(taskID)+ if s.SAdd(ctx, mk, key) {+ // Bound the manifest's own lifetime: it must not outlive the entries+ // it lists, or an abandoned task would leak one set per task forever.+ // If the TTL cannot be applied, drop the manifest immediately rather+ // than leave a TTL-less set that leaks one empty key per task.+ if !s.Expire(ctx, mk, TTL) {+ s.Delete(ctx, mk)+ }+ }+}++// PurgeTask drops every cache entry recorded for taskID, then the manifest+// itself. Call it once the task's chunks are durably persisted: from that point+// nothing can read the entries back, so holding them for the remaining TTL is+// pure waste.+//+// The manifest is deleted last: losing it first would orphan the entries it+// still lists, leaving them to expire. Best-effort — a failed reclaim only+// means the entries wait out their TTL.+func PurgeTask(ctx context.Context, s Store, taskID string) {+ if s == nil || taskID == "" {+ return+ }+ mk := manifestKey(taskID)+ keys, err := s.SMembers(ctx, mk)+ if err != nil {+ return+ }+ for _, k := range keys {+ s.Delete(ctx, k)+ }+ s.Delete(ctx, mk)+}diff --git a/internal/ingestion/chunkcache/chunkcache_test.go b/internal/ingestion/chunkcache/chunkcache_test.gonew file mode 100644index 0000000000..dedc41165f--- /dev/null+++ b/internal/ingestion/chunkcache/chunkcache_test.go@@ -0,0 +1,342 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunkcache++import (+ "context"+ "errors"+ "sort"+ "testing"+ "time"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/globals"+)++// fakeStore is an in-memory Store double: enough Redis surface for the+// manifest bookkeeping, with call counters so tests can assert the exact+// command sequence (a real Redis is an integration-tier dependency).+type fakeStore struct {+ kv map[string]string+ sets map[string]map[string]bool+ ttl map[string]time.Duration+ deleted []string+ failSet bool+ expireFails bool+}++func newFakeStore() *fakeStore {+ return &fakeStore{+ kv: map[string]string{},+ sets: map[string]map[string]bool{},+ ttl: map[string]time.Duration{},+ }+}++func (f *fakeStore) Get(_ context.Context, key string) (string, error) {+ v, ok := f.kv[key]+ if !ok {+ return "", errors.New("redis: nil")+ }+ return v, nil+}++func (f *fakeStore) Set(_ context.Context, key, value string, exp time.Duration) bool {+ if f.failSet {+ return false+ }+ f.kv[key] = value+ f.ttl[key] = exp+ return true+}++func (f *fakeStore) SAdd(_ context.Context, key, member string) bool {+ if f.sets[key] == nil {+ f.sets[key] = map[string]bool{}+ }+ f.sets[key][member] = true+ return true+}++func (f *fakeStore) SMembers(_ context.Context, key string) ([]string, error) {+ out := make([]string, 0, len(f.sets[key]))+ for m := range f.sets[key] {+ out = append(out, m)+ }+ sort.Strings(out)+ return out, nil+}++func (f *fakeStore) Delete(_ context.Context, key string) bool {+ f.deleted = append(f.deleted, key)+ delete(f.kv, key)+ delete(f.sets, key)+ return true+}++func (f *fakeStore) Expire(_ context.Context, key string, exp time.Duration) bool {+ if f.expireFails {+ return false+ }+ f.ttl[key] = exp+ return true+}++// taskCtx returns a ctx whose CanvasState carries taskID, mirroring what the+// pipeline attaches for a real run.+func taskCtx(taskID string) context.Context {+ st := runtime.NewCanvasState("", "")+ ctx := runtime.WithState(context.Background(), st)+ globals.SetTaskID(ctx, taskID)+ return ctx+}++// TestTTL_IsSevenDays pins the shared cache lifetime. Resume must be able to+// reuse per-chunk results for as long as a checkpoint can live, so this value+// and the pipeline checkpoint TTL are deliberately equal.+func TestTTL_IsSevenDays(t *testing.T) {+ if TTL != 7*24*time.Hour {+ t.Errorf("TTL = %v, want 168h", TTL)+ }+}++// TestSet_WritesValueAndRegistersManifest asserts a cache write both stores the+// value under the shared TTL and records the key on the task manifest, so the+// persist stage can reclaim exactly the keys this task produced.+func TestSet_WritesValueAndRegistersManifest(t *testing.T) {+ f := newFakeStore()+ ctx := taskCtx("task-1")+ Set(ctx, f, "kc:extractor:keywords:abc", "result")++ if got := f.kv["kc:extractor:keywords:abc"]; got != "result" {+ t.Errorf("cached value = %q, want %q", got, "result")+ }+ if got := f.ttl["kc:extractor:keywords:abc"]; got != TTL {+ t.Errorf("cached TTL = %v, want %v", got, TTL)+ }+ members, _ := f.SMembers(ctx, "kc:manifest:task-1")+ if len(members) != 1 || members[0] != "kc:extractor:keywords:abc" {+ t.Errorf("manifest members = %v, want [kc:extractor:keywords:abc]", members)+ }+ if got := f.ttl["kc:manifest:task-1"]; got != TTL {+ t.Errorf("manifest TTL = %v, want %v (must not outlive its entries)", got, TTL)+ }+}++// TestSet_ManifestExpireFailureDeletesManifest asserts that when the manifest's+// TTL cannot be applied (Expire returns false) Set removes the manifest key+// rather than leaving a TTL-less set that leaks one empty key per abandoned task+// forever. The value itself was already written and keeps its own TTL.+func TestSet_ManifestExpireFailureDeletesManifest(t *testing.T) {+ f := newFakeStore()+ f.expireFails = true+ ctx := taskCtx("task-1")+ Set(ctx, f, "kc:extractor:keywords:abc", "result")++ // Value is still cached (its own Set succeeded).+ if got := f.kv["kc:extractor:keywords:abc"]; got != "result" {+ t.Errorf("cached value = %q, want it kept", got)+ }+ // Manifest must NOT linger without a TTL.+ mk := manifestKey("task-1")+ if _, ok := f.sets[mk]; ok {+ t.Errorf("manifest %q leaked after Expire failure; want it deleted", mk)+ }+ if len(f.deleted) == 0 || f.deleted[len(f.deleted)-1] != mk {+ t.Errorf("expected manifest %q to be deleted on Expire failure, deleted=%v", mk, f.deleted)+ }+}++// TestSet_SkipsManifestWithoutTaskScope asserts a run with no task id (canvas+// debug preview, headless test) still caches but records nothing: there is no+// task whose completion could reclaim the keys, so TTL is the only reclaim+// path and a manifest would leak.+func TestSet_SkipsManifestWithoutTaskScope(t *testing.T) {+ f := newFakeStore()+ Set(context.Background(), f, "kc:extractor:keywords:abc", "result")++ if got := f.kv["kc:extractor:keywords:abc"]; got != "result" {+ t.Errorf("cached value = %q, want it cached even without task scope", got)+ }+ if len(f.sets) != 0 {+ t.Errorf("manifest sets = %v, want none without a task id", f.sets)+ }+}++// TestSet_NoStoreIsNoop asserts a Redis-less deployment degrades silently+// instead of panicking: callers pass the resolved client straight through.+func TestSet_NoStoreIsNoop(t *testing.T) {+ Set(taskCtx("task-1"), nil, "kc:extractor:keywords:abc", "result")+ if got, ok := Get(taskCtx("task-1"), nil, "kc:extractor:keywords:abc"); ok || got != "" {+ t.Errorf("Get(nil store) = (%q, %v), want (\"\", false)", got, ok)+ }+}++// TestSet_SkipsEmptyKeyOrValue asserts we never cache under an empty key (the+// caller could not derive a chunk id) and never cache an empty payload (which+// Get cannot distinguish from a miss).+func TestSet_SkipsEmptyKeyOrValue(t *testing.T) {+ f := newFakeStore()+ ctx := taskCtx("task-1")+ Set(ctx, f, "", "result")+ Set(ctx, f, "kc:extractor:keywords:abc", "")+ if len(f.kv) != 0 {+ t.Errorf("kv = %v, want no writes", f.kv)+ }+ if len(f.sets) != 0 {+ t.Errorf("manifest sets = %v, want none", f.sets)+ }+}++// TestSet_SkipsManifestWhenValueWriteFails asserts the manifest never lists a+// key whose value write failed — otherwise PurgeTask would issue deletes for+// keys that do not exist.+func TestSet_SkipsManifestWhenValueWriteFails(t *testing.T) {+ f := newFakeStore()+ f.failSet = true+ Set(taskCtx("task-1"), f, "kc:extractor:keywords:abc", "result")+ if len(f.sets) != 0 {+ t.Errorf("manifest sets = %v, want none when the value write failed", f.sets)+ }+}++// TestGet_HitAndMiss asserts the miss signal is a boolean rather than an empty+// string, so a caller can tell "not cached" from "cached empty".+func TestGet_HitAndMiss(t *testing.T) {+ f := newFakeStore()+ ctx := taskCtx("task-1")+ Set(ctx, f, "k", "v")+ if got, ok := Get(ctx, f, "k"); !ok || got != "v" {+ t.Errorf("Get(hit) = (%q, %v), want (\"v\", true)", got, ok)+ }+ if got, ok := Get(ctx, f, "absent"); ok || got != "" {+ t.Errorf("Get(miss) = (%q, %v), want (\"\", false)", got, ok)+ }+ if got, ok := Get(ctx, f, ""); ok || got != "" {+ t.Errorf("Get(empty key) = (%q, %v), want (\"\", false)", got, ok)+ }+}++// TestPurgeTask_DeletesEveryEntryThenManifest asserts persist-time reclaim+// removes all recorded entries and finally the manifest itself, leaving no+// key behind that would otherwise wait out the 7-day TTL.+func TestPurgeTask_DeletesEveryEntryThenManifest(t *testing.T) {+ f := newFakeStore()+ ctx := taskCtx("task-1")+ Set(ctx, f, "kc:extractor:keywords:a", "1")+ Set(ctx, f, "kc:meta:b", "2")+ Set(ctx, f, "kc:emb:c", "3")++ PurgeTask(context.Background(), f, "task-1")++ want := []string{"kc:emb:c", "kc:extractor:keywords:a", "kc:meta:b", "kc:manifest:task-1"}+ if len(f.deleted) != len(want) {+ t.Fatalf("deleted = %v, want %v", f.deleted, want)+ }+ // Entry deletes may arrive in any order, but the manifest must be last:+ // dropping it early would orphan the remaining entries if we crash midway.+ if f.deleted[len(f.deleted)-1] != "kc:manifest:task-1" {+ t.Errorf("last delete = %q, want the manifest key", f.deleted[len(f.deleted)-1])+ }+ entries := append([]string(nil), f.deleted[:len(f.deleted)-1]...)+ sort.Strings(entries)+ for i, w := range want[:len(want)-1] {+ if entries[i] != w {+ t.Errorf("entry deletes = %v, want %v", entries, want[:len(want)-1])+ break+ }+ }+ if len(f.kv) != 0 {+ t.Errorf("kv after purge = %v, want empty", f.kv)+ }+}++// TestPurgeTask_NoopWithoutTaskOrStore asserts the reclaim path is safe to+// call unconditionally from the persist stage.+func TestPurgeTask_NoopWithoutTaskOrStore(t *testing.T) {+ f := newFakeStore()+ PurgeTask(context.Background(), f, "")+ if len(f.deleted) != 0 {+ t.Errorf("deleted = %v, want none for an empty task id", f.deleted)+ }+ PurgeTask(context.Background(), nil, "task-1")+}++// TestKey_DiscriminatesEveryComponent asserts the key builder namespaces by+// kind and mixes every identity input, so a chat-model swap, a prompt edit or+// a different chunk can never read another entry's value.+func TestKey_DiscriminatesEveryComponent(t *testing.T) {+ base := Key("extractor:keywords", "gpt-4@openai", "chunk-1", "prompt-v1")+ cases := []struct {+ name string+ key string+ }{+ {"different kind", Key("extractor:questions", "gpt-4@openai", "chunk-1", "prompt-v1")},+ {"different model", Key("extractor:keywords", "gpt-5@openai", "chunk-1", "prompt-v1")},+ {"different chunk", Key("extractor:keywords", "gpt-4@openai", "chunk-2", "prompt-v1")},+ {"different config", Key("extractor:keywords", "gpt-4@openai", "chunk-1", "prompt-v2")},+ }+ for _, c := range cases {+ if c.key == base {+ t.Errorf("%s: key collides with base (%s)", c.name, base)+ }+ }+ if same := Key("extractor:keywords", "gpt-4@openai", "chunk-1", "prompt-v1"); same != base {+ t.Errorf("Key is not deterministic: %q vs %q", same, base)+ }+}++// TestKey_IsPrefixedByKind asserts keys stay greppable/scannable per kind in a+// live Redis, and that they share the kc: namespace the manifest uses.+func TestKey_IsPrefixedByKind(t *testing.T) {+ k := Key("emb", "bge-m3@builtin", "chunk-1")+ if want := "kc:emb:"; len(k) <= len(want) || k[:len(want)] != want {+ t.Errorf("Key = %q, want prefix %q", k, want)+ }+}++// TestKey_EmptyChunkIDYieldsNoKey asserts a chunk with no stable id produces+// no key at all, so callers skip the cache instead of sharing one bucket for+// every unidentified chunk.+func TestKey_EmptyChunkIDYieldsNoKey(t *testing.T) {+ if k := Key("emb", "bge-m3@builtin", ""); k != "" {+ t.Errorf("Key(empty chunk id) = %q, want \"\"", k)+ }+}++// TestKey_EmptyModelIDYieldsNoKey asserts the symmetric contract: an empty+// model id yields no key, so a caller that forgot to resolve the model (e.g.+// keyed on a raw, empty llm_id override) cannot collapse every model onto one+// bucket. Mirrors TestKey_EmptyChunkIDYieldsNoKey.+func TestKey_EmptyModelIDYieldsNoKey(t *testing.T) {+ if k := Key("emb", "", "chunk-1"); k != "" {+ t.Errorf("Key(empty model id) = %q, want \"\"", k)+ }+ if k := Key("extractor:questions", "", "chunk-1", "prompt"); k != "" {+ t.Errorf("Key(empty model id) = %q, want \"\"", k)+ }+}++// TestKey_FieldsAreUnambiguous asserts the identity inputs are separated when+// hashed: concatenating them differently must not collide.+func TestKey_FieldsAreUnambiguous(t *testing.T) {+ a := Key("extractor:keywords", "model", "chunk", "ab", "c")+ b := Key("extractor:keywords", "model", "chunk", "a", "bc")+ if a == b {+ t.Errorf("ambiguous field packing: %q == %q", a, b)+ }+}diff --git a/internal/ingestion/chunkcache/client_test.go b/internal/ingestion/chunkcache/client_test.gonew file mode 100644index 0000000000..be68e751cc--- /dev/null+++ b/internal/ingestion/chunkcache/client_test.go@@ -0,0 +1,47 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunkcache++import (+ "testing"++ "ragflow/internal/engine/redis"+)++// Compile-time proof that the real client still satisfies Store. Without it a+// signature change in the redis package would only surface at the call sites.+var _ Store = (*redis.Client)(nil)++// TestClient_NilInterfaceWhenRedisAbsent is the reason Client() exists. Handing+// redis.Get() straight to a Store parameter would wrap a typed nil pointer in a+// non-nil interface, defeating every `s == nil` guard and panicking on the+// first call in a Redis-less deployment. Client() must return an interface that+// compares equal to nil.+func TestClient_NilInterfaceWhenRedisAbsent(t *testing.T) {+ if redis.Get() != nil {+ t.Skip("a global Redis client is configured in this process")+ }+ if got := Client(); got != nil {+ t.Fatalf("Client() = %#v, want a nil interface", got)+ }+ // The guards must hold for the value Client() actually returns.+ Set(taskCtx("task-1"), Client(), "k", "v")+ if got, ok := Get(taskCtx("task-1"), Client(), "k"); ok || got != "" {+ t.Errorf("Get = (%q, %v), want (\"\", false)", got, ok)+ }+ PurgeTask(taskCtx("task-1"), Client(), "task-1")+}diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex f724cdae85..f8a83d063e 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -43,7 +43,6 @@ import ("sync""time"- "github.com/cespare/xxhash/v2"eschema "github.com/cloudwego/eino/schema""go.uber.org/zap""gorm.io/gorm"@@ -51,9 +50,9 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/dao"- "ragflow/internal/engine/redis""ragflow/internal/entity""ragflow/internal/entity/models"+ "ragflow/internal/ingestion/chunkcache""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer""ragflow/internal/utility"@@ -501,14 +500,27 @@ func toExtractorEinoMessages(msgs []eschema.Message) []*eschema.Message {// input map. Computed once at the top of Invoke so the rest of// the function reads as straight-line code.type extractorInputs struct {- llmID string- lang string- chunks []map[string]any+ llmID string+ // modelID is the resolved cache-key model identity (the llm_id, or — when+ // that is empty — the tenant-default chat model resolved once per run in+ // Invoke). Carrying it on the struct lets the per-chunk cache-key builders+ // reuse the single run-level resolution instead of re-resolving the default+ // model for every chunk (review finding #3). Empty means "not yet resolved"+ // and triggers a per-call fallback so direct unit-test construction of+ // extractorInputs keeps working.+ modelID string+ lang string+ chunks []map[string]any// temperature overrides the LLM temperature for this call. A// nil value leaves the request's Temperature unset so the model// (or the chat-model default) decides. The keyword/question helpers set it to// extractorTemperature (0.2) to mirror generator.py.temperature *float64+ // cache memoises per-chunk extraction results so a resumed run (which+ // re-executes every stage after the Parser checkpoint) pays no LLM cost+ // for chunks it already extracted. nil disables caching: a Redis-less+ // deployment, or a test that wants every call to reach the model.+ cache chunkcache.Store}// resolveInputs overlays per-call inputs on top of the@@ -518,6 +530,7 @@ type extractorInputs struct {func (c *ExtractorComponent) resolveInputs(inputs map[string]any) extractorInputs {out := extractorInputs{llmID: c.Param.LLMID,+ cache: chunkcache.Client(),}if inputs == nil {return out@@ -594,6 +607,10 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapreturn nil, fmt.Errorf("extractor: %w", err)}in := c.resolveInputs(inputs)+ // Resolve the cache-key model identity once for the whole run and reuse it+ // across every chunk, rather than re-resolving the (most common) default+ // model per chunk when keying the per-chunk cache. See review finding #3.+ in.modelID = extractorCacheModelID(ctx, db, in.llmID)common.Debug("extractor stage",zap.String("component", "Extractor"),zap.Int("input_chunks", len(in.chunks)),@@ -637,26 +654,51 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map}, nil}-// extractorLLMCacheKey builds a Redis key for textual extractions.-func extractorLLMCacheKey(taskType, modelID, systemPrompt, text string) string {- h := xxhash.New()- h.WriteString(taskType)- h.WriteString("\x00")- h.WriteString(modelID)- h.WriteString("\x00")- h.WriteString(systemPrompt)- h.WriteString("\x00")- h.WriteString(text)- return fmt.Sprintf("kc:extractor:%s:%x", taskType, h.Sum64())+// extractorLLMCacheKey builds the cache key for one textual extraction. The+// chunk id — not the chunk text — carries the chunk's identity: it already+// derives from the text (component.ChunkID) and is the same handle the persist+// stage uses, so the two stages agree on what "the same chunk" is. Returns ""+// when the chunk has no id, which makes the cache a no-op for that chunk.+func extractorLLMCacheKey(taskType, modelID, systemPrompt, chunkID string) string {+ return chunkcache.Key("extractor:"+taskType, modelID, chunkID, systemPrompt)}-// callTextCached wraps callText with a 24-hour Redis cache.-func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in extractorInputs, taskType, systemPrompt, chunkText string) (string, error) {- key := extractorLLMCacheKey(taskType, in.llmID, systemPrompt, chunkText)- if client := redis.Get(); client != nil {- if data, err := client.Get(ctx, key); err == nil && data != "" {- return data, nil- }+// extractorCacheModelID returns the model identity the per-chunk extraction+// cache is keyed on. The cache must capture the model that actually produced+// the text, not the raw llm_id override: when llm_id is empty the pipeline+// falls back to the tenant's default chat model (resolveExtractorChatTarget,+// extractor.go:1212), so keying on the empty override would collapse every+// tenant-default run onto one bucket and serve extractions from a model the+// tenant no longer uses. This mirrors the tokenizer's decision to key on the+// dataset-bound embd_id rather than the raw DSL string.+//+// Non-empty llm_id is already a stable tenant-model id, so it is used as-is+// (no behaviour change for the configured path). Only the empty case is+// resolved to its real default-model identity. A resolution failure returns ""+// — chunkcache.Key turns an empty modelID into an empty key, which disables+// caching for that chunk rather than risking a cross-model hit.+func extractorCacheModelID(ctx context.Context, db *gorm.DB, llmID string) string {+ if llmID != "" {+ return llmID+ }+ driver, modelName, _, _, err := resolveExtractorChatTarget(ctx, db, "")+ if err != nil || (driver == "" && modelName == "") {+ return ""+ }+ return driver + "/" + modelName+}++// callTextCached wraps callText with the per-chunk result cache.+func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in extractorInputs, taskType, systemPrompt, chunkText, chunkID string) (string, error) {+ // Prefer the run-level resolved identity; fall back only for callers that+ // build extractorInputs directly in unit tests without pre-resolving it.+ modelID := in.modelID+ if modelID == "" {+ modelID = extractorCacheModelID(ctx, db, in.llmID)+ }+ key := extractorLLMCacheKey(taskType, modelID, systemPrompt, chunkID)+ if cached, hit := chunkcache.Get(ctx, in.cache, key); hit {+ return cached, nil}res, err := c.callText(ctx, db, in, systemPrompt, chunkText)if err != nil {@@ -664,9 +706,7 @@ func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in}res = cleanExtractionResult(res)if res != "" && !strings.Contains(res, "**ERROR**") {- if client := redis.Get(); client != nil {- client.Set(ctx, key, res, 24*time.Hour)- }+ chunkcache.Set(ctx, in.cache, key, res)}return res, nil}@@ -690,9 +730,11 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, ikwTemp := extractorTemperaturekwIn := extractorInputs{llmID: in.llmID,+ modelID: in.modelID,temperature: &kwTemp,+ cache: in.cache,}- resultStr, err := c.callTextCached(ctx, db, kwIn, "keywords", systemPrompt, chunkText)+ resultStr, err := c.callTextCached(ctx, db, kwIn, "keywords", systemPrompt, chunkText, chunkCacheID(ck))if err != nil {return err}@@ -730,9 +772,11 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,qTemp := extractorTemperatureqIn := extractorInputs{llmID: in.llmID,+ modelID: in.modelID,temperature: &qTemp,+ cache: in.cache,}- resultStr, err := c.callTextCached(ctx, db, qIn, "questions", systemPrompt, chunkText)+ resultStr, err := c.callTextCached(ctx, db, qIn, "questions", systemPrompt, chunkText, chunkCacheID(ck))if err != nil {return err}@@ -776,9 +820,11 @@ func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, insumTemp := extractorTemperaturesumIn := extractorInputs{llmID: in.llmID,+ modelID: in.modelID,temperature: &sumTemp,+ cache: in.cache,}- resultStr, err := c.callTextCached(ctx, db, sumIn, "summary", systemPrompt, chunkText)+ resultStr, err := c.callTextCached(ctx, db, sumIn, "summary", systemPrompt, chunkText, chunkCacheID(ck))if err != nil {return err}@@ -798,10 +844,7 @@ func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.Dfuts := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))for i, ck := range in.chunks {i, ck := i, ck- text, _ := ck["content_with_weight"].(string)- if strings.TrimSpace(text) == "" {- text, _ = ck["text"].(string)- }+ text := extractorChunkText(ck)fn := func() error {if err := c.runAutoKeywords(ctx, db, in, ck, text); err != nil {return fmt.Errorf("chunk %d keywords: %w", i, err)@@ -826,10 +869,7 @@ func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *gofuts := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))for i, ck := range in.chunks {i, ck := i, ck- text, _ := ck["content_with_weight"].(string)- if strings.TrimSpace(text) == "" {- text, _ = ck["text"].(string)- }+ text := extractorChunkText(ck)fn := c.remainingExtractionJob(ctx, db, in, i, ck, text)f, err := extractorPool.Submit(ctx, fn)if err != nil {@@ -911,20 +951,28 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,}schemaStr := string(schemaJSON)- // LLM cache (mirrors Python get_llm_cache/set_llm_cache in- // task_executor.py:543/550 gen_metadata_task): identical (model + chunk- // text + schema) extractions are served from Redis within a 24h window so- // repeated runs / identical chunks don't re-pay the LLM call.- // Best-effort: a missing Redis client or any cache error falls through to- // a live call instead of failing the extraction.+ // Per-chunk result cache: identical (model + chunk + schema) extractions+ // are served from the cache so a resumed run — which re-executes every+ // stage after the Parser checkpoint — doesn't re-pay the LLM call.+ // Best-effort: a missing client or any cache error falls through to a live+ // call instead of failing the extraction.+ chunkID := chunkCacheID(ck)+ // Prefer the run-level resolved identity; fall back only for direct unit+ // callers that did not pre-resolve it.+ modelID := in.modelID+ if modelID == "" {+ modelID = extractorCacheModelID(ctx, db, in.llmID)+ }var parsed map[string]any- if cached, hit := getMetadataLLMCache(ctx, in.llmID, schemaStr, chunkText); hit {+ if cached, hit := getMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID); hit {parsed = cached} else {metaTemp := extractorTemperaturemetaIn := extractorInputs{llmID: in.llmID,+ modelID: in.modelID,temperature: &metaTemp,+ cache: in.cache,}systemPrompt := fmt.Sprintf(autoMetadataPrompt, schemaStr)parsed, err = c.callStructured(ctx, db, metaIn, systemPrompt, chunkText)@@ -935,7 +983,7 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,// Non-JSON or empty response — nothing to extract, not an error.return nil}- setMetadataLLMCache(ctx, in.llmID, schemaStr, chunkText, parsed)+ setMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID, parsed)}// Merge into the chunk metadata map, preserving existing keys.var meta map[string]any@@ -957,53 +1005,63 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,return nil}-// metadataLLMCacheTTL mirrors Python get_llm_cache/set_llm_cache 24h TTL.-const metadataLLMCacheTTL = 24 * time.Hour--// metadataLLMCacheKey builds a Redis key from (llm id, chunk text, "metadata",-// schema), mirroring Python get_llm_cache's xxh64(llmnm + txt + history + genconf).-func metadataLLMCacheKey(llmID, schemaJSON, chunkText string) string {- h := xxhash.New()- h.WriteString(llmID)- h.WriteString("\x00")- h.WriteString(chunkText)- h.WriteString("\x00")- h.WriteString("metadata")- h.WriteString("\x00")- h.WriteString(schemaJSON)- return fmt.Sprintf("kc:meta:%x", h.Sum64())+// extractorChunkText resolves the body an extraction is run against.+//+// "text" wins over "content_with_weight": every chunker writes the+// authoritative body to "text" (and derives the chunk id from it), while+// "content_with_weight" may survive as pass-through metadata from an upstream+// parser block. Preferring the latter would send the LLM a different body than+// the one the chunk id — and therefore the cache entry — stands for. This is+// the same priority the Tokenizer applies (normalizeChunkTextFallback) and the+// chunkers apply (itemText). The fallback keeps a chunk that only carries the+// structured field extractable rather than sending an empty body.+func extractorChunkText(ck map[string]any) string {+ if v, _ := ck["text"].(string); strings.TrimSpace(v) != "" {+ return v+ }+ v, _ := ck["content_with_weight"].(string)+ return v+}++// chunkCacheID returns the chunk's stable per-chunk id, assigned by the chunker+// (component.ChunkID over doc id + text). Returns "" when the chunk carries no+// id — e.g. chunks that reached the Extractor without passing a chunker — in+// which case callers must skip the cache instead of sharing one bucket across+// every unidentified chunk.+func chunkCacheID(ck map[string]any) string {+ id, _ := ck["id"].(string)+ return id+}++// metadataLLMCacheKey builds the cache key for one metadata extraction. The+// schema is part of the key so editing the field set invalidates results+// extracted against the previous one.+func metadataLLMCacheKey(llmID, schemaJSON, chunkID string) string {+ return chunkcache.Key("meta", llmID, chunkID, schemaJSON)}// getMetadataLLMCache returns a cached extraction for the given chunk, or-// (nil, false) on miss / Redis unavailable / decode error. Best-effort.-func getMetadataLLMCache(ctx context.Context, llmID, schemaJSON, chunkText string) (map[string]any, bool) {- client := redis.Get()- if client == nil {- return nil, false- }- data, err := client.Get(ctx, metadataLLMCacheKey(llmID, schemaJSON, chunkText))- if err != nil || data == "" {+// (nil, false) on miss / cache unavailable / decode error. Best-effort.+func getMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, schemaJSON, chunkID string) (map[string]any, bool) {+ data, hit := chunkcache.Get(ctx, store, metadataLLMCacheKey(modelID, schemaJSON, chunkID))+ if !hit {return nil, false}var parsed map[string]any- if err = json.Unmarshal([]byte(data), &parsed); err != nil {+ if err := json.Unmarshal([]byte(data), &parsed); err != nil {return nil, false}return parsed, true}-// setMetadataLLMCache stores an extraction result for 24h. Best-effort: a-// missing Redis client or marshal error is silently ignored.-func setMetadataLLMCache(ctx context.Context, llmID, schemaJSON, chunkText string, parsed map[string]any) {- client := redis.Get()- if client == nil {- return- }+// setMetadataLLMCache stores an extraction result. Best-effort: an unavailable+// cache or a marshal error is silently ignored.+func setMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, schemaJSON, chunkID string, parsed map[string]any) {data, err := json.Marshal(parsed)if err != nil {return}- client.Set(ctx, metadataLLMCacheKey(llmID, schemaJSON, chunkText), string(data), metadataLLMCacheTTL)+ chunkcache.Set(ctx, store, metadataLLMCacheKey(modelID, schemaJSON, chunkID), string(data))}// callRaw runs one chat call against the LLM (per chunk in the normal path)diff --git a/internal/ingestion/component/extractor_cache_test.go b/internal/ingestion/component/extractor_cache_test.gonew file mode 100644index 0000000000..9e160168b8--- /dev/null+++ b/internal/ingestion/component/extractor_cache_test.go@@ -0,0 +1,454 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/common"+ "ragflow/internal/ingestion/chunkcache"+ "ragflow/internal/ingestion/component/globals"+ "ragflow/internal/ingestion/component/schema"+)++// fakeCacheStore is an in-memory chunkcache.Store for the extractor cache+// tests. A real Redis would push these into the integration tier; the point+// under test is which key is read/written, not Redis itself.+type fakeCacheStore struct {+ mu sync.Mutex+ kv map[string]string+ ttl map[string]time.Duration+ sets map[string]map[string]bool+}++func newFakeCacheStore() *fakeCacheStore {+ return &fakeCacheStore{+ kv: map[string]string{},+ ttl: map[string]time.Duration{},+ sets: map[string]map[string]bool{},+ }+}++func (f *fakeCacheStore) Get(_ context.Context, key string) (string, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ v, ok := f.kv[key]+ if !ok {+ return "", errors.New("redis: nil")+ }+ return v, nil+}++func (f *fakeCacheStore) Set(_ context.Context, key, value string, exp time.Duration) bool {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.kv[key] = value+ f.ttl[key] = exp+ return true+}++func (f *fakeCacheStore) SAdd(_ context.Context, key, member string) bool {+ f.mu.Lock()+ defer f.mu.Unlock()+ if f.sets[key] == nil {+ f.sets[key] = map[string]bool{}+ }+ f.sets[key][member] = true+ return true+}++func (f *fakeCacheStore) SMembers(_ context.Context, key string) ([]string, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ out := make([]string, 0, len(f.sets[key]))+ for m := range f.sets[key] {+ out = append(out, m)+ }+ return out, nil+}++func (f *fakeCacheStore) Delete(_ context.Context, key string) bool {+ f.mu.Lock()+ defer f.mu.Unlock()+ delete(f.kv, key)+ delete(f.sets, key)+ return true+}++func (f *fakeCacheStore) Expire(_ context.Context, key string, exp time.Duration) bool {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.ttl[key] = exp+ return true+}++func (f *fakeCacheStore) get(key string) (string, bool) {+ f.mu.Lock()+ defer f.mu.Unlock()+ v, ok := f.kv[key]+ return v, ok+}++func (f *fakeCacheStore) members(key string) []string {+ out, _ := f.SMembers(context.Background(), key)+ return out+}++// cacheTaskCtx returns a ctx carrying a CanvasState with taskID, mirroring the+// pipeline so cache writes get recorded on the task manifest.+func cacheTaskCtx(taskID string) context.Context {+ ctx := runtime.WithState(context.Background(), runtime.NewCanvasState("", ""))+ globals.SetTaskID(ctx, taskID)+ return ctx+}++// TestCallTextCached_HitSkipsLLMCall asserts a cached extraction short-circuits+// the model call. This is the property that makes the single Parser-only+// checkpoint affordable: a resumed run re-walks every chunk but pays nothing.+func TestCallTextCached_HitSkipsLLMCall(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "fresh"})+ store := newFakeCacheStore()+ key := chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys-prompt")+ store.Set(context.Background(), key, "cached", chunkcache.TTL)++ c := &ExtractorComponent{}+ in := extractorInputs{llmID: "model-1", cache: store}+ got, err := c.callTextCached(t.Context(), nil, in, "keywords", "sys-prompt", "chunk body", "chunk-1")+ if err != nil {+ t.Fatalf("callTextCached: %v", err)+ }+ if got != "cached" {+ t.Errorf("result = %q, want %q", got, "cached")+ }+ if n := stub.Calls(); n != 0 {+ t.Errorf("LLM calls = %d, want 0 on a cache hit", n)+ }+}++// TestCallTextCached_KeyedByChunkIDNotText is the core of the key change. The+// chunk id already derives from the text, so hashing the text again only makes+// entries longer — and it forced every caller to keep the exact same text+// around. A hit must depend on the chunk id alone.+func TestCallTextCached_KeyedByChunkIDNotText(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "fresh"})+ store := newFakeCacheStore()+ store.Set(context.Background(), chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys"), "cached", chunkcache.TTL)++ c := &ExtractorComponent{}+ in := extractorInputs{llmID: "model-1", cache: store}+ // Same chunk id, deliberately different text: still a hit.+ got, err := c.callTextCached(t.Context(), nil, in, "keywords", "sys", "totally unrelated body", "chunk-1")+ if err != nil {+ t.Fatalf("callTextCached: %v", err)+ }+ if got != "cached" {+ t.Errorf("result = %q, want the cached value (key must ignore chunk text)", got)+ }+ if n := stub.Calls(); n != 0 {+ t.Errorf("LLM calls = %d, want 0", n)+ }+}++// TestCallTextCached_DifferentChunkIDMisses asserts two distinct chunks never+// share an entry.+func TestCallTextCached_DifferentChunkIDMisses(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "fresh"})+ store := newFakeCacheStore()+ store.Set(context.Background(), chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys"), "cached", chunkcache.TTL)++ c := &ExtractorComponent{}+ in := extractorInputs{llmID: "model-1", cache: store}+ got, err := c.callTextCached(t.Context(), nil, in, "keywords", "sys", "body", "chunk-2")+ if err != nil {+ t.Fatalf("callTextCached: %v", err)+ }+ if got != "fresh" {+ t.Errorf("result = %q, want the freshly generated value", got)+ }+ if n := stub.Calls(); n != 1 {+ t.Errorf("LLM calls = %d, want 1 on a miss", n)+ }+}++// TestCallTextCached_WriteUsesSharedTTLAndManifest asserts a fresh extraction+// is cached under the shared 7-day TTL and recorded on the task manifest, so+// the persist stage can reclaim it immediately instead of leaving it to expire.+func TestCallTextCached_WriteUsesSharedTTLAndManifest(t *testing.T) {+ withStubChatInvoker(t, stubResponse{Content: "generated"})+ store := newFakeCacheStore()+ c := &ExtractorComponent{}+ in := extractorInputs{llmID: "model-1", cache: store}++ if _, err := c.callTextCached(cacheTaskCtx("task-9"), nil, in, "keywords", "sys", "body", "chunk-1"); err != nil {+ t.Fatalf("callTextCached: %v", err)+ }++ key := chunkcache.Key("extractor:keywords", "model-1", "chunk-1", "sys")+ if got, ok := store.get(key); !ok || got != "generated" {+ t.Errorf("cached value = %q (ok=%v), want %q", got, ok, "generated")+ }+ if got := store.ttl[key]; got != chunkcache.TTL {+ t.Errorf("cache TTL = %v, want %v", got, chunkcache.TTL)+ }+ if members := store.members("kc:manifest:task-9"); len(members) != 1 || members[0] != key {+ t.Errorf("manifest = %v, want [%s]", members, key)+ }+}++// TestCallTextCached_NoChunkIDBypassesCache asserts a chunk with no stable id+// (no chunker upstream) is neither read from nor written to the cache, rather+// than every such chunk sharing one bucket.+func TestCallTextCached_NoChunkIDBypassesCache(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "generated"})+ store := newFakeCacheStore()+ c := &ExtractorComponent{}+ in := extractorInputs{llmID: "model-1", cache: store}++ got, err := c.callTextCached(cacheTaskCtx("task-9"), nil, in, "keywords", "sys", "body", "")+ if err != nil {+ t.Fatalf("callTextCached: %v", err)+ }+ if got != "generated" {+ t.Errorf("result = %q, want %q", got, "generated")+ }+ if n := stub.Calls(); n != 1 {+ t.Errorf("LLM calls = %d, want 1", n)+ }+ if len(store.kv) != 0 {+ t.Errorf("cache writes = %v, want none without a chunk id", store.kv)+ }+ if len(store.sets) != 0 {+ t.Errorf("manifest writes = %v, want none without a chunk id", store.sets)+ }+}++// TestCallTextCached_NoStoreStillCalls asserts a Redis-less deployment keeps+// working: no cache, every call reaches the model.+func TestCallTextCached_NoStoreStillCalls(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "generated"})+ c := &ExtractorComponent{}+ got, err := c.callTextCached(t.Context(), nil, extractorInputs{llmID: "model-1"}, "keywords", "sys", "body", "chunk-1")+ if err != nil {+ t.Fatalf("callTextCached: %v", err)+ }+ if got != "generated" || stub.Calls() != 1 {+ t.Errorf("result = %q, calls = %d; want (%q, 1)", got, stub.Calls(), "generated")+ }+}++// TestExtractor_ResolvesDefaultModelOncePerRun is the regression test for+// review finding #3: on the default-model path (empty llm_id) the model+// identity used to key the per-chunk cache must be resolved exactly ONCE per+// run and reused across every chunk — not re-resolved per chunk. The per-chunk+// re-resolution was a real performance regression introduced by this PR's+// correctness fix (the cache key now derives from the resolved default model,+// not the empty override), and is most visible when there is no cache at all+// (every call reaches the resolver for nothing).+//+// The invariant: the chat-target resolver override is consulted once for each+// real LLM call (callRaw resolves its target) and exactly once more for the+// whole run (the run-level cache-key model resolution). So+// resolverCalls == stub.Calls()+1. Before the fix the cache-key build+// re-resolved the model per chunk, giving 2*stub.Calls().+func TestExtractor_ResolvesDefaultModelOncePerRun(t *testing.T) {+ var resolverCalls atomic.Int32+ SetExtractorChatTargetResolverOverride(func(llmID string) (string, string, string, string, bool) {+ resolverCalls.Add(1)+ return "openai", "gpt-4o", "ak", "http://x", true+ })+ t.Cleanup(func() { SetExtractorChatTargetResolverOverride(nil) })++ stub := withStubChatInvoker(t, stubResponse{Content: `{"category":"x"}`})++ const nChunks = 5+ chunks := make([]map[string]any, nChunks)+ for i := range chunks {+ chunks[i] = map[string]any{"text": fmt.Sprintf("chunk body %d", i)}+ }++ c := &ExtractorComponent{Param: schema.ExtractorParam{+ LLMID: "", // default-model path — the one #3 is about+ Keywords: schema.KeywordExtractConfig{TopN: 3},+ Metadata: schema.MetadataExtractConfig{+ Enabled: true,+ Metadata: []common.MetadataFieldDef{{Key: "category", Type: "string"}},+ },+ }}++ if _, err := c.Invoke(t.Context(), nil, map[string]any{"chunks": chunks}); err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ // Sanity: the run actually exercised the LLM so the assertion below is+ // not vacuously true.+ if n := stub.Calls(); n == 0 {+ t.Fatal("no LLM calls happened; test did not exercise the extraction path")+ }++ want := stub.Calls() + 1+ if got := resolverCalls.Load(); got != want {+ t.Errorf("resolver calls = %d, want %d (== stub.Calls()+1: one run-level model resolution, reused across all chunks)",+ got, want)+ }+}++// TestMetadataLLMCache_KeyedByChunkID asserts the metadata extraction cache+// followed the same key change, and that a hit skips the model.+func TestMetadataLLMCache_KeyedByChunkID(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: `{"category":"fresh"}`})+ store := newFakeCacheStore()+ c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})+ in := extractorInputs{llmID: "model-1", cache: store}++ // Seed via the setter so the test pins the getter/setter agreement rather+ // than duplicating the key format.+ schemaJSON, err := json.Marshal(common.Turn2JSONSchema(c.Param.Metadata.Metadata))+ if err != nil {+ t.Fatalf("marshal metadata schema: %v", err)+ }+ setMetadataLLMCache(cacheTaskCtx("task-9"), in.cache, "model-1", string(schemaJSON), "chunk-1", map[string]any{"category": "cached"})++ ck := map[string]any{"id": "chunk-1", "text": "a body that no longer participates in the key"}+ if err := c.runEnableMetadata(t.Context(), nil, in, ck, "a body that no longer participates in the key"); err != nil {+ t.Fatalf("runEnableMetadata: %v", err)+ }+ meta, _ := ck["metadata"].(map[string]any)+ if meta["category"] != "cached" {+ t.Errorf("metadata = %v, want category=cached from the chunk-id-keyed cache", meta)+ }+ if n := stub.Calls(); n != 0 {+ t.Errorf("LLM calls = %d, want 0 on a cache hit", n)+ }+}++// TestMetadataLLMCache_DistinctChunksDoNotShare asserts the metadata cache+// discriminates chunks.+func TestMetadataLLMCache_DistinctChunksDoNotShare(t *testing.T) {+ store := newFakeCacheStore()+ in := extractorInputs{llmID: "model-1", cache: store}+ ctx := cacheTaskCtx("task-9")+ setMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-1", map[string]any{"category": "one"})++ if _, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-2"); hit {+ t.Error("chunk-2 must not read chunk-1's metadata entry")+ }+ got, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-1")+ if !hit || got["category"] != "one" {+ t.Errorf("getMetadataLLMCache(chunk-1) = %v (hit=%v), want category=one", got, hit)+ }+}++// TestMetadataLLMCache_SchemaChangeInvalidates asserts a metadata schema edit+// does not silently reuse results extracted against the old field set.+func TestMetadataLLMCache_SchemaChangeInvalidates(t *testing.T) {+ store := newFakeCacheStore()+ in := extractorInputs{llmID: "model-1", cache: store}+ ctx := cacheTaskCtx("task-9")+ setMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "chunk-1", map[string]any{"category": "one"})+ if _, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"region":"string"}`, "chunk-1"); hit {+ t.Error("a different metadata schema must not reuse the previous entry")+ }+}++// TestMetadataLLMCache_NoChunkIDBypassesCache asserts the metadata path also+// skips the cache when the chunk has no stable id.+func TestMetadataLLMCache_NoChunkIDBypassesCache(t *testing.T) {+ store := newFakeCacheStore()+ in := extractorInputs{llmID: "model-1", cache: store}+ ctx := cacheTaskCtx("task-9")+ setMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, "", map[string]any{"category": "one"})+ if len(store.kv) != 0 {+ t.Errorf("cache writes = %v, want none without a chunk id", store.kv)+ }+ if _, hit := getMetadataLLMCache(ctx, in.cache, "model-1", `{"category":"string"}`, ""); hit {+ t.Error("getMetadataLLMCache with no chunk id must miss")+ }+}++// TestTaggerCacheKey_ScopedByChunkID_ModelAndText asserts the tagger cache key+// is built from the model id, the chunk id, AND the chunk text fed to the model+// (getChunkText folds in the body and important_kwd). The key must stay+// deterministic for identical inputs, discriminate chunks/models/tag-sets/+// few-shot examples/topN, treat a missing chunk id as a no-op, and bust when the+// chunk text changes (a different important_kwd would otherwise serve stale tags).+func TestTaggerCacheKey_ScopedByChunkID_ModelAndText(t *testing.T) {+ allTags := map[string]float64{"a": 1, "b": 2}+ ex := []schema.TaggedChunk{{Content: "example one", Tags: []string{"a"}}}++ base := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, ex, 3)+ if base == "" {+ t.Fatal("taggerCacheKey returned empty for a valid chunk id")+ }+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, ex, 3); got != base {+ t.Errorf("taggerCacheKey is not deterministic: %q vs %q", got, base)+ }+ if got := taggerCacheKey("llm-1", "chunk-2", "the body", allTags, ex, 3); got == base {+ t.Error("a different chunk id must not collide")+ }+ if got := taggerCacheKey("llm-2", "chunk-1", "the body", allTags, ex, 3); got == base {+ t.Error("a different model must not collide")+ }+ if got := taggerCacheKey("llm-1", "chunk-1", "a different body", allTags, ex, 3); got == base {+ t.Error("a different chunk text (e.g. changed important_kwd) must not reuse the entry")+ }+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", map[string]float64{"a": 1}, ex, 3); got == base {+ t.Error("a different tag set must not collide")+ }+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, []schema.TaggedChunk{{Content: "example two", Tags: []string{"a"}}}, 3); got == base {+ t.Error("different few-shot examples must not collide")+ }+ if got := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, ex, 5); got == base {+ t.Error("a different topN must not collide")+ }+ if got := taggerCacheKey("llm-1", "", "the body", allTags, ex, 3); got != "" {+ t.Errorf("taggerCacheKey(no chunk id) = %q, want \"\"", got)+ }+}++// TestTaggerLLMCache_RoundTripsUnderSharedTTL asserts the tagger cache uses the+// same store/TTL/manifest contract as the other per-chunk caches.+func TestTaggerLLMCache_RoundTripsUnderSharedTTL(t *testing.T) {+ store := newFakeCacheStore()+ ctx := cacheTaskCtx("task-9")+ allTags := map[string]float64{"a": 1}+ want := map[string]int{"a": 4}++ setTaggerLLMCache(ctx, store, "llm-1", "chunk-1", "the body", allTags, nil, 3, want)++ key := taggerCacheKey("llm-1", "chunk-1", "the body", allTags, nil, 3)+ if got := store.ttl[key]; got != chunkcache.TTL {+ t.Errorf("tagger cache TTL = %v, want %v", got, chunkcache.TTL)+ }+ if members := store.members("kc:manifest:task-9"); len(members) != 1 || members[0] != key {+ t.Errorf("manifest = %v, want [%s]", members, key)+ }+ got := getTaggerLLMCache(ctx, store, "llm-1", "chunk-1", "the body", allTags, nil, 3)+ if got == nil || got["a"] != 4 {+ t.Errorf("getTaggerLLMCache = %v, want %v", got, want)+ }+ if other := getTaggerLLMCache(ctx, store, "llm-1", "chunk-2", "the body", allTags, nil, 3); other != nil {+ t.Errorf("chunk-2 read chunk-1's entry: %v", other)+ }+}diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.goindex bc3192cd8c..d83cae3dbb 100644--- a/internal/ingestion/component/extractor_tag.go+++ b/internal/ingestion/component/extractor_tag.go@@ -26,8 +26,8 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/dao"- "ragflow/internal/engine/redis""ragflow/internal/entity"+ "ragflow/internal/ingestion/chunkcache""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer"@@ -611,7 +611,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in excase <-ctx.Done():return}- llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.llmID, driver, model, apiKey, baseURL, topN, indexed)+ llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.cache, in.llmID, in.modelID, driver, model, apiKey, baseURL, topN, indexed)}(i)}wg.Wait()@@ -974,10 +974,13 @@ func getChunkText(chunk map[string]any) string {var parts []string- // 1. Extract main chunk content (prioritize content_with_weight, then text)- if v, ok := chunk["content_with_weight"].(string); ok && strings.TrimSpace(v) != "" {+ // 1. Main chunk content. "text" is preferred because every chunker writes+ // the authoritative body there and derives the chunk id from it. Legacy+ // chunks that only carry "content_with_weight" (parser-block output, many+ // unit tests) are accepted as a fallback so detection/tagging still works.+ if v, ok := chunk["text"].(string); ok && strings.TrimSpace(v) != "" {parts = append(parts, strings.TrimSpace(v))- } else if v, ok := chunk["text"].(string); ok && strings.TrimSpace(v) != "" {+ } else if v, ok := chunk["content_with_weight"].(string); ok && strings.TrimSpace(v) != "" {parts = append(parts, strings.TrimSpace(v))}@@ -1112,7 +1115,8 @@ func llmTagChunk(chunk map[string]any,allTags map[string]float64,examples []schema.TaggedChunk,- llmID, driver, model, apiKey, baseURL string,+ cache chunkcache.Store,+ llmID, modelID, driver, model, apiKey, baseURL string,topN int,idx *MemoryTagIndex,) {@@ -1120,6 +1124,12 @@ func llmTagChunk(if text == "" {return}+ chunkID := chunkCacheID(chunk)+ // Prefer the run-level resolved identity passed from runAutoTags; fall back+ // only for direct unit callers that did not pre-resolve it.+ if modelID == "" {+ modelID = extractorCacheModelID(ctx, db, llmID)+ }textHash := int64(xxhash.Sum64String(text))var picked []schema.TaggedChunk@@ -1147,7 +1157,7 @@ func llmTagChunk(}}- if cached := getTaggerLLMCache(ctx, llmID, text, allTags, picked, topN); cached != nil {+ if cached := getTaggerLLMCache(ctx, cache, modelID, chunkID, text, allTags, picked, topN); cached != nil {chunk[common.TAG_FLD] = cachedchunk["tag_kwd"] = sortedTagWeightsKeys(cached)return@@ -1198,7 +1208,7 @@ func llmTagChunk(if len(result) > 0 {chunk[common.TAG_FLD] = resultchunk["tag_kwd"] = sortedTagWeightsKeys(result)- setTaggerLLMCache(ctx, llmID, text, allTags, picked, topN, result)+ setTaggerLLMCache(ctx, cache, modelID, chunkID, text, allTags, picked, topN, result)}}@@ -1280,34 +1290,29 @@ func jsonRepairExtract(raw string) map[string]any {return obj}-func taggerCacheKey(llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) string {- hasher := xxhash.New()- hasher.Write([]byte(llmID))- hasher.Write([]byte("\x00"))- hasher.Write([]byte(text))- hasher.Write([]byte("\x00"))- tagNames := sortedTagNames(allTags)- hasher.Write([]byte(strings.Join(tagNames, ",")))- hasher.Write([]byte("\x00"))+// taggerCacheKey builds the cache key for one chunk's LLM tagging. Keyed on the+// chunk id rather than the chunk text, like the other per-chunk caches. The tag+// set, the few-shot examples and topN also participate: none of them is derived+// from the chunk, so a change in any of them yields different tags. chunkText is+// the exact text fed to the model (getChunkText folds in the chunk body and+// important_kwd): it participates in the key so a change to the chunk's keywords+// — which changes the tagging prompt — busts the cache instead of serving stale+// tags for up to the cache TTL.+func taggerCacheKey(modelID, chunkID, chunkText string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) string {+ config := make([]string, 0, 2*len(examples)+3)+ config = append(config, strings.Join(sortedTagNames(allTags), ","))for _, ex := range examples {- hasher.Write([]byte(ex.Content))- hasher.Write([]byte("\x00"))tagsJSON, _ := json.Marshal(ex.TagWeights)- hasher.Write(tagsJSON)- hasher.Write([]byte("\x00"))+ config = append(config, ex.Content, string(tagsJSON))}- hasher.Write([]byte(fmt.Sprintf("%d", topN)))- return fmt.Sprintf("tagger:%x", hasher.Sum64())+ config = append(config, strconv.Itoa(topN))+ config = append(config, chunkText)+ return chunkcache.Key("tagger", modelID, chunkID, config...)}-func getTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) map[string]int {- client := redis.Get()- if client == nil {- return nil- }- key := taggerCacheKey(llmID, text, allTags, examples, topN)- data, err := client.Get(ctx, key)- if err != nil || data == "" {+func getTaggerLLMCache(ctx context.Context, store chunkcache.Store, modelID, chunkID, chunkText string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) map[string]int {+ data, hit := chunkcache.Get(ctx, store, taggerCacheKey(modelID, chunkID, chunkText, allTags, examples, topN))+ if !hit {return nil}var result map[string]int@@ -1317,20 +1322,15 @@ func getTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[strireturn result}-func setTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int, result map[string]int) {+func setTaggerLLMCache(ctx context.Context, store chunkcache.Store, modelID, chunkID, chunkText string, allTags map[string]float64, examples []schema.TaggedChunk, topN int, result map[string]int) {if result == nil {return}- client := redis.Get()- if client == nil {- return- }- key := taggerCacheKey(llmID, text, allTags, examples, topN)data, err := json.Marshal(result)if err != nil {return}- client.Set(ctx, key, string(data), 24*time.Hour)+ chunkcache.Set(ctx, store, taggerCacheKey(modelID, chunkID, chunkText, allTags, examples, topN), string(data))}func sortedTagNames(allTags map[string]float64) []string {diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.goindex 20229b7d23..7bd908c0cd 100644--- a/internal/ingestion/component/extractor_tag_test.go+++ b/internal/ingestion/component/extractor_tag_test.go@@ -279,7 +279,7 @@ func TestLlmtagChunk_MessageFit(t *testing.T) {allTags := map[string]float64{"RAG": 1, "database": 1, "AI": 1}examples := []schema.TaggedChunk{{Content: "example one", TagWeights: map[string]int{"AI": 5}}}- llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, nil)if len(capt.req.Messages) != 2 {t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))@@ -302,7 +302,7 @@ func TestLlmtagChunk_NoContextLength_SkipsFit(t *testing.T) {allTags := map[string]float64{"RAG": 1}examples := []schema.TaggedChunk{{Content: "example", TagWeights: map[string]int{"AI": 5}}}- llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, nil)if len(capt.req.Messages) != 2 {t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))@@ -324,7 +324,7 @@ func TestLlmtagChunk_ColdStartFallback(t *testing.T) {}chunk := map[string]any{"content_with_weight": "some content"}- llmTagChunk(t.Context(), nil, capt, chunk, idx.allTags, nil, "test@test", "test_driver", "test_model", "test_key", "", 3, idx)+ llmTagChunk(t.Context(), nil, capt, chunk, idx.allTags, nil, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, idx)if len(capt.req.Messages) != 2 {t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))@@ -721,7 +721,7 @@ func TestPopulateTagKwd_LLMTagChunk(t *testing.T) {chunk := map[string]any{"content_with_weight": "some content"}allTags := map[string]float64{"RAG": 0.5, "vector database": 0.5}- llmTagChunk(t.Context(), nil, capt, chunk, allTags, nil, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, nil, nil, "test@test", "test@test", "test_driver", "test_model", "test_key", "", 3, nil)tagKwd, ok := chunk["tag_kwd"].([]string)if !ok {@@ -1166,10 +1166,10 @@ func TestTaggerCacheKey_IncludesFewShot(t *testing.T) {{Content: "sample one", TagWeights: map[string]int{"TagA": 5}},}- k1 := taggerCacheKey("llm-1", "test text", allTags, ex1, 3)- k2 := taggerCacheKey("llm-1", "test text", allTags, ex2, 3)- k3 := taggerCacheKey("llm-1", "test text", allTags, ex3, 3)- kEmpty := taggerCacheKey("llm-1", "test text", allTags, nil, 3)+ k1 := taggerCacheKey("llm-1", "test text", "body", allTags, ex1, 3)+ k2 := taggerCacheKey("llm-1", "test text", "body", allTags, ex2, 3)+ k3 := taggerCacheKey("llm-1", "test text", "body", allTags, ex3, 3)+ kEmpty := taggerCacheKey("llm-1", "test text", "body", allTags, nil, 3)if k1 == k2 {t.Fatalf("expected different cache keys for different few-shot examples: %s vs %s", k1, k2)@@ -1182,7 +1182,7 @@ func TestTaggerCacheKey_IncludesFewShot(t *testing.T) {}// Identical few-shot examples produce identical key- k1Dup := taggerCacheKey("llm-1", "test text", allTags, ex1, 3)+ k1Dup := taggerCacheKey("llm-1", "test text", "body", allTags, ex1, 3)if k1 != k1Dup {t.Fatalf("expected identical cache keys for same few-shot examples: %s vs %s", k1, k1Dup)}diff --git a/internal/ingestion/component/globals/globals.go b/internal/ingestion/component/globals/globals.goindex cdf5c1cdbe..9b64fb6e63 100644--- a/internal/ingestion/component/globals/globals.go+++ b/internal/ingestion/component/globals/globals.go@@ -53,10 +53,11 @@ func canvasStateFromContext(ctx context.Context) *runtime.CanvasState {// depends on (e.g. TokenChunker drops `name`, which Tokenizer consumes for// title embedding). Storing the shared fields in CanvasState.Globals restores// the Python behaviour without mutating every component output.-// The embedding-model id is intentionally NOT a global: it is a-// Tokenizer-scoped setup (params["setups"]["embedding_model"]). Keeping it out-// of the shared bag prevents another component (e.g. one expecting a chat-// model) from misreading a generic "model_id" global as its own.+// The embedding-model id is intentionally NOT a global: the Tokenizer resolves+// it from the dataset's own embd_id (kb_id) via its injected resolver, never+// from a run input. Keeping it out of the shared bag prevents another component+// (e.g. one expecting a chat model) from misreading a generic "model_id" global+// as its own.var GlobalMetadataKeys = []string{"name","doc_id",@@ -73,6 +74,38 @@ var GlobalMetadataKeys = []string{DebugChunkCapKey,}+// taskIDKey is the CanvasState.Globals slot carrying the ingestion task id of+// the current run. It is deliberately unexported and absent from+// GlobalMetadataKeys: the pipeline seeds it once via SetTaskID, so no run+// input and no component output can overwrite the run's task scope.+const taskIDKey = "task_id"++// SetTaskID records the ingestion task id of the current run so components can+// scope per-task bookkeeping (e.g. the per-chunk cache manifest). Called once+// by the pipeline at run start. No-op when no CanvasState is attached.+func SetTaskID(ctx context.Context, taskID string) {+ if taskID == "" {+ return+ }+ if st := canvasStateFromContext(ctx); st != nil {+ st.SetGlobal(taskIDKey, taskID)+ }+}++// TaskID returns the ingestion task id of the current run, or "" when the run+// has no task scope (headless component tests, or a canvas invoked outside the+// ingestion pipeline). Callers must treat "" as "skip per-task bookkeeping".+func TaskID(ctx context.Context) string {+ if st := canvasStateFromContext(ctx); st != nil {+ if v, ok := st.GetGlobal(taskIDKey); ok {+ if s, ok := v.(string); ok {+ return s+ }+ }+ }+ return ""+}+// DebugChunkCapKey is the run-input key (seeded into CanvasState.Globals)// carrying the canvas-debug (dry-run) chunk cap: when >= 1, a chunker node in// a debug run emits at most this many leading chunks for preview. 0 means "nodiff --git a/internal/ingestion/component/globals/globals_test.go b/internal/ingestion/component/globals/globals_test.goindex c418a42fc1..b4d83dca49 100644--- a/internal/ingestion/component/globals/globals_test.go+++ b/internal/ingestion/component/globals/globals_test.go@@ -67,6 +67,56 @@ func TestDebugChunkCap_ReadsFromGlobals(t *testing.T) {}}+// TestTaskID_EmptyWithoutState asserts TaskID degrades to "" when no+// CanvasState is attached (headless unit tests) or the key was never seeded.+// Callers use "" as "no task scope" and must skip per-task bookkeeping.+func TestTaskID_EmptyWithoutState(t *testing.T) {+ if got := TaskID(context.Background()); got != "" {+ t.Errorf("TaskID(bare ctx) = %q, want \"\"", got)+ }+ ctx := runtime.WithState(context.Background(), &runtime.CanvasState{Globals: map[string]any{}})+ if got := TaskID(ctx); got != "" {+ t.Errorf("TaskID(empty globals) = %q, want \"\"", got)+ }+}++// TestSetTaskID_RoundTrips asserts the pipeline-side seeder and the+// component-side reader agree on the storage slot.+func TestSetTaskID_RoundTrips(t *testing.T) {+ st := &runtime.CanvasState{Globals: map[string]any{}}+ ctx := runtime.WithState(context.Background(), st)+ SetTaskID(ctx, "task-42")+ if got := TaskID(ctx); got != "task-42" {+ t.Errorf("TaskID = %q, want \"task-42\"", got)+ }+}++// TestSetTaskID_NoStateIsNoop asserts seeding without a CanvasState does not+// panic — headless component tests invoke stages with a bare context.+func TestSetTaskID_NoStateIsNoop(t *testing.T) {+ SetTaskID(context.Background(), "task-42")+}++// TestTaskIDKey_NotAWhitelistedMetadataKey asserts the task id is seeded+// explicitly by the pipeline and is NOT part of GlobalMetadataKeys. Being on+// that whitelist would let any component output (or run input) carrying a+// "task_id" field overwrite the run's real task id, which would silently+// redirect per-task cache bookkeeping to the wrong task.+func TestTaskIDKey_NotAWhitelistedMetadataKey(t *testing.T) {+ for _, k := range GlobalMetadataKeys {+ if k == taskIDKey {+ t.Fatalf("GlobalMetadataKeys must not contain %q", taskIDKey)+ }+ }+ st := &runtime.CanvasState{Globals: map[string]any{}}+ ctx := runtime.WithState(context.Background(), st)+ SetTaskID(ctx, "task-real")+ SeedIngestionGlobals(ctx, map[string]any{taskIDKey: "task-spoofed"})+ if got := TaskID(ctx); got != "task-real" {+ t.Errorf("TaskID after hostile seed = %q, want \"task-real\"", got)+ }+}+// TestSeedIngestionGlobals_PicksUpDebugChunkCap asserts that adding// DebugChunkCapKey to GlobalMetadataKeys makes SeedIngestionGlobals copy the// run-input value into CanvasState.Globals — the link the executor relies ondiff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex d6bbed6eee..b85bc88fbd 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -93,6 +93,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common"+ "ragflow/internal/ingestion/chunkcache""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer"@@ -153,11 +154,14 @@ type Embedder interface {Encode(ctx context.Context, texts []string) ([]EmbeddingResult, error)}-// EmbedderResolver resolves the embedder for one tokenizer invocation.-// embeddingModel is the Tokenizer-scoped embedding-model identifier (from the-// component's setups); an empty value tells the resolver to fall back to the-// dataset's configured model.-type EmbedderResolver func(ctx context.Context, tenantID, kbID, embeddingModel string) (Embedder, error)+// EmbedderResolver resolves the embedder and its dataset-bound embedding-model+// id for one tokenizer invocation. The resolver derives the model exclusively+// from the knowledgebase's configured embd_id (see internal/ingestion/task/+// embedder.go); it never reads any embedding identifier from the DSL. The+// returned embdID is the stable string the tokenizer keys its per-chunk+// embedding cache on, so a KB whose embedding model changes yields a different+// key and never serves a stale vector.+type EmbedderResolver func(ctx context.Context, tenantID, kbID string) (Embedder, string, error)// DefaultEmbedderResolver is the production embedder resolver. It is nil in// this leaf package — which must not import internal/service (see the@@ -174,8 +178,9 @@ var DefaultEmbedderResolver EmbedderResolver// Inputs://// tenant_id (string, optional) — used to resolve the embedding model-// kb_id (string, optional) — dataset whose embd_id is used when the-// setups embedding_model is unset+// kb_id (string, optional) — dataset whose embd_id selects the embedding+// model. The model always comes from the dataset;+// the DSL never configures it.// output_format (string) — one of json/markdown/text/html/chunks// chunks (list[map]) — chunk list when output_format == "chunks"// json (list[map]) — structured parser payload when output_format == "json" or unset@@ -191,9 +196,8 @@ var DefaultEmbedderResolver EmbedderResolver// output_format — always "chunks" (matches python set_output)// _created_time / _elapsed_time — TrackElapsed bookkeepingtype TokenizerComponent struct {- param schema.TokenizerParam- resolver EmbedderResolver- embeddingModel string+ param schema.TokenizerParam+ resolver EmbedderResolver}// NewTokenizerComponent constructs a production TokenizerComponent from DSL@@ -215,7 +219,6 @@ func NewTokenizerComponentWithResolver(params map[string]any, resolver EmbedderRfunc newTokenizerComponent(params map[string]any, resolver EmbedderResolver) (runtime.Component, error) {p := schema.TokenizerParam{}.Defaults()- embeddingModel := ""if params != nil {if v, ok := params["search_method"]; ok {// Replace (not append) so a caller-supplied@@ -256,35 +259,18 @@ func newTokenizerComponent(params map[string]any, resolver EmbedderResolver) (rup.Fields = append(p.Fields, t...)}}- embeddingModel = embeddingModelFromSetups(params)}if err := p.Validate(); err != nil {return nil, fmt.Errorf("tokenizer: param check: %w", err)}- return &TokenizerComponent{param: p, resolver: resolver, embeddingModel: embeddingModel}, nil-}--// embeddingModelFromSetups extracts the embedding-model identifier from the-// component's setups map (params["setups"]["embedding_model"]). The embedding-// model id is a Tokenizer-scoped setup rather than a run-level global so it is-// never mistaken for, e.g., a chat model id shared across components. Empty-// when unset — the resolver then falls back to the dataset's configured model.-func embeddingModelFromSetups(params map[string]any) string {- setups, ok := params["setups"].(map[string]any)- if !ok {- return ""- }- if v, ok := setups["embedding_model"].(string); ok {- return strings.TrimSpace(v)- }- return ""+ return &TokenizerComponent{param: p, resolver: resolver}, nil}// Inputs returns the parameter metadata.func (c *TokenizerComponent) Inputs() map[string]string {return map[string]string{"tenant_id": "Tenant identifier used to resolve the embedding model (mirrors python self._canvas._tenant_id).",- "kb_id": "Optional knowledgebase identifier used to resolve the bound embedding model when the setups embedding_model is unset.",+ "kb_id": "Knowledgebase identifier used to resolve the bound embedding model (kb.embd_id). The embedding model is taken exclusively from the dataset; the DSL must not configure it.","output_format": "Upstream payload discriminator: json / markdown / text / html / chunks.","chunks": "List of chunk maps when output_format == \"chunks\".","json": "Structured parser payload when output_format == \"json\" or unset.",@@ -327,10 +313,6 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapname := globals.GlobalOrInput(ctx, inputs, "name", "")tenantID := globals.GlobalOrInput(ctx, inputs, "tenant_id", "")kbID := globals.GlobalOrInput(ctx, inputs, "kb_id", "")- // The embedding-model id is a Tokenizer-scoped setup (params["setups"]),- // resolved at construction, not a run-level global — see- // embeddingModelFromSetups.- embeddingModel := c.embeddingModel// decodeTokenizerFromUpstream validates `name`; carry the resolved// name into the decode input so both a Globals-backed run and a@@ -382,7 +364,7 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map// so embedding is skipped there — debug only exercises parse+chunk and// must stay side-effect free.if shouldHaveEmbedding(c.param.SearchMethod, kbID) {- chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, embeddingModel, name, chunks)+ chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, name, chunks, chunkcache.Client())if err != nil {return nil, err}@@ -408,7 +390,7 @@ func copyPipelineControlValues(output, input map[string]any) {}}-func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, embeddingModel, name string, chunks []schema.ChunkDoc) ([]schema.ChunkDoc, int, error) {+func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, name string, chunks []schema.ChunkDoc, store chunkcache.Store) ([]schema.ChunkDoc, int, error) {if len(chunks) == 0 {return chunks, 0, nil}@@ -421,7 +403,7 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, emif resolver == nil {return nil, 0, fmt.Errorf("tokenizer: embedding requested but no embedder resolver configured")}- embedder, err := resolver(ctx, tenantID, kbID, embeddingModel)+ embedder, embdID, err := resolver(ctx, tenantID, kbID)if err != nil {return nil, 0, fmt.Errorf("tokenizer: resolve embedder: %w", err)}@@ -429,21 +411,60 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, emreturn nil, 0, fmt.Errorf("tokenizer: embedding requested but encoder resolution returned nil")}+ // store may be nil (no redis configured / unit test); cache lookups then+ // degrade to a no-op and every chunk is embedded normally.++ // resolved[i] holds the content embedding vector for chunks[i] once known:+ // either served from the per-chunk cache (isHit) or produced by the batch+ // Encode below. nil means the chunk produced no embeddable text. trunc is the+ // exact text fed to the model, carried so the Set-side key matches the+ // Get-side key (the key includes the embedded text, not just the chunk id).+ type resolvedVec struct {+ content []float64+ isHit bool+ trunc string+ }+ resolved := make([]*resolvedVec, len(chunks))texts := make([]string, 0, len(chunks))pairs := make([]int, 0, len(chunks))+ // truncs[i] is the embedded text for texts[i] / pairs[i]; carried so the+ // Set-side key matches the Get-side key for freshly embedded content.+ truncs := make([]string, 0, len(chunks))for i, ck := range chunks {raw := concatFields(ck, c.param.Fields)txt := htmlTableRE.ReplaceAllString(raw, " ")txt = strings.TrimSpace(txt)- trunc := truncateForEmbedding(txt, embedder.MaxTokens())if txt == "" {continue}+ // Per-chunk embedding cache: identical (dataset embd_id, chunk, embedded+ // text) triples reuse the previous content vector, skipping the embed+ // round-trip on resume. The key includes the text that is actually+ // embedded — after field selection (c.param.Fields) and truncation — not+ // just the chunk id. A tokenizer-config or model change alters the+ // embedded input; reusing the prior vector for that case would serve a+ // stale embedding for up to the cache TTL. embdID must be non-empty or+ // every model would collapse onto one key and served vectors could come+ // from a different model.+ trunc := truncateForEmbedding(txt, embedder.MaxTokens())+ if chunkID, ok := ck.GetExtraString("id"); ok && embdID != "" && store != nil {+ if cached, hit := chunkcache.Get(ctx, store, chunkcache.Key("emb", embdID, chunkID, trunc)); hit {+ var vec []float64+ if err := json.Unmarshal([]byte(cached), &vec); err == nil && len(vec) > 0 {+ resolved[i] = &resolvedVec{content: vec, isHit: true, trunc: trunc}+ continue+ }+ }+ }texts = append(texts, trunc)+ truncs = append(truncs, trunc)pairs = append(pairs, i)}if len(texts) == 0 {- return chunks, 0, nil+ // Nothing to embed — but cache hits may still need to be written through.+ if store == nil {+ return chunks, 0, nil+ }}trimmedName := strings.TrimSpace(name)@@ -502,17 +523,39 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, em}titleWeight := c.param.FilenameEmbdWeight+ // Wire freshly embedded content into the resolved slice; cache hits already+ // carry their vector from the loop above.for i, idx := range pairs {- merged := append([]float64(nil), contentResults[i].Vector...)+ resolved[idx] = &resolvedVec{content: contentResults[i].Vector, trunc: truncs[i]}+ }+ for i, re := range resolved {+ if re == nil {+ continue+ }+ merged := append([]float64(nil), re.content...)if hasTitleVec {- merged, err = mergeEmbeddingVectors(titleVec, contentResults[i].Vector, titleWeight)+ merged, err = mergeEmbeddingVectors(titleVec, re.content, titleWeight)if err != nil {return nil, 0, fmt.Errorf("tokenizer: merge vectors: %w", err)}}- if err := chunks[idx].SetExtraValue(fmt.Sprintf("q_%d_vec", len(merged)), merged); err != nil {+ if err := chunks[i].SetExtraValue(fmt.Sprintf("q_%d_vec", len(merged)), merged); err != nil {return nil, 0, fmt.Errorf("tokenizer: vector marshal: %w", err)}+ // Backfill the per-chunk cache only for freshly embedded content; cache+ // hits are left untouched. The key intentionally omits title weighting:+ // the content vector is title-weight independent, so identical chunk+ // content reuses across runs even when only the filename weight changes.+ // The embedded text (trunc) is part of the key so a config/model change+ // that alters the embedded input forces a fresh embed rather than serving+ // a stale vector.+ if !re.isHit {+ if chunkID, ok := chunks[i].GetExtraString("id"); ok && embdID != "" && store != nil {+ if b, merr := json.Marshal(re.content); merr == nil {+ chunkcache.Set(ctx, store, chunkcache.Key("emb", embdID, chunkID, re.trunc), string(b))+ }+ }+ }}return chunks, tokenCount, nil}diff --git a/internal/ingestion/component/tokenizer_cache_test.go b/internal/ingestion/component/tokenizer_cache_test.gonew file mode 100644index 0000000000..df29a1e864--- /dev/null+++ b/internal/ingestion/component/tokenizer_cache_test.go@@ -0,0 +1,313 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "encoding/json"+ "sync"+ "testing"+ "time"++ "ragflow/internal/ingestion/chunkcache"+ "ragflow/internal/ingestion/component/schema"+)++// vecOf reads a float64 vector stored by SetExtraValue back out of the chunk.+func vecOf(t *testing.T, doc schema.ChunkDoc, key string) []float64 {+ t.Helper()+ raw, ok := doc.Extra[key]+ if !ok {+ t.Fatalf("missing extra %s", key)+ }+ var v []float64+ if err := json.Unmarshal(raw, &v); err != nil {+ t.Fatalf("unmarshal %s: %v", key, err)+ }+ return v+}++// memCacheStore is an in-memory chunkcache.Store used to exercise the+// per-chunk embedding cache without Redis.+type memCacheStore struct {+ mu sync.Mutex+ data map[string]string+ sets map[string]map[string]struct{}+}++func newMemCacheStore() *memCacheStore {+ return &memCacheStore{data: map[string]string{}, sets: map[string]map[string]struct{}{}}+}++func (m *memCacheStore) Get(_ context.Context, key string) (string, error) {+ m.mu.Lock()+ defer m.mu.Unlock()+ return m.data[key], nil+}++func (m *memCacheStore) Set(_ context.Context, key, value string, _ time.Duration) bool {+ m.mu.Lock()+ defer m.mu.Unlock()+ m.data[key] = value+ return true+}++func (m *memCacheStore) SAdd(_ context.Context, key, member string) bool {+ m.mu.Lock()+ defer m.mu.Unlock()+ if m.sets[key] == nil {+ m.sets[key] = map[string]struct{}{}+ }+ m.sets[key][member] = struct{}{}+ return true+}++func (m *memCacheStore) SMembers(_ context.Context, key string) ([]string, error) {+ m.mu.Lock()+ defer m.mu.Unlock()+ out := make([]string, 0, len(m.sets[key]))+ for k := range m.sets[key] {+ out = append(out, k)+ }+ return out, nil+}++func (m *memCacheStore) Delete(_ context.Context, key string) bool {+ m.mu.Lock()+ defer m.mu.Unlock()+ delete(m.data, key)+ delete(m.sets, key)+ return true+}++func (m *memCacheStore) Expire(_ context.Context, key string, _ time.Duration) bool {+ return true+}++func (m *memCacheStore) countEmbKeys() int {+ m.mu.Lock()+ defer m.mu.Unlock()+ n := 0+ for k := range m.data {+ if len(k) >= len("kc:emb:") && k[:len("kc:emb:")] == "kc:emb:" {+ n+++ }+ }+ return n+}++// chunkWithID builds a ChunkDoc carrying a chunk id in Extra (so the cache+// key can be derived) plus the given body text.+func chunkWithID(id, text string) schema.ChunkDoc {+ doc, _ := schema.ChunkDocFromMap(map[string]any{"id": id, "text": text})+ return doc+}++func TestEmbedChunks_PerChunkCacheMissThenHit(t *testing.T) {+ comp, stub := withStubEmbedder(t, 4)+ comp.param.Fields = []string{"text"}++ store := newMemCacheStore()+ chunks := []schema.ChunkDoc{chunkWithID("chunk-1", "hello world")}++ // First pass: cache miss -> embedder is called once.+ out1, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", chunks, store)+ if err != nil {+ t.Fatalf("embedChunks (miss): %v", err)+ }+ if got := stub.calls.Load(); got != 1 {+ t.Fatalf("expected 1 embedder call on miss, got %d", got)+ }+ vec1 := vecOf(t, out1[0], "q_4_vec")+ if store.countEmbKeys() != 1 {+ t.Fatalf("expected 1 emb cache key after miss, got %d", store.countEmbKeys())+ }++ // Second pass with the same chunk id: cache hit -> embedder NOT called again.+ out2, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", chunks, store)+ if err != nil {+ t.Fatalf("embedChunks (hit): %v", err)+ }+ if got := stub.calls.Load(); got != 1 {+ t.Fatalf("expected embedder to be skipped on cache hit, but calls=%d", got)+ }+ vec2 := vecOf(t, out2[0], "q_4_vec")+ if len(vec1) != len(vec2) {+ t.Fatalf("cached vector differs from embedded vector: %v vs %v", vec1, vec2)+ }+ for i := range vec1 {+ if vec1[i] != vec2[i] {+ t.Fatalf("cached vector differs from embedded vector: %v vs %v", vec1, vec2)+ }+ }+}++func TestEmbedChunks_CacheKeyScopedByChunkAndEmbdID(t *testing.T) {+ embdID := "embd-a"+ comp, stub := withStubEmbedderEmbdID(t, 4, &embdID)+ comp.param.Fields = []string{"text"}++ store := newMemCacheStore()++ // Two distinct chunks -> two embedder calls, two cache keys.+ a := []schema.ChunkDoc{chunkWithID("chunk-a", "alpha"), chunkWithID("chunk-b", "beta")}+ if _, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", a, store); err != nil {+ t.Fatalf("embedChunks: %v", err)+ }+ if got := stub.calls.Load(); got != 1 {+ t.Fatalf("expected 1 batch call for 2 chunks, got %d", got)+ }+ if n := store.countEmbKeys(); n != 2 {+ t.Fatalf("expected 2 emb cache keys for 2 distinct chunks, got %d", n)+ }++ // Rebinding the dataset to a different embedding model must bypass the+ // cache: the key is derived from the dataset's embd_id, so the entry+ // written under embd-a is not reusable and a stale vector is never served.+ embdID = "embd-b"+ b := []schema.ChunkDoc{chunkWithID("chunk-a", "alpha")}+ if _, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", b, store); err != nil {+ t.Fatalf("embedChunks (rebound embd_id): %v", err)+ }+ if got := stub.calls.Load(); got != 2 {+ t.Fatalf("expected a new embedder call after the dataset's embd_id changed, got calls=%d", got)+ }+ if n := store.countEmbKeys(); n != 3 {+ t.Fatalf("expected 3 emb cache keys after embd_id change, got %d", n)+ }+}++// The embedding model comes from the dataset, never from the DSL: two components+// configured with different setups.embedding_model but the same dataset embd_id+// must resolve to the same cache entry. This locks in that the DSL cannot+// influence the cache key (and would fail if setups.embedding_model were read+// again), which is what previously let a re-bound dataset serve a stale vector.+func TestEmbedChunks_CacheIgnoresDSLEmbeddingModel(t *testing.T) {+ embdID := "embd-a"+ stub := newStubEmbedder(4)+ build := func(dslModel string) *TokenizerComponent {+ comp, err := NewTokenizerComponentWithResolver(+ map[string]any{"setups": map[string]any{"embedding_model": dslModel}},+ func(_ context.Context, _, _ string) (Embedder, string, error) { return stub, embdID, nil },+ )+ if err != nil {+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)+ }+ c := comp.(*TokenizerComponent)+ c.param.Fields = []string{"text"}+ return c+ }++ store := newMemCacheStore()+ chunks := []schema.ChunkDoc{chunkWithID("chunk-1", "hello world")}++ if _, _, err := build("dsl-model-a").embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {+ t.Fatalf("embedChunks (dsl-model-a): %v", err)+ }+ if got := stub.calls.Load(); got != 1 {+ t.Fatalf("expected 1 embedder call on miss, got %d", got)+ }++ // Same dataset embd_id, different DSL model -> must hit the same entry.+ if _, _, err := build("dsl-model-b").embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {+ t.Fatalf("embedChunks (dsl-model-b): %v", err)+ }+ if got := stub.calls.Load(); got != 1 {+ t.Fatalf("DSL embedding_model must not affect the cache key: expected a hit, got calls=%d", got)+ }+ if n := store.countEmbKeys(); n != 1 {+ t.Fatalf("expected 1 emb cache key across both DSL models, got %d", n)+ }+}++// TestEmbedChunks_CacheKeyScopedByFields proves the embedding cache key captures+// the text actually embedded — which depends on c.param.Fields — so changing the+// tokenizer field selection forces a fresh embed instead of serving a stale+// vector for up to the cache TTL. The chunk id is unchanged across the two runs,+// isolating the variable to the embedded text.+func TestEmbedChunks_CacheKeyScopedByFields(t *testing.T) {+ embdID := "embd-test"+ stub := newStubEmbedder(4)+ build := func(fields []string) *TokenizerComponent {+ comp, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) {+ return stub, embdID, nil+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)+ }+ c := comp.(*TokenizerComponent)+ c.param.Fields = fields+ return c+ }++ store := newMemCacheStore()+ doc := schema.ChunkDoc{Text: "hello", Questions: "q1;q2"}+ doc.Extra = map[string]json.RawMessage{}+ if err := doc.SetExtraValue("id", "chunk-1"); err != nil {+ t.Fatalf("SetExtraValue(id): %v", err)+ }+ chunks := []schema.ChunkDoc{doc}++ // Fields=["text"]: only the body is embedded -> cache miss, 1 call.+ if _, _, err := build([]string{"text"}).embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {+ t.Fatalf("embedChunks (fields=text): %v", err)+ }+ if got := stub.calls.Load(); got != 1 {+ t.Fatalf("expected 1 embedder call with fields=text, got %d", got)+ }++ // Fields=["text","questions"]: the embedded text now includes the questions,+ // so the key must differ and the cache must be bypassed (another call).+ if _, _, err := build([]string{"text", "questions"}).embedChunks(context.Background(), "tenant", "kb", "", chunks, store); err != nil {+ t.Fatalf("embedChunks (fields=text,questions): %v", err)+ }+ if got := stub.calls.Load(); got != 2 {+ t.Fatalf("expected a fresh embed when Fields change the embedded text, got calls=%d (stale vector would otherwise be served)", got)+ }+ if n := store.countEmbKeys(); n != 2 {+ t.Fatalf("expected 2 emb cache keys (one per embedded-text variant), got %d", n)+ }+}++func TestEmbedChunks_MissingChunkIDSkipsCache(t *testing.T) {+ comp, stub := withStubEmbedder(t, 4)+ comp.param.Fields = []string{"text"}++ store := newMemCacheStore()+ // Chunk without an id: cannot be cached, always embedded.+ noID, _ := schema.ChunkDocFromMap(map[string]any{"text": "no id here"})++ first, _, err := comp.embedChunks(context.Background(), "tenant", "kb", "", []schema.ChunkDoc{noID}, store)+ if err != nil {+ t.Fatalf("embedChunks (no id): %v", err)+ }+ if stub.calls.Load() != 1 {+ t.Fatalf("expected embedder call for uncacheable chunk")+ }+ vecOf(t, first[0], "q_4_vec")+ if n := store.countEmbKeys(); n != 0 {+ t.Fatalf("expected no emb cache key for id-less chunk, got %d", n)+ }++ // Second pass: still no id -> still embedded (calls increment).+ comp.embedChunks(context.Background(), "tenant", "kb", "", []schema.ChunkDoc{noID}, store)+ if stub.calls.Load() != 2 {+ t.Fatalf("expected a second embedder call for id-less chunk (cache not used), got %d", stub.calls.Load())+ }+}++var _ chunkcache.Store = (*memCacheStore)(nil)diff --git a/internal/ingestion/component/tokenizer_nonpersist_test.go b/internal/ingestion/component/tokenizer_nonpersist_test.goindex 1d9a8c2457..7eee084374 100644--- a/internal/ingestion/component/tokenizer_nonpersist_test.go+++ b/internal/ingestion/component/tokenizer_nonpersist_test.go@@ -42,7 +42,7 @@ func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {stub := newStubEmbedder(4)cIntf, err := NewTokenizerComponentWithResolver(map[string]any{"search_method": []any{"embedding"}, "fields": []any{"text"}},- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },)if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.goindex ed6902400a..5703209b0e 100644--- a/internal/ingestion/component/tokenizer_test.go+++ b/internal/ingestion/component/tokenizer_test.go@@ -385,7 +385,7 @@ func TestTokenizerComponent_Invoke_EncoderCountMismatch(t *testing.T) {// Inject an embedder that returns the wrong number of vectors// regardless of input.wrong := &countMismatchedEmbedder{want: 1}- cIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _, _ string) (Embedder, error) { return wrong, nil })+ cIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) { return wrong, "", nil })if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)}@@ -525,13 +525,13 @@ func TestTokenizerComponent_Embedding_UsesFilenameWeight(t *testing.T) {requireTokenizerPool(t)cIntf, err := NewTokenizerComponentWithResolver(map[string]any{"filename_embd_weight": 0.25,- }, func(_ context.Context, _, _, _ string) (Embedder, error) {+ }, func(_ context.Context, _, _ string) (Embedder, string, error) {stub := newStubEmbedder(2)stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{8, 8}}, tokenCount: 3},{vectors: [][]float64{{2, 2}}, tokenCount: 5},}- return stub, nil+ return stub, "", nil})if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)@@ -735,18 +735,18 @@ func floatSliceClose(got, want []float64) bool {func TestTokenizerComponent_InstanceResolversDoNotLeakAcrossComponents(t *testing.T) {requireTokenizerPool(t)- compAIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _, _ string) (Embedder, error) {+ compAIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) {stub := newStubEmbedder(2)stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{10, 10}}, tokenCount: 1}, {vectors: [][]float64{{1, 1}}, tokenCount: 1}}- return stub, nil+ return stub, "", nil})if err != nil {t.Fatalf("NewTokenizerComponentWithResolver(A): %v", err)}- compBIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _, _ string) (Embedder, error) {+ compBIntf, err := NewTokenizerComponentWithResolver(nil, func(_ context.Context, _, _ string) (Embedder, string, error) {stub := newStubEmbedder(2)stub.resultsByCall = []embeddingCallResult{{vectors: [][]float64{{20, 20}}, tokenCount: 1}, {vectors: [][]float64{{2, 2}}, tokenCount: 1}}- return stub, nil+ return stub, "", nil})if err != nil {t.Fatalf("NewTokenizerComponentWithResolver(B): %v", err)diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex c60acb814c..3738df1d60 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -115,9 +115,23 @@ func newStubEmbedder(dim int) *stubEmbedder {// (["full_text","embedding"]); callers that need a different mode construct// the component directly via NewTokenizerComponent(NewTokenizerComponentWithResolver).func withStubEmbedder(t *testing.T, dim int) (*TokenizerComponent, *stubEmbedder) {+ // Non-empty: the per-chunk cache is skipped unless the dataset reports an+ // embd_id, so an empty value here would silently disable the cache.+ embdID := "embd-test"+ return withStubEmbedderEmbdID(t, dim, &embdID)+}++// withStubEmbedderEmbdID builds a TokenizerComponent whose resolver reports the+// value pointed to by embdID at call time. The per-chunk embedding cache keys on+// that id, so mutating *embdID between two embedChunks passes models "the+// dataset's embedding model was rebound" — the two passes must then share no+// cache entry, which is exactly what a stale vector would otherwise cause.+func withStubEmbedderEmbdID(t *testing.T, dim int, embdID *string) (*TokenizerComponent, *stubEmbedder) {t.Helper()stub := newStubEmbedder(dim)- comp, err := NewTokenizerComponentWithResolver(nil, func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil })+ comp, err := NewTokenizerComponentWithResolver(nil, func(ctx context.Context, _, _ string) (Embedder, string, error) {+ return stub, *embdID, nil+ })if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)}@@ -196,8 +210,8 @@ func TestTokenizerComponent_Invoke_NilChunks(t *testing.T) {func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {cIntf, err := NewTokenizerComponentWithResolver(map[string]any{"search_method": []any{"embedding"},- }, func(ctx context.Context, _, _, _ string) (Embedder, error) {- return newStubEmbedder(4), nil+ }, func(ctx context.Context, _, _ string) (Embedder, string, error) {+ return newStubEmbedder(4), "", nil})if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)@@ -460,7 +474,7 @@ func TestChunkOrderInt_EmbeddingOnly(t *testing.T) {stub := newStubEmbedder(3)comp, err := NewTokenizerComponentWithResolver(map[string]any{"search_method": []string{"embedding"}, "fields": []string{"text"}},- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },)if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)@@ -514,7 +528,7 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {stub := newStubEmbedder(3)comp, err := NewTokenizerComponentWithResolver(map[string]any{"search_method": []string{"embedding"}, "fields": []string{"text"}},- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },)if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)@@ -584,7 +598,7 @@ func TestChunkOrderInt_AllPathsUnconditional(t *testing.T) {t.Run(tc.name, func(t *testing.T) {comp, err := NewTokenizerComponentWithResolver(map[string]any{"search_method": tc.searchMethod, "fields": []string{"text"}},- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },)if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)@@ -627,7 +641,7 @@ func TestChunkOrderInt_A5FilterKeepsSequenceContiguous(t *testing.T) {stub := newStubEmbedder(8)comp, err := NewTokenizerComponentWithResolver(map[string]any{"search_method": []string{"full_text", "embedding"}, "fields": []string{"text"}},- func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },+ func(ctx context.Context, _, _ string) (Embedder, string, error) { return stub, "", nil },)if err != nil {t.Fatalf("NewTokenizerComponentWithResolver: %v", err)diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex a73a081073..4a6740e8fa 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -30,6 +30,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common"redis2 "ragflow/internal/engine/redis"+ "ragflow/internal/ingestion/component""ragflow/internal/ingestion/component/globals""ragflow/internal/utility"@@ -229,7 +230,10 @@ func cloneMapOrEmpty(m map[string]any) map[string]any {// defaultCheckpointTTL is the expiry applied to the eino checkpoint payload// and the RunTracker hash. A finished run's checkpoint is deleted on success;// the TTL only guards against leaks from crashed runs that never clean up.-var defaultCheckpointTTL = 24 * time.Hour+//+// It matches the per-chunk cache TTL (chunkcache.TTL) so a crashed run leaves+// both its checkpoint and the chunk cache entries to expire on the same horizon.+var defaultCheckpointTTL = 7 * 24 * time.Hour// dslKeySuffix / ovfKeySuffix derive the two DSL-fingerprint keys from a// checkpoint id. Both live in the same CheckPointStore as the eino payload@@ -437,8 +441,18 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParamcompileOpts = append(compileOpts,canvas.WithCheckPointStore(store),canvas.WithCheckPointID(p.taskID),- canvas.WithInterruptAfterNonTerminalCpn(),)+ // Single interrupt point: immediately after the Parser component. The+ // Extractor / Tokenizer stages do NOT take a checkpoint of their own —+ // a resume re-runs them, but the per-chunk cache (chunkcache) absorbs+ // the cost by serving prior LLM/embedding results. Interruption after+ // every non-terminal component was dropped on purpose to avoid one+ // needless ResumeWithData round per stage.+ if parserCpnID := ExtractParserCpnID(p.rawDSL, component.ComponentNameParser); parserCpnID != "" {+ compileOpts = append(compileOpts, canvas.WithInterruptAfter([]string{parserCpnID}))+ } else {+ compileOpts = append(compileOpts, canvas.WithInterruptAfterNonTerminalCpn())+ }}// Run-level setups (keyed by cpnID) override the DSL-baked component// setups at compile time (higher priority; see canvas.WithOverrideParams).@@ -478,6 +492,11 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam// component re-publishes `name` (and storage refs) as it derives// them mid-run.globals.SeedIngestionGlobals(runCtx, current)+ // Expose the task id on the run context so every component can register the+ // per-chunk cache keys it writes into the task manifest (chunkcache). Without+ // this the manifest is never populated and PurgeTask on success becomes a+ // no-op, leaving orphaned cache entries until TTL expiry.+ globals.SetTaskID(runCtx, p.taskID)if !resumable {return p.runPlain(runCtx, current, compiled, tracker, runState)diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.goindex a9359c7cd9..2bde20cb4c 100644--- a/internal/ingestion/task/embedder.go+++ b/internal/ingestion/task/embedder.go@@ -63,31 +63,38 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.E// newEmbedderResolver builds the production embedder resolver used by the// Tokenizer component. It always resolves the embedder from the dataset's-// configured embd_id (looked up by kbID). If the dataset has no embd_id-// configured, it returns nil (no embedding). Kept as a constructor over-// injectable deps so the resolution logic stays unit-testable without a live-// model provider / DB.+// configured embd_id (looked up by kbID) and returns that embd_id alongside the+// embedder, so the Tokenizer can key its per-chunk cache on the dataset-bound+// model. If the dataset has no embd_id configured, it returns a nil embedder and+// an empty embd_id (no embedding). Kept as a constructor over injectable deps so+// the resolution logic stays unit-testable without a live model provider / DB.func newEmbedderResolver(getKBEmbdID func(ctx context.Context, kbID string) (string, error),getEmbeddingModel func(ctx context.Context, tenantID, embdID string) (*models.EmbeddingModel, error),) componentpkg.EmbedderResolver {- return func(ctx context.Context, tenantID, kbID, _ string) (componentpkg.Embedder, error) {+ // The resolver derives the embedding model exclusively from the+ // knowledgebase's configured embd_id — never from any DSL-supplied+ // identifier. It returns embdID alongside the embedder so the tokenizer can+ // key its per-chunk cache on the dataset-bound model: when a KB's embedding+ // model changes, embdID changes, the cache key changes, and no stale vector+ // is ever served.+ return func(ctx context.Context, tenantID, kbID string) (componentpkg.Embedder, string, error) {embdID, err := getKBEmbdID(ctx, kbID)if err != nil {- return nil, fmt.Errorf("embedder: resolve kb embd_id for kb_id=%s: %w", kbID, err)+ return nil, "", fmt.Errorf("embedder: resolve kb embd_id for kb_id=%s: %w", kbID, err)}embdID = strings.TrimSpace(embdID)if embdID == "" {- return nil, nil+ return nil, "", nil}model, err := getEmbeddingModel(ctx, tenantID, embdID)if err != nil {- return nil, err+ return nil, "", err}if model == nil {- return nil, fmt.Errorf("embedder: resolved embedding model is nil for embd_id=%s", embdID)+ return nil, "", fmt.Errorf("embedder: resolved embedding model is nil for embd_id=%s", embdID)}- return &embedder{model: model}, nil+ return &embedder{model: model}, embdID, nil}}diff --git a/internal/ingestion/task/embedder_test.go b/internal/ingestion/task/embedder_test.goindex e3bbe5d730..ce288d27c4 100644--- a/internal/ingestion/task/embedder_test.go+++ b/internal/ingestion/task/embedder_test.go@@ -40,7 +40,7 @@ func TestEmbedderResolver_UsesKBEmbdID(t *testing.T) {},)ctx := t.Context()- emb, err := resolver(ctx, "tenant-1", "kb-1", "should-be-ignored")+ emb, embdID, err := resolver(ctx, "tenant-1", "kb-1")if err != nil {t.Fatalf("resolver: %v", err)}@@ -50,6 +50,9 @@ func TestEmbedderResolver_UsesKBEmbdID(t *testing.T) {if gotTenantID != "tenant-1" || gotEmbdID != "kb-embd-1" {t.Fatalf("resolver args = (%q, %q), want (tenant-1, kb-embd-1)", gotTenantID, gotEmbdID)}+ if embdID != "kb-embd-1" {+ t.Fatalf("resolver embdID = %q, want kb-embd-1", embdID)+ }}func TestEmbedderResolver_EmptyKBEmbdIDReturnsNil(t *testing.T) {@@ -63,13 +66,16 @@ func TestEmbedderResolver_EmptyKBEmbdIDReturnsNil(t *testing.T) {},)ctx := t.Context()- emb, err := resolver(ctx, "tenant-1", "kb-1", "ignored")+ emb, embdID, err := resolver(ctx, "tenant-1", "kb-1")if err != nil {t.Fatalf("resolver: %v", err)}if emb != nil {t.Fatal("expected nil embedder when kb has no embd_id")}+ if embdID != "" {+ t.Fatalf("expected empty embdID when kb has no embd_id, got %q", embdID)+ }}// stubDriver records the texts passed to Embed for verification.diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 5dc6c75f66..46e3689622 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -31,6 +31,7 @@ import ("ragflow/internal/engine"enginetypes "ragflow/internal/engine/types""ragflow/internal/entity"+ "ragflow/internal/ingestion/chunkcache""ragflow/internal/ingestion/component""ragflow/internal/ingestion/component/globals"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"@@ -323,6 +324,18 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput maps.taskCtx.Doc.ParserConfig,)+ // Persist is now fully durable: every failure-capable step above+ // (index write, compiled-product reconcile, Wiki active-MAP state) has+ // succeeded. Only now drop the per-chunk cache entries this task produced,+ // so a failure in any of those steps leaves the cache intact for the retry —+ // the retry resumes after the Parser checkpoint and would otherwise have to+ // re-pay every embedding/LLM call, which is exactly the cost this cache+ // exists to absorb. PurgeTask is best-effort and silent when no manifest+ // exists (e.g. a Redis-less run, where chunkcache.Client() is nil).+ if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" {+ chunkcache.PurgeTask(ctx, chunkcache.Client(), s.taskCtx.IngestionTask.ID)+ }+return &PipelineResult{DocID: s.taskCtx.Doc.ID,KbID: s.taskCtx.Doc.KbID,