RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Test(chunker): add golden parity harness, fixtures, and live Go<->Python tool (#17735)
Jacka399b931430d
diff --git a/go.mod b/go.modindex 363f210642..6bdaaf2206 100644--- a/go.mod+++ b/go.mod@@ -30,6 +30,7 @@ require (github.com/goccy/go-json v0.10.6github.com/golang-jwt/jwt/v5 v5.3.0github.com/gomarkdown/markdown v0.0.0-20260614204949-e08cff860f76+ github.com/google/go-cmp v0.7.0github.com/google/uuid v1.6.0github.com/gorilla/websocket v1.5.3github.com/infiniflow/infinity-go-sdk v0.0.0-00010101000000-000000000000@@ -140,7 +141,6 @@ require (github.com/gogo/protobuf v1.3.2 // indirectgithub.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirectgithub.com/golang-sql/sqlexp v0.1.0 // indirect- github.com/google/go-cmp v0.7.0 // indirectgithub.com/google/go-tpm v0.9.8 // indirectgithub.com/google/s2a-go v0.1.9 // indirectgithub.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirectdiff --git a/internal/ingestion/component/chunker/golden_parity_test.go b/internal/ingestion/component/chunker/golden_parity_test.gonew file mode 100644index 0000000000..a88851675e--- /dev/null+++ b/internal/ingestion/component/chunker/golden_parity_test.go@@ -0,0 +1,554 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++// Cross-language golden parity: Python is the reference implementation, and+// every case under testdata/parity/cases is run through both sides.+// testdata/parity/golden holds what Python produced; this test asserts the Go+// port reproduces it.+//+// Unit tier (no build tag) on purpose. The chunker package builds and passes+// under CGO_ENABLED=0: the only native-backed file is pdfcrop_cgo.go, and its+// cropImageChunks returns the chunks untouched when the PDF engine is nil —+// exactly what happens here, since the harness passes no *gorm.DB and so no+// engine is ever resolved. Running without CGO therefore costs no coverage;+// the single field it cannot populate is ChunkDoc.Image (a base64 preview).+//+// Regenerate the golden files with:+//+// python internal/ingestion/component/chunker/tool-py/capture_golden.py --all++import (+ "context"+ "encoding/json"+ "flag"+ "fmt"+ "os"+ "path"+ "path/filepath"+ "sort"+ "strconv"+ "strings"+ "testing"++ "ragflow/internal/agent/runtime"++ "github.com/google/go-cmp/cmp"+)++const (+ parityCasesDir = "testdata/parity/cases"+ parityGoldenDir = "testdata/parity/golden"+ goSnapshotDir = "testdata/parity/go_snapshot"+ knownDiffsPath = "testdata/parity/known_diffs.json"+ captureCmd = "python internal/ingestion/component/chunker/tool-py/capture_golden.py --all"+ kindExtraFields = "extra_fields"+ kindChunkCount = "chunk_count"+ kindChunkText = "chunk_text"+)++// update regenerates the Go-side snapshots for known-diff cases (see+// compareChunkCountKnownDiff). It is a test-only flag, so it is declared here+// at package scope and consumed by `go test -update`.+var update = flag.Bool("update", false, "regenerate go_snapshot files for known-diff cases")++// parityCase is one input fixture, shared verbatim by both languages.+//+// Sharing is possible because the Python upstream models declare+// populate_by_name=True with the short aliases ("json", "markdown", "text",+// "html") that the Go schema tags already use, so neither side needs a+// translation layer. Param keys line up the same way: the Python attribute+// names equal the Go json tags.+type parityCase struct {+ ID string `json:"id"`+ Component string `json:"component"`+ Notes string `json:"notes"`+ Param map[string]any `json:"param"`+ Input map[string]any `json:"input"`+}++// goldenResult is what capture_golden.py recorded for a case.+type goldenResult struct {+ CaseID string `json:"case_id"`+ Chunks []map[string]any `json:"chunks"`+ Error string `json:"error"`+}++// diffRule is one entry in known_diffs.json: an accepted, documented+// divergence from the Python baseline.+//+// The registry exists so that a difference is either fixed or written down —+// never silently tolerated. Anything not covered by a rule fails the test.+type diffRule struct {+ ID string `json:"id"`+ Tag string `json:"tag"`+ Kind string `json:"kind"`+ AppliesTo []string `json:"applies_to"`+ Fields []string `json:"fields"`+ OwnerFixSide string `json:"owner_fix_side"`+ Tracking string `json:"tracking"`+ Reason string `json:"reason"`+ Permanent bool `json:"permanent"`+}++func (r diffRule) matches(caseID string) bool {+ for _, pattern := range r.AppliesTo {+ if ok, err := path.Match(pattern, caseID); err == nil && ok {+ return true+ }+ }+ return false+}++func loadKnownDiffs(t *testing.T) []diffRule {+ t.Helper()+ raw, err := os.ReadFile(knownDiffsPath)+ if err != nil {+ t.Fatalf("read %s: %v", knownDiffsPath, err)+ }+ var registry struct {+ Version int `json:"version"`+ Rules []diffRule `json:"rules"`+ }+ if err := json.Unmarshal(raw, ®istry); err != nil {+ t.Fatalf("parse %s: %v", knownDiffsPath, err)+ }+ return registry.Rules+}++// allowedExtraFields collects the Go-only chunk keys declared for this case.+func allowedExtraFields(rules []diffRule, caseID string) map[string]string {+ allowed := make(map[string]string)+ for _, rule := range rules {+ if rule.Kind != kindExtraFields || !rule.matches(caseID) {+ continue+ }+ for _, field := range rule.Fields {+ allowed[field] = rule.ID+ }+ }+ return allowed+}++func TestChunkerGoldenParity(t *testing.T) {+ entries, err := os.ReadDir(parityCasesDir)+ if err != nil {+ t.Fatalf("read cases dir: %v", err)+ }+ rules := loadKnownDiffs(t)+ var ran int+ for _, entry := range entries {+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {+ continue+ }+ ran+++ name := strings.TrimSuffix(entry.Name(), ".json")+ t.Run(name, func(t *testing.T) {+ tc := loadCase(t, filepath.Join(parityCasesDir, entry.Name()))+ if tc.ID != name {+ t.Fatalf("case id %q does not match filename stem %q", tc.ID, name)+ }+ want := loadGolden(t, tc.ID)+ got := invokeChunker(t, tc)+ allowedExtra := allowedExtraFields(rules, tc.ID)+ if rule := matchedRatchetRule(rules, tc.ID); rule != nil {+ compareRatchetedKnownDiff(t, tc.ID, want.Chunks, got, rule, allowedExtra, extraFieldsToStrip(rules))+ return+ }+ compareChunks(t, want.Chunks, got, allowedExtra)+ })+ }+ if ran == 0 {+ t.Fatalf("no cases found under %s", parityCasesDir)+ }+}++func loadCase(t *testing.T, path string) parityCase {+ t.Helper()+ raw, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("read case: %v", err)+ }+ var tc parityCase+ if err := json.Unmarshal(raw, &tc); err != nil {+ t.Fatalf("parse case %s: %v", path, err)+ }+ return tc+}++func loadGolden(t *testing.T, caseID string) goldenResult {+ t.Helper()+ path := filepath.Join(parityGoldenDir, caseID+".json")+ raw, err := os.ReadFile(path)+ if err != nil {+ // Never skip on a missing baseline: an absent golden file means the+ // case is unverified, which is a failure, not a pass.+ t.Fatalf("missing golden file %s — capture it with:\n\t%s\n(%v)", path, captureCmd, err)+ }+ var g goldenResult+ if err := json.Unmarshal(raw, &g); err != nil {+ t.Fatalf("parse golden %s: %v", path, err)+ }+ if g.Error != "" {+ t.Fatalf("golden records a Python-side error for this case: %s", g.Error)+ }+ return g+}++// invokeChunker runs the component through the registry, which is the+// production path: it wraps every chunker in imageUploadDecorator. That+// decorator is what writes the Go-only "id" field, and with no kb_id in+// context it drops raw image bytes instead of reaching for MinIO.+func invokeChunker(t *testing.T, tc parityCase) []map[string]any {+ t.Helper()+ factory, _, _, ok := runtime.DefaultRegistry.Lookup(tc.Component)+ if !ok || factory == nil {+ t.Fatalf("component %q is not registered", tc.Component)+ }+ comp, err := factory(tc.Component, tc.Param)+ if err != nil {+ t.Fatalf("construct %s: %v", tc.Component, err)+ }+ out, err := comp.Invoke(context.Background(), nil, tc.Input)+ if err != nil {+ t.Fatalf("invoke %s: %v", tc.Component, err)+ }+ if msg, ok := out["_ERROR"].(string); ok && msg != "" {+ t.Fatalf("Go returned _ERROR while Python succeeded: %s", msg)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ return chunks+}++// compareChunks asserts the Go chunks match the Python baseline.+//+// Field handling is deliberately asymmetric, because the two directions mean+// different things:+//+// - a key Python emits that Go does not is always a defect — Go dropped+// data, so it is reported unconditionally;+// - a key Go emits that Python does not must be declared in known_diffs.json,+// so a new Go-only field has to be justified rather than silently tolerated.+//+// Deriving both sets per case rather than hardcoding an ignore list matters:+// the Go-only field set is not constant across components (TokenChunker emits+// ck_type/tk_nums/id, TitleChunker emits only id), so a fixed list would both+// over- and under-match.+func compareChunks(t *testing.T, want, got []map[string]any, allowedExtra map[string]string) {+ t.Helper()+ for _, problem := range chunkDiffs(want, got, allowedExtra) {+ t.Error(problem)+ }+}++// chunkDiffs returns every difference between the Python baseline and the Go+// output as human-readable messages; an empty result means the two sides agree.+//+// It is shared by the strict path and the known-diff ratchet so that "are these+// still diverging?" is answered by the exact same comparison that would fail the+// strict path — otherwise a rule could stay enrolled after its divergence was+// fixed, or hide a second, undeclared difference behind the first.+func chunkDiffs(want, got []map[string]any, allowedExtra map[string]string) []string {+ if len(want) != len(got) {+ // Report counts only. Diffing dozens of chunks that are misaligned by+ // one index produces pages of noise that hide the actual cause.+ return []string{fmt.Sprintf("chunk count: python=%d go=%d\npython texts: %s\ngo texts: %s",+ len(want), len(got), previewTexts(want), previewTexts(got))}+ }+ var problems []string+ for i := range want {+ pyChunk, goChunk := want[i], got[i]+ if missing := keysMissing(pyChunk, goChunk); len(missing) > 0 {+ problems = append(problems, fmt.Sprintf("chunk[%d]: Go is missing key(s) %v that Python emits", i, missing))+ }+ var undeclared []string+ for _, key := range keysMissing(goChunk, pyChunk) {+ if _, ok := allowedExtra[key]; !ok {+ undeclared = append(undeclared, key)+ }+ }+ if len(undeclared) > 0 {+ problems = append(problems, fmt.Sprintf("chunk[%d]: Go emits key(s) %v that Python does not, and no rule in %s declares them.\n"++ "Either drop the field in Go or add an extra_fields rule explaining why it must stay.",+ i, undeclared, knownDiffsPath))+ }+ for _, key := range sortedKeys(pyChunk) {+ goVal, ok := goChunk[key]+ if !ok {+ continue // already reported as missing+ }+ if diff := cmp.Diff(normalizeValue(pyChunk[key]), normalizeValue(goVal)); diff != "" {+ problems = append(problems, fmt.Sprintf("chunk[%d][%q] mismatch (-python +go):\n%s", i, key, diff))+ }+ }+ }+ return problems+}++// matchedRatchetRule returns the first snapshot-ratcheted known-diff rule that+// applies to this case, or nil if the case is compared strictly.+func matchedRatchetRule(rules []diffRule, caseID string) *diffRule {+ for i := range rules {+ r := &rules[i]+ if r.Kind != kindChunkCount && r.Kind != kindChunkText {+ continue+ }+ if !r.matches(caseID) {+ continue+ }+ return r+ }+ return nil+}++// compareRatchetedKnownDiff ratchets a documented output divergence.+//+// Unlike the strict compareChunks path, this case is recorded as a known diff,+// so the goal is not to match Python but to (1) keep diverging and (2) stay+// stable on the Go side. If the two sides ever agree again, the divergence is+// resolved and the rule must be removed; if Go's output drifts from its own+// recorded snapshot, a Go-side change broke something the registry was pinning.+// Either way the test fails loudly instead of silently passing.+//+// The rule's kind is also verified against the shape of the surviving+// divergence, so a case that changes class (same chunk count but different+// text, or vice versa) forces a re-read of the rule instead of quietly+// re-pinning under a description that no longer fits.+func compareRatchetedKnownDiff(t *testing.T, caseID string, want, got []map[string]any, rule *diffRule, allowedExtra map[string]string, strip map[string]bool) {+ t.Helper()+ if len(chunkDiffs(want, got, allowedExtra)) == 0 {+ t.Fatalf("divergence resolved: Go output now matches the Python baseline (%d chunks). "++ "Remove rule %q from %s and delete its snapshot — the known diff is gone.",+ len(got), rule.ID, knownDiffsPath)+ }+ switch rule.Kind {+ case kindChunkCount:+ if len(got) == len(want) {+ t.Fatalf("rule %q is kind %q, but both sides now emit %d chunks — only the content differs. "++ "Reclassify it as %q in %s and rewrite its reason.",+ rule.ID, kindChunkCount, len(got), kindChunkText, knownDiffsPath)+ }+ case kindChunkText:+ if len(got) != len(want) {+ t.Fatalf("rule %q is kind %q, but chunk counts now differ (python=%d go=%d). "++ "Reclassify it as %q in %s and rewrite its reason.",+ rule.ID, kindChunkText, len(want), len(got), kindChunkCount, knownDiffsPath)+ }+ }++ snapPath := filepath.Join(goSnapshotDir, caseID+".json")+ if *update {+ if err := os.MkdirAll(goSnapshotDir, 0o755); err != nil {+ t.Fatalf("create %s: %v", goSnapshotDir, err)+ }+ if err := os.WriteFile(snapPath, []byte(serializeChunks(caseID, got, strip)), 0o644); err != nil {+ t.Fatalf("write snapshot %s: %v", snapPath, err)+ }+ t.Logf("updated go snapshot %s (%d chunks)", snapPath, len(got))+ return+ }++ wantSnap, err := os.ReadFile(snapPath)+ if err != nil {+ t.Fatalf("missing go snapshot %s — generate it with `go test -update` (%v)", snapPath, err)+ }+ if gotSnap := serializeChunks(caseID, got, strip); gotSnap != string(wantSnap) {+ t.Fatalf("Go output drifted from snapshot %s.\n"++ "Either a Go-side change altered this known-diff behaviour (investigate), "++ "or run `go test -update` to accept the new snapshot.", snapPath)+ }+}++// extraFieldsToStrip collects every field declared as a Go-only extra field in+// known_diffs.json, so the ratchet snapshot can omit them (see serializeChunks).+func extraFieldsToStrip(rules []diffRule) map[string]bool {+ strip := make(map[string]bool)+ for _, r := range rules {+ if r.Kind != kindExtraFields {+ continue+ }+ for _, f := range r.Fields {+ strip[f] = true+ }+ }+ return strip+}++// serializeChunks renders Go chunks into a deterministic, human-diffable form+// used for go_snapshot files. Go's json encoder sorts map keys, so two runs+// with equal content produce byte-identical output.+//+// The Go-only decorative fields declared in known_diffs.json (id, ck_type,+// tk_nums) are stripped first: `id` is non-deterministic (derived from a+// per-run doc id), and `ck_type`/`tk_nums` are Go-internal metadata already+// covered by the extra_fields allow-list. Snapshotting them would make the+// ratchet non-reproducible across runs, defeating its purpose.+func serializeChunks(caseID string, chunks []map[string]any, strip map[string]bool) string {+ cleaned := make([]map[string]any, len(chunks))+ for i, ck := range chunks {+ c := make(map[string]any, len(ck))+ for k, v := range ck {+ if strip[k] {+ continue+ }+ c[k] = v+ }+ cleaned[i] = c+ }+ out := struct {+ CaseID string `json:"case_id"`+ Chunks []map[string]any `json:"chunks"`+ }{CaseID: caseID, Chunks: cleaned}+ raw, err := json.MarshalIndent(out, "", " ")+ if err != nil {+ // Should be impossible for these inputs; surface it rather than hide.+ panic(fmt.Sprintf("serializeChunks: %v", err))+ }+ return string(raw) + "\n"+}++// TestKnownDiffRegistryWellFormed keeps known_diffs.json from rotting into a+// junk drawer: every rule must be classifiable, justified, and point at a real+// case file. Without this gate, rules accumulate with no owner and no reason.+func TestKnownDiffRegistryWellFormed(t *testing.T) {+ rules := loadKnownDiffs(t)+ if len(rules) == 0 {+ t.Fatal("known_diffs.json has no rules; the registry must not be empty")+ }+ caseIDs := listCaseIDs(t)+ validTags := map[string]bool{+ "python_bug": true,+ "go_bug": true,+ "go_intentional": true,+ "unresolved": true,+ }+ for _, r := range rules {+ if r.ID == "" {+ t.Error("rule with empty id")+ }+ if !validTags[r.Tag] {+ t.Errorf("rule %q: tag %q not in %v", r.ID, r.Tag, validTags)+ }+ if r.Kind != kindExtraFields && r.Kind != kindChunkCount && r.Kind != kindChunkText {+ t.Errorf("rule %q: kind %q is not supported by the parity harness", r.ID, r.Kind)+ }+ if r.Reason == "" {+ t.Errorf("rule %q: reason must not be empty", r.ID)+ }+ // Either a tracking issue is recorded or the divergence is permanent.+ if r.Tracking == "" && !r.Permanent {+ t.Errorf("rule %q: must set tracking (issue link) or permanent:true", r.ID)+ }+ if r.Tracking != "" && r.Permanent {+ t.Errorf("rule %q: tracking and permanent:true are mutually exclusive", r.ID)+ }+ if !r.Permanent && r.OwnerFixSide == "" {+ t.Errorf("rule %q: non-permanent rule must set owner_fix_side", r.ID)+ }+ for _, pattern := range r.AppliesTo {+ if !patternMatchesAny(caseIDs, pattern) {+ t.Errorf("rule %q: applies_to pattern %q matches no case file under %s", r.ID, pattern, parityCasesDir)+ }+ }+ }+}++func listCaseIDs(t *testing.T) []string {+ t.Helper()+ entries, err := os.ReadDir(parityCasesDir)+ if err != nil {+ t.Fatalf("read cases dir: %v", err)+ }+ var ids []string+ for _, e := range entries {+ if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {+ continue+ }+ ids = append(ids, strings.TrimSuffix(e.Name(), ".json"))+ }+ return ids+}++func patternMatchesAny(ids []string, pattern string) bool {+ for _, id := range ids {+ if pattern == id {+ return true+ }+ if ok, err := path.Match(pattern, id); err == nil && ok {+ return true+ }+ }+ return false+}++// normalizeValue round-trips through encoding/json so numbers decoded from the+// golden file (always float64) and numbers produced natively by Go (int, int64)+// compare by value rather than by Go type.+func normalizeValue(v any) any {+ raw, err := json.Marshal(v)+ if err != nil {+ return v+ }+ var out any+ if err := json.Unmarshal(raw, &out); err != nil {+ return v+ }+ return out+}++func keysMissing(from, in map[string]any) []string {+ var out []string+ for key := range from {+ if _, ok := in[key]; !ok {+ out = append(out, key)+ }+ }+ sort.Strings(out)+ return out+}++func sortedKeys(m map[string]any) []string {+ out := make([]string, 0, len(m))+ for key := range m {+ out = append(out, key)+ }+ sort.Strings(out)+ return out+}++// previewTexts renders chunk texts compactly so a count mismatch shows where+// the two sides diverged without dumping whole documents.+func previewTexts(chunks []map[string]any) string {+ var b strings.Builder+ b.WriteByte('[')+ for i, ck := range chunks {+ if i > 0 {+ b.WriteString(", ")+ }+ if i == 6 {+ b.WriteString("...")+ break+ }+ text, _ := ck["text"].(string)+ if len(text) > 40 {+ text = text[:40] + "..."+ }+ b.WriteString(strconv.Quote(text))+ }+ b.WriteByte(']')+ return b.String()+}diff --git a/internal/ingestion/component/chunker/live_parity_test.go b/internal/ingestion/component/chunker/live_parity_test.gonew file mode 100644index 0000000000..7785ee4153--- /dev/null+++ b/internal/ingestion/component/chunker/live_parity_test.go@@ -0,0 +1,161 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++//go:build manual++package chunker++// Live Go<->Python chunker parity.+//+// Unlike golden_parity_test.go (which compares Go against committed Python+// golden fixtures), this test invokes the Python reference chunker as a+// subprocess at test time and diffs its fresh output against the Go port. The+// benefits are exactly what a live check buys you: no golden file can go+// stale, and a single command re-verifies every case against the current+// Python. The cost is a runtime dependency on the .venv Python environment+// (ragflow deps + cl100k table), so this is a manual-tier test only — it+// never runs in the default `go test ./...` CI path.+//+// Python side: tool-py/live_chunk.py <case_id> prints the case's chunks as+// JSON, reusing capture_golden.run_case so the shape matches the golden+// fixtures. It aborts (non-zero) if the tokenizer is dead, which turns a+// silently-collapsed baseline into a hard failure instead of a false pass.+//+// Comparison logic is shared with the golden harness: chunkDiffs reports every+// difference, allowedExtraFields tolerates the Go-only id/ck_type/tk_nums+// fields, and known_diffs.json is consulted so documented divergences are+// reported (not failed) rather than surprising the reader.++import (+ "encoding/json"+ "os"+ "os/exec"+ "path/filepath"+ "runtime"+ "strings"+ "testing"+)++// findRepoRoot walks up from this source file to the directory holding go.mod.+func findRepoRoot(t *testing.T) string {+ t.Helper()+ _, thisFile, _, ok := runtime.Caller(0)+ if !ok {+ t.Fatal("cannot determine test file location")+ }+ dir := filepath.Dir(thisFile)+ for i := 0; i < 12; i++ {+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {+ return dir+ }+ parent := filepath.Dir(dir)+ if parent == dir {+ break+ }+ dir = parent+ }+ t.Fatal("repo root (go.mod) not found walking up from test file")+ return ""+}++// venvPython returns the path to the ragflow .venv interpreter, or skips the+// test when it is unavailable (the live parity check needs that environment).+func venvPython(t *testing.T) string {+ t.Helper()+ root := findRepoRoot(t)+ p := filepath.Join(root, ".venv", "bin", "python")+ if _, err := os.Stat(p); err != nil {+ t.Skipf("venv python not found at %s — live Go->Python parity needs the .venv environment; skipping", p)+ }+ return p+}++// pythonChunksLive runs the Python reference chunker on one case and returns+// its chunks. A non-zero Python exit (including a dead tokenizer) fails the+// test rather than comparing against a poisoned baseline.+func pythonChunksLive(t *testing.T, caseID string) []map[string]any {+ t.Helper()+ root := findRepoRoot(t)+ py := venvPython(t)+ script := filepath.Join(root, "internal", "ingestion", "component", "chunker", "tool-py", "live_chunk.py")++ cmd := exec.Command(py, script, caseID)+ cmd.Dir = root+ cmd.Env = append(os.Environ(), "PYTHONPATH="+root, "PYTHONUNBUFFERED=1")++ out, err := cmd.Output()+ if err != nil {+ if ee, ok := err.(*exec.ExitError); ok {+ t.Fatalf("python chunker failed for %s: %v\nstderr:\n%s", caseID, err, string(ee.Stderr))+ }+ t.Fatalf("python chunker failed for %s: %v", caseID, err)+ }++ var result struct {+ CaseID string `json:"case_id"`+ Chunks []map[string]any `json:"chunks"`+ Error string `json:"error"`+ }+ if err := json.Unmarshal(out, &result); err != nil {+ t.Fatalf("parse python output for %s: %v\nraw:\n%s", caseID, err, string(out))+ }+ if result.Error != "" {+ t.Fatalf("python returned error for %s: %s", caseID, result.Error)+ }+ return result.Chunks+}++func TestChunkerLiveParity(t *testing.T) {+ entries, err := os.ReadDir(parityCasesDir)+ if err != nil {+ t.Fatalf("read cases dir: %v", err)+ }+ rules := loadKnownDiffs(t)+ var ran int+ for _, entry := range entries {+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {+ continue+ }+ ran+++ name := strings.TrimSuffix(entry.Name(), ".json")+ t.Run(name, func(t *testing.T) {+ tc := loadCase(t, filepath.Join(parityCasesDir, entry.Name()))+ if tc.ID != name {+ t.Fatalf("case id %q does not match filename stem %q", tc.ID, name)+ }+ want := pythonChunksLive(t, tc.ID) // Python reference, live+ got := invokeChunker(t, tc) // Go port+ allowedExtra := allowedExtraFields(rules, tc.ID)+ if rule := matchedRatchetRule(rules, tc.ID); rule != nil {+ // Documented divergence: Python is the truth and Go diverges by+ // design (a known go_bug). A live mismatch is expected; a live+ // match means the bug was fixed and the rule should be dropped.+ if diffs := chunkDiffs(want, got, allowedExtra); len(diffs) == 0 {+ t.Logf("known-diff %s RESOLVED live — remove rule from known_diffs.json", rule.ID)+ } else {+ t.Logf("known-diff %s still diverges live (expected): %d diff(s)", rule.ID, len(diffs))+ }+ return+ }+ for _, problem := range chunkDiffs(want, got, allowedExtra) {+ t.Errorf("%s", problem)+ }+ })+ }+ if ran == 0 {+ t.Fatalf("no cases found under %s", parityCasesDir)+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/README.md b/internal/ingestion/component/chunker/testdata/parity/README.mdnew file mode 100644index 0000000000..64b6501818--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/README.md@@ -0,0 +1,116 @@+# Chunker Python↔Go parity fixtures++Python (`rag/flow/chunker/`) is the reference implementation. Every case here is+run through both languages, and `golden_parity_test.go` asserts the Go port+reproduces what Python produced.++Chunking is deterministic — the same input always yields the same output — which+is what makes recorded baselines a valid oracle rather than a flaky snapshot.++## Layout++| Path | Contents |+|---|---|+| `cases/<id>.json` | Input fixture. Read verbatim by **both** languages. |+| `golden/<id>.json` | What Python produced for that case. |+| `known_diffs.json` | Accepted, documented divergences (see below). |+| `_manifest.json` | Capture provenance: commit, Python version, timestamp. |++The case filename stem must equal the `id` field; both the capture script and+the Go test enforce this.++## Why one input works for both languages++The Python upstream models (`rag/flow/chunker/schema.py`) declare+`populate_by_name=True` with aliases `json` / `markdown` / `text` / `html` —+the same short names the Go schema already uses as struct tags. Param keys line+up too: Python attribute names equal the Go json tags. So no translation layer+is needed, and there is no risk of the two sides drifting to different inputs.++Always write the short alias form (`"markdown"`, not `"markdown_result"`): only+the alias is accepted on both sides.++## Adding a case++1. Create `cases/<id>.json`:++ ```json+ {+ "id": "<id>",+ "component": "TokenChunker",+ "notes": "one line on what behavior this case locks",+ "param": { },+ "input": { "name": "t", "output_format": "json", "json": [] }+ }+ ```++ `component` is the registry name: `TokenChunker`, `TitleChunker`,+ `GroupTitleChunker`, or `HierarchyTitleChunker`.++2. Capture the Python baseline and run the Go side:++ ```bash+ source .venv/bin/activate && export PYTHONPATH=$(pwd)+ python internal/ingestion/component/chunker/tool-py/capture_golden.py --case <id>+ CGO_ENABLED=0 go test -count=1 ./internal/ingestion/component/chunker/ -run TestChunkerGoldenParity+ ```++3. If the case fails, that is a real divergence. Either fix the Go side or add a+ rule to `known_diffs.json` explaining why the difference stands.++## Regenerating after an intentional Python change++```bash+source .venv/bin/activate && export PYTHONPATH=$(pwd)+python internal/ingestion/component/chunker/tool-py/capture_golden.py --all+```++Check for stale baselines without writing anything:++```bash+python internal/ingestion/component/chunker/tool-py/capture_golden.py --all --check+```++Commit a regeneration on its own, with no unrelated changes — otherwise the+diff is too large to review by eye, which defeats the purpose of a baseline.++## known_diffs.json++A difference is either fixed or written down; the test never tolerates one+silently. Each rule carries:++| Field | Meaning |+|---|---|+| `id` | Stable rule name. |+| `tag` | `python_bug`, `go_bug`, `go_intentional`, or `unresolved`. |+| `kind` | Currently `extra_fields`. |+| `applies_to` | Case-id patterns (`path.Match` globs, e.g. `token__*`, `*`). |+| `fields` | For `extra_fields`: the Go-only chunk keys this rule permits. |+| `reason` | Why the difference exists, with file:line references. |+| `permanent` | `true` if the difference is by design and will not be fixed. |+| `owner_fix_side` | Which side should change: `python`, `go`, or `tbd`. |+| `tracking` | Issue link. Required unless `permanent` is true. |++Field comparison is deliberately asymmetric:++- a key Python emits that Go does not is **always** a failure — Go dropped data;+- a key Go emits that Python does not must be declared here, so every Go-only+ field is justified rather than assumed harmless.++The allowed set is derived per case rather than hardcoded, because it is not+constant across components: `TokenChunker` emits `ck_type`/`tk_nums`/`id` while+`TitleChunker` emits only `id`.++## No CGO required++The parity test is unit tier (no build tag) and passes under `CGO_ENABLED=0`.+The chunker's only native-backed file is `pdfcrop_cgo.go`, whose+`cropImageChunks` returns chunks untouched when the PDF engine is nil — which is+always the case here, since the harness passes no `*gorm.DB` and so no engine is+ever resolved. Skipping CGO therefore costs no coverage; the one field it cannot+populate is `ChunkDoc.Image`, a base64 preview.++Python has the matching seam: `restore_pdf_text_previews`+(`token_chunker.py:370`) runs just before `_finalize_json_chunks`, so image work+sits at the very end on both sides and the text-splitting logic under test+completes before it.diff --git a/internal/ingestion/component/chunker/testdata/parity/_manifest.json b/internal/ingestion/component/chunker/testdata/parity/_manifest.jsonnew file mode 100644index 0000000000..179fd095ed--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/_manifest.json@@ -0,0 +1,6 @@+{+ "captured_at": "2026-08-05T06:03:59+00:00",+ "command": "python internal/ingestion/component/chunker/tool-py/capture_golden.py --all",+ "commit": "5f0d164d69fd7e2c6aca3994a2597bea33cd3fe9",+ "python": "3.13.12"+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/title__group_simple.json b/internal/ingestion/component/chunker/testdata/parity/cases/title__group_simple.jsonnew file mode 100644index 0000000000..9c5b9f923c--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/title__group_simple.json@@ -0,0 +1,18 @@+{+ "id": "title__group_simple",+ "component": "TitleChunker",+ "notes": "Baseline group mode: two '#' headings collapse into one group. Also locks the trailing newline that joinGroupText appends.",+ "param": {+ "method": "group",+ "levels": [+ [+ "#"+ ]+ ]+ },+ "input": {+ "name": "t.md",+ "output_format": "markdown",+ "markdown": "# Chapter One\nAlpha text.\n# Chapter Two\nBeta text."+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/title__hierarchy_2level.json b/internal/ingestion/component/chunker/testdata/parity/cases/title__hierarchy_2level.jsonnew file mode 100644index 0000000000..be32a14ce4--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/title__hierarchy_2level.json@@ -0,0 +1,22 @@+{+ "id": "title__hierarchy_2level",+ "component": "TitleChunker",+ "notes": "Baseline hierarchy mode: two levels ('#' then '##') split into one chunk per heading.",+ "param": {+ "method": "hierarchy",+ "levels": [+ [+ "#"+ ],+ [+ "##"+ ]+ ],+ "hierarchy": 2+ },+ "input": {+ "name": "t.md",+ "output_format": "markdown",+ "markdown": "# A\nbody a\n## A1\nbody a1"+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__b1_count_sensitive.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__b1_count_sensitive.jsonnew file mode 100644index 0000000000..8ff3f3cb66--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__b1_count_sensitive.json@@ -0,0 +1,16 @@+{+ "id": "token__b1_count_sensitive",+ "component": "TokenChunker",+ "notes": "Tokenizer-liveness guard for PR #17712. A single long paragraph with no delimiter and a small chunk_token_size, so correct token counting splits it into many chunks by the token budget. If the Go tokenizer ever returns 0 (the offline bug), the budget is never exceeded and the paragraph collapses into a single chunk, which diverges from the multi-chunk Python baseline.",+ "param": {+ "chunk_token_size": 50,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "b1.txt",+ "output_format": "text",+ "text": "RAGFlow is a retrieval augmented generation engine that ingests heterogeneous documents and slices them into retrievable passages. The chunker must respect a token budget so that each passage fits the context window of the downstream language model without truncation. Token counting is the linchpin of this budget because every merge decision reads the per segment token length. When the encoder is missing the chunker silently reports zero tokens for every string and the budget is never exceeded so an entire document collapses into one oversized chunk. That degenerate behaviour is invisible to a test that only checks for a non empty result because zero is a number like any other. The only way to notice is to compare the actual chunk count against a reference implementation that counts tokens correctly. This case pins that comparison with a long single paragraph and a deliberately small budget so the correct count yields many chunks and a dead encoder yields exactly one. Retrieval quality depends on passages being coherent and appropriately sized so the guard is not merely cosmetic but protects the core promise of the system."+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__empty_input.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__empty_input.jsonnew file mode 100644index 0000000000..8d50bdff9a--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__empty_input.json@@ -0,0 +1,16 @@+{+ "id": "token__empty_input",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "empty.json",+ "output_format": "json",+ "json": []+ },+ "notes": "Empty json list must yield zero chunks on both sides."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__html_backtick.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__html_backtick.jsonnew file mode 100644index 0000000000..1811d8f14b--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__html_backtick.json@@ -0,0 +1,16 @@+{+ "id": "token__html_backtick",+ "component": "TokenChunker",+ "notes": "Custom-delimiter branch on the html path.",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "`\n`"+ ]+ },+ "input": {+ "name": "t",+ "output_format": "html",+ "html": "<p>one</p>\n<p>two</p>\n<p>three</p>"+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__html_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__html_default_delim.jsonnew file mode 100644index 0000000000..53b8ccc0c2--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__html_default_delim.json@@ -0,0 +1,16 @@+{+ "id": "token__html_default_delim",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "default.html",+ "output_format": "html",+ "html": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ },+ "notes": "Default delimiter ['\\n'] -> Python naive_merge('') -> 1 chunk (PY-BUG). Corpus has many '\\n' boundaries so Go segments and merges by token size into many chunks. Also a tokenizer-liveness guard: a dead tokenizer (0 tokens) collapses all segments into 1 chunk."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__html_long.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__html_long.jsonnew file mode 100644index 0000000000..282d6553e8--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__html_long.json@@ -0,0 +1,16 @@+{+ "id": "token__html_long",+ "component": "TokenChunker",+ "notes": "html counterpart of token__text_long_paragraph.",+ "param": {+ "chunk_token_size": 32,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "t",+ "output_format": "html",+ "html": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_backtick.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_backtick.jsonnew file mode 100644index 0000000000..5af494fc9f--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_backtick.json@@ -0,0 +1,25 @@+{+ "id": "token__json_backtick",+ "component": "TokenChunker",+ "notes": "Backtick-wrapped newline is the only delimiter form Python honours (_compile_delimiter_pattern, token_chunker.py:70-76). Locks the custom-delimiter branch on the json path, where chunk_token_size is bypassed and each segment becomes its own chunk.",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "`\n`"+ ]+ },+ "input": {+ "name": "t",+ "output_format": "json",+ "json": [+ {+ "text": "first segment line one\nfirst segment line two",+ "doc_type_kwd": "text"+ },+ {+ "text": "second segment line one\nsecond segment line two",+ "doc_type_kwd": "text"+ }+ ]+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_children_delim.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_children_delim.jsonnew file mode 100644index 0000000000..8bce5921d6--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_children_delim.json@@ -0,0 +1,23 @@+{+ "id": "token__json_children_delim",+ "component": "TokenChunker",+ "param": {+ "delimiters": [+ "`\n`"+ ],+ "children_delimiters": [+ "。"+ ]+ },+ "input": {+ "name": "ch.json",+ "output_format": "json",+ "json": [+ {+ "text": "第一句内容。第二句内容。第三句内容。",+ "doc_type_kwd": "text"+ }+ ]+ },+ "notes": "children_delimiters custom pattern secondary split."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_img_id.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_img_id.jsonnew file mode 100644index 0000000000..c575f364a8--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_img_id.json@@ -0,0 +1,26 @@+{+ "id": "token__json_img_id",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "img.json",+ "output_format": "json",+ "json": [+ {+ "text": "Text before the image.",+ "doc_type_kwd": "text"+ },+ {+ "text": "an image",+ "doc_type_kwd": "image",+ "img_id": "img-abc-123"+ }+ ]+ },+ "notes": "Image chunk must carry img_id through to output."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_multi_item_40.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_multi_item_40.jsonnew file mode 100644index 0000000000..92475a8ba0--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_multi_item_40.json@@ -0,0 +1,177 @@+{+ "id": "token__json_multi_item_40",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "multi.json",+ "output_format": "json",+ "json": [+ {+ "text": "Section 1. RAGFlow chunks documents for retrieval. This is sentence two of item 1.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 2. RAGFlow chunks documents for retrieval. This is sentence two of item 2.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 3. RAGFlow chunks documents for retrieval. This is sentence two of item 3.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 4. RAGFlow chunks documents for retrieval. This is sentence two of item 4.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 5. RAGFlow chunks documents for retrieval. This is sentence two of item 5.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 6. RAGFlow chunks documents for retrieval. This is sentence two of item 6.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 7. RAGFlow chunks documents for retrieval. This is sentence two of item 7.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 8. RAGFlow chunks documents for retrieval. This is sentence two of item 8.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 9. RAGFlow chunks documents for retrieval. This is sentence two of item 9.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 10. RAGFlow chunks documents for retrieval. This is sentence two of item 10.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 11. RAGFlow chunks documents for retrieval. This is sentence two of item 11.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 12. RAGFlow chunks documents for retrieval. This is sentence two of item 12.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 13. RAGFlow chunks documents for retrieval. This is sentence two of item 13.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 14. RAGFlow chunks documents for retrieval. This is sentence two of item 14.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 15. RAGFlow chunks documents for retrieval. This is sentence two of item 15.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 16. RAGFlow chunks documents for retrieval. This is sentence two of item 16.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 17. RAGFlow chunks documents for retrieval. This is sentence two of item 17.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 18. RAGFlow chunks documents for retrieval. This is sentence two of item 18.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 19. RAGFlow chunks documents for retrieval. This is sentence two of item 19.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 20. RAGFlow chunks documents for retrieval. This is sentence two of item 20.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 21. RAGFlow chunks documents for retrieval. This is sentence two of item 21.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 22. RAGFlow chunks documents for retrieval. This is sentence two of item 22.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 23. RAGFlow chunks documents for retrieval. This is sentence two of item 23.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 24. RAGFlow chunks documents for retrieval. This is sentence two of item 24.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 25. RAGFlow chunks documents for retrieval. This is sentence two of item 25.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 26. RAGFlow chunks documents for retrieval. This is sentence two of item 26.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 27. RAGFlow chunks documents for retrieval. This is sentence two of item 27.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 28. RAGFlow chunks documents for retrieval. This is sentence two of item 28.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 29. RAGFlow chunks documents for retrieval. This is sentence two of item 29.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 30. RAGFlow chunks documents for retrieval. This is sentence two of item 30.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 31. RAGFlow chunks documents for retrieval. This is sentence two of item 31.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 32. RAGFlow chunks documents for retrieval. This is sentence two of item 32.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 33. RAGFlow chunks documents for retrieval. This is sentence two of item 33.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 34. RAGFlow chunks documents for retrieval. This is sentence two of item 34.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 35. RAGFlow chunks documents for retrieval. This is sentence two of item 35.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 36. RAGFlow chunks documents for retrieval. This is sentence two of item 36.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 37. RAGFlow chunks documents for retrieval. This is sentence two of item 37.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 38. RAGFlow chunks documents for retrieval. This is sentence two of item 38.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 39. RAGFlow chunks documents for retrieval. This is sentence two of item 39.",+ "doc_type_kwd": "text"+ },+ {+ "text": "Section 40. RAGFlow chunks documents for retrieval. This is sentence two of item 40.",+ "doc_type_kwd": "text"+ }+ ]+ },+ "notes": "Lock global vs per-item merge: Python merges across items, Go merges per item."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_overlap.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_overlap.jsonnew file mode 100644index 0000000000..2d1b6147cd--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_overlap.json@@ -0,0 +1,22 @@+{+ "id": "token__json_overlap",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "\n"+ ],+ "overlapped_percent": 0.1+ },+ "input": {+ "name": "ov.json",+ "output_format": "json",+ "json": [+ {+ "text": "RAGFlow chunks documents for retrieval.\nEach chunk fits a token budget.\nOverlap preserves context across boundaries.\nThe API exposes dataset and chat endpoints.\nA web UI manages knowledge bases.\nBatch ingestion processes large corpora.",+ "doc_type_kwd": "text"+ }+ ]+ },+ "notes": "Overlapped_percent=10 on json path; both sides should apply overlap."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_single.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_single.jsonnew file mode 100644index 0000000000..d8afb7e8b0--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_single.json@@ -0,0 +1,18 @@+{+ "id": "token__json_single",+ "component": "TokenChunker",+ "notes": "Baseline: a single structured JSON item shorter than chunk_token_size passes through untouched. Locks the no-op path so harness bugs are distinguishable from real divergence.",+ "param": {+ "chunk_token_size": 128+ },+ "input": {+ "name": "t",+ "output_format": "json",+ "json": [+ {+ "text": "hello world",+ "doc_type_kwd": "text"+ }+ ]+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_single_long.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_single_long.jsonnew file mode 100644index 0000000000..a901d91ca4--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_single_long.json@@ -0,0 +1,18 @@+{+ "id": "token__json_single_long",+ "component": "TokenChunker",+ "notes": "One json item far over chunk_token_size=32 must be sub-split by token budget. Single item, so the per-item merge divergence cannot interfere — this isolates the oversize split itself.",+ "param": {+ "chunk_token_size": 32+ },+ "input": {+ "name": "t",+ "output_format": "json",+ "json": [+ {+ "text": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega second alpha beta gamma delta epsilon zeta eta theta iota kappa",+ "doc_type_kwd": "text"+ }+ ]+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_table_context.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_table_context.jsonnew file mode 100644index 0000000000..9039730ee2--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_table_context.json@@ -0,0 +1,22 @@+{+ "id": "token__json_table_context",+ "component": "TokenChunker",+ "notes": "A table item followed by a text item: locks whether the neighbouring text is folded into the table chunk or kept separate.",+ "param": {+ "chunk_token_size": 128+ },+ "input": {+ "name": "t",+ "output_format": "json",+ "json": [+ {+ "text": "A table caption.",+ "doc_type_kwd": "table"+ },+ {+ "text": "The surrounding paragraph that gives the table its context and should stay near it.",+ "doc_type_kwd": "text"+ }+ ]+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__json_two_item.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_two_item.jsonnew file mode 100644index 0000000000..a3f30d523b--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__json_two_item.json@@ -0,0 +1,22 @@+{+ "id": "token__json_two_item",+ "component": "TokenChunker",+ "notes": "Two json text items whose combined tokens still fit chunk_token_size. Minimal reproduction of the per-item vs global merge divergence (same class as json_multi_item_40, but small enough to read).",+ "param": {+ "chunk_token_size": 128+ },+ "input": {+ "name": "t",+ "output_format": "json",+ "json": [+ {+ "text": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega",+ "doc_type_kwd": "text"+ },+ {+ "text": "another long item with many words that should be merged when the token budget allows merging across items on the python side but stays separate on the go side",+ "doc_type_kwd": "text"+ }+ ]+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_backtick.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_backtick.jsonnew file mode 100644index 0000000000..7477db000c--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_backtick.json@@ -0,0 +1,16 @@+{+ "id": "token__markdown_backtick",+ "component": "TokenChunker",+ "notes": "Custom-delimiter branch on the markdown path, including blank lines so empty segments are exercised.",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "`\n`"+ ]+ },+ "input": {+ "name": "t",+ "output_format": "markdown",+ "markdown": "# Title\n\nParagraph one.\n\nParagraph two."+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_default_delim.jsonnew file mode 100644index 0000000000..0977b71479--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_default_delim.json@@ -0,0 +1,16 @@+{+ "id": "token__markdown_default_delim",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "default.markdown",+ "output_format": "markdown",+ "markdown": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ },+ "notes": "Default delimiter ['\\n'] -> Python naive_merge('') -> 1 chunk (PY-BUG). Corpus has many '\\n' boundaries so Go segments and merges by token size into many chunks. Also a tokenizer-liveness guard: a dead tokenizer (0 tokens) collapses all segments into 1 chunk."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_long.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_long.jsonnew file mode 100644index 0000000000..d8c3c3a8f4--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__markdown_long.json@@ -0,0 +1,16 @@+{+ "id": "token__markdown_long",+ "component": "TokenChunker",+ "notes": "markdown counterpart of token__text_long_paragraph.",+ "param": {+ "chunk_token_size": 32,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "t",+ "output_format": "markdown",+ "markdown": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__text_backtick.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_backtick.jsonnew file mode 100644index 0000000000..dc32f689ac--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_backtick.json@@ -0,0 +1,16 @@+{+ "id": "token__text_backtick",+ "component": "TokenChunker",+ "notes": "Backtick-wrapped newline on the text path takes the _split_text_by_pattern branch, which ignores chunk_token_size entirely. Complements token__text_default_delim, which takes the bare-delimiter branch.",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "`\n`"+ ]+ },+ "input": {+ "name": "t",+ "output_format": "text",+ "text": "first sentence here\nsecond sentence here\nthird sentence here"+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__text_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_default_delim.jsonnew file mode 100644index 0000000000..4e5e51ce79--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_default_delim.json@@ -0,0 +1,16 @@+{+ "id": "token__text_default_delim",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 128,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "default.text",+ "output_format": "text",+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ },+ "notes": "Default delimiter ['\\n']: both sides segment on '\\n' then merge by token size into 3 chunks, compared strictly (no ratchet rule). Corpus has many '\\n' boundaries. Also a tokenizer-liveness guard: a dead tokenizer (0 tokens) collapses all segments into 1 chunk, which this case would expose as a count mismatch."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__text_long_paragraph.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_long_paragraph.jsonnew file mode 100644index 0000000000..15794f6dda--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_long_paragraph.json@@ -0,0 +1,16 @@+{+ "id": "token__text_long_paragraph",+ "component": "TokenChunker",+ "notes": "One unbroken line with no newline at all, so neither side can use a delimiter and both must fall back to whitespace-atom sub-splitting. Isolates _split_oversized_unit from the delimiter divergence.",+ "param": {+ "chunk_token_size": 32,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "t",+ "output_format": "text",+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word "+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__text_overlap.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_overlap.jsonnew file mode 100644index 0000000000..c89179be71--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_overlap.json@@ -0,0 +1,17 @@+{+ "id": "token__text_overlap",+ "component": "TokenChunker",+ "notes": "overlapped_percent on the text path: locks that the overlap carried into each following chunk is byte-identical on both sides.",+ "param": {+ "chunk_token_size": 64,+ "delimiters": [+ "\n"+ ],+ "overlapped_percent": 0.1+ },+ "input": {+ "name": "t",+ "output_format": "text",+ "text": "alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \nbeta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta \ngamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma "+ }+}diff --git a/internal/ingestion/component/chunker/testdata/parity/cases/token__text_token_liveness.json b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_token_liveness.jsonnew file mode 100644index 0000000000..a43aab2d33--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/cases/token__text_token_liveness.json@@ -0,0 +1,16 @@+{+ "id": "token__text_token_liveness",+ "component": "TokenChunker",+ "param": {+ "chunk_token_size": 32,+ "delimiters": [+ "\n"+ ]+ },+ "input": {+ "name": "default.text",+ "output_format": "text",+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies.\nTemplates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ },+ "notes": "Default delimiter ['\\n'] -> Python naive_merge('') -> 1 chunk (PY-BUG). Corpus has many '\\n' boundaries so Go segments and merges by token size into many chunks. Also a tokenizer-liveness guard: a dead tokenizer (0 tokens) collapses all segments into 1 chunk."+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.jsonnew file mode 100644index 0000000000..0dd944fff8--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__html_backtick.json@@ -0,0 +1,17 @@+{+ "case_id": "token__html_backtick",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "\u003cp\u003eone\u003c/p\u003e"+ },+ {+ "doc_type_kwd": "text",+ "text": "\u003cp\u003etwo\u003c/p\u003e"+ },+ {+ "doc_type_kwd": "text",+ "text": "\u003cp\u003ethree\u003c/p\u003e"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.jsonnew file mode 100644index 0000000000..ba22d3c31f--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__markdown_backtick.json@@ -0,0 +1,17 @@+{+ "case_id": "token__markdown_backtick",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "# Title"+ },+ {+ "doc_type_kwd": "text",+ "text": "Paragraph one."+ },+ {+ "doc_type_kwd": "text",+ "text": "Paragraph two."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.json b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.jsonnew file mode 100644index 0000000000..0d30cc448f--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/go_snapshot/token__text_backtick.json@@ -0,0 +1,17 @@+{+ "case_id": "token__text_backtick",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "first sentence here"+ },+ {+ "doc_type_kwd": "text",+ "text": "second sentence here"+ },+ {+ "doc_type_kwd": "text",+ "text": "third sentence here"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/title__group_simple.json b/internal/ingestion/component/chunker/testdata/parity/golden/title__group_simple.jsonnew file mode 100644index 0000000000..a6ebab0499--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/title__group_simple.json@@ -0,0 +1,8 @@+{+ "case_id": "title__group_simple",+ "chunks": [+ {+ "text": "# Chapter One\nAlpha text.\n# Chapter Two\nBeta text.\n"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/title__hierarchy_2level.json b/internal/ingestion/component/chunker/testdata/parity/golden/title__hierarchy_2level.jsonnew file mode 100644index 0000000000..48690a0bf4--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/title__hierarchy_2level.json@@ -0,0 +1,11 @@+{+ "case_id": "title__hierarchy_2level",+ "chunks": [+ {+ "text": "# A\nbody a\n"+ },+ {+ "text": "## A1\nbody a1\n"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__b1_count_sensitive.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__b1_count_sensitive.jsonnew file mode 100644index 0000000000..94784b732a--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__b1_count_sensitive.json@@ -0,0 +1,8 @@+{+ "case_id": "token__b1_count_sensitive",+ "chunks": [+ {+ "text": "RAGFlow is a retrieval augmented generation engine that ingests heterogeneous documents and slices them into retrievable passages. The chunker must respect a token budget so that each passage fits the context window of the downstream language model without truncation. Token counting is the linchpin of this budget because every merge decision reads the per segment token length. When the encoder is missing the chunker silently reports zero tokens for every string and the budget is never exceeded so an entire document collapses into one oversized chunk. That degenerate behaviour is invisible to a test that only checks for a non empty result because zero is a number like any other. The only way to notice is to compare the actual chunk count against a reference implementation that counts tokens correctly. This case pins that comparison with a long single paragraph and a deliberately small budget so the correct count yields many chunks and a dead encoder yields exactly one. Retrieval quality depends on passages being coherent and appropriately sized so the guard is not merely cosmetic but protects the core promise of the system."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__empty_input.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__empty_input.jsonnew file mode 100644index 0000000000..9ca0e81459--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__empty_input.json@@ -0,0 +1,4 @@+{+ "case_id": "token__empty_input",+ "chunks": []+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__html_backtick.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__html_backtick.jsonnew file mode 100644index 0000000000..b8d8f868ca--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__html_backtick.json@@ -0,0 +1,14 @@+{+ "case_id": "token__html_backtick",+ "chunks": [+ {+ "text": "<p>one</p>"+ },+ {+ "text": "<p>two</p>"+ },+ {+ "text": "<p>three</p>"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__html_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__html_default_delim.jsonnew file mode 100644index 0000000000..79931bddb6--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__html_default_delim.json@@ -0,0 +1,14 @@+{+ "case_id": "token__html_default_delim",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost."+ },+ {+ "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies."+ },+ {+ "text": "Templates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__html_long.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__html_long.jsonnew file mode 100644index 0000000000..e9126dca7d--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__html_long.json@@ -0,0 +1,8 @@+{+ "case_id": "token__html_long",+ "chunks": [+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_backtick.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_backtick.jsonnew file mode 100644index 0000000000..b521ee4e77--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_backtick.json@@ -0,0 +1,21 @@+{+ "case_id": "token__json_backtick",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "first segment line one"+ },+ {+ "doc_type_kwd": "text",+ "text": "first segment line two"+ },+ {+ "doc_type_kwd": "text",+ "text": "second segment line one"+ },+ {+ "doc_type_kwd": "text",+ "text": "second segment line two"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_children_delim.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_children_delim.jsonnew file mode 100644index 0000000000..996b715b70--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_children_delim.json@@ -0,0 +1,20 @@+{+ "case_id": "token__json_children_delim",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "mom": "第一句内容。第二句内容。第三句内容。",+ "text": "第一句内容"+ },+ {+ "doc_type_kwd": "text",+ "mom": "第一句内容。第二句内容。第三句内容。",+ "text": "第二句内容"+ },+ {+ "doc_type_kwd": "text",+ "mom": "第一句内容。第二句内容。第三句内容。",+ "text": "第三句内容"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_img_id.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_img_id.jsonnew file mode 100644index 0000000000..7a711b4174--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_img_id.json@@ -0,0 +1,14 @@+{+ "case_id": "token__json_img_id",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "Text before the image."+ },+ {+ "doc_type_kwd": "image",+ "img_id": "img-abc-123",+ "text": "an image"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_multi_item_40.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_multi_item_40.jsonnew file mode 100644index 0000000000..8ba68fc67a--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_multi_item_40.json@@ -0,0 +1,29 @@+{+ "case_id": "token__json_multi_item_40",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "Section 1. RAGFlow chunks documents for retrieval. This is sentence two of item 1.\nSection 2. RAGFlow chunks documents for retrieval. This is sentence two of item 2.\nSection 3. RAGFlow chunks documents for retrieval. This is sentence two of item 3.\nSection 4. RAGFlow chunks documents for retrieval. This is sentence two of item 4.\nSection 5. RAGFlow chunks documents for retrieval. This is sentence two of item 5.\nSection 6. RAGFlow chunks documents for retrieval. This is sentence two of item 6.\nSection 7. RAGFlow chunks documents for retrieval. This is sentence two of item 7."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 8. RAGFlow chunks documents for retrieval. This is sentence two of item 8.\nSection 9. RAGFlow chunks documents for retrieval. This is sentence two of item 9.\nSection 10. RAGFlow chunks documents for retrieval. This is sentence two of item 10.\nSection 11. RAGFlow chunks documents for retrieval. This is sentence two of item 11.\nSection 12. RAGFlow chunks documents for retrieval. This is sentence two of item 12.\nSection 13. RAGFlow chunks documents for retrieval. This is sentence two of item 13.\nSection 14. RAGFlow chunks documents for retrieval. This is sentence two of item 14."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 15. RAGFlow chunks documents for retrieval. This is sentence two of item 15.\nSection 16. RAGFlow chunks documents for retrieval. This is sentence two of item 16.\nSection 17. RAGFlow chunks documents for retrieval. This is sentence two of item 17.\nSection 18. RAGFlow chunks documents for retrieval. This is sentence two of item 18.\nSection 19. RAGFlow chunks documents for retrieval. This is sentence two of item 19.\nSection 20. RAGFlow chunks documents for retrieval. This is sentence two of item 20.\nSection 21. RAGFlow chunks documents for retrieval. This is sentence two of item 21."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 22. RAGFlow chunks documents for retrieval. This is sentence two of item 22.\nSection 23. RAGFlow chunks documents for retrieval. This is sentence two of item 23.\nSection 24. RAGFlow chunks documents for retrieval. This is sentence two of item 24.\nSection 25. RAGFlow chunks documents for retrieval. This is sentence two of item 25.\nSection 26. RAGFlow chunks documents for retrieval. This is sentence two of item 26.\nSection 27. RAGFlow chunks documents for retrieval. This is sentence two of item 27.\nSection 28. RAGFlow chunks documents for retrieval. This is sentence two of item 28."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 29. RAGFlow chunks documents for retrieval. This is sentence two of item 29.\nSection 30. RAGFlow chunks documents for retrieval. This is sentence two of item 30.\nSection 31. RAGFlow chunks documents for retrieval. This is sentence two of item 31.\nSection 32. RAGFlow chunks documents for retrieval. This is sentence two of item 32.\nSection 33. RAGFlow chunks documents for retrieval. This is sentence two of item 33.\nSection 34. RAGFlow chunks documents for retrieval. This is sentence two of item 34.\nSection 35. RAGFlow chunks documents for retrieval. This is sentence two of item 35."+ },+ {+ "doc_type_kwd": "text",+ "text": "Section 36. RAGFlow chunks documents for retrieval. This is sentence two of item 36.\nSection 37. RAGFlow chunks documents for retrieval. This is sentence two of item 37.\nSection 38. RAGFlow chunks documents for retrieval. This is sentence two of item 38.\nSection 39. RAGFlow chunks documents for retrieval. This is sentence two of item 39.\nSection 40. RAGFlow chunks documents for retrieval. This is sentence two of item 40."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_overlap.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_overlap.jsonnew file mode 100644index 0000000000..c652ec0d47--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_overlap.json@@ -0,0 +1,9 @@+{+ "case_id": "token__json_overlap",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "RAGFlow chunks documents for retrieval.\nEach chunk fits a token budget.\nOverlap preserves context across boundaries.\nThe API exposes dataset and chat endpoints.\nA web UI manages knowledge bases.\nBatch ingestion processes large corpora."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_single.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_single.jsonnew file mode 100644index 0000000000..e21e8fb0a2--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_single.json@@ -0,0 +1,9 @@+{+ "case_id": "token__json_single",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "hello world"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_single_long.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_single_long.jsonnew file mode 100644index 0000000000..410bee212f--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_single_long.json@@ -0,0 +1,9 @@+{+ "case_id": "token__json_single_long",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega second alpha beta gamma delta epsilon zeta eta theta iota kappa"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_table_context.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_table_context.jsonnew file mode 100644index 0000000000..373bd676dc--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_table_context.json@@ -0,0 +1,13 @@+{+ "case_id": "token__json_table_context",+ "chunks": [+ {+ "doc_type_kwd": "table",+ "text": "A table caption."+ },+ {+ "doc_type_kwd": "text",+ "text": "The surrounding paragraph that gives the table its context and should stay near it."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__json_two_item.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_two_item.jsonnew file mode 100644index 0000000000..36940743ec--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__json_two_item.json@@ -0,0 +1,9 @@+{+ "case_id": "token__json_two_item",+ "chunks": [+ {+ "doc_type_kwd": "text",+ "text": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega\nanother long item with many words that should be merged when the token budget allows merging across items on the python side but stays separate on the go side"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_backtick.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_backtick.jsonnew file mode 100644index 0000000000..c9334aaa96--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_backtick.json@@ -0,0 +1,14 @@+{+ "case_id": "token__markdown_backtick",+ "chunks": [+ {+ "text": "# Title"+ },+ {+ "text": "Paragraph one."+ },+ {+ "text": "Paragraph two."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_default_delim.jsonnew file mode 100644index 0000000000..51a1448986--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_default_delim.json@@ -0,0 +1,14 @@+{+ "case_id": "token__markdown_default_delim",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost."+ },+ {+ "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies."+ },+ {+ "text": "Templates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_long.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_long.jsonnew file mode 100644index 0000000000..896f9c7621--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__markdown_long.json@@ -0,0 +1,8 @@+{+ "case_id": "token__markdown_long",+ "chunks": [+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__text_backtick.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_backtick.jsonnew file mode 100644index 0000000000..bf1187b5ed--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_backtick.json@@ -0,0 +1,14 @@+{+ "case_id": "token__text_backtick",+ "chunks": [+ {+ "text": "first sentence here"+ },+ {+ "text": "second sentence here"+ },+ {+ "text": "third sentence here"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__text_default_delim.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_default_delim.jsonnew file mode 100644index 0000000000..fae6e82ed4--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_default_delim.json@@ -0,0 +1,14 @@+{+ "case_id": "token__text_default_delim",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval.\nThe chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries.\nRetrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance.\nUsers tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost."+ },+ {+ "text": "Larger chunks keep more context but may dilute relevance.\nThe system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text.\nThe API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service.\nCaching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies."+ },+ {+ "text": "Templates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base.\nPlugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__text_long_paragraph.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_long_paragraph.jsonnew file mode 100644index 0000000000..cab49e714f--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_long_paragraph.json@@ -0,0 +1,8 @@+{+ "case_id": "token__text_long_paragraph",+ "chunks": [+ {+ "text": "word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__text_overlap.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_overlap.jsonnew file mode 100644index 0000000000..b44b005087--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_overlap.json@@ -0,0 +1,11 @@+{+ "case_id": "token__text_overlap",+ "chunks": [+ {+ "text": "alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha \nbeta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta beta"+ },+ {+ "text": "beta beta beta beta beta beta beta beta beta \ngamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma gamma"+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/golden/token__text_token_liveness.json b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_token_liveness.jsonnew file mode 100644index 0000000000..23d67c8199--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/golden/token__text_token_liveness.json@@ -0,0 +1,32 @@+{+ "case_id": "token__text_token_liveness",+ "chunks": [+ {+ "text": "RAGFlow is an open-source retrieval-augmented generation engine.\nIt ingests documents of many types and splits them into chunks.\nEach chunk is embedded and stored in a vector database for retrieval."+ },+ {+ "text": "The chunker honours document structure such as headings and tables.\nLong paragraphs are divided so each chunk fits a token budget.\nOverlap between chunks preserves context across boundaries."+ },+ {+ "text": "Retrieval returns the most relevant chunks for a user question.\nThe generator then composes an answer from those chunks.\nEvaluation measures faithfulness and answer relevance."+ },+ {+ "text": "Users tune chunk size to balance recall and precision.\nSmaller chunks improve recall but increase storage cost.\nLarger chunks keep more context but may dilute relevance."+ },+ {+ "text": "The system supports many languages and encodings.\nDeep parsing extracts text from PDF images and scans.\nTables are recognised and preserved as structured chunks.\nImages can be described and attached to neighbouring text."+ },+ {+ "text": "The API exposes dataset, document, and chat endpoints.\nA web UI lets users manage knowledge bases visually.\nBatch ingestion processes large corpora efficiently.\nConcurrency limits protect the embedding service."+ },+ {+ "text": "Caching avoids re-embedding unchanged content.\nLogging records ingestion progress and errors.\nMetrics help operators spot slow components.\nConfiguration controls parsers and chunking strategies."+ },+ {+ "text": "Templates encode common ingestion pipelines.\nThe engine scales horizontally behind a load balancer.\nSecurity isolates tenant data by document id.\nAuditing tracks who accessed which knowledge base."+ },+ {+ "text": "Plugins extend parsing for proprietary formats."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/testdata/parity/known_diffs.json b/internal/ingestion/component/chunker/testdata/parity/known_diffs.jsonnew file mode 100644index 0000000000..839edf6b14--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/parity/known_diffs.json@@ -0,0 +1,32 @@+{+ "version": 1,+ "rules": [+ {+ "id": "go-only-chunk-id",+ "tag": "go_intentional",+ "kind": "extra_fields",+ "applies_to": ["*"],+ "fields": ["id"],+ "permanent": true,+ "reason": "register.go MustRegisterChunker wraps every chunker in imageUploadDecorator, which writes ck[\"id\"] = common.ChunkID(doc_id, text) before any image upload so uploadChunkImage and the persist stage share one chunk identity. Python derives chunk ids outside the chunker, so its _finalize_json_chunks output has no id."+ },+ {+ "id": "go-only-token-metadata",+ "tag": "go_intentional",+ "kind": "extra_fields",+ "applies_to": ["token__*"],+ "fields": ["ck_type", "tk_nums"],+ "permanent": true,+ "reason": "Python builds both fields internally (token_chunker.py:141,144) but _finalize_json_chunks (token_chunker.py:256-259) whitelists only text/doc_type_kwd/_pdf_positions/mom/img_id, dropping them. Go keeps them because they are still live downstream: needsCrop (pdfcrop_cgo.go:151) reads ck_type, and tokenizer.go:615 forwards tk_nums. The divergence follows from Go cropping after the chunker returns, whereas Python crops inside _invoke (token_chunker.py:370) before finalizing."+ },+ {+ "id": "token-custom-delim-newline-and-field",+ "tag": "go_bug",+ "kind": "chunk_text",+ "applies_to": ["token__text_backtick", "token__markdown_backtick", "token__html_backtick"],+ "owner_fix_side": "go",+ "tracking": "TODO: file issue — Go emits an extra doc_type_kwd:'text' field on the text/markdown/html backtick path where Python emits only {'text': ...}",+ "reason": "With a backtick-wrapped delimiter (e.g. `\\n`, _compile_delimiter_pattern, token_chunker.py:70-76) Python takes the custom-pattern branch. On that branch Python's _split_text_by_pattern (rag/flow/chunker/token_chunker.py:80-95) discards the delimiter, and the text path emits a list of {'text': ...} with no doc_type_kwd (token_chunker.py:341). Go no longer keeps the delimiter either: since #17868 it uses splitDroppingDelim, so chunk text matches Python exactly. The remaining Go-only behaviour is an extra doc_type_kwd:'text' field — invokeTextPayload (token.go:323, 338-341) defaults every chunk to DocType:'text', which Python's text-path output never includes."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/tool-py/capture_golden.py b/internal/ingestion/component/chunker/tool-py/capture_golden.pynew file mode 100644index 0000000000..8d4a0e9c79--- /dev/null+++ b/internal/ingestion/component/chunker/tool-py/capture_golden.py@@ -0,0 +1,261 @@+#!/usr/bin/env python3+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Capture Python chunker outputs as golden fixtures for the Go parity test.++Python is the reference implementation; this script records what it actually+produces so `golden_parity_test.go` can assert the Go port reproduces it.++Usage (from the repository root):++ source .venv/bin/activate+ export PYTHONPATH=$(pwd)+ python internal/ingestion/component/chunker/tool-py/capture_golden.py --all++Why components are built with ``__new__`` instead of the normal constructor:+``ComponentBase.__init__`` asserts its ``canvas`` argument is a real ``Graph``,+which a fixture harness has no way to supply. Bypassing ``__init__`` and+attaching the three attributes ``_invoke`` actually reads (``_canvas``,+``_param``, ``callback``) runs the genuine chunking code path with the real+``num_tokens_from_string`` and the real ``naive_merge``.++Do NOT swap in the module stubs from ``rag/flow/tests/test_token_chunker.py``:+that file replaces ``num_tokens_from_string`` with ``lambda text: 1`` and+``naive_merge`` with a function returning ``[]``. Golden values captured under+those stubs would be systematically wrong.++``num_tokens_from_string`` swallows every exception and returns 0+(``common/token_utils.py``), so an unavailable tiktoken encoding silently turns+every token budget into "never exceeded" and collapses each case into a single+chunk. That failure looks like a legitimate baseline, so ``_assert_tokenizer_alive``+runs first and aborts the capture rather than writing a poisoned golden.+"""++from __future__ import annotations++import argparse+import asyncio+import json+import subprocess+import sys+import types+from datetime import datetime, timezone+from pathlib import Path++TOOL_DIR = Path(__file__).resolve().parent+PARITY_DIR = TOOL_DIR.parent / "testdata" / "parity"+CASES_DIR = PARITY_DIR / "cases"+GOLDEN_DIR = PARITY_DIR / "golden"+MANIFEST = PARITY_DIR / "_manifest.json"+REPO_ROOT = TOOL_DIR.parents[4]++CAPTURE_CMD = "python internal/ingestion/component/chunker/tool-py/capture_golden.py --all"+++def _build_component(component: str, param: dict):+ """Instantiate a chunker while reproducing every effect of the real constructor.++ ``__new__`` skips ``ComponentBase.__init__`` (agent/component/base.py:375),+ which is the point — that constructor asserts its canvas is a real ``Graph``.+ But skipping it means its three effects must be reproduced by hand, not just+ the ones needed to make the code run:++ 1. ``self._canvas = canvas`` — a real ``Graph`` is unobtainable in a+ fixture harness, but ``None`` is not a safe stand-in: PDF cases read+ ``canvas._doc_id`` (pdf_chunk_metadata.py:190) and would ``AttributeError``.+ A throwaway ``types.SimpleNamespace`` with the two attributes that matter+ is enough to keep those code paths alive without touching a real canvas.+ 2. ``self._param = param``+ 3. ``self._param.check()`` — the parameter validation. ``__new__`` bypasses+ the constructor, so without an explicit ``check()`` Python will accept+ values it would reject in production and emit a baseline for a+ configuration that cannot occur, which would then be used to judge Go. A+ baseline must only ever record legal behaviour.+ """+ if component == "TokenChunker":+ from rag.flow.chunker.token_chunker import TokenChunker, TokenChunkerParam++ cls, param_cls = TokenChunker, TokenChunkerParam+ elif component in ("TitleChunker", "GroupTitleChunker", "HierarchyTitleChunker"):+ from rag.flow.chunker.title_chunker.common import TitleChunkerParam+ from rag.flow.chunker.title_chunker.title_chunker import TitleChunker++ cls, param_cls = TitleChunker, TitleChunkerParam+ else:+ raise ValueError(f"unsupported component {component!r}")++ p = param_cls()+ # GroupTitleChunker / HierarchyTitleChunker are distinct registry names on+ # the Go side but a single Python class dispatching on `method`. Derive the+ # method so one case file drives both sides unchanged.+ if component == "GroupTitleChunker":+ p.method = "group"+ elif component == "HierarchyTitleChunker":+ p.method = "hierarchy"+ for key, value in param.items():+ setattr(p, key, value)+ # __new__ bypassed ComponentBase.__init__, which would normally have run+ # _param.check(); reproduce that validation so illegal params are rejected+ # here too instead of being baked into the golden baseline.+ p.check()++ comp = cls.__new__(cls)+ # None would crash PDF cases that read canvas._doc_id; a minimal namespace+ # keeps those paths alive without a real Graph.+ comp._canvas = types.SimpleNamespace(_doc_id=None, _tenant_id="t")+ comp._param = p+ comp.callback = lambda *_args, **_kwargs: None+ return comp+++def _assert_tokenizer_alive() -> None:+ """Abort unless the real tiktoken encoder is loaded and counting.++ Every token-budget decision in ``naive_merge`` reads+ ``num_tokens_from_string``. When that returns 0 the budget is never+ exceeded and every case degenerates to one chunk -- a plausible-looking+ baseline that is entirely an artefact of the capture environment.+ """+ from rag.nlp import num_tokens_from_string++ probe = "RAGFlow chunker parity tokenizer liveness probe."+ got = num_tokens_from_string(probe)+ if got <= 0:+ sys.exit(+ "tokenizer is dead: num_tokens_from_string() returned "+ f"{got} for a {len(probe)}-character probe.\n"+ "common/token_utils.py returns 0 on any encoder failure, so capturing now "+ "would write goldens where no token budget is ever exceeded.\n"+ "Fix the encoding first: uv run python3 ragflow_deps/download_deps.py "+ "(provides ragflow_deps/cl100k_base.tiktoken)."+ )+++def run_case(case: dict) -> dict:+ """Run one case and return its recorded outputs."""+ comp = _build_component(case["component"], case.get("param", {}))+ asyncio.run(comp._invoke(**case["input"]))++ outputs = comp._param.outputs+ error = outputs.get("_ERROR", {}).get("value")+ if error is not None:+ return {"case_id": case["id"], "error": error}+ chunks = outputs.get("chunks", {}).get("value") or []+ return {"case_id": case["id"], "chunks": chunks}+++def serialize(result: dict) -> str:+ return json.dumps(result, ensure_ascii=False, sort_keys=True, indent=1) + "\n"+++def load_cases(only: str | None) -> list[dict]:+ paths = sorted(CASES_DIR.glob("*.json"))+ if not paths:+ sys.exit(f"no case files found under {CASES_DIR}")+ cases = []+ for path in paths:+ case = json.loads(path.read_text(encoding="utf-8"))+ if case["id"] != path.stem:+ sys.exit(f"{path}: case id {case['id']!r} does not match filename stem {path.stem!r}")+ if only is None or case["id"] == only:+ cases.append(case)+ if only is not None and not cases:+ sys.exit(f"no case with id {only!r} under {CASES_DIR}")+ return cases+++def write_manifest() -> None:+ """Record provenance in one shared file.++ Kept out of the per-case golden files so a re-capture does not touch every+ fixture's diff with a new timestamp.+ """+ try:+ commit = subprocess.run(+ ["git", "rev-parse", "HEAD"],+ cwd=REPO_ROOT,+ capture_output=True,+ text=True,+ check=True,+ ).stdout.strip()+ except (subprocess.CalledProcessError, FileNotFoundError):+ commit = "unknown"+ MANIFEST.write_text(+ json.dumps(+ {+ "captured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),+ "commit": commit,+ "python": sys.version.split()[0],+ "command": CAPTURE_CMD,+ },+ indent=1,+ sort_keys=True,+ )+ + "\n",+ encoding="utf-8",+ )+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ group = ap.add_mutually_exclusive_group(required=True)+ group.add_argument("--all", action="store_true", help="capture every case")+ group.add_argument("--case", metavar="ID", help="capture a single case")+ ap.add_argument(+ "--check",+ action="store_true",+ help="re-run cases and compare against the golden files on disk without writing; exit non-zero on drift",+ )+ args = ap.parse_args()++ _assert_tokenizer_alive()++ GOLDEN_DIR.mkdir(parents=True, exist_ok=True)+ cases = load_cases(args.case)++ drifted: list[str] = []+ for case in cases:+ first = serialize(run_case(case))+ # Determinism guard: a chunker that is not reproducible would otherwise+ # bake its nondeterminism into the baseline, and every later parity+ # failure would be unexplainable.+ second = serialize(run_case(case))+ if first != second:+ sys.exit(f"{case['id']}: non-deterministic output across two runs; refusing to write a golden file")++ target = GOLDEN_DIR / f"{case['id']}.json"+ if args.check:+ existing = target.read_text(encoding="utf-8") if target.exists() else None+ if existing != first:+ drifted.append(case["id"])+ print(f"DRIFT {case['id']}", file=sys.stderr)+ continue+ target.write_text(first, encoding="utf-8")+ print(f"wrote {target.relative_to(REPO_ROOT)}")++ if args.check:+ if drifted:+ print(f"\n{len(drifted)} golden file(s) stale; re-run: {CAPTURE_CMD}", file=sys.stderr)+ return 1+ print(f"{len(cases)} golden file(s) up to date")+ return 0++ write_manifest()+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/internal/ingestion/component/chunker/tool-py/live_chunk.py b/internal/ingestion/component/chunker/tool-py/live_chunk.pynew file mode 100644index 0000000000..8f0d2e492c--- /dev/null+++ b/internal/ingestion/component/chunker/tool-py/live_chunk.py@@ -0,0 +1,68 @@+#!/usr/bin/env python3+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Emit one chunker case's Python output as JSON, for live Go<->Python parity.++This is the live counterpart of ``capture_golden.py``. Instead of writing a+golden fixture to disk, it runs the Python reference chunker on a single case+and prints its chunks to stdout, so a Go test can invoke it as a subprocess and+diff the result against the Go port without any committed golden file.++Usage (invoked by live_parity_test.go, not by hand):++ .venv/bin/python tool-py/live_chunk.py <case_id>++It reuses ``capture_golden.run_case`` verbatim, so the emitted chunks have+exactly the same shape as the golden fixtures. ``_assert_tokenizer_alive`` runs+first and exits non-zero if the encoder is unavailable, so a dead tokenizer+produces a hard failure instead of a plausible-looking one-chunk baseline.++Tier: this script is only ever called from a ``//go:build manual`` Go test,+because it requires the ``.venv`` Python environment with the ragflow+dependencies and the cl100k table.+"""++from __future__ import annotations++import json+import sys+from pathlib import Path++TOOL_DIR = Path(__file__).resolve().parent+sys.path.insert(0, str(TOOL_DIR))++from capture_golden import CASES_DIR, _assert_tokenizer_alive, run_case+++def main() -> int:+ if len(sys.argv) < 2:+ sys.stderr.write("usage: live_chunk.py <case_id>\n")+ return 2+ case_id = sys.argv[1]+ path = CASES_DIR / f"{case_id}.json"+ if not path.exists():+ sys.stderr.write(f"no case file {path}\n")+ return 2++ case = json.loads(path.read_text(encoding="utf-8"))+ _assert_tokenizer_alive()+ result = run_case(case)+ sys.stdout.write(json.dumps(result, ensure_ascii=False, sort_keys=True, indent=1) + "\n")+ return 0+++if __name__ == "__main__":+ sys.exit(main())