RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(embedding): cut inputs to the model window on every path (#19997)
Zhichang Yuea93fc8304e6
diff --git a/internal/entity/models/embed_limit.go b/internal/entity/models/embed_limit.gonew file mode 100644index 0000000000..7ad7f560d6--- /dev/null+++ b/internal/entity/models/embed_limit.go@@ -0,0 +1,139 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package models++import (+ "context"+ "fmt"++ "ragflow/internal/common"+ "ragflow/internal/tokenizer"+)++// EmbedWithinLimit embeds req after cutting every text to what this model accepts,+// and retries with a smaller budget if the provider still rejects an input as over+// its window. It is the embedding counterpart of RerankModel.Rerank, which cuts its+// documents the same way for the same reason.+//+// Callers use it instead of ModelDriver.Embed because a provider does NOT truncate+// an over-window input: it rejects the whole request (SiliconFlow answers 400 with+// code 20015 for an input 0.55% over spec). The document-side callers - the+// dataset-nav summary, the knowledge-compiler products, chunk and memory writes -+// have no length bound of their own, so without this cut the request is sent as-is+// and, because such a batch is retried unchanged, fails forever. Python solves it+// one layer down in BaseEmbedding.encode; Go has no such layer, so the model+// carries it. Query-side callers keep calling the driver directly: their texts are+// short, and they must keep working on a deployment whose tokenizer assets are+// missing.+//+// Only over-limit rejections are retried, each one with a strictly smaller budget:+// a rate limit belongs to the driver's own retry policy and a 5xx is not something+// a shorter input fixes. When even the batch ladder is rejected, each input is+// embedded on its own down to the floor, so one pathological text neither fails the+// whole batch nor drags its healthy neighbours down to the floor with it. A model+// that declares a tokenizer whose asset is not on disk is refused before the first+// attempt, because counting it with the calibrated cl100k estimate is exactly what+// lets an oversized request through.+func (m *EmbeddingModel) EmbedWithinLimit(ctx context.Context, req EmbedRequest, embeddingConfig *EmbeddingConfig, usage *common.ModelUsage) ([]EmbeddingData, error) {+ if m == nil || m.ModelDriver == nil {+ return nil, fmt.Errorf("embedding model: driver is nil")+ }+ if len(req.Texts) == 0 {+ return []EmbeddingData{}, nil+ }+ tokenizerID, quotaKey := m.ResolveTokenizerID(), m.QuotaKey()+ if err := tokenizer.RefuseUnavailableCounter(tokenizerID, quotaKey); err != nil {+ return nil, err+ }+ limiter := tokenizer.LimiterFor(tokenizerID, quotaKey, tokenizer.DefaultCalibration())+ counter := limiter.Counter()+ cal, calKey := limiter.Calibration()+ // maxTokens is what the provider declares and window is the local budget derived+ // from it (calibrated ratio plus margin). The first rung cuts to window, so the+ // common case is a single call; a rejection is evidence about maxTokens, which is+ // what the calibration has to be told.+ maxTokens := m.ResolveMaxTokens()+ window := limiter.Limit(maxTokens)++ for _, budget := range tokenizer.OverLimitLadder(window) {+ embeds, err := m.embedCut(ctx, req, embeddingConfig, usage, counter, budget)+ if err == nil {+ return embeds, nil+ }+ if !tokenizer.IsOverLimitError(err) {+ return nil, err+ }+ // Our count said OwnTokenMax tokens for an input the provider just rejected:+ // record it, so the next call - and the next document - starts from a tighter+ // budget instead of re-learning the same thing.+ cal.ObserveOverLimit(calKey, tokenizer.OwnTokenMax(req.Texts, counter), maxTokens)+ }++ // The batch ladder could not fit the batch, which usually means one input is+ // pathological next to healthy ones. Isolate instead of failing: each input gets+ // its own walk down to OverLimitFloorTokens, and the healthy ones keep the budget+ // they had. A failure here is reported as isolation's own - it names the input and+ // the floor it reached, while the batch rejection above says only "the whole+ // request was too big".+ return m.embedIsolating(ctx, req, embeddingConfig, usage, counter, cal, calKey, maxTokens, window)+}++// embedCut trims every text to budget and performs exactly one embedding call.+func (m *EmbeddingModel) embedCut(ctx context.Context, req EmbedRequest, embeddingConfig *EmbeddingConfig, usage *common.ModelUsage, counter tokenizer.Counter, budget int) ([]EmbeddingData, error) {+ attempt := req+ attempt.Texts = make([]string, len(req.Texts))+ for i, text := range req.Texts {+ attempt.Texts[i] = counter.TrimToLimit(text, budget)+ }+ return m.ModelDriver.Embed(ctx, m.ModelName, attempt, m.APIConfig, embeddingConfig, usage)+}++// embedIsolating embeds each input of req on its own, walking the ladder to the+// floor for the ones that need it. It returns one embedding per input, in the+// original order, and never a partial result: an input that does not fit even at the+// floor fails the call.+func (m *EmbeddingModel) embedIsolating(ctx context.Context, req EmbedRequest, embeddingConfig *EmbeddingConfig, usage *common.ModelUsage, counter tokenizer.Counter, cal *tokenizer.Calibration, calKey string, maxTokens, window int) ([]EmbeddingData, error) {+ out := make([]EmbeddingData, len(req.Texts))+ for i, text := range req.Texts {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ single := req+ single.Texts = []string{text}+ var lastErr error+ fitted := false+ for _, budget := range tokenizer.OverLimitLadderToFloor(window) {+ embeds, err := m.embedCut(ctx, single, embeddingConfig, usage, counter, budget)+ if err == nil {+ if len(embeds) != 1 {+ return nil, fmt.Errorf("embedding input %d: driver returned %d embeddings for one text", i, len(embeds))+ }+ out[i], fitted = embeds[0], true+ break+ }+ if !tokenizer.IsOverLimitError(err) {+ return nil, err+ }+ cal.ObserveOverLimit(calKey, counter.Count(counter.TrimToLimit(text, budget)), maxTokens)+ lastErr = err+ }+ if !fitted {+ return nil, fmt.Errorf("embedding input %d does not fit the model window even at %d tokens: %w", i, tokenizer.OverLimitFloorTokens, lastErr)+ }+ }+ return out, nil+}diff --git a/internal/entity/models/embed_limit_test.go b/internal/entity/models/embed_limit_test.gonew file mode 100644index 0000000000..953c575631--- /dev/null+++ b/internal/entity/models/embed_limit_test.go@@ -0,0 +1,278 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package models++import (+ "context"+ "errors"+ "strconv"+ "strings"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/tokenizer"+)++// overWindowError is what SiliconFlow answers for an input over the model window:+// a 400 whose body carries code 20015.+func overWindowError() error {+ return errors.New(`SILICONFLOW API error: 400 Bad Request, body: {"code":20015,"message":"The parameter is invalid. Please check again.","data":null}`)+}++// recordingEmbedDriver implements only the Embed half of ModelDriver: it records the+// texts it was handed and can reject them the way an over-window provider does.+type recordingEmbedDriver struct {+ ModelDriver+ attempts [][]string+ // rejectFor rejects the first N attempts.+ rejectFor int+ // rejectBatch rejects any request carrying more than one text, which is how a+ // pathological input behaves: the batch ladder cannot fit the batch, while each+ // input on its own can be embedded.+ rejectBatch bool+ // rejectLongerThan rejects any request holding a text longer than that many+ // bytes, so a small text passes while a large one fails even after the trim.+ rejectLongerThan int+ failWith error+}++func (d *recordingEmbedDriver) Embed(_ context.Context, _ *string, request EmbedRequest, _ *APIConfig, _ *EmbeddingConfig, _ *common.ModelUsage) ([]EmbeddingData, error) {+ d.attempts = append(d.attempts, request.Texts)+ if d.rejectBatch && len(request.Texts) > 1 {+ return nil, overWindowError()+ }+ if d.rejectLongerThan > 0 {+ for _, text := range request.Texts {+ if len(text) > d.rejectLongerThan {+ return nil, overWindowError()+ }+ }+ }+ if len(d.attempts) <= d.rejectFor {+ return nil, overWindowError()+ }+ if d.failWith != nil {+ return nil, d.failWith+ }+ return make([]EmbeddingData, len(request.Texts)), nil+}++// newEmbedModel pins the inputs that decide the window and the counter, so a test+// never depends on the machine's catalog or environment: the tokenizer id and the+// window come from the env overrides, and the api key (the test's own name) gives the+// model its own quota/calibration key. That last part matters because the default+// Calibration is process-global and only ever ratchets up - one test's over-limit+// observation must not shrink another test's window.+func newEmbedModel(t *testing.T, driver ModelDriver, tokenizerID string, maxTokens int) *EmbeddingModel {+ t.Helper()+ t.Setenv("TOKENIZER_EMBEDDING_TOKENIZER", tokenizerID)+ t.Setenv("TOKENIZER_EMBEDDING_MAX_TOKENS", strconv.Itoa(maxTokens))+ apiKey := t.Name()+ return NewEmbeddingModel(driver, nil, &APIConfig{ApiKey: &apiKey}, maxTokens)+}++// embedBudget is the budget a single cut would use, computed the same way+// EmbedWithinLimit computes its first rung.+func embedBudget(t *testing.T, model *EmbeddingModel) (int, tokenizer.Counter) {+ t.Helper()+ limiter := tokenizer.LimiterFor(model.ResolveTokenizerID(), model.QuotaKey(), tokenizer.DefaultCalibration())+ return limiter.Limit(model.ResolveMaxTokens()), limiter.Counter()+}++// TestEmbedWithinLimitCutsToTheWindow pins what the dataset-nav, knowledge-compiler,+// chunk and memory callers rely on: what reaches the driver fits the window, order is+// preserved, a short text is untouched, and a first-attempt success is not retried.+//+// The empty tokenizer id is the calibrated path (a model whose tokenizer the catalog+// does not name); xlmr-spm is covered when its asset is present because it is the+// counter behind the reported failure (a nav summary against bge-m3, 400/20015).+func TestEmbedWithinLimitCutsToTheWindow(t *testing.T) {+ long := strings.Repeat("中文 hello ", 5000)+ for _, id := range []string{"", tokenizer.CounterXLMRSentence} {+ if id != "" && !tokenizer.CounterExact(id) {+ t.Logf("counter %s is unavailable in this environment; skipping it", id)+ continue+ }+ driver := &recordingEmbedDriver{}+ model := newEmbedModel(t, driver, id, 8192)+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{long, "short"}}, nil, nil)+ if err != nil {+ t.Fatalf("tokenizer %q: EmbedWithinLimit: %v", id, err)+ }+ if len(embeds) != 2 {+ t.Fatalf("tokenizer %q: got %d embeddings, want 2", id, len(embeds))+ }+ if len(driver.attempts) != 1 {+ t.Fatalf("tokenizer %q: %d attempts for an input the provider accepts", id, len(driver.attempts))+ }+ got := driver.attempts[0]+ if got[1] != "short" {+ t.Errorf("tokenizer %q: a text inside the window was modified: %q", id, got[1])+ }+ if got[0] == long {+ t.Errorf("tokenizer %q: the over-window text was not cut", id)+ }+ budget, counter := embedBudget(t, model)+ if n := counter.Count(got[0]); n > budget {+ t.Errorf("tokenizer %q: the text handed to the driver counts %d tokens, budget is %d", id, n, budget)+ }+ }+}++// TestEmbedWithinLimitRetriesWithASmallerBudget is the property that turns "this+// batch fails forever" into "this batch succeeds": the retried text must be strictly+// shorter, and the caller must not see the rejection.+func TestEmbedWithinLimitRetriesWithASmallerBudget(t *testing.T) {+ driver := &recordingEmbedDriver{rejectFor: 1}+ model := newEmbedModel(t, driver, "", 8192)+ if _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{strings.Repeat("中", 20000)}}, nil, nil); err != nil {+ t.Fatalf("EmbedWithinLimit: %v", err)+ }+ if len(driver.attempts) != 2 {+ t.Fatalf("attempts = %d, want 2", len(driver.attempts))+ }+ if len(driver.attempts[1][0]) >= len(driver.attempts[0][0]) {+ t.Fatalf("the retry did not shrink the input: %d then %d bytes", len(driver.attempts[0][0]), len(driver.attempts[1][0]))+ }+}++// TestEmbedWithinLimitDoesNotRetryUnrelatedErrors keeps the retry narrow: a 5xx is+// not something a shorter input fixes, so it comes back on the first attempt.+func TestEmbedWithinLimitDoesNotRetryUnrelatedErrors(t *testing.T) {+ boom := errors.New("SILICONFLOW API error: 503 Service Unavailable")+ driver := &recordingEmbedDriver{failWith: boom}+ model := newEmbedModel(t, driver, "", 8192)+ _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{"hello"}}, nil, nil)+ if !errors.Is(err, boom) {+ t.Fatalf("error = %v, want the provider's own error", err)+ }+ if len(driver.attempts) != 1 {+ t.Fatalf("attempts = %d, want 1", len(driver.attempts))+ }+}++// TestEmbedWithinLimitRefusesAnUnavailableDeclaredTokenizer is the half of the+// contract that keeps the retry from hiding a provisioning gap: a model that declares+// a tokenizer we cannot load fails before the first attempt.+func TestEmbedWithinLimitRefusesAnUnavailableDeclaredTokenizer(t *testing.T) {+ driver := &recordingEmbedDriver{}+ model := newEmbedModel(t, driver, "no-such-family", 8192)+ _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{"hello"}}, nil, nil)+ if err == nil {+ t.Fatal("expected an error for a declared tokenizer whose asset is unavailable")+ }+ for _, want := range []string{`"no-such-family"`, "but its asset is unavailable", "refusing to count with the calibrated estimate"} {+ if !strings.Contains(err.Error(), want) {+ t.Errorf("error %q does not mention %q", err.Error(), want)+ }+ }+ if len(driver.attempts) != 0 {+ t.Errorf("the driver was called %d times despite the refusal", len(driver.attempts))+ }+}++// TestEmbedWithinLimitIsolatesAPathologicalInput is the property that keeps one bad+// input from failing the whole batch: when no batch budget fits, every input is+// embedded on its own, in order, and the healthy ones keep their content.+func TestEmbedWithinLimitIsolatesAPathologicalInput(t *testing.T) {+ driver := &recordingEmbedDriver{rejectBatch: true}+ model := newEmbedModel(t, driver, "", 8192)+ // Read the ladder length before the call: the call's own rejections ratchet the+ // calibration, which would shrink the budget this number is derived from.+ budget, _ := embedBudget(t, model)+ batchRungs := len(tokenizer.OverLimitLadder(budget))+ texts := []string{strings.Repeat("a", 40), "second"}+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: texts}, nil, nil)+ if err != nil {+ t.Fatalf("EmbedWithinLimit: %v", err)+ }+ if len(embeds) != 2 {+ t.Fatalf("got %d embeddings, want 2", len(embeds))+ }+ if len(driver.attempts) != batchRungs+2 {+ t.Fatalf("attempts = %d, want %d batch rungs plus 2 isolated calls", len(driver.attempts), batchRungs+2)+ }+ isolated := driver.attempts[batchRungs:]+ for i, attempt := range isolated {+ if len(attempt) != 1 {+ t.Fatalf("isolated call %d carried %d texts, want 1", i, len(attempt))+ }+ // Both inputs fit the first budget, so isolation must not have modified them.+ if attempt[0] != texts[i] {+ t.Errorf("isolated call %d got %q, want %q", i, attempt[0], texts[i])+ }+ }+}++// TestEmbedWithinLimitFailsWhenNoBudgetFits keeps the floor honest: the ladder ends+// above OverLimitFloorTokens and only isolation reaches it, so the error may only be+// raised after the floor was actually tried - and never with a partial result.+func TestEmbedWithinLimitFailsWhenNoBudgetFits(t *testing.T) {+ driver := &recordingEmbedDriver{rejectLongerThan: 8}+ model := newEmbedModel(t, driver, "", 8192)+ budget, _ := embedBudget(t, model)+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{strings.Repeat("中", 500)}}, nil, nil)+ if err == nil {+ t.Fatal("expected an error when not even the floor fits")+ }+ if embeds != nil {+ t.Fatalf("partial results returned: %d embeddings", len(embeds))+ }+ if !strings.Contains(err.Error(), "even at 64 tokens") {+ t.Errorf("error %q does not say the floor was tried", err)+ }+ want := len(tokenizer.OverLimitLadder(budget)) + len(tokenizer.OverLimitLadderToFloor(budget))+ if len(driver.attempts) != want {+ t.Errorf("attempts = %d, want %d (batch ladder then isolation down to the floor)", len(driver.attempts), want)+ }+}++// TestEmbedWithinLimitRecordsTheDeclaredLimitInTheCalibration pins the third argument+// of ObserveOverLimit: it is the provider's declared window, not the local budget+// derived from it. Passing the (smaller) budget would understate the inferred ratio+// and leave the calibration effectively unchanged, so the next call would repeat the+// same rejection.+func TestEmbedWithinLimitRecordsTheDeclaredLimitInTheCalibration(t *testing.T) {+ driver := &recordingEmbedDriver{rejectFor: 1}+ const declared = 8192+ model := newEmbedModel(t, driver, "", declared)+ if _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{strings.Repeat("a", 40000)}}, nil, nil); err != nil {+ t.Fatalf("EmbedWithinLimit: %v", err)+ }+ if len(driver.attempts) == 0 {+ t.Fatal("no attempt recorded")+ }+ _, counter := embedBudget(t, model)+ own := tokenizer.OwnTokenMax(driver.attempts[0], counter)+ want := float64(declared) / float64(own) * 1.01+ if got := tokenizer.DefaultCalibration().RatioUpper(model.QuotaKey()); got < want-1e-9 {+ t.Fatalf("calibration ratio = %v, want at least %v (inferred from the declared limit)", got, want)+ }+}++// TestEmbedWithinLimitEmptyTexts: nothing to embed is a no-op, not a call.+func TestEmbedWithinLimitEmptyTexts(t *testing.T) {+ driver := &recordingEmbedDriver{}+ model := newEmbedModel(t, driver, "", 8192)+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{}, nil, nil)+ if err != nil {+ t.Fatalf("EmbedWithinLimit: %v", err)+ }+ if len(embeds) != 0 || len(driver.attempts) != 0 {+ t.Fatalf("embeds = %d, attempts = %d, want 0 and 0", len(embeds), len(driver.attempts))+ }+}diff --git a/internal/entity/models/types.go b/internal/entity/models/types.goindex 07f1aab413..c27e5f82e3 100644--- a/internal/entity/models/types.go+++ b/internal/entity/models/types.go@@ -2,6 +2,7 @@ package modelsimport ("context"+ "crypto/sha256""encoding/json""errors""fmt"@@ -331,6 +332,36 @@ func (m *EmbeddingModel) ResolveTokenizerID() string {return GetEmbeddingTokenizer(name)}+// QuotaKey names the deployment this embedding model counts against: endpoint,+// region, model name and an API-key prefix. The tokenizer belongs to the model, but+// what a provider accepts is per deployment - the same model behind two endpoints+// can have different windows - so everything that learns a real/own token ratio+// (the ingest embedder, the dataset-nav embedder, the knowledge-compiler embedder)+// has to key that ratio the same way; otherwise each path re-learns the same+// rejection and none of them tightens for the others.+func (m *EmbeddingModel) QuotaKey() string {+ if m == nil {+ return ""+ }+ var baseURL, region, apiKey, modelName string+ if cfg := m.APIConfig; cfg != nil {+ if cfg.BaseURL != nil {+ baseURL = *cfg.BaseURL+ }+ if cfg.Region != nil {+ region = *cfg.Region+ }+ if cfg.ApiKey != nil {+ apiKey = *cfg.ApiKey+ }+ }+ if m.ModelName != nil {+ modelName = *m.ModelName+ }+ sum := sha256.Sum256([]byte(apiKey))+ return fmt.Sprintf("%s|%s|%s|%x", baseURL, region, modelName, sum[:8])+}+// RerankModel wraps a ModelDriver with rerank-specific configurationtype RerankModel struct {ModelDriver ModelDriverdiff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.goindex bf4ff65735..56c7f6af41 100644--- a/internal/ingestion/task/embedder.go+++ b/internal/ingestion/task/embedder.go@@ -18,7 +18,6 @@ package taskimport ("context"- "crypto/sha256""errors""fmt""math"@@ -193,22 +192,7 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.E// lets each healthy input keep its own budget. A provider that rejects even the last// proportional step therefore still gets one attempt at the floor - per input, on the// input that actually needs it.-func overLimitLadder(budget int) []int {- if budget <= 0 {- return []int{0}- }- limits := []int{budget}- for _, factor := range []float64{0.75, 0.5, 0.25, 0.125} {- next := int(float64(budget) * factor)- if next < overLimitFloorTokens {- next = overLimitFloorTokens- }- if next < limits[len(limits)-1] {- limits = append(limits, next)- }- }- return limits-}+func overLimitLadder(budget int) []int { return tokenizer.OverLimitLadder(budget) }// overLimitLadderToFloor is overLimitLadder plus the last-resort floor, for the// per-input isolation path: an input that no proportional step fits may still fit@@ -218,17 +202,11 @@ func overLimitLadder(budget int) []int {// The batch loop deliberately stops above the floor. Reaching it there would trim// EVERY input of the batch to 64 tokens just because one input is pathological,// while isolating lets each healthy input keep its own budget.-func overLimitLadderToFloor(budget int) []int {- limits := overLimitLadder(budget)- if limits[len(limits)-1] > overLimitFloorTokens {- limits = append(limits, overLimitFloorTokens)- }- return limits-}+func overLimitLadderToFloor(budget int) []int { return tokenizer.OverLimitLadderToFloor(budget) }// overLimitFloorTokens is the smallest budget worth trying before declaring the// input genuinely broken rather than merely long.-const overLimitFloorTokens = 64+const overLimitFloorTokens = tokenizer.OverLimitFloorTokens// trimAll re-trims every text to `limit` tokens. Texts handed in by the caller// are already trimmed to the full budget, and trimming a prefix again just makes@@ -404,41 +382,12 @@ func (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiterreturn out, nil}-// overLimitMarkers are the provider wordings that mean "this input is longer-// than the model accepts". 20015 is SiliconFlow's code for exactly that (it-// answers a generic "The parameter is invalid" message); the rest are the usual-// phrasings elsewhere.-var overLimitMarkers = []string{- "20015",- "too long",- "too many tokens",- "maximum context",- "context length",- "context_length",- "input length",- "token limit",- "reduce the length",- "maximum allowed",-}-// isOverLimitErr reports whether err is an over-limit rejection rather than a-// rate limit or a genuine failure. Only 4xx rejections qualify: a 5xx is the-// provider's problem and shrinking the input would not help.-func isOverLimitErr(err error) bool {- if err == nil {- return false- }- msg := strings.ToLower(err.Error())- if !strings.Contains(msg, "400") && !strings.Contains(msg, "413") && !strings.Contains(msg, "422") {- return false- }- for _, marker := range overLimitMarkers {- if strings.Contains(msg, marker) {- return true- }- }- return false-}+// rate limit or a genuine failure. The marker set and the 4xx requirement live in+// internal/tokenizer (tokenizer.IsOverLimitError): every embedder outside this+// package needs the same classification to drive its own shrink-and-retry, and one+// definition is what keeps them from drifting apart.+func isOverLimitErr(err error) bool { return tokenizer.IsOverLimitError(err) }// Retry policy for rate-limited embedding calls.//@@ -526,21 +475,16 @@ func waitOutCooldown(ctx context.Context, key providerQuotaKey) error {}}-// quotaKey derives the provider-instance key the cooldown is scoped to. The API-// key decides who owns the quota, so it is part of the key but hashed: the key-// must never carry a plaintext secret.+// quotaKey derives the provider-instance key the cooldown and the token-ratio+// calibration are scoped to. The API key decides who owns the quota, so it is part+// of the key but hashed: the key must never carry a plaintext secret. The format+// comes from the model (EmbeddingModel.QuotaKey) so every embedder - ingestion,+// dataset nav, the knowledge compiler - learns the same ratio for a deployment.func (e *embedder) quotaKey() providerQuotaKey {if e == nil || e.model == nil {return providerQuotaKey("embedder:unknown")}- var baseURL, region, apiKey string- if cfg := e.model.APIConfig; cfg != nil {- baseURL = derefString(cfg.BaseURL)- region = derefString(cfg.Region)- apiKey = derefString(cfg.ApiKey)- }- sum := sha256.Sum256([]byte(apiKey))- return providerQuotaKey(fmt.Sprintf("%s|%s|%s|%x", baseURL, region, derefString(e.model.ModelName), sum[:8]))+ return providerQuotaKey(e.model.QuotaKey())}// derefString safely dereferences an optional string.diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex ec76467572..9b9cf428c8 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -700,8 +700,10 @@ func (e *kcEmbedder) Encode(ctx context.Context, texts []string) ([][]float32, eif jobErr := ctx.Err(); jobErr != nil {return jobErr}- var embeds []models.EmbeddingData- embeds, jobErr := mdl.ModelDriver.Embed(ctx, mdl.ModelName, models.EmbedRequest{Texts: batchTexts}, mdl.APIConfig, config, nil)+ // Embed inside the model's window: compile products include the summaries+ // and entity descriptions that feed the nav index, and an over-window input+ // is a hard 400/20015 that fails the whole batch instead of being trimmed.+ embeds, jobErr := mdl.EmbedWithinLimit(ctx, models.EmbedRequest{Texts: batchTexts}, config, nil)if jobErr != nil {return fmt.Errorf("knowledge_compiler: embed: %w", jobErr)}diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex 6ecc10ac1e..02e14fbd7c 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -3060,7 +3060,9 @@ type embeddingModelEmbedder struct {func (e *embeddingModelEmbedder) Encode(ctx context.Context, texts []string) ([][]float64, error) {config := &modelModule.EmbeddingConfig{Dimension: 0}- embeds, err := e.embModel.ModelDriver.Embed(ctx, e.embModel.ModelName, modelModule.EmbedRequest{Texts: texts}, e.embModel.APIConfig, config, nil)+ // Embed inside the model's window: the caller supplies arbitrary text and the+ // provider rejects an over-window input with 400/20015 instead of truncating it.+ embeds, err := e.embModel.EmbedWithinLimit(ctx, modelModule.EmbedRequest{Texts: texts}, config, nil)if err != nil {return nil, err}diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex d6f3f5fe91..af15ff71b4 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -1414,7 +1414,9 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequesif len(questionKwd) > 0 {embeddingText = strings.Join(questionKwd, "\n")}- embeddings, err := embeddingModel.ModelDriver.Embed(ctx, embeddingModel.ModelName, models.EmbedRequest{Texts: []string{docName, embeddingText}}, embeddingModel.APIConfig, &models.EmbeddingConfig{Dimension: 0}, nil)+ // Embed inside the model's window: Content arrives from the API and can be+ // arbitrarily long, and the provider answers 400/20015 instead of truncating it.+ embeddings, err := embeddingModel.EmbedWithinLimit(ctx, models.EmbedRequest{Texts: []string{docName, embeddingText}}, &models.EmbeddingConfig{Dimension: 0}, nil)if err != nil {return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("encode chunk embedding: %v", err)}}diff --git a/internal/service/dataset/utils.go b/internal/service/dataset/utils.goindex 204a39eddc..7cc4004a52 100644--- a/internal/service/dataset/utils.go+++ b/internal/service/dataset/utils.go@@ -256,7 +256,10 @@ func datasetEncodeEmbedding(ctx context.Context, embeddingModel *modelModule.Embcleaned[i] = datasetCleanEmbeddingText(t)}embeddingConfig := &modelModule.EmbeddingConfig{Dimension: 0}- embeddings, err := embeddingModel.ModelDriver.Embed(ctx, embeddingModel.ModelName, modelModule.EmbedRequest{Texts: cleaned}, embeddingModel.APIConfig, embeddingConfig, nil)+ // Embed inside the model's window: this path takes whatever text the API handed+ // it, and the provider rejects an over-window input with 400/20015 instead of+ // truncating it.+ embeddings, err := embeddingModel.EmbedWithinLimit(ctx, modelModule.EmbedRequest{Texts: cleaned}, embeddingConfig, nil)if err != nil {return nil, err}diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.goindex fe6e696738..cd00eae65d 100644--- a/internal/service/memory_message_service.go+++ b/internal/service/memory_message_service.go@@ -329,7 +329,9 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Crreturn err}embeddingModel := models.NewEmbeddingModel(target.Driver, &target.ModelName, target.APIConfig, target.MaxTokens)- embeddings, err := embeddingModel.ModelDriver.Embed(ctx, embeddingModel.ModelName, models.EmbedRequest{Texts: contents}, embeddingModel.APIConfig, &models.EmbeddingConfig{Dimension: 0}, nil)+ // Embed inside the model's window: memory contents are caller-supplied and+ // unbounded, and the provider answers 400/20015 instead of truncating them.+ embeddings, err := embeddingModel.EmbedWithinLimit(ctx, models.EmbedRequest{Texts: contents}, &models.EmbeddingConfig{Dimension: 0}, nil)if err != nil {return fmt.Errorf("embed model: %w", err)}diff --git a/internal/service/nav_embedder.go b/internal/service/nav_embedder.goindex 16ab21de09..125b704ddd 100644--- a/internal/service/nav_embedder.go+++ b/internal/service/nav_embedder.go@@ -83,7 +83,23 @@ func (e *NavEmbedder) encode(ctx context.Context, tenantID string, texts []strinif len(nonEmpty) == 0 {return nil, nil}- embeds, err := model.ModelDriver.Embed(ctx, model.ModelName, modelModule.EmbedRequest{Texts: nonEmpty, Query: query}, model.APIConfig, nil, nil)+ // Documents go through EmbedWithinLimit: the provider does not truncate, it+ // answers 400/20015, and a nav summary is not a short string - without a tree+ // product it is every entity line of the page-index graph joined into one. The+ // model makes the cut (and retries with a smaller budget when a calibrated count+ // undershoots), which is what Python gets from BaseEmbedding.encode.+ //+ // Queries stay on the driver. A query is short, so there is nothing to cut, and+ // EmbedWithinLimit refuses to run when the model declares a tokenizer whose asset+ // is missing - a refusal that belongs to the ingest path, not to a search, which+ // has to keep answering on a deployment that never provisioned the asset.+ var embeds []modelModule.EmbeddingData+ var err error+ if query {+ embeds, err = model.ModelDriver.Embed(ctx, model.ModelName, modelModule.EmbedRequest{Texts: nonEmpty, Query: true}, model.APIConfig, nil, nil)+ } else {+ embeds, err = model.EmbedWithinLimit(ctx, modelModule.EmbedRequest{Texts: nonEmpty}, nil, nil)+ }if err != nil {return nil, err}diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.goindex e4b669dddb..795abebfdf 100644--- a/internal/tokenizer/embedlimit.go+++ b/internal/tokenizer/embedlimit.go@@ -36,6 +36,7 @@ package tokenizerimport ("fmt""math"+ "regexp""sort""strings""sync"@@ -530,6 +531,155 @@ func trimByBytes(text string, limit int) string {// utf8Start reports whether b is not a UTF-8 continuation byte.func utf8Start(b byte) bool { return b&0xC0 != 0x80 }+// ---------------------------------------------------------------------------+// Over-limit handling: the marker set, the shrink ladder, and the refusal every+// embedder shares. The embedding side of this - the loop that trims, calls the+// driver, and walks the ladder - is EmbeddingModel.EmbedWithinLimit in+// internal/entity/models, next to the RerankModel cut it mirrors.+// ---------------------------------------------------------------------------++// OverLimitFloorTokens is the smallest budget worth trying before calling an input+// genuinely broken rather than merely long.+const OverLimitFloorTokens = 64++// OverLimitMarkers are the provider wordings that mean "this input is longer than+// the model accepts". They are phrases, not codes, so a plain substring test is the+// right one for them.+var OverLimitMarkers = []string{+ "too long",+ "too many tokens",+ "maximum context",+ "context length",+ "context_length",+ "input length",+ "token limit",+ "reduce the length",+ "maximum allowed",+}++// OverLimitCodes are the provider error codes that mean the same thing. 20015 is+// SiliconFlow's: it is returned with a generic "The parameter is invalid" message, so+// the code is the only usable signal.+var OverLimitCodes = []string{"20015"}++// overLimitStatus matches an HTTP 4xx status as a delimited number: "400 Bad+// Request" matches, "1400" and "4000" do not.+var overLimitStatus = regexp.MustCompile(`\b(?:400|413|422)\b`)++// IsOverLimitError reports whether err is an over-limit rejection rather than a+// rate limit or a genuine failure. Only 4xx rejections qualify: a 5xx is the+// provider's problem and a shorter input would not fix it.+//+// Status codes and provider codes are matched as DELIMITED numbers. A substring test+// would read provider code 120015 as SiliconFlow's 20015 (and 1400 as 400), and the+// caller would answer an unrelated failure by re-embedding a silently truncated+// input — or replace the real error with a window-limit one.+func IsOverLimitError(err error) bool {+ if err == nil {+ return false+ }+ msg := strings.ToLower(err.Error())+ if !overLimitStatus.MatchString(msg) {+ return false+ }+ for _, marker := range OverLimitMarkers {+ if strings.Contains(msg, marker) {+ return true+ }+ }+ for _, code := range OverLimitCodes {+ if hasDelimitedNumber(msg, code) {+ return true+ }+ }+ return false+}++// hasDelimitedNumber reports whether s contains num as a whole number: neither+// neighbour may be a digit, so `"code":20015,` matches while 120015 and 200150 do+// not.+func hasDelimitedNumber(s, num string) bool {+ for from := 0; from+len(num) <= len(s); {+ at := strings.Index(s[from:], num)+ if at < 0 {+ return false+ }+ at += from+ end := at + len(num)+ if (at == 0 || !isASCIIDigit(s[at-1])) && (end == len(s) || !isASCIIDigit(s[end])) {+ return true+ }+ from = at + 1+ }+ return false+}++func isASCIIDigit(b byte) bool { return b >= '0' && b <= '9' }++// OverLimitLadder returns the budgets to try, in order, after a provider rejects+// an input as over its window: the caller's budget first, then progressively+// smaller fractions of it, never below OverLimitFloorTokens. Strictly decreasing,+// so the loop always makes progress.+func OverLimitLadder(budget int) []int {+ if budget <= 0 {+ return []int{0}+ }+ limits := []int{budget}+ for _, factor := range []float64{0.75, 0.5, 0.25, 0.125} {+ next := int(float64(budget) * factor)+ if next < OverLimitFloorTokens {+ next = OverLimitFloorTokens+ }+ if next < limits[len(limits)-1] {+ limits = append(limits, next)+ }+ }+ return limits+}++// OverLimitLadderToFloor is OverLimitLadder plus the last-resort floor, for the+// per-input isolation path: an input that no proportional step fits may still fit+// at the floor, and trying it there is what turns "this document cannot be+// embedded" into an embedded (truncated) chunk.+//+// A batch loop deliberately stops above the floor: reaching it there would trim+// EVERY input of the batch to 64 tokens just because one input is pathological,+// while isolating lets each healthy input keep its own budget.+func OverLimitLadderToFloor(budget int) []int {+ limits := OverLimitLadder(budget)+ if limits[len(limits)-1] > OverLimitFloorTokens {+ limits = append(limits, OverLimitFloorTokens)+ }+ return limits+}++// RefuseUnavailableCounter is the refusal every embedder shares: a model that+// DECLARES a tokenizer whose asset is not on disk must not be counted with the+// calibrated cl100k estimate, because that count belongs to a different tokenizer+// and an under-count is what makes a provider answer 400. Silently substituting it+// would trade away the exactness these counters exist for.+func RefuseUnavailableCounter(tokenizerID, calibrationKey string) error {+ if tokenizerID == "" || CounterExact(tokenizerID) {+ return nil+ }+ return fmt.Errorf(+ "embedding tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate",+ tokenizerID, calibrationKey)+}++// OwnTokenMax is the largest single input's cost in our own counter. The+// provider's window bounds each input individually, so this - not the batch total+// - is what an over-limit rejection tells us about.+func OwnTokenMax(texts []string, counter Counter) int {+ max := 0+ for _, t := range texts {+ if n := counter.Count(t); n > max {+ max = n+ }+ }+ return max+}+// ---------------------------------------------------------------------------// Counters we can build without extra assets// ---------------------------------------------------------------------------diff --git a/internal/tokenizer/embedlimit_test.go b/internal/tokenizer/embedlimit_test.goindex be5f576213..8bc24eb489 100644--- a/internal/tokenizer/embedlimit_test.go+++ b/internal/tokenizer/embedlimit_test.go@@ -17,6 +17,7 @@package tokenizerimport (+ "errors""strings""testing")@@ -270,3 +271,51 @@ func TestCL100KTrimNeverExceedsLimit(t *testing.T) {}}}++// TestIsOverLimitErrorMatchesDelimitedNumbers pins the difference between a substring+// test and a delimited one. The caller acts on this answer by re-embedding a+// truncated input, so a false positive silently replaces a real error with a+// window-limit one; 120015 merely contains 20015, and 1400 merely contains 400.+func TestIsOverLimitErrorMatchesDelimitedNumbers(t *testing.T) {+ cases := []struct {+ name string+ err string+ want bool+ }{+ {+ "siliconflow over-window",+ `SILICONFLOW API error: 400 Bad Request, body: {"code":20015,"message":"The parameter is invalid. Please check again.","data":null}`,+ true,+ },+ {+ "siliconflow code as a string",+ `SILICONFLOW API error: 400 Bad Request, body: {"code":"20015"}`,+ true,+ },+ {+ "openai wording",+ `OpenAI embeddings API error: 400 Bad Request, body: {"error":{"message":"This model's maximum context length is 8192 tokens"}}`,+ true,+ },+ {"413", `API error: 413 Request Entity Too Large, body: input is too long`, true},+ {"422", `API error: 422 Unprocessable Entity, body: too many tokens`, true},+ // Delimiter neighbours: a longer provider code that merely contains 20015, and+ // a status that merely contains 400.+ {"code 120015", `SILICONFLOW API error: 400 Bad Request, body: {"code":120015}`, false},+ {"code 200150", `SILICONFLOW API error: 400 Bad Request, body: {"code":200150}`, false},+ {"status 1400", `API error: 1400 Bad Request, body: too long`, false},+ // Rate limits and provider failures must not be mistaken for size.+ {"rate limit", `SILICONFLOW API error: 429 Too Many Requests, body: {"message":"Request was rejected due to rate limiting. Details: TPM limit reached."}`, false},+ {"server error", `SILICONFLOW API error: 500 Internal Server Error, body: too long`, false},+ {"unauthorized", `OpenAI embeddings API error: 401 Unauthorized, body: invalid api key`, false},+ {"network", `failed to send request: dial tcp: connection refused`, false},+ }+ for _, c := range cases {+ if got := IsOverLimitError(errors.New(c.err)); got != c.want {+ t.Errorf("%s: IsOverLimitError = %t, want %t (%s)", c.name, got, c.want, c.err)+ }+ }+ if IsOverLimitError(nil) {+ t.Error("IsOverLimitError(nil) = true, want false")+ }+}