Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(embedding): cut inputs to the model window on every path (#19997)

Zhichang Yuea93fc8304e6
diff --git a/internal/entity/models/embed_limit.go b/internal/entity/models/embed_limit.go
new file mode 100644
index 0000000000..7ad7f560d6
--- /dev/null
+++ b/internal/entity/models/embed_limit.go
@@ -0,0 +1,139 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "context"
+ "fmt"
+
+ "ragflow/internal/common"
+ "ragflow/internal/tokenizer"
+)
+
+// EmbedWithinLimit embeds req after cutting every text to what this model accepts,
+// and retries with a smaller budget if the provider still rejects an input as over
+// its window. It is the embedding counterpart of RerankModel.Rerank, which cuts its
+// documents the same way for the same reason.
+//
+// Callers use it instead of ModelDriver.Embed because a provider does NOT truncate
+// an over-window input: it rejects the whole request (SiliconFlow answers 400 with
+// code 20015 for an input 0.55% over spec). The document-side callers - the
+// dataset-nav summary, the knowledge-compiler products, chunk and memory writes -
+// have no length bound of their own, so without this cut the request is sent as-is
+// and, because such a batch is retried unchanged, fails forever. Python solves it
+// one layer down in BaseEmbedding.encode; Go has no such layer, so the model
+// carries it. Query-side callers keep calling the driver directly: their texts are
+// short, and they must keep working on a deployment whose tokenizer assets are
+// missing.
+//
+// Only over-limit rejections are retried, each one with a strictly smaller budget:
+// a rate limit belongs to the driver's own retry policy and a 5xx is not something
+// a shorter input fixes. When even the batch ladder is rejected, each input is
+// embedded on its own down to the floor, so one pathological text neither fails the
+// whole batch nor drags its healthy neighbours down to the floor with it. A model
+// that declares a tokenizer whose asset is not on disk is refused before the first
+// attempt, because counting it with the calibrated cl100k estimate is exactly what
+// lets an oversized request through.
+func (m *EmbeddingModel) EmbedWithinLimit(ctx context.Context, req EmbedRequest, embeddingConfig *EmbeddingConfig, usage *common.ModelUsage) ([]EmbeddingData, error) {
+ if m == nil || m.ModelDriver == nil {
+ return nil, fmt.Errorf("embedding model: driver is nil")
+ }
+ if len(req.Texts) == 0 {
+ return []EmbeddingData{}, nil
+ }
+ tokenizerID, quotaKey := m.ResolveTokenizerID(), m.QuotaKey()
+ if err := tokenizer.RefuseUnavailableCounter(tokenizerID, quotaKey); err != nil {
+ return nil, err
+ }
+ limiter := tokenizer.LimiterFor(tokenizerID, quotaKey, tokenizer.DefaultCalibration())
+ counter := limiter.Counter()
+ cal, calKey := limiter.Calibration()
+ // maxTokens is what the provider declares and window is the local budget derived
+ // from it (calibrated ratio plus margin). The first rung cuts to window, so the
+ // common case is a single call; a rejection is evidence about maxTokens, which is
+ // what the calibration has to be told.
+ maxTokens := m.ResolveMaxTokens()
+ window := limiter.Limit(maxTokens)
+
+ for _, budget := range tokenizer.OverLimitLadder(window) {
+ embeds, err := m.embedCut(ctx, req, embeddingConfig, usage, counter, budget)
+ if err == nil {
+ return embeds, nil
+ }
+ if !tokenizer.IsOverLimitError(err) {
+ return nil, err
+ }
+ // Our count said OwnTokenMax tokens for an input the provider just rejected:
+ // record it, so the next call - and the next document - starts from a tighter
+ // budget instead of re-learning the same thing.
+ cal.ObserveOverLimit(calKey, tokenizer.OwnTokenMax(req.Texts, counter), maxTokens)
+ }
+
+ // The batch ladder could not fit the batch, which usually means one input is
+ // pathological next to healthy ones. Isolate instead of failing: each input gets
+ // its own walk down to OverLimitFloorTokens, and the healthy ones keep the budget
+ // they had. A failure here is reported as isolation's own - it names the input and
+ // the floor it reached, while the batch rejection above says only "the whole
+ // request was too big".
+ return m.embedIsolating(ctx, req, embeddingConfig, usage, counter, cal, calKey, maxTokens, window)
+}
+
+// embedCut trims every text to budget and performs exactly one embedding call.
+func (m *EmbeddingModel) embedCut(ctx context.Context, req EmbedRequest, embeddingConfig *EmbeddingConfig, usage *common.ModelUsage, counter tokenizer.Counter, budget int) ([]EmbeddingData, error) {
+ attempt := req
+ attempt.Texts = make([]string, len(req.Texts))
+ for i, text := range req.Texts {
+ attempt.Texts[i] = counter.TrimToLimit(text, budget)
+ }
+ return m.ModelDriver.Embed(ctx, m.ModelName, attempt, m.APIConfig, embeddingConfig, usage)
+}
+
+// embedIsolating embeds each input of req on its own, walking the ladder to the
+// floor for the ones that need it. It returns one embedding per input, in the
+// original order, and never a partial result: an input that does not fit even at the
+// floor fails the call.
+func (m *EmbeddingModel) embedIsolating(ctx context.Context, req EmbedRequest, embeddingConfig *EmbeddingConfig, usage *common.ModelUsage, counter tokenizer.Counter, cal *tokenizer.Calibration, calKey string, maxTokens, window int) ([]EmbeddingData, error) {
+ out := make([]EmbeddingData, len(req.Texts))
+ for i, text := range req.Texts {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ single := req
+ single.Texts = []string{text}
+ var lastErr error
+ fitted := false
+ for _, budget := range tokenizer.OverLimitLadderToFloor(window) {
+ embeds, err := m.embedCut(ctx, single, embeddingConfig, usage, counter, budget)
+ if err == nil {
+ if len(embeds) != 1 {
+ return nil, fmt.Errorf("embedding input %d: driver returned %d embeddings for one text", i, len(embeds))
+ }
+ out[i], fitted = embeds[0], true
+ break
+ }
+ if !tokenizer.IsOverLimitError(err) {
+ return nil, err
+ }
+ cal.ObserveOverLimit(calKey, counter.Count(counter.TrimToLimit(text, budget)), maxTokens)
+ lastErr = err
+ }
+ if !fitted {
+ return nil, fmt.Errorf("embedding input %d does not fit the model window even at %d tokens: %w", i, tokenizer.OverLimitFloorTokens, lastErr)
+ }
+ }
+ return out, nil
+}
diff --git a/internal/entity/models/embed_limit_test.go b/internal/entity/models/embed_limit_test.go
new file mode 100644
index 0000000000..953c575631
--- /dev/null
+++ b/internal/entity/models/embed_limit_test.go
@@ -0,0 +1,278 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "context"
+ "errors"
+ "strconv"
+ "strings"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/tokenizer"
+)
+
+// overWindowError is what SiliconFlow answers for an input over the model window:
+// a 400 whose body carries code 20015.
+func overWindowError() error {
+ return errors.New(`SILICONFLOW API error: 400 Bad Request, body: {"code":20015,"message":"The parameter is invalid. Please check again.","data":null}`)
+}
+
+// recordingEmbedDriver implements only the Embed half of ModelDriver: it records the
+// texts it was handed and can reject them the way an over-window provider does.
+type recordingEmbedDriver struct {
+ ModelDriver
+ attempts [][]string
+ // rejectFor rejects the first N attempts.
+ rejectFor int
+ // rejectBatch rejects any request carrying more than one text, which is how a
+ // pathological input behaves: the batch ladder cannot fit the batch, while each
+ // input on its own can be embedded.
+ rejectBatch bool
+ // rejectLongerThan rejects any request holding a text longer than that many
+ // bytes, so a small text passes while a large one fails even after the trim.
+ rejectLongerThan int
+ failWith error
+}
+
+func (d *recordingEmbedDriver) Embed(_ context.Context, _ *string, request EmbedRequest, _ *APIConfig, _ *EmbeddingConfig, _ *common.ModelUsage) ([]EmbeddingData, error) {
+ d.attempts = append(d.attempts, request.Texts)
+ if d.rejectBatch && len(request.Texts) > 1 {
+ return nil, overWindowError()
+ }
+ if d.rejectLongerThan > 0 {
+ for _, text := range request.Texts {
+ if len(text) > d.rejectLongerThan {
+ return nil, overWindowError()
+ }
+ }
+ }
+ if len(d.attempts) <= d.rejectFor {
+ return nil, overWindowError()
+ }
+ if d.failWith != nil {
+ return nil, d.failWith
+ }
+ return make([]EmbeddingData, len(request.Texts)), nil
+}
+
+// newEmbedModel pins the inputs that decide the window and the counter, so a test
+// never depends on the machine's catalog or environment: the tokenizer id and the
+// window come from the env overrides, and the api key (the test's own name) gives the
+// model its own quota/calibration key. That last part matters because the default
+// Calibration is process-global and only ever ratchets up - one test's over-limit
+// observation must not shrink another test's window.
+func newEmbedModel(t *testing.T, driver ModelDriver, tokenizerID string, maxTokens int) *EmbeddingModel {
+ t.Helper()
+ t.Setenv("TOKENIZER_EMBEDDING_TOKENIZER", tokenizerID)
+ t.Setenv("TOKENIZER_EMBEDDING_MAX_TOKENS", strconv.Itoa(maxTokens))
+ apiKey := t.Name()
+ return NewEmbeddingModel(driver, nil, &APIConfig{ApiKey: &apiKey}, maxTokens)
+}
+
+// embedBudget is the budget a single cut would use, computed the same way
+// EmbedWithinLimit computes its first rung.
+func embedBudget(t *testing.T, model *EmbeddingModel) (int, tokenizer.Counter) {
+ t.Helper()
+ limiter := tokenizer.LimiterFor(model.ResolveTokenizerID(), model.QuotaKey(), tokenizer.DefaultCalibration())
+ return limiter.Limit(model.ResolveMaxTokens()), limiter.Counter()
+}
+
+// TestEmbedWithinLimitCutsToTheWindow pins what the dataset-nav, knowledge-compiler,
+// chunk and memory callers rely on: what reaches the driver fits the window, order is
+// preserved, a short text is untouched, and a first-attempt success is not retried.
+//
+// The empty tokenizer id is the calibrated path (a model whose tokenizer the catalog
+// does not name); xlmr-spm is covered when its asset is present because it is the
+// counter behind the reported failure (a nav summary against bge-m3, 400/20015).
+func TestEmbedWithinLimitCutsToTheWindow(t *testing.T) {
+ long := strings.Repeat("中文 hello ", 5000)
+ for _, id := range []string{"", tokenizer.CounterXLMRSentence} {
+ if id != "" && !tokenizer.CounterExact(id) {
+ t.Logf("counter %s is unavailable in this environment; skipping it", id)
+ continue
+ }
+ driver := &recordingEmbedDriver{}
+ model := newEmbedModel(t, driver, id, 8192)
+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{long, "short"}}, nil, nil)
+ if err != nil {
+ t.Fatalf("tokenizer %q: EmbedWithinLimit: %v", id, err)
+ }
+ if len(embeds) != 2 {
+ t.Fatalf("tokenizer %q: got %d embeddings, want 2", id, len(embeds))
+ }
+ if len(driver.attempts) != 1 {
+ t.Fatalf("tokenizer %q: %d attempts for an input the provider accepts", id, len(driver.attempts))
+ }
+ got := driver.attempts[0]
+ if got[1] != "short" {
+ t.Errorf("tokenizer %q: a text inside the window was modified: %q", id, got[1])
+ }
+ if got[0] == long {
+ t.Errorf("tokenizer %q: the over-window text was not cut", id)
+ }
+ budget, counter := embedBudget(t, model)
+ if n := counter.Count(got[0]); n > budget {
+ t.Errorf("tokenizer %q: the text handed to the driver counts %d tokens, budget is %d", id, n, budget)
+ }
+ }
+}
+
+// TestEmbedWithinLimitRetriesWithASmallerBudget is the property that turns "this
+// batch fails forever" into "this batch succeeds": the retried text must be strictly
+// shorter, and the caller must not see the rejection.
+func TestEmbedWithinLimitRetriesWithASmallerBudget(t *testing.T) {
+ driver := &recordingEmbedDriver{rejectFor: 1}
+ model := newEmbedModel(t, driver, "", 8192)
+ if _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{strings.Repeat("中", 20000)}}, nil, nil); err != nil {
+ t.Fatalf("EmbedWithinLimit: %v", err)
+ }
+ if len(driver.attempts) != 2 {
+ t.Fatalf("attempts = %d, want 2", len(driver.attempts))
+ }
+ if len(driver.attempts[1][0]) >= len(driver.attempts[0][0]) {
+ t.Fatalf("the retry did not shrink the input: %d then %d bytes", len(driver.attempts[0][0]), len(driver.attempts[1][0]))
+ }
+}
+
+// TestEmbedWithinLimitDoesNotRetryUnrelatedErrors keeps the retry narrow: a 5xx is
+// not something a shorter input fixes, so it comes back on the first attempt.
+func TestEmbedWithinLimitDoesNotRetryUnrelatedErrors(t *testing.T) {
+ boom := errors.New("SILICONFLOW API error: 503 Service Unavailable")
+ driver := &recordingEmbedDriver{failWith: boom}
+ model := newEmbedModel(t, driver, "", 8192)
+ _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{"hello"}}, nil, nil)
+ if !errors.Is(err, boom) {
+ t.Fatalf("error = %v, want the provider's own error", err)
+ }
+ if len(driver.attempts) != 1 {
+ t.Fatalf("attempts = %d, want 1", len(driver.attempts))
+ }
+}
+
+// TestEmbedWithinLimitRefusesAnUnavailableDeclaredTokenizer is the half of the
+// contract that keeps the retry from hiding a provisioning gap: a model that declares
+// a tokenizer we cannot load fails before the first attempt.
+func TestEmbedWithinLimitRefusesAnUnavailableDeclaredTokenizer(t *testing.T) {
+ driver := &recordingEmbedDriver{}
+ model := newEmbedModel(t, driver, "no-such-family", 8192)
+ _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{"hello"}}, nil, nil)
+ if err == nil {
+ t.Fatal("expected an error for a declared tokenizer whose asset is unavailable")
+ }
+ for _, want := range []string{`"no-such-family"`, "but its asset is unavailable", "refusing to count with the calibrated estimate"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Errorf("error %q does not mention %q", err.Error(), want)
+ }
+ }
+ if len(driver.attempts) != 0 {
+ t.Errorf("the driver was called %d times despite the refusal", len(driver.attempts))
+ }
+}
+
+// TestEmbedWithinLimitIsolatesAPathologicalInput is the property that keeps one bad
+// input from failing the whole batch: when no batch budget fits, every input is
+// embedded on its own, in order, and the healthy ones keep their content.
+func TestEmbedWithinLimitIsolatesAPathologicalInput(t *testing.T) {
+ driver := &recordingEmbedDriver{rejectBatch: true}
+ model := newEmbedModel(t, driver, "", 8192)
+ // Read the ladder length before the call: the call's own rejections ratchet the
+ // calibration, which would shrink the budget this number is derived from.
+ budget, _ := embedBudget(t, model)
+ batchRungs := len(tokenizer.OverLimitLadder(budget))
+ texts := []string{strings.Repeat("a", 40), "second"}
+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: texts}, nil, nil)
+ if err != nil {
+ t.Fatalf("EmbedWithinLimit: %v", err)
+ }
+ if len(embeds) != 2 {
+ t.Fatalf("got %d embeddings, want 2", len(embeds))
+ }
+ if len(driver.attempts) != batchRungs+2 {
+ t.Fatalf("attempts = %d, want %d batch rungs plus 2 isolated calls", len(driver.attempts), batchRungs+2)
+ }
+ isolated := driver.attempts[batchRungs:]
+ for i, attempt := range isolated {
+ if len(attempt) != 1 {
+ t.Fatalf("isolated call %d carried %d texts, want 1", i, len(attempt))
+ }
+ // Both inputs fit the first budget, so isolation must not have modified them.
+ if attempt[0] != texts[i] {
+ t.Errorf("isolated call %d got %q, want %q", i, attempt[0], texts[i])
+ }
+ }
+}
+
+// TestEmbedWithinLimitFailsWhenNoBudgetFits keeps the floor honest: the ladder ends
+// above OverLimitFloorTokens and only isolation reaches it, so the error may only be
+// raised after the floor was actually tried - and never with a partial result.
+func TestEmbedWithinLimitFailsWhenNoBudgetFits(t *testing.T) {
+ driver := &recordingEmbedDriver{rejectLongerThan: 8}
+ model := newEmbedModel(t, driver, "", 8192)
+ budget, _ := embedBudget(t, model)
+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{strings.Repeat("中", 500)}}, nil, nil)
+ if err == nil {
+ t.Fatal("expected an error when not even the floor fits")
+ }
+ if embeds != nil {
+ t.Fatalf("partial results returned: %d embeddings", len(embeds))
+ }
+ if !strings.Contains(err.Error(), "even at 64 tokens") {
+ t.Errorf("error %q does not say the floor was tried", err)
+ }
+ want := len(tokenizer.OverLimitLadder(budget)) + len(tokenizer.OverLimitLadderToFloor(budget))
+ if len(driver.attempts) != want {
+ t.Errorf("attempts = %d, want %d (batch ladder then isolation down to the floor)", len(driver.attempts), want)
+ }
+}
+
+// TestEmbedWithinLimitRecordsTheDeclaredLimitInTheCalibration pins the third argument
+// of ObserveOverLimit: it is the provider's declared window, not the local budget
+// derived from it. Passing the (smaller) budget would understate the inferred ratio
+// and leave the calibration effectively unchanged, so the next call would repeat the
+// same rejection.
+func TestEmbedWithinLimitRecordsTheDeclaredLimitInTheCalibration(t *testing.T) {
+ driver := &recordingEmbedDriver{rejectFor: 1}
+ const declared = 8192
+ model := newEmbedModel(t, driver, "", declared)
+ if _, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{Texts: []string{strings.Repeat("a", 40000)}}, nil, nil); err != nil {
+ t.Fatalf("EmbedWithinLimit: %v", err)
+ }
+ if len(driver.attempts) == 0 {
+ t.Fatal("no attempt recorded")
+ }
+ _, counter := embedBudget(t, model)
+ own := tokenizer.OwnTokenMax(driver.attempts[0], counter)
+ want := float64(declared) / float64(own) * 1.01
+ if got := tokenizer.DefaultCalibration().RatioUpper(model.QuotaKey()); got < want-1e-9 {
+ t.Fatalf("calibration ratio = %v, want at least %v (inferred from the declared limit)", got, want)
+ }
+}
+
+// TestEmbedWithinLimitEmptyTexts: nothing to embed is a no-op, not a call.
+func TestEmbedWithinLimitEmptyTexts(t *testing.T) {
+ driver := &recordingEmbedDriver{}
+ model := newEmbedModel(t, driver, "", 8192)
+ embeds, err := model.EmbedWithinLimit(t.Context(), EmbedRequest{}, nil, nil)
+ if err != nil {
+ t.Fatalf("EmbedWithinLimit: %v", err)
+ }
+ if len(embeds) != 0 || len(driver.attempts) != 0 {
+ t.Fatalf("embeds = %d, attempts = %d, want 0 and 0", len(embeds), len(driver.attempts))
+ }
+}
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index 07f1aab413..c27e5f82e3 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -2,6 +2,7 @@ package models
import (
"context"
+ "crypto/sha256"
"encoding/json"
"errors"
"fmt"
@@ -331,6 +332,36 @@ func (m *EmbeddingModel) ResolveTokenizerID() string {
return GetEmbeddingTokenizer(name)
}
+// QuotaKey names the deployment this embedding model counts against: endpoint,
+// region, model name and an API-key prefix. The tokenizer belongs to the model, but
+// what a provider accepts is per deployment - the same model behind two endpoints
+// can have different windows - so everything that learns a real/own token ratio
+// (the ingest embedder, the dataset-nav embedder, the knowledge-compiler embedder)
+// has to key that ratio the same way; otherwise each path re-learns the same
+// rejection and none of them tightens for the others.
+func (m *EmbeddingModel) QuotaKey() string {
+ if m == nil {
+ return ""
+ }
+ var baseURL, region, apiKey, modelName string
+ if cfg := m.APIConfig; cfg != nil {
+ if cfg.BaseURL != nil {
+ baseURL = *cfg.BaseURL
+ }
+ if cfg.Region != nil {
+ region = *cfg.Region
+ }
+ if cfg.ApiKey != nil {
+ apiKey = *cfg.ApiKey
+ }
+ }
+ if m.ModelName != nil {
+ modelName = *m.ModelName
+ }
+ sum := sha256.Sum256([]byte(apiKey))
+ return fmt.Sprintf("%s|%s|%s|%x", baseURL, region, modelName, sum[:8])
+}
+
// RerankModel wraps a ModelDriver with rerank-specific configuration
type RerankModel struct {
ModelDriver ModelDriver
diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.go
index bf4ff65735..56c7f6af41 100644
--- a/internal/ingestion/task/embedder.go
+++ b/internal/ingestion/task/embedder.go
@@ -18,7 +18,6 @@ package task
import (
"context"
- "crypto/sha256"
"errors"
"fmt"
"math"
@@ -193,22 +192,7 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.E
// lets each healthy input keep its own budget. A provider that rejects even the last
// proportional step therefore still gets one attempt at the floor - per input, on the
// input that actually needs it.
-func overLimitLadder(budget int) []int {
- if budget <= 0 {
- return []int{0}
- }
- limits := []int{budget}
- for _, factor := range []float64{0.75, 0.5, 0.25, 0.125} {
- next := int(float64(budget) * factor)
- if next < overLimitFloorTokens {
- next = overLimitFloorTokens
- }
- if next < limits[len(limits)-1] {
- limits = append(limits, next)
- }
- }
- return limits
-}
+func overLimitLadder(budget int) []int { return tokenizer.OverLimitLadder(budget) }
// overLimitLadderToFloor is overLimitLadder plus the last-resort floor, for the
// per-input isolation path: an input that no proportional step fits may still fit
@@ -218,17 +202,11 @@ func overLimitLadder(budget int) []int {
// The batch loop deliberately stops above the floor. Reaching it there would trim
// EVERY input of the batch to 64 tokens just because one input is pathological,
// while isolating lets each healthy input keep its own budget.
-func overLimitLadderToFloor(budget int) []int {
- limits := overLimitLadder(budget)
- if limits[len(limits)-1] > overLimitFloorTokens {
- limits = append(limits, overLimitFloorTokens)
- }
- return limits
-}
+func overLimitLadderToFloor(budget int) []int { return tokenizer.OverLimitLadderToFloor(budget) }
// overLimitFloorTokens is the smallest budget worth trying before declaring the
// input genuinely broken rather than merely long.
-const overLimitFloorTokens = 64
+const overLimitFloorTokens = tokenizer.OverLimitFloorTokens
// trimAll re-trims every text to `limit` tokens. Texts handed in by the caller
// are already trimmed to the full budget, and trimming a prefix again just makes
@@ -404,41 +382,12 @@ func (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiter
return out, nil
}
-// overLimitMarkers are the provider wordings that mean "this input is longer
-// than the model accepts". 20015 is SiliconFlow's code for exactly that (it
-// answers a generic "The parameter is invalid" message); the rest are the usual
-// phrasings elsewhere.
-var overLimitMarkers = []string{
- "20015",
- "too long",
- "too many tokens",
- "maximum context",
- "context length",
- "context_length",
- "input length",
- "token limit",
- "reduce the length",
- "maximum allowed",
-}
-
// isOverLimitErr reports whether err is an over-limit rejection rather than a
-// rate limit or a genuine failure. Only 4xx rejections qualify: a 5xx is the
-// provider's problem and shrinking the input would not help.
-func isOverLimitErr(err error) bool {
- if err == nil {
- return false
- }
- msg := strings.ToLower(err.Error())
- if !strings.Contains(msg, "400") && !strings.Contains(msg, "413") && !strings.Contains(msg, "422") {
- return false
- }
- for _, marker := range overLimitMarkers {
- if strings.Contains(msg, marker) {
- return true
- }
- }
- return false
-}
+// rate limit or a genuine failure. The marker set and the 4xx requirement live in
+// internal/tokenizer (tokenizer.IsOverLimitError): every embedder outside this
+// package needs the same classification to drive its own shrink-and-retry, and one
+// definition is what keeps them from drifting apart.
+func isOverLimitErr(err error) bool { return tokenizer.IsOverLimitError(err) }
// Retry policy for rate-limited embedding calls.
//
@@ -526,21 +475,16 @@ func waitOutCooldown(ctx context.Context, key providerQuotaKey) error {
}
}
-// quotaKey derives the provider-instance key the cooldown is scoped to. The API
-// key decides who owns the quota, so it is part of the key but hashed: the key
-// must never carry a plaintext secret.
+// quotaKey derives the provider-instance key the cooldown and the token-ratio
+// calibration are scoped to. The API key decides who owns the quota, so it is part
+// of the key but hashed: the key must never carry a plaintext secret. The format
+// comes from the model (EmbeddingModel.QuotaKey) so every embedder - ingestion,
+// dataset nav, the knowledge compiler - learns the same ratio for a deployment.
func (e *embedder) quotaKey() providerQuotaKey {
if e == nil || e.model == nil {
return providerQuotaKey("embedder:unknown")
}
- var baseURL, region, apiKey string
- if cfg := e.model.APIConfig; cfg != nil {
- baseURL = derefString(cfg.BaseURL)
- region = derefString(cfg.Region)
- apiKey = derefString(cfg.ApiKey)
- }
- sum := sha256.Sum256([]byte(apiKey))
- return providerQuotaKey(fmt.Sprintf("%s|%s|%s|%x", baseURL, region, derefString(e.model.ModelName), sum[:8]))
+ return providerQuotaKey(e.model.QuotaKey())
}
// derefString safely dereferences an optional string.
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index ec76467572..9b9cf428c8 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -700,8 +700,10 @@ func (e *kcEmbedder) Encode(ctx context.Context, texts []string) ([][]float32, e
if jobErr := ctx.Err(); jobErr != nil {
return jobErr
}
- var embeds []models.EmbeddingData
- embeds, jobErr := mdl.ModelDriver.Embed(ctx, mdl.ModelName, models.EmbedRequest{Texts: batchTexts}, mdl.APIConfig, config, nil)
+ // Embed inside the model's window: compile products include the summaries
+ // and entity descriptions that feed the nav index, and an over-window input
+ // is a hard 400/20015 that fails the whole batch instead of being trimmed.
+ embeds, jobErr := mdl.EmbedWithinLimit(ctx, models.EmbedRequest{Texts: batchTexts}, config, nil)
if jobErr != nil {
return fmt.Errorf("knowledge_compiler: embed: %w", jobErr)
}
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 6ecc10ac1e..02e14fbd7c 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -3060,7 +3060,9 @@ type embeddingModelEmbedder struct {
func (e *embeddingModelEmbedder) Encode(ctx context.Context, texts []string) ([][]float64, error) {
config := &modelModule.EmbeddingConfig{Dimension: 0}
- embeds, err := e.embModel.ModelDriver.Embed(ctx, e.embModel.ModelName, modelModule.EmbedRequest{Texts: texts}, e.embModel.APIConfig, config, nil)
+ // Embed inside the model's window: the caller supplies arbitrary text and the
+ // provider rejects an over-window input with 400/20015 instead of truncating it.
+ embeds, err := e.embModel.EmbedWithinLimit(ctx, modelModule.EmbedRequest{Texts: texts}, config, nil)
if err != nil {
return nil, err
}
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index d6f3f5fe91..af15ff71b4 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -1414,7 +1414,9 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
if len(questionKwd) > 0 {
embeddingText = strings.Join(questionKwd, "\n")
}
- embeddings, err := embeddingModel.ModelDriver.Embed(ctx, embeddingModel.ModelName, models.EmbedRequest{Texts: []string{docName, embeddingText}}, embeddingModel.APIConfig, &models.EmbeddingConfig{Dimension: 0}, nil)
+ // Embed inside the model's window: Content arrives from the API and can be
+ // arbitrarily long, and the provider answers 400/20015 instead of truncating it.
+ embeddings, err := embeddingModel.EmbedWithinLimit(ctx, models.EmbedRequest{Texts: []string{docName, embeddingText}}, &models.EmbeddingConfig{Dimension: 0}, nil)
if err != nil {
return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("encode chunk embedding: %v", err)}
}
diff --git a/internal/service/dataset/utils.go b/internal/service/dataset/utils.go
index 204a39eddc..7cc4004a52 100644
--- a/internal/service/dataset/utils.go
+++ b/internal/service/dataset/utils.go
@@ -256,7 +256,10 @@ func datasetEncodeEmbedding(ctx context.Context, embeddingModel *modelModule.Emb
cleaned[i] = datasetCleanEmbeddingText(t)
}
embeddingConfig := &modelModule.EmbeddingConfig{Dimension: 0}
- embeddings, err := embeddingModel.ModelDriver.Embed(ctx, embeddingModel.ModelName, modelModule.EmbedRequest{Texts: cleaned}, embeddingModel.APIConfig, embeddingConfig, nil)
+ // Embed inside the model's window: this path takes whatever text the API handed
+ // it, and the provider rejects an over-window input with 400/20015 instead of
+ // truncating it.
+ embeddings, err := embeddingModel.EmbedWithinLimit(ctx, modelModule.EmbedRequest{Texts: cleaned}, embeddingConfig, nil)
if err != nil {
return nil, err
}
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index fe6e696738..cd00eae65d 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -329,7 +329,9 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
return err
}
embeddingModel := models.NewEmbeddingModel(target.Driver, &target.ModelName, target.APIConfig, target.MaxTokens)
- embeddings, err := embeddingModel.ModelDriver.Embed(ctx, embeddingModel.ModelName, models.EmbedRequest{Texts: contents}, embeddingModel.APIConfig, &models.EmbeddingConfig{Dimension: 0}, nil)
+ // Embed inside the model's window: memory contents are caller-supplied and
+ // unbounded, and the provider answers 400/20015 instead of truncating them.
+ embeddings, err := embeddingModel.EmbedWithinLimit(ctx, models.EmbedRequest{Texts: contents}, &models.EmbeddingConfig{Dimension: 0}, nil)
if err != nil {
return fmt.Errorf("embed model: %w", err)
}
diff --git a/internal/service/nav_embedder.go b/internal/service/nav_embedder.go
index 16ab21de09..125b704ddd 100644
--- a/internal/service/nav_embedder.go
+++ b/internal/service/nav_embedder.go
@@ -83,7 +83,23 @@ func (e *NavEmbedder) encode(ctx context.Context, tenantID string, texts []strin
if len(nonEmpty) == 0 {
return nil, nil
}
- embeds, err := model.ModelDriver.Embed(ctx, model.ModelName, modelModule.EmbedRequest{Texts: nonEmpty, Query: query}, model.APIConfig, nil, nil)
+ // Documents go through EmbedWithinLimit: the provider does not truncate, it
+ // answers 400/20015, and a nav summary is not a short string - without a tree
+ // product it is every entity line of the page-index graph joined into one. The
+ // model makes the cut (and retries with a smaller budget when a calibrated count
+ // undershoots), which is what Python gets from BaseEmbedding.encode.
+ //
+ // Queries stay on the driver. A query is short, so there is nothing to cut, and
+ // EmbedWithinLimit refuses to run when the model declares a tokenizer whose asset
+ // is missing - a refusal that belongs to the ingest path, not to a search, which
+ // has to keep answering on a deployment that never provisioned the asset.
+ var embeds []modelModule.EmbeddingData
+ var err error
+ if query {
+ embeds, err = model.ModelDriver.Embed(ctx, model.ModelName, modelModule.EmbedRequest{Texts: nonEmpty, Query: true}, model.APIConfig, nil, nil)
+ } else {
+ embeds, err = model.EmbedWithinLimit(ctx, modelModule.EmbedRequest{Texts: nonEmpty}, nil, nil)
+ }
if err != nil {
return nil, err
}
diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.go
index e4b669dddb..795abebfdf 100644
--- a/internal/tokenizer/embedlimit.go
+++ b/internal/tokenizer/embedlimit.go
@@ -36,6 +36,7 @@ package tokenizer
import (
"fmt"
"math"
+ "regexp"
"sort"
"strings"
"sync"
@@ -530,6 +531,155 @@ func trimByBytes(text string, limit int) string {
// utf8Start reports whether b is not a UTF-8 continuation byte.
func utf8Start(b byte) bool { return b&0xC0 != 0x80 }
+// ---------------------------------------------------------------------------
+// Over-limit handling: the marker set, the shrink ladder, and the refusal every
+// embedder shares. The embedding side of this - the loop that trims, calls the
+// driver, and walks the ladder - is EmbeddingModel.EmbedWithinLimit in
+// internal/entity/models, next to the RerankModel cut it mirrors.
+// ---------------------------------------------------------------------------
+
+// OverLimitFloorTokens is the smallest budget worth trying before calling an input
+// genuinely broken rather than merely long.
+const OverLimitFloorTokens = 64
+
+// OverLimitMarkers are the provider wordings that mean "this input is longer than
+// the model accepts". They are phrases, not codes, so a plain substring test is the
+// right one for them.
+var OverLimitMarkers = []string{
+ "too long",
+ "too many tokens",
+ "maximum context",
+ "context length",
+ "context_length",
+ "input length",
+ "token limit",
+ "reduce the length",
+ "maximum allowed",
+}
+
+// OverLimitCodes are the provider error codes that mean the same thing. 20015 is
+// SiliconFlow's: it is returned with a generic "The parameter is invalid" message, so
+// the code is the only usable signal.
+var OverLimitCodes = []string{"20015"}
+
+// overLimitStatus matches an HTTP 4xx status as a delimited number: "400 Bad
+// Request" matches, "1400" and "4000" do not.
+var overLimitStatus = regexp.MustCompile(`\b(?:400|413|422)\b`)
+
+// IsOverLimitError reports whether err is an over-limit rejection rather than a
+// rate limit or a genuine failure. Only 4xx rejections qualify: a 5xx is the
+// provider's problem and a shorter input would not fix it.
+//
+// Status codes and provider codes are matched as DELIMITED numbers. A substring test
+// would read provider code 120015 as SiliconFlow's 20015 (and 1400 as 400), and the
+// caller would answer an unrelated failure by re-embedding a silently truncated
+// input — or replace the real error with a window-limit one.
+func IsOverLimitError(err error) bool {
+ if err == nil {
+ return false
+ }
+ msg := strings.ToLower(err.Error())
+ if !overLimitStatus.MatchString(msg) {
+ return false
+ }
+ for _, marker := range OverLimitMarkers {
+ if strings.Contains(msg, marker) {
+ return true
+ }
+ }
+ for _, code := range OverLimitCodes {
+ if hasDelimitedNumber(msg, code) {
+ return true
+ }
+ }
+ return false
+}
+
+// hasDelimitedNumber reports whether s contains num as a whole number: neither
+// neighbour may be a digit, so `"code":20015,` matches while 120015 and 200150 do
+// not.
+func hasDelimitedNumber(s, num string) bool {
+ for from := 0; from+len(num) <= len(s); {
+ at := strings.Index(s[from:], num)
+ if at < 0 {
+ return false
+ }
+ at += from
+ end := at + len(num)
+ if (at == 0 || !isASCIIDigit(s[at-1])) && (end == len(s) || !isASCIIDigit(s[end])) {
+ return true
+ }
+ from = at + 1
+ }
+ return false
+}
+
+func isASCIIDigit(b byte) bool { return b >= '0' && b <= '9' }
+
+// OverLimitLadder returns the budgets to try, in order, after a provider rejects
+// an input as over its window: the caller's budget first, then progressively
+// smaller fractions of it, never below OverLimitFloorTokens. Strictly decreasing,
+// so the loop always makes progress.
+func OverLimitLadder(budget int) []int {
+ if budget <= 0 {
+ return []int{0}
+ }
+ limits := []int{budget}
+ for _, factor := range []float64{0.75, 0.5, 0.25, 0.125} {
+ next := int(float64(budget) * factor)
+ if next < OverLimitFloorTokens {
+ next = OverLimitFloorTokens
+ }
+ if next < limits[len(limits)-1] {
+ limits = append(limits, next)
+ }
+ }
+ return limits
+}
+
+// OverLimitLadderToFloor is OverLimitLadder plus the last-resort floor, for the
+// per-input isolation path: an input that no proportional step fits may still fit
+// at the floor, and trying it there is what turns "this document cannot be
+// embedded" into an embedded (truncated) chunk.
+//
+// A batch loop deliberately stops above the floor: reaching it there would trim
+// EVERY input of the batch to 64 tokens just because one input is pathological,
+// while isolating lets each healthy input keep its own budget.
+func OverLimitLadderToFloor(budget int) []int {
+ limits := OverLimitLadder(budget)
+ if limits[len(limits)-1] > OverLimitFloorTokens {
+ limits = append(limits, OverLimitFloorTokens)
+ }
+ return limits
+}
+
+// RefuseUnavailableCounter is the refusal every embedder shares: a model that
+// DECLARES a tokenizer whose asset is not on disk must not be counted with the
+// calibrated cl100k estimate, because that count belongs to a different tokenizer
+// and an under-count is what makes a provider answer 400. Silently substituting it
+// would trade away the exactness these counters exist for.
+func RefuseUnavailableCounter(tokenizerID, calibrationKey string) error {
+ if tokenizerID == "" || CounterExact(tokenizerID) {
+ return nil
+ }
+ return fmt.Errorf(
+ "embedding tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate",
+ tokenizerID, calibrationKey)
+}
+
+// OwnTokenMax is the largest single input's cost in our own counter. The
+// provider's window bounds each input individually, so this - not the batch total
+// - is what an over-limit rejection tells us about.
+func OwnTokenMax(texts []string, counter Counter) int {
+ max := 0
+ for _, t := range texts {
+ if n := counter.Count(t); n > max {
+ max = n
+ }
+ }
+ return max
+}
+
// ---------------------------------------------------------------------------
// Counters we can build without extra assets
// ---------------------------------------------------------------------------
diff --git a/internal/tokenizer/embedlimit_test.go b/internal/tokenizer/embedlimit_test.go
index be5f576213..8bc24eb489 100644
--- a/internal/tokenizer/embedlimit_test.go
+++ b/internal/tokenizer/embedlimit_test.go
@@ -17,6 +17,7 @@
package tokenizer
import (
+ "errors"
"strings"
"testing"
)
@@ -270,3 +271,51 @@ func TestCL100KTrimNeverExceedsLimit(t *testing.T) {
}
}
}
+
+// TestIsOverLimitErrorMatchesDelimitedNumbers pins the difference between a substring
+// test and a delimited one. The caller acts on this answer by re-embedding a
+// truncated input, so a false positive silently replaces a real error with a
+// window-limit one; 120015 merely contains 20015, and 1400 merely contains 400.
+func TestIsOverLimitErrorMatchesDelimitedNumbers(t *testing.T) {
+ cases := []struct {
+ name string
+ err string
+ want bool
+ }{
+ {
+ "siliconflow over-window",
+ `SILICONFLOW API error: 400 Bad Request, body: {"code":20015,"message":"The parameter is invalid. Please check again.","data":null}`,
+ true,
+ },
+ {
+ "siliconflow code as a string",
+ `SILICONFLOW API error: 400 Bad Request, body: {"code":"20015"}`,
+ true,
+ },
+ {
+ "openai wording",
+ `OpenAI embeddings API error: 400 Bad Request, body: {"error":{"message":"This model's maximum context length is 8192 tokens"}}`,
+ true,
+ },
+ {"413", `API error: 413 Request Entity Too Large, body: input is too long`, true},
+ {"422", `API error: 422 Unprocessable Entity, body: too many tokens`, true},
+ // Delimiter neighbours: a longer provider code that merely contains 20015, and
+ // a status that merely contains 400.
+ {"code 120015", `SILICONFLOW API error: 400 Bad Request, body: {"code":120015}`, false},
+ {"code 200150", `SILICONFLOW API error: 400 Bad Request, body: {"code":200150}`, false},
+ {"status 1400", `API error: 1400 Bad Request, body: too long`, false},
+ // Rate limits and provider failures must not be mistaken for size.
+ {"rate limit", `SILICONFLOW API error: 429 Too Many Requests, body: {"message":"Request was rejected due to rate limiting. Details: TPM limit reached."}`, false},
+ {"server error", `SILICONFLOW API error: 500 Internal Server Error, body: too long`, false},
+ {"unauthorized", `OpenAI embeddings API error: 401 Unauthorized, body: invalid api key`, false},
+ {"network", `failed to send request: dial tcp: connection refused`, false},
+ }
+ for _, c := range cases {
+ if got := IsOverLimitError(errors.New(c.err)); got != c.want {
+ t.Errorf("%s: IsOverLimitError = %t, want %t (%s)", c.name, got, c.want, c.err)
+ }
+ }
+ if IsOverLimitError(nil) {
+ t.Error("IsOverLimitError(nil) = true, want false")
+ }
+}