RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(embedding): respect provider batch limits (#21610)
Sbaaoui Idrissf81e671c90fa
diff --git a/internal/entity/models/embed_limit.go b/internal/entity/models/embed_limit.goindex 3b70183615..93711737d2 100644--- a/internal/entity/models/embed_limit.go+++ b/internal/entity/models/embed_limit.go@@ -24,7 +24,7 @@ import ("ragflow/internal/tokenizer")-// Embed embeds req after cutting every text to what this model accepts,+// Embed splits req at the effective input-count limit, cuts every text to what this model accepts,// and retries with a smaller budget if the provider still rejects an input as over// its window. It is the embedding counterpart of RerankModel.Rerank, which cuts its// documents the same way for the same reason.@@ -57,6 +57,44 @@ func (m *EmbeddingModel) Embed(ctx context.Context, req EmbedRequest, embeddingCif len(req.Texts) == 0 {return []EmbeddingData{}, nil}+ batchSize := m.ResolveBatchSize()+ if len(req.Texts) <= batchSize {+ return m.embedBatch(ctx, req, embeddingConfig, usage)+ }+ out := make([]EmbeddingData, 0, len(req.Texts))+ var inputTokens, outputTokens, totalTokens int+ for start := 0; start < len(req.Texts); start += batchSize {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ batch := req+ batch.Texts = req.Texts[start:min(start+batchSize, len(req.Texts))]+ var batchUsage *common.ModelUsage+ if usage != nil {+ copy := *usage+ copy.InputTokens, copy.OutputTokens, copy.TotalTokens = 0, 0, 0+ batchUsage = ©+ }+ embeds, err := m.embedBatch(ctx, batch, embeddingConfig, batchUsage)+ if usage != nil {+ inputTokens += batchUsage.InputTokens+ outputTokens += batchUsage.OutputTokens+ totalTokens += batchUsage.TotalTokens+ *usage = *batchUsage+ usage.InputTokens, usage.OutputTokens, usage.TotalTokens = inputTokens, outputTokens, totalTokens+ }+ if err != nil {+ return nil, err+ }+ for i := range embeds {+ embeds[i].Index += start+ }+ out = append(out, embeds...)+ }+ return out, nil+}++func (m *EmbeddingModel) embedBatch(ctx context.Context, req EmbedRequest, embeddingConfig *EmbeddingConfig, usage *common.ModelUsage) ([]EmbeddingData, error) {tokenizerID, quotaKey := m.ResolveTokenizerID(), m.QuotaKey()if err := tokenizer.RefuseUnavailableCounter(tokenizerID, quotaKey); err != nil {return nil, err@@ -128,6 +166,7 @@ func (m *EmbeddingModel) embedIsolating(ctx context.Context, req EmbedRequest, ereturn nil, fmt.Errorf("embedding input %d: driver returned %d embeddings for one text", i, len(embeds))}out[i], fitted = embeds[0], true+ out[i].Index = ibreak}if !tokenizer.IsOverLimitError(err) {diff --git a/internal/entity/models/types.go b/internal/entity/models/types.goindex d3ba97dd78..bbe1286263 100644--- a/internal/entity/models/types.go+++ b/internal/entity/models/types.go@@ -369,7 +369,7 @@ type EmbeddingModel struct {ModelName *stringAPIConfig *APIConfigMaxTokens int // Max input tokens for the embedding model, used for text truncation- MaxBatchSize *int // Max texts per Embed request; nil means "resolve from provider capability at use site"+ MaxBatchSize *int // Max texts per driver request; Embed resolves nil from the provider capabilityinfo *ModelInfo}@@ -401,7 +401,7 @@ func (m *EmbeddingModel) Info() *ModelInfo {return cloneModelInfo(m.info)}-// ResolveBatchSize returns the safe maximum texts per Embed request. The+// ResolveBatchSize returns the safe maximum texts per driver request. The// effective limit cannot exceed either the model's declared maximum or the// provider/runtime batch capability; these values can differ (for example, a// provider model config can declare max_batch_size=32 while its driver only