Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

33.9 KB
Raw Blame
package models

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"ragflow/internal/common"
	"ragflow/internal/tokenizer"
)

const defaultMaxRerankTokens = 8196

// ErrRerankTokenLimitPolicy identifies invalid token-limit configuration or oversized rerank input.
var ErrRerankTokenLimitPolicy = errors.New("rerank token limit policy")

// Message represents a chat message with role and content
//
//	is interface{} to support different formats:
//	 - string: plain text message (e.g., "Hello")
//	 - []interface{}: multimodal content array where each element is map[string]interface{}
//	   (e.g., [{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}])
type Message struct {
	Role         string                   `json:"role"`
	Content      interface{}              `json:"content"`
	Name         interface{}              `json:"name,omitempty"`
	ToolCallID   string                   `json:"tool_call_id,omitempty"`
	ToolCalls    []map[string]interface{} `json:"tool_calls,omitempty"`
	FunctionCall interface{}              `json:"function_call,omitempty"`
	Refusal      interface{}              `json:"refusal,omitempty"`
	Audio        interface{}              `json:"audio,omitempty"`
}

// ToolCallSession mirrors Python's common.mcp_tool_call_conn.ToolCallSession protocol.
type ToolCallSession interface {
	ToolCall(name string, arguments map[string]interface{}) (string, error)
}

// ModelDriver interface for model functionality
type ModelDriver interface {
	NewInstance(baseURL map[string]string) ModelDriver

	Name() string

	// ChatWithMessages sends multiple messages synchronously
	ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error)
	// ChatStreamlyWithSender sends multiple messages asynchronously
	ChatStreamlyWithSender(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, modelConfig *ChatConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error
	// Embed a list of texts into embeddings
	Embed(ctx context.Context, modelName *string, request EmbedRequest, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig, modelUsage *common.ModelUsage) ([]EmbeddingData, error)
	// Rerank calculates similarity scores between query and texts
	Rerank(ctx context.Context, modelName *string, request RerankRequest, apiConfig *APIConfig, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error)
	// TranscribeAudio transcribe audio
	TranscribeAudio(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage) (*ASRResponse, error)
	TranscribeAudioWithSender(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error
	// AudioSpeech convert text to audio
	AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error)
	AudioSpeechWithSender(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error
	// OCRFile OCR file
	OCRFile(ctx context.Context, modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig, modelUsage *common.ModelUsage) (*OCRFileResponse, error)
	// ParseFile parse file
	ParseFile(ctx context.Context, modelName *string, content []byte, url *string, apiConfig *APIConfig, parseFileConfig *ParseFileConfig, modelUsage *common.ModelUsage) (*ParseFileResponse, error)
	// ListModels List supported models
	ListModels(ctx context.Context, apiConfig *APIConfig) ([]ListModelResponse, error)

	Balance(ctx context.Context, apiConfig *APIConfig) (map[string]interface{}, error)

	CheckConnection(ctx context.Context, apiConfig *APIConfig) error

	ListTasks(ctx context.Context, apiConfig *APIConfig) ([]ListTaskStatus, error)

	ShowTask(ctx context.Context, taskID string, apiConfig *APIConfig) (*TaskResponse, error)

	Decide(ctx context.Context, modelName *string, request *DecideRequest, apiConfig *APIConfig, modelUsage *common.ModelUsage) (*DecideResponse, error)
}

type ChatResponse struct {
	Answer        *string                  `json:"answer"`
	ReasonContent *string                  `json:"reason_content"`
	ToolCalls     []map[string]interface{} `json:"tool_calls,omitempty"`
	Usage         *TokenUsage              `json:"usage,omitempty"`
}

// TokenUsage holds token usage split for one LLM call. Consumed by
// LLMBundle for accurate Langfuse reporting and run aggregation.
// Mirrors Python's common.token_utils.usage_from_response() split.
type TokenUsage struct {
	PromptTokens     int `json:"prompt_tokens"  mapstructure:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"  mapstructure:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"  mapstructure:"total_tokens"`
}

type EmbeddingData struct {
	Embedding []float64 `json:"embedding"`
	Index     int       `json:"index"`
	// TokenCount is what this input cost, taken from the provider's reported
	// usage. Embedding APIs report usage per *request*, not per input, so the
	// ingest path distributes the request total across the inputs it sent
	// (internal/ingestion/task/embedder.go). It stays 0 for providers that
	// report no usage at all — by design, rather than inventing a number.
	TokenCount int `json:"token_count"`
}

type RerankResult struct {
	Index          int     `json:"index"`
	RelevanceScore float64 `json:"relevance_score"`
}

type RerankResponse struct {
	Data []RerankResult `json:"data"`
}

type ASRResponse struct {
	Text string `json:"text"`
}

type TTSResponse struct {
	Audio []byte `json:"audio"`
	// MediaType is the MIME type of Audio (e.g. "audio/mpeg", "audio/wav").
	// Empty means the caller's default (audio/mpeg).
	MediaType string `json:"media_type,omitempty"`
}

// OCRPage retains the source image dimensions and reading-order blocks.
// Index is the zero-based source PDF page index, including blank pages.
type OCRPage struct {
	Index  int        `json:"index"`
	Width  float64    `json:"width"`
	Height float64    `json:"height"`
	Blocks []OCRBlock `json:"blocks"`
}

// OCRBlock carries a layout region in source-image pixels. BBox is
// [left, top, right, bottom]; an absent box is unlocated. Drivers normalize
// provider coordinates to this rectangle before returning blocks.
type OCRBlock struct {
	Text  string    `json:"text"`
	Label string    `json:"label,omitempty"`
	BBox  []float64 `json:"bbox,omitempty"`
}

type OCRFileResponse struct {
	// Text is empty (but non-nil) when the response carries only media regions.
	Text *string `json:"text"`
	// Pages is optional structured output for internal parsing; nil means unavailable.
	Pages []OCRPage `json:"-"`
	// Warnings reports partial failures internally; the public response stays text-only.
	Warnings []string `json:"-"`
}

type ListModelResponse struct {
	Name          string         `json:"name"`
	ContextLength *int           `json:"context_length"`
	MaxOutput     *int           `json:"max_output"`
	ModelTypes    []string       `json:"model_types"`
	Thinking      *ModelThinking `json:"thinking"`
	MaxDimension  *int           `json:"max_dimension"`  // used by embedding models
	MaxBatchSize  *int           `json:"max_batch_size"` // used by embedding models
	Dimensions    []int          `json:"dimensions"`
}

type ParseFileResponse struct {
	TaskID string `json:"task_id"`
}

type ListTaskStatus struct {
	TaskID string `json:"task_id"`
	Status string `json:"status"`
}

type TaskSegment struct {
	Index   int    `json:"index"`
	Content string `json:"content"`
}

type TaskResponse struct {
	Segments []TaskSegment `json:"segments"`
}

type Question interface {
	isQuestion()
}

type Noul struct {
	Instructions any           `json:"instructions"`
	Criteria     *NoulCriteria `json:"criteria,omitempty"`
}

func (Noul) isQuestion() {}

func (q Noul) MarshalJSON() ([]byte, error) {
	type question Noul
	return json.Marshal(struct {
		Type string `json:"type"`
		question
	}{Type: "noul", question: question(q)})
}

type NoulCriteria struct {
	True  any `json:"true,omitzero"`
	False any `json:"false,omitzero"`
}

type Choice struct {
	Instructions any
	Criteria     any
}

func (Choice) isQuestion() {}

type Score struct {
	Instructions any
	Criteria     any
}

func (Score) isQuestion() {}

type Box struct {
	Instructions any
}

func (Box) isQuestion() {}

type DecideRequest struct {
	State     any                 `json:"state"`
	Questions map[string]Question `json:"questions"`
}

// MaxDecisionQuestions is the System One endpoint's per-request question limit.
const MaxDecisionQuestions = 16

type DecisionAnswer struct {
	Type string   `json:"type"`
	Noul *float64 `json:"noul"`
}

type DecideResponse struct {
	ID      string                    `json:"id"`
	Answers map[string]DecisionAnswer `json:"answers"`
	Usage   struct {
		InputTokens  int `json:"input_tokens"`
		OutputTokens int `json:"output_tokens"`
	} `json:"usage"`
}

type ModelListItem struct {
	ID            string `json:"id"`
	ContextLength *int   `json:"context_length"`
	Object        string `json:"object"`
	OwnedBy       string `json:"owned_by"`
}

type ModelList struct {
	Object string          `json:"object"`
	Models []ModelListItem `json:"data"`
}

// URLSuffix represents the URL suffixes for different API endpoints
type URLSuffix struct {
	Chat        string `json:"chat"`
	AsyncChat   string `json:"async_chat"`
	AsyncResult string `json:"async_result"`
	Embedding   string `json:"embedding"`
	Rerank      string `json:"rerank"`
	TTS         string `json:"tts"`
	ASR         string `json:"asr"`
	OCR         string `json:"ocr"`
	// Decision is the TypeSafe System One path: a separate endpoint from chat
	// because it takes one request carrying many questions, not a message list.
	Decision      string `json:"decision"`
	DocumentParse string `json:"doc_parse"`
	Models        string `json:"models"`
	Balance       string `json:"balance"`
	Files         string `json:"files"`
	Status        string `json:"status"`
	Tasks         string `json:"tasks"`
	Task          string `json:"task"`
}

type ChatConfig struct {
	Stream           *bool
	Vision           *bool
	Thinking         *bool
	MaxTokens        *int
	Temperature      *float64
	TopP             *float64
	PresencePenalty  *float64
	FrequencyPenalty *float64
	DoSample         *bool
	Stop             *[]string
	ModelClass       *string
	Effort           *string
	Verbosity        *string
	Tools            interface{}               `json:"tools,omitempty"`
	ToolChoice       *string                   `json:"tool_choice,omitempty"`
	ToolChoiceValue  any                       `json:"-"`
	ToolCallsResult  *[]map[string]interface{} `json:"-"`
	// UsageResult receives the token usage extracted from the final
	// streaming chunk when stream_options.include_usage is true.
	// The ChatStreamlyWithSender driver writes to this pointer (if
	// non-nil) after the stream completes; callers read it the same
	// way they read ToolCallsResult.
	UsageResult *TokenUsage `json:"-"`
}

type APIConfig struct {
	ApiKey  *string
	Region  *string
	BaseURL *string
}

type EmbedRequest struct {
	Texts  []string // for text
	Images [][]byte // for image
	Urls   []string // for image
	// Query selects the query-side encoding for providers that embed queries
	// and documents differently (Python's LLMBundle.encode_queries vs encode):
	// Cohere/Bedrock-Cohere input_type=search_query, Voyage input_type=query,
	// Jina task=retrieval.query, NVIDIA input_type=query, DashScope
	// text_type=query. Providers without an asymmetric mode ignore it.
	Query bool
}

type EmbeddingConfig struct {
	Dimension      int
	EncodingFormat string
}

type RerankRequest struct {
	Query         string  // for text question
	ImageQuery    []byte  // for image
	ImageQueryURL *string // for image

	Documents []string // for text candidates
	Images    [][]byte // for image candidates
	ImageURLs []string // for image candidates
}

type RerankConfig struct {
	TopN int
}

type ASRConfig struct {
	Params map[string]interface{} `json:"params"`
}

type TTSConfig struct {
	Format string                 `json:"format"`
	Params map[string]interface{} `json:"params"`
}

type OCRConfig struct {
	Algorithm string
	// Pages restricts OCR to these 1-indexed inclusive [from, to] ranges, the
	// parser setup's "pages" shape. Empty means the whole document. Drivers
	// whose backend cannot select pages ignore it.
	Pages [][]int
}

type ParseFileConfig struct {
	ParseMethod string `json:"parse_method"`
	Backend     string `json:"backend"`
	ServerURL   string `json:"server_url"`
}

// EmbeddingModel wraps a ModelDriver with embedding-specific configuration
type EmbeddingModel struct {
	ModelDriver  ModelDriver
	ModelName    *string
	APIConfig    *APIConfig
	MaxTokens    int  // Max input tokens for the embedding model, used for text truncation
	MaxBatchSize *int // Max texts per driver request; Embed resolves nil from the provider capability
	info         *ModelInfo
}

// NewEmbeddingModel creates a new EmbeddingModel
func NewEmbeddingModel(driver ModelDriver, modelName *string, apiConfig *APIConfig, maxTokens int) *EmbeddingModel {
	return &EmbeddingModel{
		ModelDriver: driver,
		ModelName:   modelName,
		APIConfig:   apiConfig,
		MaxTokens:   maxTokens,
	}
}

// NewEmbeddingModelWithInfo creates an embedding model with resolved metadata.
func NewEmbeddingModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, maxTokens int, info *ModelInfo) *EmbeddingModel {
	model := NewEmbeddingModel(driver, modelName, apiConfig, maxTokens)
	model.info = cloneModelInfo(info)
	if info != nil && info.Catalog != nil {
		model.MaxBatchSize = info.Catalog.MaxBatchSize
	}
	return model
}

// Info returns metadata for this selected embedding model.
func (m *EmbeddingModel) Info() *ModelInfo {
	if m == nil {
		return nil
	}
	return cloneModelInfo(m.info)
}

// ResolveBatchSize returns the safe maximum texts per driver request. The
// effective limit cannot exceed either the model's declared maximum or the
// provider/runtime batch capability; these values can differ (for example, a
// provider model config can declare max_batch_size=32 while its driver only
// accepts the default batch size of 16).
func (m *EmbeddingModel) ResolveBatchSize() int {
	var name string
	if m != nil && m.ModelName != nil {
		name = *m.ModelName
	}
	batchSize := GetEmbeddingBatchSize(name)
	if m != nil && m.MaxBatchSize != nil && *m.MaxBatchSize > 0 && *m.MaxBatchSize < batchSize {
		return *m.MaxBatchSize
	}
	return batchSize
}

// ResolveMaxTokens is ResolveBatchSize's counterpart for the input window: the
// model's own declaration wins, then the provider catalog's context_length, then
// 0, which tells the caller to apply its own default. Deliberately not 8192:
// the catalog has embedding models with 512-token windows, and overshooting a
// window is a rejected request while undershooting only truncates.
func (m *EmbeddingModel) ResolveMaxTokens() int {
	if m == nil {
		return 0
	}
	if m.MaxTokens > 0 {
		return m.MaxTokens
	}
	var name string
	if m.ModelName != nil {
		name = *m.ModelName
	}
	return GetEmbeddingMaxTokens(name)
}

// ResolveTokenizerID returns the tokenizer family declared for this model, or ""
// when the model's own tokenizer is unknown (the caller then counts with cl100k
// and a calibrated ratio).
func (m *EmbeddingModel) ResolveTokenizerID() string {
	if m == nil {
		return ""
	}
	var name string
	if m.ModelName != nil {
		name = *m.ModelName
	}
	return GetModelTokenizer(name)
}

// QuotaKey names the deployment this embedding model counts against: provider,
// endpoint, region, model name and an API-key prefix. The tokenizer belongs to the
// model, but what a provider accepts is per deployment - the same model behind two
// endpoints can have different windows - so everything that learns a real/own token
// ratio (the ingest embedder, the dataset-nav embedder, the knowledge-compiler
// embedder) has to key that ratio the same way; otherwise each path re-learns the
// same rejection and none of them tightens for the others.
func (m *EmbeddingModel) QuotaKey() string {
	if m == nil {
		return ""
	}
	return quotaKeyFor(providerNameOf(m.info), m.APIConfig, m.ModelName)
}

// providerNameOf is the provider a model was resolved from, or "" when it was built
// without catalog metadata (tests, direct construction).
func providerNameOf(info *ModelInfo) string {
	if info == nil {
		return ""
	}
	return info.ProviderName
}

// quotaKeyFor is the shared key used by every model that calibrates a real/own
// token ratio against a provider deployment: provider, resolved endpoint, region,
// model name and a short API-key digest. The tokenizer belongs to the model, but the
// accepted window belongs to the deployment, and the digest keeps two tenants of the
// same endpoint from sharing one learned ratio.
//
// providerName matters when BaseURL is empty: the driver then falls back to the
// endpoint its provider config supplies for that region, so provider + region is what
// identifies the deployment. Without it two providers that resolve to different
// default endpoints but agree on region, model and key would share one learned ratio,
// and the tighter window would over-truncate the other's requests.
func quotaKeyFor(providerName string, apiConfig *APIConfig, modelName *string) string {
	var baseURL, region, apiKey, name string
	if apiConfig != nil {
		if apiConfig.BaseURL != nil {
			baseURL = *apiConfig.BaseURL
		}
		if apiConfig.Region != nil {
			region = *apiConfig.Region
		}
		if apiConfig.ApiKey != nil {
			apiKey = *apiConfig.ApiKey
		}
	}
	if modelName != nil {
		name = *modelName
	}
	sum := sha256.Sum256([]byte(apiKey))
	return fmt.Sprintf("%s|%s|%s|%s|%x", providerName, baseURL, region, name, sum[:8])
}

// RerankModel wraps a ModelDriver with rerank-specific configuration
type RerankModel struct {
	ModelDriver ModelDriver
	ModelName   *string
	APIConfig   *APIConfig
	MaxTokens   int

	limiter    tokenizer.Limiter
	limiterErr error
	info       *ModelInfo
}

// NewRerankModel creates a new RerankModel
func NewRerankModel(driver ModelDriver, modelName *string, apiConfig *APIConfig, maxTokens int) *RerankModel {
	if maxTokens <= 0 {
		maxTokens = defaultMaxRerankTokens
	}
	model := &RerankModel{
		ModelDriver: driver,
		ModelName:   modelName,
		APIConfig:   apiConfig,
		MaxTokens:   maxTokens,
	}
	model.initLimiter()
	return model
}

// NewRerankModelWithInfo creates a rerank model with resolved metadata.
func NewRerankModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, maxTokens int, info *ModelInfo) *RerankModel {
	model := NewRerankModel(driver, modelName, apiConfig, maxTokens)
	model.info = cloneModelInfo(info)
	// Rebuild the limiter now that the provider is known: it is part of the
	// calibration key (see quotaKeyFor), and it was built without it above.
	model.initLimiter()
	return model
}

// initLimiter binds the model's declared tokenizer and its deployment calibration
// key. Safe to call more than once; the constructors call it after the model's
// metadata is set so the key can include the provider.
func (r *RerankModel) initLimiter() {
	tokenizerID := ""
	if r.ModelName != nil {
		tokenizerID = GetModelTokenizer(*r.ModelName)
	}
	// The calibration key is the deployment, not the model: rerank and embedding
	// that share an endpoint still learn separate ratios, but every rerank caller
	// of one deployment keys the same ratio and tightens it together (see
	// RerankModel.Rerank, which records over-limit rejections against it).
	quotaKey := r.QuotaKey()
	r.limiter = tokenizer.LimiterFor(tokenizerID, quotaKey, tokenizer.DefaultCalibration())
	r.limiterErr = tokenizer.RefuseUnavailableCounter(tokenizerID, quotaKey)
}

// Info returns metadata for this selected rerank model.
func (r *RerankModel) Info() *ModelInfo {
	if r == nil {
		return nil
	}
	return cloneModelInfo(r.info)
}

// QuotaKey names the deployment this rerank model counts against, so its
// calibration is keyed the same way as an embedding model on the same endpoint.
func (r *RerankModel) QuotaKey() string {
	if r == nil {
		return ""
	}
	return quotaKeyFor(providerNameOf(r.info), r.APIConfig, r.ModelName)
}

// Rerank calculates similarity between query and texts. Rerank input is measured
// and truncated with the model's declared tokenizer, just like embedding input.
// An over-limit rejection is recovered like the embedding path: the documents are
// re-cut with a smaller budget and the calibration is told, as an attempt - if every
// budget is rejected the provider's error is still returned.
//
// The window bounds the query PLUS each document, not a single text, so only the
// documents can shrink - the query is fixed. RERANK_TOKEN_LIMIT_MODE decides
// whether that is wanted at all: "passthrough" sends the input untouched, and
// "raise_error" reports an oversized document instead of trimming it, so neither
// retries; only "truncate" (the default) shrinks-and-retries.
func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) {
	if r == nil || r.ModelDriver == nil {
		return nil, errors.New("rerank model: driver is nil")
	}
	maxTokens := r.MaxTokens
	if maxTokens <= 0 {
		maxTokens = defaultMaxRerankTokens
	}
	mode := strings.ToLower(strings.TrimSpace(common.GetEnv(common.EnvRerankTokenLimitMode)))
	if mode == "" {
		mode = "truncate"
	}
	if mode != "truncate" && mode != "passthrough" && mode != "raise_error" {
		return nil, fmt.Errorf("%w: invalid %s %q; expected %q, %q, or %q", ErrRerankTokenLimitPolicy, common.EnvRerankTokenLimitMode, mode, "truncate", "passthrough", "raise_error")
	}
	// passthrough opted out of measuring the input; an empty document list has
	// nothing to cut. Neither needs the tokenizer counter, so the "declared
	// tokenizer asset is missing" refusal does not apply to them.
	if mode == "passthrough" || len(request.Documents) == 0 {
		return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)
	}
	// Both remaining modes count tokens, so an unavailable declared tokenizer is a
	// refusal rather than a silent fallback to the calibrated estimate.
	if r.limiterErr != nil {
		return nil, r.limiterErr
	}
	limiter := r.limiter
	counter := limiter.Counter()
	effectiveMaxTokens := limiter.Limit(maxTokens)

	queryTokens := counter.Count(request.Query)
	documentBudget := max(effectiveMaxTokens-queryTokens, 0)

	if mode == "raise_error" {
		for i, document := range request.Documents {
			inputTokens := queryTokens + counter.Count(document)
			if inputTokens > effectiveMaxTokens {
				return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, effectiveMaxTokens)
			}
		}
		return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)
	}

	// mode == "truncate": pre-cut each document to the window, then shrink and
	// retry if the provider still rejects an input as over its window. A provider
	// that rejects a window computed from its own declared limit is telling us the
	// declared limit was too optimistic (an estimate, or a smaller real window);
	// re-cutting keeps the call alive, and the observation tightens the next one.
	original := request.Documents
	cal, calKey := limiter.Calibration()
	var lastErr error
	for _, budget := range tokenizer.OverLimitLadder(documentBudget) {
		attempt := request
		attempt.Documents = make([]string, len(original))
		for i, document := range original {
			attempt.Documents[i] = counter.TrimToLimit(document, budget)
		}
		resp, err := r.ModelDriver.Rerank(ctx, r.ModelName, attempt, r.APIConfig, rerankConfig, modelUsage)
		if err == nil {
			return resp, nil
		}
		if !tokenizer.IsOverLimitError(err) {
			return nil, err
		}
		// The provider rejected the documents we actually sent, so the true ratio is
		// at least maxTokens / our count of that attempt's largest pair - not the
		// original's, which is longer and would understate the ratio.
		cal.ObserveOverLimit(calKey, queryTokens+tokenizer.OwnTokenMax(attempt.Documents, counter), maxTokens)
		lastErr = err
	}
	return nil, lastErr
}

// OCRModel wraps a ModelDriver with optical-character-recognition configuration.
type OCRModel struct {
	ModelDriver ModelDriver
	ModelName   *string
	APIConfig   *APIConfig
	info        *ModelInfo
}

// NewOCRModel creates an OCR model wrapper.
func NewOCRModel(driver ModelDriver, modelName *string, apiConfig *APIConfig) *OCRModel {
	return &OCRModel{ModelDriver: driver, ModelName: modelName, APIConfig: apiConfig}
}

// NewOCRModelWithInfo creates an OCR model with resolved metadata.
func NewOCRModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, info *ModelInfo) *OCRModel {
	model := NewOCRModel(driver, modelName, apiConfig)
	model.info = cloneModelInfo(info)
	return model
}

// Info returns metadata for this selected OCR model.
func (m *OCRModel) Info() *ModelInfo {
	if m == nil {
		return nil
	}
	return cloneModelInfo(m.info)
}

// OCRFile extracts text from an image or document.
func (m *OCRModel) OCRFile(ctx context.Context, content []byte, url *string, config *OCRConfig, usage *common.ModelUsage) (*OCRFileResponse, error) {
	if m == nil || m.ModelDriver == nil {
		return nil, errors.New("OCR model: driver is nil")
	}
	return m.ModelDriver.OCRFile(ctx, m.ModelName, content, url, m.APIConfig, config, usage)
}

// ASRModel wraps a ModelDriver with speech-to-text configuration.
type ASRModel struct {
	ModelDriver ModelDriver
	ModelName   *string
	APIConfig   *APIConfig
	info        *ModelInfo
}

// NewASRModel creates a new ASRModel.
func NewASRModel(driver ModelDriver, modelName *string, apiConfig *APIConfig) *ASRModel {
	return &ASRModel{ModelDriver: driver, ModelName: modelName, APIConfig: apiConfig}
}

// NewASRModelWithInfo creates an ASR model with resolved metadata.
func NewASRModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, info *ModelInfo) *ASRModel {
	model := NewASRModel(driver, modelName, apiConfig)
	model.info = cloneModelInfo(info)
	return model
}

// Info returns metadata for this selected ASR model.
func (m *ASRModel) Info() *ModelInfo {
	if m == nil {
		return nil
	}
	return cloneModelInfo(m.info)
}

// Transcribe converts audio to text.
func (m *ASRModel) Transcribe(ctx context.Context, audioFile *string, config *ASRConfig, usage *common.ModelUsage) (*ASRResponse, error) {
	return m.ModelDriver.TranscribeAudio(ctx, m.ModelName, audioFile, m.APIConfig, config, usage)
}

// TranscribeWithSender streams transcription results through sender.
func (m *ASRModel) TranscribeWithSender(ctx context.Context, audioFile *string, config *ASRConfig, usage *common.ModelUsage, sender func(*string, *string) error) error {
	return m.ModelDriver.TranscribeAudioWithSender(ctx, m.ModelName, audioFile, m.APIConfig, config, usage, sender)
}

// TTSModel wraps a ModelDriver with text-to-speech configuration.
type TTSModel struct {
	ModelDriver ModelDriver
	ModelName   *string
	APIConfig   *APIConfig
	info        *ModelInfo
}

// NewTTSModel creates a new TTSModel.
func NewTTSModel(driver ModelDriver, modelName *string, apiConfig *APIConfig) *TTSModel {
	return &TTSModel{ModelDriver: driver, ModelName: modelName, APIConfig: apiConfig}
}

// NewTTSModelWithInfo creates a TTS model with resolved metadata.
func NewTTSModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, info *ModelInfo) *TTSModel {
	model := NewTTSModel(driver, modelName, apiConfig)
	model.info = cloneModelInfo(info)
	return model
}

// Info returns metadata for this selected TTS model.
func (m *TTSModel) Info() *ModelInfo {
	if m == nil {
		return nil
	}
	return cloneModelInfo(m.info)
}

// Speech converts text to audio.
func (m *TTSModel) Speech(ctx context.Context, audioContent *string, config *TTSConfig, usage *common.ModelUsage) (*TTSResponse, error) {
	return m.ModelDriver.AudioSpeech(ctx, m.ModelName, audioContent, m.APIConfig, config, usage)
}

// SpeechWithSender streams synthesized audio through sender.
func (m *TTSModel) SpeechWithSender(ctx context.Context, audioContent *string, config *TTSConfig, usage *common.ModelUsage, sender func(*string, *string) error) error {
	return m.ModelDriver.AudioSpeechWithSender(ctx, m.ModelName, audioContent, m.APIConfig, config, usage, sender)
}

// ToolConfig bundles tool-calling configuration for a ChatModel.
type ToolConfig struct {
	Tools           string          // JSON-encoded tools list
	MaxRounds       int             // max tool-calling rounds (default: 5)
	MaxRetries      int             // max retries on failure (default: 3)
	ToolCallSession ToolCallSession // session that executes tool calls
	// TerminalTools names tools whose successful result is already the final
	// answer. When a round executes one of them, the loop stops and returns
	// that result instead of feeding it back for another model round. Mirrors
	// Python's chat_mdl.terminal_tools short-circuit (chat_model.py:619-627).
	// Empty disables the short-circuit (existing behaviour).
	TerminalTools map[string]struct{}
}

// ChatModel wraps a ModelDriver with chat-specific configuration
type ChatModel struct {
	ModelDriver ModelDriver
	ModelName   *string
	APIConfig   *APIConfig
	ToolConfig  *ToolConfig
	info        *ModelInfo
}

// NewChatModel creates a new ChatModel
func NewChatModel(driver ModelDriver, modelName *string, apiConfig *APIConfig) *ChatModel {
	return &ChatModel{
		ModelDriver: driver,
		ModelName:   modelName,
		APIConfig:   apiConfig,
	}
}

// NewChatModelWithInfo creates a chat model with resolved metadata.
func NewChatModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, info *ModelInfo) *ChatModel {
	model := NewChatModel(driver, modelName, apiConfig)
	model.info = cloneModelInfo(info)
	return model
}

// Info returns metadata for this selected chat model.
func (m *ChatModel) Info() *ModelInfo {
	if m == nil {
		return nil
	}
	return cloneModelInfo(m.info)
}

// ChatWithMessages sends a non-streaming chat request through the model wrapper.
func (m *ChatModel) ChatWithMessages(ctx context.Context, messages []Message, config *ChatConfig, usage *common.ModelUsage) (*ChatResponse, error) {
	if m == nil || m.ModelDriver == nil {
		return nil, errors.New("chat model: driver is nil")
	}
	config = m.withDefaults(config)
	if err := m.validateMaxOutput(config); err != nil {
		return nil, err
	}
	modelName := ""
	if m.ModelName != nil {
		modelName = *m.ModelName
	}
	return m.ModelDriver.ChatWithMessages(ctx, modelName, messages, m.APIConfig, config, usage)
}

// ChatStreamlyWithSender streams chat deltas through sender.
func (m *ChatModel) ChatStreamlyWithSender(ctx context.Context, messages []Message, config *ChatConfig, usage *common.ModelUsage, sender func(*string, *string) error) error {
	if m == nil || m.ModelDriver == nil {
		return errors.New("chat model: driver is nil")
	}
	config = m.withDefaults(config)
	if err := m.validateMaxOutput(config); err != nil {
		return err
	}
	modelName := ""
	if m.ModelName != nil {
		modelName = *m.ModelName
	}
	return m.ModelDriver.ChatStreamlyWithSender(ctx, modelName, messages, m.APIConfig, config, usage, sender)
}

func (m *ChatModel) withDefaults(config *ChatConfig) *ChatConfig {
	if config == nil {
		config = &ChatConfig{}
	}
	if m.info == nil {
		return config
	}
	if config.ModelClass == nil && m.info.ModelClass != "" {
		modelClass := m.info.ModelClass
		config.ModelClass = &modelClass
	}
	if config.Thinking == nil && m.info.Thinking != nil {
		thinking := m.info.Thinking.DefaultValue
		config.Thinking = &thinking
	}
	return config
}

func (m *ChatModel) validateMaxOutput(config *ChatConfig) error {
	if m == nil || m.info == nil || m.info.MaxOutput <= 0 || config == nil || config.MaxTokens == nil {
		return nil
	}
	if *config.MaxTokens > m.info.MaxOutput {
		return fmt.Errorf("chat max_tokens %d exceeds model max_output %d", *config.MaxTokens, m.info.MaxOutput)
	}
	return nil
}

// BindTools registers tools for the ChatModel to call.
// Mirrors Python's Base.bind_tools() in rag/llm/chat_model.py.
func (cm *ChatModel) BindTools(session ToolCallSession, tools interface{}) {
	// Serialize tools to JSON if it's a list/map.
	toolsJSON := ""
	switch v := tools.(type) {
	case string:
		toolsJSON = v
	case []byte:
		toolsJSON = string(v)
	default:
		if b, err := json.Marshal(tools); err == nil {
			toolsJSON = string(b)
		}
	}
	cm.ToolConfig = &ToolConfig{
		Tools:           toolsJSON,
		MaxRounds:       defaultMaxRounds,
		MaxRetries:      defaultMaxRetries,
		ToolCallSession: session,
	}
}

// SetTerminalTools marks the named tools as terminal: once one executes
// successfully, the tool loop stops and returns its result as the final answer
// rather than re-invoking the model. Mirrors Python
// `chat_mdl.mdl.terminal_tools = {...}`. Call after BindTools.
func (cm *ChatModel) SetTerminalTools(names ...string) {
	if cm.ToolConfig == nil {
		return
	}
	term := make(map[string]struct{}, len(names))
	for _, n := range names {
		term[n] = struct{}{}
	}
	cm.ToolConfig.TerminalTools = term
}

// ModelInfo describes the selected model without exposing its driver
// credentials. ContextLength is the input/context window; MaxOutput is the
// provider's maximum generated-token budget.
type ModelInfo struct {
	ID            string
	Name          string
	ProviderName  string
	InstanceID    string
	InstanceName  string
	ModelTypes    []string
	ContextLength int
	MaxOutput     int
	SupportsTools bool
	ModelClass    string
	Thinking      *ModelThinking
	Catalog       *Model
}

func cloneModelInfo(info *ModelInfo) *ModelInfo {
	if info == nil {
		return nil
	}
	copy := *info
	copy.ModelTypes = append([]string(nil), info.ModelTypes...)
	if info.Thinking != nil {
		thinking := *info.Thinking
		copy.Thinking = &thinking
	}
	if info.Catalog != nil {
		catalog := *info.Catalog
		catalog.ModelTypes = append([]string(nil), info.Catalog.ModelTypes...)
		catalog.Dimensions = append([]int(nil), info.Catalog.Dimensions...)
		catalog.Alias = append([]string(nil), info.Catalog.Alias...)
		copy.Catalog = &catalog
	}
	return &copy
}