Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

73.2 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

// Package component — Extractor component.
//
// SCOPE:
//
//   - PROVIDER-AGNOSTIC: the Extractor does NOT depend on any
//     specific LLM provider. It dispatches every chat call through
//     internal/entity/models.
//
//   - CONCURRENCY: auto extraction tasks (keywords, questions, summary, metadata)
//     run concurrently across chunks via a bounded worker pool (extractorPool).
//
//   - TIMEOUT / ELAPSED: each LLM call carries its own budget
//     (extractorChatTimeout) instead of one node-level deadline, so the
//     total work scales with the chunk count — a 3000-chunk document
//     legitimately needs longer than a 3-chunk one. A call that exceeds
//     its budget skips its chunk instead of failing the document; task
//     cancellation still aborts via ctx. Outputs are stamped with
//     _created_time / _elapsed_time by runtime.TrackElapsed upstream.
package component

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	eschema "github.com/cloudwego/eino/schema"
	"go.uber.org/zap"
	"gorm.io/gorm"

	"ragflow/internal/agent/runtime"
	"ragflow/internal/common"
	"ragflow/internal/entity"
	"ragflow/internal/entity/models"
	"ragflow/internal/ingestion/chunkcache"
	"ragflow/internal/ingestion/component/progress"
	"ragflow/internal/ingestion/component/schema"
	"ragflow/internal/modelcontext"
	"ragflow/internal/tokenizer"
	"ragflow/internal/utility"
)

// componentNameExtractor is the canonical Extractor component name; the value
// lives in common and is re-exported by the schema package so both layers
// reference the same identity.
const componentNameExtractor = schema.ComponentNameExtractor

// defaultExtractorChatTimeout bounds one Extractor LLM call, including
// retries and backoff. Tag extraction has its own timeout.
const defaultExtractorChatTimeout = 180 * time.Second

// extractorChatTimeout resolves the per-call LLM budget from
// EXTRACTOR_CHAT_TIMEOUT (seconds), falling back to
// defaultExtractorChatTimeout on unset/invalid values.
func extractorChatTimeout() time.Duration {
	if v := os.Getenv(common.EnvExtractorChatTimeout); v != "" {
		if n, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64); err == nil && n > 0 && n <= math.MaxInt64/int64(time.Second) {
			return time.Duration(n) * time.Second
		}
	}
	return defaultExtractorChatTimeout
}

// extractorRetryMax and extractorRetryDelay are package-level vars
// so tests can override them (extractorRetryDelay → time.Millisecond)
// to avoid multi-second retry sleeps. Production defaults match
// common.RetryWithBackoff defaults (3 retries, 2s initial delay).
var (
	extractorRetryMax   = common.DefaultRetryMax
	extractorRetryDelay = common.DefaultRetryDelay
)

// extractorTemperature mirrors Python's 0.2 default for keyword
// and question extraction calls (generator.py:230,245).
const extractorTemperature = 0.2

// defaultExtractorConcurrency bounds concurrent chunk extraction jobs
// process-wide to reduce provider rate-limit pressure.
const defaultExtractorConcurrency = 10

// extractorJob is one chunk's auto-extraction unit of work.
type extractorJob func() error

// extractorPool bounds chunk extraction concurrency across all Extractor
// invocations in this process: keywords, LLM tags, questions, summary, and
// metadata share the same workers. Each invocation waits for its own futures.
var extractorPool = utility.NewWorkerPool[extractorJob, struct{}](
	extractorConcurrency(),
	extractorConcurrency()*4,
	func(_ context.Context, j extractorJob) (struct{}, error) { return struct{}{}, j() },
)

// extractorConcurrency resolves the pool size from EXTRACTOR_CONCURRENCY
// with a safe default of defaultExtractorConcurrency.
func extractorConcurrency() int {
	if v := os.Getenv(common.EnvExtractorConcurrency); v != "" {
		if n, err := strconv.Atoi(v); err == nil && n > 0 {
			return n
		}
	}
	return defaultExtractorConcurrency
}

// extractorTopNPattern matches the {{ topn }} placeholder accepted in
// keyword/question system prompts.
// It is replaced with the configured top_n so the count slider stays
// authoritative even when a prompt was pre-filled by the frontend.
var extractorTopNPattern = regexp.MustCompile(`\{\{\s*topn\s*\}\}`)

// renderExtractorPrompt substitutes every {{ topn }} placeholder in the
// system prompt with the configured extraction count.
func renderExtractorPrompt(prompt string, topN int) string {
	return extractorTopNPattern.ReplaceAllString(prompt, strconv.Itoa(topN))
}

const (
	autoKeywordPrompt = `## Role
You are a text analyzer.

## Task
Extract the most important keywords/phrases of a given piece of text content.

## Requirements
- Summarize the text content, and give the top {{ topn }} important keywords/phrases.
- The keywords MUST be in the same language as the given piece of text content.
- The keywords are delimited by ENGLISH COMMA.
- Output keywords ONLY.`

	autoQuestionPrompt = `## Role
You are a text analyzer.

## Task
Propose questions about a given piece of text content.

## Requirements
- Understand and summarize the text content, and propose the top {{ topn }} important questions.
- The questions SHOULD NOT have overlapping meanings.
- The questions SHOULD cover the main content of the text as much as possible.
- The questions MUST be in the same language as the given piece of text content.
- One question per line.
- Output questions ONLY.`

	autoMetadataPrompt = `## Role: Metadata extraction expert.
## Rules:
 - Fixed Key Set: Output ONLY the keys that appear in the Schema. Never add, rename, translate, pluralize or re-case a key, and never invent an extra key even if the Content clearly carries that information. Any key that is not in the Schema MUST be omitted.
 - Strict Evidence Only: Extract a value ONLY if it is explicitly mentioned in the Content.
 - Enum Filter: For any field with an 'enum' list, the list acts as a strict filter. If no element from the list (or its direct synonym) is found in the Content, you MUST NOT extract that field.
 - No Meta-Inference: Do not infer values based on the document's nature, format, or category. If the text does not literally state the information, treat it as missing.
 - Zero-Hallucination: Never invent information or pick a "likely" value from the enum to fill a field.
 - Empty Result: If no matches are found for any field, or if the content is irrelevant, output ONLY {}.
 - Output: ONLY a valid JSON string. No Markdown, no notes.

## Schema for extraction:
%s`

	autoSummaryPrompt = `## Role
You are a precise and faithful text summarizer.

## Task
Create a concise and faithful summary of the provided text content.

## Requirements
- The summary MUST strictly rely on the provided text without hallucinating.
- The summary MUST be in the same language as the original text content.
- Be concise and focus on the main ideas, omitting redundant details.
- Output summary ONLY.`
)

// ExtractorComponent performs LLM-based extraction over a chunk
// list (or a single empty call when no chunks are wired in).
//
// The instance is safe for concurrent invocation: each Invoke
// reads Param read-only (Param is set at construction; per-call
// overrides flow through the inputs map). The single mutable
// package-level seam (extractorChatInvoker) is guarded by a
// RWMutex; tests swap it via SetExtractorChatInvoker.
type ExtractorComponent struct {
	Param schema.ExtractorParam
}

// NewExtractorComponent constructs an Extractor from a DSL param
// map. Missing keys fall back to schema.ExtractorParam.Defaults().
//
// Param map shape (all keys optional; missing → Defaults()):
//
//	{
//	  "llm_id":         string,           — optional; resolves via models.NewModelFactory
//	  "keywords":       map[string]any,   — optional auto keywords extraction config
//	  "questions":      map[string]any,   — optional auto questions extraction config
//	  "tags":           map[string]any,   — optional auto tags extraction config
//	  "summary":        map[string]any,   — optional auto summary extraction config
//	  "metadata":       map[string]any,   — optional auto metadata extraction config
//	}
//
// The keywords/questions/tags groups accept an "enabled" switch. A group
// without the key is a legacy DSL: enabled derives from top_n > 0, matching
// the pre-switch behavior of saved canvases. A group carrying an explicit
// "enabled": true with top_n < 1 is a contradictory config and fails here.
//
// errors here surface as canvas compile failures so a malformed
// param is caught at build time rather than mid-run.
func NewExtractorComponent(params map[string]any) (runtime.Component, error) {
	p := schema.ExtractorParam{}.Defaults()
	if params != nil {
		if v, ok := params["llm_id"].(string); ok {
			p.LLMID = v
		}

		// 1. Keywords
		if kwRaw, ok := params["keywords"].(map[string]any); ok {
			if v, ok := kwRaw["top_n"]; ok {
				p.Keywords.TopN = mapInt(v)
			}
			if v, ok := kwRaw["system_prompt"].(string); ok {
				p.Keywords.SystemPrompt = v
			}
			enabled, err := resolveExtractToggle(kwRaw, "keywords", p.Keywords.TopN)
			if err != nil {
				return nil, err
			}
			p.Keywords.Enabled = enabled
		}

		// 2. Questions
		if qRaw, ok := params["questions"].(map[string]any); ok {
			if v, ok := qRaw["top_n"]; ok {
				p.Questions.TopN = mapInt(v)
			}
			if v, ok := qRaw["system_prompt"].(string); ok {
				p.Questions.SystemPrompt = v
			}
			enabled, err := resolveExtractToggle(qRaw, "questions", p.Questions.TopN)
			if err != nil {
				return nil, err
			}
			p.Questions.Enabled = enabled
		}

		// 3. Tags
		if tagRaw, ok := params["tags"].(map[string]any); ok {
			if v, ok := tagRaw["top_n"]; ok {
				p.Tags.TopN = mapInt(v)
			}
			if v, ok := tagRaw["tag_file_id"].(string); ok {
				p.Tags.TagFileID = v
			}
			enabled, err := resolveExtractToggle(tagRaw, "tags", p.Tags.TopN)
			if err != nil {
				return nil, err
			}
			p.Tags.Enabled = enabled
		}

		// 4. Summary
		if sumRaw, ok := params["summary"].(map[string]any); ok {
			if v, ok := sumRaw["enabled"].(bool); ok {
				p.Summary.Enabled = v
			} else if v, ok := sumRaw["enabled"]; ok {
				p.Summary.Enabled = mapInt(v) == 1
			}
			if v, ok := sumRaw["system_prompt"].(string); ok {
				p.Summary.SystemPrompt = v
			}
		}

		// 5. Metadata
		if metaRaw, ok := params["metadata"].(map[string]any); ok {
			if v, ok := metaRaw["enabled"].(bool); ok {
				p.Metadata.Enabled = v
			} else if v, ok := metaRaw["enabled"]; ok {
				p.Metadata.Enabled = mapInt(v) == 1
			}
			if v, ok := metaRaw["metadata"]; ok {
				p.Metadata.Metadata = parseMetadataFieldDefs(v)
			}
			// BuiltInMetadata is carried for persistence/replay; LLM extraction uses Metadata only.
			if v, ok := metaRaw["built_in_metadata"]; ok {
				p.Metadata.BuiltInMetadata = parseMetadataFieldDefs(v)
			}
		}
	}
	if err := p.Validate(); err != nil {
		return nil, fmt.Errorf("extractor: param check: %w", err)
	}
	return &ExtractorComponent{Param: p}, nil
}

// resolveExtractToggle resolves the enabled switch of one extraction group
// (keywords / questions / tags).
//
// The presence of the "enabled" key distinguishes the contract version:
//   - key present: it is authoritative. enabled=true with top_n < 1 is a
//     contradictory config (nothing could be extracted) and fails fast, so
//     the canvas run dies at build time with a clear message instead of
//     silently skipping the extraction.
//   - key absent: legacy DSL saved before the switch existed — enabled
//     derives from top_n > 0, preserving the saved canvas's behavior.
//
// Accepts bool and numeric "enabled" values, mirroring the summary/metadata
// parsing below.
func resolveExtractToggle(group map[string]any, name string, topN int) (bool, error) {
	v, ok := group["enabled"]
	if !ok {
		return topN > 0, nil
	}
	enabled := false
	if b, ok := v.(bool); ok {
		enabled = b
	} else {
		enabled = mapInt(v) == 1
	}
	if enabled && topN < 1 {
		return false, fmt.Errorf("extractor: %s.enabled requires top_n >= 1", name)
	}
	return enabled, nil
}

// Inputs returns the parameter metadata. Matches the python
// Extractor._invoke kwargs plus the optional per-call llm_id
// override (python: args["llm_id"] path is implicit via
// self.chat_mdl; the Go port exposes it explicitly).
func (c *ExtractorComponent) Inputs() map[string]string {
	return map[string]string{
		"chunks": "List of map[string]any from the upstream Chunker. Each entry must carry a string 'text' field. Optional — when absent the LLM is called once with the resolved args.",
		"llm_id": "Optional per-call LLM id override. Falls back to Param.LLMID when absent.",
	}
}

// Outputs returns the public surface downstream ingestion
// consumers can wire into. Mirrors schema.ExtractorOutputs.
//
//	chunks         []map[string]any — input chunks, each augmented with
//	                                 extracted modular fields (important_kwd,
//	                                 question_kwd, tag_feas, summary, metadata).
//	output_format  string          — always "chunks". Parity with
//	                                 python set_output contract.
//	_ERROR         string          — populated on a short-circuit
//	                                 error (matches python
//	                                 set_output("_ERROR", ...)).
func (c *ExtractorComponent) Outputs() map[string]string {
	return map[string]string{
		"chunks":        "Extraction results — input chunks, each enriched with modular extraction fields (important_kwd, question_kwd, tag_feas, summary, metadata).",
		"output_format": "Always \"chunks\". Parity marker for downstream consumers.",
		"_ERROR":        "Optional short-circuit error message (reserved for the future TOC branch and other error paths).",
	}
}

// extractorChatInvoker is the seam the Extractor uses to dispatch
// its chat call. The production implementation
// (einoExtractorChatInvoker below) mirrors
// internal/agent/component/llm.go:einoChatInvoker — same factory,
// same driver resolution, but kept self-contained so the
// ingestion package does NOT pull in agent/component for a
// one-method interface.
//
// Tests swap the package-level defaultExtractorChatInvoker to inject a
// canned-response stub (see SetExtractorChatInvoker and the test
// helpers in extractor_test.go). This is the testability seam the
// Phase 2.5 spec calls out as a hard rule.
type extractorChatInvoker interface {
	Chat(ctx context.Context, req extractorChatRequest) (*extractorChatResponse, error)
}

// extractorChatRequest is the minimal surface the Extractor needs
// to dispatch a chat call. Driver is the provider key
// (e.g. "openai"); ModelName is the model id alone or composite
// "model@provider". APIKey / BaseURL are passed through so the
// driver can authenticate without re-reading the tenant config.
type extractorChatRequest struct {
	Driver      string
	ModelName   string
	APIKey      string
	BaseURL     string
	Messages    []eschema.Message
	Temperature *float64
}

// extractorChatResponse holds the LLM's text answer. Token /
// stopped flags are not consumed by the Extractor yet, so they
// remain optional / 0-valued.
type extractorChatResponse struct {
	Content string
}

// extractorChatInvokerMu guards defaultExtractorChatInvoker swaps.
var extractorChatInvokerMu sync.RWMutex

// defaultExtractorChatInvoker is the package-level seam. Production
// uses einoExtractorChatInvoker; tests inject a stub.
var defaultExtractorChatInvoker extractorChatInvoker = &einoExtractorChatInvoker{}

var extractorChatTargetResolverMu sync.RWMutex

// extractorChatTargetResolverOverride is a narrow test seam for
// integration tests that need to supply real credentials without
// teaching the production Extractor a tenant-credential lookup path.
// When set, resolveExtractorChatTarget consults it first.
var extractorChatTargetResolverOverride func(llmID string) (driver, modelName, apiKey, baseURL string, ok bool)

// SetExtractorChatInvoker swaps the package-level chat invoker
// for tests. Pass nil to restore the default. Concurrent-safe.
func SetExtractorChatInvoker(inv extractorChatInvoker) {
	extractorChatInvokerMu.Lock()
	defer extractorChatInvokerMu.Unlock()
	defaultExtractorChatInvoker = inv
}

// SetExtractorChatTargetResolverOverride swaps the package-level
// llm_id target resolver override for tests. Pass nil to restore
// the default split-only resolver. Concurrent-safe.
func SetExtractorChatTargetResolverOverride(fn func(llmID string) (driver, modelName, apiKey, baseURL string, ok bool)) {
	extractorChatTargetResolverMu.Lock()
	defer extractorChatTargetResolverMu.Unlock()
	extractorChatTargetResolverOverride = fn
}

func getExtractorChatTargetResolverOverride() func(llmID string) (driver, modelName, apiKey, baseURL string, ok bool) {
	extractorChatTargetResolverMu.RLock()
	defer extractorChatTargetResolverMu.RUnlock()
	return extractorChatTargetResolverOverride
}

// extractorResolvedTarget is the memoized result of a successful chat-target
// resolution. Only successful resolutions are cached (see target), so the
// stored entry never carries an error.
type extractorResolvedTarget struct {
	driver, modelName, apiKey, baseURL string
}

// extractorModelCache memoizes per-run chat-model resolution. Within a single
// ExtractorComponent.Invoke the llm_id is constant, so the chat target
// (driver/model/apiKey/baseURL) and the context length are resolved once and
// reused across every chunk / LLM call, avoiding thousands of redundant DAO
// reads per document. It is scoped to one run via context (never package
// global) so concurrent documents cannot leak state across tenants, and it is
// concurrency-safe because the extractor pool runs chunk jobs in parallel.
type extractorModelCache struct {
	mu      sync.Mutex
	targets map[string]extractorResolvedTarget
	ctxLens map[string]int

	// resolveTarget / resolveCtxLen are the underlying resolvers, wired to the
	// uncached implementations by newExtractorModelCache. Tests may replace
	// them to avoid a database.
	resolveTarget func(ctx context.Context, db *gorm.DB, llmID string) (string, string, string, string, error)
	resolveCtxLen func(ctx context.Context, db *gorm.DB, llmID string) int
}

func newExtractorModelCache() *extractorModelCache {
	return &extractorModelCache{
		targets:       make(map[string]extractorResolvedTarget),
		ctxLens:       make(map[string]int),
		resolveTarget: resolveExtractorChatTargetUncached,
		resolveCtxLen: extractorContextLengthUncached,
	}
}

// target returns the resolved chat target for llmID, computing and caching it
// on first use. Only successful resolutions are cached; a failed resolution is
// not memoized so a transient error is re-attempted and the run still fails
// fast on the first chunk instead of reusing a stale error.
func (m *extractorModelCache) target(ctx context.Context, db *gorm.DB, llmID string) (driver, modelName, apiKey, baseURL string, err error) {
	m.mu.Lock()
	if r, ok := m.targets[llmID]; ok {
		m.mu.Unlock()
		return r.driver, r.modelName, r.apiKey, r.baseURL, nil
	}
	m.mu.Unlock()

	driver, modelName, apiKey, baseURL, err = m.resolveTarget(ctx, db, llmID)
	if err == nil {
		m.mu.Lock()
		m.targets[llmID] = extractorResolvedTarget{driver, modelName, apiKey, baseURL}
		m.mu.Unlock()
	}
	return driver, modelName, apiKey, baseURL, err
}

// ctxLen returns the resolved context length for llmID, computing and caching
// it on first use. A zero result (unknown model / no canvas state) is a valid,
// cacheable resolution meaning "skip fitting".
func (m *extractorModelCache) ctxLen(ctx context.Context, db *gorm.DB, llmID string) int {
	m.mu.Lock()
	if v, ok := m.ctxLens[llmID]; ok {
		m.mu.Unlock()
		return v
	}
	m.mu.Unlock()

	v := m.resolveCtxLen(ctx, db, llmID)
	m.mu.Lock()
	m.ctxLens[llmID] = v
	m.mu.Unlock()
	return v
}

type extractorModelCacheKey struct{}

// withExtractorModelCache attaches a fresh per-run model cache to ctx. The
// cache is created once per ExtractorComponent.Invoke and shared by every
// downstream chunk job, so resolution happens at most once per distinct
// llm_id for the whole run.
func withExtractorModelCache(ctx context.Context) context.Context {
	return context.WithValue(ctx, extractorModelCacheKey{}, newExtractorModelCache())
}

func extractorModelCacheFromCtx(ctx context.Context) *extractorModelCache {
	if c, ok := ctx.Value(extractorModelCacheKey{}).(*extractorModelCache); ok {
		return c
	}
	return nil
}

// getExtractorChatInvoker returns the current default invoker.
func getExtractorChatInvoker() extractorChatInvoker {
	extractorChatInvokerMu.RLock()
	defer extractorChatInvokerMu.RUnlock()
	if defaultExtractorChatInvoker == nil {
		return &einoExtractorChatInvoker{}
	}
	return defaultExtractorChatInvoker
}

// einoExtractorChatInvoker is the production seam. It dispatches
// through the entity/models factory (which knows 48 of 56
// providers) and returns the assistant text via
// models.EinoChatModel.Generate. An unknown provider falls
// through to NewDummyModel in the factory's default branch — we
// surface that as a typed "no driver for %q" wrap so callers can
// decide whether to retry, route around, or log.
type einoExtractorChatInvoker struct{}

// Chat implements extractorChatInvoker for the production path.
func (e *einoExtractorChatInvoker) Chat(ctx context.Context, req extractorChatRequest) (*extractorChatResponse, error) {
	if req.ModelName == "" {
		return nil, fmt.Errorf("extractor: chat: model_name is required")
	}
	driver := strings.ToLower(strings.TrimSpace(req.Driver))
	modelName := req.ModelName
	if driver == "" && modelName != "" {
		if bare, provider, ok := splitExtractorLLIDPair(modelName); ok {
			driver = provider
			modelName = bare
		}
	}
	if driver == "" {
		return nil, fmt.Errorf("extractor: chat: no driver resolved for model %q", modelName)
	}
	common.Info(fmt.Sprintf("extractor: chat: driver=%s modelName=%s baseUrl=%s", driver, modelName, req.BaseURL))
	d, err := models.GetPreconfiguredDriver(driver, req.BaseURL)
	if err != nil {
		return nil, fmt.Errorf("extractor: resolve driver %q: %w", driver, err)
	}
	apiKey := req.APIKey
	cfg := &models.APIConfig{ApiKey: &apiKey}
	cm := models.NewChatModel(d, &modelName, cfg)
	chatCfg := &models.ChatConfig{}
	if req.Temperature != nil {
		temp := *req.Temperature
		chatCfg.Temperature = &temp
	}
	wrapper := models.NewEinoChatModel(cm, chatCfg)
	// Honour ctx cancel up front so the caller's WithTimeout(...)
	// is observed even when the driver layer doesn't take a ctx.
	common.Info(fmt.Sprintf("try to chat with message: %v", req.Messages))
	if err := ctx.Err(); err != nil {
		return nil, err
	}
	out, err := wrapper.Generate(ctx, toExtractorEinoMessages(req.Messages))
	if err != nil {
		common.Error(fmt.Sprintf("error when chat with message: %v", req.Messages), err)
		return nil, err
	}
	common.Debug(fmt.Sprintf("extractor: chat completed for model %s, response_length=%d", modelName, len(out.Content)))
	return &extractorChatResponse{Content: out.Content}, nil
}

// splitExtractorLLIDPair parses a composite llm_id "model@provider"
// mirroring agent/component/llm_credentials.go:parseLLMIDParts
// (the canonical composite form throughout the codebase). Returns
// ok=false when no "@" is present or the id is malformed.
//
//	"gpt-4o-mini@openai"           -> ("gpt-4o-mini", "openai", true)
//	"gpt-4o-mini"                  -> ("gpt-4o-mini", "", false)
//
// Kept local so the ingestion package doesn't import
// agent/component.
func splitExtractorLLIDPair(s string) (modelName, provider string, ok bool) {
	parts := strings.Split(strings.TrimSpace(s), "@")
	switch len(parts) {
	case 2:
		return parts[0], parts[1], true
	default:
		return s, "", false
	}
}

// toExtractorEinoMessages converts eschema.Message → *eschema.Message
// for the eino bridge. The user / system / assistant roles pass
// through; multi-modal content is intentionally not propagated —
// extraction prompts are text-only today.
func toExtractorEinoMessages(msgs []eschema.Message) []*eschema.Message {
	out := make([]*eschema.Message, 0, len(msgs))
	for i := range msgs {
		m := msgs[i]
		role := m.Role
		if role == "" {
			role = eschema.User
		}
		out = append(out, &eschema.Message{
			Role:    role,
			Content: m.Content,
		})
	}
	return out
}

// extractorInputs is the post-Validation view of the upstream
// input map. Computed once at the top of Invoke so the rest of
// the function reads as straight-line code.
type extractorInputs struct {
	llmID string
	// modelID is the resolved cache-key model identity (the llm_id, or — when
	// that is empty — the tenant-default chat model resolved once per run in
	// Invoke). Carrying it on the struct lets the per-chunk cache-key builders
	// reuse the single run-level resolution instead of re-resolving the default
	// model for every chunk. Empty means "not yet resolved"
	// and triggers a per-call fallback so direct unit-test construction of
	// extractorInputs keeps working.
	modelID string
	lang    string
	chunks  []map[string]any
	// temperature overrides the LLM temperature for this call. A
	// nil value leaves the request's Temperature unset so the model
	// (or the chat-model default) decides. The keyword/question helpers set it to
	// extractorTemperature (0.2) to mirror generator.py.
	temperature *float64
	// cache memoises per-chunk extraction results so a resumed run (which
	// re-executes every stage after the Parser checkpoint) pays no LLM cost
	// for chunks it already extracted. nil disables caching: a Redis-less
	// deployment, or a test that wants every call to reach the model.
	cache chunkcache.Store
}

// resolveInputs overlays per-call inputs on top of the
// component's static Param. Missing keys fall back to the
// Param-level values; per-call values win on conflict (so a
// canvas can override LLM_ID at runtime).
func (c *ExtractorComponent) resolveInputs(inputs map[string]any) extractorInputs {
	out := extractorInputs{
		llmID: c.Param.LLMID,
		cache: chunkcache.Client(),
	}
	if inputs == nil {
		return out
	}
	if v, ok := inputs["llm_id"].(string); ok && v != "" {
		out.llmID = v
	}
	if v, ok := inputs["lang"].(string); ok && v != "" {
		out.lang = v
	}
	for _, key := range extractorChunkInputOrder(inputs) {
		if chunks, ok := extractorChunkList(inputs[key]); ok {
			out.chunks = chunks
			break
		}
	}
	return out
}

func extractorChunkInputOrder(inputs map[string]any) []string {
	order := make([]string, 0, len(inputs))
	for _, preferred := range []string{"chunks", "json"} {
		if _, ok := inputs[preferred]; ok {
			order = append(order, preferred)
		}
	}
	var extra []string
	for key := range inputs {
		if key == "chunks" || key == "json" {
			continue
		}
		extra = append(extra, key)
	}
	sort.Strings(extra)
	order = append(order, extra...)
	return order
}

func extractorChunkList(v any) ([]map[string]any, bool) {
	switch list := v.(type) {
	case []map[string]any:
		return list, true
	case []schema.ChunkDoc:
		return schema.ChunkDocsToMaps(list), true
	case []any:
		out := make([]map[string]any, 0, len(list))
		for _, item := range list {
			m, ok := item.(map[string]any)
			if !ok {
				continue
			}
			out = append(out, m)
		}
		return out, true
	default:
		return nil, false
	}
}

// Invoke performs LLM-based extraction. Inputs:
//
//	chunks         (optional, []map[string]any) — upstream chunks; each must
//	                                            carry a string "text".
//	llm_id         (optional, string)            — overrides Param.LLMID.
//
// Outputs:
//
//	chunks        ([]map[string]any) — input chunks augmented with extraction results.
//	output_format (string)          — always "chunks".
//	_ERROR        (string, reserved) — populated when the component
//	                                  short-circuits with an error.
//	_created_time, _elapsed_time    — stamped by the canvas framework
//	                                 (realComponentBody), not here.
func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
	if err := c.Param.Validate(); err != nil {
		return nil, fmt.Errorf("extractor: %w", err)
	}
	// Attach a per-run model cache so the chat target and context length are
	// resolved once per distinct llm_id and reused across every chunk, instead
	// of being re-resolved in every callRaw / fitExtractorMessages.
	ctx = withExtractorModelCache(ctx)
	in := c.resolveInputs(inputs)
	// Resolve the cache-key model identity once for the whole run and reuse it
	// across every chunk, rather than re-resolving the (most common) default
	// model per chunk when keying the per-chunk cache.
	in.modelID = extractorCacheModelID(ctx, db, in.llmID)
	// Warm the cache before fan-out so the (parallel) chunk jobs read a
	// populated entry: the resolution happens exactly once here, eliminating
	// any first-batch resolution herd. A failed resolution is not cached, so
	// the first chunk's callRaw still surfaces it and the run fails fast.
	if cache := extractorModelCacheFromCtx(ctx); cache != nil {
		cache.target(ctx, db, in.llmID)
		cache.ctxLen(ctx, db, in.llmID)
	}
	common.Debug("extractor stage",
		zap.String("component", "Extractor"),
		zap.Int("input_chunks", len(in.chunks)),
	)
	if len(in.chunks) == 0 {
		return map[string]any{
			"chunks":        []map[string]any{},
			"output_format": "chunks",
		}, nil
	}

	keywordsOn := c.Param.Keywords.Enabled
	tagsOn := c.Param.Tags.Enabled
	remainingOn := c.Param.Questions.Enabled || c.Param.Summary.Enabled || c.Param.Metadata.Enabled
	phaseWindows := progress.NewSplit(keywordsOn, tagsOn, remainingOn)

	// No node-level wall-clock budget (see the package doc): each LLM call
	// carries its own budget, and an expired call skips its chunk instead
	// of failing the document. Task cancellation still aborts via ctx.
	if err := c.runAutoKeywordsPool(ctx, db, in, phaseWindows.Take(keywordsOn)); err != nil {
		return nil, fmt.Errorf("extractor: %w", err)
	}

	// Phase 2: Tag phase (if enabled), benefiting from title and freshly extracted keywords.
	if tagsOn {
		tagged, tagErr := c.runAutoTags(ctx, db, in, phaseWindows.Take(tagsOn))
		if tagErr != nil {
			return nil, fmt.Errorf("extractor: %w", tagErr)
		}
		in.chunks = tagged
	}

	// Phase 3: Remaining extractions (Questions, Summary, Metadata) via extractorPool.
	if err := c.runRemainingExtractions(ctx, db, in, phaseWindows.Take(remainingOn)); err != nil {
		return nil, fmt.Errorf("extractor: %w", err)
	}
	// The tag phase swallows per-chunk failures, so a task stopped mid-run
	// could otherwise report success. This mirrors the old node-level
	// WithTimeout post-check: never report success on a dead context.
	if err := ctx.Err(); err != nil {
		return nil, fmt.Errorf("extractor: %w", err)
	}
	common.Debug("extractor stage",
		zap.String("component", "Extractor"),
		zap.Int("output_chunks", len(in.chunks)),
	)
	return map[string]any{
		"chunks":        in.chunks,
		"output_format": "chunks",
	}, nil
}

// extractorLLMCacheKey builds the cache key for one textual extraction. The
// chunk id — not the chunk text — carries the chunk's identity: it already
// derives from the text (component.ChunkID) and is the same handle the persist
// stage uses, so the two stages agree on what "the same chunk" is. Returns ""
// when the chunk has no id, which makes the cache a no-op for that chunk.
func extractorLLMCacheKey(taskType, modelID, systemPrompt, chunkID string) string {
	return chunkcache.Key("extractor:"+taskType, modelID, chunkID, systemPrompt)
}

// extractorCacheModelID returns the model identity the per-chunk extraction
// cache is keyed on. The cache must capture the model that actually produced
// the text, not the raw llm_id override: when llm_id is empty the pipeline
// falls back to the tenant's default chat model (resolveExtractorChatTarget,
// extractor.go:1212), so keying on the empty override would collapse every
// tenant-default run onto one bucket and serve extractions from a model the
// tenant no longer uses. This mirrors the tokenizer's decision to key on the
// dataset-bound embd_id rather than the raw DSL string.
//
// Non-empty llm_id is already a stable tenant-model id, so it is used as-is
// (no behaviour change for the configured path). Only the empty case is
// resolved to its real default-model identity. A resolution failure returns ""
// — chunkcache.Key turns an empty modelID into an empty key, which disables
// caching for that chunk rather than risking a cross-model hit.
func extractorCacheModelID(ctx context.Context, db *gorm.DB, llmID string) string {
	if llmID != "" {
		return llmID
	}
	driver, modelName, _, _, err := resolveExtractorChatTarget(ctx, db, "")
	if err != nil || (driver == "" && modelName == "") {
		return ""
	}
	return driver + "/" + modelName
}

// callTextCached wraps callText with the per-chunk result cache.
func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in extractorInputs, taskType, systemPrompt, chunkText, chunkID string) (string, error) {
	// Prefer the run-level resolved identity; fall back only for callers that
	// build extractorInputs directly in unit tests without pre-resolving it.
	modelID := in.modelID
	if modelID == "" {
		modelID = extractorCacheModelID(ctx, db, in.llmID)
	}
	key := extractorLLMCacheKey(taskType, modelID, systemPrompt, chunkID)
	if cached, hit := chunkcache.Get(ctx, in.cache, key); hit {
		return cached, nil
	}
	res, err := c.callText(ctx, db, in, systemPrompt, chunkText)
	if err != nil {
		return "", err
	}
	res = cleanExtractionResult(res)
	if res != "" && !strings.Contains(res, "**ERROR**") {
		chunkcache.Set(ctx, in.cache, key, res)
	}
	return res, nil
}

// runAutoKeywords extracts keywords for the current chunk and stores
// them on ck["important_kwd"]. Keyword extraction pins
// temperature to extractorTemperature (0.2) to mirror generator.py.
func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
	if strings.TrimSpace(chunkText) == "" {
		return nil
	}
	if _, exists := ck["important_kwd"]; exists {
		return nil
	}
	if !c.Param.Keywords.Enabled {
		return nil
	}
	topN := c.Param.Keywords.TopN
	systemPrompt := strings.TrimSpace(c.Param.Keywords.SystemPrompt)
	if systemPrompt == "" {
		systemPrompt = autoKeywordPrompt
	}
	systemPrompt = renderExtractorPrompt(systemPrompt, topN)
	kwTemp := extractorTemperature
	kwIn := extractorInputs{
		llmID:       in.llmID,
		modelID:     in.modelID,
		temperature: &kwTemp,
		cache:       in.cache,
	}
	resultStr, err := c.callTextCached(ctx, db, kwIn, "keywords", systemPrompt, chunkText, chunkCacheID(ck))
	if err != nil {
		return err
	}
	if resultStr == "" {
		return nil
	}
	kwds := splitKeywords(resultStr)
	if len(kwds) == 0 {
		return nil
	}
	tok := tokenizer.New(in.lang)
	tks, tkErr := tok.Tokenize(strings.Join(kwds, " "))
	ck["important_kwd"] = kwds
	if tkErr == nil {
		ck["important_tks"] = tks
	}
	return nil
}

// runAutoQuestions extracts questions for the current chunk and stores
// them on ck["question_kwd"]. See runAutoKeywords for the temperature pin.
func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
	if strings.TrimSpace(chunkText) == "" {
		return nil
	}
	if _, exists := ck["question_kwd"]; exists {
		return nil
	}
	if !c.Param.Questions.Enabled {
		return nil
	}
	topN := c.Param.Questions.TopN
	systemPrompt := strings.TrimSpace(c.Param.Questions.SystemPrompt)
	if systemPrompt == "" {
		systemPrompt = autoQuestionPrompt
	}
	systemPrompt = renderExtractorPrompt(systemPrompt, topN)
	qTemp := extractorTemperature
	qIn := extractorInputs{
		llmID:       in.llmID,
		modelID:     in.modelID,
		temperature: &qTemp,
		cache:       in.cache,
	}
	resultStr, err := c.callTextCached(ctx, db, qIn, "questions", systemPrompt, chunkText, chunkCacheID(ck))
	if err != nil {
		return err
	}
	if resultStr == "" {
		return nil
	}
	qs := strings.Split(resultStr, "\n")
	// Filter empty lines
	var filtered []string
	for _, q := range qs {
		q = strings.TrimSpace(q)
		if q != "" {
			filtered = append(filtered, q)
		}
	}
	if len(filtered) == 0 {
		return nil
	}
	tok := tokenizer.New(in.lang)
	tks, tkErr := tok.Tokenize(strings.Join(filtered, "\n"))
	ck["question_kwd"] = filtered
	if tkErr == nil {
		ck["question_tks"] = tks
	}
	return nil
}

// runAutoSummary extracts a concise summary for the current chunk using autoSummaryPrompt
// and stores it on ck["summary"].
func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
	if strings.TrimSpace(chunkText) == "" {
		return nil
	}
	if _, exists := ck["summary"]; exists {
		return nil
	}
	if !c.Param.Summary.Enabled {
		return nil
	}
	systemPrompt := c.Param.Summary.SystemPrompt
	if strings.TrimSpace(systemPrompt) == "" {
		systemPrompt = autoSummaryPrompt
	}
	sumTemp := extractorTemperature
	sumIn := extractorInputs{
		llmID:       in.llmID,
		modelID:     in.modelID,
		temperature: &sumTemp,
		cache:       in.cache,
	}
	resultStr, err := c.callTextCached(ctx, db, sumIn, "summary", systemPrompt, chunkText, chunkCacheID(ck))
	if err != nil {
		return err
	}
	if resultStr == "" {
		return nil
	}
	ck["summary"] = resultStr
	return nil
}

// resolveChunkCallError maps one chunk job's error to the pool outcome:
//   - parent ctx done (task stop): propagate so cancellation stays fast.
//   - call budget expiry with a live parent: skip the chunk, keep the
//     document. errors.Is covers both our per-call budget and a
//     driver-level deadline. completedAny reports whether an earlier op
//     already succeeded for this chunk: a later timeout then keeps the
//     partial result instead of counting toward the all-timed-out failure.
//   - anything else: hard failure (existing fail-fast semantics).
func resolveChunkCallError(ctx context.Context, idx int, phase string, err error, completedAny bool, skipped *atomic.Int64) error {
	if err == nil {
		return nil
	}
	if ctx.Err() != nil {
		return err
	}
	if errors.Is(err, context.DeadlineExceeded) {
		if completedAny {
			common.Warn("extractor: chunk extraction partially timed out; keeping completed fields",
				zap.String("component", componentNameExtractor),
				zap.String("phase", phase),
				zap.Int("chunk_index", idx),
			)
			return nil
		}
		skipped.Add(1)
		common.Warn("extractor: chunk extraction timed out; skipping chunk",
			zap.String("component", componentNameExtractor),
			zap.String("phase", phase),
			zap.Int("chunk_index", idx),
		)
		return nil
	}
	return err
}

// checkAllChunksSkipped fails the phase when every submitted chunk timed
// out with zero completed extractions: all-hung means the model is down,
// and silently producing a document with no extractions would hide the
// outage. Chunks that timed out after a partial extraction succeed with
// the per-chunk warnings above and do not count here.
func checkAllChunksSkipped(phase string, total int, skipped int64) error {
	if total > 0 && skipped == int64(total) {
		return fmt.Errorf("extractor: %s: all %d chunks exceeded the per-call LLM timeout; the model is likely hung or unreachable", phase, total)
	}
	return nil
}

// runAutoKeywordsPool dispatches keyword extraction across all chunks concurrently
// using extractorPool before the tagging stage.
func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.DB, in extractorInputs, w progress.Window) error {
	if !c.Param.Keywords.Enabled || len(in.chunks) == 0 {
		return nil
	}
	var skipped atomic.Int64
	futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))
	for i, ck := range in.chunks {
		i, ck := i, ck
		text := extractorChunkText(ck)
		if strings.TrimSpace(text) == "" {
			continue
		}
		fn := func() error {
			err := c.runAutoKeywords(ctx, db, in, ck, text)
			completed := err == nil
			if err != nil {
				err = fmt.Errorf("chunk %d keywords: %w", i, err)
			}
			return resolveChunkCallError(ctx, i, "keywords", err, completed, &skipped)
		}
		f, err := extractorPool.Submit(ctx, fn)
		if err != nil {
			return err
		}
		futs = append(futs, f)
	}
	if err := awaitFutures(ctx, futs, w); err != nil {
		return err
	}
	return checkAllChunksSkipped("keywords", len(futs), skipped.Load())
}

// runRemainingExtractions dispatches auto questions / summary / metadata
// extractions across all chunks concurrently using extractorPool.
func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *gorm.DB, in extractorInputs, w progress.Window) error {
	if !c.Param.Questions.Enabled && !c.Param.Summary.Enabled && !c.Param.Metadata.Enabled {
		return nil
	}
	var skipped atomic.Int64
	futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))
	for i, ck := range in.chunks {
		i, ck := i, ck
		text := extractorChunkText(ck)
		if strings.TrimSpace(text) == "" {
			continue
		}
		fn := c.remainingExtractionJob(ctx, db, in, i, ck, text, &skipped)
		f, err := extractorPool.Submit(ctx, fn)
		if err != nil {
			return err
		}
		futs = append(futs, f)
	}
	if err := awaitFutures(ctx, futs, w); err != nil {
		return err
	}
	return checkAllChunksSkipped("remaining", len(futs), skipped.Load())
}

func (c *ExtractorComponent) remainingExtractionJob(ctx context.Context, db *gorm.DB, in extractorInputs, idx int, ck map[string]any, chunkText string, skipped *atomic.Int64) extractorJob {
	return func() error {
		var err error
		var completed bool
		if c.Param.Questions.Enabled {
			if qErr := c.runAutoQuestions(ctx, db, in, ck, chunkText); qErr != nil {
				err = fmt.Errorf("chunk %d questions: %w", idx, qErr)
			} else {
				completed = true
			}
		}
		if err == nil && c.Param.Summary.Enabled {
			if sErr := c.runAutoSummary(ctx, db, in, ck, chunkText); sErr != nil {
				err = fmt.Errorf("chunk %d summary: %w", idx, sErr)
			} else {
				completed = true
			}
		}
		if err == nil && c.Param.Metadata.Enabled {
			if mErr := c.runEnableMetadata(ctx, db, in, ck, chunkText); mErr != nil {
				err = fmt.Errorf("chunk %d metadata: %w", idx, mErr)
			} else {
				completed = true
			}
		}
		return resolveChunkCallError(ctx, idx, "remaining", err, completed, skipped)
	}
}

// awaitFutures waits for every submitted job and returns the first failure.
// Each settled future advances the phase's fraction window, so a chunk-heavy
// LLM extraction reports in-flight progress instead of going quiet until the
// whole phase is done.
func awaitFutures(ctx context.Context, futs []utility.WorkerPoolFuture[extractorJob, struct{}], w progress.Window) error {
	if len(futs) == 0 {
		return nil
	}
	var firstErr error
	var emu sync.Mutex
	var wg sync.WaitGroup
	var done atomic.Int64
	total := int64(len(futs))
	for _, f := range futs {
		wg.Add(1)
		go func(f utility.WorkerPoolFuture[extractorJob, struct{}]) {
			defer wg.Done()
			defer func() { w.Report(ctx, float64(done.Add(1))/float64(total)) }()
			res, werr := f.Wait(ctx)
			if werr != nil {
				emu.Lock()
				if firstErr == nil {
					firstErr = werr
				}
				emu.Unlock()
				return
			}
			if res.Err != nil {
				emu.Lock()
				if firstErr == nil {
					firstErr = res.Err
				}
				emu.Unlock()
			}
		}(f)
	}
	wg.Wait()
	return firstErr
}

// runEnableMetadata extracts structured metadata for the current chunk and
// merges the parsed JSON object into ck["metadata"]. It mirrors the
// runAutoKeywords/runAutoQuestions shape but parses a JSON object and
// merges into the chunk's metadata map.
func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
	if strings.TrimSpace(chunkText) == "" {
		return nil
	}
	if !c.Param.Metadata.Enabled || len(c.Param.Metadata.Metadata) == 0 {
		return nil
	}
	// Render the field schema into the prompt, mirroring Python's
	// turn2jsonschema(metadata_conf) rendered into the META_DATA template.
	schemaMap := common.Turn2JSONSchema(c.Param.Metadata.Metadata)
	if len(schemaMap) == 0 {
		return nil
	}
	schemaJSON, err := json.Marshal(schemaMap)
	if err != nil {
		return err
	}
	schemaStr := string(schemaJSON)

	// Per-chunk result cache: identical (model + chunk + schema) extractions
	// are served from the cache so a resumed run — which re-executes every
	// stage after the Parser checkpoint — doesn't re-pay the LLM call.
	// Best-effort: a missing client or any cache error falls through to a live
	// call instead of failing the extraction.
	chunkID := chunkCacheID(ck)
	// Prefer the run-level resolved identity; fall back only for direct unit
	// callers that did not pre-resolve it.
	modelID := in.modelID
	if modelID == "" {
		modelID = extractorCacheModelID(ctx, db, in.llmID)
	}
	var parsed map[string]any
	if cached, hit := getMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID); hit {
		parsed = cached
	} else {
		metaTemp := extractorTemperature
		metaIn := extractorInputs{
			llmID:       in.llmID,
			modelID:     in.modelID,
			temperature: &metaTemp,
			cache:       in.cache,
		}
		systemPrompt := fmt.Sprintf(autoMetadataPrompt, schemaStr)
		parsed, err = c.callStructured(ctx, db, metaIn, systemPrompt, chunkText)
		if err != nil {
			return err
		}
		if parsed == nil {
			// Non-JSON or empty response — nothing to extract, not an error.
			return nil
		}
		parsed = filterMetadataToDeclaredKeys(parsed, c.Param.Metadata.Metadata)
		if len(parsed) == 0 {
			// Every key fell outside the declared field set — nothing to keep.
			return nil
		}
		setMetadataLLMCache(ctx, in.cache, modelID, schemaStr, chunkID, parsed)
	}
	// Merge into the chunk metadata map, preserving existing keys.
	var meta map[string]any
	if existing, ok := ck["metadata"].(map[string]any); ok && existing != nil {
		meta = existing
	} else {
		meta = make(map[string]any, len(parsed))
	}
	for k, v := range parsed {
		if v == nil {
			continue
		}
		if s, ok := v.(string); ok && strings.TrimSpace(s) == "" {
			continue
		}
		meta[k] = v
	}
	ck["metadata"] = meta
	return nil
}

// filterMetadataToDeclaredKeys drops every key that is not part of the
// declared metadata field set.
//
// The rendered schema carries additionalProperties:false, but that is only a
// hint to the model: a model that renames, pluralizes or invents a field still
// returns a parseable object, and such a key would otherwise widen the
// dataset's metadata schema through mergeChunkMetadata. The declared set is
// therefore enforced here, in code.
func filterMetadataToDeclaredKeys(parsed map[string]any, fields []common.MetadataFieldDef) map[string]any {
	allowed := make(map[string]struct{}, len(fields))
	for _, f := range fields {
		allowed[f.Key] = struct{}{}
	}
	kept := make(map[string]any, len(parsed))
	for k, v := range parsed {
		if _, ok := allowed[k]; !ok {
			common.Debug("extractor stage",
				zap.String("component", componentNameExtractor),
				zap.String("dropped_metadata_key", k),
			)
			continue
		}
		kept[k] = v
	}
	return kept
}

// extractorChunkText resolves the body an extraction is run against.
//
// Canonical text wins. Media-only chunks may use their explicit surrounding
// context; chunks with neither body nor context are skipped by the extraction
// schedulers.
func extractorChunkText(ck map[string]any) string {
	if v, _ := ck["text"].(string); strings.TrimSpace(v) != "" {
		return v
	}
	var contextParts []string
	for _, key := range []string{"context_above", "context_below"} {
		if v, _ := ck[key].(string); strings.TrimSpace(v) != "" {
			contextParts = append(contextParts, strings.TrimSpace(v))
		}
	}
	if len(contextParts) > 0 {
		return strings.Join(contextParts, " ")
	}
	return ""
}

// chunkCacheID returns the chunk's stable per-chunk id, assigned by the chunker
// (component.ChunkID over doc id + text). Returns "" when the chunk carries no
// id — e.g. chunks that reached the Extractor without passing a chunker — in
// which case callers must skip the cache instead of sharing one bucket across
// every unidentified chunk.
func chunkCacheID(ck map[string]any) string {
	id, _ := ck["id"].(string)
	return id
}

// metadataLLMCacheKey builds the cache key for one metadata extraction. The
// schema is part of the key so editing the field set invalidates results
// extracted against the previous one.
func metadataLLMCacheKey(llmID, schemaJSON, chunkID string) string {
	return chunkcache.Key("meta", llmID, chunkID, schemaJSON)
}

// getMetadataLLMCache returns a cached extraction for the given chunk, or
// (nil, false) on miss / cache unavailable / decode error. Best-effort.
func getMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, schemaJSON, chunkID string) (map[string]any, bool) {
	data, hit := chunkcache.Get(ctx, store, metadataLLMCacheKey(modelID, schemaJSON, chunkID))
	if !hit {
		return nil, false
	}
	var parsed map[string]any
	if err := json.Unmarshal([]byte(data), &parsed); err != nil {
		return nil, false
	}
	return parsed, true
}

// setMetadataLLMCache stores an extraction result. Best-effort: an unavailable
// cache or a marshal error is silently ignored.
func setMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, schemaJSON, chunkID string, parsed map[string]any) {
	data, err := json.Marshal(parsed)
	if err != nil {
		return
	}
	chunkcache.Set(ctx, store, metadataLLMCacheKey(modelID, schemaJSON, chunkID), string(data))
}

// callRaw runs one chat call against the LLM (per chunk in the normal path)
// and returns the raw response. It holds the shared plumbing — driver/target
// resolution, message assembly, temperature override, retry — but deliberately
// does NOT clean or parse the response. Callers pick the view they need:
//
//   - callText wraps callRaw with the LLM-layer two-step cleanup (think +
//     tool_call) and returns a plain string.
//   - callStructured wraps callRaw with cleanup + explicit JSON parsing and
//     returns a map — the metadata path, matching Python gen_metadata.
func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (*extractorChatResponse, error) {
	if strings.TrimSpace(chunkText) == "" {
		return nil, fmt.Errorf("extractor: chunk text is empty")
	}
	driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)
	if err != nil {
		return nil, err
	}
	msgs := buildExtractorMessages(systemPrompt, chunkText)
	fitted, fitErr := fitExtractorMessages(ctx, db, in.llmID, msgs)
	if fitErr != nil {
		return nil, fitErr
	}
	msgs = fitted
	inv := getExtractorChatInvoker()
	req := extractorChatRequest{
		Driver:    driver,
		ModelName: modelName,
		APIKey:    apiKey,
		BaseURL:   baseURL,
		Messages:  msgs,
	}
	// Only override the temperature when the caller set one. A nil
	// Temperature lets the model / chat-model default decide, matching
	// Python's generic Extractor path; keyword/question helpers set
	// extractorTemperature (0.2) to mirror generator.py.
	if in.temperature != nil {
		temp := *in.temperature
		req.Temperature = &temp
	}
	var resp *extractorChatResponse
	// One hung-call ceiling per LLM request (see extractorChatTimeout).
	// RetryWithBackoff treats a budget expiry as terminal (no retry), so a
	// hung call burns at most one attempt. The pool jobs map the expiry to
	// a per-chunk skip; anything else keeps fail-fast semantics.
	budget := extractorChatTimeout()
	callCtx, cancel := context.WithTimeout(ctx, budget)
	defer cancel()
	if err := common.RetryWithBackoff(callCtx, extractorRetryMax, extractorRetryDelay, func() error {
		r, e := inv.Chat(callCtx, req)
		resp = r
		return e
	}, isRetryableLLMError); err != nil {
		return nil, err
	}
	if resp == nil {
		// Defensive: a provider adapter returning (nil, nil) would otherwise
		// panic on resp.Content below. Surface a diagnosable error instead.
		return nil, fmt.Errorf("extractor: chat: nil response from invoker")
	}
	return resp, nil
}

// toolCallRE matches a <tool_call>...</tool_call> block, mirroring Python's
// `re.sub(r"<tool_call>.*?</tool_call>", "", txt, flags=re.DOTALL)` at
// llm_service.py:461. Non-greedy so consecutive tool_call blocks are each
// stripped.
var toolCallRE = regexp.MustCompile(`(?s)<tool_call>.*?</tool_call>`)

// cleanLLMText applies the two-step LLM-layer cleanup that Python performs
// in LLMBundle.async_chat (llm_service.py:459-461) for every response,
// before any parsing:
//
//  1. Reasoning content: when the response BEGINS with `<think>`, take
//     everything after the LAST `</think>`. A `<think>` that appears after
//     a non-empty prefix is treated as ordinary content and preserved, so a
//     legitimate prefix is never stripped. Python's _remove_reasoning_content
//     (llm_service.py:332-346) finds the first `<think>` anywhere; restricting
//     to a leading `<think>` avoids deleting real content that merely contains
//     a think-looking tag. A leading `<think>` with no closing `</think>` is
//     kept unchanged.
//  2. Tool-call blocks: remove `<tool_call>...</tool_call>` spans.
//
// The result is a plain string, ready for the caller to consume as text or
// to parse explicitly.
func cleanLLMText(s string) string {
	if strings.HasPrefix(s, "<think>") {
		s = common.StripThinkTrailing(s)
	}
	s = toolCallRE.ReplaceAllString(s, "")
	return strings.TrimSpace(s)
}

// cleanExtractionResult strips `</think>` tags and rejects `**ERROR**` responses,
// matching Python's keyword_extraction and question_proposal post-processing.
func cleanExtractionResult(s string) string {
	s = common.StripThinkTrailing(s)
	s = strings.TrimSpace(s)
	if strings.Contains(s, "**ERROR**") {
		return ""
	}
	return s
}

// splitKeywords splits a comma-delimited keyword string.
func splitKeywords(s string) []string {
	parts := strings.FieldsFunc(s, func(r rune) bool {
		return r == ',' || r == ',' || r == ';' || r == ';' || r == '、' || r == '\r' || r == '\n'
	})
	result := make([]string, 0, len(parts))
	for _, p := range parts {
		p = strings.TrimSpace(p)
		if p != "" {
			result = append(result, p)
		}
	}
	return result
}

// nonRetryableStatusRE matches HTTP client-error status codes that
// signal a permanent (non-transient) condition and therefore must
// NOT be retried. The word boundaries are essential: a bare
// substring check on "400" would wrongly flag phrasing such as
// "context deadline exceeded after 400ms" (a transient timeout,
// retryable) as non-retryable. \b ensures we only match a standalone
// 3-digit status token, so "400ms" / "4000" do not match. 429 and
// 5xx are deliberately absent — they stay retryable.
var nonRetryableStatusRE = regexp.MustCompile(`\b(?:400|401|403|404|405|422)\b`)

// isRetryableLLMError classifies an LLM chat error as worth
// retrying. The production chat invoker returns opaque errors:
// configuration failures (missing model/driver) before any API
// call, and the provider SDK's raw error after the call. We treat
// context cancellation/deadline as terminal, plus a lightweight
// heuristic for non-transient auth/client errors. Anything
// unrecognized defaults to retryable so genuinely transient 5xx /
// 429 / network blips keep retrying (matching the prior blind-retry
// behavior).
func isRetryableLLMError(err error) bool {
	if err == nil {
		return true
	}
	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
		return false
	}
	msg := strings.ToLower(err.Error())
	for _, s := range []string{
		"unauthorized", "authentication", "api key",
		"bad request", "not found", "model not found", "content filter",
		"no driver resolved", "model_name is required", "resolve driver",
	} {
		if strings.Contains(msg, s) {
			return false
		}
	}
	if nonRetryableStatusRE.MatchString(msg) {
		return false
	}
	return true
}

// callText runs one chat call and returns the cleaned text response.
func (c *ExtractorComponent) callText(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (string, error) {
	resp, err := c.callRaw(ctx, db, in, systemPrompt, chunkText)
	if err != nil {
		return "", err
	}
	return cleanLLMText(resp.Content), nil
}

// callStructured runs one chat call, cleans the response, and explicitly
// parses it as a JSON object. Used by the metadata path, which needs a
// structured value to merge into chunk metadata — matching Python
// gen_metadata's json_repair.loads. A non-JSON or empty response returns
// (nil, nil) so the caller treats it as "nothing extracted", not an error.
func (c *ExtractorComponent) callStructured(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (map[string]any, error) {
	resp, err := c.callRaw(ctx, db, in, systemPrompt, chunkText)
	if err != nil {
		return nil, err
	}
	s := cleanLLMText(resp.Content)
	if s == "" {
		return nil, nil
	}
	// Second cleanup layer, mirroring Python gen_metadata's
	// re.sub(r"^.*</think>", "", ans, re.DOTALL): cleanLLMText only strips a
	// leading <think>, so a mid-text reasoning block preceded by a preamble
	// would otherwise survive into JSON parsing and silently drop the whole
	// metadata extraction that Python would have kept. No **ERROR** check here
	// — Python's gen_metadata has none either.
	s = common.StripThinkTrailing(s)
	parsed, ok := tryParseJSONObject(s)
	if !ok {
		return nil, nil
	}
	return parsed, nil
}

// resolveExtractorChatTarget resolves the chat target for llmID, reusing a
// per-run cache attached to ctx when present. Without the cache it behaves
// exactly as resolveExtractorChatTargetUncached (the historical per-call path),
// so callers and tests that do not inject a cache are unchanged.
func resolveExtractorChatTarget(ctx context.Context, db *gorm.DB, llmID string) (driver, modelName, apiKey, baseURL string, err error) {
	if c := extractorModelCacheFromCtx(ctx); c != nil {
		return c.target(ctx, db, llmID)
	}
	return resolveExtractorChatTargetUncached(ctx, db, llmID)
}

// resolveExtractorChatTargetUncached resolves the llm_id into driver / model /
// api_key / base_url without any per-run cache. The llm_id may be a bare
// tenant_model UUID or a composite "model@provider" string. Errors from DAO
// resolution are propagated so the caller sees the real failure reason.
func resolveExtractorChatTargetUncached(ctx context.Context, db *gorm.DB, llmID string) (driver, modelName, apiKey, baseURL string, err error) {
	if override := getExtractorChatTargetResolverOverride(); override != nil {
		if driver, modelName, apiKey, baseURL, ok := override(llmID); ok {
			return driver, modelName, apiKey, baseURL, nil
		}
	}

	// When llmID is empty, try tenant default chat model
	// (mirrors Python task_executor.py:573-574 fallback).
	if llmID == "" {
		if cfg := resolveExtractorChatDefaultConfig(ctx, db); cfg.driver != "" {
			return cfg.driver, cfg.modelName, cfg.apiKey, cfg.baseURL, nil
		}
	}

	cfg, cfgErr := resolveExtractorChatConfig(ctx, db, llmID)
	if cfgErr != nil {
		return "", "", "", "", cfgErr
	}
	if cfg.driver != "" {
		return cfg.driver, cfg.modelName, cfg.apiKey, cfg.baseURL, nil
	}

	// Fallback: when tenant credentials are not available
	// (no canvas state / no DB), fall back to the basic @ split
	// so callers can still use model@provider format in tests.
	if bare, provider, ok := splitExtractorLLIDPair(llmID); ok {
		return strings.ToLower(provider), bare, "", "", nil
	}
	// Nothing left to try — let Chat() surface a clear error when
	// the driver ends up empty.
	return "", llmID, "", "", nil
}

// extractorChatConfig holds the resolved chat model configuration.
type extractorChatConfig struct {
	driver    string // llm_factory
	modelName string // llm_name
	apiKey    string
	baseURL   string // api_base
}

// resolveExtractorChatConfig resolves tenant-scoped credentials for
// the given llm_id via the shared resolveModelConfig helper.
//
//   - Bare UUID → DAO lookup via resolveModelConfigByID.
//   - "model@provider" → parsed via resolveModelConfigFromProviderInstance.
//
// Returns nil error when there is no canvas state (unit tests) —
// the caller's @ split fallback handles that case.
func resolveExtractorChatConfig(ctx context.Context, db *gorm.DB, compositeLLMID string) (extractorChatConfig, error) {
	state, err := runtime.GetStateFromContext(ctx)
	if err != nil || state == nil {
		return extractorChatConfig{}, nil
	}
	tidVal, _ := state.GetGlobal("tenant_id")
	tid, _ := tidVal.(string)
	if tid == "" {
		return extractorChatConfig{}, nil
	}

	var driver models.ModelDriver
	var modelName string
	var apiConfig *models.APIConfig
	if isBareTenantModelID(compositeLLMID) {
		// Resolve the exact tenant model without a separate existence check.
		resolved, err := resolveModelConfigByID(ctx, db, tid, entity.ModelTypeChat, compositeLLMID)
		if err != nil {
			return extractorChatConfig{}, fmt.Errorf("extractor: tenant model %q not found or not usable: %w", compositeLLMID, err)
		}
		driver, modelName, apiConfig = resolved.driver, resolved.model.ModelName, resolved.apiConfig
	} else {
		// Composite "model@provider" path: delegate to the shared dispatcher.
		driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tid, entity.ModelTypeChat, compositeLLMID)
		if err != nil {
			return extractorChatConfig{}, fmt.Errorf("extractor: resolve model %q: %w", compositeLLMID, err)
		}
	}

	return newExtractorChatConfig(driver, modelName, apiConfig), nil
}

// newExtractorChatConfig builds the chat config from a resolved driver and its
// API settings: the driver name lowercased as the llm_factory, with the
// credentials dereferenced defensively (a nil apiConfig or nil fields mean
// "not configured", not a panic).
func newExtractorChatConfig(driver models.ModelDriver, modelName string, apiConfig *models.APIConfig) extractorChatConfig {
	apiKey := ""
	baseURL := ""
	if apiConfig != nil {
		if apiConfig.ApiKey != nil {
			apiKey = *apiConfig.ApiKey
		}
		if apiConfig.BaseURL != nil {
			baseURL = *apiConfig.BaseURL
		}
	}
	return extractorChatConfig{
		driver:    strings.ToLower(driver.Name()),
		modelName: modelName,
		apiKey:    apiKey,
		baseURL:   baseURL,
	}
}

// resolveExtractorChatDefaultConfig resolves the tenant's default chat
// model when no explicit llm_id is provided. Returns empty config when
// no canvas state or tenant_id is available (unit-test context).
func resolveExtractorChatDefaultConfig(ctx context.Context, db *gorm.DB) extractorChatConfig {
	state, err := runtime.GetStateFromContext(ctx)
	if err != nil || state == nil {
		return extractorChatConfig{}
	}
	tidVal, _ := state.GetGlobal("tenant_id")
	tid, _ := tidVal.(string)
	if tid == "" {
		return extractorChatConfig{}
	}

	driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tid, entity.ModelTypeChat)
	if err != nil || driver == nil {
		return extractorChatConfig{}
	}

	return newExtractorChatConfig(driver, modelName, apiConfig)
}

// isBareTenantModelID reports whether s is a 32-character hex string
// (a tenant_model primary key), as opposed to a composite "model@provider".
func isBareTenantModelID(s string) bool {
	s = strings.TrimSpace(s)
	if len(s) != 32 {
		return false
	}
	for _, r := range s {
		if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
			return false
		}
	}
	return true
}

// extractorContextLengthOverride is a narrow test seam mirroring
// extractorChatTargetResolverOverride: it lets unit tests supply a context
// length without a real tenant model row, so the message-fitting wiring in
// callRaw/llmTagChunk can be exercised without a DB. When set,
// extractorContextLength consults it first.
var (
	extractorContextLengthOverrideMu sync.RWMutex
	extractorContextLengthOverride   func(ctx context.Context, llmID string) int
)

// SetExtractorContextLengthOverride swaps the package-level context-length
// resolver for tests. Pass nil to restore the default. Concurrent-safe.
func SetExtractorContextLengthOverride(fn func(ctx context.Context, llmID string) int) {
	extractorContextLengthOverrideMu.Lock()
	defer extractorContextLengthOverrideMu.Unlock()
	extractorContextLengthOverride = fn
}

func getExtractorContextLengthOverride() func(ctx context.Context, llmID string) int {
	extractorContextLengthOverrideMu.RLock()
	defer extractorContextLengthOverrideMu.RUnlock()
	return extractorContextLengthOverride
}

// extractorContextLength returns the chat model's context window for llmID,
// reusing a per-run cache attached to ctx when present. The context-length
// override seam is still consulted first (unchanged test priority), and without
// a cache the behavior is exactly extractorContextLengthUncached.
func extractorContextLength(ctx context.Context, db *gorm.DB, llmID string) int {
	if fn := getExtractorContextLengthOverride(); fn != nil {
		return fn(ctx, llmID)
	}
	if c := extractorModelCacheFromCtx(ctx); c != nil {
		return c.ctxLen(ctx, db, llmID)
	}
	return extractorContextLengthUncached(ctx, db, llmID)
}

// extractorContextLengthUncached returns the chat model's context window
// (content_length) for llmID, or 0 when unavailable. Mirrors Python's
// chat_mdl.max_length. Used as the token budget for message fitting so
// oversized prompts are trimmed instead of rejected by the provider. When
// llm_id is empty the call falls back to the tenant default chat model (see
// resolveExtractorChatTarget), so the same model is resolved here; otherwise
// the default-model path would never get message fitting. Returns 0 (skip
// fitting) when the model is unknown (e.g. unit tests with synthetic llm_id or
// no canvas state).
func extractorContextLengthUncached(ctx context.Context, db *gorm.DB, llmID string) int {
	state, err := runtime.GetStateFromContext(ctx)
	if err != nil || state == nil {
		return 0
	}
	tidVal, _ := state.GetGlobal("tenant_id")
	tid, _ := tidVal.(string)
	if tid == "" {
		return 0
	}
	if llmID == "" {
		llmID = defaultChatModelRef(ctx, db, tid)
	}
	if llmID == "" {
		return 0
	}
	return modelcontext.ResolveModelContentLength(ctx, db, tid, llmID, "", "")
}

// defaultChatModelRef returns the tenant's default chat model reference —
// the tenant_model UUID when one is pinned, otherwise the composite
// "model@provider" id — or "" when the tenant has no default chat model.
func defaultChatModelRef(ctx context.Context, db *gorm.DB, tenantID string) string {
	if db == nil {
		// No database to read the tenant's default chat model from.
		return ""
	}
	membership, err := resolveDefaultModelMembership(ctx, db, tenantID)
	if err != nil || membership == nil {
		return ""
	}
	if membership.TenantLLMID != nil && *membership.TenantLLMID != "" {
		return *membership.TenantLLMID
	}
	return membership.LLMID
}

// extractorContextFitBudget returns 97% of the model's context window as the
// fitting budget, mirroring agent/chat's ContextFitBudget (the canonical
// helper) plus the clamp below. The
// margin leaves headroom for the difference between the cl100k tokenizer used
// for counting and the model's own tokenizer, plus per-message formatting
// overhead, so a fitted prompt stays inside the provider's real context limit
// instead of landing exactly on it.
func extractorContextFitBudget(ctxLen int) int {
	budget := int(float64(ctxLen) * 0.97)
	if budget < 1 {
		// Never hand Fit a <=0 budget: Fit treats <=0 as the 8192
		// default, which would stop trimming entirely for a tiny context.
		return 1
	}
	return budget
}

// fitExtractorMessages trims msgs to the chat model's context window using
// the shared tokenizer fitter (mirrors Python's message_fit_in), dropping
// entries the fitter removed. It returns a clear error instead of letting a
// conversation whose final user turn was trimmed to empty reach the provider:
// the proportional trim can do that when the system prompt alone exceeds the
// context budget, and providers reject empty user turns with an obscure error
// after retries.
func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs []eschema.Message) ([]eschema.Message, error) {
	ctxLen := extractorContextLength(ctx, db, llmID)
	if ctxLen <= 0 {
		return msgs, nil
	}
	fitMsgs := make([]tokenizer.Message, len(msgs))
	for i := range msgs {
		fitMsgs[i] = tokenizer.Message{Role: string(msgs[i].Role), Content: msgs[i].Content}
	}
	kept, keptIdx, _ := tokenizer.Fit(fitMsgs, extractorContextFitBudget(ctxLen))

	fitted := make([]eschema.Message, 0, len(kept))
	for j, i := range keptIdx {
		msgs[i].Content = kept[j].Content
		fitted = append(fitted, msgs[i])
	}
	if len(fitted) == 0 {
		return nil, errors.New("extractor: message fitting dropped every message; check the chat model context length setting")
	}
	// The system prompt carries the extraction contract (output format,
	// field definitions); sending without it would silently produce
	// garbage. The proportional trim can empty every system message when
	// the final user turn alone fills the budget, so require at least one
	// retained system message with non-empty content — a system message kept
	// but trimmed to empty is just as useless as a dropped one. The guard only
	// applies when the input actually had a system message: systemPrompt is
	// optional and a user-only prompt is a valid request.
	hadSystem := false
	for _, m := range msgs {
		if m.Role == eschema.System {
			hadSystem = true
			break
		}
	}
	if hadSystem {
		keptSystem := false
		for _, m := range fitted {
			if m.Role == eschema.System && strings.TrimSpace(m.Content) != "" {
				keptSystem = true
				break
			}
		}
		if !keptSystem {
			return nil, errors.New("extractor: message fitting emptied the system prompt; check the chat model context length setting or reduce the prompt size")
		}
	}
	last := fitted[len(fitted)-1]
	if last.Role != eschema.User || strings.TrimSpace(last.Content) == "" {
		return nil, errors.New("extractor: message fitting emptied the final user turn; check the chat model context length setting or reduce the prompt size")
	}
	return fitted, nil
}

// buildExtractorMessages assembles system + user messages for one extraction
// call. Normal per-chunk schedulers reject empty chunk text before reaching
// this low-level builder; the single-space fallback remains for direct
// callRaw callers that rely on the historical message shape.
// System prompt is omitted if empty or whitespace-only to avoid sending empty
// system turns to LLM providers.
func buildExtractorMessages(systemPrompt, chunkText string) []eschema.Message {
	out := make([]eschema.Message, 0, 2)
	if strings.TrimSpace(systemPrompt) != "" {
		out = append(out, eschema.Message{Role: eschema.System, Content: systemPrompt})
	}
	userContent := chunkText
	if strings.TrimSpace(userContent) == "" {
		userContent = " "
	}
	out = append(out, eschema.Message{Role: eschema.User, Content: userContent})
	return out
}

// tryParseJSONObject tries to parse s as a JSON object. Returns
// (parsed, true) on success; (nil, false) on parse error or when
// s is not a JSON object. Trims common Markdown code fences
// (```json ... ```) before parsing.
func tryParseJSONObject(s string) (map[string]any, bool) {
	trimmed := strings.TrimSpace(s)
	// Strip a surrounding Markdown code fence. Models commonly wrap JSON in
	// ```json ... ``` (language tag on the opening line) but some emit the tag
	// on its own line (```\njson\n{...}); Python's json_repair tolerates both,
	// encoding/json does not, so we drop the fence and any bare leading
	// "json"/"JSON" language line before parsing.
	if strings.HasPrefix(trimmed, "```") {
		// Drop the opening fence line (````` plus an optional inline language
		// tag such as `json`) up to and including the first newline.
		if idx := strings.IndexByte(trimmed, '\n'); idx >= 0 {
			trimmed = trimmed[idx+1:]
		} else {
			trimmed = trimmed[3:]
		}
		// Some models place the language tag on the next line
		// (`````\njson\n{...}). Drop a bare leading `json`/`JSON` line so it is
		// not mistaken for JSON content.
		if firstNL := strings.IndexByte(trimmed, '\n'); firstNL >= 0 {
			if firstLine := strings.TrimSpace(trimmed[:firstNL]); firstLine == "json" || firstLine == "JSON" {
				trimmed = trimmed[firstNL+1:]
			}
		} else if firstLine := strings.TrimSpace(trimmed); firstLine == "json" || firstLine == "JSON" {
			trimmed = ""
		}
		// Drop the closing fence if present.
		if i := strings.LastIndex(trimmed, "```"); i >= 0 {
			trimmed = trimmed[:i]
		}
		trimmed = strings.TrimSpace(trimmed)
	}
	var out map[string]any
	if err := json.Unmarshal([]byte(trimmed), &out); err != nil {
		return nil, false
	}
	if out == nil {
		return nil, false
	}
	// An empty object carries no information the caller can act on;
	// surface as "could not extract" so downstream code can route
	// it to the same fallback it would use for malformed text.
	if len(out) == 0 {
		return nil, false
	}
	return out, true
}

// parseMetadataFieldDefs converts an any value (typically []any of maps)
// to a typed []common.MetadataFieldDef slice. The parsing lives in common so the
// ingestion extractor and the agentic metadata catalog share one field contract.
func parseMetadataFieldDefs(v any) []common.MetadataFieldDef {
	return common.MetadataFieldDefsFromRaw(v)
}

// mapInt converts a JSON-compatible value to int.
func mapInt(v interface{}) int {
	switch n := v.(type) {
	case float64:
		return int(n)
	case int:
		return n
	case int64:
		return int(n)
	}
	return 0
}

// init registers Extractor under CategoryIngestion (per plan §4
// Phase 2.5). Metadata is derived from the Inputs()/Outputs()
// methods on ExtractorComponent so the API layer (Phase 4) can
// enumerate the catalog without instantiating the component.
func init() {
	c := &ExtractorComponent{}
	runtime.MustRegister(componentNameExtractor, runtime.CategoryIngestion,
		func(_ string, params map[string]any) (runtime.Component, error) {
			return NewExtractorComponent(params)
		},
		runtime.Metadata{
			Version: "1.0.0",
			Inputs:  c.Inputs(),
			Outputs: c.Outputs(),
		})
}