Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

14.3 KB
Raw Blame
package types

import (
	"database/sql/driver"
	"encoding/json"
)

// SearchTargetType represents the type of search target
type SearchTargetType string

const (
	// SearchTargetTypeKnowledgeBase - search entire knowledge base
	SearchTargetTypeKnowledgeBase SearchTargetType = "knowledge_base"
	// SearchTargetTypeKnowledge - search specific knowledge files within a knowledge base
	SearchTargetTypeKnowledge SearchTargetType = "knowledge"
)

// TagScope represents a tag-constrained retrieval scope inside one knowledge base.
type TagScope struct {
	KnowledgeBaseID string   `json:"knowledge_base_id"`
	TagIDs          []string `json:"tag_ids"`
}

// SearchTarget represents a unified search target
// Either search an entire knowledge base, or specific knowledge files within a knowledge base
type SearchTarget struct {
	// Type of search target
	Type SearchTargetType `json:"type"`
	// KnowledgeBaseID is the ID of the knowledge base to search
	KnowledgeBaseID string `json:"knowledge_base_id"`
	// TenantID is the tenant ID that owns this knowledge base
	// Required for cross-tenant shared KB queries
	TenantID uint64 `json:"tenant_id"`
	// KnowledgeIDs is the list of specific knowledge IDs to search within the knowledge base
	// Only used when Type is SearchTargetTypeKnowledge
	KnowledgeIDs []string `json:"knowledge_ids,omitempty"`
	// TagIDs limits retrieval to chunks/documents carrying any of these KB-local tags.
	TagIDs []string `json:"tag_ids,omitempty"`
	// ScopeTagIDs records the logical tag scope selected by the user. For
	// document KBs this is kept for tracing after the relation-table lookup has
	// been resolved to KnowledgeIDs; TagIDs remains the physical index filter.
	ScopeTagIDs []string `json:"scope_tag_ids,omitempty"`
	// DisableRecallThresholds keeps recall broad inside an already constrained,
	// user-selected scope. The reranker still orders candidates, but vector and
	// keyword thresholds cannot erase the whole explicit scope before reranking.
	DisableRecallThresholds bool `json:"disable_recall_thresholds,omitempty"`
}

// SearchTargets is a list of search targets, pre-computed at request entry point
type SearchTargets []*SearchTarget

// RecallThresholds returns the effective recall thresholds for this target.
func (st *SearchTarget) RecallThresholds(vectorThreshold, keywordThreshold float64) (float64, float64) {
	if st != nil && st.DisableRecallThresholds {
		return 0, 0
	}
	return vectorThreshold, keywordThreshold
}

// HasRecallThresholdOverride reports whether any target represents an
// authoritative scope whose candidates must reach reranking before filtering.
func (st SearchTargets) HasRecallThresholdOverride() bool {
	for _, target := range st {
		if target != nil && target.DisableRecallThresholds {
			return true
		}
	}
	return false
}

// HasKnowledgeRetrievalScope reports whether a request has any effective
// knowledge retrieval scope. SearchTargets are the unified runtime form and
// must be considered alongside the legacy/raw KB and knowledge ID fields so
// tag-only mentions are not mistaken for pure chat.
func HasKnowledgeRetrievalScope(
	searchTargets SearchTargets,
	knowledgeBaseIDs []string,
	knowledgeIDs []string,
) bool {
	for _, id := range knowledgeBaseIDs {
		if id != "" {
			return true
		}
	}
	for _, id := range knowledgeIDs {
		if id != "" {
			return true
		}
	}
	for _, target := range searchTargets {
		if target == nil || target.KnowledgeBaseID == "" {
			continue
		}
		if target.Type == SearchTargetTypeKnowledgeBase || len(target.KnowledgeIDs) > 0 ||
			len(target.TagIDs) > 0 || len(target.ScopeTagIDs) > 0 {
			return true
		}
	}
	return false
}

// GetAllKnowledgeBaseIDs returns all unique knowledge base IDs from the search targets
func (st SearchTargets) GetAllKnowledgeBaseIDs() []string {
	seen := make(map[string]bool)
	var result []string
	for _, t := range st {
		if t == nil || t.KnowledgeBaseID == "" {
			continue
		}
		if !seen[t.KnowledgeBaseID] {
			seen[t.KnowledgeBaseID] = true
			result = append(result, t.KnowledgeBaseID)
		}
	}
	return result
}

// GetKBTenantMap returns a map from knowledge base ID to tenant ID
func (st SearchTargets) GetKBTenantMap() map[string]uint64 {
	result := make(map[string]uint64)
	for _, t := range st {
		if t != nil && t.KnowledgeBaseID != "" {
			result[t.KnowledgeBaseID] = t.TenantID
		}
	}
	return result
}

// GetTenantIDForKB returns the tenant ID for a given knowledge base ID
// Returns 0 if not found
func (st SearchTargets) GetTenantIDForKB(kbID string) uint64 {
	for _, t := range st {
		if t != nil && t.KnowledgeBaseID == kbID {
			return t.TenantID
		}
	}
	return 0
}

// ContainsKB checks if the search targets contain a given knowledge base ID
func (st SearchTargets) ContainsKB(kbID string) bool {
	for _, t := range st {
		if t != nil && t.KnowledgeBaseID == kbID {
			return true
		}
	}
	return false
}

// MatchedImage retains an image vector hit's identity before context expansion
// replaces ImageInfo with all images in a surrounding text chunk.
type MatchedImage struct {
	ChunkID         string `json:"chunk_id"`
	KnowledgeBaseID string `json:"knowledge_base_id"`
	URL             string `json:"url"`
}

// SearchResult represents the search result
type SearchResult struct {
	// CitationSources retains independently citeable bodies after context expansion.
	CitationSources []*SearchResult `json:"-" gorm:"-"`
	// ID
	ID string `gorm:"column:id"              json:"id" extensions:"x-agent=chunk_id"`
	// Content
	Content string `gorm:"column:content"         json:"content" extensions:"x-agent,x-agent-max-runes=600"`
	// Knowledge ID
	KnowledgeID string `gorm:"column:knowledge_id"    json:"knowledge_id" extensions:"x-agent"`
	// Chunk index
	ChunkIndex int `gorm:"column:chunk_index"     json:"chunk_index" extensions:"x-agent"`
	// Knowledge title
	KnowledgeTitle string `gorm:"column:knowledge_title" json:"knowledge_title" extensions:"x-agent"`
	// Start at
	StartAt int `gorm:"column:start_at"        json:"start_at"`
	// End at
	EndAt int `gorm:"column:end_at"          json:"end_at"`
	// Seq
	Seq int `gorm:"column:seq"             json:"seq"`
	// Score
	Score float64 `                              json:"score" extensions:"x-agent"`
	// VectorScore is the engine's own similarity of a vector hit, before
	// fusion replaced Score with a rank-based one; 0 for other hits. See
	// IndexWithScore.VectorScore.
	VectorScore float64 `json:"-"`
	// Match type
	MatchType MatchType `                              json:"match_type" extensions:"x-agent"`
	// SubChunkIndex
	SubChunkID []string `                              json:"sub_chunk_id"`
	// Metadata
	Metadata map[string]string `                              json:"metadata"`

	// Chunk 类型
	ChunkType string `json:"chunk_type" extensions:"x-agent"`
	// 父 Chunk ID
	ParentChunkID string `json:"parent_chunk_id"`
	// 图片信息 (JSON 格式)
	ImageInfo string `json:"image_info"`
	// MatchedImages survives merging, deduplication and stored reference replay.
	// It contains storage references, never image bytes.
	MatchedImages []MatchedImage `json:"matched_images,omitempty" gorm:"-"`

	// Knowledge file name
	// Used for file type knowledge, contains the original file name
	KnowledgeFilename string `json:"knowledge_filename"`

	// Knowledge source
	// Used to indicate the source of the knowledge, such as "url"
	KnowledgeSource string `json:"knowledge_source"`

	// KnowledgeChannel indicates through which channel the knowledge was ingested (web, api, wechat, etc.)
	KnowledgeChannel string `json:"knowledge_channel"`

	// ChunkMetadata stores chunk-level metadata (e.g., generated questions)
	ChunkMetadata JSON `json:"chunk_metadata,omitempty"`
	// SkipGeneratedQuestions keeps generated questions out of the rerank
	// passage. Search sets it when the knowledge base's question generation
	// switch is off, including bases that were turned off before the index
	// rows were disabled.
	SkipGeneratedQuestions bool `json:"-" gorm:"-"`
	// ContextHeader is the chunk's heading breadcrumb, part of what was
	// embedded (Chunk.EmbeddingContent) but not of Content. Internal only.
	ContextHeader string `json:"-"`

	// MatchedContent is the actual content that was matched in vector search
	// For FAQ: this is the matched question text (standard or similar question)
	MatchedContent string `json:"matched_content,omitempty"`

	// KnowledgeDescription is the description of the knowledge document
	KnowledgeDescription string `json:"knowledge_description,omitempty"`

	// KnowledgeCustomMetadata is user-authored context safe to expose to models.
	KnowledgeCustomMetadata string `json:"knowledge_custom_metadata,omitempty"`

	// KnowledgeBaseID is the ID of the knowledge base this result belongs to
	KnowledgeBaseID string `json:"knowledge_base_id,omitempty"`

	// SourceLocators point back into the original file for citation
	// navigation. Merged results carry the union of their chunks' locators.
	SourceLocators SourceLocators `json:"source_locators,omitempty"`

	// ContentRevision is the chunk edit revision at retrieval time.
	// Internal only: used by the merge pipeline to decide whether source
	// coordinates are still trustworthy.
	//
	// Zero means "never edited", so any construction path that forgets to carry
	// it over fails open to the position path. Results rebuilt from JSON (stored
	// message references) always land on zero because the field is not
	// serialized; the length invariant in chunkTrusted is the remaining guard
	// there. Keep the gorm column explicit so direct row scans populate it.
	ContentRevision int `gorm:"column:content_revision" json:"-"`

	// ContentRewritten reports that the merge pipeline replaced Content
	// (parent or neighbor expansion) after retrieval, so StartAt/EndAt no
	// longer describe the body. Internal only: used by the merge pipeline,
	// never serialized.
	ContentRewritten bool `json:"-"`
}

// SearchParams represents the search parameters
type SearchParams struct {
	// QueryText is required unless query_embedding is provided, keyword matching is disabled,
	// and vector matching remains enabled.
	QueryText      string    `json:"query_text" extensions:"x-agent,x-agent-required"`
	QueryEmbedding []float32 `json:"query_embedding,omitempty"`
	// Minimum vector similarity a hit needs; 0 keeps every vector hit.
	VectorThreshold float64 `json:"vector_threshold" extensions:"x-agent"`
	// Minimum keyword score a hit needs; 0 keeps every keyword hit.
	KeywordThreshold float64 `json:"keyword_threshold" extensions:"x-agent"`
	// Number of chunks to return; 0 uses the default.
	MatchCount int `json:"match_count" extensions:"x-agent,x-agent-maximum=30"`
	// Turns keyword matching off, leaving vector matching only.
	DisableKeywordsMatch bool `json:"disable_keywords_match" extensions:"x-agent"`
	// Turns vector matching off, leaving keyword matching only.
	DisableVectorMatch bool `json:"disable_vector_match" extensions:"x-agent"`
	// Limits the search to these documents of the knowledge base.
	KnowledgeIDs    []string `json:"knowledge_ids" extensions:"x-agent,x-agent-maximum=20"`
	TagIDs          []string `json:"tag_ids"` // Tag IDs for filtering (used for FAQ priority filtering)
	ScopeTagIDs     []string `json:"scope_tag_ids,omitempty"`
	OnlyRecommended bool     `json:"only_recommended"`
	// KnowledgeBaseIDs overrides the single KB ID passed to HybridSearch,
	// allowing a single retrieval call to span multiple KBs that share the
	// same embedding model. When empty, HybridSearch uses its own id parameter.
	KnowledgeBaseIDs []string `json:"knowledge_base_ids,omitempty"`
	// SkipContextEnrichment skips fetching parent, nearby, and relation chunks
	// in processSearchResults. Used by the chat pipeline where context assembly
	// is handled separately in the merge stage.
	SkipContextEnrichment bool `json:"skip_context_enrichment,omitempty"`
	// Rerank, when present and enabled, reranks the fused candidates before
	// truncating to MatchCount. Only the hybrid-search API reads it; internal
	// callers rerank on their own.
	Rerank *RerankOptions `json:"rerank,omitempty"`
}

// Value implements the driver.Valuer interface, used to convert SearchResult to database value
func (c SearchResult) Value() (driver.Value, error) {
	return json.Marshal(c)
}

// Scan implements the sql.Scanner interface, used to convert database value to SearchResult
func (c *SearchResult) Scan(value interface{}) error {
	if value == nil {
		return nil
	}
	b, ok := value.([]byte)
	if !ok {
		return nil
	}
	return json.Unmarshal(b, c)
}

// Pagination represents the pagination parameters
type Pagination struct {
	// Page
	Page int `form:"page"      json:"page"      binding:"omitempty,min=1"`
	// Page size
	PageSize int `form:"page_size" json:"page_size" binding:"omitempty,min=1,max=1000"`
}

// GetPage gets the page number, default is 1
func (p *Pagination) GetPage() int {
	if p.Page < 1 {
		return 1
	}
	return p.Page
}

// GetPageSize gets the page size, default is 20
func (p *Pagination) GetPageSize() int {
	if p.PageSize < 1 {
		return 20
	}
	if p.PageSize > 1000 {
		return 1000
	}
	return p.PageSize
}

// Offset gets the offset for database query
func (p *Pagination) Offset() int {
	return (p.GetPage() - 1) * p.GetPageSize()
}

// Limit gets the limit for database query
func (p *Pagination) Limit() int {
	return p.GetPageSize()
}

// PageResult represents the pagination query result
type PageResult struct {
	Total    int64       `json:"total"`     // Total number of records
	Page     int         `json:"page"`      // Current page number
	PageSize int         `json:"page_size"` // Page size
	Data     interface{} `json:"data"`      // Data
}

// NewPageResult creates a new pagination result
func NewPageResult(total int64, page *Pagination, data interface{}) *PageResult {
	return &PageResult{
		Total:    total,
		Page:     page.GetPage(),
		PageSize: page.GetPageSize(),
		Data:     data,
	}
}

// SearchResult metadata for results that rest on an image matched by its own
// vector (an image_vector hit).
const (
	// MetadataKeptBy names why a result was kept other than by its rerank
	// score. A kept result rides outside the ranked top-k: truncating to
	// top-k must not drop it.
	MetadataKeptBy = "kept_by"
	// KeptByImageVector: a pictorial image hit a text reranker rejected and
	// its vector still vouched for.
	KeptByImageVector = "image_vector"
	// MetadataImageVectorMatch marks a result that stands in for an
	// image_vector hit de-duplication dropped as a copy of it, so the image
	// still reaches a model that can see it.
	MetadataImageVectorMatch = "image_vector_match"
)