Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

14.0 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Data types for OpenViking retrieval module.
"""

import queue
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Dict, List, Optional

from openviking.utils.tags import normalize_search_tags


class ContextType(str, Enum):
    """Context type for retrieval."""

    MEMORY = "memory"
    RESOURCE = "resource"
    SKILL = "skill"


class TraceEventType(str, Enum):
    """Types of trace events for retrieval process visualization."""

    # Recursive search phase
    SEARCH_DIRECTORY_START = "search_directory_start"
    SEARCH_DIRECTORY_RESULT = "search_directory_result"

    # Scoring phase
    EMBEDDING_SCORES = "embedding_scores"
    RERANK_SCORES = "rerank_scores"

    # Selection phase
    CANDIDATE_SELECTED = "candidate_selected"
    CANDIDATE_EXCLUDED = "candidate_excluded"
    DIRECTORY_QUEUED = "directory_queued"

    # Convergence
    CONVERGENCE_CHECK = "convergence_check"
    SEARCH_CONVERGED = "search_converged"

    # Summary
    SEARCH_SUMMARY = "search_summary"


@dataclass
class TraceEvent:
    """
    Single trace event for retrieval process.

    Attributes:
        event_type: Type of event
        timestamp: Relative timestamp in seconds from trace start
        message: Human-readable description
        data: Structured event data for visualization
        query_id: Optional query identifier for multi-query scenarios
    """

    event_type: TraceEventType
    timestamp: float
    message: str
    data: Dict[str, Any] = field(default_factory=dict)
    query_id: Optional[str] = None

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization."""
        result: Dict[str, Any] = {
            "event_type": self.event_type.value,
            "timestamp": round(self.timestamp, 4),
            "message": self.message,
            "data": self.data,
        }
        if self.query_id:
            result["query_id"] = self.query_id
        return result


@dataclass
class ScoreDistribution:
    """
    Score distribution statistics for visualization.

    Attributes:
        scores: List of (uri, score) tuples sorted by score descending
        min_score: Minimum score
        max_score: Maximum score
        mean_score: Mean score
        threshold: Score threshold used for filtering
    """

    scores: List[tuple]  # [(uri, score), ...]
    min_score: float = 0.0
    max_score: float = 0.0
    mean_score: float = 0.0
    threshold: float = 0.0

    @classmethod
    def from_scores(
        cls,
        uri_scores: List[tuple],
        threshold: float = 0.0,
    ) -> "ScoreDistribution":
        """Create from list of (uri, score) tuples."""
        if not uri_scores:
            return cls(scores=[], threshold=threshold)

        scores_only = [s for _, s in uri_scores]
        return cls(
            scores=sorted(uri_scores, key=lambda x: x[1], reverse=True),
            min_score=min(scores_only),
            max_score=max(scores_only),
            mean_score=sum(scores_only) / len(scores_only),
            threshold=threshold,
        )

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization."""
        return {
            "scores": [{"uri": u, "score": round(s, 4)} for u, s in self.scores],
            "min": round(self.min_score, 4),
            "max": round(self.max_score, 4),
            "mean": round(self.mean_score, 4),
            "threshold": self.threshold,
            "count": len(self.scores),
            "above_threshold": sum(1 for _, s in self.scores if s >= self.threshold),
        }


@dataclass
class ThinkingTrace:
    """
    Structured thinking trace for retrieval process visualization.

    Captures the complete retrieval decision process:
    1. Directory location reasoning
    2. Search decisions per directory
    3. Score distributions (embedding + rerank)
    4. Selection/exclusion reasons
    5. Convergence information

    Thread-safe for concurrent query execution using Queue.

    Attributes:
        _events: Queue of trace events (thread-safe)
        start_time: Trace start time (for relative timestamps)
    """

    start_time: float = field(default_factory=time.time)
    _events: queue.Queue = field(default_factory=queue.Queue, init=False, repr=False)

    def add_event(
        self,
        event_type: TraceEventType,
        message: str,
        data: Optional[Dict[str, Any]] = None,
        query_id: Optional[str] = None,
    ) -> None:
        """
        Add a trace event (thread-safe).

        Args:
            event_type: Type of event
            message: Human-readable message
            data: Event data dictionary
            query_id: Optional query identifier for multi-query scenarios
        """
        event = TraceEvent(
            event_type=event_type,
            timestamp=time.time() - self.start_time,
            message=message,
            data=data or {},
            query_id=query_id,
        )
        self._events.put(event)

    def get_events(self, query_id: Optional[str] = None) -> List[TraceEvent]:
        """
        Get all events, optionally filtered by query_id.

        Args:
            query_id: If provided, only return events for this query

        Returns:
            List of trace events (snapshot)
        """
        # Get snapshot of all events
        all_events = list(self._events.queue)

        if query_id is None:
            return all_events
        return [e for e in all_events if e.query_id == query_id]

    @property
    def events(self) -> List[TraceEvent]:
        """Get all events as list."""
        return self.get_events()

    def get_statistics(self) -> Dict[str, Any]:
        """Calculate summary statistics from events."""
        stats = {
            "total_events": len(self.events),
            "duration_seconds": 0.0,
            "directories_searched": 0,
            "candidates_collected": 0,
            "candidates_excluded": 0,
            "convergence_rounds": 0,
        }

        if self.events:
            stats["duration_seconds"] = round(self.events[-1].timestamp, 4)

        for event in self.events:
            if event.event_type == TraceEventType.SEARCH_DIRECTORY_RESULT:
                stats["directories_searched"] += 1
            elif event.event_type == TraceEventType.CANDIDATE_SELECTED:
                stats["candidates_collected"] += event.data.get("count", 1)
            elif event.event_type == TraceEventType.CANDIDATE_EXCLUDED:
                stats["candidates_excluded"] += event.data.get("count", 1)
            elif event.event_type == TraceEventType.CONVERGENCE_CHECK:
                stats["convergence_rounds"] = event.data.get("round", 0)

        return stats

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization."""
        return {
            "events": [e.to_dict() for e in self.events],
            "statistics": self.get_statistics(),
        }

    def to_messages(self) -> List[str]:
        """Convert to simple message list."""
        return [e.message for e in self.events]


@dataclass
class TypedQuery:
    """
    Query targeting a specific context type.

    Attributes:
        query: Query text
        context_type: Target context type (memory/resources/skill)
        intent: Query intent description
        priority: Priority (1-5, 1 is highest)
        target_directories: Directory URIs located by LLM
    """

    query: str
    context_type: Optional[ContextType]
    intent: str
    priority: int = 3
    target_directories: List[str] = field(default_factory=list)
    embedding_input: Optional[Any] = None
    image_query: bool = False


@dataclass
class QueryPlan:
    """
    Query plan containing multiple TypedQueries.

    Attributes:
        queries: List of typed queries
        session_context: Session context summary
        reasoning: LLM reasoning process
    """

    queries: List[TypedQuery]
    session_context: str
    reasoning: str


@dataclass
class MatchedContext:
    """Matched context from retrieval."""

    uri: str
    context_type: ContextType
    level: int = 2
    abstract: str = ""
    overview: Optional[str] = None
    category: str = ""
    score: float = 0.0
    match_reason: str = ""
    search_tags: List[str] = field(default_factory=list)
    # Recall-stage vector score and time factor, retained after model rerank.
    origin_score: Optional[float] = None
    time_score: Optional[float] = None


@dataclass
class QueryResult:
    """
    Result for a single TypedQuery.

    Attributes:
        query: Original query
        matched_contexts: List of matched contexts
        searched_directories: Directories that were searched
        thinking_trace: Structured thinking trace for visualization
    """

    query: TypedQuery
    matched_contexts: List[MatchedContext]
    searched_directories: List[str]
    thinking_trace: ThinkingTrace = field(default_factory=ThinkingTrace)

    def get_trace_messages(self) -> List[str]:
        """Get trace as simple message list."""
        return self.thinking_trace.to_messages()


@dataclass
class FindResult:
    """
    Final result from client.search().

    Attributes:
        memories: Matched memory contexts
        resources: Matched resource contexts
        skills: Matched skill contexts
        query_plan: Query plan used
        query_results: Detailed results for each query
        total: Total match count
    """

    memories: List[MatchedContext]
    resources: List[MatchedContext]
    skills: List[MatchedContext]
    query_plan: Optional[QueryPlan] = None
    query_results: Optional[List[QueryResult]] = None
    total: int = 0

    def __iter__(self):
        """Make FindResult iterable by yielding all matched contexts."""
        yield from self.memories
        yield from self.resources
        yield from self.skills

    def __post_init__(self):
        self.total = len(self.memories) + len(self.resources) + len(self.skills)

    def to_dict(self, include_provenance: bool = False) -> Dict[str, Any]:
        """Convert to dictionary format.

        Args:
            include_provenance: If True, include query_results with thinking
                trace and searched_directories for retrieval observability.
        """
        result: Dict[str, Any] = {
            "memories": [self._context_to_dict(m) for m in self.memories],
            "resources": [self._context_to_dict(r) for r in self.resources],
            "skills": [self._context_to_dict(s) for s in self.skills],
            "total": self.total,
        }

        if self.query_plan:
            result["query_plan"] = {
                "reasoning": self.query_plan.reasoning,
                "queries": [self._query_to_dict(q) for q in self.query_plan.queries],
            }

        if include_provenance and self.query_results:
            result["provenance"] = [self._query_result_to_dict(qr) for qr in self.query_results]

        return result

    def _context_to_dict(self, ctx: MatchedContext) -> Dict[str, Any]:
        """Convert MatchedContext to dict.

        Only fields the retrieval pipeline actually populates are exposed.
        ``search_tags`` is surfaced under the ``tags`` key to match the
        ``tags`` filter parameter accepted by find/search.
        """
        result: Dict[str, Any] = {
            "context_type": ctx.context_type.value,
            "uri": ctx.uri,
            "level": ctx.level,
            "score": ctx.score,
            "abstract": ctx.abstract,
            "tags": normalize_search_tags(ctx.search_tags, discard_invalid=True),
        }
        if ctx.origin_score is not None:
            result["origin_score"] = ctx.origin_score
            result["time_score"] = ctx.time_score
        return result

    def _query_to_dict(self, q: TypedQuery) -> Dict[str, Any]:
        """Convert TypedQuery to dict."""
        return {
            "query": q.query,
            "context_type": q.context_type.value if q.context_type is not None else None,
            "intent": q.intent,
            "priority": q.priority,
        }

    def _query_result_to_dict(self, qr: "QueryResult") -> Dict[str, Any]:
        """Convert QueryResult to dict with provenance data."""
        return {
            "query": qr.query.query,
            "searched_directories": qr.searched_directories,
            "matched_contexts": [
                {
                    "uri": ctx.uri,
                    "tier": f"L{ctx.level}",
                    "context_type": ctx.context_type.value,
                    "score": ctx.score,
                    **(
                        {
                            "origin_score": ctx.origin_score,
                            "time_score": ctx.time_score,
                        }
                        if ctx.origin_score is not None
                        else {}
                    ),
                    "match_reason": ctx.match_reason,
                }
                for ctx in qr.matched_contexts
            ],
            "thinking_trace": qr.thinking_trace.to_dict(),
        }

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "FindResult":
        """Construct FindResult from a dictionary (e.g. HTTP JSON response)."""

        def _parse_context(d: Dict[str, Any]) -> MatchedContext:
            return MatchedContext(
                uri=d.get("uri", ""),
                context_type=ContextType(d.get("context_type", "resource")),
                level=d.get("level", 2),
                abstract=d.get("abstract", ""),
                overview=d.get("overview"),
                category=d.get("category", ""),
                score=d.get("score", 0.0),
                origin_score=d.get("origin_score"),
                time_score=d.get("time_score"),
                match_reason=d.get("match_reason", ""),
                search_tags=list(d.get("tags") or d.get("search_tags") or []),
            )

        return cls(
            memories=[_parse_context(m) for m in data.get("memories", [])],
            resources=[_parse_context(r) for r in data.get("resources", [])],
            skills=[_parse_context(s) for s in data.get("skills", [])],
        )