Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

8.9 KB
Raw Blame
"""Lossless source ranges prepared in memory for bounded compile assignments."""

from __future__ import annotations

import json
import re
from bisect import bisect_right
from dataclasses import asdict, dataclass
from typing import Iterable
from urllib.parse import urlsplit

from openviking.parse.parsers.media.utils import get_media_type
from vikingbot.agent.tools.ov_file import VikingMultiReadTool
from vikingbot.compile.models import CompileFailure, CompileLimits

_HEADING = re.compile(r"^ {0,3}(#{1,6})[ \t]+\S")
_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})")
_FRONTMATTER = re.compile(r"\A---\r?\n.*?\r?\n(?:---|\.\.\.)(?:\r?\n|$)", re.DOTALL)
_TABLE_SEPARATOR = re.compile(r"\s*\|(?:\s*:?-{3,}:?\s*\|)+\s*")


@dataclass(frozen=True)
class CompileSourceRange:
    """An exact excerpt of one original source, with separately repeated reading context.

    ``start_char``/``end_char`` are zero-based, half-open Unicode character offsets;
    ``start_line``/``end_line`` are inclusive, one-based line numbers. ``content``
    preserves the original text, including line endings. ``context`` repeats source
    frontmatter, enclosing headings and a table header when they fit the repeat budget.
    Continuation flags identify cuts inside an oversized line or table row;
    these fragments require their adjacent ranges for complete interpretation.
    Images keep zero offsets and carry their original bytes and hash instead of text.
    """

    uri: str
    start_char: int
    end_char: int
    start_line: int
    end_line: int
    content: str
    context: str = ""
    continues_before: bool = False
    continues_after: bool = False
    image: dict | None = None
    sha256: str | None = None

    def as_assignment(self) -> dict:
        """Attach exact text ranges or bounded references to original images."""
        if self.image:
            return {"uri": self.uri, "sha256": self.sha256, "media_kind": "image"}
        return asdict(self)

    @property
    def input_chars(self) -> int:
        """Count source text and repeated context against the batch character budget."""
        if self.image:
            return len(json.dumps(self.as_assignment(), ensure_ascii=False))
        return len(self.content) + len(self.context)


async def read_source(client, uri, provider) -> str | CompileSourceRange:
    """Keep text reads intact; consume image sources through multi-read's native reader."""
    if get_media_type(urlsplit(uri).path, None) != "image":
        return await client.read_raw(uri)
    if VikingMultiReadTool._media_kind(uri) != "image":
        raise CompileFailure(
            "INVALID_ARGUMENT", f"Unsupported Compile image format: {uri}", stage="pipeline"
        )
    if not provider.supports_tool_result_media():
        raise CompileFailure(
            "UNSUPPORTED_MEDIA",
            f"Compile provider cannot deliver image evidence: {uri}",
            stage="pipeline",
        )
    result = await VikingMultiReadTool._read_image(client, uri)
    if not result["success"]:
        raise CompileFailure("SOURCE_READ_FAILED", f"{uri}: {result['content']}", stage="pipeline")
    return CompileSourceRange(uri, 0, 0, 0, 0, "", image=result["image"], sha256=result["sha256"])


def split_source(
    uri: str, content: str | CompileSourceRange, max_chars: int
) -> list[CompileSourceRange]:
    """Partition original text by characters, preferring headings, paragraphs and whole lines.

    Complete table rows fit together with their header whenever the budget permits.
    An oversized line uses explicitly marked character continuations. Repeated
    frontmatter/headings/table headers use at most one quarter of the budget; their
    original text remains in the source ranges. Empty files retain one empty range.
    """
    if isinstance(content, CompileSourceRange):
        return [content]
    if get_media_type(urlsplit(uri).path, None) == "image":
        raise CompileFailure(
            "INVALID_ARGUMENT", "Image evidence must not be split as text", stage="pipeline"
        )
    if max_chars < 1:
        raise ValueError("Source character budget must be positive")
    lines = content.splitlines(keepends=True)
    offsets = [0]
    contexts = []
    preferred = [0]
    headings = []
    table_header = ""
    match = _FRONTMATTER.match(content)
    frontmatter = match[0] if match else ""
    fence = ""
    for i, line in enumerate(lines):
        start = offsets[-1]
        offsets.append(start + len(line))
        marker = _FENCE.match(line)
        if marker:
            if not fence:
                fence = marker[1]
            elif marker[1][0] == fence[0] and len(marker[1]) >= len(fence):
                fence = ""
        if not fence and start >= len(frontmatter):
            heading = _HEADING.match(line)
            if heading:
                depth = len(heading[1])
                headings = [(level, text) for level, text in headings if level < depth]
                headings.append((depth, line.rstrip()))
                preferred.append(start)
            if _TABLE_SEPARATOR.fullmatch(line) and i:
                table_header = lines[i - 1] + line
            elif not line.lstrip().startswith("|"):
                table_header = ""
            if not line.strip():
                preferred.append(offsets[-1])
        contexts.append(
            frontmatter
            + "\n".join(text for _, text in headings)
            + ("\n" + table_header if table_header else "")
        )

    def context_at(offset: int) -> str:
        """Return enclosing source context when it fits the repeat budget."""
        if not offset or not contexts:
            return ""
        value = contexts[min(bisect_right(offsets, offset) - 1, len(contexts) - 1)]
        return value if len(value) <= max_chars // 4 else ""

    ranges = []
    start = 0
    while start < len(content) or not ranges:
        context = context_at(start)
        target = min(len(content), start + max_chars - len(context))
        end = target
        if target < len(content):
            boundary = preferred[bisect_right(preferred, target) - 1]
            line_index = bisect_right(offsets, target) - 1
            line_end = offsets[line_index]
            if boundary > start and boundary >= (start + target) // 2:
                end = boundary
            elif line_end > start:
                next_line_size = offsets[line_index + 1] - line_end
                if line_end >= (start + target) // 2 or next_line_size <= max_chars - len(
                    context_at(line_end)
                ):
                    end = line_end
        start_line = bisect_right(offsets, start)
        end_line = bisect_right(offsets, max(start, end - 1))
        ranges.append(
            CompileSourceRange(
                uri=uri,
                start_char=start,
                end_char=end,
                start_line=start_line,
                end_line=end_line,
                content=content[start:end],
                context=context,
                continues_before=start != offsets[start_line - 1],
                continues_after=end < len(content)
                and end != offsets[bisect_right(offsets, end) - 1],
            )
        )
        start = end
    return ranges


def pack_source_batches(
    sources: Iterable[tuple[str, str | CompileSourceRange]], limits: CompileLimits
) -> list[list[CompileSourceRange]]:
    """Pack exact source ranges and image references within existing text/media budgets.

    Adjacent ranges may share a batch when they fit. Small files share remaining
    capacity; original URIs remain the citation targets for all ranges. Source text
    and repeated context both count toward ``source_batch_chars``.
    """
    batches: list[list[CompileSourceRange]] = []
    batch: list[CompileSourceRange] = []
    size = 0
    image_count = media_bytes = 0
    for uri, content in sources:
        for part in split_source(uri, content, limits.source_batch_chars):
            chars, part_bytes = part.input_chars, 0
            if part.image:
                from vikingbot.agent.loop import _inline_media_bytes

                part_bytes = _inline_media_bytes([{"content": [part.image]}])
                if (
                    chars > limits.source_batch_chars
                    or part_bytes > VikingMultiReadTool._MAX_INLINE_MEDIA_BYTES
                ):
                    raise CompileFailure(
                        "MEDIA_LIMIT", "Image exceeds the source batch budget", stage="pipeline"
                    )
            if batch and (
                size + chars > limits.source_batch_chars
                or image_count + bool(part.image) > VikingMultiReadTool._MAX_INLINE_IMAGES
                or media_bytes + part_bytes > VikingMultiReadTool._MAX_INLINE_MEDIA_BYTES
            ):
                batches.append(batch)
                batch, size = [], 0
                image_count = media_bytes = 0
            batch.append(part)
            size += chars
            image_count += bool(part.image)
            media_bytes += part_bytes
    if batch:
        batches.append(batch)
    return batches