Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

14.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Rewrites Deep Research citations to gathered sources; invented or unresolved ones are stripped."""

from __future__ import annotations

import re

from markdown_it import MarkdownIt
from markdown_it.rules_inline.backticks import backtick

from core.research.redaction import _escape_link_destination


# unrolling avoids catastrophic backtracking on unclosed [Document: that would block the event loop.
_DOCUMENT_CITATION = re.compile(r"\[(?:Document|MCP):[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]")
_MARKDOWN_LINK_START = re.compile(r"\[([^\]\n]+)\]\((https?://)", re.IGNORECASE)
_SOURCES_HEADING = re.compile(
    r"^(?:#{1,6}\s+|\*\*)?"
    r"(?:Sources?|References?|Bibliography|Works\s+Cited|Source\s+List)"
    r"(?:\*\*)?\s*$",
    re.IGNORECASE | re.MULTILINE,
)
_NUMBERED_CITATION = re.compile(r"(?<!\^)\[(\d+)]")
_AUTOLINK = re.compile(r"<(https?://[^>\s]+)>", re.IGNORECASE)
# \x00 stops a URL glued to masked code from swallowing its placeholder.
# Any case: a browser reads `HTTPS://` as https too.
_RAW_URL = re.compile(r"https?://[^\s<>\x00]+", re.IGNORECASE)
# A kind missing from the restore pattern leaks a raw sentinel; [0-9] avoids other digits.
_PLACEHOLDER_KINDS = ("research-code", "research-citation")
_PLACEHOLDER = re.compile(rf"\x00(?:{'|'.join(_PLACEHOLDER_KINDS)})-[0-9]+\x00")
# For a pass that deletes prose but must carry the code through.
_CODE_PLACEHOLDER = re.compile(r"\x00research-code-[0-9]+\x00")
# Footnote lines render as prose in remark-gfm but as code in CommonMark: validate them, do not mask.
_FOOTNOTE_DEFINITION = re.compile(r" {0,3}\[\^[^\]\s]+\]:")
# Mermaid fences execute (image URLs fetched by the browser), so their URLs stay validated.
_MERMAID_FENCE = re.compile(r"mermaid\b", re.IGNORECASE)


def _citation_title(source: dict, fallback: str) -> str:
    """Strips brackets from titles, since a [PDF] prefix would make the citation unmatchable."""
    title = str(source.get("title") or fallback).replace("[", "").replace("]", "").strip()
    return title or fallback


def _trim_url_tail(raw: str) -> str:
    """Trims trailing prose punctuation and unmatched ')' like GFM autolinks, in one interleaved pass."""
    end = len(raw)
    opening, closing = raw.count("("), raw.count(")")
    while end:
        char = raw[end - 1]
        if char == ")":
            if closing <= opening:
                break
            closing -= 1
        elif char not in ".,;:!?":
            break
        end -= 1
    return raw[:end]


def _placeholder(kind: str, index: int) -> str:
    """NUL-delimited: _mask_code strips any NULs the model wrote, so a copy cannot pass as a token."""
    return f"\x00{kind}-{index}\x00"


def _record_code_span(state, silent: bool) -> bool:
    start = state.pos
    count = len(state.tokens)
    matched = backtick(state, silent)
    if (
        matched
        and not silent
        # Block's own source only: the image rule re-enters with alt text.
        and state.src is state.env.get("code_source")
        and len(state.tokens) > count
        and state.tokens[-1].type == "code_inline"
    ):
        state.env["code_spans"].append((start, state.pos))
    return matched


# Tables enabled to match the renderer, else backticks pair across cells.
_CODE_MARKDOWN = MarkdownIt("commonmark").enable("table").disable("inline")
_CODE_MARKDOWN.inline.ruler.at("backticks", _record_code_span)


def _footnote_content_lines(lines: list[str]) -> set[int]:
    """Footnote content lines per the renderer; loose on purpose, as over-collecting only masks a block."""
    covered = set()
    inside = False
    for number, line in enumerate(lines):
        if _FOOTNOTE_DEFINITION.match(line):
            inside = True
        elif not inside:
            continue
        elif not line.strip():
            covered.add(number)
        elif line.startswith(("    ", "\t")):
            covered.add(number)
        else:
            inside = False
    return covered


def _mask_code(text: str, placeholders: dict[str, str]) -> str:
    # Do this first: CommonMark maps NUL to U+FFFD, so a report cannot spell a token; offsets unchanged.
    text = text.replace("\x00", "\ufffd")
    offsets = [0, *(match.end() for match in re.finditer(r"\r\n?|\n", text))]
    if offsets[-1] != len(text):
        offsets.append(len(text))
    lines = [text[offsets[number] : offsets[number + 1]] for number in range(len(offsets) - 1)]
    footnote_content = _footnote_content_lines(lines)
    spans = []

    def record_inline(start: int, end: int) -> None:
        source = text[start:end]
        env = {"code_source": source, "code_spans": []}
        _CODE_MARKDOWN.inline.parse(source, _CODE_MARKDOWN, env, [])
        spans.extend((start + first, start + last) for first, last in env["code_spans"])

    def record_row(start: int, end: int) -> None:
        """Cuts at unescaped pipes, as the renderer splits rows, so a code span stays inside one cell."""
        cell = start
        escaped = False
        for index in range(start, end):
            if escaped:
                escaped = False
            elif text[index] == "\\":
                escaped = True
            elif text[index] == "|":
                record_inline(cell, index)
                cell = index + 1
        record_inline(cell, end)

    in_table = False
    for token in _CODE_MARKDOWN.parse(text):
        if token.type == "table_open":
            in_table = True
        elif token.type == "table_close":
            in_table = False
        if token.map is None:
            continue
        # Mask only real code: a footnote's indented block is prose; a mermaid fence is an executed diagram.
        if token.type == "code_block" and token.map[0] in footnote_content:
            continue
        if token.type == "fence" and _MERMAID_FENCE.match((token.info or "").strip()):
            continue
        start, end = (offsets[line] for line in token.map)
        if token.type in {"fence", "code_block"}:
            # Keep the following heading on its own line while the block is masked.
            end = start + len(text[start:end].rstrip("\r\n"))
            spans.append((start, end))
        elif token.type == "tr_open":
            record_row(start, end)
        elif token.type == "inline" and not in_table:
            record_inline(start, end)
    pieces = []
    cursor = 0
    for start, end in sorted(spans):
        pieces.append(text[cursor:start])
        token = _placeholder("research-code", len(placeholders))
        placeholders[token] = text[start:end]
        pieces.append(token)
        cursor = end
    pieces.append(text[cursor:])
    return "".join(pieces)


def _restore_placeholders(text: str, placeholders: dict[str, str]) -> str:
    """Restores every token in one re.sub pass; a replace per token rescans the report once per span."""
    if not placeholders:
        return text

    def restore(match: re.Match) -> str:
        token = match.group(0)
        value = placeholders.get(token)
        if value is not None:
            return value
        # Unreachable while _mask_code normalizes NULs; if skipped, drop delimiters rather than emit a NUL.
        return token.replace("\x00", "\ufffd")

    return _PLACEHOLDER.sub(restore, text)


def _validate_masked_sources(report: str, sources: list[dict], placeholders: dict[str, str]) -> str:
    source_by_url = {
        str(source.get("url") or ""): source for source in sources if source.get("url")
    }
    source_urls = list(source_by_url)

    heading = _SOURCES_HEADING.search(report)
    if heading:
        report = report[: heading.start()]

    def catalog_url(url: str) -> str | None:
        # The scheme is case-insensitive; the catalog spells it in lower case.
        lowered = re.sub(r"^https?://", lambda m: m.group(0).lower(), url, flags = re.IGNORECASE)
        return url if url in source_by_url else lowered if lowered in source_by_url else None

    def citation(url: str) -> str | None:
        url = catalog_url(url)
        if url is None:
            return None
        source = source_by_url[url]
        title = re.sub(r"(\\*)\|", r"\1\1\\|", _citation_title(source, url))
        token = _placeholder("research-citation", len(placeholders))
        placeholders[token] = f"[{title}]({_escape_link_destination(url)})"
        return token

    def replace_markdown_links(text: str) -> str:
        pieces = []
        cursor = 0
        while match := _MARKDOWN_LINK_START.search(text, cursor):
            destination_start = match.start(2)
            index = match.end(2)
            depth = 0
            escaped = False
            close = None
            destination_end = None
            while index < len(text):
                character = text[index]
                if escaped:
                    escaped = False
                elif character == "\\":
                    escaped = True
                elif character.isspace():
                    if depth != 0:
                        break
                    destination_end = index
                    title_start = index
                    while title_start < len(text) and text[title_start].isspace():
                        title_start += 1
                    if title_start < len(text) and text[title_start] in {'"', "'"}:
                        quote = text[title_start]
                        title_end = title_start + 1
                        title_escaped = False
                        while title_end < len(text):
                            if title_escaped:
                                title_escaped = False
                            elif text[title_end] == "\\":
                                title_escaped = True
                            elif text[title_end] == quote:
                                break
                            title_end += 1
                        if title_end >= len(text):
                            break
                        title_start = title_end + 1
                        while title_start < len(text) and text[title_start].isspace():
                            title_start += 1
                    if title_start < len(text) and text[title_start] == ")":
                        close = title_start
                    break
                elif character == "(":
                    depth += 1
                elif character == ")":
                    if depth == 0:
                        close = index
                        destination_end = index
                        break
                    depth -= 1
                index += 1
            if close is None:
                pieces.append(text[cursor : match.start()])
                pieces.append(match.group(1).strip())
                cursor = index
                continue
            url = text[destination_start:destination_end].replace(r"\(", "(").replace(r"\)", ")")
            pieces.append(text[cursor : match.start()])
            pieces.append(citation(url) or match.group(1).strip())
            cursor = close + 1
        pieces.append(text[cursor:])
        return "".join(pieces)

    def replace_number(match: re.Match) -> str:
        index = int(match.group(1)) - 1
        if 0 <= index < len(source_urls):
            return citation(source_urls[index]) or match.group(0)
        return match.group(0)

    def replace_autolink(match: re.Match) -> str:
        return citation(match.group(1)) or match.group(1)

    def replace_raw_url(match: re.Match) -> str:
        # Cite whole source URLs; drop other raw URLs. Whole-match avoids prefix collisions.
        raw = match.group(0)
        core = _trim_url_tail(raw)
        if catalog_url(core):
            return (citation(core) or core) + raw[len(core) :]
        # Keep the trimmed tail so dropping the URL cannot unbalance the prose.
        return raw[len(core) :]

    validated = replace_markdown_links(report)
    validated = _AUTOLINK.sub(replace_autolink, validated)
    validated = _NUMBERED_CITATION.sub(replace_number, validated)
    validated = _RAW_URL.sub(replace_raw_url, validated)
    return validated.strip()


def _validate_report_sources(report: str, sources: list[dict]) -> str:
    """canonicalize citations and remove model-authored source lists."""
    placeholders: dict[str, str] = {}
    validated = _validate_masked_sources(_mask_code(report, placeholders), sources, placeholders)
    return _restore_placeholders(validated, placeholders)


def _document_source_citation(source: dict) -> str:
    filename = str(source.get("filename") or "Document")
    if source.get("kind") == "mcp":
        return f"[MCP: {_mcp_source_label(source)}]"
    if source.get("page") is not None:
        return f"[Document: {filename}, p. {source['page']}]"
    return f"[Document: {filename}]"


def _mcp_source_label(source: dict) -> str:
    filename = str(source.get("filename") or "MCP source")
    label = " ".join(filename.replace("[", "").replace("]", "").split())
    return label or "MCP source"


def _allowed_document_citations(sources: list[dict]) -> set[str]:
    allowed = set()
    for source in sources:
        if source.get("kind") != "mcp":
            allowed.add(f"[Document: {source.get('filename') or 'Document'}]")
        allowed.add(_document_source_citation(source))
    return allowed


def _validate_masked_document_sources(
    report: str, sources: list[dict], placeholders: dict[str, str]
) -> str:
    allowed = _allowed_document_citations(sources)

    def keep_if_allowed(match: re.Match) -> str:
        # Judge it as written: the pattern spans ']' in filenames, and backticks were already masked.
        citation = _restore_placeholders(match.group(0), placeholders)
        if citation in allowed:
            return match.group(0)
        # An unsupported citation may span code; dropping it must keep the code it covers.
        return "".join(_CODE_PLACEHOLDER.findall(match.group(0)))

    return _DOCUMENT_CITATION.sub(keep_if_allowed, report)


def _validate_report_document_sources(report: str, sources: list[dict]) -> str:
    placeholders: dict[str, str] = {}
    validated = _validate_masked_document_sources(
        _mask_code(report, placeholders), sources, placeholders
    )
    return _restore_placeholders(validated, placeholders)


def _validate_report(report: str, sources: list[dict], document_sources: list[dict]) -> str:
    # Mask once: removing a URL-only line may split a paragraph and turn an indented citation into code.
    placeholders: dict[str, str] = {}
    validated = _validate_masked_sources(_mask_code(report, placeholders), sources, placeholders)
    validated = _validate_masked_document_sources(validated, document_sources, placeholders)
    return _restore_placeholders(validated, placeholders)