Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
feat(citations): render Codex source citations as links and badges
Call White029cf5f8b696
diff --git a/core/citations.py b/core/citations.pynew file mode 100644index 000000000..b76eae7fa--- /dev/null+++ b/core/citations.py@@ -0,0 +1,1352 @@+"""Source citations shared by agent backends.++Some model backends answer with opaque inline citation markers instead of+links. Codex (the OpenAI app-server) emits++ U+E200 "cite" U+E202 <ref_id> [U+E202 <ref_id> ...] U+E201++where each ``ref_id`` names one result of a web search performed earlier in the+same native thread. The results themselves arrive in separate notifications, so+a backend harvests them into a map scoped to that thread and hands the map here+when a message is about to be emitted.++This module owns the shared half of the feature: the marker grammar, the+sanitizing of untrusted titles and URLs, and the rewrite into ordinary Markdown+links plus a structured sidecar. It is deliberately transport-agnostic - the+Markdown text is what every IM platform delivers, and the sidecar is what the+Web renderer upgrades into compact citation badges.++Two rules shape every branch below: a citation we cannot resolve is *downgraded*+to a visible label, never silently dropped, and a URL is never invented from a+ref_id or from search order.+"""++from __future__ import annotations++import hashlib+import re+import secrets+from dataclasses import dataclass+from html.entities import html5 as HTML5_ENTITIES+from typing import Any, Callable, Iterable, Mapping, Optional+from urllib.parse import unquote++import idna++from core.reply_enhancer import (+ MarkdownUnit,+ markdown_link_units,+ mask_citation_slots,+ mask_hidden_and_code,+ percent_encode,+ spell_uri,+)++# The private-use delimiters the marker is wrapped in.+_START = "\ue200"+_SEP = "\ue202"+_END = "\ue201"++# The private-use delimiters an internal registration token is wrapped in.+# Deliberately not the marker's own: a token is this module's private handle on+# one registered marker, and nothing downstream may read it as citation grammar.+_TOKEN_OPEN = "\ue010"+_TOKEN_CLOSE = "\ue011"+_TOKEN_NONCE_BYTES = 16+_TOKEN_RE = re.compile(+ f"{_TOKEN_OPEN}[0-9a-f]{{{_TOKEN_NONCE_BYTES * 2}}}{_TOKEN_CLOSE}"+)++# One complete marker. The payload may not contain a start/end delimiter, so an+# unterminated marker never matches: a stream that split a marker across chunks+# keeps its raw text instead of swallowing the answer around it. Requiring the+# literal ``cite`` keyword also keeps sibling U+E200 markers (``visualize``)+# untouched.+CITATION_MARKER_RE = re.compile(f"{_START}cite{_SEP}([^{_START}{_END}]*){_END}")++# A marker and the token it is registered as are both this module's own text:+# one is replaced by the citation it asked for, the other by the marker it+# stands for, and neither reaches a reader. So neither is part of a reference+# identifier the Markdown parser matches against a definition, and the reply+# parser is told so wherever it is asked where a citation may be written.+_SLOT_TRANSPARENT_RE = re.compile(+ f"{CITATION_MARKER_RE.pattern}|{_TOKEN_RE.pattern}"+)++# Real ref_ids look like ``turn0view1``. Keep the shape permissive but bounded so+# a malformed payload falls through to the unresolved fallback.+_REF_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,64}")++_ALLOWED_SCHEMES = frozenset({"http", "https"})+# C0/C1 controls, line/paragraph separators, and the whole private-use area+# (which is where the markers themselves live), plus the invisible formatting+# characters a single-line source title has no legitimate use for: the bidi+# marks, embeddings, overrides and isolates that let a title render its own+# text backwards, and the zero-width space, word joiner and byte order mark.+#+# ZWNJ, ZWJ and the variation selectors are deliberately left in: they carry+# meaning inside a real title, joining an emoji sequence or spelling an Indic+# or Persian word, and none of them can reorder what the reader sees.+_CONTROL_RE = re.compile(+ "[\x00-\x1f\x7f-\x9f\u061c\u200b\u200e\u200f\u202a-\u202e"+ "\u2028\u2029\u2060\u2066-\u2069\ufeff\ue000-\uf8ff]"+)+# What WHATWG's URL parser deletes from its input before parsing anything: a tab+# or newline anywhere, and C0 controls or spaces at either end.+_URL_REMOVED_RE = re.compile(r"[\t\n\r]")+_URL_TRIMMED = "".join(chr(code) for code in range(0x21))+_DECIMAL_DIGITS = frozenset("0123456789")+_OCTAL_DIGITS = frozenset("01234567")+_HEX_DIGITS = frozenset("0123456789abcdefABCDEF")+# The widest an IPv4 number can be written per radix and still fit in 32 bits+# (``4294967295``, ``0xffffffff``, ``0o37777777777``), and the stand-in for one+# written wider. Every check that consumes a number rejects a value this big, so+# the exact magnitude past the cap never matters - and not converting it keeps+# CPython's bound on decimal ``int`` conversion out of an untrusted host.+_IPV4_WIDEST = {10: 10, 16: 8, 8: 11}+_IPV4_TOO_LARGE = 1 << 32++# A CommonMark character reference, which is what a link destination resolves+# before it becomes an href. Providers that lifted a URL out of HTML hand over+# ``&`` for ``&``, so one round of this is applied on the way in - and the+# same grammar decides, on the way out, which ``&`` has to be escaped so the+# renderer cannot resolve it a second time.+_REFERENCE_RE = re.compile(+ r"&(#[0-9]{1,7}|#[Xx][0-9A-Fa-f]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});"+)+# What a citation destination has to spell that plain URI spelling does not:+# an ``&`` that starts a live reference would be resolved a second time on the+# way out, turning one delivered URL into a different one. It is written as an+# escape BEFORE ``spell_uri`` runs, which keeps it untouched - so the result is+# still a fixed point, and the reference grammar stays defined once. A+# parenthesis is not here: it is part of the address, and escaping it in the URL+# names a different page. Only the link needs it guarded - see+# ``_link_spelling``.+_CITATION_UNSAFE_RE = re.compile(rf"&(?={_REFERENCE_RE.pattern[1:]})")+# The two characters of an address that can end a Markdown destination early.+_DESTINATION_PAREN_RE = re.compile(r"[()]")+# WHATWG forbidden domain code points, checked after percent-decoding. The C0+# range is there in full: a host is not allowed to hold any of it, and a+# citation whose host carries one names a page no browser opens.+_FORBIDDEN_DOMAIN = frozenset(+ "".join(chr(code) for code in range(0x20)) + " #%/:<>?@[\\]^|\x7f"+)+# The ports a browser drops from the URL it shows, because they are the ones it+# would have used anyway.+_DEFAULT_PORTS = {"http": 80, "https": 443}+_PORT_MAX = 65535+_PORT_DIGITS = len(str(_PORT_MAX))+# An IPv6 host is wrapped in brackets, which ``_canonical_uri`` percent-encodes+# along with every other character Markdown would read as syntax - so after+# spelling, a host the provider wrote ``[::1]`` and a host it wrote ``%5B::1%5D``+# are the same string. These patterns therefore recognize the wrapper in either+# spelling, and say nothing about which one the destination was written in:+# ``_authority_brackets_literal`` answers that, from the destination as Markdown+# resolved it, BEFORE spelling erased the difference.+_IPV6_OPEN_RE = re.compile(r"\[|%5[Bb]")+_IPV6_CLOSE_RE = re.compile(r"\]|%5[Dd]")+# Where an authority ends. A backslash ends one too for a special scheme, but+# ``safe_url`` has already refused every address still holding one.+_AUTHORITY_END_RE = re.compile(r"[/?#]")+_TITLE_MAX = 200+_LABEL_MAX = 64+++@dataclass(frozen=True)+class CitationSource:+ """One citable search result, as the backend received it (untrusted)."""++ ref_id: str+ title: str = ""+ url: str = ""+++@dataclass(frozen=True)+class Citation:+ """A resolved citation, numbered in first-appearance order within a message."""++ index: int+ ref_id: str+ title: str+ url: str+ label: str++ def to_payload(+ self, *, spans: list[list[int]], body_sha256: str+ ) -> dict[str, Any]:+ """The persisted sidecar shape (``message.content.citations`` entries).++ ``spans`` are half-open ``[start, end)`` ranges in UTF-16 code units,+ and ``body_sha256`` is the digest of the exact body they were measured+ in. Neither means anything without the other, so neither is optional.+ """+ return {+ "index": self.index,+ "ref_id": self.ref_id,+ "title": self.title,+ "url": self.url,+ "label": self.label,+ "spans": [list(span) for span in spans],+ "body_sha256": body_sha256,+ }+++@dataclass(frozen=True)+class WrittenLink:+ """One link a registered marker writes: a citation, or the fallback label.++ ``plain`` is the same attribution outside Markdown. Quick-reply labels and+ file labels are lifted out of the body into structured fields that no+ surface parses as Markdown, so writing a link there would show the reader+ its syntax; the domain on its own still says who is being cited.+ """++ index: Optional[int]+ text: str+ plain: str+++@dataclass(frozen=True)+class RegisteredMarker:+ """One marker registered at the native boundary, under an opaque token."""++ token: str+ # The marker exactly as it arrived, restored when the token has ended up+ # somewhere a link must not be written.+ literal: str+ lead: str+ parts: tuple[WrittenLink, ...]+++@dataclass(frozen=True)+class CitationBundle:+ """Everything one message's delivery needs to write its citations.++ Immutable all the way down, and the ``sources`` are a snapshot: the map a+ backend hands to registration is live thread state that keeps being written+ to, and a bundle that read it later would describe a different message.+ """++ citations: tuple[Citation, ...]+ markers: tuple[RegisteredMarker, ...]+ sources: tuple[CitationSource, ...]+++def clean_title(value: Any) -> str:+ """Collapse an untrusted source title to one safe single-line string."""+ if not isinstance(value, str):+ return ""+ return " ".join(_CONTROL_RE.sub(" ", value).split())[:_TITLE_MAX].strip()+++def _is_replaced_code_point(code: int) -> bool:+ """Whether the renderer substitutes U+FFFD for the numeric reference *code*.++ This is micromark's table (``micromark-util-decode-numeric-character-+ reference``), because micromark is the parser behind the Web renderer and+ the only table that can keep a persisted URL equal to the href drawn from+ it. It is wider than CommonMark's own wording, which replaces just NUL,+ surrogates and out-of-range values: micromark also replaces the C0 controls+ a URL may not hold, the whole C1 range, and the noncharacters. Notably it+ does *not* apply HTML's Windows-1252 mapping, so ``€`` becomes U+FFFD+ rather than ``€`` - resolving it to ``€`` would name a page the rendered+ link never opens.+ """+ return (+ code < 9+ or code == 11+ or 13 < code < 32+ or 126 < code < 160+ or 55295 < code < 57344+ or 64975 < code < 65008+ or code & 0xFFFF in (0xFFFE, 0xFFFF)+ or code > 0x10FFFF+ )+++def _resolve_references(value: str) -> str:+ """Resolve one round of CommonMark character references."""+ if "&" not in value:+ return value++ def replace(match: re.Match[str]) -> str:+ body = match.group(1)+ if body[0] == "#":+ try:+ code = int(body[2:], 16) if body[1] in "xX" else int(body[1:])+ except ValueError:+ return match.group(0)+ return "\ufffd" if _is_replaced_code_point(code) else chr(code)+ return HTML5_ENTITIES.get(f"{body};", match.group(0))++ return _REFERENCE_RE.sub(replace, value)+++def _normalize_destination(value: str) -> str:+ """The provider's URL string as a browser reads it, before Markdown escaping.++ The two passes are ordered the way the pipeline applies them, because they+ disagree about the same character. A *literal* tab or newline is cleaned off+ first, the way WHATWG's URL parser cleans its own input - removed wherever it+ sits, with leading and trailing C0 controls and spaces trimmed - and it has+ to be: a literal tab in a Markdown destination ends the destination, so+ ``[x](https://example.com/p<TAB>q)`` renders as no link at all.++ A tab a *character reference* spells is a different character. Providers lift+ URLs out of HTML, so one round of references is resolved after that cleanup,+ and what it produces is part of the destination: micromark percent-encodes+ it, so ``p	q``, ``p	q`` and ``p
q`` have the hrefs ``p%09q``,+ ``p%09q`` and ``p%0Aq``. Resolving before the cleanup deleted exactly those,+ pointing the citation at ``p	q`` rendered as ``pq`` - a different page,+ silently.++ Everything else a reference produces survives to be percent-encoded too,+ because that is what the browser does with it. Deleting it instead - which a+ blanket control scrub either side of the reference pass used to do - moved+ the page the same way: ``https://example.com/p€q`` became+ ``https://example.com/pq`` rather than the ``p%EF%BF%BDq`` the renderer+ produces.+ """+ return _resolve_references(_URL_REMOVED_RE.sub("", value).strip(_URL_TRIMMED))+++def _ipv4_number(part: str) -> Optional[int]:+ """One dotted part as WHATWG's IPv4 number parser reads it, or ``None``.++ A leading ``0x`` makes the part hexadecimal and a bare leading ``0`` makes+ it octal, which is how ``0x7f.1`` and ``017700000001`` both reach+ ``127.0.0.1``.++ A part written wider than 32 bits is still a number - a browser reads it and+ then fails the address - so it answers with a value past the cap rather than+ with ``None``, which would call it a domain name instead. That also keeps+ ``int`` off a host spelled with thousands of digits, which CPython refuses to+ convert at all (``sys.set_int_max_str_digits``): an unhandled ``ValueError``+ there would have escaped ``safe_url`` on nothing but an untrusted string.+ """+ if not part:+ return None+ digits, radix = _DECIMAL_DIGITS, 10+ if len(part) > 1 and part[0] == "0":+ if part[1] in "xX":+ part, radix, digits = part[2:], 16, _HEX_DIGITS+ if not part:+ return 0+ else:+ part, radix, digits = part[1:], 8, _OCTAL_DIGITS+ if not all(char in digits for char in part):+ return None+ if len(part.lstrip("0")) > _IPV4_WIDEST[radix]:+ return _IPV4_TOO_LARGE+ return int(part, radix)+++def _canonical_host(host: str) -> str:+ """*host* as a browser serializes it, or ``""`` when it names no host.++ WHATWG hands any host whose last label is a number to its IPv4 parser, so+ ``2130706433``, ``0x7f.1``, ``127.1`` and ``017700000001`` all name+ ``127.0.0.1``. An untrusted search result that spells a loopback or+ private-network address that way would otherwise be attributed to the digits+ themselves, hiding where the link goes; and a form the parser refuses+ (``256.1.1.1``, ``1.2.3.4.5``, ``example.com.0x1``) names no host at all.+ """+ parts = host.split(".")+ if parts[-1] == "" and len(parts) > 1:+ # A trailing dot is not a label. A browser keeps it on a domain+ # (``example.com.``) but ignores it when reading a number.+ parts = parts[:-1]+ last = parts[-1]+ ends_in_number = bool(last) and all(char in _DECIMAL_DIGITS for char in last)+ if not ends_in_number and _ipv4_number(last) is None:+ return host+ if len(parts) > 4:+ return ""+ numbers: list[int] = []+ for part in parts:+ number = _ipv4_number(part)+ if number is None:+ return ""+ numbers.append(number)+ if any(number > 255 for number in numbers[:-1]):+ return ""+ if numbers[-1] >= 256 ** (5 - len(numbers)):+ return ""+ address = numbers[-1]+ for offset, number in enumerate(numbers[:-1]):+ address += number * 256 ** (3 - offset)+ return ".".join(str((address >> shift) & 0xFF) for shift in (24, 16, 8, 0))+++def _canonical_uri(value: str) -> str:+ """The destination form that survives the Markdown pipeline unchanged.++ ``remark`` resolves a destination's references and escapes, then percent-+ encodes what is left. Both are applied here so the URL the backend persists+ is byte-identical to the href the renderer produces from it - which is the+ only reason a badge can be matched to its own link.++ The percent-encoding is ``spell_uri``, the one rule the renderer applies+ and every consumer of a resolved destination needs. Only what a citation+ needs on top of it is written here, and it is written as escapes that the+ shared rule then carries through unchanged.+ """+ return spell_uri(+ _CITATION_UNSAFE_RE.sub(lambda match: percent_encode(match.group()), value)+ )+++def _parse_ipv6(value: str) -> Optional[list[int]]:+ """WHATWG's IPv6 parser: eight 16-bit pieces, or ``None`` for no address.++ Python's own ``ipaddress`` cannot stand in here. It accepts a zone id+ (``fe80::1%eth0``) that a browser refuses, and it serializes an+ IPv4-mapped address back as ``::ffff:127.0.0.1`` where a browser shows+ ``::ffff:7f00:1`` - so a label drawn from it would name the address bar+ shows something the reader never sees.+ """+ address = [0] * 8+ piece_index = 0+ compress: Optional[int] = None+ pointer = 0+ length = len(value)++ def char(offset: int = 0) -> str:+ position = pointer + offset+ return value[position] if position < length else ""++ if char() == ":":+ if char(1) != ":":+ return None+ pointer += 2+ piece_index += 1+ compress = piece_index+ while pointer < length:+ if piece_index == 8:+ return None+ if char() == ":":+ if compress is not None:+ return None+ pointer += 1+ piece_index += 1+ compress = piece_index+ continue+ piece_value = 0+ piece_length = 0+ while piece_length < 4 and char() in _HEX_DIGITS:+ piece_value = piece_value * 0x10 + int(char(), 16)+ pointer += 1+ piece_length += 1+ if char() == ".":+ # A dotted tail, as in ``::ffff:127.0.0.1``. It is read as an IPv4+ # address and stored in the last two pieces, which is why the+ # browser serializes it back as hexadecimal.+ if piece_length == 0 or piece_index > 6:+ return None+ pointer -= piece_length+ numbers_seen = 0+ while pointer < length:+ ipv4_piece: Optional[int] = None+ if numbers_seen > 0:+ if char() == "." and numbers_seen < 4:+ pointer += 1+ else:+ return None+ if char() not in _DECIMAL_DIGITS:+ return None+ while char() in _DECIMAL_DIGITS:+ number = int(char())+ if ipv4_piece is None:+ ipv4_piece = number+ elif ipv4_piece == 0:+ return None+ else:+ ipv4_piece = ipv4_piece * 10 + number+ if ipv4_piece > 255:+ return None+ pointer += 1+ address[piece_index] = address[piece_index] * 0x100 + (ipv4_piece or 0)+ numbers_seen += 1+ if numbers_seen in (2, 4):+ piece_index += 1+ if numbers_seen != 4:+ return None+ break+ if char() == ":":+ pointer += 1+ if pointer >= length:+ return None+ elif char():+ return None+ address[piece_index] = piece_value+ piece_index += 1+ if compress is not None:+ swaps = piece_index - compress+ piece_index = 7+ while piece_index != 0 and swaps > 0:+ address[piece_index], address[compress + swaps - 1] = (+ address[compress + swaps - 1],+ address[piece_index],+ )+ piece_index -= 1+ swaps -= 1+ elif piece_index != 8:+ return None+ return address+++def _serialize_ipv6(address: list[int]) -> str:+ """The address as a browser writes it: lowercase, longest zero run elided."""+ compress: Optional[int] = None+ best_length = 1+ run_start: Optional[int] = None+ run_length = 0+ for piece_index in range(8):+ if address[piece_index] != 0:+ run_start = None+ run_length = 0+ continue+ if run_start is None:+ run_start = piece_index+ run_length += 1+ # Strictly greater keeps the leftmost of two equally long runs.+ if run_length > best_length:+ best_length = run_length+ compress = run_start++ out: list[str] = []+ ignore_zero = False+ for piece_index in range(8):+ if ignore_zero and address[piece_index] == 0:+ continue+ ignore_zero = False+ if compress == piece_index:+ out.append("::" if piece_index == 0 else ":")+ ignore_zero = True+ continue+ out.append(format(address[piece_index], "x"))+ if piece_index != 7:+ out.append(":")+ return "".join(out)+++_ACE_PREFIX = "xn--"+++def _ace_label(label: str) -> str:+ """*label*, an ``xn--`` spelling, when it round-trips; ``""`` otherwise."""+ try:+ decoded = label[len(_ACE_PREFIX) :].encode("ascii").decode("punycode")+ except (UnicodeError, ValueError):+ return ""+ if not decoded or decoded.isascii():+ return ""+ try:+ encoded = idna.encode(+ decoded, uts46=True, transitional=False, std3_rules=False+ ).decode("ascii")+ except (UnicodeError, ValueError):+ return ""+ return encoded if encoded == label else ""+++def _canonical_domain(host: str) -> str:+ """A non-IP host as a browser serializes it, or ``""`` when it names none.++ An internationalized label is canonicalized the way a browser does it:+ non-transitional UTS #46, with the STD3 and hyphen rules off. Python's own+ ``"idna"`` codec is IDNA 2003 *transitional* instead, which folds ``ß`` into+ ``ss`` - so ``https://faß.de/`` would be attributed to ``fass.de`` while the+ link opens ``xn--fa-hia.de``, naming a domain the reader never visits.+ """+ if not host:+ return ""+ try:+ decoded = unquote(host, errors="strict")+ except UnicodeDecodeError:+ return ""+ if any(char in _FORBIDDEN_DOMAIN for char in decoded):+ return ""+ labels: list[str] = []+ for label in decoded.split("."):+ if label.isascii() and not label.lower().startswith(_ACE_PREFIX):+ # A browser passes an ASCII label through untouched apart from case,+ # even one UTS #46 refuses: ``my_site.example.com`` holds an+ # underscore and ``ab--cd.example`` trips the hyphen rule, and both+ # resolve. Running them through IDNA would lose the citation.+ labels.append(label.lower())+ continue+ if label.isascii():+ # An ``xn--`` label is the one ASCII spelling a browser does not+ # pass through: it decodes the punycode and refuses the whole URL+ # when that fails (``https://xn--/x`` does not open). Accept one+ # only when it is the exact encoding of a label the path below+ # accepts, so a citation never names a host its link cannot reach.+ ace = _ace_label(label.lower())+ if not ace:+ return ""+ labels.append(ace)+ continue+ try:+ # A browser labels an internationalized host by the punycode it+ # actually visits, so a citation has to attribute that and not the+ # bytes the URL spells.+ labels.append(+ idna.encode(+ label, uts46=True, transitional=False, std3_rules=False+ ).decode("ascii")+ )+ except (UnicodeError, ValueError):+ # Stricter than a browser on the margins: IDNA 2008 disallows symbol+ # labels UTS #46 alone would map (``❤.example`` resolves for a+ # browser, and is dropped here). Rejection degrades the citation to+ # its unresolved label, which is the only safe direction - a label+ # that names a different domain than the link opens is the defect.+ return ""+ return _canonical_host(".".join(labels))+++def _port_accepted(value: str) -> bool:+ """Whether a browser would read *value* as a port at all.++ ``str.isdigit`` is true for ``٣`` and ``int`` reads that as 3, so a port+ checked with it would let ``:٣`` through as 3 while a browser refuses the+ URL outright. Membership in the ASCII digits is the only test that agrees.++ An empty port is not a rejection - ``https://example.com:/x`` opens - and a+ port in range is kept as the provider wrote it, default or not: it is the+ same page either way, and the label never shows a port.+ """+ if not value:+ return True+ if not all(char in _DECIMAL_DIGITS for char in value):+ return False+ trimmed = value.lstrip("0")+ # Bound the string before converting it: CPython refuses to convert a+ # decimal ``int`` past ``sys.set_int_max_str_digits``, and an untrusted+ # authority is exactly where a host spelled with thousands of digits shows+ # up. Anything wider than the maximum port is out of range regardless.+ if len(trimmed) > _PORT_DIGITS:+ return False+ return not trimmed or int(trimmed) <= _PORT_MAX+++def _authority(+ url: str, scheme: str, *, literal_brackets: bool+) -> Optional[tuple[str, str, str]]:+ """``(authority, label host, rest)`` for *url*, or ``None`` to reject it.++ *literal_brackets* says whether the destination this *url* was spelled from+ wrapped its host in LITERAL brackets. It is a required argument because the+ spelled URL no longer holds the answer - ``[`` and ``]`` are outside the+ URI-safe set, so spelling writes both ``[::1]`` and an already-escaped+ ``%5B::1%5D`` as the same ``%5B::1%5D`` - and guessing it from the spelling+ is how a destination that named no host at all acquired a live one. Ask+ ``_authority_brackets_literal`` of the resolved destination, once, before it+ is spelled.++ One function answers the whole authority, because the parts are not+ independent: which ``@`` starts the host depends on the userinfo, whether a+ ``:`` starts a port depends on whether the host is a bracketed IPv6+ literal, and whether a port is in range depends on nothing else at all.+ Asking those questions separately - which is what reading+ ``urlsplit().hostname`` and nothing else amounted to - answers each one+ against a different authority, and every shape that fell between two of+ them became a citation pointing somewhere the reader could not go.++ It answers two things at once because they are one judgment. Whether a+ browser can open this URL decides if the citation is delivered; what that+ browser resolves the host to decides what the label may claim. The+ *authority* it hands back is still the one the provider wrote: a page is+ named by the URL its source gave, and a host written ``2130706433``,+ ``EXAMPLE.com`` or ``:0080`` opens the same page the label names.++ The canonical URI is what gets parsed, not the provider's raw string, so+ this reads the same authority the renderer will hand the browser.+ """+ remainder = url[len(scheme) + 1 :]+ if not remainder.startswith("//"):+ return None+ remainder = remainder[2:]+ end = _AUTHORITY_END_RE.search(remainder)+ authority = remainder[: end.start()] if end else remainder+ rest = remainder[end.start() :] if end else ""++ # WHATWG splits userinfo at the LAST ``@``: everything before it is+ # credentials, which may hold an ``@`` of their own. They are carried+ # across untouched - case-folding a password would change it.+ userinfo, separator, host_port = authority.rpartition("@")+ if not separator:+ userinfo, host_port = "", authority++ opener = _IPV6_OPEN_RE.match(host_port)+ if opener is not None:+ if not literal_brackets:+ # The wrapper is here in the spelling and was not in the+ # destination: the provider wrote its own ``%5B``, which is not+ # authority syntax but two characters a host may not hold. Reading+ # it as a bracket would mint an address the destination never named.+ return None+ closer = _IPV6_CLOSE_RE.search(host_port, opener.end())+ if closer is None:+ return None+ inner = host_port[opener.end() : closer.start()]+ address = _parse_ipv6(inner)+ if address is None:+ return None+ # Literal brackets, not the ``%5B`` the URI encoder left in their+ # place: this host WAS written in brackets, and no URL parser opens the+ # escaped spelling, so the badge could not reach the address either.+ host = f"[{inner}]"+ label_host = f"[{_serialize_ipv6(address)}]"+ port = host_port[closer.end() :]+ if port and not port.startswith(":"):+ return None+ else:+ host, colon, port_text = host_port.partition(":")+ label_host = _canonical_domain(host)+ if not label_host:+ return None+ port = f":{port_text}" if colon else ""++ if not _port_accepted(port[1:]):+ return None+ authority = f"{host}{port}"+ return (f"{userinfo}@{authority}" if userinfo else authority), label_host, rest+++def _authority_brackets_literal(resolved: str) -> bool:+ """Whether *resolved* wraps its host in literal brackets, not escapes.++ *resolved* is the destination as a Markdown reader resolved it and BEFORE+ any URI spelling: the one moment the two spellings are still distinct. A+ provider that wrote ``https://[::1]/admin`` named a host; a provider that+ wrote ``https://%5B::1%5D/admin`` named two characters a host may not hold,+ and the difference survives nowhere else - spelling writes both the same way.++ Both ends of the wrapper have to be literal. A mixed one (``[::1%5D``,+ ``%5B::1]``) is not a bracketed host with an escape in it; it is a host that+ was never written in brackets at all, and promoting half of it into syntax+ invents an address out of the other half.++ The authority is read here the way ``_authority`` reads it there, and the+ two agree because spelling cannot move the boundaries: ``/``, ``?``, ``#``,+ ``@`` and ``:`` are all inside the URI-safe set, so they are the same+ characters in the same places before and after. What a character reference+ or a backslash escape resolved to counts as written - that resolution+ already happened, once, in ``_normalize_destination``.+ """+ scheme, separator, remainder = resolved.partition(":")+ if not separator or not remainder.startswith("//"):+ return False+ remainder = remainder[2:]+ end = _AUTHORITY_END_RE.search(remainder)+ authority = remainder[: end.start()] if end else remainder+ _, separator, host_port = authority.rpartition("@")+ if not separator:+ host_port = authority+ opener = _IPV6_OPEN_RE.match(host_port)+ if opener is None or opener.group() != "[":+ return False+ closer = _IPV6_CLOSE_RE.search(host_port, opener.end())+ return closer is not None and closer.group() == "]"+++def spell_destination(resolved: str) -> str:+ """Spell a resolved destination, keeping a literal IPv6 host readable.++ Takes the destination BEFORE spelling and returns it after, because the two+ halves of this job cannot be separated: ``[`` and ``]`` are two of the+ characters ``spell_uri`` escapes, so ``https://[::1]/x`` leaves it spelled+ ``https://%5B::1%5D/x``, which is not an address any parser opens - and by+ then nothing in the string says whether those escapes were a host's own+ syntax or the provider's own characters. A caller that could hand over only+ the spelled URL could only guess, and guessing turned+ ``https://%5B::1%5D/admin`` into a live link to the loopback interface.++ The brackets go back in the authority and nowhere else: the same two+ characters in a path, query, fragment or userinfo are data, and stay+ spelled. They go back only when the authority they belong to parses, so a+ host like ``[nope]`` stays escaped rather than becoming a different URL, and+ every address without a literal bracketed host is returned exactly as+ ``spell_uri`` wrote it.+ """+ spelled = spell_uri(resolved)+ if not _authority_brackets_literal(resolved):+ return spelled+ scheme, separator, remainder = spelled.partition(":")+ if not separator or not remainder.startswith("//"):+ return spelled+ parsed = _authority(spelled, scheme, literal_brackets=True)+ if parsed is None:+ return spelled+ return f"{scheme}://{parsed[0]}{parsed[2]}"+++def safe_url(value: Any) -> str:+ """Canonical http(s) URL usable as a Markdown destination, or ``""``.++ Anything else - ``javascript:``, ``data:``, ``file:``, a hostless URL, an+ authority no browser would open - is rejected rather than repaired, so the+ citation is reported as unresolved instead of becoming an unsafe or dead+ link.++ What survives is left as it arrived, percent-encoding aside: the URL names+ the page its source gave, and this is not the place to decide that two+ spellings of one are the same. ``source_label`` is where the host a browser+ resolves is worked out, because that is the one place the difference is+ something the reader is shown.+ """+ if not isinstance(value, str):+ return ""+ raw = _normalize_destination(value)+ if not raw:+ return ""+ # A browser reads a backslash in an http(s) authority or path as ``/`` and+ # one in a query as data, so there is no single spelling of it that names+ # the page for every reader: ``https://trusted.example\\@attacker.example/``+ # opens trusted.example and, percent-encoded, is attributed to+ # attacker.example. The citation is refused rather than guessed at. An+ # already-encoded ``%5C`` is data to every reader and is kept.+ if "\\" in raw:+ return ""+ url = _canonical_uri(raw)+ scheme, separator, _ = url.partition(":")+ # A browser lowercases the scheme, and a renderer that only knows the+ # lowercase spelling does not see a link at all: Telegram delivered+ # ``[example.com](HTTPS://Example.com/X)`` as raw Markdown.+ scheme = scheme.lower()+ if not separator or scheme not in _ALLOWED_SCHEMES:+ return ""+ # Qualified from ``raw``: the resolved destination, before ``_canonical_uri``+ # spelled it. After spelling there is no such thing as a bracket to ask about.+ parsed = _authority(+ url, scheme, literal_brackets=_authority_brackets_literal(raw)+ )+ if parsed is None:+ return ""+ return f"{scheme}://{parsed[0]}{parsed[2]}"+++def source_label(url: str, title: str = "") -> str:+ """Short link text for a citation: its domain, which is what attributes it."""+ host = ""+ if isinstance(url, str):+ scheme, separator, _ = url.partition(":")+ if separator:+ # A label is worked out from a URL ``safe_url`` already accepted,+ # which is spelled with its brackets literal if it has any.+ parsed = _authority(+ url,+ scheme.lower(),+ literal_brackets=_authority_brackets_literal(url),+ )+ if parsed is not None:+ # The port is left out on purpose. A page is attributed by the+ # site it is on, a non-default port does not change which site+ # that is, and showing it would spend six of the label's 64+ # characters saying so.+ host = parsed[1]+ if host.startswith("www."):+ host = host[4:]+ if not host:+ return title[:_LABEL_MAX].strip()+ if len(host) <= _LABEL_MAX:+ return host+ # Shorten a long host from the LEFT, keeping the tail verbatim: the label is+ # always either the host or a suffix of it, marked by the ellipsis.+ #+ # A label cut from the right names a site the link never opens -+ # ``developers.openai.com.<padding>.attacker.example`` shown as+ # ``developers.openai.com…`` - and keeping only whole labels degrades the+ # other way: ``<62 chars>.co.uk`` would be shown as ``…co.uk``, a public+ # suffix every site under it shares and the perfect place to hide a long+ # attacker label. So the budget is filled from the right instead, and only+ # the leftmost piece - the one the ellipsis is attached to - may be partial.+ #+ # That cannot compress a host into a different plausible domain, because the+ # two cases exclude each other. A short fragment only happens when whole+ # labels already fill most of the budget, and those are the rightmost labels,+ # so the registrable domain is shown whole and the fragment sits in a+ # subdomain slot. A fragment of the registrable label itself only happens+ # when little else fits, which leaves it most of the 63 characters - more+ # than a DNS label may hold minus a few, so nothing recognizable can hide in+ # what is left out.+ return "…" + host[-(_LABEL_MAX - 1) :]+++# Every character CommonMark (plus GFM) still reads as syntax inside link+# text. Two different kinds of damage, one answer: a backtick, an angle bracket+# or an unbalanced square bracket opens a construct that runs past ``](url)``+# and swallows the link itself, while emphasis, strikethrough, a character+# reference or a table cell divider quietly change which characters the reader+# is shown. The label is the source's domain - the whole attribution claim - so+# a label that reads as ``ab.example`` when the link goes to ``a*b*.example``+# is the same defect as no link at all.+_LABEL_SYNTAX_RE = re.compile(r"([\\\[\]`<*_~&|])")+++def _escape_label(value: str) -> str:+ """Escape every character a Markdown reader would not show verbatim.++ What comes out is a label that reads back exactly as the sidecar spells it+ on every surface. The Web renderer resolves the escapes itself; each IM+ formatter resolves them once, ahead of its own dialect pass, through+ ``modules.im.formatters.hold_markdown_escapes`` - so no reader is shown a+ backslash that was only ever there to keep a domain intact.+ """+ return _LABEL_SYNTAX_RE.sub(r"\\\1", value)+++def has_citation_markers(text: Optional[str]) -> bool:+ """True when ``text`` carries at least one complete citation marker."""+ return bool(text) and _START in text and bool(CITATION_MARKER_RE.search(text))+++def citation_ref_ids(text: Optional[str]) -> list[str]:+ """The refs a message asks to have attributed, in first-appearance order.++ Only markers a citation can actually be written for are requests: one+ inside a code example is literal text, one inside a hidden block leaves+ with the block, and one written into an address, a title, or a reference+ identifier has nowhere a link could be shown. This is the same question+ registration asks, asked of the same mask - a marker that will not be+ attributed must not make delivery wait for a source either, or hydrate one+ into a sidecar nothing points at. A malformed ref is not a request either -+ it can name no source, so the marker degrades to the unresolved label no+ matter what arrives later.+ """+ if not text or _START not in text:+ return []+ requested: dict[str, None] = {}+ mask = mask_citation_slots(text, transparent=_SLOT_TRANSPARENT_RE)+ for match in CITATION_MARKER_RE.finditer(mask):+ for ref in match.group(1).split(_SEP):+ if ref and _REF_ID_RE.fullmatch(ref):+ requested.setdefault(ref, None)+ return list(requested)+++def unresolved_refs(+ ref_ids: Iterable[str], sources: Mapping[str, CitationSource]+) -> list[str]:+ """Which of *ref_ids* no source has been recorded for yet, in the given order.++ This is the question a delivery boundary asks: could a search still arrive+ that changes how this message reads? A ref whose source is already present+ has reached its final form even when that source turns out to be unlinkable -+ waiting on it would hold the message forever - and a malformed ref can never+ name a source at all, so neither is reported here.+ """+ return [ref for ref in ref_ids if _REF_ID_RE.fullmatch(ref) and ref not in sources]+++def register_citations(+ text: Optional[str],+ sources: Mapping[str, CitationSource],+ *,+ unresolved_label: str,+) -> tuple[Optional[str], Optional[CitationBundle]]:+ """Give every marker this message will attribute a private identity, now.++ ``sources`` maps ref_id to the search result the backend captured for it,+ scoped to the native thread the message belongs to. Returns the text to+ carry through delivery and the bundle that finalizes it; sources repeated+ across markers share one entry (and one index) keyed by their URL.++ Registration happens against the text exactly as the backend produced it,+ because that is the only version of it whose citation grammar means what it+ says. Every later stage rewrites the text - a ``<silent>`` block leaves, a+ ``file://`` link is stripped to its label, images are appended - and a+ rewrite can splice two ordinary halves into something that reads as a+ marker. ``\\ue200ci<silent>x</silent>te\\ue202turn0view0\\ue201`` is one+ marker and one accidental one to a reader of the delivered text, and a+ stage that asked the delivered text which markers to attribute would put a+ citation on text the model never cited. So the question is asked once, here,+ and what travels from here on is an opaque token per registered marker:+ unguessable, never derived from the ref or from its order, and checked+ against the arriving text so a token can only be a token this call minted.++ A marker inside a code example, a ``<silent>`` block, or a Markdown slot+ that holds data rather than prose is not registered (one mask decides all+ three); one whose refs all fail to resolve with no label to fall back to+ keeps its literal text, exactly as before.+ """+ if not text or _START not in text:+ return text, None++ fallback = (unresolved_label or "").strip()+ citations: list[Citation] = []+ by_url: dict[str, Citation] = {}+ consulted: dict[str, CitationSource] = {}+ markers: list[RegisteredMarker] = []+ minted: set[str] = set()++ def mint_token() -> str:+ # A nonce, not a name: the token is the citation's identity for the rest+ # of delivery, so nothing about it may be reconstructible from the+ # message. Colliding with text that already arrived would hand that text+ # the identity, so the arriving text is checked too.+ while True:+ token = f"{_TOKEN_OPEN}{secrets.token_hex(_TOKEN_NONCE_BYTES)}{_TOKEN_CLOSE}"+ if token not in minted and token not in text:+ minted.add(token)+ return token++ def resolve_ref(ref: str) -> Optional[Citation]:+ if not _REF_ID_RE.fullmatch(ref):+ return None+ source = sources.get(ref)+ if source is None:+ return None+ consulted.setdefault(ref, source)+ url = safe_url(source.url)+ if not url:+ return None+ existing = by_url.get(url)+ if existing is not None:+ return existing+ title = clean_title(source.title)+ label = source_label(url, title)+ if not label:+ return None+ citation = Citation(+ index=len(citations) + 1,+ ref_id=ref,+ title=title,+ url=url,+ label=label,+ )+ citations.append(citation)+ by_url[url] = citation+ return citation++ def replace(match: re.Match[str]) -> str:+ refs = [ref for ref in match.group(1).split(_SEP) if ref]+ parts: list[WrittenLink] = []+ unresolved = not refs+ # Markers usually sit flush against the preceding word or full stop;+ # a separating space keeps the link from reading as part of the sentence.+ lead = "" if match.start() == 0 or text[match.start() - 1].isspace() else " "+ for ref in refs:+ citation = resolve_ref(ref)+ if citation is None:+ unresolved = True+ continue+ parts.append(+ WrittenLink(+ index=citation.index,+ text=_link_spelling(citation),+ plain=citation.label,+ )+ )+ if unresolved and fallback:+ parts.append(WrittenLink(index=None, text=fallback, plain=fallback))+ if not parts:+ return match.group(0)+ token = mint_token()+ markers.append(+ RegisteredMarker(+ token=token,+ literal=match.group(0),+ lead=lead,+ parts=tuple(parts),+ )+ )+ return token++ registered = _rewrite_citable_markers(text, replace)+ if not markers:+ return registered, None+ return registered, CitationBundle(+ citations=tuple(citations),+ markers=tuple(markers),+ sources=tuple(consulted.values()),+ )+++def materialize_citations(+ text: Optional[str],+ bundle: Optional[CitationBundle],+ *,+ as_markdown: bool = True,+) -> Optional[str]:+ """Write the registered citations into *text*, with no sidecar.++ This is the form every surface that carries only text needs - an IM copy, a+ Turn snapshot, an agent-run record, a live stream chunk. ``as_markdown``+ off writes each citation as its bare domain instead of a Markdown link, for+ the structured fields (quick-reply labels, file labels) that are extracted+ out of the body and are not Markdown anywhere.+ """+ if not text or bundle is None or _TOKEN_OPEN not in text:+ return text+ written, _ = _write_citations(text, bundle, as_markdown=as_markdown)+ return written+++def finalize_citations(+ text: Optional[str],+ bundle: Optional[CitationBundle],+) -> tuple[Optional[str], list[dict[str, Any]]]:+ """Write the registered citations into *text* and bind the sidecar to it.++ The returned rows describe THIS body and no other. Each carries the exact+ ``spans`` its links occupy in it and the digest of the body itself, because+ a span alone is not an identity: two ordinary links can be spelled the same,+ and a paragraph removed above one moves another into its place. Rendering a+ badge from a span the producer measured in a different version of the text+ is how an ordinary link gets to impersonate a cited one, so a consumer+ checks the digest first and the spans only after it matches.++ Only tokens this bundle minted are written; anything else in the text is+ left exactly as it arrived. A token that has landed inside code or a hidden+ block by the time delivery settles gets its original marker text back rather+ than a link or an exposed internal token - the reader is shown what the+ model wrote, and nothing claims it was cited.+ """+ if not text or bundle is None or _TOKEN_OPEN not in text:+ return text, []+ body, placed = _write_citations(text, bundle, as_markdown=True)+ if not placed:+ return body, []+ offsets = _utf16_offsets(+ body, (offset for _, start, end in placed for offset in (start, end))+ )+ spans: dict[int, list[list[int]]] = {}+ for index, start, end in placed:+ spans.setdefault(index, []).append([offsets[start], offsets[end]])+ digest = body_digest(body)+ return body, [+ citation.to_payload(spans=spans[citation.index], body_sha256=digest)+ for citation in bundle.citations+ if citation.index in spans+ ]+++def resolve_citations(+ text: Optional[str],+ sources: Mapping[str, CitationSource],+ *,+ unresolved_label: str,+) -> tuple[Optional[str], list[dict[str, Any]]]:+ """Register and finalize in one step, for a surface that transforms nothing.++ The two halves exist because delivery rewrites the text between them. When+ nothing happens in between - a probe, a test, a caller that hands its own+ output straight out - this is the same call it always was.+ """+ registered, bundle = register_citations(+ text, sources, unresolved_label=unresolved_label+ )+ return finalize_citations(registered, bundle)+++def body_digest(text: str) -> str:+ """The digest a sidecar binds its spans to: SHA-256 over UTF-16 code units.++ UTF-16 because that is the unit the spans are counted in, and a digest over+ a different encoding of the same text would let the two disagree about which+ text they mean. Little-endian, no BOM, unpaired surrogates passed through:+ exactly the bytes a consumer reading the string a code unit at a time hashes.++ This says which VERSION of the body a span was measured in. It is not a+ signature: both sides compute it locally from text they already hold, and it+ claims nothing about where that text came from.+ """+ return hashlib.sha256(text.encode("utf-16-le", "surrogatepass")).hexdigest()+++def _link_spelling(citation: Citation) -> str:+ """The exact Markdown link a citation is written as.++ A parenthesis stays in the URL and is escaped in the destination, which+ every reader resolves back to the character: an unbalanced one would+ otherwise end the link early. ``safe_url`` has refused any backslash, so an+ escape here cannot pair with one already in the address. An angle-bracket+ destination would say the same thing, but Telegram and WeChat do not read+ that form as a link.+ """+ destination = _DESTINATION_PAREN_RE.sub(lambda match: "\\" + match.group(), citation.url)+ return f"[{_escape_label(citation.label)}]({destination})"+++def _utf16_offsets(text: str, offsets: Iterable[int]) -> dict[int, int]:+ """Convert code-point offsets into UTF-16 code-unit offsets, in one pass.++ Python counts code points and every consumer of the sidecar counts UTF-16+ code units, so one emoji outside the BMP puts the two a unit apart for the+ rest of the message.+ """+ converted: dict[int, int] = {}+ cursor = 0+ units = 0+ for offset in sorted(set(offsets)):+ units += len(text[cursor:offset].encode("utf-16-le", "surrogatepass")) // 2+ cursor = offset+ converted[offset] = units+ return converted+++def _enclosing_unit_end(+ units: list[MarkdownUnit], start: int, end: int+) -> Optional[int]:+ """Where a citation for a marker inside a link or image has to go instead.++ A link nested inside a link label is not a link on any surface: CommonMark+ refuses to parse the inner one, and every IM dialect downstream spells the+ outer one as a single wrapper. So a citation for a marker written in a+ label belongs immediately after the whole unit, and after the *outermost*+ one when units nest - an image inside a link is two units, and only the+ link has an end a sibling can follow. ``units`` is ordered outermost-first+ at each offset, so the first one that contains the marker is that unit.+ """+ for unit in units:+ if unit.start > start:+ break+ if end <= unit.end:+ return unit.end+ return None+++def _separating_lead(parts: list[str]) -> str:+ """A space before a relocated citation, unless one is already written."""+ for part in reversed(parts):+ if part:+ return "" if part[-1].isspace() else " "+ return ""+++def _write_citations(+ text: str,+ bundle: CitationBundle,+ *,+ as_markdown: bool,+) -> tuple[str, list[tuple[int, int, int]]]:+ """Replace this bundle's tokens, reporting where each citation landed.++ The placements are ``(citation index, start, end)`` in code points over the+ returned text, one per link actually written.++ Where a citation can go is a question about the text as it stands now, so+ it is asked of the final text rather than remembered from registration: a+ transform may have wrapped a token in an example, and an internal token is+ the one thing that may never be shown. A token that has landed in code, in+ a hidden block, or in a slot the parser reads without showing gets its+ original marker text back and claims nothing. A token inside a link or+ image label is removed from that label and its citation written after the+ unit, which is the nearest position the reader is actually shown.++ ``as_markdown`` off is a structured field - a file label, a quick-reply+ button - that is not Markdown on any surface. There is no unit to be inside+ and no link to nest, so the attribution stays where the marker was.+ """+ registered = {marker.token: marker for marker in bundle.markers}+ if as_markdown:+ mask = mask_citation_slots(text, transparent=_SLOT_TRANSPARENT_RE)+ units = markdown_link_units(text)+ else:+ mask = mask_hidden_and_code(text)+ units = []++ # ``(offset, rank, sequence, consumed, marker, literal)``. ``rank`` puts a+ # citation relocated to an offset ahead of a token that starts there, and+ # ``sequence`` keeps two citations relocated to the same offset in the order+ # their markers were written. Both are integers, so sorting never has to+ # compare the payload.+ edits: list[tuple[int, int, int, int, Optional[RegisteredMarker], str]] = []+ sequence = 0+ for match in _TOKEN_RE.finditer(text):+ marker = registered.get(match.group(0))+ if marker is None:+ # Token-shaped text this bundle did not mint. It arrived as ordinary+ # characters and leaves as ordinary characters.+ continue+ consumed = match.end() - match.start()+ if mask[match.start() : match.end()] != match.group(0):+ edits.append(+ (match.start(), 1, sequence, consumed, None, marker.literal)+ )+ sequence += 1+ continue+ relocated = _enclosing_unit_end(units, match.start(), match.end())+ if relocated is None:+ edits.append((match.start(), 1, sequence, consumed, marker, ""))+ sequence += 1+ continue+ # The label keeps every byte that is not the token, and the citation is+ # written as a sibling of the unit rather than a link inside its label.+ edits.append((match.start(), 1, sequence, consumed, None, ""))+ sequence += 1+ edits.append((relocated, 0, sequence, 0, marker, ""))+ sequence += 1++ out: list[str] = []+ placed: list[tuple[int, int, int]] = []+ cursor = 0+ written = 0+ for offset, _rank, _sequence, consumed, marker, literal in sorted(edits):+ prefix = text[cursor:offset]+ out.append(prefix)+ written += len(prefix)+ if marker is None:+ out.append(literal)+ written += len(literal)+ else:+ # A relocated citation no longer sits where its marker did, so what+ # separates it from the text is read off what has actually been+ # written; one written in place keeps the lead registration measured.+ lead = marker.lead if consumed else _separating_lead(out)+ at = written + len(lead)+ body = lead+ for index, part in enumerate(marker.parts):+ if index:+ body += " "+ at += 1+ spelling = part.text if as_markdown else part.plain+ if part.index is not None:+ placed.append((part.index, at, at + len(spelling)))+ body += spelling+ at += len(spelling)+ out.append(body)+ written += len(body)+ cursor = offset + consumed+ out.append(text[cursor:])+ return "".join(out), placed+++def _rewrite_citable_markers(+ text: str,+ replace: Callable[[re.Match[str]], str],+) -> str:+ """Apply ``replace`` to every marker a citation could be written for.++ A marker shown inside a code example must stay literal - an agent+ explaining this very grammar is a case seen in real transcripts - and+ "inside code" is a question only a CommonMark lexer can answer. Fence+ lengths nest (a four-backtick block quoting a three-backtick one), an+ unclosed fence swallows the rest of the document, a code span may run+ across lines, and container indentation shifts all of it. A marker inside a+ ``<silent>`` block has to stay literal for a different reason: the block is+ removed before delivery, so rewriting it would spend a citation index on+ text nobody reads and lift a URL out of the block meant to hide it. Both+ A marker written into a destination, a title, a reference identifier, or an+ autolink's address has to stay literal for a third reason: there is no+ reader-facing position there at all, so a link written into one would not be+ shown - it would change where an existing link points. All three decisions+ are delegated to the reply parser's offset-preserving mask: markers are+ matched against the mask and spliced back into the original source, which+ leaves every byte this function does not replace exactly as it arrived.+ """+ mask = mask_citation_slots(text, transparent=_SLOT_TRANSPARENT_RE)+ out: list[str] = []+ cursor = 0+ # A match in the mask cannot overlap a blanked region, so the marker text+ # under it is the original text - only its surroundings may have been+ # blanked, and those are copied from ``text``, never from the mask.+ for match in CITATION_MARKER_RE.finditer(mask):+ out.append(text[cursor : match.start()])+ out.append(replace(match))+ cursor = match.end()+ out.append(text[cursor:])+ return "".join(out)diff --git a/core/controller.py b/core/controller.pyindex aa30c900f..12f1b34dd 100644--- a/core/controller.py+++ b/core/controller.py@@ -39,6 +39,7 @@ from core.handlers import ()from core.agent_auth_service import AgentAuthServicefrom core.audio_asr import AudioAsrService+from core.citations import CitationBundlefrom core.message_context import build_context_session_keyfrom core.message_dispatcher import ConsolidatedMessageDispatcherfrom core.message_output import MessageOutput@@ -3876,6 +3877,7 @@ class Controller:output: MessageOutput | None = None,terminal_error: Optional[str] = None,delivery: Any = None,+ citations: Optional[CitationBundle] = None,):"""Backward-compatible entrypoint; delegated to message dispatcher."""result = await self.message_dispatcher.emit_agent_message(@@ -3893,8 +3895,9 @@ class Controller:# ``emit_backend_failure`` does: ``message_dispatcher`` is a# substitutable collaborator (six test suites replace it), so passing# an optional diagnostic unconditionally would change the required- # signature of every stand-in.+ # signature of every stand-in. The citation sidecar rides the same way.**({"delivery": delivery} if delivery is not None else {}),+ **({"citations": citations} if citations else {}),)manager = getattr(self, "session_turns", None)complete = getattr(manager, "on_terminal_delivery_complete", None)diff --git a/core/message_dispatcher.py b/core/message_dispatcher.pyindex 40ddd1a2d..a07af7029 100644--- a/core/message_dispatcher.py+++ b/core/message_dispatcher.py@@ -10,11 +10,12 @@ import asyncioimport hashlibimport loggingimport re+import tempfileimport time-from dataclasses import replace+from dataclasses import dataclass, replacefrom datetime import datetime, timezonefrom pathlib import Path-from typing import Any, Mapping, Optional+from typing import Any, Callable, Mapping, Optional, Sequencefrom urllib.parse import urljoinfrom config.platform_registry import get_platform_descriptor@@ -23,6 +24,7 @@ from modules.im import MessageContextfrom modules.im.formatters.base_formatter import to_status_labelfrom core.delivery_evidence import STAGE_PERSIST, STAGE_SEND, STAGE_STREAM, DeliveryEvidencefrom core.delivery_target import routed_delivery_context+from core.citations import CitationBundle, materialize_citationsfrom core import failure_noticesfrom core.message_context import resolve_turn_sink_keyfrom core.message_mirror import (@@ -38,7 +40,14 @@ from core.message_output import (neutralize_mentions,output_for_message,)-from core.reply_enhancer import process_reply, strip_file_links, strip_silent_blocks+from core.reply_enhancer import (+ FileLink,+ QuickReplyButton,+ inline_links,+ process_reply,+ strip_file_links,+ strip_silent_blocks,+)from core.run_settlement import (SETTLED_BY_BACKEND_REFRESH,SETTLED_BY_STOPPED,@@ -89,6 +98,44 @@ _HARNESS_PROMPT_ECHO_I18N_KEYS = {}+def _written_buttons(+ buttons: Sequence[QuickReplyButton],+ citations: Optional[CitationBundle],+) -> Sequence[QuickReplyButton]:+ """Quick-reply buttons with their citations written as bare attribution.++ A label is lifted out of the body into a structured field no surface parses+ as Markdown, so a link written here would show the reader its syntax. The+ domain on its own still says who is being cited - and an internal token must+ never survive into a field nothing else will rewrite.+ """+ if not citations or not buttons:+ return buttons+ return [+ replace(+ button,+ text=materialize_citations(button.text, citations, as_markdown=False) or "",+ )+ for button in buttons+ ]+++def _written_files(+ files: Sequence[FileLink],+ citations: Optional[CitationBundle],+) -> Sequence[FileLink]:+ """The same for an extracted file link's label, which is uploaded as text."""+ if not citations or not files:+ return files+ return [+ replace(+ link,+ label=materialize_citations(link.label, citations, as_markdown=False) or "",+ )+ for link in files+ ]++class ActivityOutputDeliveryError(RuntimeError):"""An Activity result did not complete its local output boundary."""@@ -228,6 +275,25 @@ _WECHAT_CONSOLIDATED_SPLIT_THRESHOLD = 1700# normal fast step stays a clean label. This is the always-moving "still running"# signal that the heartbeat keeps ticking even when no new emit arrives._ACTION_TIME_HINT_S = 10.0+# Name the result attachment gets when a platform has no native Markdown upload+# and the full text has to ride its ordinary file path instead.+_RESULT_DOCUMENT_NAME = "result.md"+++@dataclass(frozen=True)+class _ResultSplit:+ """A proposed split, and whether every link in it survived.++ ``links_whole`` is False when a link unit is longer than one message can+ carry, so no boundary keeps it intact. The chunks are still the best split+ available, but a caller that can deliver the whole text another way should+ take that route BEFORE sending any of them: a half link sends fine, which+ would make the delivery look successful while the reader is shown broken+ Markdown and no way to reach the source.+ """++ chunks: list[str]+ links_whole: boolclass ConsolidatedMessageDispatcher:@@ -1772,8 +1838,14 @@ class ConsolidatedMessageDispatcher:return f"{prefix}{text[:keep]}{suffix}"@staticmethod- def _find_result_split_index(text: str, max_chars: int) -> int:- minimum_boundary = max_chars // 2+ def _find_result_split_index(text: str, max_chars: int, floor: int = 0) -> int:+ """The preferred boundary at or before ``max_chars``, above ``floor``.++ ``floor`` is a position the chunk has already committed to keeping -+ the end of a link that fits - so whitespace behind it is no longer a+ candidate. With the default it is the boundary search this always did.+ """+ minimum_boundary = max(max_chars // 2, floor)for separator in ("\n\n", "\n", " "):index = text.rfind(separator, 0, max_chars + 1)if index >= minimum_boundary:@@ -1781,56 +1853,181 @@ class ConsolidatedMessageDispatcher:return candidate if candidate <= max_chars else indexreturn max_chars- def _split_result_text(self, text: str, max_chars: int) -> list[str]:- if len(text) <= max_chars:- return [text]+ @staticmethod+ def _link_units(text: str) -> Optional[list[tuple[int, int]]]:+ """Where each Markdown link unit starts and ends in ``text``.++ A link is one unit - a label a reader taps and an address the tap goes+ to - and a boundary drawn through it delivers neither: both halves send+ successfully, so nothing falls back, and the reader is shown raw+ Markdown where the source should have been. The enumeration is the+ shared one every platform pass already holds a link with, so splitting+ and spelling agree on where a link is.++ Returns ``None`` when the scan could not run. That is not the same+ answer as "there are no links": nothing here knows where a link is+ without it, so the plan may still split on text boundaries but may not+ certify that those boundaries kept anything whole.+ """+ try:+ return [(link.start, link.end) for link in inline_links(text)]+ except Exception:+ logger.warning(+ "Link scan failed while planning a split; the plan cannot certify a whole link",+ exc_info=True,+ )+ return None+ @staticmethod+ def _straddling_link(+ split_at: int, links: Sequence[tuple[int, int]], consumed: int+ ) -> Optional[tuple[int, int]]:+ """The chunk-relative link a boundary at ``split_at`` would cut, if any.++ ``links`` are offsets into the whole text and ``consumed`` is how much+ of it earlier chunks already took.+ """+ for start, end in links:+ start -= consumed+ end -= consumed+ if end <= split_at:+ continue+ if start >= split_at:+ return None+ return max(start, 0), end+ return None++ def _boundary_keeping_links_whole(+ self,+ links: Sequence[tuple[int, int]],+ consumed: int,+ capacity: int,+ preferred: Callable[[int], int],+ ) -> tuple[int, bool]:+ """Choose a boundary in ``(0, capacity]`` that no link straddles.++ ``capacity`` is what one message actually holds, measured in the same+ characters ``preferred`` proposes a cut in; ``preferred(floor)`` is the+ platform's own boundary search above a position already committed to.++ The decision is the link's size against that capacity, never against+ the preferred cut. A link the cut lands inside can be kept three ways:+ cut before it, so it rides the next message whole; or, when it starts+ the chunk, cut after it if it fits, because a chunk that is nothing but+ the link is still a legal message. Only a link that outruns a whole+ message on its own has no boundary left, and that is the one case this+ reports as unwhole. Reading "the preferred whitespace is inside the+ link" as that case rejected links half the size of the message.++ Returns ``(split_at, links_whole)``.+ """+ floor = 0+ while True:+ split_at = preferred(floor)+ if split_at <= floor:+ # The preferred cut is behind what this chunk already keeps;+ # the whole capacity is the boundary to try instead.+ split_at = capacity+ straddling = self._straddling_link(split_at, links, consumed)+ if straddling is None:+ return split_at, True+ start, end = straddling+ if start > 0:+ return start, True+ if end <= capacity:+ # The link begins the chunk and fits in one message: keep it+ # and look for the boundary again beyond its end. ``floor``+ # strictly increases, so this settles.+ floor = end+ continue+ return capacity, False++ def _plan_result_split(self, text: str, max_chars: int) -> _ResultSplit:+ if len(text) <= max_chars:+ return _ResultSplit(chunks=[text], links_whole=True)++ scanned = self._link_units(text)+ links = scanned or []+ # A scan that could not run has not found zero links; it has found+ # nothing at all, and a plan that cannot see a link cannot promise one+ # survived. Splitting still proceeds on text boundaries - the message+ # is too long either way - but the promise is withheld.+ links_whole = scanned is not Nonechunks: list[str] = []remaining = text+ consumed = 0while len(remaining) > max_chars:- split_at = self._find_result_split_index(remaining, max_chars)- if split_at <= 0:- split_at = max_chars+ chunk = remaining+ split_at, whole = self._boundary_keeping_links_whole(+ links,+ consumed,+ max_chars,+ lambda floor: self._find_result_split_index(chunk, max_chars, floor),+ )+ links_whole = links_whole and wholechunks.append(remaining[:split_at])remaining = remaining[split_at:]+ consumed += split_atif remaining:chunks.append(remaining)- return chunks+ return _ResultSplit(chunks=chunks, links_whole=links_whole)- def _split_result_text_by_bytes(self, text: str, max_bytes: int) -> list[str]:+ def _split_result_text(self, text: str, max_chars: int) -> list[str]:+ return self._plan_result_split(text, max_chars).chunks++ def _plan_result_split_by_bytes(self, text: str, max_bytes: int) -> _ResultSplit:if self._get_text_byte_length(text) <= max_bytes:- return [text]+ return _ResultSplit(chunks=[text], links_whole=True)+ scanned = self._link_units(text)+ links = scanned or []+ links_whole = scanned is not Nonechunks: list[str] = []remaining = text+ consumed = 0while self._get_text_byte_length(remaining) > max_bytes:+ # The longest prefix this budget holds, decoded: its length is the+ # capacity in the same characters the boundary search counts, so a+ # link that ends at or before it is a link the message can carry.prefix = remaining.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore")- minimum_boundary = max(1, len(prefix) // 2)- split_at = len(prefix)- for separator in ("\n\n", "\n", " "):- index = prefix.rfind(separator)- if index >= minimum_boundary:- candidate = index + len(separator)- if self._get_text_byte_length(remaining[:candidate]) <= max_bytes:- split_at = candidate- break+ capacity = len(prefix)++ def preferred(floor: int, prefix: str = prefix, capacity: int = capacity) -> int:+ minimum_boundary = max(1, capacity // 2, floor)+ for separator in ("\n\n", "\n", " "):+ index = prefix.rfind(separator)+ if index >= minimum_boundary:+ return index + len(separator)+ return capacity++ split_at, whole = self._boundary_keeping_links_whole(+ links, consumed, capacity, preferred+ )+ links_whole = links_whole and wholechunks.append(remaining[:split_at])remaining = remaining[split_at:]+ consumed += split_atif remaining:chunks.append(remaining)- return chunks+ return _ResultSplit(chunks=chunks, links_whole=links_whole)- def _split_result_text_for_context(self, context: MessageContext, text: str) -> list[str]:+ def _split_result_text_by_bytes(self, text: str, max_bytes: int) -> list[str]:+ return self._plan_result_split_by_bytes(text, max_bytes).chunks++ def _plan_result_split_for_context(self, context: MessageContext, text: str) -> _ResultSplit:max_bytes = self._get_result_max_bytes(context)if max_bytes is not None:- return self._split_result_text_by_bytes(text, max_bytes)- return self._split_result_text(text, self._get_result_max_chars(context))+ return self._plan_result_split_by_bytes(text, max_bytes)+ return self._plan_result_split(text, self._get_result_max_chars(context))++ def _split_result_text_for_context(self, context: MessageContext, text: str) -> list[str]:+ return self._plan_result_split_for_context(context, text).chunksdef _truncate_consolidated(self, text: str, max_bytes: int) -> str:if self._get_text_byte_length(text) <= max_bytes:@@ -1847,12 +2044,32 @@ class ConsolidatedMessageDispatcher:context: MessageContext,text: str,) -> Optional[str]:+ """Send an intermediate message on a platform that cannot edit one.++ This is the second consumer of the shared split plan. It stays an+ intermediate message throughout - it settles no turn, signals no+ result lifecycle, and writes no transcript row of its own - so the+ only thing it takes from the result path is the narrow whole-document+ upload, and only for a body no boundary can keep whole.+ """target_context = self._get_target_context(context)max_bytes = self._get_consolidated_max_bytes(context)- chunks = self._split_result_text_by_bytes(text, max_bytes)+ plan = self._plan_result_split_by_bytes(text, max_bytes)+ if not plan.links_whole:+ # The same answer the result path acts on, honoured by the plan's+ # other consumer. A boundary drawn through a link delivers neither+ # half and both halves send successfully, so nothing after this+ # would notice - the whole body goes out as one document instead,+ # BEFORE any fragment, and an upload that did not happen is+ # reported as no delivery rather than as a partial one.+ logger.warning(+ "Log message split would break a link unit; delivering the whole body as a document"+ )+ return await self._upload_result_document(im_client, target_context, text)+first_message_id: Optional[str] = None- for chunk in chunks:+ for chunk in plan.chunks:try:message_id = await im_client.send_message(target_context, chunk, parse_mode="markdown")except Exception as err:@@ -2062,6 +2279,7 @@ class ConsolidatedMessageDispatcher:output: MessageOutput | None = None,terminal_error: Optional[str] = None,delivery: DeliveryEvidence | None = None,+ citations: Optional[CitationBundle] = None,) -> Optional[str]:"""Centralized dispatch for agent messages.@@ -2102,6 +2320,16 @@ class ConsolidatedMessageDispatcher:compatibility default remains one terminal result. Explicit nonterminaloutputs remain in the current Turn, while detached outputs are deliveredto the Session/IM surface without touching a newer Turn or its stream.++ ``citations`` is the registered source bundle for a message whose text+ still carries an opaque token where each cited marker was (see+ ``core.citations``). Every copy this method produces writes those tokens+ out for itself, because the copies are not the same text: file links are+ stripped for IM, media links are rewritten for the workbench, footers are+ folded, long results are truncated or split. The persisted row is the one+ that also gets the sidecar, bound to the body it was measured in, so the+ Web transcript can render compact badges while every other surface keeps+ the plain Markdown links."""settings_manager = self.controller.get_settings_manager_for_context(context)im_client = self._get_im_client(context)@@ -2194,6 +2422,12 @@ class ConsolidatedMessageDispatcher:# without guessing from transcript order or requiring a live sink.if mutates_turn_lifecycle:terminal_body = enhanced.text if enhanced and enhanced.text.strip() else text+ # A Turn snapshot is text and nothing else - no sidecar rides with it -+ # so the citations are written into it here. It does not reach+ # ``persist_agent_message``, which is where every other copy is+ # finalized, and an internal token must never be what a later steer+ # reads back as the turn's result.+ terminal_body = materialize_citations(terminal_body, citations)manager = getattr(self.controller, "session_turns", None)if manager is not None:manager.on_terminal_result(@@ -2261,7 +2495,7 @@ class ConsolidatedMessageDispatcher:# settle its origin Run without touching the current Turn.self._record_agent_run_terminal_result(context,- text,+ materialize_citations(text, citations),None,is_error=is_error,terminal_error=terminal_error,@@ -2392,7 +2626,7 @@ class ConsolidatedMessageDispatcher:)duplicate_result_text = self._accepted_message_result_text(accepted_message,- persist_text,+ materialize_citations(persist_text, citations),result_footer if mutates_turn_lifecycle else None,)self._record_agent_run_terminal_result(@@ -2483,10 +2717,17 @@ class ConsolidatedMessageDispatcher:target_context,communication_type,background_enhanced.text or persist_text,- quick_replies=[b.text for b in background_enhanced.buttons] or None,+ quick_replies=[+ b.text+ for b in _written_buttons(+ background_enhanced.buttons, citations+ )+ ]+ or None,result_footer=result_footer,metadata=output_metadata,native_message_id=native_output_id,+ citations=citations,)else:persisted_output = persist_agent_message(@@ -2496,6 +2737,7 @@ class ConsolidatedMessageDispatcher:result_footer=result_footer,metadata=output_metadata,native_message_id=native_output_id,+ citations=citations,)else:persisted_output = persist_agent_message(@@ -2504,6 +2746,7 @@ class ConsolidatedMessageDispatcher:recorded_text,metadata=output_metadata,native_message_id=native_output_id,+ citations=citations,)local_message_id = (persisted_output or {}).get("id") or (f"suppressed:{(context.platform_specific or {}).get('task_execution_id') or canonical_type}"@@ -2516,10 +2759,14 @@ class ConsolidatedMessageDispatcher:# message recorder, except an intermediate message on a Harness run,# which belongs to no output ledger entry and is recorded by neither.suppressed_trigger_kind = (context.platform_specific or {}).get("task_trigger_kind")+ # A run record carries text alone, so its copy is written here.+ # The persisted row above keeps its tokens: it is finalized with+ # a sidecar bound to the body the reader is actually shown.+ written_recorded_text = materialize_citations(recorded_text, citations)if canonical_type == "result" and suppressed_trigger_kind in HARNESS_TRIGGER_KINDS:self._record_suppressed_agent_run_terminal_result(context,- recorded_text,+ written_recorded_text,None,is_error=is_error,terminal_error=terminal_error,@@ -2528,7 +2775,7 @@ class ConsolidatedMessageDispatcher:elif canonical_type == "result" or suppressed_trigger_kind not in HARNESS_TRIGGER_KINDS:self._record_suppressed_run_message(context,- recorded_text,+ written_recorded_text,None,terminal_status=terminal_status,)@@ -2568,8 +2815,14 @@ class ConsolidatedMessageDispatcher:# ``_stream_chunk`` failure did the same thing. Whoever owes a durable# notice for this message has to be able to tell those apart from a send# that genuinely failed, so each stage reports itself.+ # The outbound copy and the live stream carry text alone; the row+ # below keeps its tokens so ``persist_agent_message`` can bind the+ # sidecar to the body it finalizes.+ notify_text = materialize_citations(text, citations)try:- message_id = await im_client.send_message(target_context, text, parse_mode=parse_mode)+ message_id = await im_client.send_message(+ target_context, notify_text, parse_mode=parse_mode+ )except Exception as err:logger.error("Failed to send notify message: %s", err, exc_info=True)if delivery is not None:@@ -2590,6 +2843,7 @@ class ConsolidatedMessageDispatcher:metadata=output_metadata,native_message_id=native_output_id,error_sink=persist_errors,+ citations=citations,)if delivery is not None:delivery.persisted_row = persisted_notify@@ -2600,7 +2854,9 @@ class ConsolidatedMessageDispatcher:# this point the message is delivered; a failure here is recorded for# diagnosis and must not be read as a delivery failure.try:- await _stream_chunk(self.controller, context, text=text, message_id=message_id, kind="notify")+ await _stream_chunk(+ self.controller, context, text=notify_text, message_id=message_id, kind="notify"+ )except Exception as err:logger.error("notify stream failed after delivery: %s", err, exc_info=True)if delivery is not None:@@ -2631,6 +2887,15 @@ class ConsolidatedMessageDispatcher:# ``enhanced`` (extracted file links + quick-reply buttons) was# computed above for persistence; reuse it for delivery.display_text = enhanced.text if enhanced.text.strip() else text+ # Everything downstream of here is a text-only copy - the IM+ # sends, the split parts, the summary, the .md attachment, the+ # status footer and the live stream - so the citations are+ # written in once, before any of them can truncate or reflow the+ # body. ``persist_text`` below still holds its tokens.+ display_text = materialize_citations(display_text, citations)+ delivery_buttons = _written_buttons(+ enhanced.buttons if enhanced else [], citations+ )# The concise done-footer (``✅ done · 248k tok``) is attached to the# fresh result message as platform subtext so the turn's final@@ -2681,7 +2946,7 @@ class ConsolidatedMessageDispatcher:im_client,target_context,display_text,- enhanced.buttons if enhanced else [],+ delivery_buttons,parse_mode,subtext=done_footer,)@@ -2704,7 +2969,7 @@ class ConsolidatedMessageDispatcher:im_client,target_context,display_text,- enhanced.buttons if enhanced else [],+ delivery_buttons,parse_mode,subtext=done_footer,)@@ -2753,22 +3018,17 @@ class ConsolidatedMessageDispatcher:logger.warning("All direct result sends failed; attempting fallback delivery")file_uploaded = False- # Fallback 1: upload full content as .md file.- if hasattr(im_client, "upload_markdown"):- try:- primary_message_id = await im_client.upload_markdown(- target_context,- title="result.md",- content=display_text,- filetype="markdown",- )- file_uploaded = True- delivered_as_attachment = True- if self._attachment_id_can_anchor_delivery(context):- scheduled_anchor_message_id = primary_message_id- logger.info("Result delivered as .md file attachment (fallback)")- except Exception as upload_err:- logger.warning("upload_markdown fallback failed: %s", upload_err)+ # Fallback 1: upload the full content as a .md file, by the+ # native markdown upload or by the plain file upload of a+ # platform that has only that one.+ attachment_id = await self._upload_result_document(im_client, target_context, display_text)+ if attachment_id:+ primary_message_id = attachment_id+ file_uploaded = True+ delivered_as_attachment = True+ if self._attachment_id_can_anchor_delivery(context):+ scheduled_anchor_message_id = primary_message_id+ logger.info("Result delivered as .md file attachment (fallback)")# Fallback 2: split into multiple messages.if not file_uploaded:@@ -2777,7 +3037,7 @@ class ConsolidatedMessageDispatcher:im_client,target_context,display_text,- enhanced.buttons if enhanced else [],+ delivery_buttons,parse_mode,subtext=done_footer,)@@ -2801,7 +3061,9 @@ class ConsolidatedMessageDispatcher:# Upload extracted file attachmentsif enhanced and enhanced.files:- await self._upload_file_links(im_client, target_context, enhanced.files)+ await self._upload_file_links(+ im_client, target_context, _written_files(enhanced.files, citations)+ )if scheduled_anchor_message_id and mutates_turn_lifecycle:try:@@ -2838,6 +3100,11 @@ class ConsolidatedMessageDispatcher:# platform. The Web message row separately persists a clean body# plus structured footer below.persisted_result_text = self._fold_footer(persist_text, folded_footer)+ # The agent-run records take text alone, so they take the written+ # copy; ``persisted_result_text`` itself still carries its tokens+ # into ``persist_agent_message``, which finalizes the row it+ # writes and binds the sidecar to it.+ written_result_text = materialize_citations(persisted_result_text, citations)run_provenance = output_semantics.provenance(context)workbench_run_waits_for_persistence = (target_context.platform == "avibe"@@ -2857,12 +3124,12 @@ class ConsolidatedMessageDispatcher:durable_output_exists = Falseactivity_was_delivered = Falsesettlement_output_semantics = output_semantics- settlement_result_text = persisted_result_text+ settlement_result_text = written_result_textif not settlement_waits_for_persistence:self._record_agent_run_terminal_result(context,- persisted_result_text,+ written_result_text,primary_message_id,is_error=is_error,terminal_error=terminal_error,@@ -2898,10 +3165,15 @@ class ConsolidatedMessageDispatcher:target_context,communication_type,avibe_enhanced.text or persist_text,- quick_replies=[b.text for b in avibe_enhanced.buttons] or None,+ quick_replies=[+ b.text+ for b in _written_buttons(avibe_enhanced.buttons, citations)+ ]+ or None,result_footer=folded_footer,metadata=output_metadata,native_message_id=native_output_id,+ citations=citations,)else:persisted_output = persist_agent_message(@@ -2911,6 +3183,7 @@ class ConsolidatedMessageDispatcher:result_footer=folded_footer,metadata=output_metadata,native_message_id=native_output_id,+ citations=citations,)if settlement_waits_for_persistence:@@ -2929,7 +3202,7 @@ class ConsolidatedMessageDispatcher:)settlement_result_text = self._accepted_message_result_text(accepted_message,- persisted_result_text,+ written_result_text,None,)activity_was_delivered = bool(@@ -3043,7 +3316,13 @@ class ConsolidatedMessageDispatcher:# Persist the intermediate log row BEFORE the mute filter so muted# assistant / tool_call messages still land in the store (product# requirement: the process log is complete even when a channel hides it).- persist_agent_message(target_context, canonical_type, persist_text)+ persist_agent_message(target_context, canonical_type, persist_text, citations=citations)++ # The row above is the only copy of an intermediate message that carries a+ # sidecar. Everything left below is what the channel is shown - the concise+ # status line, the consolidated log, the hidden-type log preview - so write+ # the citations into ``text`` once here rather than at each of them.+ text = materialize_citations(text, citations)# Target platform toolcall-delivery gate stays in FRONT of the concise# shortcut: when a turn is routed via ``delivery_override`` to a target@@ -3296,7 +3575,17 @@ class ConsolidatedMessageDispatcher:chunk (a mid-stream footer would read wrong); every chunk is a new send sothe result notifies. Returns the first chunk's id (the delivery anchor)."""- chunks = self._split_result_text_for_context(context, text)+ plan = self._plan_result_split_for_context(context, text)+ if not plan.links_whole:+ # Checked BEFORE the first send: a link that no boundary keeps whole+ # would go out as two halves that both send successfully, so nothing+ # downstream would ever fall back, and the reader would be left with+ # a broken address. Hand the whole result to the caller's fallback+ # instead of reporting a delivery that lost the source.+ logger.warning("Split would break a link unit; abandoning the split before sending")+ return None++ chunks = plan.chunksfirst_message_id: Optional[str] = Nonefor index, chunk in enumerate(chunks):@@ -3337,6 +3626,70 @@ class ConsolidatedMessageDispatcher:return first_message_id+ async def _upload_result_document(+ self,+ im_client,+ context: MessageContext,+ text: str,+ ) -> Optional[str]:+ """Deliver the whole result as a file, by whichever route the client has.++ ``upload_markdown`` is the native route. A platform can inherit it from+ ``BaseIMClient`` without implementing it and still have an ordinary file+ upload - WeChat is exactly that shape - so the same content is written+ to a temporary ``result.md`` and handed to ``upload_file_from_path``,+ the path its own file links already take.++ Returns the delivered id, or ``None`` when nothing was delivered: an+ adapter that reports failure by returning an empty id has uploaded+ nothing, and treating that as an attachment would end the turn claiming+ a result the user never received.+ """+ upload_markdown = getattr(im_client, "upload_markdown", None)+ if callable(upload_markdown):+ try:+ message_id = await upload_markdown(+ context,+ title=_RESULT_DOCUMENT_NAME,+ content=text,+ filetype="markdown",+ )+ if message_id:+ return message_id+ logger.warning("upload_markdown returned no id; nothing was attached")+ return None+ except NotImplementedError:+ logger.debug("IM client inherits upload_markdown unimplemented; trying its file upload")+ except Exception as err:+ logger.warning("upload_markdown fallback failed: %s", err)+ return None++ upload_file_from_path = getattr(im_client, "upload_file_from_path", None)+ if not callable(upload_file_from_path):+ logger.debug("IM client supports no file upload; cannot attach the full result")+ return None++ with tempfile.TemporaryDirectory(prefix="avibe-result-") as staging_dir:+ document = Path(staging_dir) / _RESULT_DOCUMENT_NAME+ try:+ document.write_text(text, encoding="utf-8")+ file_id = await upload_file_from_path(+ context,+ file_path=str(document),+ title=_RESULT_DOCUMENT_NAME,+ )+ except NotImplementedError:+ logger.debug("IM client does not implement file uploads; cannot attach the full result")+ return None+ except Exception as err:+ logger.warning("Result file upload fallback failed: %s", err)+ return None++ if not file_id:+ logger.warning("Result file upload returned no id; nothing was attached")+ return None+ return file_id+async def _upload_file_links(self,im_client,diff --git a/core/message_mirror.py b/core/message_mirror.pyindex 6e6920710..82af776be 100644--- a/core/message_mirror.py+++ b/core/message_mirror.py@@ -29,6 +29,7 @@ from typing import Any, Optionalfrom sqlalchemy.exc import IntegrityError+from core.citations import CitationBundle, finalize_citations, materialize_citationsfrom core.run_settlement import NON_COMPLETING_TURN_SETTLEMENTSfrom core.services import sessions as workbench_sessions_servicefrom modules.im.base import MessageContext@@ -290,6 +291,7 @@ def persist_agent_message(*,quick_replies: Optional[list[str]] = None,result_footer: Optional[str] = None,+ citations: Optional[CitationBundle] = None,metadata: Optional[dict[str, Any]] = None,native_message_id: Optional[str] = None,error_sink: Optional[list] = None,@@ -366,7 +368,9 @@ def persist_agent_message(platform=context.platform,event_type="tool_call",visibility="trace",- text=text,+ # A trace row carries text and no sidecar, so its copy is+ # written here rather than finalized with spans.+ text=materialize_citations(text, citations) or text,content={"kind": canonical_type or "tool_call"},metadata={"canonical_type": canonical_type or "tool_call"},agent_name=agent_name,@@ -395,6 +399,13 @@ def persist_agent_message()except Exception:logger.exception("persist_agent_message: media rewrite failed")+ # The last transform this body will see has now run, so this is+ # where the registered citations become links and the sidecar is+ # measured. Doing it any earlier would bind spans to a body the+ # reader never gets: the media rewrite above changes the text+ # around them, and on IM the delivered copy was truncated or+ # split from a different string altogether.+ text, citation_rows = finalize_citations(text, citations)content: Optional[dict] = {"kind": canonical_type} if canonical_type else None# Quick-reply buttons (avibe result): the trailing ``---\n[label]…``# block was already parsed + stripped upstream; carry the labels in@@ -407,6 +418,12 @@ def persist_agent_message(# transcript can render it beside the timestamp without guessing.if result_footer:content = {**(content or {}), "result_footer": result_footer}+ # Resolved citation sources. ``text`` already carries the links, so+ # every surface reads correctly without this; the structured copy is+ # what lets the Web transcript render compact source badges, and it+ # survives reload because it lives on the row, not in the delivery.+ if citation_rows:+ content = {**(content or {}), "citations": citation_rows}appended_row = _append_quietly(conn,scope_id=scope_id,diff --git a/core/reply_enhancer.py b/core/reply_enhancer.pyindex 4fad4b676..047bddccd 100644--- a/core/reply_enhancer.py+++ b/core/reply_enhancer.py@@ -22,18 +22,24 @@ import osimport refrom bisect import bisect_leftfrom dataclasses import dataclass, field-from typing import List, Tuple+from typing import List, Optional, Tuplefrom urllib.parse import unquote, urlparsefrom markdown_it import MarkdownItfrom markdown_it.common.normalize_url import validateLink as _validate_markdown_link-from markdown_it.common.utils import isStrSpace, unescapeAll+from markdown_it.common.utils import ESCAPE_CHAR, isStrSpace, unescapeAllfrom markdown_it.common.html_re import HTML_TAG_REfrom markdown_it.rules_inline.autolink import AUTOLINK_RE, EMAIL_RE+from markdown_it.rules_inline.autolink import autolink as _commonmark_autolinkfrom markdown_it.rules_inline.backticks import backtick as _commonmark_backtick+from markdown_it.rules_inline.entity import entity as _commonmark_entity+from markdown_it.rules_inline.html_inline import html_inline as _commonmark_html_inlinefrom markdown_it.rules_inline.image import image as _commonmark_imagefrom markdown_it.rules_inline.link import link as _commonmark_linkfrom markdown_it.rules_inline.state_inline import StateInline+from markdown_it.rules_block.reference import reference as _commonmark_reference+from markdown_it.rules_block.state_block import StateBlock+from markdown_it.token import Tokenlogger = logging.getLogger(__name__)# Block consumers use source maps/content/levels, never parsed inline children.@@ -43,6 +49,10 @@ _INLINE_CODE_RANGES_KEY = "avibe_inline_code_ranges"_INLINE_ANGLE_RANGES_KEY = "avibe_inline_angle_ranges"_INLINE_SILENT_RANGES_KEY = "avibe_inline_silent_ranges"_FILE_LINK_CAPTURES_KEY = "avibe_file_link_captures"+_LINK_CAPTURES_KEY = "avibe_link_captures"+_UNIT_CAPTURES_KEY = "avibe_unit_captures"+_DEFINITION_LINES_KEY = "avibe_definition_lines"+_RAW_HTML_CAPTURES_KEY = "avibe_raw_html_captures"def _track_inline_code(state: StateInline, silent: bool) -> bool:@@ -153,6 +163,54 @@ def _capture_file_link_rule(rule, *, is_image: bool):return capture+def _capture_inline_link(rule):+ """Wrap the CommonMark link rule and record the whole unit it accepted.++ A link is one thing to a reader: a label to tap and an address the tap+ goes to. Recording only the address leaves the brackets around it for+ someone else's parser to pair up, so the label and the offset just past+ the closing ``)`` are recorded with it.+ """++ def capture(state: StateInline, silent: bool) -> bool:+ start = state.pos+ matched = rule(state, silent)+ if not matched or silent:+ return matched+ end = state.pos+ label_end = state.md.helpers.parseLinkLabel(state, start, True)+ pos = label_end + 1+ if label_end < 0 or pos >= state.posMax or state.src[pos] != "(":+ # A reference link (``[label][ref]``) spells no destination of its+ # own, and the renderer this answers for does not read one either.+ return matched+ pos += 1+ while pos < state.posMax and (+ isStrSpace(state.src[pos]) or state.src[pos] == "\n"+ ):+ pos += 1+ destination = state.md.helpers.parseLinkDestination(+ state.src,+ pos,+ state.posMax,+ )+ if destination.ok:+ state.env.setdefault(_LINK_CAPTURES_KEY, []).append(+ (+ start,+ end,+ start + 1,+ label_end,+ pos,+ destination.pos,+ destination.str,+ )+ )+ return matched++ return capture++_INLINE_MARKDOWN.inline.ruler.disable(["autolink", "html_inline"])_INLINE_MARKDOWN.inline.ruler.before("backticks",@@ -177,6 +235,122 @@ _FILE_LINK_MARKDOWN.inline.ruler.at(_capture_file_link_rule(_commonmark_image, is_image=True),)+_LINK_MARKDOWN = MarkdownIt("commonmark")+_LINK_MARKDOWN.inline.ruler.at("link", _capture_inline_link(_commonmark_link))+++def _capture_markdown_unit(rule, *, is_image: bool = False, is_autolink: bool = False):+ """Wrap a CommonMark link/image/autolink rule and record what it displays.++ A reader is shown a link's label and an image's alt text; the address, the+ title, and the identifier that names a definition are the spelling that+ puts them there, and nothing a reader can be handed goes in them. Which of+ the two a given span is depends entirely on the form the parser matched, so+ the form is read back from the source the parser just accepted: a ``(``+ after the label is an inline destination, a second non-empty ``[`` is an+ explicit reference identifier, and anything else is a collapsed or shortcut+ reference whose label is *also* its identifier - shown to the reader and+ read back by the parser, so it belongs to both and may not be rewritten.++ The spans are recorded for every unit the parser accepts, nested ones+ included, because an image inside a link is two units and only the outer+ one has an end a following sibling can be written after.+ """++ def capture(state: StateInline, silent: bool) -> bool:+ start = state.pos+ matched = rule(state, silent)+ if not matched or silent:+ return matched+ end = state.pos+ if is_autolink:+ # ``<https://example.com>`` is an address that happens to be shown.+ # There is no label beside it to write anything into.+ captured = (start, end, -1, -1, start, end)+ else:+ label_end = state.md.helpers.parseLinkLabel(+ state, start + (1 if is_image else 0), not is_image+ )+ if label_end < 0:+ return matched+ label_start = start + (2 if is_image else 1)+ after = label_end + 1+ following = state.src[after] if after < state.posMax else ""+ if following == "(":+ captured = (start, end, label_start, label_end, after + 1, end - 1)+ elif following == "[" and end - after > 2:+ captured = (start, end, label_start, label_end, after, end)+ else:+ captured = (start, end, -1, -1, start, end)+ state.env.setdefault(_UNIT_CAPTURES_KEY, []).append(captured)+ return matched++ return capture+++def _capture_raw_html(state: StateInline, silent: bool) -> bool:+ """Run the CommonMark inline HTML rule and record the span it accepted.++ A tag, comment, processing instruction, declaration or CDATA section is+ handed to the renderer as raw HTML, not as Markdown: nothing inside it is+ ever shown to a reader as a link. Text *between* two tags is ordinary+ inline content and is not part of the span.+ """+ start = state.pos+ matched = _commonmark_html_inline(state, silent)+ if matched and not silent:+ state.env.setdefault(_RAW_HTML_CAPTURES_KEY, []).append((start, state.pos))+ return matched+++def _capture_reference_definition(rule):+ """Wrap the CommonMark reference rule and record the lines it swallowed.++ A link reference definition emits no token of its own: the block parser+ reads it, files the address under its identifier, and moves on. The lines+ are still in the source though, and every character of them - identifier,+ address, title - is spelling the reader is never shown.+ """++ def capture(+ state: StateBlock, start_line: int, end_line: int, silent: bool+ ) -> bool:+ matched = rule(state, start_line, end_line, silent)+ if matched and not silent:+ state.env.setdefault(_DEFINITION_LINES_KEY, []).append(+ (start_line, state.line)+ )+ return matched++ return capture+++_UNIT_MARKDOWN = MarkdownIt("commonmark")+# An attachment is written as an ordinary link, and CommonMark's default+# validator refuses ``file:`` outright - so without the reply parser's own+# acceptance rule this scan does not see one at all, and anything placed+# relative to "the enclosing unit" would be placed inside an attachment's label+# instead of after it. That nests a link in a link, which CommonMark does not+# allow: the outer unit stops being a link and its brackets become text the+# reader is shown.+_UNIT_MARKDOWN.validateLink = _validate_file_link_locally+_UNIT_MARKDOWN.inline.ruler.at("link", _capture_markdown_unit(_commonmark_link))+_UNIT_MARKDOWN.inline.ruler.at(+ "image", _capture_markdown_unit(_commonmark_image, is_image=True)+)+_UNIT_MARKDOWN.inline.ruler.at(+ "autolink", _capture_markdown_unit(_commonmark_autolink, is_autolink=True)+)+_UNIT_MARKDOWN.inline.ruler.at("html_inline", _capture_raw_html)++# Definitions are a block-level fact, and the identifiers they file are what+# lets the inline pass tell a reference link from bracketed prose.+_UNIT_BLOCK_MARKDOWN = MarkdownIt("commonmark").disable("inline")+_UNIT_BLOCK_MARKDOWN.validateLink = _validate_file_link_locally+_UNIT_BLOCK_MARKDOWN.block.ruler.at(+ "reference", _capture_reference_definition(_commonmark_reference)+)+# ---------------------------------------------------------------------------# Data classes# ---------------------------------------------------------------------------@@ -481,7 +655,7 @@ def _strip_silent_blocks_with_mask(text: str) -> Tuple[str, str]:if not text:return text, textif "<silent" not in text.lower():- return text, _mask_markdown_code(text)+ return text, mask_markdown_code(text)candidates = _silent_control_candidates(text)ranges, markdown_mask = _silent_control_ranges_and_mask(text, candidates)@@ -723,7 +897,7 @@ def _inline_file_link_captures(def _strip_file_links(text: str) -> str:"""Replace file links outside Markdown code with their labels."""- return _strip_file_links_with_mask(text, _mask_markdown_code(text))[0]+ return _strip_file_links_with_mask(text, mask_markdown_code(text))[0]def _strip_file_links_with_mask(text: str, markdown_mask: str) -> Tuple[str, str]:@@ -917,7 +1091,7 @@ def _extract_secret_requests("""Return ordered, de-duplicated ``$<NAME>`` markers found outside code spans."""if not text or "$<" not in text:return []- masked = markdown_mask if markdown_mask is not None else _mask_markdown_code(text)+ masked = markdown_mask if markdown_mask is not None else mask_markdown_code(text)out: List[SecretRequest] = []seen: set[str] = set()for match in _SECRET_REQUEST_RE.finditer(masked):@@ -928,8 +1102,475 @@ def _extract_secret_requests(return out-def _mask_markdown_code(text: str) -> str:- """Blank Markdown code regions without changing string offsets."""+@dataclass(frozen=True)+class InlineLink:+ """One CommonMark inline link, located in the source that spells it."""++ start: int # source offset of the opening ``[``+ end: int # source offset just past the closing ``)``+ label_start: int # source offset the label text begins at+ label_end: int # source offset just past the label text+ destination_start: int # source offset the destination begins at+ destination_end: int # source offset just past the destination+ destination: str # the destination a Markdown reader resolves+++@dataclass(frozen=True)+class MarkdownUnit:+ """One CommonMark link or image, split into what is shown and what is not.++ ``label_start``/``label_end`` are the span a reader is shown - a link's+ label, an image's alt text - and are ``-1`` when the unit displays nothing+ that is not also its own spelling. ``data_start``/``data_end`` are the span+ that only the parser reads: a destination and its title, an explicit+ reference identifier, an autolink's address, or the dual-use label of a+ collapsed or shortcut reference. They are ``-1`` when the unit spells none.+ """++ start: int # source offset the unit begins at (``[`` or ``!`` or ``<``)+ end: int # source offset just past the unit+ label_start: int+ label_end: int+ data_start: int+ data_end: int+++def _unclaimed_line_ranges(+ text: str,+ inline_ranges: List[Tuple[int, int, str]],+ code_ranges: List[Tuple[int, int]],+) -> List[Tuple[int, int, str]]:+ """Lines no inline-capable block claimed, offered one at a time.++ CommonMark hands some lines to no inline parser at all: a link reference+ definition is swallowed whole by the block parser, and an HTML block is+ passed through as raw source. The Web renderer shows what its own parser+ makes of those, but the IM converters downstream are line scanners that+ read a link wherever the characters appear -- Slack's turned a footnote+ definition's address into a link to a different site -- so a consumer that+ has to protect a link has to be able to see one there.++ Reading those lines individually is exactly the resolution the consumer+ reads them at. Lines a code block owns stay out: a fenced example is not a+ link on any surface, and that is the behaviour it already has. Lines a+ block already claimed stay out too, so a code span written across two of+ them is still one code span -- reading each line alone would find a link+ inside it that no reader is shown.+ """+ claimed = [(start, end) for start, end, _content in inline_ranges]+ claimed.extend(code_ranges)+ unclaimed: List[Tuple[int, int, str]] = []+ line_start = 0+ bounds = [+ (match.start(), match.end()) for match in re.finditer(r"\r\n|\r|\n", text)+ ]+ bounds.append((len(text), len(text)))+ for line_end, next_start in bounds:+ line = text[line_start:line_end]+ if "](" in line and not any(+ start <= line_start and line_end <= end for start, end in claimed+ ):+ unclaimed.append((line_start, line_end, line))+ line_start = next_start+ return unclaimed+++def inline_links(text: str) -> List[InlineLink]:+ """Locate every CommonMark inline link, whole unit included.++ The destination is the one a Markdown reader resolves, so a backslash+ escape or a character reference spelled inside it reads the same here as it+ does in the browser, and a title is left out of it because it is not part+ of the address. The spans are where that link was written, which is what a+ platform pass needs in order to re-spell it in its own dialect. Reference+ links are left out on purpose: they spell no destination of their own.++ The scan covers the blocks CommonMark parses inline plus the lines it hands+ to nobody, so a link written in a footnote definition or an HTML block is+ found -- see ``_unclaimed_line_ranges``. The two overlap nowhere, and what+ comes back is sorted and non-overlapping, so a caller may splice on these+ offsets in one pass.+ """+ if not text or "](" not in text:+ return []+ code_ranges, inline_ranges, _ = _markdown_block_ranges(text)+ links: List[InlineLink] = []+ for source_start, source_end, content in (+ *inline_ranges,+ *_unclaimed_line_ranges(text, inline_ranges, code_ranges),+ ):+ env: dict = {}+ _LINK_MARKDOWN.inline.parse(content, _LINK_MARKDOWN, env, [])+ captures = env.get(_LINK_CAPTURES_KEY)+ if not captures:+ continue+ offsets = _inline_source_offsets(+ text,+ source_start,+ source_end,+ content,+ )+ for capture in captures:+ located = [offsets.get(relative) for relative in capture[:-1]]+ if any(offset is None for offset in located):+ continue+ links.append(InlineLink(*located, capture[-1]))+ links.sort(key=lambda link: link.start)+ spliceable: List[InlineLink] = []+ cursor = 0+ for link in links:+ # Nested links cannot happen in CommonMark, but a capture that mapped+ # back to an overlapping span would splice the text twice; skipping is+ # the one behaviour that always leaves the source readable.+ if link.start < cursor:+ continue+ spliceable.append(link)+ cursor = link.end+ return spliceable+++def _map_unit_capture(+ capture: tuple[int, int, int, int, int, int],+ offsets: dict[int, int],+) -> MarkdownUnit | None:+ """Translate one inline-token unit capture back to the original source."""+ start, end, label_start, label_end, data_start, data_end = capture+ located: List[int] = []+ for span_start, span_end in (+ (start, end),+ (label_start, label_end),+ (data_start, data_end),+ ):+ if span_start < 0 or span_start >= span_end:+ # An empty destination (``[a]()``) names no span to protect, and a+ # unit that shows no label of its own already said so with ``-1``.+ located.extend((-1, -1))+ continue+ if span_start not in offsets or span_end - 1 not in offsets:+ return None+ located.extend((offsets[span_start], offsets[span_end - 1] + 1))+ if located[0] < 0:+ return None+ return MarkdownUnit(*located)+++def _markdown_units(+ text: str,+) -> Tuple[List[MarkdownUnit], List[Tuple[int, int]], List[Tuple[int, int]]]:+ """Every link/image unit CommonMark accepts, its definition lines, raw HTML.++ The units come back sorted outermost-first at each offset, so the first one+ that contains a given span is the one a sibling has to be written after.+ The raw HTML spans are the inline tags and the HTML blocks the parser hands+ through verbatim; neither is Markdown a reader is shown.+ """+ if not text or ("[" not in text and "<" not in text):+ return [], [], []+ code_ranges, inline_ranges, blocking = _markdown_block_ranges(text)+ raw_html = [span for span in blocking if span not in code_ranges]+ block_env: dict = {}+ _UNIT_BLOCK_MARKDOWN.parse(text, block_env)+ references = block_env.get("references") or {}+ line_offsets = [0]+ line_offsets.extend(+ match.end() for match in re.finditer(r"\r\n|\r|\n", text)+ )+ definitions: List[Tuple[int, int]] = []+ for start_line, end_line in block_env.get(_DEFINITION_LINES_KEY, []):+ start = line_offsets[start_line]+ end = (+ line_offsets[end_line] if end_line < len(line_offsets) else len(text)+ )+ definitions.append((start, end))++ units: List[MarkdownUnit] = []+ for source_start, source_end, content in inline_ranges:+ # The identifiers the block pass filed are what lets a reference link+ # resolve here; without them CommonMark reads ``[a][b]`` as prose, and+ # so would this.+ inline_env: dict = {"references": references}+ _UNIT_MARKDOWN.inline.parse(content, _UNIT_MARKDOWN, inline_env, [])+ captures = inline_env.get(_UNIT_CAPTURES_KEY)+ html_captures = inline_env.get(_RAW_HTML_CAPTURES_KEY)+ if not captures and not html_captures:+ continue+ offsets = _inline_source_offsets(text, source_start, source_end, content)+ for html_start, html_end in html_captures or ():+ if html_start in offsets and html_end - 1 in offsets:+ raw_html.append((offsets[html_start], offsets[html_end - 1] + 1))+ for capture in captures or ():+ mapped = _map_unit_capture(capture, offsets)+ if mapped is not None:+ units.append(mapped)+ units.sort(key=lambda unit: (unit.start, -unit.end))+ return units, definitions, raw_html+++def markdown_link_units(text: str) -> List[MarkdownUnit]:+ """Locate every CommonMark link and image, nested ones included."""+ return _markdown_units(text)[0]+++def _merged_ranges(ranges: List[Tuple[int, int]]) -> List[Tuple[int, int]]:+ """Sort and coalesce spans so a masking pass can walk them once."""+ merged: List[Tuple[int, int]] = []+ for start, end in sorted(ranges):+ if merged and start <= merged[-1][1]:+ merged[-1] = (merged[-1][0], max(merged[-1][1], end))+ else:+ merged.append((start, end))+ return merged+++def _reference_identifier_ranges(+ text: str,+ units: List[MarkdownUnit],+ transparent: "re.Pattern[str]",+) -> List[Tuple[int, int]]:+ """Label runs that are a reference identifier once *transparent* text goes.++ A collapsed or shortcut reference names its definition with the very+ characters it shows, so CommonMark reads one run as both - and accepts the+ unit at all only when that run matches an identifier the block pass filed.+ Text the caller replaces before anyone reads the reply is not part of that+ identifier, but the parser cannot know it: ``[Docs <marker>][]`` matches no+ definition, so the run stops being a unit and the marker inside it looks+ like ordinary label text. It is not - writing a citation there spends the+ link the definition was about to give the reader, and leaves the brackets+ beside it in the message.++ Asking the same parser the same question with that text blanked to spaces+ puts the run back in the answer: every offset still lines up, and a+ reference identifier normalises away the blanks exactly as it would have+ ignored the marker. Runs inside a unit the text already parses as are left+ alone - CommonMark has no link inside a link, so a bracket run in a label+ is shown text and nothing more.+ """+ blanked = transparent.sub(+ lambda match: " " * (match.end() - match.start()), text+ )+ ranges: List[Tuple[int, int]] = []+ for candidate in markdown_link_units(blanked):+ dual_use = (+ candidate.label_start < 0+ and candidate.data_start == candidate.start+ and candidate.data_end == candidate.end+ )+ if not dual_use:+ continue+ if any(+ unit.start <= candidate.start and candidate.end <= unit.end+ for unit in units+ ):+ continue+ ranges.append((candidate.start, candidate.end))+ return ranges+++def _markdown_data_ranges(+ text: str, transparent: Optional["re.Pattern[str]"] = None+) -> List[Tuple[int, int]]:+ """The source spans CommonMark reads but never shows anyone."""+ units, definitions, raw_html = _markdown_units(text)+ ranges = [+ (unit.data_start, unit.data_end) for unit in units if unit.data_start >= 0+ ]+ ranges.extend(definitions)+ ranges.extend(raw_html)+ if transparent is not None and transparent.search(text):+ ranges.extend(_reference_identifier_ranges(text, units, transparent))+ return _merged_ranges(ranges)+++def hidden_block_ranges(text: str) -> List[Tuple[int, int]]:+ """The source ranges delivery removes: this reply's ``<silent>`` controls.++ ``mask_hidden_and_code`` answers "would a reader be shown this offset as+ prose"; a consumer that has to count what the reader is actually handed+ needs the hidden half on its own, because a code example survives delivery+ and a silent block does not.+ """+ if not text or "<silent" not in text.lower():+ return []+ ranges, _ = _silent_control_ranges_and_mask(+ text, _silent_control_candidates(text)+ )+ return ranges+++# The characters micromark's ``normalizeUri`` leaves alone. Everything else it+# percent-encodes when it turns a parsed destination into an href.+_URI_SAFE_RE = re.compile(r"[!#$&-;=?-Z_a-z~]")+# ``%`` followed by two ASCII alphanumerics is kept as an existing escape.+_URI_ESCAPE_RE = re.compile(r"%[0-9A-Za-z]{2}")+++def percent_encode(char: str) -> str:+ """Percent-encode one character the way ``encodeURIComponent`` would."""+ if "\ud800" <= char <= "\udfff":+ # A lone surrogate is unrepresentable; the renderer substitutes U+FFFD,+ # so writing anything else would not survive it.+ char = "\ufffd"+ return "".join(f"%{byte:02X}" for byte in char.encode("utf-8", "replace"))+++def spell_uri(value: str) -> str:+ """Spell a destination the way the Web Markdown consumer spells it.++ The rule is micromark's ``normalizeUri``: the Web Markdown component's+ URI-spelling rule for the destination it has already resolved, reused here+ to write the destinations of Slack's resolved inline links. It does not+ speak for every renderer in this product. A character outside the safe set+ becomes its percent escape, and a ``%`` that already starts one is left+ alone so ``%3E`` stays ``%3E`` instead of becoming ``%253E``. A ``%`` that+ starts nothing - ``%b``, or a ``%`` at the end - is data, and is written+ ``%25``. That is not a repair policy chosen here: it is what that consumer+ does, and a destination spelled any other way names a different path.++ Malformed-looking escapes are not repaired, because that rule does not+ read them: ``%zz`` and ``%2g`` are two ASCII alphanumerics after a ``%``+ and survive verbatim, which is what the renderer produces for them too.++ Nothing about this is a claim that two spellings of an address are the+ same page - the destination is simply written the way the consumer writes+ it, so both arrive at whatever that spelling names.+ """+ out: List[str] = []+ index = 0+ while index < len(value):+ char = value[index]+ if char == "%":+ existing = _URI_ESCAPE_RE.match(value, index)+ if existing:+ out.append(existing.group(0))+ index = existing.end()+ continue+ out.append("%25")+ elif _URI_SAFE_RE.fullmatch(char):+ out.append(char)+ else:+ out.append(percent_encode(char))+ index += 1+ return "".join(out)+++def resolve_character_references(text: str) -> str:+ """Resolve CommonMark character references outside code, exactly once.++ The other half of ``unescape_markdown``: a backslash and a character+ reference are the two ways a Markdown author writes a character the reader+ must see rather than syntax the parser must read. An IM dialect resolves+ neither the same way - Slack knows three references and shows the rest+ verbatim - so a label that said ``©`` reached the reader as six+ characters instead of ``©``.++ Which spellings are references, and what each one stands for, are the+ parser's questions, so the parser's own inline rule answers both: the+ rule is run at each candidate offset and the token it pushes carries the+ text a reader is shown. A near-enough pattern paired with a decoding+ utility is a different grammar wearing the same name - the inline rule+ reads at most seven decimal or six hexadecimal digits and shows U+FFFD for+ a code point it recognizes but cannot encode, while ``unescapeAll`` takes+ eight of either and keeps the spelling, so ``�`` and ``&``+ would each come out as the opposite of what the reader sees.++ Resolving is one pass, not a loop: ``&copy;`` is the text ``©`` to+ a Markdown reader, and resolving what that produced would turn it into+ ``©`` - a character the source never wrote. Code is left alone through the+ shared mask, and so is a reference that would reach out of prose into it,+ because a reference is only one where the parser reads prose.++ Call this only where Markdown has actually been interpreted. Text a+ backslash escape protected is NOT a reference: hold it behind a placeholder+ (``hold_markdown_escapes``) and restore it after, so the escaped spelling+ and the real reference stay two different strings.+ """+ if not text or "&" not in text:+ return text+ masked = mask_markdown_code(text)+ # The state carries the source the rule reads and collects the token it+ # pushes; the rule consults no other part of it, so one state is reused+ # across candidates with its output cleared between them.+ tokens: List[Token] = []+ state = StateInline(text, _INLINE_MARKDOWN, {}, tokens)+ parts: List[str] = []+ cursor = 0+ candidate = masked.find("&")+ while candidate != -1:+ tokens.clear()+ state.tokens_meta.clear()+ state.pos = candidate+ state.posMax = len(text)+ resolved = _commonmark_entity(state, False)+ if resolved and masked[candidate : state.pos] == text[candidate : state.pos]:+ parts.append(text[cursor:candidate])+ parts.append(tokens[-1].content)+ cursor = state.pos+ candidate = masked.find("&", cursor)+ else:+ candidate = masked.find("&", candidate + 1)+ if not parts:+ return text+ parts.append(text[cursor:])+ return "".join(parts)+++def unescape_markdown(text: str, *, replace=None) -> str:+ """Resolve CommonMark backslash escapes outside code, autolinks and HTML.++ No IM dialect understands a CommonMark escape, so a ``\\_`` that made the+ Web reader see ``_`` reaches an IM reader with the backslash still showing+ -- and Telegram and Slack go further and read the escaped character as+ markup of their own. Resolving the escapes before a platform formatter+ runs hands each dialect the character the reader was meant to see;+ ``replace`` lets that formatter hide the character from its own pass.+ """+ if not text or "\\" not in text:+ return text+ masked = mask_markdown_code(text)+ terminators = {+ "-->": _substring_positions(masked, "-->"),+ "?>": _substring_positions(masked, "?>"),+ "]]>": _substring_positions(masked, "]]>"),+ ">": _substring_positions(masked, ">"),+ }+ parts: List[str] = []+ cursor = 0+ index = 0+ while index < len(text):+ char = masked[index]+ if char == "\\" and ESCAPE_CHAR.match(masked, index):+ parts.append(text[cursor:index])+ character = text[index + 1]+ parts.append(replace(character) if replace else character)+ index += 2+ cursor = index+ continue+ if char == "<":+ # An escaped ``<`` never opened one of these, and the branch above+ # already consumed it -- so reaching here means a real token start.+ token_end = _autolink_end(masked, index, terminators[">"])+ if token_end is None:+ token_end = _raw_html_end(masked, index, terminators)+ if token_end is not None:+ index = token_end+ continue+ index += 1+ parts.append(text[cursor:])+ return "".join(parts)+++def mask_markdown_code(text: str) -> str:+ """Blank Markdown code regions without changing string offsets.++ The result is the same length as *text*, so a consumer that must act on a+ literal marker only where CommonMark reads it as prose can run its own+ pattern over the mask and apply the offsets to the original source. Two+ features need exactly that: this module's ``$<NAME>`` secret requests and+ ``core.citations``' citation markers. Both have to leave a marker shown+ inside a code example alone, and neither can afford to re-derive fence,+ indented-code, and code-span lexing from patterns of its own.+ """if not any(marker in text for marker in ("`", "~~~", " ", "\t")):return textranges, _ = _markdown_code_ranges(text)@@ -939,6 +1580,62 @@ def _mask_markdown_code(text: str) -> str:return _mask_ranges(text, ranges)+def mask_hidden_and_code(text: str) -> str:+ """Blank code regions *and* silent blocks without changing string offsets.++ ``mask_markdown_code`` answers "would CommonMark read this as code"; a+ consumer that rewrites what the user will actually see needs the other half+ too, because a silent block is removed before delivery. Numbering or+ rewriting a marker inside one would spend a citation index on text nobody+ reads and, worse, lift its URL out of the block that was meant to hide it.+ Masking both here means the marker leaves with its block, untouched.+ """+ if not text:+ return text+ if "<silent" not in text.lower():+ return mask_markdown_code(text)+ candidates = _silent_control_candidates(text)+ ranges, markdown_mask = _silent_control_ranges_and_mask(text, candidates)+ if not ranges:+ return markdown_mask+ return _mask_ranges(markdown_mask, ranges)+++def mask_citation_slots(+ text: str, *, transparent: Optional["re.Pattern[str]"] = None+) -> str:+ """Blank everything a citation cannot be written into, offsets preserved.++ ``mask_hidden_and_code`` answers "will a reader be shown this offset"; a+ consumer that has to *place* something needs the rest of that question+ answered too, because a marker can be shown and still have nowhere to put a+ link. ``[a](https://x/\ue200cite\ue202turn0view0\ue201)`` shows the marker+ text to nobody: it is part of an address, and a link written into it would+ silently send the reader somewhere else. The same is true of a title, of the+ identifier that names a link reference definition, of the definition itself,+ of an autolink's address, which is its own label, and of raw HTML - a+ comment, a tag's attributes, an HTML block - which the renderer passes+ through without reading a link out of it.++ So the two halves compose: code and hidden blocks from the existing mask,+ then the slots the parser reads without showing. Both preserve every source+ offset, so a consumer still matches its own pattern against the mask and+ splices into the original source.++ *transparent* is the caller's own text - its markers, its handles on them -+ which stands in the reply now and will not when a reader sees it. It is+ matched nowhere else; it only stops a marker sitting in a collapsed or+ shortcut reference from hiding the identifier that run also is.+ """+ if not text:+ return text+ masked = mask_hidden_and_code(text)+ ranges = _markdown_data_ranges(text, transparent)+ if not ranges:+ return masked+ return _mask_ranges(masked, ranges)++def _markdown_code_ranges(text: str,) -> Tuple[List[Tuple[int, int]], List[Tuple[int, int]]]:@@ -1464,7 +2161,7 @@ def _extract_buttons(allow_unseparated: bool = True,) -> Tuple[List[QuickReplyButton], str]:"""Extract trailing quick-reply buttons and return ``(buttons, cleaned_text)``."""- mask = markdown_mask if markdown_mask is not None else _mask_markdown_code(text)+ mask = markdown_mask if markdown_mask is not None else mask_markdown_code(text)pattern = _BUTTON_BLOCK_REmasked_match = pattern.search(mask)if masked_match is None and allow_unseparated:diff --git a/docs/plans/2026-09-21-codex-source-citations.md b/docs/plans/2026-09-21-codex-source-citations.mdnew file mode 100644index 000000000..f078e0b32--- /dev/null+++ b/docs/plans/2026-09-21-codex-source-citations.md@@ -0,0 +1,717 @@+# Codex Source Citations++Codex answers a web-searched question with opaque inline markers instead of+links: `U+E200 cite U+E202 <ref_id> [U+E202 <ref_id> …] U+E201`, where each+`ref_id` names one result of a `webSearch` item the same native thread already+reported. Delivered verbatim, a reader sees an unreadable control-character+blob with no way to reach the page the answer is based on.++## Contract++- **Resolution is local.** The sources are the `results[]` Codex already sent.+ Citation handling performs no search, no page retrieval, no favicon or+ metadata fetch, and no additional model call.+- **Scope is the native thread.** A `ref_id` is unique only inside its own+ Codex thread, so harvested results are keyed by thread id, never globally and+ never per Avibe turn — a later turn may cite a search run in an earlier one.+ Ref tokens genuinely repeat across threads (1286 of 2466 distinct tokens in+ the local corpus appear in more than one thread), so a thread reads only its+ own sources.+- **The live stream is a fast path, not the record.** The in-process map is a+ bounded memo; the record is the rollout history Codex itself writes. A+ completed search is recorded under two names, so the reader accepts both: an+ older rollout names the item `WebSearch`, while the shipped codex-cli 0.154.0+ records it as `Extension` carrying `kind: "web.search"` (the live notification+ normalizes both to `webSearch`). Both are in real history — measured locally,+ 1023 old-form rows carrying results against 267 new-form ones — and every row+ the current binary writes is the new one. A thread that reaches this process+ with no live searches — after a restart, a resume (`excludeTurns`), a re-read+ (`includeTurns: False`), or a `thread/fork` that returns only an id — is read+ back from that history, located through Codex's own `threads.rollout_path`+ index rather than a guessed filename. Scope is the rollout *file*: a fork's+ file opens with the parent's rows still carrying the parent's `thread_id`, so+ the fork inherits exactly the history it carries.+ What decides whether to read is the refs the message in hand actually cites,+ never whether the thread has an entry — a partially populated thread is still+ missing exactly the refs a resume or a restart lost. That also bounds the read+ by the message instead of by a cap, so no limit can discard the one old ref+ being resolved, and an eviction costs a re-read rather than the attribution.+ A read only licenses "this ref is absent" when it reached the end of a file+ that did not change underneath it: an unreadable, unindexed, or still-growing+ history is re-read later rather than settling into permanent negative truth.+ Each absence carries the fingerprint that proved it and may suppress a re-read+ of that history only — a read directed at other refs walks the same file+ without learning anything about this one, so one shared "already read" mark+ would let it vouch for an absence it never tested.+- **One shared rewrite, two readings.** `core/citations.py` rewrites each+ marker into an ordinary Markdown link (`[domain](url)`) and returns a+ structured sidecar persisted at `message.content.citations`. Every IM+ platform delivers that Markdown through the renderer it already had — plain+ `domain (url)` where the platform has no hyperlinks, which is what WeChat's own+ formatter says a link looks like — and the Web transcript recognizes the links+ the rewrite wrote and upgrades those to a compact numbered badge with a+ title/domain preview. Neither surface re-parses markers, and the sidecar adds+ no attribution the text does not already carry.+- **A badge belongs to a link the backend wrote.** A badge is an attribution+ claim, so recognizing one is a provenance question rather than a text+ question: an answer may word its own sentence around the same page, and that+ link is the one nobody vouched for. Matching a rendered label back to a+ sidecar row cannot tell the two apart — and neither can the marker grammar,+ once delivery has run. Every stage between the model's text and the reader's+ rewrites that text, and a rewrite can splice two ordinary halves into+ something that reads as a marker. Removing a `<silent>` block turns+ `\ue200ci<silent>x</silent>te\ue202turn0view0\ue201` into a complete marker;+ flattening a `file://` attachment to its label does it too, and two such+ deletions in one reply form a marker in the IM body that the workbench body,+ which keeps the attachment, never forms. A stage that asked the delivered+ text which markers to attribute would answer with both, and would hand the+ one the model never wrote the other one's link.+- **So identity is registered once, where the grammar still means what it+ says.** `register_citations` runs at the native input boundary, over the text+ exactly as the backend produced it. It uses the same offset-preserving+ mask the rewrite always used, widened to cover the Markdown slots a citation+ cannot be shown in (`core/reply_enhancer.mask_citation_slots`), resolves every+ complete marker it may attribute — including the ones that end in the+ unresolved label — and+ replaces each with an opaque per-occurrence token: a 128-bit nonce, never+ derived from the ref or from its order, and checked against the arriving text+ so a token can only ever be one this call minted. The bundle it returns+ carries those registrations and a snapshot of the sources they consulted.+ Delivery then transforms text that contains tokens, and every text-only exit+ **materializes** them — the IM copy (before length limits, truncation and+ splitting), the Turn snapshot, the agent-run record a Harness caller reads+ back, the tool trace, the notify/intermediate/suppressed paths, the SSE+ chunk, and the quick-reply and file labels extracted out of the body, which+ are not Markdown anywhere and get the bare domain instead. A token that never+ reaches an exit is simply not a citation; a token copied within the text still+ stands for the citation it named; text spliced together out of the leftovers+ cannot become one. A token that has landed inside code or a hidden block by+ the time delivery settles gets its original marker text back, rather than a+ link or an exposed internal token. Markers that were already in code or+ hidden blocks are never registered, and a native code literal, a truncated+ marker, and a visible unresolved fallback all keep the behaviour they had.+- **A citation is only written where a link can be shown.** CommonMark has no+ link inside a link, so a source spliced into a label takes the enclosing unit+ down with it: `[Docs MARK](https://openai.com/docs)` rendered as a link+ nested in its own label, and the reader was shown brackets instead of the+ page. A marker written into a destination, a title, a reference identifier or+ an angle autolink's address is worse than that — those characters are what+ makes the unit reach somewhere, and splicing a link through them sends it to+ an address nobody wrote. Raw HTML — an inline tag and its attributes, a+ comment, a processing instruction, a declaration, CDATA, an HTML block — is+ handed through without a link read out of it, so a marker there is not+ registered either: numbering it would spend an index on a badge nobody sees.+ Text between two tags is ordinary prose and still resolves. So the parser decides, on the offset-preserving+ CommonMark units `markdown_link_units` already reports: a marker standing+ where a reader is shown something (a visible inline-link label, an explicit+ full-reference label, an image's alt text) is eligible, and its attribution+ is written immediately **after the outermost enclosing unit** — an image+ inside a link is two units, and only the link has an end a sibling can+ follow — in occurrence order, with a separating space unless one is already+ there. Only the registered token is removed from the label; the destination+ and the label's other text are untouched. A marker in a data or syntax slot+ is left exactly as the model typed it: no token is minted, no source is+ fetched, no sidecar row is emitted, and — because `citation_ref_ids` asks the+ same mask — delivery does not wait for a source it could never show. A+ shortcut or collapsed reference is the dual-use case, both the words a reader+ sees and the identifier that finds the address, so it and its definition are+ preserved whole — including when the marker is what keeps the run from+ resolving. CommonMark cannot match `[docs <marker>]` against `[docs]`, reads+ the brackets as prose, and would hand the run over as ordinary label text;+ the marker is not part of that identifier though, because citation delivery+ replaces it before a reader sees any of this. So the same parser is asked the+ same question a second time with the caller's own text — its markers and the+ tokens standing for them — blanked to spaces, which preserves every offset+ and normalises out of an identifier exactly as the marker would have been+ absent. Runs the text already parses inside a unit are left to the first+ answer: CommonMark has no link inside a link, so a bracket run in a label is+ shown text and nothing more. The decision is re-asked of the final text for+ this bundle's+ own tokens, so a token a transform moved into a label follows the rule above+ and one moved into code, a hidden block or a data slot gets its original+ marker text back. A structured field is not Markdown on any surface: a file+ label or a quick-reply button keeps its plain-domain attribution in place,+ with no link syntax and no token, because there is no unit to stand after.+ The sidecar counts only the links actually written as standalone citation+ links, and binds the body after that relocation.+- **A span is an identity only together with the body it was measured in.** The+ persisted body is finalized the same way, and each sidecar row carries+ `spans` — the exact ranges its links occupy — and `body_sha256`, one digest+ of that exact body, the same value on every row of the sidecar. Ranges alone+ can be impersonated: with `L = [example.com](https://example.com/x)`,+ `L + "\n\n" + L` measures `[0,36]` and `[38,74]`, and deleting the first+ paragraph leaves an ordinary link occupying `[0,36]` exactly. So the consumer+ verifies the digest against the full stored body first and reads the ranges+ only after it matches. Coordinates are UTF-16 code units, half-open. The+ digest is SHA-256 over the body's UTF-16LE code units, hex lowercase —+ `hashlib` on the backend, `@noble/hashes` in the browser (promoted to a direct+ dependency at the version the lockfile already pinned) — and the two agree on+ CJK, non-BMP emoji, CRLF and an unpaired surrogate. It says which version of+ a body a measurement belongs to; it is not a signature and claims nothing+ about where the body came from.+- **The reader verifies the stored body, then maps through the edits it made.**+ The Web transcript is not handed the row: `resultFooterParts` splits off a+ folded or structured result footer, and the Markdown renderer rewrites+ `@<…>` / `#<…>` mentions and `$<NAME>` secret requests into links before+ parsing. Each of those is a replacement with a measured length change, not an+ insertion, so each reports what it changed as a `TextEdit` in its own input+ coordinates and the binding is carried through in the order the passes ran —+ no pass is inferred from the text looking like a prefix of another. An edit+ that reaches into a citation's own link invalidates that citation rather than+ guessing where it moved. A badge is then drawn only where a span equals a+ parsed link node exactly. A missing, malformed or mismatched provenance+ field, an out-of-range or overlapping span, or a span that is not one whole+ link drops the entire new-contract set to plain links; it never falls back to+ matching text. Only a row carrying no provenance fields at all — persisted+ before this existed, and a shipped surface — is still recognized by its+ `(destination, label)` pair.+- **A link is held whole, and the platform spells it again.** A destination is+ data, not prose, and holding only the destination was not enough: the+ brackets left standing in the stream are still something a line scanner pairs+ up, and Slack's converter read `[^f]: [p](https://example.com/x)` as one link+ labelled `^f]: [p`. So the formatter layer holds each whole link and asks the+ platform to re-spell it from `render(label, destination)` — for Slack,+ `<url|label>`. That also repairs `[docs](url "T")`, whose title used to be+ concatenated into the destination and produced a link nobody could open. What+ a consumer scans is what the scan has to cover, so `inline_links` also reads+ the lines CommonMark hands to no inline parser — a link reference or footnote+ definition, an HTML block — one line at a time, which is the resolution those+ consumers read them at; lines a code block or another block already claimed+ stay out, so a code span written across two lines is still one code span and+ not a link found inside it. The spans come back sorted and non-overlapping,+ so a caller splices each exactly once. Code, images, and ordinary+ non-citation formatting keep the behaviour they had.+- **A link stays whole after the hold, too.** Holding the unit is where a+ platform reads it; two stages downstream still rewrite it. A result longer+ than one message is cut at whitespace on Discord and WeChat, and a cut taken+ through `[label](url)` delivers neither half — both chunks send successfully,+ so nothing falls back, and the reader is shown raw Markdown with no way to+ reach the source. The split now asks the same `inline_links` enumeration+ where a link is and chooses a boundary no link straddles; non-link text keeps+ the boundaries it had. Whether a link can be kept is decided against what one+ message actually holds, never against the preferred whitespace: a boundary is+ drawn before the link so it rides the next message whole, or — when the link+ begins the chunk — after it, because a chunk that is nothing but a link that+ fits is still a legal message, and the search then resumes past it so the+ next link is seen too. Only a link longer than a whole message has no+ boundary left. A link scan that fails answers "unknown", not "no links", so a+ plan that could not see a link never certifies one.+ When no boundary can save a link the split is abandoned **before the first+ chunk is sent** and the complete result goes out as the `result.md`+ attachment the fallback already had. WeChat has no `upload_markdown` — it+ inherits the base class's `NotImplementedError` — so that fallback also takes+ the ordinary `upload_file_from_path` route it does implement, staging the+ text in a temporary file that is removed once the upload returns; an adapter+ that reports failure with an empty id has delivered nothing, and the turn+ says so rather than claiming an attachment. The same answer is honoured by+ the plan's other consumer: an intermediate message on a platform that cannot+ edit one (WeChat) takes the unconsolidated log path, and it now takes that+ same whole-document route before sending any fragment, while staying an+ intermediate message — no terminal settlement, no result lifecycle signal, no+ transcript row of its own, and no notice copy.+ And at Slack's own serialization boundary, `&`, `<` and `>` in a label are+ encoded as Slack requires: a bare `>` closes `<url|label>` at that character,+ so the rest of the label and the whole address arrive as plain text. The+ label is finished before the wrapper closes over it, in the order that keeps+ Markdown's two spellings of a literal character apart: character references+ are resolved once after the platform converter ran (so `©` is the `©` a+ reader sees, and `&copy;` is the text `©`), then the escape+ placeholders are restored — what a backslash protected is literal text that+ merely looks like a reference — and only then is every remaining `&`, `<` and+ `>` encoded, unconditionally. So `[a&b]` and `[a\&b]` reach Slack as+ two different strings and each reader sees what CommonMark shows; a reference+ inside a code literal stays written out. A citation of+ `https://a%26amp%3B.example/x` is exactly this case: the host contains+ `&` literally, and the reader must not be told the page is on+ `a&.example`. Which spellings are references, and what each one stands for,+ are the parser's own questions — two thousand names, seven decimal digits or+ six hexadecimal ones, and U+FFFD for a code point it recognizes but cannot+ encode — so the parser's own inline rule is what answers them, run at each+ candidate offset outside code. A pattern that merely resembles the grammar,+ paired with a same-package decoding utility that takes eight digits and keeps+ an invalid spelling, is a second grammar that disagrees with every other+ Markdown surface. And because a reference spells a line break too (` `,+ `
`, `
`, ` `), the label's existing one-line rule — a+ newline inside `<url|label>` is not a link — is applied once more after+ interpretation, where the label's characters are finally known; an escaped+ `\
` is literal text with no break to fold.+ The destination is finished in the same place and for the same reason. It is+ the other half of a wrapper built out of `<`, `|` and `>`, and an address is+ allowed to contain all three: one of them arriving raw ends the unit where+ the author did not, and the reader is shown a truncated link with the rest of+ the address beside it as text. What it owes the reader is the page a tap+ opens, not the bytes that carried it there, so two jobs run in order and stay+ apart. First the address is written the way the Markdown consumer on the+ other side writes it. That rule is `core/reply_enhancer.py`'s `spell_uri`,+ promoted out of `core/citations.py` where it already existed as the+ renderer-compatible half of `_canonical_uri`: `<`, `>` and `|` leave as+ `%3C`, `%3E` and `%7C`, along with spaces, control characters, backslashes,+ backticks, bracket data and a `%` that starts no escape, while `?#&=` and an+ escape the address already spells are left standing. Then Slack's text-object+ escaping runs over the result, which is what a query separator needs:+ `?a=1&b=2` travels as `?a=1&b=2` and is `?a=1&b=2` again to the client,+ where percent-encoding that `&` would have survived every decoder and opened+ a different page.++ Picking a different safe set is not a free choice about wire bytes. A bare+ `%` and a bracket in a path survive URL parsing as distinct spellings, so+ delivering `…/a%b` where the renderer resolves `…/a%25b` is a different path+ on any server we do not control — measured, not assumed, by rendering both+ consumers and comparing `URL.href` over a bounded matrix+ (`tests/fixtures/link_destination_matrix.json`, asserted from both sides).+ The one escape that has to come back is the pair of brackets around an IPv6+ host: they are the authority's syntax rather than data, and nothing opens+ `%5B::1%5D`. But only for a host that was *written* in brackets, and the+ spelled URL can no longer answer that: `[` and `]` are outside the URI-safe+ set, so spelling writes a literal `[::1]` and a provider's own `%5B::1%5D`+ as the same characters. Repairing whatever looks escaped turned+ `https://%5B::1%5D/admin`, which names no host at all, into a live link to+ the loopback interface — and `safe_url`, the Web renderer and Slack all+ agreed on that repair, which is agreement on the bug rather than proof of+ correctness. So the destination is handed over *before* it is spelled:+ `spell_destination` does both halves, qualifying the authority from the+ resolved destination (`_authority_brackets_literal`) and then putting the+ brackets back only there, only when the authority parses, and only when both+ ends of the wrapper were literal — a mixed `[::1%5D` is not a bracketed host+ with an escape in it, it is a host that was never bracketed at all, and+ promoting half of it into syntax invents an address out of the other half.+ The blind form of the call is gone rather than deprecated, so a caller cannot+ silently forget to say which spelling it started from. A malformed-looking+ `%zz` is still left alone, because that is what the consumer's own rule does+ with it rather than a policy chosen here.+- **A backslash escape is resolved before a platform reads it.** An escape says+ one character is not syntax, and no IM dialect knows that. Telegram and Slack+ re-read the escaped character as markup of their own — Slack's converter+ turns an escaped `*` into `_`, so the reader is shown a different character+ than the writer wrote — and WeChat, whose renderer is a sequence of regex+ passes, both shows the backslash and lets an escaped `]` close a link label+ early. So a shared helper in the formatter layer resolves the escapes and+ holds what they protected behind a placeholder until the platform pass is+ done, and each of those three platforms applies it — honouring upstream's+ intent in the dialect that is actually about to parse the text. Discord and+ Feishu are left as they were: their `format_markdown` returns the text+ unchanged, so nothing here re-reads it. A destination is held the same way and+ for a stronger reason: Slack's converter scans the whole line for emphasis,+ including inside the parentheses, and turned `https://a*b*.example/x` into+ `https://a_b_.example/x` — not a formatting difference but a different site.+ A destination is opaque data to a text dialect, so it is held across the+ platform pass and comes back untouched by it — and, on a platform whose own+ wrapper it then has to fit inside, is re-spelled for that wrapper while the+ wrapper is still open, never restored into a finished one. Labels are escaped for+ every character that is active syntax in a dialect the label will be read in:+ the ones that destroy the link around it (a backtick or an angle bracket opens+ a code span, comment, processing instruction, declaration or CDATA section+ that runs past `](url)` and leaves the reader raw Markdown with nothing to+ click, and a bracket or backslash ends the label early), and the ones that+ merely re-read it as markup — `*`, `_`, `~`, `&`, `|`. The second group used+ to be left alone because the citation's identity was read back off its+ rendered text, so escaping it cost the badge; identity is registered before+ any of this runs now, so the label can be protected at no such price. It has to+ be: a host is the attribution, and `a*b*.example` shown as `ab.example` in+ italics names a site the link does not open.+- **Never invent, never silently drop.** A URL is never derived from a `ref_id`+ or from search order. An unknown, malformed, or non-http(s) source degrades to+ a visible localized label (`message.citationUnresolved`); a marker with+ nothing to fall back on keeps its raw text. A history read that fails logs the+ path it failed on and degrades the same way. Old messages stored before this+ change are not backfilled.+- **An intermediate message waits for its own sources, and only for them.** A+ `webSearch` item and the `agentMessage` that cites it are separate+ notifications, so a narration message whose ref has no source recorded yet is+ queued rather than delivered with a label it would never lose. The wait is+ bounded by the turn: the queue drains as soon as the defining search arrives,+ and is force-flushed at the turn's terminal boundary — completion, failure,+ interruption, or a non-retrying `error` — so a ref that never arrives still+ ships with its visible label. Every intermediate emit passes through the same+ queue, so holding one message cannot let the messages behind it change places,+ and a message that cites nothing is never delayed by anything except messages+ already ahead of it. A delivered message is never rewritten afterwards, and a+ superseded turn discards its queue exactly as it discards its result+ candidate.+- **Untrusted input.** Titles and URLs come from search results: a title has+ its controls, private-use characters and the invisible formatting characters+ that could render its own text backwards — the bidi marks, embeddings,+ overrides and isolates — stripped, while the joiners and variation selectors+ a real title needs to spell a family emoji, a Persian word or a text-style+ symbol are kept, and it is collapsed and bounded,+ a URL is cleaned the way a browser cleans one (see below), only http(s) with a+ host is accepted, and link labels are escaped. Whether a marker is+ code is a CommonMark question — fence lengths nest, an unclosed fence runs to+ the end, a code span may cross lines, container indentation shifts all of it —+ so the rewrite reuses the offset-preserving mask already shipped for `$<NAME>`+ secret requests instead of lexing code a second time+ (`core/reply_enhancer.mask_hidden_and_code`, which extends it with `<silent>`+ blocks: those are removed before delivery, so a marker inside one must not+ spend a citation index or lift a URL out of the block that hid it). An answer+ that explains this grammar keeps every byte of its examples.+- **One URL on both sides of the boundary.** The Web transcript matches a+ rendered `href` against the sidecar, so the persisted URL has to be the exact+ string the Markdown renderer produces. The backend therefore writes the+ canonical form up front: character references resolved once (a provider that+ lifted the URL out of HTML sends `&`) using micromark's replacement table+ rather than HTML's — micromark is the parser behind this renderer, and it+ replaces the C1 range and the noncharacters with `U+FFFD` instead of applying+ the Windows-1252 mapping that would make `€` a `€` the rendered link+ never opens. The raw string is cleaned first, exactly the way WHATWG's URL+ parser cleans its own input, because that is what decides the page: an ASCII+ tab or newline is removed wherever it sits, leading and trailing C0 controls+ and spaces are trimmed, and everything else — a C1 control, a zero-width, a+ private-use code point — survives to be percent-encoded rather than deleted.+ Deleting it moved the destination silently+ (`https://example.com/p€q` became `https://example.com/pq`). That order —+ clean the literals, then resolve the references — is the only one that keeps+ both halves of the contract, because the two spellings are not the same+ character to a Markdown parser: a literal tab or newline *ends* a destination,+ so `[x](https://example.com/p<TAB>q)` renders as no link at all, while the+ same character written as `	`, `	` or `
` is part of the+ destination and comes out of the renderer percent-encoded as `%09` / `%0A`.+ Cleaning after resolution would have deleted it and pointed the persisted URL+ at `…/pq`, a page the badge never opens. Then+ percent-encoding that matches `normalizeUri` — existing `%XX` escapes+ preserved, lone surrogates repaired to `U+FFFD`, and the result a fixed point+ of itself —+ and the scheme lowercased, since a renderer that only knows the lowercase+ spelling sees no link at all (Telegram delivered+ `[example.com](HTTPS://Example.com/X)` as raw Markdown). Parentheses are+ part of the address and stay literal in the URL and the sidecar: encoding+ them as `%28`/`%29` protected the link and named a different page. Only the+ Markdown link guards them, as `\(`/`\)` in the destination, which the Web+ renderer, Slack, Telegram and WeChat each resolve back to the character. An+ angle-bracket destination `<url>` says the same in CommonMark, but the real+ Telegram and WeChat paths do not read it as a link, so it is not used. A raw+ backslash left after normalization — literal or spelled `\`/`\` — is+ refused: WHATWG reads it as `/` in an http(s) authority and path and as data+ in a query, so no single spelling of it names the page for every reader+ (`https://trusted.example\@attacker.example/x` opens trusted.example and,+ percent-encoded, was attributed to attacker.example). An already-encoded+ `%5C` is data everywhere and is kept. Beyond that the URL+ is left as it arrived: it names the page its source gave, and `:0080`,+ `Example.COM` and an uncompressed IPv6 address all open the same page, so this+ is not the place to decide two spellings are one. What is judged is only+ whether a browser opens it at all — one authority judgment covering userinfo+ (split at the **last** `@`, never case-folded, it may be a password), a+ bracketed IPv6 literal (WHATWG's own parser, which rejects the zone id+ `ipaddress` accepts), and a port (ASCII digits only, since `int()` reads `٣`+ as 3 where a browser reads no port at all; empty is no port, in range is kept+ as written). The brackets of an IPv6 host are written literally rather than+ percent-encoded, because `%5B` is a destination the URL parser refuses+ outright — and a destination that arrived already spelled that way names no+ host, so it is rejected rather than repaired into one. Valid literal IPv6+ stays supported, userinfo and port included; nothing here is a+ loopback or private-network blacklist, which would reject addresses that are+ perfectly real. The renderer's own href is where that promise was still being+ broken: `mdast-util-to-hast` percent-encodes the brackets on the way out, so+ the anchor for a real IPv6 host rendered correctly and went nowhere. The+ shared `urlTransform` puts them back, in the authority only and only when the+ platform's own URL parser then accepts the result, so the ordinary link+ reaches the same address the badge does — and only for a destination the+ Markdown AST says was written with literal brackets. `remarkLiteralAuthority`+ asks that of `node.url`, or of the definition a reference names, where the+ parsed destination still exists, and marks the node; the mark travels with+ the occurrence rather than with the URL text, so two links in one body that+ spell the same href may disagree about it. It runs on every body, citation or+ not, over inline links, reference links, autolinks and image destinations —+ the component turns an image it will not fetch into a click-through link, so+ an image `src` becomes an `href` too. A generic link a provider wrote encoded+ keeps its invalid spelling and acquires no citation-only policy: preserving+ an invalid address is this renderer's job, minting a valid one is not. A+ badge is a presentation enhancement; it may never be the only thing able to+ navigate. An authority is read only+ where one is written: a browser repairs `https:example.com/x`, but the rule+ that keeps a hostless URL out keeps this out too, and a search result always+ writes the slashes. The label attributes the host a browser would actually reach:+ non-transitional UTS #46 per label (`例え.jp` → `xn--r8jz45g.jp`,+ `faß.de` → `xn--fa-hia.de`, where the standard library's IDNA 2003 codec would+ have said `fass.de` — a different domain than the link opens), with ASCII+ labels passed through as a browser passes them — except an `xn--` label,+ which a browser decodes and refuses the whole URL over when that fails+ (`https://xn--/x`), so one is kept only when it is the exact encoding of an+ accepted label — and a host that cannot be canonicalized rejected rather+ than guessed at. A host whose last label is a+ number goes through WHATWG's IPv4 parser for the same reason, so+ `2130706433`, `0x7f.1`, `127.1` and `017700000001` are all attributed to+ `127.0.0.1` instead of hiding a loopback destination behind the digits that+ spell it, and a form the parser refuses (`256.1.1.1`, `1.2.3.4.5`,+ `example.com.0x1`) names no host and is rejected. A number written wider than+ 32 bits is refused as a host rather than converted, which also keeps CPython's+ 4300-digit bound on decimal `int()` off an untrusted host: `https://9…9/x`+ with five thousand digits names no host instead of raising. A host too long for+ a label is elided from the **left**, keeping the host's tail verbatim — the+ label is always the last 63 characters behind an ellipsis, so it is either the+ host or a suffix of it. `developers.openai.com.<padding>.attacker.example` is+ shown as a tail that ends in `.attacker.example`, never as+ `developers.openai.com…`, which would name a site the link never opens; and+ never as `…co.uk` either, which keeping whole labels only would have produced+ for a 62-character label under `co.uk` — a public suffix every site beneath it+ shares, and the perfect place to hide a long attacker label. Only the leftmost+ piece may be partial, and a partial piece cannot read as a different domain:+ it is short only when whole labels already fill the budget, and those are the+ rightmost ones, so the registrable domain is shown whole. So neither an unsafe+ URL nor an ordinary unrelated link can become a fabricated citation. One fixture,+ `tests/fixtures/citation_url_identity.json`, is asserted by both the Python+ canonicalizer and the real renderer.+- **Surface rules.** The badge is the link itself — hover or focus reveals the+ preview, the first touch reveals instead of navigating, `Enter` opens the+ page, and external links keep `noopener noreferrer nofollow`. Its accessible+ name carries the domain the link opens beside the source's self-reported+ title (`Source 1: OpenAI docs (developers.openai.com)`), so the destination+ is never shown only in the hover preview. Only an+ agent-authored reply may draw a badge; a user bubble or a non-interactive+ preview renders the plain domain link.++## Verification++- `tests/test_citations.py`, `tests/test_codex_citations.py`,+ `tests/test_im_citation_delivery.py`, `tests/test_message_mirror.py` — grammar+ and escaping boundaries (including non-ASCII titles, nested containers, and+ legal fence lengths), thread scoping and eviction, history recovery on the+ real consumption path (fresh handler, cross-turn, fork inheritance, same-name+ ref isolation, both recorded item shapes in one thread, a partially populated+ thread, a bound that must not discard the ref being resolved, an empty or+ absent or unreadable history that later becomes readable, a ref proven+ absent not being rescanned, and an absence not outliving the history that+ proved it), the URL identity table including the IDN mapping cases and the+ authority table (IPv6 literals, ports, userinfo), which links each citation+ wrote and which prose links it did not, the real `WeChatBot.format_markdown`+ boundary rather than its formatter alone, message readiness at+ every terminal boundary, hidden blocks, event ordering and repeated+ completions, IM delivery, persistence and reload.+- `tests/e2e/test_codex_citation_contract.py` — the installed `codex` binary+ relays markers byte-for-byte, and its real notifications drive the product+ chain (event handler → `emit_result_message` → dispatcher → SQLite) to a+ delivered message and a persisted sidecar. Both halves are real bytes,+ including `results[]`: the binary's standalone search tool posts to+ `<provider base_url>/alpha/search`, so the loopback provider answers the+ search itself once `model_providers.<p>.supports_standalone_web_search` and+ the under-development `features.standalone_web_search` are set, and the+ `webSearch` item the test replays is the captured native notification.+- `tests/test_markdown_escape_delivery.py` — the same escaped text through+ Telegram, Slack and WeChat: the reader sees the character and no backslash,+ an escape inside a code span stays a backslash, and an escaped backtick in a+ label no longer takes the link and the code span after it.+- `ui/src/components/ui/markdown.test.tsx` — renderer matching and provenance:+ a word-for-word prose link left alone, a badge on each link one source wrote,+ a body that is not the one the ranges were measured in degrading to plain+ links, a range covering only part of a link matching nothing, the same exact+ spelling inside a code span, a fenced block, a footnote definition, a table+ cell or an image alt moving nothing, an image / autolink / reference link+ spelling the same page never badged, a legacy row still recognized, plus+ hover/focus/touch behavior and accessible naming.+- `tests/fixtures/citation_authority_matrix.json` — one table for the brackets+ an authority may keep, read by three separate surfaces so that none of them is+ the oracle for another: `tests/test_citations.py` asserts `safe_url` of the+ destination (a rejection recorded as `null`, not omitted because constructing+ the URL throws), `ui/src/components/ui/markdown.test.tsx` renders the same+ Markdown through the real component and asserts the anchor's exact `href` and+ then the address a browser resolves from it — or that a browser refuses it —+ and `tests/test_link_unit_delivery.py` decodes the destination back out of the+ real Slack wrapper. Agreement between the three on repairing an encoded+ wrapper would be agreement on the bug, which is why the columns are measured+ rather than derived. The rows cover literal brackets, an upper- and a+ lowercase encoded wrapper, each mixed wrapper, a double-encoded one, userinfo+ and an empty userinfo, an in-range and an out-of-range port, a malformed IPv6+ host, bracket data in path, query and fragment, and the escape and+ character-reference spellings that resolve to literal brackets. One row is two+ links in the SAME body whose hrefs are identical after naive spelling, which is+ what proves the qualification is kept per occurrence rather than per URL.+- `tests/test_citations.py::TestWhereACitationCanBeShown` — the producer half of+ where a citation may land: a source written after the unit for a marker at the+ start, middle and end of a label, in an image's alt text, inside an image+ inside a link, in an explicit full-reference label, and after non-ASCII text+ whose spans are counted in UTF-16; two markers in two labels as two spans of+ one source in reading order; and, byte for byte unchanged with an empty+ sidecar and an empty `citation_ref_ids`, a marker in a destination, a title,+ an angle destination, an angle autolink, an image `src`, a reference+ identifier and its definition, a shortcut and a collapsed reference whose+ definition repeats the marker, and the four forms whose definition does not —+ shortcut and collapsed, link and image — where the marker alone is what stops+ the run resolving. Then+ the tokens a transform moved, judged where they ended up, and the structured+ fields — a file label and a quick-reply button — keeping plain attribution in+ place. `TestRegisteredIdentity` already covered the copied, deleted, spliced+ and moved-into-code tokens and is unchanged.+- `ui/src/lib/citations.test.ts` — the digest and the remap in isolation: the+ UTF-16LE digest against CJK, a non-BMP emoji, CRLF and an unpaired surrogate;+ and `remapCitations` over deletions, insertions and length changes before,+ after and across a span, where crossing one invalidates that citation.+ `ui/src/lib/mentions.test.ts` does the same for the edits the mention pass+ reports. `tests/fixtures/citation_body_digest.json` is the one table both+ ends of the digest are asserted against.+- `tests/test_citation_exits.py` — the exit census. One real producer run is+ driven through the real `ConsolidatedMessageDispatcher` against a temporary+ SQLite home, per surface and per level (delivered result, silent result,+ intermediate, notify, suppressed delivery, Harness terminal, tool trace,+ SSE), and every copy that leaves is checked: no internal token at any exit,+ the attribution present in each, and three private-use literals the product+ must NOT touch — a code-fenced marker, a truncated marker, and an unrelated+ loose `U+F8FF` — surviving verbatim.+- `tests/test_link_unit_delivery.py` — the whole-link hold, on the consumers+ that would swallow it: a footnote definition, a table row, an HTML block,+ ordinary prose and prose wrapped across lines, plus a code span written+ across two lines that must stay one code span. Then what happens to the unit+ after the hold: a label spelling `&`, `<` or `>` — written plainly, written+ as a backslash escape restored after the converter ran, or looking like a+ mention. What Slack shows is then+ asserted against CommonMark itself: each named and numeric reference is+ paired with its escaped spelling (`&`/`\&`, `<`/`\<`,+ `>`, `©`/`\©`, `&`, `©`), a real `markdown-it` render+ says what a reader should see, and Slack's wire form is decoded back and+ required to equal it — with a reference inside a code literal left written+ out and a nested-looking one decoded only once. Two producer-side citations+ of `https://a%26amp%3B.example/x` and `https://a%26lt%3B.example/x` run the+ same assertion end to end, over both the visible label and the destination.+ The oracle is asked for every name the parser knows — the whole entity table+ — and for a bounded numeric matrix: decimal and hexadecimal widths on either+ side of the grammar's limit, code points that are valid, recognized but+ unencodable, or past the last one, and each of those escaped, nested-looking,+ or inside a code literal. A label is then required to be one line however its+ break is spelled: a source LF or CRLF, ` `, `
`, `
`,+ ` `, and `\
` staying literal, with the address left alone.+ The destination has a corpus of its own, run as one batch through the real+ adapter: each wrapper delimiter written raw, backslash-escaped and as a+ character reference; a reference spelling a reference and the escape that+ spells the same five characters; references that produce a control character+ or a space; `%xx` that was already spelled and a `%b` that is not an escape+ at all; a query, a fragment, a bracketed IPv6 authority with userinfo and+ port, a non-ASCII host and path, brackets, a backslash, backticks, and a+ `mailto:` address. Every row asserts the exact wire and, separately, the+ address that wire resolves to — the wrapper is parsed structurally first,+ before a single pass of Slack's three decodings over each field, so a+ permissive decoder cannot make a delimiter that ended the unit early look+ like a working link, and the count of units is part of the assertion. An+ empty label exercises the bare `<url>` branch and two adjacent links the+ composition. The same corpus is rendered by the Web `Markdown` component in+ `ui/src/components/ui/markdown.test.tsx`, whose anchors are the independent+ answer for what address each link names; the two agree on every row except+ the three the adapter deliberately leaves alone.+ `ui/src/components/ui/markdown.test.tsx` asks the real Web renderer the same+ eight reference questions, so the two surfaces are measured against one+ answer rather than against each other's expectations.+ Then the boundary: a fitting link crossing a proposed boundary, a fitting+ link at the head of a chunk, two adjacent links, and a multibyte byte budget,+ each keeping every link whole and every chunk within the platform limit —+ and a link scan made to fail, which must not certify the plan it could not+ read. Then the delivery itself, through the real dispatcher with only the+ network replaced: a text-only client sending a fitting link as ordinary text,+ with no attachment and no delivery-failure notice; Discord attaching the+ complete result rather than fragmenting a link no message can hold; the real+ WeChat adapter carrying it over `upload_file_from_path` with the staging file+ cleaned up; a refused CDN upload reported as no delivery rather than as an+ attachment; and the same three answers on the intermediate log path, driven+ from a registered citation through the real WeChat adapter — in every+ fallback case not one fragment sent before it.+- `ui/src/components/workbench/CitationStoredRows.test.tsx` — the rows+ `tests/citation_bridge.py` wrote through the real dispatcher, read by the+ real `MessageRow` and the real activity card: an IM row whose footer is+ folded into the stored body, the same row without one, a workbench row that+ stores its footer apart, a narration row, and the combination row where the+ backend rewrote an attachment before measuring and the reader then edits the+ body twice more (footer strip, secure-input card) above the citation.+- `tests/citation_bridge.py` → `tests/fixtures/citation_consumer_bridge.json` —+ one recording of what the real producer and the real delivery pass emit for+ each case, so the two ends of the contract are measured against one run+ instead of against each other's assumptions. Nothing in it is hand-authored:+ the body, the sidecar and the links a reader must end up with are derived from+ the run, and regenerating it (`python -m tests.citation_bridge`) is how a+ deliberate change is recorded — including the counterexamples that decide+ where identity comes from: a `<silent>` strip splicing two halves into a+ marker, and a `file://` flattening plus a `<silent>` strip cancelling out so+ that the IM body gains a complete marker the workbench body never forms. Both+ stay literal text, and the sidecar claims the one occurrence the model wrote;+ the test runs a resolve over the delivered body to show what a stage placed+ after delivery would have produced instead. The recording also carries the+ rows the dispatcher stored, read back out of SQLite, which is what the Web+ consumers are handed. `tests/test_citation_consumers.py` asks the+ real Slack, Telegram and WeChat renderers what they show of that recording —+ and Discord and Feishu that they still pass it through — asserting the label+ read and the address reached rather than that a URL appears somewhere; no+ message is sent to any platform. It also drives the real dispatcher with real+ persistence, so the stored row it checks is the one SQLite returns, including+ the `file://` rewrite that runs after delivery and before the sidecar is+ measured. `ui/src/components/ui/citation-bridge.test.tsx`+ asks the real `Markdown` component the same questions, with the sidecar,+ without it, with a stale one and with a pre-provenance one, and again with the+ mention and secret-request rewrites running beside it — those edit the source+ text before Markdown parses it, which is exactly what a positional contract+ can be broken by. Nine cases were added for where a citation may be written:+ a marker in a label and at three positions in one, the data and syntax slots,+ the reference slots including the dual-use shortcut and collapsed forms, an+ image's alt and `src`, an image inside a link, three real `<silent>` strips+ that move a token into a label, a destination and a title, an open fence, and+ a file label beside quick-reply buttons. The recording carries two separate+ questions about its anchors, because a case may answer yes to one and no to+ the other: whether they are every address a reader is given (so a link+ pointing anywhere else is a link nobody wrote), and whether they are also in+ the order shown. An image, an autolink or a reference definition names an+ address the delivery pass's inline-link scan cannot resolve, so those cases+ record the first as false and are asked only which links must appear; every+ case that was recorded before this is unchanged, including the GFM footnote+ that relocates its link.+- `ui/e2e/citations/` — real-browser layout and pointer routing on desktop,+ mobile Chromium, and mobile WebKit, in English and Chinese, including the+ falsifiable form of the no-fetch rule: zero requests to the cited site after+ render and preview, exactly one after an explicit open, and a prose link to a+ cited page that stays an ordinary anchor in a real browser.++## Known-by-design++- **A ref whose search never arrives still degrades to a label.** Holding an+ intermediate message is bounded by its turn, not by a promise about ordering:+ when the turn reaches a terminal boundary with the ref still undefined, the+ message ships with the visible unresolved label. Nothing is re-resolved or+ backfilled after delivery, because rewriting a message after the fact would+ let an old answer change under the reader.+ The corpus measurement that once justified doing nothing here is retained as+ frequency evidence, not as a protocol invariant: across 344 rollout files,+ 8,893 cited refs had a `WebSearch` row in the same file and **0** of them+ appeared after the citing row. Nothing in the transport enforces that, so the+ queue above covers the inversion rather than assuming it away; the history+ read still covers the commoner case where the row exists but this process+ never saw the live event.+- **A final result message is not queued.** `turn/completed` is already the+ boundary the queue would wait for, so the turn's result is resolved and+ delivered there with whatever attribution exists at that moment.+- **A badge is a Web affordance; IM gets the link.** Every IM surface delivers+ the ordinary Markdown link the rewrite wrote, and the sidecar never travels+ with it. The attribution a reader on IM has is the label and the address, and+ that is the whole contract there.+- **The whole-link hold does not re-open Slack's other seams.** Held units are+ inline links; a link written inside a cross-line inline code span, and an+ image, keep exactly the behaviour they had before — including Slack's+ pre-existing mangling of the former. Reading those lines individually would+ find a "link" no reader is shown, which is the worse failure, and fixing+ Slack's inline-code converter is not this change.+- **The stored row's re-measurement is asserted on the backend side.** A+ `file://` attachment is rewritten to a media-proxy URL inside the same+ transaction that writes the row, so the stored body is not the delivered one+ and its spans and digest are measured after that rewrite. The proxy id is+ minted per registration, so the recorded fixture cannot pin it: the recording+ blanks that one id (and the digest that covers it) and the rebinding itself is+ asserted in Python, where the same run can compare the delivered measurement+ against the stored one.+- **An attachment is a link, so the unit parser had to be told so.**+ `markdown-it`'s default `validateLink` rejects `file:` outright, so the+ CommonMark instance that locates units could not see an attachment at all —+ and a citation placed relative to "the enclosing unit" would have been written+ *inside* an attachment's label, which is the defect this whole rule exists to+ prevent. The unit parsers now use `_validate_file_link_locally`, the+ acceptance rule the reply parser already applied to the same links, rather+ than a second policy about schemes.+- **A token moved into code is asserted directly, not through a delivery+ case.** The transforms that actually run between registration and+ materialization — the `<silent>` strip, the `file://` flattening — can move a+ token into a label, a destination or a title, and each of those is a recorded+ bridge case. None of them can wrap a token in a code span or a fence, so that+ branch of the re-evaluation is asserted where it is reachable, by calling+ materialization with the token already there (`TestRegisteredIdentity`). It is+ not claimed as delivery-path coverage.+- **Two product calls are covered by argument rather than by a test.** The+ dispatcher materializes once more inside the duplicate-result short-circuit,+ where `_accepted_message_result_text` only reads the fallback text when the+ accepted message is not a mapping — reachable with no token in it. It is kept+ because removing it would leave a shape that leaks a token the moment that+ branch changes. On the Web side, `resultFooterParts` emits only trailing cuts+ for every input a stored row can have, so the remap through its edits is+ either a no-op or a whole-body invalidation there; the remap itself is+ asserted directly in `ui/src/lib/citations.test.ts`. Neither is claimed as+ census coverage.diff --git a/modules/agents/base.py b/modules/agents/base.pyindex 90f74a04c..87629eaf9 100644--- a/modules/agents/base.py+++ b/modules/agents/base.py@@ -13,6 +13,7 @@ from modules.im import MessageContextfrom modules.im.base import FileAttachmentfrom core.agent_input import AgentInputMetadatafrom core.agent_session_context import resolve_context_agent_session_target+from core.citations import CitationBundlefrom core.message_output import MessageOutput, terminal_turn_outputfrom core.reply_enhancer import strip_silent_blocksfrom core.runtime_activation import RuntimeActivationIdentity@@ -592,7 +593,17 @@ class BaseAgent(ABC):suffix: Optional[str] = None,request: Optional[AgentRequest] = None,output: MessageOutput | None = None,+ citations: Optional[CitationBundle] = None,) -> Optional[str]:+ """Emit a terminal result.++ ``citations`` is the registered source bundle for the answer body (see+ ``core.citations``): the body carries an opaque token where each cited+ marker was, and the bundle is what writes them out at the end of+ delivery. Each surface writes its own copy, because each rewrites the+ text differently on the way there - so the badge sidecar the Web+ transcript renders describes the body that reader actually has.+ """if output is None and request is not None:output = request.outputif output is None:@@ -645,6 +656,12 @@ class BaseAgent(ABC):parse_mode=parse_mode,is_error=is_error,output=output,+ # Forwarded ONLY when there is a sidecar to carry, for the+ # reason ``controller.emit_agent_message`` gives when it+ # forwards the same value: the controller is a substitutable+ # collaborator, and an always-present keyword would change the+ # required signature of every stand-in that has none.+ **({"citations": citations} if citations else {}),)else:# No visible text (show_duration off + empty result/suffix) is still@@ -699,6 +716,7 @@ class BaseAgent(ABC):is_error=is_error,result_footer=result_footer,output=output,+ **({"citations": citations} if citations else {}),)# Remove ack reaction after result is sentdiff --git a/modules/agents/codex/event_handler.py b/modules/agents/codex/event_handler.pyindex ebcaffb8d..a10a2bcf3 100644--- a/modules/agents/codex/event_handler.py+++ b/modules/agents/codex/event_handler.py@@ -10,13 +10,31 @@ import loggingimport osimport reimport tempfile+from collections import OrderedDictfrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Collection, Mappingfrom vibe.i18n import t as i18n_tfrom core.backend_failure import emit_backend_failure+from core.citations import (+ CitationBundle,+ CitationSource,+ citation_ref_ids,+ has_citation_markers,+ register_citations,+ unresolved_refs,+)from core.processing_indicator import STOPPED_REACTION_EMOJIfrom core.reply_enhancer import strip_silent_blocks+from modules.agents.codex.search_history import (+ SAFE_THREAD_ID_RE,+ HistoryFingerprint,+ ThreadSearchState,+ harvest_search_results,+ is_web_search_item,+ read_thread_search_sources,+)+from modules.agents.codex.turn_state import CodexHeldMessage, CodexTurnStateif TYPE_CHECKING:from modules.agents.base import AgentRequest@@ -24,9 +42,16 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)_GENERATED_IMAGE_EXTENSIONS = {".jpeg", ".jpg", ".png", ".webp"}-_SAFE_THREAD_ID_RE = re.compile(r"^[A-Za-z0-9_-]+$")_ATTACHMENT_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(attachment://[^)\s]+\)")_ImageSnapshot = dict[Path, tuple[int, int]]+# Bounds for the citation source cache. Generous enough that an ordinary+# multi-search conversation always resolves, small enough that a long-lived+# process cannot grow it without limit.+_MAX_CITATION_THREADS = 16+_MAX_CITATION_SOURCES_PER_THREAD = 512+# Refs one thread is known not to define. Bounded like the sources it mirrors;+# forgetting one costs a re-scan of that thread's history, never a citation.+_MAX_CITATION_ABSENT_REFS_PER_THREAD = 512class CodexEventHandler:@@ -41,6 +66,14 @@ class CodexEventHandler:self._agent = agentself._image_snapshots_by_turn: dict[str, tuple[str, _ImageSnapshot]] = {}self._pending_image_snapshots_by_session: dict[str, tuple[str, _ImageSnapshot]] = {}+ # Citable web-search results, keyed by Codex thread then ref_id. A ref_id+ # is only unique inside its own native thread, and a later Avibe turn can+ # cite a search run in an earlier one, so the scope is the thread - never+ # the turn, and never one map shared across threads. This is a memo over+ # Codex's recorded history rather than the record itself: a thread absent+ # here is read back from that history on first use, so neither a restart+ # nor an eviction loses attribution.+ self._search_sources_by_thread: OrderedDict[str, ThreadSearchState] = OrderedDict()def snapshot_generated_images(self, thread_id: str, base_session_id: str) -> None:"""Record generated images present before a Codex turn starts."""@@ -147,6 +180,9 @@ class CodexEventHandler:if not turn_state:logger.debug("Ignoring interrupted completion for unknown turn %s", turn_id)return+ # The turn is over, so nothing it narrated is waiting for a source+ # any more; it ships with whatever attribution it has.+ await self._drain_narration(turn_state, force=True)self._clear_generated_image_snapshot(params)cleanup_request = self._agent._turn_registry.claim_indicator_cleanup(turn_id)self._agent._turn_registry.pop_turn(turn_id)@@ -170,6 +206,9 @@ class CodexEventHandler:if not turn_state:logger.info("Ignoring failed completion for unknown turn %s", turn_id)return+ # What the turn narrated before it failed is still what happened, and+ # it belongs ahead of the error rather than lost behind it.+ await self._drain_narration(turn_state, force=True)error_msg = turn_state.terminal_error if turn_state else Nonealready_notified = turn_state.terminal_error_notified if turn_state else Falseerror_was_user_visible = already_notified@@ -209,15 +248,22 @@ class CodexEventHandler:if not turn_state:logger.debug("Ignoring completion for unknown turn %s", turn_id)return+ # A turn whose result is suppressed narrates nothing either; its held+ # messages are discarded with it, exactly as its result candidate is.+ turn_state.pending_narration.clear()self._clear_generated_image_snapshot(params)self._agent._turn_registry.pop_turn(turn_id)logger.debug("Ignoring inactive turn/completed for turn %s", turn_id)self._release_stream_turn(tracked_request.context)return+ await self._drain_narration(turn_state, force=True)await asyncio.to_thread(self._persist_turn_generated_images, params)pending = turn_state.pending_assistant if turn_state else Nonepending_text = pending[0] if pending else None+ pending_text, citations = await self._register_citations(+ pending_text, params, tracked_request+ )result_text = self._append_generated_images(pending_text, params, tracked_request)self._agent._turn_registry.pop_turn(turn_id)if pending and (pending[0] or "").strip():@@ -229,6 +275,7 @@ class CodexEventHandler:started_at=tracked_request.started_at,parse_mode=pending_parse_mode or "markdown",request=tracked_request,+ citations=citations,)else:await self._agent.emit_result_message(@@ -238,6 +285,7 @@ class CodexEventHandler:started_at=tracked_request.started_at,parse_mode="markdown",request=tracked_request,+ citations=citations,)thread_id = self._extract_thread_id(params) or self._agent._session_mgr.get_thread_id(tracked_request.base_session_id@@ -255,6 +303,7 @@ class CodexEventHandler:returnturn_state = self._agent._turn_registry.get_turn(turn_id) if turn_id else None+ thread_id = self._extract_thread_id(params)if item_type == "agentMessage":text = item.get("text", "")@@ -264,11 +313,8 @@ class CodexEventHandler:prev_is_visible = bool(prev and strip_silent_blocks(prev[0]).strip())if text_is_visible and prev_is_visible:prev_text, prev_pm = prev- await self._agent.controller.emit_agent_message(- request.context,- "assistant",- prev_text,- parse_mode=prev_pm or "markdown",+ await self._narrate(+ request, turn_state, thread_id, "assistant", prev_text, prev_pm)# A silent terminal response may settle the turn, but it must# not displace an earlier visible result candidate.@@ -290,12 +336,7 @@ class CodexEventHandler:"output": output[:500] if output else "",},)- await self._agent.controller.emit_agent_message(- request.context,- "toolcall",- toolcall,- parse_mode="markdown",- )+ await self._narrate(request, turn_state, thread_id, "toolcall", toolcall)elif item_type == "fileChange":changes = item.get("changes", [])@@ -309,12 +350,7 @@ class CodexEventHandler:"file_change",{"file": file_path, "type": change_kind},)- await self._agent.controller.emit_agent_message(- request.context,- "toolcall",- toolcall,- parse_mode="markdown",- )+ await self._narrate(request, turn_state, thread_id, "toolcall", toolcall)elif item_type == "reasoning":# Extract from summary array (list of strings) or content array@@ -328,16 +364,240 @@ class CodexEventHandler:parts.append(c)text = "\n".join(parts)if text:- await self._agent.controller.emit_agent_message(- request.context,- "assistant",- f"_🧠 {text}_",- parse_mode="markdown",+ await self._narrate(+ request, turn_state, thread_id, "assistant", f"_🧠 {text}_")+ # One predicate for both readings, so the live path and the recorded one+ # cannot disagree about what a completed search looks like.+ elif is_web_search_item(item):+ self._record_search_sources(params, item)+ # A message held for one of these refs can go out now.+ await self._drain_narration(turn_state)+elif item_type == "imageGeneration":await asyncio.to_thread(self._persist_generated_image, params, item)+ def _record_search_sources(self, params: dict[str, Any], item: dict[str, Any]) -> None:+ """Harvest a completed web search's results so later markers can cite them.++ The live stream is the fast path only. Codex records the same results in+ the thread's rollout file, and ``_sources_for_thread`` reads them back+ when this handler never saw them, so an eviction here costs a re-read+ rather than the attribution.+ """+ thread_id = self._extract_thread_id(params)+ if not thread_id:+ return+ harvested = harvest_search_results(item)+ # A search with nothing citable adds nothing; leave the thread's entry+ # (and its hydration state) exactly as it was.+ if not harvested:+ return+ self._merge_search_sources(thread_id, harvested)++ def _thread_state(self, thread_id: str) -> ThreadSearchState:+ """This thread's record, created on first use and kept most-recently-used."""+ state = self._search_sources_by_thread.get(thread_id)+ if state is None:+ state = ThreadSearchState()+ self._search_sources_by_thread[thread_id] = state+ self._search_sources_by_thread.move_to_end(thread_id)+ while len(self._search_sources_by_thread) > _MAX_CITATION_THREADS:+ self._search_sources_by_thread.popitem(last=False)+ return state++ def _merge_search_sources(+ self,+ thread_id: str,+ harvested: list[CitationSource],+ ) -> ThreadSearchState:+ """Fold sources into a thread's record, newest definition of a ref winning."""+ state = self._thread_state(thread_id)+ for source in harvested:+ state.sources[source.ref_id] = source+ state.sources.move_to_end(source.ref_id)+ # It is defined after all, so the recorded absence is now wrong.+ state.absent.pop(source.ref_id, None)+ while len(state.sources) > _MAX_CITATION_SOURCES_PER_THREAD:+ state.sources.popitem(last=False)+ return state++ @staticmethod+ def _remember_absent(+ state: ThreadSearchState,+ refs: Collection[str],+ proved_at: HistoryFingerprint | None,+ ) -> None:+ """Record refs a complete read of *proved_at* does not define.++ The fingerprint travels with the ref, so the absence can only ever+ suppress a re-read of the very history that proved it.+ """+ if proved_at is None:+ return+ for ref in refs:+ state.absent[ref] = proved_at+ state.absent.move_to_end(ref)+ while len(state.absent) > _MAX_CITATION_ABSENT_REFS_PER_THREAD:+ state.absent.popitem(last=False)++ async def _sources_for_thread(+ self, thread_id: str, wanted: Collection[str]+ ) -> Mapping[str, CitationSource]:+ """The sources that define *wanted* in this thread, history included.++ A thread reaches this handler with no live searches recorded whenever the+ process restarted, the conversation resumed (``excludeTurns``), it was+ re-read (``includeTurns: False``), or it was forked - ``thread/fork``+ hands back an id and nothing else. A thread that *has* live searches can+ still be missing the very refs those cases lost, so what decides whether+ to read the recorded history is the refs this message asks for, never+ whether the thread has an entry.++ The read is skipped only when every missing ref was already looked for in+ exactly one history, and that history is the one on disk now - a ref whose+ absence was never proved, or was proved against an older file, is reason+ enough to walk it again. Its results are returned alongside the cache even+ if the bounded cache immediately evicted one of them, so an eviction costs+ a re-read and never the attribution.+ """+ if not thread_id or not wanted:+ return {}+ state = self._thread_state(thread_id)+ missing = {ref for ref in wanted if ref not in state.sources}+ if not missing:+ return state.sources+ proofs = {state.absent.get(ref) for ref in missing}+ unchanged = proofs.pop() if len(proofs) == 1 else None+ read = await asyncio.to_thread(+ read_thread_search_sources, thread_id, wanted=missing, unchanged=unchanged+ )+ if not read.scanned:+ return state.sources+ self._merge_search_sources(thread_id, list(read.sources.values()))+ # Only a complete read leaves an absence behind it: an unreadable,+ # unindexed, or still-growing history proved nothing, so its refs stay+ # re-readable rather than settling into negative truth.+ if read.complete:+ self._remember_absent(state, missing - read.sources.keys(), read.fingerprint)+ return {**state.sources, **read.sources}++ async def _prepare_citations(+ self,+ text: str | None,+ thread_id: str,+ request: AgentRequest,+ ) -> tuple[str | None, CitationBundle | None, list[str]]:+ """Register one message's markers and say which refs are still undefined.++ The third value is what a delivery boundary needs: a ref no source has+ been recorded for yet may still be on its way, so the message that cites+ it is not finished. A ref whose source is present but unlinkable is not+ in it - that citation has already reached its final form.++ What comes back is the text with each attributable marker replaced by an+ opaque token, plus the bundle that writes those tokens out at the end of+ delivery. This is the last point where the markers mean what they say:+ every stage after it rewrites the text, and two of those rewrites can+ splice a marker into existence (see ``core.citations``).+ """+ if not has_citation_markers(text):+ return text, None, []+ refs = citation_ref_ids(text)+ sources = await self._sources_for_thread(thread_id, refs)+ registered, bundle = register_citations(+ text,+ sources,+ unresolved_label=self._t("message.citationUnresolved", request),+ )+ return registered, bundle, unresolved_refs(refs, sources)++ async def _register_citations(+ self,+ text: str | None,+ params: dict[str, Any],+ request: AgentRequest,+ ) -> tuple[str | None, CitationBundle | None]:+ """Register an outgoing message's citation markers for delivery.++ Registration happens at emit time, not when the text arrives, so a search+ result that lands between the message item and the turn completing is+ still available to the message that cites it.+ """+ registered, bundle, _ = await self._prepare_citations(+ text, self._extract_thread_id(params), request+ )+ return registered, bundle++ async def _narrate(+ self,+ request: AgentRequest,+ turn_state: CodexTurnState | None,+ thread_id: str,+ role: str,+ text: str,+ parse_mode: str | None = "markdown",+ ) -> None:+ """Deliver one intermediate message, or hold it until it can be attributed.++ Every intermediate emit goes through here, whether or not it cites+ anything, because holding one message may not let the messages behind it+ change places: the reader sees the turn's own order or nothing.++ Without a turn there is no boundary that could end a wait, so a message+ outside one is delivered as it arrives - the pre-citation behaviour.+ """+ if turn_state is None:+ resolved, citations, _ = await self._prepare_citations(text, thread_id, request)+ await self._emit_narration(request, role, resolved, parse_mode, citations)+ return+ turn_state.pending_narration.append(+ CodexHeldMessage(role=role, text=text, parse_mode=parse_mode, thread_id=thread_id)+ )+ await self._drain_narration(turn_state)++ async def _emit_narration(+ self,+ request: AgentRequest,+ role: str,+ text: str | None,+ parse_mode: str | None,+ citations: CitationBundle | None,+ ) -> None:+ await self._agent.controller.emit_agent_message(+ request.context,+ role,+ text,+ parse_mode=parse_mode or "markdown",+ # Forwarded only when there is a sidecar, so a message without+ # citations keeps the call it always had.+ **({"citations": citations} if citations else {}),+ )++ async def _drain_narration(+ self, turn_state: CodexTurnState | None, *, force: bool = False+ ) -> None:+ """Deliver held narration in order, stopping at the first one still waiting.++ *force* is the turn's terminal boundary: the wait is bounded by the turn,+ so a ref that never arrives still ships, carrying its visible unresolved+ label instead of holding the message forever.+ """+ if turn_state is None:+ return+ held = turn_state.pending_narration+ request = turn_state.request+ while held:+ entry = held[0]+ resolved, citations, missing = await self._prepare_citations(+ entry.text, entry.thread_id, request+ )+ if missing and not force:+ return+ held.pop(0)+ await self._emit_narration(request, entry.role, resolved, entry.parse_mode, citations)+def _persist_turn_generated_images(self, params: dict[str, Any]) -> None:turn = params.get("turn")if not isinstance(turn, dict):@@ -372,6 +632,9 @@ class CodexEventHandler:logger.info("Ignoring Codex error for unknown turn %s: %s", turn_id, message)return+ # A terminal error ends the turn, so the wait ends with it. A+ # ``willRetry`` error returned above: the turn is still running.+ await self._drain_narration(turn_state, force=True)turn_state.terminal_error = messageif (self._agent._turn_registry.should_emit_terminal_error(turn_id)@@ -544,7 +807,7 @@ class CodexEventHandler:return imagesdef _generated_images_dir(self, thread_id: str) -> Path | None:- if not _SAFE_THREAD_ID_RE.fullmatch(thread_id):+ if not SAFE_THREAD_ID_RE.fullmatch(thread_id):logger.warning("Ignoring unsafe Codex thread id for generated images: %s", thread_id)return Nonecodex_home = Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex")diff --git a/modules/agents/codex/search_history.py b/modules/agents/codex/search_history.pynew file mode 100644index 000000000..caeaa06d3--- /dev/null+++ b/modules/agents/codex/search_history.py@@ -0,0 +1,260 @@+"""Citable web-search results, read from Codex's own recorded thread history.++The live ``item/completed`` notification is the cheap path to a thread's search+results, but it is not a complete one. A restarted process starts with an empty+handler; a resume sends ``excludeTurns: True``; ``thread/read`` sends+``includeTurns: False``; ``thread/fork`` returns only the new thread's id. None+of those replay the searches an earlier answer already cited, so a later answer+in the same conversation would lose its attribution. Codex has already written+the results down - every turn is appended to the thread's rollout file, and its+own ``state_5.sqlite`` index maps a thread id to that file - so recovering them+needs no extra search, page fetch, or model call.++Scope is the rollout *file*, deliberately, and not the ``thread_id`` recorded+inside each row. A forked thread's file opens with the parent's rows copied+verbatim, and those copies keep the *parent's* thread_id (measured on a local+fork: 909 inherited rows under the parent id ahead of 403 of the fork's own).+Reading the file therefore inherits exactly the history the fork carries, while+a thread whose file this is not stays invisible - which is what keeps the refs+apart, because a ref_id is only unique inside one thread (measured in a local+corpus: 1286 of 2466 distinct tokens appear in more than one thread's history).++Every failure here is a degradation, never an invention: an unreadable or+missing file logs what went wrong and yields no sources, which leaves the+citation to fall back to its unresolved label.+"""++from __future__ import annotations++import json+import logging+import re+from collections import OrderedDict+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Collection, Mapping++from core.citations import CitationSource+from modules.agents.native_sessions.codex import CodexNativeSessionProvider++logger = logging.getLogger(__name__)++# A thread id reaches the filesystem and a SQL lookup, so it is validated+# before either. Real ids are UUIDs; this also admits the shorter synthetic+# ids used in tests.+SAFE_THREAD_ID_RE = re.compile(r"^[A-Za-z0-9_-]+$")++# Rollout files reach hundreds of megabytes (397MB in one local sample), so the+# scan streams and parses only the lines that can possibly carry a search. The+# generic ``read_json_lines`` helper reads a whole file into memory and parses+# every row, which is right for the few-KB previews it was written for and+# wrong here.+# A completed web search has been recorded under two names. Codex moved the+# search tool into an extension namespace, so a current rollout (codex-cli+# 0.154.0) writes ``{"type": "Extension", "kind": "web.search"}`` where an older+# one wrote ``{"type": "WebSearch"}``; the live notification normalizes both to+# ``webSearch``. Both are present in real history - measured on the local corpus:+# 1023 old-form rows carrying results against 267 new-form ones - and every row+# the shipped binary writes from now on is the new form, so reading only one name+# loses attribution on one side of the rename or the other.+_WEB_SEARCH_TOKENS = ('"WebSearch"', '"webSearch"', '"web.search"')+_WEB_SEARCH_ITEM_TYPES = frozenset({"websearch"})+_WEB_SEARCH_EXTENSION_KIND = "web.search"+++def is_web_search_item(item: Any) -> bool:+ """True when *item* is one completed web search, in any shape Codex writes."""+ if not isinstance(item, Mapping):+ return False+ if str(item.get("type") or "").lower() in _WEB_SEARCH_ITEM_TYPES:+ return True+ return str(item.get("kind") or "").lower() == _WEB_SEARCH_EXTENSION_KIND+++def harvest_search_results(item: Mapping[str, Any]) -> list[CitationSource]:+ """Read one completed web search's citable results.++ Shared by the live notification and the recorded-history reader so the two+ readings of the same payload cannot drift apart. A result missing either a+ ref_id or a URL is skipped rather than repaired: there is nothing to link+ to, and a URL is never invented from a ref_id or from search order.+ """+ results = item.get("results")+ if not isinstance(results, list):+ return []+ harvested: list[CitationSource] = []+ for result in results:+ if not isinstance(result, dict):+ continue+ ref_id = result.get("ref_id") or result.get("refId")+ url = result.get("url")+ if not isinstance(ref_id, str) or not ref_id:+ continue+ if not isinstance(url, str) or not url:+ continue+ title = result.get("title")+ harvested.append(+ CitationSource(+ ref_id=ref_id,+ title=title if isinstance(title, str) else "",+ url=url,+ )+ )+ return harvested+++def thread_history_path(thread_id: str) -> Path | None:+ """Locate the rollout file Codex recorded for *thread_id*."""+ if not thread_id or not SAFE_THREAD_ID_RE.fullmatch(thread_id):+ logger.debug("Skipping Codex citation history for unusable thread id")+ return None+ try:+ return CodexNativeSessionProvider().rollout_path(thread_id)+ except Exception as exc:+ logger.warning("Could not locate Codex history for thread %s: %s", thread_id, exc)+ return None+++@dataclass(frozen=True)+class HistoryFingerprint:+ """What a thread's rollout file looked like at one moment.++ A rollout file only ever grows, so identity plus length plus mtime is enough+ to tell "the file I already read" from "the file has more in it now". It is+ measured before *and* after a scan: a file Codex appended to mid-read was+ never read whole, and a partial read must not be remembered as a complete+ one.+ """++ path: str+ exists: bool+ size: int+ mtime_ns: int+++@dataclass(frozen=True)+class ThreadSearchRead:+ """One reading of a thread's recorded searches, and how much it proves.++ ``scanned`` says whether the file was walked at all; ``complete`` says the+ walk reached the end of a file that did not change underneath it. Only a+ complete scan licenses a caller to remember that a ref is *absent* - an+ unreadable, unindexed, or still-being-written history has learned nothing,+ and must stay re-readable rather than settle into permanent negative truth.+ """++ sources: dict[str, CitationSource] = field(default_factory=dict)+ fingerprint: HistoryFingerprint | None = None+ scanned: bool = False+ complete: bool = False+++@dataclass+class ThreadSearchState:+ """What one process knows about a single thread's citable sources.++ One record rather than parallel maps, because the facts are one lifecycle:+ the sources seen so far, and the refs a *complete* read proved the recorded+ history does not define - each one carrying the fingerprint of the history+ that proved it. A history that grows invalidates the absences it justified,+ so an absence outliving its evidence would turn "not written down yet" into+ permanent negative truth. The proof is per ref and never shared, because a+ read is directed at the refs one message cites: a later scan looking for+ other refs walks the grown file without learning anything about this one, and+ a single "history I have already read" mark would let it vouch for an absence+ it never tested. Both maps are insertion-ordered so the holder can bound them+ least-recently-used; forgetting an entry costs a re-read, never a citation.+ """++ sources: "OrderedDict[str, CitationSource]" = field(default_factory=OrderedDict)+ absent: "OrderedDict[str, HistoryFingerprint]" = field(default_factory=OrderedDict)+++def _fingerprint(path: Path) -> HistoryFingerprint:+ """Measure *path*, treating an unstattable file as an absent one."""+ try:+ stat = path.stat()+ except OSError:+ return HistoryFingerprint(path=str(path), exists=False, size=-1, mtime_ns=-1)+ return HistoryFingerprint(+ path=str(path), exists=True, size=stat.st_size, mtime_ns=stat.st_mtime_ns+ )+++def read_thread_search_sources(+ thread_id: str,+ *,+ wanted: Collection[str],+ unchanged: HistoryFingerprint | None = None,+) -> ThreadSearchRead:+ """Find the searches that define *wanted* in one thread's recorded history.++ Blocking: callers on an event loop should hand this to a worker thread. The+ read is directed by the refs a message actually cites, which is what bounds+ it - a rollout file reaches hundreds of megabytes and may define thousands of+ refs, but a message cites a handful, so nothing is kept that was not asked+ for and no bound can discard the very ref being resolved. A ref defined more+ than once resolves to its latest definition, the same last-wins rule the live+ cache applies, so a hydrated thread behaves like one that streamed.++ Pass *unchanged* to skip the walk when the file is still exactly the one that+ fingerprint describes; the result then reports ``scanned=False``.+ """+ if not wanted:+ return ThreadSearchRead(complete=True)++ path = thread_history_path(thread_id)+ if path is None:+ logger.debug("No Codex history file recorded for thread %s", thread_id)+ return ThreadSearchRead()++ before = _fingerprint(path)+ if unchanged is not None and unchanged == before:+ # Same file, same length, same mtime: a rescan could only find what the+ # scan behind that fingerprint already found.+ return ThreadSearchRead(fingerprint=before, complete=True)++ wanted_refs = frozenset(wanted)+ found: dict[str, CitationSource] = {}+ readable = True+ try:+ with path.open(encoding="utf-8", errors="replace") as handle:+ for line in handle:+ if not any(token in line for token in _WEB_SEARCH_TOKENS):+ continue+ try:+ row = json.loads(line)+ except ValueError:+ # Codex may still be appending: the last line can be a+ # partial write, and a row we cannot parse is not evidence+ # about the rows around it.+ continue+ if not isinstance(row, dict):+ continue+ payload = row.get("payload")+ item = payload.get("item") if isinstance(payload, dict) else None+ if not isinstance(item, dict):+ continue+ if not is_web_search_item(item):+ continue+ for source in harvest_search_results(item):+ if source.ref_id in wanted_refs:+ found[source.ref_id] = source+ except OSError as exc:+ # Say why the rest is missing; the unread refs degrade to the unresolved+ # label, and the incomplete read below keeps them re-readable.+ readable = False+ logger.warning("Could not read Codex citation history %s: %s", path, exc)++ after = _fingerprint(path)+ complete = readable and after == before+ logger.debug(+ "Recovered %d of %d requested source(s) from Codex history for thread %s (complete=%s)",+ len(found),+ len(wanted_refs),+ thread_id,+ complete,+ )+ return ThreadSearchRead(+ sources=found, fingerprint=after, scanned=True, complete=complete+ )diff --git a/modules/agents/codex/turn_state.py b/modules/agents/codex/turn_state.pyindex a2ae7de5e..eaded1529 100644--- a/modules/agents/codex/turn_state.py+++ b/modules/agents/codex/turn_state.py@@ -2,18 +2,38 @@from __future__ import annotations-from dataclasses import dataclass-from typing import TYPE_CHECKING, Optional, Tuple+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, List, Optional, Tupleif TYPE_CHECKING:from modules.agents.base import AgentRequest+@dataclass+class CodexHeldMessage:+ """One intermediate message a turn narrated but has not delivered yet.++ Held because a ref it cites has no source recorded for it so far, or because+ a message ahead of it is. ``thread_id`` travels with it because a ref_id only+ names a source inside its own Codex thread.+ """++ role: str+ text: str+ parse_mode: Optional[str]+ thread_id: str++@dataclassclass CodexTurnState:turn_id: strrequest: AgentRequestpending_assistant: Optional[Tuple[str, Optional[str]]] = None+ # Narration this turn has produced but not yet delivered, oldest first,+ # because the source that will attribute part of it has not arrived. Held as+ # one queue per turn so the order the turn narrated in is the order the user+ # reads, and so a boundary that ends the turn can settle all of it at once.+ pending_narration: List[CodexHeldMessage] = field(default_factory=list)terminal_error: Optional[str] = Noneterminal_error_notified: bool = Falsevisible_to_user: bool = True@@ -153,6 +173,9 @@ class CodexTurnRegistry:state.visible_to_user = Falsestate.pending_assistant = None+ # A hidden turn says nothing further, so everything it was still holding+ # back goes with it rather than surfacing under its successor.+ state.pending_narration.clear()state.terminal_error = Nonestate.terminal_error_notified = Falseif self._active_turns.get(state.request.base_session_id) == turn_id:diff --git a/modules/agents/native_sessions/codex.py b/modules/agents/native_sessions/codex.pyindex 9ef89c3a1..0401f5af9 100644--- a/modules/agents/native_sessions/codex.py+++ b/modules/agents/native_sessions/codex.py@@ -67,6 +67,34 @@ class CodexNativeSessionProvider(NativeSessionProvider):logger.warning("Failed to list Codex sessions for %s: %s", working_path, exc)return items+ def rollout_path(self, native_session_id: str) -> Path | None:+ """Return the native history file Codex recorded for one thread.++ ``threads.rollout_path`` is Codex's own index of where a thread's+ transcript lives, so a caller that needs to read that history never has+ to guess a filename or walk the sessions tree. Deliberately unfiltered+ by ``cwd``/``archived``: the thread id already identifies the thread,+ and history stays readable after the thread is archived.+ """+ if not native_session_id or not self.db_path.exists():+ return None+ try:+ with self._connect() as conn:+ row = conn.execute(+ "SELECT rollout_path FROM threads WHERE id = ? LIMIT 1",+ (native_session_id,),+ ).fetchone()+ except Exception as exc:+ logger.warning(+ "Failed to read Codex rollout path for %s: %s", native_session_id, exc+ )+ return None+ raw = str((row[0] if row else "") or "").strip()+ if not raw:+ return None+ path = Path(raw).expanduser()+ return path if path.is_file() else None+def hydrate_preview(self, item: NativeResumeSession) -> NativeResumeSession:preview = ""rollout_path_raw = str(item.locator.get("rollout_path") or "").strip()diff --git a/modules/im/formatters/__init__.py b/modules/im/formatters/__init__.pyindex 830cf893e..637d1fdec 100644--- a/modules/im/formatters/__init__.py+++ b/modules/im/formatters/__init__.py@@ -1,5 +1,19 @@-from .base_formatter import BaseMarkdownFormatter-from .slack_formatter import SlackFormatter+# A platform pass needs CommonMark's own semantics as well as its dialect's:+# a character reference is resolved where the reader's characters are decided,+# next to the escapes and the links this package already hands out.+from core.reply_enhancer import resolve_character_references++from .base_formatter import (+ BaseMarkdownFormatter,+ hold_links,+ hold_markdown_escapes,+ restore_held,+)+from .slack_formatter import (+ SlackFormatter,+ encode_slack_delimiters,+ spell_uri_escapes,+)from .discord_formatter import DiscordFormatterfrom .telegram_formatter import TelegramFormatterfrom .feishu_formatter import FeishuFormatter@@ -8,7 +22,13 @@ from .avibe_formatter import AvibeFormatter__all__ = ["BaseMarkdownFormatter",+ "hold_links",+ "hold_markdown_escapes",+ "resolve_character_references",+ "restore_held","SlackFormatter",+ "encode_slack_delimiters",+ "spell_uri_escapes","DiscordFormatter","TelegramFormatter","FeishuFormatter",diff --git a/modules/im/formatters/base_formatter.py b/modules/im/formatters/base_formatter.pyindex d8c2654b6..a0ec30222 100644--- a/modules/im/formatters/base_formatter.py+++ b/modules/im/formatters/base_formatter.py@@ -1,12 +1,113 @@from abc import ABC, abstractmethod-from typing import Optional, List, Tuple, Any, Dict+from typing import Callable, Optional, List, Tuple, Any, Dictimport jsonimport loggingimport re+import uuid++from core.reply_enhancer import inline_links, unescape_markdownlogger = logging.getLogger(__name__)+def hold_markdown_escapes(+ text: str,+ *,+ render: Optional[Callable[[str], str]] = None,+) -> Tuple[str, Dict[str, str]]:+ """Resolve CommonMark backslash escapes behind per-platform placeholders.++ One Markdown text is delivered to every platform, and a backslash escape in+ it means "show this character, do not read it as syntax". No IM dialect+ knows that: Slack and Telegram read the escaped character as markup of+ their own, and WeChat does not forward Markdown at all - its own pass+ converts the text locally, turning+ ``[a\\*b\\*.example](https://a*b*.example/x)`` into+ ``a*b*.example (https://a*b*.example/x)``, so an escaped bracket or star is+ a syntax question for that conversion too. Discord and Feishu were measured+ here only as local pass-through - what their renderers show for a stray+ backslash has not been observed on those platforms. So the escape is+ resolved here, before a platform formatter runs, and the character it+ protected is held behind a placeholder until that formatter is done - which+ is the same reason it was escaped upstream, honoured in the dialect that is+ actually about to parse the text.++ ``render`` is how the platform wants the character to come back: Telegram+ needs it HTML-escaped, the others want it verbatim. Restore with+ ``restore_held`` once the platform pass is finished.+ """+ held: Dict[str, str] = {}++ def hold(character: str) -> str:+ token = _placeholder()+ held[token] = render(character) if render else character+ return token++ return unescape_markdown(text, replace=hold), held+++def hold_links(+ text: str,+ *,+ render: Callable[[str, str], str],+) -> Tuple[str, Dict[str, str]]:+ """Hide every inline link from a platform's text scanner, whole.++ A link is one unit: a label a reader taps and an address the tap goes to.+ A platform converter does not see it that way - it scans a line for+ patterns - and both halves are its to get wrong. Slack's third-party+ mrkdwn converter read ``https://a*b*.example/x`` as emphasis and delivered+ a link to ``https://a_b_.example/x``, a different site; holding only the+ address left the brackets around it in the stream, where the same pass+ paired them with the wrong text and sent+ ``[^f]: [p](https://example.com/x)`` out as one link labelled ``^f]: [p``.++ So the whole unit is held and the platform is asked to spell it again from+ what a Markdown reader actually resolves: ``render(label, destination)``.+ The destination is the resolved one - angle-bracket form, backslash escapes+ and character references already applied, and a title left out, because it+ is not part of the address and no IM dialect has a place to put it. Both+ fields may still hold the placeholders of an escape pass that ran first, so+ ``render`` is where those come back: a character restored after the unit is+ spelled is a character its dialect can no longer encode. Restore with+ ``restore_held`` BEFORE restoring that pass, so the text around the units+ is the only thing left for it to reach.++ ``inline_links`` returns non-overlapping spans in source order, so this+ splices each one exactly once.+ """+ links = inline_links(text)+ if not links:+ return text, {}+ held: Dict[str, str] = {}+ parts: List[str] = []+ cursor = 0+ for link in links:+ token = _placeholder()+ held[token] = render(text[link.label_start : link.label_end], link.destination)+ parts.append(text[cursor : link.start])+ parts.append(token)+ cursor = link.end+ parts.append(text[cursor:])+ return "".join(parts), held+++def _placeholder() -> str:+ """An inert stand-in no platform's formatter reads as markup of its own.++ The shape is shared by every hold pass so one ``restore_held`` call can put+ back whichever of them wrote the token it finds.+ """+ return f"\ue000MD{uuid.uuid4().hex}\ue001"+++def restore_held(text: str, held: Dict[str, str]) -> str:+ """Put back what ``hold_markdown_escapes`` (or a sibling pass) held."""+ for token, replacement in held.items():+ text = text.replace(token, replacement)+ return text++def _truncate_status(text: str, max_len: int) -> str:"""Truncate ``text`` to ``max_len`` chars with a trailing ellipsis.diff --git a/modules/im/formatters/slack_formatter.py b/modules/im/formatters/slack_formatter.pyindex e12ed88c8..4c770a485 100644--- a/modules/im/formatters/slack_formatter.py+++ b/modules/im/formatters/slack_formatter.py@@ -1,6 +1,63 @@+from core.citations import spell_destination+from .base_formatter import BaseMarkdownFormatter+def encode_slack_delimiters(text: str) -> str:+ """Spell ``&``, ``<`` and ``>`` so Slack shows them instead of reading them.++ For text whose Markdown has already been interpreted - a link label, whose+ backslash escapes have been restored and whose character references have+ been resolved, so every one of these three characters left in it is one+ the reader is meant to see. Slack reads all three as markup of its own, and+ inside ``<url|label>`` a bare ``>`` ends the link early, so each is encoded+ here, once and unconditionally.++ Encoding every one of them is what keeps the two spellings apart. A label+ that MEANS ``&`` arrives here as ``&``; a label that SPELLS ``&``+ arrives as those five characters and has to leave as ``&amp;`` to be+ shown as five. Leaving an already-spelled reference alone would deliver one+ string for both, which is the caller's distinction to keep, not this+ function's to guess from the finished text.++ Not the same job as ``escape_special_chars``, which encodes text that was+ never Markdown (a path, a tool name) - the same encoding, reached without+ any Markdown interpretation in between.+ """+ return text.replace("&", "&").replace("<", "<").replace(">", ">")+++def spell_uri_escapes(url: str) -> str:+ """Write a resolved destination the way its Markdown consumer spells it.++ Not an encoder for a URL that was never one, and not a canonicaliser: this+ is the address a Markdown reader already resolved, on its way into a place+ that cannot hold every character. ``spell_uri`` is the rule the renderer on+ the other side of that boundary applies to the same destination, so the two+ write the same address rather than two spellings nobody has checked name+ one page. A ``%3E`` that arrived spelled stays spelled once; a ``%`` that+ starts no escape is data and is written ``%25``, because that is what the+ consumer reads it as.++ What it buys inside ``<url|label>`` falls out of the same rule. ``>`` ends+ the link early and ``|`` starts the label early, so a destination holding+ either arrives truncated with the rest shown as plain text; both are+ outside the safe set and leave as ``%3E`` and ``%7C``. So do a space, a+ backslash, a backtick and every C0 control - including the newline a+ character reference can put in an address, which no single-line wrapper+ survives.++ A bracketed IPv6 authority is the one place an escape has to come back:+ those two brackets are the host's own syntax rather than data, and nothing+ opens ``%5B::1%5D``. Which is why the destination is handed over unspelled+ and ``spell_destination`` does both halves - once spelled, a host written+ ``[::1]`` and a provider's own ``%5B::1%5D`` are the same characters, and+ putting brackets back into the second one would deliver a live link to an+ address the provider never named.+ """+ return spell_destination(url)++class SlackFormatter(BaseMarkdownFormatter):"""Slack mrkdwn formatterdiff --git a/modules/im/formatters/telegram_formatter.py b/modules/im/formatters/telegram_formatter.pyindex 43e912d17..a5f965707 100644--- a/modules/im/formatters/telegram_formatter.py+++ b/modules/im/formatters/telegram_formatter.py@@ -11,7 +11,11 @@ import htmlimport reimport uuid-from .base_formatter import BaseMarkdownFormatter+from .base_formatter import (+ BaseMarkdownFormatter,+ hold_markdown_escapes,+ restore_held,+)class TelegramFormatter(BaseMarkdownFormatter):@@ -80,7 +84,12 @@ class TelegramFormatter(BaseMarkdownFormatter):continueurl_start = label_end + 2- if not text.startswith(("http://", "https://"), url_start):+ # A scheme is case-insensitive, and a link whose scheme is spelled+ # ``HTTPS://`` is still a link: missing it here left the raw+ # Markdown in the message instead of a hyperlink.+ if not text[url_start : url_start + 8].lower().startswith(+ ("http://", "https://")+ ):rendered_parts.append(self._apply_inline_formatting(text[cursor : start + 1]))cursor = start + 1continue@@ -154,12 +163,23 @@ class TelegramFormatter(BaseMarkdownFormatter):def render_inline_code(match: re.Match[str]) -> str:return f"<code>{html.escape(match.group(1))}</code>"- rendered = self._CODE_BLOCK_RE.sub(lambda m: stash(render_code_block(m)), text)+ # First, because every pass below reads the text as markup and a+ # backslash escape exists to say one character is not: an escaped+ # bracket would otherwise end a link label here and leak raw Markdown,+ # and an escaped ``*`` or backtick would be read as emphasis or code+ # that the writer had explicitly turned off.+ rendered, escaped = hold_markdown_escapes(+ text,+ render=lambda character: stash(html.escape(character)),+ )++ rendered = self._CODE_BLOCK_RE.sub(lambda m: stash(render_code_block(m)), rendered)rendered = self._INLINE_CODE_RE.sub(lambda m: stash(render_inline_code(m)), rendered)rendered = html.escape(rendered)rendered = self._render_links(rendered)+ rendered = restore_held(rendered, escaped)for token, replacement in placeholders.items():rendered = rendered.replace(token, replacement)return rendereddiff --git a/modules/im/slack.py b/modules/im/slack.pyindex e27fb4f6e..640009c4e 100644--- a/modules/im/slack.py+++ b/modules/im/slack.py@@ -5,7 +5,7 @@ import loggingimport reimport timeimport aiohttp-from typing import Dict, Any, Optional, Callable, List, Tuple+from typing import Dict, Any, Callable, List, Mapping, Optional, Tuplefrom slack_sdk.web.async_client import AsyncWebClientfrom slack_sdk.socket_mode.aiohttp import SocketModeClientfrom slack_sdk.socket_mode.request import SocketModeRequest@@ -29,7 +29,15 @@ from .message_facts import (from .download_target import open_download_targetfrom config.v2_config import SlackConfigfrom core.auth import AuthResult-from .formatters import SlackFormatter+from .formatters import (+ SlackFormatter,+ encode_slack_delimiters,+ hold_links,+ hold_markdown_escapes,+ resolve_character_references,+ restore_held,+ spell_uri_escapes,+)from .slack_modal import parse_routing_modal_selectionfrom vibe.i18n import get_supported_languages, t as i18n_tfrom vibe.proxy import resolve_proxy@@ -53,6 +61,9 @@ _SLACK_SECTION_TEXT_LIMIT = 3000_SLACK_MARKDOWN_TEXT_LIMIT = 12000_BARE_HTTP_URL_RE = re.compile(r"https?://[^\s<>\|]+")_TRAILING_URL_PUNCTUATION = ".,!?;:"+# A line break inside a link label: one space to a Markdown reader, and the+# only way a two-line label can reach Slack as one link.+_SOFT_LINE_BREAK_RE = re.compile(r"[ \t]*(?:\r\n|\r|\n)[ \t]*")_EVENT_TASK_SHUTDOWN_DRAIN_TIMEOUT_SECONDS = 70.0@@ -374,6 +385,61 @@ class SlackBot(BaseIMClient):pattern = rf"^\s*<@{re.escape(user_id)}>\s*"return re.sub(pattern, "", text, count=1).strip()+ def _render_slack_link(self, label: str, destination: str, escaped: Mapping[str, str]) -> str:+ """Spell one Markdown link the way Slack reads one.++ ``<destination|label>`` is a single unit, so whatever formatting the+ label carried has to be mrkdwn by the time it lands inside: the+ converter formats a label in place, and it no longer sees this one.+ A label written across two lines is one line to a Markdown reader and+ has to be one here too, because a newline inside the angle brackets is+ not a link on Slack. A link spelling no label at all is sent bare+ rather than with an empty one.++ BOTH halves are finished here rather than left half-built, and for the+ same reason. Slack reads ``&``, ``<`` and ``>`` as markup of its own -+ a bare ``>`` ends the link at that character, so the rest of the link+ is shown as plain text - and the characters a backslash escape+ protected only come back when ``escaped`` is restored. Restoring them+ inside the wrapper's fields, before those fields are spelled for Slack,+ is the difference between the reader seeing ``a > b`` and seeing the+ link fall apart: restoring them after the wrapper is built puts a raw+ delimiter inside it with nothing left to encode it. Finishing only the+ label left exactly that hole on the other side, where the address is.++ A destination is not text, though, so it is not finished the way a+ label is. Its characters were never markup to interpret; what it needs+ is to be spelled as a URI may spell it, which is where ``>``, ``|`` and+ a control character go, and to have Slack's own three read as+ characters rather than markup. Those are two jobs and stay two: the+ escape keeps the address the reader reaches, and the encoding keeps+ Slack from reading it. A query's ``&`` survives both - the first leaves+ a separator alone, the second spells it ``&``, and one pass of+ Slack's decoding hands the reader back the address that was resolved.+ """+ label = self.markdown_converter.convert(_SOFT_LINE_BREAK_RE.sub(" ", label))+ # Markdown's own two spellings of a literal character, resolved in the+ # order that keeps them apart. A character reference is resolved after+ # the converter ran - before it, ``<em>`` would be handed to the+ # converter as a tag it reads rather than as the text it is - and only+ # then are the escaped characters restored, because what a backslash+ # protected is literal text that merely looks like a reference.+ label = resolve_character_references(label)+ label = restore_held(label, escaped)+ # `` `` and ``
`` spell a line break, so the label is only+ # known to be one line once every spelling of its characters has been+ # read. The same policy runs again over the finished text - an escaped+ # ``\
`` stayed literal and has no break to fold.+ label = _SOFT_LINE_BREAK_RE.sub(" ", label)+ label = encode_slack_delimiters(label)+ # The address, restored before the wrapper closes over it rather than+ # after. What a backslash protected is a character of the address, so+ # it is put back as it stands and not read again - the destination a+ # Markdown reader resolved already interpreted its references.+ destination = restore_held(destination, escaped)+ destination = encode_slack_delimiters(spell_uri_escapes(destination))+ return f"<{destination}|{label}>" if label else f"<{destination}>"+def _convert_markdown_to_slack_mrkdwn(self, text: str) -> str:"""Convert standard markdown to Slack mrkdwn format using third-party library@@ -387,9 +453,30 @@ class SlackBot(BaseIMClient):- Headers, lists, quotes, and more"""try:- # Use the third-party converter for comprehensive markdown to mrkdwn conversion- converted_text = self.markdown_converter.convert(text)- return converted_text+ # A CommonMark backslash escape means the character after it is not+ # syntax, and the converter does not know that: it reads an escaped+ # ``*`` as emphasis and emits ``_`` for it, changing the characters+ # the reader sees. Resolving the escapes first and holding what they+ # protected keeps the converter off exactly the text that was marked+ # as literal.+ held_text, escaped = hold_markdown_escapes(text)+ # A link is not text to format either, and the converter scanned+ # both halves of one: ``https://a*b*.example/x`` came back as+ # ``https://a_b_.example/x``, a link to a different site, and a+ # bracket it paired with the wrong text sent a footnote definition+ # out labelled ``^f]: [p``. Slack spells a link itself below.+ held_text, links = hold_links(+ held_text,+ render=lambda label, destination: self._render_slack_link(label, destination, escaped),+ )+ converted_text = self.markdown_converter.convert(held_text)+ # Links first, and by then a rendered link owes this pass nothing:+ # both of its fields resolved their own escapes while the wrapper+ # was open, so the last restoration reaches only the text around+ # them. A placeholder left inside a wrapper would come back as a+ # raw character with nothing able to encode it any more.+ converted_text = restore_held(converted_text, links)+ return restore_held(converted_text, escaped)except Exception as e:logger.warning(f"Error converting markdown to mrkdwn: {e}, using original text")# Fallback to original text if conversion failsdiff --git a/modules/im/wechat.py b/modules/im/wechat.pyindex 52c8ecce6..3ece9dd06 100644--- a/modules/im/wechat.py+++ b/modules/im/wechat.py@@ -38,6 +38,10 @@ from modules.im.message_facts import (is_original_human_wechat_text,wechat_message_kind,)+from modules.im.formatters.base_formatter import (+ hold_markdown_escapes,+ restore_held,+)from modules.im.formatters.wechat_formatter import WeChatFormatterlogger = logging.getLogger(__name__)@@ -146,7 +150,7 @@ _MD_ITALIC_UNDER = re.compile(r"_(.+?)_")_MD_STRIKETHROUGH = re.compile(r"~~(.+?)~~")_MD_INLINE_CODE = re.compile(r"`([^`]+)`")_MD_CODE_FENCE = re.compile(r"```[\w]*\n?(.*?)```", re.DOTALL)-_MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)")+_MD_LINK = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")_MD_HEADING = re.compile(r"^#{1,6}\s+", re.MULTILINE)_MD_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]+\)")_MD_HR = re.compile(r"^---+$", re.MULTILINE)@@ -493,14 +497,54 @@ class WeChatBot(BaseIMClient):"""WeChat DMs have no thread concept."""return False+ def _plain_link(self, label: str, destination: str) -> str:+ """One markdown link as the plain text this platform can show.++ The wording comes from ``WeChatFormatter`` rather than a second opinion+ here, so the adapter and the formatter cannot disagree about what a link+ looks like on a platform that has none.+ """+ label = label.strip()+ destination = destination.strip()+ if destination.startswith("<") and destination.endswith(">"):+ destination = destination[1:-1].strip()+ elif destination:+ # A markdown destination cannot hold unescaped whitespace, so+ # anything past it is the optional title, which is not an address.+ destination = destination.split(maxsplit=1)[0]+ if not destination or destination == label:+ return label or destination+ return self.formatter.format_link(label, destination)+def format_markdown(self, text: str) -> str:- """Strip markdown formatting for WeChat plain text rendering."""+ """Strip markdown formatting for WeChat plain text rendering.++ WeChat personal chat has no hyperlinks, so a link survives only as+ readable text - and an answer whose source the reader cannot open has+ lost its attribution, which is exactly what a cited answer is for. The+ address is therefore kept, and held behind a placeholder for the rest of+ the pass the way ``TelegramFormatter.render`` holds its code: a URL is+ full of characters the emphasis strippers match - ``_``, ``*``, a+ backtick - and rewriting a destination is worse than not showing one.+ """if not text:return text- # Order matters: code fences before inline code, bold before italic- result = _MD_CODE_FENCE.sub(r"\1", text)+ # Order matters: escapes before everything (a backslash escape says the+ # character it protects is not syntax, and every pass below is a+ # syntax pass - an escaped ``]`` would end a link label early and leak+ # the raw Markdown), then code fences before inline code, bold before+ # italic.+ result, escaped = hold_markdown_escapes(text)+ result = _MD_CODE_FENCE.sub(r"\1", result)result = _MD_IMAGE.sub(r"\1", result)- result = _MD_LINK.sub(r"\1", result)+ held: Dict[str, str] = {}++ def hold_link(match: "re.Match[str]") -> str:+ token = f"\ue000WX{uuid.uuid4().hex}\ue001"+ held[token] = self._plain_link(match.group(1), match.group(2))+ return token++ result = _MD_LINK.sub(hold_link, result)result = _MD_BOLD.sub(r"\1", result)result = _MD_STRIKETHROUGH.sub(r"\1", result)result = _MD_ITALIC_STAR.sub(r"\1", result)@@ -508,7 +552,9 @@ class WeChatBot(BaseIMClient):result = _MD_INLINE_CODE.sub(r"\1", result)result = _MD_HEADING.sub("", result)result = _MD_HR.sub("---", result)- return result.strip()+ result = result.strip()+ result = restore_held(result, held)+ return restore_held(result, escaped)# ------------------------------------------------------------------# Callback registrationdiff --git a/pyproject.toml b/pyproject.tomlindex c98fa8d70..1d399927e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -51,6 +51,9 @@ dependencies = ["aiohttp>=3.8.0","ijson>=3.5.1","markdown-it-py>=4.2.0",+ # citation labels need non-transitional UTS #46; the standard library's+ # "idna" codec is IDNA 2003 and folds a different domain out of some hosts+ "idna>=3.4","markdown_to_mrkdwn>=0.2.0","typing_extensions>=4.12.2","PyYAML>=6.0",diff --git a/storage/agent_activity_service.py b/storage/agent_activity_service.pyindex e0346e0d2..645a3e201 100644--- a/storage/agent_activity_service.py+++ b/storage/agent_activity_service.py@@ -329,6 +329,13 @@ def _timeline(conn, session_id: str, *, include_text: bool) -> list[dict[str, An),"row_kind": "assistant","text": msg.get("text") if include_text else None,+ # An intermediate assistant message is rendered through Activity,+ # not ``MessageRow``, so its citation sidecar has to travel with+ # the row or the badge the transcript would draw degrades to the+ # plain domain link here (see ``core.citations``).+ "citations": (+ (msg.get("content") or {}).get("citations") if include_text else None+ ),# The silent marker is a terminal that is INVISIBLE in the transcript,# so a group closing on it must anchor to the (visible) turn trigger# rather than the marker itself; ``terminal_status`` is resolved here so@@ -472,6 +479,13 @@ def _make_group("created_at": item["created_at"],# Migrated event ids are hashes; only storage has the original clock."order_micros": item["sort"],+ # Omitted entirely when there is nothing to carry, so an ordinary+ # row keeps the wire shape it always had.+ **(+ {"citations": item["citations"]}+ if isinstance(item.get("citations"), list) and item["citations"]+ else {}+ ),}for item in pending]diff --git a/tests/citation_bridge.py b/tests/citation_bridge.pynew file mode 100644index 000000000..c5464b0a0--- /dev/null+++ b/tests/citation_bridge.py@@ -0,0 +1,802 @@+"""One producer run every citation consumer is measured against.++A citation is written by ``core.citations`` in Python and recognized again by+``ui/src/lib/citations.ts`` in TypeScript, over two different Markdown parsers,+after the delivery pass in ``core.reply_enhancer`` has had its turn. Each of+those three boundaries is a place the two ends can quietly stop agreeing, and a+test that hand-writes what the backend "would have" emitted cannot see it+happen: it measures the expectation, not the product.++So this module runs the real producer once and writes what it actually emitted -+the delivered text, the sidecar, and the text each surface is really handed -+into ``tests/fixtures/citation_consumer_bridge.json``. Both consumer suites read+that one file: ``tests/test_citation_consumers.py`` drives the real Slack,+Telegram, WeChat, Discord and Feishu renderers with it, and+``ui/src/components/ui/citation-bridge.test.tsx`` drives the real ``Markdown``+component with it. When the producer changes, one regeneration moves both.++The expectations travel with it, and they are derived rather than authored: the+anchors a case should end up showing are read back out of the delivered text+with the same ``inline_links`` the delivery pass itself uses, and each one is+marked cited by the provenance rule the sidecar states. A consumer test then+asks a real renderer whether the reader sees exactly those labels pointing at+exactly those addresses - which is a question about the renderer, because the+answer it is checked against came from the producer, not from this file.++Regenerate with::++ .venv/bin/python -m tests.citation_bridge+"""++from __future__ import annotations++import asyncio+import json+import os+import re+import sys+import tempfile+from pathlib import Path+from typing import Any, Iterable, Mapping+from unittest import mock++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from sqlalchemy import select # noqa: E402++from core.citations import ( # noqa: E402+ _END,+ _SEP,+ _START,+ CitationSource,+ body_digest,+ finalize_citations,+ materialize_citations,+ register_citations,+ resolve_citations,+)+from core.message_dispatcher import ConsolidatedMessageDispatcher # noqa: E402+from core.reply_enhancer import ( # noqa: E402+ inline_links,+ markdown_link_units,+ process_reply,+ unescape_markdown,+)+from modules.im import MessageContext # noqa: E402+from storage.db import create_sqlite_engine, dispose_cached_sqlite_engines # noqa: E402+from storage.importer import ensure_sqlite_state # noqa: E402+from storage.models import agent_sessions, messages # noqa: E402+from storage.settings_service import upsert_scope # noqa: E402+from tests.test_message_dispatcher_platform_limits import _StubController # noqa: E402++FIXTURE = Path(__file__).resolve().parent / "fixtures" / "citation_consumer_bridge.json"++# What an unresolvable marker degrades to. The real backend passes a localized+# string; the exact wording is not what any of this is measuring.+UNRESOLVED_LABEL = "(source unavailable)"+++def marker(*refs: str) -> str:+ """One citation marker naming *refs*, spelled the way Codex spells it."""+ return f"{_START}cite{_SEP}{_SEP.join(refs)}{_END}"+++def source(ref: str, url: str, title: str) -> CitationSource:+ return CitationSource(ref_id=ref, url=url, title=title)+++# Each case is one agent reply. ``text`` is what the model wrote, before any+# Avibe pass touched it; ``{m0}``/``{m1}`` are citation markers and ``{link0}``+# is the complete Markdown link the producer will write for the first source -+# substituted from a probe run, so a case can repeat a citation's own exact+# spelling without this file guessing what that spelling is.+CASES: tuple[dict[str, Any], ...] = (+ {+ "key": "star_host",+ "why": "A host holding Markdown emphasis: the label must read as the host "+ "it names, and the address must arrive character for character.",+ "sources": {"turn0view0": ("https://a*b*.example/x", "Star Host")},+ "text": "Cited. {m0}",+ },+ {+ "key": "entity_host",+ "why": "A host that spells a character reference once decoded; the reader "+ "must see the reference, not whatever it would expand to.",+ "sources": {"turn0view0": ("https://a%26copy%3B.example/x", "Entity Host")},+ "text": "Cited. {m0}",+ },+ {+ "key": "underscore_path",+ "why": "Underscores in the path and the query - not CommonMark emphasis, "+ "but Slack mrkdwn and Telegram both read them as italics.",+ "sources": {"turn0view0": ("https://example.com/a_b_c?q=_u_&r=1", "Underscores")},+ "text": "Cited. {m0}",+ },+ {+ "key": "backtick_query",+ "why": "A backtick and an asterisk inside the query string, where a "+ "converter that scans destinations rewrites the address itself.",+ "sources": {"turn0view0": ("https://example.com/p?q=%60code%60&r=a*b*", "Query")},+ "text": "Cited. {m0}",+ },+ {+ "key": "format_chars",+ "why": "A zero-width space in the query, percent-encoded: invisible "+ "characters must not silently drop out of an address.",+ "sources": {"turn0view0": ("https://example.com/p?q=a%E2%80%8Bb", "Invisible")},+ "text": "Cited. {m0}",+ },+ {+ "key": "ipv6_port",+ "why": "A literal IPv6 authority with a port. The brackets are reserved "+ "authority syntax, and a link nobody can open is not attribution.",+ "sources": {"turn0view0": ("https://[::1]:8443/x?q=1", "Loopback")},+ "text": "Cited. {m0}",+ },+ {+ "key": "paren_path",+ "why": "Parentheses in the address, balanced, unbalanced and already "+ "percent-encoded, one of them beside a literal IPv6 host. Each is the "+ "page its source named, so the URL keeps it as written; only the link "+ "guards it, and every reader has to land on the same address.",+ "sources": {+ "turn0view0": ("https://en.wikipedia.org/wiki/Foo_(bar)", "Balanced"),+ "turn0view1": ("https://example.com/a_(b?q=c)d", "Unbalanced"),+ "turn0view2": ("https://example.com/a%28b%29", "Encoded"),+ "turn0view3": ("https://[::1]:8443/x(y", "Loopback"),+ },+ "text": "One {m0}, two {m1}, three {m2}, four {m3}.",+ },+ {+ "key": "footnote",+ "why": "A GFM footnote definition holding a link to the same page. One "+ "renderer reads a link there and the other reads a definition, which is "+ "exactly the disagreement that cost the real citation its badge.",+ "sources": {"turn0view0": ("https://example.com/x", "Footnoted")},+ "text": "[^f]: [p](https://example.com/x)\n\nCited. {m0} [^f]",+ },+ {+ "key": "code_fence",+ "why": "The citation's own exact spelling shown as a code example. It is "+ "not a link to any reader, but it is the same characters, and both ends "+ "have to count it the same way.",+ "sources": {"turn0view0": ("https://example.com/x", "Fenced")},+ "text": "```\n{link0}\n```\n\nCited. {m0}",+ },+ {+ "key": "table",+ "why": "The same spelling inside a GFM table cell, where `|` ends a cell "+ "and a label that spells one would split the row.",+ "sources": {"turn0view0": ("https://example.com/x", "Tabled")},+ "text": "| Source |\n| --- |\n| {link0} |\n\nCited. {m0}",+ },+ {+ "key": "near_html",+ "why": "The same spelling next to inline HTML, which each renderer is "+ "free to keep, escape or drop.",+ "sources": {"turn0view0": ("https://example.com/x", "Inline HTML")},+ "text": "Cited. {m0} <span>{link0}</span>",+ },+ {+ "key": "prose_repeat",+ "why": "The agent points at the cited page in its own prose, before and "+ "after, spelled identically. Only the middle one is a citation, and an "+ "ordinary link must never be impersonated.",+ "sources": {"turn0view0": ("https://example.com/x", "Repeated")},+ "text": "Background: {link0}\n\nCited. {m0}\n\nAlso {link0}",+ },+ {+ "key": "emoji_prefix",+ "why": "Non-ASCII text before the marker: offsets counted in characters "+ "on one side and in UTF-16 units on the other stop agreeing here.",+ "sources": {"turn0view0": ("https://example.com/x", "Emoji")},+ "text": "🔎 出处 {m0}",+ },+ {+ "key": "silent_strip",+ "why": "A silent control block holding the same spelling. Delivery removes "+ "it, so the consumer never counts it - and neither may the producer.",+ "sources": {"turn0view0": ("https://example.com/x", "Silent")},+ "text": "<silent>note to self: {link0}</silent>\nCited. {m0}",+ },+ {+ "key": "file_link",+ "why": "A file:// attachment link in the same reply: IM strips it to a "+ "plain label and the workbench keeps it, so the two surfaces get "+ "different text and the citation has to survive both.",+ "sources": {"turn0view0": ("https://example.com/x", "With attachment")},+ "text": "Cited. {m0}\n\n[report](file:///tmp/avibe-citation-bridge/report.txt)",+ },+ {+ "key": "quick_replies",+ "why": "A trailing quick-reply block, which delivery strips from the body "+ "on both surfaces.",+ "sources": {"turn0view0": ("https://example.com/x", "With buttons")},+ "text": "Cited. {m0}\n\n---\n[👌 继续] | [✅ 完成]",+ },+ {+ "key": "marker_shaped",+ "why": "The workbench rewrites `$<NAME>` into a secure-input card and "+ "`@<\u2026>`/`#<\u2026>` into chips by editing the source text before Markdown "+ "parses it - so a reply carrying those markers, and a source address "+ "spelled like one, is where an inserted card could move or swallow the "+ "link the producer counted.",+ "sources": {+ "turn0view0": ("https://example.com/p?q=$<openAiKey>&a=@<claude>", "Marker Shaped"),+ },+ "text": "Provide $<openAiKey>, then ping @<claude> in #<ses6jr7c5h2q6>.\n\nCited. {m0}",+ },+ {+ "key": "multi_source",+ "why": "Two sources in one reply, each with its own index and its own "+ "spelling.",+ "sources": {+ "turn0view0": ("https://example.com/x", "First"),+ "turn0view1": ("https://a*b*.example/y", "Second"),+ },+ "text": "First {m0}, second {m1}.",+ },+ {+ "key": "link_label",+ "why": "The marker written inside a link's own label. A citation is a "+ "link, and a link inside a link is not one - CommonMark makes the "+ "outer brackets text again - so the source is shown after the unit "+ "that encloses it, with that unit's destination and the label's other "+ "words untouched. Both links name the same page here, which is where "+ "a binding that matched on the address would badge the wrong one.",+ "sources": {"turn0view0": ("https://openai.com/docs", "OpenAI Docs")},+ "text": "See [the docs {m0}](https://openai.com/docs) for details.",+ },+ {+ "key": "link_label_positions",+ "why": "One source cited from inside three labels - opening one, in "+ "the middle of the next, closing the third. One row, three spans, and "+ "each source after its own unit rather than after the last of them.",+ "sources": {"turn0view0": ("https://example.com/s", "Thrice")},+ "text": "[{m0} a](https://u.example/1), [b {m0} c](https://u.example/2), "+ "[d {m0}](https://u.example/3).",+ },+ {+ "key": "data_slots",+ "why": "The slots of a link that are address or syntax rather than "+ "anything a reader is shown: a destination, a title, an angle-bracketed "+ "destination, and an autolink, which is all address. A marker there is "+ "part of what makes the unit work, so it stays the characters the model "+ "typed - no token minted, no source fetched, no link spliced in - while "+ "the one in the prose beside them is still a citation.",+ "sources": {"turn0view0": ("https://example.com/s", "Prose")},+ "text": "Cited. {m0}\n\n"+ "[a](https://example.com/{m0}path) [b](https://example.com/y \"T {m0}\") "+ "[c](<https://example.com/z{m0}>) <https://example.com/auto{m0}>",+ },+ {+ "key": "reference_slots",+ "why": "A reference link written in full shows its label and names its "+ "definition separately, so the label is a place a citation can go. "+ "Written short, the same brackets are both - the text a reader sees and "+ "the identifier that finds the address - so editing them would leave a "+ "link pointing at nothing. Those stay exactly as written, definition "+ "included.",+ "sources": {"turn0view0": ("https://example.com/s", "Referenced")},+ "text": "[the docs {m0}][ref], [dual {m0}] and [dual {m0}][].\n\n"+ "[ref]: https://r.example/page\n[dual {m0}]: https://r.example/dual",+ },+ {+ "key": "image_slots",+ "why": "An image's alt text is what a reader is given when the picture "+ "is not there; its src is an address. One marker is a citation and the "+ "other is not, in one line. The component does not fetch a foreign "+ "image - it renders a click-through link instead - so both are links on "+ "this surface.",+ "sources": {"turn0view0": ("https://example.com/s", "Illustrated")},+ "text": " and "+ "",+ },+ {+ "key": "image_in_link",+ "why": "An image inside a link: two units, one holding the other. The "+ "source belongs after the outer one. Placed after the image instead it "+ "would sit in the link's label and take the link down with it.",+ "sources": {"turn0view0": ("https://example.com/s", "Nested")},+ "text": "[](https://out.example/page)",+ },+ {+ "key": "moved_by_a_transform",+ "why": "Delivery removes silent blocks, and what closes up behind them "+ "leaves a registered token somewhere the model never put it. Each one "+ "is judged where it ends up rather than where it started: inside a "+ "label it is still a citation and is shown after that link, inside a "+ "destination or a title it is data again and the marker the model typed "+ "comes back - the alternative is a link nobody can open and a source "+ "claimed where no reader can see it.",+ "sources": {"turn0view0": ("https://example.com/s", "Moved")},+ "text": "A [lab <silent>](https://z.example) x</silent>{m0}](https://out.example/page) B"+ "\n\nC [two](https://z.example<silent>) y</silent>{m0}) D"+ "\n\nE [three](https://z.example \"t<silent>\") y</silent>{m0}\") F",+ },+ {+ "key": "open_fence",+ "why": "A fence the answer never closed takes the rest of the reply "+ "into code. The citation's own spelling in there is not a link to any "+ "renderer here, and a marker in there is not a citation either - which "+ "is the same rule the closed fence above gets, decided by the parser "+ "rather than by whether a closing line happens to exist.",+ "sources": {"turn0view0": ("https://example.com/s", "Unclosed")},+ "text": "Cited. {m0}\n\n```\n{link0}\n{m0} still open",+ },+ {+ "key": "file_label_and_quick_reply",+ "why": "A marker in an attachment's label and in a quick-reply button. "+ "Neither is Markdown a reader parses - one titles a file card, the "+ "other is a button - so the attribution is plain text left where it "+ "stands, with no link syntax and no token reaching a structured field. "+ "In the body the attachment is still a link, and CommonMark refuses "+ "``file:`` outright: a scan that cannot see that unit would put the "+ "source inside its label and break the card.",+ "sources": {"turn0view0": ("https://example.com/s", "Attached")},+ "text": "Cited. {m0}\n\n[report {m0}](file:///tmp/avibe-citation-bridge/report.txt)"+ "\n\n---\n[\U0001f44c \u7ee7\u7eed {m0}] | [\u2705 \u5b8c\u6210]",+ },+ {+ "key": "splice",+ "why": "Stripping a silent block splices two ordinary halves into "+ "characters that read as a citation marker. Only the first marker here "+ "was ever one, and the delivered text has to say so.",+ "sources": {"turn0view0": ("https://example.com/x", "Spliced")},+ "text": "Actual. {m0}\n\nSynthetic. \ue200ci<silent>private</silent>te"+ "\ue202turn0view0\ue201",+ },+ {+ "key": "file_delete_splice",+ "why": "Two deletions that cancel out. Removing the attachment link to "+ "its label and removing the silent block leave characters that read as "+ "a marker on IM and do not on the workbench - one reply, two bodies, "+ "and a marker shape that was never in the input either was.",+ "sources": {"turn0view0": ("https://example.com/x", "Cancelled out")},+ "text": "Actual. {m0}\n\nSynthetic. \ue200ci"+ "[te](file:///tmp/avibe-citation-bridge/report.txt)"+ "\ue202turn0view0<silent>note</silent>\ue201",+ },+ {+ "key": "file_link_above",+ "why": "A file:// attachment above the citation: IM flattens it to a "+ "label and the workbench keeps it, so one reply puts the same citation "+ "at two different offsets and each body needs its own measurement.",+ "sources": {"turn0view0": ("https://example.com/x", "Shifted")},+ "text": "[report](file:///tmp/avibe-citation-bridge/report.txt)\n\nCited. {m0}",+ },+ {+ "key": "unresolved",+ "why": "A marker naming a source the backend never captured. It degrades "+ "to a visible label rather than vanishing, and writes no sidecar row.",+ "sources": {},+ "text": "Claimed. {m0}",+ },+)+++def _resolve(text: str, sources: Mapping[str, CitationSource]):+ return resolve_citations(text, sources, unresolved_label=UNRESOLVED_LABEL)+++def _link_probe(sources: Mapping[str, CitationSource], refs: list[str]) -> list[str]:+ """The exact Markdown link the producer writes for each ref, measured."""+ spellings: list[str] = []+ for ref in refs:+ body, sidecar = _resolve(marker(ref), sources)+ spellings.append(utf16_slice(body, *sidecar[0]["spans"][0]) if sidecar else "")+ return spellings+++def utf16_len(text: str) -> int:+ """The length of *text* in UTF-16 code units, the unit a span counts in."""+ return len(text.encode("utf-16-le", "surrogatepass")) // 2+++def utf16_slice(text: str, start: int, end: int) -> str:+ """``text[start:end]`` read in UTF-16 code units.++ A Python slice counts code points, so the two disagree from the first+ non-BMP character onward - and a test that asserts what a span covers by+ slicing the string directly asserts the wrong characters exactly where it+ matters most.+ """+ units = text.encode("utf-16-le", "surrogatepass")+ return units[start * 2 : end * 2].decode("utf-16-le", "surrogatepass")+++def _anchors(text: str, citations: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:+ """The links a reader of *text* is shown, and which of them are citations.++ The labels and addresses come from the same inline-link scan the delivery+ pass runs, so they are what the product itself reads out of this text rather+ than what this file expects to be there. ``cited`` then applies the rule the+ sidecar states about THIS text: the row was measured in a body with this+ digest, and this link occupies one of the spans it names. A consumer has to+ reach the same answer the same way.++ This is a floor, not a census: the scan reads CommonMark, and a consumer may+ legitimately show links it does not see (a GFM footnote definition) or fewer+ (a ``file://`` attachment, which is not a web link on any surface). Each+ case records whether its anchors are exhaustive so a consumer test knows+ which of the two questions it may ask.+ """+ digest = body_digest(text)+ spans = {+ (span[0], span[1])+ for row in citations+ if row.get("body_sha256") == digest+ for span in row.get("spans") or ()+ }+ inner = _nested_unit_spans(text)+ anchors: list[dict[str, Any]] = []+ for link in inline_links(text):+ # A citation's span covers the whole link it wrote, so a link that+ # spells anything more - a title, say - simply matches no span.+ cited = (+ utf16_len(text[: link.start]),+ utf16_len(text[: link.end]),+ ) in spans+ # A label holding an image is not a string of text, and what a renderer+ # puts inside that anchor is the picture rather than the Markdown that+ # spelled it. Recording the source here would be recording something no+ # reader is shown, so the label is left unsaid and the link is known by+ # its address alone.+ label = unescape_markdown(text[link.label_start : link.label_end])+ if any(+ link.label_start <= start and end <= link.label_end for start, end in inner+ ):+ label = None+ anchors.append({"label": label, "url": link.destination, "cited": cited})+ return anchors+++def _nested_unit_spans(text: str) -> list[tuple[int, int]]:+ """The spans of every link or image written inside another one."""+ units = markdown_link_units(text)+ return [+ (unit.start, unit.end)+ for unit in units+ if any(+ other is not unit and other.start <= unit.start and unit.end <= other.end+ for other in units+ )+ ]+++# A link reference definition, and the address it names. Every other address a+# renderer can reach is spelled inside a unit the scan below already walks.+_DEFINITION = re.compile(r"^ {0,3}\[[^\]\n]*\]:[ \t]*(\S+)", re.MULTILINE)+_ADDRESS = re.compile(r"https?://[^\s<>()\[\]\"']+")+++def _addresses_cover_the_surface(text: str, anchors: list[dict[str, Any]]) -> bool:+ """Whether an address a reader of *text* reaches must be one of *anchors*.++ ``inline_links`` reads inline links, which is what the delivery pass needs+ and less than a reader gets: an image, a reference link or an autolink is a+ link to a reader and not to that scan, and a reference definition holds an+ address no unit spells. So everything it did not resolve is read back for+ the addresses it spells - an attachment names none, and neither does a+ reference that only points at a definition - and each one has to be an+ address already recorded. When it is not, the anchors are a floor: a+ consumer may be shown somewhere this file does not name, so it may be asked+ which links it must show and never which it must not.+ """+ recorded = {anchor["url"] for anchor in anchors}+ scanned = {(link.start, link.end) for link in inline_links(text)}+ unresolved = [+ text[unit.start : unit.end]+ for unit in markdown_link_units(text)+ if (unit.start, unit.end) not in scanned+ ]+ unresolved.extend(match.group(1) for match in _DEFINITION.finditer(text))+ return all(+ address in recorded+ for spelled in unresolved+ for address in _ADDRESS.findall(spelled)+ )+++# ----- The rows the dispatcher actually stored ------------------------------+#+# The cases above stop where the delivery pass does: at the text each surface is+# handed. A transcript row is one step further on, and it is not that text. The+# dispatcher folds a result footer into the body it stores for IM and keeps it+# beside the body it stores for the workbench, and the Web reader then splits+# the footer back off before Markdown ever sees the body. That split is a real+# edit to the exact body the ranges were measured in, so a consumer that cannot+# carry a measurement across an edit loses the badge on every ordinary IM+# transcript row - which is a regression a fixture stopping at the delivered+# text could not see.+#+# So these rows are read back out of a real SQLite home after a real+# ``ConsolidatedMessageDispatcher`` wrote them. Nothing here is composed by this+# file; it only says which reply was sent and on which surface.++ROW_SESSION = "ses_bridge_rows"+ROW_NOW = "2026-09-21T12:00:00Z"+ROW_SOURCES = {"turn0view0": ("https://example.com/x", "Cited page")}+# Prose pointing at the cited page in the answer's own words, then the citation+# itself - the same spelling twice, so a consumer that loses the measurement and+# falls back to matching text would badge the wrong one and say so. The emoji+# ahead of both is a surrogate pair: it makes a range counted in characters and+# a range counted in UTF-16 code units disagree from the first link onward.+ROW_TEXT = "🔎 Background: {link0}\n\nCited. {m0}"+# The same answer carrying every transform a stored body can stack above its+# citation: an attachment the two surfaces rewrite differently, and a secret+# marker the reader turns into a card as it renders.+ATTACHED_AND_CARDED = (+ "[report](file:///tmp/avibe-citation-bridge/report.txt)\n\n"+ "Provide $<deployKey> first.\n\n" + ROW_TEXT+)++ROW_CASES: tuple[dict[str, Any], ...] = (+ {+ "key": "im_row_with_footer",+ "why": "An ordinary IM transcript row. The dispatcher folds the footer "+ "into the stored body and stores it structured as well, so the Web "+ "reader removes a trailing range before rendering - the case where "+ "adding a body digest could have cost every IM row its badge.",+ "platform": "slack",+ "type": "result",+ "footer": "✅ ⏱️ 5s · 🪙 1.2k tok",+ },+ {+ "key": "im_row_without_footer",+ "why": "The same row with nothing to strip: the body the ranges were "+ "measured in is the body the renderer is handed.",+ "platform": "slack",+ "type": "result",+ "footer": None,+ },+ {+ "key": "workbench_row_with_footer",+ "why": "The workbench stores a clean body and its footer apart, so the "+ "same reply reaches the same reader through a different split.",+ "platform": "avibe",+ "type": "result",+ "footer": "✅ ⏱️ 2m 24s",+ },+ {+ "key": "activity_row",+ "why": "An interim narration row, which the activity card renders "+ "verbatim - no footer, no split, and its own consumer.",+ "platform": "avibe",+ "type": "assistant",+ "footer": None,+ },+ {+ "key": "workbench_row_rewritten_and_carded",+ "why": "Every transform a stored row can stack, in the order they "+ "happen and all of them above the citation: an attachment link is "+ "rewritten to the media proxy before the sidecar is measured, a footer "+ "is folded in and stripped back out by the reader, and a `$<NAME>` "+ "marker becomes a secure-input card as the reader renders it. Each one "+ "moves the citation, and only one of them was measured by the backend.",+ "platform": "avibe",+ "type": "result",+ "footer": "✅ ⏱️ 2m 24s",+ "text": ATTACHED_AND_CARDED,+ },+ {+ "key": "im_row_flattened_carded_and_folded",+ "why": "The same reply down the IM path, where the transforms stack "+ "differently: the attachment is flattened to a bare label instead of a "+ "proxy link, and the footer is folded into the stored body. So the "+ "reader runs two edits over one body - a trailing strip and an "+ "inserted secure-input card above the citation - and the badge has to "+ "survive their composition.",+ "platform": "slack",+ "type": "result",+ "footer": "✅ ⏱️ 5s",+ "text": ATTACHED_AND_CARDED,+ },+)+++def _row_reply(sources: Mapping[str, CitationSource], text: str = ROW_TEXT) -> str:+ refs = sorted(sources)+ return text.format(+ **{f"m{n}": marker(ref) for n, ref in enumerate(refs)},+ **{f"link{n}": spelling for n, spelling in enumerate(_link_probe(sources, refs))},+ )+++def _seed_home() -> None:+ """A temporary Avibe home holding one session the rows can belong to."""+ dispose_cached_sqlite_engines()+ ensure_sqlite_state()+ engine = create_sqlite_engine()+ try:+ with engine.begin() as conn:+ scope_id = upsert_scope(+ conn,+ platform="avibe",+ scope_type="project",+ native_id="proj_bridge_rows",+ now=ROW_NOW,+ )+ conn.execute(+ agent_sessions.insert().values(+ id=ROW_SESSION,+ scope_id=scope_id,+ agent_backend="codex",+ agent_variant="default",+ session_anchor=f"anchor_{ROW_SESSION}",+ native_session_id="",+ status="active",+ metadata_json="{}",+ created_at=ROW_NOW,+ updated_at=ROW_NOW,+ last_active_at=ROW_NOW,+ )+ )+ finally:+ engine.dispose()+++def _stored_rows(seen: set[str]) -> list[dict[str, Any]]:+ """Rows this store has gained since the last call, oldest first."""+ engine = create_sqlite_engine()+ try:+ with engine.connect() as conn:+ fresh = [+ {"type": row[1], "text": row[2], "content": json.loads(row[3] or "null")}+ for row in conn.execute(+ select(+ messages.c.id,+ messages.c.type,+ messages.c.content_text,+ messages.c.content_json,+ ).order_by(messages.c.id)+ )+ if row[0] not in seen and not seen.add(row[0])+ ]+ finally:+ engine.dispose()+ return fresh+++def build_rows() -> list[dict[str, Any]]:+ """Drive the real dispatcher once per surface and read the rows back."""+ sources = {ref: source(ref, url, title) for ref, (url, title) in ROW_SOURCES.items()}+ rows: list[dict[str, Any]] = []++ with tempfile.TemporaryDirectory() as home:+ with mock.patch.dict(os.environ, {"AVIBE_HOME": home}):+ _seed_home()+ seen: set[str] = set()+ for case in ROW_CASES:+ reply = _row_reply(sources, case.get("text", ROW_TEXT))+ registered, bundle = register_citations(+ reply, sources, unresolved_label=UNRESOLVED_LABEL+ )+ controller = _StubController(case["platform"])+ controller.config.reply_enhancements = True+ context = MessageContext(+ user_id="workbench" if case["platform"] == "avibe" else "u1",+ channel_id=ROW_SESSION if case["platform"] == "avibe" else "c1",+ platform=case["platform"],+ platform_specific=(+ {"agent_session_id": ROW_SESSION}+ if case["platform"] == "avibe"+ else None+ ),+ )+ asyncio.run(+ ConsolidatedMessageDispatcher(controller).emit_agent_message(+ context,+ case["type"],+ registered,+ result_footer=case["footer"],+ citations=bundle,+ )+ )+ written = _stored_rows(seen)+ # One emit is one logical message, so one row. A surface that+ # wrote none (a send that failed, a type that is not stored)+ # would otherwise be recorded as a case with nothing in it.+ if len(written) != 1:+ raise AssertionError(f"{case['key']} wrote {len(written)} rows, expected 1")+ rows.append(+ {+ **{+ k: v+ for k, v in case.items()+ if k not in ("footer", "text")+ },+ "row": written[0],+ }+ )+ dispose_cached_sqlite_engines()+ return rows+++def build() -> dict[str, Any]:+ """Run the real producer and delivery pass over every case."""+ cases: list[dict[str, Any]] = []+ for case in CASES:+ refs = sorted(case["sources"])+ sources = {+ ref: source(ref, url, title) for ref, (url, title) in case["sources"].items()+ }+ markers = {f"m{n}": marker(ref) for n, ref in enumerate(refs)} or {"m0": marker("turn0view0")}+ links = {f"link{n}": spelling for n, spelling in enumerate(_link_probe(sources, refs))}+ agent_text = case["text"].format(**markers, **links)++ # Registration happens where the markers still mean what they say, and+ # what travels from here is an opaque token per marker. The tokens are+ # nonces, so nothing recorded below may contain one: this file records+ # the bodies each surface finally shows.+ registered, bundle = register_citations(+ agent_text, sources, unresolved_label=UNRESOLVED_LABEL+ )+ # The two surfaces, reached exactly as ``core.message_dispatcher`` reaches+ # them: the workbench keeps its file links so it can rewrite them for+ # inline rendering, IM gets them flattened to plain labels. Each body is+ # then written separately, because they are different bodies.+ web = process_reply(registered, keep_file_links=True)+ im = process_reply(registered)+ web_text, web_citations = finalize_citations(web.text, bundle)+ # IM carries no sidecar; its rows are measured here only to say which of+ # its links a reader is owed as a citation.+ im_text = materialize_citations(im.text, bundle)+ im_citations = finalize_citations(im.text, bundle)[1]+ web_anchors = _anchors(web_text, web_citations)+ im_anchors = _anchors(im_text, im_citations)+ cover = _addresses_cover_the_surface(+ web_text, web_anchors+ ) and _addresses_cover_the_surface(im_text, im_anchors)+ cases.append(+ {+ "key": case["key"],+ "why": case["why"],+ "sources": {+ ref: {"url": url, "title": title}+ for ref, (url, title) in case["sources"].items()+ },+ "agent_text": agent_text,+ "web_citations": web_citations,+ "im_citations": im_citations,+ "web_text": web_text,+ "im_text": im_text,+ "quick_replies": [+ materialize_citations(button.text, bundle, as_markdown=False)+ for button in web.buttons+ ],+ "files": [+ {+ "label": materialize_citations(f.label, bundle, as_markdown=False),+ "path": f.path,+ }+ for f in web.files+ ],+ "web_anchors": web_anchors,+ "im_anchors": im_anchors,+ # Two different questions, and a case may answer yes to the+ # first and no to the second. ``cover`` says the addresses below+ # are every address a reader is given, so a link pointing+ # anywhere else is a link nobody wrote. ``exhaustive`` says they+ # are also in the order they are shown - which a GFM footnote+ # definition breaks by itself, because its link is rendered+ # where the footnotes go rather than where it was written.+ "anchors_cover_destinations": cover,+ "anchors_exhaustive": cover and "[^" not in web_text,+ }+ )+ return {+ "generated_by": "tests/citation_bridge.py",+ "cases": cases,+ "rows": build_rows(),+ }+++def main() -> None:+ FIXTURE.parent.mkdir(parents=True, exist_ok=True)+ FIXTURE.write_text(json.dumps(build(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")+ print(f"wrote {FIXTURE}")+++if __name__ == "__main__":+ main()diff --git a/tests/e2e/test_codex_citation_contract.py b/tests/e2e/test_codex_citation_contract.pynew file mode 100644index 000000000..a9255b14e--- /dev/null+++ b/tests/e2e/test_codex_citation_contract.py@@ -0,0 +1,404 @@+"""The real installed Codex CLI's citation wire contract, end to end.++Every other citation test starts from a marker someone typed into a fixture.+This one starts from the real ``codex`` app-server binary: a loopback provider+serves both halves of a cited answer - the web search the native process runs,+and the model text that cites it - and the notifications Avibe actually receives+are captured. Those captured notifications - not hand-written ones - are then+replayed through the product path (``CodexEventHandler`` ->+``BaseAgent.emit_result_message`` -> ``ConsolidatedMessageDispatcher`` ->+``persist_agent_message``) and read back out of SQLite.++Both halves are reproducible against a loopback provider, which is worth stating+because it was once assumed only the first was:++* the marker is ordinary text in the model's answer. That half is worth proving+ on real bytes because the delimiters are private-use codepoints+ (U+E200..U+E202), exactly the kind of thing a UTF-8 round trip, a JSON escape,+ or a whitespace trim could quietly destroy.+* ``results[]`` comes from the binary's standalone search tool, which posts to+ ``<provider base_url>/alpha/search`` - a path this provider answers itself.+ Activating it takes ``model_providers.<p>.supports_standalone_web_search``+ plus the under-development ``features.standalone_web_search``; with both set+ the binary offers the ``web`` namespace, calls ``web.run``, and reports this+ provider's own results back as a genuine search item. So the search item+ replayed below is a captured native notification, not a schema fixture.+"""++from __future__ import annotations++import asyncio+import itertools+import json+import subprocess+import types+from http import HTTPStatus+from pathlib import Path+from urllib.parse import urlsplit++import pytest++from config.v2_config import ModelHubBackendModelConfig+from core.citations import _TOKEN_OPEN as TOKEN_OPEN+from core.citations import body_digest+from core.message_dispatcher import ConsolidatedMessageDispatcher+from modules.agents.base import BaseAgent+from modules.agents.codex.event_handler import CodexEventHandler+from modules.agents.codex.search_history import is_web_search_item+from modules.agents.codex.transport import CodexTransport+from modules.agents.model_hub import ModelHubLaunch, build_codex_hub_launch+from modules.im import MessageContext+from storage import messages_service+from storage.db import create_sqlite_engine+from storage.importer import ensure_sqlite_state+from storage.models import agent_sessions+from storage.settings_service import upsert_scope+from tests.e2e.drivers import mock_llm_upstream as upstream+from tests.e2e.test_model_hub_catalog_consumer import (+ codex_catalog_runtime, # noqa: F401 -- fixture dependency+ rejected_external_proxy, # noqa: F401 -- fixture dependency+)+from tests.scenario_harness.message_delivery import MessageDeliveryController+from tests.test_codex_event_handler import _StubAgent+from vibe.backend_model_catalog import _codex_hub_catalog_bytes++pytestmark = pytest.mark.e2e_model_hub++TOKEN = "isolated-citation-contract-fixture"+MODEL = "citation-route-alias"+SESSION_ID = "sescitation01"++START, SEP, END = "", "", ""+GUIDE_URL = "https://developers.openai.com/api/docs/guides/tools-web-search"+PROBE_URL = "https://example.com/citation-probe-source"+UNRESOLVED = "(source unavailable)"++SEARCH_QUERY = "native web search guide"+# The standalone search tool joins this to the provider base_url the Hub launch+# builds (``<gateway>/v1``), so the loopback gateway can answer it.+SEARCH_PATH = "/v1/alpha/search"+# ``codex_api::search::SearchResponse``: prose for the model, plus the citable+# results the native process reports back as the search item's ``results[]``.+SEARCH_RESPONSE = {+ "output": "Two sources on native web search.",+ "results": [+ {"ref_id": "turn0view0", "title": "Web search - OpenAI API", "url": GUIDE_URL},+ {"ref_id": "turn0view1", "title": "引用探针来源 — Café", "url": PROBE_URL},+ ],+}++# One answer exercising every boundary at once: a resolvable ref, a multi-ref+# marker whose second ref is unknown, the grammar quoted inside a fenced code+# example, and a marker the stream cut off before its terminator.+ANSWER = (+ f"Native web search is documented.{START}cite{SEP}turn0view0{END}\n"+ f"Two sources, one unknown.{START}cite{SEP}turn0view1{SEP}turn9view9{END}\n\n"+ "The marker grammar itself:\n\n"+ f"```\n{START}cite{SEP}turn0view0{END}\n```\n\n"+ f"Truncated tail {START}cite{SEP}turn0view0"+)+++def citation_runtime(codex_catalog_runtime, record_property):+ binary, runtime, raw_catalog = codex_catalog_runtime+ version = subprocess.run(+ [binary, "--version"], cwd=runtime.home, env=runtime.env,+ capture_output=True, text=True, check=True, timeout=15,+ ).stdout.strip()+ record_property("codex_version", version)+ Path(runtime.env["CODEX_HOME"], "config.toml").write_text(+ 'cli_auth_credentials_store = "file"\n'+ # The search tool has to be offered before the model can call it.+ "[tools]\nweb_search = true\n"+ )+ catalog_path = runtime.home / "citation-models.json"+ catalog_path.write_bytes(_codex_hub_catalog_bytes(+ raw_catalog,+ [ModelHubBackendModelConfig(id=MODEL, context_window=128_000).to_payload()],+ ))+ return binary, runtime, catalog_path+++def _substitute(node):+ """Swap the loopback's canned answer for the cited one, wherever it appears."""+ if isinstance(node, str):+ return ANSWER if node == "mock response" else node+ if isinstance(node, list):+ return [_substitute(item) for item in node]+ if isinstance(node, dict):+ return {key: _substitute(value) for key, value in node.items()}+ return node+++def _search_call_frames(model: str) -> list[bytes]:+ """The first model response: one call into the binary's ``web`` namespace."""+ call = {+ "type": "function_call",+ "id": "fc_search_001",+ "call_id": "call_search_001",+ "name": "run",+ "namespace": "web",+ "arguments": json.dumps({"search_query": [{"q": SEARCH_QUERY}]}),+ "status": "completed",+ }+ response = {"id": "resp_search_001", "object": "response", "model": model}+ return [+ upstream._sse("response.created", {+ "type": "response.created", "sequence_number": 0,+ "response": {**response, "status": "in_progress"},+ }),+ upstream._sse("response.output_item.done", {+ "type": "response.output_item.done",+ "sequence_number": 1, "output_index": 0, "item": call,+ }),+ upstream._sse("response.completed", {+ "type": "response.completed", "sequence_number": 2,+ "response": {+ **response, "status": "completed", "output": [call],+ "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},+ },+ }),+ ]+++def searched_answer_upstream(monkeypatch) -> list[object]:+ """Serve one real web search, then the answer citing it; record the search.++ Two loopback behaviours, both needed before the native process will produce a+ search item of its own: the first model response calls ``web.run``, and the+ provider answers the standalone search request the binary then posts to its+ own base URL.+ """+ original_frames = upstream._responses_stream_frames+ original_post = upstream.MockLLMUpstreamHandler.do_POST+ responses = itertools.count(1)+ searched: list[object] = []++ def frames(model):+ if next(responses) == 1:+ return _search_call_frames(model)+ rebuilt = []+ for frame in original_frames(model):+ head, _, data = frame.partition(b"data: ")+ event = head.decode("utf-8").removeprefix("event: ").strip() or None+ rebuilt.append(upstream._sse(event, _substitute(json.loads(data))))+ return rebuilt++ def do_POST(handler): # noqa: N802 -- BaseHTTPRequestHandler's own name+ if urlsplit(handler.path).path != SEARCH_PATH:+ original_post(handler)+ return+ searched.append(handler._read_json_body())+ handler._write_json(HTTPStatus.OK, SEARCH_RESPONSE)++ monkeypatch.setattr(upstream, "_responses_stream_frames", frames)+ monkeypatch.setattr(upstream.MockLLMUpstreamHandler, "do_POST", do_POST)+ return searched+++def _run_real_turn(fixture, gateway) -> list[tuple[str, dict]]:+ """Run one real native turn and return every notification it produced."""+ binary, runtime, catalog_path = fixture+ launch = ModelHubLaunch(+ backend="codex", channel="hub", requested_model=MODEL,+ runtime_model=MODEL, target_model="unrelated-target",+ gateway_base_url=gateway.url, gateway_token=TOKEN,+ )+ args, env = build_codex_hub_launch([], runtime.env, launch, model_catalog_path=catalog_path)+ provider = next(+ value.split('"')[1] for value in args if value.startswith('model_provider="')+ )+ args = args + [+ # Standalone search is what makes ``results[]`` real: the tool posts to+ # this provider's own endpoint instead of a hosted service. It is still+ # under development, so the provider flag alone does not enable it.+ "-c", f"model_providers.{provider}.supports_standalone_web_search=true",+ "-c", "features.standalone_web_search=true",+ "-c", "suppress_unstable_features_warning=true",+ ]++ async def probe():+ transport = CodexTransport(+ binary=binary, cwd=str(runtime.home), runtime_args=args, runtime_env=env,+ )+ notifications: list[tuple[str, dict]] = []+ completed: asyncio.Queue = asyncio.Queue()++ async def notify(method, params):+ notifications.append((method, params))+ if method == "turn/completed":+ await completed.put(params)++ transport.on_notification(notify)+ try:+ await transport.start()+ thread = await transport.send_request(+ "thread/start",+ {"model": MODEL, "cwd": transport._cwd, "approvalPolicy": "never"},+ )+ await transport.send_request(+ "turn/start",+ {+ "threadId": thread["thread"]["id"],+ "input": [{"type": "text", "text": "Search the web and cite the guide."}],+ },+ )+ result = await asyncio.wait_for(completed.get(), timeout=60)+ assert result["turn"]["status"] == "completed", result+ finally:+ await transport.stop()+ return notifications++ return asyncio.run(probe())+++def _captured(fixture) -> tuple[dict, dict, dict]:+ """The real search, answer, and ``turn/completed`` notifications for one turn."""+ with upstream.MockLLMUpstream() as gateway:+ gateway.configure(protocol="openai_responses")+ notifications = _run_real_turn(fixture, gateway)++ def completed(match):+ return next(+ params+ for method, params in notifications+ if method == "item/completed" and match(params.get("item") or {})+ )++ # Recognized by the production predicate, so the shape the binary happens to+ # name its search item is the shape the product path accepts.+ search = completed(lambda item: is_web_search_item(item) and item.get("results"))+ message = completed(lambda item: item.get("type") == "agentMessage")+ completion = next(params for method, params in notifications if method == "turn/completed")+ return search, message, completion+++def test_real_codex_relays_citation_markers_verbatim(citation_runtime, searched_answer_upstream):+ """The private-use marker must reach Avibe's parse point byte for byte."""+ search, message, completion = _captured(citation_runtime)++ assert message["item"]["text"] == ANSWER+ assert f"{START}cite{SEP}turn0view0{END}" in message["item"]["text"]+ # The sources are the binary's own: it called the search tool, the tool+ # posted the model's query to this provider, and the results came back as one+ # native item - untouched, including a non-ASCII title.+ assert searched_answer_upstream, "the binary never posted a standalone search"+ assert SEARCH_QUERY in json.dumps(searched_answer_upstream[0], ensure_ascii=False)+ assert [result["ref_id"] for result in search["item"]["results"]] == [+ "turn0view0",+ "turn0view1",+ ]+ assert search["item"]["results"][1]["title"] == "引用探针来源 — Café"+ # The completion notification is what triggers resolution, and it must name+ # the same thread and turn: that scoping is the only reason a ref_id - which+ # is unique nowhere else - is safe to look up at all.+ assert message["threadId"] and message["turnId"]+ assert search["threadId"] == message["threadId"]+ assert completion.get("threadId") == message["threadId"]+ assert completion["turn"]["id"] == message["turnId"]+++def test_real_notifications_deliver_and_persist_resolved_citations(+ citation_runtime, searched_answer_upstream, monkeypatch, tmp_path,+):+ """Replay the captured real notifications through the real product path."""+ search, message, completion = _captured(citation_runtime)+ thread_id, turn_id = message["threadId"], message["turnId"]++ monkeypatch.setenv("AVIBE_HOME", str(tmp_path / "avibe-home"))+ ensure_sqlite_state()+ engine = create_sqlite_engine()+ now = "2026-09-21T00:00:00Z"+ with engine.begin() as conn:+ scope_id = upsert_scope(+ conn, platform="slack", scope_type="channel", native_id="C-citations", now=now+ )+ conn.execute(agent_sessions.insert().values(+ id=SESSION_ID, scope_id=scope_id, agent_backend="codex", agent_variant="default",+ session_anchor=f"anchor_{SESSION_ID}", native_session_id=thread_id, status="active",+ metadata_json="{}", created_at=now, updated_at=now, last_active_at=now,+ ))++ # Real dispatcher behind the controller surface, exactly as the live wiring+ # has it: the agent emits onto the controller, which delegates to dispatch.+ controller = MessageDeliveryController(platform="slack")+ controller.config.show_duration = False+ dispatcher = ConsolidatedMessageDispatcher(controller)+ controller.emit_agent_message = dispatcher.emit_agent_message++ agent = _StubAgent()+ agent.controller = controller+ agent.config = controller.config+ agent._calculate_duration_ms = lambda started_at: 0+ # The real result emit, so the sidecar crosses the real agent boundary.+ agent.emit_result_message = types.MethodType(BaseAgent.emit_result_message, agent)++ handler = CodexEventHandler(agent)+ context = MessageContext(+ user_id="U1", channel_id="C-citations", platform="slack",+ platform_specific={"agent_session_id": SESSION_ID},+ )+ request = types.SimpleNamespace(+ base_session_id=SESSION_ID,+ session_key="slack::channel::C-citations",+ working_path=str(tmp_path),+ context=context,+ started_at=0,+ output=None,+ )+ agent._turn_registry.register_turn(turn_id, request)++ async def replay():+ # The search arrives in its own notification, ahead of the answer that+ # cites it - as captured, ids and all.+ await handler._on_item_completed(search, request)+ await handler._on_item_completed(message, request)+ await handler._on_turn_completed(completion, request)++ asyncio.run(replay())++ delivered = "\n".join(controller.im_probe.rendered_texts())+ assert f"[developers.openai.com]({GUIDE_URL})" in delivered+ assert f"[example.com]({PROBE_URL})" in delivered+ # The unknown ref is labelled, not deleted, and never guessed into a URL.+ assert UNRESOLVED in delivered+ assert "turn9view9" not in delivered+ # A marker shown as a code example, and one the stream cut off, stay literal.+ assert f"```\n{START}cite{SEP}turn0view0{END}\n```" in delivered+ assert delivered.endswith(f"Truncated tail {START}cite{SEP}turn0view0")++ with engine.connect() as conn:+ transcript = messages_service.list_session_messages(+ conn, session_id=SESSION_ID, types=("result",)+ )+ row = transcript["messages"][-1]+ citations = (row["content"] or {})["citations"]++ assert [(c["index"], c["ref_id"], c["url"]) for c in citations] == [+ (1, "turn0view0", GUIDE_URL),+ (2, "turn0view1", PROBE_URL),+ ]+ assert citations[1]["title"] == "引用探针来源 — Café"+ # Every sidecar row describes the stored body and nothing else: one digest+ # of that exact text, and spans that land on the links the backend wrote,+ # counted in UTF-16 code units because that is what the renderer counts+ # (ui/src/lib/citations.ts). It verifies the digest before it paints+ # anything, so both numbers here are ones it has to reach independently.+ body = row["text"]+ assert {c["body_sha256"] for c in citations} == {body_digest(body)}+ units = body.encode("utf-16-le", "surrogatepass")+ for citation in citations:+ assert citation["spans"]+ for start, end in citation["spans"]:+ # These labels need no escaping, so the link reads exactly as spelled.+ assert units[start * 2 : end * 2].decode("utf-16-le", "surrogatepass") == (+ f"[{citation['label']}]({citation['url']})"+ )++ # The identity the message travelled under is internal. A reader may see the+ # model's own marker characters - quoted in a code example, or cut off+ # mid-stream, both above - but never a token this delivery minted.+ assert TOKEN_OPEN not in delivered+ assert TOKEN_OPEN not in bodydiff --git a/tests/fixtures/citation_authority_matrix.json b/tests/fixtures/citation_authority_matrix.jsonnew file mode 100644index 000000000..8bfdfd5b3--- /dev/null+++ b/tests/fixtures/citation_authority_matrix.json@@ -0,0 +1,248 @@+{+ "description": "One shared table for the brackets a destination's authority is allowed to keep.\n\n`[` and `]` are the syntax of an IPv6 host and ordinary data everywhere else in a URL, and a URI is spelled before it is delivered - `spell_uri` on the backend, `normalizeUri` in the Web pipeline - which writes every bracket that is data as `%5B`/`%5D`. Deciding afterwards which ones to put back cannot work: by then `https://[::1]/admin`, whose host is a literal address, and `https://%5B::1%5D/admin`, whose host is the nine characters `%5B::1%5D`, are the same string. Restoring both gives the second one a live loopback address its author never wrote. So the question is asked of the destination the Markdown parser resolved, before spelling, and the answer is carried through to the surface.\n\nThree readers measure three separate facts about each row, and a row is only evidence because they are separate:\n\n- `tests/test_citations.py` calls `safe_url(destination)` and asserts `safe_url`. `null` means the value is rejected outright - reported as an unresolved citation - rather than repaired into a link.\n- `ui/src/components/ui/markdown.test.tsx` renders `markdown` through the real Markdown component and asserts the anchor's `href` exactly, then asserts `navigates`: the URL a browser resolves from that href, or `null` when a browser refuses it. `null` rows are asserted as refusals, not skipped - `new URL('https://%5B::1%5D/admin')` throws, and omitting the row because the assertion is awkward is how the repair got in.\n- `tests/test_link_unit_delivery.py` renders `markdown` through the real Slack adapter and asserts `slack`, the destination a client decodes back out of the `<url|label>` wrapper.\n\n`destination` and `markdown` differ only in the row whose `why` says so: `safe_url` receives a citation's url as data, the other two receive Markdown. Agreement between the three on repairing an encoded wrapper would be agreement on the bug, so each column is measured against its own surface and none is derived from another.",+ "cases": [+ {+ "why": "a valid IPv6 literal is an address a browser opens, and spelling a destination must not be what takes it away",+ "destination": "https://[::1]/admin",+ "markdown": "[h](https://[::1]/admin)",+ "safe_url": "https://[::1]/admin",+ "href": "https://[::1]/admin",+ "navigates": "https://[::1]/admin",+ "slack": "https://[::1]/admin"+ },+ {+ "why": "%5B is a character of the host, not a bracket the destination ever had: restoring it would mint a live loopback address nobody wrote",+ "destination": "https://%5B::1%5D/admin",+ "markdown": "[h](https://%5B::1%5D/admin)",+ "safe_url": null,+ "href": "https://%5B::1%5D/admin",+ "navigates": null,+ "slack": "https://%5B::1%5D/admin"+ },+ {+ "why": "the escape is the same escape in lower case, and so is the address it does not name",+ "destination": "https://%5b::1%5d/admin",+ "markdown": "[h](https://%5b::1%5d/admin)",+ "safe_url": null,+ "href": "https://%5b::1%5d/admin",+ "navigates": null,+ "slack": "https://%5b::1%5d/admin"+ },+ {+ "why": "one literal bracket is not a wrapper, and half of one is not completed into a whole",+ "destination": "https://[::1%5D/admin",+ "markdown": "[h](https://[::1%5D/admin)",+ "safe_url": null,+ "href": "https://%5B::1%5D/admin",+ "navigates": null,+ "slack": "https://%5B::1%5D/admin"+ },+ {+ "why": "the same from the other end - the closing bracket alone buys nothing either",+ "destination": "https://%5B::1]/admin",+ "markdown": "[h](https://%5B::1]/admin)",+ "safe_url": null,+ "href": "https://%5B::1%5D/admin",+ "navigates": null,+ "slack": "https://%5B::1%5D/admin"+ },+ {+ "why": "mixing the two spellings of the escape changes neither answer",+ "destination": "https://%5B::1%5d/admin",+ "markdown": "[h](https://%5B::1%5d/admin)",+ "safe_url": null,+ "href": "https://%5B::1%5d/admin",+ "navigates": null,+ "slack": "https://%5B::1%5d/admin"+ },+ {+ "why": "%255B spells the text %5B, which is one layer further from a bracket still",+ "destination": "https://%255B::1%255D/admin",+ "markdown": "[h](https://%255B::1%255D/admin)",+ "safe_url": null,+ "href": "https://%255B::1%255D/admin",+ "navigates": null,+ "slack": "https://%255B::1%255D/admin"+ },+ {+ "why": "userinfo and a port belong to a valid literal authority, and keeping brackets is not a reason to drop them",+ "destination": "https://u:p@[::1]:8443/x",+ "markdown": "[h](https://u:p@[::1]:8443/x)",+ "safe_url": "https://u:p@[::1]:8443/x",+ "href": "https://u:p@[::1]:8443/x",+ "navigates": "https://u:p@[::1]:8443/x",+ "slack": "https://u:p@[::1]:8443/x"+ },+ {+ "why": "userinfo and a port in front of an encoded wrapper do not make it a host",+ "destination": "https://u:p@%5B::1%5D:8443/x",+ "markdown": "[h](https://u:p@%5B::1%5D:8443/x)",+ "safe_url": null,+ "href": "https://u:p@%5B::1%5D:8443/x",+ "navigates": null,+ "slack": "https://u:p@%5B::1%5D:8443/x"+ },+ {+ "why": "an empty userinfo is carried by the renderer and dropped by the citation's canonical form; the literal address both name is the same one",+ "destination": "https://@[::1]/x",+ "markdown": "[h](https://@[::1]/x)",+ "safe_url": "https://[::1]/x",+ "href": "https://@[::1]/x",+ "navigates": "https://[::1]/x",+ "slack": "https://[::1]/x"+ },+ {+ "why": "a port a browser accepts leaves the authority valid, brackets and all",+ "destination": "https://[::1]:8443/x",+ "markdown": "[h](https://[::1]:8443/x)",+ "safe_url": "https://[::1]:8443/x",+ "href": "https://[::1]:8443/x",+ "navigates": "https://[::1]:8443/x",+ "slack": "https://[::1]:8443/x"+ },+ {+ "why": "a port past 65535 makes the whole authority one no parser reads, so there is no valid literal left to preserve",+ "destination": "https://[::1]:99999/x",+ "markdown": "[h](https://[::1]:99999/x)",+ "safe_url": null,+ "href": "https://%5B::1%5D:99999/x",+ "navigates": null,+ "slack": "https://%5B::1%5D:99999/x"+ },+ {+ "why": "an empty port is no port, and the literal address survives it",+ "destination": "https://[::1]:/x",+ "markdown": "[h](https://[::1]:/x)",+ "safe_url": "https://[::1]:/x",+ "href": "https://[::1]:/x",+ "navigates": "https://[::1]/x",+ "slack": "https://[::1]:/x"+ },+ {+ "why": "literal brackets around something that is not an address are not an authority, so they are spelled like any other data",+ "destination": "https://[nope]/x",+ "markdown": "[h](https://[nope]/x)",+ "safe_url": null,+ "href": "https://%5Bnope%5D/x",+ "navigates": null,+ "slack": "https://%5Bnope%5D/x"+ },+ {+ "why": "and the encoded spelling of the same non-address is not repaired into one",+ "destination": "https://%5Bnope%5D/x",+ "markdown": "[h](https://%5Bnope%5D/x)",+ "safe_url": null,+ "href": "https://%5Bnope%5D/x",+ "navigates": null,+ "slack": "https://%5Bnope%5D/x"+ },+ {+ "why": "an unclosed bracket is not an address",+ "destination": "https://[::1/x",+ "markdown": "[h](https://[::1/x)",+ "safe_url": null,+ "href": "https://%5B::1/x",+ "navigates": null,+ "slack": "https://%5B::1/x"+ },+ {+ "why": "a dotted IPv4 tail is a valid literal address; how a browser compresses it for display is a separate question",+ "destination": "https://[::ffff:1.2.3.4]/x",+ "markdown": "[h](https://[::ffff:1.2.3.4]/x)",+ "safe_url": "https://[::ffff:1.2.3.4]/x",+ "href": "https://[::ffff:1.2.3.4]/x",+ "navigates": "https://[::ffff:102:304]/x",+ "slack": "https://[::ffff:1.2.3.4]/x"+ },+ {+ "why": "a bracket in the path is data - it is spelled, never preserved",+ "destination": "https://example.com/[x]",+ "markdown": "[h](https://example.com/[x])",+ "safe_url": "https://example.com/%5Bx%5D",+ "href": "https://example.com/%5Bx%5D",+ "navigates": "https://example.com/%5Bx%5D",+ "slack": "https://example.com/%5Bx%5D"+ },+ {+ "why": "path data already spelled is left exactly as written rather than decoded first",+ "destination": "https://example.com/%5Bx%5D",+ "markdown": "[h](https://example.com/%5Bx%5D)",+ "safe_url": "https://example.com/%5Bx%5D",+ "href": "https://example.com/%5Bx%5D",+ "navigates": "https://example.com/%5Bx%5D",+ "slack": "https://example.com/%5Bx%5D"+ },+ {+ "why": "query and fragment are data too, and structure around them survives",+ "destination": "https://example.com/p?x=[c]#f[d]",+ "markdown": "[h](https://example.com/p?x=[c]#f[d])",+ "safe_url": "https://example.com/p?x=%5Bc%5D#f%5Bd%5D",+ "href": "https://example.com/p?x=%5Bc%5D#f%5Bd%5D",+ "navigates": "https://example.com/p?x=%5Bc%5D#f%5Bd%5D",+ "slack": "https://example.com/p?x=%5Bc%5D#f%5Bd%5D"+ },+ {+ "why": "the host's wrapper is preserved while a bracket in the userinfo beside it is still spelled: only the authority's brackets qualify, and only where they are structure",+ "destination": "https://a[b@[::1]/x",+ "markdown": "[h](https://a[b@[::1]/x)",+ "safe_url": "https://a%5Bb@[::1]/x",+ "href": "https://a%5Bb@[::1]/x",+ "navigates": "https://a%5Bb@[::1]/x",+ "slack": "https://a%5Bb@[::1]/x"+ },+ {+ "why": "a bracket in the userinfo is no reason to promote an encoded wrapper into a host",+ "destination": "https://a[b@%5B::1%5D/x",+ "markdown": "[h](https://a[b@%5B::1%5D/x)",+ "safe_url": null,+ "href": "https://a%5Bb@%5B::1%5D/x",+ "navigates": null,+ "slack": "https://a%5Bb@%5B::1%5D/x"+ },+ {+ "why": "a numeric character reference resolves to a literal bracket before any spelling happens, so the address it names is the address delivered",+ "destination": "https://[::1]/admin",+ "markdown": "[h](https://[::1]/admin)",+ "safe_url": "https://[::1]/admin",+ "href": "https://[::1]/admin",+ "navigates": "https://[::1]/admin",+ "slack": "https://[::1]/admin"+ },+ {+ "why": "the one row where the two readers are handed different inputs: a Markdown escape resolves to a literal bracket for a renderer parsing Markdown, while safe_url is given a citation's url as data, where a backslash is a character of the host and no address",+ "destination": "https://\\[::1\\]/admin",+ "markdown": "[h](https://\\[::1\\]/admin)",+ "safe_url": null,+ "href": "https://[::1]/admin",+ "navigates": "https://[::1]/admin",+ "slack": "https://[::1]/admin"+ },+ {+ "why": "the path's bracket is spelled and the encoded wrapper is still not a host: one destination, two brackets, two answers",+ "destination": "https://%5B::1%5D/a[b]",+ "markdown": "[h](https://%5B::1%5D/a[b])",+ "safe_url": null,+ "href": "https://%5B::1%5D/a%5Bb%5D",+ "navigates": null,+ "slack": "https://%5B::1%5D/a%5Bb%5D"+ }+ ],+ "per_occurrence": {+ "why": "Two links in one body whose hrefs are identical after naive spelling. A qualification kept per occurrence keeps them apart; anything keyed by the spelled URL - a cache, a set, a second pass over the finished text - collapses them into whichever answer it saw first.",+ "markdown": "[lit](https://[::1]/admin) and [enc](https://%5B::1%5D/admin)",+ "links": [+ {+ "href": "https://[::1]/admin",+ "label": "lit"+ },+ {+ "href": "https://%5B::1%5D/admin",+ "label": "enc"+ }+ ],+ "slack": [+ "https://[::1]/admin",+ "https://%5B::1%5D/admin"+ ]+ }+}diff --git a/tests/fixtures/citation_body_digest.json b/tests/fixtures/citation_body_digest.jsonnew file mode 100644index 000000000..4c07dc315--- /dev/null+++ b/tests/fixtures/citation_body_digest.json@@ -0,0 +1,42 @@+[+ {+ "name": "empty",+ "text": "",+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "name": "ascii-answer",+ "text": "Documented. [developers.openai.com](https://developers.openai.com/api/docs/guides/tools-web-search)",+ "sha256": "37df97b8515651bb4def4d91ac931dffc965ba5bcc8f611b4aa49a03a6c94a68"+ },+ {+ "name": "chinese",+ "text": "\u4e2d\u6587\u6a19\u984c\uff1a\u5df2\u5f15\u7528\u3002",+ "sha256": "227381de452d1e18dffce25aec8fc09cd182a1b10a297549e5e29f77ed803b5a"+ },+ {+ "name": "astral-emoji",+ "text": "\ud83d\ude42 done \u2705",+ "sha256": "d0bbed34afdad402430a900852380a2bb1d443c206bc50ea431e5b8cdda8af79"+ },+ {+ "name": "crlf",+ "text": "line one\r\nline two\r\n",+ "sha256": "aef727977006d394bfdaa5b3e2452614b26b447d6e1e2a43948e99588ba9c8d8"+ },+ {+ "name": "lone-high-surrogate",+ "text": "before \ud83d after",+ "sha256": "ca78c781d7c80e9927bed69e82f8bc93a87bd0d9dcd9d2497ee8553ea2af1f7f"+ },+ {+ "name": "lone-low-surrogate",+ "text": "before \ude42 after",+ "sha256": "8c846b9c049b19c34cb9a373e04fce8eb8b4a64826c7e99a6b714aa2f7f28ea7"+ },+ {+ "name": "mixed",+ "text": "\u4e2d\u6587\ud83d\ude42\r\n\ud800\uff5eend",+ "sha256": "75947d497b2bf2b406f8c2724be32e54d0c0af6d1011e8fb1c159daf16add50d"+ }+]diff --git a/tests/fixtures/citation_consumer_bridge.json b/tests/fixtures/citation_consumer_bridge.jsonnew file mode 100644index 000000000..ab3f95def--- /dev/null+++ b/tests/fixtures/citation_consumer_bridge.json@@ -0,0 +1,2505 @@+{+ "generated_by": "tests/citation_bridge.py",+ "cases": [+ {+ "key": "star_host",+ "why": "A host holding Markdown emphasis: the label must read as the host it names, and the address must arrive character for character.",+ "sources": {+ "turn0view0": {+ "url": "https://a*b*.example/x",+ "title": "Star Host"+ }+ },+ "agent_text": "Cited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Star Host",+ "url": "https://a*b*.example/x",+ "label": "a*b*.example",+ "spans": [+ [+ 7,+ 47+ ]+ ],+ "body_sha256": "eb9ddd8a0ef42c6e11907a0236f56ba62035d47bcdece9c2ccb1c6f4a0869115"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Star Host",+ "url": "https://a*b*.example/x",+ "label": "a*b*.example",+ "spans": [+ [+ 7,+ 47+ ]+ ],+ "body_sha256": "eb9ddd8a0ef42c6e11907a0236f56ba62035d47bcdece9c2ccb1c6f4a0869115"+ }+ ],+ "web_text": "Cited. [a\\*b\\*.example](https://a*b*.example/x)",+ "im_text": "Cited. [a\\*b\\*.example](https://a*b*.example/x)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "a*b*.example",+ "url": "https://a*b*.example/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "a*b*.example",+ "url": "https://a*b*.example/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "entity_host",+ "why": "A host that spells a character reference once decoded; the reader must see the reference, not whatever it would expand to.",+ "sources": {+ "turn0view0": {+ "url": "https://a%26copy%3B.example/x",+ "title": "Entity Host"+ }+ },+ "agent_text": "Cited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Entity Host",+ "url": "https://a%26copy%3B.example/x",+ "label": "a©.example",+ "spans": [+ [+ 7,+ 56+ ]+ ],+ "body_sha256": "70d0ad0977951744eecc54b7c52f7b63f7ffd25c80506afab29da20ddc76248e"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Entity Host",+ "url": "https://a%26copy%3B.example/x",+ "label": "a©.example",+ "spans": [+ [+ 7,+ 56+ ]+ ],+ "body_sha256": "70d0ad0977951744eecc54b7c52f7b63f7ffd25c80506afab29da20ddc76248e"+ }+ ],+ "web_text": "Cited. [a\\©.example](https://a%26copy%3B.example/x)",+ "im_text": "Cited. [a\\©.example](https://a%26copy%3B.example/x)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "a©.example",+ "url": "https://a%26copy%3B.example/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "a©.example",+ "url": "https://a%26copy%3B.example/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "underscore_path",+ "why": "Underscores in the path and the query - not CommonMark emphasis, but Slack mrkdwn and Telegram both read them as italics.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/a_b_c?q=_u_&r=1",+ "title": "Underscores"+ }+ },+ "agent_text": "Cited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Underscores",+ "url": "https://example.com/a_b_c?q=_u_&r=1",+ "label": "example.com",+ "spans": [+ [+ 7,+ 57+ ]+ ],+ "body_sha256": "d60ba5fe9949c542b856924a537416a64ba949f88fedbc5da02f54e05e8bd1cc"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Underscores",+ "url": "https://example.com/a_b_c?q=_u_&r=1",+ "label": "example.com",+ "spans": [+ [+ 7,+ 57+ ]+ ],+ "body_sha256": "d60ba5fe9949c542b856924a537416a64ba949f88fedbc5da02f54e05e8bd1cc"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/a_b_c?q=_u_&r=1)",+ "im_text": "Cited. [example.com](https://example.com/a_b_c?q=_u_&r=1)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/a_b_c?q=_u_&r=1",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/a_b_c?q=_u_&r=1",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "backtick_query",+ "why": "A backtick and an asterisk inside the query string, where a converter that scans destinations rewrites the address itself.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/p?q=%60code%60&r=a*b*",+ "title": "Query"+ }+ },+ "agent_text": "Cited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Query",+ "url": "https://example.com/p?q=%60code%60&r=a*b*",+ "label": "example.com",+ "spans": [+ [+ 7,+ 63+ ]+ ],+ "body_sha256": "7e8a6d3c876e54aceccd94efc38b593356a4dca9f42a3b9832f42b33946da0e5"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Query",+ "url": "https://example.com/p?q=%60code%60&r=a*b*",+ "label": "example.com",+ "spans": [+ [+ 7,+ 63+ ]+ ],+ "body_sha256": "7e8a6d3c876e54aceccd94efc38b593356a4dca9f42a3b9832f42b33946da0e5"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/p?q=%60code%60&r=a*b*)",+ "im_text": "Cited. [example.com](https://example.com/p?q=%60code%60&r=a*b*)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/p?q=%60code%60&r=a*b*",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/p?q=%60code%60&r=a*b*",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "format_chars",+ "why": "A zero-width space in the query, percent-encoded: invisible characters must not silently drop out of an address.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/p?q=a%E2%80%8Bb",+ "title": "Invisible"+ }+ },+ "agent_text": "Cited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Invisible",+ "url": "https://example.com/p?q=a%E2%80%8Bb",+ "label": "example.com",+ "spans": [+ [+ 7,+ 57+ ]+ ],+ "body_sha256": "f10a639c01574e9f9008ec6d8f5f883433c1c1410aea56764e0a1317c4a25bb8"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Invisible",+ "url": "https://example.com/p?q=a%E2%80%8Bb",+ "label": "example.com",+ "spans": [+ [+ 7,+ 57+ ]+ ],+ "body_sha256": "f10a639c01574e9f9008ec6d8f5f883433c1c1410aea56764e0a1317c4a25bb8"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/p?q=a%E2%80%8Bb)",+ "im_text": "Cited. [example.com](https://example.com/p?q=a%E2%80%8Bb)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/p?q=a%E2%80%8Bb",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/p?q=a%E2%80%8Bb",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "ipv6_port",+ "why": "A literal IPv6 authority with a port. The brackets are reserved authority syntax, and a link nobody can open is not attribution.",+ "sources": {+ "turn0view0": {+ "url": "https://[::1]:8443/x?q=1",+ "title": "Loopback"+ }+ },+ "agent_text": "Cited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Loopback",+ "url": "https://[::1]:8443/x?q=1",+ "label": "[::1]",+ "spans": [+ [+ 7,+ 42+ ]+ ],+ "body_sha256": "0b2ea3f074161b055c45398217c9c36b6be3340d24a47754b62fbe6062093e99"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Loopback",+ "url": "https://[::1]:8443/x?q=1",+ "label": "[::1]",+ "spans": [+ [+ 7,+ 42+ ]+ ],+ "body_sha256": "0b2ea3f074161b055c45398217c9c36b6be3340d24a47754b62fbe6062093e99"+ }+ ],+ "web_text": "Cited. [\\[::1\\]](https://[::1]:8443/x?q=1)",+ "im_text": "Cited. [\\[::1\\]](https://[::1]:8443/x?q=1)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "[::1]",+ "url": "https://[::1]:8443/x?q=1",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "[::1]",+ "url": "https://[::1]:8443/x?q=1",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "paren_path",+ "why": "Parentheses in the address, balanced, unbalanced and already percent-encoded, one of them beside a literal IPv6 host. Each is the page its source named, so the URL keeps it as written; only the link guards it, and every reader has to land on the same address.",+ "sources": {+ "turn0view0": {+ "url": "https://en.wikipedia.org/wiki/Foo_(bar)",+ "title": "Balanced"+ },+ "turn0view1": {+ "url": "https://example.com/a_(b?q=c)d",+ "title": "Unbalanced"+ },+ "turn0view2": {+ "url": "https://example.com/a%28b%29",+ "title": "Encoded"+ },+ "turn0view3": {+ "url": "https://[::1]:8443/x(y",+ "title": "Loopback"+ }+ },+ "agent_text": "One citeturn0view0, two citeturn0view1, three citeturn0view2, four citeturn0view3.",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Balanced",+ "url": "https://en.wikipedia.org/wiki/Foo_(bar)",+ "label": "en.wikipedia.org",+ "spans": [+ [+ 4,+ 65+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ },+ {+ "index": 2,+ "ref_id": "turn0view1",+ "title": "Unbalanced",+ "url": "https://example.com/a_(b?q=c)d",+ "label": "example.com",+ "spans": [+ [+ 71,+ 118+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ },+ {+ "index": 3,+ "ref_id": "turn0view2",+ "title": "Encoded",+ "url": "https://example.com/a%28b%29",+ "label": "example.com",+ "spans": [+ [+ 126,+ 169+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ },+ {+ "index": 4,+ "ref_id": "turn0view3",+ "title": "Loopback",+ "url": "https://[::1]:8443/x(y",+ "label": "[::1]",+ "spans": [+ [+ 176,+ 210+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Balanced",+ "url": "https://en.wikipedia.org/wiki/Foo_(bar)",+ "label": "en.wikipedia.org",+ "spans": [+ [+ 4,+ 65+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ },+ {+ "index": 2,+ "ref_id": "turn0view1",+ "title": "Unbalanced",+ "url": "https://example.com/a_(b?q=c)d",+ "label": "example.com",+ "spans": [+ [+ 71,+ 118+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ },+ {+ "index": 3,+ "ref_id": "turn0view2",+ "title": "Encoded",+ "url": "https://example.com/a%28b%29",+ "label": "example.com",+ "spans": [+ [+ 126,+ 169+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ },+ {+ "index": 4,+ "ref_id": "turn0view3",+ "title": "Loopback",+ "url": "https://[::1]:8443/x(y",+ "label": "[::1]",+ "spans": [+ [+ 176,+ 210+ ]+ ],+ "body_sha256": "1074ba2db78640f7e7ce0668e7efa7f227f5d8c1cb464e1cc671f1a6b672700d"+ }+ ],+ "web_text": "One [en.wikipedia.org](https://en.wikipedia.org/wiki/Foo_\\(bar\\)), two [example.com](https://example.com/a_\\(b?q=c\\)d), three [example.com](https://example.com/a%28b%29), four [\\[::1\\]](https://[::1]:8443/x\\(y).",+ "im_text": "One [en.wikipedia.org](https://en.wikipedia.org/wiki/Foo_\\(bar\\)), two [example.com](https://example.com/a_\\(b?q=c\\)d), three [example.com](https://example.com/a%28b%29), four [\\[::1\\]](https://[::1]:8443/x\\(y).",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "en.wikipedia.org",+ "url": "https://en.wikipedia.org/wiki/Foo_(bar)",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/a_(b?q=c)d",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/a%28b%29",+ "cited": true+ },+ {+ "label": "[::1]",+ "url": "https://[::1]:8443/x(y",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "en.wikipedia.org",+ "url": "https://en.wikipedia.org/wiki/Foo_(bar)",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/a_(b?q=c)d",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/a%28b%29",+ "cited": true+ },+ {+ "label": "[::1]",+ "url": "https://[::1]:8443/x(y",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "footnote",+ "why": "A GFM footnote definition holding a link to the same page. One renderer reads a link there and the other reads a definition, which is exactly the disagreement that cost the real citation its badge.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Footnoted"+ }+ },+ "agent_text": "[^f]: [p](https://example.com/x)\n\nCited. citeturn0view0 [^f]",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Footnoted",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 41,+ 77+ ]+ ],+ "body_sha256": "1d358a30bdb3319d4ab2c3ba1c1b9a83765b072976de5a780d01c945912f8532"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Footnoted",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 41,+ 77+ ]+ ],+ "body_sha256": "1d358a30bdb3319d4ab2c3ba1c1b9a83765b072976de5a780d01c945912f8532"+ }+ ],+ "web_text": "[^f]: [p](https://example.com/x)\n\nCited. [example.com](https://example.com/x) [^f]",+ "im_text": "[^f]: [p](https://example.com/x)\n\nCited. [example.com](https://example.com/x) [^f]",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "p",+ "url": "https://example.com/x",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "p",+ "url": "https://example.com/x",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": false+ },+ {+ "key": "code_fence",+ "why": "The citation's own exact spelling shown as a code example. It is not a link to any reader, but it is the same characters, and both ends have to count it the same way.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Fenced"+ }+ },+ "agent_text": "```\n[example.com](https://example.com/x)\n```\n\nCited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Fenced",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 53,+ 89+ ]+ ],+ "body_sha256": "196f4c941cdff7392175738fca699d1fcae22b399f6c2bcebf7dbf96cdf15b25"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Fenced",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 53,+ 89+ ]+ ],+ "body_sha256": "196f4c941cdff7392175738fca699d1fcae22b399f6c2bcebf7dbf96cdf15b25"+ }+ ],+ "web_text": "```\n[example.com](https://example.com/x)\n```\n\nCited. [example.com](https://example.com/x)",+ "im_text": "```\n[example.com](https://example.com/x)\n```\n\nCited. [example.com](https://example.com/x)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "table",+ "why": "The same spelling inside a GFM table cell, where `|` ends a cell and a label that spells one would split the row.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Tabled"+ }+ },+ "agent_text": "| Source |\n| --- |\n| [example.com](https://example.com/x) |\n\nCited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Tabled",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 68,+ 104+ ]+ ],+ "body_sha256": "c8084f3b9db805f5e8185a9e80378e01014d6faf9d298bec207f0e1fa57da81a"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Tabled",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 68,+ 104+ ]+ ],+ "body_sha256": "c8084f3b9db805f5e8185a9e80378e01014d6faf9d298bec207f0e1fa57da81a"+ }+ ],+ "web_text": "| Source |\n| --- |\n| [example.com](https://example.com/x) |\n\nCited. [example.com](https://example.com/x)",+ "im_text": "| Source |\n| --- |\n| [example.com](https://example.com/x) |\n\nCited. [example.com](https://example.com/x)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "near_html",+ "why": "The same spelling next to inline HTML, which each renderer is free to keep, escape or drop.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Inline HTML"+ }+ },+ "agent_text": "Cited. citeturn0view0 <span>[example.com](https://example.com/x)</span>",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Inline HTML",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "88379004dd3ee944926328fda72ebabcc25fd1992e0ba5a82f652af6d2d62449"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Inline HTML",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "88379004dd3ee944926328fda72ebabcc25fd1992e0ba5a82f652af6d2d62449"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/x) <span>[example.com](https://example.com/x)</span>",+ "im_text": "Cited. [example.com](https://example.com/x) <span>[example.com](https://example.com/x)</span>",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "prose_repeat",+ "why": "The agent points at the cited page in its own prose, before and after, spelled identically. Only the middle one is a citation, and an ordinary link must never be impersonated.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Repeated"+ }+ },+ "agent_text": "Background: [example.com](https://example.com/x)\n\nCited. citeturn0view0\n\nAlso [example.com](https://example.com/x)",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Repeated",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 57,+ 93+ ]+ ],+ "body_sha256": "9a299f42bc345b737f2bfbf872b5138de742155daee254895c7763362f298c95"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Repeated",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 57,+ 93+ ]+ ],+ "body_sha256": "9a299f42bc345b737f2bfbf872b5138de742155daee254895c7763362f298c95"+ }+ ],+ "web_text": "Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)\n\nAlso [example.com](https://example.com/x)",+ "im_text": "Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)\n\nAlso [example.com](https://example.com/x)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": false+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "emoji_prefix",+ "why": "Non-ASCII text before the marker: offsets counted in characters on one side and in UTF-16 units on the other stop agreeing here.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Emoji"+ }+ },+ "agent_text": "🔎 出处 citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Emoji",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 6,+ 42+ ]+ ],+ "body_sha256": "ba197e0279ad263511d1e508d3eb689079a1304d48d722d3ecf245f897c59b19"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Emoji",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 6,+ 42+ ]+ ],+ "body_sha256": "ba197e0279ad263511d1e508d3eb689079a1304d48d722d3ecf245f897c59b19"+ }+ ],+ "web_text": "🔎 出处 [example.com](https://example.com/x)",+ "im_text": "🔎 出处 [example.com](https://example.com/x)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "silent_strip",+ "why": "A silent control block holding the same spelling. Delivery removes it, so the consumer never counts it - and neither may the producer.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Silent"+ }+ },+ "agent_text": "<silent>note to self: [example.com](https://example.com/x)</silent>\nCited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Silent",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "c72f94d17bbddfd9039b915493398ecd3903cf39e2cb39cbdd92be4c3d71bf6b"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Silent",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "c72f94d17bbddfd9039b915493398ecd3903cf39e2cb39cbdd92be4c3d71bf6b"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/x)",+ "im_text": "Cited. [example.com](https://example.com/x)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "file_link",+ "why": "A file:// attachment link in the same reply: IM strips it to a plain label and the workbench keeps it, so the two surfaces get different text and the citation has to survive both.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "With attachment"+ }+ },+ "agent_text": "Cited. citeturn0view0\n\n[report](file:///tmp/avibe-citation-bridge/report.txt)",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "With attachment",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "3f9c32506f24ea62a4396ba0fc4cbfc5272ee30ef04d00c740eb757321e0d02d"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "With attachment",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "f68ec6e8a1d71f8934f8fd3569c471916996ff250c7a695ae718849d29bd1095"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/x)\n\n[report](file:///tmp/avibe-citation-bridge/report.txt)",+ "im_text": "Cited. [example.com](https://example.com/x)\n\nreport",+ "quick_replies": [],+ "files": [+ {+ "label": "report",+ "path": "/tmp/avibe-citation-bridge/report.txt"+ }+ ],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "quick_replies",+ "why": "A trailing quick-reply block, which delivery strips from the body on both surfaces.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "With buttons"+ }+ },+ "agent_text": "Cited. citeturn0view0\n\n---\n[👌 继续] | [✅ 完成]",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "With buttons",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "c72f94d17bbddfd9039b915493398ecd3903cf39e2cb39cbdd92be4c3d71bf6b"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "With buttons",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "c72f94d17bbddfd9039b915493398ecd3903cf39e2cb39cbdd92be4c3d71bf6b"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/x)",+ "im_text": "Cited. [example.com](https://example.com/x)",+ "quick_replies": [+ "👌 继续",+ "✅ 完成"+ ],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "marker_shaped",+ "why": "The workbench rewrites `$<NAME>` into a secure-input card and `@<…>`/`#<…>` into chips by editing the source text before Markdown parses it - so a reply carrying those markers, and a source address spelled like one, is where an inserted card could move or swallow the link the producer counted.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/p?q=$<openAiKey>&a=@<claude>",+ "title": "Marker Shaped"+ }+ },+ "agent_text": "Provide $<openAiKey>, then ping @<claude> in #<ses6jr7c5h2q6>.\n\nCited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Marker Shaped",+ "url": "https://example.com/p?q=$%3CopenAiKey%3E&a=@%3Cclaude%3E",+ "label": "example.com",+ "spans": [+ [+ 71,+ 142+ ]+ ],+ "body_sha256": "6513e2bd02eb906da3150bbe37b8314b6ccc000a25c128d78abc37660b82017e"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Marker Shaped",+ "url": "https://example.com/p?q=$%3CopenAiKey%3E&a=@%3Cclaude%3E",+ "label": "example.com",+ "spans": [+ [+ 71,+ 142+ ]+ ],+ "body_sha256": "6513e2bd02eb906da3150bbe37b8314b6ccc000a25c128d78abc37660b82017e"+ }+ ],+ "web_text": "Provide $<openAiKey>, then ping @<claude> in #<ses6jr7c5h2q6>.\n\nCited. [example.com](https://example.com/p?q=$%3CopenAiKey%3E&a=@%3Cclaude%3E)",+ "im_text": "Provide $<openAiKey>, then ping @<claude> in #<ses6jr7c5h2q6>.\n\nCited. [example.com](https://example.com/p?q=$%3CopenAiKey%3E&a=@%3Cclaude%3E)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/p?q=$%3CopenAiKey%3E&a=@%3Cclaude%3E",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/p?q=$%3CopenAiKey%3E&a=@%3Cclaude%3E",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "multi_source",+ "why": "Two sources in one reply, each with its own index and its own spelling.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "First"+ },+ "turn0view1": {+ "url": "https://a*b*.example/y",+ "title": "Second"+ }+ },+ "agent_text": "First citeturn0view0, second citeturn0view1.",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "First",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 6,+ 42+ ]+ ],+ "body_sha256": "692dd53dbdc2ece7fa2bde05c6b8b321828f02f51586299c27df55c1b75e1b02"+ },+ {+ "index": 2,+ "ref_id": "turn0view1",+ "title": "Second",+ "url": "https://a*b*.example/y",+ "label": "a*b*.example",+ "spans": [+ [+ 51,+ 91+ ]+ ],+ "body_sha256": "692dd53dbdc2ece7fa2bde05c6b8b321828f02f51586299c27df55c1b75e1b02"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "First",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 6,+ 42+ ]+ ],+ "body_sha256": "692dd53dbdc2ece7fa2bde05c6b8b321828f02f51586299c27df55c1b75e1b02"+ },+ {+ "index": 2,+ "ref_id": "turn0view1",+ "title": "Second",+ "url": "https://a*b*.example/y",+ "label": "a*b*.example",+ "spans": [+ [+ 51,+ 91+ ]+ ],+ "body_sha256": "692dd53dbdc2ece7fa2bde05c6b8b321828f02f51586299c27df55c1b75e1b02"+ }+ ],+ "web_text": "First [example.com](https://example.com/x), second [a\\*b\\*.example](https://a*b*.example/y).",+ "im_text": "First [example.com](https://example.com/x), second [a\\*b\\*.example](https://a*b*.example/y).",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ },+ {+ "label": "a*b*.example",+ "url": "https://a*b*.example/y",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ },+ {+ "label": "a*b*.example",+ "url": "https://a*b*.example/y",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "link_label",+ "why": "The marker written inside a link's own label. A citation is a link, and a link inside a link is not one - CommonMark makes the outer brackets text again - so the source is shown after the unit that encloses it, with that unit's destination and the label's other words untouched. Both links name the same page here, which is where a binding that matched on the address would badge the wrong one.",+ "sources": {+ "turn0view0": {+ "url": "https://openai.com/docs",+ "title": "OpenAI Docs"+ }+ },+ "agent_text": "See [the docs citeturn0view0](https://openai.com/docs) for details.",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "OpenAI Docs",+ "url": "https://openai.com/docs",+ "label": "openai.com",+ "spans": [+ [+ 41,+ 78+ ]+ ],+ "body_sha256": "9bd459fd35f007f8c109c5b009ac890a807491960fad853a2b1a94abe5d30eb5"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "OpenAI Docs",+ "url": "https://openai.com/docs",+ "label": "openai.com",+ "spans": [+ [+ 41,+ 78+ ]+ ],+ "body_sha256": "9bd459fd35f007f8c109c5b009ac890a807491960fad853a2b1a94abe5d30eb5"+ }+ ],+ "web_text": "See [the docs ](https://openai.com/docs) [openai.com](https://openai.com/docs) for details.",+ "im_text": "See [the docs ](https://openai.com/docs) [openai.com](https://openai.com/docs) for details.",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "the docs ",+ "url": "https://openai.com/docs",+ "cited": false+ },+ {+ "label": "openai.com",+ "url": "https://openai.com/docs",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "the docs ",+ "url": "https://openai.com/docs",+ "cited": false+ },+ {+ "label": "openai.com",+ "url": "https://openai.com/docs",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "link_label_positions",+ "why": "One source cited from inside three labels - opening one, in the middle of the next, closing the third. One row, three spans, and each source after its own unit rather than after the last of them.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Thrice"+ }+ },+ "agent_text": "[citeturn0view0 a](https://u.example/1), [b citeturn0view0 c](https://u.example/2), [d citeturn0view0](https://u.example/3).",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Thrice",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 26,+ 62+ ],+ [+ 92,+ 128+ ],+ [+ 156,+ 192+ ]+ ],+ "body_sha256": "992e162c046453519eb0dc150c297ceaa10270ee234bd3ee67d1514c95073be3"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Thrice",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 26,+ 62+ ],+ [+ 92,+ 128+ ],+ [+ 156,+ 192+ ]+ ],+ "body_sha256": "992e162c046453519eb0dc150c297ceaa10270ee234bd3ee67d1514c95073be3"+ }+ ],+ "web_text": "[ a](https://u.example/1) [example.com](https://example.com/s), [b c](https://u.example/2) [example.com](https://example.com/s), [d ](https://u.example/3) [example.com](https://example.com/s).",+ "im_text": "[ a](https://u.example/1) [example.com](https://example.com/s), [b c](https://u.example/2) [example.com](https://example.com/s), [d ](https://u.example/3) [example.com](https://example.com/s).",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": " a",+ "url": "https://u.example/1",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "b c",+ "url": "https://u.example/2",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "d ",+ "url": "https://u.example/3",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": " a",+ "url": "https://u.example/1",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "b c",+ "url": "https://u.example/2",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "d ",+ "url": "https://u.example/3",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "data_slots",+ "why": "The slots of a link that are address or syntax rather than anything a reader is shown: a destination, a title, an angle-bracketed destination, and an autolink, which is all address. A marker there is part of what makes the unit work, so it stays the characters the model typed - no token minted, no source fetched, no link spliced in - while the one in the prose beside them is still a citation.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Prose"+ }+ },+ "agent_text": "Cited. citeturn0view0\n\n[a](https://example.com/citeturn0view0path) [b](https://example.com/y \"T citeturn0view0\") [c](<https://example.com/zciteturn0view0>) <https://example.com/autociteturn0view0>",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Prose",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "a384366aa828fe9aeb90f87aab6eb2d0fbb18a657fd6c59353c046ee02c8f8ab"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Prose",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "a384366aa828fe9aeb90f87aab6eb2d0fbb18a657fd6c59353c046ee02c8f8ab"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/s)\n\n[a](https://example.com/citeturn0view0path) [b](https://example.com/y \"T citeturn0view0\") [c](<https://example.com/zciteturn0view0>) <https://example.com/autociteturn0view0>",+ "im_text": "Cited. [example.com](https://example.com/s)\n\n[a](https://example.com/citeturn0view0path) [b](https://example.com/y \"T citeturn0view0\") [c](<https://example.com/zciteturn0view0>) <https://example.com/autociteturn0view0>",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "a",+ "url": "https://example.com/citeturn0view0path",+ "cited": false+ },+ {+ "label": "b",+ "url": "https://example.com/y",+ "cited": false+ },+ {+ "label": "c",+ "url": "https://example.com/zciteturn0view0",+ "cited": false+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "a",+ "url": "https://example.com/citeturn0view0path",+ "cited": false+ },+ {+ "label": "b",+ "url": "https://example.com/y",+ "cited": false+ },+ {+ "label": "c",+ "url": "https://example.com/zciteturn0view0",+ "cited": false+ }+ ],+ "anchors_cover_destinations": false,+ "anchors_exhaustive": false+ },+ {+ "key": "reference_slots",+ "why": "A reference link written in full shows its label and names its definition separately, so the label is a place a citation can go. Written short, the same brackets are both - the text a reader sees and the identifier that finds the address - so editing them would leave a link pointing at nothing. Those stay exactly as written, definition included.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Referenced"+ }+ },+ "agent_text": "[the docs citeturn0view0][ref], [dual citeturn0view0] and [dual citeturn0view0][].\n\n[ref]: https://r.example/page\n[dual citeturn0view0]: https://r.example/dual",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Referenced",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 17,+ 53+ ]+ ],+ "body_sha256": "e0e4bfa542b198c14f0171bcc3be1e1b596b1e0a4659044cd5ebc47cef7d7617"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Referenced",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 17,+ 53+ ]+ ],+ "body_sha256": "e0e4bfa542b198c14f0171bcc3be1e1b596b1e0a4659044cd5ebc47cef7d7617"+ }+ ],+ "web_text": "[the docs ][ref] [example.com](https://example.com/s), [dual citeturn0view0] and [dual citeturn0view0][].\n\n[ref]: https://r.example/page\n[dual citeturn0view0]: https://r.example/dual",+ "im_text": "[the docs ][ref] [example.com](https://example.com/s), [dual citeturn0view0] and [dual citeturn0view0][].\n\n[ref]: https://r.example/page\n[dual citeturn0view0]: https://r.example/dual",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "anchors_cover_destinations": false,+ "anchors_exhaustive": false+ },+ {+ "key": "image_slots",+ "why": "An image's alt text is what a reader is given when the picture is not there; its src is an address. One marker is a citation and the other is not, in one line. The component does not fetch a foreign image - it renders a click-through link instead - so both are links on this surface.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Illustrated"+ }+ },+ "agent_text": " and ",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Illustrated",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 39,+ 75+ ]+ ],+ "body_sha256": "de5142f360c26f90728c7e4ad2d7588ed5e3b837524908a6c9e5c15d89884d7a"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Illustrated",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 39,+ 75+ ]+ ],+ "body_sha256": "de5142f360c26f90728c7e4ad2d7588ed5e3b837524908a6c9e5c15d89884d7a"+ }+ ],+ "web_text": " [example.com](https://example.com/s) and ",+ "im_text": " [example.com](https://example.com/s) and ",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "anchors_cover_destinations": false,+ "anchors_exhaustive": false+ },+ {+ "key": "image_in_link",+ "why": "An image inside a link: two units, one holding the other. The source belongs after the outer one. Placed after the image instead it would sit in the link's label and take the link down with it.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Nested"+ }+ },+ "agent_text": "[](https://out.example/page)",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Nested",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 67,+ 103+ ]+ ],+ "body_sha256": "c8d053801e35938d8a6b465e979825b9b732666fa3aab28c5477b961f84318b5"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Nested",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 67,+ 103+ ]+ ],+ "body_sha256": "c8d053801e35938d8a6b465e979825b9b732666fa3aab28c5477b961f84318b5"+ }+ ],+ "web_text": "[](https://out.example/page) [example.com](https://example.com/s)",+ "im_text": "[](https://out.example/page) [example.com](https://example.com/s)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": null,+ "url": "https://out.example/page",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": null,+ "url": "https://out.example/page",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "anchors_cover_destinations": false,+ "anchors_exhaustive": false+ },+ {+ "key": "moved_by_a_transform",+ "why": "Delivery removes silent blocks, and what closes up behind them leaves a registered token somewhere the model never put it. Each one is judged where it ends up rather than where it started: inside a label it is still a citation and is shown after that link, inside a destination or a title it is data again and the marker the model typed comes back - the alternative is a link nobody can open and a source claimed where no reader can see it.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Moved"+ }+ },+ "agent_text": "A [lab <silent>](https://z.example) x</silent>citeturn0view0](https://out.example/page) B\n\nC [two](https://z.example<silent>) y</silent>citeturn0view0) D\n\nE [three](https://z.example \"t<silent>\") y</silent>citeturn0view0\") F",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Moved",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 35,+ 71+ ]+ ],+ "body_sha256": "695f872cc2e6bce140460a0e0f174c94370c6bb2905e90ffd43ac1e3dc12aa7a"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Moved",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 35,+ 71+ ]+ ],+ "body_sha256": "695f872cc2e6bce140460a0e0f174c94370c6bb2905e90ffd43ac1e3dc12aa7a"+ }+ ],+ "web_text": "A [lab ](https://out.example/page) [example.com](https://example.com/s) B\n\nC [two](https://z.exampleciteturn0view0) D\n\nE [three](https://z.example \"tciteturn0view0\") F",+ "im_text": "A [lab ](https://out.example/page) [example.com](https://example.com/s) B\n\nC [two](https://z.exampleciteturn0view0) D\n\nE [three](https://z.example \"tciteturn0view0\") F",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "lab ",+ "url": "https://out.example/page",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "two",+ "url": "https://z.exampleciteturn0view0",+ "cited": false+ },+ {+ "label": "three",+ "url": "https://z.example",+ "cited": false+ }+ ],+ "im_anchors": [+ {+ "label": "lab ",+ "url": "https://out.example/page",+ "cited": false+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "two",+ "url": "https://z.exampleciteturn0view0",+ "cited": false+ },+ {+ "label": "three",+ "url": "https://z.example",+ "cited": false+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "open_fence",+ "why": "A fence the answer never closed takes the rest of the reply into code. The citation's own spelling in there is not a link to any renderer here, and a marker in there is not a citation either - which is the same rule the closed fence above gets, decided by the parser rather than by whether a closing line happens to exist.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Unclosed"+ }+ },+ "agent_text": "Cited. citeturn0view0\n\n```\n[example.com](https://example.com/s)\nciteturn0view0 still open",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Unclosed",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "55f17c8c69883e01ff3aa65ae70d8149f2dc53dd14b30c3f76065d4203eadd6b"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Unclosed",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ]+ ],+ "body_sha256": "55f17c8c69883e01ff3aa65ae70d8149f2dc53dd14b30c3f76065d4203eadd6b"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/s)\n\n```\n[example.com](https://example.com/s)\nciteturn0view0 still open",+ "im_text": "Cited. [example.com](https://example.com/s)\n\n```\n[example.com](https://example.com/s)\nciteturn0view0 still open",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "file_label_and_quick_reply",+ "why": "A marker in an attachment's label and in a quick-reply button. Neither is Markdown a reader parses - one titles a file card, the other is a button - so the attribution is plain text left where it stands, with no link syntax and no token reaching a structured field. In the body the attachment is still a link, and CommonMark refuses ``file:`` outright: a scan that cannot see that unit would put the source inside its label and break the card.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/s",+ "title": "Attached"+ }+ },+ "agent_text": "Cited. citeturn0view0\n\n[report citeturn0view0](file:///tmp/avibe-citation-bridge/report.txt)\n\n---\n[👌 继续 citeturn0view0] | [✅ 完成]",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Attached",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ],+ [+ 101,+ 137+ ]+ ],+ "body_sha256": "7c0134b567efd0e1b38c11d0b26bff8a157a41719cf5b3f31b4342d5713eea16"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Attached",+ "url": "https://example.com/s",+ "label": "example.com",+ "spans": [+ [+ 7,+ 43+ ],+ [+ 52,+ 88+ ]+ ],+ "body_sha256": "5fe6fc09cbd662d232422fffd26ad244b476c2d4aaa501aacda5587cf52ec8a8"+ }+ ],+ "web_text": "Cited. [example.com](https://example.com/s)\n\n[report ](file:///tmp/avibe-citation-bridge/report.txt) [example.com](https://example.com/s)",+ "im_text": "Cited. [example.com](https://example.com/s)\n\nreport [example.com](https://example.com/s)",+ "quick_replies": [+ "👌 继续 example.com",+ "✅ 完成"+ ],+ "files": [+ {+ "label": "report example.com",+ "path": "/tmp/avibe-citation-bridge/report.txt"+ }+ ],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ },+ {+ "label": "example.com",+ "url": "https://example.com/s",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "splice",+ "why": "Stripping a silent block splices two ordinary halves into characters that read as a citation marker. Only the first marker here was ever one, and the delivered text has to say so.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Spliced"+ }+ },+ "agent_text": "Actual. citeturn0view0\n\nSynthetic. ci<silent>private</silent>teturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Spliced",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 8,+ 44+ ]+ ],+ "body_sha256": "9bcdee138d4e0ae2e733ea5a2a7607cfcc01a34530a64595bd20d8bc11cc8a07"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Spliced",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 8,+ 44+ ]+ ],+ "body_sha256": "9bcdee138d4e0ae2e733ea5a2a7607cfcc01a34530a64595bd20d8bc11cc8a07"+ }+ ],+ "web_text": "Actual. [example.com](https://example.com/x)\n\nSynthetic. citeturn0view0",+ "im_text": "Actual. [example.com](https://example.com/x)\n\nSynthetic. citeturn0view0",+ "quick_replies": [],+ "files": [],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "file_delete_splice",+ "why": "Two deletions that cancel out. Removing the attachment link to its label and removing the silent block leave characters that read as a marker on IM and do not on the workbench - one reply, two bodies, and a marker shape that was never in the input either was.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Cancelled out"+ }+ },+ "agent_text": "Actual. citeturn0view0\n\nSynthetic. ci[te](file:///tmp/avibe-citation-bridge/report.txt)turn0view0<silent>note</silent>",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cancelled out",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 8,+ 44+ ]+ ],+ "body_sha256": "a7de4b71ffbc3a7224deb9ec92a71756eb37e011afa62a2469d98c9d1d0c66a3"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cancelled out",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 8,+ 44+ ]+ ],+ "body_sha256": "9bcdee138d4e0ae2e733ea5a2a7607cfcc01a34530a64595bd20d8bc11cc8a07"+ }+ ],+ "web_text": "Actual. [example.com](https://example.com/x)\n\nSynthetic. ci[te](file:///tmp/avibe-citation-bridge/report.txt)turn0view0",+ "im_text": "Actual. [example.com](https://example.com/x)\n\nSynthetic. citeturn0view0",+ "quick_replies": [],+ "files": [+ {+ "label": "te",+ "path": "/tmp/avibe-citation-bridge/report.txt"+ }+ ],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "file_link_above",+ "why": "A file:// attachment above the citation: IM flattens it to a label and the workbench keeps it, so one reply puts the same citation at two different offsets and each body needs its own measurement.",+ "sources": {+ "turn0view0": {+ "url": "https://example.com/x",+ "title": "Shifted"+ }+ },+ "agent_text": "[report](file:///tmp/avibe-citation-bridge/report.txt)\n\nCited. citeturn0view0",+ "web_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Shifted",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 63,+ 99+ ]+ ],+ "body_sha256": "630a79407e8e7ea5926302abc0ebb12a6791dc904d3d8fba51a97cf9ee209225"+ }+ ],+ "im_citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Shifted",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 15,+ 51+ ]+ ],+ "body_sha256": "97bace9967a0f0f9bada602b695161fa12d57fa0696453b7323220e2a68e7b3b"+ }+ ],+ "web_text": "[report](file:///tmp/avibe-citation-bridge/report.txt)\n\nCited. [example.com](https://example.com/x)",+ "im_text": "report\n\nCited. [example.com](https://example.com/x)",+ "quick_replies": [],+ "files": [+ {+ "label": "report",+ "path": "/tmp/avibe-citation-bridge/report.txt"+ }+ ],+ "web_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "im_anchors": [+ {+ "label": "example.com",+ "url": "https://example.com/x",+ "cited": true+ }+ ],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ },+ {+ "key": "unresolved",+ "why": "A marker naming a source the backend never captured. It degrades to a visible label rather than vanishing, and writes no sidecar row.",+ "sources": {},+ "agent_text": "Claimed. citeturn0view0",+ "web_citations": [],+ "im_citations": [],+ "web_text": "Claimed. (source unavailable)",+ "im_text": "Claimed. (source unavailable)",+ "quick_replies": [],+ "files": [],+ "web_anchors": [],+ "im_anchors": [],+ "anchors_cover_destinations": true,+ "anchors_exhaustive": true+ }+ ],+ "rows": [+ {+ "key": "im_row_with_footer",+ "why": "An ordinary IM transcript row. The dispatcher folds the footer into the stored body and stores it structured as well, so the Web reader removes a trailing range before rendering - the case where adding a body digest could have cost every IM row its badge.",+ "platform": "slack",+ "type": "result",+ "row": {+ "type": "result",+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)\n\n✅ ⏱️ 5s · 🪙 1.2k tok",+ "content": {+ "kind": "result",+ "result_footer": "✅ ⏱️ 5s · 🪙 1.2k tok",+ "citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cited page",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 60,+ 96+ ]+ ],+ "body_sha256": "667f719a1b5f9adc6002fb8b5f12b73441a6c0256bf2065accffbbfcfe7f3c2c"+ }+ ],+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)\n\n✅ ⏱️ 5s · 🪙 1.2k tok"+ }+ }+ },+ {+ "key": "im_row_without_footer",+ "why": "The same row with nothing to strip: the body the ranges were measured in is the body the renderer is handed.",+ "platform": "slack",+ "type": "result",+ "row": {+ "type": "result",+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)",+ "content": {+ "kind": "result",+ "citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cited page",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 60,+ 96+ ]+ ],+ "body_sha256": "222df127c10b93d7f38a3bf37a27695e91c6c8add0d8b1617fb90f69ed56143a"+ }+ ],+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)"+ }+ }+ },+ {+ "key": "workbench_row_with_footer",+ "why": "The workbench stores a clean body and its footer apart, so the same reply reaches the same reader through a different split.",+ "platform": "avibe",+ "type": "result",+ "row": {+ "type": "result",+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)",+ "content": {+ "kind": "result",+ "result_footer": "✅ ⏱️ 2m 24s",+ "citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cited page",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 60,+ 96+ ]+ ],+ "body_sha256": "222df127c10b93d7f38a3bf37a27695e91c6c8add0d8b1617fb90f69ed56143a"+ }+ ],+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)"+ }+ }+ },+ {+ "key": "activity_row",+ "why": "An interim narration row, which the activity card renders verbatim - no footer, no split, and its own consumer.",+ "platform": "avibe",+ "type": "assistant",+ "row": {+ "type": "assistant",+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)",+ "content": {+ "kind": "assistant",+ "citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cited page",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 60,+ 96+ ]+ ],+ "body_sha256": "222df127c10b93d7f38a3bf37a27695e91c6c8add0d8b1617fb90f69ed56143a"+ }+ ],+ "text": "🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)"+ }+ }+ },+ {+ "key": "workbench_row_rewritten_and_carded",+ "why": "Every transform a stored row can stack, in the order they happen and all of them above the citation: an attachment link is rewritten to the media proxy before the sidecar is measured, a footer is folded in and stripped back out by the reader, and a `$<NAME>` marker becomes a secure-input card as the reader renders it. Each one moves the citation, and only one of them was measured by the backend.",+ "platform": "avibe",+ "type": "result",+ "row": {+ "type": "result",+ "text": "[report](/api/media/7EPiDrihQUpry6om5gCmSg)\n\nProvide $<deployKey> first.\n\n🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)",+ "content": {+ "kind": "result",+ "result_footer": "✅ ⏱️ 2m 24s",+ "citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cited page",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 134,+ 170+ ]+ ],+ "body_sha256": "0a527ac2acef9af97986b634a9d0913cdcefb058e50e8fa3bef90f9300fa1be5"+ }+ ],+ "text": "[report](/api/media/7EPiDrihQUpry6om5gCmSg)\n\nProvide $<deployKey> first.\n\n🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)"+ }+ }+ },+ {+ "key": "im_row_flattened_carded_and_folded",+ "why": "The same reply down the IM path, where the transforms stack differently: the attachment is flattened to a bare label instead of a proxy link, and the footer is folded into the stored body. So the reader runs two edits over one body - a trailing strip and an inserted secure-input card above the citation - and the badge has to survive their composition.",+ "platform": "slack",+ "type": "result",+ "row": {+ "type": "result",+ "text": "report\n\nProvide $<deployKey> first.\n\n🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)\n\n✅ ⏱️ 5s",+ "content": {+ "kind": "result",+ "result_footer": "✅ ⏱️ 5s",+ "citations": [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Cited page",+ "url": "https://example.com/x",+ "label": "example.com",+ "spans": [+ [+ 97,+ 133+ ]+ ],+ "body_sha256": "257292188fb8f7f9721c283a12b2b9f6c56c41cde67b8e1159303fcfa94bd5df"+ }+ ],+ "text": "report\n\nProvide $<deployKey> first.\n\n🔎 Background: [example.com](https://example.com/x)\n\nCited. [example.com](https://example.com/x)\n\n✅ ⏱️ 5s"+ }+ }+ }+ ]+}diff --git a/tests/fixtures/citation_url_identity.json b/tests/fixtures/citation_url_identity.jsonnew file mode 100644index 000000000..a5aae4c96--- /dev/null+++ b/tests/fixtures/citation_url_identity.json@@ -0,0 +1,341 @@+{+ "description": "One shared table for the URL a citation is allowed to carry.\n\nA citation is delivered twice: as the destination of an ordinary Markdown link every surface can follow, and as the `url` of a `message.content.citations` row. The Web renderer draws a badge only when the two agree exactly, so the backend has to write the destination the real Markdown pipeline (remark-parse -> remark-rehype's normalizeUri -> defaultUrlTransform) produces from it - a fixed point - and has to label it with the host a browser actually resolves.\n\nBoth sides of that boundary read this file: tests/test_citations.py asserts core.citations produces `url`/`label` from `raw`, and ui/src/components/ui/markdown.test.tsx renders `url` and asserts the DOM href and the WHATWG host come back unchanged. Neither side can drift without the other failing. `url: null` means the raw value must be rejected outright rather than repaired.\n\n`authority` is the same table for the authority - userinfo, host and port - where the browser is the assertion rather than a reference: every accepted row's `url` is one `new URL` opens rather than refuses, and `new URL(url).hostname` is exactly that row's `label`. The authority stays as the provider wrote it - letter case, a leading zero in a port, an uncompressed IPv6 address - because each of those spellings opens the same page, and the label is where the browser's own answer is shown instead. A `url: null` row is one a browser refuses, apart from the single deliberate exception whose `why` says so. A new authority shape is a row here, not a branch.",+ "cases": [+ {+ "why": "an ordinary result needs no rewriting at all",+ "raw": "https://developers.openai.com/api/docs/guides/tools-web-search",+ "url": "https://developers.openai.com/api/docs/guides/tools-web-search",+ "label": "developers.openai.com"+ },+ {+ "why": "a non-ASCII path: the renderer percent-encodes it, so the backend must too",+ "raw": "https://example.com/中文",+ "url": "https://example.com/%E4%B8%AD%E6%96%87",+ "label": "example.com"+ },+ {+ "why": "a non-ASCII host resolves through IDNA, so the label is the punycode the browser visits",+ "raw": "https://例え.jp/x",+ "url": "https://%E4%BE%8B%E3%81%88.jp/x",+ "label": "xn--r8jz45g.jp"+ },+ {+ "why": "P1: IDNA 2003 folds ss out of a German host, naming a domain the link never opens",+ "raw": "https://faß.de/",+ "url": "https://fa%C3%9F.de/",+ "label": "xn--fa-hia.de"+ },+ {+ "why": "UTS #46 maps a fullwidth host onto ASCII, so no punycode is left to show",+ "raw": "https://EXAMPLE.com/x",+ "url": "https://%EF%BC%A5%EF%BC%B8%EF%BC%A1%EF%BC%AD%EF%BC%B0%EF%BC%AC%EF%BC%A5.com/x",+ "label": "example.com"+ },+ {+ "why": "a Cyrillic look-alike is named by the punycode it is, never by the brand it imitates",+ "raw": "https://аррӏе.com/",+ "url": "https://%D0%B0%D1%80%D1%80%D3%8F%D0%B5.com/",+ "label": "xn--80ak6aa92e.com"+ },+ {+ "why": "UTS #46 ignores a soft hyphen inside a host, so the label is the host without it",+ "raw": "https://a\u00adb.example/",+ "url": "https://a%C2%ADb.example/",+ "label": "ab.example"+ },+ {+ "why": "only a non-ASCII label takes the IDNA path; UTS #46 rejects an ASCII one a browser keeps",+ "raw": "https://my_site.例え.jp/",+ "url": "https://my_site.%E4%BE%8B%E3%81%88.jp/",+ "label": "my_site.xn--r8jz45g.jp"+ },+ {+ "why": "an escaped host keeps its case through urlsplit, and a browser lowercases it anyway",+ "raw": "https://例え.JP/x",+ "url": "https://%E4%BE%8B%E3%81%88.JP/x",+ "label": "xn--r8jz45g.jp"+ },+ {+ "why": "IDNA 2008 disallows a symbol host a browser still resolves, and no label beats a wrong one",+ "raw": "https://❤.example/",+ "url": null,+ "label": null+ },+ {+ "why": "parentheses are part of the address and stay literal; only the Markdown link escapes them",+ "raw": "https://en.wikipedia.org/wiki/Cite_(disambiguation)",+ "url": "https://en.wikipedia.org/wiki/Cite_(disambiguation)",+ "label": "en.wikipedia.org"+ },+ {+ "why": "a space cannot survive a Markdown destination unencoded",+ "raw": "https://example.com/a b",+ "url": "https://example.com/a%20b",+ "label": "example.com"+ },+ {+ "why": "an escape that already looks like one is preserved, never double-encoded",+ "raw": "https://example.com/p%41q",+ "url": "https://example.com/p%41q",+ "label": "example.com"+ },+ {+ "why": "a literal percent is not an escape",+ "raw": "https://example.com/100%",+ "url": "https://example.com/100%25",+ "label": "example.com"+ },+ {+ "why": "two characters that are not both alphanumeric do not make an escape",+ "raw": "https://example.com/%-a",+ "url": "https://example.com/%25-a",+ "label": "example.com"+ },+ {+ "why": "P1: a raw backslash in the authority sends urlsplit and the browser to different hosts, so it names no source",+ "raw": "https://attacker.example\\@developers.openai.com/page",+ "url": null,+ "label": null+ },+ {+ "why": "P1: a backslash inside the host itself is not a host a browser will resolve",+ "raw": "https://attacker.example\\developers.openai.com/",+ "url": null,+ "label": null+ },+ {+ "why": "P1: a numeric host names the address a browser resolves, not the digits it spells",+ "raw": "https://2130706433/p",+ "url": "https://2130706433/p",+ "label": "127.0.0.1"+ },+ {+ "why": "P1: hexadecimal and shortened IPv4 forms reach the same loopback address",+ "raw": "https://0x7f.1/p",+ "url": "https://0x7f.1/p",+ "label": "127.0.0.1"+ },+ {+ "why": "an octal host is still the address it encodes",+ "raw": "https://017700000001/p",+ "url": "https://017700000001/p",+ "label": "127.0.0.1"+ },+ {+ "why": "a leading zero makes a part octal, so the label is not the digits read as decimal",+ "raw": "https://010.0.0.1/p",+ "url": "https://010.0.0.1/p",+ "label": "8.0.0.1"+ },+ {+ "why": "an already canonical address is left exactly as it is",+ "raw": "https://1.2.3.4/p",+ "url": "https://1.2.3.4/p",+ "label": "1.2.3.4"+ },+ {+ "why": "a numeric host the IPv4 parser refuses names no host a browser will reach",+ "raw": "https://256.1.1.1/p",+ "url": null,+ "label": null+ },+ {+ "why": "a trailing number turns the whole host numeric, and this one does not parse",+ "raw": "https://example.com.0x1/p",+ "url": null,+ "label": null+ },+ {+ "why": "P1: a host of digits wider than an address is refused rather than converted",+ "raw": "https://99999999999/x",+ "url": null,+ "label": null+ },+ {+ "why": "the check is the address, not the width: ten digits can overflow too",+ "raw": "https://9999999999/x",+ "url": null,+ "label": null+ },+ {+ "why": "the widest address that does fit is still read as one",+ "raw": "https://4294967295/x",+ "url": "https://4294967295/x",+ "label": "255.255.255.255"+ },+ {+ "why": "a hexadecimal part too wide is a number the parser refuses, never a domain name",+ "raw": "https://example.0xFFFFFFFFFFFFFFFF/x",+ "url": null,+ "label": null+ },+ {+ "why": "a leading zero makes the part octal however many zeros follow it",+ "raw": "https://00000000000000000000127/x",+ "url": "https://00000000000000000000127/x",+ "label": "0.0.0.87"+ },+ {+ "why": "CommonMark resolves an entity in a destination, so the stored URL is the resolved one",+ "raw": "https://example.com/s?a=1&b=2",+ "url": "https://example.com/s?a=1&b=2",+ "label": "example.com"+ },+ {+ "why": "a destination that would resolve twice is pinned after the first resolution",+ "raw": "https://example.com/s?a=1&amp;b=2",+ "url": "https://example.com/s?a=1%26amp;b=2",+ "label": "example.com"+ },+ {+ "why": "P2: the renderer replaces a C1 reference with U+FFFD, so the stored URL carries that",+ "raw": "https://example.com/p€q",+ "url": "https://example.com/p%EF%BF%BDq",+ "label": "example.com"+ },+ {+ "why": "a noncharacter reference is replaced the same way rather than kept",+ "raw": "https://example.com/pq",+ "url": "https://example.com/p%EF%BF%BDq",+ "label": "example.com"+ },+ {+ "why": "a zero-width reference is escaped, not deleted, so the path still matches",+ "raw": "https://example.com/p​q",+ "url": "https://example.com/p%E2%80%8Bq",+ "label": "example.com"+ },+ {+ "why": "a reference spelling a marker delimiter is escaped out of harm rather than dropped",+ "raw": "https://example.com/pq",+ "url": "https://example.com/p%EE%88%80q",+ "label": "example.com"+ },+ {+ "why": "P1: a named tab reference is part of the destination, so the renderer escapes it",+ "raw": "https://example.com/p	q",+ "url": "https://example.com/p%09q",+ "label": "example.com"+ },+ {+ "why": "the same tab spelled numerically, which a literal tab in a destination cannot reach",+ "raw": "https://example.com/p	q",+ "url": "https://example.com/p%09q",+ "label": "example.com"+ },+ {+ "why": "P1: a newline reference is escaped too; removing it pointed the citation elsewhere",+ "raw": "https://example.com/p
q",+ "url": "https://example.com/p%0Aq",+ "label": "example.com"+ },+ {+ "why": "a carriage return reference goes the same way rather than being dropped",+ "raw": "https://example.com/p
q",+ "url": "https://example.com/p%0Dq",+ "label": "example.com"+ },+ {+ "why": "brackets are Markdown label punctuation and are encoded on both sides",+ "raw": "https://example.com/a[b]",+ "url": "https://example.com/a%5Bb%5D",+ "label": "example.com"+ },+ {+ "why": "angle brackets and quotes would make a pointy destination or break out of an attribute",+ "raw": "https://example.com/<x>\"y\"",+ "url": "https://example.com/%3Cx%3E%22y%22",+ "label": "example.com"+ },+ {+ "why": "a tab is dropped before parsing, exactly as a browser drops it",+ "raw": "https://example.com/a\tb",+ "url": "https://example.com/ab",+ "label": "example.com"+ },+ {+ "why": "a lone surrogate has no encoding; the renderer substitutes U+FFFD",+ "raw": "https://example.com/\ud800",+ "url": "https://example.com/%EF%BF%BD",+ "label": "example.com"+ },+ {+ "why": "the host is named without its www prefix, which is not what distinguishes the source",+ "raw": "https://www.example.com/x",+ "url": "https://www.example.com/x",+ "label": "example.com"+ },+ {+ "why": "P1: a host too long to show is elided from the left, keeping the registrable suffix",+ "raw": "https://developers.openai.com.padding.padding.padding.padding.padding.padding.attacker.example/x",+ "url": "https://developers.openai.com.padding.padding.padding.padding.padding.padding.attacker.example/x",+ "label": "\u2026adding.padding.padding.padding.padding.padding.attacker.example"+ },+ {+ "why": "P1: whole labels alone left only the public suffix a long label was hiding behind",+ "raw": "https://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.co.uk/x",+ "url": "https://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.co.uk/x",+ "label": "\u2026aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.co.uk"+ },+ {+ "why": "a browser lowercases the host but not the path, so the label is lowercase and the URL is not",+ "raw": "https://Example.COM/X",+ "url": "https://Example.COM/X",+ "label": "example.com"+ },+ {+ "why": "P2: a scheme is case-insensitive, and only its lowercase spelling is recognized everywhere",+ "raw": "HTTPS://Example.com/X",+ "url": "https://Example.com/X",+ "label": "example.com"+ },+ {+ "why": "an entity must not be able to smuggle a scheme past the check",+ "raw": "javascript:alert(1)",+ "url": null,+ "label": null+ },+ {+ "why": "an unsafe scheme is rejected, not repaired",+ "raw": "javascript:alert(1)",+ "url": null,+ "label": null+ },+ {+ "why": "a URL with no host names no source",+ "raw": "https:///path",+ "url": null,+ "label": null+ }+ ],+ "authority": [+ {"why": "an IPv6 literal is a host a browser resolves, so a citation naming one is delivered instead of dropped", "raw": "https://[2001:db8::1]/docs", "url": "https://[2001:db8::1]/docs", "label": "[2001:db8::1]"},+ {"why": "the URL keeps the address its source wrote, while the label is the serialized form the address bar shows", "raw": "https://[2001:0db8:0000:0000:0000:0000:0000:0001]/x", "url": "https://[2001:0db8:0000:0000:0000:0000:0000:0001]/x", "label": "[2001:db8::1]"},+ {"why": "hexadecimal pieces are lowercased for the label, not for the URL", "raw": "https://[2001:DB8::1]/x", "url": "https://[2001:DB8::1]/x", "label": "[2001:db8::1]"},+ {"why": "an all-zero address is labelled with the empty compression", "raw": "https://[0:0:0:0:0:0:0:0]/x", "url": "https://[0:0:0:0:0:0:0:0]/x", "label": "[::]"},+ {"why": "only the longest zero run compresses, so the shorter one beside it stays written out", "raw": "https://[1:0:0:2:0:0:0:3]/x", "url": "https://[1:0:0:2:0:0:0:3]/x", "label": "[1:0:0:2::3]"},+ {"why": "a dotted IPv4 tail is labelled as hexadecimal pieces, which is the host the address bar names", "raw": "https://[::127.0.0.1]/x", "url": "https://[::127.0.0.1]/x", "label": "[::7f00:1]"},+ {"why": "an IPv4-mapped address is no exception - Python's ipaddress writes it back dotted, a browser does not", "raw": "https://[::ffff:127.0.0.1]/x", "url": "https://[::ffff:127.0.0.1]/x", "label": "[::ffff:7f00:1]"},+ {"why": "a port belongs to the URL but never to the attribution: the label names the host", "raw": "https://[::1]:8080/x", "url": "https://[::1]:8080/x", "label": "[::1]"},+ {"why": "the scheme's default port is kept - it opens the same page - and no port reaches the label", "raw": "https://[::1]:443/x", "url": "https://[::1]:443/x", "label": "[::1]"},+ {"why": "a zone id names an interface only the local machine has, and a browser refuses it", "raw": "https://[fe80::1%25eth0]/x", "url": null, "label": null},+ {"why": "an unclosed bracket is not an address", "raw": "https://[2001:db8::1/x", "url": null, "label": null},+ {"why": "a port is carried as it was written, leading zeros and all: it is the same port", "raw": "https://example.com:0080/x", "url": "https://example.com:0080/x", "label": "example.com"},+ {"why": "the scheme's own default port is no different - dropping it would edit a URL that already opens", "raw": "https://example.com:443/x", "url": "https://example.com:443/x", "label": "example.com"},+ {"why": "another scheme's default is just a port here", "raw": "https://example.com:80/x", "url": "https://example.com:80/x", "label": "example.com"},+ {"why": "an empty port is no port, and refusing the URL over it would lose a source a browser opens", "raw": "https://example.com:/x", "url": "https://example.com:/x", "label": "example.com"},+ {"why": "a port past 16 bits opens nothing, so the citation is refused rather than delivered broken", "raw": "https://example.com:99999/x", "url": null, "label": null},+ {"why": "a port must be ASCII digits: int() reads an Arabic-Indic three as 3, a browser reads no port at all", "raw": "https://example.com:%D9%A3/x", "url": null, "label": null},+ {"why": "a C0 control the host spells percent-encoded is a forbidden domain code point once decoded", "raw": "https://exa%01mple.com/x", "url": null, "label": null},+ {"why": "userinfo is case-folded nowhere - it may be a password - and neither is the host beside it; the label is where the browser's answer shows", "raw": "https://user:[email protected]/x", "url": "https://user:[email protected]/x", "label": "example.com"},+ {"why": "the label names the host and never the userinfo, so a username cannot dress a link up as a domain", "raw": "https://[email protected]/x", "url": "https://[email protected]/x", "label": "example.com"},+ {"why": "userinfo hides an IPv6 host no better", "raw": "https://attacker.example@[::1]/x", "url": "https://attacker.example@[::1]/x", "label": "[::1]"},+ {"why": "a well-formed ACE label is the host a browser opens, so it keeps its citation", "raw": "https://xn--fa-hia.de/x", "url": "https://xn--fa-hia.de/x", "label": "xn--fa-hia.de"},+ {"why": "an ACE prefix in any case is still that label, and the label names it lowercased", "raw": "https://XN--80AK6AA92E.com/x", "url": "https://XN--80AK6AA92E.com/x", "label": "xn--80ak6aa92e.com"},+ {"why": "an ACE label with no punycode after the prefix makes a browser refuse the URL", "raw": "https://xn--/x", "url": null, "label": null},+ {"why": "an ACE label whose punycode decodes to a control character is not a host", "raw": "https://xn--abc/x", "url": null, "label": null},+ {"why": "an ACE label must round-trip to a non-ASCII label: xn--ascii- decodes to plain ascii, and refusing what a browser might still open is the safe direction", "raw": "https://xn--ascii-/x", "url": null, "label": null},+ {"why": "an ACE label that does not round-trip is refused even beside a valid one", "raw": "https://xn--a.xn--fa-hia/x", "url": null, "label": null},+ {"why": "an authority is read only where one is written: a browser repairs the missing slashes, but the rule that keeps a hostless URL out keeps this out too, and a search result always writes them", "raw": "https:example.com/x", "url": null, "label": null}+ ]+}diff --git a/tests/fixtures/link_destination_matrix.json b/tests/fixtures/link_destination_matrix.jsonnew file mode 100644index 000000000..3d1674cde--- /dev/null+++ b/tests/fixtures/link_destination_matrix.json@@ -0,0 +1,441 @@+{+ "description": "One shared table for the address a link destination arrives at.\n\nA destination is delivered to readers with different dialects, and each writes it in the spelling its own surface can hold. Two spellings are not evidence that they name one page, so this file asserts the relationship instead of listing two strings: tests/test_link_unit_delivery.py renders `markdown` through the real Slack adapter and asserts the address it decodes out of the wrapper is `address`, and ui/src/components/ui/markdown.test.tsx renders the same `markdown` through the real Markdown component and asserts the browser URL parser resolves that anchor's href and `address` to the same URL. Neither side can drift without the other failing.\n\nNavigation equivalence is all this table claims. The exact bytes each surface puts on its own wire are asserted separately, beside the adapter that writes them.",+ "cases": [+ {+ "markdown": "[x](https://example.com/p[q)",+ "address": "https://example.com/p%5Bq"+ },+ {+ "markdown": "[x](https://example.com/p]q)",+ "address": "https://example.com/p%5Dq"+ },+ {+ "markdown": "[x](https://example.com/p%q)",+ "address": "https://example.com/p%25q"+ },+ {+ "markdown": "[x](https://example.com/p%2q)",+ "address": "https://example.com/p%2q"+ },+ {+ "markdown": "[x](https://example.com/p%3Eq)",+ "address": "https://example.com/p%3Eq"+ },+ {+ "markdown": "[x](https://example.com/p%zzq)",+ "address": "https://example.com/p%zzq"+ },+ {+ "markdown": "[x](https://example.com/p%2gq)",+ "address": "https://example.com/p%2gq"+ },+ {+ "markdown": "[x](https://example.com/p\\>q)",+ "address": "https://example.com/p%3Eq"+ },+ {+ "markdown": "[x](https://example.com/p\\<q)",+ "address": "https://example.com/p%3Cq"+ },+ {+ "markdown": "[x](https://example.com/p\\|q)",+ "address": "https://example.com/p%7Cq"+ },+ {+ "markdown": "[x](https://example.com/p\\\\q)",+ "address": "https://example.com/p%5Cq"+ },+ {+ "markdown": "[x](https://example.com/p\\`q)",+ "address": "https://example.com/p%60q"+ },+ {+ "markdown": "[x](https://example.com/p^q)",+ "address": "https://example.com/p%5Eq"+ },+ {+ "markdown": "[x](https://example.com/p{q)",+ "address": "https://example.com/p%7Bq"+ },+ {+ "markdown": "[x](https://example.com/p}q)",+ "address": "https://example.com/p%7Dq"+ },+ {+ "markdown": "[x](https://example.com/p\"q)",+ "address": "https://example.com/p%22q"+ },+ {+ "markdown": "[x](https://example.com/p'q)",+ "address": "https://example.com/p'q"+ },+ {+ "markdown": "[x](https://example.com/p&q)",+ "address": "https://example.com/p&q"+ },+ {+ "markdown": "[x](https://example.com/p\\&q)",+ "address": "https://example.com/p&q"+ },+ {+ "markdown": "[x](https://example.com/p&amp;q)",+ "address": "https://example.com/p&q"+ },+ {+ "markdown": "[x](https://example.com/p q)",+ "address": "https://example.com/p%0Aq"+ },+ {+ "markdown": "[x](https://example.com/p	q)",+ "address": "https://example.com/p%09q"+ },+ {+ "markdown": "[x](https://example.com/p\u00a0q)",+ "address": "https://example.com/p%C2%A0q"+ },+ {+ "markdown": "[x](https://example.com/p\u200bq)",+ "address": "https://example.com/p%E2%80%8Bq"+ },+ {+ "markdown": "[x](https://example.com/p\u4e2dq)",+ "address": "https://example.com/p%E4%B8%ADq"+ },+ {+ "markdown": "[x](https://example.com/p\ud83d\ude00q)",+ "address": "https://example.com/p%F0%9F%98%80q"+ },+ {+ "markdown": "[x](https://example.com/p\ud800q)",+ "address": "https://example.com/p%EF%BF%BDq"+ },+ {+ "markdown": "[x](https://example.com/p?x=p[q&y=1)",+ "address": "https://example.com/p?x=p%5Bq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p]q&y=1)",+ "address": "https://example.com/p?x=p%5Dq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p%q&y=1)",+ "address": "https://example.com/p?x=p%25q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p%2q&y=1)",+ "address": "https://example.com/p?x=p%2q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p%3Eq&y=1)",+ "address": "https://example.com/p?x=p%3Eq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p%zzq&y=1)",+ "address": "https://example.com/p?x=p%zzq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p%2gq&y=1)",+ "address": "https://example.com/p?x=p%2gq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\\>q&y=1)",+ "address": "https://example.com/p?x=p%3Eq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\\<q&y=1)",+ "address": "https://example.com/p?x=p%3Cq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\\|q&y=1)",+ "address": "https://example.com/p?x=p%7Cq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\\\\q&y=1)",+ "address": "https://example.com/p?x=p%5Cq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\\`q&y=1)",+ "address": "https://example.com/p?x=p%60q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p^q&y=1)",+ "address": "https://example.com/p?x=p%5Eq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p{q&y=1)",+ "address": "https://example.com/p?x=p%7Bq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p}q&y=1)",+ "address": "https://example.com/p?x=p%7Dq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\"q&y=1)",+ "address": "https://example.com/p?x=p%22q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p'q&y=1)",+ "address": "https://example.com/p?x=p'q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p&q&y=1)",+ "address": "https://example.com/p?x=p&q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\\&q&y=1)",+ "address": "https://example.com/p?x=p&q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p&amp;q&y=1)",+ "address": "https://example.com/p?x=p&q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p q&y=1)",+ "address": "https://example.com/p?x=p%0Aq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p	q&y=1)",+ "address": "https://example.com/p?x=p%09q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\u00a0q&y=1)",+ "address": "https://example.com/p?x=p%C2%A0q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\u200bq&y=1)",+ "address": "https://example.com/p?x=p%E2%80%8Bq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\u4e2dq&y=1)",+ "address": "https://example.com/p?x=p%E4%B8%ADq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\ud83d\ude00q&y=1)",+ "address": "https://example.com/p?x=p%F0%9F%98%80q&y=1"+ },+ {+ "markdown": "[x](https://example.com/p?x=p\ud800q&y=1)",+ "address": "https://example.com/p?x=p%EF%BF%BDq&y=1"+ },+ {+ "markdown": "[x](https://example.com/p#p[q)",+ "address": "https://example.com/p#p%5Bq"+ },+ {+ "markdown": "[x](https://example.com/p#p]q)",+ "address": "https://example.com/p#p%5Dq"+ },+ {+ "markdown": "[x](https://example.com/p#p%q)",+ "address": "https://example.com/p#p%25q"+ },+ {+ "markdown": "[x](https://example.com/p#p%2q)",+ "address": "https://example.com/p#p%2q"+ },+ {+ "markdown": "[x](https://example.com/p#p%3Eq)",+ "address": "https://example.com/p#p%3Eq"+ },+ {+ "markdown": "[x](https://example.com/p#p%zzq)",+ "address": "https://example.com/p#p%zzq"+ },+ {+ "markdown": "[x](https://example.com/p#p%2gq)",+ "address": "https://example.com/p#p%2gq"+ },+ {+ "markdown": "[x](https://example.com/p#p\\>q)",+ "address": "https://example.com/p#p%3Eq"+ },+ {+ "markdown": "[x](https://example.com/p#p\\<q)",+ "address": "https://example.com/p#p%3Cq"+ },+ {+ "markdown": "[x](https://example.com/p#p\\|q)",+ "address": "https://example.com/p#p%7Cq"+ },+ {+ "markdown": "[x](https://example.com/p#p\\\\q)",+ "address": "https://example.com/p#p%5Cq"+ },+ {+ "markdown": "[x](https://example.com/p#p\\`q)",+ "address": "https://example.com/p#p%60q"+ },+ {+ "markdown": "[x](https://example.com/p#p^q)",+ "address": "https://example.com/p#p%5Eq"+ },+ {+ "markdown": "[x](https://example.com/p#p{q)",+ "address": "https://example.com/p#p%7Bq"+ },+ {+ "markdown": "[x](https://example.com/p#p}q)",+ "address": "https://example.com/p#p%7Dq"+ },+ {+ "markdown": "[x](https://example.com/p#p\"q)",+ "address": "https://example.com/p#p%22q"+ },+ {+ "markdown": "[x](https://example.com/p#p'q)",+ "address": "https://example.com/p#p'q"+ },+ {+ "markdown": "[x](https://example.com/p#p&q)",+ "address": "https://example.com/p#p&q"+ },+ {+ "markdown": "[x](https://example.com/p#p\\&q)",+ "address": "https://example.com/p#p&q"+ },+ {+ "markdown": "[x](https://example.com/p#p&amp;q)",+ "address": "https://example.com/p#p&q"+ },+ {+ "markdown": "[x](https://example.com/p#p q)",+ "address": "https://example.com/p#p%0Aq"+ },+ {+ "markdown": "[x](https://example.com/p#p	q)",+ "address": "https://example.com/p#p%09q"+ },+ {+ "markdown": "[x](https://example.com/p#p\u00a0q)",+ "address": "https://example.com/p#p%C2%A0q"+ },+ {+ "markdown": "[x](https://example.com/p#p\u200bq)",+ "address": "https://example.com/p#p%E2%80%8Bq"+ },+ {+ "markdown": "[x](https://example.com/p#p\u4e2dq)",+ "address": "https://example.com/p#p%E4%B8%ADq"+ },+ {+ "markdown": "[x](https://example.com/p#p\ud83d\ude00q)",+ "address": "https://example.com/p#p%F0%9F%98%80q"+ },+ {+ "markdown": "[x](https://example.com/p#p\ud800q)",+ "address": "https://example.com/p#p%EF%BF%BDq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p[q)",+ "address": "https://[::1]:8443/p%5Bq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p]q)",+ "address": "https://[::1]:8443/p%5Dq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p%q)",+ "address": "https://[::1]:8443/p%25q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p%2q)",+ "address": "https://[::1]:8443/p%2q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p%3Eq)",+ "address": "https://[::1]:8443/p%3Eq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p%zzq)",+ "address": "https://[::1]:8443/p%zzq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p%2gq)",+ "address": "https://[::1]:8443/p%2gq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\\>q)",+ "address": "https://[::1]:8443/p%3Eq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\\<q)",+ "address": "https://[::1]:8443/p%3Cq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\\|q)",+ "address": "https://[::1]:8443/p%7Cq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\\\\q)",+ "address": "https://[::1]:8443/p%5Cq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\\`q)",+ "address": "https://[::1]:8443/p%60q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p^q)",+ "address": "https://[::1]:8443/p%5Eq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p{q)",+ "address": "https://[::1]:8443/p%7Bq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p}q)",+ "address": "https://[::1]:8443/p%7Dq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\"q)",+ "address": "https://[::1]:8443/p%22q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p'q)",+ "address": "https://[::1]:8443/p'q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p&q)",+ "address": "https://[::1]:8443/p&q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\\&q)",+ "address": "https://[::1]:8443/p&q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p&amp;q)",+ "address": "https://[::1]:8443/p&q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p q)",+ "address": "https://[::1]:8443/p%0Aq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p	q)",+ "address": "https://[::1]:8443/p%09q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\u00a0q)",+ "address": "https://[::1]:8443/p%C2%A0q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\u200bq)",+ "address": "https://[::1]:8443/p%E2%80%8Bq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\u4e2dq)",+ "address": "https://[::1]:8443/p%E4%B8%ADq"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\ud83d\ude00q)",+ "address": "https://[::1]:8443/p%F0%9F%98%80q"+ },+ {+ "markdown": "[x](https://[::1]:8443/p\ud800q)",+ "address": "https://[::1]:8443/p%EF%BF%BDq"+ },+ {+ "markdown": "[x](https://u:p@[::1]:8443/a[b]?x=[c]#f[d])",+ "address": "https://u:p@[::1]:8443/a%5Bb%5D?x=%5Bc%5D#f%5Bd%5D"+ }+ ]+}diff --git a/tests/test_agent_activity_service.py b/tests/test_agent_activity_service.pyindex 46d31c2b9..f5fb801bd 100644--- a/tests/test_agent_activity_service.py+++ b/tests/test_agent_activity_service.py@@ -62,7 +62,20 @@ def _seed_session(conn, *, session_id="ses_act"):return scope_id-def _msg(conn, scope_id, session_id, *, mid, mtype, author, created_at, text="", source="agent", metadata=None):+def _msg(+ conn,+ scope_id,+ session_id,+ *,+ mid,+ mtype,+ author,+ created_at,+ text="",+ source="agent",+ metadata=None,+ content=None,+):conn.execute(messages.insert().values(id=mid,@@ -73,7 +86,7 @@ def _msg(conn, scope_id, session_id, *, mid, mtype, author, created_at, text="",type=mtype,source=source,content_text=text,- content_json="{}",+ content_json=json.dumps(content or {}),metadata_json=json.dumps(metadata or {}),created_at=created_at,updated_at=created_at,@@ -219,6 +232,49 @@ def test_rows_merge_across_tables_by_parsed_timestamp(isolated_state):]+def test_a_narration_row_carries_its_citation_sidecar(isolated_state):+ """An intermediate cited answer renders through Activity, not ``MessageRow``.++ Its sources are persisted in ``content.citations`` (see ``core.citations``),+ so the row contract has to carry them or the Web badge silently degrades to+ the plain domain link the text already holds. A row with nothing to carry+ keeps the wire shape it always had.+ """+ engine = create_sqlite_engine()+ sid = "ses_cited"+ sidecar = [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Web search - OpenAI API",+ "url": "https://developers.openai.com/api/docs/guides/tools-web-search",+ "label": "developers.openai.com",+ }+ ]+ with engine.begin() as conn:+ scope = _seed_session(conn, session_id=sid)+ _msg(conn, scope, sid, mid="m_u", mtype="user", author="user", created_at="2026-06-01T10:00:00.000000+00:00", text="q", source="user")+ _msg(+ conn, scope, sid, mid="m_cited", mtype="assistant", author="agent",+ created_at="2026-06-01T10:00:01.000000+00:00",+ text="Documented. [developers.openai.com](https://developers.openai.com/api/docs/guides/tools-web-search)",+ content={"kind": "assistant", "citations": sidecar},+ )+ _msg(+ conn, scope, sid, mid="m_plain", mtype="assistant", author="agent",+ created_at="2026-06-01T10:00:02.000000+00:00", text="Still working.",+ )+ _msg(conn, scope, sid, mid="m_r", mtype="result", author="agent", created_at="2026-06-01T10:00:03.000000+00:00", text="done")++ with engine.connect() as conn:+ detail = agent_activity_service.get_turn_group(conn, session_id=sid, group_id="m_cited")++ assert detail is not None+ rows = {row["id"]: row for row in detail["rows"]}+ assert rows["m_cited"]["citations"] == sidecar+ assert "citations" not in rows["m_plain"]++def test_agent_annotation_marks_are_not_activity(isolated_state):engine = create_sqlite_engine()sid = "ses_sp"diff --git a/tests/test_agent_silent_result.py b/tests/test_agent_silent_result.pyindex 7e490642f..cc38a246c 100644--- a/tests/test_agent_silent_result.py+++ b/tests/test_agent_silent_result.py@@ -7,6 +7,7 @@ from types import SimpleNamespacesys.path.insert(0, str(Path(__file__).resolve().parents[1]))+from core.citations import CitationSource, register_citationsfrom modules.agents.base import AgentRequest, BaseAgentfrom modules.im import MessageContextfrom modules.im.formatters.slack_formatter import SlackFormatter@@ -95,6 +96,53 @@ class AgentSilentResultTests(unittest.IsolatedAsyncioTestCase):# Promoted to body, so no separate subtext footer is passed.self.assertEqual(controller.result_footers, [None])+ async def test_an_uncited_result_does_not_require_the_citation_keyword(self):+ """The controller is substitutable, so an empty sidecar stays unsent.++ ``_StubController`` above deliberately keeps the pre-citation signature:+ an ordinary uncited turn must reach any stand-in that never heard of+ citations, on both the duration-on and duration-off paths.+ """+ controller = _StubController()+ agent = _StubAgent(controller)+ context = MessageContext(user_id="U1", channel_id="C1", platform="slack")++ for show_duration in (True, False):+ controller.config.show_duration = show_duration+ message_id = await agent.emit_result_message(context, "Plain answer.")++ self.assertEqual(message_id, "message-id")+ self.assertEqual([m[0] for m in controller.messages], ["result", "result"])++ async def test_a_cited_result_forwards_its_sidecar(self):+ """The keyword is withheld only when there is nothing to carry."""+ seen: list[object] = []++ class _CitationAwareController(_StubController):+ async def emit_agent_message(self, *args, citations=None, **kwargs):+ seen.append(citations)+ return await super().emit_agent_message(*args, **kwargs)++ controller = _CitationAwareController()+ agent = _StubAgent(controller)+ context = MessageContext(user_id="U1", channel_id="C1", platform="slack")+ registered, bundle = register_citations(+ "Cited answer.\ue200cite\ue202turn0view0\ue201",+ {+ "turn0view0": CitationSource(+ ref_id="turn0view0", title="Example", url="https://example.com/x"+ )+ },+ unresolved_label="(source unavailable)",+ )++ await agent.emit_result_message(context, registered, citations=bundle)++ # The bundle is forwarded, not a copy of it: the tokens in the text are+ # only meaningful against the one that minted them.+ self.assertEqual(seen, [bundle])+ self.assertIs(seen[0], bundle)+class AgentSessionIdContextTests(unittest.TestCase):def test_bind_agent_session_id_attaches_returned_public_session_id(self):diff --git a/tests/test_citation_consumers.py b/tests/test_citation_consumers.pynew file mode 100644index 000000000..9c199847b--- /dev/null+++ b/tests/test_citation_consumers.py@@ -0,0 +1,714 @@+"""What a reader on each IM platform actually sees of a cited answer.++Every case here comes out of one real producer run recorded by+``tests/citation_bridge.py``: the text ``core.citations`` wrote, the sidecar it+wrote beside it, and the body ``core.reply_enhancer`` hands each surface. This+file then puts that body through each platform's own renderer - the real+``SlackBot`` conversion, the real ``TelegramBot``/``WeChatBot``+``format_markdown``, the real Discord and Feishu pass-throughs - and asks what+the reader ends up with.++The question is deliberately narrow, because attribution is a narrow promise: a+cited answer must show the source's name and reach the source's address. So the+assertions are on the visible label and the navigable destination, never on+"the URL appears somewhere". A citation whose host reads ``ab.example`` because+a converter ate the asterisks has kept the URL and lost the attribution, and+that is the bug this file exists to catch.++``ui/src/components/ui/citation-bridge.test.tsx`` reads the same fixture and+asks the same question of the real Web renderer. Discord and Feishu are measured+locally as pass-throughs only: nothing here sends a message, so nothing here can+claim how those platforms render one.+"""++from __future__ import annotations++import html+import json+import os+import re+import sys+import tempfile+import unittest+from pathlib import Path+from typing import Any, Mapping+from unittest import mock+from urllib.parse import unquote++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from config.v2_config import (+ DiscordConfig,+ LarkConfig,+ SlackConfig,+ TelegramConfig,+ WeChatConfig,+)+from core.citations import (+ CitationSource,+ body_digest,+ citation_ref_ids,+ register_citations,+ resolve_citations,+)+from core.message_dispatcher import ConsolidatedMessageDispatcher+from core.reply_enhancer import inline_links, unescape_markdown+from modules.im import MessageContext+from modules.im.discord import DiscordBot+from modules.im.feishu import FeishuBot+from modules.im.slack import SlackBot+from modules.im.telegram import TelegramBot+from modules.im.wechat import WeChatBot+from storage import messages_service+from storage.db import create_sqlite_engine, dispose_cached_sqlite_engines+from storage.importer import ensure_sqlite_state+from storage.models import agent_sessions+from storage.settings_service import upsert_scope+from tests.citation_bridge import FIXTURE, UNRESOLVED_LABEL, build, utf16_slice+from tests.test_message_dispatcher_platform_limits import _StubController++BRIDGE: dict[str, Any] = json.loads(FIXTURE.read_text(encoding="utf-8"))+CASES: list[dict[str, Any]] = BRIDGE["cases"]+ROWS: list[dict[str, Any]] = BRIDGE["rows"]+++def case(key: str) -> dict[str, Any]:+ return next(row for row in CASES if row["key"] == key)+++def stored_row(key: str) -> dict[str, Any]:+ return next(row["row"] for row in ROWS if row["key"] == key)+++def sources_of(row: Mapping[str, Any]) -> dict[str, CitationSource]:+ """The search results the backend captured, as the producer receives them."""+ return {+ ref: CitationSource(ref_id=ref, url=meta["url"], title=meta["title"])+ for ref, meta in row["sources"].items()+ }+++def registered(row: Mapping[str, Any]):+ """Re-run the producer for one recorded case.++ The recording cannot carry the bundle itself: the tokens in it are nonces,+ minted per run and never written to disk. So a test that needs the+ dispatcher's real input registers the same agent text against the same+ sources again, and checks the bodies that come out against the recording.+ """+ return register_citations(+ row["agent_text"], sources_of(row), unresolved_label=UNRESOLVED_LABEL+ )+++MEDIA_TOKEN_RE = re.compile(r"/api/media/[A-Za-z0-9_-]+")+++def without_media_ids(payload: Any) -> Any:+ """The one thing in a recorded row that is minted fresh on every run.++ Persisting a ``file://`` attachment registers it and writes a proxy URL+ carrying a new random id, so the stored body - and therefore its digest -+ differs between two identical runs. Both are blanked here so the rest of+ the row is still compared byte for byte; that the digest describes the body+ it was written with is asserted on the recorded row itself, below.+ """+ if isinstance(payload, dict):+ blanked = {key: without_media_ids(value) for key, value in payload.items()}+ if isinstance(blanked.get("text"), str) and MEDIA_TOKEN_RE.search(blanked["text"]):+ blanked["text"] = MEDIA_TOKEN_RE.sub("/api/media/<id>", blanked["text"])+ for cite in blanked.get("citations") or ():+ cite["body_sha256"] = "<digest of a body holding a fresh media id>"+ return blanked+ if isinstance(payload, list):+ return [without_media_ids(item) for item in payload]+ return payload+++class BridgeFixtureTests(unittest.TestCase):+ def test_the_recorded_producer_output_is_what_the_producer_emits_today(self):+ """The fixture is a recording, so a stale one would test a past product.++ Regenerate with ``.venv/bin/python -m tests.citation_bridge``.+ """+ self.assertEqual(+ without_media_ids(json.loads(json.dumps(build(), ensure_ascii=False))),+ without_media_ids(BRIDGE),+ )++ def test_every_recorded_row_is_a_row_a_reader_could_lose_a_badge_on(self):+ """The rows are the Web consumers' input, so they have to be real ones.++ A row whose sidecar does not describe its own stored text would make+ the Web test assert a refusal; a set where no row carries a folded+ footer would never exercise the split that makes the measurement have+ to travel. Both would pass quietly.+ """+ folded = 0+ for row in ROWS:+ with self.subTest(case=row["key"]):+ stored = row["row"]+ self.assertTrue(row["why"].strip())+ content = stored["content"]+ [cite] = content["citations"]+ self.assertEqual(cite["body_sha256"], body_digest(stored["text"]))+ self.assertEqual(+ utf16_slice(stored["text"], *cite["spans"][0]),+ f"[{cite['label']}]({cite['url']})",+ )+ footer = content.get("result_footer")+ if footer and stored["text"].rstrip().endswith(footer):+ folded += 1+ # The strip removes a range AFTER the citation here. That+ # is what makes the row survivable; a fixture where the+ # footer preceded it would be a different question.+ self.assertLess(cite["spans"][0][1], len(stored["text"]))+ self.assertTrue(folded, "no recorded row folds a footer into its stored body")++ def test_the_rows_include_one_a_reader_edits_twice_more_after_the_backend(self):+ """The combination row, checked to still be one.++ Its value is entirely in what has already happened to it: the same+ reply persisted down both paths, where the attachment is rewritten+ before the sidecar is measured - to a proxy link on the workbench, to a+ bare label on IM - and a ``$<NAME>`` marker sits above the citation for+ the reader to turn into a card. A bridge edit that dropped any of those+ would leave the Web test passing on a plainer row than it names.+ """+ rewritten = stored_row("workbench_row_rewritten_and_carded")+ self.assertRegex(rewritten["text"], r"^\[report\]\(/api/media/[^)]+\)")+ # The workbench stores its footer apart, so the reader's only edit here+ # is the card; the IM row below gets the strip as well.+ self.assertNotIn(rewritten["content"]["result_footer"], rewritten["text"])++ flattened = stored_row("im_row_flattened_carded_and_folded")+ self.assertTrue(flattened["text"].startswith("report\n\n"))+ self.assertNotIn("](", flattened["text"].split("\n\n")[0])+ self.assertTrue(+ flattened["text"].rstrip().endswith(flattened["content"]["result_footer"])+ )++ for row in (rewritten, flattened):+ [cite] = row["content"]["citations"]+ # Every one of those sits above the citation, which is the only+ # place an edit can move it.+ before = utf16_slice(row["text"], 0, cite["spans"][0][0])+ self.assertIn("report", before)+ self.assertIn("$<deployKey>", before)+ self.assertEqual(cite["body_sha256"], body_digest(row["text"]))++ def test_every_case_carries_an_expectation_a_consumer_can_fail(self):+ for row in CASES:+ with self.subTest(case=row["key"]):+ self.assertTrue(row["why"].strip())+ if row["im_citations"]:+ self.assertTrue(+ any(anchor["cited"] for anchor in row["im_anchors"]),+ "a cited case whose delivered text shows no citation proves nothing",+ )++ def test_a_citation_claims_exactly_the_links_the_producer_wrote(self):+ """The binding contract, read back off the recording.++ Each sidecar row names the spans its links occupy in one exact body;+ the anchors derived from that body must add up to at most that many,+ and every other anchor must stay ordinary.+ """+ for row in CASES:+ with self.subTest(case=row["key"]):+ claimed = sum(len(cite["spans"]) for cite in row["im_citations"])+ shown = [anchor for anchor in row["im_anchors"] if anchor["cited"]]+ # A claimed occurrence the reader is not shown is one inside a+ # code example: counted by both ends, rendered as a link by+ # neither. It may never be more than that.+ self.assertLessEqual(len(shown), claimed)+ for anchor in shown:+ self.assertIn(+ (anchor["url"], anchor["label"]),+ [(cite["url"], cite["label"]) for cite in row["im_citations"]],+ )++ def test_two_deletions_that_cancel_out_do_not_make_a_second_citation(self):+ """The counterexample that decides where a citation gets its identity.++ ``file_delete_splice`` is one reply that cited one page once. Delivery+ removes two ranges from it - an attachment link collapses to its label,+ a ``<silent>`` block leaves - and the characters left behind close up+ into a second complete marker the model never wrote. On the workbench+ the attachment survives as a link, so the same reply splices on one+ surface and not on the other.++ Resolving the delivered body, which is all a stage placed after+ delivery can do, is run here against the real recording: it writes a+ second link to the source and puts the badge on it, leaving the+ sentence the model actually cited an ordinary anchor. Registration+ happens before either deletion instead, so the shipped sidecar claims+ the one occurrence that was cited and the spliced shape stays literal.+ """+ row = case("file_delete_splice")+ spliced = "\ue200cite\ue202turn0view0\ue201"+ # The model wrote one marker, in the first sentence. The second+ # sentence held halves that were not one: an attachment link sat+ # between ``ci`` and ``te``, a ``<silent>`` block before the+ # terminator.+ self.assertEqual(row["agent_text"].count(spliced), 1)+ self.assertTrue(row["agent_text"].startswith(f"Actual. {spliced}"))+ # After delivery the marker the model wrote is a link, and the only+ # one still standing is the one it did not write.+ self.assertEqual(row["im_text"].count(spliced), 1)+ self.assertTrue(row["im_text"].endswith(f"Synthetic. {spliced}"))+ self.assertNotIn(spliced, row["web_text"])+ self.assertEqual(citation_ref_ids(row["im_text"]), ["turn0view0"])+ self.assertEqual(citation_ref_ids(row["web_text"]), [])++ [shipped] = row["im_citations"]+ self.assertEqual(len(shipped["spans"]), 1)+ cited = utf16_slice(row["im_text"], *shipped["spans"][0])+ self.assertEqual(cited, f"[{shipped['label']}]({shipped['url']})")+ # The one link the sidecar claims is the sentence that carried the+ # marker in the model's own text, not the one the deletions formed.+ self.assertTrue(row["im_text"].startswith(f"Actual. {cited}"))++ late_body, late = resolve_citations(+ row["im_text"], sources_of(row), unresolved_label=UNRESOLVED_LABEL+ )+ [late_cite] = late+ self.assertEqual(late_body.count(cited), 2)+ self.assertEqual(len(late_cite["spans"]), 1)+ self.assertNotEqual(late_cite["spans"], shipped["spans"])+ self.assertTrue(late_body.endswith(f"Synthetic. {cited}"))+++# A private marker is not an address. Where the model wrote one in a+# destination the producer leaves it exactly as typed - claiming a source there+# would be claiming one no reader can see - and every renderer past that point+# is free to spell those characters as percent escapes. So a rendered address is+# read back through that one spelling before it is compared, and through nothing+# else: a substituted host is still a link nobody wrote.+_MARKER_ESCAPE = re.compile(r"(?:%EE%88%8[0-2])+", re.IGNORECASE)+++def as_written(text: str) -> str:+ return _MARKER_ESCAPE.sub(lambda match: unquote(match.group()), text)+++class PlatformConsumerTests(unittest.TestCase):+ """One platform's real renderer, asked what the reader sees.++ Subclasses supply ``render`` and the two readings a platform-specific+ assertion needs: ``links`` (what the reader can click, as+ ``(destination, label)``) and ``readable`` (the output with the platform's+ own escaping resolved, so a destination can be searched for as written).+ """++ __test__ = False++ def render(self, text: str) -> str: # pragma: no cover - overridden+ raise NotImplementedError++ def links(self, rendered: str) -> list[tuple[str, str]]: # pragma: no cover+ raise NotImplementedError++ def readable(self, rendered: str) -> str:+ return rendered++ def test_a_citation_shows_its_source_and_reaches_its_address(self):+ """The whole promise: the reader sees the host and lands on the page."""+ for row in CASES:+ rendered = self.render(row["im_text"])+ clickable = self.links(rendered)+ for anchor in row["im_anchors"]:+ if not anchor["cited"]:+ continue+ with self.subTest(case=row["key"], url=anchor["url"]):+ self.assertIn(+ (anchor["url"], anchor["label"]),+ [(as_written(url), label) for url, label in clickable],+ )++ def test_no_link_points_anywhere_the_producer_did_not_write(self):+ """A converter that reads a destination as text rewrites the address.++ Slack's mrkdwn converter did exactly that: ``https://a*b*.example/x``+ came back as ``https://a_b_.example/x``, a link to a different site that+ still looks like a citation.+ """+ for row in CASES:+ # Only where the recording names every address a reader can reach.+ # An image, a reference link or an autolink is a link this scan does+ # not resolve, and a body holding one says so: its anchors are the+ # links a consumer must show, not the only ones it may.+ if not row["anchors_cover_destinations"]:+ continue+ rendered = self.render(row["im_text"])+ written = {anchor["url"] for anchor in row["im_anchors"]}+ for destination, _ in self.links(rendered):+ with self.subTest(case=row["key"], destination=destination):+ self.assertIn(as_written(destination), written)++ def test_every_address_the_answer_carried_survives_the_renderer(self):+ """Including the ones this platform does not turn into a link.++ A code example, a table cell or a footnote is rendered differently by+ every dialect here; what none of them may do is lose or alter the+ address the answer pointed at.+ """+ for row in CASES:+ readable = as_written(self.readable(self.render(row["im_text"])))+ for anchor in row["im_anchors"]:+ with self.subTest(case=row["key"], url=anchor["url"]):+ self.assertIn(anchor["url"], readable)+++class SlackConsumerTests(PlatformConsumerTests):+ __test__ = True++ # ``<destination|label>``, and the bare ``<destination>`` Slack also accepts.+ _LINK = re.compile(r"<([^<>|]+)(?:\|([^<>]*))?>")+ # The three references Slack resolves on the way in, and the only ones: a+ # label spelling ``©`` is shown as those six characters, not ``(c)``.+ # A label arrives spelled this way because ``&``, ``<`` and ``>`` are+ # Slack's own markup, so reading back what the reader sees means resolving+ # these three and nothing else.+ _RESOLVED = {"&": "&", "<": "<", ">": ">"}+ _REFERENCE = re.compile(r"&(?:amp|lt|gt);")++ def setUp(self):+ self.bot = SlackBot(SlackConfig(bot_token="xoxb-test"))++ def render(self, text: str) -> str:+ return self.bot._convert_markdown_to_slack_mrkdwn(text)++ def _resolve(self, text: str) -> str:+ return self._REFERENCE.sub(lambda match: self._RESOLVED[match.group()], text)++ def readable(self, rendered: str) -> str:+ return self._resolve(rendered)++ def links(self, rendered: str) -> list[tuple[str, str]]:+ found = []+ for match in self._LINK.finditer(rendered):+ # Structure first, then one pass of the three resolutions over each+ # field. Both halves travel as Slack text, so both are read back+ # the way Slack reads them - a query ``&`` is sent as ``&``+ # precisely so the client resolves it to the separator again, and+ # an oracle that refused to resolve the destination would be+ # measuring the spelling instead of the page the reader reaches.+ # The wrapper is parsed before any of that, so a delimiter that+ # ended the unit early still shows up as a broken unit here.+ destination = self._resolve(match.group(1))+ if not destination.startswith(("http://", "https://")):+ continue # ``<@U1>``/``<#C1>`` and stray angle brackets in prose+ label = match.group(2)+ found.append((destination, self._resolve(label) if label is not None else destination))+ return found++ def test_the_converter_alone_would_have_sent_readers_to_another_site(self):+ """The counterexample, measured on both sides of the fix.++ ``markdown_to_mrkdwn`` scans the whole line for emphasis, destination+ included. Holding the destinations across its pass is what keeps the+ address the search engine returned.+ """+ from markdown_to_mrkdwn import SlackMarkdownConverter++ star = case("star_host")+ url = star["im_citations"][0]["url"]++ unheld = SlackMarkdownConverter().convert(star["im_text"])+ held = self.render(star["im_text"])++ self.assertIn("https://a_b_.example/x", unheld)+ self.assertNotIn(url, unheld)+ self.assertIn(f"<{url}|a*b*.example>", held)++ def test_a_destination_is_held_through_the_query_string_too(self):+ """The query survives the converter, and its ``&`` survives the wrapper.++ Two assertions, because they are two different promises. The exact+ wire says the separator is spelled the way a Slack text object spells+ one - not percent-encoded, which would survive every decoder and leave+ the reader on a page with one parameter instead of two. The reading+ says that spelling resolves back to the address the producer wrote.+ """+ query = case("backtick_query")+ url = query["im_citations"][0]["url"]+ rendered = self.render(query["im_text"])++ self.assertIn("<https://example.com/p?q=%60code%60&r=a*b*|example.com>", rendered)+ self.assertIn((url, "example.com"), self.links(rendered))+++class TelegramConsumerTests(PlatformConsumerTests):+ __test__ = True++ _ANCHOR = re.compile(r'<a href="([^"]*)">(.*?)</a>', re.DOTALL)++ def setUp(self):+ self.bot = TelegramBot(TelegramConfig(bot_token="123456:test-token"))++ def render(self, text: str) -> str:+ return self.bot.format_markdown(text)++ def readable(self, rendered: str) -> str:+ return html.unescape(rendered)++ def links(self, rendered: str) -> list[tuple[str, str]]:+ return [+ (html.unescape(match.group(1)), html.unescape(match.group(2)))+ for match in self._ANCHOR.finditer(rendered)+ ]++ def test_the_label_is_not_re_read_as_emphasis(self):+ """A host holding ``*`` was shown as ``ab.example`` with an italic ``b``."""+ star = case("star_host")++ rendered = self.render(star["im_text"])++ self.assertIn('<a href="https://a*b*.example/x">a*b*.example</a>', rendered)+ self.assertNotIn("<i>", rendered)+++class WeChatConsumerTests(PlatformConsumerTests):+ __test__ = True++ def setUp(self):+ self.bot = WeChatBot(WeChatConfig(bot_token="test-token"))++ def render(self, text: str) -> str:+ return self.bot.format_markdown(text)++ def links(self, rendered: str) -> list[tuple[str, str]]:+ """WeChat has no hyperlinks, so ``label (address)`` is the link.++ Reading it back needs the labels the answer actually used - a general+ scan of "text before parentheses" would call the whole sentence a label.+ """+ found = []+ for row in CASES:+ for anchor in row["im_anchors"]:+ flattened = f"{anchor['label']} ({anchor['url']})"+ for _ in range(rendered.count(flattened)):+ found.append((anchor["url"], anchor["label"]))+ return found++ def test_the_address_stays_readable_because_nothing_is_clickable(self):+ star = case("star_host")++ self.assertEqual(+ self.render(star["im_text"]),+ "Cited. a*b*.example (https://a*b*.example/x)",+ )+++class PassThroughConsumerTests(unittest.TestCase):+ """Discord and Feishu render Markdown themselves and are handed it as-is.++ That is all this measures: the adapter changes nothing. What those two+ platforms then draw is not observable from here, and no message is sent.+ """++ def setUp(self):+ self.discord = DiscordBot(DiscordConfig(bot_token="test-token"))+ self.feishu = FeishuBot(LarkConfig(app_id="app-id", app_secret="app-secret"))++ def test_the_adapter_hands_the_body_over_unchanged(self):+ for row in CASES:+ with self.subTest(case=row["key"]):+ self.assertEqual(self.discord.format_markdown(row["im_text"]), row["im_text"])+ self.assertEqual(self.feishu.format_markdown(row["im_text"]), row["im_text"])+++class TranscriptPersistenceTests(unittest.IsolatedAsyncioTestCase):+ """The workbench row the Web renderer reads back, read back out of SQLite.++ Everything above measures the producer in memory. This drives the real+ ``ConsolidatedMessageDispatcher`` with real persistence into a temporary+ SQLite home, then reads the rows back through ``list_session_messages`` -+ the same query the transcript endpoint runs - and checks them against the+ recording. That equality is what makes the Web half of the bridge a test of+ the stored row: ``citation-bridge.test.tsx`` renders ``web_text`` and+ ``web_citations``, and these are the bytes the database returned.++ IM and the transcript are handed different bodies on purpose - a ``file://``+ link is flattened for IM and kept for the workbench media proxy - so the+ recording carries both.+ """++ SESSION = "ses_bridge"+ NOW = "2026-09-21T12:00:00Z"++ def setUp(self):+ self._home = tempfile.TemporaryDirectory()+ self._env = mock.patch.dict(os.environ, {"AVIBE_HOME": self._home.name})+ self._env.start()+ dispose_cached_sqlite_engines()+ ensure_sqlite_state()+ engine = create_sqlite_engine()+ try:+ with engine.begin() as conn:+ scope_id = upsert_scope(+ conn,+ platform="avibe",+ scope_type="project",+ native_id="proj_bridge",+ now=self.NOW,+ )+ conn.execute(+ agent_sessions.insert().values(+ id=self.SESSION,+ scope_id=scope_id,+ agent_backend="codex",+ agent_variant="default",+ session_anchor=f"anchor_{self.SESSION}",+ native_session_id="",+ status="active",+ metadata_json="{}",+ created_at=self.NOW,+ updated_at=self.NOW,+ last_active_at=self.NOW,+ )+ )+ finally:+ engine.dispose()++ def tearDown(self):+ dispose_cached_sqlite_engines()+ self._env.stop()+ self._home.cleanup()++ async def deliver(self, row: Mapping[str, Any], platform: str):+ controller = _StubController(platform)+ controller.config.reply_enhancements = True+ dispatcher = ConsolidatedMessageDispatcher(controller)+ avibe = platform == "avibe"+ context = MessageContext(+ user_id="workbench" if avibe else "u1",+ channel_id=self.SESSION if avibe else "c1",+ platform=platform,+ platform_specific={"agent_session_id": self.SESSION} if avibe else None,+ )+ text, bundle = registered(row)+ await dispatcher.emit_agent_message(context, "result", text, citations=bundle)+ return controller.im_client.sent++ def stored(self) -> list[tuple[str, Any]]:+ engine = create_sqlite_engine()+ try:+ with engine.connect() as conn:+ transcript = messages_service.list_session_messages(+ conn, session_id=self.SESSION+ )+ finally:+ engine.dispose()+ return [+ (message["text"], (message["content"] or {}).get("citations"))+ for message in transcript["messages"]+ ]++ def assertDescribesItsOwnBody(self, text: str, citations):+ """Each row measures THIS body: its digest, and its own link at its span.++ What a span covers is read with the delivery pass's own link scanner+ rather than compared to a rebuilt spelling, because the label in the+ body is Markdown-escaped and the one in the sidecar is the host. The+ property is that the span is exactly one whole link, to this source.+ """+ for cite in citations or ():+ self.assertEqual(cite["body_sha256"], body_digest(text))+ for span in cite["spans"]:+ spelled = utf16_slice(text, *span)+ found = list(inline_links(spelled))+ self.assertEqual(len(found), 1)+ self.assertEqual((found[0].start, found[0].end), (0, len(spelled)))+ self.assertEqual(found[0].destination, cite["url"])+ self.assertEqual(+ unescape_markdown(spelled[found[0].label_start : found[0].label_end]),+ cite["label"],+ )++ async def test_the_row_the_web_renderer_is_handed_is_the_one_sqlite_returns(self):+ for row in CASES:+ await self.deliver(row, "avibe")+ stored = self.stored()+ self.assertEqual(len(stored), len(CASES))++ for row, (text, citations) in zip(CASES, stored):+ with self.subTest(case=row["key"]):+ self.assertDescribesItsOwnBody(text, citations)+ if "file://" in row["web_text"]:+ # Rewritten on its way to disk; measured in its own test+ # below, where what changed is the point.+ continue+ self.assertEqual(text, row["web_text"])+ self.assertEqual(citations, row["web_citations"] or None)++ async def test_the_media_rewrite_moves_the_measurement_it_walks_past(self):+ """The one transform that runs after delivery and before the sidecar.++ A ``file://`` attachment is registered and swapped for a media-proxy URL+ inside the same transaction that writes the row, so the stored body is+ not the one delivery produced - it is shorter, and a citation below the+ attachment sits somewhere else in it. The sidecar is measured after that+ rewrite for exactly this reason: a span carried over from the delivered+ text would point into the wrong characters, and the digest would say so.+ """+ for key in ("file_link", "file_link_above"):+ await self.deliver(case(key), "avibe")+ stored = self.stored()++ for key, (text, citations) in zip(("file_link", "file_link_above"), stored):+ row = case(key)+ with self.subTest(case=key):+ self.assertNotIn("file://", text)+ self.assertRegex(text, r"\[report\]\(/api/media/[^)]+\)")+ self.assertDescribesItsOwnBody(text, citations)+ # Same citation, same link, different body: the recorded+ # delivery measurement does not describe what was stored.+ recorded = row["web_citations"][0]+ self.assertEqual(citations[0]["url"], recorded["url"])+ self.assertNotEqual(citations[0]["body_sha256"], recorded["body_sha256"])++ # Above the citation the rewrite shortens the text, so the span moves;+ # below it nothing before the link changed, so it does not.+ self.assertEqual(stored[0][1][0]["spans"], case("file_link")["web_citations"][0]["spans"])+ self.assertNotEqual(+ stored[1][1][0]["spans"], case("file_link_above")["web_citations"][0]["spans"]+ )++ async def test_im_receives_the_flattened_body_and_never_the_sidecar(self):+ for row in CASES:+ with self.subTest(case=row["key"]):+ sent = await self.deliver(row, "slack")++ delivered = "".join(text for _, _, text, _ in sent)+ self.assertEqual(delivered, row["im_text"])+ for cite in row["im_citations"]:+ # The sidecar's own fields: a reader on IM is shown links,+ # and the measurement that binds them to a body stays on+ # the row. A ``ref_id`` is NOT asserted absent - a marker+ # the producer refused to register keeps its literal text,+ # spelled ref and all (``splice``), which is the point of+ # refusing it.+ self.assertNotIn(cite["body_sha256"], delivered)+ self.assertNotIn("spans", delivered)++ async def test_the_extracted_buttons_and_labels_carry_their_own_copy(self):+ """Quick-reply labels and file labels leave the body and stay text."""+ row = case("quick_replies")+ await self.deliver(row, "avibe")++ engine = create_sqlite_engine()+ try:+ with engine.connect() as conn:+ transcript = messages_service.list_session_messages(+ conn, session_id=self.SESSION+ )+ finally:+ engine.dispose()+ content = transcript["messages"][0]["content"] or {}+ self.assertEqual(content.get("quick_replies"), row["quick_replies"])+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_citation_exits.py b/tests/test_citation_exits.pynew file mode 100644index 000000000..7b84c399e--- /dev/null+++ b/tests/test_citation_exits.py@@ -0,0 +1,511 @@+"""Every copy of a cited answer that leaves the dispatcher, asked for the token.++A registered citation travels through delivery as an opaque nonce: an identity+no later stage can guess back, minted at the native input boundary where the+marker grammar still means what it says. Each surface then writes that token+out into the form that surface can show - a Markdown link in a body, a bare+host in a structured field - because each surface holds a different text.++That only works if *every* exit writes it. A token that survives into an IM+message, a stored row, an agent-run record, a Turn snapshot or an upload title+is a private handle showing up in text a person reads, and nothing downstream+will ever turn it back into a link. So this file drives the real+``ConsolidatedMessageDispatcher`` over one real producer run against a+temporary SQLite home, collects every string that run handed to something+outside the dispatcher, and asks all of them the same question at once.++The question is NOT "did every private-use character disappear". This answer+carries three that must come through verbatim: the marker grammar shown as a+code example, a marker the stream cut in half, and an unrelated private-use+character the model simply wrote. A sweep that removed those would pass a+census written the lazy way while deleting the answer's own text.+"""++from __future__ import annotations++import json+import os+import sys+import tempfile+import unittest+from pathlib import Path+from typing import Any, Optional+from unittest import mock++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from sqlalchemy import select++from core.citations import _SEP, _START, _TOKEN_RE, CitationSource, register_citations+from core.message_dispatcher import ConsolidatedMessageDispatcher+from modules.im import MessageContext+from storage.db import create_sqlite_engine, dispose_cached_sqlite_engines+from storage.importer import ensure_sqlite_state+from storage.models import agent_events, agent_sessions, messages+from storage.settings_service import upsert_scope+from tests.citation_bridge import UNRESOLVED_LABEL, marker+from tests.test_message_dispatcher_platform_limits import _StubController, _StubIMClient++REF = "turn0view0"+SOURCE_URL = "https://example.com/x"+SOURCES = {REF: CitationSource(ref_id=REF, url=SOURCE_URL, title="Cited page")}++# A marker the stream cut in half. It is not a complete marker, so it is not+# registered and nothing rewrites it - existing behaviour, asserted here so the+# census cannot be satisfied by deleting it.+TRUNCATED = f"{_START}cite{_SEP}{REF}"+# A private-use character that is simply part of the answer. It is neither+# citation grammar nor an internal token.+LOOSE_PUA = ""+++def agent_text(report: Path) -> str:+ """One agent reply, spelled the way the backend hands it to registration."""+ cite = marker(REF)+ return (+ # The citation: one complete marker in prose. The only thing here+ # registration may claim.+ f"Cited. {cite}\n\n"+ # The same grammar shown as a code example. The reader is being shown+ # what a marker looks like, so it has to stay what it looks like.+ f"```\n{cite}\n```\n\n"+ f"Cut off: {TRUNCATED}\n\n"+ f"Loose private use: {LOOSE_PUA}\n\n"+ # Two markers inside text that is lifted OUT of the body into fields no+ # surface parses as Markdown - an upload title and a quick-reply label.+ # They are the exits where a token would land somewhere nothing+ # downstream ever rewrites, so they are the ones worth writing a reply+ # around.+ f"[report {cite}](file://{report})\n\n"+ f"---\n[Open {cite}] | [Done]"+ )+++class _Exits:+ """Every string one dispatcher run handed to something outside itself."""++ def __init__(self) -> None:+ self.rows: list[tuple[str, str]] = []++ def note(self, where: str, value: Any) -> None:+ if isinstance(value, str) and value:+ self.rows.append((where, value))++ def of(self, *where: str) -> list[str]:+ return [value for name, value in self.rows if name in where]++ def names(self) -> set[str]:+ return {name for name, _ in self.rows}+++class _RecordingIMClient(_StubIMClient):+ """The platform stub, plus the send shapes a cited result actually uses."""++ def __init__(self, exits: _Exits, **kwargs: Any) -> None:+ super().__init__(**kwargs)+ self.exits = exits++ async def send_message(self, context, text, parse_mode=None, reply_to=None, subtext=None):+ self.exits.note("im.send", text)+ self.exits.note("im.subtext", subtext)+ return await super().send_message(context, text, parse_mode=parse_mode)++ async def send_message_with_buttons(+ self, context, text, keyboard, parse_mode=None, subtext=None+ ):+ self.exits.note("im.send", text)+ self.exits.note("im.subtext", subtext)+ for row in getattr(keyboard, "buttons", None) or []:+ for button in row:+ self.exits.note("im.button", getattr(button, "text", None))+ self.exits.note("im.button", getattr(button, "callback_data", None))+ return await super().send_message(context, text, parse_mode=parse_mode)++ async def edit_message(self, context, message_id, text=None, keyboard=None, parse_mode=None):+ self.exits.note("im.edit", text)+ return await super().edit_message(+ context, message_id, text=text, keyboard=keyboard, parse_mode=parse_mode+ )++ async def upload_markdown(self, context, title, content, filetype=None):+ self.exits.note("im.attachment.title", title)+ self.exits.note("im.attachment", content)+ return "bot-attachment-1"++ async def upload_file_from_path(self, context, file_path, title=None, **kwargs):+ self.exits.note("im.upload.title", title)+ return "bot-upload-1"+++class _RecordingRunStore:+ """The Harness run ledger, as the dispatcher reaches it."""++ def __init__(self, exits: _Exits) -> None:+ self.exits = exits++ def record_turn_run_outputs(self, run_ids, *, output_id, text, **kwargs):+ self.exits.note("run.output", text)+ self.exits.note("run.output.error", kwargs.get("error"))++ def record_run_message(self, run_id, *, text, message_id=None, terminal_status=None):+ self.exits.note("run.message", text)++ def close(self):+ return None+++class _RecordingTurns:+ """The Turn snapshot a later steer reads the result back out of."""++ def __init__(self, exits: _Exits) -> None:+ self.exits = exits++ def on_terminal_result(self, context, *, is_error, settled_by, terminal_evidence):+ self.exits.note("turn.snapshot", terminal_evidence.get("result_text"))+ self.exits.note("turn.snapshot.error", terminal_evidence.get("terminal_error"))++ def accepted_agent_run_ids_for_turn(self, turn_id):+ return []++ def on_terminal_delivery_complete(self, context):+ return None+++class _RecordingController(_StubController):+ """A stub controller that also owns a live SSE sink and a Turn manager."""++ def __init__(+ self,+ platform: str,+ exits: _Exits,+ *,+ max_bytes: int | None = None,+ progress_style: str = "verbose",+ ) -> None:+ super().__init__(platform, max_bytes=max_bytes)+ self.exits = exits+ self._progress_style_value = progress_style+ self.im_client = _RecordingIMClient(+ exits, max_bytes=max_bytes, supports_editing=platform != "wechat"+ )+ self.config.reply_enhancements = True+ self.session_turns = _RecordingTurns(exits)++ async def on_chunk(chunk):+ exits.note("sse.chunk", chunk.get("text"))++ self._sink = {"on_chunk": on_chunk}++ def get_turn_sink(self, key):+ return self._sink++ def get_progress_style_for_context(self, context):+ return self._progress_style_value+++class CitationExitTests(unittest.IsolatedAsyncioTestCase):+ """One reply, every exit, one question.++ Each test drives a different delivery shape - inline, split, summarized and+ uploaded, suppressed, notified, logged - because they are different code+ paths that each make their own copy of the body, and a copy made before the+ tokens are written is exactly the defect this is looking for.+ """++ SESSION = "ses_exits"+ NOW = "2026-09-21T12:00:00Z"++ def setUp(self):+ self._home = tempfile.TemporaryDirectory()+ self._env = mock.patch.dict(os.environ, {"AVIBE_HOME": self._home.name})+ self._env.start()+ dispose_cached_sqlite_engines()+ ensure_sqlite_state()+ engine = create_sqlite_engine()+ try:+ with engine.begin() as conn:+ scope_id = upsert_scope(+ conn,+ platform="avibe",+ scope_type="project",+ native_id="proj_exits",+ now=self.NOW,+ )+ conn.execute(+ agent_sessions.insert().values(+ id=self.SESSION,+ scope_id=scope_id,+ agent_backend="codex",+ agent_variant="default",+ session_anchor=f"anchor_{self.SESSION}",+ native_session_id="",+ status="active",+ metadata_json="{}",+ created_at=self.NOW,+ updated_at=self.NOW,+ last_active_at=self.NOW,+ )+ )+ finally:+ engine.dispose()++ self._files = tempfile.TemporaryDirectory()+ self.report = Path(self._files.name) / "report.txt"+ self.report.write_text("attached\n", encoding="utf-8")+ self.exits = _Exits()+ self._seen_rows: set[str] = set()++ def tearDown(self):+ dispose_cached_sqlite_engines()+ self._env.stop()+ self._home.cleanup()+ self._files.cleanup()++ # -- driving -----------------------------------------------------------++ def registered(self, *, pad: str = "") -> tuple[str, Any]:+ """The real producer's output for this reply, tokens and all."""+ text, bundle = register_citations(+ agent_text(self.report) + pad, SOURCES, unresolved_label=UNRESOLVED_LABEL+ )+ # A census over a body that never held a token proves nothing, and the+ # three literals must be present to be able to survive.+ self.assertTrue(_TOKEN_RE.search(text))+ self.assertIn(f"```\n{marker(REF)}\n```", text)+ self.assertIn(TRUNCATED, text)+ self.assertIn(LOOSE_PUA, text)+ return text, bundle++ def context(self, platform: str, **spec: Any) -> MessageContext:+ avibe = platform == "avibe"+ payload: dict[str, Any] = dict(spec)+ if avibe:+ payload.setdefault("agent_session_id", self.SESSION)+ return MessageContext(+ user_id="workbench" if avibe else "u1",+ channel_id=self.SESSION if avibe else "c1",+ platform=platform,+ platform_specific=payload or None,+ )++ async def drive(+ self,+ platform: str,+ message_type: str = "result",+ *,+ pad: str = "",+ spec: Optional[dict[str, Any]] = None,+ max_bytes: int | None = None,+ progress_style: str = "verbose",+ **kwargs: Any,+ ) -> _RecordingController:+ text, bundle = self.registered(pad=pad)+ controller = _RecordingController(+ platform, self.exits, max_bytes=max_bytes, progress_style=progress_style+ )+ dispatcher = ConsolidatedMessageDispatcher(controller)+ store = _RecordingRunStore(self.exits)+ with mock.patch(+ "core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=store+ ):+ await dispatcher.emit_agent_message(+ self.context(platform, **(spec or {})),+ message_type,+ text,+ citations=bundle,+ **kwargs,+ )+ self.collect_rows()+ return controller++ def collect_rows(self) -> None:+ """Whatever this run left in the store is an exit too.++ Read back by id so a test that drives twice records each row once:+ the store is cumulative and the exits are per-run.+ """+ engine = create_sqlite_engine()+ try:+ with engine.connect() as conn:+ for table, kind in ((messages, "row"), (agent_events, "trace")):+ for row in conn.execute(+ select(table.c.id, table.c.content_text, table.c.content_json)+ ):+ if row[0] in self._seen_rows:+ continue+ self._seen_rows.add(row[0])+ self.exits.note(f"{kind}.text", row[1])+ self.exits.note(f"{kind}.content", row[2])+ finally:+ engine.dispose()++ # -- the question ------------------------------------------------------++ def assertNoInternalToken(self, *expected_exits: str) -> None:+ """No copy of this reply carries the private handle it travelled as."""+ self.assertTrue(self.exits.rows, "the drive recorded no exits at all")+ for where, value in self.exits.rows:+ with self.subTest(exit=where):+ self.assertIsNone(+ _TOKEN_RE.search(value),+ f"{where} carries an internal citation token",+ )+ # An exit nobody reached cannot leak, so the census only means something+ # if the run actually reached the ones it claims to cover.+ self.assertLessEqual(set(expected_exits), self.exits.names())++ def assertLiteralsSurvive(self, body: str) -> None:+ """The three private-use passages that are not this module's to touch."""+ self.assertIn(f"```\n{marker(REF)}\n```", body)+ self.assertIn(TRUNCATED, body)+ self.assertIn(LOOSE_PUA, body)++ def assertAttributes(self, body: str) -> None:+ """The reader is given the source, not a handle to it."""+ self.assertIn(SOURCE_URL, body)++ # -- the drives --------------------------------------------------------++ async def test_an_im_result_writes_every_copy_it_hands_out(self):+ # The run ids travel with the message because a delivered IM result+ # also writes the agent-run record a Harness caller reads back, and+ # that copy is written from the body+footer fold rather than from the+ # row - a separate exit with a separate write.+ await self.drive(+ "slack",+ result_footer="✅ ⏱️ 5s",+ spec={"turn_token": "t1", "accepted_agent_run_ids": ["run-1"]},+ )++ self.assertNoInternalToken(+ "im.send",+ "im.button",+ "im.upload.title",+ "turn.snapshot",+ "run.output",+ "sse.chunk",+ "row.text",+ )+ [sent] = self.exits.of("im.send")+ self.assertLiteralsSurvive(sent)+ self.assertAttributes(sent)+ # The button and the upload title are structured fields: no surface+ # parses them as Markdown, so the attribution there is the bare host.+ self.assertEqual(self.exits.of("im.button")[0], "Open example.com")+ self.assertEqual(self.exits.of("im.upload.title"), ["report example.com"])+ # The Turn snapshot is read back by a later steer, and it is text with+ # no sidecar riding along.+ [snapshot] = self.exits.of("turn.snapshot")+ self.assertLiteralsSurvive(snapshot)+ self.assertAttributes(snapshot)+ # The agent-run record keeps the footer folded back in, so it is a+ # third spelling of the same answer and is written on its own.+ [recorded] = self.exits.of("run.output")+ self.assertLiteralsSurvive(recorded)+ self.assertAttributes(recorded)+ self.assertIn("⏱️ 5s", recorded)++ async def test_a_long_im_result_is_written_before_it_is_split(self):+ # Discord splits rather than summarizes, so the body is cut into pieces+ # AFTER the citations are written. A copy made before the split would+ # put a token in every part.+ await self.drive("discord", pad="\n\n" + "filler. " * 400)++ self.assertNoInternalToken("im.send", "row.text")+ parts = self.exits.of("im.send")+ self.assertGreater(len(parts), 1)+ self.assertLiteralsSurvive("".join(parts))+ self.assertAttributes("".join(parts))++ async def test_a_result_too_large_to_send_is_written_before_it_is_cut(self):+ # Slack neither splits nor sends this inline: the reader gets a summary+ # and the full body as an attachment. Both are copies.+ await self.drive("slack", pad="\n\n" + "filler. " * 5000)++ self.assertNoInternalToken("im.send", "im.attachment", "row.text")+ [summary] = self.exits.of("im.send")+ self.assertTrue(summary.startswith("Result too long"))+ # The summary keeps the head of the body, which is where the citation+ # and the literals are; the attachment keeps all of it.+ self.assertLiteralsSurvive(summary)+ self.assertAttributes(summary)+ [attachment] = self.exits.of("im.attachment")+ self.assertLiteralsSurvive(attachment)+ self.assertAttributes(attachment)++ async def test_the_workbench_row_carries_a_sidecar_and_never_a_token(self):+ await self.drive("avibe", spec={"accepted_agent_run_ids": ["run-1"]})++ self.assertNoInternalToken("row.text", "row.content", "sse.chunk", "run.output")+ [body] = self.exits.of("row.text")+ self.assertLiteralsSurvive(body)+ self.assertAttributes(body)+ [content] = self.exits.of("row.content")+ stored = json.loads(content)+ # The row is the one copy that also carries the measurement, and the+ # labels lifted out of the body sit beside it.+ self.assertEqual([row["url"] for row in stored["citations"]], [SOURCE_URL])+ self.assertEqual(stored["quick_replies"], ["Open example.com", "Done"])++ async def test_a_notify_writes_its_outbound_and_its_stream_copy(self):+ await self.drive("slack", "notify")++ self.assertNoInternalToken("im.send", "sse.chunk", "row.text")+ for copy in self.exits.of("im.send", "sse.chunk", "row.text"):+ self.assertLiteralsSurvive(copy)+ self.assertAttributes(copy)++ async def test_an_intermediate_log_row_its_tool_trace_and_its_bubble(self):+ # Three copies of an intermediate emit, and they are not the same text:+ # the row keeps the body, the trace event keeps the body, and the+ # concise bubble keeps a one-line label DERIVED from it.+ await self.drive("slack", "assistant")+ await self.drive("slack", "toolcall")+ await self.drive("slack", "assistant", progress_style="concise")++ self.assertNoInternalToken("im.send", "row.text", "trace.text")+ [trace] = self.exits.of("trace.text")+ self.assertLiteralsSurvive(trace)+ self.assertAttributes(trace)+ for logged in self.exits.of("im.send"):+ self.assertAttributes(logged)++ async def test_a_silent_result_still_writes_the_record_a_steer_reads(self):+ # Nobody is shown this answer: a silent-level result returns before any+ # delivery, persistence or streaming. Two copies still leave - the Turn+ # snapshot a later steer reads back, and the agent-run record that+ # settles the run - and neither has a sidecar to travel with.+ await self.drive(+ "slack",+ level="silent",+ spec={"turn_token": "t2", "accepted_agent_run_ids": ["run-2"]},+ )++ self.assertNoInternalToken("turn.snapshot", "run.output")+ self.assertEqual(self.exits.of("im.send"), [])+ for copy in self.exits.of("turn.snapshot", "run.output"):+ self.assertLiteralsSurvive(copy)+ self.assertAttributes(copy)++ async def test_a_suppressed_harness_result_writes_its_run_records(self):+ # Nothing is delivered outward: the run ledger and the stored row are+ # the only places this answer exists, which is why a token reaching+ # them would never be corrected by a later stage.+ await self.drive(+ "slack",+ spec={+ "suppress_delivery": True,+ "task_trigger_kind": "scheduled",+ "task_execution_id": "run-9",+ },+ )++ self.assertNoInternalToken("run.output", "row.text")+ self.assertEqual(self.exits.of("im.send"), [])+ [recorded] = self.exits.of("run.output")+ self.assertLiteralsSurvive(recorded)+ self.assertAttributes(recorded)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_citations.py b/tests/test_citations.pynew file mode 100644index 000000000..7e96d5d7e--- /dev/null+++ b/tests/test_citations.py@@ -0,0 +1,1524 @@+"""The shared citation grammar, sanitizer, and Markdown rewrite.++Fixture-driven on purpose: the marker shape and the ref_id vocabulary+(``turn0view0``, ``turn0view1``, ``turn1view0``) are the ones a real Codex+0.154.0 turn emitted, so a table entry here is a recorded observation rather+than an invented example.++Two properties carry the feature and are asserted separately below rather than+left implicit in the happy path:++* attribution is never silently deleted - an unresolvable marker degrades to a+ visible label, and with no label to fall back to the raw marker survives;+* a URL is never invented - an unknown ref, an unsafe scheme, and a hostless+ URL all resolve to nothing rather than to a guess.+"""++from __future__ import annotations++import hashlib+import json+import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from core.citations import (+ _TOKEN_CLOSE as TOKEN_CLOSE,+)+from core.citations import (+ _TOKEN_OPEN as TOKEN_OPEN,+)+from core.citations import (+ CitationSource,+ body_digest,+ citation_ref_ids,+ clean_title,+ finalize_citations,+ has_citation_markers,+ materialize_citations,+ register_citations,+ resolve_citations,+ safe_url,+ source_label,+ unresolved_refs,+)+from core.reply_enhancer import strip_silent_blocks+from markdown_it import MarkdownIt++# The reference parser for anything asserted about rendered Markdown, so a+# claim that a label breaks its own link is a measurement and not a reading.+MARKDOWN = MarkdownIt("commonmark")++START, SEP, END = "\ue200", "\ue202", "\ue201"+UNRESOLVED = "(source unavailable)"+++def marker(*ref_ids: str) -> str:+ """The exact wire shape: U+E200 cite U+E202 ref [U+E202 ref ...] U+E201."""+ return f"{START}cite{SEP}{SEP.join(ref_ids)}{END}"+++def utf16_slice(text: str, start: int, end: int) -> str:+ """The substring a span names - sliced the way the consumer slices it.++ Spans are UTF-16 code units because that is what a browser counts;+ ``text[start:end]`` would silently disagree the moment an astral character+ appears above the span.+ """+ units = text.encode("utf-16-le", "surrogatepass")+ return units[start * 2 : end * 2].decode("utf-16-le", "surrogatepass")+++def charcodeat_bytes(text: str) -> bytes:+ """The bytes a ``charCodeAt`` loop in the browser would hash.++ Written out unit by unit rather than handed to a codec, so the fixture is+ checked against what JavaScript does and not against the same Python call+ the implementation makes.+ """+ units: list[int] = []+ for char in text:+ code = ord(char)+ if code > 0xFFFF:+ code -= 0x10000+ units.append(0xD800 + (code >> 10))+ units.append(0xDC00 + (code & 0x3FF))+ else:+ units.append(code)+ return b"".join(unit.to_bytes(2, "little") for unit in units)+++# The two sources a real logged turn produced, plus the loopback probe's.+GUIDE = CitationSource(+ ref_id="turn0view0",+ title="Web search - OpenAI API",+ url="https://developers.openai.com/api/docs/guides/tools-web-search",+)+GUIDE_AGAIN = CitationSource(ref_id="turn1view0", title=GUIDE.title, url=GUIDE.url)+PROBE = CitationSource(+ ref_id="turn0view1",+ title="Citation probe source",+ url="https://example.com/citation-probe-source",+)+SOURCES = {s.ref_id: s for s in (GUIDE, GUIDE_AGAIN, PROBE)}+++def resolve(text: str, sources=None, *, unresolved_label: str = UNRESOLVED):+ return resolve_citations(+ text,+ SOURCES if sources is None else sources,+ unresolved_label=unresolved_label,+ )+++class TestResolution:+ def test_real_marker_becomes_a_markdown_link_and_one_sidecar_entry(self):+ text, citations = resolve(f"Native search is documented.{marker('turn0view0')}")++ assert text == (+ "Native search is documented. "+ "[developers.openai.com](https://developers.openai.com/api/docs/guides/tools-web-search)"+ )+ assert citations == [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Web search - OpenAI API",+ "url": "https://developers.openai.com/api/docs/guides/tools-web-search",+ "label": "developers.openai.com",+ # Where the link sits in THIS body, and the digest that binds+ # the two together. See TestBodyBinding.+ "spans": [[29, len(text)]],+ "body_sha256": body_digest(text),+ }+ ]++ def test_multiple_refs_in_one_marker_each_become_a_link(self):+ text, citations = resolve(f"Two sources.{marker('turn0view0', 'turn0view1')}")++ assert text.endswith(+ "[developers.openai.com](https://developers.openai.com/api/docs/guides/tools-web-search) "+ "[example.com](https://example.com/citation-probe-source)"+ )+ assert [c["index"] for c in citations] == [1, 2]+ assert [c["ref_id"] for c in citations] == ["turn0view0", "turn0view1"]++ def test_index_follows_first_appearance_across_markers(self):+ _, citations = resolve(+ f"B{marker('turn0view1')} then A{marker('turn0view0')} then B again{marker('turn0view1')}"+ )++ assert [(c["index"], c["label"]) for c in citations] == [+ (1, "example.com"),+ (2, "developers.openai.com"),+ ]++ def test_two_refs_for_the_same_page_share_one_index(self):+ """``turn0view0`` and ``turn1view0`` are the same URL from two searches."""+ text, citations = resolve(f"First{marker('turn0view0')} second{marker('turn1view0')}")++ assert len(citations) == 1+ assert citations[0]["index"] == 1+ assert text.count("[developers.openai.com]") == 2++ def test_every_sidecar_entry_matches_a_link_the_text_actually_carries(self):+ """The Web renderer matches on (href, link text), so the pair must agree."""+ text, citations = resolve(+ f"A{marker('turn0view0')} B{marker('turn0view1')} C{marker('turn0view0', 'turn9view9')}"+ )++ for citation in citations:+ assert f"[{citation['label']}]({citation['url']})" in text++ def test_markers_survive_a_final_message_that_repeats_an_earlier_one(self):+ """Resolving twice is idempotent: the second pass sees no markers left."""+ first, citations = resolve(f"Answer.{marker('turn0view0')}")+ second, again = resolve(first)++ assert second == first+ assert again == []+ assert len(citations) == 1+++class TestAttributionIsNeverDeleted:+ def test_unknown_ref_degrades_to_the_visible_label(self):+ text, citations = resolve(f"Claimed.{marker('turn7view7')}")++ assert text == f"Claimed. {UNRESOLVED}"+ assert citations == []++ def test_partially_resolved_marker_keeps_its_link_and_labels_the_rest_once(self):+ text, citations = resolve(f"Mixed.{marker('turn0view0', 'turn7view7', 'turn8view8')}")++ assert text == (+ "Mixed. "+ "[developers.openai.com](https://developers.openai.com/api/docs/guides/tools-web-search) "+ f"{UNRESOLVED}"+ )+ assert text.count(UNRESOLVED) == 1+ assert len(citations) == 1++ def test_raw_marker_survives_when_there_is_no_label_to_fall_back_to(self):+ raw = f"Claimed.{marker('turn7view7')}"++ text, citations = resolve(raw, unresolved_label="")++ assert text == raw+ assert citations == []++ def test_marker_with_an_empty_payload_is_reported_not_dropped(self):+ text, _ = resolve(f"Claimed.{START}cite{SEP}{END}")++ assert text == f"Claimed. {UNRESOLVED}"++ @pytest.mark.parametrize(+ "sources",+ [{}, {"turn0view0": CitationSource(ref_id="turn0view0")}],+ ids=["no-results-captured", "result-without-a-url"],+ )+ def test_missing_metadata_is_labelled_rather_than_guessed(self, sources):+ text, citations = resolve(f"Claimed.{marker('turn0view0')}", sources)++ assert text == f"Claimed. {UNRESOLVED}"+ assert citations == []+++class TestUntrustedMetadata:+ @pytest.mark.parametrize(+ "url",+ [+ "javascript:alert(1)",+ "data:text/html,<script>alert(1)</script>",+ "file:///etc/passwd",+ "mailto:[email protected]",+ "avibe-secret:OPENAI_KEY",+ "http:///no-host",+ "https://",+ " ",+ "not a url at all",+ ],+ )+ def test_unsafe_or_hostless_urls_are_rejected(self, url):+ assert safe_url(url) == ""++ text, citations = resolve(+ f"Claimed.{marker('turn0view0')}",+ {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=url)},+ )+ assert text == f"Claimed. {UNRESOLVED}"+ assert citations == []++ @pytest.mark.parametrize(+ "url",+ ["http://example.com/x", "https://example.com/x"],+ )+ def test_http_and_https_are_accepted_verbatim(self, url):+ assert safe_url(url) == url++ def test_an_uppercase_scheme_is_normalized_rather_than_carried(self):+ """A scheme is case-insensitive, but a renderer that only knows its+ lowercase spelling sees no link at all: Telegram delivered+ ``[example.com](HTTPS://Example.com/X)`` as raw Markdown."""+ assert safe_url("HTTPS://Example.com/X") == "https://Example.com/X"++ @pytest.mark.parametrize(+ "url",+ [+ "https://2130706433/p",+ "https://0x7f.1/p",+ "https://127.1/p",+ "https://017700000001/p",+ ],+ ids=["decimal", "hexadecimal", "shortened", "octal"],+ )+ def test_a_numeric_host_is_named_by_the_address_a_browser_reaches(self, url):+ """Every one of these opens the loopback address. Attributing the+ citation to the digits instead would hide that from the reader."""+ assert source_label(safe_url(url)) == "127.0.0.1"++ @pytest.mark.parametrize(+ "url",+ ["https://256.1.1.1/p", "https://1.2.3.4.5/p", "https://example.com.0x1/p"],+ ids=["out-of-range", "too-many-parts", "domain-shaped"],+ )+ def test_a_numeric_host_the_ipv4_parser_refuses_names_no_source(self, url):+ """A browser will not navigate to these, so there is no host to name."""+ assert safe_url(url) == ""++ @pytest.mark.parametrize(+ "code,replaced",+ [+ (0x08, True),+ (0x09, False),+ (0x0A, False),+ (0x0B, True),+ (0x0C, False),+ (0x0D, False),+ (0x0E, True),+ (0x1F, True),+ (0x20, False),+ (0x7E, False),+ (0x7F, True),+ (0x80, True),+ (0x9F, True),+ (0xA0, False),+ (0xD7FF, False),+ (0xD800, True),+ (0xDFFF, True),+ (0xE000, False),+ (0xFDCF, False),+ (0xFDD0, True),+ (0xFDEF, True),+ (0xFDF0, False),+ (0xFFFE, True),+ (0xFFFF, True),+ (0x1FFFE, True),+ (0x10FFFD, False),+ (0x10FFFF, True),+ (0x110000, True),+ ],+ )+ def test_the_replacement_table_is_the_one_the_renderer_uses(self, code, replaced):+ """micromark's table, boundary by boundary.++ The persisted URL has to equal the href drawn from it, so a numeric+ reference has to resolve the way the parser behind the Web renderer+ resolves it - which is not HTML's Windows-1252 mapping: ``€`` is+ U+FFFD there, never ``€``.+ """+ from core.citations import _is_replaced_code_point++ assert _is_replaced_code_point(code) is replaced++ def test_a_long_host_is_elided_from_the_left(self):+ """A label cut from the right reads as a site the link never opens."""+ host = "developers.openai.com." + "padding." * 6 + "attacker.example"++ label = source_label(f"https://{host}/x")++ assert label.startswith("…")+ assert label.endswith(".attacker.example")+ assert not label.startswith("developers.openai.com")+ assert host.endswith(label[1:])+ assert len(label) == 64++ def test_an_elision_shows_a_site_and_not_only_a_public_suffix(self):+ """``…co.uk`` is every site under it, which is where a long label hides.++ Keeping whole labels only was what produced that: the one label left of+ the suffix did not fit, so all of it was dropped. The budget is filled+ from the right instead, and the ellipsis marks the partial piece.+ """+ host = "a" * 62 + ".co.uk"++ label = source_label(f"https://{host}/x")++ assert label != "…co.uk"+ assert label == "…" + host[-63:]+ assert len(label) == 64++ @pytest.mark.parametrize(+ "host",+ [+ "a" * 70 + ".uk",+ "a" * 62 + ".co.uk",+ "a" * 100,+ "developers.openai.com." + "padding." * 6 + "attacker.example",+ "x" * 30 + "." + "y" * 30 + ".example.com",+ ],+ )+ def test_an_elided_label_is_always_a_tail_of_the_host(self, host):+ """The one invariant both surfaces assert: never a fabricated middle."""+ label = source_label(f"https://{host}/x")++ assert label.startswith("…")+ assert len(label) == 64+ assert host.endswith(label[1:])++ @pytest.mark.parametrize(+ "raw, expected",+ [+ ("p	q", "p%09q"),+ ("p	q", "p%09q"),+ ("p	q", "p%09q"),+ ("p
q", "p%0Aq"),+ ("p q", "p%0Aq"),+ ("p
q", "p%0Aq"),+ ("p
q", "p%0Dq"),+ # A literal one never reaches the renderer as part of a destination:+ # it ends the destination, so ``[x](...p<TAB>q)`` is not a link.+ ("p\tq", "pq"),+ ("p\nq", "pq"),+ ("p\rq", "pq"),+ ],+ )+ def test_a_tab_a_reference_spells_is_escaped_and_a_literal_one_is_dropped(+ self, raw, expected+ ):+ """The two are different characters, and the order of the passes decides.++ WHATWG's input cleanup removes a literal tab or newline, so it runs on+ the raw string. What a character reference resolves to afterwards is part+ of the destination, and micromark percent-encodes it - measured, not+ assumed: ``[x](https://example.com/p	q)`` renders+ ``https://example.com/p%09q``. Resolving before the cleanup deleted+ exactly those characters, silently moving the page.+ """+ assert safe_url(f"https://example.com/{raw}") == f"https://example.com/{expected}"++ @pytest.mark.parametrize(+ "host, expected",+ [+ # Wider than 32 bits: a browser reads the number and then fails the+ # address, so this names no source rather than naming a domain.+ ("99999999999", ""),+ ("9999999999", ""),+ ("0x" + "f" * 20, ""),+ ("0" + "7" * 20, ""),+ ("example.0x" + "F" * 16, ""),+ ("1.2.3." + "9" * 4400, ""),+ # Long enough that CPython refuses the decimal conversion outright.+ ("9" * 5000, ""),+ # The boundary that does fit, and an octal part however many zeros+ # pad it - both measured against ``new URL`` in a real browser.+ ("4294967295", "255.255.255.255"),+ ("0" * 20 + "127", "0.0.0.87"),+ ],+ )+ def test_a_numeric_host_wider_than_an_address_is_refused_not_converted(+ self, host, expected+ ):+ """An untrusted host of digits must not reach ``int`` unbounded.++ CPython refuses to convert a decimal string past+ ``sys.int_info.default_max_str_digits`` (4300), so a host spelled with+ thousands of digits raised ``ValueError`` out of ``safe_url`` - on+ nothing but a search result. The magnitude is what matters anyway: every+ check an IPv4 number reaches rejects a value this big.+ """+ url = f"https://{host}/x"++ assert source_label(url) == expected+ assert bool(safe_url(url)) is bool(expected)++ def test_markdown_punctuation_is_protected_in_the_link_and_not_in_the_url(self):+ """A parenthesis is part of the address, and only Markdown needs it guarded.++ Percent-encoding it in the URL protected the link and moved the page:+ ``Foo_%28bar%29`` is not the path the source gave. The URL keeps it, and+ the link the answer carries escapes it, which every reader resolves back.+ """+ url = 'https://en.wikipedia.org/wiki/Foo_(bar)?q=a b&t="x"<y>'++ canonical = safe_url(url)++ assert canonical == (+ "https://en.wikipedia.org/wiki/Foo_(bar)?q=a%20b&t=%22x%22%3Cy%3E"+ )+ text, citations = resolve(+ f"Claimed.{marker('turn0view0')}",+ {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=url)},+ )+ assert text == (+ "Claimed. [en.wikipedia.org]"+ "(https://en.wikipedia.org/wiki/Foo_\\(bar\\)?q=a%20b&t=%22x%22%3Cy%3E)"+ )+ assert citations[0]["url"] == canonical++ @pytest.mark.parametrize(+ "raw, expected",+ [+ ("https://en.wikipedia.org/wiki/Foo_(bar)", "https://en.wikipedia.org/wiki/Foo_(bar)"),+ ("https://example.com/a_(b", "https://example.com/a_(b"),+ ("https://example.com/a)b", "https://example.com/a)b"),+ ("https://example.com/a%28b%29", "https://example.com/a%28b%29"),+ ("https://example.com/a(b)", "https://example.com/a(b)"),+ ("https://[::1]:8443/x(y?q=(1)", "https://[::1]:8443/x(y?q=(1)"),+ ],+ ids=["balanced", "unbalanced-open", "unbalanced-close", "already-encoded",+ "character-reference", "ipv6"],+ )+ def test_a_parenthesis_reaches_the_reader_as_the_source_wrote_it(self, raw, expected):+ """The sidecar, the link and what a Markdown reader resolves all agree."""+ assert safe_url(raw) == expected++ text, citations = resolve(+ f"Claimed.{marker('turn0view0')}",+ {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=raw)},+ )++ assert citations[0]["url"] == expected+ tokens = MARKDOWN.parseInline(text)[0].children+ hrefs = [token.attrGet("href") for token in tokens if token.type == "link_open"]+ assert hrefs == [MARKDOWN.normalizeLink(expected)]+ assert text.endswith(")")++ @pytest.mark.parametrize(+ "url",+ [+ "https://trusted.example\\@attacker.example/x",+ "https://example.com/a\\b",+ "https://example.com/p?q=a\\b",+ "https://example.com/a\b",+ "https://example.com/a\b",+ ],+ ids=["authority", "path", "query", "numeric-reference", "named-reference"],+ )+ def test_a_backslash_left_in_the_address_names_no_source(self, url):+ """A browser reads ``\\`` as ``/`` in an http(s) authority and path, and+ as data in a query, so no one spelling of it names the page for every+ reader - ``trusted.example\\@attacker.example`` opens trusted.example+ in a browser and was attributed to attacker.example. It is refused."""+ assert safe_url(url) == ""++ text, citations = resolve(+ f"Claimed.{marker('turn0view0')}",+ {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=url)},+ )+ assert text == f"Claimed. {UNRESOLVED}"+ assert citations == []++ def test_an_encoded_backslash_is_an_ordinary_character_of_the_address(self):+ """``%5C`` is data to every reader, so it is kept exactly as written."""+ url = "https://example.com/a%5Cb?q=c%5Cd"++ assert safe_url(url) == url+ _, citations = resolve(+ f"Claimed.{marker('turn0view0')}",+ {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=url)},+ )+ assert citations[0]["url"] == url++ def test_non_ascii_title_is_preserved_and_collapsed_to_one_line(self):+ source = CitationSource(+ ref_id="turn0view0",+ title=" 中文標題 — Café äöü\n\tsecond line ",+ url="https://example.com/x",+ )++ _, citations = resolve(f"Claimed.{marker('turn0view0')}", {"turn0view0": source})++ assert citations[0]["title"] == "中文標題 — Café äöü second line"++ def test_control_and_private_use_characters_are_stripped_from_a_title(self):+ """A title must not be able to smuggle a marker back into the sidecar."""+ assert clean_title(f"A{START}cite{SEP}turn0view0{END}BC\x07D") == "A cite turn0view0 B C D"++ @pytest.mark.parametrize("title", [None, 42, b"bytes", {"t": "x"}])+ def test_non_string_titles_collapse_to_empty(self, title):+ assert clean_title(title) == ""++ def test_title_is_bounded(self):+ assert len(clean_title("x" * 5000)) == 200++ def test_label_is_the_domain_lowercased_without_www(self):+ assert source_label("https://WWW.Example.COM/path") == "example.com"+ assert source_label("https://sub.example.co.uk/x") == "sub.example.co.uk"++ def test_label_falls_back_to_the_title_when_there_is_no_host(self):+ assert source_label("not-a-url", "Some Title") == "Some Title"++ def test_markdown_label_punctuation_is_escaped(self):+ """Unreachable through a valid URL (a host has no ``[``), kept as a guard+ because the label falls back to an untrusted title when parsing degrades."""+ from core.citations import _escape_label++ assert _escape_label("a]b[c\\d") == "a\\]b\\[c\\\\d"++ def test_a_title_may_still_spell_an_emoji_or_an_indic_word(self):+ """The sanitizer removes what reorders text, not what joins it.++ A zero-width joiner and a variation selector are how a real title+ spells a family emoji, a Persian word or a text-style symbol. Deleting+ them silently rewrites the title; only the bidi controls, which can+ render a title's own text backwards, have no legitimate use on one+ line.+ """+ assert clean_title("👨\u200d👩\u200d👧 ☎\ufe0f افغانی\u200c ها") == (+ "👨\u200d👩\u200d👧 ☎\ufe0f افغانی\u200c ها"+ )+ assert clean_title("Report\u202egnp.\u202c pdf") == "Report gnp. pdf"++ @pytest.mark.parametrize(+ "label, closer",+ [+ ("ex`ample.com", "`code`"),+ ("a<!--b", "--> and"),+ ("a<?x", "?> and"),+ ("a<!X", "> and"),+ ("a<![CDATA[b", "]]> and"),+ ],+ ids=["code-span", "comment", "instruction", "declaration", "cdata"],+ )+ def test_a_label_cannot_swallow_the_link_it_labels(self, label, closer):+ """Measured against CommonMark: each of these, left raw, takes the link.++ A backtick or an angle bracket in link text does not merely change how+ the text reads. It opens a code span, an HTML comment, a processing+ instruction, a declaration or a CDATA section, which then runs past+ ``](url)`` looking for its closer - and markdown-it renders the whole+ thing as literal text, so the reader is shown raw Markdown with nothing+ to click. Escaping is what keeps the link a link.+ """+ from core.citations import _escape_label++ document = f"Claim. [{_escape_label(label)}](https://s.example/p) then {closer} prose."++ assert MARKDOWN.render(document).count('<a href="https://s.example/p">') == 1++ @pytest.mark.parametrize(+ "label, shown",+ [+ ("a*b*.example", "a<em>b</em>.example"),+ ("a©.example", "a\u00a9.example"),+ ],+ ids=["emphasis", "entity"],+ )+ def test_a_label_reads_as_the_host_it_names(self, label, shown):+ """A label is attribution, so it has to name the host character for+ character - and left raw, each of these names a different one.++ These do not break the link the way a backtick does; they change what+ the reader is told the source IS. ``shown`` records what CommonMark+ makes of the raw label, measured rather than assumed, and the escaped+ label has to render as the host instead. The IM dialects were measured+ too (tests/test_citation_consumers.py): none of them shows the+ backslash.+ """+ from core.citations import _escape_label++ url = "https://s.example/p"+ raw = MARKDOWN.render(f"[{label}]({url})")+ escaped = MARKDOWN.render(f"[{_escape_label(label)}]({url})")++ assert f'<a href="{url}">{shown}</a>' in raw+ assert f'<a href="{url}">{label.replace("&", "&")}</a>' in escaped++ def test_a_label_also_marks_what_only_another_dialect_reads_as_syntax(self):+ """The reader is not always reading CommonMark.++ CommonMark leaves an intraword ``_``, a single ``~`` and a ``|`` alone,+ so the test above cannot show them changing anything. Slack mrkdwn+ reads ``_italic_`` and ``~strike~``, and a ``|`` ends a GFM table cell -+ and a label is untrusted text that lands in all of them. Escaping is+ the one spelling every dialect agrees means "literal"; what each one+ actually shows is measured in tests/test_citation_consumers.py.+ """+ from core.citations import _escape_label++ assert _escape_label("a_b~c|d") == "a\\_b\\~c\\|d"++ @pytest.mark.parametrize(+ "ref_id",+ ["turn0view0 with spaces", "a" * 65, "ref/id", "refid", ""],+ ids=["spaces", "too-long", "slash", "zero-width", "empty"],+ )+ def test_malformed_ref_ids_never_reach_the_source_map(self, ref_id):+ text, citations = resolve(+ f"Claimed.{marker(ref_id)}",+ {ref_id: CitationSource(ref_id=ref_id, title="T", url="https://example.com/x")},+ )++ assert "example.com" not in text+ assert citations == []+++class TestCodeAndIncompleteMarkers:+ """Where a marker counts as code is a CommonMark question, not a regex one.++ The cases below are the ones local patterns get wrong: fence lengths nest,+ a closing fence may be longer than its opener, an unclosed fence runs to+ the end of the document, a code span may cross lines, and container+ indentation shifts all of it. Both directions are asserted - a literal+ marker keeps every byte, and a marker in prose still resolves even when+ code sits nearby - because over-masking silently drops attribution.+ """++ @pytest.mark.parametrize(+ "template",+ [+ "Inline `{m}` stays literal.",+ "```\n{m}\n```",+ "~~~text\n{m}\n~~~",+ " {m}",+ "\t{m}",+ "``{m}``",+ "`` ` {m} ``",+ "`line one\n{m}`",+ "````markdown\n```\n{m}\n```\n````",+ "`````\n````\n{m}\n````\n`````",+ "```text\n{m}",+ "~~~\n{m}",+ "- item\n\n ```\n {m}\n ```\n",+ "> ```\n> {m}\n> ```\n",+ "> - a\n>\n> ```\n> {m}\n> ```\n",+ "- item\n\n {m}\n",+ ],+ ids=[+ "inline",+ "fenced",+ "tilde-fenced",+ "indented",+ "tab-indented",+ "double-backtick-span",+ "double-backtick-span-holding-a-backtick",+ "code-span-across-lines",+ "four-backtick-fence-quoting-three",+ "five-backtick-fence-quoting-four",+ "unclosed-fence-runs-to-end",+ "unclosed-tilde-fence-runs-to-end",+ "fence-inside-a-list-item",+ "fence-inside-a-blockquote",+ "fence-inside-a-blockquoted-list",+ "indented-code-inside-a-list-item",+ ],+ )+ def test_a_marker_inside_a_code_example_is_untouched(self, template):+ raw = template.format(m=marker("turn0view0"))++ text, citations = resolve(raw)++ assert text == raw+ assert citations == []++ @pytest.mark.parametrize(+ "template",+ [+ "- item {m}\n",+ "> quoted {m}\n",+ "```\ncode\n`````\nanswer{m}",+ "````\n```\n````\nanswer{m}",+ ],+ ids=[+ "inside-a-list-item",+ "inside-a-blockquote",+ "after-a-longer-closing-fence",+ "after-a-fence-that-quoted-a-fence",+ ],+ )+ def test_a_marker_in_prose_resolves_even_next_to_code(self, template):+ """Code lexing decides what to skip; it must not swallow real prose."""+ raw = template.format(m=marker("turn0view0"))++ text, citations = resolve(raw)++ assert marker("turn0view0") not in text+ assert "[developers.openai.com]" in text+ assert [c["ref_id"] for c in citations] == ["turn0view0"]++ @pytest.mark.parametrize(+ "template",+ [+ "Hi <!-- {m} --> then",+ 'a <span title="{m}">shown</span>',+ "a <x y='{m}'> b",+ "a <?php {m} ?> b",+ "a <![CDATA[ {m} ]]> b",+ "a <!DOCTYPE {m}> b",+ "<!--\n{m}\n-->\n",+ '<div title="{m}">\n\nafter',+ "<div>\n{m}\n</div>\n\n",+ ],+ ids=[+ "inline-comment",+ "inline-attribute",+ "single-quoted-attribute",+ "processing-instruction",+ "cdata",+ "declaration",+ "comment-block",+ "html-block-attribute",+ "html-block-body",+ ],+ )+ def test_a_marker_inside_raw_html_is_untouched(self, template):+ """The renderer passes raw HTML through without reading a link from it.++ A source numbered there would be a badge nobody sees, and the next+ visible citation would start the transcript at 2.+ """+ raw = template.format(m=marker("turn0view0")) + f" later{marker('turn0view1')}"++ text, citations = resolve(raw)++ assert template.format(m=marker("turn0view0")) in text+ assert [(c["index"], c["ref_id"]) for c in citations] == [(1, "turn0view1")]++ @pytest.mark.parametrize(+ "template",+ ["a <b>{m}</b> end", "a <x {m}> b"],+ ids=["between-two-tags", "not-a-tag"],+ )+ def test_a_marker_the_reader_is_shown_beside_html_still_resolves(self, template):+ raw = template.format(m=marker("turn0view0"))++ _text, citations = resolve(raw)++ assert [c["ref_id"] for c in citations] == ["turn0view0"]++ def test_prose_around_a_code_example_still_resolves(self):+ raw = f"See{marker('turn0view0')}\n\n```\n{marker('turn0view0')}\n```\n\nAnd{marker('turn0view1')}"++ text, citations = resolve(raw)++ assert f"```\n{marker('turn0view0')}\n```" in text+ assert text.count("[developers.openai.com]") == 1+ assert [c["label"] for c in citations] == ["developers.openai.com", "example.com"]++ @pytest.mark.parametrize(+ "raw",+ [+ f"split {START}cite{SEP}turn0view0",+ f"split cite{SEP}turn0view0{END}",+ f"{START}visualize{SEP}" '{"a":1}' f"{END} keep",+ ],+ ids=["no-terminator", "no-opener", "sibling-marker"],+ )+ def test_an_incomplete_or_foreign_marker_is_left_alone(self, raw):+ assert resolve(raw) == (raw, [])++ def test_a_restarted_marker_resolves_the_complete_half_and_keeps_the_truncated_one(self):+ """Chunked delivery can strip a terminator; the well-formed half still counts."""+ truncated = f"{START}cite{SEP}turn0view0"++ text, citations = resolve(f"nested {truncated}{marker('turn0view1')}")++ assert text == (+ f"nested {truncated} [example.com](https://example.com/citation-probe-source)"+ )+ assert [c["ref_id"] for c in citations] == ["turn0view1"]+++class TestDetectionAndSpacing:+ @pytest.mark.parametrize(+ "text, expected",+ [+ (None, False),+ ("", False),+ ("plain text", False),+ (f"{START}cite{SEP}turn0view0", False),+ (f"{START}visualize{SEP}x{END}", False),+ (f"has{marker('turn0view0')}", True),+ ],+ )+ def test_has_citation_markers(self, text, expected):+ assert has_citation_markers(text) is expected++ @pytest.mark.parametrize(+ "raw, expected",+ [+ ("word{m}", "word [example.com](https://example.com/citation-probe-source)"),+ ("word {m}", "word [example.com](https://example.com/citation-probe-source)"),+ ("{m}", "[example.com](https://example.com/citation-probe-source)"),+ ("word.\n{m}", "word.\n[example.com](https://example.com/citation-probe-source)"),+ ],+ ids=["abutting", "already-spaced", "at-start", "after-newline"],+ )+ def test_one_separating_space_is_added_only_when_needed(self, raw, expected):+ text, _ = resolve(raw.format(m=marker("turn0view1")))++ assert text == expected++ @pytest.mark.parametrize("text", [None, "", "no markers here"])+ def test_text_without_markers_is_returned_unchanged_with_no_sidecar(self, text):+ assert resolve(text) == (text, [])+++class TestHiddenBlocks:+ """A citation inside a ``<silent>`` block belongs to no delivered message.++ The block is removed downstream of this rewrite, so numbering a marker+ inside one attributes a source the reader never sees: the sidecar opens at+ index 2, the badge it describes has no link, and the visible citation is+ misnumbered. Hidden markers are therefore skipped exactly the way code is -+ left literal, so they leave with the block that hid them.+ """++ def test_a_hidden_marker_is_neither_numbered_nor_rewritten(self):+ raw = (+ f"<silent>internal note{marker('turn0view0')}</silent>"+ f"Visible.{marker('turn0view1')}"+ )++ text, citations = resolve(raw)++ assert marker("turn0view0") in text+ assert text.endswith("Visible. [example.com](https://example.com/citation-probe-source)")+ assert [(c["index"], c["ref_id"]) for c in citations] == [(1, "turn0view1")]++ def test_stripping_the_block_leaves_exactly_the_visible_citation(self):+ raw = (+ f"<silent>hidden{marker('turn0view0')}</silent>"+ f"Visible.{marker('turn0view1')}"+ )++ text, _ = resolve(raw)++ assert strip_silent_blocks(text) == (+ "Visible. [example.com](https://example.com/citation-probe-source)"+ )++ def test_an_entirely_hidden_message_carries_no_sidecar(self):+ raw = f"<silent>only this{marker('turn0view0')}</silent>"++ text, citations = resolve(raw)++ assert text == raw+ assert citations == []++ def test_a_hidden_block_inside_a_code_example_hides_nothing(self):+ """``<silent>`` shown as code is not a control, so the prose marker resolves."""+ raw = f"`<silent>`\nVisible.{marker('turn0view0')}"++ text, citations = resolve(raw)++ assert "[developers.openai.com]" in text+ assert [c["ref_id"] for c in citations] == ["turn0view0"]+++class TestRequestedRefs:+ """What a message asks for, which is what a backend has to go looking for.++ A backend reads its recorded history to fill the gaps in a message's+ attribution, so it needs the refs that message actually requests - not every+ ref it ever saw. The same eligibility rules apply as to the rewrite: code and+ hidden blocks ask for nothing, and a malformed ref is not a request.+ """++ def test_the_requested_refs_are_reported_in_first_appearance_order(self):+ text = f"A{marker('turn0view1')} B{marker('turn0view0', 'turn0view1')}"++ assert citation_ref_ids(text) == ["turn0view1", "turn0view0"]++ @pytest.mark.parametrize(+ "text",+ [None, "", "plain text", f"{START}cite{SEP}turn0view0", f"{START}visualize{SEP}x{END}"],+ ids=["none", "empty", "no-marker", "unterminated", "sibling-marker"],+ )+ def test_text_that_asks_for_nothing_reports_nothing(self, text):+ assert citation_ref_ids(text) == []++ def test_a_marker_in_code_asks_for_nothing(self):+ assert citation_ref_ids(f"```\n{marker('turn0view0')}\n```") == []++ def test_a_marker_in_a_hidden_block_asks_for_nothing(self):+ assert citation_ref_ids(f"<silent>{marker('turn0view0')}</silent>") == []++ def test_a_malformed_ref_is_not_a_request(self):+ assert citation_ref_ids(f"A{marker('has space', 'turn0view0', 'x' * 65)}") == [+ "turn0view0"+ ]+++class TestUnresolvedRefs:+ """Which requested refs are still waiting for a source, and which never will be.++ A backend holds a message while a ref it names may still arrive. A ref whose+ source is already present but unusable is not waiting for anything - holding+ it would delay the message forever - so it is reported as resolved-as-far-as-+ it-goes and degrades to the visible label instead.+ """++ def test_a_ref_with_no_source_is_unresolved(self):+ assert unresolved_refs(["turn0view0", "turn9view9"], SOURCES) == ["turn9view9"]++ def test_a_fully_sourced_message_is_not_waiting(self):+ assert unresolved_refs(["turn0view0", "turn0view1"], SOURCES) == []++ @pytest.mark.parametrize(+ "url",+ ["javascript:alert(1)", "", "https:///path", "not-a-url"],+ ids=["unsafe-scheme", "empty", "hostless", "unparseable-authority"],+ )+ def test_a_source_that_can_never_be_linked_is_not_waited_for(self, url):+ sources = {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=url)}++ assert unresolved_refs(["turn0view0"], sources) == []++ def test_nothing_requested_means_nothing_unresolved(self):+ assert unresolved_refs([], SOURCES) == []+++class TestRegisteredIdentity:+ """Which text in a delivered message is allowed to be a citation at all.++ A marker means what it says exactly once: in the text the backend produced.+ Everything downstream rewrites that text - a ``<silent>`` block leaves, a+ ``file://`` link is stripped to its label, images are appended - and a+ rewrite can splice two ordinary halves into characters that read as a+ marker. So the markers are registered against the native text and travel as+ opaque per-occurrence tokens; the stage that finally writes the links reads+ only those tokens and never the grammar again.+ """++ GUIDE_URL = "https://developers.openai.com/api/docs/guides/tools-web-search"+ GUIDE_LINK = f"[developers.openai.com]({GUIDE_URL})"++ def test_a_marker_a_later_transform_splices_together_is_not_a_citation(self):+ """The counterexample that decides where registration has to happen.++ Only the first of these is a citation. Strip the hidden block - which+ every delivered copy does - and the second half is character for+ character a legal marker too. A stage that read the grammar here would+ attribute a page to text the model never cited.+ """+ native = (+ f"Actual.{marker('turn0view0')}\n\n"+ f"Synthetic. {START}ci<silent>private</silent>te{SEP}turn0view0{END}"+ )+ registered, bundle = register_citations(+ native, SOURCES, unresolved_label=UNRESOLVED+ )+ delivered = strip_silent_blocks(registered)+ body, citations = finalize_citations(delivered, bundle)++ assert body.count(self.GUIDE_LINK) == 1+ assert body.endswith(f"Synthetic. {marker('turn0view0')}")+ assert len(citations) == 1+ assert citations[0]["spans"] == [+ [body.index(self.GUIDE_LINK), body.index(self.GUIDE_LINK) + len(self.GUIDE_LINK)]+ ]++ def test_a_copy_of_a_registered_token_still_speaks_for_its_citation(self):+ """A transform that duplicates text duplicates the citation with it -+ the copy is the same registered occurrence, not a new claim."""+ registered, bundle = register_citations(+ f"Cited.{marker('turn0view0')}", SOURCES, unresolved_label=UNRESOLVED+ )+ body, citations = finalize_citations(f"{registered}\n\n{registered}", bundle)++ assert body.count(self.GUIDE_LINK) == 2+ assert len(citations) == 1+ assert [utf16_slice(body, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK,+ self.GUIDE_LINK,+ ]++ def test_token_shaped_text_this_bundle_never_minted_is_left_alone(self):+ """An identity is a nonce this call handed out, not a shape."""+ registered, bundle = register_citations(+ f"Cited.{marker('turn0view0')}", SOURCES, unresolved_label=UNRESOLVED+ )+ forged = f"{TOKEN_OPEN}{'0' * 32}{TOKEN_CLOSE}"+ body, citations = finalize_citations(f"{forged} {registered}", bundle)++ assert body.startswith(forged)+ assert len(citations) == 1+ assert [utf16_slice(body, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK+ ]++ def test_a_token_a_transform_deleted_describes_nothing(self):+ registered, bundle = register_citations(+ f"Cited.{marker('turn0view0')}", SOURCES, unresolved_label=UNRESOLVED+ )+ body, citations = finalize_citations("Nothing left.", bundle)++ assert TOKEN_OPEN in registered+ assert body == "Nothing left."+ assert citations == []++ @pytest.mark.parametrize(+ "template",+ ["`{token}`", "```\n{token}\n```", "<silent>{token}</silent>"],+ ids=["code-span", "code-fence", "hidden"],+ )+ def test_a_token_a_transform_moved_into_code_shows_the_model_s_own_text(+ self, template+ ):+ """An internal token is the one thing a reader may never be shown, and+ a code example is not an attribution either - so the marker the model+ wrote comes back verbatim and no badge is claimed for it."""+ registered, bundle = register_citations(+ f"Cited.{marker('turn0view0')}", SOURCES, unresolved_label=UNRESOLVED+ )+ token = registered[len("Cited.") :]+ body, citations = finalize_citations(+ template.format(token=token), bundle+ )++ assert body == template.format(token=marker("turn0view0"))+ assert citations == []+++class TestWhereACitationCanBeShown:+ """Which part of a link may become a citation, and which is only its wiring.++ A citation is a link, and CommonMark has no link inside a link: written into+ a label it takes the enclosing unit down with it, and the reader is shown+ the brackets instead of the page the answer meant to point at. Written into+ a destination, a title, a reference identifier or an autolink's address it+ is not shown to anyone at all - that text is what makes the unit reach+ somewhere - and splicing a link through it sends the unit to an address+ nobody wrote.++ So the parser decides: a marker standing where a reader is shown something+ is a citation, and its source is written immediately after the outermost+ unit that encloses it; a marker standing in a slot only the parser reads+ stays the characters the model typed, mints no token, and claims nothing.+ """++ GUIDE_LINK = f"[developers.openai.com]({GUIDE.url})"++ def written(self, native: str) -> tuple[str, list[dict]]:+ """What delivery finally writes for *native*, and what it claims."""+ registered, bundle = register_citations(+ native, SOURCES, unresolved_label=UNRESOLVED+ )+ return finalize_citations(registered, bundle)++ @pytest.mark.parametrize(+ ("native", "expected"),+ [+ (+ f"See [the docs {marker('turn0view0')}](https://openai.com/docs) for details.",+ "See [the docs ](https://openai.com/docs) {link} for details.",+ ),+ (+ f"[{marker('turn0view0')} a](https://u.example/1)",+ "[ a](https://u.example/1) {link}",+ ),+ (+ f"[a {marker('turn0view0')} b](https://u.example/1)",+ "[a b](https://u.example/1) {link}",+ ),+ (+ f"",+ " {link}",+ ),+ (+ f"[](https://out.example/page)",+ "[](https://out.example/page) {link}",+ ),+ (+ f"[the docs {marker('turn0view0')}][ref]\n\n[ref]: https://r.example/page",+ "[the docs ][ref] {link}\n\n[ref]: https://r.example/page",+ ),+ (+ f"\u65e5\u672c\u8a9e [\u30e9\u30d9\u30eb {marker('turn0view0')}](https://u.example/1)",+ "\u65e5\u672c\u8a9e [\u30e9\u30d9\u30eb ](https://u.example/1) {link}",+ ),+ ],+ ids=[+ "label-middle",+ "label-start",+ "label-inner",+ "image-alt",+ "image-inside-a-link",+ "explicit-reference-label",+ "non-ascii-before-it",+ ],+ )+ def test_a_source_is_written_after_the_unit_whose_label_asked_for_it(+ self, native, expected+ ):+ """The destination the answer wrote survives, and so does its wording.++ An image inside a link is the case that makes "the outermost unit" the+ rule rather than "the unit it was in": place the source after the image+ and it lands in the link's label, which is the defect this describes.+ """+ body, citations = self.written(native)++ assert body == expected.format(link=self.GUIDE_LINK)+ assert [utf16_slice(body, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK+ ]++ @pytest.mark.parametrize(+ "native",+ [+ f"[a](https://example.com/{marker('turn0view0')}path)",+ f'[b](https://example.com/y "T {marker("turn0view0")}")',+ f"[c](<https://example.com/z{marker('turn0view0')}>)",+ f"<https://example.com/auto{marker('turn0view0')}>",+ f"}q.png)",+ f"[the docs][{marker('turn0view0')}]\n\n[{marker('turn0view0')}]: https://r.example/p",+ f"[dual {marker('turn0view0')}]\n\n[dual {marker('turn0view0')}]: https://r.example/d",+ f"[dual {marker('turn0view0')}][]\n\n[dual {marker('turn0view0')}]: https://r.example/d",+ f"[docs {marker('turn0view0')}]\n\n[docs]: https://r.example/d",+ f"[docs {marker('turn0view0')}][]\n\n[docs]: https://r.example/d",+ f"![docs {marker('turn0view0')}]\n\n[docs]: https://i.example/d.png",+ f"![docs {marker('turn0view0')}][]\n\n[docs]: https://i.example/d.png",+ ],+ ids=[+ "destination",+ "title",+ "angle-destination",+ "angle-autolink",+ "image-source",+ "reference-identifier-and-definition",+ "shortcut-reference",+ "collapsed-reference",+ "shortcut-reference-the-marker-unresolves",+ "collapsed-reference-the-marker-unresolves",+ "shortcut-image-the-marker-unresolves",+ "collapsed-image-the-marker-unresolves",+ ],+ )+ def test_a_slot_only_the_parser_reads_keeps_the_model_s_own_text(self, native):+ """Nothing is minted, nothing is written, and nothing is waited for.++ A shortcut or collapsed reference is the dual-use case: those brackets+ are both the words a reader sees and the identifier that finds the+ address, so editing them would leave a link pointing at nothing.++ A marker the definition does not repeat is the same run all the same.+ CommonMark cannot match ``[docs <marker>]`` against ``[docs]`` and so+ reads the brackets as prose, but the run is still what names that+ definition - and a citation written into it costs the reader the link+ the definition was about to give them, image or link alike.+ """+ body, citations = self.written(native)++ assert body == native+ assert citations == []+ # The same decision on both sides of the boundary: a marker that cannot+ # become a link must not hold the message waiting for a source either.+ assert citation_ref_ids(native) == []+ assert unresolved_refs(citation_ref_ids(native), {}) == []++ def test_two_markers_in_two_labels_are_two_sources_in_reading_order(self):+ native = (+ f"[a {marker('turn0view0')}](https://u.example/1)"+ f" and [b {marker('turn0view0')}](https://u.example/2)"+ )+ body, citations = self.written(native)++ assert body == (+ f"[a ](https://u.example/1) {self.GUIDE_LINK}"+ f" and [b ](https://u.example/2) {self.GUIDE_LINK}"+ )+ # One source cited twice is one row and two spans - the numbering counts+ # sources a reader can see, not markers the model wrote.+ assert len(citations) == 1+ assert [utf16_slice(body, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK,+ self.GUIDE_LINK,+ ]++ @pytest.mark.parametrize(+ ("moved", "expected"),+ [+ ("[lab {token}](https://out.example/page)", "[lab ](https://out.example/page) {link}"),+ ("[a](https://example.com/{token}path)", "[a](https://example.com/{marker}path)"),+ ('[b](https://example.com/y "T {token}")', '[b](https://example.com/y "T {marker}")'),+ ],+ ids=["into-a-label", "into-a-destination", "into-a-title"],+ )+ def test_a_token_a_transform_moved_is_judged_where_it_ended_up(+ self, moved, expected+ ):+ """Delivery closes gaps, and a token can land somewhere the model never+ put it. Writing a link into a destination on the strength of where the+ marker started is how an answer acquires an address nobody wrote."""+ registered, bundle = register_citations(+ f"Cited.{marker('turn0view0')}", SOURCES, unresolved_label=UNRESOLVED+ )+ token = registered[len("Cited.") :]+ body, citations = finalize_citations(moved.format(token=token), bundle)++ assert body == expected.format(+ link=self.GUIDE_LINK, marker=marker("turn0view0")+ )+ assert bool(citations) is ("{link}" in expected)++ def test_a_field_that_is_not_markdown_keeps_plain_attribution_in_place(self):+ """An attachment's title and a quick-reply button are not Markdown a+ reader parses, so there is no unit to place a source after and no link+ syntax to write: the source's name stands where the marker stood."""+ registered, bundle = register_citations(+ f"report {marker('turn0view0')}", SOURCES, unresolved_label=UNRESOLVED+ )+ token = registered[len("report ") :]++ assert (+ materialize_citations(registered, bundle, as_markdown=False)+ == "report developers.openai.com"+ )+ assert (+ materialize_citations(f"OK {token}", bundle, as_markdown=False)+ == "OK developers.openai.com"+ )+ assert TOKEN_OPEN not in materialize_citations(+ registered, bundle, as_markdown=False+ )+++class TestBodyBinding:+ """Which body the sidecar describes, and where in it each citation sits.++ A badge is an attribution claim, so it belongs to a link this module wrote+ and not to any link the answer's own prose happens to point at the same+ page. The rows therefore carry the exact ``spans`` their links occupy -+ UTF-16 code units, half-open, the coordinates the browser counts in.++ A span alone is not an identity. Two ordinary links can be spelled the+ same, and deleting a paragraph above one moves another into the+ coordinates it vacated: measured with react-markdown, ``L + "\\n\\n" + L``+ puts links at ``[0, 36]`` and ``[38, 74]``, so dropping the first+ paragraph leaves an ordinary link sitting exactly where a citation was+ measured. So every row also carries the digest of the body it was measured+ in, and a consumer that cannot reproduce that digest paints no badge.+ """++ GUIDE_URL = "https://developers.openai.com/api/docs/guides/tools-web-search"+ GUIDE_LINK = f"[developers.openai.com]({GUIDE_URL})"++ def test_a_citation_spans_the_link_it_wrote(self):+ text, citations = resolve(f"Documented.{marker('turn0view0')}")++ assert [utf16_slice(text, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK+ ]+ assert citations[0]["body_sha256"] == body_digest(text)++ def test_the_lead_space_and_the_fallback_label_belong_to_no_citation(self):+ """The span is the link, not the punctuation around it, and text that+ stands in for a source nobody could resolve is nobody's link."""+ text, citations = resolve(+ f"Both.{marker('turn0view0', 'turn9view9')}"+ )++ assert text == f"Both. {self.GUIDE_LINK} {UNRESOLVED}"+ assert [utf16_slice(text, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK+ ]++ def test_prose_that_spells_the_same_link_is_not_the_one_described(self):+ """Character for character the same link, so the reader cannot tell+ them apart either - and neither side pretends to. The badge stays on+ the one this module wrote."""+ text, citations = resolve(f"See {self.GUIDE_LINK} first.{marker('turn0view0')}")++ assert text.count(self.GUIDE_LINK) == 2+ assert citations[0]["spans"] == [+ [text.rindex(self.GUIDE_LINK), len(text)]+ ]++ def test_one_source_cited_twice_spans_both_of_its_links(self):+ text, citations = resolve(f"One.{marker('turn0view0')} Two.{marker('turn1view0')}")++ assert len(citations) == 1+ assert [utf16_slice(text, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK,+ self.GUIDE_LINK,+ ]++ def test_a_citation_to_another_page_moves_neither_of_its_neighbours(self):+ text, citations = resolve(+ f"A.{marker('turn0view0')} B.{marker('turn0view1')} C.{marker('turn1view0')}"+ )++ assert [+ [utf16_slice(text, *span) for span in c["spans"]] for c in citations+ ] == [+ [self.GUIDE_LINK, self.GUIDE_LINK],+ [f"[example.com]({PROBE.url})"],+ ]++ def test_spans_are_counted_in_utf16_code_units(self):+ """What the browser counts, not what Python iterates: an astral emoji+ is one character here and two code units there, and the consumer reads+ the second number."""+ text, citations = resolve(f"🙂🙂 Documented.{marker('turn0view0')}")+ start, end = citations[0]["spans"][0]++ assert start == text.index(self.GUIDE_LINK) + 2+ assert utf16_slice(text, start, end) == self.GUIDE_LINK++ def test_every_row_of_one_body_carries_that_body_s_digest(self):+ text, citations = resolve(+ f"A.{marker('turn0view0')} B.{marker('turn0view1')}"+ )++ assert len(citations) == 2+ assert {c["body_sha256"] for c in citations} == {body_digest(text)}++ def test_a_transform_between_registration_and_delivery_moves_the_spans(self):+ """The reason the two halves exist. The hidden block leaves before the+ links are written, so the spans are measured in the body that ships."""+ registered, bundle = register_citations(+ f"<silent>Private.</silent>Shown.{marker('turn0view0')}",+ SOURCES,+ unresolved_label=UNRESOLVED,+ )+ body, citations = finalize_citations(strip_silent_blocks(registered), bundle)++ assert body == f"Shown. {self.GUIDE_LINK}"+ assert [utf16_slice(body, *span) for span in citations[0]["spans"]] == [+ self.GUIDE_LINK+ ]+ assert citations[0]["body_sha256"] == body_digest(body)++ def test_a_link_that_outlives_the_citation_beside_it_is_described_by_nobody(self):+ """The impersonation counterexample, in the one place it can be stopped.++ Delete the paragraph a citation was written into and the answer's own+ prose link slides into the coordinates it held. Nothing downstream can+ tell the difference by looking; the sidecar simply never describes it,+ because the token that spoke for the citation left with the paragraph.+ """+ registered, bundle = register_citations(+ f"Cited.{marker('turn0view0')}\n\nSee {self.GUIDE_LINK} too.",+ SOURCES,+ unresolved_label=UNRESOLVED,+ )+ body, citations = finalize_citations(registered.split("\n\n", 1)[1], bundle)++ assert body == f"See {self.GUIDE_LINK} too."+ assert citations == []+++class TestBodyDigest:+ """One digest definition, asserted on both sides of the boundary.++ SHA-256 over the body's UTF-16 code units, little-endian, no BOM - the+ bytes ``charCodeAt`` yields in the browser and ``surrogatepass`` yields+ here. The vectors live in a fixture the renderer's test reads too, so the+ two implementations are held to the same table rather than to each other's+ reputation.+ """++ CASES = json.loads(+ (Path(__file__).parent / "fixtures" / "citation_body_digest.json").read_text(+ encoding="utf-8"+ )+ )++ @pytest.mark.parametrize("case", CASES, ids=[c["name"] for c in CASES])+ def test_the_backend_computes_the_digest_the_renderer_will_check(self, case):+ assert body_digest(case["text"]) == case["sha256"]++ @pytest.mark.parametrize("case", CASES, ids=[c["name"] for c in CASES])+ def test_the_recorded_digest_is_sha256_of_the_bytes_charcodeat_yields(self, case):+ assert case["sha256"] == hashlib.sha256(charcodeat_bytes(case["text"])).hexdigest()+++class TestUrlIdentityAcrossTheBoundary:+ """One URL table, asserted on both sides of the backend/renderer boundary.++ The sidecar is matched against the ``href`` the Markdown renderer produced,+ so the two have to agree byte for byte. The renderer normalizes its link+ destinations (percent-encoding, escape resolution, surrogate repair) and the+ backend cannot see that happen, so every case here is also rendered by+ ``ui/src/components/ui/markdown.test.tsx`` from this same fixture. A ``url``+ of ``null`` means the raw value must be rejected rather than canonicalized.++ The fixture's ``authority`` half is the same table for the userinfo, host+ and port, and it is read straight into the same cases: an authority is one+ thing a browser decides all at once, so a new shape belongs in that table+ rather than in a branch of its own.+ """++ FIXTURE = json.loads(+ (Path(__file__).resolve().parent / "fixtures/citation_url_identity.json").read_text(+ encoding="utf-8"+ )+ )+ CASES = [*FIXTURE["cases"], *FIXTURE["authority"]]++ @pytest.mark.parametrize("case", CASES, ids=[c["why"] for c in CASES])+ def test_the_backend_produces_the_url_the_renderer_will_keep(self, case):+ assert safe_url(case["raw"]) == (case["url"] or "")++ @pytest.mark.parametrize(+ "case", [c for c in CASES if c["url"]], ids=[c["why"] for c in CASES if c["url"]]+ )+ def test_the_label_attributes_the_host_a_browser_would_reach(self, case):+ assert source_label(case["url"]) == case["label"]++ @pytest.mark.parametrize(+ "case", [c for c in CASES if c["url"]], ids=[c["why"] for c in CASES if c["url"]]+ )+ def test_canonicalizing_a_canonical_url_changes_nothing(self, case):+ """The rewrite runs once per message; it must be a fixed point regardless."""+ assert safe_url(case["url"]) == case["url"]++ def test_a_rewritten_message_carries_the_canonical_url_in_both_halves(self):+ raw = "https://example.com/中文"+ sources = {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=raw)}++ text, citations = resolve(f"Cited.{marker('turn0view0')}", sources)++ assert citations[0]["url"] == "https://example.com/%E4%B8%AD%E6%96%87"+ assert f"({citations[0]['url']})" in text+++class TestAuthorityBracketsAcrossTheBoundary:+ """The citation's column of the shared authority table.++ ``[`` and ``]`` are an IPv6 host's syntax and data everywhere else, and+ ``_canonical_uri`` writes every bracket it is handed as ``%5B``/``%5D``.+ Asking afterwards which ones to restore is asking a string that no longer+ holds the answer: ``https://[::1]/admin`` and ``https://%5B::1%5D/admin``+ arrive at the same spelling, and putting brackets back into both hands the+ second one a live loopback address its source never named. So the question+ is settled on the resolved destination, before spelling, and the answer is+ carried through.++ The Web renderer and the Slack adapter are measured on the same rows from+ their own side - see the fixture's description - because three surfaces+ agreeing on a repair is agreement, not proof.+ """++ FIXTURE = json.loads(+ (Path(__file__).resolve().parent / "fixtures/citation_authority_matrix.json").read_text(+ encoding="utf-8"+ )+ )+ CASES = FIXTURE["cases"]++ @pytest.mark.parametrize("case", CASES, ids=[c["why"] for c in CASES])+ def test_only_a_valid_literal_authority_keeps_its_brackets(self, case):+ assert safe_url(case["destination"]) == (case["safe_url"] or "")++ @pytest.mark.parametrize(+ "case", [c for c in CASES if c["safe_url"]], ids=[c["why"] for c in CASES if c["safe_url"]]+ )+ def test_the_accepted_spelling_is_still_a_fixed_point(self, case):+ """A restored bracket must not be spelled away by the next pass."""+ assert safe_url(case["safe_url"]) == case["safe_url"]diff --git a/tests/test_codex_citations.py b/tests/test_codex_citations.pynew file mode 100644index 000000000..9d95a42ae--- /dev/null+++ b/tests/test_codex_citations.py@@ -0,0 +1,1258 @@+"""Citation handling in the Codex backend: source capture and emit-time resolution.++The grammar itself is covered by ``tests/test_citations.py``. What is specific to+this backend, and asserted here, is *where* a ``ref_id`` is allowed to mean+something: a ref is only unique inside its own native Codex thread, arrives in a+notification separate from the answer that cites it, and survives the ways a+conversation loses its live stream - a restarted process, a resume, a re-read, a+native fork - by being read back from the history Codex already recorded.++Every case runs against an isolated ``CODEX_HOME``. Hydration is part of the+path under test, so a case that did not build its own rollout fixture must see+an empty one rather than the developer's real Codex state.+"""++from __future__ import annotations++import json+import os+import sqlite3+import sys+import tempfile+import unittest+from pathlib import Path+from types import SimpleNamespace+from unittest import mock+from unittest.mock import AsyncMock++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from core.citations import body_digest, finalize_citations, materialize_citations+from core.reply_enhancer import strip_silent_blocks+from modules.agents.codex import search_history+from modules.agents.codex.event_handler import (+ _MAX_CITATION_SOURCES_PER_THREAD,+ _MAX_CITATION_THREADS,+)+# The handler harness loads ``event_handler.py`` under its own module name, so a+# patch aimed at the real import path would miss the copy under test.+from tests.test_codex_event_handler import _MODULE as codex_event_handler+from tests.test_codex_event_handler import CodexEventHandler, _StubAgent++START, SEP, END = "\ue200", "\ue202", "\ue201"+UNRESOLVED = "(source unavailable)"++GUIDE_URL = "https://developers.openai.com/api/docs/guides/tools-web-search"+PROBE_URL = "https://example.com/citation-probe-source"+++def marker(*ref_ids: str) -> str:+ return f"{START}cite{SEP}{SEP.join(ref_ids)}{END}"+++def written(call, at: int = 1) -> str | None:+ """The body a delivery boundary writes from one emit call.++ The handler hands the boundary an opaque token per citation plus the bundle+ that writes it out (see ``core.citations``), because every surface rewrites+ the text on the way to its reader. What the reader is shown is the+ boundary's own copy, so that is what these cases assert.+ """+ return materialize_citations(call.args[at], call.kwargs.get("citations"))+++def sidecar(call, at: int = 1) -> list[dict]:+ """The sidecar rows a persisted copy of one emit call would carry."""+ return finalize_citations(call.args[at], call.kwargs.get("citations"))[1]+++def cached_sources(handler, thread_id: str) -> dict:+ """The sources one thread's cache entry currently holds.++ Reads the handler's own bookkeeping, which carries a thread's readiness+ alongside its sources, so a test about the bound does not have to know which+ of the two it is looking at.+ """+ entry = handler._search_sources_by_thread.get(thread_id)+ if entry is None:+ return {}+ return dict(getattr(entry, "sources", entry))+++class IsolatedCodexHome:+ """Bind ``CODEX_HOME`` to an empty directory for the duration of one test."""++ def isolate_codex_home(self) -> Path:+ home = Path(tempfile.mkdtemp(prefix="avibe-codex-home-"))+ self.addCleanup(lambda: __import__("shutil").rmtree(home, ignore_errors=True))+ patcher = mock.patch.dict(os.environ, {"CODEX_HOME": str(home)})+ patcher.start()+ self.addCleanup(patcher.stop)+ return home++ def recorded_search(self, *results: dict, owner: str, turn: str = "turn-0") -> dict:+ """One ``item_completed`` row exactly as a rollout file records it.++ ``owner`` is the ``payload.thread_id`` the row carries, which is the+ *parent's* id for the rows a fork inherits - the reason scope is the+ file and not this field.+ """+ return {+ "timestamp": "2026-09-21T00:00:00.000Z",+ "ordinal": 1,+ "type": "event_msg",+ "payload": {+ "type": "item_completed",+ "thread_id": owner,+ "turn_id": turn,+ "item": {+ "type": "WebSearch",+ "id": f"ws_{turn}",+ "query": "native web search",+ "action": {"type": "search", "queries": ["native web search"]},+ "results": list(results),+ },+ "completed_at_ms": 1,+ },+ }++ def recorded_extension_search(+ self, *results: dict, owner: str, turn: str = "turn-0"+ ) -> dict:+ """The same row as the shipped binary writes it today.++ The native search tool lives in an extension namespace now, so a current+ rollout records its completed item as ``Extension`` carrying ``kind:+ "web.search"`` rather than naming the item ``WebSearch``. Both forms are+ in the local corpus (1023 old-form rows with results against 267 new-form+ ones), and every row codex-cli 0.154.0 writes is this one - captured from+ an isolated native turn against a loopback provider, not invented here.+ """+ row = self.recorded_search(*results, owner=owner, turn=turn)+ item = row["payload"]["item"]+ row["payload"]["item"] = {+ "type": "Extension",+ "kind": "web.search",+ "id": item["id"],+ "query": item["query"],+ "action": item["action"],+ "results": item["results"],+ }+ return row++ def record_history(+ self,+ home: Path,+ thread_id: str,+ rows: list,+ *,+ trailing: str = "",+ ) -> Path:+ """Write a rollout file for *thread_id* and index it the way Codex does."""+ sessions = home / "sessions" / "2026" / "09" / "21"+ sessions.mkdir(parents=True, exist_ok=True)+ path = sessions / f"rollout-2026-09-21T00-00-00-{thread_id}.jsonl"+ body = "".join(f"{json.dumps(row)}\n" for row in rows) + trailing+ path.write_text(body, encoding="utf-8")+ self.index_history(home, thread_id, str(path))+ return path++ def index_history(self, home: Path, thread_id: str, rollout_path: str) -> None:+ connection = sqlite3.connect(home / "state_5.sqlite")+ with connection:+ connection.execute(+ "CREATE TABLE IF NOT EXISTS threads (id TEXT PRIMARY KEY, rollout_path TEXT)"+ )+ connection.execute(+ "INSERT OR REPLACE INTO threads (id, rollout_path) VALUES (?, ?)",+ (thread_id, rollout_path),+ )+ connection.close()++ def web_result(self, ref_id: str, url: str, title: str = "T") -> dict:+ """One ``results[]`` entry, keeping the fields the native tool sends."""+ return {+ "type": "text_result",+ "thumbnail_url": "https://images.example.com/thumb.png",+ "domain": "example.com",+ "ref_id": ref_id,+ "snippet": "snippet",+ "title": title,+ "url": url,+ }+++def web_search(thread_id: str, *results: dict, turn_id: str = "turn-1") -> dict:+ """An ``item/completed`` notification for a finished web search."""+ return {+ "threadId": thread_id,+ "turnId": turn_id,+ "item": {"type": "webSearch", "query": "native web search", "results": list(results)},+ }+++def agent_message(thread_id: str, text: str, turn_id: str = "turn-1") -> dict:+ return {+ "threadId": thread_id,+ "turnId": turn_id,+ "item": {"type": "agentMessage", "text": text},+ }+++def turn_completed(thread_id: str, turn_id: str = "turn-1") -> dict:+ return {"threadId": thread_id, "turn": {"id": turn_id, "status": "completed"}}+++def _request(session_id: str = "session-1"):+ return SimpleNamespace(+ base_session_id=session_id,+ session_key=f"slack::channel::{session_id}",+ working_path="/repo",+ context=SimpleNamespace(platform_specific={}),+ started_at=0,+ )+++class CodexCitationCaptureTests(IsolatedCodexHome, unittest.IsolatedAsyncioTestCase):+ """``_record_search_sources``: what gets into the cache, and under which key."""++ def setUp(self):+ self.isolate_codex_home()+ self.handler = CodexEventHandler(_StubAgent())++ def sources(self, thread_id: str) -> dict:+ return cached_sources(self.handler, thread_id)++ def test_a_completed_search_is_stored_under_its_thread(self):+ params = web_search(+ "thread-a",+ {"ref_id": "turn0view0", "title": "Web search - OpenAI API", "url": GUIDE_URL},+ )++ self.handler._record_search_sources(params, params["item"])++ source = self.sources("thread-a")["turn0view0"]+ self.assertEqual((source.ref_id, source.title, source.url), ("turn0view0", "Web search - OpenAI API", GUIDE_URL))++ def test_the_camel_case_alias_is_accepted(self):+ """The app-server camel-cases most fields; ``ref_id`` is the observed one."""+ params = web_search("thread-a", {"refId": "turn0view0", "title": "T", "url": GUIDE_URL})++ self.handler._record_search_sources(params, params["item"])++ self.assertIn("turn0view0", self.sources("thread-a"))++ async def test_a_ref_id_means_nothing_outside_its_own_thread(self):+ """Two threads both start at ``turn0view0``; neither may see the other's."""+ for thread_id, url in (("thread-a", GUIDE_URL), ("thread-b", PROBE_URL)):+ params = web_search(thread_id, {"ref_id": "turn0view0", "title": "T", "url": url})+ self.handler._record_search_sources(params, params["item"])++ self.assertEqual(self.sources("thread-a")["turn0view0"].url, GUIDE_URL)+ self.assertEqual(self.sources("thread-b")["turn0view0"].url, PROBE_URL)++ registered, bundle = await self.handler._register_citations(+ f"Cited.{marker('turn0view0')}", {"threadId": "thread-b"}, _request()+ )+ text, citations = finalize_citations(registered, bundle)+ self.assertEqual(text, f"Cited. [example.com]({PROBE_URL})")+ self.assertEqual([c["url"] for c in citations], [PROBE_URL])++ async def test_a_thread_that_never_searched_resolves_nothing(self):+ params = web_search("thread-a", {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL})+ self.handler._record_search_sources(params, params["item"])++ registered, bundle = await self.handler._register_citations(+ f"Cited.{marker('turn0view0')}", {"threadId": "thread-zzz"}, _request()+ )+ text, citations = finalize_citations(registered, bundle)++ self.assertEqual(text, f"Cited. {UNRESOLVED}")+ self.assertEqual(citations, [])++ def test_a_later_search_result_supersedes_the_same_ref_id(self):+ for url in (GUIDE_URL, PROBE_URL):+ params = web_search("thread-a", {"ref_id": "turn0view0", "title": "T", "url": url})+ self.handler._record_search_sources(params, params["item"])++ self.assertEqual(self.sources("thread-a")["turn0view0"].url, PROBE_URL)++ def test_unusable_results_are_skipped_rather_than_stored_partially(self):+ params = web_search(+ "thread-a",+ {"ref_id": "turn0view0", "url": GUIDE_URL}, # keeps: no title is fine+ {"ref_id": "no-url"},+ {"ref_id": "blank-url", "url": ""},+ {"ref_id": "", "url": GUIDE_URL},+ {"ref_id": 7, "url": GUIDE_URL},+ {"ref_id": "bad-url-type", "url": ["https://example.com"]},+ "not-a-dict",+ None,+ )++ self.handler._record_search_sources(params, params["item"])++ self.assertEqual(list(self.sources("thread-a")), ["turn0view0"])+ self.assertEqual(self.sources("thread-a")["turn0view0"].title, "")++ def test_a_notification_without_a_thread_or_a_result_list_is_ignored(self):+ for params in (+ {"item": {"type": "webSearch", "results": [{"ref_id": "r", "url": GUIDE_URL}]}},+ web_search("thread-a"),+ {"threadId": "thread-a", "item": {"type": "webSearch"}},+ {"threadId": "thread-a", "item": {"type": "webSearch", "results": "nope"}},+ ):+ self.handler._record_search_sources(params, params["item"])++ self.assertEqual(self.handler._search_sources_by_thread, {})++ def test_sources_are_evicted_oldest_first_within_a_thread(self):+ limit = _MAX_CITATION_SOURCES_PER_THREAD+ params = web_search(+ "thread-a",+ *[+ {"ref_id": f"turn0view{i}", "title": "T", "url": f"https://example.com/{i}"}+ for i in range(limit + 5)+ ],+ )++ self.handler._record_search_sources(params, params["item"])++ stored = list(self.sources("thread-a"))+ self.assertEqual(len(stored), limit)+ self.assertNotIn("turn0view0", stored)+ self.assertIn(f"turn0view{limit + 4}", stored)++ def test_a_reused_source_is_kept_over_an_untouched_one(self):+ """LRU, so the ref a long conversation keeps citing survives eviction."""+ for i in range(3):+ params = web_search("thread-a", {"ref_id": f"r{i}", "title": "T", "url": f"https://e.com/{i}"})+ self.handler._record_search_sources(params, params["item"])+ refreshed = web_search("thread-a", {"ref_id": "r0", "title": "T", "url": "https://e.com/0"})+ self.handler._record_search_sources(refreshed, refreshed["item"])++ self.assertEqual(list(self.sources("thread-a")), ["r1", "r2", "r0"])++ def test_threads_are_evicted_oldest_first(self):+ limit = _MAX_CITATION_THREADS+ for i in range(limit + 2):+ params = web_search(f"thread-{i}", {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL})+ self.handler._record_search_sources(params, params["item"])++ threads = list(self.handler._search_sources_by_thread)+ self.assertEqual(len(threads), limit)+ self.assertNotIn("thread-0", threads)+ self.assertIn(f"thread-{limit + 1}", threads)++ def test_an_active_thread_outlives_an_idle_one(self):+ for i in range(3):+ params = web_search(f"thread-{i}", {"ref_id": "r", "title": "T", "url": GUIDE_URL})+ self.handler._record_search_sources(params, params["item"])+ again = web_search("thread-0", {"ref_id": "r2", "title": "T", "url": PROBE_URL})+ self.handler._record_search_sources(again, again["item"])++ self.assertEqual(list(self.handler._search_sources_by_thread), ["thread-1", "thread-2", "thread-0"])+++class CodexCitationResolutionTests(IsolatedCodexHome, unittest.IsolatedAsyncioTestCase):+ """The product path: notifications in, delivered message plus sidecar out."""++ def setUp(self):+ self.isolate_codex_home()+ self.agent = _StubAgent()+ self.handler = CodexEventHandler(self.agent)+ self.request = _request()+ self.agent._turn_registry.register_turn("turn-1", self.request)++ def result_call(self):+ self.agent.emit_result_message.assert_awaited()+ return self.agent.emit_result_message.await_args++ async def test_a_search_that_lands_after_the_answer_still_resolves_it(self):+ """Resolution is deferred to emit time precisely for this ordering."""+ await self.handler._on_item_completed(+ agent_message("thread-a", f"Native search exists.{marker('turn0view0')}"), self.request+ )+ await self.handler._on_item_completed(+ web_search("thread-a", {"ref_id": "turn0view0", "title": "Web search - OpenAI API", "url": GUIDE_URL}),+ self.request,+ )+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ call = self.result_call()+ body = f"Native search exists. [developers.openai.com]({GUIDE_URL})"+ self.assertEqual(written(call), body)+ self.assertEqual(+ sidecar(call),+ [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Web search - OpenAI API",+ "url": GUIDE_URL,+ "label": "developers.openai.com",+ # Where this citation's link sits in *this* body, and the+ # digest that binds the two: a reader holding any other+ # text has no numbered badge to paint.+ # See test_citations.TestBodyBinding.+ "spans": [[body.index("[developers"), len(body)]],+ "body_sha256": body_digest(body),+ }+ ],+ )++ async def test_a_search_from_an_earlier_turn_is_still_citable(self):+ """Thread-scoped, not turn-scoped: the cache outlives the turn that filled it."""+ await self.handler._on_item_completed(+ web_search("thread-a", {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL}),+ self.request,+ )+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)+ self.agent.emit_result_message.reset_mock()++ later = _request()+ self.agent._turn_registry.register_turn("turn-2", later)+ await self.handler._on_item_completed(+ agent_message("thread-a", f"As established.{marker('turn0view0')}", turn_id="turn-2"), later+ )+ await self.handler._on_turn_completed(turn_completed("thread-a", turn_id="turn-2"), later)++ self.assertEqual(written(self.result_call()), f"As established. [developers.openai.com]({GUIDE_URL})")++ async def test_an_answer_without_markers_carries_no_sidecar(self):+ await self.handler._on_item_completed(agent_message("thread-a", "Plain answer."), self.request)+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ call = self.result_call()+ self.assertEqual(written(call), "Plain answer.")+ self.assertIsNone(call.kwargs["citations"])++ async def test_an_unresolvable_ref_is_labelled_and_carries_no_sidecar(self):+ await self.handler._on_item_completed(+ agent_message("thread-a", f"Claimed.{marker('turn9view9')}"), self.request+ )+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ call = self.result_call()+ self.assertEqual(written(call), f"Claimed. {UNRESOLVED}")+ self.assertEqual(sidecar(call), [])++ async def test_a_repeated_completion_does_not_duplicate_the_links(self):+ await self.handler._on_item_completed(+ web_search("thread-a", {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL}),+ self.request,+ )+ await self.handler._on_item_completed(+ agent_message("thread-a", f"Cited.{marker('turn0view0')}"), self.request+ )+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ self.assertEqual(self.agent.emit_result_message.await_count, 1)+ self.assertEqual(written(self.result_call()).count(f"[developers.openai.com]({GUIDE_URL})"), 1)++ async def test_an_intermediate_message_is_flushed_with_its_own_sidecar(self):+ await self.handler._on_item_completed(+ web_search("thread-a", {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL}),+ self.request,+ )+ await self.handler._on_item_completed(+ agent_message("thread-a", f"Progress.{marker('turn0view0')}"), self.request+ )+ await self.handler._on_item_completed(agent_message("thread-a", "Final answer."), self.request)++ self.agent.controller.emit_agent_message.assert_awaited_once()+ call = self.agent.controller.emit_agent_message.await_args+ self.assertEqual(written(call, 2), f"Progress. [developers.openai.com]({GUIDE_URL})")+ self.assertEqual([c["url"] for c in sidecar(call, 2)], [GUIDE_URL])++ async def test_an_intermediate_message_without_citations_keeps_its_original_call(self):+ """No empty sidecar kwarg: an uncited flush is the call it always was."""+ await self.handler._on_item_completed(agent_message("thread-a", "Progress."), self.request)+ await self.handler._on_item_completed(agent_message("thread-a", "Final answer."), self.request)++ self.agent.controller.emit_agent_message.assert_awaited_once_with(+ self.request.context, "assistant", "Progress.", parse_mode="markdown"+ )++ async def test_a_generated_image_appended_after_resolution_keeps_both(self):+ """Citations are resolved before the image footer is appended, not instead of it."""+ await self.handler._on_item_completed(+ web_search("thread-a", {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL}),+ self.request,+ )+ await self.handler._on_item_completed(+ agent_message("thread-a", f"Cited.{marker('turn0view0')}"), self.request+ )+ self.handler._append_generated_images = lambda text, params, request: f"{text}\n\n"++ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ call = self.result_call()+ self.assertEqual(written(call), f"Cited. [developers.openai.com]({GUIDE_URL})\n\n")+ self.assertEqual(len(sidecar(call)), 1)++ async def test_an_untrusted_source_cannot_inject_a_link(self):+ """Titles and URLs come from the open web; the sidecar must stay clean."""+ await self.handler._on_item_completed(+ web_search(+ "thread-a",+ {"ref_id": "turn0view0", "title": "Badactor\n\n", "url": "javascript:alert(1)"},+ {"ref_id": "turn0view1", "title": "中文標題", "url": PROBE_URL},+ ),+ self.request,+ )+ await self.handler._on_item_completed(+ agent_message("thread-a", f"Two.{marker('turn0view0', 'turn0view1')}"), self.request+ )+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ call = self.result_call()+ body = f"Two. [example.com]({PROBE_URL}) {UNRESOLVED}"+ self.assertEqual(written(call), body)+ link = f"[example.com]({PROBE_URL})"+ self.assertEqual(+ sidecar(call),+ [+ {+ "index": 1,+ "ref_id": "turn0view1",+ "title": "中文標題",+ "url": PROBE_URL,+ "label": "example.com",+ # The rejected source wrote no link at all, so the surviving+ # citation spans only its own link — the fallback text that+ # follows belongs to no source and is never covered.+ "spans": [[body.index(link), body.index(link) + len(link)]],+ "body_sha256": body_digest(body),+ }+ ],+ )++ async def test_a_hidden_block_neither_numbers_nor_leaks_its_source(self):+ """``<silent>`` content is removed downstream, so it may not be attributed.++ Numbering a hidden marker would open the reader's sidecar at index 2 and+ describe a badge whose link was stripped with the block.+ """+ await self.handler._on_item_completed(+ web_search(+ "thread-a",+ {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL},+ {"ref_id": "turn0view1", "title": "T", "url": PROBE_URL},+ ),+ self.request,+ )+ await self.handler._on_item_completed(+ agent_message(+ "thread-a",+ f"<silent>internal{marker('turn0view0')}</silent>"+ f"Visible.{marker('turn0view1')}",+ ),+ self.request,+ )+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ call = self.result_call()+ self.assertEqual(+ [(c["index"], c["url"]) for c in sidecar(call)], [(1, PROBE_URL)]+ )+ self.assertEqual(+ materialize_citations(+ strip_silent_blocks(call.args[1]), call.kwargs["citations"]+ ),+ f"Visible. [example.com]({PROBE_URL})",+ )++ async def test_the_unresolved_label_is_localized(self):+ self.agent.controller._t = lambda key: "(来源不可用)" if key == "message.citationUnresolved" else key++ await self.handler._on_item_completed(+ agent_message("thread-a", f"Claimed.{marker('turn9view9')}"), self.request+ )+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ self.assertEqual(written(self.result_call()), "Claimed. (来源不可用)")++ async def test_an_empty_answer_still_completes_the_turn(self):+ self.agent.emit_result_message = AsyncMock()++ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ call = self.result_call()+ self.assertIsNone(written(call))+ self.assertIsNone(call.kwargs["citations"])+++class CodexCitationReadinessTests(IsolatedCodexHome, unittest.IsolatedAsyncioTestCase):+ """When a narration message that cites something may be delivered.++ A search and the answer that cites it are two notifications, and nothing in+ the transport promises the search arrives first. An intermediate message is+ therefore held while a ref it names has no source yet - and only then. The+ hold ends at the first terminal boundary the turn reaches, so it is bounded+ by the turn rather than by a guess about ordering; it never reorders the+ narration around it, and a message that cites nothing is never delayed by+ anything except messages already ahead of it.+ """++ def setUp(self):+ self.isolate_codex_home()+ self.agent = _StubAgent()+ self.handler = CodexEventHandler(self.agent)+ self.request = _request()+ self.agent._turn_registry.register_turn("turn-1", self.request)+ self.emitted = self.agent.controller.emit_agent_message+ # One list for both surfaces: several cases below are about the order a+ # reader sees, which per-mock await lists cannot express.+ self.delivered: list[str] = []+ self.emitted.side_effect = lambda *a, **k: self.delivered.append(+ materialize_citations(a[2], k.get("citations"))+ )+ self.agent.emit_result_message.side_effect = lambda *a, **k: self.delivered.append(+ materialize_citations(a[1], k.get("citations"))+ )++ async def item(self, params: dict) -> None:+ await self.handler._on_item_completed(params, self.request)++ def guide_search(self, ref_id: str = "turn0view0") -> dict:+ return web_search("thread-a", {"ref_id": ref_id, "title": "T", "url": GUIDE_URL})++ async def test_a_cited_narration_waits_for_the_search_that_defines_it(self):+ """Review C's sequence: the citing message is flushed before the search."""+ await self.item(agent_message("thread-a", f"Progress.{marker('turn0view0')}"))+ await self.item(agent_message("thread-a", "Final answer."))++ self.assertEqual(self.delivered, [])++ await self.item(self.guide_search())++ self.assertEqual(self.delivered, [f"Progress. [developers.openai.com]({GUIDE_URL})"])+ self.assertEqual(+ [c["url"] for c in sidecar(self.emitted.await_args, 2)], [GUIDE_URL]+ )++ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ self.assertEqual(self.delivered[-1], "Final answer.")++ async def test_a_held_narration_is_delivered_when_the_turn_completes(self):+ """The wait is bounded by the turn: a ref that never arrives still ships."""+ await self.item(agent_message("thread-a", f"Progress.{marker('turn9view9')}"))+ await self.item(agent_message("thread-a", "Final answer."))+ self.assertEqual(self.delivered, [])++ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ self.assertEqual(self.delivered, [f"Progress. {UNRESOLVED}", "Final answer."])++ async def test_an_interrupted_turn_still_delivers_what_it_narrated(self):+ await self.item(agent_message("thread-a", f"Progress.{marker('turn9view9')}"))+ await self.item(agent_message("thread-a", "Final answer."))++ await self.handler._on_turn_completed(+ {"threadId": "thread-a", "turn": {"id": "turn-1", "status": "interrupted"}},+ self.request,+ )++ self.assertEqual(self.delivered, [f"Progress. {UNRESOLVED}"])++ async def test_a_failed_turn_delivers_its_narration_before_the_error(self):+ await self.item(agent_message("thread-a", f"Progress.{marker('turn9view9')}"))+ await self.item(agent_message("thread-a", "Final answer."))++ await self.handler._on_turn_completed(+ {+ "threadId": "thread-a",+ "turn": {"id": "turn-1", "status": "failed", "error": {"message": "boom"}},+ },+ self.request,+ )++ self.assertEqual(self.delivered[0], f"Progress. {UNRESOLVED}")++ async def test_a_terminal_error_delivers_the_held_narration(self):+ await self.item(agent_message("thread-a", f"Progress.{marker('turn9view9')}"))+ await self.item(agent_message("thread-a", "Final answer."))++ await self.handler._on_error({"turnId": "turn-1", "error": {"message": "boom"}}, self.request)++ self.assertEqual(self.delivered[0], f"Progress. {UNRESOLVED}")++ async def test_a_retried_error_is_not_a_boundary(self):+ """``willRetry`` means the turn is still running, so nothing settles."""+ await self.item(agent_message("thread-a", f"Progress.{marker('turn0view0')}"))+ await self.item(agent_message("thread-a", "Final answer."))++ await self.handler._on_error(+ {"turnId": "turn-1", "willRetry": True, "error": {"message": "transient"}},+ self.request,+ )+ self.assertEqual(self.delivered, [])++ await self.item(self.guide_search())++ self.assertEqual(self.delivered, [f"Progress. [developers.openai.com]({GUIDE_URL})"])++ async def test_a_superseded_turn_settles_without_speaking(self):+ """A hidden turn's queue is discarded, exactly as its result candidate is."""+ await self.item(agent_message("thread-a", f"Progress.{marker('turn9view9')}"))+ await self.item(agent_message("thread-a", "Final answer."))++ self.handler.clear_pending("turn-1")+ await self.handler._on_turn_completed(turn_completed("thread-a"), self.request)++ self.assertEqual(self.delivered, [])++ async def test_an_uncited_narration_is_never_delayed(self):+ await self.item(agent_message("thread-a", "Progress."))+ await self.item(agent_message("thread-a", "Final answer."))++ self.assertEqual(self.delivered, ["Progress."])+ self.emitted.assert_awaited_once_with(+ self.request.context, "assistant", "Progress.", parse_mode="markdown"+ )++ async def test_a_resolvable_citation_is_not_delayed_either(self):+ await self.item(self.guide_search())+ await self.item(agent_message("thread-a", f"Progress.{marker('turn0view0')}"))+ await self.item(agent_message("thread-a", "Final answer."))++ self.assertEqual(self.delivered, [f"Progress. [developers.openai.com]({GUIDE_URL})"])++ async def test_nothing_overtakes_a_held_narration(self):+ """Holding one message must not let the messages behind it change places."""+ self.agent._get_formatter = lambda context: SimpleNamespace(+ format_toolcall=lambda name, payload: f"[{name}]"+ )+ await self.item(agent_message("thread-a", f"First.{marker('turn0view0')}"))+ await self.item(agent_message("thread-a", "Second."))+ await self.item(+ {+ "threadId": "thread-a",+ "turnId": "turn-1",+ "item": {+ "type": "commandExecution",+ "command": "ls",+ "status": "completed",+ "exitCode": 0,+ "aggregatedOutput": "",+ },+ }+ )+ await self.item(agent_message("thread-a", "Third."))++ self.assertEqual(self.delivered, [])++ await self.item(self.guide_search())++ self.assertEqual(+ self.delivered,+ [f"First. [developers.openai.com]({GUIDE_URL})", "[bash]", "Second."],+ )++ async def test_a_second_held_message_resolves_against_its_own_ref(self):+ """The queue drains as far as it can, and stops at the first unready entry."""+ await self.item(agent_message("thread-a", f"First.{marker('turn0view0')}"))+ await self.item(agent_message("thread-a", f"Second.{marker('turn0view1')}"))+ await self.item(agent_message("thread-a", "Third."))++ await self.item(+ web_search("thread-a", {"ref_id": "turn0view1", "title": "T", "url": PROBE_URL})+ )+ self.assertEqual(self.delivered, [])++ await self.item(self.guide_search())++ self.assertEqual(+ self.delivered,+ [+ f"First. [developers.openai.com]({GUIDE_URL})",+ f"Second. [example.com]({PROBE_URL})",+ ],+ )+++class CodexCitationHistoryTests(IsolatedCodexHome, unittest.IsolatedAsyncioTestCase):+ """Recovering sources a live stream never delivered.++ A conversation loses the ``item/completed`` notifications behind its earlier+ answers whenever the process restarts, the thread is resumed with+ ``excludeTurns: True``, it is re-read with ``includeTurns: False``, or it is+ forked - ``thread/fork`` returns an id and nothing more. Each case below runs+ the real consumption path (item in, delivered message and sidecar out)+ against a handler that never saw those notifications.+ """++ def setUp(self):+ self.home = self.isolate_codex_home()+ self.agent = _StubAgent()+ self.handler = CodexEventHandler(self.agent)+ self.request = _request()+ self.agent._turn_registry.register_turn("turn-1", self.request)++ async def answer(self, thread_id: str, text: str, *, turn_id: str = "turn-1", request=None):+ """Deliver one answer through the ordinary item/completed + turn path."""+ request = request or self.request+ await self.handler._on_item_completed(+ agent_message(thread_id, text, turn_id=turn_id), request+ )+ await self.handler._on_turn_completed(+ turn_completed(thread_id, turn_id=turn_id), request+ )+ self.agent.emit_result_message.assert_awaited()+ return self.agent.emit_result_message.await_args++ async def next_answer(self, thread_id: str, text: str):+ """A later Avibe turn in the same conversation, with its own request."""+ self._turns = getattr(self, "_turns", 1) + 1+ turn_id = f"turn-{self._turns}"+ request = _request(f"session-{self._turns}")+ self.agent._turn_registry.register_turn(turn_id, request)+ return await self.answer(thread_id, text, turn_id=turn_id, request=request)++ async def test_a_fresh_handler_cites_a_search_it_never_saw(self):+ """The restart case: the notification is gone, the recorded result is not."""+ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )++ call = await self.answer("thread-a", f"As established.{marker('turn0view0')}")++ self.assertEqual(written(call), f"As established. [developers.openai.com]({GUIDE_URL})")+ self.assertEqual([c["url"] for c in sidecar(call)], [GUIDE_URL])++ async def test_the_history_shape_the_shipped_binary_writes_is_read(self):+ """The current native rollout names the item ``Extension``/``web.search``.++ Reproduced against the real 0.154.0 app-server: with standalone search+ active, the recorded ``item_completed`` row carries the extension name+ and kind, not ``WebSearch``. A reader that only knows the older name+ recovers nothing from any history this binary writes.+ """+ self.record_history(+ self.home,+ "thread-a",+ [+ self.recorded_extension_search(+ self.web_result("turn0search0", PROBE_URL), owner="thread-a"+ )+ ],+ )++ call = await self.answer("thread-a", f"Probed.{marker('turn0search0')}")++ self.assertEqual(written(call), f"Probed. [example.com]({PROBE_URL})")+ self.assertEqual([c["url"] for c in sidecar(call)], [PROBE_URL])++ async def test_both_recorded_shapes_are_read_from_one_history(self):+ """A conversation spanning the rename keeps every source it recorded."""+ self.record_history(+ self.home,+ "thread-a",+ [+ self.recorded_search(+ self.web_result("turn0view0", GUIDE_URL), owner="thread-a"+ ),+ self.recorded_extension_search(+ self.web_result("turn1search0", PROBE_URL),+ owner="thread-a",+ turn="turn-1",+ ),+ ],+ )++ call = await self.answer(+ "thread-a",+ f"Old.{marker('turn0view0')} New.{marker('turn1search0')}",+ )++ self.assertEqual(+ [c["url"] for c in sidecar(call)], [GUIDE_URL, PROBE_URL]+ )++ async def test_a_fork_cites_the_parent_history_it_carries(self):+ """A fork's file opens with the parent's rows, still under the parent's id."""+ self.record_history(+ self.home,+ "thread-fork",+ [+ self.recorded_search(+ self.web_result("turn0view0", GUIDE_URL),+ owner="thread-parent",+ turn="turn-parent",+ ),+ self.recorded_search(+ self.web_result("turn1view0", PROBE_URL),+ owner="thread-fork",+ turn="turn-own",+ ),+ ],+ )++ call = await self.answer(+ "thread-fork", f"Both.{marker('turn0view0', 'turn1view0')}"+ )++ self.assertEqual(+ written(call),+ f"Both. [developers.openai.com]({GUIDE_URL}) [example.com]({PROBE_URL})",+ )++ async def test_the_same_ref_id_in_another_thread_keeps_its_own_source(self):+ """Ref tokens repeat across threads, so each thread reads only its own file."""+ for thread_id, url in (("thread-a", GUIDE_URL), ("thread-b", PROBE_URL)):+ self.record_history(+ self.home,+ thread_id,+ [self.recorded_search(self.web_result("turn0view0", url), owner=thread_id)],+ )++ first = await self.answer("thread-a", f"A.{marker('turn0view0')}")+ self.assertEqual([c["url"] for c in sidecar(first)], [GUIDE_URL])++ second_request = _request("session-2")+ self.agent._turn_registry.register_turn("turn-2", second_request)+ second = await self.answer(+ "thread-b", f"B.{marker('turn0view0')}", turn_id="turn-2", request=second_request+ )+ self.assertEqual([c["url"] for c in sidecar(second)], [PROBE_URL])++ async def test_history_is_read_once_per_thread_not_once_per_message(self):+ """Cross-turn: the read is a memo, so a long conversation pays for it once."""+ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )++ with mock.patch.object(+ codex_event_handler,+ "read_thread_search_sources",+ wraps=search_history.read_thread_search_sources,+ ) as reader:+ await self.answer("thread-a", f"First.{marker('turn0view0')}")+ later = _request("session-2")+ self.agent._turn_registry.register_turn("turn-2", later)+ call = await self.answer(+ "thread-a", f"Second.{marker('turn0view0')}", turn_id="turn-2", request=later+ )++ self.assertEqual(reader.call_count, 1)+ self.assertEqual(written(call), f"Second. [developers.openai.com]({GUIDE_URL})")++ async def test_an_evicted_thread_is_read_back_rather_than_lost(self):+ """The cache is a memo over history, so eviction costs a read, not the link."""+ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )+ params = web_search("thread-a", {"ref_id": "turn0view0", "title": "T", "url": GUIDE_URL})+ self.handler._record_search_sources(params, params["item"])+ for i in range(_MAX_CITATION_THREADS + 1):+ noise = web_search(f"other-{i}", {"ref_id": "r", "title": "T", "url": PROBE_URL})+ self.handler._record_search_sources(noise, noise["item"])+ self.assertNotIn("thread-a", self.handler._search_sources_by_thread)++ call = await self.answer("thread-a", f"Still cited.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Still cited. [developers.openai.com]({GUIDE_URL})")++ async def test_a_partially_written_last_row_does_not_lose_the_rest(self):+ """Codex may be mid-append; an unparseable tail is not evidence about the head."""+ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ trailing='{"type": "event_msg", "payload": {"item": {"type": "WebSea',+ )++ call = await self.answer("thread-a", f"Cited.{marker('turn0view0')}")++ self.assertEqual([c["url"] for c in sidecar(call)], [GUIDE_URL])++ async def test_a_thread_with_no_recorded_history_degrades_to_the_label(self):+ """Nothing indexed, nothing invented."""+ call = await self.answer("thread-a", f"Claimed.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Claimed. {UNRESOLVED}")+ self.assertEqual(sidecar(call), [])++ async def test_a_recorded_result_without_a_url_is_not_given_one(self):+ """A ref_id is a name, never a source: an entry with no URL stays unresolved."""+ self.record_history(+ self.home,+ "thread-a",+ [+ self.recorded_search(+ {"type": "web", "ref_id": "turn0view0", "title": "No link", "snippet": "s"},+ owner="thread-a",+ )+ ],+ )++ call = await self.answer("thread-a", f"Claimed.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Claimed. {UNRESOLVED}")+ self.assertEqual(sidecar(call), [])++ @unittest.skipIf(+ hasattr(os, "geteuid") and os.geteuid() == 0,+ "root can read a mode-000 file, so the failure branch cannot be reached",+ )+ async def test_an_unreadable_history_file_is_reported_and_degrades(self):+ """A read that fails leaves real evidence in the log and invents nothing."""+ path = self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )+ path.chmod(0o000)+ self.addCleanup(path.chmod, 0o600)++ with self.assertLogs(search_history.logger, level="WARNING") as logs:+ call = await self.answer("thread-a", f"Claimed.{marker('turn0view0')}")++ self.assertTrue(any(str(path) in line for line in logs.output), logs.output)+ self.assertEqual(written(call), f"Claimed. {UNRESOLVED}")+ self.assertEqual(sidecar(call), [])++ async def test_a_live_search_does_not_hide_the_recorded_ones(self):+ """A cache entry says what this process saw, not what the thread contains.++ One live search creates the thread's entry. The refs an earlier answer+ cited are not in it, and they are the ones a resume or a restart lost -+ so a live result must not be read as "this thread is fully known".+ """+ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )+ live = web_search("thread-a", self.web_result("turn1view0", PROBE_URL))+ self.handler._record_search_sources(live, live["item"])++ call = await self.answer("thread-a", f"Both.{marker('turn0view0', 'turn1view0')}")++ self.assertEqual(+ written(call),+ f"Both. [developers.openai.com]({GUIDE_URL}) [example.com]({PROBE_URL})",+ )++ async def test_the_source_bound_never_discards_the_ref_being_resolved(self):+ """Bounding the cache may cost a re-read; it may never cost the link."""+ results = [+ self.web_result(f"turn0view{i}", f"https://example.com/{i}")+ for i in range(_MAX_CITATION_SOURCES_PER_THREAD + 1)+ ]+ self.record_history(+ self.home, "thread-a", [self.recorded_search(*results, owner="thread-a")]+ )+ live = web_search("thread-a", *results)+ self.handler._record_search_sources(live, live["item"])+ self.assertNotIn("turn0view0", cached_sources(self.handler, "thread-a"))++ call = await self.answer("thread-a", f"The oldest one.{marker('turn0view0')}")++ self.assertEqual(written(call), "The oldest one. [example.com](https://example.com/0)")++ async def test_an_empty_history_that_later_records_a_search_is_re_read(self):+ """A history is a growing file, so "not there yet" is not "not there"."""+ path = self.record_history(self.home, "thread-a", [])++ first = await self.answer("thread-a", f"First.{marker('turn0view0')}")+ self.assertEqual(written(first), f"First. {UNRESOLVED}")++ with path.open("a", encoding="utf-8") as handle:+ row = self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")+ handle.write(f"{json.dumps(row)}\n")++ call = await self.next_answer("thread-a", f"Second.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Second. [developers.openai.com]({GUIDE_URL})")++ async def test_a_history_that_appears_later_is_found(self):+ """Codex indexes the thread when it writes it; a lookup miss is not final."""+ first = await self.answer("thread-a", f"First.{marker('turn0view0')}")+ self.assertEqual(written(first), f"First. {UNRESOLVED}")++ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )++ call = await self.next_answer("thread-a", f"Second.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Second. [developers.openai.com]({GUIDE_URL})")++ @unittest.skipIf(+ hasattr(os, "geteuid") and os.geteuid() == 0,+ "root can read a mode-000 file, so the failure branch cannot be reached",+ )+ async def test_an_unreadable_history_is_re_read_once_it_can_be_read(self):+ """A read that failed learned nothing, so it may not be remembered as absence.++ Restoring the mode changes neither size nor mtime, so nothing about the+ file itself says to look again: it is the incomplete read that must not+ settle into negative truth.+ """+ path = self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )+ path.chmod(0o000)+ self.addCleanup(path.chmod, 0o600)++ with self.assertLogs(search_history.logger, level="WARNING"):+ first = await self.answer("thread-a", f"First.{marker('turn0view0')}")+ self.assertEqual(written(first), f"First. {UNRESOLVED}")++ path.chmod(0o600)++ call = await self.next_answer("thread-a", f"Second.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Second. [developers.openai.com]({GUIDE_URL})")++ async def test_a_missing_ref_does_not_rescan_an_unchanged_history(self):+ """The cost of a ref that is genuinely absent is paid once per history."""+ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )+ scans: list[bool] = []+ real = search_history.read_thread_search_sources++ def spy(*args, **kwargs):+ read = real(*args, **kwargs)+ scans.append(read.scanned)+ return read++ with mock.patch.object(codex_event_handler, "read_thread_search_sources", spy):+ await self.answer("thread-a", f"First.{marker('turn9view9')}")+ call = await self.next_answer("thread-a", f"Second.{marker('turn9view9')}")++ self.assertEqual(scans, [True, False])+ self.assertEqual(written(call), f"Second. {UNRESOLVED}")++ async def test_an_absence_does_not_outlive_the_history_that_proved_it(self):+ """A scan directed at one ref may not vindicate another ref's absence.++ The second answer asks only about ``turn9view9``, so its scan of the+ grown file learns nothing about ``turn0view0`` - it never looked. If both+ absences lean on one "history I have already read" mark, that scan moves+ the mark forward and the third answer takes the grown file for the one+ that proved ``turn0view0`` missing, leaving the search now sitting in it+ unread. Only the history an absence was actually proved against may+ suppress a re-read.+ """+ path = self.record_history(self.home, "thread-a", [])++ first = await self.answer("thread-a", f"First.{marker('turn0view0')}")+ self.assertEqual(written(first), f"First. {UNRESOLVED}")++ with path.open("a", encoding="utf-8") as handle:+ row = self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")+ handle.write(f"{json.dumps(row)}\n")++ second = await self.next_answer("thread-a", f"Second.{marker('turn9view9')}")+ self.assertEqual(written(second), f"Second. {UNRESOLVED}")++ call = await self.next_answer("thread-a", f"Third.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Third. [developers.openai.com]({GUIDE_URL})")++ async def test_the_latest_recorded_definition_of_a_ref_wins(self):+ """A rollout file may redefine a ref; the newest row is the trustworthy one."""+ self.record_history(+ self.home,+ "thread-a",+ [+ self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a"),+ self.recorded_search(+ self.web_result("turn0view0", PROBE_URL), owner="thread-a", turn="turn-1"+ ),+ ],+ )++ call = await self.answer("thread-a", f"Cited.{marker('turn0view0')}")++ self.assertEqual(written(call), f"Cited. [example.com]({PROBE_URL})")++ def test_a_read_carries_only_the_refs_it_was_asked_for(self):+ """The read is directed by the message, which is what bounds it."""+ self.record_history(+ self.home,+ "thread-a",+ [+ self.recorded_search(+ *[+ self.web_result(f"turn0view{i}", f"https://example.com/{i}")+ for i in range(20)+ ],+ owner="thread-a",+ )+ ],+ )++ read = search_history.read_thread_search_sources("thread-a", wanted={"turn0view3"})++ self.assertEqual(+ {ref: source.url for ref, source in read.sources.items()},+ {"turn0view3": "https://example.com/3"},+ )+ self.assertTrue(read.scanned)+ self.assertTrue(read.complete)++ def test_an_unchanged_history_is_not_scanned_again(self):+ self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )++ first = search_history.read_thread_search_sources("thread-a", wanted={"nope"})+ self.assertTrue(first.scanned)+ self.assertEqual(first.sources, {})++ again = search_history.read_thread_search_sources(+ "thread-a", wanted={"nope"}, unchanged=first.fingerprint+ )++ self.assertFalse(again.scanned)+ self.assertEqual(again.fingerprint, first.fingerprint)++ def test_a_changed_history_is_scanned_again(self):+ path = self.record_history(+ self.home,+ "thread-a",+ [self.recorded_search(self.web_result("turn0view0", GUIDE_URL), owner="thread-a")],+ )+ first = search_history.read_thread_search_sources("thread-a", wanted={"turn1view0"})++ with path.open("a", encoding="utf-8") as handle:+ row = self.recorded_search(self.web_result("turn1view0", PROBE_URL), owner="thread-a")+ handle.write(f"{json.dumps(row)}\n")++ again = search_history.read_thread_search_sources(+ "thread-a", wanted={"turn1view0"}, unchanged=first.fingerprint+ )++ self.assertTrue(again.scanned)+ self.assertEqual(again.sources["turn1view0"].url, PROBE_URL)++ def test_an_unsafe_thread_id_never_reaches_the_filesystem(self):+ """The id is interpolated into a lookup and a path, so it is validated first."""+ for thread_id in ("../../etc/passwd", "a/b", "thread a", "", "thread;drop"):+ read = search_history.read_thread_search_sources(thread_id, wanted={"turn0view0"})+ self.assertEqual(read.sources, {}, thread_id)+ self.assertFalse(read.scanned, thread_id)++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_codex_event_handler.py b/tests/test_codex_event_handler.pyindex 8fba7ad8c..90e993644 100644--- a/tests/test_codex_event_handler.py+++ b/tests/test_codex_event_handler.py@@ -24,6 +24,7 @@ class _TurnState:self.turn_id = turn_idself.request = requestself.pending_assistant = None+ self.pending_narration = []self.terminal_error = Noneself.terminal_error_notified = Falseself.visible_to_user = True@@ -67,6 +68,7 @@ class _StubTurnRegistry:return Nonestate.visible_to_user = Falsestate.pending_assistant = None+ state.pending_narration = []state.terminal_error = Nonestate.terminal_error_notified = Falseif self._active_turns.get(state.request.base_session_id) == turn_id:diff --git a/tests/test_codex_turn_state.py b/tests/test_codex_turn_state.pyindex db2eee1d6..2c7e50c83 100644--- a/tests/test_codex_turn_state.py+++ b/tests/test_codex_turn_state.py@@ -80,3 +80,36 @@ def test_indicator_cleanup_has_exactly_one_owner():assert registry.claim_indicator_cleanup("turn-1") is requestassert registry.claim_indicator_cleanup("turn-1") is None+++def test_hiding_a_turn_discards_everything_it_had_not_said_yet():+ """A superseded turn speaks no further, so both held buffers go with it.++ ``pending_narration`` holds intermediate messages waiting for the search+ that attributes them. It is the same kind of undelivered candidate as+ ``pending_assistant``, so hiding the turn has to settle both or a hidden+ turn would still narrate once its sources arrived.+ """+ registry = CodexTurnRegistry()+ request = SimpleNamespace(base_session_id="session-1")+ state = registry.register_turn("turn-1", request)+ state.pending_assistant = ("Final answer.", "markdown")+ state.pending_narration.append(("Progress.", "markdown"))++ hidden = registry.hide_turn("turn-1")++ assert hidden is state+ assert state.pending_assistant is None+ assert state.pending_narration == []+++def test_each_turn_gets_its_own_narration_queue():+ registry = CodexTurnRegistry()+ first = SimpleNamespace(base_session_id="session-1")+ second = SimpleNamespace(base_session_id="session-2")++ one = registry.register_turn("turn-1", first)+ two = registry.register_turn("turn-2", second)+ one.pending_narration.append(("Only mine.", "markdown"))++ assert two.pending_narration == []diff --git a/tests/test_im_citation_delivery.py b/tests/test_im_citation_delivery.pynew file mode 100644index 000000000..76c5e5cfb--- /dev/null+++ b/tests/test_im_citation_delivery.py@@ -0,0 +1,276 @@+"""IM delivery of cited answers.++The Web transcript upgrades a citation into a badge from the structured sidecar.+Every IM platform instead gets exactly what it always got: ordinary Markdown+links in the message body, rendered to that platform's own clickable link+syntax by the formatter it already had. This file pins both halves of that -+the sidecar reaches persistence and never the wire, and the links survive each+platform's renderer with their source attribution intact.+"""++from __future__ import annotations++import sys+import unittest+from pathlib import Path+from unittest import mock++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from core.citations import (+ CitationSource,+ finalize_citations,+ materialize_citations,+ register_citations,+ resolve_citations,+)+from core.message_dispatcher import ConsolidatedMessageDispatcher+from modules.im import MessageContext+from modules.im.formatters.avibe_formatter import AvibeFormatter+from modules.im.formatters.discord_formatter import DiscordFormatter+from modules.im.formatters.feishu_formatter import FeishuFormatter+from modules.im.formatters.telegram_formatter import TelegramFormatter+from modules.im.formatters.wechat_formatter import WeChatFormatter+from modules.im.wechat import WeChatBot, WeChatConfig+from tests.test_message_dispatcher_platform_limits import _StubController++URL = "https://developers.openai.com/api/docs/guides/tools-web-search"+LINK = f"[developers.openai.com]({URL})"+ANSWER = "Native web search is documented."+CITED = f"{ANSWER} {LINK}"+MARKER = "\ue200cite\ue202turn0view0\ue201"+SOURCE = CitationSource(ref_id="turn0view0", title="Web search - OpenAI API", url=URL)+++def cited(text: str = f"{ANSWER}{MARKER}"):+ """A registered answer and its bundle, exactly as the backend hands them on.++ What travels through delivery is an opaque token, not the link: every+ surface below is measured on what it makes of that, which is the only way+ a test can see a surface leak one or write the wrong body.+ """+ return register_citations(+ text, {SOURCE.ref_id: SOURCE}, unresolved_label="(source unavailable)"+ )+++class CitationDeliveryTests(unittest.IsolatedAsyncioTestCase):+ async def deliver(self, platform: str, text: str, **kwargs):+ controller = _StubController(platform)+ dispatcher = ConsolidatedMessageDispatcher(controller)+ context = MessageContext(user_id="u1", channel_id="c1", platform=platform)+ with mock.patch("core.message_dispatcher.persist_agent_message") as persist:+ await dispatcher.emit_agent_message(context, "result", text, **kwargs)+ return controller.im_client.sent, persist++ async def test_every_platform_receives_the_links_and_never_the_sidecar(self):+ registered, bundle = cited()+ for platform in ("slack", "telegram", "discord", "lark", "wechat"):+ with self.subTest(platform=platform):+ sent, _ = await self.deliver(platform, registered, citations=bundle)++ delivered = "".join(text for _, _, text, _ in sent)+ self.assertEqual(delivered, CITED)+ self.assertNotIn("ref_id", delivered)+ self.assertNotIn("turn0view0", delivered)++ async def test_the_sidecar_reaches_persistence(self):+ registered, bundle = cited()+ _, persist = await self.deliver("slack", registered, citations=bundle)++ persist.assert_called_once()+ # The body is persisted still holding its tokens: the mirror writes the+ # links and measures the spans after the last transform it will see.+ self.assertIs(persist.call_args.kwargs["citations"], bundle)+ body, citations = finalize_citations(persist.call_args.args[2], bundle)+ self.assertEqual(body, CITED)+ self.assertEqual([c["url"] for c in citations], [URL])++ async def test_an_uncited_answer_still_persists_without_a_sidecar(self):+ _, persist = await self.deliver("slack", "Plain answer.")++ self.assertIsNone(persist.call_args.kwargs["citations"])++ async def test_the_sidecar_does_not_change_how_a_long_answer_is_chunked(self):+ """Chunking reads the text only, so the two runs must be byte-identical."""+ controller = _StubController("wechat", max_bytes=1900)+ dispatcher = ConsolidatedMessageDispatcher(controller)+ context = MessageContext(user_id="wechat-user", channel_id="wechat-user", platform="wechat")+ registered, bundle = cited(f"{'详' * 1200}\n\n{ANSWER}{MARKER}\n\n{'细' * 1200}")+ long_text = materialize_citations(registered, bundle)++ with mock.patch("core.message_dispatcher.persist_agent_message"):+ await dispatcher.emit_agent_message(+ context, "result", registered, citations=bundle+ )+ with_sidecar = [text for _, _, text, _ in controller.im_client.sent]+ controller.im_client.sent.clear()+ await dispatcher.emit_agent_message(context, "result", long_text)+ without_sidecar = [text for _, _, text, _ in controller.im_client.sent]++ # A token is shorter than the link it stands for, so chunking it would+ # split the answer somewhere else: the links are written before the+ # body is measured, and the two runs come out identical.+ self.assertGreater(len(with_sidecar), 1)+ self.assertEqual(with_sidecar, without_sidecar)+ self.assertEqual("".join(with_sidecar), long_text)++ async def test_the_declared_parse_mode_is_untouched(self):+ registered, bundle = cited()+ sent, _ = await self.deliver(+ "slack", registered, parse_mode="markdown", citations=bundle+ )++ self.assertEqual([parse_mode for _, _, _, parse_mode in sent], ["markdown"])+++class CitationRenderingPerPlatformTests(unittest.TestCase):+ """Each platform's own renderer, fed the exact text the backend emits."""++ def test_telegram_renders_an_anchor_carrying_the_source_domain(self):+ rendered = TelegramFormatter().render(CITED)++ self.assertIn(f'<a href="{URL}">developers.openai.com</a>', rendered)++ def test_telegram_keeps_a_cited_url_holding_markdown_punctuation_intact(self):+ """The link escapes the parens and Telegram opens the address as written."""+ for url in (+ "https://en.wikipedia.org/wiki/Foo_(bar)",+ "https://example.com/a_(b",+ "https://example.com/a%28b%29",+ ):+ with self.subTest(url=url):+ text, citations = resolve_citations(+ "Claimed.\ue200cite\ue202turn0view0\ue201",+ {"turn0view0": CitationSource(ref_id="turn0view0", title="T", url=url)},+ unresolved_label="[source unavailable]",+ )++ rendered = TelegramFormatter().render(text)++ self.assertEqual(citations[0]["url"], url)+ self.assertIn(f'<a href="{url}">', rendered)+ self.assertTrue(rendered.endswith("</a>"))++ def test_telegram_leaves_a_cited_link_inside_a_code_block_literal(self):+ rendered = TelegramFormatter().render(f"```\n{LINK}\n```")++ self.assertIn("developers.openai.com](https", rendered)+ self.assertNotIn("<a href", rendered)++ def test_telegram_recognizes_a_link_whose_scheme_is_not_lowercase(self):+ """A scheme is case-insensitive. The link parser only knew the lowercase+ spelling, so an uppercase one was delivered as raw Markdown - which any+ link the agent writes in its own prose hits too, not only a citation."""+ rendered = TelegramFormatter().render("See [example.com](HTTPS://Example.com/X)")++ self.assertIn('<a href="HTTPS://Example.com/X">example.com</a>', rendered)++ def test_a_source_that_shouted_its_scheme_is_still_delivered_as_an_anchor(self):+ """The whole path, not just the formatter: an untrusted result spells+ the scheme in uppercase, and Telegram still shows a labeled hyperlink."""+ text, citations = resolve_citations(+ "Claimed.\ue200cite\ue202turn0view0\ue201",+ {+ "turn0view0": CitationSource(+ ref_id="turn0view0", title="T", url="HTTPS://Example.com/X"+ )+ },+ unresolved_label="[source unavailable]",+ )++ rendered = TelegramFormatter().render(text)++ self.assertEqual(citations[0]["url"], "https://Example.com/X")+ self.assertIn('<a href="https://Example.com/X">example.com</a>', rendered)++ def test_markdown_native_platforms_pass_the_link_through(self):+ for formatter in (DiscordFormatter(), FeishuFormatter(), AvibeFormatter()):+ with self.subTest(formatter=type(formatter).__name__):+ self.assertEqual(+ formatter.format_link("developers.openai.com", URL),+ LINK,+ )++ def test_wechat_keeps_the_url_visible_because_it_has_no_hyperlinks(self):+ """Plain text, so attribution has to survive as readable text."""+ rendered = WeChatFormatter().format_link("developers.openai.com", URL)++ self.assertIn("developers.openai.com", rendered)+ self.assertIn(URL, rendered)+++class WeChatAdapterRenderingTests(unittest.TestCase):+ """The real send path, not just the formatter it is supposed to agree with.++ ``WeChatBot.send_message`` puts the text through ``format_markdown`` before+ it reaches the API, so that method - and not ``WeChatFormatter`` on its own -+ decides what a WeChat user can read. A formatter test that never crosses this+ boundary cannot see the adapter drop the very address it renders.+ """++ def setUp(self):+ self.bot = WeChatBot(WeChatConfig(bot_token="test-token"))++ def test_the_adapter_keeps_the_destination_it_flattens(self):+ flattened = self.bot.format_markdown(CITED)++ self.assertIn(URL, flattened)+ self.assertEqual(flattened, f"Native web search is documented. developers.openai.com ({URL})")++ def test_the_adapter_writes_a_link_the_way_its_own_formatter_does(self):+ """One platform, one answer about how a link looks in plain text."""+ self.assertEqual(+ self.bot.format_markdown(LINK),+ WeChatFormatter().format_link("developers.openai.com", URL),+ )++ def test_flattening_a_link_never_lengthens_the_message(self):+ """``text (url)`` is one character shorter than ``[text](url)``.++ Chunking runs on the text the dispatcher holds, before this conversion,+ so keeping the address can only ever be safe if it cannot grow a chunk.+ """+ for text in (CITED, f"{LINK} {LINK}", f"**Bold** {LINK}", f"# Head\n{LINK}"):+ with self.subTest(text=text):+ self.assertLessEqual(len(self.bot.format_markdown(text)), len(text))++ def test_the_emphasis_passes_cannot_rewrite_a_destination(self):+ """A URL is full of characters the markdown strippers would eat."""+ url = "https://example.com/a_b_c/d*e*f/g`h"++ flattened = self.bot.format_markdown(f"See [example.com]({url}) now")++ self.assertEqual(flattened, f"See example.com ({url}) now")++ def test_a_self_titled_link_is_not_written_twice(self):+ url = "https://example.com/x"++ self.assertEqual(self.bot.format_markdown(f"[{url}]({url})"), url)++ def test_ordinary_markdown_is_still_reduced_to_plain_text(self):+ text = "# Title\n**bold** *italic* ~~gone~~ `code` and [a](https://example.com/a)"++ self.assertEqual(+ self.bot.format_markdown(text),+ "Title\nbold italic gone code and a (https://example.com/a)",+ )++ def test_an_image_is_still_reduced_to_its_alt_text(self):+ self.assertEqual(+ self.bot.format_markdown(""),+ "a screenshot",+ )+++class SlackCitationRenderingTests(unittest.TestCase):+ def test_slack_converts_the_link_to_mrkdwn(self):+ from markdown_to_mrkdwn import SlackMarkdownConverter++ rendered = SlackMarkdownConverter().convert(CITED)++ self.assertIn(f"<{URL}|developers.openai.com>", rendered)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_link_unit_delivery.py b/tests/test_link_unit_delivery.pynew file mode 100644index 000000000..f06d173fd--- /dev/null+++ b/tests/test_link_unit_delivery.py@@ -0,0 +1,1309 @@+"""Where a link is, and what a platform owes the whole of it.++A Markdown link is one unit: a label a reader taps and an address the tap goes+to. Every IM dialect spells that unit its own way, and the passes that do it+are line scanners - Slack's ``markdown_to_mrkdwn`` reads a whole line for+emphasis and for brackets alike. Two things went wrong there. It read+``https://a*b*.example/x`` as emphasis and delivered a link to+``https://a_b_.example/x``, a different site; and once the address was held+behind a placeholder, the brackets still in the stream were its to mis-pair -+``[^f]: [p](https://example.com/x)`` went out as one link labelled ``^f]: [p``.++So the unit is located once, in the shared layer, and each platform is asked to+spell it again from the label and the destination a Markdown reader resolves.+Both halves are measured here: which source ranges a link can be found in -+including the ones CommonMark hands to no inline parser, because the consumer+downstream reads them anyway - and what the real Slack client sends for each.+"""++from __future__ import annotations++import html+import json+import re+import sys+import unittest+from pathlib import Path+from unittest import mock++from markdown_it import MarkdownIt+from markdown_it.common.entities import entities++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from config.v2_config import WeChatConfig+from core.citations import (+ _END,+ _SEP,+ _START,+ CitationSource,+ register_citations,+ resolve_citations,+)+from core.message_dispatcher import ConsolidatedMessageDispatcher+from core.reply_enhancer import inline_links+from modules.im import MessageContext+from modules.im.formatters import hold_links, restore_held+from modules.im.slack import SlackBot, SlackMarkdownConverter+from modules.im.wechat import WeChatBot, wechat_api, wechat_cdn+from tests.test_message_dispatcher_result_fallback import _StubController++STAR = "https://a*b*.example/x"+PLAIN = "https://example.com/x"+++def located(text: str) -> list[tuple[str, str, str]]:+ """Every link in *text* as ``(whole unit, label, resolved destination)``."""+ return [+ (+ text[link.start : link.end],+ text[link.label_start : link.label_end],+ link.destination,+ )+ for link in inline_links(text)+ ]+++def slack(text: str) -> str:+ # The conversion is the whole unit under test; the rest of the client is+ # network plumbing this never reaches.+ bot = SlackBot.__new__(SlackBot)+ bot.markdown_converter = SlackMarkdownConverter()+ return bot._convert_markdown_to_slack_mrkdwn(text)+++# Slack's own grammar for the unit, and the only way to read one honestly:+# structure first. A field stops at a raw ``<``, ``>`` or ``|`` because that is+# where Slack stops reading it, so a destination that let one through does not+# parse into a field here at all - which is the point. Decoding before parsing+# would quietly turn a truncated link into a plausible one and let a broken+# delimiter pass as a passing test.+_SLACK_WRAPPER_RE = re.compile(r"<([^<>|]*)(?:\|([^<>]*))?>")+# The three references Slack resolves on the way in, and nothing else: what the+# reader is shown for a wire string is that string with these three decoded, in+# one pass.+_SLACK_DECODED = {"&": "&", "<": "<", ">": ">"}+_SLACK_REFERENCE_RE = re.compile(r"&(?:amp|lt|gt);")+_ANCHOR_TEXT_RE = re.compile(r"<a [^>]*>(.*)</a>", re.S)+++def all_links(wire: str) -> list[tuple[str, str]]:+ """Every ``(destination, label)`` in *wire*, exactly as the wire spells them."""+ return _SLACK_WRAPPER_RE.findall(wire)+++def one_link(wire: str) -> tuple[str, str]:+ """The ``(destination, label)`` of the single wrapper in *wire*, undecoded.++ Asserting there is exactly one is half the contract: a destination that+ ended the wrapper early leaves a second ``<...>`` behind it, or none that+ parses at all, and either way the count says so before any field is read.+ """+ units = all_links(wire)+ assert len(units) == 1, wire+ return units[0]+++def slack_label(markdown: str) -> str:+ """The label half of the one link *markdown* is delivered as."""+ return one_link(slack(markdown))[1]+++def slack_decodes(wire: str) -> str:+ """What a Slack reader is shown for *wire*."""+ return _SLACK_REFERENCE_RE.sub(lambda match: _SLACK_DECODED[match.group()], wire)+++def slack_reads(wire: str) -> tuple[str, str]:+ """The ``(destination, label)`` a Slack client resolves out of *wire*.++ One pass of the three decodings over each field, once the structure is+ settled. The destination here is the address a tap opens, which is the+ thing the wire owes the reader - not the bytes it took to spell it.+ """+ destination, label = one_link(wire)+ return slack_decodes(destination), slack_decodes(label)+++def commonmark_shows(label: str) -> str:+ """What a Markdown reader is shown for *label* written as a link's text.++ The oracle for every reference case below: CommonMark decides whether a+ label means a character or spells one, and Slack has to show the reader+ the same thing. Code literals are excluded - their rendering is a ``<code>``+ element rather than text - and are asserted directly instead.+ """+ rendered = MarkdownIt().renderInline(f"[{label}]({PLAIN})")+ match = _ANCHOR_TEXT_RE.search(rendered)+ assert match is not None, rendered+ return html.unescape(match.group(1))+++def one_line(text: str) -> str:+ """A label's documented policy: a break, and the space around it, is a space.++ The oracle above answers what a Markdown reader sees; a Slack label answers+ that on one line, because a newline inside ``<url|label>`` is not a link.+ """+ return re.sub(r"[ \t]*(?:\r\n|\r|\n)[ \t]*", " ", text)+++def citation_body(url: str, title: str = "Source") -> str:+ """The delivered text a real citation of *url* produces."""+ body, _ = resolve_citations(+ f"Cited. {_START}cite{_SEP}turn0view0{_END}",+ {"turn0view0": CitationSource(ref_id="turn0view0", url=url, title=title)},+ unresolved_label="(source unavailable)",+ )+ return body+++def converter_alone(text: str) -> str:+ """What the third-party converter does with no protection at all.++ Used only where the contract is "this text is NOT ours to change" - code and+ images, compared against the converter of whatever version is installed. The+ mangling that motivated this change is documented in prose rather than+ asserted: pinning one vendor version's particular wrong output would make an+ upstream fix read as a regression here, and the delivered link is the thing+ this module owes the reader either way.+ """+ return SlackMarkdownConverter().convert(text)+++class TestWhereALinkIs:+ """The source ranges a link has to be findable in.++ A block CommonMark parses inline is the easy half. The other half is the+ lines it parses inline for nobody - a link reference definition is consumed+ whole by the block parser and an HTML block is passed through raw - which+ the Web renderer treats its own way but the IM converters read as ordinary+ lines of text.+ """++ def test_a_link_in_prose_is_located_whole(self):+ text = f"Read [the docs]({STAR}) first."++ assert located(text) == [(f"[the docs]({STAR})", "the docs", STAR)]++ def test_a_link_in_a_footnote_definition_is_found(self):+ """The line CommonMark hands to no inline parser at all.++ Nothing found it before, so nothing protected it, and Slack's converter+ paired the definition's own bracket with the link's - the reader got+ one link labelled ``^f]: [p`` pointing at a rewritten address.+ """+ text = f"[^f]: [p]({STAR})\n\nCited. [q]({PLAIN}) [^f]"++ assert located(text) == [+ (f"[p]({STAR})", "p", STAR),+ (f"[q]({PLAIN})", "q", PLAIN),+ ]++ def test_a_link_in_an_html_block_is_found(self):+ text = f"<div>\n[x]({STAR})\n</div>"++ assert located(text) == [(f"[x]({STAR})", "x", STAR)]++ def test_a_link_in_a_table_cell_is_found(self):+ text = f"| Source |\n| --- |\n| [t]({STAR}) |"++ assert located(text) == [(f"[t]({STAR})", "t", STAR)]++ def test_a_link_written_across_two_lines_is_one_unit(self):+ text = f"See [the\nguide]({STAR}) here."++ assert located(text) == [(f"[the\nguide]({STAR})", "the\nguide", STAR)]++ def test_a_link_inside_a_code_span_that_spans_two_lines_is_not_a_link(self):+ """The counterexample to reading each line on its own.++ A backtick pair is a code span across the whole paragraph, and the link+ it holds is an example rather than an address. A pass that scanned+ every line separately - or masked backticks one line at a time - would+ find one here, and would then hand a platform a link no reader is+ shown. Lines a block already claimed are read as that block.+ """+ text = f"Text `code\n[x]({STAR})` more"++ assert located(text) == []++ def test_a_link_inside_code_on_one_line_is_not_a_link(self):+ for text in (+ f"An example: `[x]({STAR})`",+ f"```\n[x]({STAR})\n```",+ f" [x]({STAR})\n",+ ):+ assert located(text) == []++ def test_an_image_is_not_a_link(self):+ assert located(f"") == []++ def test_a_reference_link_spells_no_destination(self):+ assert located(f"[a][ref]\n\n[ref]: {STAR}") == []++ def test_neighbouring_links_are_each_located_once(self):+ """Two units touching, so a caller may splice on these offsets in one pass."""+ text = f"[a]({PLAIN})[b]({STAR})"+ links = inline_links(text)++ assert located(text) == [(f"[a]({PLAIN})", "a", PLAIN), (f"[b]({STAR})", "b", STAR)]+ assert links[0].end == links[1].start++ def test_a_title_is_not_part_of_the_address(self):+ text = f'[docs]({PLAIN} "T")'++ assert located(text) == [(text, "docs", PLAIN)]+++class TestHoldingTheUnit:+ def test_each_unit_is_replaced_once_and_restored_verbatim(self):+ text = f"See [one]({PLAIN}) and [two]({STAR})."++ held, tokens = hold_links(text, render=lambda label, url: f"<{url}|{label}>")++ assert "](" not in held+ assert len(tokens) == 2+ assert restore_held(held, tokens) == (+ f"See <{PLAIN}|one> and <{STAR}|two>."+ )++ def test_the_platform_is_handed_the_label_and_the_resolved_destination(self):+ """Resolved: angle brackets and a title are spelling, not address."""+ seen: list[tuple[str, str]] = []++ hold_links(+ f'[a](<{STAR}>) and [b]({PLAIN} "T")',+ render=lambda label, url: seen.append((label, url)) or "",+ )++ assert seen == [("a", STAR), ("b", PLAIN)]++ def test_a_text_with_no_link_is_handed_back_untouched(self):+ text = "Just *em* and **bold** and `code`."++ assert hold_links(text, render=lambda label, url: "") == (text, {})+++class TestSlackSendsTheWholeUnit:+ """The real client, on each range the converter reads as a line of text."""++ def test_a_footnote_definition_reaches_slack_as_the_link_it_spells(self):+ text = f"[^f]: [p]({STAR})"++ assert slack(text) == f"[^f]: <{STAR}|p>"++ def test_an_html_block_keeps_the_address_it_holds(self):+ text = f"<div>\n[x]({STAR})\n</div>"++ assert slack(text) == f"<div>\n<{STAR}|x>\n</div>"++ def test_a_table_cell_becomes_a_link_instead_of_raw_markdown(self):+ text = f"| Source |\n| --- |\n| [t]({STAR}) |"++ assert slack(text) == f"*Source*\n<{STAR}|t>"++ def test_a_label_written_across_two_lines_arrives_as_one_link(self):+ """A newline inside ``<url|label>`` is not a link, and a soft break is a space.++ The converter's link pattern never crosses a line, so on its own it+ left the Markdown standing - with the address already rewritten by the+ emphasis pass that does not stop at one either.+ """+ text = f"See [the\nguide]({STAR}) here."++ assert slack(text) == f"See <{STAR}|the guide> here."++ def test_a_titled_link_no_longer_sends_the_title_as_part_of_the_address(self):+ text = f'[docs]({PLAIN} "T")'++ assert slack(text) == f"<{PLAIN}|docs>"++ def test_a_formatted_label_is_still_mrkdwn(self):+ """Holding the unit takes the label out of the converter's reach, so the+ platform that spells the unit formats the label itself."""+ assert slack(f"[**bold**]({PLAIN})") == f"<{PLAIN}|*bold*>"++ def test_a_link_with_no_label_is_sent_bare(self):+ assert slack(f"[]({PLAIN})") == f"<{PLAIN}>"++ def test_prose_around_a_link_is_formatted_as_before(self):+ assert slack(f"See [one]({PLAIN}) and *em* text.") == (+ f"See <{PLAIN}|one> and _em_ text."+ )++ def test_a_code_example_keeps_the_behaviour_it_had(self):+ """Code is not a link on any surface, and holding stops at its edge."""+ for text in (f"```\n[x]({STAR})\n```", f"Text `code\n[x]({STAR})` more"):+ assert slack(text) == converter_alone(text)++ def test_an_image_keeps_the_behaviour_it_had(self):+ text = f""++ assert slack(text) == converter_alone(text)+++class TestSlackSpellsTheWholeLabel:+ """What a label owes Slack once the unit is Slack's to serialize.++ ``<url|label>`` ends at the first ``>``, so a label that means one+ literally takes the address down with it: the reader is shown the rest of+ the label and the raw URL, with nothing to click. Slack reads ``&`` and+ ``<`` as markup of its own for the same reason, and a label that came from+ a backslash escape only becomes those characters at restore time - after+ the wrapper used to be built.+ """++ def test_a_label_that_means_a_delimiter_arrives_as_one(self):+ assert slack(f"[a > b]({PLAIN})") == f"<{PLAIN}|a > b>"+ assert slack(f"[Tom & Jerry]({PLAIN})") == f"<{PLAIN}|Tom & Jerry>"+ assert slack(f"[5 < 6]({PLAIN})") == f"<{PLAIN}|5 < 6>"++ def test_a_backslash_escaped_delimiter_is_encoded_not_restored_raw(self):+ """The ordering case: these characters come back after the converter ran.++ An escape says the character is not syntax, so it is held through the+ platform pass and restored afterwards. Restored into a finished+ ``<url|label>`` it would be a raw delimiter inside the wrapper with+ nothing left to encode it - the link would end at the ``>``.+ """+ assert slack(rf"[a \> b \& c \< d]({PLAIN})") == f"<{PLAIN}|a > b & c < d>"++ def test_a_reference_the_source_spells_arrives_as_the_character_it_names(self):+ """``&`` means ``&`` to the reader, and Slack spells ``&`` that way too."""+ assert slack(f"[AT&T]({PLAIN})") == f"<{PLAIN}|AT&T>"+ assert slack(f"[a <b> c]({PLAIN})") == f"<{PLAIN}|a <b> c>"++ def test_a_label_that_looks_like_a_mention_is_delivered_as_text(self):+ """A label reading ``<@U123>`` is text somebody wrote, not a mention."""+ assert slack(f"[<@U123>]({PLAIN})") == f"<{PLAIN}|<@U123>>"++ def test_emphasis_in_a_label_survives_the_encoding(self):+ assert slack(f"[**bold** & <em>]({PLAIN})") == f"<{PLAIN}|*bold* & <em>>"++ def test_the_destination_reaches_the_address_it_named(self):+ """The address is not text - but the wire that carries it is.++ Slack reads ``&`` in a text object as the start of one of the three+ references it resolves, so a query separator travels spelled the way+ that object spells one. What the reader is owed is the address a tap+ opens, not the bytes that carried it there: ``?a=1&b=2`` on the+ wire is ``?a=1&b=2`` to the client, which is the page the Markdown+ named. Percent-encoding that ``&`` instead would survive every decoder+ and open a different page - one parameter ``a`` holding ``1&b=2``.+ """+ query = "https://example.com/s?a=1&b=2"+ wire = slack(f"[q]({query})")++ assert wire == "<https://example.com/s?a=1&b=2|q>"+ assert slack_reads(wire) == (query, "q")++ def test_an_autolink_is_not_this_serializers_unit(self):+ """``<https://...>`` arrives already wrapped, and is left as it was."""+ query = "https://example.com/s?a=1&b=2"++ assert slack(f"<{query}>") == f"<{query}>"++ def test_text_outside_a_link_keeps_the_behaviour_it_had(self):+ """Only the link wrapper is this change's boundary."""+ for text in ("a > b & c", "5 < 6", "plain & text"):+ assert slack(text) == converter_alone(text)+++# The destination half of the unit, as one batch: the Markdown a reader wrote,+# the exact bytes Slack is sent, and the address a tap on it opens. The wire is+# asserted on its own so a permissive decoder cannot make a broken delimiter+# look like a working link, and the address on its own because that - not the+# spelling - is what the reader was promised.+# The bounded matrix both consumers are measured on. Shared with+# ui/src/components/ui/markdown.test.tsx, which renders the same sources+# through the real Markdown component; see the file's own description.+_DESTINATION_MATRIX = json.loads(+ (Path(__file__).resolve().parent / "fixtures/link_destination_matrix.json").read_text(+ encoding="utf-8"+ )+)["cases"]++_DESTINATION_CORPUS = (+ # A delimiter of the wrapper, however the source spells it. Raw, escaped+ # and written as a reference all resolve to the same address, and all three+ # have to leave it whole: a ``>`` ends the link early, a ``|`` starts the+ # label early, and a ``<`` opens a second unit inside the first.+ (r"[h](https://example.com/a>b)", "<https://example.com/a%3Eb|h>", "https://example.com/a%3Eb"),+ (r"[h](https://example.com/a\>b)", "<https://example.com/a%3Eb|h>", "https://example.com/a%3Eb"),+ ("[h](https://example.com/a>b)", "<https://example.com/a%3Eb|h>", "https://example.com/a%3Eb"),+ ("[h](https://example.com/a>b)", "<https://example.com/a%3Eb|h>", "https://example.com/a%3Eb"),+ (r"[h](https://example.com/a<b)", "<https://example.com/a%3Cb|h>", "https://example.com/a%3Cb"),+ (r"[h](https://example.com/a\<b)", "<https://example.com/a%3Cb|h>", "https://example.com/a%3Cb"),+ ("[h](https://example.com/a<b)", "<https://example.com/a%3Cb|h>", "https://example.com/a%3Cb"),+ (r"[h](https://example.com/a|b)", "<https://example.com/a%7Cb|h>", "https://example.com/a%7Cb"),+ (r"[h](https://example.com/a\|b)", "<https://example.com/a%7Cb|h>", "https://example.com/a%7Cb"),+ # ``&`` is the one Slack control character an address genuinely needs: it+ # separates query parameters. So it is spelled, not escaped away, and the+ # separator comes back when the client resolves the reference.+ ("[h](https://example.com/s?a=1&b=2)", "<https://example.com/s?a=1&b=2|h>", "https://example.com/s?a=1&b=2"),+ ("[h](https://example.com/a&b)", "<https://example.com/a&b|h>", "https://example.com/a&b"),+ # A reference that stands for the text of a reference, paired with the+ # escape that spells the same thing. Both name an address holding the five+ # characters ``&``, and one decoding pass has to leave them standing.+ ("[h](https://example.com/a&amp;b)", "<https://example.com/a&amp;b|h>", "https://example.com/a&b"),+ (r"[h](https://example.com/a\&b)", "<https://example.com/a&amp;b|h>", "https://example.com/a&b"),+ # A reference can put a control character in an address. A newline would+ # end the one line this wrapper is allowed to be.+ ("[h](https://example.com/p q)", "<https://example.com/p%0Aq|h>", "https://example.com/p%0Aq"),+ ("[h](https://example.com/p	q)", "<https://example.com/p%09q|h>", "https://example.com/p%09q"),+ ("[h](<https://example.com/a b>)", "<https://example.com/a%20b|h>", "https://example.com/a%20b"),+ # An escape the address already spells is left spelled once. Encoding the+ # ``%`` again would send ``%253E``, which is a path holding the text+ # ``%3E`` rather than the one the author wrote.+ ("[h](https://example.com/a%3Eb)", "<https://example.com/a%3Eb|h>", "https://example.com/a%3Eb"),+ ("[h](https://example.com/a%26amp%3Bb)", "<https://example.com/a%26amp%3Bb|h>", "https://example.com/a%26amp%3Bb"),+ # ``%b`` starts no escape, so that ``%`` is a character of the path and is+ # written ``%25``. Leaving it bare is not the gentler answer: the consumer+ # on the other side reads a bare ``%`` as data too, so a bare one delivers+ # a path nobody asked for. ``%zz`` is two ASCII alphanumerics after a+ # ``%``, which the same rule reads as an escape already spelled - so it is+ # left standing rather than repaired into something it never said.+ ("[h](https://example.com/a%b)", "<https://example.com/a%25b|h>", "https://example.com/a%25b"),+ ("[h](https://example.com/a%zzb)", "<https://example.com/a%zzb|h>", "https://example.com/a%zzb"),+ # Brackets are the syntax of an IPv6 host and data everywhere else, and the+ # destination has to say which it means. Only the authority keeps them; the+ # same two characters in a path, query or fragment are spelled - and+ # userinfo, port, query and fragment structure survive either way.+ (r"[h](https://[::1]:8443/a\>b)", "<https://[::1]:8443/a%3Eb|h>", "https://[::1]:8443/a%3Eb"),+ (+ "[h](https://u:p@[::1]:8443/a?x=1&y=2)",+ "<https://u:p@[::1]:8443/a?x=1&y=2|h>",+ "https://u:p@[::1]:8443/a?x=1&y=2",+ ),+ (+ "[h](https://u:p@[::1]:8443/a[b]?x=[c]#f[d])",+ "<https://u:p@[::1]:8443/a%5Bb%5D?x=%5Bc%5D#f%5Bd%5D|h>",+ "https://u:p@[::1]:8443/a%5Bb%5D?x=%5Bc%5D#f%5Bd%5D",+ ),+ ("[h](https://example.com/a[b])", "<https://example.com/a%5Bb%5D|h>", "https://example.com/a%5Bb%5D"),+ # A bracketed host no URL parser accepts stays spelled. Putting brackets+ # back there would invent an address the source never wrote, and the Web+ # renderer refuses the same repair for the same reason.+ ("[h](https://[nope]/x)", "<https://%5Bnope%5D/x|h>", "https://%5Bnope%5D/x"),+ ("[h](https://example.com/p?x=1#f%20g)", "<https://example.com/p?x=1#f%20g|h>", "https://example.com/p?x=1#f%20g"),+ # Characters with no meaning to a URI and none to Slack either, but every+ # one of them outside the set the consumer leaves standing.+ (r"[h](https://example.com/a\\b)", "<https://example.com/a%5Cb|h>", "https://example.com/a%5Cb"),+ ("[h](https://example.com/q=`x`)", "<https://example.com/q=%60x%60|h>", "https://example.com/q=%60x%60"),+ # Non-ASCII is spelled, in the host as much as in the path, because that is+ # what the consumer writes. A URL parser reads that host back as the+ # punycode it was always going to resolve - a third spelling, and the one+ # the connection is actually made to.+ (+ "[h](https://例子.测试/路径?q=a&b=c)",+ "<https://%E4%BE%8B%E5%AD%90.%E6%B5%8B%E8%AF%95/%E8%B7%AF%E5%BE%84?q=a&b=c|h>",+ "https://%E4%BE%8B%E5%AD%90.%E6%B5%8B%E8%AF%95/%E8%B7%AF%E5%BE%84?q=a&b=c",+ ),+ # A scheme that is not http, already delivered today.+ (+ r"[h](mailto:[email protected]?subject=a\>b)",+ "<mailto:[email protected]?subject=a%3Eb|h>",+ "mailto:[email protected]?subject=a%3Eb",+ ),+)+++class TestSlackSpellsTheWholeDestination:+ """The other half of the unit, and the half a broken wrapper takes with it.++ ``<url|label>`` is built out of ``<``, ``|`` and ``>``, and an address is+ allowed to contain all three. One of them in the destination ends the unit+ where the author did not: the reader is shown a truncated link followed by+ the rest of the address as plain text, and the tap goes somewhere else or+ nowhere. Escaped or spelled as a reference, it is the same character by the+ time the wrapper is built - so the destination is finished before the+ wrapper closes over it, the same as the label.++ Two jobs, kept apart. First the address is written the way the Markdown+ consumer on the other side writes it - ``spell_uri``, so ``<``, ``>`` and+ ``|`` leave as ``%3C``, ``%3E`` and ``%7C`` along with spaces, controls,+ backslashes, backticks, brackets that are data and a ``%`` that starts no+ escape, while ``?#&=`` and an escape already spelled are left standing.+ Then Slack's own text-object escaping runs over the result, which is what a+ query ``&`` needs and percent-encoding would destroy.++ The wire is asserted here, exactly, because it is what Slack receives. That+ those bytes and the Web renderer's own href reach one address is a separate+ claim, and it is measured rather than asserted: see+ ``test_the_matrix_arrives_at_the_address_the_web_consumer_resolves``.+ """++ def test_every_destination_arrives_as_the_address_it_named(self):+ for markdown, wire, address in _DESTINATION_CORPUS:+ assert slack(markdown) == wire, markdown+ assert slack_reads(wire) == (address, "h"), markdown++ def test_the_matrix_arrives_at_the_address_the_web_consumer_resolves(self):+ """Half of a relationship, and the half this language can measure.++ Listing the Slack spelling beside the Web spelling and calling them+ equivalent proves nothing about where either one goes. So one bounded+ matrix is shared with the Web suite: this side asserts what the real+ adapter decodes out of the wrapper, that side renders the same source+ through the real Markdown component and asserts a browser URL parser+ resolves its anchor and this address to one URL. Either side drifting+ fails one of the two.+ """+ for case in _DESTINATION_MATRIX:+ wire = slack(case["markdown"])++ assert slack_reads(wire)[0] == case["address"], case["markdown"]++ def test_a_destination_that_breaks_out_leaves_more_than_one_unit_behind(self):+ """Why the wire is asserted and not just the decoding: the failure shape.++ A raw ``>`` in the destination does not corrupt a field, it ends the+ wrapper - and what follows is text, not a link. Reading the fields of+ whatever parsed first would find a plausible ``(destination, label)``+ and say nothing was wrong, so the shape of the wire is checked first.+ """+ for markdown, _, _ in _DESTINATION_CORPUS:+ rendered = slack(markdown)++ assert (rendered.count("<"), rendered.count("|"), rendered.count(">")) == (+ 1,+ 1,+ 1,+ ), markdown+ assert len(all_links(rendered)) == 1, markdown++ broken = "<https://example.com/a>b|h>"++ assert all_links(broken) == [("https://example.com/a", "")]+ assert all_links(slack_decodes(broken)) == [("https://example.com/a", "")]++ def test_an_empty_label_leaves_the_bare_wrapper_intact(self):+ """No label, so no ``|`` - and the destination is still finished first."""+ wire = slack(r"[](https://example.com/a\>b)")++ assert wire == "<https://example.com/a%3Eb>"+ assert slack_reads(wire) == ("https://example.com/a%3Eb", "")++ def test_two_links_side_by_side_stay_two_links(self):+ """Composition: each unit closes over its own address, and only its own."""+ wire = slack(r"[a](https://example.com/1\>x) [b](https://example.com/2|y)")++ assert wire == "<https://example.com/1%3Ex|a> <https://example.com/2%7Cy|b>"+ assert all_links(wire) == [+ ("https://example.com/1%3Ex", "a"),+ ("https://example.com/2%7Cy", "b"),+ ]+++# The brackets half of the same table, measured on this side. See+# tests/fixtures/citation_authority_matrix.json: the citation suite asserts+# ``safe_url`` and the Web suite asserts the anchor, so a repair the three+# surfaces agree on still has to be agreed on three times.+_AUTHORITY_MATRIX = json.loads(+ (Path(__file__).resolve().parent / "fixtures/citation_authority_matrix.json").read_text(+ encoding="utf-8"+ )+)+++class TestSlackKeepsOnlyTheBracketsThatWereAHost:+ """A bracket is a host's syntax or it is data, and spelling loses which.++ ``spell_uri`` writes every bracket it is given as ``%5B``/``%5D``, so by the+ time the wrapper is built ``https://[::1]/admin`` and+ ``https://%5B::1%5D/admin`` are one string. A pass that then restores+ brackets to whatever looks like an authority gives the second destination a+ loopback address its author never wrote - and a tap on it opens the+ reader's own machine. So the destination is qualified before it is spelled,+ and only a literal authority a URL parser accepts gets its brackets back.+ """++ def test_every_authority_arrives_as_the_destination_it_named(self):+ for case in _AUTHORITY_MATRIX["cases"]:+ wire = slack(case["markdown"])++ assert len(all_links(wire)) == 1, case["markdown"]+ assert slack_reads(wire) == (case["slack"], "h"), case["why"]++ def test_two_destinations_that_spell_alike_are_still_two_destinations(self):+ """Per occurrence, not per spelling.++ Both links reach ``https://%5B::1%5D/admin`` if the answer is looked up+ by the finished text, so the body carries one of each and the pair has+ to come back apart.+ """+ pair = _AUTHORITY_MATRIX["per_occurrence"]+ wire = slack(pair["markdown"])++ assert [link[0] for link in all_links(wire)] == pair["slack"]+ assert [link[1] for link in all_links(wire)] == [+ link["label"] for link in pair["links"]+ ]+++class TestSlackTellsAReferenceFromItsSpelling:+ """Two labels, one wire: a reference and the text that spells one.++ ``&`` in a Markdown label is the character ``&``; ``\\&`` is the+ five characters ``&``. The escape is what tells them apart, and it is+ held behind a placeholder for the whole platform pass - so the distinction+ is still there to use, right up to the moment the label is serialized. A+ pass that restores the escape first and then guesses from the finished+ string cannot tell the two apart any more, and picked one meaning for both.++ A real citation writes both spellings: ``https://a%26amp%3B.example/x`` is+ a host that literally contains ``&``, and a reader shown ``a&.example``+ is being told the page is on a different site.+ """++ def test_every_reference_is_the_character_commonmark_shows(self):+ """The oracle, paired: each reference and the escaped text that spells it."""+ for label in (+ "a&b",+ r"a\&b",+ "a<b",+ r"a\<b",+ "a>b",+ r"a\>b",+ "a©b",+ r"a\©b",+ "a&b",+ r"a\&b",+ "a<b",+ "a©b",+ "a&nope;b",+ ):+ assert slack_decodes(slack_label(f"[{label}]({PLAIN})")) == commonmark_shows(label), label++ def test_the_two_spellings_do_not_arrive_as_the_same_wire_string(self):+ """Encoding is not the same as resolving, and the wire has to say which."""+ assert slack_label(f"[a&b]({PLAIN})") == "a&b"+ assert slack_label(rf"[a\&b]({PLAIN})") == "a&amp;b"+ assert slack_label(f"[a<b]({PLAIN})") == "a<b"+ assert slack_label(rf"[a\<b]({PLAIN})") == "a&lt;b"++ def test_a_reference_slack_cannot_resolve_is_resolved_here(self):+ """``©`` is a character to a Markdown reader, whatever Slack knows.++ Slack resolves only three references, so the character has to arrive+ already resolved. The escaped spelling is the literal-text case, and it+ arrives as the six characters it names.+ """+ assert slack_label(f"[a©b]({PLAIN})") == "a©b"+ assert slack_label(rf"[a\©b]({PLAIN})") == "a&copy;b"++ def test_a_reference_is_resolved_once_and_not_again(self):+ """``&copy;`` is ``©`` to a reader, not ``©``."""+ assert slack_label(f"[a&copy;b]({PLAIN})") == "a&copy;b"+ assert slack_decodes(slack_label(f"[a&copy;b]({PLAIN})")) == "a©b"++ def test_a_reference_inside_a_code_literal_stays_written_out(self):+ """Code is a literal, so its ``&`` is five characters the reader sees."""+ assert slack_label(f"[`&`]({PLAIN})") == "`&amp;`"+ assert slack_decodes(slack_label(f"[`&`]({PLAIN})")) == "`&`"++ def test_a_citation_host_that_spells_a_reference_reaches_slack_whole(self):+ """The producer end: a real citation of a host that contains ``&``.++ The label a citation writes is escaped character by character, so the+ host's own ``&`` is literal text. The address arrives from citation+ ingestion already percent-encoded and holds no character Slack reads,+ so here the wire and the address a tap opens are the same string - and+ both are asserted, because a decoder permissive enough to agree with a+ broken wire would have agreed with this one too.+ """+ for url, shown in (+ ("https://a%26amp%3B.example/x", "a&.example"),+ ("https://a%26lt%3B.example/x", "a<.example"),+ ("https://a%26copy%3B.example/x", "a©.example"),+ ):+ wire = slack(citation_body(url))+ destination, label = one_link(wire)++ assert destination == url, url+ assert slack_reads(wire) == (url, shown), url++ def test_every_name_the_parser_knows_reaches_slack_as_that_character(self):+ """The whole entity table, against the parser that owns it.++ Which names are references at all, and what each one stands for, is one+ table with two thousand rows; a helper that reconstructs it from a+ pattern of its own is a second grammar that will disagree somewhere. So+ the oracle is the real inline parser, asked for every name it knows.+ """+ for name in entities:+ label = f"a&{name};b"++ assert slack_decodes(slack_label(f"[{label}]({PLAIN})")) == one_line(+ commonmark_shows(label)+ ), label++ def test_a_numeric_reference_is_read_at_the_width_the_grammar_allows(self):+ """Seven decimal digits and six hexadecimal ones - the eighth is text.++ A decoding utility of the same package accepts eight of either. It is+ not the inline rule, and a label is what the inline rule reads: padded+ past the limit, ``&`` is the thirteen characters a reader+ sees, not ``&``.+ """+ widths = [f"a&#{'0' * pad}38;b" for pad in range(0, 7)]+ widths += [f"a&#x{'0' * pad}26;b" for pad in range(0, 6)]++ for label in widths:+ assert slack_decodes(slack_label(f"[{label}]({PLAIN})")) == commonmark_shows(+ label+ ), label++ assert slack_label(f"[a&b]({PLAIN})") == "a&#00000038;b"+ assert slack_label(f"[a&b]({PLAIN})") == "a&#x0000026;b"++ def test_a_code_point_the_parser_cannot_encode_is_the_character_it_substitutes(self):+ """Recognized is not the same as valid, and the rule answers both.++ ``�``, a C1 control, a surrogate half and anything past the last+ code point are all references the grammar recognizes; CommonMark shows+ U+FFFD for each. Leaving the spelling standing would show the reader a+ different thing than every other Markdown surface does.+ """+ for label in (+ "a�b",+ "a€b",+ "a�b",+ "a�b",+ "a�b",+ "a�b",+ "ab",+ "a�b",+ ):+ assert slack_decodes(slack_label(f"[{label}]({PLAIN})")) == commonmark_shows(+ label+ ), label++ assert slack_label(f"[a�b]({PLAIN})") == "a\ufffdb"++ def test_a_malformed_or_unknown_reference_stays_the_text_it_is(self):+ for label in ("a&#;b", "a&#x;b", "a&#xZZ;b", "a&nope;b", "a& b"):+ assert slack_decodes(slack_label(f"[{label}]({PLAIN})")) == commonmark_shows(+ label+ ), label++ def test_an_escaped_numeric_reference_is_literal_text(self):+ """The escape still decides, and a numeric spelling is no different."""+ assert slack_label(rf"[a\&b]({PLAIN})") == "a&#38;b"+ assert slack_label(rf"[a\�b]({PLAIN})") == "a&#0;b"+ assert slack_label(rf"[a\&b]({PLAIN})") == "a&#00000038;b"++ def test_a_nested_looking_numeric_reference_is_resolved_once(self):+ assert slack_decodes(slack_label(f"[a&#38;b]({PLAIN})")) == "a&b"++ def test_a_numeric_reference_inside_a_code_literal_stays_written_out(self):+ assert slack_decodes(slack_label(f"[`�`]({PLAIN})")) == "`�`"++ def test_emphasis_and_a_reference_in_one_label_both_survive(self):+ assert slack(f"[**bold** & ©]({PLAIN})") == f"<{PLAIN}|*bold* & ©>"+++class TestALabelIsOneLineHoweverTheBreakIsSpelled:+ """``<url|label>`` ends at a newline, and Markdown spells one three ways.++ A source line break was already folded to a space before the platform pass,+ which was the whole of the invariant while a break could only be written as+ one. A character reference writes one too - `` ``, ``
``,+ ``
``, `` `` - and those become characters only after the+ reference is resolved. So the label is measured for being one line once its+ characters are decided, not before they are.+ """++ def test_a_source_line_break_is_still_a_space(self):+ """The case that already held, restated where the policy now ends."""+ assert slack(f"[the\nguide]({PLAIN})") == f"<{PLAIN}|the guide>"+ assert slack(f"[the\r\nguide]({PLAIN})") == f"<{PLAIN}|the guide>"++ def test_a_reference_that_spells_a_break_is_folded_the_same_way(self):+ for label in ("a b", "a
b", "a
b", "a b", "a b"):+ assert slack(f"[{label}]({PLAIN})") == f"<{PLAIN}|a b>", label++ def test_an_escaped_break_reference_is_literal_text_with_no_break_to_fold(self):+ assert slack(rf"[a\
b]({PLAIN})") == f"<{PLAIN}|a&NewLine;b>"+ assert slack(rf"[a\ b]({PLAIN})") == f"<{PLAIN}|a&#10;b>"++ def test_the_destination_is_not_touched_by_the_fold(self):+ """A percent-encoded break in the address is address, not label."""+ url = "https://example.com/a%0Ab"++ assert slack(f"[x]({url})") == f"<{url}|x>"++ def test_text_outside_a_link_keeps_the_lines_it_had(self):+ assert slack("first\nsecond") == converter_alone("first\nsecond")+++class _TextOnlyClient:+ """A client that can only send text, and whose sends all succeed.++ The success matters: a fragment delivered here is reported as a delivered+ result, so nothing downstream would ever fall back to sending the whole+ thing another way.+ """++ def __init__(self):+ self.sent: list[str] = []+ self._next_id = 1++ def should_use_thread_for_reply(self):+ return False++ async def send_message(self, context, text, parse_mode=None, reply_to=None, subtext=None):+ self.sent.append(text)+ message_id = f"msg-{self._next_id}"+ self._next_id += 1+ return message_id+++class _DiscordLikeClient(_TextOnlyClient):+ """Adds the native Markdown upload Discord implements."""++ def __init__(self, *, upload_id: str = "discord-file-1"):+ super().__init__()+ self.uploads: list[tuple[str, str]] = []+ self._upload_id = upload_id++ async def upload_markdown(self, context, title, content, filetype="markdown"):+ self.uploads.append((title, content))+ return self._upload_id+++def discord_context() -> MessageContext:+ return MessageContext(user_id="U1", channel_id="C1", platform="discord")+++def wechat_context() -> MessageContext:+ # ``context_token`` is spelled here so the adapter resolves the recipient+ # from the message instead of reading its on-disk token cache.+ return MessageContext(+ user_id="wechat-user",+ channel_id="wechat-user",+ platform="wechat",+ platform_specific={"context_token": "ctx-token"},+ )+++def dispatcher_for(platform: str, client) -> ConsolidatedMessageDispatcher:+ return ConsolidatedMessageDispatcher(_StubController(platform=platform, im_client=client))+++class TestTheSplitKeepsALinkWhole:+ """Where the shared splitter is allowed to draw a boundary.++ A result longer than one message is cut into several, and the cut used to+ be taken at whitespace with no idea what it was cutting. A boundary drawn+ through ``[label](url)`` delivers neither half: both chunks send fine, so+ nothing falls back, and the reader is shown raw Markdown and no way to+ reach the source.+ """++ def test_a_fitting_link_crossing_a_boundary_moves_whole(self):+ link = f"[a b]({PLAIN})"+ text = "x" * 1890 + link + " tail"++ chunks = dispatcher_for("discord", _DiscordLikeClient())._split_result_text(text, 1900)++ assert "".join(chunks) == text+ assert sum(chunk.count(link) for chunk in chunks) == 1+ assert all(len(chunk) <= 1900 for chunk in chunks)++ def test_adjacent_links_are_each_kept_whole(self):+ first, second = f"[a]({PLAIN})", f"[b c]({STAR})"+ text = "x" * (1900 - len(first) - 3) + first + second + " tail"++ chunks = dispatcher_for("discord", _DiscordLikeClient())._split_result_text(text, 1900)++ assert "".join(chunks) == text+ assert [chunk.count(first) for chunk in chunks] == [1, 0]+ assert [chunk.count(second) for chunk in chunks] == [0, 1]+ assert all(len(chunk) <= 1900 for chunk in chunks)++ def test_a_multibyte_budget_keeps_a_link_whole_and_within_the_limit(self):+ """WeChat counts bytes, so the boundary moves on a different ruler."""+ link = f"[文 档]({PLAIN})"+ text = "字" * 629 + link + "字" * 100++ chunks = dispatcher_for("wechat", _DiscordLikeClient())._split_result_text_by_bytes(text, 1900)++ assert "".join(chunks) == text+ assert sum(chunk.count(link) for chunk in chunks) == 1+ assert all(len(chunk.encode("utf-8")) <= 1900 for chunk in chunks)++ def test_a_link_that_fits_the_message_is_not_called_unsplittable(self):+ """Capacity is the message, not the last space before the link ends.++ The preferred cut is whitespace, and here every space inside the+ budget is inside the label. Pulling the boundary back to the link's+ start lands on ``0`` - the link begins the chunk - which said nothing+ about whether the link fits: it fits with 426 characters to spare, and+ the cut belongs after it.+ """+ dispatcher = dispatcher_for("discord", _DiscordLikeClient())+ link = "[" + ("word " * 220) + "](https://example.com/" + ("x" * 350) + ")"+ text = link + "x" * 1000+ assert (len(link), len(text)) == (1474, 2474)++ for plan in (+ dispatcher._plan_result_split(text, 1900),+ dispatcher._plan_result_split_by_bytes(text, 1900),+ ):+ assert plan.links_whole is True+ assert "".join(plan.chunks) == text+ assert len(plan.chunks) > 1+ assert sum(chunk.count(link) for chunk in plan.chunks) == 1+ assert all(len(chunk) <= 1900 for chunk in plan.chunks)+ assert all(len(chunk.encode("utf-8")) <= 1900 for chunk in plan.chunks)++ def test_a_multibyte_link_that_fits_is_measured_on_its_own_ruler(self):+ """Same shape on the byte budget, where capacity is not the character count."""+ dispatcher = dispatcher_for("wechat", _DiscordLikeClient())+ link = f"[{'文 档 ' * 100}]({PLAIN})"+ text = link + "字" * 1000++ plan = dispatcher._plan_result_split_by_bytes(text, 1900)++ assert plan.links_whole is True+ assert "".join(plan.chunks) == text+ assert sum(chunk.count(link) for chunk in plan.chunks) == 1+ assert all(len(chunk.encode("utf-8")) <= 1900 for chunk in plan.chunks)++ def test_two_links_at_the_head_of_a_chunk_are_both_kept_whole(self):+ """The search resumes past the first link, so the second is seen too.++ The only whitespace inside the budget is inside the first link's+ label. Once that link is allowed to stay, the boundary has to be+ looked for again beyond it - and what it finds there is the second+ link, which must not be cut either.+ """+ dispatcher = dispatcher_for("discord", _DiscordLikeClient())+ first = "[" + ("word " * 220) + "](https://example.com/" + ("x" * 350) + ")"+ second = f"[ab]({STAR})"+ text = first + second + "x" * 1000++ plan = dispatcher._plan_result_split(text, 1900)++ assert plan.links_whole is True+ assert "".join(plan.chunks) == text+ assert sum(chunk.count(first) for chunk in plan.chunks) == 1+ assert sum(chunk.count(second) for chunk in plan.chunks) == 1+ assert all(len(chunk) <= 1900 for chunk in plan.chunks)++ def test_a_scan_that_fails_does_not_certify_a_plan(self):+ """No answer is not the answer "there are no links".++ The enumeration is the only thing that knows where a link is. When it+ cannot run, the plan still has to make progress - the text is longer+ than one message either way - but it may not claim the boundaries it+ drew kept anything whole.+ """+ dispatcher = dispatcher_for("discord", _DiscordLikeClient())+ text = "word " * 800++ with mock.patch(+ "core.message_dispatcher.inline_links", side_effect=RuntimeError("scanner down")+ ):+ plan = dispatcher._plan_result_split(text, 1900)+ by_bytes = dispatcher._plan_result_split_by_bytes(text, 1900)++ assert plan.links_whole is False+ assert by_bytes.links_whole is False+ assert "".join(plan.chunks) == text+ assert all(len(chunk) <= 1900 for chunk in plan.chunks)++ def test_a_link_longer_than_one_message_leaves_no_whole_boundary(self):+ """No cut keeps this one intact, and the plan says so instead of pretending."""+ dispatcher = dispatcher_for("discord", _DiscordLikeClient())+ oversized = f"[{'label ' * 400}]({PLAIN})"++ assert dispatcher._plan_result_split(f"lead\n\n{oversized}", 1900).links_whole is False+ assert dispatcher._plan_result_split_by_bytes(f"lead\n\n{oversized}", 1900).links_whole is False+ assert dispatcher._plan_result_split("x" * 4000, 1900).links_whole is True+++class TestALinkTooLongToSplitIsDeliveredWhole(unittest.IsolatedAsyncioTestCase):+ """The consumer end: what the platform is actually asked to send.++ Persistence is stubbed out here - the workbench row is a different contract+ and writing one would reach real local state - so what remains is exactly+ the delivery the reader gets.+ """++ async def test_discord_attaches_the_whole_result_instead_of_fragmenting_a_link(self):+ client = _DiscordLikeClient()+ dispatcher = dispatcher_for("discord", client)+ text = f"Source:\n\n[{'label ' * 400}]({PLAIN})"++ with mock.patch("core.message_dispatcher.persist_agent_message"):+ message_id = await dispatcher.emit_agent_message(discord_context(), "result", text)++ assert client.uploads == [("result.md", text)]+ assert message_id == "discord-file-1"+ # Nothing of the result went out as text: no fragment, and no claim of+ # a delivery the reader could not follow.+ assert client.sent == [dispatcher._t("info.resultDeliveredAsAttachment")]++ async def test_discord_still_splits_when_every_link_fits(self):+ client = _DiscordLikeClient()+ dispatcher = dispatcher_for("discord", client)+ link = f"[a b]({PLAIN})"+ text = "x" * 1890 + link + " tail " + "y" * 1900++ with mock.patch("core.message_dispatcher.persist_agent_message"):+ message_id = await dispatcher.emit_agent_message(discord_context(), "result", text)++ assert message_id == "msg-1"+ assert client.uploads == []+ assert "".join(client.sent) == text+ assert sum(sent.count(link) for sent in client.sent) == 1++ async def test_a_fitting_link_is_delivered_as_text_by_a_client_with_no_file_route(self):+ """The case that used to end in a failure notice with nothing sent.++ A text-only client has no attachment to fall back to, so a plan that+ wrongly reports the link unsplittable costs the reader the whole+ message. There is a legal boundary here, and taking it delivers+ everything as text.+ """+ client = _TextOnlyClient()+ dispatcher = dispatcher_for("discord", client)+ link = "[" + ("word " * 220) + "](https://example.com/" + ("x" * 350) + ")"+ text = link + "x" * 1000++ with mock.patch("core.message_dispatcher.persist_agent_message"):+ message_id = await dispatcher.emit_agent_message(discord_context(), "result", text)++ assert message_id == "msg-1"+ assert "".join(client.sent) == text+ assert sum(sent.count(link) for sent in client.sent) == 1+ assert dispatcher._t("error.resultDeliveryFailed") not in client.sent+ assert dispatcher._t("info.resultDeliveredAsAttachment") not in client.sent++ async def test_a_fitting_link_does_not_cost_an_attachment(self):+ """A client that CAN attach must not be asked to, for a link that fits."""+ client = _DiscordLikeClient()+ dispatcher = dispatcher_for("discord", client)+ link = "[" + ("word " * 220) + "](https://example.com/" + ("x" * 350) + ")"+ text = link + "x" * 1000++ with mock.patch("core.message_dispatcher.persist_agent_message"):+ message_id = await dispatcher.emit_agent_message(discord_context(), "result", text)++ assert message_id == "msg-1"+ assert client.uploads == []+ assert "".join(client.sent) == text++ async def test_a_platform_with_no_file_route_reports_no_delivery(self):+ """Honest failure beats a half link the user is told arrived."""+ client = _TextOnlyClient()+ dispatcher = dispatcher_for("discord", client)+ text = f"Source:\n\n[{'label ' * 400}]({PLAIN})"++ with mock.patch("core.message_dispatcher.persist_agent_message"):+ message_id = await dispatcher.emit_agent_message(discord_context(), "result", text)++ assert message_id is None+ assert client.sent == [dispatcher._t("error.resultDeliveryFailed")]+++def wechat_dispatcher() -> tuple[WeChatBot, ConsolidatedMessageDispatcher]:+ bot = WeChatBot(WeChatConfig(bot_token="token"))+ return bot, dispatcher_for("wechat", bot)+++def text_items(calls: list) -> list[str]:+ return [+ item["text_item"]["text"]+ for call in calls+ for item in call+ if item.get("type") == 1+ ]+++async def deliver_through_wechat(+ dispatcher,+ text: str,+ *,+ cdn_meta,+ message_type: str = "result",+ citations=None,+):+ """Emit *text* through the real adapter with only its network replaced."""+ calls: list = []+ uploaded: dict = {}++ async def fake_upload_to_cdn(base_url, token, cdn_base_url, user_id, file_path, proxy=None):+ uploaded["path"] = file_path+ uploaded["content"] = Path(file_path).read_text(encoding="utf-8")+ return cdn_meta++ async def fake_send_message(base_url, token, to_user_id, context_token, item_list, proxy=None):+ calls.append(item_list)+ return {"message_id": f"wc-{len(calls)}"}++ with (+ mock.patch.object(wechat_cdn, "upload_file_to_cdn", fake_upload_to_cdn),+ mock.patch.object(wechat_api, "send_message", fake_send_message),+ mock.patch("core.message_dispatcher.persist_agent_message"),+ ):+ message_id = await dispatcher.emit_agent_message(+ wechat_context(), message_type, text, citations=citations+ )++ return message_id, calls, uploaded+++class TestTheWeChatAdapterCarriesTheWholeResult(unittest.IsolatedAsyncioTestCase):+ """The real adapter, with only its network calls replaced.++ WeChat inherits ``upload_markdown`` from ``BaseIMClient`` without+ implementing it - it raises - and implements ``upload_file_from_path``+ instead. A fallback that only knows the first name has no route here, which+ is exactly how a link too long to split used to go out in halves.+ """++ async def test_the_whole_result_rides_the_file_upload_the_adapter_has(self):+ bot, dispatcher = wechat_dispatcher()+ text = f"来源:\n\n[{'标签' * 600}]({PLAIN})"++ message_id, calls, uploaded = await deliver_through_wechat(+ dispatcher,+ text,+ cdn_meta={"encrypt_query_param": "q", "aes_key": "k", "file_size": 12, "file_id": "wc-file-1"},+ )++ assert message_id == "wc-file-1"+ assert uploaded["content"] == text+ assert Path(uploaded["path"]).name == "result.md"+ # The staging file is the transport's, not the machine's: it is gone+ # once the upload returns.+ assert not Path(uploaded["path"]).exists()+ # One file item, and the only text sent is the notice - no fragment of+ # the link was delivered before it.+ assert [item["type"] for call in calls for item in call].count(4) == 1+ assert text_items(calls) == [+ bot.format_markdown(dispatcher._t("info.resultDeliveredAsAttachment"))+ ]++ async def test_a_failed_upload_is_not_reported_as_a_delivery(self):+ """The CDN refuses, so ``upload_file_from_path`` answers with an empty id."""+ bot, dispatcher = wechat_dispatcher()+ text = f"来源:\n\n[{'标签' * 600}]({PLAIN})"++ message_id, calls, uploaded = await deliver_through_wechat(dispatcher, text, cdn_meta=None)++ assert message_id is None+ assert not Path(uploaded["path"]).exists()+ assert text_items(calls) == [+ bot.format_markdown(dispatcher._t("error.resultDeliveryFailed"))+ ]++ async def test_a_result_whose_links_all_fit_is_still_split_as_before(self):+ """Still split, and the chunk boundary now falls outside the link.++ WeChat renders the unit its own way - ``label (url)`` - so what has to+ survive the split is that rendering, whole and in one message.+ """+ bot, dispatcher = wechat_dispatcher()+ text = "字" * 629 + f"[文 档]({PLAIN})" + "字" * 400++ message_id, calls, uploaded = await deliver_through_wechat(+ dispatcher,+ text,+ cdn_meta={"file_id": "wc-file-1"},+ )++ assert message_id == "wc-1"+ assert uploaded == {}+ chunks = text_items(calls)+ assert len(chunks) > 1+ assert sum(chunk.count(bot.format_markdown(f"[文 档]({PLAIN})")) for chunk in chunks) == 1+ assert sum(chunk.count("字") for chunk in chunks) == 1029+ assert all(len(chunk.encode("utf-8")) <= 1900 for chunk in chunks)+++class TestTheLogPathCarriesTheWholeMessage(unittest.IsolatedAsyncioTestCase):+ """The plan's other consumer, on the path an intermediate message takes.++ A WeChat message cannot be edited, so an assistant message on the way to a+ result is delivered as its own sends rather than by editing one bubble.+ That path splits with the same shared planner and used to read only the+ chunks out of it - so the link-integrity answer the planner had already+ computed was discarded, and a citation whose address outran one message+ went out in pieces that all sent successfully.++ This is the same narrow whole-document route the result path uses, taken+ before any fragment. Nothing else about an intermediate message changes:+ it settles no turn, signals no result, and writes no second transcript row.+ """++ async def test_an_unsplittable_citation_rides_the_file_instead_of_fragments(self):+ bot, dispatcher = wechat_dispatcher()+ url = "https://example.com/" + ("x" * 2000)+ text, bundle = register_citations(+ f"Source: {_START}cite{_SEP}turn0search0{_END}",+ {"turn0search0": CitationSource(ref_id="turn0search0", url=url)},+ unresolved_label="(source unavailable)",+ )++ message_id, calls, uploaded = await deliver_through_wechat(+ dispatcher,+ text,+ cdn_meta={"file_id": "wc-file-1"},+ message_type="assistant",+ citations=bundle,+ )++ assert message_id == "wc-file-1"+ # The document carries the whole address the fragments cut in half.+ assert url in uploaded["content"]+ assert Path(uploaded["path"]).name == "result.md"+ assert not Path(uploaded["path"]).exists()+ assert [item["type"] for call in calls for item in call].count(4) == 1+ # An intermediate message has no notice of its own, and no fragment of+ # it was sent before the document.+ assert text_items(calls) == []++ async def test_a_failed_upload_is_not_reported_as_a_log_delivery(self):+ bot, dispatcher = wechat_dispatcher()+ url = "https://example.com/" + ("x" * 2000)+ text, bundle = register_citations(+ f"Source: {_START}cite{_SEP}turn0search0{_END}",+ {"turn0search0": CitationSource(ref_id="turn0search0", url=url)},+ unresolved_label="(source unavailable)",+ )++ message_id, calls, uploaded = await deliver_through_wechat(+ dispatcher,+ text,+ cdn_meta=None,+ message_type="assistant",+ citations=bundle,+ )++ assert message_id is None+ assert not Path(uploaded["path"]).exists()+ assert text_items(calls) == []++ async def test_a_log_message_whose_links_fit_is_still_split_as_before(self):+ bot, dispatcher = wechat_dispatcher()+ text = "字" * 629 + f"[文 档]({PLAIN})" + "字" * 400++ message_id, calls, uploaded = await deliver_through_wechat(+ dispatcher,+ text,+ cdn_meta={"file_id": "wc-file-1"},+ message_type="assistant",+ )++ assert message_id == "wc-1"+ assert uploaded == {}+ chunks = text_items(calls)+ assert len(chunks) > 1+ assert sum(chunk.count(bot.format_markdown(f"[文 档]({PLAIN})")) for chunk in chunks) == 1+ assert sum(chunk.count("字") for chunk in chunks) == 1029+ assert all(len(chunk.encode("utf-8")) <= 1900 for chunk in chunks)diff --git a/tests/test_markdown_escape_delivery.py b/tests/test_markdown_escape_delivery.pynew file mode 100644index 000000000..05fb96eb8--- /dev/null+++ b/tests/test_markdown_escape_delivery.py@@ -0,0 +1,141 @@+"""What a CommonMark backslash escape becomes on each IM platform.++One Markdown text is delivered everywhere, and an escape in it means exactly+one thing: show this character, do not read it as syntax. No IM dialect knows+that. Telegram and Slack read the escaped character as markup of their own -+Slack's converter turns an escaped ``*`` into ``_``, so the reader is shown a+different character than the writer wrote - and WeChat, whose renderer is a+sequence of regex passes, lets an escaped ``]`` close a link label early and+leaks the raw Markdown around it.++So the escape is resolved once, in the shared layer, before any platform pass+runs, and the character it protected is held behind a placeholder until that+pass is done. Every case below is the same question on three dialects: does the+reader see the character, and nothing else?+"""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++from modules.im.formatters.telegram_formatter import TelegramFormatter+from modules.im.formatters.wechat_formatter import WeChatFormatter+from modules.im.slack import SlackBot, SlackMarkdownConverter+from modules.im.wechat import WeChatBot++LINK = "https://s.example/p"+++def telegram(text: str) -> str:+ return TelegramFormatter().render(text)+++def slack(text: str) -> str:+ # The conversion is the whole unit under test; the rest of the client is+ # network plumbing this never reaches.+ bot = SlackBot.__new__(SlackBot)+ bot.markdown_converter = SlackMarkdownConverter()+ return bot._convert_markdown_to_slack_mrkdwn(text)+++def wechat(text: str) -> str:+ bot = WeChatBot.__new__(WeChatBot)+ bot.formatter = WeChatFormatter()+ return bot.format_markdown(text)+++DIALECTS = pytest.mark.parametrize(+ "render", [telegram, slack, wechat], ids=["telegram", "slack", "wechat"]+)+++class TestEscapedProse:+ """An escape the agent wrote in ordinary prose, which predates citations."""++ @DIALECTS+ @pytest.mark.parametrize(+ "raw, shown",+ [+ (r"plain \*literal\* text", "plain *literal* text"),+ (r"a \_b\_ c", "a _b_ c"),+ (r"tick \` tick", "tick ` tick"),+ (r"a \~b\~ c", "a ~b~ c"),+ (r"a \\ b", "a \\ b"),+ ],+ ids=["asterisk", "underscore", "backtick", "tilde", "backslash"],+ )+ def test_the_reader_sees_the_character_and_no_backslash(self, render, raw, shown):+ assert render(raw) == shown++ @DIALECTS+ def test_an_escape_inside_a_code_span_is_still_code(self, render):+ """A backslash is not an escape inside code, so it stays a backslash."""+ assert "\\_" in render(r"`a \_ b`")+++class TestEscapedLinkLabel:+ """A label the citation rewrite escaped, which is why the escapes are there.++ The rewrite escapes the characters that would otherwise let a label swallow+ the link around it. That is a CommonMark statement, so every platform that+ does not speak CommonMark has to be told the same thing in its own terms -+ and the link has to survive the telling.+ """++ @pytest.mark.parametrize(+ "raw, label",+ [+ (r"[a\]b](%s) tail" % LINK, "a]b"),+ (r"[ex\`ample.com](%s) tail" % LINK, "ex`ample.com"),+ (r"[a\*b](%s) tail" % LINK, "a*b"),+ (r"[a\[b](%s) tail" % LINK, "a[b"),+ ],+ ids=["bracket", "backtick", "asterisk", "open-bracket"],+ )+ def test_telegram_links_the_label_verbatim(self, raw, label):+ assert telegram(raw) == f'<a href="{LINK}">{label}</a> tail'++ @pytest.mark.parametrize(+ "raw, label",+ [+ (r"[a\]b](%s) tail" % LINK, "a]b"),+ (r"[ex\`ample.com](%s) tail" % LINK, "ex`ample.com"),+ (r"[a\*b](%s) tail" % LINK, "a*b"),+ (r"[a\[b](%s) tail" % LINK, "a[b"),+ ],+ ids=["bracket", "backtick", "asterisk", "open-bracket"],+ )+ def test_slack_links_the_label_verbatim(self, raw, label):+ assert slack(raw) == f"<{LINK}|{label}> tail"++ @pytest.mark.parametrize(+ "raw, label",+ [+ (r"[a\]b](%s) tail" % LINK, "a]b"),+ (r"[ex\`ample.com](%s) tail" % LINK, "ex`ample.com"),+ (r"[a\*b](%s) tail" % LINK, "a*b"),+ (r"[a\[b](%s) tail" % LINK, "a[b"),+ ],+ ids=["bracket", "backtick", "asterisk", "open-bracket"],+ )+ def test_wechat_shows_the_label_and_the_address_it_cannot_link(self, raw, label):+ """WeChat has no links, so attribution survives as label plus address."""+ assert wechat(raw) == f"{label} ({LINK}) tail"++ @DIALECTS+ def test_an_escaped_backtick_in_a_label_does_not_take_the_code_after_it(self, render):+ """The defect the escape exists to prevent, seen on each platform.++ Unescaped, the backtick opens a code span that runs past ``](url)`` to+ the next one and the link is never parsed at all.+ """+ rendered = render(rf"[ex\`ample.com]({LINK}) and `code` prose.")++ assert "ex`ample.com" in rendered+ assert LINK in rendered+ assert "](" not in rendereddiff --git a/tests/test_message_dispatcher_platform_limits.py b/tests/test_message_dispatcher_platform_limits.pyindex 7d4a648c2..6565a64dc 100644--- a/tests/test_message_dispatcher_platform_limits.py+++ b/tests/test_message_dispatcher_platform_limits.py@@ -25,7 +25,10 @@ class _StubIMClient:def supports_message_editing(self, context=None):return self._supports_editing- async def send_message(self, context, text, parse_mode=None, reply_to=None):+ async def send_message(self, context, text, parse_mode=None, reply_to=None, subtext=None):+ # ``subtext`` is part of the platform interface (a de-emphasized footer+ # Slack and Discord render below the body); a stub that rejected it+ # would fail every send the dispatcher makes for a result with a footer.if self._max_bytes is not None and len(text.encode("utf-8")) > self._max_bytes:raise RuntimeError("message too large")self.sent.append((context.channel_id, context.thread_id, text, parse_mode))diff --git a/tests/test_message_mirror.py b/tests/test_message_mirror.pyindex 405a5d062..8b34df3e4 100644--- a/tests/test_message_mirror.py+++ b/tests/test_message_mirror.py@@ -26,6 +26,12 @@ from sqlalchemy import selectsys.path.insert(0, str(Path(__file__).resolve().parents[1]))+from core.citations import (+ CitationSource,+ body_digest,+ finalize_citations,+ register_citations,+)from core.message_mirror import (agent_message_exists,mirror_harness_inbound,@@ -176,6 +182,123 @@ def test_persist_agent_keeps_result_footer_as_structured_content(isolated_state)assert content["result_footer"] == "✅ ⏱️ 5s · 🪙 1.2k tok"+# The wire shape the backend produces, and the source it resolves against —+# a registered answer rather than a hand-written sidecar, because the rows this+# test reads back are only ever written from one.+CITED_MARKER = "\ue200cite\ue202turn0view0\ue201"+CITED_LINK = "[developers.openai.com](https://developers.openai.com/x)"+++def _registered(text: str, title: str = "Web search — OpenAI API"):+ source = CitationSource(+ ref_id="turn0view0", title=title, url="https://developers.openai.com/x"+ )+ return register_citations(+ text, {source.ref_id: source}, unresolved_label="(source unavailable)"+ )+++def test_persist_agent_keeps_citations_as_structured_content(isolated_state):+ """The sidecar rides the row, so it is not lost when the delivery is."""+ ctx = _slack_ctx()+ mirror_inbound(ctx, "ping")+ registered, bundle = _registered(+ f"Native search exists.{CITED_MARKER}", title="Web search — OpenAI API 中文"+ )+ persist_agent_message(ctx, "result", registered, citations=bundle)++ engine = create_sqlite_engine()+ with engine.connect() as conn:+ row = conn.execute(+ select(messages).where(messages.c.author == "agent")+ ).mappings().one()++ content = json.loads(row["content_json"])+ # The row is written from the finalized body: the text keeps the plain+ # Markdown links every surface already renders, and the sidecar describes+ # THAT text - never a version of it the reader cannot see.+ body = row["content_text"]+ assert body == f"Native search exists. {CITED_LINK}"+ assert content["citations"] == [+ {+ "index": 1,+ "ref_id": "turn0view0",+ "title": "Web search — OpenAI API 中文",+ "url": "https://developers.openai.com/x",+ "label": "developers.openai.com",+ "spans": [[body.index(CITED_LINK), len(body)]],+ "body_sha256": body_digest(body),+ }+ ]+++def test_persist_agent_without_citations_writes_no_sidecar_key(isolated_state):+ ctx = _slack_ctx()+ mirror_inbound(ctx, "ping")+ persist_agent_message(ctx, "result", "pong", citations=None)++ engine = create_sqlite_engine()+ with engine.connect() as conn:+ row = conn.execute(+ select(messages).where(messages.c.author == "agent")+ ).mappings().one()++ assert "citations" not in json.loads(row["content_json"])+++def test_citations_survive_reload_on_the_answer_they_belong_to(isolated_state):+ """Streamed live and re-read after a restart must agree, per message."""+ from core import inbox_events+ from storage import messages_service++ registered, bundle = _registered(f"Cited answer.{CITED_MARKER}")+ cited_body, citations = finalize_citations(registered, bundle)+ engine = create_sqlite_engine()+ now = "2026-05-30T12:00:00Z"+ with engine.begin() as conn:+ scope_id = upsert_scope(+ conn, platform="avibe", scope_type="project", native_id="proj_cite", now=now+ )+ conn.execute(+ agent_sessions.insert().values(+ id="ses_cite", scope_id=scope_id, agent_backend="codex", agent_variant="default",+ session_anchor="anchor_ses_cite", native_session_id="", status="active",+ metadata_json="{}", created_at=now, updated_at=now, last_active_at=now,+ )+ )++ ctx = MessageContext(+ user_id="workbench", channel_id="ses_cite", platform="avibe",+ platform_specific={"agent_session_id": "ses_cite"},+ )++ async def scenario():+ sub_id, queue = inbox_events.bus.subscribe()+ try:+ persist_agent_message(ctx, "output", "Uncited answer.")+ first = await _drain_published(queue)+ persist_agent_message(ctx, "result", registered, citations=bundle)+ return first, await _drain_published(queue)+ finally:+ inbox_events.bus.unsubscribe(sub_id)++ uncited_events, cited_events = asyncio.run(scenario())++ # Live: the open browser gets the sidecar on the event, on that message only.+ assert cited_events["message.new"]["content"]["citations"] == citations+ assert "citations" not in (uncited_events["message.new"]["content"] or {})++ # Reload: the same association comes back off disk, unattached to its sibling.+ with engine.connect() as conn:+ transcript = messages_service.list_session_messages(conn, session_id="ses_cite")+ assert [+ (row["text"], (row["content"] or {}).get("citations")) for row in transcript["messages"]+ ] == [+ ("Uncited answer.", None),+ (cited_body, citations),+ ]++def test_agent_message_receipt_lookup_returns_text_footer_and_batch(isolated_state):ctx = _slack_ctx()mirror_inbound(ctx, "ping")diff --git a/ui/e2e/citations/README.md b/ui/e2e/citations/README.mdnew file mode 100644index 000000000..37a778c29--- /dev/null+++ b/ui/e2e/citations/README.md@@ -0,0 +1,26 @@+# Source Citation Regression++Run `npm run test:citations` from `ui/` after installing Chromium and WebKit with+`npx playwright install --with-deps chromium webkit`.++The fixture renders the production `Markdown` renderer with a citation sidecar+shaped exactly as the agent-reply bubble passes it, using real CSS, the real+`Popover`, and the shipped i18n bundles. No Avibe service is used and every cited+destination is served by a local route stub. English and Chinese checks run on+desktop Chromium, mobile Chromium, and mobile WebKit.++They cover what a browser has to answer and jsdom cannot: badge height against a+reference `Badge`, three citations in one sentence staying on one line, hover+preview on desktop, first-tap-reveals / second-tap-opens on touch, keyboard focus+revealing the preview and `Enter` opening the source in a new page, the marker+grammar staying literal inside a code example, an unresolved ref staying plain+prose, and the count of requests to the cited domains — zero until the reader+explicitly opens the page, which is how the "no automatic page, favicon, or+metadata fetching" constraint is asserted instead of assumed.++Screenshots and failure traces go under `e2e/.artifacts/citations/`.++The renderer's own matching rules (which links become badges, which stay ordinary+anchors, and how a malformed stored sidecar degrades) are covered in+`src/components/ui/markdown.test.tsx`. Full-service local Incus acceptance and+native-device touch checks remain separate manual validation.diff --git a/ui/e2e/citations/citations.spec.ts b/ui/e2e/citations/citations.spec.tsnew file mode 100644index 000000000..82b98b26c--- /dev/null+++ b/ui/e2e/citations/citations.spec.ts@@ -0,0 +1,160 @@+import { expect, test } from '@playwright/test';+import { copy } from '../support/copy';++// What a browser has to answer that jsdom cannot: do several source badges stay+// on one line, does a mouse hover and a finger tap each reach the preview, is+// the original page reachable from the keyboard, and — the constraint this+// feature was given — does rendering a citation fetch nothing from the cited+// site until the reader explicitly asks for it.+const GUIDE = 'https://developers.openai.com/api/docs/guides/tools-web-search';+const SOURCES = [+ { index: 1, title: 'Web search — OpenAI API', label: 'developers.openai.com' },+ { index: 2, title: '引用探针来源 — Café', label: 'example.com' },+ // A search that returned no title is attributed by its domain instead.+ { index: 3, title: 'reference.invalid', label: 'reference.invalid' },+];++for (const locale of ['en', 'zh'] as const) {+ test.describe(locale, () => {+ let citedSiteRequests = 0;++ test.beforeEach(async ({ page, context }) => {+ citedSiteRequests = 0;+ await page.addInitScript((language) => localStorage.setItem('i18nextLng', language), locale);+ // Every cited destination is served locally. A real request would make this+ // suite depend on the open internet; counting them is also how the+ // "nothing is fetched" property below is asserted rather than assumed.+ await context.route(/developers\.openai\.com|example\.com|reference\.invalid/, (route) => {+ citedSiteRequests += 1;+ return route.fulfill({ contentType: 'text/html', body: '<title>Cited source stub</title>' });+ });+ await page.goto('/e2e/citations/fixture.html');+ });++ const badge = (page: import('@playwright/test').Page, source: typeof SOURCES[number]) =>+ page.getByRole('link', {+ name: copy('chat.citation.badge', { index: source.index, source: source.title }, locale),+ exact: true,+ }).first();++ test(`badges stay compact and inline (${locale})`, async ({ page }, testInfo) => {+ const reference = (await page.getByText('Reference badge', { exact: true }).boundingBox())!;++ for (const source of SOURCES) {+ const bounds = (await badge(page, source).boundingBox())!;+ expect(bounds.height).toBeLessThanOrEqual(reference.height);+ expect(bounds.height).toBeLessThan(30);+ }++ // The property that makes them "compact": three sources in one sentence+ // leave that line exactly as tall as an uncited line of the same answer.+ const plain = (await page.getByText('Plain answer line without any citation.').boundingBox())!;+ const row = (await page.getByText('Three in a row.').boundingBox())!;+ expect(row.height).toBeCloseTo(plain.height, 0);++ // The grammar quoted in a code example is text, not a badge.+ await expect(page.locator('pre code')).toContainText('cite');+ await expect(page.locator('pre a')).toHaveCount(0);+ // An unresolved ref keeps its label as plain prose — never as a link.+ await expect(page.getByText('(source unavailable)')).toBeVisible();+ await expect(page.getByRole('link', { name: /source unavailable/ })).toHaveCount(0);+ // A link the answer worded itself points at source 1's page and is still+ // not a badge: the attribution belongs to the link the backend wrote, not+ // to every link that happens to go there.+ const prose = page.getByRole('link', { name: 'the web search guide', exact: true });+ await expect(prose).toHaveAttribute('href', GUIDE);+ await expect(prose).not.toHaveAttribute('data-citation-index', /.*/);+ await expect(page.locator('a[data-citation-index]')).toHaveCount(5);++ await page.screenshot({ path: testInfo.outputPath(`compact-${locale}.png`), scale: 'css' });+ });++ test(`the cited page is reachable and nothing is fetched before that (${locale})`, async ({+ page, context, isMobile,+ }, testInfo) => {+ const first = badge(page, SOURCES[0]);+ await expect(first).toBeVisible();++ if (isMobile) {+ // No hover on a phone: the first tap must reveal the preview instead of+ // navigating, or the reader can never see what they are about to open.+ await first.tap();+ } else {+ await first.hover();+ }+ await expect(page.getByText(SOURCES[0].title, { exact: true })).toBeVisible();+ await expect(page.getByText(SOURCES[0].label, { exact: true }).last()).toBeVisible();+ await expect(page).toHaveURL(/fixture\.html$/);+ // `animations: 'disabled'` finishes the panel's fade-in first, so the shot+ // shows the settled preview instead of whatever opacity it was passing through.+ await page.screenshot({ path: testInfo.outputPath(`preview-${locale}.png`), scale: 'css', animations: 'disabled' });++ // Rendering and previewing a citation reads only what the backend already+ // sent: no page, no favicon, no metadata request to the cited site.+ expect(citedSiteRequests).toBe(0);++ const opened = context.waitForEvent('page');+ if (isMobile) {+ // …and the next tap follows the link.+ await first.tap();+ } else {+ await page.getByRole('link', { name: copy('chat.citation.open', undefined, locale), exact: true })+ .click();+ }+ const source = await opened;+ await expect(source).toHaveURL(GUIDE);+ expect(citedSiteRequests).toBe(1);+ await source.close();+ });++ test(`a tap on a badge never reaches its neighbour (${locale})`, async ({ page, isMobile }) => {+ test.skip(!isMobile, 'the touch hit area only exists below the md breakpoint');+ // Three chips separated by one space each. The mobile hit area has to be+ // big enough to tap and small enough that it stays inside the space the+ // chip owns - a box that reaches its neighbour opens the WRONG source,+ // which is worse than a small target.+ const row = page.locator('p', { hasText: 'Three in a row.' }).last();+ const chips = row.getByRole('link');+ await expect(chips).toHaveCount(3);++ const owners = await row.evaluate((node) => {+ const badges = Array.from(node.querySelectorAll('a[data-citation-index]')) as HTMLElement[];+ return badges.flatMap((badge) => {+ const box = badge.getBoundingClientRect();+ const points: Array<[number, number]> = [+ [box.left + 1, box.top + box.height / 2],+ [box.right - 1, box.top + box.height / 2],+ [box.left + box.width / 2, box.top + 1],+ [box.left + box.width / 2, box.bottom - 1],+ ];+ return points.map(([x, y]) => {+ const hit = document.elementFromPoint(x, y) as HTMLElement | null;+ const owner = hit?.closest('a[data-citation-index]') as HTMLElement | null;+ return `${badge.dataset.citationIndex}->${owner?.dataset.citationIndex ?? 'none'}`;+ });+ });+ });+ expect(owners.filter((pair) => pair !== `${pair.split('->')[0]}->${pair.split('->')[0]}`)).toEqual([]);++ // …and the edge a neighbour used to cover still opens its own source.+ const box = (await chips.first().boundingBox())!;+ await chips.first().tap({ position: { x: box.width - 1, y: box.height / 2 } });+ await expect(page.getByText(SOURCES[0].title, { exact: true })).toBeVisible();+ await expect(page.getByText(SOURCES[1].title, { exact: true })).toHaveCount(0);+ });++ test(`a badge opens its source from the keyboard (${locale})`, async ({ page, context }) => {+ const first = badge(page, SOURCES[0]);+ await first.focus();++ // Focus alone reveals the preview, and focus stays on the badge — so the+ // portalled panel never has to be tabbed into to reach the page.+ await expect(page.getByText(SOURCES[0].title, { exact: true })).toBeVisible();+ await expect(first).toBeFocused();++ const opened = context.waitForEvent('page');+ await page.keyboard.press('Enter');+ await expect(await opened).toHaveURL(GUIDE);+ });+ });+}diff --git a/ui/e2e/citations/fixture.html b/ui/e2e/citations/fixture.htmlnew file mode 100644index 000000000..c922eafc4--- /dev/null+++ b/ui/e2e/citations/fixture.html@@ -0,0 +1,12 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="UTF-8" />+ <meta name="viewport" content="width=device-width, initial-scale=1" />+ <title>Source Citation Regression</title>+ </head>+ <body>+ <div id="root"></div>+ <script type="module" src="./fixture.tsx"></script>+ </body>+</html>diff --git a/ui/e2e/citations/fixture.tsx b/ui/e2e/citations/fixture.tsxnew file mode 100644index 000000000..a2253e28c--- /dev/null+++ b/ui/e2e/citations/fixture.tsx@@ -0,0 +1,79 @@+import { StrictMode } from 'react';+import { createRoot } from 'react-dom/client';+import { Badge } from '../../src/components/ui/badge';+import { Markdown } from '../../src/components/ui/markdown';+import { bindCitations, bodyDigest, type CitationSource } from '../../src/lib/citations';+import '../../src/i18n';+import '../../src/index.css';++// The production Markdown renderer with a citation sidecar, exactly as the+// agent-reply bubble passes it (ChatPage: agent-authored rows only). Nothing is+// mocked but the message itself — real CSS, real Popover, real i18n — because+// what this fixture exists to check is browser layout and pointer routing, which+// jsdom cannot answer.+const GUIDE = 'https://developers.openai.com/api/docs/guides/tools-web-search';+const PROBE = 'https://example.com/citation-probe-source';+const REFERENCE = 'https://reference.invalid/three';++const SOURCES = [+ { index: 1, ref_id: 'turn0view0', title: 'Web search — OpenAI API', url: GUIDE, label: 'developers.openai.com' },+ { index: 2, ref_id: 'turn0view1', title: '引用探针来源 — Café', url: PROBE, label: 'example.com' },+ { index: 3, ref_id: 'turn0view2', title: '', url: REFERENCE, label: 'reference.invalid' },+] as const satisfies readonly CitationSource[];++// The answer as core.citations delivers it, assembled the way core.citations+// assembles it: a source in this list is a link the backend wrote, and it+// records the exact range it wrote it at. Everything else is the answer's own+// prose — including the sentence in its own words that points at the same page+// as source 1, which must stay an ordinary anchor no matter how closely it+// matches, and the marker grammar left literal inside a code example.+//+// Paragraph-separated on purpose: a cited line and an uncited one are then two+// comparable blocks, which is how the spec asserts that citing a source does not+// make the answer's lines taller.+const PARTS: Array<string | CitationSource> = [+ 'Plain answer line without any citation.\n\n',+ 'Native web search is documented. ', SOURCES[0], '\n\n',+ 'Two sources, one unknown. ', SOURCES[1], ' (source unavailable)\n\n',+ 'Three in a row. ', SOURCES[0], ' ', SOURCES[1], ' ', SOURCES[2], '\n\n',+ `In the answer's own words, see [the web search guide](${GUIDE}).\n\n`,+ 'The marker grammar itself:\n\n',+ '```\n\uE200cite\uE202turn0view0\uE201\n```',+];++const build = () => {+ let text = '';+ const measured = new Map<CitationSource, number[][]>();+ for (const part of PARTS) {+ if (typeof part === 'string') {+ text += part;+ continue;+ }+ const start = text.length;+ text += `[${part.label}](${part.url})`;+ measured.set(part, [...(measured.get(part) ?? []), [start, text.length]]);+ }+ const body_sha256 = bodyDigest(text);+ return {+ text,+ citations: [...measured].map(([source, spans]) => ({ ...source, spans, body_sha256 })),+ };+};++const { text: ANSWER, citations } = build();+// ChatPage reads the sidecar against the body it was measured in and hands the+// renderer the result; the fixture goes through that same door.+const BINDING = bindCitations(citations, ANSWER);++export function Fixture() {+ return <div className="min-h-dvh bg-background p-4 text-foreground">+ <div className="mx-auto max-w-2xl space-y-4">+ <Badge variant="secondary">Reference badge</Badge>+ <div className="rounded-2xl border border-border bg-surface p-3 text-[13px] leading-relaxed">+ <Markdown content={ANSWER} citations={BINDING} className="vr-markdown--inherit-size" />+ </div>+ </div>+ </div>;+}++createRoot(document.getElementById('root')!).render(<StrictMode><Fixture /></StrictMode>);diff --git a/ui/e2e/citations/tsconfig.json b/ui/e2e/citations/tsconfig.jsonnew file mode 100644index 000000000..bbf316cee--- /dev/null+++ b/ui/e2e/citations/tsconfig.json@@ -0,0 +1,16 @@+{+ "extends": "../../tsconfig.app.json",+ "compilerOptions": {+ "types": [+ "vite/client",+ "node"+ ]+ },+ "include": [+ "*.ts",+ "*.tsx",+ "../../playwright.citations.config.ts",+ "../../agentation.d.ts"+ ],+ "exclude": []+}diff --git a/ui/package-lock.json b/ui/package-lock.jsonindex e17c19581..fb0497ce8 100644--- a/ui/package-lock.json+++ b/ui/package-lock.json@@ -17,6 +17,7 @@"@lexical/react": "^0.45.0","@monaco-editor/react": "^4.7.0","@noble/curves": "^2.2.0",+ "@noble/hashes": "^2.2.0","@radix-ui/react-dialog": "^1.1.15","@radix-ui/react-label": "^2.1.8","@radix-ui/react-popover": "^1.1.15",diff --git a/ui/package.json b/ui/package.jsonindex f6784fac0..9a570eace 100644--- a/ui/package.json+++ b/ui/package.json@@ -20,6 +20,7 @@"e2e": "playwright test","e2e:headed": "playwright test --headed","test:chat-paging": "tsc -p e2e/chat-paging/tsconfig.json --noEmit && playwright test --config playwright.chat-paging.config.ts",+ "test:citations": "tsc -p e2e/citations/tsconfig.json --noEmit && playwright test --config playwright.citations.config.ts","test:inbox-return": "tsc -p e2e/inbox-return/tsconfig.json --noEmit && playwright test --config playwright.inbox-return.config.ts","test:badge-triggers": "tsc -p e2e/badge-triggers/tsconfig.json --noEmit && playwright test --config playwright.badge-triggers.config.ts","test:project-order": "tsc -p e2e/project-order/tsconfig.json --noEmit && playwright test --config playwright.project-order.config.ts",@@ -43,6 +44,7 @@"@lexical/react": "^0.45.0","@monaco-editor/react": "^4.7.0","@noble/curves": "^2.2.0",+ "@noble/hashes": "^2.2.0","@radix-ui/react-dialog": "^1.1.15","@radix-ui/react-label": "^2.1.8","@radix-ui/react-popover": "^1.1.15",diff --git a/ui/playwright.citations.config.ts b/ui/playwright.citations.config.tsnew file mode 100644index 000000000..36c545f53--- /dev/null+++ b/ui/playwright.citations.config.ts@@ -0,0 +1,19 @@+import { defineConfig, devices } from '@playwright/test';++export default defineConfig({+ testDir: './e2e/citations',+ outputDir: './e2e/.artifacts/citations',+ workers: 1,+ retries: 0,+ use: { baseURL: 'http://127.0.0.1:5218', locale: 'en-US', trace: 'retain-on-failure' },+ webServer: {+ command: 'npm run dev -- --host 127.0.0.1 --port 5218 --strictPort',+ env: { VIBE_UI_BACKEND: 'http://127.0.0.1:9' },+ url: 'http://127.0.0.1:5218/e2e/citations/fixture.html',+ },+ projects: [+ { name: 'desktop', use: { ...devices['Desktop Chrome'] } },+ { name: 'mobile', use: { ...devices['iPhone 13'], browserName: 'chromium' } },+ { name: 'mobile-webkit', use: { ...devices['iPhone 13'] } },+ ],+});diff --git a/ui/src/components/ui/badge-variants.ts b/ui/src/components/ui/badge-variants.tsindex 8d6ecc909..72c06232f 100644--- a/ui/src/components/ui/badge-variants.ts+++ b/ui/src/components/ui/badge-variants.ts@@ -34,6 +34,17 @@ export const badgeVariants = cva(export const interactiveBadgeTriggerClassName ="relative cursor-pointer justify-center before:absolute before:left-1/2 before:top-1/2 before:h-full before:min-h-11 before:w-full before:min-w-11 before:-translate-x-1/2 before:-translate-y-1/2 before:content-[''] md:before:hidden";+// The same idea for a badge that lives INSIDE a sentence. An inline chip cannot+// claim 44px without taking taps from the words and chips around it - a citation+// run is separated by a single space, and a 44px box reaches into the prose lines+// above and below - so its hit area grows only into the space it actually owns:+// the line box, and half the gap to its neighbour. WCAG 2.2 SC 2.5.8 exempts+// exactly this case ("the target is in a sentence ... constrained by the+// line-height of non-target text"); a target that opens the WRONG source has no+// exemption.+export const inlineBadgeTriggerClassName =+ "relative cursor-pointer justify-center before:absolute before:inset-x-[-2px] before:inset-y-[-3px] before:content-[''] md:before:hidden";+// Mobile badge popovers sit below shell headers and must reserve the notch area// in both their origin and their available height.export const mobileHeaderPopoverClassName =diff --git a/ui/src/components/ui/citation-badge.tsx b/ui/src/components/ui/citation-badge.tsxnew file mode 100644index 000000000..953a76435--- /dev/null+++ b/ui/src/components/ui/citation-badge.tsx@@ -0,0 +1,134 @@+import * as React from 'react';+import { useTranslation } from 'react-i18next';+import { ExternalLink } from 'lucide-react';++import { badgeVariants, inlineBadgeTriggerClassName } from './badge-variants';+import { Popover, PopoverAnchor, PopoverContent } from './popover';+import type { CitationSource } from '@/lib/citations';+import { cn } from '@/lib/utils';++// One numbered source badge for a citation in an agent answer.+//+// The badge IS the link, which is what keeps it usable without trapping focus:+// focus reveals the preview and Enter opens the page, so the portalled panel+// never has to be tabbed into, and its `aria-label` already carries the full+// attribution — the panel is what a sighted reader hovers for, not where the+// information only lives. (InfoHint can't be reused here for exactly that+// reason: its trigger is a <button>, so a link in its panel would be the only+// way out and keyboard-unreachable.)+//+// A mouse hovers to preview and clicks to open. Touch has no hover, so the FIRST+// tap reveals the preview instead of navigating and every later tap follows the+// link; the panel repeats the destination as an explicit action so the way out+// is never a guess. Non-modal on purpose — an inline citation sits in transcript+// prose, not inside a Dialog.+export const CitationBadge: React.FC<{ citation: CitationSource; className?: string }> = ({+ citation,+ className,+}) => {+ const { t } = useTranslation();+ const [open, setOpen] = React.useState(false);+ const pointerType = React.useRef<string>('mouse');+ // Whether this badge has already shown its preview once. Deliberately a ref+ // and not the `open` state: an outside tap closes the panel on pointerdown+ // before the click lands, so a second tap reading `open` would re-open instead+ // of navigating and the page would be unreachable on a phone.+ const revealed = React.useRef(false);+ // Hover-close is deferred so the pointer can travel from the badge into the+ // panel — which is portalled, so no wrapper can hold both.+ const closeTimer = React.useRef<number | null>(null);+ const cancelClose = React.useCallback(() => {+ if (closeTimer.current !== null) {+ window.clearTimeout(closeTimer.current);+ closeTimer.current = null;+ }+ }, []);+ const scheduleClose = React.useCallback(() => {+ cancelClose();+ closeTimer.current = window.setTimeout(() => setOpen(false), 120);+ }, [cancelClose]);+ React.useEffect(() => cancelClose, [cancelClose]);++ // A source whose search returned no title is attributed by its domain, which+ // is the one thing every citation has.+ const title = citation.title?.trim() || citation.label;+ // The title is whatever the search result said about itself; the domain is+ // where the link actually goes. A screen reader hears both, so the+ // destination is never something only the hover preview shows.+ const accessibleName = title === citation.label+ ? t('chat.citation.badge', { index: citation.index, source: title })+ : t('chat.citation.badgeWithDomain', {+ index: citation.index,+ source: title,+ domain: citation.label,+ });++ return (+ <Popover open={open} onOpenChange={setOpen}>+ <PopoverAnchor asChild>+ <a+ href={citation.url}+ target="_blank"+ rel="noopener noreferrer nofollow"+ aria-label={accessibleName}+ data-citation-index={citation.index}+ onPointerDown={(event) => { pointerType.current = event.pointerType; }}+ // Only a mouse hovers; a tap reports pointerType 'touch' and is left to+ // the click handler, so a phone gets one reveal instead of open-then-close.+ onPointerEnter={(event) => {+ if (event.pointerType === 'mouse') { cancelClose(); setOpen(true); }+ }}+ onPointerLeave={(event) => {+ if (event.pointerType === 'mouse') scheduleClose();+ }}+ onFocus={() => { cancelClose(); setOpen(true); }}+ onBlur={scheduleClose}+ onClick={(event) => {+ if (pointerType.current === 'touch' && !revealed.current) {+ event.preventDefault();+ revealed.current = true;+ cancelClose();+ setOpen(true);+ }+ }}+ className={cn(+ badgeVariants({ variant: 'secondary' }),+ inlineBadgeTriggerClassName,+ // Pin the chip's own leading. It is baseline-aligned inline content, so+ // its box has to fit inside the prose line box or every line carrying a+ // citation grows taller than the uncited lines around it; inheriting+ // `.vr-markdown`'s 1.6 makes it do exactly that. (Same reason the+ // `recommendation` badge variant fixes its own leading.)+ 'vr-citation align-baseline px-1.5 font-mono leading-none tabular-nums focus-visible:ring-1 focus-visible:ring-cyan/60',+ className,+ )}+ >+ {citation.index}+ </a>+ </PopoverAnchor>+ <PopoverContent+ align="start"+ sideOffset={6}+ collisionPadding={12}+ // Don't steal focus from the badge on hover-open: the badge is what opens+ // the page, and moving focus into the portal would break tab order.+ onOpenAutoFocus={(event) => event.preventDefault()}+ onPointerEnter={cancelClose}+ onPointerLeave={scheduleClose}+ className="w-64 space-y-1.5 p-3"+ >+ <p className="text-[12px] font-medium leading-snug text-foreground">{title}</p>+ <p className="truncate text-[11px] leading-snug text-muted">{citation.label}</p>+ <a+ href={citation.url}+ target="_blank"+ rel="noopener noreferrer nofollow"+ className="inline-flex items-center gap-1 text-[11px] font-medium text-cyan-ink hover:underline"+ >+ {t('chat.citation.open')}+ <ExternalLink className="size-3" aria-hidden="true" />+ </a>+ </PopoverContent>+ </Popover>+ );+};diff --git a/ui/src/components/ui/citation-bridge.test.tsx b/ui/src/components/ui/citation-bridge.test.tsxnew file mode 100644index 000000000..b1259a227--- /dev/null+++ b/ui/src/components/ui/citation-bridge.test.tsx@@ -0,0 +1,410 @@+/* @vitest-environment jsdom */++// The Web half of one shared recording.+//+// `tests/citation_bridge.py` runs the real Python producer and delivery pass+// over every case and writes what they actually emitted - the body the+// workbench is handed, the sidecar beside it, and the links a reader of that+// body must end up with. `tests/test_citation_consumers.py` asks each IM+// platform's renderer what it shows of that. This file asks the real `Markdown`+// component the same question, so the two ends of the contract are measured+// against one producer run instead of against each other's assumptions.+//+// Every assertion here is about what the reader gets: the label shown, the+// address the anchor navigates to, and which link carries the badge. A citation+// that lands on the wrong link is a false attribution, and a badge that is the+// only thing able to reach the source means the plain link underneath it is+// broken - so the sidecar is also removed, mismatched and downgraded to its+// pre-provenance shape, and the answer must stay readable and clickable in all+// four.++import { readFileSync } from 'node:fs';+import { resolve } from 'node:path';++import { createInstance } from 'i18next';+import { cleanup, render } from '@testing-library/react';+import { I18nextProvider, initReactI18next } from 'react-i18next';+import { afterEach, describe, expect, it } from 'vitest';++import en from '@/i18n/en.json';+import { bindCitations, type CitationSource } from '@/lib/citations';+import type { MentionReference } from '@/lib/mentions';+import { RouteSurfaceActiveContext } from '../../lib/routeSurfaceActivity';+import { Markdown } from './markdown';++afterEach(cleanup);++type Anchor = { label: string | null; url: string; cited: boolean };+type BridgeCase = {+ key: string;+ why: string;+ web_text: string;+ web_citations: CitationSource[];+ /** The same reply as IM delivers it, measured in that body. */+ im_citations: CitationSource[];+ web_anchors: Anchor[];+ anchors_exhaustive: boolean;+};++// Read at runtime rather than imported: the fixture is produced by the Python+// suite and lives outside Vite's module graph.+const { cases } = JSON.parse(+ readFileSync(resolve(process.cwd(), '../tests/fixtures/citation_consumer_bridge.json'), 'utf8'),+) as { cases: BridgeCase[] };++const i18n = createInstance();+void i18n.use(initReactI18next).init({+ lng: 'en',+ fallbackLng: 'en',+ resources: { en: { translation: en } },+ interpolation: { escapeValue: false },+});++// Exactly what ChatPage does with a stored row: read the sidecar against the+// body it was measured in, and hand the renderer the result. Passing the raw+// rows and the text separately would let this file decide they match, which is+// the decision under test.+const renderCase = (content: string, citations?: CitationSource[]) => render(+ <I18nextProvider i18n={i18n}>+ <RouteSurfaceActiveContext.Provider value>+ <Markdown content={content} citations={bindCitations(citations, content)} interactive />+ </RouteSurfaceActiveContext.Provider>+ </I18nextProvider>,+);++// Only web links: a `file://` attachment is rewritten to a media-proxy URL+// before the workbench ever renders it, so the raw form recorded here is not a+// destination this component is meant to reach.+const webLinks = (container: HTMLElement) =>+ Array.from(container.querySelectorAll<HTMLAnchorElement>('a[href^="http"]'));++const badged = (anchor: HTMLAnchorElement) => anchor.hasAttribute('data-citation-index');++// A private marker is not an address. Where the model wrote one in a+// destination the producer leaves it exactly as typed - claiming a source+// there would be claiming one no reader can see - and every renderer past that+// point is free to spell those characters as percent escapes. So a rendered+// address is read back through that one spelling before it is compared, and+// through nothing else: a substituted host is still a link nobody wrote.+const asWritten = (href: string | null) => (href ?? '').replace(/(?:%EE%88%8[0-2])+/gi, decodeURIComponent);++// A label holding an image is not a string of text, and the producer says so by+// leaving it unsaid; that link is known by its address alone.+const shows = (links: HTMLAnchorElement[], anchor: Anchor) => links.some(+ (node) => asWritten(node.getAttribute('href')) === anchor.url+ && (anchor.label === null || node.textContent === anchor.label),+);++/** The characters a citation's first span covers, in the body it was measured in. */+const spelling = (row: BridgeCase, entry: CitationSource) => {+ const [start, end] = (entry.spans as number[][])[0];+ // A JavaScript string index is a UTF-16 code unit, which is the unit the+ // backend counted in.+ return row.web_text.slice(start, end);+};++describe('citation bridge: the real renderer, on one real producer run', () => {+ const cited = cases.map((row) => [row.key, row] as const);+ const withSidecar = cited.filter(([, row]) => row.web_citations.length > 0);++ it.each(cited)('%s shows every source the producer cited', (_key, row) => {+ const { container } = renderCase(row.web_text, row.web_citations);+ const links = webLinks(container);+ const expected = row.web_anchors.filter((anchor) => anchor.cited);++ // A badge is the citation's rendering, so it must carry the destination and+ // name the source - the label is shown in its preview, not in the strip of+ // text, which is the whole point of a compact badge.+ const badges = links.filter(badged);+ expect(badges).toHaveLength(expected.length);+ expect(badges.map((anchor) => asWritten(anchor.getAttribute('href')))).toEqual(+ expected.map((anchor) => anchor.url),+ );+ badges.forEach((anchor, at) => {+ const source = row.web_citations.find((entry) => entry.url === expected[at].url);+ const title = source?.title?.trim() || source?.label;+ expect(anchor.getAttribute('aria-label')).toBe(+ title === source?.label+ ? `Source ${source?.index}: ${title}`+ : `Source ${source?.index}: ${title} (${source?.label})`,+ );+ });+ });++ it.each(cited)('%s leaves every link the answer wrote itself ordinary', (_key, row) => {+ const { container } = renderCase(row.web_text, row.web_citations);+ const ordinary = webLinks(container).filter((anchor) => !badged(anchor));+ const expected = row.web_anchors.filter((anchor) => !anchor.cited);++ // The answer's own prose may point at the cited page with the very same+ // link. It is still the answer talking, not the search result, and it has+ // to read as what the agent wrote.+ for (const anchor of expected) {+ expect(shows(ordinary, anchor)).toBe(true);+ }+ if (row.anchors_exhaustive) expect(ordinary).toHaveLength(expected.length);+ });++ it.each(cited.filter(([, row]) => row.anchors_exhaustive))(+ '%s badges the occurrence the producer wrote and no other',+ (_key, row) => {+ const { container } = renderCase(row.web_text, row.web_citations);++ // Position, not just count: three identical links where the middle one is+ // the citation is exactly the case a count cannot tell apart.+ expect(webLinks(container).map((anchor) => ({+ url: asWritten(anchor.getAttribute('href')),+ cited: badged(anchor),+ }))).toEqual(row.web_anchors.map((anchor) => ({ url: anchor.url, cited: anchor.cited })));+ },+ );++ it.each(cited)('%s stays readable and clickable with no sidecar at all', (_key, row) => {+ const { container } = renderCase(row.web_text);+ const links = webLinks(container);++ // An older row, a failed load, a transcript from before the feature: the+ // badge is an enhancement, so its absence costs the reader nothing but the+ // preview. Every link still shows its label and still opens its page.+ expect(links.filter(badged)).toHaveLength(0);+ for (const anchor of row.web_anchors) {+ expect(shows(links, anchor)).toBe(true);+ }+ });++ it.each(withSidecar)(+ '%s refuses to guess when the sidecar no longer matches the text',+ (_key, row) => {+ // The stored text and the stored sidecar disagree - an edit after the row+ // was written, a truncation, a row written for another message. The spans+ // still land on real links here, which is exactly why the digest is what+ // decides: guessing is how an ordinary link gets impersonated.+ const edited = `${row.web_text}\n\nEdited.`;+ const { container } = renderCase(edited, row.web_citations);+ const links = webLinks(container);++ expect(links.filter(badged)).toHaveLength(0);+ for (const anchor of row.web_anchors) {+ expect(shows(links, anchor)).toBe(true);+ }+ },+ );++ it.each(withSidecar)('%s refuses a sidecar it cannot locate in this text', (_key, row) => {+ // Same body, but the measurement is damaged: a span past the end, or a row+ // that names the contract without stating it. One bad row drops the whole+ // set rather than leaving the rest to be matched some other way.+ for (const broken of [+ row.web_citations.map((entry) => ({ ...entry, spans: [[0, row.web_text.length + 1]] })),+ row.web_citations.map((entry) => ({ ...entry, spans: undefined })),+ ]) {+ const { container } = renderCase(row.web_text, broken as CitationSource[]);+ expect(webLinks(container).filter(badged)).toHaveLength(0);+ cleanup();+ }+ });++ it.each(withSidecar)(+ '%s still renders a row persisted before provenance existed',+ (_key, row) => {+ // Rows already on disk carry no measurement at all, and the match they+ // were written for is the exact destination and link text. It is less+ // precise - every identical link in the answer gets a badge - and it+ // keeps working.+ const legacy = row.web_citations.map(+ ({ spans: _spans, body_sha256: _digest, ...rest }) => rest,+ );+ const { container } = renderCase(row.web_text, legacy as CitationSource[]);+ const links = webLinks(container);+ const matches = row.web_anchors.filter((anchor) => legacy.some(+ (entry) => entry.url === anchor.url && entry.label === anchor.label,+ ));++ expect(links.filter(badged).length).toBeGreaterThan(0);+ if (row.anchors_exhaustive) expect(links.filter(badged)).toHaveLength(matches.length);+ },+ );+});++describe('citation bridge: the counterexamples this contract was written for', () => {+ const only = (key: string) => cases.find((row) => row.key === key) as BridgeCase;++ it('reaches a literal IPv6 source whether or not a badge is drawn', () => {+ const row = only('ipv6_port');+ const [url] = row.web_anchors.map((anchor) => anchor.url);++ const withBadge = webLinks(renderCase(row.web_text, row.web_citations).container);+ cleanup();+ const without = webLinks(renderCase(row.web_text).container);++ // The brackets are authority syntax, and percent-encoding them makes an+ // address no browser will open. A badge may not be the only thing that+ // navigates: both anchors end at the same place, and that place is the+ // source.+ expect(withBadge[0].getAttribute('href')).toBe(url);+ expect(without[0].getAttribute('href')).toBe(url);+ expect(new URL(without[0].href).hostname).toBe('[::1]');+ expect(new URL(without[0].href).port).toBe('8443');+ });++ it('keeps a footnote definition ordinary and badges the real citation', () => {+ const row = only('footnote');+ const { container } = renderCase(row.web_text, row.web_citations);+ const links = webLinks(container);++ // Both links point at the same page and only one was written by the+ // backend. Counting parsed links disagreed across the two parsers here and+ // the badge was silently lost; the span the producer measured does not.+ expect(links.filter(badged)).toHaveLength(1);+ expect(links.filter((anchor) => !badged(anchor)).map((anchor) => anchor.textContent))+ .toContain('p');+ });++ it('shows a host spelled with Markdown emphasis as that host', () => {+ const row = only('star_host');+ const { container } = renderCase(row.web_text, row.web_citations);+ const [badge] = webLinks(container);++ expect(badge.getAttribute('href')).toBe('https://a*b*.example/x');+ expect(badge.getAttribute('aria-label')).toContain('Star Host');+ expect(container.querySelector('em')).toBeNull();+ });++ it('shows a host that spells a character reference verbatim', () => {+ const row = only('entity_host');+ cleanup();+ const { container } = renderCase(row.web_text);++ expect(container.querySelector('a')?.textContent).toBe('a©.example');+ });++ it('does not hand the badge to a surviving copy when the citation is deleted', () => {+ const row = only('prose_repeat');+ const [entry] = row.web_citations;+ const trimmed = row.web_text.replace(`Cited. ${spelling(row, entry)}\n\n`, '');++ const { container } = renderCase(trimmed, row.web_citations);+ const links = webLinks(container);++ // An edit that drops the sentence, or a quotation that copies only part of+ // the answer: what is left is the agent's own prose pointing at the same+ // page, character for character the citation's own spelling, and shifted+ // into the range it used to occupy. Inheriting the badge would credit the+ // search result for what the answer said, so the sidecar and the text+ // disagreeing costs the reader the preview and nothing else.+ expect(trimmed).not.toContain('Cited.');+ expect(trimmed.split(spelling(row, entry))).toHaveLength(3);+ expect(links).toHaveLength(2);+ expect(links.filter(badged)).toHaveLength(0);+ expect(links.map((anchor) => anchor.getAttribute('href'))).toEqual([entry.url, entry.url]);+ });++ it('does not read one surface of a reply with the other surface\'s measurement', () => {+ // One producer run writes two bodies - the workbench keeps the `file://`+ // attachment and IM flattens it to a label - so the same citation sits at+ // [63, 99) in one and [15, 51) in the other. Nothing in a bare range says+ // which body it was counted in; the digest does, and it is the only reason+ // a row cannot be read against the wrong one.+ const row = only('file_link_above');++ expect(row.im_citations[0].spans).not.toEqual(row.web_citations[0].spans);+ expect(webLinks(renderCase(row.web_text, row.im_citations).container).filter(badged))+ .toHaveLength(0);+ cleanup();+ expect(webLinks(renderCase(row.web_text, row.web_citations).container).filter(badged))+ .toHaveLength(1);+ });+});++// ChatPage hands ONE `Markdown` the citation sidecar, the mention sidecar and+// the secret-request opt-in together, and the last two take effect by REWRITING+// the source text before Markdown parses it: `$<NAME>` becomes a secure-input+// card, `@<…>` / `#<…>` become chips. That moves the very link the producer+// counted, which is the one thing a positional contract can be silently broken+// by — so every case is rendered a second time through that production+// combination, and the `marker_shaped` case, whose own producer-written body+// carries all three markers before the citation, is where the shift is real.+//+// The markers are not pasted around the recorded bodies: adding characters+// makes a different body, the digest says so, and the run would prove only that+// a refusal is a refusal. The rewrite has to be measured on a body the producer+// actually wrote.+//+// The archived-transcript card is the one drawn here because it needs no live+// vault client. The rewrite under test runs before either card exists.+const REFERENCES: MentionReference[] = [+ { kind: 'agent', name: 'claude' },+ { kind: 'session', session_id: 'ses6jr7c5h2q6', title: 'Bridge' },+];++const renderWithSurfaceRewrites = (content: string, citations?: CitationSource[]) => render(+ <I18nextProvider i18n={i18n}>+ <RouteSurfaceActiveContext.Provider value>+ <Markdown+ content={content}+ citations={bindCitations(citations, content)}+ references={REFERENCES}+ secretRequests+ readOnly+ interactive+ />+ </RouteSurfaceActiveContext.Provider>+ </I18nextProvider>,+);++describe('citation bridge: the rewrites that run beside it on the real surface', () => {+ const cited = cases.map((row) => [row.key, row] as const);++ it.each(cited)('%s badges the same links once chips and cards are enabled', (_key, row) => {+ const { container } = renderWithSurfaceRewrites(row.web_text, row.web_citations);+ const links = webLinks(container);++ expect(links.filter(badged).map((anchor) => asWritten(anchor.getAttribute('href')))).toEqual(+ row.web_anchors.filter((anchor) => anchor.cited).map((anchor) => anchor.url),+ );+ if (row.anchors_exhaustive) {+ expect(links.map((anchor) => ({+ url: asWritten(anchor.getAttribute('href')),+ cited: badged(anchor),+ }))).toEqual(row.web_anchors.map((anchor) => ({ url: anchor.url, cited: anchor.cited })));+ }+ });++ it('carries the citation across the chips and the card minted before it', () => {+ const row = cases.find((entry) => entry.key === 'marker_shaped') as BridgeCase;+ const [entry] = row.web_citations;+ const { container } = renderWithSurfaceRewrites(row.web_text, row.web_citations);+ const [badge] = webLinks(container);++ // Three markers stand in the prose ahead of the citation, and each is+ // replaced by something of a different length - so the link the backend+ // measured at [71, 142) is not written at 71 by the time Markdown parses+ // it. If no marker were rewritten nothing would have moved and this would+ // prove nothing, so the absence of the raw markers is asserted too.+ expect((entry.spans as number[][])[0][0]).toBe(71);+ expect(container.textContent).not.toContain('@<claude>');+ expect(container.textContent).not.toContain('#<ses6jr7c5h2q6>');+ expect(container.textContent).not.toContain('$<openAiKey>');+ // The chips carry the session's title rather than its id, so the text they+ // replaced the markers with is a different length in both directions.+ expect(container.textContent).toContain('@claude');+ expect(container.textContent).toContain('#Bridge');++ expect(badged(badge)).toBe(true);+ expect(badge.getAttribute('href')).toBe(entry.url);+ });++ it('leaves an address spelled like those markers alone', () => {+ const row = cases.find((entry) => entry.key === 'marker_shaped') as BridgeCase;+ const { container } = renderWithSurfaceRewrites(row.web_text, row.web_citations);+ const [badge] = webLinks(container);++ // `$<…>` and `@<…>` inside a destination are not markers to rewrite, and a+ // card minted inside the link would take the address apart. The producer's+ // percent-encoding is what keeps the two apart, so it is asserted here as+ // the address the reader reaches, not as a spelling rule.+ expect(badge.getAttribute('href')).toBe(row.web_citations[0].url);+ expect(new URL(badge.href).searchParams.get('q')).toBe('$<openAiKey>');+ expect(badge.querySelector('button')).toBeNull();+ });+});diff --git a/ui/src/components/ui/markdown.test.tsx b/ui/src/components/ui/markdown.test.tsxindex 3433e2b9a..6700dac4d 100644--- a/ui/src/components/ui/markdown.test.tsx+++ b/ui/src/components/ui/markdown.test.tsx@@ -1,12 +1,222 @@/* @vitest-environment jsdom */-import { cleanup, render } from '@testing-library/react';+import { readFileSync } from 'node:fs';+import { resolve } from 'node:path';++import { createInstance } from 'i18next';+import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';+import { I18nextProvider, initReactI18next } from 'react-i18next';import { afterEach, describe, expect, it } from 'vitest';+import en from '@/i18n/en.json';+import { bindCitations, bodyDigest, type CitationSource } from '@/lib/citations';+import { RouteSurfaceActiveContext } from '../../lib/routeSurfaceActivity';import { Markdown } from './markdown';afterEach(cleanup);+// A character reference is the other way Markdown writes a character the+// reader must see, and every surface has to agree on which spellings are+// references and what each one stands for. This renderer is the reference+// answer the IM adapters are measured against, so the cases where a pattern+// that merely looks like the grammar disagrees with it are asserted here too.+describe('Markdown character references in a link label', () => {+ it.each([+ ['a&b', 'a&b'],+ ['a©b', 'a\u00A9b'],+ // Past the grammar's width - seven decimal digits, six hexadecimal ones -+ // the spelling is the text.+ ['a&b', 'a&b'],+ ['a&b', 'a&b'],+ // Recognized, but not a code point that can be encoded.+ ['a�b', 'a\uFFFDb'],+ ['a€b', 'a\uFFFDb'],+ ['a�b', 'a\uFFFDb'],+ ['a�b', 'a\uFFFDb'],+ ])('shows %j as the label %j', (label, shown) => {+ const { container } = render(+ <Markdown content={`[${label}](https://example.com/x)`} />,+ );+ const anchor = container.querySelector('a');++ expect(anchor?.textContent).toBe(shown);+ expect(anchor?.getAttribute('href')).toBe('https://example.com/x');+ });+});++// The address half of the same unit. This renderer resolves a destination the+// way a Markdown reader does - escapes, character references and all - and+// then writes it as an href, so it is the independent answer for what address+// each of these links names. Two things are checked, and they are not the same+// check: the rows below pin the exact spelling this renderer writes, while the+// shared matrix further down compares the destination the Slack adapter+// delivers, decoded out of its wrapper, against the address an actual anchor+// here navigates to. Destinations that differ on the wire are not thereby the+// same address - only that measured navigation comparison says where each one+// arrives, and nothing here speaks for what a target server would accept.+describe('Markdown link destinations', () => {+ it.each([+ // A character the Slack wrapper is built out of, however Markdown spells it.+ [String.raw`[h](https://example.com/a>b)`, 'https://example.com/a%3Eb'],+ [String.raw`[h](https://example.com/a\>b)`, 'https://example.com/a%3Eb'],+ ['[h](https://example.com/a>b)', 'https://example.com/a%3Eb'],+ ['[h](https://example.com/a>b)', 'https://example.com/a%3Eb'],+ [String.raw`[h](https://example.com/a<b)`, 'https://example.com/a%3Cb'],+ [String.raw`[h](https://example.com/a\<b)`, 'https://example.com/a%3Cb'],+ ['[h](https://example.com/a<b)', 'https://example.com/a%3Cb'],+ [String.raw`[h](https://example.com/a|b)`, 'https://example.com/a%7Cb'],+ [String.raw`[h](https://example.com/a\|b)`, 'https://example.com/a%7Cb'],+ // A query separator is structure, and stays one.+ ['[h](https://example.com/s?a=1&b=2)', 'https://example.com/s?a=1&b=2'],+ ['[h](https://example.com/a&b)', 'https://example.com/a&b'],+ // A reference that spells the text of a reference, and the escape that+ // spells the same five characters.+ ['[h](https://example.com/a&amp;b)', 'https://example.com/a&b'],+ [String.raw`[h](https://example.com/a\&b)`, 'https://example.com/a&b'],+ // A reference can put a control character or a space in an address.+ ['[h](https://example.com/p q)', 'https://example.com/p%0Aq'],+ ['[h](https://example.com/p	q)', 'https://example.com/p%09q'],+ ['[h](<https://example.com/a b>)', 'https://example.com/a%20b'],+ // An escape already spelled stays spelled once.+ ['[h](https://example.com/a%3Eb)', 'https://example.com/a%3Eb'],+ ['[h](https://example.com/a%26amp%3Bb)', 'https://example.com/a%26amp%3Bb'],+ // Structure a URI is made of, including a bracketed IPv6 authority.+ [String.raw`[h](https://[::1]:8443/a\>b)`, 'https://[::1]:8443/a%3Eb'],+ ['[h](https://u:p@[::1]:8443/a?x=1&y=2)', 'https://u:p@[::1]:8443/a?x=1&y=2'],+ ['[h](https://example.com/p?x=1#f%20g)', 'https://example.com/p?x=1#f%20g'],+ [String.raw`[h](https://example.com/a\\b)`, 'https://example.com/a%5Cb'],+ ['[h](https://example.com/q=`x`)', 'https://example.com/q=%60x%60'],+ // A scheme that is not http.+ [+ String.raw`[h](mailto:[email protected]?subject=a\>b)`,+ 'mailto:[email protected]?subject=a%3Eb',+ ],+ // A `%` that starts no escape is data and is spelled; one that starts an+ // escape already spelled is left alone, malformed-looking or not.+ ['[h](https://example.com/a%b)', 'https://example.com/a%25b'],+ ['[h](https://example.com/a%zzb)', 'https://example.com/a%zzb'],+ // Brackets are the host's syntax in an authority and data anywhere else,+ // and a bracketed host this platform rejects is left spelled rather than+ // repaired into an address the source never wrote.+ ['[h](https://example.com/a[b])', 'https://example.com/a%5Bb%5D'],+ [+ '[h](https://u:p@[::1]:8443/a[b]?x=[c]#f[d])',+ 'https://u:p@[::1]:8443/a%5Bb%5D?x=%5Bc%5D#f%5Bd%5D',+ ],+ ['[h](https://[nope]/x)', 'https://%5Bnope%5D/x'],+ [+ '[h](https://例子.测试/路径?q=a&b=c)',+ 'https://%E4%BE%8B%E5%AD%90.%E6%B5%8B%E8%AF%95/%E8%B7%AF%E5%BE%84?q=a&b=c',+ ],+ ])('opens %j at %j', (markdown, href) => {+ const { container } = render(<Markdown content={markdown} />);+ const anchor = container.querySelector('a');++ expect(anchor?.getAttribute('href')).toBe(href);+ expect(anchor?.textContent).toBe('h');+ });++ // One shared table with tests/test_link_unit_delivery.py, read at runtime so+ // the fixture stays outside Vite's module graph. That suite asserts what the+ // real Slack adapter decodes out of its wrapper for each `markdown`; this one+ // renders the same source through the real component and asks the platform's+ // own URL parser whether the anchor it produced and that address are one URL.+ // Neither suite can drift without the other failing, and neither one claims+ // the two wires are equal - only that they arrive in the same place.+ type DestinationCase = { markdown: string; address: string };+ const destinationMatrix = (JSON.parse(+ readFileSync(resolve(process.cwd(), '../tests/fixtures/link_destination_matrix.json'), 'utf8'),+ ) as { cases: DestinationCase[] }).cases;++ it.each(destinationMatrix)(+ 'resolves $markdown to the address the Slack adapter delivers',+ ({ markdown, address }) => {+ const { container } = render(<Markdown content={markdown} />);+ const href = container.querySelector('a')?.getAttribute('href');++ expect(href).toBeDefined();+ expect(new URL(href as string).href).toBe(new URL(address).href);+ },+ );++ it('keeps a bare destination and two neighbouring links whole', () => {+ const empty = render(<Markdown content={String.raw`[](https://example.com/a\>b)`} />);++ expect(empty.container.querySelector('a')?.getAttribute('href'))+ .toBe('https://example.com/a%3Eb');+ expect(empty.container.querySelector('a')?.textContent).toBe('');++ const pair = render(+ <Markdown content={String.raw`[a](https://example.com/1\>x) [b](https://example.com/2|y)`} />,+ );++ expect([...pair.container.querySelectorAll('a')].map((a) => a.getAttribute('href')))+ .toEqual(['https://example.com/1%3Ex', 'https://example.com/2%7Cy']);+ });++ // The brackets half of the same boundary, and the third shared table. A+ // bracket is an IPv6 host's syntax in the authority and data everywhere+ // else, and `normalizeUri` writes both as `%5B`/`%5D` - so by the time an+ // href exists, `https://[::1]/admin` and `https://%5B::1%5D/admin` are one+ // string, and restoring brackets to whatever looks like an authority hands+ // the second one a live loopback address its author never wrote. The+ // distinction is taken from the parsed destination, before that spelling,+ // and carried to the href.+ //+ // Two facts per row, and they are not the same fact: `href` is the exact+ // attribute this component writes, and `navigates` is what a browser's URL+ // parser resolves from it - `null` where it refuses to. The refusals are+ // asserted rather than skipped: `new URL('https://%5B::1%5D/admin')` throws,+ // and leaving those rows out because the assertion is awkward is exactly how+ // an invented address goes unnoticed. The citation and Slack suites assert+ // their own columns of this table; see the fixture's description.+ type AuthorityCase = {+ why: string;+ markdown: string;+ href: string;+ navigates: string | null;+ };+ const authorityMatrix = JSON.parse(+ readFileSync(resolve(process.cwd(), '../tests/fixtures/citation_authority_matrix.json'), 'utf8'),+ ) as {+ cases: AuthorityCase[];+ per_occurrence: {+ markdown: string;+ links: { href: string; label: string }[];+ };+ };++ it.each(authorityMatrix.cases)('keeps only a real authority\'s brackets: $why', ({+ markdown,+ href,+ navigates,+ }) => {+ const { container } = render(<Markdown content={markdown} />);+ const anchors = [...container.querySelectorAll('a')];++ expect(anchors).toHaveLength(1);+ expect(anchors[0].getAttribute('href')).toBe(href);++ if (navigates === null) {+ expect(() => new URL(href)).toThrow();+ } else {+ expect(new URL(href).href).toBe(navigates);+ }+ });++ it('keeps two destinations that spell alike apart', () => {+ // Both hrefs are `https://%5B::1%5D/admin` if the answer is looked up by+ // the finished text rather than kept per occurrence.+ const { markdown, links } = authorityMatrix.per_occurrence;+ const { container } = render(<Markdown content={markdown} />);++ expect([...container.querySelectorAll('a')].map((a) => ({+ href: a.getAttribute('href'),+ label: a.textContent,+ }))).toEqual(links);+ });+});+describe('Markdown emphasis', () => {it.each([['', 'p'],@@ -79,3 +289,415 @@ describe('Markdown emphasis', () => {expect(container.querySelectorAll('td')).toHaveLength(2);});});++// Source citations: the backend already rewrote each marker into an ordinary+// Markdown link, so every surface is readable without this renderer. What is+// asserted here is the upgrade — recognizing exactly those links and drawing+// them as compact numbered badges — and, just as importantly, what must NOT be+// upgraded: a link the agent wrote in its own prose, and a stored sidecar row+// that no longer describes a safe destination.+describe('Markdown source citations', () => {+ const i18n = createInstance();+ void i18n.use(initReactI18next).init({+ lng: 'en',+ fallbackLng: 'en',+ resources: { en: { translation: en } },+ interpolation: { escapeValue: false },+ });++ const GUIDE = 'https://developers.openai.com/api/docs/guides/tools-web-search';+ const PROBE = 'https://example.com/citation-probe-source';+ // What core/citations.py's _escape_label writes: every character a Markdown+ // reader - or an IM dialect - would not show verbatim.+ const escapeLabel = (label: string) => label.replace(/([\\[\]`<*_~&|])/g, '\\$1');+ // And what its _link_spelling writes: the URL keeps its parentheses, and the+ // destination escapes them so an unbalanced one cannot end the link early.+ const link = (citation: { label: string; url: string }) =>+ `[${escapeLabel(citation.label)}](${citation.url.replace(/[()]/g, '\\$&')})`;+ const guide = (over: Partial<CitationSource> = {}): CitationSource => ({+ index: 1,+ ref_id: 'turn0view0',+ title: 'Web search — OpenAI API',+ url: GUIDE,+ label: 'developers.openai.com',+ ...over,+ });++ /**+ * One reply, assembled the way the backend assembles one.+ *+ * The backend writes each citation's link itself, so it knows the exact range+ * it wrote it at and the exact body it ended up in. A test says the same+ * thing by building the body out of its parts: a `string` is prose the answer+ * wrote (including a link the answer worded itself), and a source is a link+ * the backend wrote, which contributes both its spelling and its range. Every+ * row then carries the digest of the finished body, because the ranges are+ * only meaningful in it.+ */+ const body = (...parts: Array<string | CitationSource>) => {+ let text = '';+ const measured = new Map<CitationSource, number[][]>();+ for (const part of parts) {+ if (typeof part === 'string') {+ text += part;+ continue;+ }+ const start = text.length;+ text += link(part);+ measured.set(part, [...(measured.get(part) ?? []), [start, text.length]]);+ }+ const body_sha256 = bodyDigest(text);+ const citations = [...measured].map(([source, spans]) => ({ ...source, spans, body_sha256 }));+ return { text, citations };+ };++ const renderMarkdown = (content: string, citations?: unknown[], interactive = true) => render(+ <I18nextProvider i18n={i18n}>+ <RouteSurfaceActiveContext.Provider value>+ <Markdown+ content={content}+ citations={bindCitations(citations, content)}+ interactive={interactive}+ />+ </RouteSurfaceActiveContext.Provider>+ </I18nextProvider>,+ );++ const badges = (container: HTMLElement) =>+ Array.from(container.querySelectorAll<HTMLAnchorElement>('a[data-citation-index]'));++ // One shared table with tests/test_citations.py: the backend writes `url`,+ // and this asserts the real renderer hands back exactly that href, so a badge+ // cannot quietly stop matching the link it describes. Read at runtime rather+ // than imported so the fixture stays outside Vite's module graph.+ type IdentityCase = { why: string; raw: string; url: string | null; label: string | null };+ const fixture = JSON.parse(+ readFileSync(resolve(process.cwd(), '../tests/fixtures/citation_url_identity.json'), 'utf8'),+ ) as { cases: IdentityCase[]; authority: IdentityCase[] };+ const urlIdentity = [...fixture.cases, ...fixture.authority]+ .filter((entry): entry is { why: string; raw: string; url: string; label: string } =>+ Boolean(entry.url));++ it.each(urlIdentity)('keeps a canonical citation URL identical through the renderer ($why)', ({ url, label }) => {+ const { text, citations } = body('Cited. ', guide({ url, label }));+ const [badge] = badges(renderMarkdown(text, citations).container);++ expect(badge).toBeDefined();+ expect(badge.getAttribute('href')).toBe(url);+ });++ it.each(urlIdentity)('labels a citation with the host a browser resolves ($why)', ({ url, label }) => {+ // Attribution names the host the browser reaches - punycode included, and+ // an IPv4 form serialized the way the URL parser serializes it - not the+ // bytes the URL spells. A host too long for a badge is elided from the+ // LEFT and keeps the host's tail verbatim, so the shown text is exactly the+ // last 63 characters: a label cut from the right would read as a domain the+ // link never opens, and keeping whole labels only left `…co.uk`, a public+ // suffix shared by every site a long label could hide behind.+ // `hostname`, not `host`: a port belongs to the URL and never to the+ // attribution, so a label beside `:8080` still names the site.+ const host = new URL(url).hostname;+ const elided = label.startsWith('…');+ const shown = elided ? label.slice(1) : label;++ expect(label.length).toBeLessThanOrEqual(64);+ if (elided) {+ expect(shown).toBe(host.slice(-63));+ expect(label.length).toBe(64);+ } else {+ expect(host === shown || host === `www.${shown}`).toBe(true);+ }+ });++ it.each([+ // Measured rather than assumed: micromark - the parser behind this renderer+ // - replaces the whole C1 range with U+FFFD, so `€` is NOT HTML's+ // Windows-1252 `€`.+ ['€', '%EF%BF%BD'],+ // The character a reference spells is part of the destination, so micromark+ // percent-encodes it. Only a LITERAL tab or newline is removed - and that+ // one never reaches here, because it ends the destination and leaves no+ // link at all. The backend cleans literals off first and resolves+ // references after, which is the only order that keeps these two hrefs.+ ['	', '%09'],+ ['	', '%09'],+ ['
', '%0A'],+ ['
', '%0D'],+ ])('resolves a %s reference in a destination the way the backend does', (reference, encoded) => {+ // The backend never writes a live character reference into a destination -+ // it percent-encodes the character, or escapes the `&` that would start one+ // - so this is the boundary being held rather than a shape to recognize:+ // both sides must agree on what the destination says.+ const { container } = renderMarkdown(+ `Cited. [developers.openai.com](https://example.com/p${reference}q)`,+ );++ expect(container.querySelector('a')?.getAttribute('href'))+ .toBe(`https://example.com/p${encoded}q`);+ });++ it('renders a cited link as a numbered badge that is itself the link', () => {+ const { text, citations } = body('Documented. ', guide());+ const { container } = renderMarkdown(text, citations);+ const [badge] = badges(container);++ expect(badge.textContent).toBe('1');+ expect(badge.getAttribute('href')).toBe(GUIDE);+ // Safe external-link behavior, and the full attribution on the accessible+ // name — the source's own title and the domain the link opens — so the+ // preview panel is an affordance, not the only carrier.+ expect(badge.getAttribute('target')).toBe('_blank');+ expect(badge.getAttribute('rel')).toBe('noopener noreferrer nofollow');+ expect(badge.getAttribute('aria-label')).toBe(+ 'Source 1: Web search — OpenAI API (developers.openai.com)',+ );+ expect(container.textContent).toBe('Documented. 1');+ });++ it('keeps several citations compact and in the order the backend numbered them', () => {+ const second = guide({+ index: 2, ref_id: 'turn0view1', title: '引用探针来源', url: PROBE, label: 'example.com',+ });+ const { text, citations } = body('Two sources. ', guide(), ' ', second);+ const { container } = renderMarkdown(text, citations);++ expect(badges(container).map((badge) => badge.textContent)).toEqual(['1', '2']);+ expect(container.querySelectorAll('a')).toHaveLength(2);+ });++ it('leaves a link the agent worded itself as an ordinary anchor', () => {+ // Same page, different characters, and in a range the backend never claimed.+ const { text, citations } = body(`See [the web search guide](${GUIDE}) and `, guide(), '.');+ const { container } = renderMarkdown(text, citations);+ const anchors = Array.from(container.querySelectorAll('a'));++ expect(anchors.map((anchor) => anchor.textContent)).toEqual(['the web search guide', '1']);+ expect(anchors[0].hasAttribute('data-citation-index')).toBe(false);+ });++ it('collapses to plain text on a non-interactive surface', () => {+ const { text, citations } = body('Documented. ', guide());+ const { container } = renderMarkdown(text, citations, false);++ // A badge is an anchor; nested inside a clickable preview row it would be+ // invalid interactive content, so the link text stands in as prose.+ expect(container.querySelector('a')).toBeNull();+ expect(container.textContent).toBe('Documented. developers.openai.com');+ });++ it.each([+ ['an unsafe scheme', { url: 'javascript:alert(1)' }],+ ['a missing label', { label: undefined }],+ ['an unusable index', { index: Number.NaN }],+ ])('degrades a stored row with %s to the link the text already carries', (_case, over) => {+ const { text, citations } = body('Documented. ', guide());+ const { container } = renderMarkdown(text, citations.map((row) => ({ ...row, ...over })));+ const anchor = container.querySelector('a');++ expect(badges(container)).toHaveLength(0);+ expect(anchor?.getAttribute('href')).toBe(GUIDE);+ expect(anchor?.textContent).toBe('developers.openai.com');+ });++ it('previews the source title and domain on hover, and offers the page explicitly', () => {+ const { text, citations } = body('Documented. ', guide());+ const { container } = renderMarkdown(text, citations);++ fireEvent.pointerOver(badges(container)[0], { pointerType: 'mouse' });++ expect(screen.getByText('Web search — OpenAI API')).toBeTruthy();+ expect(screen.getByText('developers.openai.com')).toBeTruthy();+ expect(screen.getByText('Open source page').closest('a')?.getAttribute('href')).toBe(GUIDE);+ });++ it('reveals the same preview on keyboard focus without moving focus off the link', () => {+ const { text, citations } = body('Documented. ', guide());+ const { container } = renderMarkdown(text, citations);+ const [badge] = badges(container);++ // A real focus() rather than a synthetic event: the point of this test is+ // that the element the browser focuses is the element that opens the page.+ act(() => badge.focus());++ expect(screen.getByText('Web search — OpenAI API')).toBeTruthy();+ // Focus stays on the badge, so Enter opens the page instead of tabbing into+ // a portalled panel first.+ expect(document.activeElement).toBe(badge);+ });++ it('lets the first tap reveal the preview and the next one follow the link', () => {+ const { text, citations } = body('Documented. ', guide());+ const { container } = renderMarkdown(text, citations);+ const [badge] = badges(container);++ fireEvent.pointerDown(badge, { pointerType: 'touch' });++ expect(fireEvent.click(badge)).toBe(false);+ expect(screen.getByText('Web search — OpenAI API')).toBeTruthy();++ fireEvent.pointerDown(badge, { pointerType: 'touch' });++ expect(fireEvent.click(badge)).toBe(true);+ });++ // A badge is an attribution claim, so it belongs to a link the backend wrote.+ // Recognizing one used to mean matching the rendered label back to a sidecar+ // row, which cannot tell that link apart from one the answer's own prose+ // wrote to the same page - and got it wrong in the direction that matters,+ // since the prose link is the one nobody vouched for.+ //+ // So the row carries the ranges the backend wrote its links at, in the body+ // it wrote them into - named by its digest, because a range means nothing+ // without the version of the text it was counted in. A badge is drawn only on+ // a link whose parsed position IS one of those ranges, exactly.+ describe('provenance', () => {+ it('leaves a prose link alone even when it is word-for-word the citation', () => {+ const citation = guide();+ const { text, citations } = body('See ', link(citation), ' first. ', citation);+ const { container } = renderMarkdown(text, citations);+ const anchors = Array.from(container.querySelectorAll('a'));++ expect(anchors).toHaveLength(2);+ expect(anchors[0].hasAttribute('data-citation-index')).toBe(false);+ expect(anchors[0].textContent).toBe('developers.openai.com');+ expect(anchors[1].getAttribute('data-citation-index')).toBe('1');+ });++ it('draws a badge on each link one source wrote', () => {+ const citation = guide();+ const { text, citations } = body('One. ', citation, ' Two. ', citation);+ const { container } = renderMarkdown(text, citations);++ expect(citations[0].spans).toHaveLength(2);+ expect(badges(container).map((badge) => badge.textContent)).toEqual(['1', '1']);+ });++ it('locates each source independently, so one between two others shifts none', () => {+ const first = guide();+ const second = guide({+ index: 2, ref_id: 'turn0view1', title: '引用探针来源', url: PROBE, label: 'example.com',+ });+ const { text, citations } = body('A. ', first, ' B. ', second, ' C. ', first);+ const { container } = renderMarkdown(text, citations);++ expect(badges(container).map((badge) => badge.textContent)).toEqual(['1', '2', '1']);+ });++ it('degrades to the plain link when the body is not the one that was measured', () => {+ // The ranges still land on real links here - the edit is at the end. What+ // says they are no longer this body's ranges is the digest, and the honest+ // answer is the link the reader already has rather than a badge on+ // whichever link happens to sit in that position now.+ const { text, citations } = body('Documented. ', guide());+ const { container } = renderMarkdown(`${text} Edited.`, citations);++ expect(badges(container)).toHaveLength(0);+ expect(container.querySelector('a')?.getAttribute('href')).toBe(GUIDE);+ });++ it('draws nothing on a link a range covers only part of', () => {+ // A range is the WHOLE `[label](url)` the backend wrote. One that starts a+ // character early describes something that is not a link at all, so it+ // matches no link - a near miss is not a match.+ const { text, citations } = body('Documented. ', guide());+ const [[start, end]] = citations[0].spans;+ const { container } = renderMarkdown(+ text,+ [{ ...citations[0], spans: [[start - 1, end]] }],+ );++ expect(badges(container)).toHaveLength(0);+ expect(container.querySelector('a')?.getAttribute('href')).toBe(GUIDE);+ });++ it.each([+ ['an image', ``],+ ['an autolink', `<${'https://developers.openai.com/api/docs/guides/tools-web-search'}>`],+ ['a reference link', '[dup][k]'],+ ])('does not badge %s, which spells the destination some other way', (_why, decoy) => {+ // Same page, different characters, and none of them written by the+ // backend. An image is not a link, and an autolink and a reference link+ // are links whose position is their own - never a range the backend+ // measured, because the backend wrote an inline link.+ const { text, citations } = body(`${decoy} Shown. `, guide(), `\n\n[k]: ${GUIDE}`);+ const { container } = renderMarkdown(text, citations);++ expect(badges(container).map((badge) => badge.textContent)).toEqual(['1']);+ });++ it.each([+ ['a code span', (l: string) => `\`${l}\``],+ ['a fenced block', (l: string) => `\`\`\`\n${l}\n\`\`\``],+ ['a footnote definition', (l: string) => `[^f]: ${l}`],+ ['a table cell', (l: string) => `| a |\n| - |\n| ${l} |`],+ ['an image alt', (l: string) => `!${l}`],+ ])('is unmoved by the same exact spelling inside %s', (_why, wrap) => {+ // This is the case that used to break: the two sides disagreed about+ // whether a footnote definition holds a link, their counts diverged, and+ // the real citation silently lost its badge. Neither side counts anything+ // now - the backend says where it wrote, and the parser says where this+ // link is.+ const citation = guide();+ const { text, citations } = body(`${wrap(link(citation))}\n\nShown. `, citation);+ const badged = badges(renderMarkdown(text, citations).container);++ expect(badged.map((badge) => badge.textContent)).toEqual(['1']);+ expect(badged[0].getAttribute('href')).toBe(GUIDE);+ });++ it('still recognizes a row persisted before provenance existed', () => {+ // History is a shipped surface: a message stored by an earlier release+ // carries no ranges at all, so it keeps the match it was written for -+ // the exact destination and link text.+ const legacy = guide();+ const { container } = renderMarkdown(`Documented. ${link(legacy)}`, [legacy]);++ expect(badges(container).map((badge) => badge.textContent)).toEqual(['1']);+ });++ it.each([+ ['a bare literal', 'https://[2001:db8::1]/x', '[2001:db8::1]'],+ ['a port', 'https://[2001:db8::1]:8443/x', '[2001:db8::1]'],+ ['userinfo in front of it', 'https://attacker.example@[::1]/x', '[::1]'],+ ])('reaches an IPv6 source with %s, badge or not', (_why, url, label) => {+ // An IPv6 host is REQUIRED to be bracketed, and `[`/`]` are two of the+ // characters mdast-util-to-hast percent-encodes into an href: the link+ // rendered, looked right, and went nowhere. A badge is a presentation+ // upgrade, so the ORDINARY link has to reach the page on its own and the+ // badge has to agree with it - same address, whether or not a sidecar+ // ever arrives.+ const { text, citations } = body('Documented. ', guide({ url, label }));+ const { container } = renderMarkdown(text, citations);+ const plain = renderMarkdown(text);+ const badged = badges(container)[0].getAttribute('href') as string;+ const ordinary = plain.container.querySelector('a')?.getAttribute('href') as string;++ expect(badged).toBe(url);+ expect(ordinary).toBe(url);+ expect(new URL(ordinary).hostname).toBe(label);+ expect(new URL(badged).href).toBe(new URL(ordinary).href);+ });++ it('recognizes a citation whose label had to be escaped, and keeps the code after it', () => {+ // The escape is what stops the backtick opening a code span that runs+ // past `](url)` to the next one and takes the link with it.+ const { text, citations } = body(+ 'Documented. ', guide({ label: 'ex`ample.com' }), ' and `code` prose.',+ );+ const { container } = renderMarkdown(text, citations);++ expect(badges(container).map((badge) => badge.textContent)).toEqual(['1']);+ expect(container.querySelector('code')?.textContent).toBe('code');+ });+ });++ it('attributes a source whose search returned no title by its domain', () => {+ const { text, citations } = body('Documented. ', guide({ title: '' }));+ const { container } = renderMarkdown(text, citations);++ fireEvent.pointerOver(badges(container)[0], { pointerType: 'mouse' });++ expect(badges(container)[0].getAttribute('aria-label')).toBe('Source 1: developers.openai.com');+ expect(screen.getAllByText('developers.openai.com').length).toBeGreaterThan(0);+ });+});diff --git a/ui/src/components/ui/markdown.tsx b/ui/src/components/ui/markdown.tsxindex 83b88e19f..83c09ef6e 100644--- a/ui/src/components/ui/markdown.tsx+++ b/ui/src/components/ui/markdown.tsx@@ -11,6 +11,15 @@ import { Badge } from '@/components/ui/badge';import { ChatImage, LinkedImageContent, LinkedImageProvider } from '@/components/ui/chat-image';import { FileCard } from '@/components/ui/file-card';import { SecretRequestCard } from '@/components/ui/secret-request-card';+import { CitationBadge } from '@/components/ui/citation-badge';+import {+ findCitation,+ remapCitations,+ remarkCitationSpans,+ type CitationBinding,+ type EditedText,+ type TextEdit,+} from '@/lib/citations';import { inAppChatPath } from '@/lib/applicationRoutes';import { isProxyMediaUrl, readMediaDims } from '@/lib/mediaProxy';import { isAbsoluteWindowsFileHref, resolveLocalFileLink, type LocalFileLinkTarget } from '@/lib/localFileLinks';@@ -50,38 +59,179 @@ const SECRET_REQUEST_RE = /\$<([A-Za-z_][A-Za-z0-9_]*)>/g;const CODE_SPAN_RE = /```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`/g;const INDENTED_CODE_LINE_RE = /^(?: {4}|\t)/;-function linkifySecretRequests(text: string): string {- if (!text.includes('$<')) return text;+// Like ``linkifyMentions``, this replaces marker text with a link of a different+// length, so it reports each replacement in the coordinates of its input — that+// is what lets a citation measured against this text survive the pass.+function linkifySecretRequests(text: string): EditedText {+ const edits: TextEdit[] = [];+ if (!text.includes('$<')) return { text, edits };// Within a non-fenced/non-inline-code segment, still skip indented code lines.- const rewrite = (segment: string) =>- segment+ const rewrite = (segment: string, base: number) => {+ let at = base;+ return segment.split('\n')- .map((line) =>- INDENTED_CODE_LINE_RE.test(line)- ? line- : line.replace(SECRET_REQUEST_RE, (_m, name) => `[${name}](${SECRET_LINK_SCHEME}:${name})`),- )+ .map((line) => {+ const lineAt = at;+ at += line.length + 1; // the newline ``split`` consumed+ if (INDENTED_CODE_LINE_RE.test(line)) return line;+ return line.replace(SECRET_REQUEST_RE, (marker: string, name: string, offset: number) => {+ const card = `[${name}](${SECRET_LINK_SCHEME}:${name})`;+ edits.push({ start: lineAt + offset, end: lineAt + offset + marker.length, inserted: card.length });+ return card;+ });+ }).join('\n');+ };// Partition on code spans; rewrite only the non-code segments (code stays verbatim).let result = '';let last = 0;CODE_SPAN_RE.lastIndex = 0;for (let m = CODE_SPAN_RE.exec(text); m; m = CODE_SPAN_RE.exec(text)) {- result += rewrite(text.slice(last, m.index)) + m[0];+ result += rewrite(text.slice(last, m.index), last) + m[0];last = m.index + m[0].length;}- return result + rewrite(text.slice(last));+ return { text: result + rewrite(text.slice(last), last), edits };+}++// The visible text of a rendered link, flattened back to a string. Only a+// citation row persisted before provenance existed is still matched on it (see+// lib/citations); a current one is matched on which link the backend wrote.+function linkText(children: React.ReactNode): string {+ if (typeof children === 'string') return children;+ if (typeof children === 'number') return String(children);+ if (Array.isArray(children)) return children.map(linkText).join('');+ return '';+}++// An IPv6 host is REQUIRED to be written in brackets — and `[`/`]` are two of+// the characters mdast-util-to-hast percent-encodes when it turns a parsed+// destination into an href. So `https://[::1]/x` arrives at the href below+// spelled `https://%5B::1%5D/x`, which is not a URL any browser will parse: the+// link is rendered, looks right, and goes nowhere. Put the brackets back — in+// the authority only, never across the rest of the href, and only when the+// platform itself then accepts the result as a URL.+//+// Only for a destination that was WRITTEN with brackets, which is the one thing+// the href no longer says: the same encoder writes a literal `[::1]` and a+// provider's own `%5B::1%5D` as the same characters, so repairing whatever looks+// escaped turned `https://%5B::1%5D/admin` — an address with no host, which this+// renderer should leave exactly as invalid as it arrived — into a live link to+// the loopback interface. `remarkLiteralAuthority` answers that question up in+// the AST, where the parsed destination still exists, and marks the node; this+// runs only for a node it marked.+// Userinfo is part of the authority and may precede the host, so the match+// runs to the last `@` before the path rather than assuming the host is first.+const ENCODED_IPV6_AUTHORITY = /^(https?:\/\/(?:[^/?#]*@)?)(%5B[^/?#]*%5D[^/?#]*)/i;++function repairIpv6Authority(url: string): string {+ const match = ENCODED_IPV6_AUTHORITY.exec(url);+ if (!match) return url;+ const authority = match[2].replace(/%5B/gi, '[').replace(/%5D/gi, ']');+ const repaired = `${match[1]}${authority}${url.slice(match[0].length)}`;+ try {+ // Not a shape check: the browser's own parser decides, so a host it would+ // reject (`https://[nope]/`) stays encoded rather than becoming a new URL.+ new URL(repaired);+ } catch {+ return url;+ }+ return repaired;+}++// The mark `remarkLiteralAuthority` leaves on a node whose destination was+// written with a literal bracketed host. It is read back off the hast element+// react-markdown hands `urlTransform`, so the qualification travels with the+// occurrence rather than with the URL text — two links in one message may spell+// the same href and disagree about this.+const LITERAL_AUTHORITY = 'dataLiteralAuthority';++type MdastNode = {+ type?: string;+ url?: unknown;+ identifier?: unknown;+ data?: { hProperties?: Record<string, unknown> };+ children?: unknown;+};++function eachMdastNode(node: MdastNode, visit: (node: MdastNode) => void): void {+ visit(node);+ if (!Array.isArray(node.children)) return;+ for (const child of node.children) eachMdastNode(child as MdastNode, visit);+}++// Whether a destination, as the Markdown parser resolved it and before anything+// spelled it as a URI, wraps its host in literal brackets. Both ends have to be+// literal: a mixed wrapper (`[::1%5D`) is not a bracketed host with an escape in+// it, it is a host that was never bracketed, and promoting half of it into+// syntax invents an address out of the other half. The boundaries are read the+// same way a URL parser reads them — `/?#` end the authority, the LAST `@`+// starts the host — and none of those characters are touched by URI spelling,+// so both ends of the pipeline see them in the same places.+function hasLiteralAuthorityBrackets(url: string): boolean {+ const colon = url.indexOf(':');+ if (colon < 0) return false;+ const afterScheme = url.slice(colon + 1);+ if (!afterScheme.startsWith('//')) return false;+ const rest = afterScheme.slice(2);+ const end = rest.search(/[/?#]/);+ const authority = end < 0 ? rest : rest.slice(0, end);+ const at = authority.lastIndexOf('@');+ const hostPort = at < 0 ? authority : authority.slice(at + 1);+ const opener = /^(?:\[|%5[Bb])/.exec(hostPort);+ if (!opener || opener[0] !== '[') return false;+ const closer = /\]|%5[Dd]/.exec(hostPort.slice(opener[0].length));+ return closer !== null && closer[0] === ']';+}++// Carry that one bit from the parsed destination to the href consumer.+// `node.url` (or, for a reference, the url of the definition it names) is the+// destination mdast still holds; by the time `urlTransform` runs, hast has+// spelled it and the answer is gone. Inline links, autolinks, reference links+// and image destinations all pass through here — the component turns an image+// this renderer will not fetch into a click-through link of its own, so an+// image `src` reaches an `href` too. Nothing else about the node is read or+// written, and the mark is consumed by `mentionUrlTransform` rather than+// rendered: `a` and `img` below take the props they name, not a spread.+function remarkLiteralAuthority() {+ return (tree: unknown) => {+ const definitions = new Map<string, string>();+ eachMdastNode(tree as MdastNode, (node) => {+ if (node.type !== 'definition') return;+ if (typeof node.identifier === 'string' && typeof node.url === 'string') {+ definitions.set(node.identifier, node.url);+ }+ });+ eachMdastNode(tree as MdastNode, (node) => {+ let url: string | undefined;+ if (node.type === 'link' || node.type === 'image') {+ if (typeof node.url === 'string') url = node.url;+ } else if (node.type === 'linkReference' || node.type === 'imageReference') {+ if (typeof node.identifier === 'string') url = definitions.get(node.identifier);+ } else {+ return;+ }+ if (url === undefined || !hasLiteralAuthorityBrackets(url)) return;+ const data = (node.data ??= {});+ const properties = (data.hProperties ??= {});+ properties[LITERAL_AUTHORITY] = true;+ });+ };}// Keep react-markdown's URL sanitizer from stripping our custom schemes (it allows// only http/https/mailto/tel/relative by default).-function mentionUrlTransform(url: string, allowLocalFiles: boolean): string {+function mentionUrlTransform(+ url: string,+ allowLocalFiles: boolean,+ node?: { properties?: Record<string, unknown> },+): string {if (url.startsWith(`${MENTION_LINK_SCHEME}:`)|| url.startsWith(`${SECRET_LINK_SCHEME}:`)|| (allowLocalFiles && isAbsoluteWindowsFileHref(url))) return url;- return defaultUrlTransform(url);+ const literalAuthority = node?.properties?.[LITERAL_AUTHORITY] === true;+ return defaultUrlTransform(literalAuthority ? repairIpv6Authority(url) : url);}// A fenced code block with a hover/tap copy button. The button lives on a@@ -180,6 +330,14 @@ export const Markdown: React.FC<{* or quoted text could mint an "agent asked for this secret" card that creates a vault* secret on click. */secretRequests?: boolean;+ /** Citation binding — the sidecar already read against the exact text this+ * renderer is handed (`bindCitations`, then `remapCitations` through whatever+ * the caller cut out of the stored row). The links it names render as compact+ * numbered source badges instead of bare domain links; everything else stays+ * an ordinary link. ONLY the agent-reply surface passes one: the badge is an+ * attribution claim, so a user bubble or a quoted preview must not be able to+ * mint one. The text already reads correctly without it. */+ citations?: CitationBinding | null;/** Agent-reply opt-in: `/abs/path` and `./relative/path` Markdown links open* in Avibe's Editor. Relative paths resolve from the owning Session workdir. */localFileWorkdir?: string | null;@@ -197,10 +355,45 @@ export const Markdown: React.FC<{softBreaks = false,references,secretRequests = false,+ citations,localFileWorkdir,onOpenLocalFile,readOnly = false,}) => {+ // Mention markers are rewritten to `avibe-mention:` links BEFORE markdown sees+ // them, and only when a sidecar is present — agent replies (no references) skip+ // this so their code spans are never touched. The links render as chips via the+ // `a` map. Secret `$<NAME>` markers are rewritten too, but only on the agent-reply+ // surface (secretRequests) — user bubbles / previews / docs keep them as plain text.+ //+ // Both are replacements, so both move the text after them. The caller measured+ // its citations against `content`; each pass hands back what it changed, and the+ // binding is carried through in the same order the passes ran, so what finally+ // reaches ReactMarkdown is a binding stated in the coordinates of the text+ // ReactMarkdown is parsing. A pass that edited into a citation's own link drops+ // that citation rather than letting the badge land on rewritten characters.+ const prepared = React.useMemo(() => {+ let text = content;+ let binding = citations ?? null;+ if (references && references.length) {+ const pass = linkifyMentions(text, references);+ text = pass.text;+ binding = remapCitations(binding, pass.edits);+ }+ if (secretRequests) {+ const pass = linkifySecretRequests(text);+ text = pass.text;+ binding = remapCitations(binding, pass.edits);+ }+ return { text, binding };+ }, [content, references, secretRequests, citations]);++ // The citation annotation is only ever read back on the surface that renders+ // badges, so the walk that writes it is attached only there — every other+ // markdown surface parses exactly what it parsed before. A legacy row needs no+ // annotation: it is matched on the link's own spelling, not on where it sits.+ const annotateCitations = interactive && !!prepared.binding?.bound.length;+// Stable ``remarkPlugins`` + ``components`` identities across re-renders.// ReactMarkdown keys its rendered tree on the component functions it is handed;// the old inline object minted fresh functions every render, so ReactMarkdown@@ -214,10 +407,16 @@ export const Markdown: React.FC<{// the editor-preview caller that lacks that wrapper.)const remarkPlugins = React.useMemo(// CJK punctuation can touch emphasis markers without spaces between words.- () => (softBreaks- ? [remarkGfm, remarkCjkFriendly, remarkBreaks]- : [remarkGfm, remarkCjkFriendly]),- [softBreaks],+ () => [+ remarkGfm,+ remarkCjkFriendly,+ // Unconditional: which destinations were written with a bracketed host is+ // a fact about this text, not about whether it carries citations.+ remarkLiteralAuthority,+ ...(softBreaks ? [remarkBreaks] : []),+ ...(annotateCitations ? [remarkCitationSpans] : []),+ ],+ [softBreaks, annotateCitations],);const components = React.useMemo<Components>(() => ({@@ -241,7 +440,7 @@ export const Markdown: React.FC<{// download card (filename + type + download / preview). Other links keep// the normal anchor (interactive) or collapse to plain text inside a// clickable row (non-interactive).- a: ({ href, children }) => {+ a: ({ href, children, node }) => {const url = href ? String(href) : '';// @-agent / #-session mention chips (see lib/mentions). Rendered in both// interactive and non-interactive contexts — a chip is a span, safe inside@@ -267,6 +466,15 @@ export const Markdown: React.FC<{const name = url.slice(SECRET_LINK_SCHEME.length + 1);return secretRequests ? <SecretRequestCard name={name} readOnly={readOnly} /> : <span>{children}</span>;}+ // A backend-resolved source citation → compact numbered badge with a+ // title/domain preview. Only on the interactive surface: the badge is an+ // anchor, which would be invalid interactive content inside a clickable+ // row — the ``!interactive`` branch below already renders the plain+ // domain text, which still attributes the source.+ const citation = interactive+ ? findCitation(prepared.binding, node, url, linkText(children))+ : null;+ if (citation) return <CitationBadge citation={citation} />;if (interactive && url && isProxyMediaUrl(url)) {return <FileCard href={url}>{children}</FileCard>;}@@ -332,17 +540,11 @@ export const Markdown: React.FC<{),}),}),- [interactive, secretRequests, readOnly, localFileWorkdir, onOpenLocalFile],+ [interactive, secretRequests, prepared.binding, readOnly, localFileWorkdir, onOpenLocalFile],);- // Mention markers are rewritten to `avibe-mention:` links BEFORE markdown sees- // them, and only when a sidecar is present — agent replies (no references) skip- // this so their code spans are never touched. The links render as chips via the- // `a` map. Secret `$<NAME>` markers are rewritten too, but only on the agent-reply surface- // (secretRequests) — user bubbles / previews / docs keep them as plain text.- let rendered = references && references.length ? linkifyMentions(content, references) : content;- if (secretRequests) rendered = linkifySecretRequests(rendered);- const urlTransform = (url: string) => mentionUrlTransform(url, Boolean(onOpenLocalFile));+ const urlTransform = (url: string, _key: string, node: { properties?: Record<string, unknown> }) =>+ mentionUrlTransform(url, Boolean(onOpenLocalFile), node);return (<div className={cn('vr-markdown', className)}><ReactMarkdown@@ -350,7 +552,7 @@ export const Markdown: React.FC<{components={components}urlTransform={urlTransform}>- {rendered}+ {prepared.text}</ReactMarkdown></div>);diff --git a/ui/src/components/workbench/AgentActivityGroup.test.tsx b/ui/src/components/workbench/AgentActivityGroup.test.tsxindex d6a39de2e..9b63d3cb4 100644--- a/ui/src/components/workbench/AgentActivityGroup.test.tsx+++ b/ui/src/components/workbench/AgentActivityGroup.test.tsx@@ -59,6 +59,58 @@ describe('shell tool-call display', () => {});});+describe('narration citations', () => {+ it.each(['live', 'history'])('renders a cited narration row as a source badge in %s', (surface) => {+ // The sidecar reaches Markdown only if the row carries it; without one the+ // same link must stay an ordinary link rather than claim a source badge.+ const rows = [+ {+ id: 'cited',+ kind: 'assistant' as const,+ text: 'Found it. [example.com](https://example.com/x)',+ created_at: '2026-09-21T02:00:00Z',+ citations: [{ index: 1, ref_id: 'turn0view0', title: 'Example page', url: 'https://example.com/x', label: 'example.com' }],+ },+ {+ id: 'plain',+ kind: 'assistant' as const,+ text: 'Also see [example.org](https://example.org/y)',+ created_at: '2026-09-21T02:00:01Z',+ },+ ];+ const { container } = render(surface === 'live' ? (+ <ActivityCard+ rows={rows}+ startedAtMs={Date.now()}+ expanded+ onToggleExpanded={vi.fn()}+ showToolCalls+ onToggleTools={vi.fn()}+ />+ ) : (+ <ActivityChip+ group={{+ id: 'completed-turn', anchorMessageId: 'reply', anchorPosition: 'before',+ open: false, status: 'done', steps: 2, durationMs: 1000, rows,+ }}+ expanded+ loading={false}+ onToggle={vi.fn()}+ showToolCalls+ onToggleTools={vi.fn()}+ />+ ));++ const badges = Array.from(container.querySelectorAll<HTMLAnchorElement>('a[data-citation-index]'));+ expect(badges).toHaveLength(1);+ expect(badges[0].getAttribute('href')).toBe('https://example.com/x');+ expect(badges[0].textContent).toBe('1');+ const plain = container.querySelector<HTMLAnchorElement>('a[href="https://example.org/y"]');+ expect(plain?.hasAttribute('data-citation-index')).toBe(false);+ expect(plain?.textContent).toBe('example.org');+ });+});+describe('ActivityCard display shortcut', () => {it('shows hour and day units when a live run crosses those boundaries', () => {vi.useFakeTimers();diff --git a/ui/src/components/workbench/AgentActivityGroup.tsx b/ui/src/components/workbench/AgentActivityGroup.tsxindex 2b82bdf3c..d9ecf7f3f 100644--- a/ui/src/components/workbench/AgentActivityGroup.tsx+++ b/ui/src/components/workbench/AgentActivityGroup.tsx@@ -29,6 +29,7 @@ import clsx from 'clsx';import { Markdown } from '../ui/markdown';import { Dialog, DialogContent } from '../ui/dialog';import { copyTextToClipboard } from '../../lib/utils';+import { bindCitations } from '../../lib/citations';import {filterActivityRows,formatActivityElapsedClock,@@ -328,18 +329,27 @@ const ActivityToolRow: React.FC<{ row: ActivityRow }> = ({ row }) => {// ----- One interim assistant row: sparkles + full text (owner decision), the// existing Markdown renderer in a compact style. ------const ActivityAssistantRow: React.FC<{ row: ActivityRow }> = ({ row }) => (- <div className="flex items-start gap-2 px-1.5 py-1">- <Sparkles className="mt-0.5 size-3.5 shrink-0 text-mint-ink" aria-hidden="true" />- <div className="min-w-0 flex-1 text-[12px] leading-relaxed text-foreground/90 [&_p]:my-0.5 [&_pre]:max-w-full [&_pre]:overflow-x-auto">- {row.text ? (- <Markdown content={row.text} className="vr-markdown--inherit-size" />- ) : (- <span className="text-muted">—</span>- )}+const ActivityAssistantRow: React.FC<{ row: ActivityRow }> = ({ row }) => {+ // Narration rows are the agent's own words, so the same authorship gate the+ // transcript applies is satisfied here by construction: an assistant row may+ // draw a source badge. Unlike a transcript row this one is rendered verbatim,+ // so the body the sidecar was measured in IS the body handed to the renderer.+ // Without a sidecar — or with one that no longer describes this text — the+ // link stays the plain domain the backend wrote.+ const citations = useMemo(() => bindCitations(row.citations, row.text), [row.citations, row.text]);+ return (+ <div className="flex items-start gap-2 px-1.5 py-1">+ <Sparkles className="mt-0.5 size-3.5 shrink-0 text-mint-ink" aria-hidden="true" />+ <div className="min-w-0 flex-1 text-[12px] leading-relaxed text-foreground/90 [&_p]:my-0.5 [&_pre]:max-w-full [&_pre]:overflow-x-auto">+ {row.text ? (+ <Markdown content={row.text} className="vr-markdown--inherit-size" citations={citations} />+ ) : (+ <span className="text-muted">—</span>+ )}+ </div></div>- </div>-);+ );+};const ActivityRowItem = memo(function ActivityRowItem({ row }: { row: ActivityRow }) {return row.kind === 'tool_call' ? <ActivityToolRow row={row} /> : <ActivityAssistantRow row={row} />;diff --git a/ui/src/components/workbench/ChatPage.tsx b/ui/src/components/workbench/ChatPage.tsxindex be57e86f0..fe76448ac 100644--- a/ui/src/components/workbench/ChatPage.tsx+++ b/ui/src/components/workbench/ChatPage.tsx@@ -133,6 +133,7 @@ import { useCoalescedWrite } from '../../lib/useCoalescedWrite';import { hasInAppBackEntry } from '../../lib/navigationHistory';import { Composer, type ComposerAttachment, type ComposerHandle, type ComposerProps } from './Composer';import type { MentionReference } from '../../lib/mentions';+import { bindCitations, remapCitations } from '../../lib/citations';import { QuickReplies } from './QuickReplies';import { QueuedAttachmentGroup, QueuedAttachmentSheet } from './QueuedAttachments';import { ActivityCard, ActivityChip } from './AgentActivityGroup';@@ -4706,7 +4707,20 @@ export const MessageRow = memo(function MessageRow({const triggerLink = isHarness ? chatTriggerLink(message, t('chat.source.agentFallback')) : null;const vaultStatusKey = isVaultCallback(message) ? vaultCallbackStatusKey(message) : null;const messageFontStyle = { fontSize: `${normalizeChatMessageFontSize(messageFontSize)}px` };- const resultPresentation = resultFooterParts(message);+ const resultPresentation = useMemo(() => resultFooterParts(message), [message]);++ // A citation names the links the backend wrote, as ranges in the body it+ // stored — so it is verified against that stored body, `message.text`, and+ // only then carried onto the body this row actually renders. The footer strip+ // above is a real edit with real coordinates, which is what makes that+ // possible without guessing whether the rendered body is "a prefix of" the+ // stored one. Read here rather than inside the renderer: only the agent's own+ // reply may draw a badge, and only this row knows who wrote it.+ const citationBinding = useMemo(() => {+ if (!agentAuthored) return null;+ const stored = (message.content as { citations?: unknown[] } | null)?.citations;+ return remapCitations(bindCitations(stored, message.text), resultPresentation.edits);+ }, [agentAuthored, message.content, message.text, resultPresentation]);// User-uploaded attachments ride in ``content.attachments`` (agent-reply media// is rewritten inline into the text instead, handled by the Markdown renderer).@@ -4790,6 +4804,11 @@ export const MessageRow = memo(function MessageRow({// card). Keyed to authorship, not to the card family, so the agent's reverse annotation// keeps the card it had before that row had its own type.secretRequests={agentAuthored}+ // A numbered source badge asserts that the ANSWER cited that page, so only+ // the agent's own reply may draw one — and only where the sidecar still+ // describes this exact text. Otherwise the link still renders, as the plain+ // domain the backend wrote.+ citations={citationBinding}localFileWorkdir={agentAuthored ? session.workdir : undefined}onOpenLocalFile={agentAuthored ? onOpenLocalFile : undefined}// …and on an archived transcript the card is locked: archiving EXPIRED thediff --git a/ui/src/components/workbench/CitationStoredRows.test.tsx b/ui/src/components/workbench/CitationStoredRows.test.tsxnew file mode 100644index 000000000..3bf62c782--- /dev/null+++ b/ui/src/components/workbench/CitationStoredRows.test.tsx@@ -0,0 +1,274 @@+/* @vitest-environment jsdom */++// The rows the dispatcher stored, read by the surfaces that show them.+//+// `tests/citation_bridge.py` drives the real `ConsolidatedMessageDispatcher`+// over one cited reply against a temporary SQLite home and records the rows it+// wrote — text, sidecar and footer exactly as they land on disk. This file+// hands those rows to the two components a reader actually meets them through:+// the transcript row and the activity card.+//+// The question is narrow and it is the one a body digest could have broken. A+// citation names ranges in the body it was measured in, and for an IM result+// that body is the one with the footer folded into it — which is not the body+// the transcript renders, because the reader is shown the footer separately.+// If the measurement did not survive that split, every ordinary IM transcript+// row would silently drop to a plain link.++import { readFileSync } from 'node:fs';+import { resolve } from 'node:path';++import { cleanup, render } from '@testing-library/react';+import { createInstance } from 'i18next';+import type { ReactElement } from 'react';+import { I18nextProvider, initReactI18next } from 'react-i18next';+import { MemoryRouter } from 'react-router-dom';+import { afterEach, describe, expect, it, vi } from 'vitest';++// The wire is the only stand-in here: a row carrying ``$<NAME>`` renders the+// real secure-input card, which reads the api context. The card is only ever+// rendered in these tests, never opened, so nothing is fetched - but it has to+// find a provider, and the provider has to find a fetch.+const apiFetch = vi.hoisted(() => vi.fn(async () => ({})));+vi.mock('../../lib/apiFetch', () => ({ apiFetch }));++// ``ChatPage`` reaches the composer's mention editor at import time, which reads+// ``matchMedia`` before any test body runs.+vi.hoisted(() => {+ Object.defineProperty(window, 'matchMedia', {+ configurable: true,+ value: vi.fn().mockReturnValue({+ matches: false,+ addEventListener: () => {},+ removeEventListener: () => {},+ }),+ });+});++import en from '../../i18n/en.json';+import { ApiProvider } from '../../context/ApiContext';+import type { WorkbenchMessage, WorkbenchSession } from '../../context/ApiContext';+import { ToastProvider } from '../../context/ToastProvider';+import { activityRowFromMessage } from '../../lib/agentActivity';+import type { CitationSource } from '../../lib/citations';+import { ActivityCard, ActivityChip } from './AgentActivityGroup';+import { MessageRow } from './ChatPage';++type StoredRow = {+ key: string;+ why: string;+ platform: string;+ type: 'result' | 'assistant';+ row: {+ type: string;+ text: string;+ content: {+ citations: CitationSource[];+ result_footer?: string;+ };+ };+};++const { rows: STORED } = JSON.parse(+ readFileSync(resolve(process.cwd(), '../tests/fixtures/citation_consumer_bridge.json'), 'utf8'),+) as { rows: StoredRow[] };++const i18n = createInstance();+void i18n.use(initReactI18next).init({+ lng: 'en',+ fallbackLng: 'en',+ resources: { en: { translation: en } },+ interpolation: { escapeValue: false },+});++afterEach(cleanup);++const mount = (ui: ReactElement) => render(+ <I18nextProvider i18n={i18n}>+ <ToastProvider>+ <ApiProvider>+ <MemoryRouter>{ui}</MemoryRouter>+ </ApiProvider>+ </ToastProvider>+ </I18nextProvider>,+);++const session = (): WorkbenchSession =>+ ({+ id: 'ses_bridge_rows',+ scope_id: 'scope-1',+ project_id: 'proj-1',+ title: 'Citations',+ agent_id: 'agt-1',+ agent_name: 'codex',+ agent_backend: 'codex',+ status: 'active',+ agent_status: 'idle',+ created_at: '2026-09-21T12:00:00Z',+ updated_at: '2026-09-21T12:00:00Z',+ metadata: {},+ }) as WorkbenchSession;++/** The stored row as the workbench receives it over the wire. */+const message = (entry: StoredRow, text = entry.row.text): WorkbenchMessage =>+ ({+ id: `msg_${entry.key}`,+ type: entry.row.type,+ author: 'agent',+ source: 'agent',+ text,+ content: entry.row.content,+ metadata: {},+ created_at: '2026-09-21T12:00:00Z',+ }) as unknown as WorkbenchMessage;++const links = (container: HTMLElement) =>+ Array.from(container.querySelectorAll<HTMLAnchorElement>('a[href^="http"]'));++const badged = (anchor: HTMLAnchorElement) => anchor.hasAttribute('data-citation-index');++const results = STORED.filter((entry) => entry.type === 'result').map(+ (entry) => [entry.key, entry] as const,+);++describe('a stored transcript row', () => {+ it.each(results)('%s badges the link the backend wrote and no other', (_key, entry) => {+ const { container } = mount(+ <MessageRow message={message(entry)} session={session()} messageFontSize={13} />,+ );+ const [cite] = entry.row.content.citations;+ const anchors = links(container);++ // Both links point at the same page and are spelled identically; only the+ // second one was written by the backend. A consumer that lost the ranges+ // and matched on text would badge both, and one of them would be crediting+ // the search result for a sentence the agent wrote itself.+ expect(anchors).toHaveLength(2);+ expect(anchors.map((anchor) => anchor.getAttribute('href'))).toEqual([cite.url, cite.url]);+ expect(anchors.map(badged)).toEqual([false, true]);+ expect(anchors[0].textContent).toBe(cite.label);+ });++ it.each(results)('%s shows its footer outside the body it measured', (_key, entry) => {+ const { container } = mount(+ <MessageRow message={message(entry)} session={session()} messageFontSize={13} />,+ );+ const footer = entry.row.content.result_footer;+ if (!footer) {+ expect(container.textContent).not.toContain('⏱️');+ return;+ }++ // The footer is shown once — as the row's own footer, never as part of the+ // answer — and the badge above survived it being taken out of the body.+ const shown = footer.replace(/^✅\s*/, '');+ expect(container.textContent?.split(shown)).toHaveLength(2);+ expect(container.querySelector('.vr-markdown')?.textContent ?? '').not.toContain(shown);+ expect(links(container).filter(badged)).toHaveLength(1);+ });++ it.each(results)('%s keeps every link when the row no longer matches', (_key, entry) => {+ // An edit after the row was written, a truncation, a row read against+ // another message: the ranges still land on real links here, which is+ // exactly why the digest is what decides.+ const { container } = mount(+ <MessageRow+ message={message(entry, `${entry.row.text}\n\nEdited.`)}+ session={session()}+ messageFontSize={13}+ />,+ );+ const anchors = links(container);++ expect(anchors).toHaveLength(2);+ expect(anchors.filter(badged)).toHaveLength(0);+ expect(anchors.every((anchor) => anchor.getAttribute('href') === entry.row.content.citations[0].url))+ .toBe(true);+ });++ it('carries the measurement through every edit stacked above the citation', () => {+ // Two rows, one reply, and the transforms that ran on it in order. The+ // backend rewrote an attachment before it measured anything — to a proxy+ // link on the workbench, to a bare label on IM — so the ranges it wrote+ // already describe a body the model never produced. Then the reader edits+ // that body again: a `$<NAME>` marker becomes a secure-input card above the+ // citation, and an IM row's folded footer is cut off below it. The badge+ // has to come out of all of that on the link the backend actually wrote.+ for (const key of ['workbench_row_rewritten_and_carded', 'im_row_flattened_carded_and_folded']) {+ const entry = STORED.find((row) => row.key === key) as StoredRow;+ const { container } = mount(+ <MessageRow message={message(entry)} session={session()} messageFontSize={13} />,+ );+ const body = container.querySelector('.vr-markdown') as HTMLElement;++ // The card replaced the marker with something 23 code units longer, so a+ // range carried across unchanged would miss its link by that much.+ expect(body.textContent).not.toContain('$<deployKey>');+ expect(body.querySelector('a[href^="avibe-secret:"], button')).not.toBeNull();+ expect(body.textContent).toContain('deployKey');+ // …and the attachment, in whichever form this surface stored it.+ const attachment = body.querySelector<HTMLAnchorElement>('a[href^="/api/media/"]');+ if (entry.platform === 'avibe') expect(attachment).not.toBeNull();+ else expect(attachment).toBeNull();+ expect(body.textContent).toContain('report');++ const anchors = links(container);+ expect(anchors.map(badged)).toEqual([false, true]);+ expect(anchors[1].getAttribute('href')).toBe(entry.row.content.citations[0].url);+ expect(anchors[1].textContent).toBe(String(entry.row.content.citations[0].index));+ cleanup();+ }+ });++ it.each(results)('%s draws no badge on a row the agent did not write', (_key, entry) => {+ // The sidecar travels in ``content``, and a quoted or user-authored row can+ // carry the same content shape. A numbered source badge asserts that the+ // ANSWER cited that page, so authorship gates it before the ranges matter.+ const { container } = mount(+ <MessageRow+ message={{ ...message(entry), author: 'user', type: 'user' } as WorkbenchMessage}+ session={session()}+ messageFontSize={13}+ />,+ );+ expect(links(container).filter(badged)).toHaveLength(0);+ });+});++describe('a stored narration row', () => {+ const entry = STORED.find((row) => row.type === 'assistant') as StoredRow;++ it.each(['live', 'history'])('badges the same link on the %s activity surface', (surface) => {+ const rows = [activityRowFromMessage(message(entry))];+ const { container } = mount(surface === 'live' ? (+ <ActivityCard+ rows={rows}+ startedAtMs={Date.parse('2026-09-21T12:00:00Z')}+ expanded+ onToggleExpanded={vi.fn()}+ showToolCalls+ onToggleTools={vi.fn()}+ />+ ) : (+ <ActivityChip+ group={{+ id: 'turn', anchorMessageId: 'reply', anchorPosition: 'before',+ open: false, status: 'done', steps: 1, durationMs: 1000, rows,+ }}+ expanded+ loading={false}+ onToggle={vi.fn()}+ showToolCalls+ onToggleTools={vi.fn()}+ />+ ));+ const [cite] = entry.row.content.citations;+ const anchors = links(container);++ // A narration row is rendered verbatim — no footer, no split — so the body+ // the sidecar was measured in IS the body handed to the renderer.+ expect(anchors.map(badged)).toEqual([false, true]);+ expect(anchors[1].getAttribute('href')).toBe(cite.url);+ expect(anchors[1].textContent).toBe(String(cite.index));+ });+});diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.jsonindex 37c34e93b..f6150578c 100644--- a/ui/src/i18n/en.json+++ b/ui/src/i18n/en.json@@ -4196,6 +4196,11 @@"vaultExpired": "Expired","agentFallback": "agent"},+ "citation": {+ "badge": "Source {{index}}: {{source}}",+ "badgeWithDomain": "Source {{index}}: {{source}} ({{domain}})",+ "open": "Open source page"+ },"annotation": {"titleUser": "User annotation","titleAgent": "Agent annotation",diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.jsonindex 6e539644a..056f9a23c 100644--- a/ui/src/i18n/zh.json+++ b/ui/src/i18n/zh.json@@ -4196,6 +4196,11 @@"vaultExpired": "已过期","agentFallback": "智能体"},+ "citation": {+ "badge": "来源 {{index}}:{{source}}",+ "badgeWithDomain": "来源 {{index}}:{{source}}({{domain}})",+ "open": "打开来源页面"+ },"annotation": {"titleUser": "用户批注","titleAgent": "Agent 批注",diff --git a/ui/src/index.css b/ui/src/index.cssindex e29a7d0cc..db735e512 100644--- a/ui/src/index.css+++ b/ui/src/index.css@@ -861,6 +861,20 @@ pre,text-decoration: underline;text-underline-offset: 2px;}+/* A citation badge is an <a> too (see citation-badge.tsx), so the rule above+ would repaint it cyan and underline it — the one thing a compact numbered chip+ must not look like. Unlayered rules outrank Tailwind's utilities layer, so the+ badge's text color has to be restated here; every other class it carries+ (border, background, shape, focus ring) is unaffected and stays in the+ component. */+.vr-markdown a.vr-citation {+ color: var(--muted);+ text-decoration: none;+}+.vr-markdown a.vr-citation:hover,+.vr-markdown a.vr-citation:focus-visible {+ color: var(--foreground);+}.vr-markdown code {font-family: var(--font-mono);font-size: 0.88em;diff --git a/ui/src/lib/agentActivity.test.ts b/ui/src/lib/agentActivity.test.tsindex 56dd3c020..93dc73c14 100644--- a/ui/src/lib/agentActivity.test.ts+++ b/ui/src/lib/agentActivity.test.ts@@ -353,6 +353,34 @@ describe('groupFromWire', () => {expect(group.rows?.[0].kind).toBe('assistant');});+ it('carries an assistant row\'s citation sidecar through the wire shape', () => {+ // A narration row is rendered through Activity, so its sources have to+ // survive the wire or the badge degrades to the bare domain link.+ const group = groupFromWire({+ id: 'm_c1',+ anchor_message_id: 'm_r1',+ anchor_position: 'before',+ open: false,+ status: 'done',+ steps: 2,+ duration_ms: 1000,+ rows: [+ {+ id: 'm_c1',+ kind: 'assistant',+ text: 'Cited [example.com](https://example.com/x)',+ created_at: '2026-06-01T10:00:01Z',+ citations: [{ index: 1, ref_id: 'turn0view0', url: 'https://example.com/x', label: 'example.com' }],+ },+ { id: 'm_c2', kind: 'assistant', text: 'Plain', created_at: '2026-06-01T10:00:02Z' },+ ],+ });+ expect(group.rows?.[0].citations).toEqual([+ { index: 1, ref_id: 'turn0view0', url: 'https://example.com/x', label: 'example.com' },+ ]);+ expect(group.rows?.[1].citations).toBeUndefined();+ });+it('maps an open interrupted group anchored after its trigger', () => {const group = groupFromWire({id: 'e_t1',@@ -420,6 +448,22 @@ describe('activityRowFromMessage', () => {const tool = activityRowFromMessage({ id: 'e1', type: 'tool_call', text: '🔧 `Bash`', created_at: 't2' } as WorkbenchMessage);expect(tool.kind).toBe('tool_call');});++ it('lifts the citation sidecar out of the message content envelope', () => {+ const cited = activityRowFromMessage({+ id: 'm2',+ type: 'assistant',+ text: 'Cited [example.com](https://example.com/x)',+ created_at: 't3',+ content: { citations: [{ index: 1, ref_id: 'turn0view0', url: 'https://example.com/x', label: 'example.com' }] },+ } as unknown as WorkbenchMessage);+ expect(cited.citations).toEqual([+ { index: 1, ref_id: 'turn0view0', url: 'https://example.com/x', label: 'example.com' },+ ]);+ expect(+ activityRowFromMessage({ id: 'm3', type: 'assistant', text: 'Plain', created_at: 't4' } as WorkbenchMessage).citations,+ ).toBeUndefined();+ });});describe('liveActivityReducer (generation invariant)', () => {diff --git a/ui/src/lib/agentActivity.ts b/ui/src/lib/agentActivity.tsindex d34ca9f48..044cbf21e 100644--- a/ui/src/lib/agentActivity.ts+++ b/ui/src/lib/agentActivity.ts@@ -1,4 +1,5 @@import type { WorkbenchMessage } from '../context/ApiContext';+import type { CitationSource } from './citations';import { specFor } from './messageTypes';import { timestampOrderTimeMs } from './transcriptOrder';@@ -12,6 +13,9 @@ export type ActivityRow = {text: string;created_at: string;order_micros?: number; // authoritative durable key; absent on live SSE rows+ // Source sidecar for an agent narration row whose text carries citation links+ // (see lib/citations). Absent on every other row, live or durable.+ citations?: CitationSource[];};// Storage owns persisted order, including clocks recovered from migration metadata.@@ -83,7 +87,14 @@ export type TurnActivityGroupWire = {duration_ms: number | null;started_at?: string | null;ended_at?: string | null;- rows?: Array<{ id: string; kind: 'assistant' | 'tool_call'; text: string; created_at: string; order_micros?: number }>;+ rows?: Array<{+ id: string;+ kind: 'assistant' | 'tool_call';+ text: string;+ created_at: string;+ order_micros?: number;+ citations?: CitationSource[];+ }>;};export const groupFromWire = (wire: TurnActivityGroupWire): ActivityGroup => ({@@ -97,6 +108,7 @@ export const groupFromWire = (wire: TurnActivityGroupWire): ActivityGroup => ({startedAt: wire.started_at ?? null,rows: wire.rows?.map((r) => ({id: r.id, kind: r.kind, text: r.text, created_at: r.created_at, order_micros: r.order_micros,+ citations: r.citations,})),});@@ -107,6 +119,7 @@ export const activityRowFromMessage = (msg: WorkbenchMessage): ActivityRow => ({kind: msg.type === 'tool_call' ? 'tool_call' : 'assistant',text: msg.text ?? '',created_at: msg.created_at,+ citations: (msg.content as { citations?: CitationSource[] } | null | undefined)?.citations,});// ===== Live running-card buffer: a pure state machine (state, not timing) =====diff --git a/ui/src/lib/citations.test.ts b/ui/src/lib/citations.test.tsnew file mode 100644index 000000000..8bd9481e4--- /dev/null+++ b/ui/src/lib/citations.test.ts@@ -0,0 +1,219 @@+// The citation binding, tested as the pure decision it is.+//+// `citation-bridge.test.tsx` renders a real producer run through the real+// component, which is what proves the contract end to end. This file covers the+// answers that file cannot reach from a recording: a digest agreeing with+// Python's over text no producer case contains, a span the renderer must refuse,+// and an edit that reaches into a citation — which the real passes, on the real+// producer's output, never do (the producer percent-encodes `<`/`>`, so only+// prose outside the link ever holds a live marker). The rule still has to hold,+// because the next transform added to that surface may not be so careful.++import { readFileSync } from 'node:fs';+import { resolve } from 'node:path';++import { describe, expect, it } from 'vitest';++import {+ bindCitations,+ bodyDigest,+ remapCitations,+ type CitationSource,+ type TextEdit,+} from './citations';++// Written by the Python side, which computes the same digest with hashlib over+// the same UTF-16LE code units. Read at runtime: it holds unpaired surrogates,+// which is the whole reason it exists, and they survive as JSON escapes.+const DIGEST_VECTORS = JSON.parse(+ readFileSync(resolve(process.cwd(), '../tests/fixtures/citation_body_digest.json'), 'utf8'),+) as Array<{ name: string; text: string; sha256: string }>;++const BODY = 'Cited. [example.com](https://example.com/x) and more.';+const SPAN: [number, number] = [7, 43];++const row = (over: Partial<CitationSource> = {}): CitationSource => ({+ index: 1,+ ref_id: 'turn0view0',+ title: 'Example',+ url: 'https://example.com/x',+ label: 'example.com',+ spans: [[...SPAN]],+ body_sha256: bodyDigest(BODY),+ ...over,+});++describe('bodyDigest', () => {+ it.each(DIGEST_VECTORS.map((v) => [v.name, v] as const))(+ 'agrees with the backend on %s',+ (_name, vector) => {+ expect(bodyDigest(vector.text)).toBe(vector.sha256);+ },+ );++ it('measures the span the sidecar names', () => {+ // The unit the two ends count in: a JavaScript string index is a UTF-16 code+ // unit, which is what `core.citations` encodes before it measures.+ expect(BODY.slice(...SPAN)).toBe('[example.com](https://example.com/x)');+ });++ it('separates two bodies that differ only past the astral plane', () => {+ expect(bodyDigest('🙂')).not.toBe(bodyDigest('🙃'));+ });+});++describe('bindCitations', () => {+ it('binds a row measured in this exact body', () => {+ const binding = bindCitations([row()], BODY);++ expect(binding?.bound).toHaveLength(1);+ expect(binding?.bound[0].spans).toEqual([SPAN]);+ expect(binding?.legacy).toEqual([]);+ });++ it('has nothing to say without a sidecar', () => {+ expect(bindCitations(undefined, BODY)).toBeNull();+ expect(bindCitations([], BODY)).toBeNull();+ });++ it('refuses the whole sidecar when the body is not the one it describes', () => {+ // The paragraph above the citation was deleted, and the surviving link now+ // sits exactly where the citation used to. Only the digest can tell.+ expect(bindCitations([row()], `${BODY} Edited.`)).toBeNull();+ expect(bindCitations([row({ body_sha256: undefined })], BODY)).toBeNull();+ });++ it('refuses a span this body cannot hold', () => {+ for (const spans of [+ [[7]],+ [[7, 43, 44]],+ [[-1, 43]],+ [[43, 7]],+ [[7, 7]],+ [[7, BODY.length + 1]],+ [[7.5, 43]],+ [],+ 'nope',+ ]) {+ expect(bindCitations([row({ spans: spans as number[][] })], BODY)).toBeNull();+ }+ });++ it('refuses two rows that claim the same characters', () => {+ // One link is one citation. Overlapping claims mean the measurement is not+ // describing this text, whatever else it describes.+ const overlap = [row(), row({ index: 2, spans: [[20, 43]] })];++ expect(bindCitations(overlap, BODY)).toBeNull();+ });++ it('accepts two rows that sit side by side', () => {+ const body = `${BODY}\n\n[example.org](https://example.org/y)`;+ const first = row({ body_sha256: bodyDigest(body) });+ const second = row({+ index: 2,+ url: 'https://example.org/y',+ label: 'example.org',+ spans: [[54, 90]],+ body_sha256: bodyDigest(body),+ });++ expect(bindCitations([first, second], body)?.bound).toHaveLength(2);+ });++ it('drops a row that could never be an anchor, and keeps the rest', () => {+ const body = `${BODY}\n\n[example.org](https://example.org/y)`;+ const digest = bodyDigest(body);+ const unusable = row({ index: 2, url: 'javascript:alert(1)', body_sha256: digest });++ const binding = bindCitations([row({ body_sha256: digest }), unusable], body);++ // It names no link this renderer will draw, so its absence changes nothing+ // about the body — unlike a row that disagrees with the body it names.+ expect(binding?.bound.map((entry) => entry.source.index)).toEqual([1]);+ });++ it('reads a row persisted before the contract existed', () => {+ const legacy = row({ spans: undefined, body_sha256: undefined });++ expect(bindCitations([legacy], BODY)).toEqual({ bound: [], legacy: [legacy] });+ });++ it('refuses a sidecar that mixes the two', () => {+ // Half of it describes this body and half of it describes nothing. Picking+ // the half to believe is the guess this contract exists to avoid.+ const mixed = [row(), row({ index: 2, spans: undefined, body_sha256: undefined })];++ expect(bindCitations(mixed, BODY)).toBeNull();+ });+});++describe('remapCitations', () => {+ const bound = () => bindCitations([row()], BODY);++ const remapped = (edits: TextEdit[]) => remapCitations(bound(), edits)?.bound[0]?.spans;++ it('carries a span across an edit that lands before it', () => {+ // A chip minted from a 6-unit marker in the prose above becomes a 40-unit+ // link, and the citation moves the 34 units that pass added.+ expect(remapped([{ start: 0, end: 6, inserted: 40 }])).toEqual([[41, 77]]);+ });++ it('carries a span across an edit that shortens the text', () => {+ expect(remapped([{ start: 0, end: 7, inserted: 2 }])).toEqual([[2, 38]]);+ });++ it('leaves a span alone when the edit is after it', () => {+ expect(remapped([{ start: 44, end: 52, inserted: 0 }])).toEqual([SPAN]);+ });++ it('composes several edits in one pass', () => {+ expect(remapped([+ { start: 0, end: 2, inserted: 5 },+ { start: 44, end: 45, inserted: 9 },+ ])).toEqual([[10, 46]]);+ });++ it('treats an insertion at the span edges as outside it', () => {+ // Written where the link begins, so the link begins just after it...+ expect(remapped([{ start: SPAN[0], end: SPAN[0], inserted: 4 }])).toEqual([[11, 47]]);+ // ...and written where it ends, so the link is unmoved.+ expect(remapped([{ start: SPAN[1], end: SPAN[1], inserted: 4 }])).toEqual([SPAN]);+ });++ it('invalidates a citation an edit reached into', () => {+ // A card minted inside the destination would take the address apart, and+ // there is no honest answer to which half kept the attribution.+ for (const edit of [+ { start: 10, end: 14, inserted: 30 },+ { start: 6, end: 8, inserted: 2 },+ { start: 42, end: 44, inserted: 2 },+ { start: 0, end: BODY.length, inserted: 3 },+ { start: 20, end: 20, inserted: 7 },+ ]) {+ expect(remapCitations(bound(), [edit])).toBeNull();+ }+ });++ it('drops one citation whole when any of its spans is invalidated', () => {+ const body = `${BODY}\n\n[example.com](https://example.com/x)`;+ const twice = row({ spans: [[7, 43], [54, 90]], body_sha256: bodyDigest(body) });++ expect(remapCitations(bindCitations([twice], body), [{ start: 60, end: 62, inserted: 0 }]))+ .toBeNull();+ });++ it('keeps a legacy row, which is matched on text rather than position', () => {+ const legacy = row({ spans: undefined, body_sha256: undefined });++ expect(remapCitations(bindCitations([legacy], BODY), [{ start: 0, end: 9, inserted: 41 }]))+ .toEqual({ bound: [], legacy: [legacy] });+ });++ it('passes an empty edit list straight through', () => {+ const binding = bound();++ expect(remapCitations(binding, [])).toBe(binding);+ expect(remapCitations(null, [{ start: 0, end: 1, inserted: 0 }])).toBeNull();+ });+});diff --git a/ui/src/lib/citations.ts b/ui/src/lib/citations.tsnew file mode 100644index 000000000..b0f45e91f--- /dev/null+++ b/ui/src/lib/citations.ts@@ -0,0 +1,341 @@+// Shared contract for agent source citations.+//+// Some agent backends answer with opaque inline citation markers instead of+// links. The backend resolves each marker against the search results it already+// received and rewrites it into an ordinary Markdown link, plus a structured+// sidecar in `message.content.citations` (see core/citations.py). The delivered+// text is therefore already complete on every surface — an IM platform shows a+// normal clickable link — so the sidecar adds no attribution of its own: it is+// what lets this renderer RECOGNIZE those links and draw them as compact+// numbered badges with a source preview, with nothing re-parsed and nothing+// fetched.+//+// Recognizing one is a provenance question rather than a text question. A badge+// is an attribution claim, so it belongs to a link the backend actually wrote —+// not to any link the answer's own prose happens to point at the same page. The+// backend knows exactly which characters it wrote and where, so what travels is+// where those links SIT: half-open `[start, end)` ranges in UTF-16 code units.+//+// A range alone is not an identity, though. Two ordinary links can be spelled+// the same, and deleting a paragraph moves one of them into the range another+// occupied: with `[example.com](https://example.com/x)` written twice, the+// second link sits at `[38, 74)` and the first at `[0, 36)`, so dropping the+// first paragraph leaves an ordinary link at `[0, 36)` wearing the citation's+// coordinates. So every row also carries `body_sha256`, the digest of the exact+// body its ranges were measured in, and this file verifies that digest BEFORE+// it trusts a single range. The digest says which VERSION of the body is meant;+// it is computed locally on both ends and claims nothing about where the text+// came from.+//+// The body the renderer is finally handed is not always the body the backend+// measured: the transcript strips a generated result footer, and `@<…>` chips+// and `$<NAME>` cards are minted by rewriting the source before Markdown parses+// it. Each of those passes reports what it changed as `TextEdit`s, and the+// ranges are carried across them (`remapCitations`). A pass that edits INTO a+// citation's own range invalidates that citation rather than guessing which+// half of the result the attribution belongs to.++import { sha256 } from '@noble/hashes/sha2.js';+import { bytesToHex } from '@noble/hashes/utils.js';++export type CitationSource = {+ /** 1-based first-appearance order within the message; the badge's label. */+ index: number;+ /** The backend-native reference id the marker carried (diagnostics only). */+ ref_id?: string;+ /** The cited page's title, as the backend's search returned it. Untrusted. */+ title?: string;+ /** Exactly the destination the backend wrote into the Markdown link. */+ url: string;+ /** Exactly the link text the backend used — the source's domain. */+ label: string;+ /** Half-open `[start, end)` ranges, in UTF-16 code units, of the links this+ * citation wrote in the body `body_sha256` names. Absent — together with+ * `body_sha256` — on rows persisted before this contract existed. */+ spans?: number[][];+ /** SHA-256 over that body's UTF-16LE code units, lowercase hex. */+ body_sha256?: string;+};++/**+ * One replacement a pass made to the text it was handed.+ *+ * `start`/`end` are half-open coordinates in that pass's INPUT — not in the+ * original body and not in the pass's output — and `inserted` is how many code+ * units were written in place of the `end - start` that were removed. A+ * deletion inserts 0, an insertion removes 0, and either side may be longer:+ * the two linkify passes are replacements, not insertions, so a length change+ * of any sign has to be carried.+ *+ * Passes compose in the order they ran: each pass's edits are expressed in the+ * output of the pass before it, so `remapCitations` is called once per pass,+ * in that same order.+ */+export type TextEdit = {+ start: number;+ end: number;+ inserted: number;+};++/** The result of a pass that rewrites text and says what it changed. */+export type EditedText = {+ text: string;+ edits: TextEdit[];+};++/** One citation, located in the text a renderer is currently holding. */+export type BoundCitation = {+ source: CitationSource;+ /** Half-open `[start, end)` ranges in that text, in UTF-16 code units. */+ spans: Array<[number, number]>;+};++/**+ * What a renderer needs to draw badges on ONE exact text.+ *+ * `bound` rows are matched positionally and are only meaningful for the text+ * they were last remapped onto. `legacy` rows carry no ranges at all — they+ * were persisted before this contract existed — and keep the match they were+ * written for, the exact (destination, link text) pair.+ */+export type CitationBinding = {+ bound: BoundCitation[];+ legacy: CitationSource[];+};++/** The UTF-16LE bytes of `text`, unpaired surrogates included. */+function utf16leBytes(text: string): Uint8Array {+ const bytes = new Uint8Array(text.length * 2);+ for (let at = 0; at < text.length; at += 1) {+ // `charCodeAt` is the code UNIT, which is what the ranges count and what+ // Python's `surrogatepass` encode writes for an unpaired one.+ const unit = text.charCodeAt(at);+ bytes[at * 2] = unit & 0xff;+ bytes[at * 2 + 1] = unit >>> 8;+ }+ return bytes;+}++/**+ * The digest a sidecar binds its ranges to: SHA-256 over UTF-16LE code units,+ * lowercase hex — byte for byte what `core.citations.body_digest` computes.+ */+export function bodyDigest(text: string): string {+ return bytesToHex(sha256(utf16leBytes(text)));+}++/** Reject anything a stored row could hold that must never become an anchor. */+function isUsable(value: unknown): value is CitationSource {+ if (!value || typeof value !== 'object') return false;+ const candidate = value as Partial<CitationSource>;+ if (typeof candidate.url !== 'string' || typeof candidate.label !== 'string') return false;+ if (!Number.isFinite(candidate.index)) return false;+ // The backend already allows only http(s); re-check here because this row came+ // back from storage, and a bad one must degrade to plain text, not to a link.+ return /^https?:\/\//i.test(candidate.url);+}++/** True once a row claims the positional contract, however badly. */+function carriesProvenance(row: CitationSource): boolean {+ return row.spans !== undefined || row.body_sha256 !== undefined;+}++/** The row's ranges, or null if any of them is not one this text can hold. */+function readSpans(raw: unknown, limit: number): Array<[number, number]> | null {+ if (!Array.isArray(raw) || raw.length === 0) return null;+ const spans: Array<[number, number]> = [];+ for (const span of raw) {+ if (!Array.isArray(span) || span.length !== 2) return null;+ const [start, end] = span as unknown[];+ if (!Number.isInteger(start) || !Number.isInteger(end)) return null;+ if ((start as number) < 0 || (end as number) <= (start as number) || (end as number) > limit) {+ return null;+ }+ spans.push([start as number, end as number]);+ }+ return spans;+}++/**+ * Read a stored sidecar against the exact body it was measured in.+ *+ * Returns null when nothing may be drawn, which is the same answer as "no+ * sidecar": the delivered text already carries every link and every label, so+ * refusing costs the reader the preview and nothing else. It is the answer for+ * a digest that does not match, a range outside the body, two ranges that+ * overlap, a row that claims the contract but cannot state it, and a sidecar+ * that mixes rows that claim it with rows that do not — in every one of those+ * the sidecar and the body disagree, and choosing a survivor would be a guess.+ * A failure here never falls back to the legacy match; that match is only for+ * rows that never carried the contract at all.+ */+export function bindCitations(+ citations: readonly unknown[] | undefined,+ body: string,+): CitationBinding | null {+ if (!citations?.length) return null;+ const usable = citations.filter(isUsable);+ if (!usable.length) return null;++ const provenance = usable.filter(carriesProvenance);+ if (!provenance.length) return { bound: [], legacy: usable };+ if (provenance.length !== usable.length) return null;++ const digest = bodyDigest(body);+ const bound: BoundCitation[] = [];+ const claimed: Array<[number, number]> = [];+ for (const source of provenance) {+ if (source.body_sha256 !== digest) return null;+ const spans = readSpans(source.spans, body.length);+ if (!spans) return null;+ bound.push({ source, spans });+ claimed.push(...spans);+ }+ // Each range is one link, so no two of them may share a character.+ claimed.sort((left, right) => left[0] - right[0]);+ for (let at = 1; at < claimed.length; at += 1) {+ if (claimed[at][0] < claimed[at - 1][1]) return null;+ }+ return { bound, legacy: [] };+}++/** Where `[start, end)` lands after `edits`, or null if an edit reached into it. */+function remapSpan(+ start: number,+ end: number,+ ordered: readonly TextEdit[],+): [number, number] | null {+ let shift = 0;+ for (const edit of ordered) {+ // Entirely before: the range moves by however much this edit changed the+ // length. An insertion AT `start` is before the range — it is written where+ // the range begins, so the link still begins just after it.+ if (edit.end <= start) {+ shift += edit.inserted - (edit.end - edit.start);+ continue;+ }+ // Entirely after, and the rest are further right still.+ if (edit.start >= end) break;+ return null;+ }+ return [start + shift, end + shift];+}++/**+ * Carry a binding across one pass's edits, into the text that pass produced.+ *+ * A citation whose range the pass edited into is dropped: the characters the+ * backend pointed at are no longer the characters on screen, and deciding+ * which part of the replacement inherited the attribution would be guessing.+ * `legacy` rows match on text rather than position and are unaffected.+ */+export function remapCitations(+ binding: CitationBinding | null | undefined,+ edits: readonly TextEdit[],+): CitationBinding | null {+ if (!binding) return null;+ if (!edits.length) return binding;+ const ordered = [...edits].sort((left, right) => left.start - right.start);+ const bound: BoundCitation[] = [];+ for (const entry of binding.bound) {+ const moved: Array<[number, number]> = [];+ for (const [start, end] of entry.spans) {+ const next = remapSpan(start, end, ordered);+ if (!next) {+ moved.length = 0;+ break;+ }+ moved.push(next);+ }+ if (moved.length) bound.push({ source: entry.source, spans: moved });+ }+ if (!bound.length && !binding.legacy.length) return null;+ return { bound, legacy: binding.legacy };+}++// The annotation travels from the remark plugin to the renderer as hast data+// attributes, which is the one channel mdast → hast preserves verbatim.+const SPAN_START = 'dataCitationStart';+const SPAN_END = 'dataCitationEnd';++/** The parts of an mdast node this counts on, none of which it can assume. */+type MarkdownNode = {+ type?: unknown;+ position?: { start?: { offset?: unknown }; end?: { offset?: unknown } };+ data?: { hProperties?: Record<string, unknown> };+ children?: unknown;+};++/** Every link node in document order (pre-order is source order for inlines). */+function eachLink(node: MarkdownNode, visit: (link: MarkdownNode) => void): void {+ if (node.type === 'link') visit(node);+ if (!Array.isArray(node.children)) return;+ for (const child of node.children) eachLink(child as MarkdownNode, visit);+}++/**+ * Record where in the source each inline link is written, so `findCitation`+ * can recognize the links the backend wrote.+ *+ * Unist offsets are already UTF-16 code units — a JavaScript string index —+ * which is the unit the sidecar's ranges are counted in. Only an inline link+ * is annotated: a reference link, an autolink and an image spell their+ * destination some other way, and the backend did not write any of them.+ */+export function remarkCitationSpans() {+ return (tree: unknown) => {+ eachLink(tree as MarkdownNode, (link) => {+ const start = link.position?.start?.offset;+ const end = link.position?.end?.offset;+ if (typeof start !== 'number' || typeof end !== 'number') return;+ const data = (link.data ??= {});+ const properties = (data.hProperties ??= {});+ properties[SPAN_START] = start;+ properties[SPAN_END] = end;+ });+ };+}++/** The range `remarkCitationSpans` left on the rendered link. */+function linkSpan(node: unknown): [number, number] | null {+ const properties = (node as { properties?: Record<string, unknown> } | undefined)?.properties;+ if (!properties) return null;+ const start = properties[SPAN_START];+ const end = properties[SPAN_END];+ if (typeof start !== 'number' || typeof end !== 'number') return null;+ return [start, end];+}++/**+ * The citation a rendered Markdown link stands for, or null for an ordinary link.+ *+ * `node` is the hast element the renderer was handed; `href` and `text` are the+ * destination it resolved and the link's visible text, used only by the legacy+ * match. A bound row is recognized by position alone, and the position has to+ * be the WHOLE link: the backend measured the complete `[label](url)` it wrote,+ * so a node covering anything else — a longer link that merely starts there, a+ * fragment of one — simply matches no range.+ */+export function findCitation(+ binding: CitationBinding | null | undefined,+ node: unknown,+ href: string,+ text: string,+): CitationSource | null {+ if (!binding) return null;+ const span = linkSpan(node);+ if (span) {+ for (const entry of binding.bound) {+ for (const [start, end] of entry.spans) {+ if (start === span[0] && end === span[1]) return entry.source;+ }+ }+ }+ // A row persisted before this contract existed still has to render, so it+ // keeps the match it was written for: the exact (destination, link text) pair.+ if (!href || !text) return null;+ for (const candidate of binding.legacy) {+ if (candidate.url === href && candidate.label === text) return candidate;+ }+ return null;+}diff --git a/ui/src/lib/mentions.test.ts b/ui/src/lib/mentions.test.tsnew file mode 100644index 000000000..8c3aa3a5c--- /dev/null+++ b/ui/src/lib/mentions.test.ts@@ -0,0 +1,95 @@+import { describe, expect, it } from 'vitest';++import { linkifyMentions, type MentionReference } from './mentions';++const REFERENCES: MentionReference[] = [+ { kind: 'agent', name: 'claude' },+ { kind: 'session', session_id: 'ses6jr7c5h2q6', title: 'Bridge' },+];++/**+ * Check that the pass's edits describe the rewrite it performed.+ *+ * A citation's ranges cross this pass on the strength of these numbers alone+ * (see lib/citations), so "close enough" is not a thing they can be. Each edit+ * names a range of the INPUT and how many units replaced it; walking them in+ * order therefore reconstructs which parts of the output are untouched input,+ * and those parts have to be exactly the input's own.+ */+const verifyEdits = (input: string, output: string, edits: readonly {+ start: number; end: number; inserted: number;+}[]) => {+ let read = 0;+ let write = 0;+ const replaced: Array<[string, string]> = [];+ for (const edit of edits) {+ expect(edit.start).toBeGreaterThanOrEqual(read);+ expect(edit.end).toBeGreaterThanOrEqual(edit.start);+ // Everything since the previous edit was carried over verbatim.+ expect(output.slice(write, write + (edit.start - read)))+ .toBe(input.slice(read, edit.start));+ write += edit.start - read;+ replaced.push([input.slice(edit.start, edit.end), output.slice(write, write + edit.inserted)]);+ read = edit.end;+ write += edit.inserted;+ }+ expect(output.slice(write)).toBe(input.slice(read));+ expect(output.length).toBe(write + (input.length - read));+ return replaced;+};++describe('linkifyMentions', () => {+ it('reports every marker it rewrote, in the coordinates it was handed', () => {+ const input = 'Ask @<claude> about #<ses6jr7c5h2q6>, then @<claude> again.';+ const { text, edits } = linkifyMentions(input, REFERENCES);++ const replaced = verifyEdits(input, text, edits);+ expect(replaced.map(([from]) => from))+ .toEqual(['@<claude>', '#<ses6jr7c5h2q6>', '@<claude>']);+ // The session chip shows the title rather than the id, so the two markers+ // change length by different amounts and neither can be assumed.+ expect(replaced.map(([, to]) => to)).toEqual([+ '[@claude](avibe-mention:agent:claude)',+ '[#Bridge](avibe-mention:session:ses6jr7c5h2q6)',+ '[@claude](avibe-mention:agent:claude)',+ ]);+ });++ it('reports nothing when there is nothing to rewrite', () => {+ const input = 'No markers here, just an [email protected] and a # sign.';+ const { text, edits } = linkifyMentions(input, REFERENCES);++ expect(text).toBe(input);+ expect(edits).toEqual([]);+ });++ it('skips markers inside code, and still measures the ones outside it', () => {+ // The offsets after a skipped code span are the ones a naive per-segment+ // counter gets wrong, so a marker on each side of one is the case worth+ // measuring.+ const input = 'Type `@<claude>` to reach @<claude>, or ```\n#<ses6jr7c5h2q6>\n``` first.';+ const { text, edits } = linkifyMentions(input, REFERENCES);++ const replaced = verifyEdits(input, text, edits);+ expect(replaced.map(([from]) => from)).toEqual(['@<claude>']);+ expect(text).toContain('`@<claude>`');+ expect(text).toContain('#<ses6jr7c5h2q6>\n```');+ });++ it('measures in UTF-16 code units, past an astral character', () => {+ const input = '🙂 ask @<claude>.';+ const { text, edits } = linkifyMentions(input, REFERENCES);++ verifyEdits(input, text, edits);+ expect(edits[0].start).toBe(7);+ expect(input.slice(edits[0].start, edits[0].end)).toBe('@<claude>');+ });++ it('falls back to the session id when no title was captured', () => {+ const input = 'See #<ses6jr7c5h2q6>.';+ const { text, edits } = linkifyMentions(input);++ const replaced = verifyEdits(input, text, edits);+ expect(replaced[0][1]).toBe('[#ses6jr7c5h2q6](avibe-mention:session:ses6jr7c5h2q6)');+ });+});diff --git a/ui/src/lib/mentions.ts b/ui/src/lib/mentions.tsindex 5deac2ccb..ab0f5b643 100644--- a/ui/src/lib/mentions.ts+++ b/ui/src/lib/mentions.ts@@ -10,6 +10,10 @@// the text BEFORE react-markdown sees it (we rewrite markers to mention links),// so it never collides with markdown's `<tag>` HTML handling.+// Rewriting a marker moves the text after it, so this pass reports what it+// changed in the form the citation binding carries its measurements across.+import type { EditedText, TextEdit } from './citations';+export type AgentReference = {kind: 'agent';/** Display name — the `vibe agent run --agent <name>` handle and the chip label. */@@ -65,31 +69,47 @@ function escapeMarkdownLabel(value: string): string {* `avibe-mention:` scheme, so the shared `Markdown` renderer can show them as* chips via its `a` component map. Session labels use the title from* `references` when available, else the raw id.+ *+ * A marker becomes a link of a different length, which moves everything after+ * it — including any source citation the backend measured in this same text.+ * So each rewrite is reported as a `TextEdit` in the coordinates of `text`,+ * letting the caller carry those measurements across this pass (see+ * lib/citations). The rewritten text itself is unchanged by that bookkeeping.*/// Inline code spans and fenced code blocks — markers inside these must render// literally (e.g. `` `@<T>` ``), so linkify skips them.const CODE_SEGMENT_RE = /(```[\s\S]*?```|`[^`]*`)/g;-export function linkifyMentions(text: string, references?: MentionReference[]): string {+export function linkifyMentions(text: string, references?: MentionReference[]): EditedText {const sessionTitles = new Map<string, string>();for (const ref of references ?? []) {if (ref.kind === 'session' && ref.title) sessionTitles.set(ref.session_id, ref.title);}- const rewrite = (segment: string): string =>- segment.replace(MENTION_MARKER_RE, (_full, trigger: string, inner: string) => {- if (trigger === '@') {- const label = escapeMarkdownLabel(`@${inner}`);- return `[${label}](${MENTION_LINK_SCHEME}:agent:${encodeURIComponent(inner)})`;- }- const label = escapeMarkdownLabel(`#${sessionTitles.get(inner) || inner}`);- return `[${label}](${MENTION_LINK_SCHEME}:session:${encodeURIComponent(inner)})`;+ const edits: TextEdit[] = [];+ const rewrite = (segment: string, base: number): string =>+ segment.replace(MENTION_MARKER_RE, (full: string, trigger: string, inner: string, at: number) => {+ const label = trigger === '@'+ ? escapeMarkdownLabel(`@${inner}`)+ : escapeMarkdownLabel(`#${sessionTitles.get(inner) || inner}`);+ const kind = trigger === '@' ? 'agent' : 'session';+ const link = `[${label}](${MENTION_LINK_SCHEME}:${kind}:${encodeURIComponent(inner)})`;+ edits.push({ start: base + at, end: base + at + full.length, inserted: link.length });+ return link;});// Split out code spans/blocks (the odd capture-group chunks) and rewrite markers// only in the surrounding prose, so marker-shaped text inside code stays literal.- return text+ // The chunks concatenate back to `text`, so a running cursor over them is each+ // chunk's own offset in the input.+ let cursor = 0;+ const rewritten = text.split(CODE_SEGMENT_RE)- .map((chunk) => (chunk.startsWith('`') ? chunk : rewrite(chunk)))+ .map((chunk) => {+ const base = cursor;+ cursor += chunk.length;+ return chunk.startsWith('`') ? chunk : rewrite(chunk, base);+ }).join('');+ return { text: rewritten, edits };}/** Parse an `avibe-mention:<kind>:<value>` href back into its parts. */diff --git a/ui/src/lib/resultFooter.test.ts b/ui/src/lib/resultFooter.test.tsindex 0a0a562bc..16259e747 100644--- a/ui/src/lib/resultFooter.test.ts+++ b/ui/src/lib/resultFooter.test.ts@@ -18,9 +18,14 @@ const message = (describe('resultFooterParts', () => {it('uses structured content and removes an older folded copy', () => {const footer = '✅ ⏱️ 5s · 🪙 1.2k tok';- expect(resultFooterParts(message(`Answer\n\n${footer}`, { result_footer: footer }))).toEqual({+ const text = `Answer\n\n${footer}`;+ expect(resultFooterParts(message(text, { result_footer: footer }))).toEqual({body: 'Answer',footer: '⏱️ 5s · 🪙 1.2k tok',+ // Counted in UTF-16 code units, the unit a citation's ranges also use:+ // the coin is a surrogate pair, so this is not the footer's length in+ // characters.+ edits: [{ start: 6, end: text.length, inserted: 0 }],});});@@ -29,6 +34,7 @@ describe('resultFooterParts', () => {expect(resultFooterParts(message('Answer', { result_footer: footer }))).toEqual({body: 'Answer',footer: '⏱️ 2m 24s',+ edits: [],});});@@ -39,9 +45,11 @@ describe('resultFooterParts', () => {['❌ 🪙 12.3k tok', '❌ 🪙 12.3k tok'],['✅ 🪙 1.4M tok', '🪙 1.4M tok'],]) {- expect(resultFooterParts(message(`Answer\n\n${footer}`))).toEqual({+ const text = `Answer\n\n${footer}`;+ expect(resultFooterParts(message(text))).toEqual({body: 'Answer',footer: displayedFooter,+ edits: [{ start: 6, end: text.length, inserted: 0 }],});}});@@ -51,21 +59,50 @@ describe('resultFooterParts', () => {expect(resultFooterParts(message(footer))).toEqual({body: '',footer: '⏱️ 5s · 🪙 1.2k tok',+ edits: [{ start: 0, end: footer.length, inserted: 0 }],});});it('does not move authored text or non-Agent result content', () => {const text = 'Answer\n\n✅ Looks good';- expect(resultFooterParts(message(text))).toEqual({ body: text, footer: null });+ expect(resultFooterParts(message(text))).toEqual({ body: text, footer: null, edits: [] });const coinText = 'Answer\n\n✅ 🪙 deployment complete';- expect(resultFooterParts(message(coinText))).toEqual({ body: coinText, footer: null });+ expect(resultFooterParts(message(coinText)))+ .toEqual({ body: coinText, footer: null, edits: [] });expect(resultFooterParts(message('✅ 🪙 deployment complete'))).toEqual({body: '✅ 🪙 deployment complete',footer: null,+ edits: [],});expect(resultFooterParts(message(text, {}, { author: 'user', type: 'user' }))).toEqual({body: text,footer: null,+ edits: [],});});++ // The stored row is the body a citation's ranges were measured in, and the+ // renderer is handed `body` instead - so `edits` is how those measurements+ // cross this split (see lib/citations). It is only worth carrying if it says+ // what actually happened, which is a property of every case above rather+ // than a case of its own.+ it.each([+ ['a structured footer folded into the text', 'Answer\n\n✅ ⏱️ 5s', { result_footer: '✅ ⏱️ 5s' }],+ ['a structured footer stored apart', 'Answer', { result_footer: '✅ ⏱️ 2m 24s' }],+ ['a legacy footer paragraph', 'Answer\n\n✅ ⏱️ 0s', {}],+ ['a body that is only a footer', '✅ 🪙 1.4M tok', {}],+ ['an emoji body the pattern must not claim', 'Answer\n\n✅ Looks good', {}],+ ['an authored body with no footer at all', 'Just an answer.', {}],+ ])('describes the cut it made in %s', (_why, text, content) => {+ const { body, edits } = resultFooterParts(message(text, content));++ // Every edit here is a removal, so the body is what is left of the stored+ // text once they are taken out - in the stored text's own coordinates.+ let rebuilt = text;+ for (const edit of [...edits].sort((left, right) => right.start - left.start)) {+ expect(edit.inserted).toBe(0);+ rebuilt = rebuilt.slice(0, edit.start) + rebuilt.slice(edit.end);+ }+ expect(rebuilt).toBe(body);+ });});diff --git a/ui/src/lib/resultFooter.ts b/ui/src/lib/resultFooter.tsindex 6380b2987..311753e54 100644--- a/ui/src/lib/resultFooter.ts+++ b/ui/src/lib/resultFooter.ts@@ -1,4 +1,5 @@import type { WorkbenchMessage } from '@/context/ApiContext';+import type { TextEdit } from '@/lib/citations';const LEGACY_RESULT_FOOTER_RE =/^(?:✅|⚠️|❌) (?:⏱️ (?:\d+m )?\d+s(?: · 🪙 \d+(?:\.\d+)?[kM]? tok)?|🪙 \d+(?:\.\d+)?[kM]? tok)$/u;@@ -6,40 +7,55 @@ const LEGACY_RESULT_FOOTER_RE =export type ResultFooterParts = {body: string;footer: string | null;+ /** How `body` was cut out of `message.text`, so measurements taken against+ * the stored row (a citation's spans) can be carried onto the body the+ * renderer is handed. Empty when the two are the same text. */+ edits: TextEdit[];};function webResultFooter(footer: string): string {return footer.replace(/^✅\s+/u, '');}+/** The whole `[at, message.text.length)` tail, removed. */+function cut(at: number, length: number): TextEdit[] {+ return [{ start: at, end: length, inserted: 0 }];+}+/** Separate Avibe's generated duration/token summary from an Agent reply body. */export function resultFooterParts(message: Pick<WorkbenchMessage, 'author' | 'type' | 'text' | 'content'>,): ResultFooterParts {if (message.author !== 'agent' || (message.type !== 'result' && message.type !== 'error')) {- return { body: message.text, footer: null };+ return { body: message.text, footer: null, edits: [] };}const structured = (message.content as { result_footer?: unknown } | null)?.result_footer;if (typeof structured === 'string' && structured.trim()) {const footer = structured.trim();const suffix = `\n\n${footer}`;+ const stripped = message.text.endsWith(suffix);return {- body: message.text.endsWith(suffix) ? message.text.slice(0, -suffix.length) : message.text,+ body: stripped ? message.text.slice(0, -suffix.length) : message.text,footer: webResultFooter(footer),+ edits: stripped ? cut(message.text.length - suffix.length, message.text.length) : [],};}if (LEGACY_RESULT_FOOTER_RE.test(message.text)) {- return { body: '', footer: webResultFooter(message.text) };+ return { body: '', footer: webResultFooter(message.text), edits: cut(0, message.text.length) };}// Rows created before result_footer became structured still contain the exact// generated footer as their final paragraph. Match only that closed format so// authored prose ending in ordinary emoji/text is never moved.const splitAt = message.text.lastIndexOf('\n\n');- if (splitAt < 0) return { body: message.text, footer: null };+ if (splitAt < 0) return { body: message.text, footer: null, edits: [] };const candidate = message.text.slice(splitAt + 2);- if (!LEGACY_RESULT_FOOTER_RE.test(candidate)) return { body: message.text, footer: null };- return { body: message.text.slice(0, splitAt), footer: webResultFooter(candidate) };+ if (!LEGACY_RESULT_FOOTER_RE.test(candidate)) return { body: message.text, footer: null, edits: [] };+ return {+ body: message.text.slice(0, splitAt),+ footer: webResultFooter(candidate),+ edits: cut(splitAt, message.text.length),+ };}diff --git a/uv.lock b/uv.lockindex ee4a14e98..d38f6a436 100644--- a/uv.lock+++ b/uv.lock@@ -308,6 +308,7 @@ dependencies = [{ name = "discord-py" },{ name = "fastapi" },{ name = "httpx", extra = ["socks"] },+ { name = "idna" },{ name = "ijson" },{ name = "imagesize" },{ name = "lark-oapi" },@@ -350,6 +351,7 @@ requires-dist = [{ name = "discord-py", specifier = ">=2.4.0" },{ name = "fastapi", specifier = ">=0.110" },{ name = "httpx", extras = ["socks"], specifier = ">=0.27" },+ { name = "idna", specifier = ">=3.4" },{ name = "ijson", specifier = ">=3.5.1" },{ name = "imagesize", specifier = ">=1.4.1" },{ name = "lark-oapi", specifier = ">=1.4.0" },diff --git a/vibe/i18n/en.json b/vibe/i18n/en.jsonindex 749c2bcd2..1ed6bc1f9 100644--- a/vibe/i18n/en.json+++ b/vibe/i18n/en.json@@ -749,7 +749,8 @@},"message": {"ack": "{agent} received, processing...",- "quickReplyNote": "Reply: {text}"+ "quickReplyNote": "Reply: {text}",+ "citationUnresolved": "(source unavailable)"},"backend": {"claude": "ClaudeCode",diff --git a/vibe/i18n/zh.json b/vibe/i18n/zh.jsonindex 3868da596..97d27f527 100644--- a/vibe/i18n/zh.json+++ b/vibe/i18n/zh.json@@ -749,7 +749,8 @@},"message": {"ack": "{agent} 已收到,正在处理...",- "quickReplyNote": "回复:{text}"+ "quickReplyNote": "回复:{text}",+ "citationUnresolved": "(来源不可用)"},"backend": {"claude": "ClaudeCode",